diff --git a/Cargo.lock b/Cargo.lock index 1508b3ee0..2b2af875d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1629,20 +1629,24 @@ version = "0.1.0" dependencies = [ "aes", "bytemuck", + "chrono", "craft-stream-codecs", "flate2", "hayro", "hayro-cmap", "hayro-interpret", "hayro-syntax", + "js-sys", "krilla", "kurbo 0.13.1", "md-5 0.11.0", "proptest", + "serde_json", "sha2 0.11.1", "stringprep", "thiserror 2.0.21", "ttf-parser", + "wasm-bindgen", "wasm-bindgen-test", "zeroize", ] diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml index ba9a737d6..193aeb9df 100644 --- a/crates/pdf/Cargo.toml +++ b/crates/pdf/Cargo.toml @@ -19,6 +19,7 @@ write = ["dep:krilla", "dep:thiserror"] write-pdf = ["write", "krilla/pdf"] write-simple-text = ["write", "krilla/simple-text"] filters = ["dep:craft-stream-codecs", "dep:flate2", "dep:thiserror"] +cos = ["crypt", "filters", "dep:chrono", "dep:js-sys", "dep:wasm-bindgen"] crypt = ["dep:aes", "dep:md-5", "dep:sha2", "dep:stringprep", "dep:zeroize", "dep:thiserror"] [dependencies] @@ -37,8 +38,17 @@ md-5 = { version = "0.11", optional = true } sha2 = { version = "0.11", optional = true } stringprep = { version = "0.1", optional = true } zeroize = { version = "1", optional = true } +# PDF date strings in the object layer (`cos`). +chrono = { workspace = true, optional = true } + +# On the web, a rewrite's fresh file identifier (`cos`) takes its entropy from the browser +# (`crypto.getRandomValues`); native builds compile neither. +[target.'cfg(target_arch = "wasm32")'.dependencies] +js-sys = { version = "0.3", optional = true } +wasm-bindgen = { version = "0.2", optional = true } [dev-dependencies] +serde_json = { workspace = true } hayro-syntax = "=0.7.2" hayro-interpret = { version = "=0.7.0", default-features = false } hayro-cmap = "=0.1.0" @@ -50,3 +60,11 @@ proptest = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3" + +[[example]] +name = "cos_info" +required-features = ["cos"] + +[[example]] +name = "cos_protect" +required-features = ["cos"] diff --git a/crates/pdf/README.md b/crates/pdf/README.md index 5b0f65e43..18d3d25a9 100644 --- a/crates/pdf/README.md +++ b/crates/pdf/README.md @@ -14,6 +14,7 @@ No feature is enabled by default. | `write-simple-text` | `write` plus Krilla's optional text shaper and `draw_text` | | `filters` | The `/Filter` chain for editing and writing PDF files: decode and encode, with limits and accounting. No reader or writer dependency | | `crypt` | The standard security handler: decrypt, encrypt and create encryption. No reader or writer dependency | +| `cos` | `crypt` and `filters` plus the object layer: open, edit and save PDF files (incremental and full saves). No reader or writer dependency | ## Reading and rendering @@ -111,6 +112,50 @@ read-only consumer, then repeat with `--target wasm32-unknown-unknown` and the t These backend checks supplement the root workspace gates; excluded vendor crates are not covered by `cargo test --workspace` alone. +## Object layer + +`cos` is an object layer for apps that edit PDF files (ISO 32000-2 §7). It parses the object +graph lazily and tolerantly, keeps edits in a copy-on-write overlay, and writes the file back +incrementally (the original bytes untouched) or as a full, garbage-collected rewrite. It uses +`crypt` and `filters`; the reader (`read`) and writer (`write`) are separate and do not use it. + +```rust +use craft_pdf::cos::{Document, ObjRef, Object, SaveOptions, write_full, write_incremental}; + +let doc = Document::open(bytes)?; // Arc> +let doc = Document::open_with_password(bytes, Some("owner"))?; // encrypted files +let obj = doc.get(ObjRef::new(12, 0)); // loaded on first use, cached +let data = stream.decoded()?; // filters applied on demand +doc.set(r, Object::Dict(d)); // edits land in the overlay +let r = doc.add(obj); +let bytes = write_incremental(&doc, &SaveOptions::default())?; // append a revision +let bytes = write_full(&doc, &SaveOptions::default())?; // rewrite what the trailer reaches +``` + +- Reads every cross-reference form (tables, streams, hybrid `/XRefStm`) and object streams. A + file whose cross-reference data is wrong or missing is rebuilt by scanning for object headers, + and each repair is recorded (`repair_log`). Malformed input gives `CosError`, never a panic. +- Encryption on open and save through `crypt`: revisions 2 to 6, crypt filters, `/EFF` and + `/EncryptMetadata`. `set_encryption` and `remove_encryption` change it. +- `write_incremental` appends the edited objects with a cross-reference section in the style of + the file's last one, so earlier revisions and their signatures stay as they were; it falls back + to `write_full` when an edit cannot be appended. `write_full` renumbers what the trailer + reaches and, by default, packs objects into object streams. Both return the bytes. +- `Document` is cheap to clone: the bytes, index and caches are shared, so a clone is a + snapshot for undo or a background job. +- Images: `ImageBudget::decode` gives an image's samples after its filters, CCITT fax and JBIG2 + included, all images of one operation spending from one allowance. DCT and JPX are not decoded + here. +- Not here: streaming files larger than memory, linearized output. + +The code comes from PdfCraft's `pdfcraft-cos` crate (revision +`1d2af9907ba772b368f6b54811359805fa5428e9`), unchanged apart from module paths and doc +comments. Its test helpers come along: `src/cos/testkit.rs` (synthetic files, from +`pdfcraft-testkit`) and `tests/cos/oracle.rs` (qpdf runs, from `pdfcraft-oracle`; a missing +`qpdf` skips those tests, or fails them when `CRAFT_PDF_ORACLES_REQUIRED` is set). Tests: +`cargo test -p craft-pdf --no-default-features --features cos`. Examples: `cos_info` and +`cos_protect` (`--features cos`). + ## Stream filters `filters` is the `/Filter` chain of a PDF stream (ISO 32000-2 §7.4) for code that edits or diff --git a/crates/pdf/examples/cos_info.rs b/crates/pdf/examples/cos_info.rs new file mode 100644 index 000000000..51e71e1ce --- /dev/null +++ b/crates/pdf/examples/cos_info.rs @@ -0,0 +1,38 @@ +//! Print how a file was opened: revisions, repair notes, trailer, security and the catalog. +//! `cargo run -p craft-pdf --features cos --example cos_info f.pdf [password] [object-number…]` +fn show(o: &craft_pdf::cos::Object) -> String { + let mut t = Vec::new(); + craft_pdf::cos::serialize(o, &mut t); + let s = String::from_utf8_lossy(&t).into_owned(); + s.chars().take(400).collect() +} + +fn main() { + let mut args = std::env::args().skip(1); + let path = args.next().expect("usage: cosinfo [password] [obj…]"); + let password = args.next().filter(|p| !p.is_empty()); + let objs: Vec = args.filter_map(|a| a.parse().ok()).collect(); + let bytes = std::sync::Arc::new(std::fs::read(&path).expect("readable")); + match craft_pdf::cos::Document::open_with_password(bytes, password.as_deref()) { + Ok(doc) => { + println!("version {} revisions {:?}", doc.version(), doc.revisions()); + for r in doc.repair_log() { + println!("repair: {r}"); + } + println!("trailer {}", show(&craft_pdf::cos::Object::Dict(doc.trailer().clone()))); + if let Some(s) = doc.security() { + println!("security: auth {:?}, key {} bytes, {:?}", s.auth(), s.file_key().len(), s.permissions()); + } + if let Some(r) = doc.root() { + println!("catalog {}", show(&doc.get(r))); + } + for n in objs { + match doc.try_get(n) { + Ok(o) => println!("{n} 0 obj {}", show(&o)), + Err(e) => println!("{n} 0 obj error: {e}"), + } + } + } + Err(e) => println!("error: {e}"), + } +} diff --git a/crates/pdf/examples/cos_protect.rs b/crates/pdf/examples/cos_protect.rs new file mode 100644 index 000000000..c40bccdce --- /dev/null +++ b/crates/pdf/examples/cos_protect.rs @@ -0,0 +1,35 @@ +//! Encrypt a PDF: `cargo run -p craft-pdf --features cos --example cos_protect in.pdf out.pdf rc4-40|rc4-128|aes-128|aes-256 USER OWNER` +/// 32 fresh bytes for the file key. `RandomState` is seeded by the operating system (as the +/// engine's `entropy` uses it), so every run encrypts with a different key. +fn seed() -> [u8; 32] { + use std::hash::{BuildHasher, Hasher}; + let mut out = [0u8; 32]; + for (i, chunk) in out.chunks_mut(8).enumerate() { + let mut h = std::collections::hash_map::RandomState::new().build_hasher(); + h.write_usize(i); + chunk.copy_from_slice(&h.finish().to_le_bytes()); + } + out +} + +fn main() { + let a: Vec = std::env::args().skip(1).collect(); + let [input, output, alg, user, owner] = &a[..] else { panic!("usage: protect in out alg user owner") }; + let algorithm = match alg.as_str() { + "rc4-40" => craft_pdf::cos::Algorithm::Rc4_40, + "rc4-128" => craft_pdf::cos::Algorithm::Rc4_128, + "aes-128" => craft_pdf::cos::Algorithm::Aes128, + _ => craft_pdf::cos::Algorithm::Aes256, + }; + let mut doc = craft_pdf::cos::Document::open(std::sync::Arc::new(std::fs::read(input).unwrap())).unwrap(); + doc.set_encryption(&craft_pdf::cos::NewEncryption { + algorithm, + user_password: user, + owner_password: owner, + permissions: -1, + encrypt_metadata: true, + seed: seed(), + }) + .unwrap(); + std::fs::write(output, craft_pdf::cos::write_full(&doc, &Default::default()).unwrap()).unwrap(); +} diff --git a/crates/pdf/src/cos.rs b/crates/pdf/src/cos.rs new file mode 100644 index 000000000..1084c8bf4 --- /dev/null +++ b/crates/pdf/src/cos.rs @@ -0,0 +1,58 @@ +//! The PDF object layer (COS), for code that edits and saves PDF files. It uses [`crate::crypt`] +//! and [`crate::filters`]; the reader (`read`) is separate and does not use it. +//! +//! Parses the COS object graph lazily and tolerantly, keeps edits in a copy-on-write overlay +//! (cheap snapshots for undo), and writes documents back either incrementally (original bytes +//! untouched) or as a full, garbage-collected rewrite. An app can use the object graph as its +//! document model. +//! +//! Done: every cross-reference form (tables, streams, hybrid `/XRefStm`), object streams, repair +//! by scanning, encryption on load and save (R2–R6, crypt filters, `/EFF`), incremental writing, +//! and full saves that pack objects into compressed object streams with a cross-reference stream. +//! Not yet: streaming very large files within a memory budget, and linearized output. + +mod budget; +mod bytes; +mod document; +mod object; +pub mod page_labels; +mod parser; +mod security; +mod writer; + +#[cfg(test)] +pub(crate) mod testkit; + +pub use crate::crypt::{ + Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, +}; +pub use budget::{DecodeBudget, ImageBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS, MAX_REDACTED_IMAGE_PIXELS}; +pub use bytes::Bytes; +pub use document::{DECODE_PARMS_KEYS, Document, ReadState, Revision, SourceIdentity, XrefEntry}; +pub use object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, Name, ObjRef, Object, PdfString, Stream, TextChars}; +pub use parser::{Lexer, is_whitespace, parse_indirect}; +pub use writer::{SaveOptions, civil_from_days, new_file_id, pdf_date, reachable_objects, serialize, write_full, write_incremental}; + +#[derive(Debug, thiserror::Error, Clone, PartialEq)] +pub enum CosError { + #[error("not a PDF file (no %PDF- header)")] + NotPdf, + #[error("syntax error at byte {offset}: {detail}")] + Syntax { offset: usize, detail: String }, + #[error("object {0} is missing")] + MissingObject(u32), + #[error("object {0} is not a dictionary")] + NotADictionary(ObjRef), + #[error("stream data could not be decoded: {0}")] + Filter(String), + #[error("the document is protected by a password")] + NeedsPassword, + #[error("the password is incorrect")] + WrongPassword, + #[error("unsupported security: {0}")] + Security(String), + #[error("internal lock poisoned")] + Poisoned, + #[error("this document was opened read-only with a decode limit and cannot be saved")] + ReadOnlyLimit, +} diff --git a/crates/pdf/src/cos/budget.rs b/crates/pdf/src/cos/budget.rs new file mode 100644 index 000000000..4e0795ad0 --- /dev/null +++ b/crates/pdf/src/cos/budget.rs @@ -0,0 +1,1188 @@ +//! A shared allowance for operations that decode many streams. + +use std::collections::HashSet; + +use crate::filters::FilterError; + +use crate::cos::{CosError, Document, ObjRef, Object, Stream}; + +/// The steps of decoding an operation over a document's images may take whatever the file's size. +/// A step of JBIG2 decoding ([`hayro_jbig2::Image::decode_within`]) is about a pixel, and takes 6 to +/// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the +/// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to +/// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A +/// byte Flate reads is [`crate::filters::FLATE_READ_WORK`] steps besides, since inflating blocks +/// that make nothing takes up to 21 ns a byte: about 7 ns a step. +const IMAGE_WORK: u64 = 1 << 34; +/// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a +/// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored +/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to +/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; + +/// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every +/// form XObject it enters, each occurrence counted again. +pub const MAX_PAGE_DECODED: usize = 256 << 20; +/// Form XObjects one page's content walk may enter, each entry counted again. +pub const MAX_PAGE_FORMS: usize = 50_000; + +/// The most pixels an image redaction decodes, to clear the ones under a mark or to store again +/// without the `/JBIG2Globals` it shares with a redacted image (an app's redaction). +/// An optimizer that has JBIG2 images share a dictionary should share it only within this many +/// pixels, so that redaction can store each of them again: one it couldn't would refuse the +/// redaction. +pub const MAX_REDACTED_IMAGE_PIXELS: u64 = 64 << 20; + +/// The steps a sample of a JPEG costs: a few bytes can hold a page of one grey, and decoding, +/// resampling and storing it took about 20 to 45 ns a sample. +const JPEG_WORK: u64 = 2; +/// The steps a byte of a JPEG costs to read: its tables took 24 ns a byte before a size was known, +/// and images can share one, so reading it again for each has to count. +const JPEG_READ_WORK: u64 = 4; + +/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data and +/// whether it stopped at damage, or the error, each with that count. +type Decoder = fn(&Stream, usize) -> Result<(Vec, bool, usize), (FilterError, usize)>; + +/// What one operation over a document's images may still spend decoding them: exporting them, +/// optimizing them, and storing LZW streams with Flate as it does, clearing them for redaction. +/// +/// An image has its own limit, which doesn't bound a file of thousands of images made to reach it: +/// a few hundred bytes of JBIG2 can ask for seconds of decoding, 4 KB of Flate inside Flate for a +/// gigabyte, and an image that shares `/JBIG2Globals` decodes them again. So images share this +/// allowance, whatever they do with what they decode and whether or not it works, and past it they +/// aren't decoded. A caller then treats them as it does an image it can't read, and +/// [`ImageBudget::refused`] says how many there were. +/// +/// An image also pays for its pixels once decoded, as what reads it goes through them one by one +/// however few bytes coded them: blank rows of fax or Flate take almost none. It pays for no more +/// than its data holds, a bit a pixel at least, so an image declaring a size its data doesn't fill +/// costs what a smaller one would, not the run's allowance. A JPEG pays for its samples when +/// it is decoded ([`ImageBudget::pay_jpeg`]), and is otherwise not charged, as JPEG 2000 isn't: +/// their decoders don't run here. +#[derive(Debug)] +pub struct ImageBudget { + work: u64, + refused: usize, +} + +impl ImageBudget { + /// The allowance for an operation on `doc`, which a longer file makes larger. The streams made + /// or changed since it was opened count as the file's bytes do, so a document built in memory + /// (combined, created, or given pages of scans) gets what it would once saved and opened again. + pub fn new(doc: &Document) -> Self { + let length = |n: usize| u64::try_from(n).unwrap_or(u64::MAX); + let edited = doc + .scan_objects_subset(doc.modified_objects()) + .filter_map(|(_, object)| match &*object.ok()? { + Object::Stream(s) => Some(length(s.raw.len())), + _ => None, + }) + .fold(0u64, u64::saturating_add); + let bytes = length(doc.bytes().len()).saturating_add(edited); + Self::with_work(IMAGE_WORK.saturating_add(bytes.saturating_mul(IMAGE_WORK_PER_BYTE))) + } + + /// An allowance of `work` steps of decoding. + pub fn with_work(work: u64) -> Self { + Self { work, refused: 0 } + } + + /// The samples of image `s`, decoded and charged to the allowance: the one way to decode an + /// image outside the renderer, so that an operation over many images can't decode them without + /// one. + /// + /// They go through its filters ([`Stream::decoded`]), or, for a CCITT fax or JBIG2 image (its + /// last filter), through the filters before it and then [`crate::filters::decode_ccitt_within`] + /// with its [`Document::fax_params`], or [`crate::filters::decode_jbig2_within`] with its + /// `/JBIG2Globals`, as `CCITTFaxDecode` and `JBIG2Decode` give them (ISO 32000-2 §7.4.6, + /// §7.4.7). Parameters given by reference count, for every filter + /// ([`Document::with_direct_parms`]). A fax image's `/Columns` has to be its `/Width`, and a + /// JBIG2 image's page its `/Width` × `/Height`: a PDF renderer draws the whole page into + /// the image. Up to its `/Height` rows are decoded, whatever a fax image's `/Rows` says; damaged + /// fax data gives fewer, so a caller checks it has the rows it needs. Damaged JBIG2 data, and + /// globals that can't be read (within [`MAX_JBIG2_GLOBALS`](crate::cos::MAX_JBIG2_GLOBALS)), are an + /// error. + /// + /// No stage, the filters before a fax or JBIG2 decoder included, may produce more than `max` + /// bytes, nor more than [`MAX_DECODED`](crate::cos::MAX_DECODED). A JBIG2 image's `/JBIG2Globals` + /// are another stream, shared between images, and are decoded within `MAX_JBIG2_GLOBALS` + /// whatever `max` is. Each filter as it runs, the globals when they have to be read, and the fax + /// or JBIG2 decoder spend from the allowance, and the image then pays for its pixels; with + /// nothing left an image is refused before anything is read, its globals included. + /// + /// ``` + /// # let doc = craft_pdf::cos::Document::new_empty(); + /// # let s = craft_pdf::cos::Stream::from_raw(craft_pdf::cos::Dict::new(), Vec::new()); + /// let samples = craft_pdf::cos::ImageBudget::new(&doc).decode(&doc, &s, 1 << 20); + /// assert_eq!(samples.unwrap(), b""); + /// ``` + /// + /// `Document` has no way of its own to decode an image: + /// + /// ```compile_fail + /// # let doc = craft_pdf::cos::Document::new_empty(); + /// # let s = craft_pdf::cos::Stream::from_raw(craft_pdf::cos::Dict::new(), Vec::new()); + /// let _ = doc.decoded_image(&s); + /// ``` + /// + /// ```compile_fail + /// # let doc = craft_pdf::cos::Document::new_empty(); + /// # let s = craft_pdf::cos::Stream::from_raw(craft_pdf::cos::Dict::new(), Vec::new()); + /// let _ = doc.decoded_image_within(&s, 1 << 20); + /// ``` + pub fn decode(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { + let decoded = doc.decoded_image_spending(s, max, &mut self.work); + self.count_refused(s, &decoded); + self.pay_size(s, decoded?) + } + + /// Image `s` through its filters alone, charged to the allowance: [`Stream::decoded_within`] + /// with its `/DecodeParms` given by reference read too ([`Document::with_direct_parms`]), for an + /// image that has no fax or JBIG2 decoder to go through. + pub fn decode_filters(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { + let decoded = doc.with_direct_parms(s).decoded_spending(max, &mut self.work); + self.count_refused(s, &decoded); + let decoded = decoded?; + if s.filters().iter().all(|(f, _)| f.is_decodable()) { self.pay_size(s, decoded) } else { Ok(decoded) } + } + + /// `data` through `chain`, any corruption being an error ([`crate::filters::decode_within`]), + /// for a stream a run rewrites rather than an image it reads, charged to the allowance. + pub fn decode_chain( + &mut self, + chain: &[(crate::filters::Filter, crate::filters::Params)], + data: &[u8], + max: usize, + ) -> Result, FilterError> { + crate::filters::decode_within(chain, data, max, &mut self.work) + } + + /// Pays `steps` of other work on decoded data, or spends all there is and says so. + pub fn spend(&mut self, steps: u64) -> bool { + let left = self.work.checked_sub(steps); + self.work = left.unwrap_or(0); + left.is_some() + } + + /// Pays for `pixels` a decode goes on to work through, or spends all there is, and the image + /// is refused. + pub fn pay_pixels(&mut self, pixels: u64) -> bool { + let paid = self.spend(pixels); + self.refused += usize::from(!paid); + paid + } + + /// Pays for reading JPEG `data` and decoding its `samples`, as [`ImageBudget::pay_pixels`] does. + /// A caller pays before its tables are parsed, which can be most of the file. + pub fn pay_jpeg(&mut self, samples: u64, data: &[u8]) -> bool { + let reading = (data.len() as u64).saturating_mul(JPEG_READ_WORK); + self.pay_pixels(samples.saturating_mul(JPEG_WORK).saturating_add(reading)) + } + + /// Pays for the pixels of image `s` that its `decoded` data holds, those of its `/Width` and + /// `/Height` but no more than a bit of the data each, unless it is JBIG2, whose decoder pays for + /// its page, or isn't decoded here. What can't be paid for refuses the image. + fn pay_size(&mut self, s: &Stream, decoded: Vec) -> Result, CosError> { + if !charged(s) || matches!(s.filters().last(), Some((crate::filters::Filter::Jbig2, _))) { + return Ok(decoded); + } + let side = |key: &[u8]| s.dict.int(key).and_then(|v| u64::try_from(v).ok()).unwrap_or(0); + let held = u64::try_from(decoded.len()).unwrap_or(u64::MAX).saturating_mul(8); + if self.pay_pixels(side(b"Width").saturating_mul(side(b"Height")).min(held)) { + Ok(decoded) + } else { + Err(CosError::Filter(FilterError::Spent.to_string())) + } + } + + /// An image that failed with nothing left was refused, unless it has a filter that costs + /// nothing here, which fails whatever is left. + fn count_refused(&mut self, s: &Stream, decoded: &Result, CosError>) { + if decoded.is_err() && self.work == 0 && charged(s) { + self.refused += 1; + } + } + + /// The decodes refused for lack of allowance: an image read twice over is counted twice. + pub fn refused(&self) -> usize { + self.refused + } + + /// Charges `work` steps paid for out of another allowance, by a thread working on its own. + pub fn charge(&mut self, work: u64) { + self.work = self.work.saturating_sub(work); + } + + /// The steps of decoding still allowed. + pub fn work_left(&self) -> u64 { + self.work + } +} + +/// Whether `s` can be decoded here and so charged: general filters, a fax or JBIG2 decoder last at +/// most. Those decoders anywhere else can't run. +fn charged(s: &Stream) -> bool { + use crate::filters::Filter; + s.readable_filters().is_ok_and(|chain| { + chain.split_last().is_none_or(|((last, _), before)| { + (last.is_decodable() || matches!(last, Filter::Jbig2 | Filter::CcittFax)) && before.iter().all(|(f, _)| f.is_decodable()) + }) + }) +} + +/// What one operation may still decode and enter across all the streams it reads. +/// +/// [`MAX_DECODED`](crate::cos::MAX_DECODED) bounds one decode. It does not bound a page whose +/// `/Contents` names the same compressed stream thousands of times, or a form drawn thousands of +/// times from other forms: each occurrence would get a fresh allowance. A `DecodeBudget` is +/// shared by every decode of one operation, so the total decoded bytes and form entries stay +/// under fixed limits however the file repeats its references. A decode that runs past the +/// remaining allowance spends all of it, so a bomb referenced many times is inflated once. +#[derive(Debug)] +pub struct DecodeBudget { + /// The whole allowance, for errors. + limit: usize, + bytes: usize, + forms: usize, + seen: HashSet, + /// Decodes refused for lack of allowance ([`DecodeBudget::refused`]). + refusals: usize, +} + +impl DecodeBudget { + /// An allowance of `bytes` decoded bytes and `forms` form entries. + pub fn new(bytes: usize, forms: usize) -> Self { + Self { limit: bytes, bytes, forms, seen: HashSet::new(), refusals: 0 } + } + + /// The allowance for one page: [`MAX_PAGE_DECODED`] bytes and [`MAX_PAGE_FORMS`] forms. + pub fn page() -> Self { + Self::new(MAX_PAGE_DECODED, MAX_PAGE_FORMS) + } + + /// Decoded bytes still allowed. + pub fn remaining(&self) -> usize { + self.bytes + } + + /// How many decodes were refused for lack of allowance: one that came once it was spent, or + /// ran past what was left. A decode that spent exactly what was left wasn't refused, so an + /// allowance used up to the byte has none. + pub fn refused(&self) -> usize { + self.refusals + } + + /// Whether the allowance is spent and `stream`, which has data, would be refused, noting the + /// refusal ([`DecodeBudget::refused`]): a caller asks this before resolving the stream's + /// parameters, work the allowance doesn't count. A stream without data still reads. + pub fn refuses(&mut self, stream: &Stream) -> bool { + let refuses = self.bytes == 0 && !stream.raw.is_empty(); + self.refusals += usize::from(refuses); + refuses + } + + /// Whether a decode was refused for lack of allowance or a form entry for lack of visits. + pub fn exhausted(&self) -> bool { + self.bytes == 0 || self.forms == 0 + } + + /// `stream` decoded like [`Stream::decoded`], charged to the allowance. Fails, without + /// decoding, once the allowance is spent. A stream damaged partway gives what decoded before + /// the damage, so code that writes the stream back reads it with + /// [`DecodeBudget::decode_with_damage`] instead. + pub fn decode(&mut self, stream: &Stream) -> Result, CosError> { + self.charge(stream, usize::MAX, Stream::decoded_within_counted).map(|(data, _)| data) + } + + /// Like [`DecodeBudget::decode`], for a stream with a tighter bound of its own: decoding + /// past `max` bytes is an error that charges `max`, the work it took to find out. + pub fn decode_within(&mut self, stream: &Stream, max: usize) -> Result, CosError> { + self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) + } + + /// Like [`DecodeBudget::decode_within`], with every filter stage held to `max`, as + /// [`Stream::decoded_within`] holds it, and the stages together to `max` for each of them, the + /// stream's `/DecodeParms` given by reference read in `doc` through a view of it that shares + /// its data and copies no more of its dictionary than its filters and parameters + /// ([`DecodeBudget::decode_with_damage`] reads it so). A + /// stream so reads here when it reads without an allowance and the allowance can pay for + /// what its stages decode, the output of one stage on top of another's. A stage past `max` + /// is an error that charges what the stages before it decoded and `max`, the work it took to + /// find out; past the stages' total it charges that total, and past what is left it spends + /// all of it, as [`DecodeBudget::decode_within`] does. + pub fn decode_stages_within(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result, CosError> { + // Before the parameters are resolved, work the allowance doesn't count. + if self.refuses(stream) { + return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); + } + let stream = doc.decoding_view(stream); + let total = max.saturating_mul(stream.filters().len().max(1)); + let room = total.min(self.bytes); + match stream.decoded_within_each_counted(max, room) { + Ok((data, _, spent)) => { + self.bytes = self.bytes.saturating_sub(spent); + Ok(data) + } + // A stage stopped at `max`, below what was left of the room. + Err((FilterError::LimitExceeded(_), spent)) if room.saturating_sub(spent) > max => { + self.bytes = self.bytes.saturating_sub(spent.saturating_add(max)); + Err(CosError::Filter(FilterError::LimitExceeded(max).to_string())) + } + // At the stages' total, with more left. + Err((FilterError::LimitExceeded(_), _)) if total < self.bytes => { + self.bytes = self.bytes.saturating_sub(total); + Err(CosError::Filter(FilterError::LimitExceeded(total).to_string())) + } + Err((FilterError::LimitExceeded(_), _)) => { + self.bytes = 0; + self.refusals += 1; + Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())) + } + Err((e, spent)) => { + self.bytes = self.bytes.saturating_sub(spent); + Err(CosError::Filter(e.to_string())) + } + } + } + + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), with the stream's + /// `/DecodeParms` given by reference read in `doc`, also saying whether the stream is damaged: + /// `true` when a filter stopped at corrupt data, so the bytes are only what decoded before it. + /// A stream written back from them loses everything after the damage, which other readers may + /// still read. + pub fn decode_with_damage(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { + self.charge(&doc.decoding_view(stream), max, Stream::decoded_within_counted) + } + + /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to + /// the allowance, with its `/DecodeParms` given by reference read in `doc`. + pub fn decode_strict(&mut self, doc: &Document, stream: &Stream) -> Result, CosError> { + let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); + self.charge(&doc.decoding_view(stream), usize::MAX, strict).map(|(data, _)| data) + } + + /// Decodes `stream` within `max` and the allowance, charging what every filter stage + /// decoded: a stage's output counts even when a later stage shrinks it, and a corrupt stream + /// pays for the bytes decoded before the error, so referencing a stream again and again + /// spends the allowance by the work each decode does. + fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result<(Vec, bool), CosError> { + if self.refuses(stream) { + return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); + } + if max < self.bytes { + let result = decode(stream, max); + let spent = match &result { + Ok((_, _, spent)) => *spent, + Err((FilterError::LimitExceeded(_), _)) => max, + Err((_, spent)) => *spent, + }; + self.bytes = self.bytes.saturating_sub(spent); + return result.map(|(data, damaged, _)| (data, damaged)).map_err(|(e, _)| CosError::Filter(e.to_string())); + } + match decode(stream, self.bytes) { + Ok((data, damaged, spent)) => { + self.bytes = self.bytes.saturating_sub(spent); + Ok((data, damaged)) + } + Err((FilterError::LimitExceeded(_), _)) => { + self.bytes = 0; + self.refusals += 1; + Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())) + } + Err((e, spent)) => { + self.bytes = self.bytes.saturating_sub(spent); + Err(CosError::Filter(e.to_string())) + } + } + } + + /// Counts one form entry. `false` once [`DecodeBudget::new`]'s `forms` entries are spent: + /// the caller leaves that form out. + pub fn enter_form(&mut self) -> bool { + match self.forms.checked_sub(1) { + Some(left) => { + self.forms = left; + true + } + None => false, + } + } + + /// `true` the first time `r` is passed to this budget, for callers that need each + /// referenced stream only once. + pub fn first_visit(&mut self, r: ObjRef) -> bool { + self.seen.insert(r) + } + + /// Each stream `contents` names, decoded against the allowance, in order, its `/DecodeParms` + /// given by reference read in `doc`. `contents` is a page's `/Contents`: one stream, or an + /// array whose non-stream entries are left out. A stream that fails to decode, or arrives + /// after the allowance is spent, is an `Err` in its place, so the caller chooses between + /// skipping it and giving up. + pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { + self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() + } + + /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway + /// ([`DecodeBudget::decode_with_damage`], which reads `/DecodeParms` given by reference). + pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { + match &*doc.resolve(contents) { + Object::Stream(s) => vec![self.decode_with_damage(doc, s, usize::MAX)], + Object::Array(list) => list + .iter() + .filter_map(|entry| match &*doc.resolve(entry) { + Object::Stream(s) => Some(self.decode_with_damage(doc, s, usize::MAX)), + _ => None, + }) + .collect(), + _ => Vec::new(), + } + } +} + +#[cfg(test)] +mod image_tests { + use super::*; + use crate::cos::Dict; + + fn jbig2(parms: Option) -> Stream { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(16)); + dict.set(b"Height".to_vec(), Object::Int(2)); + dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + if let Some(parms) = parms { + dict.set(b"DecodeParms".to_vec(), parms); + } + Stream::from_raw(dict, crate::filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) + } + + /// What a Flate stage pays to read `n` bytes, besides what it makes. + fn read(n: usize) -> u64 { + crate::filters::FLATE_READ_WORK * n as u64 + } + + /// Each image spends what it took from one allowance, and those past it are refused and + /// counted, whatever their filters. What can't be decoded here costs nothing, and fails + /// without being one of these. + #[test] + fn the_images_of_an_operation_share_its_allowance() { + let doc = Document::new_empty(); + let image = jbig2(None); + let mut budget = ImageBudget::with_work(u64::MAX); + let samples = budget.decode(&doc, &image, 1 << 20).unwrap(); + let cost = u64::MAX - budget.work_left(); + assert!(cost > 0); + let mut budget = ImageBudget::with_work(3 * cost); + for left in [2, 1, 0] { + assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); + assert_eq!(budget.work_left(), left * cost); + } + assert_eq!(budget.refused(), 0, "the allowance held exactly three"); + for refused in [1, 2] { + assert!(budget.decode(&doc, &image, 1 << 20).is_err()); + assert_eq!(budget.refused(), refused); + } + assert!(budget.decode(&doc, &Stream::flate(Dict::new(), b"samples"), 1 << 20).is_err()); + assert!(budget.decode(&doc, &Stream::from_raw(Dict::new(), b"samples".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.refused(), 4, "other images share it"); + for name in ["NoSuchFilter", "DCTDecode", "JPXDecode"] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name(name)); + assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.refused(), 4, "{name} isn't one of these"); + } + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Int(5)); + assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.refused(), 4, "a /Filter that can't be read isn't either"); + } + + /// What a file of images each inflating a megabyte asks for: 4 KB of Flate inside Flate can ask + /// for a gigabyte, and these aren't JBIG2, so without the allowance nothing would stop them. + #[test] + fn filter_bombs_stop_at_the_allowance() { + let doc = Document::new_empty(); + let samples = vec![0; 1 << 20]; + let inner = crate::filters::encode_flate(&samples); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); + let bomb = Stream::from_raw(dict, crate::filters::encode_flate(&inner)); + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); + let mut budget = ImageBudget::with_work(2 * cost + 1); + for left in [cost + 1, 1] { + assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); + assert_eq!(budget.work_left(), left); + } + assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1), "the first it can't pay for ends it"); + } + + /// Filters in front of a JBIG2 decoder pay as they run, on top of what the decoder takes, and + /// one that fails has still paid: nothing about the JBIG2 data is known by then, and each image + /// of a file can do this. + #[test] + fn filters_in_front_of_jbig2_pay_whether_or_not_they_work() { + let doc = Document::new_empty(); + let alone = jbig2(None); + let mut budget = ImageBudget::with_work(u64::MAX); + let samples = budget.decode(&doc, &alone, 1 << 20).unwrap(); + let decoding = u64::MAX - budget.work_left(); + let behind = |raw: Vec| { + let mut dict = alone.dict.clone(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("JBIG2Decode")])); + Stream::from_raw(dict, raw) + }; + + let image = behind(crate::filters::encode_flate(&alone.raw)); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); + + // Past this image's limit: an error, with what finding out took paid and the rest left. + let bomb = behind(crate::filters::encode_flate(&vec![0; 1 << 16])); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode(&doc, &bomb, 1000).is_err()); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); + assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); + + // A small allowance is a limit too: it stops there, and the image is refused. + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); + assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + } + + /// Globals that are read again for every image pay every time, and fail having paid. Kept + /// ones have no filters left to undo, so they pay for being read and nothing more. + #[test] + fn globals_pay_for_their_filters_each_time_they_are_decoded() { + let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; + let raw = crate::filters::encode_flate(&skipped); + let stream = |doc: &Document, r: ObjRef| match &*doc.get(r) { + Object::Stream(s) => s.clone(), + _ => panic!("not a stream"), + }; + let spent = |doc: &Document, s: &Stream| { + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(doc, s, 1 << 20).unwrap(); + u64::MAX - budget.work_left() + }; + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(dict, raw.clone()))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + let image = doc.add(Object::Stream(jbig2(Some(Object::Dict(parms))))); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Image".to_vec(), Object::Ref(image))).unwrap(); + + // Added in memory, so not kept: the filters are undone for each read. + let s = stream(&doc, image); + let first = spent(&doc, &s); + assert_eq!(spent(&doc, &s), first, "not kept, so paid for again"); + + // Read from a file, so kept: the second read has no filters to undo. + let file = crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap(); + let doc = Document::open(std::sync::Arc::new(file)).unwrap(); + let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); + let s = stream(&doc, image); + let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); + assert_eq!(spent(&doc, &s), first, "the first read undoes them"); + assert_eq!(spent(&doc, &s), first - filters, "kept"); + + // Globals that fail pay too, with the image not refused: these go past their own limit. + let over = crate::filters::encode_flate(&vec![0; crate::cos::MAX_JBIG2_GLOBALS + 1]); + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(dict, over.clone()))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + let image = jbig2(Some(Object::Dict(parms))); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::cos::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); + assert_eq!(budget.refused(), 0); + } + + /// A fax image pays for what comes before its decoder and for the rows it asks for, and past + /// the allowance it is refused like any other. + #[test] + fn fax_images_share_the_allowance() { + let doc = Document::new_empty(); + let (w, h) = (4096u32, 64u32); + let coded = crate::filters::encode_g4(&vec![0xFF; 512 * h as usize], w, h).unwrap(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(i64::from(w))); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(i64::from(w))); + dict.set(b"Height".to_vec(), Object::Int(i64::from(h))); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("CCITTFaxDecode")])); + dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); + let image = Stream::from_raw(dict, crate::filters::encode_flate(&coded)); + let pixels = u64::from(w) * u64::from(h); + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; + let mut budget = ImageBudget::with_work(cost); + assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); + assert_eq!(budget.work_left(), 0, "exactly that"); + assert!(budget.decode(&doc, &image, 1 << 20).is_err()); + assert_eq!(budget.refused(), 1); + } + + /// An image pays for its pixels however few bytes hold them, or a blank page of fax or Flate + /// would cost a run a second for a few bytes' worth of allowance; a JBIG2 image doesn't pay + /// twice, and one this doesn't decode pays nothing. Past what is left it isn't read. + #[test] + fn an_image_pays_for_its_pixels() { + let doc = Document::new_empty(); + let blank = |filter: Option<&str>, data: Vec| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(4096)); + dict.set(b"Height".to_vec(), Object::Int(1024)); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + Stream::from_raw(dict, data) + }; + let rows = vec![0xFFu8; 512 * 1024]; + let flate = blank(Some("FlateDecode"), crate::filters::encode_flate(&rows)); + let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; + let mut budget = ImageBudget::with_work(cost); + assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); + assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); + // Its filters alone pay the same. + let mut budget = ImageBudget::with_work(cost); + assert_eq!(budget.decode_filters(&doc, &flate, 1 << 24).unwrap(), rows); + assert_eq!(budget.work_left(), 0); + // Short of its pixels it is refused, having paid for its filters. + let mut budget = ImageBudget::with_work(4096 * 1024 - 1); + assert!(budget.decode(&doc, &flate, 1 << 24).unwrap_err().to_string().contains("more image decoding than")); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + let mut budget = ImageBudget::with_work(4096 * 1024 - 1); + assert!(budget.decode_filters(&doc, &flate, 1 << 24).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + // What isn't decoded here pays for nothing, and fails as it would. + let mut budget = ImageBudget::with_work(10); + assert!(budget.decode(&doc, &blank(Some("DCTDecode"), b"data".to_vec()), 1 << 24).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (10, 0)); + // Unfiltered data pays too, and a size that isn't one is no size. + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &blank(None, rows.clone()), 1 << 24).unwrap(); + assert_eq!(u64::MAX - budget.work_left(), rows.len() as u64 + 4096 * 1024); + let mut odd = Dict::new(); + odd.set(b"Width".to_vec(), Object::Int(-4)); + odd.set(b"Height".to_vec(), Object::Int(9)); + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &Stream::from_raw(odd, vec![1, 2]), 1 << 24).unwrap(); + assert_eq!(u64::MAX - budget.work_left(), 2); + // A JPEG pays twice over for each sample, and four times for each byte it is read from. + let mut budget = ImageBudget::with_work(30); + assert!(budget.pay_jpeg(9, b"ab")); + assert_eq!((budget.work_left(), budget.refused()), (4, 0)); + assert!(!budget.pay_jpeg(0, b"ab")); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + } + + /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays + /// little for it: a scan-like picture stored in under half its size costs no more than half + /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so + /// a letterhead on hundreds of pages has to fit about as often as it did unpriced (here, at + /// most half again the cost). + #[test] + fn an_image_an_encoder_made_pays_little_for_its_reading() { + let doc = Document::new_empty(); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let samples: Vec = (0..256 * 256usize) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + ((i % 256) as u8 / 16) ^ (seed as u8 & 3) + }) + .collect(); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + let image = Stream::flate(dict, &samples); + assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); + let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &image, 1 << 20).unwrap(); + let paid = u64::MAX - budget.work_left(); + assert_eq!(paid, unpriced + read(image.raw.len())); + assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); + } + + /// An image pays for the pixels its data holds, not those its size declares: one claiming + /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the + /// images after it are decoded as they would be without it. + #[test] + fn an_image_pays_for_no_more_pixels_than_its_data_holds() { + let doc = Document::new_empty(); + let sized = |side: i64, data: &[u8]| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(side)); + dict.set(b"Height".to_vec(), Object::Int(side)); + Stream::flate(dict, data) + }; + let claims = sized(131_072, &[0; 16]); + let image = sized(64, &[7; 64 * 64]); + let cost = |s: &Stream| { + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, s, 1 << 24).unwrap(); + u64::MAX - budget.work_left() + }; + assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); + assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); + for decode in [ImageBudget::decode, ImageBudget::decode_filters] { + let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); + assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); + assert_eq!(decode(&mut budget, &doc, &image, 1 << 24).unwrap(), [7; 64 * 64], "the image after it is decoded"); + assert_eq!((budget.work_left(), budget.refused()), (0, 0)); + } + } + + /// A stream a run rewrites is read strictly and charged, and what the caller does with it after + /// comes out of the same allowance: what can't be paid for spends all of it. + #[test] + fn a_rewritten_stream_shares_the_allowance() { + use crate::filters::{Filter, FilterError, Params}; + let samples = vec![7u8; 1000]; + let raw = crate::filters::encode_flate(&samples); + let chain = [(Filter::Flate, Params::default())]; + let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); + let mut budget = ImageBudget::with_work(cost + 10); + assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); + assert_eq!(budget.work_left(), 10); + assert!(budget.spend(4)); + assert_eq!(budget.work_left(), 6); + assert!(budget.spend(6), "exactly what is left"); + assert_eq!(budget.work_left(), 0); + assert!(budget.spend(0)); + let mut budget = ImageBudget::with_work(5); + assert!(!budget.spend(6)); + assert_eq!(budget.work_left(), 0, "spends all of it"); + assert!(matches!(budget.decode_chain(&chain, &raw, 1 << 20), Err(FilterError::Spent))); + let cut = raw.get(..raw.len() / 2).unwrap(); + assert!(matches!(ImageBudget::with_work(u64::MAX).decode_chain(&chain, cut, 1 << 20), Err(FilterError::Corrupt { .. })), "strictly"); + } + + /// An image read through its filters alone reads parameters given by reference, is charged, and + /// is refused like the others. It isn't given to a decoder. + #[test] + fn filters_alone_are_charged_too() { + let mut doc = Document::new_empty(); + let p = crate::filters::Params { predictor: 12, columns: 4, ..crate::filters::Params::default() }; + let rows = vec![7u8; 400]; + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(4)); + let parms = doc.add(Object::Dict(parms)); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + let raw = crate::filters::encode(&crate::filters::Filter::Flate, &p, &rows).unwrap(); + let image = Stream::from_raw(dict, raw); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); + let cost = u64::MAX - budget.work_left(); + // The tag byte of each of its 100 rows counts, which the predictor takes off. + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); + let mut budget = ImageBudget::with_work(cost - 1); + assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); + assert_eq!((budget.work_left(), budget.refused()), (0, 1)); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode_filters(&doc, &jbig2(None), 1 << 20).is_err()); + assert_eq!(budget.work_left(), u64::MAX, "no decoder here, so nothing was read"); + // Nor for a fax image, nor for a decoder that isn't last, however large they say they are. + for chain in [vec!["CCITTFaxDecode"], vec!["CCITTFaxDecode", "FlateDecode"], vec!["JBIG2Decode", "FlateDecode"]] { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(1000)); + dict.set(b"Height".to_vec(), Object::Int(1000)); + dict.set(b"Filter".to_vec(), Object::Array(chain.iter().map(|f| Object::name(f)).collect())); + let image = Stream::from_raw(dict, b"data".to_vec()); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); + assert_eq!(budget.work_left(), u64::MAX, "{chain:?}"); + } + // Through `decode` a decoder that isn't last pays nothing either; one that is does. + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(1000)); + dict.set(b"Height".to_vec(), Object::Int(1000)); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("JBIG2Decode"), Object::name("FlateDecode")])); + let mut budget = ImageBudget::with_work(u64::MAX); + assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); + assert_eq!(budget.work_left(), u64::MAX); + } + + /// PDFBOX-3932: an image and soft mask declared `/Predictor 15` that their writer never + /// applied, so their rows carry no PNG filter-type byte. Exported, they were "shorter than + /// their size says"; they are read as the samples they are, and tagged rows as before. + #[test] + fn an_image_whose_rows_carry_no_png_tags_decodes_to_its_samples() { + let doc = Document::new_empty(); + let samples: Vec = (0..36u8).map(|i| 200 - i).collect(); + let image = |height: Option, raw: Vec| { + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(15)); + parms.set(b"Colors".to_vec(), Object::Int(3)); + parms.set(b"Columns".to_vec(), Object::Int(3)); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(3)); + if let Some(h) = height { + dict.set(b"Height".to_vec(), Object::Int(h)); + } + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + Stream::from_raw(dict, raw) + }; + let untagged = crate::filters::encode_flate(&samples); + let mut budget = ImageBudget::with_work(u64::MAX); + assert_eq!(budget.decode(&doc, &image(Some(4), untagged.clone()), 1 << 20).unwrap(), samples); + // Without a height to tell, the predictor is undone, as before. + assert_ne!(budget.decode(&doc, &image(None, untagged.clone()), 1 << 20).ok(), Some(samples.clone())); + assert_ne!(budget.decode(&doc, &image(Some(5), untagged), 1 << 20).ok(), Some(samples.clone())); + let p = crate::filters::Params { predictor: 15, colors: 3, columns: 3, ..crate::filters::Params::default() }; + let tagged = crate::filters::encode(&crate::filters::Filter::Flate, &p, &samples).unwrap(); + assert_eq!(budget.decode(&doc, &image(Some(4), tagged), 1 << 20).unwrap(), samples); + } + + /// The images of a file share its globals, so an image refused doesn't read them: this one's + /// can't be read, which is how to tell. + #[test] + fn an_image_past_the_allowance_does_not_read_its_globals() { + let mut doc = Document::new_empty(); + let mut unreadable = Dict::new(); + unreadable.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let globals = doc.add(Object::Stream(Stream::from_raw(unreadable, b"data".to_vec()))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + let image = jbig2(Some(Object::Dict(parms))); + let read = ImageBudget::with_work(u64::MAX).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); + assert!(read.contains("globals"), "{read}"); + let refused = ImageBudget::with_work(0).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); + assert!(refused.contains("more image decoding than"), "{refused}"); + } + + /// A longer file gets more, by its length, and an empty one the fixed part. + #[test] + fn a_longer_file_allows_more_decoding() { + assert_eq!(ImageBudget::new(&Document::new_empty()).work_left(), IMAGE_WORK); + let pdf = b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF".to_vec(); + let len = pdf.len() as u64; + let doc = Document::open(std::sync::Arc::new(pdf)).unwrap(); + assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + len * IMAGE_WORK_PER_BYTE); + } + + /// A document made in memory, as Combine, Create and inserted scans make one, gets for the + /// streams it was given what it would get for them once saved: what isn't a stream adds nothing. + #[test] + fn streams_made_in_memory_allow_more_decoding_too() { + let mut doc = Document::new_empty(); + doc.add(Object::Stream(Stream::from_raw(Dict::new(), vec![0; 3000]))); + doc.add(Object::Dict(Dict::new())); + doc.add(Object::Stream(Stream::flate(Dict::new(), &[1; 100]))); + let flate = crate::filters::encode_flate(&[1; 100]).len() as u64; + assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + (3000 + flate) * IMAGE_WORK_PER_BYTE); + // A stream of an opened file changed in memory counts with the file. + let pdf = b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF".to_vec(); + let len = pdf.len() as u64; + let mut doc = Document::open(std::sync::Arc::new(pdf)).unwrap(); + doc.add(Object::Stream(Stream::from_raw(Dict::new(), vec![0; 500]))); + assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + (len + 500) * IMAGE_WORK_PER_BYTE); + } + + // What the constants' comments promise, for a page scanned at 300 dpi: about 1,000 of them + // whatever the file's size, and a page paying for itself with 2 KB of the file. + const PAGE: u64 = 17_000_000; + const _: () = assert!(IMAGE_WORK >= 1_000 * PAGE); + const _: () = assert!(IMAGE_WORK_PER_BYTE * 2_100 >= PAGE); + // And that a stream can be decoded about 8 times at Flate's best, 1,032 to one, and the byte in. + const _: () = assert!(IMAGE_WORK_PER_BYTE >= 7 * 1_033); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cos::Dict; + + #[test] + fn repeated_references_share_one_allowance() { + let mut doc = Document::new_empty(); + let bomb = doc.add(Object::Stream(Stream::flate(Dict::new(), &vec![b' '; 1 << 20]))); + let contents = Object::Array(vec![Object::Ref(bomb); 64]); + let mut budget = DecodeBudget::new(10 << 20, 1); + let decoded = budget.decode_contents(&doc, &contents); + assert_eq!(decoded.len(), 64); + assert_eq!(decoded.iter().filter(|d| d.is_ok()).count(), 10); + let total: usize = decoded.iter().flatten().map(Vec::len).sum(); + assert_eq!(total, 10 << 20); + assert!(budget.exhausted()); + // A decode past the allowance spends it, so a bomb is inflated at most once more. + let mut budget = DecodeBudget::new((1 << 20) - 1, 1); + assert!(budget.decode_contents(&doc, &contents).iter().all(Result::is_err)); + assert_eq!(budget.remaining(), 0); + let mut budget = DecodeBudget::new((1 << 20) - 1, 1); + let Object::Stream(stream) = &*doc.get(bomb) else { panic!("not a stream") }; + assert!(budget.decode_strict(&doc, stream).is_err()); + assert_eq!(budget.remaining(), 0); + + let mut budget = DecodeBudget::new(10 << 20, 1); + assert!(budget.decode_within(stream, 1000).is_err()); + assert_eq!(budget.remaining(), (10 << 20) - 1000); + assert_eq!(budget.decode_within(stream, 2 << 20).map(|d| d.len()).ok(), Some(1 << 20)); + assert_eq!(budget.remaining(), (9 << 20) - 1000); + + assert!(budget.enter_form()); + assert!(!budget.enter_form()); + assert!(budget.first_visit(bomb)); + assert!(!budget.first_visit(bomb)); + } + + /// A corrupt stream fails a strict decode after decoding part of it: each reference pays + /// for that part, so references to it can't decode without limit. + #[test] + fn failed_strict_decodes_charge_what_they_decoded() { + let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); + let cut = whole.raw.len() / 2; + let corrupt = Stream::from_raw(whole.dict.clone(), whole.raw.get(..cut).unwrap_or_default().to_vec()); + let partial = corrupt.decoded().map(|d| d.len()).unwrap_or(0); + assert!(partial > 0, "a cut flate stream decodes in part"); + let doc = Document::new_empty(); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert!(budget.decode_strict(&doc, &corrupt).is_err()); + assert_eq!(budget.remaining(), (10 << 20) - partial); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert!(budget.decode_within(&corrupt, 1 << 30).is_ok()); + assert_eq!(budget.remaining(), (10 << 20) - partial); + } + + /// A stream damaged partway decodes to what came before the damage, and says so, so that code + /// writing it back can leave it as it is. What every reader takes is no damage: a missing + /// checksum, no filter at all. + #[test] + fn a_stream_damaged_partway_says_so() { + let doc = Document::new_empty(); + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(&doc, s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); + let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); + let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); + assert!(partial > 0, "a cut flate stream decodes in part"); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &cut, usize::MAX), Some((partial, true))); + assert_eq!(budget.remaining(), (10 << 20) - partial); + // Within a bound of its own, under what is left. + assert_eq!(read(&mut budget, &cut, 4 << 20), Some((partial, true))); + assert_eq!(budget.decode(&cut).map(|d| d.len()).ok(), Some(partial)); + + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &whole, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &whole, 4 << 20), Some((1 << 20, false))); + let unchecked = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() - 4).unwrap_or_default().to_vec()); + assert_eq!(read(&mut budget, &unchecked, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &Stream::from_raw(Dict::new(), b"0 0 m".to_vec()), usize::MAX), Some((5, false))); + // A later stage's damage is the stream's. + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); + let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); + assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + // So is an earlier stage's, though the stage after it reads all it was given: a stored + // block of whole hex, then a block of the reserved type (RFC 1951 §3.2.3). + let readable = b"4142>"; + let len = u16::try_from(readable.len()).unwrap_or_default(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + let flate_damaged = Stream::from_raw(hex.dict.clone(), raw); + assert_eq!(budget.decode_with_damage(&doc, &flate_damaged, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + + /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a + /// predicted stream decodes as with them direct. Of the stream, decoding copies no more than + /// its filters and their parameters. + #[test] + fn parameters_given_by_reference_are_read() { + let content = b"q 10 0 0 10 0 0 cm /Im Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let raw = Stream::flate(Dict::new(), &rows).raw; + let mut doc = Document::new_empty(); + let direct: Dict = [(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect(); + let by_reference = Object::Ref(doc.add(Object::Dict(direct.clone()))); + let predictor = doc.add(Object::Int(12)); + let inner = Object::Dict([(b"Predictor".to_vec(), Object::Ref(predictor))].into_iter().collect()); + for parms in [Object::Dict(direct), by_reference, inner] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), parms.clone()); + dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); + let s = Stream::from_raw(dict, raw.to_vec()); + assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_strict(&doc, &s).ok(), Some(content.to_vec()), "{parms:?}"); + let r = doc.add(Object::Stream(s.clone())); + let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); + assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_contents(&doc, &Object::Ref(r)), [Ok(content.to_vec())], "{parms:?}"); + let view = doc.decoding_view(&s); + assert!(matches!(view, std::borrow::Cow::Borrowed(_)) || view.dict.len() == 2, "{:?}", view.dict); + } + } + + /// A page's content streams each say whether they are damaged, in their places. + #[test] + fn a_pages_streams_each_say_whether_they_are_damaged() { + let mut doc = Document::new_empty(); + let whole = doc.add(Object::Stream(Stream::flate(Dict::new(), b"0 0 m 1 1 l S"))); + let Object::Stream(s) = &*doc.get(whole) else { panic!("not a stream") }; + let cut = Stream::from_raw(s.dict.clone(), s.raw.get(..s.raw.len() - 6).unwrap_or_default().to_vec()); + let cut = doc.add(Object::Stream(cut)); + let contents = Object::Array(vec![Object::Ref(whole), Object::Null, Object::Ref(cut)]); + let mut budget = DecodeBudget::page(); + let read: Vec<_> = budget.decode_contents_with_damage(&doc, &contents).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(read, [Some(false), Some(true)]); + let one: Vec<_> = + budget.decode_contents_with_damage(&doc, &Object::Ref(cut)).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(one, [Some(true)]); + assert_eq!(budget.decode_contents(&doc, &contents).iter().filter(|r| r.is_ok()).count(), 2, "read up to the damage"); + } + + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, + /// gives nothing back, and each reference still pays for what it decoded. + #[test] + fn a_decode_that_gives_nothing_back_still_pays() { + let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; + data.extend(vec![0u8; 0x0601]); + data.push(0x07); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let stream = Stream::from_raw(dict, data); + let mut budget = DecodeBudget::new(10 * 0x0601, 1); + assert!(budget.decode(&stream).unwrap().is_empty()); + assert_eq!(budget.remaining(), 9 * 0x0601); + } + + /// A chain whose later stage shrinks the data pays for every stage's output: a Flate stage + /// that inflates to hex text, which ASCIIHex then halves, charges both. + #[test] + fn decodes_charge_every_stage_of_a_chain() { + let hex: Vec = std::iter::repeat_n(b"ab".as_slice(), 1 << 19).flatten().copied().collect(); + let flate = Stream::flate(Dict::new(), &hex); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); + let chained = Stream::from_raw(dict, flate.raw.to_vec()); + assert_eq!(chained.decoded().map(|d| d.len()).ok(), Some(1 << 19)); + let charged = (1 << 20) + (1 << 19); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(budget.decode(&chained).map(|d| d.len()).ok(), Some(1 << 19)); + assert_eq!(budget.remaining(), (10 << 20) - charged); + assert!(budget.decode_within(&chained, 2 << 20).is_ok()); + assert_eq!(budget.remaining(), (10 << 20) - 2 * charged); + assert!(budget.decode_strict(&Document::new_empty(), &chained).is_ok()); + assert_eq!(budget.remaining(), (10 << 20) - 3 * charged); + } + + /// `decode_stages_within` holds every stage to the limit, as a decode without an allowance + /// does, and the stages together to the limit for each of them. A stream behind ASCIIHex and + /// Flate whose stages each stay within it reads, the hex stage's output charged on top; one + /// whose Flate stage would make more stops there, charged what the hex stage made and the + /// limit, the rest of the allowance left; a Flate stage making hex text past the limit leaves + /// the stream unread though the total has room, as `Stream::decoded_within` leaves it; and a + /// stream past what is left spends all of it. + #[test] + fn each_stage_is_held_to_the_limit() { + let chained = |data: &[u8], filters: [&str; 2]| { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(filters.into_iter().map(Object::name).collect())); + Stream::from_raw(dict, data.to_vec()) + }; + let hexed = |data: &[u8]| { + let flate = Stream::flate(Dict::new(), data); + let hex: Vec = flate.raw.iter().flat_map(|b| format!("{b:02X}").into_bytes()).chain(*b">").collect(); + (chained(&hex, ["ASCIIHexDecode", "FlateDecode"]), flate.raw.len()) + }; + let max = 1 << 16; + let (fits, inner) = hexed(&vec![0u8; max]); + let mut budget = DecodeBudget::new(1 << 20, 0); + assert_eq!(budget.decode_stages_within(&Document::new_empty(), &fits, max).map(|d| d.len()).ok(), Some(max)); + assert_eq!(budget.remaining(), (1 << 20) - inner - max); + let (bomb, inner) = hexed(&vec![0u8; 8 * max]); + let mut budget = DecodeBudget::new(1 << 20, 0); + assert!(budget.decode_stages_within(&Document::new_empty(), &bomb, max).is_err()); + assert_eq!(budget.remaining(), (1 << 20) - inner - max, "the Flate stage stopped at the limit"); + // Flate making 1.2 times the limit of hex text, which ASCIIHex halves: the stages together + // make 1.8 times the limit, within twice it, but the Flate stage alone is past it. + let text = "00".repeat(max * 6 / 10).into_bytes(); + let wide = chained(&Stream::flate(Dict::new(), &text).raw, ["FlateDecode", "ASCIIHexDecode"]); + assert!(wide.decoded_within(max).is_err()); + assert!(DecodeBudget::new(1 << 20, 0).decode_stages_within(&Document::new_empty(), &wide, max).is_err()); + let mut budget = DecodeBudget::new(max, 0); + assert!(budget.decode_stages_within(&Document::new_empty(), &fits, max).is_err()); + assert_eq!(budget.remaining(), 0); + } + + /// What `decode_stages_within` charges, and which limit it reports, on each way a read stops: + /// a stage at the stream's limit with more of the room left charges what came before and the + /// limit; at the stages' total, with more of the allowance left, it charges that total; at + /// what is left of the allowance it spends all of it and reports the allowance's limit, as + /// `decode_within` does. Each boundary between them is read on both sides. + #[test] + fn a_stage_past_its_limit_charges_what_came_before_and_the_limit() { + let max = 1 << 12; + let hex = |data: &[u8]| data.iter().flat_map(|b| format!("{b:02X}").into_bytes()).chain(*b">").collect::>(); + let two = |raw: &[u8]| { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("FlateDecode")])); + Stream::from_raw(dict, hex(raw)) + }; + let says = |read: Result, CosError>, limit: usize| { + read.err().map(|e| e.to_string()) == Some(CosError::Filter(FilterError::LimitExceeded(limit).to_string()).to_string()) + }; + // Flate data making eight times the limit: the hex stage makes `inner` bytes of it, and + // the Flate stage stops at the limit. + let bomb = Stream::flate(Dict::new(), &vec![0u8; 8 * max]).raw.to_vec(); + let inner = bomb.len(); + // A byte more of the room than the hex stage and the limit: the Flate stage stops at the + // limit, so that is what it charges. + let mut budget = DecodeBudget::new(inner + max + 1, 0); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&bomb), max), max)); + assert_eq!(budget.remaining(), 1); + // Exactly the hex stage and the limit: what was left stopped it as much as the limit did, + // so it is past the allowance, spends it and reports the allowance's limit. + let mut budget = DecodeBudget::new(inner + max, 0); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&bomb), max), inner + max)); + assert_eq!(budget.remaining(), 0); + // The same data padded to exactly the limit: the hex stage makes all of it, and the Flate + // stage, at the limit, reaches the stages' total. With more of the allowance left, that + // total is what it charges and reports. + let mut padded = bomb.clone(); + padded.resize(max, 0); + let mut budget = DecodeBudget::new(10 * max, 0); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), 2 * max)); + assert_eq!(budget.remaining(), 8 * max); + // With exactly the total left (of an allowance of three times the limit), it is past the + // allowance: all of it is spent, and the allowance's limit reported. + let mut budget = DecodeBudget::new(3 * max, 0); + budget.decode(&Stream::from_raw(Dict::new(), vec![0; max])).unwrap(); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), 3 * max)); + assert_eq!(budget.remaining(), 0); + // With less than the total left, the same. + let mut budget = DecodeBudget::new(max, 0); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), max)); + assert_eq!(budget.remaining(), 0); + // With nothing left, a stream with data is refused before it is decoded, even one that + // decodes to nothing; an empty one reads. + let mut budget = DecodeBudget::new(0, 0); + assert!(says(budget.decode_stages_within(&Document::new_empty(), &Stream::flate(Dict::new(), b""), max), 0)); + assert_eq!(budget.decode_stages_within(&Document::new_empty(), &Stream::from_raw(Dict::new(), Vec::new()), max).ok(), Some(Vec::new())); + } +} diff --git a/crates/pdf/src/cos/bytes.rs b/crates/pdf/src/cos/bytes.rs new file mode 100644 index 000000000..f6ad1b8e8 --- /dev/null +++ b/crates/pdf/src/cos/bytes.rs @@ -0,0 +1,98 @@ +//! Stream data that can share a larger buffer instead of copying out of it. + +use std::ops::{Deref, Range}; +use std::sync::Arc; + +/// A stream's bytes: a buffer of their own, or a range of a larger shared buffer. Streams read +/// from a file point into the file's bytes, which the document keeps anyway, so caching a +/// parsed stream costs its dictionary and not a second copy of its data. +#[derive(Clone, Default)] +pub struct Bytes { + buf: Arc>, + range: Range, +} + +impl Bytes { + /// `range` of `buf`, without copying; `None` when the range is not within `buf`. + pub(crate) fn view(buf: &Arc>, range: Range) -> Option { + (range.start <= range.end && range.end <= buf.len()).then(|| Self { buf: buf.clone(), range }) + } + + /// Whether these bytes keep alive a larger buffer than themselves that isn't `owner`. + pub(crate) fn borrows_other_than(&self, owner: &Arc>) -> bool { + self.range.len() < self.buf.len() && !Arc::ptr_eq(&self.buf, owner) + } + + /// Whether these bytes are (a view into) `buf`, holding no memory of their own. + pub(crate) fn shares(&self, buf: &Arc>) -> bool { + Arc::ptr_eq(&self.buf, buf) + } + + /// The same bytes, in a buffer of their own. + pub(crate) fn detached(&self) -> Self { + Self::from(self.to_vec()) + } +} + +impl From> for Bytes { + fn from(v: Vec) -> Self { + let range = 0..v.len(); + Self { buf: Arc::new(v), range } + } +} + +impl From>> for Bytes { + fn from(buf: Arc>) -> Self { + let range = 0..buf.len(); + Self { buf, range } + } +} + +impl Deref for Bytes { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + // `view` and `From` only build in-bounds ranges; an empty slice keeps this panic-free. + self.buf.get(self.range.clone()).unwrap_or_default() + } +} + +impl AsRef<[u8]> for Bytes { + fn as_ref(&self) -> &[u8] { + self + } +} + +impl PartialEq for Bytes { + fn eq(&self, other: &Self) -> bool { + (Arc::ptr_eq(&self.buf, &other.buf) && self.range == other.range) || **self == **other + } +} + +impl Eq for Bytes {} + +impl std::fmt::Debug for Bytes { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + (**self).fmt(f) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn views_share_their_buffer_and_compare_by_content() { + let file = Arc::new(b"0123456789".to_vec()); + let view = Bytes::view(&file, 2..5).expect("in range"); + assert_eq!(&*view, b"234"); + assert_eq!(view.as_ptr(), file[2..].as_ptr(), "no copy"); + assert_eq!(view, Bytes::from(b"234".to_vec())); + assert!(Bytes::view(&file, 5..11).is_none() && Bytes::view(&file, Range { start: 6, end: 5 }).is_none()); + let other = Arc::new(Vec::new()); + assert!(view.borrows_other_than(&other) && !view.borrows_other_than(&file)); + let own = view.detached(); + assert!(!own.borrows_other_than(&other) && own == view && own.as_ptr() != view.as_ptr()); + assert!(!Bytes::from(file.clone()).borrows_other_than(&other), "a whole buffer is its own"); + } +} diff --git a/crates/pdf/src/cos/document.rs b/crates/pdf/src/cos/document.rs new file mode 100644 index 000000000..6082f4165 --- /dev/null +++ b/crates/pdf/src/cos/document.rs @@ -0,0 +1,98 @@ +//! Document: cross-reference data, lazy object loading, and a copy-on-write overlay of edits. +//! +//! `Document` is cheap to clone (the original bytes, xref index and parse caches are shared), so +//! a clone is a snapshot: undo/redo and background jobs hold clones while the UI edits another. +//! Edits never touch the original bytes; the writer appends them (incremental save) or rewrites +//! the file (full save). + +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::{Arc, Mutex, Weak}; + +use crate::cos::CosError; +use crate::cos::object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream}; +use crate::cos::parser::{EndstreamIndex, Lexer, is_whitespace, parse_indirect, parse_indirect_shared}; + +mod objects; +mod open; +mod reader; +mod state; +mod xref; + +pub use objects::DECODE_PARMS_KEYS; +pub(crate) use objects::Held; +#[cfg(test)] +pub(crate) use reader::find; +#[cfg(test)] +use reader::rfind; +use state::{ObjStm, Slot, Unreadable}; +pub(crate) use state::{ObjectReader, held_bytes}; +pub use state::{ReadState, SourceIdentity}; +#[cfg(test)] +use state::{SECTION_READS, objstm_member}; +use xref::XrefIndex; +pub use xref::{Revision, XrefEntry}; + +#[derive(Clone)] +pub struct Document { + data: Arc>, + /// Where `endstream` occurs in `data`; see [`EndstreamIndex`]. + endstreams: Arc, + source_identity: Arc<()>, + entries: Arc, + trailer: Dict, + revisions: Arc>, + /// See [`Self::xref_oddities`]. + xref_oddities: Arc>, + /// Where the file's object headers are, as (offset, generation) of the last header of each + /// number. Filled only when the cross-reference data proved inconsistent; see + /// `fill_xref_gaps`. + located: Arc>, + repair_log: Arc>, + /// See [`Self::rebuilt_unlinked_update`]. + unlinked_update: bool, + /// What loading each file object gave, a failure as well: a damaged object is otherwise + /// loaded again on every read, and a member of an object stream that can't be decoded + /// decodes the stream again (`Document::load`). + cache: Arc>, + objstms: Arc>>>, + /// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the + /// source, and is reset wherever they are. + unreadable: Arc>, + /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan + /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). + jbig2_globals: Arc>>>>, + stream_limit: usize, + overlay: BTreeMap, + next_num: u32, + /// Position of the `%PDF-` header (some files have junk before it). + header_offset: usize, + version: String, + /// The authenticated security handler of an encrypted document. + security: Option>, + /// Object number of the `/Encrypt` dictionary (never encrypted itself). + encrypt_num: Option, + /// What that dictionary and `/ID` refer to, which the key comes from, and so is as stored. + encrypt_members: Arc>, + /// Object numbers a scan of the file didn't find: the file doesn't change, so it never will. + unfound: Arc>>, + /// Encryption was added, changed or removed since opening: only a full save can apply it. + encryption_changed: bool, + /// Set by edits that must not leave earlier revisions in the file (redaction). + full_save: bool, + /// The next full save also gives the file a new identifier (`/ID`), see + /// [`Document::require_full_save_with_new_id`]. + new_id: bool, + /// The handler and `/Encrypt` object number that saves use, when encryption changed. + out_security: Option>, + out_encrypt_num: Option, +} + +impl std::fmt::Debug for Document { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Document").field("bytes", &self.data.len()).field("xref", &self.entries).field("edits", &self.overlay.len()).finish() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/pdf/src/cos/document/objects.rs b/crates/pdf/src/cos/document/objects.rs new file mode 100644 index 000000000..2b8df8eb1 --- /dev/null +++ b/crates/pdf/src/cos/document/objects.rs @@ -0,0 +1,1121 @@ +use std::collections::HashSet; + +use super::reader::find; +use super::state::{ + LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadFrame, ReadState, ScanLengths, Slot, SourceIdentity, loaded_bytes, loading, note_overlay_read, + note_read, objstm_member, overlay_reads, +}; +use super::xref::{Revision, XrefEntry, XrefIndex}; +use super::{ + Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, +}; + +/// The most decoded `/JBIG2Globals` a document keeps for its images to share (see +/// `Document::jbig2_globals`): two streams at [`MAX_JBIG2_GLOBALS`]. +const JBIG2_GLOBALS_HELD: usize = 2 * MAX_JBIG2_GLOBALS; + +impl Document { + pub fn bytes(&self) -> &Arc> { + &self.data + } + + /// Identity of the original bytes, xref and authenticated decode context. Edits and + /// clones preserve it; reopening (including after any save) produces a new identity. + pub fn source_identity(&self) -> SourceIdentity { + SourceIdentity(Arc::downgrade(&self.source_identity)) + } + + /// What the document reads as now (see [`ReadState`]): unlike [`Self::source_identity`], it + /// changes with every edit, so a value derived from the document can be cached against it. + pub fn read_state(&self) -> ReadState { + ReadState { source: self.source_identity(), trailer: self.trailer.clone(), overlay: self.overlay.clone() } + } + + pub fn version(&self) -> &str { + &self.version + } + + /// Set the PDF header version (`"1.4"`, `"1.7"`, `"2.0"`, …) written by a full save. + pub fn set_version(&mut self, version: impl Into) { + self.version = version.into(); + } + + pub fn trailer(&self) -> &Dict { + &self.trailer + } + + pub fn trailer_mut(&mut self) -> &mut Dict { + &mut self.trailer + } + + pub fn revisions(&self) -> &[Revision] { + &self.revisions + } + + /// Position of the `%PDF-` header in the file: 0 unless bytes come before it. + pub fn header_offset(&self) -> usize { + self.header_offset + } + + /// There is no cross-reference chain to append to: the index was rebuilt from object + /// headers because the file's own was unusable (see [`Self::repair_log`]), or the document + /// was built from scratch ([`Self::new_empty`]). A save writes such a document in full, even + /// when nothing was edited: for a repaired file that is what puts the repair on disk. + pub fn reconstructed(&self) -> bool { + self.revisions.is_empty() + } + + /// Where each revision of the file ends (oldest first): the byte after the `%%EOF` (and its + /// end-of-line) that closes its cross-reference section. `data[..end]` is that revision as it + /// was saved. A linearized file's first-page section belongs to the revision it precedes, so + /// ends only ever increase. Empty for a reconstructed file. + pub fn revision_ends(&self) -> Vec { + let d = &self.data[..]; + let mut ends: Vec = Vec::with_capacity(self.revisions.len()); + for r in self.revisions.iter() { + let from = (r.xref_offset as usize).saturating_add(self.header_offset).min(d.len()); + let Some(at) = d[from..].windows(5).position(|w| w == b"%%EOF") else { continue }; + let mut end = from + at + 5; + while end < d.len() && matches!(d[end], b'\r' | b'\n') && end - (from + at + 5) < 2 { + end += 1; + } + match ends.last_mut() { + Some(last) if end <= *last => {} + _ => ends.push(end), + } + } + ends + } + + pub fn repair_log(&self) -> &[String] { + &self.repair_log + } + + /// Whether the file was rebuilt from its object headers because its last update has no `/Prev` + /// and so cut its page tree off the cross-reference chain (ISO 32000-2 §7.5.6, §7.5.8.2). The + /// renderer rebuilds such a file under the same conditions, so the two read the same objects. + pub fn rebuilt_unlinked_update(&self) -> bool { + self.unlinked_update + } + + /// Objects of the file that could not be read, lowest number first, as lines such as + /// "object 11 0 could not be read: syntax error at byte 512: ...". [`Self::get`] reads each + /// as null (ISO 32000-2 §7.3.10), so a document with one opens, but what it held is lost: + /// unlike [`Self::repair_log`], nothing was repaired. Lists the objects read so far, by this + /// document, its clones or a scan of it, that have not since been edited; past 100, a last + /// line says there were more. Opening again, authenticating or rebuilding the index starts + /// the list afresh, as it does the object cache. + pub fn unreadable_objects(&self) -> Vec { + let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() }; + unreadable.lines(|num| self.overlay.contains_key(&num)) + } + + /// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file + /// whose broken object is asked for again and again pays for it once. + fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { + // A poisoned lock only means the object goes unlisted; the read itself still fails. + let Ok(mut unreadable) = self.unreadable.lock() else { return }; + let hidden = |n: u32| self.overlay.contains_key(&n); + if unreadable.wants(num, hidden) { + unreadable.note(num, self.generation(num), &reason(), hidden); + } + } + + /// Why an object listed at `offset` could not be read: what parsing it there, without resolving + /// an indirect `/Length`, found. A best account, not the only one: the reader also tried + /// elsewhere. + fn why_unreadable(&self, offset: usize) -> String { + match parse_indirect_shared(&self.data, offset, &|_| None, &self.endstreams) { + Err(e) => e.to_string(), + Ok(_) => "no readable copy was found at its cross-reference offset or elsewhere in the file".into(), + } + } + + /// Add a line to [`Self::repair_log`]: a leniency applied while reading that the user should + /// know about (fidelity: never silent). + pub fn note_repair(&mut self, line: String) { + let mut log = self.repair_log.as_ref().clone(); + log.push(line); + self.repair_log = Arc::new(log); + } + + /// `true` when there are unsaved edits. Removing an `/Encrypt` dictionary written inline in + /// the trailer (as MuPDF writes it) changes no object, and is an edit all the same. + pub fn is_modified(&self) -> bool { + !self.overlay.is_empty() || self.encryption_changed + } + + /// Object numbers changed since the document was opened (or last saved). + pub fn modified_objects(&self) -> Vec { + self.overlay.keys().copied().collect() + } + + /// Where object `num` is stored in the file (ignoring unsaved edits). + pub fn xref_entry(&self, num: u32) -> Option { + self.entries.lookup(num, &self.data) + } + + /// Cross-reference entries this reader did not take at face value, as (byte offset of the + /// section, object number): an in-use entry at offset 0, a second and different entry for + /// one object in the same section, a reserved cross-reference stream entry type, or a + /// hybrid /XRefStm entry that disagrees with the table. The reader sets such an entry aside + /// and keeps what it already had for the object. Another reader, the page renderer among + /// them, may follow it instead, so the two can then show different objects for one number. + /// Code that compares revisions (signature validation) must not call such an object + /// unchanged. + pub fn xref_oddities(&self) -> &[(u64, u32)] { + &self.xref_oddities + } + + /// Whether the cross-reference entry of `num` locates that object itself: its `num G obj` + /// header at exactly the recorded offset, or the member with that number at the recorded + /// position of its object stream. When it does not, [`Self::get`] still finds the object by + /// searching, but another reader may resolve the number differently. Ignores unsaved edits. + pub fn xref_entry_is_exact(&self, num: u32) -> bool { + match self.xref_entry(num) { + None | Some(XrefEntry::Free { .. }) => true, + Some(XrefEntry::InFile { offset, generation }) => { + let at = |off: usize| matches!(parse_indirect_shared(&self.data, off, &|_| None, &self.endstreams), Ok((id, _)) if id == ObjRef::new(num, generation)); + // Offsets may count from a header that does not start the file. + usize::try_from(offset).is_ok_and(|off| at(off) || (self.header_offset != 0 && at(off.saturating_add(self.header_offset)))) + } + Some(XrefEntry::InStream { stream, index }) => self + .objstm(stream) + .is_ok_and(|stm| usize::try_from(index).ok().and_then(|i| stm.index.get(i)).is_some_and(|m| m.is_some_and(|(n, _)| n == num))), + } + } + + /// Object `num` has an unsaved edit. + pub fn is_edited(&self, num: u32) -> bool { + self.overlay.contains_key(&num) + } + + pub fn root(&self) -> Option { + self.trailer.reference(b"Root") + } + + // ── object access ─────────────────────────────────────────────────────────────────────── + + /// Fetch an object by reference. Undefined references are `Null` (ISO 32000-2 §7.3.10). + /// A live file slot also accepts its actual header generation when a damaged xref row + /// disagrees; the disagreement is recorded in [`Self::repair_log`]. Free slots never do. + pub fn get(&self, r: ObjRef) -> Arc { + self.try_get_reference(r).unwrap_or_else(|_| Arc::new(Object::Null)) + } + + pub(super) fn reference_matches(&self, r: ObjRef) -> bool { + match self.overlay.get(&r.num) { + Some(Slot::Set(generation, _)) => r.generation == *generation, + Some(Slot::RepairedSet(generation, alias, _)) => r.generation == *generation || r.generation == *alias, + Some(Slot::Freed(_)) => false, + None => match self.entries.lookup(r.num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + r.generation == generation || self.indexed_header_reference(r.num, offset) == Some(r) + } + Some(XrefEntry::InStream { .. }) => r.generation == 0, + None | Some(XrefEntry::Free { .. }) => false, + }, + } + } + + /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), + /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged + /// file slot (or a repair-aware edit of one) also the generation its object header carries. + /// Code comparing revisions (signature validation) uses it to tell whether the references + /// that reached an object in one revision still reach the same slot in another. + pub fn accepted_generations(&self, num: u32) -> Vec { + // Mirrors `reference_matches`, which stays allocation-free for every lookup. + let mut generations = match self.overlay.get(&num) { + Some(Slot::Set(generation, _)) => vec![*generation], + Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], + Some(Slot::Freed(_)) => Vec::new(), + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() + } + Some(XrefEntry::InStream { .. }) => vec![0], + None | Some(XrefEntry::Free { .. }) => Vec::new(), + }, + }; + generations.sort_unstable(); + generations.dedup(); + generations + } + + pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { + if !self.reference_matches(r) { + // An edit of `r.num` can make it match, so a load that read null here read it too; + // with `r.num` edited (freed, or set at another generation) the null is the edit's. + if self.overlay.contains_key(&r.num) { + note_overlay_read(); + } + note_read(r.num, None); + return Ok(Arc::new(Object::Null)); + } + self.try_get(r.num) + } + + /// A repair probe, not a whole-file search: only the indexed header (or the same offset + /// relative to a shifted PDF header) can establish this additional generation. A small + /// window bounds hostile whitespace/tokens; ordinary xref-matching reads are unchanged. + pub(super) fn indexed_header_reference(&self, num: u32, offset: u64) -> Option { + let at = |off: usize| { + let bytes = self.data.get(off..off.saturating_add(128).min(self.data.len()))?; + let mut lexer = Lexer::new(bytes, 0); + lexer.skip_ws(); + let number = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; + lexer.skip_ws(); + let generation = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; + (number == num && lexer.eat_keyword(b"obj")).then_some(ObjRef::new(number, generation)) + }; + let off = usize::try_from(offset).ok()?; + at(off).or_else(|| if self.header_offset == 0 { None } else { off.checked_add(self.header_offset).and_then(at) }) + } + + /// Add at most one bounded diagnostic per object. False stops diagnostic probing only; + /// parsers must still read the remaining xref rows needed to build their index. + pub(super) fn inspect_xref_generation_repair(&self, num: u32, entry: XrefEntry, recorded: &mut HashSet, log: &mut Vec) -> bool { + let XrefEntry::InFile { offset, generation } = entry else { return true }; + if recorded.contains(&num) { + return true; + } + let Some(header) = self.indexed_header_reference(num, offset) else { return true }; + if header.generation == generation { + return true; + } + if recorded.len() == 256 { + log.push("additional xref/header generation disagreements omitted after 256 objects".into()); + return false; + } + recorded.insert(num); + log.push(format!("object {num}: xref generation {generation} disagrees with object header generation {}", header.generation)); + true + } + + /// Diagnose once at open without expanding compact ranges or decoding an xref twice. + /// Materialized readers retain bounded notes while they read accepted sections, before + /// newer rows hide older ones. Discarded parse attempts/rebuilt indexes carry no notes. + pub(super) fn note_xref_generation_repairs(&self, log: &mut Vec) { + match &*self.entries { + XrefIndex::Materialized(_, notes) => log.extend(notes.iter().cloned()), + XrefIndex::Compact { revisions, .. } => { + let mut recorded = HashSet::new(); + for revision in revisions { + for section in std::iter::once(&revision.primary).chain(revision.hybrid.iter()) { + for num in section.numbers() { + if let Some(entry) = section.lookup(num, &self.data) + && !self.inspect_xref_generation_repair(num, entry, &mut recorded, log) + { + return; + } + } + } + } + } + } + } + + /// Follow a reference if `o` is one; otherwise return `o` itself. An object can be nothing but + /// a reference to another, which is followed too, up to [`MAX_REFERENCE_HOPS`] of them. + pub fn resolve(&self, o: &Object) -> Arc { + let Object::Ref(r) = o else { return Arc::new(o.clone()) }; + let mut held = self.get(*r); + for _ in 0..MAX_REFERENCE_HOPS { + let Object::Ref(r) = &*held else { break }; + held = self.get(*r); + } + held + } + + /// `s` with the `/DecodeParms` its filters use made direct, so [`Stream::decoded`] reads them: + /// an indirect dictionary resolved, and of an array (itself perhaps indirect), the entries at + /// the filters' places (`/Crypt` counts, ISO 32000-2 §7.4.1) resolved, the rest dropped. Of + /// each dictionary only the entries the decoders read are kept, numbers and flags given by + /// reference (`/Predictor 7 0 R`) resolved; `/JBIG2Globals` stays a reference. A `/Crypt` + /// entry, which no decoder reads ([`Stream::filters`] leaves it out), is null. `s` itself + /// when none of those is indirect, or when the other filters are more than any decoder takes + /// ([`crate::filters::MAX_CHAIN`]). Only small values are copied, so a file can't have a + /// large object copied once for each key and filter. + pub fn with_direct_parms<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut s = s.clone(); + s.dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(s) + } + + /// [`Document::with_direct_parms`] for decoding alone: where that copies `s`, this makes a + /// stream of `s`'s data, shared, and a dictionary of its `/Filter` and the parameters made + /// direct, all its filters read. A stream named again and again, as a page's `/Contents` may + /// list one, isn't copied whole each time. + pub(crate) fn decoding_view<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut dict = Dict::new(); + if let Some(f) = s.dict.get(b"Filter") { + dict.set(b"Filter".to_vec(), f.clone()); + } + dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(Stream { dict, raw: s.raw.clone() }) + } + + /// The `/DecodeParms` [`Document::with_direct_parms`] gives `s`, or `None` when it keeps `s`. + fn direct_parms(&self, s: &Stream) -> Option { + let p = s.dict.get(b"DecodeParms")?; + let crypt = |f: &Object| matches!(f.as_name(), Some(b"Crypt")); + let filters: Vec = match s.dict.get(b"Filter") { + Some(Object::Array(a)) => a.iter().map(crypt).collect(), + Some(f) => vec![crypt(f)], + None => Vec::new(), + }; + if filters.iter().filter(|c| !**c).count() > crate::filters::MAX_CHAIN { + return None; + } + let mut changed = matches!(p, Object::Ref(_)); + // Its numbers and flags kept, resolved; anything else is no filter's parameters (null). + let one = |e: &Object, changed: &mut bool| -> Object { + *changed |= matches!(e, Object::Ref(_)); + let Object::Dict(d) = &*self.held(e) else { return Object::Null }; + let mut made = Dict::new(); + for key in DECODE_PARMS_KEYS { + let Some(v) = d.get(key) else { continue }; + *changed |= matches!(v, Object::Ref(_)); + let v = self.held(v); + if matches!(*v, Object::Int(_) | Object::Real(_) | Object::Bool(_)) { + made.set(key.to_vec(), (*v).clone()); + } + } + if let Some(g @ Object::Ref(_)) = d.get(b"JBIG2Globals") { + made.set(b"JBIG2Globals".to_vec(), g.clone()); + } + Object::Dict(made) + }; + let made = match &*self.held(p) { + Object::Array(a) => { + Object::Array(a.iter().zip(&filters).map(|(e, crypt)| if *crypt { Object::Null } else { one(e, &mut changed) }).collect()) + } + _ => one(p, &mut changed), + }; + changed.then_some(made) + } + + /// `o`, or the object it refers to, without copying either. + pub(crate) fn held<'o>(&self, o: &'o Object) -> Held<'o> { + match o { + Object::Ref(_) => Held::Resolved(self.resolve(o)), + o => Held::Direct(o), + } + } + + /// Resolve and return a dictionary (a stream's dictionary counts). + pub fn dict(&self, o: &Object) -> Option { + self.resolve(o).as_dict().cloned() + } + + /// The CCITT parameters of image `s` when its last filter is `CCITTFaxDecode`: its + /// `/DecodeParms` (resolved; the dictionary of a lone filter, or its entry in an array), with the + /// defaults of ISO 32000-2 Table 11 for what that leaves out. `None` for another image, and for + /// one whose `/Filter` isn't a name or an array of names. + pub fn fax_params(&self, s: &Stream) -> Option { + let chain = s.readable_filters().ok()?; + if !matches!(chain.last(), Some((crate::filters::Filter::CcittFax, _))) { + return None; + } + let parms = self.last_filter_parms(s); + let int = |key: &[u8], default: i64| parms.get(key).and_then(|v| self.resolve(v).as_int()).unwrap_or(default); + let flag = |key: &[u8], default: bool| match parms.get(key).map(|v| self.resolve(v)).as_deref() { + Some(Object::Bool(b)) => *b, + _ => default, + }; + let d = crate::filters::CcittParams::default(); + Some(crate::filters::CcittParams { + k: int(b"K", d.k), + columns: int(b"Columns", d.columns), + rows: int(b"Rows", d.rows), + end_of_line: flag(b"EndOfLine", d.end_of_line), + encoded_byte_align: flag(b"EncodedByteAlign", d.encoded_byte_align), + end_of_block: flag(b"EndOfBlock", d.end_of_block), + black_is_1: flag(b"BlackIs1", d.black_is_1), + }) + } + + /// The `/DecodeParms` dictionary of stream `s`'s last filter, resolved: that of a lone filter, + /// or its entry in an array. Empty when there is none. + fn last_filter_parms(&self, s: &Stream) -> Dict { + // Its place in /Filter, /Crypt included, pairs it with its /DecodeParms entry (§7.4.1): the + // last filter but a /Crypt, which `filters` leaves out wherever it stands. + let index = match s.dict.get(b"Filter") { + Some(Object::Array(a)) => a.iter().rposition(|f| f.as_name() != Some(b"Crypt".as_slice())).unwrap_or(0), + _ => 0, + }; + match s.dict.get(b"DecodeParms").map(|p| self.resolve(p)).as_deref() { + Some(Object::Array(a)) => a.get(index).and_then(|p| self.dict(p)), + Some(p) if index == 0 => p.as_dict().cloned(), + _ => None, + } + .unwrap_or_default() + } + + /// [`Document::decoded_image_spending`] with no limit on the work, for tests of what an image + /// decodes to: everything else decodes images through an [`ImageBudget`](crate::cos::ImageBudget). + #[cfg(test)] + pub(crate) fn decoded_image(&self, s: &Stream) -> Result, CosError> { + self.decoded_image_within(s, MAX_DECODED) + } + + /// [`Document::decoded_image`] within `max` bytes. + #[cfg(test)] + pub(crate) fn decoded_image_within(&self, s: &Stream, max: usize) -> Result, CosError> { + let mut unbounded = u64::MAX; + self.decoded_image_spending(s, max, &mut unbounded) + } + + /// The samples of image `s`, decoded within `max` bytes ([`MAX_DECODED`] at most), spending from + /// `work`: see [`ImageBudget::decode`](crate::cos::ImageBudget::decode), the way in from outside this + /// crate, which says what this decodes and how. + pub(crate) fn decoded_image_spending(&self, s: &Stream, max: usize, work: &mut u64) -> Result, CosError> { + let max = max.min(MAX_DECODED); + let s = &*self.with_direct_parms(s); + let filter = |e: crate::filters::FilterError| CosError::Filter(e.to_string()); + let chain = s.readable_filters().map_err(filter)?; + // The filters before the image codec, such as ASCII85Decode, are undone first. + let undo = |before: &[(crate::filters::Filter, crate::filters::Params)], work: &mut u64| match before { + [] => Ok(Cow::Borrowed(&s.raw[..])), + _ => crate::filters::decode_tolerant_within(before, &s.raw, max, work).map(Cow::Owned).map_err(filter), + }; + // `/Width` and `/Height` are read as given directly, as every caller reads them first: one + // given by reference has the image refused there, before it gets here. + let height = s.dict.int(b"Height").and_then(|h| u32::try_from(h).ok()); + if let Some(((crate::filters::Filter::Jbig2, _), before)) = chain.split_last() { + let (Some(columns), Some(height)) = (s.dict.int(b"Width").and_then(|w| u32::try_from(w).ok()), height) else { + return Err(CosError::Filter("a JBIG2 image's /Width or /Height isn't a size".into())); + }; + // Nothing to read for what can't be decoded: the images of a file share its globals. + let globals = if *work == 0 { None } else { self.jbig2_globals(s, work)? }; + let data = if *work == 0 { Cow::Borrowed(&[][..]) } else { undo(before, work)? }; + return crate::filters::decode_jbig2_within(&data, globals.as_deref().map(Vec::as_slice), columns, height, max, work).map_err(filter); + } + let Some(p) = self.fax_params(s) else { + // The last filter gives the image's rows: rows without the tags of the PNG predictor + // they declare are read as they are (`crate::filters::decode_image_tolerant_within`). + return match height { + Some(rows) => s.decoded_image_rows_spending(max, rows, work), + None => s.decoded_spending(max, work), + }; + }; + // As many rows as the image has: a /Rows below /Height would cut the picture short, and + // rows past /Height aren't drawn. + let p = crate::filters::CcittParams { rows: 0, ..p }; + if s.dict.int(b"Width") != Some(p.columns) { + return Err(CosError::Filter("a fax image's /Columns aren't its /Width".into())); + } + let stride = usize::try_from(p.columns).unwrap_or(usize::MAX).div_ceil(8).max(1); + let rows = u32::try_from(max / stride).unwrap_or(u32::MAX).min(height.unwrap_or(0)); + let data = undo(chain.split_last().map_or(&[][..], |(_, before)| before), work)?; + crate::filters::decode_ccitt_within(&data, &p, rows, work).map_err(filter) + } + + /// What the `/JBIG2Globals` stream of JBIG2 image `s` (in its last filter's `/DecodeParms`) + /// decodes to, within [`MAX_JBIG2_GLOBALS`]; `None` without one (a reference to no object is + /// one to null, ISO 32000-2 §7.3.10, which is an absent entry, §7.3.7). The image is then read + /// without them, as the renderer draws it, so an image that needed them mostly fails to decode + /// and can decode to other pixels. A stream that is there and can't be read is an error, rather + /// than the image decoded without it. + /// + /// The images of a scan share one globals stream, so a stream of the file, not edited since + /// it was read (the same in every clone) and with its `/DecodeParms` direct (by reference they + /// are another object, which can change while this one doesn't), has its filters undone once a + /// document and is kept, up to [`JBIG2_GLOBALS_HELD`] in all; others are read each time. That + /// saves the filters only: the segments it holds are decoded again for every image. Undoing + /// the filters spends from `work`, so globals read for every image pay every time. + fn jbig2_globals(&self, s: &Stream, work: &mut u64) -> Result>>, CosError> { + let Some(entry) = self.last_filter_parms(s).get(b"JBIG2Globals").cloned() else { return Ok(None) }; + let held = match entry { + Object::Ref(r) if !self.overlay.contains_key(&r.num) => Some(r.num), + _ => None, + }; + if let Some(hit) = held.and_then(|num| self.jbig2_globals.lock().ok()?.get(&num).cloned()) { + return Ok(Some(hit)); + } + let Object::Stream(g) = &*self.resolve(&entry) else { return Ok(None) }; + let readable = self.with_direct_parms(g); + let held = held.filter(|_| matches!(readable, Cow::Borrowed(_))); + let mut data = readable.decoded_spending(MAX_JBIG2_GLOBALS, work).map_err(|e| { + let detail = match e { + CosError::Filter(detail) => detail, + e => e.to_string(), + }; + CosError::Filter(format!("the JBIG2 globals can't be read: {detail}")) + })?; + // What is kept counts by its length, so that's all it may hold on to. + data.shrink_to_fit(); + let data = Arc::new(data); + // A poisoned lock only means nothing is kept. + if let (Some(num), Ok(mut kept)) = (held, self.jbig2_globals.lock()) { + let total = kept.values().map(|g| g.len()).fold(data.len(), usize::saturating_add); + if total <= JBIG2_GLOBALS_HELD { + kept.insert(num, data.clone()); + } + } + Ok(Some(data)) + } + + pub fn try_get(&self, num: u32) -> Result, CosError> { + if let Some(slot) = self.overlay.get(&num) { + note_read(num, None); + note_overlay_read(); + return Ok(match slot { + Slot::Set(_, o) | Slot::RepairedSet(_, _, o) => o.clone(), + Slot::Freed(_) => Arc::new(Object::Null), + }); + } + // Clones share the cache: what a load read that this document has edited since (the + // member's object stream, a `/Length`) makes the entry another document's. + if let Some((loaded, reads)) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) + && !reads.touch(&self.overlay) + { + note_read(num, Some(reads)); + return loaded.clone(); + } + note_read(num, None); + // Loading can re-enter (indirect stream `/Length`, object streams); damaged files make + // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. + let _guard = + LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; + let _scope = crate::cos::security::FollowScope::enter(); + let edits_read = overlay_reads(); + let frame = ReadFrame::enter(); + let loaded = self.load(num, 0).map(Arc::new); + let reads = frame.reads(); + drop(frame); + if let Err(e) = &loaded + && !matches!(e, CosError::Poisoned) + { + self.note_unreadable(num, || e.to_string()); + } + // A failure is kept like an object, or every read of a damaged object loads it again. A + // poisoned lock says nothing about the object. Clones share the cache, so a load that read + // this document's edits (an edited object stream or `/Length`) is not kept. + if overlay_reads() == edits_read && !matches!(loaded, Err(CosError::Poisoned)) { + // The editing cache is bounded like the scan reader's: re-parsing a file-backed view + // is cheap, and returned objects outlive eviction. + self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone(), loaded_bytes(&loaded, &self.data), reads); + self.evict_editor_cache_if_over_budget(num)?; + } + loaded + } + + /// Bounds for the editing cache: roomier than the scan reader's (128 objects / 8 MB) + /// so interactive editing does not thrash, but still capped so touching a large + /// document cannot pin its whole object graph. Only owned (decrypted/copied) bytes + /// count; file-backed views re-parse for free. + /// + /// Worst case per document: input bytes + `EDITOR_CACHE_BYTES` (32 MB) of owned object + /// data + `EDITOR_OBJSTM_BYTES` (32 MB) + unsaved `overlay` edits. Each object is measured + /// once, as it is cached ([`ObjectCache`](super::state::ObjectCache)), so both caps hold + /// after every insertion. + /// Clones share these caches, so one clone's eviction drops another's hot set; + /// that only costs re-parsing, never correctness. Eviction keeps just the newcomer: + /// file-backed views re-parse cheaply and editor working sets stay far below 1024. + pub(super) const EDITOR_CACHE_OBJECTS: usize = 1024; + + pub(super) const EDITOR_CACHE_BYTES: usize = 32 * 1024 * 1024; + + /// Object streams decode to megabytes each, so fewer are kept than plain objects. + const EDITOR_OBJSTM_OBJECTS: usize = 256; + + const EDITOR_OBJSTM_BYTES: usize = 32 * 1024 * 1024; + + fn evict_editor_cache_if_over_budget(&self, fresh_num: u32) -> Result<(), CosError> { + let mut cache = self.cache.lock().map_err(|_| CosError::Poisoned)?; + if cache.len() <= Self::EDITOR_CACHE_OBJECTS && cache.owned() <= Self::EDITOR_CACHE_BYTES { + return Ok(()); + } + // Eviction keeps just the newcomer: file-backed views re-parse cheaply. + cache.retain(|n| *n == fresh_num); + Ok(()) + } + + fn evict_editor_streams_if_over_budget(&self, current: u32) -> Result<(), CosError> { + // The caller inserts `current` first, so retaining it keeps a live entry. + let over = { + let streams = self.objstms.lock().map_err(|_| CosError::Poisoned)?; + let bytes = streams.values().fold(0usize, |bytes, stream| { + bytes + .saturating_add(stream.data.capacity()) + .saturating_add(stream.index.capacity().saturating_mul(std::mem::size_of::>())) + }); + bytes > Self::EDITOR_OBJSTM_BYTES || streams.len() > Self::EDITOR_OBJSTM_OBJECTS + }; + if over { + self.objstms.lock().map_err(|_| CosError::Poisoned)?.retain(|n, _| *n == current); + } + Ok(()) + } + + /// Visit every current object without filling this document's editing cache. + /// + /// Full-document searches (for example, standalone timestamps) need to inspect objects + /// that may never be used again. Keep their parse caches private and release them in + /// batches. Returned objects own their data and remain valid after the next batch. + pub fn scan_objects(&self) -> impl Iterator)> + use<> { + let numbers = self.object_numbers(); + let mut reader = self.object_reader(); + numbers.into_iter().map(move |num| { + let reference = ObjRef::new(num, reader.document.generation(num)); + (reference, reader.get(reference)) + }) + } + + /// Like [`Self::scan_objects`], preserving read errors for callers that must distinguish + /// incomplete discovery from an actual null object. Caches remain private and bounded. + pub fn scan_objects_checked(&self) -> impl Iterator, CosError>)> + use<> { + self.scan_objects_subset(self.object_numbers()) + } + + /// Inspect selected current objects with the same bounded cache as a whole-file scan. + /// Missing/freed numbers yield `Null`; malformed objects return their read error. + pub fn scan_objects_subset(&self, numbers: Vec) -> impl Iterator, CosError>)> + use<> { + let mut reader = self.object_reader(); + numbers.into_iter().map(move |num| { + let reference = ObjRef::new(num, reader.document.generation(num)); + (reference, reader.try_get(reference)) + }) + } + + /// Inspect selected objects in the immutable source, ignoring all unsaved edits. This + /// allows a consumer to check an edit's dependencies without retaining an old document. + pub fn scan_original_objects_subset(&self, numbers: Vec) -> impl Iterator, CosError>)> + use<> { + let mut reader = self.object_reader(); + reader.document.overlay.clear(); + numbers.into_iter().map(move |num| { + let reference = ObjRef::new(num, reader.document.generation(num)); + (reference, reader.try_get(reference)) + }) + } + + /// Opened with a tighter stream limit than editing uses ([`Document::open_with_stream_limit`]). + pub(crate) fn stream_limited(&self) -> bool { + self.stream_limit < crate::cos::object::MAX_DECODED + } + + pub(crate) fn object_reader(&self) -> ObjectReader { + let mut document = self.clone(); + document.cache = Arc::default(); + document.objstms = Arc::default(); + ObjectReader { document, decoded_count: 0 } + } + + fn load(&self, num: u32, depth: u32) -> Result { + if depth > 16 { + return Err(CosError::Syntax { offset: 0, detail: "reference cycle while loading".into() }); + } + match self.entries.lookup(num, &self.data) { + None | Some(XrefEntry::Free { .. }) => Ok(Object::Null), + Some(XrefEntry::InFile { offset, .. }) => { + let resolve = |r: ObjRef| self.stream_length(r); + let off = offset as usize; + let at = |off: usize| match parse_indirect_shared(&self.data, off, &resolve, &self.endstreams) { + Ok((id, o)) if id.num == num => Some(self.decrypted(id, o)), + _ => None, + }; + // Offsets relative to a shifted header, or simply wrong: try both fixes. + at(off) + .or_else(|| if self.header_offset == 0 { None } else { at(off.saturating_add(self.header_offset)) }) + .or_else(|| self.located.get(&num).and_then(|(o, _)| usize::try_from(*o).ok()).and_then(at)) + .or_else(|| self.scan_for(num).map(|(id, o)| self.decrypted(id, o))) + .ok_or_else(|| { + self.note_unreadable(num, || self.why_unreadable(off)); + CosError::MissingObject(num) + }) + } + Some(XrefEntry::InStream { stream, index }) => { + let stm = self.objstm(stream)?; + // The member at the entry's position, else wherever the stream lists `num`. + let at = usize::try_from(index).ok().and_then(|i| stm.index.get(i)).copied().flatten().filter(|(n, _)| *n == num); + let (_, off) = at.or_else(|| stm.index.iter().flatten().find(|(n, _)| *n == num).copied()).ok_or(CosError::MissingObject(num))?; + Lexer::new(&stm.data, off).object() + } + } + } + + pub(super) fn objstm(&self, num: u32) -> Result, CosError> { + // An edited or freed stream is this document's alone: clones share the parsed streams. + let edited = self.overlay.contains_key(&num); + if !edited + && let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) + && !s.reads.touch(&self.overlay) + { + note_read(num, Some(&s.reads)); + return Ok(s.clone()); + } + let frame = ReadFrame::enter(); + // A file entry that cannot place the stream (one that says it is itself compressed + // inside another stream, as in #2367, or points nowhere) is tried by the stream's own + // header, as a plain object with a wrong offset is. An edited or freed stream, in the + // overlay or in a saved revision's table, is what that says it is: the file's copy + // must not come back. + let edits_read = overlay_reads(); + let loaded = match self.try_get(num) { + Ok(o) if matches!(*o, Object::Stream(_)) => o, + _ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => { + return Err(CosError::MissingObject(num)); + } + _ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?), + }; + let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) }; + let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize; + let first = usize::try_from(s.dict.int(b"First").unwrap_or(0).max(0)).ok(); + // Its parameters can be given by reference, and can sit in another object stream. A scan of + // the file's object streams comes here without a read around it. + let _scope = crate::cos::security::FollowScope::enter(); + let readable = crate::cos::security::following(Cow::Borrowed(s), || self.with_direct_parms(s)); + #[cfg(test)] + super::tests::OBJSTM_DECODES.with(|decodes| decodes.set(decodes.get() + 1)); + let data = readable.decoded_within(self.stream_limit)?; + let mut lx = Lexer::new(&data, 0); + let mut index = Vec::with_capacity(n); + for _ in 0..n { + let (Some(Object::Int(onum)), Some(Object::Int(off))) = (lx.object().ok(), lx.object().ok()) else { break }; + index.push(objstm_member(onum, off, first, data.len())); + } + let stm = Arc::new(ObjStm { index, data: Arc::new(data), reads: frame.reads() }); + // A stream whose load or decoding read this document's edits (its `/Length`, its + // `/DecodeParms`) is as much its own as an edited one. + if edited || overlay_reads() != edits_read { + return Ok(stm); + } + // Same bound as the object cache: decoded object streams can be megabytes each. + // Insert first so the retained `num` is actually in the map. + self.objstms.lock().map_err(|_| CosError::Poisoned)?.insert(num, stm.clone()); + self.evict_editor_streams_if_over_budget(num)?; + Ok(stm) + } + + /// An indirect stream `/Length`, read through the loader's guard against cycles. Kept in + /// [`LENGTHS`] only during a load, which clears it when it ends: a scan of the file from + /// [`Self::objstm`] can come here without one. + fn stream_length(&self, r: ObjRef) -> Option { + if self.overlay.contains_key(&r.num) { + note_overlay_read(); + } + let cached = loading(); + // Each reuse counts as a read of the length and of what resolving it read, and as an edit + // read when that went through this document's edits. + if cached && let Some((known, edits, reads)) = LENGTHS.with(|m| m.borrow().get(&r).cloned()) { + if edits { + note_overlay_read(); + } + note_read(r.num, Some(&reads)); + return known; + } + note_read(r.num, None); + let edits_read = overlay_reads(); + let frame = ReadFrame::enter(); + let length = self.try_get_reference(r).ok().and_then(|o| o.as_int()); + if cached { + LENGTHS.with(|m| m.borrow_mut().insert(r, (length, overlay_reads() != edits_read, frame.reads()))); + } + length + } + + /// Last-resort lookup: scan the file for `num G obj`. What isn't there is remembered: each miss + /// reads the whole file, and a file can refer to such an object from everywhere. + fn scan_for(&self, num: u32) -> Option<(ObjRef, Object)> { + if self.unfound.lock().is_ok_and(|unfound| unfound.contains(&num)) { + return None; + } + #[cfg(test)] + super::tests::FILE_SCANS.with(|scans| scans.set(scans.get() + 1)); + let needle = format!("{num} "); + let data = &self.data; + let budget = ScanLengths::enter(); + let resolve = |r: ObjRef| if budget.take() { self.stream_length(r) } else { None }; + let mut found = None; + let mut i = 0; + while let Some(p) = find(data, needle.as_bytes(), i, data.len()) { + i = p + 1; + if p > 0 && !is_whitespace(data[p - 1]) { + continue; + } + // An indirect `/Length` is resolved, as at the object's offset, or a stream that ends + // in a misspelt `endstream` is searched on into a later object's. A header that parses + // as far as the `/Length` starts with `num` itself, after white space. A budget shared + // with the scans this leads to bounds how often the file is read again ([`ScanLengths`]). + if let Ok((id, o)) = parse_indirect_shared(data, p, &resolve, &self.endstreams) + && id.num == num + { + found = Some((id, o)); // keep the last (newest) definition + } + } + if found.is_none() + && let Ok(mut unfound) = self.unfound.lock() + { + unfound.insert(num); + } + found + } + + // ── editing ───────────────────────────────────────────────────────────────────────────── + + /// Stream data still pointing into another document's file would keep that whole file + /// alive for as long as this edit exists: give it a buffer of its own (just its bytes). + fn adopt(&self, obj: Object) -> Object { + match obj { + Object::Stream(mut s) if s.raw.borrows_other_than(&self.data) => { + s.raw = s.raw.detached(); + Object::Stream(s) + } + other => other, + } + } + + /// Replace an object (keeping its generation). + pub fn set(&mut self, r: ObjRef, obj: impl Into) { + let obj = self.adopt(obj.into()); + self.overlay.insert(r.num, Slot::Set(r.generation, Arc::new(obj))); + if r.num >= self.next_num { + self.next_num = r.num.saturating_add(1); + } + } + + /// Replace an accepted existing reference without losing a damaged source slot's aliases. + /// Unlike [`Self::set`], this preserves both xref/header generations when they disagree. + /// Such an edit forces a full rewrite so the output has consistent references, rather than + /// appending a new contradictory xref row. Missing, free and stale references are errors. + pub fn set_existing(&mut self, r: ObjRef, obj: impl Into) -> Result<(), CosError> { + if !self.reference_matches(r) { + return Err(CosError::MissingObject(r.num)); + } + let alias = self.alternate_generation(r); + if let Some(alias) = alias { + let obj = self.adopt(obj.into()); + self.overlay.insert(r.num, Slot::RepairedSet(r.generation, alias, Arc::new(obj))); + } else { + self.set(r, obj); + } + Ok(()) + } + + /// Other accepted generation for a known-live reference to a damaged slot. + pub(super) fn alternate_generation(&self, r: ObjRef) -> Option { + match self.overlay.get(&r.num) { + Some(Slot::RepairedSet(generation, alias, _)) => Some(if r.generation == *generation { *alias } else { *generation }), + Some(_) => None, + None => match self.entries.lookup(r.num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => self.indexed_header_reference(r.num, offset).and_then(|header| { + (header.generation != generation).then_some(if r.generation == generation { header.generation } else { generation }) + }), + _ => None, + }, + } + } + + /// Add a new indirect object and return its reference. + pub fn add(&mut self, obj: impl Into) -> ObjRef { + let r = ObjRef::new(self.allocate_num(), 0); + let obj = self.adopt(obj.into()); + self.overlay.insert(r.num, Slot::Set(0, Arc::new(obj))); + r + } + + /// Find a never-used number, retaining the high-water mark for incremental /Size. + /// Exhausted hints fall back to gaps without replacing source or overlay objects. + fn allocate_num(&mut self) -> u32 { + let mut num = if self.next_num == u32::MAX { 1 } else { self.next_num.max(1) }; + while self.entries.lookup(num, &self.data).is_some() || self.overlay.contains_key(&num) { + num = if num == u32::MAX - 1 { 1 } else { num + 1 }; + } + self.next_num = self.next_num.max(num.saturating_add(1)); + num + } + + /// Delete an object (its number becomes free with the next generation). + pub fn free(&mut self, r: ObjRef) { + self.overlay.insert(r.num, Slot::Freed(r.generation.saturating_add(1))); + } + + /// Free what was added since the file was read and nothing refers to any more: the objects + /// [`SaveOptions::drop_unlinked`](crate::cos::SaveOptions::drop_unlinked) leaves out of an + /// incremental save, freed as it writes them (the generation stays). For a writer about to + /// add an object that nothing refers to either, which that option would leave out too. + pub fn free_unlinked_additions(&mut self) { + for num in crate::cos::writer::unlinked_additions(self) { + let generation = self.generation(num); + self.overlay.insert(num, Slot::Freed(generation)); + } + } + + /// Record `current`'s object graph as unsaved edits of this document. + /// + /// Used when the working snapshot was restored from an earlier revision and the next + /// incremental save must still append to this file (the last saved revision). Objects + /// that already match are left untouched, except members of an object stream of this file + /// whose stream number `current` frees or reuses: those are written again on their own. + /// Reads go through private readers so neither document's editing cache is filled. + pub fn overlay_changes_from(&mut self, current: &Document) -> Result<(), CosError> { + let saved_nums = self.object_numbers(); + let current_nums = current.object_numbers(); + let mut freed = Vec::new(); + let mut changed = Vec::new(); + { + let mut saved_reader = self.object_reader(); + let mut current_reader = current.object_reader(); + for &num in &saved_nums { + if current_nums.binary_search(&num).is_err() { + freed.push(num); + } + } + for &num in ¤t_nums { + let generation = current.generation(num); + let same = saved_nums.binary_search(&num).is_ok() + && self.generation(num) == generation + && self.alternate_generation(ObjRef::new(num, generation)) == current.alternate_generation(ObjRef::new(num, generation)) + && saved_reader.try_get(ObjRef::new(num, generation))? == current_reader.try_get(ObjRef::new(num, generation))?; + if !same { + changed.push(num); + } + } + } + // An object stream of this file that is freed or replaced takes its members with it, so + // a member that is otherwise unchanged is written again on its own. + let gone = |n: u32| freed.binary_search(&n).is_ok() || changed.binary_search(&n).is_ok(); + let orphaned: Vec = current_nums + .iter() + .copied() + .filter(|&num| changed.binary_search(&num).is_err()) + .filter(|&num| { + let packed_in = match self.xref_entry(num) { + Some(XrefEntry::InStream { stream, .. }) => Some(stream), + _ => None, + }; + packed_in.is_some_and(gone) + }) + .collect(); + changed.extend(orphaned); + for num in freed { + let generation = self.generation(num); + self.free(ObjRef::new(num, generation)); + } + let mut current_reader = current.object_reader(); + for num in changed { + let generation = current.generation(num); + let object = current_reader.try_get(ObjRef::new(num, generation))?; + if let Some(alias) = current.alternate_generation(ObjRef::new(num, generation)) { + let object = self.adopt((*object).clone()); + self.overlay.insert(num, Slot::RepairedSet(generation, alias, Arc::new(object))); + self.next_num = self.next_num.max(num.saturating_add(1)); + } else { + self.set(ObjRef::new(num, generation), (*object).clone()); + } + } + *self.trailer_mut() = current.trailer().clone(); + Ok(()) + } + + /// Edit a dictionary (or stream dictionary) object in place. + pub fn update_dict(&mut self, r: ObjRef, f: impl FnOnce(&mut Dict)) -> Result<(), CosError> { + let mut obj = (*self.get(r)).clone(); + let d = obj.as_dict_mut().ok_or(CosError::NotADictionary(r))?; + f(d); + self.set_existing(r, obj) + } + + /// Current generation of an object number. + pub fn generation(&self, num: u32) -> u16 { + match self.overlay.get(&num) { + Some(Slot::Set(g, _) | Slot::RepairedSet(g, _, _)) => *g, + Some(Slot::Freed(g)) => *g, + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { generation, .. }) => generation, + Some(XrefEntry::Free { next_generation }) => next_generation, + _ => 0, + }, + } + } + + pub(crate) fn overlay_entries(&self) -> impl Iterator>)> + '_ { + self.overlay.iter().map(|(n, s)| match s { + Slot::Set(g, o) | Slot::RepairedSet(g, _, o) => (*n, *g, Some(o.clone())), + Slot::Freed(g) => (*n, *g, None), + }) + } + + pub(crate) fn next_num(&self) -> u32 { + self.next_num + } + + /// All object numbers known (file and overlay), excluding freed ones. + pub fn object_numbers(&self) -> Vec { + let source_numbers = self.entries.object_numbers(&self.data); + let mut numbers = Vec::with_capacity(source_numbers.len().max(self.overlay.len())); + let mut source = source_numbers.into_iter().peekable(); + for (&num, slot) in &self.overlay { + while source.peek().is_some_and(|n| *n < num) { + if let Some(n) = source.next() { + numbers.push(n); + } + } + if source.peek().is_some_and(|n| *n == num) { + source.next(); + } + if matches!(slot, Slot::Set(..) | Slot::RepairedSet(..)) { + numbers.push(num); + } + } + numbers.extend(source); + numbers + } + + /// Rebase this document on bytes written by the writer (after a save): edits become part of + /// the file and the overlay is cleared, so the next incremental save appends only new edits. + pub fn reopen_after_save(&self, bytes: Arc>) -> Result { + Document::open(bytes) + } + + // ── cross-reference parsing ───────────────────────────────────────────────────────────── + + pub(super) fn root_is_catalog(&self) -> bool { + let Some(r) = self.root() else { return false }; + matches!(self.try_get_reference(r).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog") || d.contains(b"Pages")) + } +} + +/// The `/DecodeParms` entries the decoders read (ISO 32000-2 Tables 8, 11 and 13: the Flate and +/// LZW predictors, CCITT, DCT's colour transform): numbers and flags a file may give by +/// reference, which [`Dict::int`] doesn't follow. +pub const DECODE_PARMS_KEYS: [&[u8]; 13] = [ + b"Predictor", + b"Colors", + b"BitsPerComponent", + b"Columns", + b"EarlyChange", + b"K", + b"Rows", + b"EndOfLine", + b"EncodedByteAlign", + b"EndOfBlock", + b"BlackIs1", + b"DamagedRowsBeforeError", + b"ColorTransform", +]; + +/// How many references in a row [`Document::resolve`] follows: enough for any file, and it ends a +/// cycle of them. +const MAX_REFERENCE_HOPS: usize = 32; + +/// An object read in place, or the object a reference names as the document holds it: either way +/// nothing is copied ([`Document::held`]). +pub(crate) enum Held<'o> { + Direct(&'o Object), + Resolved(Arc), +} + +impl std::ops::Deref for Held<'_> { + type Target = Object; + + fn deref(&self) -> &Object { + match self { + Held::Direct(o) => o, + Held::Resolved(o) => o, + } + } +} diff --git a/crates/pdf/src/cos/document/open.rs b/crates/pdf/src/cos/document/open.rs new file mode 100644 index 000000000..46a6423a9 --- /dev/null +++ b/crates/pdf/src/cos/document/open.rs @@ -0,0 +1,503 @@ +use super::reader::find; +use super::state::{Credentials, Slot}; +use super::xref::XrefIndex; +use super::{Arc, BTreeMap, CosError, Dict, Document, HashSet, ObjRef, Object}; + +/// More objects than `/Encrypt` and `/ID` refer to in any file, so that one which refers to the page +/// tree, say, doesn't have a whole document read to open it. +const MAX_ENCRYPT_MEMBERS: usize = 64; + +impl Document { + /// A new document with an empty page tree (for split / extract / combine). It has no + /// original bytes, so saving always writes a full file. + pub fn new_empty() -> Self { + let mut doc = Document { + data: Arc::new(Vec::new()), + endstreams: Arc::default(), + source_identity: Arc::new(()), + entries: Arc::new(XrefIndex::empty()), + trailer: Dict::new(), + revisions: Arc::new(Vec::new()), + xref_oddities: Arc::default(), + located: Arc::default(), + repair_log: Arc::new(Vec::new()), + unlinked_update: false, + cache: Arc::default(), + objstms: Arc::default(), + unreadable: Arc::default(), + jbig2_globals: Arc::default(), + stream_limit: crate::cos::object::MAX_DECODED, + overlay: BTreeMap::new(), + next_num: 1, + header_offset: 0, + version: "1.7".into(), + security: None, + encrypt_num: None, + encrypt_members: Arc::default(), + unfound: Arc::default(), + encryption_changed: false, + full_save: false, + new_id: false, + out_security: None, + out_encrypt_num: None, + }; + let mut pages = Dict::new(); + pages.set(b"Type".to_vec(), Object::name("Pages")); + pages.set(b"Kids".to_vec(), Object::Array(Vec::new())); + pages.set(b"Count".to_vec(), Object::Int(0)); + let pages = doc.add(pages); + let mut catalog = Dict::new(); + catalog.set(b"Type".to_vec(), Object::name("Catalog")); + catalog.set(b"Pages".to_vec(), Object::Ref(pages)); + let root = doc.add(catalog); + doc.trailer.set(b"Root".to_vec(), Object::Ref(root)); + doc + } + + /// Parse a document. Damaged cross-reference data is reconstructed; see `repair_log`. + /// Encrypted documents open only if their user password is empty; see `open_with_password`. + pub fn open(data: Arc>) -> Result { + Self::open_with_password(data, None) + } + + /// Parse a document, authenticating encrypted ones with `password` (owner or user; `None` + /// tries the empty password). Fails with `NeedsPassword` / `WrongPassword` as appropriate. + pub fn open_with_password(data: Arc>, password: Option<&str>) -> Result { + Self::open_with_stream_limit(data, password, crate::cos::object::MAX_DECODED) + } + + /// Open with a tighter decompression bound for object and cross-reference streams. + /// Inspection can use this without changing the editing reader's default limit. Such a + /// document is read-only: objects in a larger object stream read as missing, so the + /// writers refuse to save it ([`CosError::ReadOnlyLimit`]). + pub fn open_with_stream_limit(data: Arc>, password: Option<&str>, stream_limit: usize) -> Result { + Self::open_as(data, Credentials::Password(password), stream_limit) + } + + /// Open `data`, an earlier revision of this file (the bytes a signature covers) or another + /// save of it, with this document's credentials: an encrypted one must have the same + /// `/Encrypt` dictionary and `/ID` as this document, whose file key is then reused, so no + /// password is needed or kept; unencrypted data opens as [`open`](Self::open) does. Data + /// encrypted differently fails with `NeedsPassword`. + pub fn open_with_same_credentials(&self, data: Arc>) -> Result { + let credentials = match &self.security { + Some(handler) => Credentials::Handler { handler, id0: self.first_id() }, + None => Credentials::Password(None), + }; + Self::open_as(data, credentials, self.stream_limit) + } + + fn open_as(data: Arc>, credentials: Credentials<'_>, stream_limit: usize) -> Result { + // Viewers accept files whose header is missing or damaged as long as the body looks + // like PDF; so do we (a note goes to the repair log). + let header = find(&data, b"%PDF-", 0, 1024); + let headerless = header.is_none(); + if headerless && find(&data, b" obj", 0, 4096).is_none() { + return Err(CosError::NotPdf); + } + let header_offset = header.unwrap_or(0); + let version = match header { + Some(h) => String::from_utf8_lossy(&data[(h + 5).min(data.len())..(h + 8).min(data.len())]).trim().to_string(), + None => "1.4".to_string(), + }; + let mut doc = Document { + data, + endstreams: Arc::default(), + source_identity: Arc::new(()), + entries: Arc::new(XrefIndex::empty()), + trailer: Dict::new(), + revisions: Arc::new(Vec::new()), + xref_oddities: Arc::default(), + located: Arc::default(), + repair_log: Arc::new(Vec::new()), + unlinked_update: false, + cache: Arc::default(), + objstms: Arc::default(), + unreadable: Arc::default(), + jbig2_globals: Arc::default(), + stream_limit: stream_limit.min(crate::cos::object::MAX_DECODED), + overlay: BTreeMap::new(), + next_num: 1, + header_offset, + version, + security: None, + encrypt_num: None, + encrypt_members: Arc::default(), + unfound: Arc::default(), + encryption_changed: false, + full_save: false, + new_id: false, + out_security: None, + out_encrypt_num: None, + }; + let mut log = Vec::new(); + if headerless { + log.push("the %PDF- header is missing; reading the file as PDF 1.4".into()); + } + let parsed = doc.read_xref_chain(&mut log); + let mut authenticated = false; + // The newest section cut the page tree off from the chain (see `update_cuts_off_the_pages`). + let mut cut_off = false; + let ok = match parsed { + Ok((entries, trailer, revisions, oddities)) => { + doc.entries = Arc::new(entries); + doc.trailer = trailer; + doc.revisions = Arc::new(revisions); + doc.xref_oddities = Arc::new(oddities); + doc.fill_xref_gaps(&mut log); + // Authenticate before checking the catalog: it may sit in an encrypted object + // stream, which reads as garbage until the key is known. + if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() { + doc.authenticate(&enc, &credentials)?; + authenticated = true; + } + let catalog = doc.root_is_catalog(); + cut_off = catalog && doc.update_cuts_off_the_pages(); + catalog && !cut_off + } + Err(e) => { + log.push(format!("cross-reference data unreadable ({e}); rebuilding from object headers")); + false + } + }; + if cut_off { + log.push(UNLINKED_UPDATE.into()); + doc.unlinked_update = true; + } else if !ok && !doc.entries.is_empty() { + log.push("cross-reference data does not lead to a valid catalog; rebuilding from object headers".into()); + } + if !ok { + doc.reconstruct(&mut log)?; + } + // Header generations are plaintext, including in encrypted files. Inspect only the + // bounded headers, not object bodies, and retain the compact xref representation. + doc.note_xref_generation_repairs(&mut log); + doc.repair_log = Arc::new(log); + // Reconstruction may have found a different trailer: (re)authenticate against it. + if (!ok || !authenticated) + && let Some(enc) = doc.trailer.get(b"Encrypt").cloned() + { + doc.authenticate(&enc, &credentials)?; + } + if Arc::make_mut(&mut doc.entries).remove_exhausted() { + Arc::make_mut(&mut doc.repair_log).push("skipped exhausted xref object number 4294967295".into()); + } + let max = doc.entries.max_num(); + let size = u32::try_from(doc.trailer.int(b"Size").unwrap_or(0).max(0)).unwrap_or(u32::MAX); + if size == u32::MAX { + Arc::make_mut(&mut doc.repair_log).push("exhausted trailer /Size; new objects use unused object numbers".into()); + } + doc.next_num = max.saturating_add(1).max(size).max(1); + Ok(doc) + } + + /// Whether the chain of cross-reference sections ends at the newest one (no `/Prev`, or one + /// that isn't an offset) although the file holds earlier sections, and the catalog's `/Pages` + /// is not among the objects that one section reaches. ISO 32000-2 §7.5.6 and §7.5.8.2 require + /// the `/Prev`; without it the earlier revisions, page tree and all, are cut off, so the file + /// is rebuilt from its object headers as a damaged one is (the newest definition of each + /// object wins), and the repair log says why. (A file whose `/Root` itself is missing is + /// rebuilt anyway.) + fn update_cuts_off_the_pages(&self) -> bool { + if self.revisions.len() != 1 { + return false; + } + let Some(root) = self.root() else { return false }; + let Ok(catalog) = self.try_get_reference(root) else { return false }; + let Object::Dict(catalog) = &*catalog else { return false }; + // The page tree first: the scan for an earlier `startxref` reads the whole file. + !catalog.get(b"Pages").is_some_and(|pages| matches!(&*self.resolve(pages), Object::Dict(_))) && earlier_sections(&self.data) + } + + fn authenticate(&mut self, enc: &Object, credentials: &Credentials<'_>) -> Result<(), CosError> { + // Authentication can change how the same input is decoded. + self.source_identity = Arc::new(()); + let (num, dict) = match enc { + Object::Ref(r) => (Some(r.num), self.get(*r).as_dict().cloned()), + Object::Dict(d) => (None, Some(d.clone())), + _ => (None, None), + }; + let Some(dict) = dict else { + // A dangling /Encrypt: viewers treat the file as unencrypted, and so do we. + let mut log = self.repair_log.as_ref().clone(); + log.push("the trailer names an /Encrypt dictionary that does not exist; reading the file as unencrypted".into()); + self.repair_log = Arc::new(log); + self.trailer.remove(b"Encrypt"); + return Ok(()); + }; + let params = crate::cos::security::encrypt_dict(self, &dict); + let id0 = self.first_id(); + // Read here, while nothing is decrypted yet. + let members = self.encryption_members(enc); + let handler = match credentials { + Credentials::Password(password) => match crate::crypt::SecurityHandler::open(params.clone(), &id0, *password) { + Ok(handler) => Arc::new(handler), + // No password, or the empty one: what only the embedded files need isn't asked. + Err(crate::crypt::CryptError::WrongPassword) => match self.without_password(&dict, params) { + Some(handler) if password.is_none_or(str::is_empty) => Arc::new(handler), + _ if password.is_none() => return Err(CosError::NeedsPassword), + _ => return Err(CosError::WrongPassword), + }, + Err(other) => return Err(CosError::Security(other.to_string())), + }, + // The same encryption as the document whose key this is: that key opens it. A handler + // without the key opens only what it would open by itself. + Credentials::Handler { handler, id0: known } if *handler.dict() == params && *known == id0 => { + if !handler.has_key() && !crate::cos::security::efopen_only(self, &dict) { + return Err(CosError::NeedsPassword); + } + Arc::clone(handler) + } + Credentials::Handler { .. } => return Err(CosError::NeedsPassword), + }; + self.security = Some(handler); + self.encrypt_num = num; + self.encrypt_members = Arc::new(members); + // Objects read while locating the catalog were read before decryption was possible. + self.cache = Arc::default(); + self.objstms = Arc::default(); + self.unreadable = Arc::default(); + Ok(()) + } + + /// The handler for a document opened without a password that only its embedded files need + /// (see [`crate::cos::security::efopen_only`]). Not a repair: ISO 32000-2 §7.6.6 has such a + /// document open so; [`Document::embedded_files_locked`] tells it. + fn without_password(&self, dict: &Dict, params: crate::crypt::EncryptDict) -> Option { + crate::cos::security::efopen_only(self, dict).then(|| crate::crypt::SecurityHandler::without_key(params)).flatten() + } + + /// A document opened without the password its embedded files need takes either password + /// later: with the file key its embedded files decrypt, so what was read as stored is read + /// again. Refused while an edited stream holds data read as stored, which the key would + /// encrypt a second time. + fn unlock_embedded_files(&mut self, password: &str) -> Result<(), CosError> { + let Some(current) = self.security.as_deref() else { return Ok(()) }; + if self.overlay_entries().any(|(_, _, o)| matches!(o.as_deref(), Some(Object::Stream(s)) if self.reads_as_stored(s))) { + return Err(CosError::Security("an embedded file was edited before the password was given: save or undo that edit first".into())); + } + let handler = crate::crypt::SecurityHandler::open(current.dict().clone(), &self.first_id(), Some(password)).map_err(|e| match e { + crate::crypt::CryptError::WrongPassword => CosError::WrongPassword, + other => CosError::Security(other.to_string()), + })?; + self.security = Some(Arc::new(handler)); + self.source_identity = Arc::new(()); + self.cache = Arc::default(); + self.objstms = Arc::default(); + Ok(()) + } + + /// Whether the document opened without the password its embedded files need (`/AuthEvent + /// /EFOpen`): their data is held, and saved, as stored, and isn't to be read as the files. + pub fn embedded_files_locked(&self) -> bool { + self.security.as_deref().is_some_and(|h| !h.has_key()) + } + + /// Whether `s`, a stream of this document, holds encrypted data that reads as stored because the + /// document opened without the password its embedded files need (see + /// [`Document::embedded_files_locked`]). Such data isn't the file it encrypts. + pub fn reads_as_stored(&self, s: &crate::cos::Stream) -> bool { + self.security.as_deref().is_some_and(|h| crate::cos::security::stream_reads_as_stored(self, h, s)) + } + + /// The first file identifier, part of the encryption key (empty when there is none). + fn first_id(&self) -> Vec { + let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); + let first = id.as_deref().and_then(Object::as_array).and_then(|a| a.first()).map(|s| self.resolve(s)); + first.as_deref().and_then(Object::as_string).map(|s| s.bytes.clone()).unwrap_or_default() + } + + /// The objects `/Encrypt`, given as `enc`, and `/ID` refer to, the ones they refer to, and so + /// on, up to [`MAX_ENCRYPT_MEMBERS`]. Their strings aren't encrypted (ISO 32000-2 §7.6.2): the + /// key is made from them, so they have to read as stored, and stay so. A number with no object + /// behind it, and a stream, which those dictionaries never hold, isn't one: an object added + /// under that number later, or a page's content a hostile `/Encrypt` names, is encrypted. + fn encryption_members(&self, enc: &Object) -> HashSet { + let mut pending = Vec::new(); + crate::cos::writer::collect_refs(enc, &mut |r| pending.push(r)); + if let Some(id) = self.trailer.get(b"ID") { + crate::cos::writer::collect_refs(id, &mut |r| pending.push(r)); + } + let mut members = HashSet::new(); + while let Some(r) = pending.pop() { + if members.len() >= MAX_ENCRYPT_MEMBERS || members.contains(&r.num) { + continue; + } + let held = self.get(r); + if matches!(&*held, Object::Null | Object::Stream(_)) { + continue; + } + members.insert(r.num); + crate::cos::writer::collect_refs(&held, &mut |x| pending.push(x)); + } + members + } + + /// Whether object `num` is written as it is held, being one of [`Document::encryption_members`], + /// unedited, and the save writing the same `/Encrypt`. What was edited or added is encrypted + /// whatever its number. + pub(crate) fn stored_as_is(&self, num: u32) -> bool { + !self.encryption_changed && !self.is_edited(num) && self.encrypt_members.contains(&num) + } + + /// Lift the restrictions of a document opened with its user password by entering the + /// owner (permissions) password (§7.6.4). Fails with `WrongPassword` for any other + /// password. Does nothing when the document is unencrypted or already opened as owner. + /// Both passwords unlock the same file key, so nothing read so far is decoded again. + pub fn unlock(&mut self, password: &str) -> Result<(), CosError> { + let Some(current) = self.security.as_deref() else { return Ok(()) }; + if !current.has_key() { + return self.unlock_embedded_files(password); + } + if current.auth() == crate::crypt::Auth::Owner { + return Ok(()); + } + let owner = current.authenticate_owner(&self.first_id(), password).map_err(|e| match e { + crate::crypt::CryptError::WrongPassword => CosError::WrongPassword, + other => CosError::Security(other.to_string()), + })?; + self.security = Some(Arc::new(owner)); + Ok(()) + } + + /// The security handler, when the document is encrypted. + pub fn security(&self) -> Option<&crate::crypt::SecurityHandler> { + self.security.as_deref() + } + + /// What the opening password allows (`None` for unencrypted documents: everything). + pub fn permissions(&self) -> Option { + self.security().map(|s| s.permissions()) + } + + /// Make the next save a full rewrite: earlier revisions (which still hold what an edit + /// removed) must not stay in the file. Redaction requires this. + pub fn require_full_save(&mut self) { + self.full_save = true; + } + + /// Like [`Document::require_full_save`], and the rewritten file also gets a new file identifier + /// (`/ID`), so that a redacted or sanitized copy never claims to be its source (ISO 32000-2 + /// §14.4). An encrypted file keeps the first string of its identifier, which its key depends on. + pub fn require_full_save_with_new_id(&mut self) { + self.full_save = true; + self.new_id = true; + } + + /// The next save must rewrite the whole file (see [`Document::require_full_save`]). + /// Repair-aware edits retaining two accepted generations cannot use one incremental slot. + pub fn full_save_required(&self) -> bool { + self.full_save || self.overlay.values().any(|slot| matches!(slot, Slot::RepairedSet(..))) + } + + /// The next full save gives the file a new identifier (see [`Document::require_full_save_with_new_id`]). + pub fn new_id_required(&self) -> bool { + self.new_id + } + + /// Security was set or removed since the document was opened (the next save applies it). + pub fn encryption_changed(&self) -> bool { + self.encryption_changed + } + + /// The handler used to write: the new one after `set_encryption` / `remove_encryption`, + /// otherwise the one the document was opened with. + pub(crate) fn output_security(&self) -> (Option<&crate::crypt::SecurityHandler>, Option) { + if self.encryption_changed { (self.out_security.as_deref(), self.out_encrypt_num) } else { (self.security.as_deref(), self.encrypt_num) } + } + + /// The security the next save writes: protection applied with `set_encryption`, none after + /// `remove_encryption`, otherwise the security the document was opened with. + pub fn output_handler(&self) -> Option<&crate::crypt::SecurityHandler> { + self.output_security().0 + } + + /// Protect the document with a password (§7.6.4). Takes effect on the next save, which is + /// always a full rewrite. Returns the handler (authenticated as owner). + pub fn set_encryption(&mut self, params: &crate::crypt::NewEncryption) -> Result<(), CosError> { + // The file identifier is part of the key; make sure it exists and keep it. + let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); + let id0 = match id.as_deref() { + Some(Object::Array(a)) if a.len() == 2 => self.first_id(), + _ => { + let id = crate::crypt::file_id(¶ms.seed); + let s = Object::String(crate::cos::PdfString { bytes: id.clone(), hex: true }); + self.trailer.set(b"ID".to_vec(), Object::Array(vec![s.clone(), s])); + id + } + }; + let h = crate::crypt::create(params, &id0).map_err(|e| CosError::Security(e.to_string()))?; + let d = h.dict(); + let s = |b: &[u8]| Object::String(crate::cos::PdfString { bytes: b.to_vec(), hex: true }); + let mut e = Dict::new(); + e.set(b"Filter".to_vec(), Object::name("Standard")); + e.set(b"V".to_vec(), Object::Int(d.v)); + e.set(b"R".to_vec(), Object::Int(d.r)); + e.set(b"Length".to_vec(), Object::Int(d.length_bits)); + e.set(b"O".to_vec(), s(&d.o)); + e.set(b"U".to_vec(), s(&d.u)); + e.set(b"P".to_vec(), Object::Int(i64::from(d.p))); + if d.v >= 4 { + let mut cf = Dict::new(); + let mut std_cf = Dict::new(); + std_cf.set(b"Type".to_vec(), Object::name("CryptFilter")); + let (cfm, len) = if d.v >= 5 { ("AESV3", 32) } else { ("AESV2", 16) }; + std_cf.set(b"CFM".to_vec(), Object::name(cfm)); + std_cf.set(b"Length".to_vec(), Object::Int(len)); + std_cf.set(b"AuthEvent".to_vec(), Object::name("DocOpen")); + cf.set(b"StdCF".to_vec(), Object::Dict(std_cf)); + e.set(b"CF".to_vec(), Object::Dict(cf)); + e.set(b"StmF".to_vec(), Object::name("StdCF")); + e.set(b"StrF".to_vec(), Object::name("StdCF")); + if !d.encrypt_metadata { + e.set(b"EncryptMetadata".to_vec(), Object::Bool(false)); + } + } + if d.v >= 5 { + e.set(b"OE".to_vec(), s(&d.oe)); + e.set(b"UE".to_vec(), s(&d.ue)); + e.set(b"Perms".to_vec(), s(&d.perms)); + } + if let Some(Object::Ref(old)) = self.trailer.get(b"Encrypt").cloned() { + self.free(old); + } + let r = self.add(e); + self.trailer.set(b"Encrypt".to_vec(), Object::Ref(r)); + // Objects are still read with the original handler; saves use the new one. + self.out_security = Some(Arc::new(h)); + self.out_encrypt_num = Some(r.num); + self.encryption_changed = true; + Ok(()) + } + + /// Remove password protection (requires the document to be open — any authenticated + /// password; callers should require the owner password, §7.6.4). Applied by a full save. + pub fn remove_encryption(&mut self) { + if let Some(Object::Ref(r)) = self.trailer.get(b"Encrypt").cloned() { + self.free(r); + } + self.trailer.remove(b"Encrypt"); + self.out_security = None; + self.out_encrypt_num = None; + self.encryption_changed = true; + } + + /// Decrypt a freshly parsed indirect object if the document is encrypted. + pub(super) fn decrypted(&self, id: ObjRef, o: Object) -> Object { + match &self.security { + Some(h) if Some(id.num) != self.encrypt_num && !self.encrypt_members.contains(&id.num) => { + crate::cos::security::transform(self, h, &o, id.num, id.generation, crate::cos::security::Pass::Decrypt) + } + _ => o, + } + } +} + +/// The repair-log line for a file rebuilt because its last update has no `/Prev`. +const UNLINKED_UPDATE: &str = "the file's last update has no /Prev link to its earlier revisions, so their objects (the page tree among them) are unreachable through the cross-reference chain; rebuilding from object headers"; + +/// Whether `data` has more than one `startxref`: an update was appended to it, or it is +/// linearized (whose first-page section links to the main one). +fn earlier_sections(data: &[u8]) -> bool { + find(data, b"startxref", 0, data.len()).and_then(|at| find(data, b"startxref", at.saturating_add(9), data.len())).is_some() +} diff --git a/crates/pdf/src/cos/document/reader.rs b/crates/pdf/src/cos/document/reader.rs new file mode 100644 index 000000000..3e0b9684c --- /dev/null +++ b/crates/pdf/src/cos/document/reader.rs @@ -0,0 +1,957 @@ +#[cfg(test)] +use super::state::SECTION_READS; +use super::state::Unreadable; +use super::xref::{ + MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData, +}; +use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, HashSet, Lexer, ObjRef, Object, is_whitespace, parse_indirect, parse_indirect_shared}; + +impl Document { + #[allow(clippy::type_complexity)] + pub(super) fn read_xref_chain(&self, log: &mut Vec) -> Result<(XrefIndex, Dict, Vec, Vec<(u64, u32)>), CosError> { + let mut compact_log = Vec::new(); + if let Ok(Some((index, trailer, revisions))) = self.read_compact_xref_chain(&mut compact_log) { + let size = trailer.int(b"Size").unwrap_or(0).max(0); + if size > 0 && size <= i64::from(u32::MAX) && u64::from(index.max_num()) < size as u64 { + log.extend(compact_log); + // The compact reader declines any section with set-aside entries, so there are none here. + return Ok((index, trailer, revisions, Vec::new())); + } + } + let (entries, trailer, revisions, oddities, generation_repairs) = self.read_xref_chain_materialized(log)?; + Ok((XrefIndex::Materialized(Arc::new(entries), Arc::new(generation_repairs)), trailer, revisions, oddities)) + } + + fn read_compact_xref_chain(&self, log: &mut Vec) -> Result)>, CosError> { + let data = &self.data; + let tail_start = data.len().saturating_sub(4096); + let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?; + let mut lx = Lexer::new(data, sx + 9); + let first = match lx.object() { + Ok(Object::Int(n)) if n >= 0 => n as u64, + _ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }), + }; + let mut sections = Vec::new(); + let mut retained_rows = 0usize; + let mut retained_bytes = 0usize; + let mut trailer: Option = None; + let mut revisions = Vec::new(); + let mut visited = HashSet::new(); + let mut next = Some(first); + while let Some(off) = next.take() { + if !visited.insert(off) || visited.len() > 10_000 { + log.push(format!("cross-reference chain loops at offset {off}; stopped")); + break; + } + let ((primary, sect_trailer, is_stream), header_relative) = match self.read_compact_section(off as usize) { + Ok(Some(v)) => (v, false), + Ok(None) => return Ok(None), + Err(e) if self.header_offset > 0 => { + log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header")); + match self.read_compact_section(off as usize + self.header_offset)? { + Some(v) => (v, true), + None => return Ok(None), + } + } + // An unreadable older section loses every object only it locates: the + // materialized reader logs it and the file is rebuilt from object headers (#1656). + Err(e) => return Err(e), + }; + revisions.push(Revision { xref_offset: off, is_stream, header_relative }); + if primary.renumbered { + log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0")); + } + let Some(rows) = retained_rows.checked_add(primary.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) }; + let Some(bytes) = retained_bytes.checked_add(primary.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES) else { + return Ok(None); + }; + retained_rows = rows; + retained_bytes = bytes; + let hybrid = if let Some(x) = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0) { + match self.read_compact_section(x as usize) { + Ok(Some((section, _, _))) => { + let Some(rows) = retained_rows.checked_add(section.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { + return Ok(None); + }; + let Some(bytes) = retained_bytes.checked_add(section.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES) + else { + return Ok(None); + }; + retained_rows = rows; + retained_bytes = bytes; + Some(section) + } + Ok(None) => return Ok(None), + Err(e) => { + log.push(format!("hybrid /XRefStm at {x} unreadable: {e}")); + None + } + } + } else { + None + }; + if let Some(hybrid) = &hybrid { + // A hybrid entry that disagrees with the table is set aside by the materialized reader. + let mut numbers = Vec::new(); + hybrid.extend_numbers(&mut numbers); + for num in numbers { + let Some(entry) = hybrid.lookup(num, &self.data) else { continue }; + if primary.lookup(num, &self.data).is_some_and(|old| !matches!(old, XrefEntry::Free { .. }) && old != entry) { + return Ok(None); + } + } + } + sections.push(XrefRevisionSection { primary, hybrid }); + next = prev_offset(data, §_trailer, off, |num| sections.iter().find_map(|s| s.lookup(num, data)), log); + if trailer.is_none() { + trailer = Some(sect_trailer); + } + } + revisions.reverse(); + let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?; + for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { + trailer.remove(k); + } + let mut max_num = 0; + let mut has_entries = false; + for revision in §ions { + max_num = max_num.max(revision.primary.max_num); + has_entries |= revision.primary.has_entries; + if let Some(hybrid) = &revision.hybrid { + max_num = max_num.max(hybrid.max_num); + has_entries |= hybrid.has_entries; + } + } + Ok(Some((XrefIndex::Compact { revisions: sections, max_num, has_entries }, trailer, revisions))) + } + + fn read_compact_section(&self, off: usize) -> Result, CosError> { + let data = &self.data; + let mut lx = Lexer::new(data, off); + if lx.eat_keyword(b"xref") { + let mut ranges = Vec::new(); + let mut layout_ok = true; + let mut max_num = 0; + let mut has_entries = false; + let mut previous_end = None; + let mut row_count = 0usize; + let mut renumbered = false; + loop { + lx.skip_ws(); + if lx.eat_keyword(b"trailer") { + let t = lx.object()?; + let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?; + if !layout_ok { + return Ok(None); + } + return Ok(Some((XrefSection { data: XrefSectionData::Table, ranges, row_count, max_num, has_entries, renumbered }, d, false))); + } + let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { + return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() }); + }; + if !valid_xref_range(start, count) || count > 10_000_000 { + return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() }); + } + let shifted = subsection_start(data, self.header_offset, lx.pos, start, count); + renumbered |= shifted != start; + let start = shifted; + let Some(next_rows) = usize::try_from(count).ok().and_then(|count| row_count.checked_add(count)) else { return Ok(None) }; + if next_rows > MAX_COMPACT_XREF_ROWS { + return Ok(None); + } + row_count = next_rows; + let mut range_offset = None; + let mut stride = None; + let mut previous_record = None; + for i in 0..count { + lx.skip_ws(); + let record = lx.pos; + if range_offset.is_none() { + range_offset = Some(record); + } + if let Some(previous) = previous_record { + let Some(delta) = record.checked_sub(previous) else { + layout_ok = false; + continue; + }; + if stride.is_some_and(|known| known != delta) { + layout_ok = false; + } + stride.get_or_insert(delta); + } + previous_record = Some(record); + let o = lx.token(); + lx.skip_ws(); + let _generation = lx.token(); + lx.skip_ws(); + let kind = lx.token(); + let num = (start + i) as u32; + let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::().ok()); + match (kind, offset) { + (b"n", Some(offset)) if offset != 0 => { + has_entries = true; + max_num = max_num.max(num); + } + (b"n", Some(_)) => { + // Set aside by the materialized reader, which records it. + if num != 0 { + return Ok(None); + } + } + (b"f", _) => { + has_entries = true; + max_num = max_num.max(num); + } + _ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }), + } + } + if count > 0 { + let range_start = start as u32; + let range_count = count as u32; + let end = u64::from(range_start) + u64::from(range_count); + if previous_end.is_some_and(|previous| u64::from(range_start) < previous) { + layout_ok = false; + } + previous_end = Some(end); + ranges.push(XrefRange { + start: range_start, + count: range_count, + offset: range_offset.unwrap_or(lx.pos), + stride: stride.unwrap_or(1), + }); + } + } + } + let (_, obj) = parse_indirect(data, off, &|_| None)?; + let Object::Stream(stream) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) }; + if stream.dict.name(b"Type") != Some(b"XRef") { + return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() }); + } + let w: Vec = stream + .dict + .get(b"W") + .and_then(Object::as_array) + .map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect()) + .unwrap_or_default(); + if w.len() < 3 { + return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() }); + } + let widths = [w[0], w[1], w[2]]; + let row = widths.iter().sum::(); + if row == 0 { + return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() }); + } + let size = stream.dict.int(b"Size").unwrap_or(0).max(0); + let index: Vec = match stream.dict.get(b"Index").and_then(Object::as_array) { + Some(a) => a.iter().filter_map(Object::as_int).collect(), + None => vec![0, size], + }; + let mut requested_rows = 0usize; + let mut previous_end = None; + for pair in index.chunks(2) { + let [start, count] = pair else { break }; + if !valid_xref_range(*start, *count) { + return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() }); + } + let Ok(count) = usize::try_from(*count) else { return Ok(None) }; + let Some(next) = requested_rows.checked_add(count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) }; + if count > 0 { + let end = (*start as u64).saturating_add(count as u64); + if previous_end.is_some_and(|previous| (*start as u64) < previous) { + return Ok(None); + } + previous_end = Some(end); + } + requested_rows = next; + } + if requested_rows.checked_mul(row).is_none_or(|bytes| bytes > MAX_COMPACT_XREF_BYTES) { + return Ok(None); + } + let raw = stream.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit).min(MAX_COMPACT_XREF_BYTES))?; + if raw.capacity() > MAX_COMPACT_XREF_BYTES { + return Ok(None); + } + let mut ranges = Vec::new(); + let mut row_start = 0usize; + let available_rows = raw.len() / row; + for pair in index.chunks(2) { + let [start, count] = pair else { break }; + let range_start = *start as u32; + let Ok(requested) = usize::try_from(*count) else { return Ok(None) }; + let range_count = requested.min(available_rows.saturating_sub(row_start)); + if range_count > 0 { + let Ok(range_count_u32) = u32::try_from(range_count) else { return Ok(None) }; + let Some(byte_offset) = row_start.checked_mul(row) else { return Ok(None) }; + ranges.push(XrefRange { start: range_start, count: range_count_u32, offset: byte_offset, stride: row }); + } + let Some(next) = row_start.checked_add(range_count) else { return Ok(None) }; + row_start = next; + } + // Rows the materialized reader sets aside (reserved types, offset 0) are left to it. + for range in &ranges { + for i in 0..range.count as usize { + let at = range.offset + i * row; + let Some(r) = raw.get(at..at + row) else { break }; + let field = |from: usize, width: usize, default: u64| -> u64 { + if width == 0 { + return default; + } + r[from..from + width].iter().fold(0u64, |acc, byte| acc << 8 | u64::from(*byte)) + }; + let kind = field(0, widths[0], 1); + let offset = field(widths[0], widths[1], 0); + if kind > 2 || (kind == 1 && offset == 0 && range.start + i as u32 != 0) { + return Ok(None); + } + } + } + let max_num = ranges.iter().filter_map(|range| range.start.checked_add(range.count.saturating_sub(1))).max().unwrap_or(0); + let row_count = row_start; + let section = XrefSection { + data: XrefSectionData::Stream { bytes: Arc::new(raw), widths }, + ranges, + row_count, + max_num, + has_entries: false, + renumbered: false, + }; + let has_entries = section.has_any_entry(); + let section = XrefSection { has_entries, ..section }; + Ok(Some((section, stream.dict.clone(), true))) + } + + /// Read the cross-reference chain from `startxref` into one map of entries, newer sections + /// winning, with the trailer, the revisions oldest first and the set-aside entries. A chain + /// that lists more than [`MAX_XREF_OBJECTS`] rows in all is refused with + /// [`CosError::Syntax`], and the caller rebuilds the file from object headers (#5082). + #[allow(clippy::type_complexity)] + pub(super) fn read_xref_chain_materialized( + &self, + log: &mut Vec, + ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { + self.read_xref_chain_materialized_with_limit(log, MAX_XREF_OBJECTS) + } + + /// [`Self::read_xref_chain_materialized`] refusing a chain past `row_limit` rows in all + /// instead of [`MAX_XREF_OBJECTS`], so tests can reach the limit with a few rows. + #[allow(clippy::type_complexity)] + pub(super) fn read_xref_chain_materialized_with_limit( + &self, + log: &mut Vec, + row_limit: usize, + ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { + let data = &self.data; + let tail_start = data.len().saturating_sub(4096); + let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?; + let mut lx = Lexer::new(data, sx + 9); + let first = match lx.object() { + Ok(Object::Int(n)) if n >= 0 => n as u64, + _ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }), + }; + let mut entries = BTreeMap::new(); + let mut trailer: Option = None; + let mut revisions = Vec::new(); + let mut oddities = Vec::new(); + let mut visited = HashSet::new(); + // Hybrid streams already read, by offset: their entries, or `None` when unreadable. A newer + // section has merged every entry of one into `entries`, so an older section that names it + // again only has its own table compared with them, without decoding the stream again. + // At most MAX_XREF_OBJECTS entries are kept in all; a stream past that is read again. + let mut hybrids: HashMap>> = HashMap::new(); + let (mut hybrid_rows, mut noted) = (0usize, HashSet::new()); + let mut generation_repairs = Vec::new(); + let mut recorded = HashSet::new(); + let mut diagnose = true; + // Every row the sections (and their hybrid streams) list, counted by `read_section` as it + // reads them, before any are merged or deduplicated. + let mut merged_rows = 0usize; + let mut next = Some(first); + while let Some(off) = next.take() { + if !visited.insert(off) || visited.len() > 10_000 { + log.push(format!("cross-reference chain loops at offset {off}; stopped")); + break; + } + // This section's own entries; older sections only fill what newer ones left out. + let mut sect = BTreeMap::new(); + // Object numbers whose entry in this section was set aside ([`Self::xref_oddities`]). + let mut odd = Vec::new(); + let rows_before = merged_rows; + let ((sect_trailer, is_stream), header_relative) = match self.read_section(off as usize, &mut sect, &mut odd, &mut merged_rows, log) { + Ok(v) => (v, false), + Err(e) if self.header_offset > 0 => { + log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header")); + sect.clear(); + odd.clear(); + merged_rows = rows_before; + (self.read_section(off as usize + self.header_offset, &mut sect, &mut odd, &mut merged_rows, log)?, true) + } + Err(e) => { + // An unreadable older section loses every object only it locates (the page + // tree, often): rebuild from object headers, as readers do (#1656). + if !revisions.is_empty() { + log.push(format!("older cross-reference section at {off} is unreadable: {e}")); + } + return Err(e); + } + }; + revisions.push(Revision { xref_offset: off, is_stream, header_relative }); + for (&num, &entry) in § { + if !diagnose { + break; + } + diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); + } + // Hybrid files (§7.5.8.4): the /XRefStm entries supplement the table and replace the + // free entries it gives objects only a cross-reference stream can locate. + let hybrid = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0); + if let Some((x, read)) = hybrid.and_then(|x| hybrids.get(&x).map(|read| (x, read))) { + if noted.insert(x) { + log.push(format!("hybrid /XRefStm at {x} named again by the section at {off}; read once")); + } + if let Some(stm) = read { + for (num, old) in § { + // The table places the object somewhere other than the stream does. + if !matches!(old, XrefEntry::Free { .. }) && stm.get(num).is_some_and(|e| e != old) { + odd.push(*num); + } + } + } + } else if let Some(x) = hybrid { + let mut stm = BTreeMap::new(); + let read = match self.read_section(x as usize, &mut stm, &mut odd, &mut merged_rows, log) { + Ok(_) => { + for (&num, &entry) in &stm { + if !diagnose { + break; + } + diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); + } + for (&num, &e) in &stm { + match sect.get(&num) { + None | Some(XrefEntry::Free { .. }) => { + sect.insert(num, e); + } + // The table already places the object somewhere else. + Some(old) if *old != e => odd.push(num), + Some(_) => {} + } + } + Some(stm) + } + Err(e) => { + log.push(format!("hybrid /XRefStm at {x} unreadable: {e}")); + None + } + }; + let rows = read.as_ref().map_or(0, BTreeMap::len); + if let Some(total) = hybrid_rows.checked_add(rows).filter(|total| *total <= MAX_XREF_OBJECTS) { + hybrid_rows = total; + hybrids.insert(x, read); + } + } + // Each section's rows are capped, so the chain is too: at most `row_limit` rows + // (MAX_XREF_OBJECTS outside tests) read in all, or the file is rebuilt from object + // headers, as when one section's stream decodes past that. This counts every row + // read, before deduplication, not the map's size: a chain that relists the same + // objects, or a stream whose `/Index` repeats a range (`[0 4 0 4]`), costs as much to + // merge (#5082). Rows set aside are among the rows read, and a row is set aside at + // most twice (in its section and against a hybrid stream), so they stay bounded too. A + // section's rows (its hybrid stream's included) never outnumber the rows the compact + // reader counts for it, and that reader stops at about a quarter of MAX_XREF_OBJECTS: + // no chain within its limit is refused. + if merged_rows > row_limit { + return Err(CosError::Syntax { + offset: usize::try_from(off).unwrap_or(usize::MAX), + detail: format!("cross-reference chain lists more than {row_limit} entries"), + }); + } + oddities.extend(odd.into_iter().map(|num| (off, num))); + for (num, e) in sect { + entries.entry(num).or_insert(e); + } + next = prev_offset(data, §_trailer, off, |num| entries.get(&num).copied(), log); + if trailer.is_none() { + trailer = Some(sect_trailer); + } + } + revisions.reverse(); + let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?; + for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { + trailer.remove(k); + } + Ok((entries, trailer, revisions, oddities, generation_repairs)) + } + + /// Read one section at `off`, adding entries not already known (newer sections win). `odd` + /// collects the object numbers whose entry here is set aside ([`Self::xref_oddities`]), and + /// `counted` grows by every row read, repeated and set-aside rows included (#5082). + fn read_section( + &self, + off: usize, + entries: &mut BTreeMap, + odd: &mut Vec, + counted: &mut usize, + log: &mut Vec, + ) -> Result<(Dict, bool), CosError> { + #[cfg(test)] + SECTION_READS.with(|c| c.set(c.get() + 1)); + let data = &self.data; + let mut lx = Lexer::new(data, off); + if lx.eat_keyword(b"xref") { + loop { + lx.skip_ws(); + if lx.eat_keyword(b"trailer") { + let t = lx.object()?; + let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?; + return Ok((d, false)); + } + let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { + return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() }); + }; + if !valid_xref_range(start, count) || count > 10_000_000 { + return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() }); + } + let shifted = subsection_start(data, self.header_offset, lx.pos, start, count); + if shifted != start { + log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0")); + } + let start = shifted; + for i in 0..count { + *counted = counted.saturating_add(1); + lx.skip_ws(); + let o = lx.token(); + lx.skip_ws(); + let g = lx.token(); + lx.skip_ws(); + let kind = lx.token(); + let num = (start + i) as u32; + let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::().ok()); + let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0).min(u16::MAX as u32) as u16; + let entry = match (kind, offset) { + (b"n", Some(offset)) => XrefEntry::InFile { offset, generation }, + (b"f", _) => XrefEntry::Free { next_generation: generation }, + _ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }), + }; + // Offset 0 "in use" entries are a known producer bug: ignore them. (Object 0 + // is never a real object, so its entry cannot mislead anyone.) + if matches!(entry, XrefEntry::InFile { offset: 0, .. }) { + if num != 0 { + odd.push(num); + } + continue; + } + if *entries.entry(num).or_insert(entry) != entry { + odd.push(num); + } + } + } + } + // Cross-reference stream. + let (_, obj) = parse_indirect(data, off, &|_| None)?; + let Object::Stream(s) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) }; + if s.dict.name(b"Type") != Some(b"XRef") { + return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() }); + } + let w: Vec = s + .dict + .get(b"W") + .and_then(Object::as_array) + .map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect()) + .unwrap_or_default(); + if w.len() < 3 { + return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() }); + } + let size = s.dict.int(b"Size").unwrap_or(0).max(0); + let index: Vec = match s.dict.get(b"Index").and_then(Object::as_array) { + Some(a) => a.iter().filter_map(Object::as_int).collect(), + None => vec![0, size], + }; + let row = w[0] + w[1] + w[2]; + if row == 0 { + return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() }); + } + // One row per object, and a document has at most 8,388,607 indirect objects (the classic + // implementation limit): decoding more than that (about 200 MB at the widest rows) is a + // decompression bomb, not cross-reference data. The section is then reconstructed. + let raw = s.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit))?; + let field = |r: &[u8], from: usize, len: usize, default: u64| -> u64 { + if len == 0 { + return default; + } + r[from..from + len].iter().fold(0u64, |acc, b| acc << 8 | *b as u64) + }; + let mut rows = raw.chunks_exact(row); + for pair in index.chunks(2) { + let [start, count] = pair else { break }; + if !valid_xref_range(*start, *count) { + return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() }); + } + for i in 0..*count { + let Some(r) = rows.next() else { break }; + *counted = counted.saturating_add(1); + let t = field(r, 0, w[0], 1); + let a = field(r, w[0], w[1], 0); + let b = field(r, w[0] + w[1], w[2], 0); + let num = (*start + i) as u32; + let entry = match t { + 0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 }, + 1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 }, + 2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 }, + // Reserved types are treated as null references (§7.5.8.3). + _ => { + odd.push(num); + continue; + } + }; + if matches!(entry, XrefEntry::InFile { offset: 0, .. }) { + if num != 0 { + odd.push(num); + } + continue; + } + if *entries.entry(num).or_insert(entry) != entry { + odd.push(num); + } + } + } + Ok((s.dict.clone(), true)) + } + + /// Cross-reference data that leaves out objects the file contains: a subsection numbered one + /// too low (object 0 in use, the last object without an entry), or an in-use entry at offset 0 + /// that nothing else replaces. (A number no section lists is not evidence: it is a null + /// object, §7.3.10, and valid files have them below an oversized `/Size`.) Viewers rebuild + /// the table and show these objects; reading them as null would drop them from every copy. + /// When one of those shows, the file is scanned once for object headers, and wrong offsets + /// are looked up there (`load`). Only the numbers the damage leaves out take the last + /// header's location: one whose entry was set aside, or the last object of a subsection + /// numbered one too low. Any other unlisted number stays null, and existing + /// entries are never replaced: an object freed by an update stays freed. + pub(super) fn fill_xref_gaps(&mut self, log: &mut Vec) { + let entries = &self.entries; + let shifted = matches!(entries.lookup(0, &self.data), Some(XrefEntry::InFile { .. })); + let set_aside = self.xref_oddities.iter().any(|(_, num)| entries.lookup(*num, &self.data).is_none()); + if !(shifted || set_aside) { + return; + } + let mut located = BTreeMap::new(); + for (num, offset, generation, _) in object_headers(&self.data) { + located.insert(num, (offset, generation)); + } + let mut filled = entries.to_map(&self.data); + // A subsection `0 N` whose rows are those of objects 1 to N leaves out object N, whatever + // newer sections say about its neighbours. + let past_shift: HashSet = if shifted { + self.revisions + .iter() + .filter(|r| !r.is_stream) + .filter_map(|r| usize::try_from(r.xref_offset).ok().map(|o| if r.header_relative { o.saturating_add(self.header_offset) } else { o })) + .filter_map(|at| shifted_subsection_end(&self.data, at)) + .collect() + } else { + HashSet::new() + }; + let odd: HashSet = self.xref_oddities.iter().map(|(_, n)| *n).collect(); + let mut found = 0usize; + for (&num, &(offset, generation)) in &located { + let left_out = past_shift.contains(&num) || odd.contains(&num); + if num != 0 && !filled.contains_key(&num) && left_out { + filled.insert(num, XrefEntry::InFile { offset, generation }); + found += 1; + } + } + if found > 0 { + log.push(format!("the cross-reference data leaves out {found} objects the file contains; located them by their object headers")); + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(filled), Arc::default())); + } + self.located = Arc::new(located); + } + + /// Rebuild the cross-reference index by scanning for `N G obj` headers (§7.5.4 recovery). + pub(super) fn reconstruct(&mut self, log: &mut Vec) -> Result<(), CosError> { + let data = self.data.clone(); + let mut entries: BTreeMap = BTreeMap::new(); + let mut objstms = Vec::new(); + // Offsets of objects that may be cross-reference streams: their dictionaries are the + // trailers of files that have no `trailer` keyword (§7.5.8). + let mut xref_streams = Vec::new(); + let headers = object_headers(&data); + // Where `/XRef` occurs in the file, found in one pass. + let mut xref_marks = Vec::new(); + let mut mark_from = 0; + while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { + xref_marks.push(p); + mark_from = p + 1; + } + for &(num, offset, generation, body) in &headers { + entries.insert(num, XrefEntry::InFile { offset, generation }); + objstms.push(num); + // `/XRef` within the 1024 bytes after the header (the first one that is there). + let first = xref_marks.partition_point(|&p| p < body); + if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) + && let Ok(at) = usize::try_from(offset) + { + xref_streams.push(at); + } + } + if entries.is_empty() { + return Err(CosError::Syntax { offset: 0, detail: "no objects found".into() }); + } + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()), Arc::default())); + self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); + // Objects inside object streams. + for num in objstms { + if let Ok(o) = self.try_get(num) + && let Object::Stream(s) = &*o + && s.dict.name(b"Type") == Some(b"ObjStm") + && let Ok(stm) = self.objstm(num) + { + for (idx, member) in stm.index.iter().enumerate() { + if let Some((inner, _)) = member { + entries.entry(*inner).or_insert(XrefEntry::InStream { stream: num, index: idx as u32 }); + } + } + } + } + let object_count = entries.len(); + self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries), Arc::default())); + // Failures from the scan stand: the members added change no scanned entry, and a + // `/Length` found only now cannot make a stream unreadable. + self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a + // usable /Root (last in the file, whichever kind), else the catalog itself. One whose + // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another + // dictionary, such as /Info, wherever it is; an FDF's /Root (no /Type, no /Pages) still + // counts when nothing better is there. + // A file whose only trailers are cross-reference streams (and whose `startxref` is gone + // or points nowhere) keeps its /Info, /ID and /Encrypt this way. + let rank = |at: usize, d: &Dict| { + let root = self.try_get_reference(d.reference(b"Root")?).ok()?; + let Object::Dict(c) = &*root else { return None }; + Some((c.name(b"Type") == Some(b"Catalog") || c.contains(b"Pages"), at)) + }; + let mut trailer: Option<((bool, usize), Dict)> = None; + let mut from = 0; + while let Some(p) = find(&data, b"trailer", from, data.len()) { + from = p + 7; + let mut lx = Lexer::new(&data, from); + if let Ok(Object::Dict(d)) = lx.object() + && let Some(r) = rank(p, &d) + && trailer.as_ref().is_none_or(|(best, _)| r > *best) + { + trailer = Some((r, d)); + } + } + for at in xref_streams { + if trailer.as_ref().is_some_and(|((catalog, p), _)| *catalog && *p > at) { + continue; + } + if let Ok((_, Object::Stream(s))) = parse_indirect_shared(&data, at, &|_| None, &self.endstreams) + && s.dict.name(b"Type") == Some(b"XRef") + && let Some(r) = rank(at, &s.dict) + && trailer.as_ref().is_none_or(|(best, _)| r > *best) + { + trailer = Some((r, s.dict)); + } + } + let trailer = match trailer { + Some((_, t)) => t, + None => { + let catalog = self + .entries + .object_numbers(&self.data) + .into_iter() + .find(|n| matches!(self.try_get(*n).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog"))); + let Some(c) = catalog else { return Err(CosError::Syntax { offset: 0, detail: "no document catalog found".into() }) }; + let mut t = Dict::new(); + t.set(b"Root".to_vec(), Object::Ref(ObjRef::new(c, self.generation(c)))); + t + } + }; + let mut trailer = trailer; + // What a cross-reference stream's dictionary carries for itself goes, as the chain + // readers drop it: a save writes its own. + for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { + trailer.remove(k); + } + self.trailer = trailer; + self.revisions = Arc::new(Vec::new()); + log.push(format!("rebuilt cross-reference index from {object_count} object headers")); + Ok(()) + } +} + +/// For a classic table at `at` whose first subsection is `0 N` and opens with an in-use row +/// (the rows of objects 1 to N), the number N its rows really end at. +fn shifted_subsection_end(data: &[u8], at: usize) -> Option { + let mut lx = Lexer::new(data, at); + if !lx.eat_keyword(b"xref") { + return None; + } + let (Ok(Object::Int(0)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { return None }; + lx.skip_ws(); + let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); + lx.skip_ws(); + lx.token(); + lx.skip_ws(); + match (offset, lx.token()) { + (Some(offset), b"n") if offset != 0 => u32::try_from(count).ok().filter(|n| *n > 0), + _ => None, + } +} + +/// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position +/// after `obj`). +fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { + let mut headers = Vec::new(); + let mut i = 0; + while i < data.len() { + // Object headers start at a line start (or after whitespace). + let at_start = i == 0 || is_whitespace(data[i - 1]) || data[i - 1] == b'>'; + if at_start && data[i].is_ascii_digit() { + let mut lx = Lexer::new(data, i); + let n = lx.token(); + lx.skip_ws(); + let g = lx.token(); + if !n.is_empty() && !g.is_empty() && n.iter().all(u8::is_ascii_digit) && g.iter().all(u8::is_ascii_digit) && lx.eat_keyword(b"obj") { + let num = std::str::from_utf8(n).ok().and_then(|s| s.parse::().ok()); + let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0); + if let Some(num) = num { + headers.push((num, i as u64, generation, lx.pos)); + } + i = lx.pos; + continue; + } + } + i += 1; + } + headers +} + +/// The offset of the section before the one whose trailer this is (`/Prev`), if any. Some writers +/// give it as a reference, `N 0 R`. When the sections read so far (`lookup`) place that object +/// and it holds an offset (iText's `indirectPrev.pdf`), that is the offset; otherwise, as the +/// tiff2pdf files in JHOVE's PDF-HUL-157 cases need, the sections before are at N. Either way the +/// reference is noted. Any other value that is not an offset is noted and ends the chain. +fn prev_offset(data: &[u8], trailer: &Dict, off: u64, lookup: impl Fn(u32) -> Option, log: &mut Vec) -> Option { + match trailer.get(b"Prev") { + Some(Object::Int(p)) if *p >= 0 => Some(*p as u64), + Some(Object::Ref(r)) => { + let held = match lookup(r.num) { + Some(XrefEntry::InFile { offset, generation }) if generation == r.generation => { + usize::try_from(offset).ok().and_then(|at| parse_indirect(data, at, &|_| None).ok()).and_then(|(at, value)| match value { + Object::Int(p) if at == *r && p >= 0 => Some(p as u64), + _ => None, + }) + } + _ => None, + }; + match held { + Some(p) => { + log.push(format!( + "the trailer at {off} gives /Prev as the reference {} {} R; reading the offset {p} it holds", + r.num, r.generation + )); + Some(p) + } + None => { + log.push(format!( + "the trailer at {off} gives /Prev as the reference {} {} R; reading it as offset {}", + r.num, r.generation, r.num + )); + Some(u64::from(r.num)) + } + } + } + Some(_) => { + log.push(format!("the trailer at {off} has a /Prev that is not an offset; the sections before it are unreachable")); + None + } + None => None, + } +} + +pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option { + let to = to.min(hay.len()); + if from >= to || needle.is_empty() || to - from < needle.len() { + return None; + } + let hay = &hay[from..to]; + // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. + const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; + let (k, rare) = + needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); + let mut i = k; + while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { + let start = i + p - k; + if hay.get(start..start + needle.len()) == Some(needle) { + return Some(start + from); + } + i += p + 1; + } + None +} + +/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which +/// the compiler turns into vector compares; only a block that holds the byte is searched. +fn position_of(hay: &[u8], byte: u8) -> Option { + const BLOCK: usize = 32; + let (blocks, rest) = hay.as_chunks::(); + for (i, block) in blocks.iter().enumerate() { + if block.iter().fold(false, |any, &b| any | (b == byte)) { + return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); + } + } + rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) +} + +pub(super) fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { + if hay.len() < needle.len() { + return None; + } + (from..=hay.len() - needle.len()).rev().find(|&i| &hay[i..i + needle.len()] == needle) +} + +/// The object number a classic xref subsection really starts at. A common producer bug heads the +/// table `1 n` and then writes object 0's free entry (`0000000000 65535 f`) as its first row, so +/// every row would be read one number too high and object 1 as free. Such a subsection is read +/// from 0 when its second row, read that way, locates an object that is indeed numbered 1, at +/// that offset or, as `load` reads offsets, relative to a header shifted by `header_offset`. +/// `rows` is the position just after the subsection header. +fn subsection_start(data: &[u8], header_offset: usize, rows: usize, start: i64, count: i64) -> i64 { + if start != 1 || count < 2 { + return start; + } + let mut lx = Lexer::new(data, rows); + let mut row = || { + lx.skip_ws(); + let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); + lx.skip_ws(); + let generation = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); + lx.skip_ws(); + (offset, generation, lx.token() == b"f") + }; + let (Some(0), Some(65535), true) = row() else { return start }; + let (Some(second), _, false) = row() else { return start }; + let object_one = |at: usize| { + if at >= data.len() { + return false; + } + let mut obj = Lexer::new(data, at); + matches!((obj.object(), obj.object()), (Ok(Object::Int(1)), Ok(Object::Int(_)))) && obj.eat_keyword(b"obj") + }; + let Ok(second) = usize::try_from(second) else { return start }; + if object_one(second) || (header_offset != 0 && object_one(second.saturating_add(header_offset))) { 0 } else { start } +} + +/// Object numbers must fit the representation used by references and the xref map. +/// Validate the whole subsection before adding entries, so neither arithmetic overflow nor +/// truncation can turn a damaged range into entries for unrelated objects. +fn valid_xref_range(start: i64, count: i64) -> bool { + u32::try_from(start).is_ok() && count >= 0 && start.checked_add(count).is_some_and(|end| end <= i64::from(u32::MAX) + 1) +} + +/// The most indirect objects a document can have (the classic implementation limit, ISO +/// 32000-1 Annex C): it bounds how much data a cross-reference stream can hold. +const MAX_XREF_OBJECTS: usize = 8_388_607; diff --git a/crates/pdf/src/cos/document/state.rs b/crates/pdf/src/cos/document/state.rs new file mode 100644 index 000000000..62ad2f8a3 --- /dev/null +++ b/crates/pdf/src/cos/document/state.rs @@ -0,0 +1,505 @@ +use super::xref::XrefEntry; +use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, ObjRef, Object, Weak}; + +thread_local! { + /// The objects being loaded on this thread, innermost last (see `Document::try_get`). + static LOADING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + /// Indirect `/Length` values resolved during the current outermost load: a damaged file + /// retries a parse several ways, and each retry would resolve the same lengths again + /// (exponential in the chain of streams whose lengths point at further streams). Each is + /// kept with whether resolving it read unsaved edits. + pub(super) static LENGTHS: std::cell::RefCell> = std::cell::RefCell::new(HashMap::new()); + /// What the loads running on this thread have read ([`ReadFrame`]): how many frames are + /// open, the object numbers read inside them, and how often that list was full. + static READS: std::cell::RefCell<(u32, Vec, u64)> = const { std::cell::RefCell::new((0, Vec::new(), 0)) }; + /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). + static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// How deep `Document::scan_for` is nested on this thread, and how many indirect `/Length` + /// values the outermost scan and those inside it have resolved ([`ScanLengths`]). + static SCAN_LENGTHS: std::cell::Cell<(u32, u32)> = const { std::cell::Cell::new((0, 0)) }; +} + +/// The indirect `/Length` values a scan of the file for an object may resolve, shared with the +/// scans it leads to. Each header in the file that starts with the object's number is parsed, and +/// a `/Length` that names an object the cross-reference table cannot place scans the file for that +/// one: a damaged file with many such headers, each naming another missing object, would read the +/// whole file once per header. Past the budget a `/Length` is left unresolved, and the stream is +/// read as before resolution, by its `endstream`. +pub(super) struct ScanLengths; + +impl ScanLengths { + /// A real object is defined once per revision, and its `/Length` once per definition. + pub(super) const MAX: u32 = 32; + + /// Marks a scan as running; the outermost one starts the budget afresh. + pub(super) fn enter() -> Self { + SCAN_LENGTHS.with(|c| { + let (depth, used) = c.get(); + c.set((depth.saturating_add(1), if depth == 0 { 0 } else { used })); + }); + ScanLengths + } + + /// Whether one more `/Length` may be resolved, counting it if so. + pub(super) fn take(&self) -> bool { + SCAN_LENGTHS.with(|c| { + let (depth, used) = c.get(); + if used >= Self::MAX { + return false; + } + c.set((depth, used.saturating_add(1))); + true + }) + } +} + +impl Drop for ScanLengths { + fn drop(&mut self) { + SCAN_LENGTHS.with(|c| { + let (depth, used) = c.get(); + c.set((depth.saturating_sub(1), used)); + }); + } +} + +/// A resolved `/Length` in [`LENGTHS`]: its value, whether resolving it read unsaved edits, and +/// what it read. +pub(super) type Length = (Option, bool, Reads); + +/// The other objects a cached load read: an object stream for a member, an indirect `/Length`, +/// and what those read in turn. Clones share the caches, so a document whose edits include one of +/// them reads the object again instead (`Document::try_get`); `None` is a load that read more +/// than [`Reads::MAX`], which such a document always reads again. +#[derive(Clone, Debug, Default)] +pub(super) struct Reads(Option>); + +impl Reads { + /// Objects one entry keeps: a member's stream and a length or two, and what they read. + const MAX: usize = 64; + /// Object numbers the open frames keep between them. + const HELD: usize = 4096; + + /// Whether `overlay` (a document's edits) holds any of these objects. + pub(super) fn touch(&self, overlay: &BTreeMap) -> bool { + !overlay.is_empty() && self.0.as_ref().is_none_or(|nums| nums.iter().any(|n| overlay.contains_key(n))) + } + + fn bytes(&self) -> usize { + self.0.as_ref().map_or(0, |nums| nums.len().saturating_mul(std::mem::size_of::())) + } +} + +/// Counts `num` as read by the loads open on this thread, with what loading it read (`reads`). +pub(super) fn note_read(num: u32, reads: Option<&Reads>) { + READS.with(|r| { + let (depth, held, full) = &mut *r.borrow_mut(); + if *depth == 0 { + return; + } + let more = reads.map_or(Some(&[][..]), |r| r.0.as_deref()); + match more { + Some(more) if held.len().saturating_add(more.len()) < Reads::HELD => { + held.push(num); + held.extend_from_slice(more); + } + _ => *full = full.wrapping_add(1), + } + }); +} + +/// Collects what one load reads, for [`Reads`]; frames nest, and each read counts for every +/// frame open around it. +pub(super) struct ReadFrame { + start: usize, + full: u64, +} + +impl ReadFrame { + pub(super) fn enter() -> Self { + READS.with(|r| { + let (depth, held, full) = &mut *r.borrow_mut(); + *depth = depth.saturating_add(1); + ReadFrame { start: held.len(), full: *full } + }) + } + + /// What was read since the frame opened. + pub(super) fn reads(&self) -> Reads { + READS.with(|r| { + let (_, held, full) = &*r.borrow(); + if *full != self.full { + return Reads(None); + } + let mut nums = held.get(self.start..).unwrap_or_default().to_vec(); + nums.sort_unstable(); + nums.dedup(); + Reads((nums.len() <= Reads::MAX).then(|| Arc::from(nums))) + }) + } +} + +impl Drop for ReadFrame { + fn drop(&mut self) { + READS.with(|r| { + let (depth, held, _) = &mut *r.borrow_mut(); + *depth = depth.saturating_sub(1); + if *depth == 0 { + held.clear(); + } + }); + } +} + +/// Unsaved edits read so far on this thread: a load during which this changes depended on one. +pub(super) fn overlay_reads() -> u64 { + OVERLAY_READS.with(std::cell::Cell::get) +} + +pub(super) fn note_overlay_read() { + OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1))); +} + +/// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream +/// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is +/// bounded as well. +pub(super) struct LoadGuard; + +impl LoadGuard { + const MAX: usize = 32; + + pub(super) fn enter(num: u32) -> Option { + LOADING.with(|l| { + let mut l = l.borrow_mut(); + if l.len() >= Self::MAX || l.contains(&num) { + return None; + } + l.push(num); + Some(LoadGuard) + }) + } +} + +/// Whether an object is being loaded on this thread: [`LENGTHS`] holds only while one is, since +/// it is cleared when the outermost load ends. +pub(super) fn loading() -> bool { + LOADING.with(|l| !l.borrow().is_empty()) +} + +/// What loading a file object gave, kept in the document's cache either way. +pub(super) type Loaded = Result, CosError>; + +/// The bytes a load result holds of its own ([`held_bytes`]); a failure holds none. +pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc>) -> usize { + loaded.as_ref().map_or(0, |o| held_bytes(o, data, 0)) +} + +impl Drop for LoadGuard { + fn drop(&mut self) { + let outermost = LOADING.with(|l| { + let mut l = l.borrow_mut(); + l.pop(); + l.is_empty() + }); + if outermost { + LENGTHS.with(|m| m.borrow_mut().clear()); + } + } +} + +#[derive(Clone, Debug)] +pub(super) struct ObjStm { + /// One slot per member in stream order, since cross-reference entries name a member by its + /// position: (object number, byte offset within `data`), or `None` for a member whose number + /// or offset is unusable ([`objstm_member`]). + pub(super) index: Vec>, + pub(super) data: Arc>, + /// What parsing it read: the stream itself and its `/Length`, `/DecodeParms` and so on. + pub(super) reads: Reads, +} + +/// Where member `num` of an object stream starts: its `offset` past `/First` (`first`), when the +/// number is an object number and the start lies inside the decoded `len` bytes. Checked on every +/// target, so a 32-bit build neither truncates nor wraps a large `/First` or offset (#1815). +pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: usize) -> Option<(u32, usize)> { + let num = u32::try_from(num).ok()?; + let start = first?.checked_add(usize::try_from(offset).ok()?)?; + (start < len).then_some((num, start)) +} + +/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). +/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that +/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole +/// list holds at most `MAX` plus the objects edited in clones. +#[derive(Debug, Default)] +pub(super) struct Unreadable { + pub(super) objects: BTreeMap, + pub(super) more: bool, +} + +impl Unreadable { + pub(super) const MAX: usize = 100; + /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. + const REASON: usize = 160; + + /// Whether `num` is to be recorded: not listed yet and there is room among the entries the + /// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]), + /// so a clone's edits never use up its own allowance. A full list notes that there were more, + /// so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool { + if self.objects.contains_key(&num) { + return false; + } + if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX { + self.more = true; + return false; + } + true + } + + /// Record `num` once; the first reason given is kept. + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) { + if self.wants(num, hidden) { + self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); + } + } + + /// One line per recorded object not `hidden`, then one saying there were more, if there were. + pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { + // Clones share the list and each hides its own edits, so the list may hold more than + // `MAX` entries in all; each document still shows at most `MAX`. + let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num)); + let mut lines: Vec = shown + .by_ref() + .take(Self::MAX) + .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) + .collect(); + if self.more || shown.next().is_some() { + lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); + } + lines + } +} + +/// Where an overlay slot came from. +#[derive(Clone, Debug, PartialEq)] +pub(super) enum Slot { + Set(u16, Arc), + /// Both generations accepted from a damaged live source slot survive a repair-aware edit. + /// A full rewrite is required: a new xref slot cannot encode two generations. + RepairedSet(u16, u16, Arc), + Freed(u16), +} + +/// An immutable input/decode context shared by document clones. Holding this identity +/// retains neither the input bytes nor the parsed document. Weak ownership prevents an +/// expired identity from matching a later allocation at the same address. +#[derive(Clone, Debug)] +pub struct SourceIdentity(pub(super) Weak<()>); + +impl PartialEq for SourceIdentity { + fn eq(&self, other: &Self) -> bool { + self.0.ptr_eq(&other.0) + } +} + +impl Eq for SourceIdentity {} + +/// What a document reads as: its source ([`SourceIdentity`]), its trailer and every object changed +/// since it was opened. A clone, or a snapshot that undo restores, has the same state, and every +/// change makes a different one. Changed objects are compared as allocations, which the state +/// keeps alive, so a later object can't take an address one of them had. +#[derive(Clone)] +pub struct ReadState { + pub(super) source: SourceIdentity, + pub(super) trailer: Dict, + pub(super) overlay: BTreeMap, +} + +impl ReadState { + /// Whether `doc` reads as this state. + pub fn matches(&self, doc: &Document) -> bool { + self.source == doc.source_identity() + && self.trailer == doc.trailer + && self.overlay.len() == doc.overlay.len() + && self.overlay.iter().zip(&doc.overlay).all(|((a, x), (b, y))| { + a == b + && match (x, y) { + (Slot::Set(g, o), Slot::Set(h, p)) => g == h && Arc::ptr_eq(o, p), + (Slot::RepairedSet(g, a, o), Slot::RepairedSet(h, b, p)) => g == h && a == b && Arc::ptr_eq(o, p), + (Slot::Freed(g), Slot::Freed(h)) => g == h, + _ => false, + } + }) + } +} + +impl std::fmt::Debug for ReadState { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ReadState").field("changed_objects", &self.overlay.len()).finish_non_exhaustive() + } +} + +/// A temporary reader for whole-document work. Its caches never populate the editor or +/// undo snapshots. Returned objects survive cache eviction; a stream read from the file is a +/// view into the input bytes (it keeps the whole input alive while held, not a copy). +pub(crate) struct ObjectReader { + pub(crate) document: Document, + pub(super) decoded_count: usize, +} + +impl ObjectReader { + pub(super) const OBJECTS: usize = 128; + pub(super) const STREAM_BYTES: usize = 8 * 1024 * 1024; + + pub(crate) fn get(&mut self, reference: ObjRef) -> Arc { + self.try_get(reference).unwrap_or_else(|_| Arc::new(Object::Null)) + } + + pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { + let object = self.document.try_get_reference(reference); + // A missed eviction here only costs a re-parse, so never fail the read on a poisoned + // lock. The editing-cache evictors propagate instead; they guard user edits. + if let Ok(mut streams) = self.document.objstms.lock() { + // Only insertion changes this private cache between reads. Most neighboring + // objects reuse a decoded stream: do not sum the whole store for every object. + if streams.len() != self.decoded_count { + let bytes = streams.values().fold(0usize, |bytes, stream| { + bytes + .saturating_add(stream.data.capacity()) + .saturating_add(stream.index.capacity().saturating_mul(std::mem::size_of::>())) + }); + if bytes > Self::STREAM_BYTES || streams.len() >= Self::OBJECTS { + // Retain one oversized active stream so its next object does not require + // decoding it again. That stream still obeys the document's decode cap. + let current = match self.document.xref_entry(reference.num) { + Some(XrefEntry::InStream { stream, .. }) => Some(stream), + _ => None, + }; + streams.retain(|num, _| Some(*num) == current); + } + self.decoded_count = streams.len(); + } + } + // Parsed objects stay cached (an indirect /Length or a dictionary read again costs a + // lookup, not a parse) until the cache passes its object count or holds more than + // STREAM_BYTES of data of its own. Streams read from the file are views into the + // input bytes, which the document keeps anyway, so they don't count; decrypted + // streams and strings do. Checked after failed loads too: resolving a /Length may + // have cached large objects. + if let Ok(mut cache) = self.document.cache.lock() + && (cache.len() >= Self::OBJECTS || cache.owned() > Self::STREAM_BYTES) + { + cache.clear(); + } + object + } +} + +#[cfg(test)] +thread_local! { + /// Cross-reference sections read on this thread. + pub(super) static SECTION_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// What loading each object gave (a failure as well), by number, each with what it holds ([`held_bytes`]) measured once, when it goes +/// in, and their sum kept beside them. +/// +/// Both caches are bounded by those bytes. Summing them on every read walked every cached object +/// whole each time: a file whose page dictionaries carry kilobytes of private data (InDesign's +/// `/PieceInfo`) took seconds to open, almost all of it re-measuring objects already measured +/// (#5873). +#[derive(Default)] +pub(crate) struct ObjectCache { + entries: HashMap, + owned: usize, +} + +impl ObjectCache { + /// What loading `num` gave, and what that load read. + pub(super) fn get(&self, num: &u32) -> Option<(&Loaded, &Reads)> { + self.entries.get(num).map(|(object, _, reads)| (object, reads)) + } + + #[cfg(test)] + pub(crate) fn contains_key(&self, num: &u32) -> bool { + self.entries.contains_key(num) + } + + /// Keep what loading `num` gave, which holds `owned` bytes ([`loaded_bytes`]), and what it read. + pub(super) fn insert(&mut self, num: u32, object: Loaded, owned: usize, reads: Reads) { + let owned = owned.saturating_add(reads.bytes()); + if let Some((_, before, _)) = self.entries.insert(num, (object, owned, reads)) { + self.owned = self.owned.saturating_sub(before); + } + self.owned = self.owned.saturating_add(owned); + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + #[cfg(test)] + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// The bytes the cached objects hold between them. + pub(crate) fn owned(&self) -> usize { + self.owned + } + + pub(crate) fn clear(&mut self) { + self.entries.clear(); + self.owned = 0; + } + + pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) { + self.entries.retain(|num, _| keep(num)); + self.owned = self.entries.values().fold(0usize, |n, (_, owned, _)| n.saturating_add(*owned)); + } + + #[cfg(test)] + pub(crate) fn values(&self) -> impl Iterator> { + self.entries.values().filter_map(|(object, _, _)| object.as_ref().ok()) + } +} + +/// Nesting [`held_bytes`] walks: past the parser's own nesting limit, so every parsed object +/// is measured in full. +const OWNED_BYTES_DEPTH: usize = 256; + +/// What keeping `object` parsed costs: string and stream data that is not a view into `data` +/// (the input file), and the parsed tree around it -- an [`Object`] per value and a key per +/// dictionary entry. An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured and +/// counts as too large to keep. +/// +/// The tree counts. Measured by its strings and streams alone, a page dictionary carrying +/// InDesign's `/PieceInfo` -- thousands of small numeric arrays, parsed into tens of thousands of +/// values -- weighed nothing, so the caches kept a thousand of them: a 35 MB file held 1.5 GB +/// (#5873). +pub(crate) fn held_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { + if depth > OWNED_BYTES_DEPTH { + return usize::MAX; + } + let node = std::mem::size_of::(); + // A key, its slot beside the value, and room in the index a large dictionary keeps. + let entry = |key: &[u8]| key.len().saturating_add(std::mem::size_of::<(crate::cos::Name, Object)>()).saturating_add(16); + let dict = |d: &crate::cos::Dict, start: usize| { + d.iter().fold(start, |n, (k, v)| n.saturating_add(entry(k)).saturating_add(held_bytes(v, data, depth + 1))) + }; + match object { + Object::String(s) => node.saturating_add(s.bytes.len()), + Object::Name(n) => node.saturating_add(n.len()), + Object::Stream(s) => dict(&s.dict, node.saturating_add(if s.raw.shares(data) { 0 } else { s.raw.len() })), + Object::Array(a) => a.iter().fold(node, |n, v| n.saturating_add(held_bytes(v, data, depth + 1))), + Object::Dict(d) => dict(d, node), + _ => node, + } +} + +/// How an encrypted document is opened: with a password, or with the file key of a document +/// that has the same encryption (an earlier revision of the same file). +pub(super) enum Credentials<'a> { + Password(Option<&'a str>), + Handler { handler: &'a Arc, id0: Vec }, +} diff --git a/crates/pdf/src/cos/document/tests.rs b/crates/pdf/src/cos/document/tests.rs new file mode 100644 index 000000000..836b8a0b3 --- /dev/null +++ b/crates/pdf/src/cos/document/tests.rs @@ -0,0 +1,2713 @@ +use super::*; + +/// What [`crate::cos::reachable_objects`] gives is what a full save keeps: not an object nothing refers to, nor one +/// a reference only dangles at, and a ring of references ends. +#[test] +fn what_is_reachable_is_what_a_full_save_keeps() { + let mut doc = Document::new_empty(); + let root = doc.root().unwrap(); + let kept = doc.add(Object::Int(7)); + let dangling = ObjRef::new(4_000, 0); + let list = doc.add(Object::Null); + let ring = doc.add(Object::Array(vec![Object::Ref(list)])); + doc.set(list, Object::Array(vec![Object::Ref(kept), Object::Ref(ring), Object::Ref(dangling)])); + let orphan = doc.add(Object::Int(8)); + doc.update_dict(root, |d| d.set(b"Held".to_vec(), Object::Ref(list))).unwrap(); + let found = crate::cos::reachable_objects(&doc).unwrap(); + assert!([root, kept, list, ring].iter().all(|r| found.contains(&r.num))); + assert!(!found.contains(&orphan.num) && !found.contains(&dangling.num)); + let saved = Document::open(Arc::new(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap())).unwrap(); + let holds = |n: i64| saved.scan_objects().any(|(_, o)| matches!(*o, Object::Int(v) if v == n)); + assert!(holds(7) && !holds(8)); +} + +std::thread_local! { + /// How many times `Document::scan_for` read the file on this thread. + pub(super) static FILE_SCANS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// How many times `Document::objstm` decoded an object stream on this thread. + pub(super) static OBJSTM_DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// A scan that doesn't find an object isn't made again: each reads the whole file, and a file can +/// refer to such an object from every object it has. +#[test] +fn what_a_scan_of_the_file_did_not_find_is_remembered() { + let body = b"%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n"; + let mut out = body.to_vec(); + let xref = out.len(); + out.extend_from_slice(b"xref\n0 4\n0000000000 65535 f \n"); + out.extend_from_slice(format!("{:010} 00000 n \n", 9).as_bytes()); + out.extend_from_slice(format!("{:010} 00000 n \n", body.iter().position(|b| *b == b'\n').unwrap() + 49).as_bytes()); + out.extend_from_slice(b"0000000001 00000 n \n"); + out.extend_from_slice(format!("trailer << /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + FILE_SCANS.with(|scans| scans.set(0)); + for _ in 0..5 { + assert!(doc.try_get(3).is_err()); + } + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1); + let clone = doc.clone(); + assert!(clone.try_get(3).is_err()); + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "a clone has the same file"); +} + +/// An object found by a scan of the file, because its cross-reference offset is wrong, has its +/// indirect `/Length` resolved as at the right offset. Unresolved, a stream ending in a misspelt +/// `endstream` (`endstrea`) was searched on into a later object's `endstream`, and took that +/// object's header and data as its own. +#[test] +fn a_scanned_stream_resolves_its_indirect_length() { + let mut out = build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /Length 4 0 R >>\nstream\nHELLO\nendstrea", + "5", + "<< /Length 5 >>\nstream\nWORLD\nendstream", + ], + "/Root 1 0 R", + ); + // Point object 3's entry at the catalog, so only the scan finds it. + let at = |needle: &[u8], out: &[u8]| out.windows(needle.len()).position(|w| w == needle).unwrap(); + let catalog = format!("{:010} 00000 n ", at(b"1 0 obj", &out)); + let entry = at(format!("{:010} 00000 n ", at(b"3 0 obj", &out)).as_bytes(), &out); + out.splice(entry..entry + catalog.len(), catalog.bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + let stream = doc.try_get(3).unwrap(); + let Object::Stream(s) = &*stream else { panic!("stream expected: {stream:?}") }; + assert_eq!(&s.raw[..], b"HELLO"); +} + +/// A scan of the file resolves only so many indirect `/Length` values. Each header that starts +/// with the sought number is parsed, and a `/Length` naming an object the table cannot place scans +/// the file for that one: with a hundred such headers, each naming another misplaced object, one +/// lookup read the whole file a hundred and one times. +#[test] +fn a_scan_of_the_file_resolves_a_bounded_number_of_lengths() { + const HEADERS: u32 = 100; + let mut out = b"%PDF-1.7\n".to_vec(); + let catalog = out.len(); + out.extend_from_slice(b"1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n"); + let pages = out.len(); + out.extend_from_slice(b"2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n"); + // Object 3 is defined only by headers inside this string, each with its own misplaced /Length. + out.extend_from_slice(b"4 0 obj (\n"); + for k in 0..HEADERS { + out.extend_from_slice(format!("3 0 obj << /Length {} 0 R >> stream\nX\nendstream endobj\n", 10 + k).as_bytes()); + } + out.extend_from_slice(b") endobj\n"); + let size = 10 + HEADERS; + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); + for num in 1..size { + let at = if num == 2 { pages } else { catalog }; + out.extend_from_slice(format!("{at:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + FILE_SCANS.with(|scans| scans.set(0)); + let three = doc.try_get(3).unwrap(); + assert!(matches!(&*three, Object::Stream(_)), "{three:?}"); + let scans = FILE_SCANS.with(std::cell::Cell::get); + assert!(scans <= 1 + super::state::ScanLengths::MAX as usize, "{scans} scans of the file"); +} + +/// An object can be nothing but a reference to another. A chain of them is followed, and one that +/// goes round ends in what isn't a dictionary or a name, without hanging. +#[test] +fn references_to_references_are_followed() { + let mut doc = Document::new_empty(); + let name = doc.add(Object::name("Target")); + let one = doc.add(Object::Ref(name)); + let two = doc.add(Object::Ref(one)); + assert_eq!(doc.resolve(&Object::Ref(two)).as_name(), Some(b"Target".as_slice())); + let a = doc.add(Object::Null); + let b = doc.add(Object::Ref(a)); + doc.set(a, Object::Ref(b)); + assert_eq!(doc.resolve(&Object::Ref(a)).as_name(), None); +} + +/// An object stream's `/DecodeParms` can be given by reference: without them a predictor isn't +/// undone, so its members were lost. +#[test] +fn an_object_stream_with_parameters_by_reference_is_read() { + let members = b"3 0 << /Hidden (yes) >>"; + // PNG Sub, one row. + let mut rows = vec![1u8]; + let mut before = 0u8; + for b in members { + rows.push(b.wrapping_sub(before)); + before = *b; + } + let coded = crate::filters::encode_flate(&rows); + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 7]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /DecodeParms 6 0 R /Length {} >>\nstream\n", coded.len()).into_bytes(); + stm.extend_from_slice(&coded); + stm.extend_from_slice(b"\nendstream"); + obj(&mut out, 4, &stm); + let data = [2u8, 0, 4, 0]; + let mut xs = format!("<< /Type /XRef /Size 7 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); + xs.extend_from_slice(&data); + xs.extend_from_slice(b"\nendstream"); + obj(&mut out, 5, &xs); + obj(&mut out, 6, format!("<< /Predictor 11 /Columns {} >>", members.len()).as_bytes()); + let table = out.len(); + out.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); + for (n, off) in offs.iter().enumerate().skip(1) { + let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(format!("trailer << /Size 7 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + let hidden = doc.get(ObjRef::new(3, 0)); + assert_eq!(hidden.as_dict().and_then(|d| d.get(b"Hidden")).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"yes".to_vec())); +} + +/// Each entry of an `/Encrypt` dictionary, and of a crypt filter in it, reads the same by reference. +/// Read as absent, `/StmF` or `/CFM` was `/Identity`, so a whole file's streams stayed ciphertext. +#[test] +fn encrypt_dictionary_entries_read_the_same_by_reference() { + let string = |s: &str| Object::String(crate::cos::PdfString::text(s)); + let mut filter = Dict::new(); + filter.set(b"CFM".to_vec(), Object::name("AESV2")); + filter.set(b"Length".to_vec(), Object::Int(5)); + let cf = |filter: Dict| { + let mut cf = Dict::new(); + cf.set(b"StdCF".to_vec(), Object::Dict(filter)); + cf.set(b"Other".to_vec(), Object::Dict(Dict::new())); + Object::Dict(cf) + }; + let mut d = Dict::new(); + let entries = [ + ("Filter", Object::name("Standard")), + ("V", Object::Int(4)), + ("R", Object::Int(4)), + ("Length", Object::Int(128)), + ("P", Object::Int(-44)), + ("O", string("owner")), + ("U", string("user")), + ("EncryptMetadata", Object::Bool(false)), + ("CF", cf(filter.clone())), + ("StmF", Object::name("StdCF")), + ("StrF", Object::name("Other")), + ("EFF", Object::name("StdCF")), + ]; + for (key, value) in &entries { + d.set(key.as_bytes().to_vec(), value.clone()); + } + let mut doc = Document::new_empty(); + let direct = crate::cos::security::encrypt_dict(&doc, &d); + assert_eq!((direct.stm_f.as_slice(), direct.str_f.as_slice(), direct.ef_f.as_slice()), (&b"StdCF"[..], &b"Other"[..], &b"StdCF"[..])); + assert_eq!(direct.length_bits, 40, "the filter's own length, in bytes"); + assert!(!direct.encrypt_metadata); + assert_eq!(direct.crypt_filters.iter().filter(|(_, m)| *m == crate::crypt::Method::Aes128).count(), 1); + for (key, value) in &entries { + let mut one = d.clone(); + let r = doc.add(value.clone()); + one.set(key.as_bytes().to_vec(), Object::Ref(r)); + assert_eq!(crate::cos::security::encrypt_dict(&doc, &one), direct, "/{key}"); + } + for key in ["CFM", "Length"] { + let mut by_reference = filter.clone(); + let r = doc.add(filter.get(key.as_bytes()).unwrap().clone()); + by_reference.set(key.as_bytes().to_vec(), Object::Ref(r)); + let mut one = d.clone(); + one.set(b"CF".to_vec(), cf(by_reference)); + assert_eq!(crate::cos::security::encrypt_dict(&doc, &one), direct, "/{key} in the filter"); + } +} + +/// /DecodeParms given by reference, whole, as array entries or as the numbers inside, are read +/// once made direct: here a Flate predictor, which the stream alone doesn't see. Only the entries the filters use are +/// resolved, and nothing is for a chain longer than a decoder takes. +#[test] +fn decode_parms_given_by_reference_are_read_once_direct() { + let rows = [1u8, 2, 3, 4, 5, 6, 7, 8]; + let params = crate::filters::Params { predictor: 12, colors: 1, bits_per_component: 8, columns: 4, early_change: 1 }; + let coded = crate::filters::encode(&crate::filters::Filter::Flate, ¶ms, &rows).unwrap(); + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(4)); + let mut doc = Document::new_empty(); + let r = doc.add(Object::Dict(parms.clone())); + let array = doc.add(Object::Array(vec![Object::Ref(r)])); + let stream = |filters: usize, p: Object| { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"); filters])); + d.set(b"DecodeParms".to_vec(), p); + Stream::from_raw(d, coded.clone()) + }; + for p in [Object::Ref(r), Object::Array(vec![Object::Ref(r)]), Object::Ref(array)] { + let s = stream(1, p.clone()); + assert_ne!(s.decoded().unwrap(), rows, "{p:?} read as it is"); + assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "{p:?}"); + } + // A number inside given by reference, which `Dict::int` doesn't follow: in the dictionary + // itself, and in an array's entry. + let twelve = doc.add(Object::Int(12)); + let mut inner = parms.clone(); + inner.set(b"Predictor".to_vec(), Object::Ref(twelve)); + for p in [Object::Dict(inner.clone()), Object::Array(vec![Object::Dict(inner)])] { + let s = stream(1, p.clone()); + assert_ne!(s.decoded().unwrap(), rows, "{p:?} read as it is"); + assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "{p:?}"); + } + // Only numbers and flags are copied: a key naming a big array, or an entry that is no + // dictionary, is left out (a file could name one big object from every key and filter). + let big = doc.add(Object::Array(vec![Object::Int(1); 100_000])); + let mut hostile = Dict::new(); + hostile.set(b"Predictor".to_vec(), Object::Ref(big)); + hostile.set(b"Columns".to_vec(), Object::Ref(big)); + hostile.set(b"Colors".to_vec(), Object::Ref(twelve)); + let entry = doc.add(Object::Dict(hostile)); + let two = stream(2, Object::Array(vec![Object::Ref(entry), Object::Ref(big)])); + let made = doc.with_direct_parms(&two); + let made = made.dict.get(b"DecodeParms").and_then(Object::as_array).unwrap(); + let first = made[0].as_dict().unwrap(); + assert_eq!((first.contains(b"Predictor"), first.contains(b"Columns"), first.int(b"Colors")), (false, false, Some(12))); + assert!(matches!(made[1], Object::Null), "{:?}", made[1]); + let direct = stream(1, Object::Dict(parms)); + assert!(matches!(doc.with_direct_parms(&direct), Cow::Borrowed(_)), "nothing to resolve"); + assert_eq!(direct.decoded().unwrap(), rows); + // Entries past the filters' are left out, unresolved; so is everything past the chain limit. + let many = stream(1, Object::Array(vec![Object::Ref(r); 1000])); + let made = doc.with_direct_parms(&many); + assert_eq!(made.dict.get(b"DecodeParms").and_then(Object::as_array).map(Vec::len), Some(1)); + let long = stream(crate::filters::MAX_CHAIN + 1, Object::Array(vec![Object::Ref(r); 9])); + assert!(matches!(doc.with_direct_parms(&long), Cow::Borrowed(_))); + // A chain as long as a decoder takes is resolved, and a `/Crypt` in it doesn't count, as + // the decoders leave it out: ASCIIHex as often as the limit allows, then the predictor's + // Flate, its parameters by reference. + let hex = |b: &[u8]| b.iter().flat_map(|x| format!("{x:02X}").into_bytes()).chain(*b">").collect::>(); + let wrapped = (1..crate::filters::MAX_CHAIN).fold(coded.clone(), |data, _| hex(&data)); + for crypt in [false, true] { + let mut names = vec![Object::name("ASCIIHexDecode"); crate::filters::MAX_CHAIN - 1]; + names.push(Object::name("FlateDecode")); + if crypt { + names.insert(0, Object::name("Crypt")); + } + let mut entries = vec![Object::Null; names.len() - 1]; + entries.push(Object::Ref(r)); + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::Array(names)); + d.set(b"DecodeParms".to_vec(), Object::Array(entries)); + let s = Stream::from_raw(d, wrapped.clone()); + assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "with /Crypt: {crypt}"); + } +} + +/// A CCITT fax image reads through its /DecodeParms, given directly, in an array or by +/// reference (its numbers too), with /BlackIs1, and behind other filters; one whose /Columns +/// aren't its /Width is refused, /Height rows are decoded whatever /Rows says, and other images +/// decode through their filters. +#[test] +fn fax_images_decode_with_their_parameters() { + // 10 × 3: the first five samples of each row white, the rest black. + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = crate::filters::encode_g4(&bits, 10, 3).unwrap(); + let mut doc = Document::new_empty(); + let mut parms = Dict::new(); + parms.set(b"K".to_vec(), Object::Int(-1)); + parms.set(b"Columns".to_vec(), Object::Int(10)); + let indirect = doc.add(Object::Dict(parms.clone())); + let image = |doc: &Document, parms: Option, width: i64, height: i64| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(width)); + d.set(b"Height".to_vec(), Object::Int(height)); + d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + if let Some(p) = parms { + d.set(b"DecodeParms".to_vec(), p); + } + doc.decoded_image(&Stream::from_raw(d, coded.clone())) + }; + for p in [ + Object::Dict(parms.clone()), + Object::Array(vec![Object::Dict(parms.clone())]), + Object::Ref(indirect), + Object::Array(vec![Object::Ref(indirect)]), + ] { + assert_eq!(image(&doc, Some(p.clone()), 10, 3).unwrap(), bits, "{p:?}"); + } + let mut inverted = parms.clone(); + inverted.set(b"BlackIs1".to_vec(), Object::Bool(true)); + assert_eq!( + image(&doc, Some(Object::Dict(inverted)), 10, 3).unwrap(), + [0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000] + ); + assert_eq!(image(&doc, Some(Object::Dict(parms.clone())), 10, 2).unwrap(), bits[..4], "at most /Height rows"); + // Within a bound: no more rows than it holds. + let mut bounded = Dict::new(); + bounded.set(b"Width".to_vec(), Object::Int(10)); + bounded.set(b"Height".to_vec(), Object::Int(3)); + bounded.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + bounded.set(b"DecodeParms".to_vec(), Object::Dict(parms.clone())); + assert_eq!(doc.decoded_image_within(&Stream::from_raw(bounded, coded.clone()), 5).unwrap(), bits[..4], "two rows of two bytes in five"); + let mut by_reference = Dict::new(); + by_reference.set(b"K".to_vec(), Object::Ref(doc.add(Object::Int(-1)))); + by_reference.set(b"Columns".to_vec(), Object::Ref(doc.add(Object::Int(10)))); + assert_eq!(image(&doc, Some(Object::Dict(by_reference)), 10, 3).unwrap(), bits, "/K and /Columns by reference"); + let mut understated = parms.clone(); + understated.set(b"Rows".to_vec(), Object::Int(1)); + assert_eq!(image(&doc, Some(Object::Dict(understated.clone())), 10, 3).unwrap(), bits, "/Height rows, whatever /Rows says"); + // More /Rows than /Height (PDFBOX-771: 1,084,975,104), as the renderer reads it too. + let mut overstated = parms.clone(); + overstated.set(b"Rows".to_vec(), Object::Int(1_000_000_000)); + assert_eq!(image(&doc, Some(Object::Dict(overstated)), 10, 3).unwrap(), bits, "/Height rows, not /Rows past it"); + // The parameters as the file gives them, /Rows included, and Table 11's defaults. + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); + d.set(b"DecodeParms".to_vec(), Object::Dict(understated)); + let p = doc.fax_params(&Stream::from_raw(d.clone(), Vec::new())).unwrap(); + assert_eq!((p.k, p.columns, p.rows, p.end_of_block, p.black_is_1), (-1, 10, 1, true, false)); + d.set(b"Filter".to_vec(), Object::name("FlateDecode")); + assert!(doc.fax_params(&Stream::from_raw(d, Vec::new())).is_none(), "not a fax image"); + assert!(matches!(image(&doc, Some(Object::Dict(parms.clone())), 12, 3), Err(CosError::Filter(_))), "/Columns other than /Width"); + assert!(matches!(image(&doc, None, 10, 3), Err(CosError::Filter(_))), "the default 1728 /Columns"); + let mut plain = Dict::new(); + plain.set(b"Filter".to_vec(), Object::name("FlateDecode")); + assert_eq!(doc.decoded_image(&Stream::from_raw(plain, crate::filters::encode_flate(&bits))).unwrap(), bits); + + // Behind other filters, its parameters at its own place in the array (/Crypt counts). + let behind = |filters: &[Object], parms: &[Object], data: Vec| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(10)); + d.set(b"Height".to_vec(), Object::Int(3)); + d.set(b"Filter".to_vec(), Object::Array(filters.to_vec())); + d.set(b"DecodeParms".to_vec(), Object::Array(parms.to_vec())); + doc.decoded_image(&Stream::from_raw(d, data)) + }; + let (fax, null, own) = (Object::name("CCITTFaxDecode"), Object::Null, Object::Dict(parms.clone())); + let hex = coded.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); + let flated = crate::filters::encode_flate(&coded); + let ahx = [Object::name("AHx"), fax.clone()]; + assert_eq!(behind(&ahx, &[null.clone(), Object::Ref(indirect)], hex.clone()).unwrap(), bits, "behind ASCIIHexDecode"); + assert_eq!( + behind(&[Object::name("FlateDecode"), Object::name("CCF")], &[null.clone(), own.clone()], flated.clone()).unwrap(), + bits, + "behind Flate" + ); + assert_eq!(behind(&[Object::name("Crypt"), fax.clone()], &[null.clone(), own.clone()], coded.clone()).unwrap(), bits, "after /Crypt"); + // A /Crypt belongs first (§7.4.10); one after the fax filter leaves it its own entry. + assert_eq!(behind(&[fax.clone(), Object::name("Crypt")], &[own.clone(), null.clone()], coded.clone()).unwrap(), bits, "before /Crypt"); + assert!(behind(&ahx, &[own.clone(), null.clone()], hex).is_err(), "the first entry is the other filter's"); + assert!(behind(&[fax.clone(), Object::name("FlateDecode")], &[own.clone(), null.clone()], flated).is_err(), "CCITT, then Flate"); + assert!(behind(&[Object::Int(5), fax], &[null, own], coded.clone()).is_err(), "a /Filter entry that isn't a name"); +} + +/// Images that share a `/JBIG2Globals` stream of the file decode it once a document: it is kept +/// for the next, in every clone. A globals stream edited since is read afresh (the kept one is the +/// file's), and objects added in memory aren't kept. +#[test] +fn shared_jbig2_globals_are_read_once() { + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = crate::filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); + let mut doc = Document::new_empty(); + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; + let globals = doc.add(Object::Stream(Stream::from_raw(d, crate::filters::encode_flate(&skipped)))); + let image = |doc: &mut Document| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(10)); + d.set(b"Height".to_vec(), Object::Int(3)); + d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + d.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + doc.add(Object::Stream(Stream::from_raw(d, coded.clone()))) + }; + let (a, b) = (image(&mut doc), image(&mut doc)); + let stream = |doc: &Document, r: ObjRef| match &*doc.get(r) { + Object::Stream(s) => s.clone(), + _ => panic!("not a stream"), + }; + assert_eq!(doc.decoded_image(&stream(&doc, a)).unwrap(), bits); + assert!(doc.jbig2_globals.lock().unwrap().is_empty(), "an object added in memory isn't kept"); + + // The same objects read from a file, the catalog holding on to the images (a full save may + // number them anew). + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Images".to_vec(), Object::Array(vec![Object::Ref(a), Object::Ref(b)]))).unwrap(); + let file = crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap(); + let mut doc = Document::open(Arc::new(file)).unwrap(); + let images = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"Images").unwrap().as_array().unwrap().clone(); + let (a, b) = (images[0].as_ref().unwrap(), images[1].as_ref().unwrap()); + let parms = stream(&doc, a).dict.get(b"DecodeParms").unwrap().as_dict().unwrap().clone(); + let globals = parms.reference(b"JBIG2Globals").unwrap(); + let clone = doc.clone(); + assert_eq!(doc.decoded_image(&stream(&doc, a)).unwrap(), bits); + assert_eq!(doc.jbig2_globals.lock().unwrap().len(), 1, "kept"); + let kept = doc.jbig2_globals.lock().unwrap()[&globals.num].clone(); + assert_eq!(clone.decoded_image(&stream(&clone, b)).unwrap(), bits, "the clone's other image"); + let shared = clone.jbig2_globals.lock().unwrap()[&globals.num].clone(); + assert!(Arc::ptr_eq(&kept, &shared), "the clone uses the one kept, not a second decode"); + doc.set(globals, Object::Stream(Stream::from_raw(Dict::new(), b"not JBIG2".to_vec()))); + assert!(doc.decoded_image(&stream(&doc, b)).is_err(), "the edited globals are read afresh"); + assert_eq!(clone.decoded_image(&stream(&clone, b)).unwrap(), bits, "the clone keeps the file's"); +} + +/// A JBIG2 image's size is read as given directly, as every caller reads it first, and an image +/// with neither is an error, not an image of no rows. +#[test] +fn a_jbig2_image_needs_its_size_given_directly() { + let coded = crate::filters::encode_jbig2_generic(&[0xFF, 0xFF], 8, 2).unwrap(); + let mut doc = Document::new_empty(); + let size = doc.add(Object::Int(2)); + let image = |width: Object, height: Object| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), width); + d.set(b"Height".to_vec(), height); + d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + doc.decoded_image(&Stream::from_raw(d, coded.clone())) + }; + assert!(image(Object::Int(8), Object::Int(2)).is_ok()); + assert!(image(Object::Ref(size), Object::Int(2)).is_err(), "/Width by reference"); + assert!(image(Object::Int(8), Object::Ref(size)).is_err(), "/Height by reference"); +} + +/// A document that kept globals of 64 MiB keeps no more, and the images past that still read: +/// three streams of 22 MiB, shared by three images, are two kept and one decoded each time. +#[test] +fn kept_jbig2_globals_are_bounded_in_total() { + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = crate::filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); + let length = 22 * 1024 * 1024; + let mut skipped = vec![0, 0, 0, 0, 0x3E, 0, 0]; + skipped.extend_from_slice(&u32::try_from(4 + length).unwrap().to_be_bytes()); + skipped.extend_from_slice(&[0, 0, 0, 1]); + skipped.resize(skipped.len() + length, 0); + let mut doc = Document::new_empty(); + let mut images = Vec::new(); + for _ in 0..3 { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(d, crate::filters::encode_flate(&skipped)))); + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(10)); + d.set(b"Height".to_vec(), Object::Int(3)); + d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + d.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + images.push(Object::Ref(doc.add(Object::Stream(Stream::from_raw(d, coded.clone()))))); + } + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Images".to_vec(), Object::Array(images))).unwrap(); + let doc = Document::open(Arc::new(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap())).unwrap(); + let images = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"Images").unwrap().as_array().unwrap().clone(); + for image in &images { + let Object::Stream(s) = &*doc.get(image.as_ref().unwrap()) else { panic!("not a stream") }; + assert_eq!(doc.decoded_image(s).unwrap(), bits, "every image reads"); + } + let kept = doc.jbig2_globals.lock().unwrap(); + assert_eq!(kept.len(), 2, "two fit in 64 MiB, a third doesn't"); + assert!(kept.values().all(|g| g.capacity() == g.len()), "what is kept holds no more than it counts"); +} + +/// Globals whose `/DecodeParms` are given by reference aren't kept: that's another object, so +/// editing it changes what they decode to while the stream itself, the key kept by, stays the same. +#[test] +fn jbig2_globals_with_parameters_by_reference_are_read_afresh() { + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = crate::filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); + let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; + let params = crate::filters::Params { predictor: 12, colors: 1, bits_per_component: 8, columns: 15, early_change: 1 }; + let mut doc = Document::new_empty(); + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(15)); + let parms = doc.add(Object::Dict(parms)); + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::name("FlateDecode")); + d.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + let predicted = crate::filters::encode(&crate::filters::Filter::Flate, ¶ms, &skipped).unwrap(); + let globals = doc.add(Object::Stream(Stream::from_raw(d, predicted))); + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(10)); + d.set(b"Height".to_vec(), Object::Int(3)); + d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); + let mut image_parms = Dict::new(); + image_parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + d.set(b"DecodeParms".to_vec(), Object::Dict(image_parms)); + let image = doc.add(Object::Stream(Stream::from_raw(d, coded))); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Image".to_vec(), Object::Ref(image))).unwrap(); + let mut doc = Document::open(Arc::new(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap())).unwrap(); + let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); + let stream = |doc: &Document| match &*doc.get(image) { + Object::Stream(s) => s.clone(), + _ => panic!("not a stream"), + }; + assert_eq!(doc.decoded_image(&stream(&doc)).unwrap(), bits); + assert!(doc.jbig2_globals.lock().unwrap().is_empty(), "not kept"); + let globals = stream(&doc).dict.get(b"DecodeParms").unwrap().as_dict().unwrap().reference(b"JBIG2Globals").unwrap(); + let Object::Stream(g) = &*doc.get(globals) else { panic!("not a stream") }; + let parms = g.dict.reference(b"DecodeParms").unwrap(); + doc.set(parms, Object::Dict(Dict::new())); + assert!(doc.decoded_image(&stream(&doc)).is_err(), "without the predictor they read otherwise"); +} + +/// A JBIG2 image reads through its decoder, behind other filters too, with the `/JBIG2Globals` of +/// its own `/DecodeParms` (a dictionary or its place in an array, /Crypt counting, by reference +/// too): globals that can't be read are an error, and ones at another filter's place aren't its +/// own, nor are those of a lone dictionary when other filters come first. Its page has to be its +/// /Width × /Height, and fit the bound. +#[test] +fn jbig2_images_decode_with_their_globals() { + // 10 × 3: the first five samples of each row white, the rest black. + let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; + let coded = crate::filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); + let hex = coded.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); + let mut doc = Document::new_empty(); + // An extension segment (ISO/IEC 14492 7.4.14), which a decoder skips, Flate-encoded; and data + // that isn't JBIG2. + let globals = |doc: &mut Document, data: &[u8]| { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let globals = doc.add(Object::Stream(Stream::from_raw(d, crate::filters::encode_flate(data)))); + let mut parms = Dict::new(); + parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); + Object::Dict(parms) + }; + let (good, bad) = (globals(&mut doc, &[0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]), globals(&mut doc, b"not JBIG2")); + let indirect = Object::Ref(doc.add(bad.clone())); + let image = |filters: &[&str], parms: Option, data: &[u8], (width, height): (i64, i64), max: usize| { + let mut d = Dict::new(); + d.set(b"Width".to_vec(), Object::Int(width)); + d.set(b"Height".to_vec(), Object::Int(height)); + d.set(b"Filter".to_vec(), Object::Array(filters.iter().map(|f| Object::name(f)).collect())); + if let Some(p) = parms { + d.set(b"DecodeParms".to_vec(), p); + } + doc.decoded_image_within(&Stream::from_raw(d, data.to_vec()), max) + }; + let jbig2 = ["JBIG2Decode"]; + let full = (10, 3); + assert_eq!(image(&jbig2, None, &coded, full, MAX_DECODED).unwrap(), bits); + assert_eq!(image(&jbig2, Some(good.clone()), &coded, full, MAX_DECODED).unwrap(), bits, "globals it skips"); + for parms in [bad.clone(), Object::Array(vec![bad.clone()]), indirect.clone(), Object::Array(vec![indirect])] { + let e = image(&jbig2, Some(parms.clone()), &coded, full, MAX_DECODED).unwrap_err(); + assert!(e.to_string().contains("JBIG2"), "{parms:?}: {e}"); + } + let behind = ["ASCIIHexDecode", "JBIG2Decode"]; + assert_eq!(image(&behind, None, &hex, full, MAX_DECODED).unwrap(), bits, "behind ASCIIHexDecode"); + assert!(image(&behind, Some(Object::Array(vec![Object::Null, bad.clone()])), &hex, full, MAX_DECODED).is_err(), "its own globals"); + assert_eq!(image(&behind, Some(Object::Array(vec![bad.clone(), Object::Null])), &hex, full, MAX_DECODED).unwrap(), bits, "another's"); + assert_eq!(image(&behind, Some(bad.clone()), &hex, full, MAX_DECODED).unwrap(), bits, "a lone dictionary, the first filter's"); + let after_crypt = ["Crypt", "JBIG2Decode"]; + assert!(image(&after_crypt, Some(Object::Array(vec![Object::Null, bad.clone()])), &coded, full, MAX_DECODED).is_err(), "after /Crypt"); + assert_eq!(image(&after_crypt, Some(Object::Array(vec![bad, Object::Null])), &coded, full, MAX_DECODED).unwrap(), bits, "/Crypt's"); + + for size in [(11, 3), (9, 3), (10, 2), (10, 4)] { + assert!(image(&jbig2, None, &coded, size, MAX_DECODED).is_err(), "a page other than {size:?}"); + } + assert_eq!(image(&jbig2, None, &coded, full, 6).unwrap(), bits, "the page within the bound"); + assert!(image(&jbig2, None, &coded, full, 5).is_err(), "the page past the bound"); +} + +#[test] +fn a_missing_file_id_is_derived_from_the_encryption_seed() { + let mut doc = Document::new_empty(); + let seed = [9u8; 32]; + doc.set_encryption(&crate::crypt::NewEncryption { + algorithm: crate::crypt::Algorithm::Aes256, + user_password: "user", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed, + }) + .unwrap(); + let id = doc.trailer().get(b"ID").and_then(|o| o.as_array()).and_then(|a| a.first()).and_then(|o| o.as_string()).map(|s| s.bytes.clone()); + assert_eq!(id.as_deref(), Some(crate::crypt::file_id(&seed).as_slice())); + + let mut kept = Document::new_empty(); + let fixed = Object::String(crate::cos::PdfString { bytes: b"0123456789abcdef".to_vec(), hex: true }); + kept.trailer_mut().set(b"ID".to_vec(), Object::Array(vec![fixed.clone(), fixed])); + kept.set_encryption(&crate::crypt::NewEncryption { + algorithm: crate::crypt::Algorithm::Aes256, + user_password: "user", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed, + }) + .unwrap(); + let still = kept.trailer().get(b"ID").and_then(|o| o.as_array()).and_then(|a| a.first()).and_then(|o| o.as_string()).map(|s| s.bytes.clone()); + assert_eq!(still.as_deref(), Some(b"0123456789abcdef".as_slice())); +} + +#[test] +fn a_read_state_matches_a_clone_and_no_changed_document() { + let mut doc = Document::new_empty(); + let a = doc.add(Object::Int(1)); + let state = doc.read_state(); + let snapshot = doc.clone(); + assert!(state.matches(&doc) && state.matches(&snapshot)); + // The same value set again is still a change: another allocation. + doc.set(a, Object::Int(1)); + assert!(!state.matches(&doc) && state.matches(&snapshot)); + let edited = doc.read_state(); + doc.free(a); + assert!(!edited.matches(&doc)); + let mut trailer_changed = snapshot.clone(); + trailer_changed.trailer_mut().set(b"Info".to_vec(), Object::Null); + assert!(!state.matches(&trailer_changed)); + // Another document's state never matches, even with the same changes. + let mut other = Document::new_empty(); + other.add(Object::Int(1)); + assert!(!other.read_state().matches(&snapshot)); +} + +/// Build a classic-xref file from object bodies (object i+1 = bodies[i]). +pub(crate) fn build(bodies: &[&str], trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, b) in bodies.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{b}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", bodies.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} {trailer} >>\nstartxref\n{xref}\n%%EOF\n", bodies.len() + 1).as_bytes()); + out +} + +fn assert_compact_matches_materialized(doc: &Document) { + assert!(matches!(doc.entries.as_ref(), XrefIndex::Compact { .. }), "valid xrefs should not retain a per-entry map"); + let (entries, _, revisions, _, _) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); + assert_eq!(revisions.as_slice(), doc.revisions()); + let max = entries.keys().next_back().copied().unwrap_or(0); + assert_eq!(doc.entries.max_num(), max); + for num in 0..=max { + assert_eq!(doc.xref_entry(num), entries.get(&num).copied(), "object {num}"); + } + let expected: Vec<_> = entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect(); + assert_eq!(doc.object_numbers(), expected); +} + +#[test] +fn compact_xref_tables_streams_and_revisions_match_the_materialized_reader() { + let classic = Document::open(Arc::new(build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 9 >>"], + "/Root 1 0 R", + ))) + .unwrap(); + assert_compact_matches_materialized(&classic); + + let mut sparse = b"%PDF-1.7\n".to_vec(); + let root_offset = sparse.len(); + sparse.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 5 0 R >>\nendobj\n"); + let pages_offset = sparse.len(); + sparse.extend_from_slice(b"5 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); + let xref = sparse.len(); + sparse.extend_from_slice( + format!( + "xref\n0 1\n0000000000 65535 f \n1 1\n{root_offset:010} 00000 n \n5 1\n{pages_offset:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n" + ) + .as_bytes(), + ); + let sparse = Document::open(Arc::new(sparse)).unwrap(); + assert_eq!(sparse.object_numbers(), vec![1, 5]); + assert_compact_matches_materialized(&sparse); + + let mut sparse_stream = b"%PDF-1.7\n".to_vec(); + let mut offsets = [0u32; 8]; + offsets[1] = sparse_stream.len() as u32; + sparse_stream.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 5 0 R >>\nendobj\n"); + offsets[5] = sparse_stream.len() as u32; + sparse_stream.extend_from_slice(b"5 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); + offsets[6] = sparse_stream.len() as u32; + sparse_stream.extend_from_slice(b"6 0 obj\n<< /Unused true >>\nendobj\n"); + offsets[7] = sparse_stream.len() as u32; + let xref_row = |kind: u8, offset: u32, generation: u16| { + let mut row = Vec::with_capacity(7); + row.push(kind); + row.extend_from_slice(&offset.to_be_bytes()); + row.extend_from_slice(&generation.to_be_bytes()); + row + }; + let mut rows = Vec::new(); + rows.extend(xref_row(0, 0, u16::MAX)); + rows.extend(xref_row(1, offsets[1], 0)); + rows.extend(xref_row(1, offsets[5], 0)); + rows.extend(xref_row(1, offsets[6], 0)); + rows.extend(xref_row(1, offsets[7], 0)); + sparse_stream.extend_from_slice( + format!("7 0 obj\n<< /Type /XRef /Size 8 /Root 1 0 R /W [1 4 2] /Index [0 2 5 3] /Length {} >>\nstream\n", rows.len()).as_bytes(), + ); + sparse_stream.extend_from_slice(&rows); + sparse_stream.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{}\n%%EOF\n", offsets[7]).as_bytes()); + let sparse_stream = Document::open(Arc::new(sparse_stream)).unwrap(); + assert_eq!(sparse_stream.object_numbers(), vec![1, 5, 6, 7]); + assert_compact_matches_materialized(&sparse_stream); + + let stream_bytes = crate::cos::write_full(&classic, &crate::cos::SaveOptions::default()).unwrap(); + let stream = Document::open(Arc::new(stream_bytes)).unwrap(); + assert!(stream.revisions().last().is_some_and(|revision| revision.is_stream)); + assert_compact_matches_materialized(&stream); + + let mut edited = classic.clone(); + edited.update_dict(ObjRef::new(1, 0), |dict| dict.set(b"Lang".to_vec(), Object::String(crate::cos::PdfString::literal("en")))).unwrap(); + edited.free(ObjRef::new(3, 0)); + let incremental = crate::cos::write_incremental(&edited, &crate::cos::SaveOptions::default()).unwrap(); + let updated = Document::open(Arc::new(incremental)).unwrap(); + assert_eq!(updated.revisions().len(), 2); + assert_compact_matches_materialized(&updated); +} + +#[test] +fn malformed_size_uses_the_materialized_compatibility_path() { + let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 9 >>"], "/Root 1 0 R"); + let marker = b"/Size 4"; + let at = bytes.windows(marker.len()).position(|window| window == marker).unwrap(); + bytes[at..at + marker.len()].copy_from_slice(b"/Size 2"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(matches!(doc.entries.as_ref(), XrefIndex::Materialized(..))); + assert_eq!(doc.next_num, 4); + assert_eq!(doc.object_numbers(), vec![1, 2, 3]); +} + +/// Entries the reader sets aside are remembered, so a caller comparing revisions can tell +/// that another reader may resolve the object differently. +#[test] +fn cross_reference_entries_that_are_set_aside_are_recorded() { + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let plain = Document::open(Arc::new(base.clone())).unwrap(); + assert!(plain.xref_oddities().is_empty()); + assert!((1..=3).all(|n| plain.xref_entry_is_exact(n))); + let prev = String::from_utf8_lossy(&base).rfind("xref\n0 4").unwrap(); + let third = plain.xref_entry(3).unwrap(); + + // A later section: object 3 "in use" at offset 0, object 2 listed twice with different + // offsets, and object 0 at offset 0 (the usual producer slip, which names no object). + let mut bytes = base.clone(); + let section = bytes.len(); + bytes.extend_from_slice( + format!("xref\n0 1\n0000000000 65535 n \n3 1\n0000000000 00000 n \n2 1\n0000000058 00000 n \n2 1\n0000000009 00000 n \ntrailer\n<< /Size 4 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.xref_oddities(), [(section as u64, 3), (section as u64, 2)]); + // What the reader resolves is what it resolved before: the first entry it kept. + assert_eq!(doc.xref_entry(3), Some(third)); + assert_eq!(doc.xref_entry(2), Some(XrefEntry::InFile { offset: 58, generation: 0 })); +} + +/// An entry that misses its object still resolves (by searching) but is not exact. +#[test] +fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() { + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let text = String::from_utf8_lossy(&base).into_owned(); + let at = text.find("3 0 obj").unwrap(); + let bytes = text.replacen(&format!("{at:010} 00000 n"), &format!("{:010} 00000 n", at + 2), 1).into_bytes(); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.get(ObjRef { num: 3, generation: 0 }).as_string().map(|s| s.bytes.clone()), Some(b"three".to_vec())); + assert!(!doc.xref_entry_is_exact(3)); + assert!(doc.xref_entry_is_exact(1) && doc.xref_entry_is_exact(2)); +} + +/// An object that can't be read at its entry or anywhere else is searched for in the whole file +/// once: every later read, in a clone too, gives the same error without searching again. An +/// edit of that number still answers for it. +#[test] +fn an_unreadable_object_is_searched_for_once() { + let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"]; + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap(); + FILE_SCANS.with(|scans| scans.set(0)); + let first = doc.try_get(3); + assert!(first.is_err(), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + } + let mut edited = doc.clone(); + assert_eq!(edited.try_get(3), first); + assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "searches for an object that isn't there"); + let report = doc.unreadable_objects(); + assert_eq!(report.len(), 1, "cached failures must stay listed exactly once: {report:?}"); + assert_eq!(edited.unreadable_objects(), report, "clones retain the cached failure diagnostic"); + edited.set(ObjRef::new(3, 0), Object::Int(7)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Int(7))); + assert_eq!(doc.try_get(3), first, "the edit is the clone's alone"); + edited.free(ObjRef::new(3, 0)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); +} + +/// A member of an object stream that can't be decoded (here, one that decodes past the document's +/// stream limit) is decoded once: every later read, in a clone too, gives the same error without +/// decoding the stream again. The stream itself loads, so no search of the file is involved. +#[test] +fn a_member_of_an_undecodable_object_stream_is_decoded_once() { + let file = object_stream_file("3 0 ", &format!("<< /Hidden ({}) >>", "x".repeat(4096)), 0); + let doc = Document::open_with_stream_limit(Arc::new(file), None, 1024).unwrap(); + OBJSTM_DECODES.with(|decodes| decodes.set(0)); + let first = doc.try_get(3); + assert!(matches!(first, Err(CosError::Filter(_))), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + } + assert_eq!(doc.clone().try_get(3), first); + assert_eq!(OBJSTM_DECODES.with(std::cell::Cell::get), 1, "decodes an object stream that can't be decoded"); +} + +/// Clones share the cache, and an edit is one clone's alone: a member of an object stream that +/// one clone freed or replaced still reads, in the others, from the file's stream. +#[test] +fn an_edited_object_stream_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); + let doc = Document::open(file.clone()).unwrap(); + let mut freed = doc.clone(); + freed.free(ObjRef::new(4, 0)); + assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4))); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the free reached another clone"); + let doc = Document::open(file).unwrap(); + let mut replaced = doc.clone(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + replaced.set(ObjRef::new(4, 0), crate::cos::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec())); + assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no")); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); +} + +/// A member another clone read from the file's object stream is not what a clone that replaced +/// the stream reads, nor is one the document read itself before replacing it. +#[test] +fn a_replaced_object_stream_hides_the_members_read_before() { + let replacement = || { + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + crate::cos::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec()) + }; + let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); + let doc = Document::open(file.clone()).unwrap(); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); + let mut replaced = doc.clone(); + replaced.set(ObjRef::new(4, 0), replacement()); + assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no"), "another clone's read of the file's stream"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); + let mut doc = Document::open(file.clone()).unwrap(); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); + doc.set(ObjRef::new(4, 0), replacement()); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("no"), "the document's own read before the edit"); + let doc = Document::open(file).unwrap(); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); + let mut freed = doc.clone(); + freed.free(ObjRef::new(4, 0)); + assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4)), "another clone's read before the free"); +} + +/// An object stream another clone framed by the file's indirect `/Length` is not what a clone +/// that edited that length reads. +#[test] +fn an_edited_object_stream_length_hides_the_members_read_before() { + let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); + let doc = Document::open(file).unwrap(); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here")); + let mut edited = doc.clone(); + edited.set(ObjRef::new(6, 0), Object::Int("3 0 << /Hidden (cut".len() as i64)); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "another clone's framing of the stream"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + +/// A stream another clone read by the file's indirect `/Length` is not what a clone that edited +/// that length reads. +#[test] +fn an_edited_stream_length_hides_the_stream_read_before() { + let bodies = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /Length 4 0 R >>\nstream\nshort\nendstream long\nendstream", + "20", + ]; + let mut out = b"%PDF-1.4\n".to_vec(); + let mut table = String::from("xref\n0 5\n0000000000 65535 f \n"); + for (n, body) in bodies.iter().enumerate() { + table.push_str(&format!("{:010} 00000 n \n", out.len())); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", n + 1).as_bytes()); + } + let start = out.len(); + out.extend_from_slice(format!("{table}trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{start}\n%%EOF\n").as_bytes()); + let data = |doc: &Document| match &*doc.get(ObjRef::new(3, 0)) { + Object::Stream(s) => s.raw.to_vec(), + other => panic!("{other:?}"), + }; + let doc = Document::open(Arc::new(out)).unwrap(); + assert_eq!(data(&doc), b"short\nendstream long"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(4, 0), Object::Int(5)); + assert_eq!(data(&edited), b"short", "another clone's read by the file's length"); + assert_eq!(data(&doc), b"short\nendstream long", "the edited length reached another clone"); +} + +/// A file whose object 3 sits in an object stream PNG-predicted by `/DecodeParms 6 1 R`, a +/// generation the file doesn't have, and the parameters that make it decode. +fn predicted_object_stream() -> (Vec, Dict) { + let members = b"3 0 << /Hidden (yes) >>"; + // PNG Sub, one row. + let mut rows = vec![1u8]; + let mut before = 0u8; + for b in members { + rows.push(b.wrapping_sub(before)); + before = *b; + } + let coded = crate::filters::encode_flate(&rows); + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 7]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /DecodeParms 6 1 R /Length {} >>\nstream\n", coded.len()).into_bytes(); + stm.extend_from_slice(&coded); + stm.extend_from_slice(b"\nendstream"); + obj(&mut out, 4, &stm); + let data = [2u8, 0, 4, 0]; + let mut xs = format!("<< /Type /XRef /Size 7 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); + xs.extend_from_slice(&data); + xs.extend_from_slice(b"\nendstream"); + obj(&mut out, 5, &xs); + obj(&mut out, 6, b"null"); + let table = out.len(); + out.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); + for (n, off) in offs.iter().enumerate().skip(1) { + let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(format!("trailer << /Size 7 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(11)); + parms.set(b"Columns".to_vec(), Object::Int(members.len() as i64)); + (out, parms) +} + +fn hidden(doc: &Document) -> Option> { + doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.get(b"Hidden")).and_then(Object::as_string).map(|s| s.bytes.clone()) +} + +/// An object stream whose `/DecodeParms` names a generation the file doesn't have decodes without +/// them; a clone that sets that generation decodes with them, not by the stream another clone read. +#[test] +fn parameters_an_edit_makes_match_hide_the_object_stream_read_before() { + let (file, parms) = predicted_object_stream(); + let doc = Document::open(Arc::new(file)).unwrap(); + assert_eq!(hidden(&doc), None, "no object 6 1: no predictor undone"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(6, 1), Object::Dict(parms)); + assert_eq!(hidden(&edited), Some(b"yes".to_vec()), "another clone's stream, decoded without them"); +} + +/// The other way round: an object stream a clone decoded by its edited parameters stays that +/// clone's, so the document it was cloned from still decodes by the file's. +#[test] +fn an_object_stream_decoded_by_edited_parameters_is_not_shared() { + let (file, parms) = predicted_object_stream(); + let doc = Document::open(Arc::new(file)).unwrap(); + let mut edited = doc.clone(); + edited.set(ObjRef::new(6, 1), Object::Dict(parms)); + assert_eq!(hidden(&edited), Some(b"yes".to_vec())); + assert_eq!(hidden(&doc), None, "the clone's decoding, read from the shared streams"); +} + +/// What a load read is kept only up to a bound; past it the load counts as having read every +/// object, so a document with any edit reads it again. +#[test] +fn a_load_that_read_too_much_counts_as_reading_everything() { + use super::state::{ReadFrame, note_read}; + let mut overlay = BTreeMap::new(); + overlay.insert(999_999, Slot::Freed(1)); + let frame = ReadFrame::enter(); + for n in 0..8 { + note_read(n, None); + } + assert!(!frame.reads().touch(&overlay), "a few reads, none edited"); + assert!(!frame.reads().touch(&BTreeMap::new()), "nothing is edited"); + for n in 8..100 { + note_read(n, None); + } + assert!(frame.reads().touch(&overlay), "more objects than an entry keeps"); + drop(frame); + let outer = ReadFrame::enter(); + let inner = ReadFrame::enter(); + for n in 0..5_000 { + note_read(n % 4, None); + } + assert!(inner.reads().touch(&overlay), "more reads than the thread keeps"); + drop(inner); + assert!(outer.reads().touch(&overlay), "an outer frame missed them too"); +} + +/// A member that failed to load from the file's object stream is kept as a failure, but replacing +/// or freeing that stream forgets it: the member then reads from the edit. +#[test] +fn an_edited_object_stream_forgets_its_members_failures() { + let file = Arc::new(object_stream_file("3 0 ", ") broken", 0)); + let mut doc = Document::open(file.clone()).unwrap(); + assert!(doc.try_get(3).is_err(), "the file's member does not parse"); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + doc.set(ObjRef::new(4, 0), crate::cos::Stream::from_raw(dict, b"3 0 << /Hidden (fixed) >>".to_vec())); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("fixed"), "the cached failure outlived the edit"); + assert!(doc.try_get(3).is_ok()); + let mut doc = Document::open(file).unwrap(); + assert!(matches!(doc.try_get(3), Err(CosError::Syntax { .. })), "the file's member does not parse"); + doc.free(ObjRef::new(4, 0)); + assert_eq!(doc.try_get(3), Err(CosError::MissingObject(4)), "the freed stream's member"); +} + +/// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: +/// the others still read its members within the file's length. +#[test] +fn an_edited_object_stream_length_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); + let doc = Document::open(file).unwrap(); + let mut edited = doc.clone(); + // Ends the stream at the `endstream` inside the member's string. + let cut = "3 0 << /Hidden (cut".len(); + edited.set(ObjRef::new(6, 0), Object::Int(cut as i64)); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + +/// A `/Length` read through one clone's edits and reused within the same load still keeps that +/// load to the clone. Here the file's offsets are header-relative behind junk, so object 10's +/// offset first lands on object 12, whose length (object 8, in the edited object stream 9) is +/// read; object 10's own length then loads object stream 4, framed by object 8 too. +#[test] +fn a_reused_edited_length_leaves_other_clones_members_alone() { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 13]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let stream = |dict: &str, data: &str| format!("<< {dict} >>\nstream\n{data}\nendstream").into_bytes(); + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let inner = "11 0 3 2 5 << /Hidden (cut endstream here) >>"; + obj(&mut out, 4, &stream("/Type /ObjStm /N 2 /First 9 /Length 8 0 R", inner)); + let lengths = format!("8 0 {}", inner.len()); + obj(&mut out, 9, &stream(&format!("/Type /ObjStm /N 1 /First 4 /Length {}", lengths.len()), &lengths)); + obj(&mut out, 12, &stream("/Length 8 0 R", &"x".repeat(40))); + obj(&mut out, 10, &stream("/Length 11 0 R", "PDATA")); + // A cross-reference stream: a hybrid file's /XRefStm is not looked for behind junk. + let compressed = [(3, 4, 1), (8, 9, 0), (11, 4, 0)]; + let mut data = Vec::new(); + for (n, off) in offs.iter().enumerate() { + let row = match compressed.iter().find(|c| c.0 == n) { + Some(&(_, stream, index)) => [2, 0, stream, index], + None if *off == 0 => [0, 0, 0, 0], + None => { + let [.., hi, lo] = u32::try_from(*off).unwrap().to_be_bytes(); + [1, hi, lo, 0] + } + }; + data.extend_from_slice(&row); + } + let table = out.len(); + let dict = format!("<< /Type /XRef /Size 14 /W [1 2 1] /Index [0 13] /Root 1 0 R /Length {} >>", data.len()); + out.extend_from_slice(format!("13 0 obj\n{dict}\nstream\n").as_bytes()); + out.extend_from_slice(&data); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{table}\n%%EOF\n").as_bytes()); + let mut file = vec![b'J'; offs[10] - offs[12] - 1]; + file.push(b'\n'); + file.extend_from_slice(&out); + let doc = Document::open(Arc::new(file)).unwrap(); + let mut edited = doc.clone(); + let cut = "11 0 3 2 5 << /Hidden (cut".len(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + edited.set(ObjRef::new(9, 0), crate::cos::Stream::from_raw(dict, format!("8 0 {cut}").into_bytes())); + assert!(matches!(&*edited.get(ObjRef::new(10, 0)), Object::Stream(s) if &s.raw[..] == b"PDATA")); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + +/// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered +/// `1 n` instead of `0 n` (#2676): every row is one number off. +fn misnumbered_table(start: u32) -> Vec { + let bodies = ["<< /Marker (first object) >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Type /Catalog /Pages 2 0 R >>"]; + let mut out = b"%PDF-1.7 +" + .to_vec(); + let mut offsets = Vec::new(); + for (i, body) in bodies.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice( + format!( + "{} 0 obj +{body} +endobj +", + i + 1 + ) + .as_bytes(), + ); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n{start} 4\n0000000000 65535 f \n").as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size 4 /Root 3 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + out +} + +#[test] +fn a_table_numbered_from_1_that_begins_with_object_0_is_read_from_0() { + let doc = Document::open(Arc::new(misnumbered_table(1))).unwrap(); + let first = doc.get(ObjRef::new(1, 0)).as_dict().cloned().expect("object 1 is not free"); + assert!(first.contains(b"Marker")); + assert_eq!(doc.root(), Some(ObjRef::new(3, 0))); + assert!(doc.repair_log().iter().any(|l| l.contains("numbered from 1")), "{:?}", doc.repair_log()); + // A correct table reads the same and logs nothing. + let fine = Document::open(Arc::new(misnumbered_table(0))).unwrap(); + assert!(fine.get(ObjRef::new(1, 0)).as_dict().is_some_and(|d| d.contains(b"Marker"))); + assert!(!fine.repair_log().iter().any(|l| l.contains("numbered from 1"))); +} + +#[test] +fn a_table_that_really_starts_at_1_keeps_its_numbers() { + // Object 1 deleted for good (generation 65535) in a subsection that does start at 1: its + // second row locates object 2, so nothing is shifted. + let mut bytes = b"%PDF-1.7\n".to_vec(); + let pages = bytes.len(); + bytes.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); + let catalog = bytes.len(); + bytes.extend_from_slice(b"3 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); + let xref = bytes.len(); + bytes.extend_from_slice( + format!( + "xref\n0 1\n0000000000 65535 f \n1 3\n0000000000 65535 f \n{pages:010} 00000 n \n{catalog:010} 00000 n \ntrailer\n<< /Size 4 /Root 3 0 R >>\nstartxref\n{xref}\n%%EOF\n" + ) + .as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(doc.get(ObjRef::new(2, 0)).as_dict().is_some_and(|d| d.name(b"Type") == Some(b"Pages"))); + assert!(doc.get(ObjRef::new(1, 0)).as_dict().is_none(), "object 1 stays free"); + assert!(!doc.repair_log().iter().any(|l| l.contains("numbered from 1"))); +} + +/// A one-page scan: the page draws image 5 with content stream 4. +fn scanned_page() -> Vec { + build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 10 10] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>", + "<< /Length 21 >>\nstream\n10 0 0 10 0 0 cm /Im0 Do\nendstream", + "<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>\nstream\nA\nendstream", + ], + "/Root 1 0 R", + ) +} + +/// `bytes` with its cross-reference table replaced by `table` (the lines after `xref`). +fn with_table(bytes: &[u8], table: &[String], size: usize) -> Vec { + let text = String::from_utf8_lossy(bytes).into_owned(); + let at = text.rfind("\nxref\n").unwrap() + 1; + let mut out = bytes[..at].to_vec(); + out.extend_from_slice(format!("xref\n{}\ntrailer\n<< /Size {size} /Root 1 0 R >>\nstartxref\n{at}\n%%EOF\n", table.join("\n")).as_bytes()); + out +} + +/// The table's entry lines, object 0 first. +fn table_entries(bytes: &[u8]) -> Vec { + let text = String::from_utf8_lossy(bytes).into_owned(); + let at = text.rfind("\nxref\n").unwrap() + 1; + text[at..].lines().skip(2).take_while(|l| !l.starts_with("trailer")).map(str::to_string).collect() +} + +/// The classic off-by-one: the subsection says it starts at 0 but lists objects 1 to 5, so +/// the last object has no entry. Viewers rebuild the table and show the page; a reader that +/// took the missing entry for a null object dropped the image when pages were copied (#1628). +#[test] +fn an_off_by_one_xref_subsection_still_finds_the_last_object() { + let base = scanned_page(); + let entries = table_entries(&base); + let mut table = vec!["0 5".to_string()]; + table.extend(entries[1..6].iter().cloned()); + let doc = Document::open(Arc::new(with_table(&base, &table, 5))).unwrap(); + let image = doc.get(ObjRef::new(5, 0)); + assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "{image:?}"); + assert!(matches!(&*image, Object::Stream(_))); + assert!(matches!(&*doc.get(ObjRef::new(4, 0)), Object::Stream(s) if &s.raw[..] == b"10 0 0 10 0 0 cm /Im0 Do")); + assert!(doc.repair_log().iter().any(|l| l.contains("object headers")), "{:?}", doc.repair_log()); +} + +/// An "in use" entry at offset 0 says nothing about where the object is: the object is +/// found by its header, and the repair is recorded. +#[test] +fn an_object_listed_at_offset_zero_is_found_by_its_header() { + let base = scanned_page(); + for num in [4usize, 5] { + let mut table = vec!["0 6".to_string()]; + table.extend(table_entries(&base)); + table[num + 1] = "0000000000 00000 n ".into(); + let doc = Document::open(Arc::new(with_table(&base, &table, 6))).unwrap(); + assert!(matches!(&*doc.get(ObjRef::new(num as u32, 0)), Object::Stream(_)), "object {num}"); + assert!(doc.repair_log().iter().any(|l| l.contains("object headers")), "object {num}: {:?}", doc.repair_log()); + } +} + +/// A valid table that simply does not list every number below `/Size` is not repaired: an +/// unlisted number is a null object (§7.3.10), even when an old body with that number is +/// still in the file. +#[test] +fn a_consistent_table_with_unlisted_numbers_is_taken_as_it_is() { + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let at = String::from_utf8_lossy(&base).rfind("\nxref\n").unwrap() + 1; + let mut bytes = base[..at].to_vec(); + bytes.extend_from_slice(b"4 0 obj\n(stale)\nendobj\n"); + let mut table = vec!["0 4".to_string()]; + table.extend(table_entries(&base)); + let doc = Document::open(Arc::new(with_table(&[&bytes[..], &base[at..]].concat(), &table, 6))).unwrap(); + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + assert_eq!(*doc.get(ObjRef::new(4, 0)), Object::Null); + assert!(matches!(&*doc.get(ObjRef::new(3, 0)), Object::String(_))); +} + +/// Filling the gaps of a damaged table never brings back an object that an incremental +/// update freed: its body is still in the file, but the newer section says it is gone. +#[test] +fn an_object_freed_by_an_update_stays_freed_when_the_table_is_repaired() { + let base = scanned_page(); + // The original table lists the content stream at offset 0, so the gaps get filled. + let mut table = vec!["0 6".to_string()]; + table.extend(table_entries(&base)); + table[5] = "0000000000 00000 n ".into(); + let mut bytes = with_table(&base, &table, 6); + let prev = String::from_utf8_lossy(&bytes).rfind("\nxref\n").unwrap() + 1; + let section = bytes.len(); + bytes.extend_from_slice( + format!("xref\n5 1\n0000000000 00001 f \ntrailer\n<< /Size 6 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(matches!(&*doc.get(ObjRef::new(4, 0)), Object::Stream(_)), "the gap is filled"); + assert_eq!(*doc.get(ObjRef::new(5, 0)), Object::Null, "the freed image stays freed"); + assert_eq!(doc.xref_entry(5), Some(XrefEntry::Free { next_generation: 1 })); +} + +/// Repairing a damaged table fills only the numbers the damage leaves out: an old body of a +/// number no section lists, under an oversized `/Size`, stays a null object. +#[test] +fn a_repaired_table_leaves_an_unlisted_number_null() { + let base = scanned_page(); + let at = String::from_utf8_lossy(&base).rfind("\nxref\n").unwrap() + 1; + let entries = table_entries(&base); + let mut offset_zero = vec!["0 6".to_string()]; + offset_zero.extend(entries.iter().cloned()); + offset_zero[5] = "0000000000 00000 n ".into(); + let mut off_by_one = vec!["0 5".to_string()]; + off_by_one.extend(entries[1..6].iter().cloned()); + for (name, table, found) in [("offset 0", offset_zero, 4), ("off by one", off_by_one, 5)] { + let bytes = [&base[..at], b"7 0 obj\n(stale)\nendobj\n", &base[at..]].concat(); + let doc = Document::open(Arc::new(with_table(&bytes, &table, 9))).unwrap(); + assert!(matches!(&*doc.get(ObjRef::new(found, 0)), Object::Stream(_)), "{name}: the gap is filled"); + assert_eq!(*doc.get(ObjRef::new(7, 0)), Object::Null, "{name}: {:?}", doc.repair_log()); + assert_eq!(doc.xref_entry(7), None, "{name}"); + } +} + +/// The last object of a subsection numbered one too low is filled from that subsection, +/// whatever later happens to the row before it: an incremental save that rewrites object 4, +/// or object 4's own row at offset 0. +#[test] +fn the_last_object_of_a_shifted_subsection_is_filled_whatever_its_neighbour_says() { + let base = scanned_page(); + let entries = table_entries(&base); + let mut off_by_one = vec!["0 5".to_string()]; + off_by_one.extend(entries[1..6].iter().cloned()); + let shifted = with_table(&base, &off_by_one, 5); + let prev = String::from_utf8_lossy(&shifted).rfind("\nxref\n").unwrap() + 1; + let mut updated = shifted.clone(); + let body = updated.len(); + updated.extend_from_slice(b"4 0 obj\n<< /Length 21 >>\nstream\n10 0 0 10 0 0 cm /Im0 Do\nendstream\nendobj\n"); + let section = updated.len(); + updated.extend_from_slice( + format!("xref\n4 1\n{body:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), + ); + let mut neighbour_at_zero = off_by_one.clone(); + neighbour_at_zero[5] = "0000000000 00000 n ".into(); + for (name, bytes) in [("updated", updated), ("row 4 at offset 0", with_table(&base, &neighbour_at_zero, 5))] { + let doc = Document::open(Arc::new(bytes)).unwrap(); + let image = doc.get(ObjRef::new(5, 0)); + assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "{name}: {:?}", doc.repair_log()); + } +} + +/// The image the first page draws as `/Im1`, followed through the page tree. +fn first_page_image(doc: &Document) -> Arc { + let catalog = doc.get(doc.root().unwrap()); + let pages = doc.resolve(catalog.as_dict().unwrap().get(b"Pages").unwrap()); + let page = doc.resolve(&pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap()[0]); + let resources = doc.resolve(page.as_dict().unwrap().get(b"Resources").unwrap()); + let xobjects = doc.resolve(resources.as_dict().unwrap().get(b"XObject").unwrap()); + doc.resolve(xobjects.as_dict().unwrap().get(b"Im1").unwrap()) +} + +/// The opposite off-by-one (#1628): the subsection says it starts at 1, but its first entry +/// is object 0's free-list head, so every entry lands one number too high and object 1, the +/// image, reads as free. The offset the table gives object 2 holds `1 0 obj`, so the +/// subsection is read from 0, and the image survives a full save. +#[test] +fn a_subsection_from_one_that_opens_with_the_free_list_head_is_read_from_zero() { + let base = build( + &[ + "<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Length 12 >>\nstream\nABCDEFGHIJKL\nendstream", + "<< /Length 29 >>\nstream\nq 72 0 0 72 0 0 cm /Im1 Do Q\nendstream", + "<< /Type /Page /Parent 4 0 R /MediaBox [0 0 72 72] /Resources << /XObject << /Im1 1 0 R >> >> /Contents 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Catalog /Pages 4 0 R >>", + "<< /Subject (synthetic) >>", + ], + "/Root 5 0 R /Info 6 0 R", + ); + // /Size 7 is one short of the shifted numbers; /Size 8 covers them all. Junk before the + // header moves every object past the offsets the table gives; it is not a comment, which + // the lexer would skip on its way to `1 0 obj` anyway. + for (size, junk) in [(7, ""), (8, ""), (7, "Content-Type: application/pdf\n\n")] { + let text = String::from_utf8_lossy(&base).replace("\nxref\n0 7\n", "\nxref\n1 7\n").replace("/Size 7 ", &format!("/Size {size} ")); + let doc = Document::open(Arc::new(format!("{junk}{text}").into_bytes())).unwrap(); + let image = first_page_image(&doc); + assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "/Size {size} {junk:?}: {image:?}"); + assert_eq!(doc.xref_entry(0), Some(XrefEntry::Free { next_generation: 65535 }), "/Size {size} {junk:?}"); + assert!(doc.repair_log().iter().any(|l| l.contains("from 0")), "/Size {size} {junk:?}: {:?}", doc.repair_log()); + let saved = Document::open(Arc::new(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap())).unwrap(); + let image = first_page_image(&saved); + assert!(matches!(&*image, Object::Stream(s) if &s.raw[..] == b"ABCDEFGHIJKL"), "/Size {size} {junk:?}: {image:?}"); + } +} + +/// Object 1 really freed: the subsection from 1 opens with a free entry, but the next entry's +/// offset holds object 2, not object 1. The table is taken as it is, and the old body of +/// object 1 still in the file stays freed. +#[test] +fn a_subsection_from_one_whose_next_offset_holds_object_two_is_taken_as_it_is() { + let base = build(&["(freed)", "<< /Type /Catalog /Pages 3 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 2 0 R"); + let entries = table_entries(&base); + let mut table = vec!["1 3".to_string(), "0000000000 65535 f ".to_string()]; + table.extend(entries[2..].iter().cloned()); + let text = String::from_utf8_lossy(&base).into_owned(); + let at = text.rfind("\nxref\n").unwrap() + 1; + for junk in ["", "Content-Type: application/pdf\n\n"] { + let mut bytes = junk.as_bytes().to_vec(); + bytes.extend_from_slice(&base[..at]); + bytes.extend_from_slice(format!("xref\n{}\ntrailer\n<< /Size 4 /Root 2 0 R >>\nstartxref\n{at}\n%%EOF\n", table.join("\n")).as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.xref_entry(1), Some(XrefEntry::Free { next_generation: 65535 }), "{junk:?}"); + assert_eq!(*doc.get(ObjRef::new(1, 0)), Object::Null, "{junk:?}: the freed object stays freed"); + assert!(matches!(&*doc.get(ObjRef::new(2, 0)), Object::Dict(d) if d.name(b"Type") == Some(&b"Catalog"[..])), "{junk:?}"); + assert!(!doc.repair_log().iter().any(|l| l.contains("from 0")), "{junk:?}: {:?}", doc.repair_log()); + assert!(!junk.is_empty() || doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + } +} + +#[test] +fn opening_an_exhausted_classic_xref_repairs_and_allocates() { + // The extra in-use row need not be loaded to exhaust the next object number. + let bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\nxref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n4294967295 1\n0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\nstartxref\n110\n%%EOF\n"; + assert_eq!(bytes.len(), 266); + let mut doc = Document::open(Arc::new(bytes.to_vec())).unwrap(); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + assert!(doc.repair_log().iter().any(|line| line.contains("exhausted xref"))); + assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); +} + +#[test] +fn opening_an_exhausted_xref_stream_repairs_and_allocates() { + let mut bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n".to_vec(); + let offset = bytes.len(); + bytes.extend_from_slice(b"3 0 obj\n<< /Type /XRef /Size 4 /Root 1 0 R /W [1 4 2] /Index [0 3 4294967295 1] /Length 28 >>\nstream\n"); + for (kind, position, generation) in [(0_u8, 0_u32, 65535_u16), (1, 9, 0), (1, 58, 0), (1, 58, 0)] { + bytes.push(kind); + bytes.extend_from_slice(&position.to_be_bytes()); + bytes.extend_from_slice(&generation.to_be_bytes()); + } + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{offset}\n%%EOF\n").as_bytes()); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + assert!(doc.repair_log().iter().any(|line| line.contains("exhausted xref"))); + assert_eq!(doc.add(Object::Null), ObjRef::new(4, 0)); +} + +#[test] +fn exhausted_size_and_high_edits_allocate_without_overwriting_objects() { + let ordinary = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + for size in ["4294967295", "4294967296", "9223372036854775807"] { + let bytes = String::from_utf8(ordinary.clone()).unwrap().replace("/Size 3", &format!("/Size {size}")); + let mut doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert!(doc.repair_log().iter().any(|line| line.contains("exhausted trailer"))); + assert_eq!(doc.add(Object::Int(42)), ObjRef::new(3, 0)); + doc.set(ObjRef::new(u32::MAX, 0), Object::Int(99)); + doc.set(ObjRef::new(u32::MAX - 1, 0), Object::Int(98)); + assert_eq!(doc.add(Object::Int(43)), ObjRef::new(4, 0)); + assert_eq!(doc.get(ObjRef::new(3, 0)).as_int(), Some(42)); + assert_eq!(doc.get(ObjRef::new(u32::MAX, 0)).as_int(), Some(99)); + assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); + } + let mut doc = Document::new_empty(); + doc.set(ObjRef::new(u32::MAX - 2, 0), Object::Null); + assert_eq!(doc.add(Object::Null), ObjRef::new(u32::MAX - 1, 0)); + assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); + assert_eq!(doc.add(Object::Null), ObjRef::new(4, 0)); +} + +#[test] +fn exhausted_xref_controls_preserve_ordinary_ignored_and_repaired_entries() { + let ordinary = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + for (extra, repaired) in + [("", false), ("4294967295 1\n0000000000 00000 n \n", false), ("4294967295 2\n0000000058 00000 n \n0000000058 00000 n \n", true)] + { + let bytes = String::from_utf8(ordinary.clone()).unwrap().replacen("trailer\n", &format!("{extra}trailer\n"), 1); + let mut doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); + assert_eq!(!doc.repair_log().is_empty(), repaired); + if repaired { + assert!(doc.repair_log().iter().any(|line| line.contains("xref") && line.contains("range"))); + } + assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); + } +} + +#[test] +fn invalid_xref_object_ranges_are_reconstructed() { + // Original in-memory PDF: an intact catalog followed by a damaged xref section. + for stream in [false, true] { + for (start, count) in [(i64::MAX, 2), (-1, 2), (4_294_967_296, 1), (4_294_967_295, 2)] { + let mut bytes = + b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n".to_vec(); + let offset = bytes.len(); + if stream { + bytes.extend_from_slice( + format!("3 0 obj\n<< /Type /XRef /Root 1 0 R /Size 4 /W [1 1 1] /Index [{start} {count}] /Length 6 >>\nstream\n").as_bytes(), + ); + bytes.extend_from_slice(&[0, 0, 0, 0, 0, 0]); + bytes.extend_from_slice(b"\nendstream\nendobj\n"); + } else { + bytes.extend_from_slice( + format!("xref\n{start} {count}\n0000000000 65535 f \n0000000000 65535 f \ntrailer\n<< /Root 1 0 R /Size 3 >>\n").as_bytes(), + ); + } + bytes.extend_from_slice(format!("startxref\n{offset}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!( + doc.repair_log().iter().any(|line| line.contains("xref") && line.contains("range")), + "stream={stream}, start={start}, count={count}: {:?}", + doc.repair_log() + ); + assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); + } + } +} + +#[test] +fn objects_only_an_unreadable_older_section_locates_are_found_by_scanning() { + // The newest section lists the catalog only; its /Prev points past the end of the file, + // so the page tree is reachable only by scanning object headers, as readers do (#1656). + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, body) in + [(1, "<< /Type /Catalog /Pages 2 0 R >>"), (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), (3, "<< /Type /Page /Parent 2 0 R >>")] + { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); + } + let xref = bytes.len(); + bytes.extend_from_slice( + format!( + "xref\n0 2\n0000000000 65535 f \n{:010} 00000 n \ntrailer\n<< /Root 1 0 R /Size 4 /Prev 999999 >>\nstartxref\n{xref}\n%%EOF\n", + offsets[0] + ) + .as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.get(ObjRef::new(2, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(b"Pages".as_slice())); + assert_eq!(doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(b"Page".as_slice())); + assert!(doc.repair_log().iter().any(|l| l.contains("999999")), "{:?}", doc.repair_log()); +} + +#[test] +fn self_referential_length_does_not_overflow() { + let bytes = build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Length 3 0 R >>\nstream\nabc\nendstream"], + "/Root 1 0 R", + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + // Either recovered via the endstream search or reported — but never a stack overflow. + let o = doc.get(ObjRef::new(3, 0)); + if let Object::Stream(s) = o.as_ref() { + assert_eq!(&s.raw[..], b"abc"); + } +} + +#[test] +fn mutually_referential_lengths_do_not_overflow() { + let bytes = build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /Length 4 0 R >>\nstream\nabc\nendstream", + "<< /Length 3 0 R >>\nstream\nxyz\nendstream", + ], + "/Root 1 0 R", + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let _ = doc.get(ObjRef::new(3, 0)); + let _ = doc.get(ObjRef::new(4, 0)); +} + +#[test] +fn reads_classic_revision_and_edits_are_copy_on_write() { + let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.revisions().len(), 1); + assert!(doc.repair_log().is_empty()); + let mut edited = doc.clone(); + edited.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(crate::cos::PdfString::literal("en")))).unwrap(); + assert!(edited.is_modified() && !doc.is_modified(), "clone is an independent snapshot"); + assert!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().get(b"Lang").is_none()); +} + +#[test] +fn object_numbers_merge_source_and_overlay_in_order() { + let mut doc = Document::new_empty(); + doc.overlay.clear(); + doc.entries = Arc::new(XrefIndex::Materialized( + Arc::new(BTreeMap::from([ + (0, XrefEntry::Free { next_generation: 65535 }), + (2, XrefEntry::InFile { offset: 10, generation: 7 }), + (4, XrefEntry::Free { next_generation: 9 }), + (6, XrefEntry::InStream { stream: 80, index: 1 }), + (8, XrefEntry::InFile { offset: 20, generation: 5 }), + (10, XrefEntry::InFile { offset: 30, generation: 3 }), + (u32::MAX, XrefEntry::InFile { offset: 40, generation: 1 }), + ])), + Arc::default(), + )); + doc.set(ObjRef::new(1, 2), Object::Int(1)); // before the first live source slot + doc.set(ObjRef::new(2, 8), Object::Int(2)); // override, without duplication + doc.set(ObjRef::new(4, 9), Object::Int(4)); // reuse a source free slot + doc.free(ObjRef::new(6, 0)); // remove a compressed source object + doc.free(ObjRef::new(7, 2)); // free a number absent from the source + doc.free(ObjRef::new(10, 3)); + doc.set(ObjRef::new(11, 4), Object::Int(11)); + assert_eq!(doc.object_numbers(), vec![1, 2, 4, 8, 11, u32::MAX]); + assert_eq!(doc.generation(2), 8); + assert_eq!(doc.generation(4), 9); + assert_eq!(doc.generation(6), 1); + assert_eq!(doc.generation(8), 5); + assert_eq!(doc.generation(10), 4); + doc.overlay.clear(); + assert_eq!(doc.object_numbers(), vec![2, 6, 8, 10, u32::MAX]); + doc.entries = Arc::new(XrefIndex::empty()); + assert!(doc.object_numbers().is_empty()); + doc.set(ObjRef::new(5, 7), Object::Int(5)); + assert_eq!(doc.object_numbers(), vec![5]); +} + +#[test] +fn full_object_scans_preserve_edits_without_retaining_the_scan() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>".to_owned(), "<< /Type /Pages /Kids [] /Count 0 >>".to_owned()]; + bodies.extend((0..400).map(|i| format!("<< /Value {i} >>"))); + let refs: Vec<_> = bodies.iter().map(String::as_str).collect(); + let mut doc = Document::open(Arc::new(build(&refs, "/Root 1 0 R"))).unwrap(); + doc.set(ObjRef::new(3, 0), Object::Int(900)); + doc.free(ObjRef::new(4, 0)); + let added = doc.add(Object::Int(901)); + let cached = doc.cache.lock().unwrap().len(); + let held: Vec<_> = doc.scan_objects().collect(); + assert_eq!(held.len(), 402); + assert_eq!(doc.cache.lock().unwrap().len(), cached, "scanning must not populate the editor cache"); + assert!(doc.objstms.lock().unwrap().is_empty()); + assert!(!held.iter().any(|(r, _)| r.num == 4)); + assert_eq!(held.iter().find(|(r, _)| r.num == 3).unwrap().1.as_int(), Some(900)); + assert_eq!(held.iter().find(|(r, _)| *r == added).unwrap().1.as_int(), Some(901)); + assert_eq!(held.iter().find(|(r, _)| r.num == 400).unwrap().1.as_dict().unwrap().int(b"Value"), Some(397)); + // Previously returned objects survive cache eviction and dropping the source. + drop(doc); + assert_eq!(held.iter().find(|(r, _)| r.num == 128).unwrap().1.as_dict().unwrap().int(b"Value"), Some(125)); +} + +#[test] +fn temporary_reader_keeps_file_views_and_evicts_data_of_its_own() { + // A stream read from the file is a view into the input: caching it costs no copy, so + // even a large one stays cached (its /Length object too) instead of being re-parsed. + let body = "x".repeat(ObjectReader::STREAM_BYTES + 1); + let stream = format!("<< /Length 4 0 R >>\nstream\n{body}\nendstream"); + let length = body.len().to_string(); + let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", &stream, &length], "/Root 1 0 R"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let mut reader = doc.object_reader(); + let read = reader.get(ObjRef::new(3, 0)); + let Object::Stream(s) = &*read else { panic!("stream") }; + assert!(s.raw.shares(&reader.document.data), "no copy of the file's bytes"); + assert!(reader.document.cache.lock().unwrap().contains_key(&3), "a view costs nothing to keep"); + // Data of its own (decrypted streams, strings) counts: past the budget the cache is + // cleared after the next read, including a failed one. + let owned = Object::Stream(crate::cos::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); + let size = held_bytes(&owned, &reader.document.data, 0); + reader.document.cache.lock().unwrap().insert(3, Ok(Arc::new(owned)), size, Default::default()); + assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); + assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); +} + +#[test] +fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { + let mut source = Document::new_empty(); + let objects: Vec<_> = (0..260) + .map(|i| { + let mut value = vec![b'x'; 48 * 1024]; + value[..4].copy_from_slice(&(i as u32).to_le_bytes()); + Object::Ref(source.add(Object::String(crate::cos::PdfString::literal(value)))) + }) + .collect(); + source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); + let bytes = Arc::new(crate::cos::write_full(&source, &crate::cos::SaveOptions::default()).unwrap()); + let doc = Document::open(bytes).unwrap(); + let root = doc.get(doc.root().unwrap()); + let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap(); + let snapshot = doc.clone(); + let cached = doc.cache.lock().unwrap().len(); + let streams = doc.objstms.lock().unwrap().len(); + let mut reader = doc.object_reader(); + let mut held = None; + for (i, reference) in references.iter().enumerate() { + let Object::Ref(reference) = reference else { panic!("indirect object") }; + let object = reader.get(*reference); + assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); + if i == 0 { + held = Some(object); + } + assert!(reader.document.cache.lock().unwrap().len() < ObjectReader::OBJECTS); + let decoded = reader.document.objstms.lock().unwrap(); + assert!(decoded.len() < ObjectReader::OBJECTS); + let bytes: usize = decoded.values().map(|s| s.data.capacity() + s.index.capacity() * std::mem::size_of::>()).sum(); + assert!(bytes <= ObjectReader::STREAM_BYTES || decoded.len() == 1, "only one oversized stream may remain"); + } + drop(reader); + assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes(), "owned results survive evictions"); + for object_streams in [false, true] { + let saved = crate::cos::write_full(&doc, &crate::cos::SaveOptions { object_streams, ..Default::default() }).unwrap(); + assert!(Document::open(Arc::new(saved)).unwrap().repair_log().is_empty()); + assert_eq!(doc.cache.lock().unwrap().len(), cached, "full saves must not fill the editor cache"); + assert_eq!(doc.objstms.lock().unwrap().len(), streams); + assert!(Arc::ptr_eq(&doc.cache, &snapshot.cache), "do not detach or clear caller snapshots"); + } +} + +#[test] +fn streams_without_length_share_one_endstream_search() { + let n = 2_000; + let mut file = b"%PDF-1.7\n".to_vec(); + for k in 1..=n { + file.extend_from_slice(format!("{k} 0 obj\n<< >>\nstream\nxx\n").as_bytes()); + } + file.extend_from_slice(b"endstream\nendobj\n"); + file.extend_from_slice( + format!("{0} 0 obj\n<< /Type /Catalog /Pages {1} 0 R >>\nendobj\n{1} 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n", n + 1, n + 2) + .as_bytes(), + ); + file.extend_from_slice(format!("trailer\n<< /Root {} 0 R /Size {} >>\n%%EOF\n", n + 1, n + 3).as_bytes()); + let len = file.len(); + crate::cos::parser::ENDSTREAM_SCAN.with(|c| c.set(0)); + let doc = Document::open(Arc::new(file)).unwrap(); + for k in [1, n / 2, n] { + assert!(matches!(doc.try_get(k).as_deref(), Ok(Object::Stream(_))), "object {k}"); + } + let scanned = crate::cos::parser::ENDSTREAM_SCAN.with(|c| c.get()); + assert!(scanned <= 4 * len, "{scanned} bytes searched in a {len}-byte file"); +} + +#[test] +fn editor_cache_counts_deeply_nested_strings() { + let mut nested = Object::String(crate::cos::PdfString::literal(vec![b'x'; 1 << 20])); + for _ in 0..8 { + nested = Object::Array(vec![nested]); + } + let mut source = Document::new_empty(); + let objects: Vec<_> = (0..48).map(|_| Object::Ref(source.add(nested.clone()))).collect(); + source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); + let bytes = Arc::new(crate::cos::write_full(&source, &crate::cos::SaveOptions::default()).unwrap()); + let doc = Document::open(bytes).unwrap(); + let held = held_bytes(&nested, &doc.data, 0); + assert!((1 << 20..(1 << 20) + 4096).contains(&held), "the string and a little tree: {held}"); + let root = doc.get(doc.root().unwrap()); + for reference in root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap() { + let Object::Ref(reference) = reference else { panic!("indirect object") }; + drop(doc.get(*reference)); + let cache = doc.cache.lock().unwrap(); + let owned = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); + let strings = cache.values().filter(|o| o.as_array().is_some()).count(); + assert!(strings << 20 <= Document::EDITOR_CACHE_BYTES + (1 << 20), "{strings} MiB strings cached, {owned} bytes counted"); + } +} + +#[test] +fn editor_cache_stays_bounded() { + let mut source = Document::new_empty(); + let objects: Vec<_> = (0..(Document::EDITOR_CACHE_OBJECTS + 64)) + .map(|i| { + let mut value = vec![b'x'; 1024]; + value[..4].copy_from_slice(&(i as u32).to_le_bytes()); + Object::Ref(source.add(Object::String(crate::cos::PdfString::literal(value)))) + }) + .collect(); + source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); + let bytes = Arc::new(crate::cos::write_full(&source, &crate::cos::SaveOptions::default()).unwrap()); + let doc = Document::open(bytes).unwrap(); + let root = doc.get(doc.root().unwrap()); + let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap().clone(); + let mut held = None; + for (i, reference) in references.iter().enumerate() { + let Object::Ref(reference) = reference else { panic!("indirect object") }; + let object = doc.get(*reference); + assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); + if i == 0 { + held = Some(object); + } + assert!(doc.cache.lock().unwrap().len() <= Document::EDITOR_CACHE_OBJECTS, "editing must not pin the whole object graph"); + // Eviction keeps the newcomer. + assert!(doc.cache.lock().unwrap().contains_key(&reference.num), "eviction must keep the newcomer"); + } + // Returned objects survive eviction, through both handles of the shared cache. + assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); + let snapshot = doc.clone(); + let first = references[0].as_ref().unwrap(); + assert_eq!(&snapshot.get(first).as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); +} + +#[test] +fn editor_cache_byte_bound_fires() { + // 40 × 1 MB strings: far below the count cap, above the 32 MB byte budget, which is + // checked after every insertion. + let mut source = Document::new_empty(); + let objects: Vec<_> = (0..40) + .map(|i| { + let mut value = vec![b'y'; 1024 * 1024]; + value[..4].copy_from_slice(&(i as u32).to_le_bytes()); + Object::Ref(source.add(Object::String(crate::cos::PdfString::literal(value)))) + }) + .collect(); + source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); + let bytes = Arc::new(crate::cos::write_full(&source, &crate::cos::SaveOptions::default()).unwrap()); + let doc = Document::open(bytes).unwrap(); + let root = doc.get(doc.root().unwrap()); + let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap().clone(); + let mut held = None; + for (i, reference) in references.iter().enumerate() { + let Object::Ref(reference) = reference else { panic!("indirect object") }; + let object = doc.get(*reference); + assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); + if i == 0 { + held = Some(object); + } + // Eviction keeps the newcomer. + assert!(doc.cache.lock().unwrap().contains_key(&reference.num), "eviction must keep the newcomer"); + } + assert!(doc.cache.lock().unwrap().len() < references.len(), "the byte budget must evict even below the count cap"); + let last = references[references.len() - 1].as_ref().unwrap(); + assert!(doc.cache.lock().unwrap().contains_key(&last.num), "eviction must keep the newcomer"); + assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); +} + +#[test] +fn a_stream_limited_document_cannot_be_saved() { + let bytes = Arc::new(crate::cos::write_full(&Document::new_empty(), &crate::cos::SaveOptions::default()).unwrap()); + let doc = Document::open_with_stream_limit(bytes.clone(), None, 1 << 20).unwrap(); + for saved in + [crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()), crate::cos::write_incremental(&doc, &crate::cos::SaveOptions::default())] + { + assert!(matches!(saved, Err(CosError::ReadOnlyLimit)), "{saved:?}"); + } + let doc = Document::open(bytes).unwrap(); + assert!(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).is_ok()); +} + +#[test] +fn inspection_stream_limit_applies_before_loading_compressed_objects() { + let mut doc = Document::new_empty(); + let large = doc.add(Object::String(crate::cos::PdfString::literal(vec![b'x'; 4096]))); + doc.update_dict(doc.root().unwrap(), |catalog| catalog.set(b"Extension".to_vec(), Object::Ref(large))).unwrap(); + let bytes = Arc::new(crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap()); + assert!(Document::open(bytes.clone()).is_ok()); + assert!(Document::open_with_stream_limit(bytes, None, 512).is_err()); +} + +/// A /Prev chain that loops back on itself stops at the loop (and says so). +#[test] +fn looping_revision_chains_stop() { + let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + // An update whose /Prev points at itself. + let at = bytes.len(); + bytes.extend_from_slice( + format!("xref\n0 1\n0000000000 65535 f \ntrailer << /Size 3 /Root 1 0 R /Prev {at} >>\nstartxref\n{at}\n%%EOF\n").as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + assert!(doc.repair_log().iter().any(|l| l.contains("loops")), "{:?}", doc.repair_log()); +} + +/// A hybrid-reference file: the classic table leaves object 3 out; the `/XRefStm` stream +/// says it lives in object stream 4. +#[test] +fn hybrid_reference_files_read_the_xref_stream_too() { + let doc = Document::open(Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0))).unwrap(); + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + assert_compact_matches_materialized(&doc); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); +} + +/// A hybrid-reference file whose classic table leaves object 3 out, and whose `/XRefStm` +/// stream puts it at position `member` of object stream 4, which holds `header` (the number +/// and offset pairs) and then `members`. +fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { + object_stream_file_with(header, members, member, false) +} + +/// [`object_stream_file`], with the stream's `/Length` in object 6 when `indirect_length`. +fn object_stream_file_with(header: &str, members: &str, member: u8, indirect_length: bool) -> Vec { + let mut out = b"%PDF-1.5\n".to_vec(); + let size = if indirect_length { 7 } else { 6 }; + let mut offs = vec![0usize; size]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let count = header.split_whitespace().count() / 2; + let inner = format!("{header}{members}"); + let length = if indirect_length { "6 0 R".to_string() } else { inner.len().to_string() }; + let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {length} >>\nstream\n", header.len()).into_bytes(); + stm.extend_from_slice(inner.as_bytes()); + stm.extend_from_slice(b"\nendstream"); + obj(&mut out, 4, &stm); + let data = [2u8, 0, 4, member]; + let mut xs = format!("<< /Type /XRef /Size 6 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); + xs.extend_from_slice(&data); + xs.extend_from_slice(b"\nendstream"); + obj(&mut out, 5, &xs); + if indirect_length { + obj(&mut out, 6, inner.len().to_string().as_bytes()); + } + let table = out.len(); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); + for (n, off) in offs.iter().enumerate().skip(1) { + let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; + out.extend_from_slice(line.as_bytes()); + } + out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + out +} + +/// Many revisions naming one hybrid /XRefStm read it once. +#[test] +fn a_hybrid_stream_named_by_many_revisions_is_read_once() { + let mut out = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); + let stream = find(&out, b"5 0 obj", 0, out.len()).unwrap(); + let mut prev = rfind(&out, b"\nxref\n", 0).unwrap() + 1; + let revisions = 200; + for _ in 0..revisions { + let at = out.len(); + out.extend_from_slice(format!("xref\ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {prev} >>\nstartxref\n{at}\n%%EOF\n").as_bytes()); + prev = at; + } + let doc = Document::open(Arc::new(out)).unwrap(); + SECTION_READS.with(|c| c.set(0)); + let (entries, ..) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); + let reads = SECTION_READS.with(|c| c.get()); + assert!(reads <= revisions + 2, "{reads} sections read for {} revisions", revisions + 1); + assert!(matches!(entries.get(&3), Some(XrefEntry::InStream { stream: 4, .. })), "{:?}", entries.get(&3)); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); + assert_compact_matches_materialized(&doc); +} + +/// An older section that names an /XRefStm a newer section already read still has its own +/// table compared with that stream, so an entry it sets aside is recorded for it. +#[test] +fn a_repeated_hybrid_stream_records_each_sections_conflicts() { + let mut out = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); + let stream = find(&out, b"5 0 obj", 0, out.len()).unwrap(); + let catalog = find(&out, b"1 0 obj", 0, out.len()).unwrap(); + let base = rfind(&out, b"\nxref\n", 0).unwrap() + 1; + // This section's table places object 3 at the catalog; the stream puts it in object stream 4. + let older = out.len(); + out.extend_from_slice( + format!("xref\n3 1\n{catalog:010} 00000 n \ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {base} >>\nstartxref\n{older}\n%%EOF\n") + .as_bytes(), + ); + let newer = out.len(); + out.extend_from_slice(format!("xref\ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {older} >>\nstartxref\n{newer}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + assert_eq!(doc.xref_oddities(), [(older as u64, 3)]); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); +} + +fn text_of(doc: &Document, num: u32, key: &[u8]) -> Option { + doc.get(ObjRef::new(num, 0)).as_dict().and_then(|d| d.get(key).and_then(|v| v.as_string().map(|s| s.to_text()))) +} + +/// An object stream member numbered 2^32 + 3 is not object 3 (#1815): the number used to be +/// cast to `u32`, which dropped the high bits. +#[test] +fn object_stream_numbers_beyond_u32_are_not_truncated() { + let control = Document::open(Arc::new(object_stream_file("3 0 ", "<< /Member (yes) >>", 0))).unwrap(); + assert_eq!(text_of(&control, 3, b"Member").as_deref(), Some("yes")); + let doc = Document::open(Arc::new(object_stream_file("4294967299 0 ", "<< /Member (yes) >>", 0))).unwrap(); + assert!(matches!(*doc.get(ObjRef::new(3, 0)), Object::Null), "{:?}", doc.get(ObjRef::new(3, 0))); + assert!(!doc.xref_entry_is_exact(3)); +} + +/// Member starts are checked rather than cast, so they cannot wrap on any target (#1815). +#[test] +fn object_stream_member_starts_are_checked() { + assert_eq!(objstm_member(3, 4, Some(10), 20), Some((3, 14))); + assert_eq!(objstm_member(3, 9, Some(10), 20), Some((3, 19)), "the last byte"); + assert_eq!(objstm_member(3, 10, Some(10), 20), None, "at the end of the data"); + assert_eq!(objstm_member(3, 1, Some(usize::MAX), 20), None, "/First plus the offset overflows"); + assert_eq!(objstm_member(3, i64::MAX, Some(10), 20), None); + assert_eq!(objstm_member(3, -1, Some(10), 20), None); + assert_eq!(objstm_member(3, 0, None, 20), None, "/First does not fit in usize"); + assert_eq!(objstm_member(i64::from(u32::MAX), 0, Some(0), 1), Some((u32::MAX, 0))); + assert_eq!(objstm_member(i64::from(u32::MAX) + 1, 0, Some(0), 1), None); + assert_eq!(objstm_member(-1, 0, Some(0), 1), None); +} + +/// A member at a negative offset is unusable, not the stream's first member (#1815): the +/// offset used to be clamped to zero, which read the object at `/First`. +#[test] +fn object_stream_negative_offsets_are_refused() { + let doc = Document::open(Arc::new(object_stream_file("7 0 3 -1 ", "<< /First (yes) >>", 1))).unwrap(); + assert!(matches!(*doc.get(ObjRef::new(3, 0)), Object::Null), "{:?}", doc.get(ObjRef::new(3, 0))); + let control = Document::open(Arc::new(object_stream_file("7 0 3 19 ", "<< /First (yes) >> << /Second (yes) >>", 1))).unwrap(); + assert_eq!(text_of(&control, 3, b"Second").as_deref(), Some("yes")); +} + +#[test] +fn revision_ends_split_incremental_updates() { + let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + let first = bytes.len(); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert_eq!(doc.revision_ends(), vec![first]); + let mut edited = doc.clone(); + edited.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(crate::cos::PdfString::literal("en")))).unwrap(); + bytes = crate::cos::write_incremental(&edited, &crate::cos::SaveOptions::default()).unwrap(); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + let ends = doc.revision_ends(); + assert_eq!(ends.len(), 2, "{ends:?}"); + assert!(ends[0] >= first && ends[0] < bytes.len() && ends[1] == bytes.len(), "{ends:?} of {}", bytes.len()); + // The first revision opens on its own, without the update. + let old = Document::open(Arc::new(bytes[..ends[0]].to_vec())).unwrap(); + assert!(old.get(ObjRef::new(1, 0)).as_dict().unwrap().get(b"Lang").is_none()); +} + +/// qpdf's `xref-compressed-in-compressed.pdf` in miniature: object 3 is an object stream +/// holding object 4, and the cross-reference stream says object 3 itself lives in stream 99. +fn compressed_in_compressed_fixture() -> Document { + let mut out = b"%PDF-1.5\n".to_vec(); + let at = |out: &mut Vec, num: u32, body: &str| { + let off = out.len(); + out.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); + off as u16 + }; + let o1 = at(&mut out, 1, "<< /Type /Catalog /Pages 2 0 R /Outlines 4 0 R >>"); + let o2 = at(&mut out, 2, "<< /Type /Pages /Kids [] /Count 0 >>"); + let inner = "<< /Type /Outlines /Count 0 >>"; + at(&mut out, 3, &format!("<< /Type /ObjStm /N 1 /First 4 /Length {} >>\nstream\n4 0 {inner}\nendstream", 4 + inner.len())); + // /W [1 2 1]: object 3 is said to live in stream 99 (which does not exist), 4 in stream 3. + let rows: Vec<[u8; 4]> = + vec![[0, 0, 0, 255], [1, 0, o1 as u8, 0], [1, 0, o2 as u8, 0], [2, 0, 99, 0], [2, 0, 3, 0], [1, (out.len() >> 8) as u8, out.len() as u8, 0]]; + let rows: Vec = rows.concat(); + let xref = out.len(); + out.extend_from_slice( + format!("5 0 obj\n<< /Type /XRef /W [1 2 1] /Index [0 6] /Size 6 /Root 1 0 R /Length {} >>\nstream\n", rows.len()).as_bytes(), + ); + out.extend_from_slice(&rows); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + Document::open(Arc::new(out)).unwrap() +} + +/// An object stream whose own cross-reference entry says it is compressed inside another +/// stream (which the format forbids, and which does not exist here: qpdf's +/// `xref-compressed-in-compressed.pdf`, #2367) could not be loaded, so every object in it +/// read as null and a full save wrote `/Outlines null`. The stream is now found by its own +/// header, as a plain object with a wrong offset is. +#[test] +fn an_object_stream_marked_as_compressed_is_found_by_its_header() { + let doc = compressed_in_compressed_fixture(); + assert_eq!(doc.revisions().len(), 1, "the chain itself reads: {:?}", doc.repair_log()); + let outlines = doc.get(ObjRef::new(4, 0)); + assert_eq!(outlines.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Outlines"[..]), "{outlines:?}"); + let saved = crate::cos::write_full(&doc, &crate::cos::SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let outlines = back.resolve(catalog.as_dict().unwrap().get(b"Outlines").unwrap()); + assert_eq!(outlines.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Outlines"[..]), "the full save keeps it: {outlines:?}"); +} + +/// The header fallback is for file entries only: once the stream is freed (or replaced) in +/// the overlay, its objects must not come back from the file's copy of it. +#[test] +fn a_freed_object_stream_is_not_found_by_its_header() { + let doc = compressed_in_compressed_fixture(); + assert!(doc.get(ObjRef::new(4, 0)).as_dict().is_some()); + let mut edited = doc.clone(); + edited.free(ObjRef::new(3, 0)); + let fresh = edited.object_reader().document; + assert!(matches!(*fresh.get(ObjRef::new(4, 0)), Object::Null), "{:?}", fresh.get(ObjRef::new(4, 0))); + let mut replaced = doc.clone(); + replaced.set(ObjRef::new(3, 0), Object::Int(0)); + let fresh = replaced.object_reader().document; + assert!(matches!(*fresh.get(ObjRef::new(4, 0)), Object::Null), "{:?}", fresh.get(ObjRef::new(4, 0))); +} + +/// The same once the free is saved: the revision's table says the stream is free, and +/// neither reading the file back nor rewriting it in full brings its objects back. +#[test] +fn a_container_freed_in_a_saved_revision_is_not_found_by_its_header() { + let mut edited = compressed_in_compressed_fixture(); + edited.free(ObjRef::new(3, 0)); + let incremental = crate::cos::write_incremental(&edited, &crate::cos::SaveOptions::default()).unwrap(); + let reopened = Document::open(Arc::new(incremental)).unwrap(); + assert_eq!(reopened.revisions().len(), 2); + assert!(matches!(reopened.xref_entry(3), Some(XrefEntry::Free { .. }))); + assert!(matches!(*reopened.get(ObjRef::new(4, 0)), Object::Null), "{:?}", reopened.get(ObjRef::new(4, 0))); + for object_streams in [true, false] { + let opts = crate::cos::SaveOptions { object_streams, ..Default::default() }; + let full = crate::cos::write_full(&reopened, &opts).unwrap(); + let back = Document::open(Arc::new(full)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let outlines = catalog.as_dict().unwrap().get(b"Outlines").map(|o| back.resolve(o)); + assert!(outlines.as_ref().is_none_or(|o| matches!(**o, Object::Null)), "the full save keeps it out: {outlines:?}"); + } +} + +/// A trailer whose /Prev is written as the reference `N 0 R` (tiff2pdf did this; the files +/// are JHOVE's PDF-HUL-157 cases, #2366) used to end the cross-reference chain silently, so +/// every object of the earlier sections read as null: a four-page file showed one page, and +/// a full save wrote the three missing kids as null. The offset is read from the reference. +#[test] +fn a_prev_written_as_a_reference_still_reaches_the_earlier_sections() { + let base = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R >>"], "/Root 1 0 R"); + let prev = String::from_utf8_lossy(&base).rfind("xref\n0 4").unwrap(); + let update = |prev_entry: &str| { + let mut bytes = base.clone(); + let at = bytes.len(); + bytes.extend_from_slice(b"4 0 obj\n<< /Title (Update) >>\nendobj\n"); + let xref = bytes.len(); + bytes.extend_from_slice( + format!("xref\n4 1\n{at:010} 00000 n \ntrailer\n<< /Size 5 /Root 1 0 R /Info 4 0 R /Prev {prev_entry} >>\nstartxref\n{xref}\n%%EOF\n") + .as_bytes(), + ); + Document::open(Arc::new(bytes)).unwrap() + }; + let good = update(&prev.to_string()); + let bad = update(&format!("{prev} 0 R")); + assert!(good.repair_log().is_empty(), "{:?}", good.repair_log()); + assert_eq!(bad.revisions().len(), 2, "{:?}", bad.repair_log()); + assert_eq!(bad.repair_log().len(), 1, "a note about the /Prev: {:?}", bad.repair_log()); + assert_eq!(bad.xref_entry(3), good.xref_entry(3), "the page of the first section is reached"); + assert_eq!(bad.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); + assert_eq!( + bad.get(ObjRef::new(4, 0)).as_dict().and_then(|d| d.get(b"Title").and_then(|t| t.as_string().map(|s| s.to_text()))), + Some("Update".into()) + ); +} + +/// iText's `indirectPrev.pdf` in miniature: the update's trailer gives `/Prev 4 0 R`, and +/// object 4, listed in that update, holds the offset of the first section. Read as offset 4 +/// (inside the header), the chain broke and the file was rebuilt without its revisions. +#[test] +fn a_prev_reference_to_an_object_holding_the_offset_reads_that_offset() { + let mut bytes = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R >>"], "/Root 1 0 R"); + let prev = String::from_utf8_lossy(&bytes).rfind("xref\n0 4").unwrap(); + let mut at = Vec::new(); + for (num, body) in + [(2, "<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>".to_string()), (4, prev.to_string()), (5, "<< /Type /Page /Parent 2 0 R >>".into())] + { + at.push(bytes.len()); + bytes.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); + } + let xref = bytes.len(); + bytes.extend_from_slice( + format!( + "xref\n2 1\n{:010} 00000 n \n4 2\n{:010} 00000 n \n{:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R /Prev 4 0 R >>\nstartxref\n{xref}\n%%EOF\n", + at[0], at[1], at[2] + ) + .as_bytes(), + ); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.revisions().len(), 2, "{:?}", doc.repair_log()); + assert_eq!(doc.repair_log().len(), 1, "a note about the /Prev, and no rebuild: {:?}", doc.repair_log()); + assert!(doc.repair_log()[0].contains(&format!("the offset {prev}")), "{:?}", doc.repair_log()); + assert_eq!(doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); + assert_compact_matches_materialized(&doc); +} + +/// A file whose only trailers are cross-reference stream dictionaries, with its `startxref` +/// out of reach (here a second file appended after it, cut off before its own +/// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used +/// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after +/// editing one Info field dropped every other. +#[test] +fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { + let table = build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Kept) /Creator (Me) >>"], + "/Root 1 0 R /Info 3 0 R", + ); + let packed = crate::cos::write_full(&Document::open(Arc::new(table)).unwrap(), &crate::cos::SaveOptions::default()).unwrap(); + let text = String::from_utf8_lossy(&packed); + assert!(text.contains("/XRef") && !text.contains("trailer"), "a cross-reference stream is the only trailer"); + let mut bytes = packed.clone(); + let appended = format!("%PDF-1.3\n90 0 obj\n<< /Length 5000 >>\nstream\n{}\nendstream\nendobj\n", "x".repeat(5000)); + bytes.extend_from_slice(appended.as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(doc.revisions().is_empty(), "rebuilt: {:?}", doc.repair_log()); + let info = doc.trailer().reference(b"Info").expect("the cross-reference stream's /Info"); + let field = |d: &Document, key: &[u8]| d.get(info).as_dict().unwrap().get(key).unwrap().as_string().unwrap().bytes.clone(); + assert_eq!(field(&doc, b"Title"), b"Kept"); + assert!(doc.trailer().get(b"ID").is_some(), "and its /ID"); + let mut edited = doc.clone(); + edited.update_dict(info, |d| d.set(b"Subject".to_vec(), Object::String(crate::cos::PdfString::literal("new")))).unwrap(); + let saved = crate::cos::write_full(&edited, &crate::cos::SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + let info = back.trailer().reference(b"Info").expect("the full save keeps /Info"); + let field = |key: &[u8]| back.get(info).as_dict().unwrap().get(key).unwrap().as_string().unwrap().bytes.clone(); + assert_eq!((field(b"Title"), field(b"Creator"), field(b"Subject")), (b"Kept".to_vec(), b"Me".to_vec(), b"new".to_vec())); +} + +/// A rebuild prefers a trailer whose /Root is the catalog (as `root_is_catalog` judges it): a +/// later cross-reference stream whose /Root points at the Info dictionary does not replace an +/// earlier good trailer, or a full save would write the Info dictionary as /Root. +#[test] +fn reconstruction_skips_a_trailer_whose_root_is_not_the_catalog() { + let mut bytes = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Kept) >>"], "/Root 1 0 R /Info 3 0 R"); + bytes.extend_from_slice(b"4 0 obj\n<< /Type /XRef /Size 5 /W [1 1 1] /Root 3 0 R /Length 3 >>\nstream\nabc\nendstream\nendobj\n"); + let appended = format!("%PDF-1.3\n90 0 obj\n<< /Length 5000 >>\nstream\n{}\nendstream\nendobj\n", "x".repeat(5000)); + bytes.extend_from_slice(appended.as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(doc.repair_log().iter().any(|l| l.contains("rebuilt")), "{:?}", doc.repair_log()); + assert_eq!(doc.trailer().reference(b"Root").map(|r| r.num), Some(1), "the catalog, not the Info dictionary"); + assert_eq!(doc.trailer().reference(b"Info").map(|r| r.num), Some(3)); +} + +#[test] +fn damaged_xref_is_reconstructed() { + let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); + let n = bytes.len(); + bytes[n - 12..n - 6].copy_from_slice(b"999999"); // point startxref into nowhere + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(doc.revisions().is_empty()); + assert!(!doc.repair_log().is_empty()); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); +} + +#[test] +fn a_decompression_bomb_in_an_xref_stream_is_refused_and_the_file_reconstructed() { + // 32 MiB of rows behind two FlateDecode filters (a few hundred bytes in the file) for + // /W [1 1 1]: at 3 bytes a row that is more than 8,388,607 objects' worth, so it is not + // decoded (in full: the limit applies while inflating) and the objects are found by + // reconstruction instead. + let bomb = crate::filters::encode_flate(&crate::filters::encode_flate(&vec![0u8; 32 << 20])); + let mut bytes = b"%PDF-1.7\n".to_vec(); + let o1 = bytes.len(); + bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); + let o2 = bytes.len(); + bytes.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); + let x = bytes.len(); + bytes.extend_from_slice( + format!("3 0 obj\n<< /Type /XRef /Size 4 /W [1 1 1] /Root 1 0 R /Filter [/FlateDecode /FlateDecode] /Length {} >>\nstream\n", bomb.len()) + .as_bytes(), + ); + bytes.extend_from_slice(&bomb); + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); + assert!(o1 < o2 && o2 < x); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + assert!(doc.repair_log().iter().any(|l| l.contains("limit")), "{:?}", doc.repair_log()); +} + +/// A chain of cross-reference streams whose sections each stay well under the per-section +/// limit but together list more than 8,388,607 rows (#5082). Too many rows for the compact +/// reader, the chain was merged whole; it is now refused and the file rebuilt from object +/// headers. The bulk rows are set aside (in use at offset 0), so the test holds a few bytes a row. +#[test] +fn a_cross_reference_chain_past_the_total_cap_is_refused_and_the_file_reconstructed() { + const BULK: usize = 4_200_000; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for body in + ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>"] + { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); + } + // Objects 4 and 5: /W [1 0 0] rows of type 1 at offset 0, each a set-aside entry. + let bulk = crate::filters::encode_flate(&vec![1u8; BULK]); + let size = 16 + 2 * BULK; + let mut prev = None; + for (num, start) in [(4, 16 + BULK), (5, 16)] { + offsets.push(bytes.len()); + let prev_key = prev.map(|p| format!(" /Prev {p}")).unwrap_or_default(); + bytes.extend_from_slice( + format!( + "{num} 0 obj\n<< /Type /XRef /Size {size} /W [1 0 0] /Index [{start} {BULK}]{prev_key} /Filter /FlateDecode /Length {} >>\nstream\n", + bulk.len() + ) + .as_bytes(), + ); + bytes.extend_from_slice(&bulk); + bytes.extend_from_slice(b"\nendstream\nendobj\n"); + prev = offsets.last().copied(); + } + // Object 6, the newest section: objects 0 to 6 at their offsets. + let x = bytes.len(); + offsets.push(x); + let mut rows = vec![0u8, 0, 0, 0, 0, 255]; + for o in &offsets { + rows.push(1); + rows.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); + rows.push(0); + } + bytes.extend_from_slice( + format!("6 0 obj\n<< /Type /XRef /Size {size} /Root 1 0 R /W [1 4 1] /Index [0 7] /Prev {} /Length {} >>\nstream\n", offsets[4], rows.len()) + .as_bytes(), + ); + bytes.extend_from_slice(&rows); + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); + + let doc = Document::open(Arc::new(bytes)).unwrap(); + let log = doc.repair_log(); + assert!( + log.iter().any(|l| l.contains("cross-reference chain lists more than 8388607 entries") && l.contains("rebuilding from object headers")), + "{log:?}" + ); + assert!(log.iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{log:?}"); + assert!(doc.revisions().is_empty()); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + let page = doc.try_get(3).unwrap(); + assert_eq!(page.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); + // The chain read itself is refused at the second bulk section, which each decode within its own limit. + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized(&mut Vec::new()) else { panic!("the chain must be refused") }; + assert_eq!(offset, offsets[3]); + assert!(detail.contains("more than 8388607 entries"), "{detail}"); +} + +/// Three classic tables listing 2, 1 and 1 rows, newest first: objects 1 and 2, then object 3 +/// (or object 1 again when `overlap`), then object 4. +fn three_table_chain(overlap: bool) -> (Vec, usize) { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, body) in + ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 3 >>", "<< /Value 4 >>"].iter().enumerate() + { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", num + 1).as_bytes()); + } + let older = bytes.len(); + bytes.extend_from_slice(format!("xref\n4 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R >>\n", offsets[3]).as_bytes()); + let middle = bytes.len(); + let middle_num = if overlap { 1 } else { 3 }; + bytes.extend_from_slice( + format!("xref\n{middle_num} 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {older} >>\n", offsets[middle_num - 1]).as_bytes(), + ); + let newer = bytes.len(); + bytes.extend_from_slice( + format!( + "xref\n1 1\n{:010} 00000 n \n2 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {middle} >>\nstartxref\n{newer}\n%%EOF\n", + offsets[0], offsets[1], + ) + .as_bytes(), + ); + (bytes, older) +} + +/// The chain's row limit counts every row merged (#5082): four rows over three sections pass a +/// limit of 4 and are refused at the oldest section by a limit of 3, both when the sections +/// list different objects and when one relists an object a newer section has (a map of three +/// entries, but four rows merged). +#[test] +fn the_chain_row_limit_counts_disjoint_and_relisted_rows() { + for overlap in [false, true] { + let (bytes, older) = three_table_chain(overlap); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, ..) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 4).unwrap(); + assert_eq!(entries.len(), if overlap { 3 } else { 4 }); + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 3) else { + panic!("four rows must be refused by a limit of 3 (overlap: {overlap})") + }; + assert_eq!(offset, older); + assert!(detail.contains("more than 3 entries"), "{detail}"); + } +} + +/// Rows set aside count against the chain's row limit (#5082): a hybrid file whose table puts +/// object 3 at an offset while its /XRefStm puts it in an object stream merges six rows and sets +/// one aside, so a limit of 7 passes and a limit of 6 refuses it. +#[test] +fn the_chain_row_limit_counts_hybrid_rows_set_aside() { + let mut bytes = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); + // Object 3's row in the table (`xref\n0 6\n`, 20-byte rows): free, now in use at offset 9. + let row = find(&bytes, b"xref\n0 6\n", 0, bytes.len()).unwrap() + 9 + 3 * 20; + assert_eq!(&bytes[row..row + 20], b"0000000000 65535 f \n"); + bytes[row..row + 20].copy_from_slice(b"0000000009 00000 n \n"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7).unwrap(); + assert_eq!(entries.len(), 6); + assert_eq!(oddities.iter().map(|(_, num)| *num).collect::>(), vec![3]); + let Err(CosError::Syntax { detail, .. }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 6) else { + panic!("six rows and one set aside must be refused by a limit of 6") + }; + assert!(detail.contains("more than 6 entries"), "{detail}"); +} + +/// A cross-reference stream whose `/Index [0 4 0 4]` lists objects 0 to 3 twice, the same rows +/// both times: eight rows read, four entries (#5082). The chain's row limit counts the rows +/// read, before deduplication, so a limit of 8 passes and a limit of 7 refuses it. +#[test] +fn the_chain_row_limit_counts_rows_an_xref_stream_index_repeats() { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for body in ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"] { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); + } + let x = bytes.len(); + offsets.push(x); + let mut once = vec![0u8, 0, 0, 0, 0, 255]; + for o in &offsets { + once.push(1); + once.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); + once.push(0); + } + let rows = [once.clone(), once].concat(); + bytes.extend_from_slice( + format!("3 0 obj\n<< /Type /XRef /Size 4 /Root 1 0 R /W [1 4 1] /Index [0 4 0 4] /Length {} >>\nstream\n", rows.len()).as_bytes(), + ); + bytes.extend_from_slice(&rows); + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 8).unwrap(); + assert_eq!(entries.len(), 4); + assert!(oddities.is_empty(), "{oddities:?}"); + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7) else { + panic!("eight rows listing four objects must be refused by a limit of 7") + }; + assert_eq!(offset, x); + assert!(detail.contains("more than 7 entries"), "{detail}"); +} + +// Not on the web: it times the load on a thread, and wasm32-unknown-unknown has none. +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { + // From the nightly fuzz job: `6 0 obj << /Length 6 0 R >> stream` with no `endstream` + // after it. Every parse attempt failed, and each attempt resolved the length by loading + // object 6 again, retrying the same ways down to the nesting limit: about 2^32 parses. + // Objects being loaded are now a seen-set, lengths are memoised, and the second parse + // only runs when the header is shifted. + let mut bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n".to_vec(); + bytes.extend_from_slice(b"6 0 obj\n<< /Length 6 0 R >>\nstream\nno end marker follows\n"); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + if let Ok(doc) = Document::open(Arc::new(bytes)) { + let _ = doc.try_get(6); + } + let _ = tx.send(()); + }); + rx.recv_timeout(std::time::Duration::from_secs(20)).expect("loading must not retry exponentially"); +} + +/// A file whose object 3 (its `/Info`) can't be parsed: the rest reads. +fn with_unreadable_info() -> Vec { + build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Author ) >>"], + "/Root 1 0 R /Info 3 0 R", + ) +} + +/// An object that can't be parsed reads as null and is listed once, by number and generation, +/// with why. Clones and scans share the list, an edit takes the object off it (in that document +/// only), and a sound file lists nothing. +#[test] +fn unreadable_objects_are_listed_with_the_reason() { + let doc = Document::open(Arc::new(with_unreadable_info())).unwrap(); + assert!(doc.unreadable_objects().is_empty(), "nothing has read object 3 yet"); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert!(doc.try_get(3).is_err()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "read twice, listed once: {lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); + let scanned = doc.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let unread = Document::open(Arc::new(with_unreadable_info())).unwrap(); + let scanned = unread.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(unread.unreadable_objects(), lines, "a clone's scan reaches the original"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); + assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); + assert_eq!(doc.unreadable_objects(), lines); + let sound = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) >>"], "/Root 1 0 R /Info 3 0 R"); + let sound = Document::open(Arc::new(sound)).unwrap(); + assert_eq!(sound.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 0); + assert!(sound.unreadable_objects().is_empty()); +} + +/// A file rebuilt from its object headers still lists an object the scan could not read. +#[test] +fn a_rebuilt_file_lists_the_objects_its_scan_could_not_read() { + let text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n3 0 obj\n<< /A ) >>\nendobj\n%%EOF\n"; + let doc = Document::open(Arc::new(text.as_bytes().to_vec())).unwrap(); + assert!(doc.repair_log().iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{:?}", doc.repair_log()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); +} + +/// The list keeps at most 100 objects and then says there were more; an object whose +/// cross-reference offset holds no object is listed with that as its reason. +#[test] +fn the_unreadable_object_list_is_bounded() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 120)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 120); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 101, "{:?}", lines.last()); + assert!(lines[0].starts_with("object 3 0 could not be read"), "{}", lines[0]); + assert_eq!(lines[100], "more objects could not be read; only the first 100 are listed"); + + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let text = String::from_utf8_lossy(&base).into_owned(); + let gone = text.replacen("3 0 obj\n(three)", "9 0 obj\n(three)", 1); + let doc = Document::open(Arc::new(gone.into_bytes())).unwrap(); + assert!(doc.try_get(3).is_err()); + assert_eq!( + doc.unreadable_objects(), + ["object 3 0 could not be read: no readable copy was found at its cross-reference offset or elsewhere in the file"] + ); +} + +/// The unreadable list is shared by clones, and each hides what it has edited. A clone that edited +/// every listed object still lists its own failures: hidden entries don't use up its allowance. +#[test] +fn objects_a_clone_edited_do_not_use_up_its_unreadable_allowance() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 110)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + for num in 3..103 { + assert!(doc.try_get(num).is_err()); + } + assert_eq!(doc.unreadable_objects().len(), 100); + let mut edited = doc.clone(); + for num in 3..103 { + edited.set(ObjRef::new(num, 0), Object::Null); + } + for num in 103..113 { + assert!(edited.try_get(num).is_err()); + } + let lines = edited.unreadable_objects(); + assert_eq!(lines.len(), 10, "{lines:?}"); + assert!(lines[0].starts_with("object 103 0 could not be read"), "{}", lines[0]); + let original = doc.unreadable_objects(); + assert_eq!(original.len(), 101, "the original still shows at most 100, then says there were more"); + assert_eq!(original[100], "more objects could not be read; only the first 100 are listed"); +} + +#[test] +fn a_restored_revision_keeps_what_the_saved_file_held_in_object_streams() { + // The earlier revision: an empty document read back from a file whose catalog and pages + // sit in an object stream. + let earlier = Document::open(Arc::new(crate::cos::write_full(&Document::new_empty(), &crate::cos::SaveOptions::default()).unwrap())).unwrap(); + // The saved revision adds an object the catalog links to, so a full save numbers its objects + // anew; an undo then returns to the earlier graph. + let mut later = earlier.clone(); + let extra = later.add(Object::Int(7)); + let root = later.root().unwrap(); + later.update_dict(root, |c| c.set(b"Extra".to_vec(), Object::Ref(extra))).unwrap(); + let saved = crate::cos::write_full(&later, &crate::cos::SaveOptions::default()).unwrap(); + let mut base = Document::open(Arc::new(saved)).unwrap(); + assert!( + base.object_numbers().iter().any(|&n| matches!(base.xref_entry(n), Some(XrefEntry::InStream { .. }))), + "the saved file holds objects in a stream" + ); + base.overlay_changes_from(&earlier).unwrap(); + let file = crate::cos::write_incremental(&base, &crate::cos::SaveOptions::default()).unwrap(); + let reread = Document::open(Arc::new(file)).unwrap(); + let catalog = reread.get(reread.root().unwrap()).as_dict().unwrap().clone(); + assert!(catalog.get(b"Extra").is_none(), "the restored catalog"); + let pages = reread.resolve(catalog.get(b"Pages").unwrap()); + assert_eq!(pages.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Pages"[..]), "the pages survive: {pages:?}"); +} + +/// The cache keeps the sum of what its objects hold as they come and go, so neither reader has +/// to measure the cache again on every read (#5873). +#[test] +fn the_object_cache_keeps_its_total_as_objects_come_and_go() { + let data = Arc::new(Vec::new()); + let one = Arc::new(Object::String(crate::cos::PdfString::literal(vec![b'x'; 100]))); + let two = Arc::new(Object::Array(vec![Object::Int(1); 10])); + let (a, b) = (held_bytes(&one, &data, 0), held_bytes(&two, &data, 0)); + let mut cache = super::state::ObjectCache::default(); + cache.insert(1, Ok(one.clone()), a, Default::default()); + cache.insert(2, Ok(two), b, Default::default()); + assert_eq!(cache.owned(), a + b); + cache.insert(2, Ok(one), a, Default::default()); + assert_eq!(cache.owned(), a + a, "a replaced object no longer counts"); + cache.retain(|n| *n == 1); + assert_eq!((cache.len(), cache.owned()), (1, a)); + cache.clear(); + assert_eq!((cache.len(), cache.owned()), (0, 0)); +} + +/// A dictionary of numbers owns no string or stream bytes, but parsed it is tens of thousands +/// of values: InDesign's per-page `/PieceInfo`. Counting only strings and streams, the editing +/// cache kept a thousand such pages and a 35 MB book held 1.5 GB (#5873). What the parsed tree +/// holds counts now, so the byte budget bounds them. +#[test] +fn the_editor_cache_charges_a_dictionary_for_the_values_it_holds() { + let entries: String = (0..2000).map(|i| format!("/{i}[{i} 141 3 116.22 -375.993 1.0]")).collect(); + let dict = format!("<< /PieceInfo << /InDesign << /Map << {entries} >> >> >> >>"); + let count = 300; + let refs: String = (0..count).map(|i| format!("{} 0 R ", i + 2)).collect(); + let mut bodies = vec![format!("<< /Type /Catalog /Extension [{refs}] >>")]; + bodies.extend((0..count).map(|_| dict.clone())); + let bodies: Vec<&str> = bodies.iter().map(String::as_str).collect(); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + let one = doc.get(crate::cos::ObjRef::new(2, 0)); + let held = held_bytes(&one, &doc.data, 0); + assert!(held > 2000 * std::mem::size_of::(), "the parsed tree counts: {held} bytes"); + for i in 0..count { + drop(doc.get(crate::cos::ObjRef::new(i as u32 + 2, 0))); + } + let cache = doc.cache.lock().unwrap(); + assert!(cache.owned() <= Document::EDITOR_CACHE_BYTES, "{} bytes held", cache.owned()); + assert!(cache.len() < count, "the byte budget evicted some, below the count cap: {} kept", cache.len()); + let sum = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); + assert_eq!(cache.owned(), sum, "the kept total is the cache's own"); +} + +/// The width (`/MediaBox` x1) of each of `pages`, in order. +fn page_widths(pages: &[Dict]) -> Vec> { + pages.iter().map(|p| p.get(b"MediaBox").and_then(Object::as_array).and_then(|b| b.get(2)).and_then(Object::as_int)).collect() +} + +/// The pages of `doc` in page-tree order, as the dictionaries the document holds. +fn page_tree_pages(doc: &Document) -> Vec { + let catalog = doc.get(doc.root().expect("root")); + let pages = doc.resolve(catalog.as_dict().and_then(|c| c.get(b"Pages")).expect("/Pages")); + let kids = pages.as_dict().and_then(|p| p.get(b"Kids")).and_then(Object::as_array).cloned().unwrap_or_default(); + kids.iter().filter_map(|k| doc.resolve(k).as_dict().cloned()).collect() +} + +/// ISO 32000-2 §7.5.6 and §7.5.8.2: an update reaches the revisions before it through `/Prev`. +/// One without it cuts them off, page tree and all. The file is rebuilt from its object headers +/// as a damaged one is, so its page tree is found, the newest definition of each object wins, and +/// the repair log says why; a save rewrites it in full, with every page. +#[test] +fn an_update_without_prev_is_rebuilt_with_its_page_tree_and_a_repair_note() { + let doc = Document::open(Arc::new(crate::cos::testkit::update_without_prev(false))).expect("the unlinked file opens"); + let pages = page_tree_pages(&doc); + assert_eq!(pages.len(), 3); + assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page wins: {pages:?}"); + assert!(doc.reconstructed() && doc.rebuilt_unlinked_update()); + assert!(doc.repair_log().iter().any(|l| l.contains("no /Prev")), "{:?}", doc.repair_log()); + let catalog = doc.get(doc.root().expect("root")); + assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog wins"); + + let saved = crate::cos::write_incremental(&doc, &crate::cos::SaveOptions::default()).expect("saves"); + let reopened = Document::open(Arc::new(saved)).expect("the saved file opens"); + let pages = page_tree_pages(&reopened); + assert_eq!(page_widths(&pages), [Some(200), Some(300), Some(400)], "every page is saved, in order"); + assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page is saved: {pages:?}"); + let catalog = reopened.get(reopened.root().expect("root")); + assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog is saved"); + assert!(reopened.repair_log().is_empty(), "{:?}", reopened.repair_log()); + + // The same file with `/Prev` reads through its chain, as before. + let linked = Document::open(Arc::new(crate::cos::testkit::update_without_prev(true))).expect("the linked file opens"); + assert_eq!(page_tree_pages(&linked).len(), 3); + assert!(!linked.reconstructed() && !linked.rebuilt_unlinked_update()); + assert_eq!(linked.revisions().len(), 2); + assert!(linked.repair_log().is_empty(), "{:?}", linked.repair_log()); +} + +/// A hybrid update (ISO 32000-2 §7.5.8.4) without `/Prev`: its `/XRefStm` stream belongs to the +/// update, so the earlier revision is cut off just the same, and the file is rebuilt. +#[test] +fn a_hybrid_update_without_prev_is_rebuilt_too() { + let doc = Document::open(Arc::new(crate::cos::testkit::hybrid_update_without_prev(false))).expect("the unlinked file opens"); + let pages = page_tree_pages(&doc); + assert_eq!(pages.len(), 3); + assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page wins: {pages:?}"); + assert!(doc.rebuilt_unlinked_update(), "{:?}", doc.repair_log()); + assert!(doc.repair_log().iter().any(|l| l.contains("no /Prev")), "{:?}", doc.repair_log()); + let catalog = doc.get(doc.root().expect("root")); + assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog wins"); + let linked = Document::open(Arc::new(crate::cos::testkit::hybrid_update_without_prev(true))).expect("the linked file opens"); + assert_eq!(page_tree_pages(&linked).len(), 3); + assert!(!linked.rebuilt_unlinked_update() && linked.repair_log().is_empty(), "{:?}", linked.repair_log()); + assert_eq!(linked.revisions().len(), 2); +} + +#[test] +fn find_agrees_with_a_plain_window_search() { + let naive = |hay: &[u8], needle: &[u8], from: usize, to: usize| { + if from >= to || needle.is_empty() || to - from < needle.len() { + return None; + } + hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + }; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let alphabet = b"/XRef t\n<>0"; + for _ in 0..20_000 { + let hay: Vec = (0..next() % 200).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); + let needles: [&[u8]; 5] = [b"/XRef", b"trailer", b"R", b"ff", b"<<"]; + let needle = needles[(next() % 5) as usize]; + let from = (next() % (hay.len() as u64 + 1)) as usize; + let to = from + (next() % (hay.len() as u64 + 1 - from as u64)) as usize; + assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}"); + } +} diff --git a/crates/pdf/src/cos/document/xref.rs b/crates/pdf/src/cos/document/xref.rs new file mode 100644 index 000000000..771df28b8 --- /dev/null +++ b/crates/pdf/src/cos/document/xref.rs @@ -0,0 +1,275 @@ +use super::{Arc, BTreeMap, Lexer}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum XrefEntry { + Free { next_generation: u16 }, + InFile { offset: u64, generation: u16 }, + InStream { stream: u32, index: u32 }, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct XrefRange { + pub(super) start: u32, + pub(super) count: u32, + pub(super) offset: usize, + pub(super) stride: usize, +} + +#[derive(Clone, Debug)] +pub(super) enum XrefSectionData { + Table, + Stream { bytes: Arc>, widths: [usize; 3] }, +} + +#[derive(Clone, Debug)] +pub(super) struct XrefSection { + pub(super) data: XrefSectionData, + pub(super) ranges: Vec, + pub(super) row_count: usize, + pub(super) max_num: u32, + pub(super) has_entries: bool, + /// A table subsection was read one number lower than it says ([`subsection_start`]). + pub(super) renumbered: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct XrefRevisionSection { + pub(super) primary: XrefSection, + pub(super) hybrid: Option, +} + +#[derive(Clone, Debug)] +pub(super) enum XrefIndex { + Compact { revisions: Vec, max_num: u32, has_entries: bool }, + Materialized(Arc>, Arc>), +} + +pub(super) const MAX_COMPACT_XREF_ROWS: usize = 2_000_000; + +pub(super) const MAX_COMPACT_XREF_BYTES: usize = 64 * 1024 * 1024; + +impl XrefSection { + pub(super) fn has_any_entry(&self) -> bool { + match &self.data { + XrefSectionData::Table => self.has_entries, + XrefSectionData::Stream { bytes, widths } => { + let row_width = widths.iter().sum::(); + if row_width == 0 { + return false; + } + for range in &self.ranges { + let available = bytes.len().saturating_sub(range.offset) / row_width; + for i in 0..available.min(range.count as usize) { + let Some(offset) = i.checked_mul(row_width).and_then(|delta| range.offset.checked_add(delta)) else { continue }; + let Some(row) = bytes.get(offset..offset.saturating_add(row_width)) else { continue }; + let read_field = |from: usize, width: usize, default: u64| -> Option { + if width == 0 { + return Some(default); + } + let end = from.checked_add(width)?; + Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte))) + }; + let Some(kind) = read_field(0, widths[0], 1) else { continue }; + if kind == 0 || kind == 2 { + return true; + } + if kind == 1 && read_field(widths[0], widths[1], 0).is_some_and(|offset| offset != 0) { + return true; + } + } + } + false + } + } + } + + pub(super) fn retained_bytes(&self) -> usize { + match &self.data { + XrefSectionData::Table => 0, + XrefSectionData::Stream { bytes, .. } => bytes.capacity(), + } + } + + pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { + let after = self.ranges.partition_point(|range| range.start <= num); + let range = self.ranges.get(after.checked_sub(1)?)?; + let end = u64::from(range.start).checked_add(u64::from(range.count))?; + if u64::from(num) >= end { + return None; + } + let row = usize::try_from(num.checked_sub(range.start)?).ok()?; + let offset = range.offset.checked_add(row.checked_mul(range.stride)?)?; + match &self.data { + XrefSectionData::Table => { + let mut lexer = Lexer::new(file, offset); + lexer.skip_ws(); + let object_offset = lexer.token(); + lexer.skip_ws(); + let generation = lexer.token(); + lexer.skip_ws(); + let kind = lexer.token(); + match kind { + b"n" => { + let object_offset = std::str::from_utf8(object_offset).ok()?.parse::().ok()?; + if object_offset == 0 { + return None; + } + let generation = std::str::from_utf8(generation).ok()?.parse::().unwrap_or(0).min(u16::MAX as u32) as u16; + Some(XrefEntry::InFile { offset: object_offset, generation }) + } + b"f" => { + let generation = std::str::from_utf8(generation).ok()?.parse::().unwrap_or(0).min(u16::MAX as u32) as u16; + Some(XrefEntry::Free { next_generation: generation }) + } + _ => None, + } + } + XrefSectionData::Stream { bytes, widths } => { + let row_width = widths.iter().sum::(); + let end = offset.checked_add(row_width)?; + let row = bytes.get(offset..end)?; + let field = |from: usize, width: usize, default: u64| -> Option { + if width == 0 { + return Some(default); + } + let end = from.checked_add(width)?; + Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte))) + }; + let kind = field(0, widths[0], 1)?; + let a = field(widths[0], widths[1], 0)?; + let b = field(widths[0] + widths[1], widths[2], 0)?; + let entry = match kind { + 0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 }, + 1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 }, + 2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 }, + _ => return None, + }; + (!matches!(entry, XrefEntry::InFile { offset: 0, .. })).then_some(entry) + } + } + } + + pub(super) fn numbers(&self) -> impl Iterator + '_ { + self.ranges.iter().flat_map(|range| { + let available = match &self.data { + XrefSectionData::Table => range.count as usize, + XrefSectionData::Stream { bytes, widths } => { + let row_width = widths.iter().sum::(); + bytes.len().saturating_sub(range.offset).checked_div(row_width).unwrap_or(0) + } + }; + (0..available.min(range.count as usize)).filter_map(move |i| u32::try_from(i).ok().and_then(|i| range.start.checked_add(i))) + }) + } + + pub(super) fn extend_numbers(&self, numbers: &mut Vec) { + numbers.extend(self.numbers()); + } +} + +impl XrefRevisionSection { + pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { + let primary = self.primary.lookup(num, file); + let hybrid = self.hybrid.as_ref().and_then(|section| section.lookup(num, file)); + match primary { + None | Some(XrefEntry::Free { .. }) => hybrid.or(primary), + entry => entry, + } + } + + fn extend_numbers(&self, numbers: &mut Vec) { + self.primary.extend_numbers(numbers); + if let Some(hybrid) = &self.hybrid { + hybrid.extend_numbers(numbers); + } + } +} + +impl XrefIndex { + pub(super) fn empty() -> Self { + Self::Materialized(Arc::new(BTreeMap::new()), Arc::default()) + } + + pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { + match self { + Self::Compact { revisions, .. } => revisions.iter().find_map(|revision| revision.lookup(num, file)), + Self::Materialized(entries, _) => entries.get(&num).copied(), + } + } + + pub(super) fn is_empty(&self) -> bool { + match self { + Self::Compact { has_entries, .. } => !has_entries, + Self::Materialized(entries, _) => entries.is_empty(), + } + } + + /// Drop an entry for object number 4294967295, which no incremental `/Size` can cover. + /// Only a materialized index can hold one: the compact reader requires `max_num < /Size`. + pub(super) fn remove_exhausted(&mut self) -> bool { + match self { + Self::Compact { .. } => false, + Self::Materialized(entries, _) => Arc::make_mut(entries).remove(&u32::MAX).is_some(), + } + } + + /// Every entry, newest section first, as the materialized reader keeps them. + pub(super) fn to_map(&self, file: &[u8]) -> BTreeMap { + match self { + Self::Materialized(entries, _) => entries.as_ref().clone(), + Self::Compact { revisions, .. } => { + let mut numbers = Vec::new(); + for revision in revisions { + revision.extend_numbers(&mut numbers); + } + numbers.into_iter().filter_map(|num| self.lookup(num, file).map(|entry| (num, entry))).collect() + } + } + } + + pub(super) fn max_num(&self) -> u32 { + match self { + Self::Compact { max_num, .. } => *max_num, + Self::Materialized(entries, _) => entries.keys().next_back().copied().unwrap_or(0), + } + } + + pub(super) fn object_numbers(&self, file: &[u8]) -> Vec { + match self { + Self::Materialized(entries, _) => { + entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect() + } + Self::Compact { revisions, .. } => { + if let [revision] = revisions.as_slice() + && revision.hybrid.is_none() + { + let mut numbers = Vec::with_capacity(revision.primary.row_count); + revision.primary.extend_numbers(&mut numbers); + numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. }))); + return numbers; + } + let mut numbers = Vec::new(); + for revision in revisions { + revision.extend_numbers(&mut numbers); + } + numbers.sort_unstable(); + numbers.dedup(); + numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. }))); + numbers + } + } + } +} + +/// One cross-reference section of the file (one per save; the first is the original). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Revision { + /// Offset of the xref section (`startxref` value). + pub xref_offset: u64, + /// `true` if the section is a cross-reference stream. + pub is_stream: bool, + /// `true` if `xref_offset` counts from the `%PDF-` header rather than from the start of the + /// file. The two differ only in a file with bytes before its header, and an update appended + /// to such a file must count the same way as the section it points back to. + pub header_relative: bool, +} diff --git a/crates/pdf/src/cos/object.rs b/crates/pdf/src/cos/object.rs new file mode 100644 index 000000000..05b64eee2 --- /dev/null +++ b/crates/pdf/src/cos/object.rs @@ -0,0 +1,833 @@ +//! The PDF object model (ISO 32000-2 §7.3). +//! +//! Dictionaries keep their key order so rewritten objects stay recognisable in diffs and +//! byte-stable when re-serialized. Streams keep their *encoded* bytes; decoding is on demand. + +use crate::filters::{Filter, Params}; + +use crate::cos::{Bytes, CosError}; + +/// An indirect reference: object number and generation. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct ObjRef { + pub num: u32, + pub generation: u16, +} + +impl ObjRef { + pub const fn new(num: u32, generation: u16) -> Self { + Self { num, generation } + } +} + +impl std::fmt::Display for ObjRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} {} R", self.num, self.generation) + } +} + +/// A string object. `hex` records how it was written, so it is re-emitted the same way. +#[derive(Clone, Debug, PartialEq)] +pub struct PdfString { + pub bytes: Vec, + pub hex: bool, +} + +impl PdfString { + pub fn literal(bytes: impl Into>) -> Self { + Self { bytes: bytes.into(), hex: false } + } + + /// A text string (ISO 32000-2 §7.9.2.2): PDFDocEncoding when possible, else UTF-16BE with BOM. + pub fn text(s: &str) -> Self { + if s.chars().all(|c| (' '..='~').contains(&c) || c == '\n' || c == '\r' || c == '\t') { + return Self::literal(s.as_bytes().to_vec()); + } + let mut bytes = vec![0xFE, 0xFF]; + for u in s.encode_utf16() { + bytes.extend_from_slice(&u.to_be_bytes()); + } + Self { bytes, hex: false } + } + + /// Decode a text string: UTF-16BE (BOM), UTF-8 (BOM, PDF 2.0) or PDFDocEncoding. + /// + /// Lenient beyond the spec, as readers are: invalid sequences after a BOM decode lossily + /// instead of dropping the string, a BOM-less string whose bytes are valid UTF-8 with + /// non-ASCII bytes (written by some producers) is read as UTF-8 rather than as + /// PDFDocEncoding mojibake, and stray leading U+FEFF (doubled BOMs) are removed. + pub fn to_text(&self) -> String { + self.text_chars().collect() + } + + /// The characters [`PdfString::to_text`] decodes, one at a time, without building a string. + pub fn text_chars(&self) -> TextChars<'_> { + let decode = match self.bytes.as_slice() { + [0xFE, 0xFF, rest @ ..] => { + let units: fn(&[u8; 2]) -> u16 = |c| u16::from_be_bytes(*c); + Decode::Utf16(char::decode_utf16(rest.as_chunks::<2>().0.iter().map(units))) + } + [0xEF, 0xBB, 0xBF, rest @ ..] => Decode::Utf8Lossy { chunks: rest.utf8_chunks(), valid: "".chars(), replace: false }, + b => match std::str::from_utf8(b) { + // Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch. + Ok(utf8) if !b.is_ascii() => Decode::Utf8(utf8.chars()), + _ => Decode::PdfDoc(b.iter()), + }, + }; + TextChars { decode, leading: true } + } + + /// Whether the string decodes to `s` ([`PdfString::to_text`]), compared without allocating. + pub fn text_eq(&self, s: &str) -> bool { + self.text_chars().eq(s.chars()) + } +} + +/// The characters of a text string: [`PdfString::text_chars`]. +pub struct TextChars<'a> { + decode: Decode<'a>, + /// Still at the start, where stray U+FEFF (doubled BOMs) are dropped. + leading: bool, +} + +/// The UTF-16BE code units after a BOM (an odd last byte is left out). +type Utf16Units<'a> = std::iter::Map, fn(&[u8; 2]) -> u16>; + +enum Decode<'a> { + Utf16(std::char::DecodeUtf16>), + /// After a UTF-8 BOM: each invalid sequence is one U+FFFD, as `String::from_utf8_lossy` has it. + Utf8Lossy { + chunks: std::str::Utf8Chunks<'a>, + valid: std::str::Chars<'a>, + replace: bool, + }, + Utf8(std::str::Chars<'a>), + PdfDoc(std::slice::Iter<'a, u8>), +} + +impl Iterator for Decode<'_> { + type Item = char; + + fn next(&mut self) -> Option { + match self { + Decode::Utf16(units) => units.next().map(|u| u.unwrap_or(char::REPLACEMENT_CHARACTER)), + Decode::Utf8Lossy { chunks, valid, replace } => loop { + if let Some(c) = valid.next() { + return Some(c); + } + if *replace { + *replace = false; + return Some(char::REPLACEMENT_CHARACTER); + } + let chunk = chunks.next()?; + *valid = chunk.valid().chars(); + *replace = !chunk.invalid().is_empty(); + }, + Decode::Utf8(chars) => chars.next(), + Decode::PdfDoc(bytes) => bytes.next().map(|&c| crate::crypt::pdfdoc_char(c)), + } + } +} + +impl Iterator for TextChars<'_> { + type Item = char; + + fn next(&mut self) -> Option { + loop { + let c = self.decode.next()?; + if self.leading && c == '\u{FEFF}' { + continue; + } + self.leading = false; + return Some(c); + } + } +} + +/// A name object (without the leading `/`, with `#xx` escapes already decoded). +pub type Name = Vec; + +#[cfg(test)] +thread_local! { + /// Key comparisons and index lookups made by [`Dict`] on this thread. + static KEY_COMPARES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Entries from which a [`Dict`] keeps a key index. Smaller dictionaries (nearly all of them) +/// search their entries; a parsed file controls the key count, and a linear search per +/// inserted key would make a dictionary of many distinct keys quadratic to build. +const INDEX_FROM: usize = 32; + +/// An ordered dictionary. +#[derive(Clone, Default)] +pub struct Dict { + entries: Vec<(Name, Object)>, + /// Key to position in `entries`, kept once the dictionary reaches [`INDEX_FROM`] entries. + /// Boxed so that every `Dict`, and so every `Object`, grows by one pointer, not by a map. + #[allow(clippy::box_collection)] + index: Option>>, +} + +impl PartialEq for Dict { + fn eq(&self, other: &Self) -> bool { + self.entries == other.entries + } +} + +impl std::fmt::Debug for Dict { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Dict").field("entries", &self.entries).finish() + } +} + +impl Dict { + pub fn new() -> Self { + Self::default() + } + + fn position(&self, key: &[u8]) -> Option { + if let Some(index) = &self.index { + #[cfg(test)] + KEY_COMPARES.with(|c| c.set(c.get() + 1)); + return index.get(key).copied(); + } + self.entries.iter().position(|(k, _)| { + #[cfg(test)] + KEY_COMPARES.with(|c| c.set(c.get() + 1)); + k == key + }) + } + + pub fn get(&self, key: &[u8]) -> Option<&Object> { + self.position(key).and_then(|i| self.entries.get(i)).map(|(_, v)| v) + } + + pub fn get_mut(&mut self, key: &[u8]) -> Option<&mut Object> { + let i = self.position(key)?; + self.entries.get_mut(i).map(|(_, v)| v) + } + + /// Insert or replace, keeping the position of an existing key. + pub fn set(&mut self, key: impl Into, value: impl Into) { + let key = key.into(); + let value = value.into(); + match self.position(&key).and_then(|i| self.entries.get_mut(i)) { + Some((_, v)) => *v = value, + None => { + let at = self.entries.len(); + if let Some(index) = &mut self.index { + index.insert(key.clone(), at); + } else if at + 1 >= INDEX_FROM { + let index = self.entries.iter().enumerate().map(|(i, (k, _))| (k.clone(), i)).chain([(key.clone(), at)]).collect(); + self.index = Some(Box::new(index)); + } + self.entries.push((key, value)); + } + } + } + + pub fn remove(&mut self, key: &[u8]) -> Option { + let i = self.position(key)?; + if let Some(index) = &mut self.index { + index.remove(key); + index.values_mut().filter(|at| **at > i).for_each(|at| *at -= 1); + } + Some(self.entries.remove(i).1) + } + + pub fn contains(&self, key: &[u8]) -> bool { + self.get(key).is_some() + } + + pub fn iter(&self) -> impl Iterator { + self.entries.iter().map(|(k, v)| (k, v)) + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub fn name(&self, key: &[u8]) -> Option<&[u8]> { + self.get(key).and_then(Object::as_name) + } + + pub fn int(&self, key: &[u8]) -> Option { + self.get(key).and_then(Object::as_int) + } + + pub fn reference(&self, key: &[u8]) -> Option { + self.get(key).and_then(Object::as_ref) + } +} + +impl FromIterator<(Name, Object)> for Dict { + fn from_iter>(iter: T) -> Self { + let mut d = Dict::new(); + for (k, v) in iter { + d.set(k, v); + } + d + } +} + +/// A stream: dictionary plus its data *as stored* (still filtered). +#[derive(Clone, Debug, PartialEq)] +pub struct Stream { + pub dict: Dict, + pub raw: Bytes, +} + +/// Upper bound for a single decoded stream (decompression-bomb defence). +pub const MAX_DECODED: usize = 1 << 30; + +/// The most an image's `/JBIG2Globals` stream may decode to: the symbol and pattern dictionaries an +/// encoder shares between pages, themselves coded, so far less than any stream may decode to +/// ([`MAX_DECODED`]). They are decoded before the image's JBIG2 page is read, so the page's size +/// can't bound them: they have a limit of their own. +pub const MAX_JBIG2_GLOBALS: usize = 32 << 20; + +impl Stream { + /// A new stream from already-encoded bytes (`/Length` is set at write time). + pub fn from_raw(dict: Dict, raw: Vec) -> Self { + Self { dict, raw: raw.into() } + } + + /// A new stream holding `data` compressed with Flate. + pub fn flate(mut dict: Dict, data: &[u8]) -> Self { + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + dict.remove(b"DecodeParms"); + Self { dict, raw: crate::filters::encode_flate(data).into() } + } + + /// The filter chain declared in the dictionary. + pub fn filters(&self) -> Vec<(Filter, Params)> { + let names: Vec<&[u8]> = match self.dict.get(b"Filter") { + Some(Object::Name(n)) => vec![n], + Some(Object::Array(a)) => a.iter().filter_map(Object::as_name).collect(), + _ => Vec::new(), + }; + let parms: Vec> = match self.dict.get(b"DecodeParms") { + Some(Object::Dict(d)) => vec![Some(d)], + Some(Object::Array(a)) => a.iter().map(Object::as_dict).collect(), + _ => Vec::new(), + }; + names + .into_iter() + .enumerate() + // `/Crypt` is applied by the security handler when the object is loaded (§7.4.10). + .filter(|(_, n)| *n != b"Crypt") + .map(|(i, n)| { + let p = parms.get(i).copied().flatten(); + let get = |k: &[u8], d: i64| p.and_then(|p| p.int(k)).unwrap_or(d); + let params = Params { + predictor: get(b"Predictor", 1), + colors: get(b"Colors", 1), + bits_per_component: get(b"BitsPerComponent", 8), + columns: get(b"Columns", 1), + early_change: get(b"EarlyChange", 1), + }; + (Filter::from_name(n), params) + }) + .collect() + } + + /// [`Stream::filters`], or an error when `/Filter` is present but is not a name or an + /// array of names (an indirect reference, say): the data is not unfiltered, so it must not + /// read as decoded. + pub(crate) fn readable_filters(&self) -> Result, crate::filters::FilterError> { + let readable = match self.dict.get(b"Filter") { + None | Some(Object::Null | Object::Name(_)) => true, + Some(Object::Array(a)) => a.iter().all(|f| f.as_name().is_some()), + Some(_) => false, + }; + if !readable { + return Err(crate::filters::FilterError::Unsupported("/Filter is not a name or an array of names".into())); + } + Ok(self.filters()) + } + + /// Decoded data, tolerating truncated or corrupt encodings the way viewers do (the data + /// decoded before the damage is returned). Decoding past + /// [`crate::cos::budget::MAX_PAGE_DECODED`] (256 MiB), the most the renderer reads from one + /// stream, is an error. Streams with image codecs fail with + /// `CosError::Filter` (keep them encoded). Use `decoded_strict` to detect damage. + pub fn decoded(&self) -> Result, CosError> { + self.decoded_within(crate::cos::budget::MAX_PAGE_DECODED) + } + + /// Like [`Stream::decoded`], for a stream whose decoded size has a tighter bound than + /// [`MAX_DECODED`]: decoding past `max` bytes is an error. + pub fn decoded_within(&self, max: usize) -> Result, CosError> { + let decode = + |chain: &[(Filter, Params)], data: &[u8], max| crate::filters::decode_tolerant(chain, data, max).map(|(v, _)| (v, 0)).map_err(|e| (e, 0)); + self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) + } + + /// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept + /// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter + /// stages together, and the result gives whether the data stopped at damage (it is then what + /// decoded before it) and the bytes the stages decoded (the work done, on success and on + /// failure). + pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, bool, usize), (crate::filters::FilterError, usize)> { + let mut damaged = false; + let decode = |chain: &[(Filter, Params)], data: &[u8], max| { + crate::filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| { + damaged = partial; + (v, spent) + }) + }; + self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) + } + + /// [`Stream::decoded_within_counted`] with every filter stage also held to `each`, as + /// [`Stream::decoded_within`] holds it, while `max` bounds the stages together + /// ([`crate::filters::decode_tolerant_counted_each`]). Without filters the data is held to + /// both. + pub(crate) fn decoded_within_each_counted( + &self, + each: usize, + max: usize, + ) -> Result<(Vec, bool, usize), (crate::filters::FilterError, usize)> { + let mut damaged = false; + let decode = |chain: &[(Filter, Params)], data: &[u8], max| { + crate::filters::decode_tolerant_counted_each(chain, data, each.min(MAX_DECODED), max).map(|(v, partial, spent)| { + damaged = partial; + (v, spent) + }) + }; + let max = if self.readable_filters().is_ok_and(|chain| chain.is_empty()) { max.min(each) } else { max }; + self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) + } + + /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a + /// document's images ([`crate::filters::decode_tolerant_within`]). + pub(crate) fn decoded_spending(&self, max: usize, work: &mut u64) -> Result, CosError> { + let filter = |e: crate::filters::FilterError| CosError::Filter(e.to_string()); + let chain = self.readable_filters().map_err(filter)?; + crate::filters::decode_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), work).map_err(filter) + } + + /// [`Stream::decoded_spending`] for the samples of an image `rows` high (its `/Height`): + /// [`crate::filters::decode_image_tolerant_within`]. + pub(crate) fn decoded_image_rows_spending(&self, max: usize, rows: u32, work: &mut u64) -> Result, CosError> { + let filter = |e: crate::filters::FilterError| CosError::Filter(e.to_string()); + let chain = self.readable_filters().map_err(filter)?; + crate::filters::decode_image_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), rows, work).map_err(filter) + } + + /// The stream decoded through its filters by `decode`, or copied when it has none, within + /// `max` and [`MAX_DECODED`]. + fn decode_chain( + &self, + max: usize, + decode: impl FnOnce(&[(Filter, Params)], &[u8], usize) -> Result<(Vec, usize), (crate::filters::FilterError, usize)>, + ) -> Result<(Vec, usize), (crate::filters::FilterError, usize)> { + let max = max.min(MAX_DECODED); + let chain = self.readable_filters().map_err(|e| (e, 0))?; + if chain.is_empty() { + return self + .raw_within_typed(max) + .map(|v| { + let n = v.len(); + (v, n) + }) + .map_err(|e| (e, 0)); + } + decode(&chain, &self.raw, max) + } + + /// Decoded data, bounded by [`MAX_DECODED`]; any corruption is an error. + pub fn decoded_strict(&self) -> Result, CosError> { + self.decoded_strict_within(MAX_DECODED) + } + + /// Like [`Stream::decoded_strict`], bounded by `max` (and [`MAX_DECODED`]): for data that + /// will be copied elsewhere, where a damaged stream must not pass for a shorter one. + pub fn decoded_strict_within(&self, max: usize) -> Result, CosError> { + let decode = |chain: &[(Filter, Params)], data: &[u8], max| crate::filters::decode(chain, data, max).map(|v| (v, 0)).map_err(|e| (e, 0)); + self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) + } + + /// [`Stream::decoded_strict`] bounded by `max` and counted like + /// [`Stream::decoded_within_counted`]. + pub(crate) fn decoded_strict_within_counted(&self, max: usize) -> Result<(Vec, usize), (crate::filters::FilterError, usize)> { + self.decode_chain(max, crate::filters::decode_counted) + } + + /// Data after the first filter alone, bounded by `max`: the rest of the chain stays encoded. + /// Any corruption in that first filter is an error. + pub fn decoded_first_filter_strict(&self, max: usize) -> Result, CosError> { + let mut chain = self.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?; + chain.truncate(1); + if chain.is_empty() { + return self.raw_within(max); + } + crate::filters::decode(&chain, &self.raw, max.min(MAX_DECODED)).map_err(|e| CosError::Filter(e.to_string())) + } + + fn raw_within(&self, max: usize) -> Result, CosError> { + self.raw_within_typed(max).map_err(|e| CosError::Filter(e.to_string())) + } + + fn raw_within_typed(&self, max: usize) -> Result, crate::filters::FilterError> { + let max = max.min(MAX_DECODED); + // An empty decoder chain still produces an owned output buffer: check its limit + // before cloning, including streams whose /Crypt filter was applied on load. + if self.raw.len() > max { + return Err(crate::filters::FilterError::LimitExceeded(max)); + } + Ok(self.raw.to_vec()) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Object { + Null, + Bool(bool), + Int(i64), + Real(f64), + String(PdfString), + Name(Name), + Array(Vec), + Dict(Dict), + Stream(Stream), + Ref(ObjRef), +} + +impl Object { + pub fn name(n: &str) -> Self { + Object::Name(n.as_bytes().to_vec()) + } + + pub fn as_int(&self) -> Option { + match self { + Object::Int(i) => Some(*i), + Object::Real(r) if r.fract() == 0.0 && r.is_finite() => Some(*r as i64), + _ => None, + } + } + + pub fn as_f64(&self) -> Option { + match self { + Object::Int(i) => Some(*i as f64), + Object::Real(r) => Some(*r), + _ => None, + } + } + + pub fn as_name(&self) -> Option<&[u8]> { + match self { + Object::Name(n) => Some(n), + _ => None, + } + } + + pub fn as_ref(&self) -> Option { + match self { + Object::Ref(r) => Some(*r), + _ => None, + } + } + + pub fn as_dict(&self) -> Option<&Dict> { + match self { + Object::Dict(d) => Some(d), + Object::Stream(s) => Some(&s.dict), + _ => None, + } + } + + pub fn as_dict_mut(&mut self) -> Option<&mut Dict> { + match self { + Object::Dict(d) => Some(d), + Object::Stream(s) => Some(&mut s.dict), + _ => None, + } + } + + pub fn as_array(&self) -> Option<&Vec> { + match self { + Object::Array(a) => Some(a), + _ => None, + } + } + + pub fn as_string(&self) -> Option<&PdfString> { + match self { + Object::String(s) => Some(s), + _ => None, + } + } +} + +impl From for Object { + fn from(v: i64) -> Self { + Object::Int(v) + } +} +impl From for Object { + fn from(v: i32) -> Self { + Object::Int(v as i64) + } +} +impl From for Object { + fn from(v: f64) -> Self { + Object::Real(v) + } +} +impl From for Object { + fn from(v: bool) -> Self { + Object::Bool(v) + } +} +impl From for Object { + fn from(v: ObjRef) -> Self { + Object::Ref(v) + } +} +impl From for Object { + fn from(v: Dict) -> Self { + Object::Dict(v) + } +} +impl From> for Object { + fn from(v: Vec) -> Self { + Object::Array(v) + } +} +impl From for Object { + fn from(v: PdfString) -> Self { + Object::String(v) + } +} +impl From for Object { + fn from(v: Stream) -> Self { + Object::Stream(v) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_filter_that_is_not_a_direct_name_is_not_decoded_as_absent() { + let stream = |filter: Object| { + let mut d = Dict::new(); + d.set(b"Filter".to_vec(), filter); + Stream::from_raw(d, b"48 69>".to_vec()) + }; + let hex = Object::Name(b"ASCIIHexDecode".to_vec()); + assert_eq!(stream(hex.clone()).decoded_strict().ok(), Some(b"Hi".to_vec())); + assert_eq!(stream(Object::Array(vec![hex.clone()])).decoded().ok(), Some(b"Hi".to_vec())); + assert_eq!(stream(Object::Null).decoded_strict().ok(), Some(b"48 69>".to_vec())); + for filter in [ + Object::Ref(ObjRef::new(5, 0)), + Object::Array(vec![Object::Ref(ObjRef::new(5, 0))]), + Object::Int(3), + Object::Array(vec![hex, Object::Int(1)]), + ] { + let s = stream(filter.clone()); + assert!(s.decoded_strict().is_err(), "{filter:?}"); + assert!(s.decoded().is_err(), "{filter:?}"); + assert!(s.decoded_first_filter_strict(1 << 20).is_err(), "{filter:?}"); + } + } + + #[test] + fn parsing_many_distinct_keys_is_not_quadratic() { + let n = 20_000; + let body: String = (0..n).map(|i| format!("/K{i} {i} ")).collect(); + let text = format!("<< {body}/K7 true >>"); + KEY_COMPARES.with(|c| c.set(0)); + let Ok(Object::Dict(d)) = crate::cos::parser::Lexer::new(text.as_bytes(), 0).object() else { panic!("not a dict") }; + let compares = KEY_COMPARES.with(|c| c.get()); + assert!(compares < 8 * n, "{compares} key comparisons for {n} keys"); + // Order and replace-in-place semantics stay those of a plain ordered dictionary. + assert_eq!(d.len(), n); + assert_eq!(d.iter().next().map(|(k, _)| k.as_slice()), Some(&b"K0"[..])); + assert_eq!(d.get(b"K7"), Some(&Object::Bool(true))); + assert_eq!(d.iter().nth(7).map(|(_, v)| v), Some(&Object::Bool(true))); + let mut d = d; + assert_eq!(d.remove(b"K3"), Some(Object::Int(3))); + assert_eq!(d.get(b"K4"), Some(&Object::Int(4))); + assert_eq!(d.iter().nth(3).map(|(k, _)| k.as_slice()), Some(&b"K4"[..])); + d.set(b"K3".to_vec(), Object::Null); + assert_eq!(d.iter().last().map(|(k, _)| k.as_slice()), Some(&b"K3"[..])); + assert!(d.get(b"K20000").is_none()); + } + + #[test] + fn text_strings_round_trip() { + for s in ["Plain ASCII", "Café — naïve", "日本語", "emoji 📄"] { + assert_eq!(PdfString::text(s).to_text(), s); + } + assert!(!PdfString::text("abc").bytes.starts_with(&[0xFE, 0xFF])); + } + + #[test] + fn text_string_encodings() { + // UTF-16BE with BOM. + let mut utf16 = vec![0xFE, 0xFF]; + utf16.extend("第一章 概述".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(PdfString::literal(utf16).to_text(), "第一章 概述"); + // UTF-8 with BOM (PDF 2.0): the BOM is not part of the text. + let mut utf8 = vec![0xEF, 0xBB, 0xBF]; + utf8.extend_from_slice("第一章".as_bytes()); + assert_eq!(PdfString::literal(utf8).to_text(), "第一章"); + // Invalid UTF-8 after the BOM decodes lossily instead of vanishing. + assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, b'A', 0xFF, b'B']).to_text(), "A\u{FFFD}B"); + // BOM-less raw UTF-8 from producers that skip the BOM. + assert_eq!(PdfString::literal("目录".as_bytes()).to_text(), "目录"); + // PDFDocEncoding Latin-1 (not valid UTF-8) stays PDFDocEncoding. + assert_eq!(PdfString::literal(vec![b'C', b'a', b'f', 0xE9]).to_text(), "Café"); + // Doubled BOMs leave no U+FEFF behind. + assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A']).to_text(), "A"); + assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF, b'A']).to_text(), "A"); + // Odd trailing byte and empty strings never panic. + assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0x00, b'A', 0x00]).to_text(), "A"); + assert_eq!(PdfString::literal(vec![0xFE, 0xFF]).to_text(), ""); + assert_eq!(PdfString::literal(Vec::new()).to_text(), ""); + } + + #[test] + fn text_eq_compares_what_to_text_decodes() { + let utf16 = |s: &str| [0xFE, 0xFF].into_iter().chain(s.encode_utf16().flat_map(u16::to_be_bytes)).collect::>(); + // Each invalid UTF-8 sequence is one U+FFFD, as String::from_utf8_lossy has it. + let lossy = [0xEF, 0xBB, 0xBF, 0xC3, 0x28, b'x', 0xF0, 0x9F, 0x92, 0xFF]; + assert_eq!(PdfString::literal(lossy.to_vec()).to_text(), String::from_utf8_lossy(&lossy[3..])); + let cases: Vec> = vec![ + Vec::new(), + b"Plain".to_vec(), + vec![b'C', b'a', b'f', 0xE9, 0x18], + "目录".as_bytes().to_vec(), + utf16("第一章"), + vec![0xFE, 0xFF, 0xD8, 0x00, 0x00, b'A'], + vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A', 0xFE, 0xFF], + lossy.to_vec(), + ]; + for bytes in cases { + let s = PdfString::literal(bytes); + let text = s.to_text(); + assert!(s.text_eq(&text), "{text:?}"); + assert!(!s.text_eq(&format!("{text}x")), "{text:?}"); + assert!(text.is_empty() || !s.text_eq(&text[..text.len() - text.chars().last().map_or(0, char::len_utf8)]), "{text:?}"); + } + // A U+FEFF after the first character is text and stays. + assert!(PdfString::literal(utf16("A\u{FEFF}")).text_eq("A\u{FEFF}")); + } + + #[test] + fn pdfdoc_encoding_high_range() { + assert_eq!(PdfString::literal(vec![0x84, 0x92, 0xA0, b'A']).to_text(), "—™€A"); + } + + #[test] + fn dict_preserves_order_and_replaces_in_place() { + let mut d = Dict::new(); + d.set(b"B".to_vec(), 1); + d.set(b"A".to_vec(), 2); + d.set(b"B".to_vec(), 3); + let keys: Vec<_> = d.iter().map(|(k, _)| k.clone()).collect(); + assert_eq!(keys, vec![b"B".to_vec(), b"A".to_vec()]); + assert_eq!(d.int(b"B"), Some(3)); + } + + #[test] + fn raw_streams_refuse_decoding_past_a_caller_limit() { + let stream = Stream::from_raw(Dict::new(), b"abc".to_vec()); + for max in [0, 2] { + assert!(matches!(stream.decoded_within(max), Err(CosError::Filter(_)))); + } + } + + #[test] + fn strict_decoding_is_bounded_by_the_callers_remaining_allowance() { + let stream = Stream::flate(Dict::new(), &vec![0u8; 1 << 20]); + assert!(matches!(stream.decoded_strict_within(1000), Err(CosError::Filter(_)))); + assert_eq!(stream.decoded_strict_within(1 << 20).map(|v| v.len()).ok(), Some(1 << 20)); + } + + #[test] + fn empty_filter_arrays_refuse_decoding_past_a_caller_limit() { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(Vec::new())); + let stream = Stream::from_raw(dict, b"abc".to_vec()); + assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); + } + + #[test] + fn crypt_only_streams_refuse_decoding_past_a_caller_limit() { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("Crypt")); + let mut parms = Dict::new(); + parms.set(b"Name".to_vec(), Object::name("Identity")); + dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + let stream = Stream::from_raw(dict, b"abc".to_vec()); + assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); + } + + #[test] + fn identity_streams_preserve_bytes_within_a_caller_limit() { + let mut array = Dict::new(); + array.set(b"Filter".to_vec(), Object::Array(Vec::new())); + let mut crypt = Dict::new(); + crypt.set(b"Filter".to_vec(), Object::name("Crypt")); + for dict in [Dict::new(), array, crypt] { + let stream = Stream::from_raw(dict.clone(), b"abc".to_vec()); + assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); + assert_eq!(stream.decoded_within(usize::MAX).unwrap(), b"abc"); + assert_eq!(stream.decoded().unwrap(), b"abc"); + assert_eq!(stream.decoded_strict().unwrap(), b"abc"); + assert!(Stream::from_raw(dict, Vec::new()).decoded_within(0).unwrap().is_empty()); + } + } + + #[test] + fn filtered_streams_preserve_the_caller_limit() { + let stream = Stream::flate(Dict::new(), b"abc"); + assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); + assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); + assert_eq!(stream.decoded_strict().unwrap(), b"abc"); + } + + #[test] + fn raw_flate_fallback_streams_preserve_the_caller_limit() { + // Raw deflate with a stored byte and a final empty block. 78 01 also + // passes the zlib header check, so decoding must retry the whole input. + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let stream = Stream::from_raw(dict, vec![0x78, 0x01, 0x00, 0xfe, 0xff, b'A', 0x01, 0x00, 0x00, 0xff, 0xff]); + let result = stream.decoded_within(0); + assert!(matches!(&result, Err(CosError::Filter(detail)) if detail.contains("exceeds the limit of 0 bytes")), "{result:?}"); + assert_eq!(stream.decoded_within(1).unwrap(), b"A"); + assert_eq!(stream.decoded_strict().unwrap(), b"A"); + } + + #[test] + fn decoded_stops_at_renderer_cap_of_256_mib() { + let data = vec![0u8; (256 << 20) + 1]; + let stream = Stream::flate(Dict::new(), &data); + assert!(matches!(stream.decoded(), Err(CosError::Filter(_)))); + } + + #[test] + fn first_filter_decodes_only_the_outer_wrapper() { + let data = "0 0 m 100 100 l S\n".repeat(64).into_bytes(); + let inner = Stream::flate(Dict::new(), &data).raw.to_vec(); + for (outer, name, corrupt) in [(Filter::Ascii85, "ASCII85Decode", &b"{~>"[..]), (Filter::AsciiHex, "ASCIIHexDecode", &b"G0>"[..])] { + let encoded = crate::filters::encode(&outer, &Params::default(), &inner).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name(name), Object::name("FlateDecode")])); + let stream = Stream::from_raw(dict.clone(), encoded); + assert_eq!(stream.decoded_first_filter_strict(usize::MAX).unwrap(), inner); + assert_eq!(stream.decoded_strict().unwrap(), data); + assert!(matches!(stream.decoded_first_filter_strict(inner.len() - 1), Err(CosError::Filter(_)))); + assert!(matches!(Stream::from_raw(dict, corrupt.to_vec()).decoded_first_filter_strict(usize::MAX), Err(CosError::Filter(_)))); + } + } +} diff --git a/crates/pdf/src/cos/page_labels.rs b/crates/pdf/src/cos/page_labels.rs new file mode 100644 index 000000000..557929f8b --- /dev/null +++ b/crates/pdf/src/cos/page_labels.rs @@ -0,0 +1,96 @@ +//! Bounded numbering for page labels (ISO 32000-2 §12.4.2). +//! Pure helpers shared by inspection and editing; no object graph or renderer dependency. + +/// A complete custom label, including its prefix, may contain this many UTF-8 bytes. +pub const MAX_LABEL_BYTES: usize = 1024; +/// Raw prefixes are bounded before decoding; this admits a full UTF-16BE ASCII label. +pub const MAX_PREFIX_BYTES: usize = 2 * MAX_LABEL_BYTES + 2; +/// Sum of the UTF-8 bytes of custom labels for one document. +pub const MAX_LABEL_TOTAL_BYTES: usize = 4 << 20; +/// Sum of the UTF-8 bytes of the decoded prefixes of one `/PageLabels` tree. Each range decodes +/// its own prefix, and ranges may share one prefix object, so without this a small file could ask +/// for a copy per range: [`MAX_LABEL_TREE_WORK`] ranges naming one 1,024-byte prefix would hold +/// about 200 MiB. Real documents stay far below it. +pub const MAX_PREFIX_TOTAL_BYTES: usize = 4 << 20; +/// The most number-tree entries (nodes, kids and range pairs) read from one `/PageLabels` +/// tree. Scaled for real documents: one range per page on a ten-thousand-page PDF happens +/// (prefix labels; #307's file has 9,156 pages), so the cap is twenty times that while still +/// bounding hostile trees. +pub const MAX_LABEL_TREE_WORK: usize = 200_000; +/// Maximum nesting of a label number tree. +pub const MAX_LABEL_TREE_DEPTH: usize = 32; + +/// Lowercase Roman numerals, or `None` if the output exceeds the remaining label budget. +/// The budget is capped at [`MAX_LABEL_BYTES`], including when `n` is hostile. +pub fn roman(mut n: u64, max_bytes: usize) -> Option { + const PARTS: [(u64, &str); 13] = [ + (1000, "m"), + (900, "cm"), + (500, "d"), + (400, "cd"), + (100, "c"), + (90, "xc"), + (50, "l"), + (40, "xl"), + (10, "x"), + (9, "ix"), + (5, "v"), + (4, "iv"), + (1, "i"), + ]; + let max_bytes = max_bytes.min(MAX_LABEL_BYTES); + let mut out = String::new(); + for (value, part) in PARTS { + let count = usize::try_from(n / value).ok()?; + let bytes = count.checked_mul(part.len())?; + if bytes > max_bytes.saturating_sub(out.len()) { + return None; + } + // The quotient is checked against the tiny byte budget before repeating. + out.push_str(&part.repeat(count)); + n %= value; + } + Some(out) +} + +/// a..z, aa..zz, aaa.. (the repeated-letter numbering specified by PDF). +/// Returns `None` instead of truncating a label that exceeds the remaining byte budget. +pub fn alpha(n: u64, max_bytes: usize) -> Option { + let n = n.max(1) - 1; + let count = usize::try_from(n / 26 + 1).ok()?; + if count > max_bytes.min(MAX_LABEL_BYTES) { + return None; + } + let letter = char::from(b'a' + (n % 26) as u8); + Some(std::iter::repeat_n(letter, count).collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn bounded_page_labels_follow_pdf_numbering() { + assert_eq!(roman(4, MAX_LABEL_BYTES).as_deref(), Some("iv")); + assert_eq!(roman(1994, MAX_LABEL_BYTES).as_deref(), Some("mcmxciv")); + for (n, expected) in [(0, "a"), (1, "a"), (26, "z"), (27, "aa"), (52, "zz"), (53, "aaa")] { + assert_eq!(alpha(n, MAX_LABEL_BYTES).as_deref(), Some(expected)); + } + } + + #[test] + fn bounded_page_labels_check_lengths_before_repeating() { + assert_eq!(roman(1_024_000, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES); + assert!(roman(1_024_001, MAX_LABEL_BYTES).is_none()); + assert_eq!(alpha(26 * 1024, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES); + assert!(alpha(26 * 1024 + 1, MAX_LABEL_BYTES).is_none()); + assert_eq!(roman(4, 2).as_deref(), Some("iv")); + assert!(roman(4, 1).is_none()); + assert_eq!(alpha(27, 2).as_deref(), Some("aa")); + assert!(alpha(27, 1).is_none()); + for max in [0, MAX_LABEL_BYTES, usize::MAX] { + assert!(roman(u64::MAX, max).is_none()); + assert!(alpha(u64::MAX, max).is_none()); + } + } +} diff --git a/crates/pdf/src/cos/parser.rs b/crates/pdf/src/cos/parser.rs new file mode 100644 index 000000000..763a5e514 --- /dev/null +++ b/crates/pdf/src/cos/parser.rs @@ -0,0 +1,886 @@ +//! Lexer and object parser (ISO 32000-2 §7.2–7.3), tolerant of common real-world damage. +//! +//! The parser works on a byte slice and a position; it never allocates for skipped data and +//! never panics on malformed input (every index is bounds-checked). Depth is limited to defend +//! against pathological nesting. + +use std::sync::Arc; + +use crate::cos::object::{Dict, ObjRef, Object, PdfString, Stream}; +use crate::cos::{Bytes, CosError}; + +const MAX_DEPTH: usize = 128; +/// Objects one top-level [`Lexer::object`] call may build. Real arrays (page trees, parent +/// trees, widths) stay far below; a decompressed object stream of `0 0 0 ...` does not. +const MAX_NODES: usize = 1 << 22; + +/// One of the six PDF whitespace characters (ISO 32000-2 §7.2.3, Table 1). +pub fn is_whitespace(b: u8) -> bool { + matches!(b, b'\0' | b'\t' | b'\n' | b'\x0C' | b'\r' | b' ') +} + +pub(crate) fn is_delimiter(b: u8) -> bool { + matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') +} + +fn is_regular(b: u8) -> bool { + !is_whitespace(b) && !is_delimiter(b) +} + +/// A cursor over PDF bytes. +pub struct Lexer<'a> { + pub data: &'a [u8], + pub pos: usize, + /// Objects built by the current top-level [`Lexer::object`] call. + nodes: usize, +} + +impl<'a> Lexer<'a> { + pub fn new(data: &'a [u8], pos: usize) -> Self { + Self { data, pos: pos.min(data.len()), nodes: 0 } + } + + fn peek(&self) -> Option { + self.data.get(self.pos).copied() + } + + /// Skip whitespace and comments. + pub fn skip_ws(&mut self) { + while let Some(b) = self.peek() { + if is_whitespace(b) { + self.pos += 1; + } else if b == b'%' { + while let Some(c) = self.peek() { + if c == b'\n' || c == b'\r' { + break; + } + self.pos += 1; + } + } else { + break; + } + } + } + + /// Read a run of regular characters (a keyword or number). + pub fn token(&mut self) -> &'a [u8] { + let start = self.pos.min(self.data.len()); + while self.peek().is_some_and(is_regular) { + self.pos += 1; + } + &self.data[start..self.pos] + } + + /// `true` (and consumes it) if the next token is exactly `kw`. + pub fn eat_keyword(&mut self, kw: &[u8]) -> bool { + self.skip_ws(); + let end = self.pos + kw.len(); + if self.data.get(self.pos..end) == Some(kw) && self.data.get(end).is_none_or(|b| !is_regular(*b)) { + self.pos = end; + true + } else { + false + } + } + + fn err(&self, what: &str) -> CosError { + CosError::Syntax { offset: self.pos, detail: what.to_string() } + } + + /// Parse one direct object. References `N G R` are recognised. + pub fn object(&mut self) -> Result { + self.nodes = 0; + self.object_depth(0) + } + + fn over_budget(&self) -> bool { + self.nodes > MAX_NODES + } + + fn object_depth(&mut self, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(self.err("nesting too deep")); + } + self.nodes = self.nodes.saturating_add(1); + if self.over_budget() { + return Err(self.err("object too large")); + } + self.skip_ws(); + let Some(b) = self.peek() else { return Err(self.err("unexpected end of data")) }; + match b { + b'/' => { + self.pos += 1; + Ok(Object::Name(self.name_body())) + } + b'(' => { + self.pos += 1; + Ok(Object::String(PdfString { bytes: self.literal_string()?, hex: false })) + } + b'<' if self.data.get(self.pos + 1) == Some(&b'<') => { + self.pos += 2; + Ok(Object::Dict(self.dict_body(depth)?)) + } + b'<' => { + self.pos += 1; + Ok(Object::String(PdfString { bytes: self.hex_string()?, hex: true })) + } + b'[' => { + self.pos += 1; + let mut items = Vec::new(); + loop { + self.skip_ws(); + match self.peek() { + None => return Err(self.err("unterminated array")), + Some(b']') => { + self.pos += 1; + break; + } + _ => { + let before = self.pos; + match self.object_depth(depth + 1) { + Ok(o) => items.push(o), + Err(e) if self.over_budget() => return Err(e), + // Tolerate junk inside arrays by skipping one byte. + Err(_) if self.pos == before => self.pos += 1, + Err(e) => return Err(e), + } + } + } + } + Ok(Object::Array(items)) + } + b'+' | b'-' | b'.' | b'0'..=b'9' => self.number_or_ref(), + _ => { + let start = self.pos; + let tok = self.token(); + match tok { + b"true" => Ok(Object::Bool(true)), + b"false" => Ok(Object::Bool(false)), + b"null" => Ok(Object::Null), + b"" => { + self.pos = start + 1; + Err(CosError::Syntax { offset: start, detail: format!("unexpected byte 0x{b:02x}") }) + } + other => Err(CosError::Syntax { offset: start, detail: format!("unexpected keyword {:?}", String::from_utf8_lossy(other)) }), + } + } + } + } + + fn name_body(&mut self) -> Vec { + let mut out = Vec::new(); + while let Some(b) = self.peek() { + if !is_regular(b) { + break; + } + self.pos += 1; + if b == b'#' + && let (Some(h), Some(l)) = (self.data.get(self.pos).and_then(|c| hexval(*c)), self.data.get(self.pos + 1).and_then(|c| hexval(*c))) + { + out.push(h << 4 | l); + self.pos += 2; + continue; + } + out.push(b); + } + out + } + + fn literal_string(&mut self) -> Result, CosError> { + let mut out = Vec::new(); + let mut depth = 1usize; + while let Some(b) = self.peek() { + self.pos += 1; + match b { + b'(' => { + depth += 1; + out.push(b); + } + b')' => { + depth -= 1; + if depth == 0 { + return Ok(out); + } + out.push(b); + } + b'\\' => { + let Some(e) = self.peek() else { break }; + self.pos += 1; + match e { + b'n' => out.push(b'\n'), + b'r' => out.push(b'\r'), + b't' => out.push(b'\t'), + b'b' => out.push(8), + b'f' => out.push(12), + b'(' | b')' | b'\\' => out.push(e), + b'\r' => { + if self.peek() == Some(b'\n') { + self.pos += 1; + } + } + b'\n' => {} + b'0'..=b'7' => { + let mut v = (e - b'0') as u32; + for _ in 0..2 { + match self.peek() { + Some(d @ b'0'..=b'7') => { + v = v * 8 + (d - b'0') as u32; + self.pos += 1; + } + _ => break, + } + } + out.push((v & 0xFF) as u8); + } + other => out.push(other), + } + } + // End-of-line markers inside strings are normalised to \n (§7.3.4.2). + b'\r' => { + if self.peek() == Some(b'\n') { + self.pos += 1; + } + out.push(b'\n'); + } + other => out.push(other), + } + } + Err(self.err("unterminated string")) + } + + fn hex_string(&mut self) -> Result, CosError> { + let mut out = Vec::new(); + let mut hi: Option = None; + while let Some(b) = self.peek() { + self.pos += 1; + if b == b'>' { + if let Some(h) = hi { + out.push(h << 4); + } + return Ok(out); + } + if is_whitespace(b) { + continue; + } + let Some(v) = hexval(b) else { return Err(self.err("invalid hex digit")) }; + match hi.take() { + None => hi = Some(v), + Some(h) => out.push(h << 4 | v), + } + } + Err(self.err("unterminated hex string")) + } + + fn dict_body(&mut self, depth: usize) -> Result { + let mut d = Dict::new(); + loop { + self.skip_ws(); + match self.peek() { + None => return Err(self.err("unterminated dictionary")), + Some(b'>') if self.data.get(self.pos + 1) == Some(&b'>') => { + self.pos += 2; + return Ok(d); + } + Some(b'/') => { + self.pos += 1; + let key = self.name_body(); + self.skip_ws(); + // A key directly followed by `>>` has no value: treat as null. (A following + // `/` is a name *value*, so it cannot signal a missing value.) + if self.peek() == Some(b'>') && self.data.get(self.pos + 1) == Some(&b'>') { + d.set(key, Object::Null); + continue; + } + let value = self.object_depth(depth + 1)?; + d.set(key, value); + } + Some(_) => { + // Tolerate garbage between entries (seen in damaged files): skip a token. + let before = self.pos; + let _ = self.object_depth(depth + 1); + if self.over_budget() { + return Err(self.err("object too large")); + } + if self.pos == before { + self.pos += 1; + } + } + } + } + } + + fn number_or_ref(&mut self) -> Result { + let start = self.pos; + let first = self.number()?; + // Look ahead for "G R" to form a reference. + if let Object::Int(n) = first + && n >= 0 + { + let save = self.pos; + self.skip_ws(); + if self.peek().is_some_and(|b| b.is_ascii_digit()) { + let gstart = self.pos; + let g = self.token(); + if g.iter().all(u8::is_ascii_digit) && self.eat_keyword(b"R") { + let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0); + if n <= u32::MAX as i64 { + return Ok(Object::Ref(ObjRef::new(n as u32, generation.min(u16::MAX as u32) as u16))); + } + } + let _ = gstart; + } + self.pos = save; + } + let _ = start; + Ok(first) + } + + fn number(&mut self) -> Result { + let start = self.pos; + let tok = self.token(); + let s = std::str::from_utf8(tok).map_err(|_| self.err("bad number"))?; + if s.contains('.') { + // Tolerate forms like "--5", "5-" or "1.2.3" by parsing the longest valid prefix. + // Only a doubled sign needs a cleaned copy: a page of InDesign `/PieceInfo` is + // thousands of plain reals, and building a copy and a substring search for each + // was a large part of reading one (#5873). + let t = s.trim_start_matches('+'); + let cleaned = if t.as_bytes().windows(2).any(|w| w == b"--") { std::borrow::Cow::Owned(t.replacen("--", "-", 1)) } else { t.into() }; + let v = cleaned.get(..real_prefix_len(cleaned.as_bytes())).and_then(|p| parse_counted::(p).ok()); + return Ok(Object::Real(v.unwrap_or(0.0))); + } + // Tolerate a doubled sign ("--280") like the real branch does. + let t = s.trim_start_matches('+'); + let cleaned = t.strip_prefix('-').filter(|r| r.starts_with('-')).unwrap_or(t); + match cleaned.parse::() { + Ok(v) => Ok(Object::Int(v)), + Err(_) => { + // Out-of-range integers are treated as reals (§7.3.3 allows implementation limits). + cleaned.parse::().map(Object::Real).map_err(|_| CosError::Syntax { offset: start, detail: format!("bad number {s:?}") }) + } + } + } +} + +/// Length of the longest prefix of `s` that reads as a decimal number: a sign, digits with at +/// most one `.`, and an exponent with digits. Zero when that prefix has no digit. Found in one +/// pass so a long damaged token is converted once, not once per shorter prefix. +fn real_prefix_len(s: &[u8]) -> usize { + let digits = |from: usize| s.get(from..).map_or(0, |r| r.iter().take_while(|b| b.is_ascii_digit()).count()); + let mut at = usize::from(matches!(s.first(), Some(b'+' | b'-'))); + let int = digits(at); + at += int; + let mut frac = 0; + if s.get(at) == Some(&b'.') { + frac = digits(at + 1); + at += 1 + frac; + } + if int + frac == 0 { + return 0; + } + if matches!(s.get(at), Some(b'e' | b'E')) { + let sign = usize::from(matches!(s.get(at + 1), Some(b'+' | b'-'))); + let exp = digits(at + 1 + sign); + if exp > 0 { + at += 1 + sign + exp; + } + } + at +} + +#[cfg(test)] +thread_local! { + /// Bytes searched for `endstream` on this thread. + pub(crate) static ENDSTREAM_SCAN: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Bytes handed to number conversion on this thread. + static NUMBER_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +fn parse_counted(s: &str) -> Result { + #[cfg(test)] + NUMBER_BYTES.with(|c| c.set(c.get() + s.len())); + s.parse() +} + +fn hexval(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +/// Parse an indirect object at `offset`: `N G obj [stream…endstream] endobj`. +/// `resolve_length` resolves an indirect `/Length`. Returns the object and its (num, gen). +pub fn parse_indirect(data: &[u8], offset: usize, resolve_length: &dyn Fn(ObjRef) -> Option) -> Result<(ObjRef, Object), CosError> { + parse_indirect_in(data, None, offset, resolve_length, None) +} + +/// [`parse_indirect`] for a document's own bytes: stream data points into `data` instead of +/// being copied out of it. +/// `endstreams` is the document's [`EndstreamIndex`] for `data`. +pub(crate) fn parse_indirect_shared( + data: &Arc>, + offset: usize, + resolve_length: &dyn Fn(ObjRef) -> Option, + endstreams: &EndstreamIndex, +) -> Result<(ObjRef, Object), CosError> { + parse_indirect_in(data, Some(data), offset, resolve_length, Some(endstreams)) +} + +/// Where `endstream` occurs in a document's bytes, found once, the first time a stream's +/// /Length is wrong or missing. Searching the rest of the file for each such stream instead +/// makes many of them ending at one late `endstream` quadratic to read. +pub(crate) type EndstreamIndex = std::sync::OnceLock>; + +fn endstream_offsets(data: &[u8]) -> Vec { + #[cfg(test)] + ENDSTREAM_SCAN.with(|c| c.set(c.get() + data.len())); + data.windows(9).enumerate().filter(|(i, w)| *w == b"endstream" && endstream_at(data, *i)).map(|(i, _)| i).collect() +} + +fn parse_indirect_in( + data: &[u8], + shared: Option<&Arc>>, + offset: usize, + resolve_length: &dyn Fn(ObjRef) -> Option, + endstreams: Option<&EndstreamIndex>, +) -> Result<(ObjRef, Object), CosError> { + let mut lx = Lexer::new(data, offset); + lx.skip_ws(); + let num_tok = lx.token(); + lx.skip_ws(); + let gen_tok = lx.token(); + let parse_u = |t: &[u8]| std::str::from_utf8(t).ok().and_then(|s| s.parse::().ok()); + let (Some(num), Some(generation)) = (parse_u(num_tok), parse_u(gen_tok)) else { + return Err(CosError::Syntax { offset, detail: "expected object header".into() }); + }; + if !lx.eat_keyword(b"obj") { + return Err(CosError::Syntax { offset: lx.pos, detail: "expected 'obj'".into() }); + } + let id = ObjRef::new(num.min(u32::MAX as u64) as u32, generation.min(u16::MAX as u64) as u16); + let obj = match lx.object() { + Ok(o) => o, + // "N G obj endobj" (empty) is treated as null. + Err(_) if lx.eat_keyword(b"endobj") => return Ok((id, Object::Null)), + Err(e) => return Err(e), + }; + lx.skip_ws(); + if let Object::Dict(dict) = &obj + && lx.eat_keyword(b"stream") + { + // The keyword is followed by CRLF or LF (a lone CR is tolerated). Microsoft Reporting + // Services writes a space first (`stream \r\n`); spaces and tabs count only when a line + // break follows, so they are never taken from the data. + let mut eol = lx.pos; + while matches!(data.get(eol), Some(b' ' | b'\t')) { + eol += 1; + } + if matches!(data.get(eol), Some(b'\r' | b'\n')) { + lx.pos = eol; + } + match (data.get(lx.pos), data.get(lx.pos + 1)) { + (Some(b'\r'), Some(b'\n')) => lx.pos += 2, + (Some(b'\n'), _) | (Some(b'\r'), _) => lx.pos += 1, + _ => {} + } + let start = lx.pos; + let declared = match dict.get(b"Length") { + Some(Object::Int(n)) => Some(*n), + Some(Object::Ref(r)) => resolve_length(*r), + _ => None, + }; + // A /Length that fits in the file (not negative, and no wrap on a 32-bit target). + let fits = declared.and_then(|l| usize::try_from(l).ok()).filter(|l| start.checked_add(*l).is_some_and(|end| end <= data.len())); + // What follows the /Length bytes, white space aside. + let after = |len: usize| { + let mut after = Lexer::new(data, start.saturating_add(len)); + after.skip_ws(); + data.get(after.pos..).unwrap_or_default() + }; + let len = match fits { + // The /Length ends at `endstream`, or where it would be: misspelt (`endstrea`), or + // left out before `endobj` or the end of the file. The stream is the bytes its /Length + // gives, "how many bytes of the PDF file are used for the stream's data" (ISO 32000-2 + // §7.3.8.2), as other readers take it. A producer that wrote `endstrea` after every + // stream lost every font and every page's content, and searching on would run into a + // later object's `endstream`. + Some(l) if at_stream_end(after(l)) => l, + // Wrong or missing /Length: search for "endstream" (§7.3.8.1 recovery). With none + // left in the file, a /Length that fits is still the stream's extent. + _ => match find_endstream(data, start, endstreams) { + Some(end) => end - start, + None => fits.ok_or_else(|| CosError::Syntax { offset: start, detail: "stream without endstream".into() })?, + }, + }; + let range = start..start + len; + let raw = match shared { + Some(buf) => Bytes::view(buf, range), + None => data.get(range).map(|r| Bytes::from(r.to_vec())), + } + .ok_or_else(|| CosError::Syntax { offset: start, detail: "stream data past the end of the file".into() })?; + let mut dict = dict.clone(); + dict.set(b"Length".to_vec(), Object::Int(raw.len() as i64)); + return Ok((id, Object::Stream(Stream { dict, raw }))); + } + Ok((id, obj)) +} + +/// Whether `rest`, what follows a stream's /Length bytes with white space skipped, is where the +/// stream ends. A whole `endstream` token is: a token runs on through regular characters (ISO +/// 32000-2 §7.2.3), so payload such as `endstreamer` is not one. A misspelt `endstream` +/// (`endstrea`), an `endobj` with the `endstream` left out, or both, end it only where the object +/// ends as well, before the end of the data, another object's `N G obj` header, or `xref`, +/// `trailer` or `startxref`: payload such as `endobj more` or `endobject`, where a wrong /Length +/// stops, is not, and the search for `endstream` runs instead. So does anything else but the end +/// of the data. +fn at_stream_end(rest: &[u8]) -> bool { + let mut lx = Lexer::new(rest, 0); + if lx.eat_keyword(b"endstream") { + return true; + } + let misspelt = lx.eat_keyword(b"endstrea"); + let endobj = lx.eat_keyword(b"endobj"); + lx.skip_ws(); + if lx.pos >= rest.len() { + return true; + } + if !(misspelt || endobj) { + return false; + } + if [&b"xref"[..], b"trailer", b"startxref"].iter().any(|kw| Lexer::new(rest, lx.pos).eat_keyword(kw)) { + return true; + } + let number = |lx: &mut Lexer| { + lx.skip_ws(); + let t = lx.token(); + !t.is_empty() && t.iter().all(u8::is_ascii_digit) + }; + number(&mut lx) && number(&mut lx) && lx.eat_keyword(b"obj") +} + +/// Whether a whole `endstream` token starts at `at` in `data`: one followed by a regular +/// character (`endstreamer`) is payload (ISO 32000-2 §7.2.3). +fn endstream_at(data: &[u8], at: usize) -> bool { + data.get(at..).is_some_and(|d| d.starts_with(b"endstream")) && !data.get(at.saturating_add(9)).copied().is_some_and(is_regular) +} + +/// Offset where stream data ends (before the EOL preceding "endstream"). +fn find_endstream(data: &[u8], from: usize, endstreams: Option<&EndstreamIndex>) -> Option { + let at = match endstreams { + Some(index) => { + let index = index.get_or_init(|| endstream_offsets(data)); + index.get(index.partition_point(|&p| p < from)).copied()? + } + None => { + let hay = data.get(from..)?; + let found = hay.windows(9).enumerate().position(|(i, w)| w == b"endstream" && endstream_at(hay, i)); + #[cfg(test)] + ENDSTREAM_SCAN.with(|c| c.set(c.get() + found.map_or(hay.len(), |p| p + 9))); + found? + from + } + }; + let mut end = at; + if end > from && data[end - 1] == b'\n' { + end -= 1; + } + if end > from && data[end - 1] == b'\r' { + end -= 1; + } + Some(end) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn numbers_with_multibyte_garbage_do_not_panic() { + // From `cargo xtask fuzz`: "2 0 obj\n.\u{2014}" (an em dash after a dot). + let doc = crate::cos::Document::open(std::sync::Arc::new("2 0 obj\n.\u{2014}\n".as_bytes().to_vec())); + assert!(doc.is_err()); + for t in [".\u{2014}", "1.\u{e9}5", "-.\u{1F600}", "+.5\u{2014}"] { + let _ = Lexer::new(t.as_bytes(), 0).object(); + } + assert_eq!(Lexer::new("1.5\u{2014}".as_bytes(), 0).object().unwrap(), Object::Real(1.5)); + } + + #[test] + fn a_damaged_real_is_converted_once() { + let token = format!("1.{}{}", "5".repeat(2_000), "x".repeat(20_000)); + NUMBER_BYTES.with(|c| c.set(0)); + let Ok(Object::Real(v)) = Lexer::new(token.as_bytes(), 0).object() else { panic!("not a real") }; + let converted = NUMBER_BYTES.with(|c| c.get()); + assert!(converted <= token.len(), "{converted} bytes converted for a {}-byte token", token.len()); + assert!((v - 1.5555).abs() < 1e-3); + // The longest numeric prefix wins, as before. + for (t, want) in + [("1.2.3", 1.2), ("--5.", -5.0), ("+.5x", 0.5), ("5.-", 5.0), ("1.5e3", 1500.0), ("1.5e+", 1.5), ("-.x", 0.0), (".", 0.0), ("-inf.", 0.0)] + { + assert_eq!(Lexer::new(t.as_bytes(), 0).object().ok(), Some(Object::Real(want)), "{t}"); + } + } + + fn obj(s: &str) -> Object { + Lexer::new(s.as_bytes(), 0).object().unwrap_or_else(|e| panic!("{s}: {e}")) + } + + #[test] + fn doubled_sign_integer_reads_like_the_real_form() { + // Some writers emit a TJ kerning as "--280"; the real form "--280.0" is already read. + assert_eq!(obj("--280.0"), Object::Real(-280.0)); + assert_eq!(obj("--280"), Object::Int(-280)); + match obj("[(Hello) --280 (World)]") { + Object::Array(items) => { + assert_eq!(items.len(), 3); + assert_eq!(items[1], Object::Int(-280)); + } + other => panic!("not an array: {other:?}"), + } + } + + #[test] + fn scalars() { + assert_eq!(obj("true"), Object::Bool(true)); + assert_eq!(obj("null"), Object::Null); + assert_eq!(obj(" -17 "), Object::Int(-17)); + assert_eq!(obj("+42"), Object::Int(42)); + assert_eq!(obj("-.5"), Object::Real(-0.5)); + assert_eq!(obj("4."), Object::Real(4.0)); + assert_eq!(obj("99999999999999999999"), Object::Real(1e20)); + assert_eq!(obj("12 0 R"), Object::Ref(ObjRef::new(12, 0))); + assert_eq!(obj("12 0 obj").as_int(), Some(12), "12 followed by 0 obj is not a reference"); + } + + #[test] + fn names_decode_hex_escapes() { + assert_eq!(obj("/A#20B"), Object::Name(b"A B".to_vec())); + assert_eq!(obj("/Adobe#"), Object::Name(b"Adobe#".to_vec())); + assert_eq!(obj("/"), Object::Name(Vec::new())); + } + + #[test] + fn literal_strings() { + let s = |x: &str| obj(x).as_string().expect("string").bytes.clone(); + assert_eq!(s("(a (nested) b)"), b"a (nested) b"); + assert_eq!(s(r"(\n\t\\\(\))"), b"\n\t\\()"); + assert_eq!(s(r"(\101\1011)"), b"AA1"); + assert_eq!(s("(line\\\ncontinued)"), b"linecontinued"); + assert_eq!(s("(cr\r\nlf)"), b"cr\nlf"); + } + + #[test] + fn hex_strings() { + let s = obj("<48 65 6C6c6f7>"); + assert_eq!(s.as_string().unwrap().bytes, b"Hellop"); + assert!(s.as_string().unwrap().hex); + } + + #[test] + fn arrays_and_dicts() { + let o = obj("<< /Type /Page /Kids [1 0 R 2 0 R] /Box [0 0 612.5 792] /Nested << /A true >> /Empty >>"); + let d = o.as_dict().unwrap(); + assert_eq!(d.name(b"Type"), Some(&b"Page"[..])); + assert_eq!(d.get(b"Kids").unwrap().as_array().unwrap().len(), 2); + assert_eq!(d.get(b"Box").unwrap().as_array().unwrap()[2], Object::Real(612.5)); + assert_eq!(d.get(b"Empty"), Some(&Object::Null)); + } + + #[test] + fn comments_are_whitespace() { + assert_eq!(obj("% hello\n [1 % two\n 3]"), Object::Array(vec![Object::Int(1), Object::Int(3)])); + } + + #[test] + fn indirect_stream_with_wrong_length_recovers() { + let data = b"7 0 obj\n<< /Length 999 >>\nstream\nHELLO\nendstream\nendobj\n"; + let (id, o) = parse_indirect(data, 0, &|_| None).unwrap(); + assert_eq!(id, ObjRef::new(7, 0)); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO"); + assert_eq!(s.dict.int(b"Length"), Some(5)); + } + + /// With no `endstream` left in the file, as a producer that writes `endstrea` leaves every + /// stream, a stream is the bytes its /Length gives, "how many bytes of the PDF file are used + /// for the stream's data" (ISO 32000-2 §7.3.8.2): it was an error, and the object null, so + /// such a file lost every font and page content. A /Length past the file, or none, is still + /// no stream. + #[test] + fn indirect_stream_without_endstream_is_its_length() { + for end in [&b"\r\nendstrea\rendobj\r\n"[..], b"\nendobj\n", b""] { + let mut data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO".to_vec(); + data.extend_from_slice(end); + let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO", "{end:?}"); + } + assert!(parse_indirect(b"7 0 obj\n<< /Length 6 >>\nstream\nHELLO", 0, &|_| None).is_err(), "past the file"); + assert!(parse_indirect(b"7 0 obj\n<< >>\nstream\nHELLO\nendstrea\nendobj\n", 0, &|_| None).is_err(), "no /Length"); + } + + /// A stream ending in `endstrea` is its /Length even when a later object does have an + /// `endstream`: the search for one would otherwise run on into that object and take its + /// header and data as this stream's. + #[test] + fn a_stream_without_endstream_is_not_read_into_a_later_one() { + let data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO\r\nendstrea\rendobj\r\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n"; + let second = data.windows(7).position(|w| w == b"8 0 obj").unwrap(); + for (offset, want) in [(0, &b"HELLO"[..]), (second, b"WORLD")] { + let (_, o) = parse_indirect(data, offset, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected at {offset}") }; + assert_eq!(&s.raw[..], want, "at {offset}"); + } + // Through the loader's shared index of `endstream` offsets as well. + let index = EndstreamIndex::new(); + let (_, o) = parse_indirect_in(data, None, 0, &|_| None, Some(&index)).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO"); + } + + /// The same for a stream whose `endstream` is left out before `endobj`: the later object's + /// `endstream` is not this stream's end. + #[test] + fn a_stream_ending_at_endobj_is_not_read_into_a_later_one() { + let data = b"7 0 obj\n<< /Length 5 >>\nstream\nHELLO\nendobj\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n"; + let (_, o) = parse_indirect(data, 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO"); + } + + /// A wrong /Length that stops before payload bytes beginning like an end marker (`endobject`, + /// `endstreak`, `endstreamer`) is not taken for the stream's end: a token runs on through + /// regular characters (ISO 32000-2 §7.2.3), so the search for `endstream` finds the whole + /// payload, and skips `endstreamer` too, by itself and through the loader's shared index. + #[test] + fn a_length_ending_inside_a_longer_word_is_not_an_end_marker() { + for word in ["endobject", "endstreak", "endstreamer"] { + let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {word} more\nendstream\nendobj\n"); + let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}"); + let index = EndstreamIndex::new(); + let (_, o) = parse_indirect_in(pdf.as_bytes(), None, 0, &|_| None, Some(&index)).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}, indexed"); + } + // Whole markers still end it, followed by white space, a delimiter or the end of the data. + for tail in ["endobj\n", "endstrea%", "endobj"] { + let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{tail}"); + let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO", "{tail:?}"); + } + } + + /// A whole `endobj` or `endstrea` token where a wrong /Length stops is payload, not the + /// stream's end, unless the object ends there too: what follows is the end of the data, + /// another object's header, or `xref`, `trailer` or `startxref`. Otherwise the search for + /// `endstream` finds the whole payload. + #[test] + fn an_end_marker_inside_the_payload_does_not_end_the_stream() { + for payload in ["endobj more", "endstrea more", "endstrea endobj more", "endobj 12 more"] { + let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {payload}\nendstream\nendobj\n"); + let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(s.raw[..], *format!("HELLO {payload}").as_bytes(), "{payload}"); + } + // Where the object does end, the /Length is the stream, also with an `endstream` later. + for next in ["8 0 obj\n<< >>", "%c\n12 0 obj", "xref", "trailer", "startxref"] { + for marker in ["endobj", "endstrea\nendobj", "endstrea"] { + let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{marker}\n{next}\nstream\nWORLD\nendstream\nendobj\n"); + let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO", "{marker:?} then {next:?}"); + } + } + } + /// Without an `endstream`, the /Length is trusted as written when it fits, also when it is + /// wrong, and an indirect one is resolved first; one that doesn't fit, a negative one and one + /// too large for any file are errors rather than wrapped into a short read. + #[test] + fn a_stream_without_endstream_takes_its_length_only_when_it_fits() { + let stream = |length: &str, tail: &str| format!("7 0 obj\n<< /Length {length} >>\nstream\nHELLO{tail}"); + let raw = |pdf: &str, lookup: &dyn Fn(ObjRef) -> Option| match parse_indirect(pdf.as_bytes(), 0, lookup) { + Ok((_, Object::Stream(s))) => Ok(s.raw.to_vec()), + Ok((_, other)) => panic!("{pdf:?} is {other:?}"), + Err(e) => Err(e), + }; + let none = |_: ObjRef| None; + // Wrong but fitting: the bytes it gives, not the bytes up to `endstrea`. + assert_eq!(raw(&stream("3", "\nendstrea\nendobj\n"), &none).unwrap(), b"HEL"); + // Exactly to the end of the file, and nothing at all. + assert_eq!(raw(&stream("5", ""), &none).unwrap(), b"HELLO"); + assert_eq!(raw(&stream("0", "\nendstrea\nendobj\n"), &none).unwrap(), b""); + // Indirect, resolved through the lookup. + let lookup = |r: ObjRef| (r == ObjRef::new(9, 0)).then_some(4); + assert_eq!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &lookup).unwrap(), b"HELL"); + assert!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &none).is_err(), "an unresolved /Length"); + // One past the end, negative, the largest integer and one beyond it. + for length in ["6", "-1", "-9223372036854775808", "9223372036854775807", "18446744073709551621"] { + assert!(raw(&stream(length, ""), &none).is_err(), "/Length {length}"); + } + } + + #[test] + fn indirect_stream_with_a_space_before_the_line_break() { + // `stream \r\n` as Microsoft Reporting Services writes it: the data starts after the + // line break, so the /Length is still right and the data keeps no leading space. + for eol in [&b" \r\n"[..], b" \n", b"\t \r"] { + let mut data = b"7 0 obj\n<< /Length 5 >>\nstream".to_vec(); + data.extend_from_slice(eol); + data.extend_from_slice(b"HELLO\r\nendstream\nendobj\n"); + let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b"HELLO", "{eol:?}"); + } + // Spaces with no line break after them are data, not a keyword terminator. + let data = b"7 0 obj\n<< /Length 7 >>\nstream HELLO\nendstream\nendobj\n"; + let (_, o) = parse_indirect(data, 0, &|_| None).unwrap(); + let Object::Stream(s) = o else { panic!("stream expected") }; + assert_eq!(&s.raw[..], b" HELLO"); + } + + #[test] + fn indirect_length_is_resolved() { + let data = b"7 0 obj\n<< /Length 8 0 R >>\nstream\r\nAB\r\nendstream\nendobj\n"; + let (_, o) = parse_indirect(data, 0, &|r| (r.num == 8).then_some(2)).unwrap(); + let Object::Stream(s) = o else { panic!() }; + assert_eq!(&s.raw[..], b"AB"); + } + + #[test] + fn garbage_never_panics() { + for input in ["<<", "[", "(", "<", "<< /A", "[1 2", ")", "]", ">>", "<<>>>", "1 0", "\0\0", "/A#", "", "((("] { + let _ = Lexer::new(input.as_bytes(), 0).object(); + } + let deep = "[".repeat(10_000); + assert!(Lexer::new(deep.as_bytes(), 0).object().is_err()); + } + + #[test] + fn one_object_has_a_node_budget() { + // A wide array (from a tiny compressed object stream) must not grow without bound. + let wide = |n: usize| format!("[{}]", "0 ".repeat(n)); + let at_cap = Lexer::new(wide(MAX_NODES - 1).as_bytes(), 0).object(); + assert!(matches!(at_cap, Ok(Object::Array(ref items)) if items.len() == MAX_NODES - 1)); + assert!(Lexer::new(wide(MAX_NODES).as_bytes(), 0).object().is_err()); + // The same inside a dictionary value and behind tolerated junk. + assert!(Lexer::new(format!("<< /A {} >>", wide(MAX_NODES)).as_bytes(), 0).object().is_err()); + assert!(Lexer::new(format!("[[{}] ]", "0 ".repeat(MAX_NODES)).as_bytes(), 0).object().is_err()); + // The budget is per top-level object: a lexer reused for many operands keeps working. + let two = wide(MAX_NODES - 1).repeat(2); + let mut lx = Lexer::new(two.as_bytes(), 0); + assert!(lx.object().is_ok() && lx.object().is_ok()); + } + + /// A plain real is read without a cleaned copy of its token; the forms that need one read + /// as they did (#5873). + #[test] + fn reals_read_the_same_with_and_without_a_doubled_sign() { + for (token, want) in [("1.5", 1.5), ("-375.993", -375.993), ("+2.25", 2.25), ("--5.5", -5.5), ("1.2.3", 1.2), ("-.5", -0.5), ("0.0", 0.0)] { + assert_eq!(Lexer::new(token.as_bytes(), 0).object().unwrap(), Object::Real(want), "{token}"); + } + } +} diff --git a/crates/pdf/src/cos/security.rs b/crates/pdf/src/cos/security.rs new file mode 100644 index 000000000..69b59f39c --- /dev/null +++ b/crates/pdf/src/cos/security.rs @@ -0,0 +1,376 @@ +//! Bridging the COS object model and [`crate::crypt`]: reading `/Encrypt` dictionaries and +//! decrypting / encrypting the strings and streams of an object (§7.6.2). +//! +//! Not encrypted, per the spec: the `/Encrypt` dictionary itself, cross-reference streams, the +//! trailer `/ID`, objects inside object streams (the object stream is encrypted as a whole), +//! the `/Contents` of signature dictionaries, and metadata streams when `/EncryptMetadata` is +//! false. Streams naming a crypt filter (`/Crypt` in `/Filter`) use that filter; embedded files +//! use `/EFF`. + +use std::cell::Cell; + +use crate::crypt::{EncryptDict, Method, SecurityHandler, StreamKind}; +use crate::filters::MAX_CHAIN; + +use crate::cos::document::Held; +use crate::cos::{Dict, Document, Object, PdfString, Stream}; + +/// How many references one read or write of an object may follow, with all that it loads on the +/// way. How an object is treated can be given by reference, and what that loads can do the same: a +/// file could make one read a tree of them, each object read again once the cache has let it go. +/// An object takes a few of these, and two dozen if every part of it is given by reference; past +/// this they read as absent. The parameters of an object stream, read together, count as one. +const MAX_FOLLOWED: u32 = 32; + +std::thread_local! { + /// How many [`FollowScope`]s this thread is inside. + static SCOPES: Cell = const { Cell::new(0) }; + /// The references followed in the outermost of them. + static FOLLOWED: Cell = const { Cell::new(0) }; + /// The references refused in the outermost of them, past [`MAX_FOLLOWED`]. + static REFUSED: Cell = const { Cell::new(0) }; +} + +/// One read or write of an object, with everything it loads on the way: what [`MAX_FOLLOWED`] counts. +pub(crate) struct FollowScope; + +impl FollowScope { + pub(crate) fn enter() -> Self { + if SCOPES.replace(SCOPES.get().saturating_add(1)) == 0 { + FOLLOWED.set(0); + REFUSED.set(0); + } + FollowScope + } +} + +impl Drop for FollowScope { + fn drop(&mut self) { + SCOPES.set(SCOPES.get().saturating_sub(1)); + } +} + +/// What `f` gives, counted against [`MAX_FOLLOWED`], or `spent` past it. Called inside a +/// [`FollowScope`], which is what starts the count again. +pub(crate) fn following(spent: T, f: impl FnOnce() -> T) -> T { + if FOLLOWED.get() >= MAX_FOLLOWED { + REFUSED.set(REFUSED.get().saturating_add(1)); + return spent; + } + FOLLOWED.set(FOLLOWED.get() + 1); + f() +} + +/// `o`, or the object it refers to, for what decides how a stream or dictionary is treated. Past +/// [`MAX_FOLLOWED`] a reference reads as absent. +fn followed<'o>(doc: &Document, o: &'o Object) -> Held<'o> { + match o { + Object::Ref(_) => following(Held::Direct(&Object::Null), || doc.held(o)), + o => Held::Direct(o), + } +} + +fn bytes_of(doc: &Document, d: &Dict, key: &[u8]) -> Vec { + d.get(key).map(|v| doc.resolve(v)).and_then(|v| v.as_string().map(|s| s.bytes.clone())).unwrap_or_default() +} + +/// Read the handler parameters from an `/Encrypt` dictionary. +pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { + let int = |k: &[u8], default: i64| d.get(k).map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(default); + let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec)); + let mut crypt_filters = Vec::new(); + if let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) { + for (k, v) in cf.iter() { + let Some(f) = doc.resolve(v).as_dict().cloned() else { continue }; + let method = match name(&f, b"CFM").as_deref() { + Some(b"V2") => Method::Rc4, + Some(b"AESV2") => Method::Aes128, + Some(b"AESV3") => Method::Aes256, + _ => Method::Identity, + }; + crypt_filters.push((k.clone(), method)); + } + } + let v = int(b"V", 0); + // V4: the key length comes from the standard crypt filter's /Length (bytes, or bits in + // some files); /Length on the dictionary is often a stale 40. A length that isn't + // positive means nothing and reads as if absent. + let cf_len = d + .get(b"CF") + .map(|c| doc.resolve(c)) + .and_then(|c| c.as_dict().and_then(|c| c.get(b"StdCF").map(|f| doc.resolve(f)))) + .and_then(|f| f.as_dict().and_then(|f| f.get(b"Length").map(|n| doc.resolve(n)))) + .and_then(|n| n.as_int()) + .filter(|n| *n > 0); + let length_bits = match (v, cf_len) { + (5.., _) => 256, + (4, Some(n)) if n <= 32 => n * 8, + (4, Some(n)) => n, + (4, None) => 128, + _ => int(b"Length", 40), + }; + EncryptDict { + filter: name(d, b"Filter").unwrap_or_default(), + v, + r: int(b"R", 0), + length_bits, + o: bytes_of(doc, d, b"O"), + u: bytes_of(doc, d, b"U"), + oe: bytes_of(doc, d, b"OE"), + ue: bytes_of(doc, d, b"UE"), + perms: bytes_of(doc, d, b"Perms"), + p: int(b"P", -1) as i32, + encrypt_metadata: !matches!(d.get(b"EncryptMetadata").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(false))), + crypt_filters, + stm_f: name(d, b"StmF").unwrap_or_else(|| b"Identity".to_vec()), + str_f: name(d, b"StrF").unwrap_or_else(|| b"Identity".to_vec()), + ef_f: name(d, b"EFF").unwrap_or_default(), + } +} + +/// Whether `d`, an `/Encrypt` dictionary, has crypt filters that encrypt (`/CFM` `V2`, `AESV2` or +/// `AESV3`), each authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO +/// 32000-2 §7.6.6, Table 25): what a document asks for its embedded files alone, which it opens +/// without (§7.6.6: "EFOpen Authorization shall be required when accessing embedded files"). The +/// files use them through `/EFF` or through a `/Crypt` filter of their own. Its strings and streams +/// have to be in the clear too, which [`SecurityHandler::without_key`] checks. +pub(crate) fn efopen_only(doc: &Document, d: &Dict) -> bool { + let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec)); + let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) else { return false }; + let mut encrypting = Vec::new(); + for (k, v) in cf.iter() { + let Some(f) = doc.resolve(v).as_dict().cloned() else { continue }; + if !matches!(name(&f, b"CFM").as_deref(), Some(b"V2" | b"AESV2" | b"AESV3")) { + continue; + } + if name(&f, b"AuthEvent").as_deref() != Some(b"EFOpen") { + return false; + } + encrypting.push(k.clone()); + } + !encrypting.is_empty() +} + +/// Whether `s`, read from a document whose handler has no file key, holds data that handler would +/// decrypt, which therefore reads as stored: still encrypted. +pub(crate) fn stream_reads_as_stored(doc: &Document, h: &SecurityHandler, s: &Stream) -> bool { + if h.has_key() { + return false; + } + let _scope = FollowScope::enter(); + let named = named_crypt_filter(doc, &s.dict); + let ty = s.dict.get(b"Type").map(|t| followed(doc, t)); + let kind = match (&named, ty.as_deref().and_then(Object::as_name)) { + (Some(n), _) => StreamKind::Named(n), + (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, + _ => StreamKind::Normal, + }; + h.decrypts(kind) +} + +/// Whether `f`, an entry of a `/Filter` chain or its only filter, is `/Crypt`. +fn is_crypt(doc: &Document, f: &Object) -> bool { + followed(doc, f).as_name() == Some(b"Crypt") +} + +/// The crypt filter a stream names in its `/Filter` chain (`/Crypt` + `/DecodeParms /Name`), each +/// given directly or by reference, in an array or not. A chain longer than any decoder takes names +/// none: nothing can read that stream anyway, and every stream of a file can share one long array. +/// A `/Name` that reads as absent means `/Identity`, unless a reference on the way to it was refused +/// (see [`MAX_FOLLOWED`]): that says nothing about the stream, so it names none, and the document's +/// own filter is used. +fn named_crypt_filter(doc: &Document, dict: &Dict) -> Option> { + let refused = REFUSED.get(); + let filter = followed(doc, dict.get(b"Filter")?); + let i = match &*filter { + Object::Array(a) if a.len() <= MAX_CHAIN + 1 => a.iter().position(|f| is_crypt(doc, f))?, + f if is_crypt(doc, f) => 0, + _ => return None, + }; + let parms = dict.get(b"DecodeParms").map(|p| followed(doc, p)); + let entry = match parms.as_deref() { + Some(Object::Array(a)) => a.get(i).map(|p| followed(doc, p)), + Some(p) if i == 0 => Some(Held::Direct(p)), + _ => None, + }; + let name = entry.as_deref().and_then(Object::as_dict).and_then(|p| p.get(b"Name")).map(|n| followed(doc, n)); + match name.as_deref().and_then(Object::as_name) { + Some(name) => Some(name.to_vec()), + // /Identity here would read ciphertext as data, and write the stream in the clear. + None if REFUSED.get() > refused => None, + None => Some(b"Identity".to_vec()), + } +} + +/// A signature dictionary (field signature, document timestamp or usage-rights signature), +/// whose `/Contents` is never encrypted (ISO 32000-2 §7.6.2, §12.8.1): the signer writes a +/// placeholder, hashes the file around it and patches the value in place, so encrypting it +/// would make signing impossible. Identified as other readers do: by `/Type`, or by a string +/// `/Contents` alongside an array `/ByteRange` (`/Type` is optional). A signature dictionary +/// has no `/Subtype`; a dictionary with one (an annotation, whose `/Contents` is the comment +/// text) or with another `/Type` is never exempt, whatever else it carries. +/// With `doc` its entries are read through references; a dictionary nested in another object +/// has none (see [`transform`]). +fn is_signature_dict(doc: Option<&Document>, d: &Dict) -> bool { + // All this decides is what happens to such a /Contents. + if !matches!(d.get(b"Contents"), Some(Object::String(_))) { + return false; + } + // A null entry, or one that refers to nothing, is an absent one (ISO 32000-2 §7.3.7, §7.3.10). + let entry = |key: &[u8]| { + let value = d.get(key)?; + let held = doc.map_or(Held::Direct(value), |doc| followed(doc, value)); + (!matches!(*held, Object::Null)).then_some(held) + }; + if entry(b"Subtype").is_some() { + return false; + } + match entry(b"Type") { + Some(t) => matches!(t.as_name(), Some(b"Sig" | b"DocTimeStamp")), + None => matches!(entry(b"ByteRange").as_deref(), Some(Object::Array(_))), + } +} + +/// Longer than any filter's name. A name can be of any length, and a chain every stream shares can +/// hold a long one. +const MAX_FILTER_NAME: usize = 127; + +/// The dictionary of `source` without the `/Crypt` entries of its filter chain and their decode +/// parameters. The two come out direct when given by reference, which every stream of a file can +/// share, so of what was shared only small values are copied: filter names, and the parameters +/// decoders read ([`Document::with_direct_parms`]). The chain is short, as [`named_crypt_filter`] +/// found a filter in it. +fn drop_crypt_filter(doc: &Document, source: &Stream) -> Dict { + let entry = |f: &Object| match f { + f if is_crypt(doc, f) => Object::name("Crypt"), + Object::Name(n) if n.len() <= MAX_FILTER_NAME => f.clone(), + Object::Ref(_) => f.clone(), + _ => Object::Null, + }; + let filter = source.dict.get(b"Filter").map(|f| followed(doc, f)); + let chain: Vec = match filter.as_deref() { + Some(Object::Array(a)) => a.iter().map(entry).collect(), + Some(f) => vec![entry(f)], + None => Vec::new(), + }; + let crypt: Vec = chain.iter().map(|f| f.as_name() == Some(b"Crypt")).collect(); + let mut chained = Stream::from_raw(source.dict.clone(), Vec::new()); + chained.dict.set(b"Filter".to_vec(), Object::Array(chain.clone())); + // A lone parameter dictionary belongs to the first filter (as `named_crypt_filter` reads it). + let parms = match doc.with_direct_parms(&chained).dict.get(b"DecodeParms") { + Some(Object::Array(p)) => Some(Object::Array(p.iter().zip(&crypt).filter(|(_, c)| !**c).map(|(p, _)| p.clone()).collect())), + Some(Object::Dict(_)) if crypt.first() == Some(&true) => None, + _ => source.dict.get(b"DecodeParms").cloned(), + }; + let mut dict = source.dict.clone(); + match parms { + Some(p) => dict.set(b"DecodeParms".to_vec(), p), + None => { + dict.remove(b"DecodeParms"); + } + } + let filters: Vec = chain.into_iter().zip(&crypt).filter(|(_, c)| !**c).map(|(f, _)| f).collect(); + if filters.is_empty() { + dict.remove(b"Filter"); + if matches!(dict.get(b"DecodeParms"), Some(Object::Array(p)) if p.is_empty()) { + dict.remove(b"DecodeParms"); + } + } else { + dict.set(b"Filter".to_vec(), Object::Array(filters)); + } + dict +} + +/// What [`transform`] does to an object. +#[derive(Clone, Copy, PartialEq, Eq)] +pub(crate) enum Pass { + /// Decrypt, with the handler the document was opened with. + Decrypt, + /// Encrypt with the handler the document was opened with: streams keep their crypt filters. + Encrypt, + /// Encrypt with protection applied by `set_encryption`. A stream's crypt filter came from + /// the source's `/Encrypt`; unless the new one defines that name it is dropped, so the + /// stream follows the new policy instead of reading as `/Identity` and staying in clear. + EncryptReplacing, +} + +/// Decrypt or encrypt every string and stream in an indirect object, except a signature +/// dictionary's `/Contents` (see [`is_signature_dict`]). +/// +/// `o` is as `doc` holds it: what a stream's dictionary gives by reference is read from `doc`, so +/// its references have to be the document's, not a save's renumbered ones. +pub(crate) fn transform(doc: &Document, h: &SecurityHandler, o: &Object, num: u32, generation: u16, pass: Pass) -> Object { + let _scope = FollowScope::enter(); + let decrypt = pass == Pass::Decrypt; + let strings = |s: &PdfString| { + let bytes = if decrypt { h.decrypt_string(num, generation, &s.bytes) } else { h.encrypt_string(num, generation, &s.bytes) }; + PdfString { bytes, hex: s.hex || !decrypt } + }; + // References are followed here only, for the object itself, never for what is nested in it. + // `walk` goes deep, and a load from down there adds that depth again for each object loaded, + // which overflowed the stack on a 10 KB file. A stream can't be nested, and a dictionary that + // is reads as it says directly. + match o { + Object::Stream(s) => stream(doc, s, &strings, h, num, generation, pass), + Object::Dict(d) if is_signature_dict(Some(doc), d) => signature_dict(d, &strings), + o => walk(o, &strings), + } +} + +/// `d`, a signature dictionary, with the strings of everything but its `/Contents` transformed. +fn signature_dict(d: &Dict, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { + Object::Dict( + d.iter() + .map(|(k, v)| match (k.as_slice(), v) { + (b"Contents", Object::String(_)) => (k.clone(), v.clone()), + _ => (k.clone(), walk(v, strings)), + }) + .collect(), + ) +} + +fn walk(o: &Object, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { + match o { + Object::String(s) => Object::String(strings(s)), + Object::Array(a) => Object::Array(a.iter().map(|x| walk(x, strings)).collect()), + Object::Dict(d) if is_signature_dict(None, d) => signature_dict(d, strings), + Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect()), + other => other.clone(), + } +} + +fn stream( + doc: &Document, + s: &Stream, + strings: &dyn Fn(&PdfString) -> PdfString, + h: &SecurityHandler, + num: u32, + generation: u16, + pass: Pass, +) -> Object { + let decrypt = pass == Pass::Decrypt; + let walked = |d: &Dict| -> Dict { d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect() }; + let ty = s.dict.get(b"Type").map(|t| followed(doc, t)); + let ty = ty.as_deref().and_then(Object::as_name); + // Cross-reference streams are never encrypted (§7.5.8), but only reading may trust the + // label: a stream a producer calls /XRef stays plaintext if writing believes it, and the + // writer builds its own cross-reference streams outside this transform. + if (decrypt && ty == Some(b"XRef")) || (ty == Some(b"Metadata") && !h.encrypts_metadata()) { + return Object::Stream(Stream { dict: walked(&s.dict), raw: s.raw.clone() }); + } + let mut named = named_crypt_filter(doc, &s.dict); + // Before the walk, which then encrypts what is in the dictionary this leaves. + let foreign = pass == Pass::EncryptReplacing && named.as_deref().is_some_and(|n| !h.dict().crypt_filters.iter().any(|(f, _)| f == n)); + let dropped = foreign.then(|| drop_crypt_filter(doc, s)); + if dropped.is_some() { + named = None; + } + let dict = walked(dropped.as_ref().unwrap_or(&s.dict)); + let kind = match (&named, ty) { + (Some(n), _) => StreamKind::Named(n), + (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, + _ => StreamKind::Normal, + }; + let raw = if decrypt { h.decrypt_stream(num, generation, &s.raw, kind) } else { h.encrypt_stream(num, generation, &s.raw, kind) }; + Object::Stream(Stream { dict, raw: raw.into() }) +} diff --git a/crates/pdf/src/cos/testkit.rs b/crates/pdf/src/cos/testkit.rs new file mode 100644 index 000000000..44e07b702 --- /dev/null +++ b/crates/pdf/src/cos/testkit.rs @@ -0,0 +1,358 @@ +//! Synthetic PDF fixtures for the object layer's tests: one way to write the bytes, so tests +//! differ only where they mean to (from PdfCraft's `pdfcraft-testkit`). +//! +//! [`Pdf`] writes the envelope around the object bodies a test gives it: object numbers, offsets, +//! a classic cross-reference table and a trailer. It hands back bytes rather than a parsed +//! document, so a test checks the parser rather than the builder. +//! +//! ```text +//! let bytes = Pdf::new() +//! .obj("<< /Type /Catalog /Pages 2 0 R >>") +//! .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") +//! .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 300] /Contents 4 0 R >>") +//! .stream("", b"BT /F1 12 Tf (hi) Tj ET") +//! .build(); +//! ``` +//! +//! Test scaffolding: compiled into the unit tests (`cfg(test)`) and into the `cos` integration +//! test binary, never into the library. +#![allow(dead_code)] + +/// A small PDF with a classic cross-reference table, built one object at a time. +/// +/// Objects are numbered from 1 in the order they are added, all at generation 0, so the first +/// object added is `1 0 R`. The trailer names `1 0 R` as `/Root`, which is what every caller of +/// this builder wants; [`Pdf::trailer_entry`] adds the rest. +#[derive(Debug, Clone)] +pub struct Pdf { + version: String, + objects: Vec>, + trailer: Vec, +} + +impl Default for Pdf { + fn default() -> Self { + Self::new() + } +} + +impl Pdf { + /// An empty PDF 1.7 file. Add objects with [`Pdf::obj`]. + pub fn new() -> Self { + Self { version: "1.7".to_owned(), objects: Vec::new(), trailer: Vec::new() } + } + + /// Set the header version, as it appears after `%PDF-` (for example `"1.4"`). + #[must_use] + pub fn version(mut self, version: &str) -> Self { + self.version = version.to_owned(); + self + } + + /// Append one object body — everything between `N 0 obj` and `endobj`. + #[must_use] + pub fn obj(mut self, body: impl AsRef<[u8]>) -> Self { + self.objects.push(body.as_ref().to_vec()); + self + } + + /// Append several object bodies, in order. + #[must_use] + pub fn objs, T: AsRef<[u8]>>(mut self, bodies: I) -> Self { + self.objects.extend(bodies.into_iter().map(|b| b.as_ref().to_vec())); + self + } + + /// Append a stream object: `dict` holds any entries besides `/Length`, which is computed. + #[must_use] + pub fn stream(self, dict: &str, data: impl AsRef<[u8]>) -> Self { + self.obj(stream_obj(dict, data)) + } + + /// Add an entry to the trailer dictionary, such as `"/Info 9 0 R"`. `/Size` and `/Root` are + /// always written. + #[must_use] + pub fn trailer_entry(mut self, entry: &str) -> Self { + self.trailer.push(entry.to_owned()); + self + } + + /// The number the next object added will get, so a caller can refer forward to it. + /// + /// ``` + /// # use crate::cos::testkit::Pdf; + /// let mut p = Pdf::new().obj("<< /Type /Catalog /Pages 2 0 R >>"); + /// assert_eq!(p.next_obj(), 2); + /// ``` + pub fn next_obj(&self) -> usize { + self.objects.len().saturating_add(1) + } + + /// Serialise the file. + pub fn build(&self) -> Vec { + let mut out = format!("%PDF-{}\n", self.version).into_bytes(); + let mut offsets = Vec::with_capacity(self.objects.len()); + for (i, body) in self.objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i.saturating_add(1)).as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + } + let startxref = out.len(); + let size = self.objects.len().saturating_add(1); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); + for offset in offsets { + out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + let extra = self.trailer.iter().map(|e| format!(" {e}")).collect::(); + out.extend_from_slice(format!("trailer\n<< /Size {size} /Root 1 0 R{extra} >>\nstartxref\n{startxref}\n%%EOF\n").as_bytes()); + out + } +} + +/// A stream object body with `/Length` filled in, for callers assembling object lists themselves. +/// +/// `dict` holds any entries besides `/Length`; an empty one gives `<< /Length N >>` rather than +/// the doubled space the hand-rolled copies produced, which parsers accept but readers trip over. +pub fn stream_obj(dict: &str, data: impl AsRef<[u8]>) -> Vec { + let data = data.as_ref(); + let dict = dict.trim(); + let sep = if dict.is_empty() { "" } else { " " }; + let mut out = format!("<<{sep}{dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + out.extend_from_slice(data); + out.extend_from_slice(b"\nendstream"); + out +} + +/// A three-page file (200, 300 and 400 points wide, in that order, the second with a link to +/// `https://example.com/`) whose page tree sits in an object stream of its first revision, plus an update that rewrites the catalog and the first +/// page in a cross-reference stream: with `/Prev` when `linked`, without it otherwise (the shape +/// of the signed PDFBOX-2720 and PDFBOX-2942 files). Without `/Prev` a reader that follows the +/// chain (ISO 32000-2 §7.5.6, §7.5.8.2) finds no page tree, and a scan for page dictionaries +/// outside object streams finds only the rewritten page. +pub fn update_without_prev(linked: bool) -> Vec { + let (mut out, eight) = unlinked_original(); + // The update: the catalog and the first page again, and a section for them and itself. + let (catalog, first) = unlinked_rewrites(&mut out); + let nine = out.len(); + let mut rows = Vec::new(); + xref_row(&mut rows, 1, catalog, 0); + xref_row(&mut rows, 1, first, 0); + xref_row(&mut rows, 1, nine, 0); + let prev = if linked { format!(" /Prev {eight}") } else { String::new() }; + out.extend_from_slice( + format!("9 0 obj\n<< /Type /XRef /Size 11 /Root 1 0 R /Index [1 1 3 1 9 1] /W [1 4 2]{prev} /Length {} >>\nstream\n", rows.len()).as_bytes(), + ); + out.extend_from_slice(&rows); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{nine}\n%%EOF\n").as_bytes()); + out +} + +/// [`update_without_prev`] with the update written as a hybrid-reference section (ISO 32000-2 +/// §7.5.8.4): a classic table whose trailer names a cross-reference stream with `/XRefStm`. That +/// stream belongs to the same update; it is not an earlier section, so without `/Prev` the page +/// tree is cut off just the same. +pub fn hybrid_update_without_prev(linked: bool) -> Vec { + let (mut out, eight) = unlinked_original(); + let (catalog, first) = unlinked_rewrites(&mut out); + // The update's cross-reference stream, locating only itself. + let nine = out.len(); + let mut rows = Vec::new(); + xref_row(&mut rows, 1, nine, 0); + out.extend_from_slice(format!("9 0 obj\n<< /Type /XRef /Size 11 /Index [9 1] /W [1 4 2] /Length {} >>\nstream\n", rows.len()).as_bytes()); + out.extend_from_slice(&rows); + out.extend_from_slice(b"\nendstream\nendobj\n"); + let table = out.len(); + out.extend_from_slice(format!("xref\n1 1\n{catalog:010} 00000 n \n3 1\n{first:010} 00000 n \n").as_bytes()); + let prev = if linked { format!(" /Prev {eight}") } else { String::new() }; + out.extend_from_slice(format!("trailer\n<< /Size 11 /Root 1 0 R /XRefStm {nine}{prev} >>\nstartxref\n{table}\n%%EOF\n").as_bytes()); + out +} + +/// One row of a cross-reference stream with `/W [1 4 2]`. +fn xref_row(out: &mut Vec, kind: u8, field: usize, index: u16) { + out.push(kind); + out.extend_from_slice(&u32::try_from(field).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(&index.to_be_bytes()); +} + +/// The update both fixtures append: the catalog and the first page again. Their offsets. +fn unlinked_rewrites(out: &mut Vec) -> (usize, usize) { + let catalog = out.len(); + out.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R /Lang (en) >>\nendobj\n"); + let first = out.len(); + out.extend_from_slice(b"3 0 obj\n<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Annots [] >>\nendobj\n"); + (catalog, first) +} + +/// The first revision of [`update_without_prev`], and the offset of its cross-reference stream. +fn unlinked_original() -> (Vec, usize) { + let row = xref_row; + let page = |width: u32, extra: &str| format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {width} 200] /Contents 4 0 R{extra} >>"); + let mut out = b"%PDF-1.7\n".to_vec(); + let one = out.len(); + out.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); + let four = out.len(); + out.extend_from_slice(b"4 0 obj\n<< /Length 14 >>\nstream\n0 0 50 50 re f\nendstream\nendobj\n"); + // Objects 2 (the page tree), 3, 6 and 7 (the pages) and 10 (the link) in object stream 5. + let members = [ + (2, "<< /Type /Pages /Kids [3 0 R 6 0 R 7 0 R] /Count 3 >>".to_string()), + (3, page(200, "")), + (6, page(300, " /Annots [10 0 R]")), + (7, page(400, "")), + (10, "<< /Type /Annot /Subtype /Link /Rect [10 10 60 60] /A << /S /URI /URI (https://example.com/) >> >>".to_string()), + ]; + let (mut header, mut body) = (String::new(), String::new()); + for (num, member) in &members { + header.push_str(&format!("{num} {} ", body.len())); + body.push_str(member); + body.push(' '); + } + let five = out.len(); + out.extend_from_slice( + format!( + "5 0 obj\n<< /Type /ObjStm /N 5 /First {} /Length {} >>\nstream\n{header}{body}\nendstream\nendobj\n", + header.len(), + header.len() + body.len() + ) + .as_bytes(), + ); + let eight = out.len(); + let mut rows = Vec::new(); + row(&mut rows, 0, 0, 65535); + row(&mut rows, 1, one, 0); + row(&mut rows, 2, 5, 0); + row(&mut rows, 2, 5, 1); + row(&mut rows, 1, four, 0); + row(&mut rows, 1, five, 0); + row(&mut rows, 2, 5, 2); + row(&mut rows, 2, 5, 3); + row(&mut rows, 1, eight, 0); + row(&mut rows, 0, 0, 0); + row(&mut rows, 2, 5, 4); + out.extend_from_slice(format!("8 0 obj\n<< /Type /XRef /Size 11 /Root 1 0 R /W [1 4 2] /Length {} >>\nstream\n", rows.len()).as_bytes()); + out.extend_from_slice(&rows); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{eight}\n%%EOF\n").as_bytes()); + (out, eight) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The serialiser the workspace had 111 copies of, kept here verbatim as the thing [`Pdf`] has + /// to agree with. Migrating a crate to the builder must not change a single byte of its + /// fixtures, or the test that fixture feeds silently starts testing something else. + fn hand_rolled(version: &str, objs: &[Vec], trailer_extra: &str) -> Vec { + let mut out = format!("%PDF-{version}\n").into_bytes(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R{trailer_extra} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out + } + + fn sample_objects() -> Vec> { + vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /Contents 4 0 R >>".to_vec(), + stream_obj("", b"BT /F1 24 Tf 20 150 Td (Page 1) Tj ET"), + ] + } + + #[test] + fn the_builder_is_byte_identical_to_the_serialiser_it_replaces() { + let objs = sample_objects(); + for version in ["1.4", "1.7", "2.0"] { + let built = Pdf::new().version(version).objs(objs.clone()).build(); + assert_eq!(built, hand_rolled(version, &objs, ""), "version {version}"); + } + // ...including the trailer entries the copies varied on. + let built = Pdf::new().objs(objs.clone()).trailer_entry("/Info 5 0 R").build(); + assert_eq!(built, hand_rolled("1.7", &objs, " /Info 5 0 R")); + } + + /// Offsets must point at the `N 0 obj` that starts each object, counted from the file's first + /// byte. A fixture whose table is wrong tests the repair path, not the path it meant to. + #[test] + fn every_offset_in_the_table_points_at_its_object() { + let bytes = Pdf::new().objs(sample_objects()).build(); + let text = String::from_utf8_lossy(&bytes); + let table = text.split("xref\n0 ").nth(1).expect("an xref table"); + let rows: Vec<&str> = table.lines().skip(1).take_while(|l| l.ends_with(" n ") || l.ends_with(" f ")).collect(); + assert_eq!(rows.len(), 5, "free entry plus four objects: {rows:?}"); + for (i, row) in rows.iter().enumerate().skip(1) { + let offset: usize = row.split_whitespace().next().and_then(|n| n.parse().ok()).expect("an offset"); + let expected = format!("{i} 0 obj\n"); + assert!(bytes.get(offset..).is_some_and(|rest| rest.starts_with(expected.as_bytes())), "object {i} at {offset}"); + } + } + + /// `/Size` is one more than the object count (object 0 is the free head), and `startxref` + /// points at the table. + #[test] + fn the_trailer_agrees_with_the_body() { + let bytes = Pdf::new().objs(sample_objects()).build(); + let text = String::from_utf8_lossy(&bytes); + assert!(text.contains("/Size 5"), "{text}"); + let startxref: usize = text.rsplit("startxref\n").next().and_then(|t| t.lines().next()).and_then(|n| n.parse().ok()).expect("startxref"); + assert!(bytes.get(startxref..).is_some_and(|rest| rest.starts_with(b"xref\n")), "startxref {startxref}"); + } + + #[test] + fn stream_length_counts_the_data_not_the_keywords() { + let obj = stream_obj("/Filter /FlateDecode", b"abc"); + assert_eq!(obj, b"<< /Filter /FlateDecode /Length 3 >>\nstream\nabc\nendstream".to_vec()); + // Binary data, including NUL and bytes that are not UTF-8, survives untouched. + let data = [0u8, 0xFF, b'\n', 0x80]; + let obj = stream_obj("", data); + assert!(obj.ends_with(b"stream\n\0\xff\n\x80\nendstream"), "{obj:?}"); + // An empty dict gives one space, not two. + assert!(obj.starts_with(b"<< /Length 4 >>"), "{obj:?}"); + assert_eq!(stream_obj(" /Type /XObject ", b""), stream_obj("/Type /XObject", b"")); + } + + /// An empty file is still a parseable shape rather than something that panics or truncates. + #[test] + fn a_file_with_no_objects_is_still_well_formed() { + let bytes = Pdf::new().build(); + let text = String::from_utf8_lossy(&bytes); + assert!(text.starts_with("%PDF-1.7\n"), "{text}"); + assert!(text.contains("xref\n0 1\n0000000000 65535 f \n"), "{text}"); + assert!(text.contains("/Size 1 /Root 1 0 R"), "{text}"); + assert!(text.ends_with("%%EOF\n"), "{text}"); + } + + #[test] + fn next_obj_tracks_what_has_been_added() { + let p = Pdf::new(); + assert_eq!(p.next_obj(), 1); + let p = p.obj("<< >>").obj("<< >>"); + assert_eq!(p.next_obj(), 3); + // Forward references are the reason this exists: the page can name a stream not yet added. + let n = p.next_obj(); + let bytes = p.obj(format!("<< /Contents {n} 0 R >>")).build(); + assert!(String::from_utf8_lossy(&bytes).contains("/Contents 3 0 R")); + } + + /// Hostile and awkward object bodies must not panic or corrupt the offsets. + #[test] + fn odd_object_bodies_keep_the_table_honest() { + let bodies: Vec> = vec![Vec::new(), b"\n".to_vec(), vec![0u8; 1024], "<< /T (\u{1F4A5}) >>".as_bytes().to_vec(), b"endobj".to_vec()]; + let bytes = Pdf::new().objs(bodies.clone()).build(); + assert_eq!(bytes, hand_rolled("1.7", &bodies, "")); + // The offsets still land on their objects even when a body contains `endobj` itself. + let text = String::from_utf8_lossy(&bytes); + assert!(text.contains("/Size 6"), "{text}"); + } +} diff --git a/crates/pdf/src/cos/writer.rs b/crates/pdf/src/cos/writer.rs new file mode 100644 index 000000000..4d8981dd5 --- /dev/null +++ b/crates/pdf/src/cos/writer.rs @@ -0,0 +1,1214 @@ +//! Serialization and saving (ISO 32000-2 §7.5). +//! +//! * `write_incremental` appends the edited objects, a cross-reference section in the same style +//! as the file's last one (table or stream) and a trailer with `/Prev`. The original bytes are +//! never modified, which keeps digital signatures over earlier revisions valid. With +//! `SaveOptions::drop_unlinked`, objects added since the file was read that nothing refers to +//! any more are left out (written as free). +//! * `write_full` writes only objects reachable from the trailer, renumbered densely, with a +//! cross-reference table or stream — the "Save As" / garbage-collecting path. Unknown trailer +//! entries are copied, including objects reachable only from them. + +use std::borrow::Cow; +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::io::Write; +use std::sync::Arc; + +use crate::cos::CosError; +use crate::cos::document::{Document, ObjectReader, XrefEntry, held_bytes}; +use crate::cos::object::{Dict, ObjRef, Object, PdfString, Stream}; + +/// Serialize one object. Deterministic: same object → same bytes. +pub fn serialize(o: &Object, out: &mut Vec) { + match o { + Object::Null => out.extend_from_slice(b"null"), + Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), + Object::Int(i) => write_int(*i, out), + Object::Real(r) => write_real(*r, out), + Object::String(s) => write_string(s, out), + Object::Name(n) => write_name(n, out), + Object::Array(a) => { + out.push(b'['); + for (i, x) in a.iter().enumerate() { + if i > 0 { + out.push(b' '); + } + serialize(x, out); + } + out.push(b']'); + } + Object::Dict(d) => write_dict(d, out), + Object::Stream(s) => write_stream(s, out), + Object::Ref(r) => { + write_digits(u64::from(r.num), out); + out.push(b' '); + write_digits(u64::from(r.generation), out); + out.extend_from_slice(b" R"); + } + } +} + +/// Upper-case hexadecimal digits. +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. +fn write_digits(mut v: u64, out: &mut Vec) { + // `u64::MAX` has 20 digits, so `at` never goes below 0. + let mut buf = [0u8; 20]; + let mut at = buf.len(); + loop { + at -= 1; + buf[at] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { + break; + } + } + out.extend_from_slice(&buf[at..]); +} + +/// A signed integer in decimal, as `write!(out, "{i}")` writes it. +fn write_int(i: i64, out: &mut Vec) { + if i < 0 { + out.push(b'-'); + } + write_digits(i.unsigned_abs(), out); +} + +/// `b` as two upper-case hexadecimal digits. +fn hex_pair(b: u8) -> [u8; 2] { + [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] +} + +fn write_real(r: f64, out: &mut Vec) { + if !r.is_finite() { + out.push(b'0'); + return; + } + // A whole number is written as an integer only within the integer range Annex C gives + // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. + if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { + write_int(i64::from(r as i32), out); + return; + } + // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the + // shortest string that parses back to the same value, never in exponent form, so values + // round-trip exactly (rounding to a fixed number of places visibly shifted shading colours). + let s = format!("{r}"); + out.extend_from_slice(if s == "-0" { b"0" } else { s.as_bytes() }); + // A whole number past that range prints without a point; the trailing `.` keeps it a real. + if !s.contains('.') && s != "-0" { + out.push(b'.'); + } +} + +fn write_name(n: &[u8], out: &mut Vec) { + out.push(b'/'); + for &b in n { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + out.push(b); + } else { + out.push(b'#'); + out.extend_from_slice(&hex_pair(b)); + } + } +} + +fn write_string(s: &PdfString, out: &mut Vec) { + let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + if s.hex || binary * 4 > s.bytes.len().max(1) { + out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); + out.push(b'<'); + for &b in &s.bytes { + out.extend_from_slice(&hex_pair(b)); + } + out.push(b'>'); + return; + } + out.push(b'('); + for &b in &s.bytes { + match b { + b'(' | b')' | b'\\' => { + out.push(b'\\'); + out.push(b); + } + b'\n' => out.extend_from_slice(b"\\n"), + b'\r' => out.extend_from_slice(b"\\r"), + b'\t' => out.extend_from_slice(b"\\t"), + 0x08 => out.extend_from_slice(b"\\b"), + 0x0C => out.extend_from_slice(b"\\f"), + b' '..=b'~' => out.push(b), + _ => { + out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); + } + } + } + out.push(b')'); +} + +fn write_dict(d: &Dict, out: &mut Vec) { + out.extend_from_slice(b"<<"); + for (k, v) in d.iter() { + write_name(k, out); + out.push(b' '); + serialize(v, out); + } + out.extend_from_slice(b">>"); +} + +fn write_stream(s: &Stream, out: &mut Vec) { + let mut d = s.dict.clone(); + d.set(b"Length".to_vec(), Object::Int(s.raw.len() as i64)); + write_dict(&d, out); + out.extend_from_slice(b"\nstream\n"); + out.extend_from_slice(&s.raw); + out.extend_from_slice(b"\nendstream"); +} + +/// A document opened without the password its embedded files need holds their data as stored and +/// can't encrypt: a save keeps its `/Encrypt` as it is, and writes no edited or added stream that +/// would have to be encrypted (an attached file put in, say), which would be written in the clear. +fn keyless_check(doc: &Document) -> Result<(), CosError> { + if !doc.embedded_files_locked() { + return Ok(()); + } + let refuse = + |what: &str| Err(CosError::Security(format!("{what} without the password of the document's embedded files: open it with the password"))); + if doc.encryption_changed() || doc.trailer().get(b"Encrypt").is_none() { + return refuse("the document's security can't change"); + } + if doc.overlay_entries().any(|(_, _, o)| matches!(o.as_deref(), Some(Object::Stream(s)) if doc.reads_as_stored(s))) { + return refuse("an edited or added embedded file can't be encrypted"); + } + Ok(()) +} + +/// The object as written: encrypted with its (output) number when the document is encrypted, +/// except the `/Encrypt` dictionary itself (`source_num` is its number in the source document). +/// `o` has the document's numbers in its references ([`crate::cos::security::transform`]). +fn prepared<'a>(doc: &Document, source_num: u32, num: u32, generation: u16, o: &'a Object) -> Cow<'a, Object> { + match doc.output_security() { + (Some(h), encrypt_num) if Some(source_num) != encrypt_num && !doc.stored_as_is(source_num) => { + let pass = if doc.encryption_changed() { crate::cos::security::Pass::EncryptReplacing } else { crate::cos::security::Pass::Encrypt }; + Cow::Owned(crate::cos::security::transform(doc, h, o, num, generation, pass)) + } + _ => Cow::Borrowed(o), + } +} + +fn write_indirect(num: u32, generation: u16, o: &Object, out: &mut Vec) { + let _ = writeln!(out, "{num} {generation} obj"); + serialize(o, out); + out.extend_from_slice(b"\nendobj\n"); +} + +/// Options for saving. +#[derive(Clone, Debug)] +pub struct SaveOptions { + /// Replace or add `/Info /ModDate` with this PDF date string (e.g. from `pdf_date`). + pub mod_date: Option, + /// Seed for a new file identifier when the document has none (deterministic in tests). + pub id_seed: u64, + /// Full saves: pack non-stream objects into compressed object streams with a cross-reference + /// stream (PDF 1.5+, §7.5.7–7.5.8). Off writes a classic table that any reader accepts. + pub object_streams: bool, + /// Incremental saves: leave out what was added since the file was read and nothing refers to + /// any more, such as a comment added and deleted again or a redaction mark added and cleared + /// (written as free). Off by default, because a writer may add an object only it finds again + /// by scanning, as a standalone document timestamp is; such a writer frees the leftovers + /// before adding it, with [`Document::free_unlinked_additions`]. Full saves always leave + /// these out. + pub drop_unlinked: bool, + /// Entropy for the file identifier of a rewrite that asks for a new one (see + /// [`Document::require_full_save_with_new_id`]). Given, the identifier depends only on this and + /// on the written bytes, so the same save gives the same file; tests use it to pin output. + /// Without it, fresh entropy from the host (the operating system on native targets, the + /// browser on WebAssembly) is mixed in. + pub id_entropy: Option<[u8; 16]>, +} + +impl Default for SaveOptions { + fn default() -> Self { + Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true, drop_unlinked: false, id_entropy: None } + } +} + +/// Objects per object stream (Acrobat and qpdf use 100–200). +const OBJSTM_SIZE: usize = 100; +/// Keep a batch's serialized body small as well as limiting its object count. One object +/// may exceed this target; it is emitted in its own object stream without retaining a batch. +const OBJSTM_BYTES: usize = 8 * 1024 * 1024; + +/// Where an object lives, for the cross-reference section. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Row { + /// Free, with the generation the number gets when reused. + Free(u16), + /// At a byte offset. + InFile(u64, u16), + /// Inside object stream `stream`, at position `index`. + InStream(u32, u32), +} + +/// Append the document's edits to its original bytes (incremental update, §7.5.6). +/// +/// A document that had to be reconstructed (no usable cross-reference chain) cannot be updated +/// incrementally — other readers would follow the broken chain — so it is rewritten in full, +/// which also repairs the file (what users expect after "the file was damaged and repaired"). +pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + keyless_check(doc)?; + // Reconstructed files have no chain to append to; added or removed encryption must + // rewrite every object; redaction must not leave the old revision behind. All need a full save. + if doc.reconstructed() || doc.encryption_changed() || doc.full_save_required() { + return write_full(doc, opts); + } + let mut doc = doc.clone(); + stamp_mod_date(&mut doc, opts); + let original = doc.bytes().clone(); + // The revision is written on its own and joined to the original at the end, so the result is + // allocated once at its exact size: growing a buffer that holds a large original past a + // guessed capacity would copy the whole file again and keep up to as much again unused. + let newline = !matches!(original.last(), Some(b'\n' | b'\r')); + // A file with bytes before its `%PDF-` header may count its offsets from the header. The new + // section then has to as well: readers take `/Prev` and `startxref` in one convention, and a + // mix sends them into reconstruction (#2364). + let shift = if doc.revisions().last().is_some_and(|r| r.header_relative) { doc.header_offset() } else { 0 }; + let base = (original.len() + usize::from(newline)).saturating_sub(shift) as u64; + let mut out = Vec::new(); + let mut offsets: BTreeMap = BTreeMap::new(); // num → (offset, gen, in use) + let unlinked = if opts.drop_unlinked { unlinked_additions(&doc) } else { HashSet::new() }; + for (num, generation, obj) in doc.overlay_entries() { + match obj { + // Written as free, so `/Size` still covers every number this document handed out. The + // generation stays: the number was never in use, and an undo can bring it back as is. + Some(_) if unlinked.contains(&num) => { + offsets.insert(num, (0, generation, false)); + } + Some(o) => { + offsets.insert(num, (base + out.len() as u64, generation, true)); + write_indirect(num, generation, &prepared(&doc, num, num, generation, &o), &mut out); + } + None => { + offsets.insert(num, (0, generation, false)); + } + } + } + let prev = doc.revisions().last().map(|r| r.xref_offset); + let as_stream = doc.revisions().last().is_some_and(|r| r.is_stream); + let mut trailer = doc.trailer().clone(); + if doc.output_security().0.is_none() { + ensure_id(&mut trailer, opts, &original); // an encrypted file's ID is part of its key + } + if let Some(p) = prev { + trailer.set(b"Prev".to_vec(), Object::Int(p as i64)); + } + let size = doc.next_num().max(trailer.int(b"Size").map_or(0, |n| n.clamp(0, i64::from(u32::MAX)) as u32)); + if as_stream { + let rows = offsets.iter().map(|(n, (o, g, used))| (*n, if *used { Row::InFile(*o, *g) } else { Row::Free(*g) })).collect(); + write_xref_stream(&mut out, base, &mut trailer, rows, size); + } else { + trailer.set(b"Size".to_vec(), Object::Int(size as i64)); + let xref_at = base + out.len() as u64; + write_xref_table(&mut out, &offsets, prev.is_none()); + out.extend_from_slice(b"trailer\n"); + write_dict(&trailer, &mut out); + let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); + } + let mut file = Vec::with_capacity(original.len() + usize::from(newline) + out.len()); + file.extend_from_slice(&original); + if newline { + file.push(b'\n'); + } + file.extend_from_slice(&out); + Ok(file) +} + +/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and +/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets +/// (aliases of one slot included) and, in visit order, whether it is a stream and the parsed +/// object where the budget allowed keeping it. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>, Vec>>); + +/// Number the objects reachable from the trailer. `retain` keeps parsed objects for the write +/// pass, up to the budget. A caller that needs only the numbers passes `false`, so the traversal +/// holds no parsed objects beyond the current one. +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict, retain: bool) -> Result { + // Breadth-first walk from the trailer, assigning new numbers in visit order. + let mut map: HashMap = HashMap::new(); + // Record the type while traversing so classifying packed objects needs no second parse. + let mut order: Vec<(ObjRef, bool)> = Vec::new(); + // The parsed object of each entry in `order`, kept for the write pass while the budget + // allows it. Past the budget the write pass reads the object again. + let mut kept: Vec>> = Vec::new(); + let mut kept_budget = KeptObjects::default(); + let mut queue: VecDeque = VecDeque::new(); + let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); + for key in [&b"Root"[..], b"Info", b"Encrypt"] { + if let Some(r) = trailer_in.reference(key) { + queue.push_back(r); + } + } + // A direct /Encrypt can refer to objects, /CF or /StmF, say. + if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { + visit_refs(encrypt, &mut queue); + } + // Unknown trailer entries are document data. Objects they alone reference must be written + // too, or a full rewrite would keep the key and drop its value. + for (key, value) in trailer_in.iter() { + if !trailer_key_rewritten(key) { + collect_refs(value, &mut |r| queue.push_back(r)); + } + } + while let Some(r) = queue.pop_front() { + if map.contains_key(&r) { + continue; + } + let o = reader.get(r); + if matches!(*o, Object::Null) { + continue; // dangling reference: written as null by renumbering below + } + // Accepted xref/header aliases denote one slot, including page-tree cycles. + // Check for null first: an unrelated or freed generation must remain dangling. + let canonical = ObjRef::new(r.num, reader.document.generation(r.num)); + if let Some(&num) = map.get(&canonical) { + map.insert(r, num); + continue; + } + let num = output_number(order.len())?; + map.insert(r, num); + map.insert(canonical, num); + order.push((canonical, matches!(*o, Object::Stream(_)))); + visit_refs(&o, &mut queue); + kept.push((retain && kept_budget.admit(&o, reader.document.bytes())).then_some(o)); + } + Ok((map, order, kept)) +} + +/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] +/// reaches from the trailer, object streams included. For a check that something removed from a +/// document cannot reach the saved file through a reference nobody looked for. +pub fn reachable_objects(doc: &Document) -> Result, CosError> { + let mut reader = doc.object_reader(); + let trailer_in = reader.document.trailer().clone(); + let (_, order, _) = full_save_order(&mut reader, &trailer_in, false)?; + Ok(order.into_iter().map(|(r, _)| r.num).collect()) +} + +/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. +pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + keyless_check(doc)?; + // Renumbering breaks a key that mixes in the object number. + if doc.security().is_some_and(|h| !h.has_key() && !h.survives_renumbering()) { + return Err(CosError::Security( + "the embedded files are encrypted and the document was opened without their password: open it with the password to save it in full" + .into(), + )); + } + // A full traversal must not fill the editor's shared caches (which undo snapshots also + // own). Parsed objects and decoded object streams are temporary working data here. + let mut reader = doc.object_reader(); + stamp_mod_date(&mut reader.document, opts); + let trailer_in = reader.document.trailer().clone(); + let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in, true)?; + let version = reader.document.version().chars().take(3).collect::(); + let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; + let mut out = Vec::new(); + let _ = writeln!(out, "%PDF-{version}"); + // A binary comment (raw high bytes) marks the file as binary for transfer tools. + out.extend_from_slice(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n']); + // The /Encrypt dictionary and everything it references (e.g. indirect /CF crypt-filter + // dictionaries) are needed before anything can be decrypted: never inside object streams. + let mut encryption_objects: std::collections::HashSet = std::collections::HashSet::new(); + let mut pending: Vec = Vec::new(); + if let Some(encrypt) = trailer_in.get(b"Encrypt") { + collect_refs(encrypt, &mut |r| pending.push(r)); + } + while let Some(r) = pending.pop() { + if let Some(&num) = map.get(&r) + && encryption_objects.insert(num) + { + collect_refs(&reader.get(r), &mut |x| pending.push(x)); + } + } + // Acrobat refuses an encrypted file whose catalog sits in an object stream ("the file is + // damaged and could not be repaired", before it even asks for the password), while other + // readers accept it. qpdf keeps the catalog out of object streams too. Verified with + // Acrobat DC on macOS (#774). + let root_outside = + if reader.document.output_security().0.is_some() { trailer_in.reference(b"Root").and_then(|r| map.get(&r)).copied() } else { None }; + let mut rows: BTreeMap = BTreeMap::new(); + // Keep only references, not renamed copies of the entire object graph. Standalone + // objects still precede object streams, preserving the established output order. + let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); + for (i, (r, is_stream)) in order.iter().enumerate() { + let num = output_number(i)?; + let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(&num) && root_outside != Some(num); + if in_stream { + packed.push((num, *r, kept[i].take())); + } else { + let o = written_object(&mut reader, *r, *is_stream, kept[i].take())?; + let o = renumber(&prepared(&reader.document, r.num, num, 0, &o), &map); + rows.insert(num, Row::InFile(out.len() as u64, 0)); + write_indirect(num, 0, &o, &mut out); + } + } + // Object streams take the numbers after the last object. Their contents are not encrypted + // individually: the whole stream is (§7.5.7, §7.6.2). + let mut next = output_number(order.len())?; + let mut batch = ObjectStreamBatch::default(); + for (num, reference, parsed) in packed { + let mut encoded = Vec::new(); + serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); + if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { + batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; + } + let _ = write!(batch.header, "{} {} ", num, batch.body.len()); + batch.body.extend_from_slice(&encoded); + batch.body.push(b'\n'); + drop(encoded); + rows.insert(num, Row::InStream(next, batch.count as u32)); + batch.count += 1; + if batch.count == OBJSTM_SIZE || batch.body.len() >= OBJSTM_BYTES { + batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; + } + } + batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; + let mut trailer = full_save_trailer(&reader.document, &trailer_in, &map, opts, &out); + if opts.object_streams { + write_xref_stream(&mut out, 0, &mut trailer, rows, next); + } else { + trailer.set(b"Size".to_vec(), Object::Int(next as i64)); + let offsets: BTreeMap = + rows.iter().filter_map(|(n, r)| if let Row::InFile(o, g) = r { Some((*n, (*o, *g, true))) } else { None }).collect(); + let xref_at = out.len(); + write_xref_table(&mut out, &offsets, true); + out.extend_from_slice(b"trailer\n"); + write_dict(&trailer, &mut out); + let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); + } + // The size isn't known until the end, and the buffer grew by doubling. The result is often + // kept as a document's working bytes for as long as it's open: drop the unused capacity. + out.shrink_to_fit(); + Ok(out) +} + +/// What `write_full` keeps of the objects it parsed while numbering them, measured with +/// [`held_bytes`] (the parsed tree and any data that is not a view into the file). Beyond the +/// budget objects are read again when written, which bounds the memory of a save of a huge +/// document. +#[derive(Default)] +struct KeptObjects { + bytes: usize, +} + +impl KeptObjects { + const BYTES: usize = 64 * 1024 * 1024; + + /// Whether `object` still fits the budget; if so it is counted against it. + fn admit(&mut self, object: &Object, data: &Arc>) -> bool { + let bytes = self.bytes.saturating_add(held_bytes(object, data, 0)); + if bytes > Self::BYTES { + return false; + } + self.bytes = bytes; + true + } +} + +/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read +/// again. +fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { + match kept { + Some(o) => Ok(o), + None => reread(reader, r, is_stream), + } +} + +/// Assemble the full rewrite's trailer and apply its identifier policy. +fn full_save_trailer(doc: &Document, trailer_in: &Dict, map: &HashMap, opts: &SaveOptions, out: &[u8]) -> Dict { + let mut trailer = Dict::new(); + for key in [&b"Root"[..], b"Info", b"Encrypt"] { + if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) { + trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0))); + } + } + if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { + trailer.set(b"Encrypt".to_vec(), renumber(encrypt, map)); + } + if let Some(id) = trailer_in.get(b"ID") { + trailer.set(b"ID".to_vec(), direct_id(doc, id)); + } + for (key, value) in trailer_in.iter() { + if !trailer_key_rewritten(key) { + trailer.set(key.clone(), renumber(value, map)); + } + } + if doc.output_security().0.is_none() { + ensure_id(&mut trailer, opts, doc.bytes()); + } + if doc.new_id_required() { + regenerate_id(&mut trailer, opts, out, doc.output_security().0.is_some()); + } + trailer +} + +/// Object `r` read again for writing. The traversal read it once and only kept its number, so +/// a second read that fails, finds nothing or finds a different kind of object must fail the +/// save rather than write `null` in its place. +fn reread(reader: &mut ObjectReader, r: ObjRef, is_stream: bool) -> Result, CosError> { + let o = reader.try_get(r)?; + if matches!(*o, Object::Null) || matches!(*o, Object::Stream(_)) != is_stream { + return Err(CosError::Syntax { offset: 0, detail: format!("object {} read differently while saving; nothing was written", r.num) }); + } + Ok(o) +} + +#[derive(Default)] +struct ObjectStreamBatch { + header: Vec, + body: Vec, + count: usize, +} + +impl ObjectStreamBatch { + fn flush(&mut self, doc: &Document, next: &mut u32, rows: &mut BTreeMap, out: &mut Vec) -> Result<(), CosError> { + if self.count == 0 { + return Ok(()); + } + let following = next.checked_add(1).ok_or_else(|| CosError::Syntax { offset: 0, detail: "too many objects to save".into() })?; + // Release capacity after each batch too: an unusually large object must not leave + // a large scratch buffer alive for the rest of the save. + let Self { mut header, body, count } = std::mem::take(self); + let first = header.len(); + header.extend_from_slice(&body); + drop(body); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("ObjStm")); + d.set(b"N".to_vec(), Object::Int(count as i64)); + d.set(b"First".to_vec(), Object::Int(first as i64)); + let stm = Object::Stream(Stream::flate(d, &header)); + rows.insert(*next, Row::InFile(out.len() as u64, 0)); + write_indirect(*next, 0, &prepared(doc, u32::MAX, *next, 0, &stm), out); + *next = following; + Ok(()) + } +} + +/// `/ID` for the trailer, direct, as it has to be with `/Encrypt`: nothing else in the file holds +/// what it refers to now. Given by reference it is two entries at most, however many references a +/// file gave, so that one large string isn't copied for each. An entry that isn't a string stays in +/// its place as an empty one, which is how the key read it. +fn direct_id(doc: &Document, id: &Object) -> Object { + if matches!(id, Object::Array(a) if !a.iter().any(|s| matches!(s, Object::Ref(_)))) { + return id.clone(); + } + let entry = |s: &Object| match &*doc.resolve(s) { + s @ Object::String(_) => s.clone(), + _ => Object::String(crate::cos::PdfString { bytes: Vec::new(), hex: true }), + }; + let held = doc.resolve(id); + Object::Array(held.as_array().into_iter().flatten().take(2).map(entry).collect()) +} + +/// Trailer keys a full rewrite generates itself. Everything else is copied, with references +/// renumbered, so a save does not drop metadata the reader accepted. +fn trailer_key_rewritten(key: &[u8]) -> bool { + matches!( + key, + b"Size" | b"Prev" | b"XRefStm" | b"Type" | b"W" | b"Index" | b"Filter" | b"DecodeParms" | b"Length" | b"Root" | b"Info" | b"Encrypt" | b"ID" + ) +} + +fn output_number(index: usize) -> Result { + u32::try_from(index).ok().and_then(|n| n.checked_add(1)).ok_or_else(|| CosError::Syntax { offset: 0, detail: "too many objects to save".into() }) +} + +/// Objects added since the file was read (their numbers are not in use in it) that nothing an +/// incremental update writes refers to: what one edit added and a later one unlinked, such as a +/// deleted comment or a cleared redaction mark. The file never held them, so leaving them out +/// loses nothing, while writing them would keep what the user removed. Only the update is walked: +/// from the trailer and the file's changed objects through additions. An unchanged object of the +/// file is not followed: a reference it holds to such a number already reads as null in the file +/// (a damaged file's dangling reference), and it still does. +pub(crate) fn unlinked_additions(doc: &Document) -> HashSet { + let added = |num: u32| matches!(doc.xref_entry(num), None | Some(XrefEntry::Free { .. })); + let mut pending: HashMap> = + doc.overlay_entries().filter_map(|(num, _, o)| Some((num, o?))).filter(|(num, _)| added(*num)).collect(); + if pending.is_empty() { + return HashSet::new(); + } + let mut stack: Vec = Vec::new(); + doc.trailer().iter().for_each(|(_, v)| collect_refs(v, &mut |r| stack.push(r))); + for (num, _, o) in doc.overlay_entries() { + if let Some(o) = o + && !added(num) + { + collect_refs(&o, &mut |r| stack.push(r)); + } + } + while let Some(r) = stack.pop() { + if let Some(o) = pending.remove(&r.num) { + collect_refs(&o, &mut |r| stack.push(r)); + } + } + pending.into_keys().collect() +} + +pub(crate) fn collect_refs(o: &Object, f: &mut impl FnMut(ObjRef)) { + match o { + Object::Ref(r) => f(*r), + Object::Array(a) => a.iter().for_each(|x| collect_refs(x, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| collect_refs(v, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| collect_refs(v, f)), + _ => {} + } +} + +fn renumber(o: &Object, map: &HashMap) -> Object { + match o { + Object::Ref(r) => map.get(r).map(|n| Object::Ref(ObjRef::new(*n, 0))).unwrap_or(Object::Null), + Object::Array(a) => Object::Array(a.iter().map(|x| renumber(x, map)).collect()), + Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), renumber(v, map))).collect()), + Object::Stream(s) => Object::Stream(Stream { dict: s.dict.iter().map(|(k, v)| (k.clone(), renumber(v, map))).collect(), raw: s.raw.clone() }), + other => other.clone(), + } +} + +/// Classic xref table with one subsection per run of consecutive numbers (§7.5.4). +fn write_xref_table(out: &mut Vec, offsets: &BTreeMap, include_zero: bool) { + out.extend_from_slice(b"xref\n"); + let mut all: Vec<(u32, (u64, u16, bool))> = offsets.iter().map(|(k, v)| (*k, *v)).collect(); + if include_zero { + all.insert(0, (0, (0, 65535, false))); + } + // A section is one or more subsections. `xref` followed immediately by `trailer` is not a + // table (qpdf: "xref syntax invalid") and is what an incremental save of a classic-table + // file used to append when nothing was edited. A zero-length subsection names no objects, + // so the free list and every in-use entry stay as they were, while a trailer-only change + // (a missing `/ID`, or a caller edit of the trailer) is still written. + if all.is_empty() { + let _ = writeln!(out, "0 0"); + return; + } + let mut i = 0; + while i < all.len() { + let start = all[i].0; + let mut j = i; + while j + 1 < all.len() && all[j + 1].0 == all[j].0 + 1 { + j += 1; + } + let _ = writeln!(out, "{} {}", start, j - i + 1); + for (_, (off, generation, used)) in &all[i..=j] { + let _ = writeln!(out, "{:010} {:05} {} ", off, generation, if *used { 'n' } else { 'f' }); + } + i = j + 1; + } +} + +/// A cross-reference stream section (§7.5.8) that takes object number `num`, ending the file. +/// `out` holds the file from byte offset `base` on. Rows are Flate-compressed with the PNG Up +/// predictor, and field widths fit the largest value. +fn write_xref_stream(out: &mut Vec, base: u64, trailer: &mut Dict, mut rows: BTreeMap, num: u32) { + let xref_at = base + out.len() as u64; + rows.insert(num, Row::InFile(xref_at, 0)); + if trailer.get(b"Prev").is_none() { + rows.entry(0).or_insert(Row::Free(65535)); + } + let fields = |r: &Row| match *r { + Row::Free(g) => (0u64, 0u64, u64::from(g)), + Row::InFile(o, g) => (1, o, u64::from(g)), + Row::InStream(s, i) => (2, u64::from(s), u64::from(i)), + }; + let width = |v: u64| ((64 - v.leading_zeros()).div_ceil(8)).max(1) as usize; + let (w2, w3) = rows.values().map(fields).fold((1, 1), |(a, b), (_, x, y)| (width(x).max(a), width(y).max(b))); + let cols = 1 + w2 + w3; + let mut index = Vec::new(); + let mut numbers = rows.keys().copied().peekable(); + while let Some(start) = numbers.next() { + let mut last = start; + let mut count = 1i64; + while numbers.peek().copied().is_some_and(|next| last.checked_add(1) == Some(next)) { + if let Some(next) = numbers.next() { + last = next; + count += 1; + } + } + index.push(Object::Int(i64::from(start))); + index.push(Object::Int(count)); + } + let mut data = Vec::with_capacity(rows.len().saturating_mul(cols + 1)); + // Type is one byte, and each remaining field is at most one u64. Reuse the + // predictor row on the stack instead of allocating a Vec for every xref entry. + let mut prev = [0u8; 17]; + for r in rows.values() { + let (t, a, b) = fields(r); + let a = a.to_be_bytes(); + let b = b.to_be_bytes(); + let fields = std::iter::once(t as u8).chain(a.into_iter().skip(8 - w2)).chain(b.into_iter().skip(8 - w3)); + data.push(2); // PNG "Up" + for (value, previous) in fields.zip(&mut prev) { + data.push(value.wrapping_sub(*previous)); + *previous = value; + } + } + let mut d = trailer.clone(); + d.set(b"Type".to_vec(), Object::name("XRef")); + d.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); + d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(w2 as i64), Object::Int(w3 as i64)])); + d.set(b"Index".to_vec(), Object::Array(index)); + let mut stream = Stream::flate(d, &data); + let mut parms = Dict::new(); + parms.set(b"Predictor".to_vec(), Object::Int(12)); + parms.set(b"Columns".to_vec(), Object::Int(cols as i64)); + stream.dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); + write_indirect(num, 0, &Object::Stream(stream), out); + let _ = write!(out, "startxref\n{xref_at}\n%%EOF\n"); + trailer.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); +} + +fn stamp_mod_date(doc: &mut Document, opts: &SaveOptions) { + let Some(date) = &opts.mod_date else { return }; + let date = Object::String(PdfString::literal(date.as_bytes().to_vec())); + match doc.trailer().get(b"Info").cloned() { + Some(Object::Ref(r)) => { + let _ = doc.update_dict(r, |d| d.set(b"ModDate".to_vec(), date.clone())); + } + _ => { + let mut info = Dict::new(); + info.set(b"ModDate".to_vec(), date); + let r = doc.add(info); + doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(r)); + } + } +} + +/// Keep any existing two-element `/ID` array as it is, even a malformed one such as `[<> <>]` or `[1 2]` (an ordinary +/// save preserves what it doesn't understand; Save as PDF/A replaces or completes it), or create one with +/// [`new_file_id`] when there is none or it isn't a two-element array. +fn ensure_id(trailer: &mut Dict, opts: &SaveOptions, content: &[u8]) { + if matches!(trailer.get(b"ID"), Some(Object::Array(a)) if a.len() == 2) { + return; + } + trailer.set(b"ID".to_vec(), new_file_id(content, opts.id_seed)); +} + +/// A new file identifier for a file whose bytes are `content` (ISO 32000-1 14.4): an array of +/// two equal 16-byte strings, the same for the same content and `seed`. +pub fn new_file_id(content: &[u8], seed: u64) -> Object { + // FNV-1a over a sample of the content and the seed: stable, dependency-free. + let mut h: u64 = 0xcbf2_9ce4_8422_2325 ^ seed; + for b in content.iter().step_by((content.len() / 4096).max(1)).chain(content.iter().rev().take(64)) { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + let mut id = Vec::with_capacity(16); + id.extend_from_slice(&h.to_be_bytes()); + id.extend_from_slice(&h.rotate_left(29).wrapping_mul(0x9E37_79B9_7F4A_7C15).to_be_bytes()); + let s = Object::String(PdfString { bytes: id, hex: true }); + Object::Array(vec![s.clone(), s]) +} + +/// Give a rewritten file a new identifier (§14.4), so that a redacted or sanitized copy does not +/// claim to be its source. An encrypted file keeps the first string, which its key is derived from, +/// and gets a new second one. When there is no such string (no `/ID`, one entry, or a first entry +/// that is not a string) the reader takes the empty string as the key input (`Document::first_id`), +/// so the empty string is written first and the key stays valid. +fn regenerate_id(trailer: &mut Dict, opts: &SaveOptions, written: &[u8], encrypted: bool) { + let fresh = Object::String(PdfString { bytes: new_id_bytes(opts, written), hex: true }); + let pair = if encrypted { + let first = match trailer.get(b"ID") { + Some(Object::Array(a)) => match a.first() { + Some(Object::String(s)) => Object::String(s.clone()), + _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), + }, + _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), + }; + vec![first, fresh] + } else { + vec![fresh.clone(), fresh] + }; + trailer.set(b"ID".to_vec(), Object::Array(pair)); +} + +/// FNV-1a over 64 bits, with its published offset basis and prime. The identifier is built from it +/// rather than from `std`'s `DefaultHasher`, whose algorithm the standard library does not specify. +const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; +const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + +fn fnv1a(state: u64, bytes: &[u8]) -> u64 { + bytes.iter().fold(state, |h, b| (h ^ u64::from(*b)).wrapping_mul(FNV_PRIME)) +} + +/// Sixteen bytes for a file identifier, one 8-byte half per lane. The bytes are a pure function of +/// `SaveOptions::id_entropy`, `id_seed` and the written file: the hash is FNV-1a (above) and every +/// integer is written as a fixed-width big-endian value, so the same input gives the same identifier +/// on every platform and toolchain. Without entropy only the salt changes (see `id_salt`). FNV is +/// not collision-resistant against chosen content; the identifier only has to be unique, not secret. +/// Every written byte is hashed, once, in a single traversal of the output. +fn new_id_bytes(opts: &SaveOptions, written: &[u8]) -> Vec { + let salt = id_salt(opts); + let length = u64::try_from(written.len()).unwrap_or(u64::MAX); + let mut halves = [0u8, 1].map(|lane| { + let mut h = fnv1a(FNV_OFFSET, &salt.to_be_bytes()); + h = fnv1a(h, &[lane]); + h = fnv1a(h, &opts.id_seed.to_be_bytes()); + fnv1a(h, &length.to_be_bytes()) + }); + for chunk in written.chunks(4096) { + for h in &mut halves { + *h = fnv1a(*h, chunk); + } + } + halves.iter().flat_map(|h| h.to_be_bytes()).collect() +} + +/// The salt of an identifier. With caller entropy it is a fixed hash of that entropy (deterministic, +/// for tests and callers who pin output). Without it the salt is fresh: on WebAssembly from the +/// browser ([`browser_salt`], so two fresh web apps that write the same bytes cannot share an +/// identifier), and elsewhere from the standard library's random hash keys — seeded by the operating +/// system, so separate processes differ — mixed with a process-wide counter, which separates +/// successive calls in one process. +fn id_salt(opts: &SaveOptions) -> u64 { + use std::hash::BuildHasher; + use std::sync::atomic::{AtomicU64, Ordering}; + static COUNTER: AtomicU64 = AtomicU64::new(0); + match &opts.id_entropy { + Some(entropy) => fnv1a(FNV_OFFSET, entropy), + None => { + #[cfg(target_arch = "wasm32")] + if let Some(salt) = browser_salt() { + return salt; + } + std::collections::hash_map::RandomState::new().hash_one(COUNTER.fetch_add(1, Ordering::Relaxed)) + } + } +} + +/// One byte from one `Math.random` draw: the draw (in [0, 1)) scaled to [0, 256), integer part +/// taken. The quotient is in [0, 256), so a saturating cast never wraps; a NaN or infinite draw +/// (which a conforming engine never returns) folds in as 0. No input can panic here. +#[cfg(any(target_arch = "wasm32", test))] +fn draw_byte(random: f64) -> u8 { + (random * 256.0).floor().rem_euclid(256.0) as u8 +} + +/// The `Math.random` fallback's mixing: one salt byte per draw (see [`draw_byte`]), folded through +/// FNV-1a exactly like the Web Crypto bytes, so all sixteen bytes of the fold carry entropy from +/// the draws. Pure and platform-independent, so the native test target can exercise it; on wasm +/// each draw comes from `Math.random`. +#[cfg(any(target_arch = "wasm32", test))] +fn salt_from_draws(draws: &[f64; 16]) -> u64 { + let mut bytes = [0u8; 16]; + for (byte, draw) in bytes.iter_mut().zip(draws) { + *byte = draw_byte(*draw); + } + fnv1a(FNV_OFFSET, &bytes) +} + +/// Fresh salt entropy from the browser, on WebAssembly. `crypto.getRandomValues` (Web Crypto, the +/// browser's secure generator, available in windows and workers alike) gives sixteen bytes, folded +/// into one salt. When that API is missing or refuses the call, `Math.random` is mixed instead +/// ([`salt_from_draws`]: sixteen draws, so the salt's full width carries entropy) — browsers seed +/// it afresh for every document, so a fresh instance still cannot reproduce another's salt. Only +/// where neither is reachable does the caller fall back to its `RandomState` salt, whose keys are +/// fixed at compile time on `wasm32-unknown-unknown` — the state before browser entropy was wired +/// in. +#[cfg(target_arch = "wasm32")] +fn browser_salt() -> Option { + use wasm_bindgen::JsCast; + let crypto = js_sys::Reflect::get(&js_sys::global(), &wasm_bindgen::JsValue::from_str("crypto")) + .ok() + .filter(|crypto| !crypto.is_undefined() && !crypto.is_null()); + if let Some(crypto) = crypto { + let fill = js_sys::Reflect::get(&crypto, &wasm_bindgen::JsValue::from_str("getRandomValues")) + .ok() + .and_then(|fill| fill.dyn_into::().ok()); + if let Some(fill) = fill { + let bytes = js_sys::Uint8Array::new_with_length(16); + if fill.call1(&crypto, &bytes).is_ok() { + return Some(fnv1a(FNV_OFFSET, &bytes.to_vec())); + } + } + } + // Web Crypto is missing or refused the call: degrade to `Math.random` rather than giving up. + let mut draws = [0.0; 16]; + for draw in &mut draws { + *draw = js_sys::Math::random(); + } + Some(salt_from_draws(&draws)) +} + +/// A PDF date string for "now" given seconds since the Unix epoch (UTC). +pub fn pdf_date(unix_secs: i64) -> String { + let (days, secs) = (unix_secs.div_euclid(86_400), unix_secs.rem_euclid(86_400)); + let (y, m, d) = civil_from_days(days); + // `{y:04}` rather than `%Y`, which writes a `+` before years past 9999 (and a PDF date is a + // bare four-plus-digit year, never signed). No year clamp — the old code wrote `y` as-is. + format!("D:{y:04}{m:02}{d:02}{:02}{:02}{:02}Z", secs / 3600, secs / 60 % 60, secs % 60) +} + +/// Days since 1970-01-01 → proleptic Gregorian (year, month, day). Total for any `i64` count: +/// dates past `NaiveDate`'s ±262143-year range keep their proleptic Gregorian year instead of +/// falling back to the epoch. +/// +/// The Gregorian calendar repeats every 400 years (146_097 days), so chrono converts the day +/// within one cycle and whole cycles go back into the year (an `i128` so the extremes can't +/// overflow), keeping chrono as the only calendar logic. 1970-01-01 is `num_days_from_ce` 719_163. +pub fn civil_from_days(days: i64) -> (i64, u32, u32) { + use chrono::Datelike; + let (cycles, rest) = (days.div_euclid(146_097), days.rem_euclid(146_097)); + // `rest + 719_163` is 719_163..865_259, always a valid CE day. + match i32::try_from(rest + 719_163).ok().and_then(chrono::NaiveDate::from_num_days_from_ce_opt) { + Some(d) => { + let y = i128::from(d.year()) + i128::from(cycles) * 400; + (i64::try_from(y).unwrap_or(if y < 0 { i64::MIN } else { i64::MAX }), d.month(), d.day()) + } + None => (1970, 1, 1), // unreachable + } +} + +#[cfg(test)] +mod tests { + /// `/ID` by reference comes out direct, each entry where it was: the key is made from the first. + #[test] + fn a_file_identifier_keeps_its_entries_in_their_places() { + let doc = Document::new_empty(); + let string = |bytes: &[u8]| Object::String(crate::cos::PdfString { bytes: bytes.to_vec(), hex: true }); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + assert_eq!(super::direct_id(&doc, &Object::Array(vec![nowhere, string(b"ab")])), Object::Array(vec![string(b""), string(b"ab")])); + } + + // Not built for the web: proptest is a dev-dependency of the other targets only. + #[cfg(not(target_arch = "wasm32"))] + use proptest::prelude::*; + + #[test] + fn the_same_entropy_and_bytes_give_the_same_identifier_on_every_platform() { + // Computed from the formula outside this crate: fixed-width big-endian fields and FNV-1a. + let opts = super::SaveOptions { id_entropy: Some([0x42; 16]), id_seed: 7, ..super::SaveOptions::default() }; + let want = [0x96, 0xf8, 0xcb, 0x7d, 0x5b, 0xb0, 0x09, 0xc8, 0x92, 0xc9, 0xb7, 0x51, 0xdf, 0xe6, 0xab, 0xd1]; + assert_eq!(super::new_id_bytes(&opts, b"%PDF-1.7\n"), want); + } + + #[test] + fn fnv1a_is_the_published_64_bit_algorithm() { + // Published FNV-1a (64 bit) test vectors, so the identifier can be reproduced outside this crate. + assert_eq!(super::fnv1a(super::FNV_OFFSET, b""), 0xcbf2_9ce4_8422_2325); + assert_eq!(super::fnv1a(super::FNV_OFFSET, b"a"), 0xaf63_dc4c_8601_ec8c); + assert_eq!(super::fnv1a(super::FNV_OFFSET, b"foobar"), 0x8594_4171_f739_67e8); + } + + #[test] + fn math_random_draws_fill_the_whole_salt_width() { + // Draw i = (i + 1/2)/256 scales to the integer i, so the fold's byte i proves draw i landed + // there: all sixteen bytes carry their own draw's entropy (the fallback once shifted four + // draws into one word, leaving the other four bytes of the fold constant). + let draws: [f64; 16] = std::array::from_fn(|i| (i as f64 + 0.5) / 256.0); + let bytes: Vec = (0..16).collect(); + assert_eq!(super::salt_from_draws(&draws), super::fnv1a(super::FNV_OFFSET, &bytes)); + } + + #[test] + fn every_math_random_draw_reaches_the_salt() { + let base = [0.5; 16]; + let baseline = super::salt_from_draws(&base); + for i in 0..16 { + let mut draws = base; + draws[i] = 0.75; + assert_ne!(super::salt_from_draws(&draws), baseline, "draw {i} did not reach the salt"); + } + } + + #[test] + fn a_math_random_draw_outside_its_range_still_mixes_without_panicking() { + // A conforming `Math.random` stays in [0, 1), but the mixing must not panic outside it. + assert_eq!(super::draw_byte(0.0), 0); + assert_eq!(super::draw_byte(0.5), 128); + assert_eq!(super::draw_byte(255.5 / 256.0), 255); + assert_eq!(super::draw_byte(-0.5 / 256.0), 255); + assert_eq!(super::draw_byte(-1.0), 0); + assert_eq!(super::draw_byte(2.0), 0); + assert_eq!(super::draw_byte(f64::NAN), 0); + assert_eq!(super::draw_byte(f64::INFINITY), 0); + } + + use super::*; + use crate::cos::Lexer; + + fn roundtrip(o: &Object) -> Object { + let mut out = Vec::new(); + serialize(o, &mut out); + Lexer::new(&out, 0).object().expect("parses") + } + + #[test] + fn kept_objects_stop_at_their_byte_budget() { + let data = Arc::new(Vec::new()); + let string = Object::String(PdfString::literal(vec![b'x'; 1000])); + let array = Object::Array(vec![Object::Int(1), Object::Null]); + let mut kept = KeptObjects::default(); + assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); + assert_eq!(kept.bytes, held_bytes(&string, &data, 0) + held_bytes(&array, &data, 0)); + let mut full = KeptObjects { bytes: KeptObjects::BYTES - held_bytes(&string, &data, 0) + 1 }; + let before = full.bytes; + assert!(!full.admit(&string, &data), "one byte over the budget"); + assert_eq!(full.bytes, before, "a refused object is not counted"); + let mut deep = Object::Null; + for _ in 0..300 { + deep = Object::Array(vec![deep]); + } + assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); + } + + #[test] + fn kept_objects_count_names_and_dictionary_keys() { + let data = Arc::new(Vec::new()); + let name = Object::Name(vec![b'n'; KeptObjects::BYTES + 1]); + let mut kept = KeptObjects::default(); + assert!(!kept.admit(&name, &data), "a name over the byte budget is not kept"); + assert_eq!(kept.bytes, 0); + let mut dict = Dict::new(); + dict.set(vec![b'k'; KeptObjects::BYTES + 1], Object::Null); + assert!(!kept.admit(&Object::Dict(dict), &data), "nor a dictionary key"); + assert_eq!(kept.bytes, 0); + } + + #[test] + fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { + let mut doc = Document::new_empty(); + let dict = doc.add(Object::Dict(Dict::new())); + let mut reader = doc.object_reader(); + assert!(reread(&mut reader, dict, false).is_ok()); + assert!(reread(&mut reader, dict, true).is_err(), "a dictionary where a stream was traversed"); + assert!(reread(&mut reader, ObjRef::new(9999, 0), false).is_err(), "nothing where an object was traversed"); + } + + #[test] + fn xref_predictor_rows_preserve_variable_width_fields_and_sparse_runs() { + for offset in [0, 255, 256, 65535, 65536, u32::MAX as u64, u32::MAX as u64 + 1, u64::MAX] { + for index in [0, 255, 256, u32::MAX] { + for incremental in [false, true] { + let mut trailer = Dict::new(); + if incremental { + trailer.set(b"Prev".to_vec(), Object::Int(17)); + } + let rows = BTreeMap::from([(1, Row::InFile(offset, 32768)), (2, Row::Free(65535)), (4, Row::InStream(65536, index))]); + let mut bytes = Vec::new(); + write_xref_stream(&mut bytes, 0, &mut trailer, rows, 5); + let (_, object) = crate::cos::parser::parse_indirect(&bytes, 0, &|_| None).unwrap(); + let Object::Stream(stream) = object else { panic!("xref stream") }; + let widths: Vec<_> = stream.dict.get(b"W").unwrap().as_array().unwrap().iter().map(|o| o.as_int().unwrap() as usize).collect(); + assert_eq!(widths[0], 1); + assert!((3..=8).contains(&widths[1])); + assert!((2..=4).contains(&widths[2])); + let runs: Vec<_> = stream.dict.get(b"Index").unwrap().as_array().unwrap().iter().map(|o| o.as_int().unwrap()).collect(); + assert_eq!(runs, if incremental { vec![1, 2, 4, 2] } else { vec![0, 3, 4, 2] }); + let mut expected = vec![(1, offset, 32768), (0, 0, 65535), (2, 65536, u64::from(index)), (1, 0, 0)]; + if !incremental { + expected.insert(0, (0, 0, 65535)); + } + let decoded = stream.decoded().unwrap(); + let columns: usize = widths.iter().sum(); + assert_eq!(decoded.len(), columns * expected.len()); + for (row, (kind, first, second)) in decoded.chunks_exact(columns).zip(expected) { + let values: Vec<_> = [0..1, 1..1 + widths[1], 1 + widths[1]..columns] + .into_iter() + .map(|range| row[range].iter().fold(0u64, |value, byte| value * 256 + u64::from(*byte))) + .collect(); + assert_eq!(values, vec![kind, first, second]); + } + } + } + } + } + + #[test] + fn pdf_dates_are_utc_and_byte_identical() { + // `D:YYYYMMDDHHmmSSZ` (ISO 32000-2 §7.9.4), the same strings the hand-rolled code wrote. + assert_eq!(pdf_date(0), "D:19700101000000Z"); + assert_eq!(pdf_date(1_790_000_000), "D:20260921141320Z"); + // Leap days, including the 400-year rule, and the last second of a year. + assert_eq!(pdf_date(1_709_164_800), "D:20240229000000Z"); + assert_eq!(pdf_date(951_782_400), "D:20000229000000Z"); + assert_eq!(pdf_date(1_767_225_599), "D:20251231235959Z"); + // Before the epoch (seconds into a day are counted forward from midnight UTC). + assert_eq!(pdf_date(-1), "D:19691231235959Z"); + // Years past 9999 keep the plain zero-padded year (no `+`), as the hand-rolled code did. + assert_eq!(pdf_date(253_402_300_800), "D:100000101000000Z"); + // Past chrono's ±262143-year range the proleptic Gregorian date is kept, not the epoch. + // Pinned to the old hand-rolled output (which had no year clamp — a negative year keeps + // its sign, exactly as before). + assert_eq!(pdf_date(10_000_000_000_000), "D:3188570520174640Z"); + assert_eq!(pdf_date(-10_000_000_000_000), "D:-3149180813061320Z"); + assert_eq!(crate::cos::civil_from_days(0), (1970, 1, 1)); + assert_eq!(crate::cos::civil_from_days(-1), (1969, 12, 31)); + assert_eq!(crate::cos::civil_from_days(11_016), (2000, 2, 29)); + assert_eq!(crate::cos::civil_from_days(19_782), (2024, 2, 29)); + assert_eq!(crate::cos::civil_from_days(i64::MAX), (25_252_734_927_768_524, 7, 27)); + assert_eq!(crate::cos::civil_from_days(i64::MIN), (-25_252_734_927_764_585, 6, 7)); + } + + #[test] + fn reals_keep_full_precision_without_exponents() { + for r in [0.1, -0.000_000_123_456_789, 1e-12, 123_456.789_012_345, 1.0 / 3.0, -2.5e7 + 0.25] { + let mut out = Vec::new(); + serialize(&Object::Real(r), &mut out); + assert!(!out.contains(&b'e') && !out.contains(&b'E'), "{}", String::from_utf8_lossy(&out)); + assert_eq!(roundtrip(&Object::Real(r)), Object::Real(r)); + } + } + + #[test] + fn whole_reals_past_the_integer_limit_stay_reals() { + for (r, text) in [ + (2_147_483_647.0, "2147483647"), + (-2_147_483_647.0, "-2147483647"), + (2_147_483_648.0, "2147483648."), + (4e9, "4000000000."), + (-4e9, "-4000000000."), + (1e30, "1000000000000000000000000000000."), + (595.0, "595"), + (-0.0, "0"), + ] { + let mut out = Vec::new(); + serialize(&Object::Real(r), &mut out); + assert_eq!(String::from_utf8_lossy(&out), text, "{r}"); + let back = roundtrip(&Object::Real(r)); + if r.abs() > f64::from(i32::MAX) { + assert_eq!(back, Object::Real(r), "{r} must read back as a real"); + } + } + } + + #[cfg(not(target_arch = "wasm32"))] + proptest! { + #[test] + fn any_finite_real_round_trips(r in proptest::num::f64::NORMAL | proptest::num::f64::SUBNORMAL | proptest::num::f64::ZERO) { + let back = roundtrip(&Object::Real(r)); + match back { + Object::Real(b) => prop_assert_eq!(b, r), + Object::Int(i) => { + prop_assert!(i.unsigned_abs() <= i32::MAX as u64, "{r} came back as integer {i}"); + prop_assert_eq!(i as f64, r); + } + other => prop_assert!(false, "unexpected {other:?}"), + } + } + + #[test] + fn strings_and_names_round_trip(bytes in proptest::collection::vec(any::(), 0..64)) { + let s = Object::String(PdfString { bytes: bytes.clone(), hex: false }); + prop_assert_eq!(roundtrip(&s).as_string().map(|s| s.bytes.clone()), Some(bytes.clone())); + if !bytes.is_empty() && !bytes.contains(&0) { + prop_assert_eq!(roundtrip(&Object::Name(bytes.clone())), Object::Name(bytes)); + } + } + } +} diff --git a/crates/pdf/src/filters.rs b/crates/pdf/src/filters.rs index 71a939dea..0fd5792e4 100644 --- a/crates/pdf/src/filters.rs +++ b/crates/pdf/src/filters.rs @@ -5,9 +5,9 @@ //! TIFF/PNG predictors, `ASCIIHexDecode`, `ASCII85Decode`, `RunLengthDecode`). //! Image codecs (`DCTDecode`, `JPXDecode`, `JBIG2Decode`, `CCITTFaxDecode`), //! `Crypt` and unknown filters are reported as [`FilterError::Unsupported`] so -//! callers keep such data encoded. CCITT fax images decode on their own -//! ([`decode_ccitt_within`]), spending from an allowance the caller shares across a run of -//! images. +//! callers keep such data encoded. CCITT fax and JBIG2 images decode on their own +//! ([`decode_ccitt_within`], [`decode_jbig2_within`]), each spending from an allowance the +//! caller shares across a run of images. //! //! The codecs themselves are `craft-stream-codecs`, re-exported here; this module adds Flate, //! the chain (stage order, limits across stages, strict and tolerant decoding) and its @@ -24,7 +24,7 @@ use std::borrow::Cow; pub use craft_stream_codecs::{ CcittParams, CodecError as FilterError, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, Params, SharedJbig2, - decode_ccitt_within, encode_g4, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared, + decode_ccitt_within, decode_jbig2_within, encode_g4, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared, }; use craft_stream_codecs::{Failure, Step, pay}; use craft_stream_codecs::{ascii85, asciihex, lzw, predictor, runlength}; diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 3585c20e4..e052c7304 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -4,6 +4,8 @@ //! Advanced backend re-exports are coupled to the pinned backend, not a universal document model. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(feature = "cos")] +pub mod cos; #[cfg(feature = "crypt")] pub mod crypt; #[cfg(feature = "filters")] diff --git a/crates/pdf/tests/cos/bounded_writer.rs b/crates/pdf/tests/cos/bounded_writer.rs new file mode 100644 index 000000000..b8fe79370 --- /dev/null +++ b/crates/pdf/tests/cos/bounded_writer.rs @@ -0,0 +1,236 @@ +//! Full rewrites must preserve the graph when temporary parse/packing caches are evicted. +//! Fixtures are generated here; no third-party PDF assets are used. + +use std::collections::{BTreeMap, HashSet}; +use std::sync::Arc; + +use craft_pdf::cos::{Algorithm, Dict, Document, NewEncryption, ObjRef, Object, PdfString, SaveOptions, XrefEntry, write_full}; + +const NODES: usize = 260; +const EDITED: usize = 129; + +fn node(i: usize) -> ObjRef { + ObjRef::new(301 + i as u32, (i % 5 + 1) as u16) +} + +fn stream_body(i: usize) -> Vec { + format!("opaque stream {i}: preserve these bytes\n").into_bytes() +} + +/// Nonzero generations, sparse numbers, shared references, a cycle and interleaved streams. +/// A raw classic xref avoids depending on the writer under test to construct the input. +fn fixture() -> Vec { + let mut objects = vec![ + (ObjRef::new(10, 2), format!("<< /Type /Catalog /Pages 20 0 R /Extension {} /Shared 42 3 R /Freed 43 8 R /Missing 9000 0 R >>", node(0))), + (ObjRef::new(20, 0), "<< /Type /Pages /Kids [] /Count 0 >>".into()), + (ObjRef::new(42, 3), "<< /Token (shared extension value) >>".into()), + (ObjRef::new(43, 8), "<< /Token (deleted extension value) >>".into()), + (ObjRef::new(70, 4), "<< /Title (original title) >>".into()), + ]; + for i in 0..NODES { + let fields = format!("/Value {i} /Next {} /Shared 42 3 R /Opaque << /Nested [42 3 R (retained data)] >>", node((i + 1) % NODES)); + let body = if i % 19 == 0 { + let raw = stream_body(i); + format!("<< {fields} /Length {} >>\nstream\n{}endstream", raw.len(), String::from_utf8(raw).unwrap()) + } else { + format!("<< {fields} >>") + }; + objects.push((node(i), body)); + } + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = BTreeMap::new(); + for (reference, body) in objects { + offsets.insert(reference.num, (bytes.len(), reference.generation)); + bytes.extend_from_slice(format!("{} {} obj\n{body}\nendobj\n", reference.num, reference.generation).as_bytes()); + } + let size = offsets.last_key_value().unwrap().0 + 1; + let xref = bytes.len(); + bytes.extend_from_slice(format!("xref\n0 {size}\n").as_bytes()); + for n in 0..size { + match offsets.get(&n) { + Some((offset, generation)) => bytes.extend_from_slice(format!("{offset:010} {generation:05} n \n").as_bytes()), + None => bytes.extend_from_slice(b"0000000000 65535 f \n"), + } + } + bytes.extend_from_slice(format!("trailer\n<< /Size {size} /Root 10 2 R /Info 70 4 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + bytes +} + +fn edited_document() -> Document { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + assert!(doc.repair_log().is_empty()); + assert_eq!(doc.root(), Some(ObjRef::new(10, 2))); + assert_eq!(doc.generation(node(EDITED).num), node(EDITED).generation); + doc.update_dict(node(EDITED), |d| d.set(b"Value".to_vec(), Object::Int(8129))).unwrap(); + doc.free(ObjRef::new(43, 8)); + let added = doc.add(Object::String(PdfString::text("added extension value"))); + doc.update_dict(doc.root().unwrap(), |d| d.set(b"Added".to_vec(), Object::Ref(added))).unwrap(); + doc +} + +fn check_graph(doc: &Document) { + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + let catalog = doc.get(doc.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + let shared = catalog.reference(b"Shared").unwrap(); + let shared_object = doc.get(shared); + assert_eq!(shared_object.as_dict().unwrap().get(b"Token").unwrap().as_string().unwrap().to_text(), "shared extension value"); + assert_eq!(catalog.get(b"Freed"), Some(&Object::Null)); + assert_eq!(catalog.get(b"Missing"), Some(&Object::Null)); + assert_eq!(doc.resolve(catalog.get(b"Added").unwrap()).as_string().unwrap().to_text(), "added extension value"); + let first = catalog.reference(b"Extension").unwrap(); + let mut current = first; + let mut visited = HashSet::new(); + for i in 0..NODES { + assert!(visited.insert(current), "cycle closed before node {i}"); + let object = doc.get(current); + let dict = object.as_dict().unwrap(); + assert_eq!(dict.int(b"Value"), Some(if i == EDITED { 8129 } else { i as i64 })); + assert_eq!(dict.reference(b"Shared"), Some(shared)); + let opaque = dict.get(b"Opaque").unwrap().as_dict().unwrap().get(b"Nested").unwrap().as_array().unwrap(); + assert_eq!(opaque.first(), Some(&Object::Ref(shared))); + assert_eq!(opaque.get(1).unwrap().as_string().unwrap().to_text(), "retained data"); + if i % 19 == 0 { + let Object::Stream(stream) = object.as_ref() else { panic!("node {i} lost its stream") }; + assert_eq!(stream.decoded().unwrap(), stream_body(i)); + } else { + assert!(matches!(object.as_ref(), Object::Dict(_))); + } + current = dict.reference(b"Next").unwrap(); + } + assert_eq!(current, first, "the extension cycle is preserved"); + assert!(doc.object_numbers().into_iter().all(|n| doc.generation(n) == 0)); +} + +#[test] +fn full_rewrites_preserve_graph_across_multiple_object_streams_and_evictions() { + for object_streams in [false, true] { + let doc = edited_document(); + let opts = SaveOptions { object_streams, mod_date: Some("D:20261008000000Z".into()), ..SaveOptions::default() }; + let saved = write_full(&doc, &opts).unwrap(); + assert_eq!(saved, write_full(&doc, &opts).unwrap(), "unchanged inputs produce identical bytes"); + let back = Document::open(Arc::new(saved)).unwrap(); + check_graph(&back); + let info = back.resolve(back.trailer().get(b"Info").unwrap()); + let info = info.as_dict().unwrap(); + assert_eq!(info.get(b"Title").unwrap().as_string().unwrap().to_text(), "original title"); + assert_eq!(info.get(b"ModDate").unwrap().as_string().unwrap().to_text(), "D:20261008000000Z"); + let containers: HashSet<_> = back + .object_numbers() + .into_iter() + .filter_map(|num| match back.xref_entry(num) { + Some(XrefEntry::InStream { stream, .. }) => Some(stream), + _ => None, + }) + .collect(); + if object_streams { + assert!(containers.len() >= 3, "fixture must span multiple packed-object chunks"); + } else { + assert!(containers.is_empty()); + } + // Read an already packed input as well: eviction must not lose decoded ObjStm data. + check_graph(&Document::open(Arc::new(write_full(&back, &opts).unwrap())).unwrap()); + // Writing must not apply the date stamp or consume the caller's edits. + let original_info = doc.resolve(doc.trailer().get(b"Info").unwrap()); + assert!(!original_info.as_dict().unwrap().contains(b"ModDate")); + assert!(doc.is_modified()); + assert_eq!(doc.get(node(EDITED)).as_dict().unwrap().int(b"Value"), Some(8129)); + } +} + +fn protect(doc: &mut Document, algorithm: Algorithm) { + doc.set_encryption(&NewEncryption { + algorithm, + user_password: "user", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed: [23; 32], + }) + .unwrap(); +} + +#[test] +fn encrypted_full_rewrites_preserve_data_across_evictions() { + for algorithm in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = edited_document(); + protect(&mut doc, algorithm); + let encrypted = Arc::new(write_full(&doc, &SaveOptions::default()).unwrap()); + let cold = Document::open_with_password(encrypted, Some("user")).unwrap(); + // The second rewrite must decrypt lazily from the input, then encrypt the new streams. + let rewritten = Arc::new(write_full(&cold, &SaveOptions::default()).unwrap()); + assert!(Document::open_with_password(rewritten.clone(), Some("wrong")).is_err()); + let back = Document::open_with_password(rewritten, Some("owner")).unwrap(); + assert!(back.security().is_some()); + check_graph(&back); + } +} + +#[test] +fn encryption_dependencies_stay_outside_object_streams_across_evictions() { + let mut doc = edited_document(); + protect(&mut doc, Algorithm::Aes256); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + let filters = doc.get(encrypt).as_dict().unwrap().get(b"CF").cloned().unwrap(); + let filters = doc.add(filters); + let mut next = filters; + for i in (0..140).rev() { + let mut dict = Dict::new(); + dict.set(b"Value".to_vec(), Object::Int(i)); + dict.set(b"Next".to_vec(), Object::Ref(next)); + next = doc.add(dict); + } + doc.update_dict(encrypt, |dict| { + dict.set(b"CF".to_vec(), Object::Ref(filters)); + dict.set(b"PrivateChain".to_vec(), Object::Ref(next)); + }) + .unwrap(); + let saved = Arc::new(write_full(&doc, &SaveOptions::default()).unwrap()); + let cold = Document::open_with_password(saved, Some("user")).unwrap(); + let rewritten = Arc::new(write_full(&cold, &SaveOptions::default()).unwrap()); + let back = Document::open_with_password(rewritten, Some("user")).unwrap(); + check_graph(&back); + let encrypt = back.trailer().reference(b"Encrypt").unwrap(); + assert!(matches!(back.xref_entry(encrypt.num), Some(XrefEntry::InFile { .. }))); + let encrypt = back.get(encrypt); + let encrypt = encrypt.as_dict().unwrap(); + let filters = encrypt.reference(b"CF").unwrap(); + assert!(matches!(back.xref_entry(filters.num), Some(XrefEntry::InFile { .. }))); + let mut current = encrypt.reference(b"PrivateChain").unwrap(); + for i in 0..140 { + assert!(matches!(back.xref_entry(current.num), Some(XrefEntry::InFile { .. }))); + let object = back.get(current); + let dict = object.as_dict().unwrap(); + assert_eq!(dict.int(b"Value"), Some(i)); + current = dict.reference(b"Next").unwrap(); + } + assert_eq!(current, filters); +} + +#[test] +fn large_packed_strings_round_trip_across_byte_limited_batches() { + let mut doc = Document::new_empty(); + let mut references = Vec::new(); + // More than 8 MiB in fewer than 100 objects: a byte cap may flush before the count cap. + for i in 0..70u8 { + let mut bytes = vec![b'a' + i % 26; 192 * 1024]; + bytes[0] = i; + references.push(Object::Ref(doc.add(Object::String(PdfString::literal(bytes))))); + } + doc.update_dict(doc.root().unwrap(), |dict| dict.set(b"Extension".to_vec(), Object::Array(references))).unwrap(); + let source = Arc::new(write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap()); + drop(doc); + let cold = Document::open(source).unwrap(); + let back = Document::open(Arc::new(write_full(&cold, &SaveOptions::default()).unwrap())).unwrap(); + assert!(back.repair_log().is_empty()); + let catalog = back.get(back.root().unwrap()); + let values = catalog.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap(); + assert_eq!(values.len(), 70); + for (i, reference) in values.iter().enumerate() { + let value = back.resolve(reference); + let bytes = &value.as_string().unwrap().bytes; + assert_eq!(bytes.len(), 192 * 1024); + assert_eq!(bytes[0], i as u8); + assert!(bytes[1..].iter().all(|&b| b == b'a' + i as u8 % 26)); + } +} diff --git a/crates/pdf/tests/cos/corpus.rs b/crates/pdf/tests/cos/corpus.rs new file mode 100644 index 000000000..802f46105 --- /dev/null +++ b/crates/pdf/tests/cos/corpus.rs @@ -0,0 +1,180 @@ +//! Corpus parity: every file hayro-syntax can open must also open with our reader, agree on the +//! page count, and survive an incremental save that touches the catalog (the appended revision +//! must be readable by both readers and leave the original bytes untouched). +//! +//! Runs over `corpus/pdfjs/test/pdfs` at the repository root when present (PdfCraft's +//! `cargo xtask corpus` fetches and verifies it; copy or link its `corpus/pdfjs`); when it is +//! absent these are a no-op, so CI without the corpus stays green. Run with +//! `--ignored --nocapture`. +//! +//! A corpus that is present but **unverified** is a failure, not a skip. `xtask corpus` writes +//! `corpus/pdfjs/.pdfcraft-corpus.json` only after the checkout matches the pinned commit and +//! manifest sha256, so these tests refuse to run against a tree nobody has checked. Without that, +//! a half-finished or tampered checkout would quietly produce green results. + +use std::sync::Arc; + +use craft_pdf::cos::{Document, Object, SaveOptions, write_full, write_incremental}; + +/// `corpus/pdfjs`, or `None` when nothing has been fetched. +/// +/// Panics when the checkout exists without the stamp that `cargo xtask corpus` writes on a +/// successful verification: an unverified corpus must fail loudly rather than skip. +fn checkout() -> Option { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pdfjs"); + if !dir.is_dir() { + return None; + } + let stamp = dir.join(".pdfcraft-corpus.json"); + assert!( + stamp.is_file(), + "{} exists but {} does not: the corpus has not been verified against the pin. \ + Run `cargo xtask corpus` (it fetches and verifies), or delete corpus/pdfjs to skip these tests.", + dir.display(), + stamp.display() + ); + Some(dir) +} + +/// Leaf pages reachable from the catalog (cycle-safe), like viewers count them. +fn page_count(doc: &Document) -> Option { + let root = doc.get(doc.root()?); + let mut stack = vec![root.as_dict()?.get(b"Pages")?.clone()]; + let (mut seen, mut n) = (std::collections::HashSet::new(), 0); + while let Some(o) = stack.pop() { + if let Object::Ref(r) = o + && !seen.insert(r) + { + continue; + } + let node = doc.resolve(&o); + let Some(d) = node.as_dict() else { continue }; + match d.get(b"Kids").map(|k| doc.resolve(k)) { + Some(k) if d.name(b"Type") != Some(b"Page") => stack.extend(k.as_array().cloned().unwrap_or_default().into_iter().rev()), + _ => n += 1, + } + } + Some(n) +} + +#[test] +#[ignore = "needs corpus/; run with --ignored"] +fn corpus_parity() { + let Some(root) = checkout() else { + eprintln!("corpus not fetched; skipping (cargo xtask corpus)"); + return; + }; + let dir = root.join("test/pdfs"); + let entries = std::fs::read_dir(&dir).unwrap_or_else(|e| panic!("verified corpus has no {}: {e}", dir.display())); + let mut files: Vec<_> = entries.flatten().map(|e| e.path()).filter(|p| p.extension().is_some_and(|e| e == "pdf")).collect(); + files.sort(); + let (mut checked, mut encrypted, mut failures) = (0, 0, Vec::new()); + for path in &files { + let bytes = Arc::new(std::fs::read(path).unwrap()); + let Ok(reference) = std::panic::catch_unwind(|| hayro_syntax::Pdf::new(bytes.clone())) else { continue }; + let Ok(reference) = reference else { continue }; // hayro can't open it either: not a parity case + let name = path.file_name().unwrap().to_string_lossy().into_owned(); + if std::env::var("COS_ONLY").is_ok_and(|o| o != name) { + continue; + } + if std::env::var_os("COS_TRACE").is_some() { + eprintln!("{name}"); + } + let expected = reference.pages().len(); + let result = std::panic::catch_unwind(|| -> Result<(), String> { + let mut doc = match Document::open(bytes.clone()) { + Ok(d) => d, + Err(craft_pdf::cos::CosError::NeedsPassword) => return Err("encrypted".into()), + Err(e) => return Err(format!("open: {e}")), + }; + // hayro may repair a broken /Count; compare against the leaf walk count loosely. + if page_count(&doc) != Some(expected) && expected > 0 { + return Err(format!("page count {:?} vs hayro {expected}", page_count(&doc))); + } + let root = doc.root().ok_or("no root")?; + doc.update_dict(root, |d| d.set(b"PdfCraftTest".to_vec(), Object::Bool(true))).map_err(|e| e.to_string())?; + let saved = write_incremental(&doc, &SaveOptions::default()).map_err(|e| format!("save: {e}"))?; + // Reconstructed files are rewritten in full; everything else must keep its bytes. + if !doc.revisions().is_empty() && saved.get(..bytes.len()) != Some(&bytes[..]) { + return Err("prefix modified".into()); + } + let again = Document::open(Arc::new(saved.clone())).map_err(|e| format!("reopen: {e}"))?; + let flag = again.get(again.root().ok_or("no root after save")?).as_dict().and_then(|d| d.get(b"PdfCraftTest").cloned()); + if flag != Some(Object::Bool(true)) { + return Err("edit lost after reopen".into()); + } + let hay = hayro_syntax::Pdf::new(Arc::new(saved)).map_err(|e| format!("hayro reopen: {e:?}"))?; + if hay.pages().len() != expected { + return Err(format!("hayro sees {} pages after save, expected {expected}", hay.pages().len())); + } + let full = write_full(&doc, &SaveOptions::default()).map_err(|e| format!("full: {e}"))?; + let hay = hayro_syntax::Pdf::new(Arc::new(full)).map_err(|e| format!("hayro full: {e:?}"))?; + if hay.pages().len() != expected { + return Err(format!("hayro sees {} pages after full save, expected {expected}", hay.pages().len())); + } + Ok(()) + }); + match result { + Ok(Ok(())) => checked += 1, + Ok(Err(e)) if e == "encrypted" => encrypted += 1, + Ok(Err(e)) => failures.push(format!("{name}: {e}")), + Err(_) => failures.push(format!("{name}: PANIC")), + } + } + eprintln!("cos corpus parity: {checked} ok, {encrypted} encrypted (skipped), {} failures", failures.len()); + for f in &failures { + eprintln!(" {f}"); + } + assert!(failures.iter().all(|f| !f.ends_with("PANIC")), "no panics"); +} + +/// Password-protected corpus files (passwords from pdf.js's test manifest): the right password +/// opens them and the edit/save round trip still works; no or a wrong password is reported. +#[test] +#[ignore = "needs corpus/; run with --ignored"] +fn corpus_passwords() { + let Some(root) = checkout() else { + eprintln!("corpus not fetched; skipping (cargo xtask corpus)"); + return; + }; + let dir = root.join("test"); + let manifest = + std::fs::read_to_string(dir.join("test_manifest.json")).unwrap_or_else(|e| panic!("verified corpus has no test_manifest.json: {e}")); + let entries: Vec = serde_json::from_str(&manifest).expect("manifest is JSON"); + let cases: Vec<(String, String)> = + entries.iter().filter_map(|e| Some((e.get("file")?.as_str()?.to_string(), e.get("password")?.as_str()?.to_string()))).collect(); + assert!(cases.len() >= 5, "manifest parsed ({} cases)", cases.len()); + let mut failures = Vec::new(); + for (file, pw) in &cases { + let Ok(bytes) = std::fs::read(dir.join(file)) else { continue }; + let bytes = Arc::new(bytes); + if !matches!(Document::open(bytes.clone()), Err(craft_pdf::cos::CosError::NeedsPassword)) { + failures.push(format!("{file}: opened without a password")); + } + if !matches!(Document::open_with_password(bytes.clone(), Some("definitely wrong")), Err(craft_pdf::cos::CosError::WrongPassword)) { + failures.push(format!("{file}: wrong password not rejected")); + } + match Document::open_with_password(bytes.clone(), Some(pw)) { + Ok(mut doc) => { + let pages = page_count(&doc); + if pages.unwrap_or(0) == 0 { + failures.push(format!("{file}: no pages after decryption")); + continue; + } + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"PdfCraftTest".to_vec(), Object::Bool(true))).unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + match Document::open_with_password(Arc::new(saved.clone()), Some(pw)) { + Ok(again) if page_count(&again) == pages => {} + other => failures.push(format!("{file}: reopen after save failed ({:?})", other.err())), + } + } + Err(e) => failures.push(format!("{file}: {e}")), + } + } + eprintln!("password corpus: {} cases, {} failures", cases.len(), failures.len()); + for f in &failures { + eprintln!(" {f}"); + } + assert!(failures.is_empty()); +} diff --git a/crates/pdf/tests/cos/efopen.rs b/crates/pdf/tests/cos/efopen.rs new file mode 100644 index 000000000..30cce32fe --- /dev/null +++ b/crates/pdf/tests/cos/efopen.rs @@ -0,0 +1,224 @@ +//! A document that encrypts only its embedded files, authorised when one is opened (`/AuthEvent +//! /EFOpen`, ISO 32000-2 §7.6.6, Table 25): it opens without a password, its pages read as stored, +//! and its embedded files keep their encrypted bytes, flagged as such, through every save. + +use std::sync::Arc; + +use craft_pdf::cos::{Algorithm, CosError, Document, NewEncryption, ObjRef, Object, SaveOptions, write_full, write_incremental}; +use craft_pdf::crypt::StreamKind; + +const PAGE: &str = "BT /F1 12 Tf 20 100 Td (Visible page) Tj ET"; +const ATTACHED: &[u8] = b"the attached file's own words"; +const PASSWORD: &str = "attach pw"; +const EMBEDDED: ObjRef = ObjRef { num: 6, generation: 0 }; +const CONTENT: ObjRef = ObjRef { num: 4, generation: 0 }; + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02X}")).collect() +} + +/// One page in the clear and one attachment encrypted with `/StdCF`, whose `/AuthEvent` is +/// `auth_event`: AES-256 (V5, R6) or AES-128 (V4, R4), `/StmF` and `/StrF` `/Identity`. +fn fixture(algorithm: Algorithm, auth_event: &str) -> Vec { + fixture_with(algorithm, auth_event, false) +} + +/// [`fixture`], the attachment naming `/StdCF` in a `/Crypt` filter of its own when `named`, with +/// no `/EFF` (ISO 32000-2 §7.4.10). +fn fixture_with(algorithm: Algorithm, auth_event: &str, named: bool) -> Vec { + let id0 = [7u8; 16]; + let params = + NewEncryption { algorithm, user_password: PASSWORD, owner_password: "owner pw", permissions: -4, encrypt_metadata: true, seed: [3; 32] }; + let handler = craft_pdf::crypt::create(¶ms, &id0).unwrap(); + let d = handler.dict(); + let kind = if named { StreamKind::Named(b"StdCF") } else { StreamKind::EmbeddedFile }; + let cipher = handler.encrypt_stream(EMBEDDED.num, 0, ATTACHED, kind); + let (eff, crypt) = if named { ("", " /Filter [/Crypt] /DecodeParms [<< /Name /StdCF >>]") } else { (" /EFF /StdCF", "") }; + let (version, cfm, r6) = match algorithm { + Algorithm::Aes256 => ("/V 5 /R 6", "/AESV3 /Length 32", format!("/OE <{}> /UE <{}> /Perms <{}>", hex(&d.oe), hex(&d.ue), hex(&d.perms))), + _ => ("/V 4 /R 4 /Length 128", "/AESV2 /Length 16", String::new()), + }; + let encrypt = format!( + "<< /Filter /Standard {version} /O <{}> /U <{}> {r6} /P {} /CF << /StdCF << /AuthEvent /{auth_event} /CFM {cfm} >> >> /StmF /Identity /StrF /Identity{eff} >>", + hex(&d.o), + hex(&d.u), + d.p + ); + let mut objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /Names << /EmbeddedFiles << /Names [(note.txt) 5 0 R] >> >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 200] >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /Contents 4 0 R >>".to_vec(), + format!("<< /Length {} >>\nstream\n{PAGE}\nendstream", PAGE.len()).into_bytes(), + b"<< /Type /Filespec /F (note.txt) /UF (note.txt) /EF << /F 6 0 R >> >>".to_vec(), + ]; + let mut stream = format!("<< /Type /EmbeddedFile{crypt} /Length {} >>\nstream\n", cipher.len()).into_bytes(); + stream.extend_from_slice(&cipher); + stream.extend_from_slice(b"\nendstream"); + objs.push(stream); + objs.push(encrypt.into_bytes()); + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + let trailer = format!( + "trailer\n<< /Size {} /Root 1 0 R /Encrypt 7 0 R /ID [<{}> <{}>] >>\nstartxref\n{xref}\n%%EOF\n", + objs.len() + 1, + hex(&id0), + hex(&id0) + ); + out.extend_from_slice(trailer.as_bytes()); + out +} + +fn stream(doc: &Document, r: ObjRef) -> craft_pdf::cos::Stream { + match &*doc.get(r) { + Object::Stream(s) => s.clone(), + other => panic!("object {} isn't a stream: {other:?}", r.num), + } +} + +/// The attachment as the password reads it from `bytes`. +fn attachment_with_password(bytes: Vec) -> Vec { + let doc = Document::open_with_password(Arc::new(bytes), Some(PASSWORD)).unwrap(); + assert!(!doc.embedded_files_locked()); + let embedded = doc.root().and_then(|r| doc.get(r).as_dict().cloned()).unwrap(); + assert!(embedded.get(b"Names").is_some()); + stream(&doc, EMBEDDED).decoded().unwrap() +} + +#[test] +fn a_document_encrypting_only_its_embedded_files_opens_without_a_password() { + let bytes = fixture(Algorithm::Aes256, "EFOpen"); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert!(doc.embedded_files_locked()); + // Not a repair: the specification has it open so. + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + assert!(doc.trailer().get(b"Encrypt").is_some()); + // The page reads as stored; the attachment keeps its encrypted bytes and says so. + let content = stream(&doc, CONTENT); + assert_eq!(content.decoded().unwrap(), PAGE.as_bytes()); + assert!(!doc.reads_as_stored(&content)); + let embedded = stream(&doc, EMBEDDED); + assert!(doc.reads_as_stored(&embedded)); + assert_ne!(embedded.decoded().unwrap(), ATTACHED); + // The password still opens it, and reads the attachment. + assert_eq!(attachment_with_password(bytes.clone()), ATTACHED); + // A wrong password is still refused. + assert!(matches!(Document::open_with_password(Arc::new(bytes), Some("nope")), Err(CosError::WrongPassword))); +} + +#[test] +fn a_crypt_filter_authorised_when_the_document_opens_still_needs_the_password() { + let bytes = fixture(Algorithm::Aes256, "DocOpen"); + assert!(matches!(Document::open(Arc::new(bytes.clone())), Err(CosError::NeedsPassword))); + assert_eq!(attachment_with_password(bytes), ATTACHED); +} + +#[test] +fn saves_keep_the_encrypted_embedded_files_as_stored() { + for algorithm in [Algorithm::Aes256, Algorithm::Aes128] { + let mut doc = Document::open(Arc::new(fixture(algorithm, "EFOpen"))).unwrap(); + assert!(doc.embedded_files_locked(), "{algorithm:?}"); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"PageMode".to_vec(), Object::name("UseAttachments"))).unwrap(); + // An incremental save keeps /Encrypt, and the attachment decrypts with the password. + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert_eq!(attachment_with_password(saved), ATTACHED, "{algorithm:?}"); + // A full save renumbers: AES-256 keys don't depend on the number, AES-128 keys do. + match algorithm { + Algorithm::Aes256 => { + let full = write_full(&doc, &SaveOptions::default()).unwrap(); + let reopened = Document::open(Arc::new(full.clone())).unwrap(); + assert!(reopened.embedded_files_locked()); + let doc = Document::open_with_password(Arc::new(full), Some(PASSWORD)).unwrap(); + let names = doc.root().and_then(|r| doc.get(r).as_dict().cloned()).unwrap(); + let tree = doc.resolve(names.get(b"Names").unwrap()).as_dict().cloned().unwrap(); + let files = doc.resolve(tree.get(b"EmbeddedFiles").unwrap()).as_dict().cloned().unwrap(); + let list = doc.resolve(files.get(b"Names").unwrap()).as_array().cloned().unwrap(); + let spec = doc.resolve(&list[1]).as_dict().cloned().unwrap(); + let ef = doc.resolve(spec.get(b"EF").unwrap()).as_dict().cloned().unwrap(); + let file = match &*doc.resolve(ef.get(b"F").unwrap()) { + Object::Stream(s) => s.decoded().unwrap(), + other => panic!("{other:?}"), + }; + assert_eq!(file, ATTACHED); + } + _ => assert!(matches!(write_full(&doc, &SaveOptions::default()), Err(CosError::Security(_))), "{algorithm:?}"), + } + } +} + +#[test] +fn security_cannot_change_without_the_embedded_files_password() { + let mut doc = Document::open(Arc::new(fixture(Algorithm::Aes256, "EFOpen"))).unwrap(); + doc.remove_encryption(); + assert!(matches!(write_full(&doc, &SaveOptions::default()), Err(CosError::Security(_)))); + assert!(matches!(write_incremental(&doc, &SaveOptions::default()), Err(CosError::Security(_)))); +} + +/// From the review of the EFOpen change: the empty password opens it too, and the credentials of a +/// document opened without the key open no other. +#[test] +fn the_empty_password_opens_it_and_keyless_credentials_open_nothing_more() { + let bytes = fixture(Algorithm::Aes256, "EFOpen"); + let doc = Document::open_with_password(Arc::new(bytes.clone()), Some("")).unwrap(); + assert!(doc.embedded_files_locked()); + assert!(doc.open_with_same_credentials(Arc::new(bytes)).unwrap().embedded_files_locked()); + // The same /Encrypt values and /ID, authorised when the document opens. + let doc_open = fixture(Algorithm::Aes256, "DocOpen"); + assert!(matches!(doc.open_with_same_credentials(Arc::new(doc_open.clone())), Err(CosError::NeedsPassword))); + assert!(matches!(Document::open_with_password(Arc::new(doc_open), Some("")), Err(CosError::WrongPassword))); +} + +/// An embedded file that names an EFOpen crypt filter of its own, with no `/EFF`, opens the same way. +#[test] +fn an_embedded_file_naming_its_own_crypt_filter_opens_without_a_password() { + let bytes = fixture_with(Algorithm::Aes256, "EFOpen", true); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert!(doc.embedded_files_locked()); + assert!(doc.reads_as_stored(&stream(&doc, EMBEDDED))); + assert_eq!(stream(&doc, CONTENT).decoded().unwrap(), PAGE.as_bytes()); + assert_eq!(attachment_with_password(bytes), ATTACHED); +} + +/// A save can't encrypt without the key: removing `/Encrypt` through the trailer, or putting in an +/// embedded file, is refused rather than written in the clear. +#[test] +fn saves_refuse_what_would_need_the_key() { + let open = || Document::open(Arc::new(fixture(Algorithm::Aes256, "EFOpen"))).unwrap(); + let mut doc = open(); + doc.trailer_mut().remove(b"Encrypt"); + assert!(matches!(write_incremental(&doc, &SaveOptions::default()), Err(CosError::Security(_)))); + assert!(matches!(write_full(&doc, &SaveOptions::default()), Err(CosError::Security(_)))); + let mut doc = open(); + let mut replaced = stream(&doc, EMBEDDED); + replaced.raw = b"new words".to_vec().into(); + doc.set(EMBEDDED, Object::Stream(replaced)); + assert!(matches!(write_incremental(&doc, &SaveOptions::default()), Err(CosError::Security(_)))); +} + +/// The password given after opening unlocks the embedded files; a wrong one doesn't. +#[test] +fn the_password_given_later_unlocks_the_embedded_files() { + let mut doc = Document::open(Arc::new(fixture(Algorithm::Aes256, "EFOpen"))).unwrap(); + assert!(matches!(doc.unlock("nope"), Err(CosError::WrongPassword))); + assert!(doc.embedded_files_locked()); + doc.unlock(PASSWORD).unwrap(); + assert!(!doc.embedded_files_locked()); + assert_eq!(stream(&doc, EMBEDDED).decoded().unwrap(), ATTACHED); + assert_eq!(stream(&doc, CONTENT).decoded().unwrap(), PAGE.as_bytes()); + // Not while an embedded file read as stored has been edited. + let mut doc = Document::open(Arc::new(fixture(Algorithm::Aes256, "EFOpen"))).unwrap(); + let edited = stream(&doc, EMBEDDED); + doc.set(EMBEDDED, Object::Stream(edited)); + assert!(matches!(doc.unlock(PASSWORD), Err(CosError::Security(_)))); +} diff --git a/crates/pdf/tests/cos/encryption.rs b/crates/pdf/tests/cos/encryption.rs new file mode 100644 index 000000000..923b04e39 --- /dev/null +++ b/crates/pdf/tests/cos/encryption.rs @@ -0,0 +1,1319 @@ +//! Encryption end to end: protect a document with every algorithm, save it, and read it back +//! with our reader and with an independent one (hayro-syntax's decryption). Also editing +//! encrypted documents incrementally, changing and removing passwords, and permissions. + +use std::io::Write; +use std::sync::Arc; + +use craft_pdf::cos::{Algorithm, Auth, CosError, Document, NewEncryption, ObjRef, Object, PdfString, SaveOptions, write_full, write_incremental}; + +const SECRET: &str = "BT /F1 18 Tf 20 100 Td (Secret page) Tj ET"; + +fn fixture() -> Vec { + fixture_with("") +} + +/// The fixture with `extra` added to the content stream's dictionary. +fn fixture_with(extra: &str) -> Vec { + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /Metadata 6 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 200] >>".to_string(), + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".to_string(), + format!("<< /Length {}{extra} >>\nstream\n{SECRET}\nendstream", SECRET.len()), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_string(), + "<< /Type /Metadata /Subtype /XML /Length 22 >>\nstream\nmeta\nendstream".replace("22", "26"), + "<< /Title (Quarterly numbers) >>".to_string(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R /Info 7 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn protect(alg: Algorithm, user: &str, owner: &str, permissions: i32) -> Vec { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: user, + owner_password: owner, + permissions, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + write_full(&doc, &SaveOptions::default()).unwrap() +} + +fn content(doc: &Document) -> String { + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let pages = doc.get(catalog.reference(b"Pages").unwrap()).as_dict().cloned().unwrap(); + let page = doc.resolve(&pages.get(b"Kids").unwrap().as_array().unwrap()[0]).as_dict().cloned().unwrap(); + match doc.resolve(page.get(b"Contents").unwrap()).as_ref() { + Object::Stream(s) => String::from_utf8_lossy(&s.decoded().unwrap()).trim().to_string(), + _ => panic!("no content stream"), + } +} + +fn title(doc: &Document) -> Option { + let info = doc.resolve(doc.trailer().get(b"Info")?); + info.as_dict()?.get(b"Title").and_then(|t| doc.resolve(t).as_string().map(|s| s.to_text())) +} + +const ALL: [Algorithm; 4] = [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256]; + +#[test] +fn every_algorithm_round_trips_and_hides_the_content() { + for alg in ALL { + let bytes = protect(alg, "user", "owner", -1); + let raw = String::from_utf8_lossy(&bytes); + assert!(!raw.contains("Secret page") && !raw.contains("Quarterly"), "{alg:?}: plaintext leaked"); + let bytes = Arc::new(bytes); + assert!(matches!(Document::open(bytes.clone()), Err(CosError::NeedsPassword)), "{alg:?}"); + assert!(matches!(Document::open_with_password(bytes.clone(), Some("guess")), Err(CosError::WrongPassword)), "{alg:?}"); + let user = Document::open_with_password(bytes.clone(), Some("user")).unwrap(); + assert_eq!(user.security().unwrap().auth(), Auth::User); + assert_eq!(content(&user), SECRET, "{alg:?}"); + assert_eq!(title(&user).as_deref(), Some("Quarterly numbers"), "{alg:?}"); + let owner = Document::open_with_password(bytes.clone(), Some("owner")).unwrap(); + assert_eq!(owner.security().unwrap().auth(), Auth::Owner); + // Independent check: hayro decrypts our output. hayro-syntax 0.7 has no owner-password + // authentication for R2–R4 (Algorithm 7), so for those qpdf is the owner-password oracle + // (`qpdf_agrees`); hayro checks every user password and R6 owner passwords. + for pw in if alg == Algorithm::Aes256 { &["user", "owner"][..] } else { &["user"][..] } { + let hay = hayro_syntax::Pdf::new_with_password(bytes.as_ref().clone(), pw).unwrap_or_else(|e| panic!("{alg:?}/{pw}: {e:?}")); + let page = hay.pages().first().unwrap().page_stream().map(|s| String::from_utf8_lossy(s).trim().to_string()); + assert_eq!(page.as_deref(), Some(SECRET), "{alg:?}/{pw}"); + } + assert!(hayro_syntax::Pdf::new(bytes.as_ref().clone()).is_err(), "{alg:?}: hayro must not open it without a password"); + } +} + +#[test] +fn empty_user_password_opens_but_reports_restrictions() { + let bytes = Arc::new(protect(Algorithm::Aes256, "", "owner", 0b0100)); // print only + let doc = Document::open(bytes.clone()).unwrap(); + let p = doc.permissions().unwrap(); + assert!(p.print() && !p.modify() && !p.copy() && !p.assemble()); + let owner = Document::open_with_password(bytes, Some("owner")).unwrap(); + assert!(owner.permissions().unwrap().modify(), "the owner may do everything"); + assert!(Document::open(Arc::new(fixture())).unwrap().permissions().is_none(), "unencrypted: no restrictions"); +} + +#[test] +fn the_permissions_password_unlocks_an_open_document() { + for alg in ALL { + // Only a permissions password: opens without asking, restricted (#995). + let mut doc = Document::open(Arc::new(protect(alg, "", "perms", 0b0100))).unwrap(); + // Read something before unlocking: it must stay readable afterwards. + assert_eq!(content(&doc), SECRET, "{alg:?}"); + assert_eq!(doc.unlock("").unwrap_err(), CosError::WrongPassword, "{alg:?}"); + assert_eq!(doc.unlock("nope").unwrap_err(), CosError::WrongPassword, "{alg:?}"); + assert!(!doc.permissions().unwrap().modify(), "{alg:?}: still restricted"); + doc.unlock("perms").unwrap(); + assert_eq!(doc.security().unwrap().auth(), Auth::Owner, "{alg:?}"); + assert!(doc.permissions().unwrap().modify(), "{alg:?}"); + assert_eq!(content(&doc), SECRET, "{alg:?}"); + assert_eq!(title(&doc).as_deref(), Some("Quarterly numbers"), "{alg:?}"); + // Unlocked, security can be removed. + doc.remove_encryption(); + let plain = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(Document::open(Arc::new(plain)).unwrap().security().is_none(), "{alg:?}"); + } + // Opened with a separate open password: that password doesn't unlock. + let mut doc = Document::open_with_password(Arc::new(protect(Algorithm::Aes256, "open", "perms", 0b0100)), Some("open")).unwrap(); + assert_eq!(doc.unlock("open").unwrap_err(), CosError::WrongPassword); + doc.unlock("perms").unwrap(); + assert!(doc.permissions().unwrap().modify()); + // Nothing to unlock: unencrypted, or already the owner. + Document::open(Arc::new(fixture())).unwrap().unlock("anything").unwrap(); + doc.unlock("anything").unwrap(); +} + +#[test] +fn incremental_edits_of_encrypted_documents_stay_encrypted() { + for alg in ALL { + let original = protect(alg, "user", "owner", -1); + let mut doc = Document::open_with_password(Arc::new(original.clone()), Some("user")).unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d| d.set(b"Title".to_vec(), Object::String(PdfString::text("Revised figures")))).unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert_eq!(&saved[..original.len()], &original[..], "{alg:?}: incremental"); + assert!(!String::from_utf8_lossy(&saved).contains("Revised"), "{alg:?}: new string encrypted"); + let again = Document::open_with_password(Arc::new(saved.clone()), Some("user")).unwrap(); + assert_eq!(title(&again).as_deref(), Some("Revised figures"), "{alg:?}"); + assert_eq!(content(&again), SECRET); + assert!(hayro_syntax::Pdf::new_with_password(saved, "user").is_ok(), "{alg:?}"); + } +} + +#[test] +fn changing_and_removing_passwords() { + let bytes = protect(Algorithm::Rc4_128, "old", "old-owner", -1); + // Change: decrypt with the old handler, encrypt with the new one. + let mut doc = Document::open_with_password(Arc::new(bytes), Some("old-owner")).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "new", + owner_password: "new-owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + let changed = Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap()); // falls back to a full save + assert!(Document::open_with_password(changed.clone(), Some("old")).is_err()); + let doc = Document::open_with_password(changed.clone(), Some("new")).unwrap(); + assert_eq!(content(&doc), SECRET); + assert_eq!(doc.security().unwrap().dict().r, 6); + // Remove. + let mut doc = Document::open_with_password(changed, Some("new-owner")).unwrap(); + doc.remove_encryption(); + let plain = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(String::from_utf8_lossy(&plain).contains("Secret page")); + let doc = Document::open(Arc::new(plain.clone())).unwrap(); + assert!(doc.security().is_none() && !doc.trailer().contains(b"Encrypt")); + assert_eq!(title(&doc).as_deref(), Some("Quarterly numbers")); + assert!(hayro_syntax::Pdf::new(plain).is_ok()); +} + +#[test] +fn unencrypted_metadata_is_left_readable() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "o", + permissions: -1, + encrypt_metadata: false, + seed: [5; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(String::from_utf8_lossy(&bytes).contains("meta"), "search engines can read XMP"); + assert!(!String::from_utf8_lossy(&bytes).contains("Secret page")); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let meta = doc.get(catalog.reference(b"Metadata").unwrap()); + let Object::Stream(s) = meta.as_ref() else { panic!() }; + assert_eq!(s.decoded().unwrap(), b"meta"); + let _ = ObjRef::new(1, 0); +} + +/// The second independent oracle: qpdf must accept both passwords, identify the owner password, +/// and find no structural errors. For R2-R4 this is the *only* check of owner-password +/// authentication (Algorithm 7) that does not go through our own code, so it is the one place a +/// silent skip would cost the most -- see `oracle.rs`, which makes absence loud. +#[test] +fn qpdf_agrees() { + let qpdf = crate::oracle::or_skip!(crate::oracle::qpdf()); + let dir = private_temp_dir("pdfcraft-enc"); + for alg in ALL { + let path = dir.join(format!("{alg:?}.pdf")); + std::fs::File::create_new(&path).and_then(|mut f| f.write_all(&protect(alg, "user", "owner", -1))).unwrap(); + for (pw, expect) in [("user", "user password"), ("owner", "owner password")] { + let out = qpdf.command().arg(format!("--password={pw}")).arg("--show-encryption").arg(&path).output().unwrap(); + let text = String::from_utf8_lossy(&out.stdout); + assert!(text.contains(&format!("Supplied password is {expect}")), "{alg:?}/{pw}: {text}"); + let check = qpdf.command().arg(format!("--password={pw}")).arg("--check").arg(&path).output().unwrap(); + assert!(check.status.success(), "{alg:?}/{pw}: {}", String::from_utf8_lossy(&check.stdout)); + } + } + let _ = std::fs::remove_dir_all(dir); +} + +/// A new directory for oracle files, private to this user: an unpredictable name (`RandomState` +/// is seeded by the operating system), created by this call or not at all. +fn private_temp_dir(prefix: &str) -> std::path::PathBuf { + use std::hash::{BuildHasher, Hasher}; + let mut name = std::collections::hash_map::RandomState::new().build_hasher(); + name.write_u32(std::process::id()); + let dir = std::env::temp_dir().join(format!("{prefix}-{:016x}", name.finish())); + #[cfg(unix)] + let builder = { + let mut builder = std::fs::DirBuilder::new(); + std::os::unix::fs::DirBuilderExt::mode(&mut builder, 0o700); + builder + }; + #[cfg(not(unix))] + let builder = std::fs::DirBuilder::new(); + builder.create(&dir).unwrap(); + dir +} + +/// Oracle files go in a directory this test created: another local account cannot have +/// prepared it (or symlinks inside it) in advance. +#[test] +fn oracle_files_go_in_a_fresh_directory() { + let (a, b) = (private_temp_dir("pdfcraft-enc-probe"), private_temp_dir("pdfcraft-enc-probe")); + assert_ne!(a, b); + assert_eq!(std::fs::read_dir(&a).unwrap().count(), 0); + let _ = (std::fs::remove_dir(a), std::fs::remove_dir(b)); +} + +/// A signature dictionary's `/Contents` is never encrypted (§7.6.2): signers patch it in place +/// after hashing the file around it. Its other strings, and the `/Contents` of anything that +/// is not a signature dictionary, are encrypted as usual, and reading leaves it untouched. +#[test] +fn signature_contents_stay_unencrypted() { + for alg in ALL { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let hex = |b: &[u8]| Object::String(PdfString { bytes: b.to_vec(), hex: true }); + let byte_range = Object::Array([0, 10, 20, 30].into_iter().map(Object::Int).collect()); + let mut sig = craft_pdf::cos::Dict::new(); + sig.set(b"Type".to_vec(), Object::name("Sig")); + sig.set(b"Contents".to_vec(), hex(&[0x30, 0x82, 0xDE, 0xAD])); + sig.set(b"ByteRange".to_vec(), byte_range.clone()); + sig.set(b"Name".to_vec(), PdfString::literal(b"Signer Name".to_vec())); + let sig = doc.add(Object::Dict(sig)); + // No /Type: still a signature dictionary by its /Contents and /ByteRange. + let mut untyped = craft_pdf::cos::Dict::new(); + untyped.set(b"Contents".to_vec(), hex(&[0x30, 0x82, 0xBE, 0xEF])); + untyped.set(b"ByteRange".to_vec(), byte_range); + let untyped = doc.add(Object::Dict(untyped)); + let mut note = craft_pdf::cos::Dict::new(); + note.set(b"Type".to_vec(), Object::name("Annot")); + note.set(b"Contents".to_vec(), PdfString::literal(b"Private note".to_vec())); + let note = doc.add(Object::Dict(note)); + doc.update_dict(doc.root().unwrap(), |c| { + c.set(b"Test".to_vec(), Object::Array(vec![Object::Ref(sig), Object::Ref(untyped), Object::Ref(note)])) + }) + .unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let raw = String::from_utf8_lossy(&bytes); + assert!(raw.contains("<3082DEAD>") && raw.contains("<3082BEEF>"), "{alg:?}: signature /Contents written as is"); + assert!(!raw.contains("Signer Name") && !raw.contains("Private note"), "{alg:?}: other strings are encrypted"); + let back = Document::open(Arc::new(bytes)).unwrap(); + let test = back.get(back.root().unwrap()).as_dict().unwrap().get(b"Test").unwrap().as_array().unwrap().clone(); + let string = |i: usize, key: &[u8]| { + let o = back.resolve(&test[i]); + o.as_dict().unwrap().get(key).unwrap().as_string().unwrap().bytes.clone() + }; + assert_eq!(string(0, b"Contents"), [0x30, 0x82, 0xDE, 0xAD], "{alg:?}: read back without decrypting"); + assert_eq!(string(0, b"Name"), b"Signer Name", "{alg:?}"); + assert_eq!(string(1, b"Contents"), [0x30, 0x82, 0xBE, 0xEF], "{alg:?}"); + assert_eq!(string(2, b"Contents"), b"Private note", "{alg:?}"); + } +} + +/// Only a signature dictionary keeps its `/Contents` in clear. An annotation that carries a +/// `/ByteRange` (or even `/Type /Sig`) is still an annotation, and its comment is encrypted. +#[test] +fn a_byte_range_does_not_exempt_an_annotation() { + for (ty, subtype) in [(Some("Annot"), "Text"), (None, "FreeText"), (Some("Sig"), "Text")] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut note = craft_pdf::cos::Dict::new(); + if let Some(ty) = ty { + note.set(b"Type".to_vec(), Object::name(ty)); + } + note.set(b"Subtype".to_vec(), Object::name(subtype)); + note.set(b"Contents".to_vec(), PdfString::literal(b"Private note".to_vec())); + note.set(b"ByteRange".to_vec(), Object::Array(Vec::new())); + let note = doc.add(Object::Dict(note)); + doc.update_dict(doc.root().unwrap(), |c| c.set(b"Test".to_vec(), Object::Ref(note))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("Private note"), "{ty:?} {subtype}: comment encrypted"); + let back = Document::open(Arc::new(bytes)).unwrap(); + let note = back.get(back.root().unwrap()).as_dict().unwrap().reference(b"Test").unwrap(); + assert_eq!(back.get(note).as_dict().unwrap().get(b"Contents").unwrap().as_string().unwrap().bytes, b"Private note"); + } +} + +/// Protecting a document puts every stream under the new handler. A stream that selects a +/// crypt filter of its own loses the selector: the new `/Encrypt` defines only `/StdCF`, so an +/// inherited name (or `/Identity`) would leave the stream in clear. +#[test] +fn protecting_drops_inherited_crypt_filter_selectors() { + for selector in [" /Filter /Crypt /DecodeParms << /Name /MyCF >>", " /Filter [/Crypt] /DecodeParms [<< /Name /Identity >>]", " /Filter /Crypt"] { + let mut doc = Document::open(Arc::new(fixture_with(selector))).unwrap(); + assert_eq!(content(&doc), SECRET, "{selector}"); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("Secret page"), "{selector}: stream encrypted"); + assert_eq!(content(&Document::open(Arc::new(bytes.clone())).unwrap()), SECRET, "{selector}"); + assert!(hayro_syntax::Pdf::new(bytes).is_ok(), "{selector}"); + } + // Under the handler the document was opened with, `/Identity` is the author's choice (a + // stream meant to stay readable) and is kept. + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let pages = doc.get(catalog.reference(b"Pages").unwrap()).as_dict().cloned().unwrap(); + let page = doc.resolve(&pages.get(b"Kids").unwrap().as_array().unwrap()[0]).as_dict().cloned().unwrap(); + let contents = page.reference(b"Contents").unwrap(); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Filter".to_vec(), Object::name("Crypt")); + doc.set(contents, Object::Stream(s)); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(String::from_utf8_lossy(&saved).contains("Secret page"), "written as the stream asks"); + assert_eq!(content(&Document::open(Arc::new(saved)).unwrap()), SECRET); +} + +#[test] +fn a_nonpositive_crypt_filter_length_reads_as_the_default_key_length() { + // A V4 crypt filter's /Length counts bytes (≤ 32) or bits. A zero or negative value is + // meaningless and is read as if absent (128 bits), not multiplied into an overflow (#1816). + let original = protect(Algorithm::Aes128, "", "owner", -1); + for length in [i64::MIN, i64::MIN / 8 - 1, i64::MIN / 8, -1, 0, 16, 128] { + let mut doc = Document::open(Arc::new(original.clone())).unwrap(); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + doc.update_dict(encrypt, |e| { + let Some(Object::Dict(cf)) = e.get(b"CF") else { panic!("no /CF") }; + let Some(Object::Dict(std_cf)) = cf.get(b"StdCF") else { panic!("no /StdCF") }; + let (mut cf, mut std_cf) = (cf.clone(), std_cf.clone()); + std_cf.set(b"Length".to_vec(), Object::Int(length)); + cf.set(b"StdCF".to_vec(), Object::Dict(std_cf)); + e.set(b"CF".to_vec(), Object::Dict(cf)); + }) + .unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let again = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(content(&again), SECRET, "/Length {length}"); + } +} + +#[test] +fn an_earlier_revision_opens_with_the_documents_own_credentials() { + // The bytes a signature covers are an earlier revision of the file: the document that is + // open can read them with its own key, so no password has to be kept or asked for again. + for alg in ALL { + let original = protect(alg, "user", "owner", -1); + let mut doc = Document::open_with_password(Arc::new(original.clone()), Some("user")).unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d| d.set(b"Title".to_vec(), Object::String(PdfString::text("Revised")))).unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let current = Document::open_with_password(Arc::new(saved), Some("user")).unwrap(); + assert!(matches!(Document::open(Arc::new(original.clone())), Err(CosError::NeedsPassword)), "{alg:?}: not without a password"); + let earlier = current.open_with_same_credentials(Arc::new(original.clone())).unwrap(); + assert_eq!(content(&earlier), SECRET, "{alg:?}: the earlier revision is readable"); + assert_eq!(title(&earlier).as_deref(), Some("Quarterly numbers"), "{alg:?}: and it is the earlier one"); + // Another file, encrypted with other keys: the document's key is no use, and it says so. + let other = { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + write_full(&doc, &SaveOptions::default()).unwrap() + }; + assert!(matches!(current.open_with_same_credentials(Arc::new(other)), Err(CosError::NeedsPassword)), "{alg:?}"); + // An unencrypted file opens as it always does. + assert_eq!(content(¤t.open_with_same_credentials(Arc::new(fixture())).unwrap()), SECRET, "{alg:?}"); + } +} + +/// `protect`, with the `/Encrypt` dictionary written inline in the trailer, as MuPDF writes it, +/// instead of as an indirect object. +fn protect_inline(alg: Algorithm, user: &str, owner: &str) -> Vec { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: user, + owner_password: owner, + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let dict = doc.get(doc.trailer().reference(b"Encrypt").unwrap()).as_dict().cloned().unwrap(); + doc.trailer_mut().set(b"Encrypt".to_vec(), Object::Dict(dict)); + write_full(&doc, &SaveOptions::default()).unwrap() +} + +/// MuPDF writes `/Encrypt` inline in the trailer, so removing it changes no object. It is still +/// an unsaved change, and saving must write a file that opens without a password (#2661). +#[test] +fn removing_an_inline_encrypt_dictionary_leaves_an_unencrypted_file() { + for alg in ALL { + let bytes = Arc::new(protect_inline(alg, "user", "owner")); + assert!(matches!(Document::open(bytes.clone()), Err(CosError::NeedsPassword)), "{alg:?}"); + let mut doc = Document::open_with_password(bytes, Some("owner")).unwrap(); + assert!(matches!(doc.trailer().get(b"Encrypt"), Some(Object::Dict(_))), "{alg:?}: inline"); + assert!(!doc.is_modified(), "{alg:?}"); + doc.remove_encryption(); + assert!(doc.is_modified(), "{alg:?}: removing security is an unsaved change"); + let plain = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let again = Document::open(Arc::new(plain.clone())).unwrap_or_else(|e| panic!("{alg:?}: {e:?}")); + assert!(again.security().is_none() && !again.trailer().contains(b"Encrypt"), "{alg:?}"); + assert_eq!(content(&again), SECRET, "{alg:?}"); + assert!(hayro_syntax::Pdf::new(plain).is_ok(), "{alg:?}"); + } +} + +/// A content stream that calls itself `/Type /XRef` is still encrypted: only reading trusts the label. +#[test] +fn a_stream_labelled_xref_is_encrypted_when_saved() { + let mut doc = Document::open(Arc::new(fixture_with(" /Type /XRef"))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "u", + owner_password: "o", + permissions: -4, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the labelled stream was written in the clear"); +} + +/// The content stream's reference. +fn contents_of(doc: &Document) -> ObjRef { + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let pages = doc.get(catalog.reference(b"Pages").unwrap()).as_dict().cloned().unwrap(); + let page = doc.resolve(&pages.get(b"Kids").unwrap().as_array().unwrap()[0]).as_dict().cloned().unwrap(); + page.reference(b"Contents").unwrap() +} + +/// The content stream's data as loaded, whatever its filters say. +fn raw_content(doc: &Document) -> String { + match &*doc.get(contents_of(doc)) { + Object::Stream(s) => String::from_utf8_lossy(&s.raw).trim().to_string(), + _ => panic!("no content stream"), + } +} + +/// Gives the content stream the `/Filter` and `/DecodeParms` of `selector`. +fn select(doc: &mut Document, (filter, parms): (Object, Object)) { + let contents = contents_of(doc); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Filter".to_vec(), filter); + s.dict.set(b"DecodeParms".to_vec(), parms); + doc.set(contents, Object::Stream(s)); +} + +/// `/Crypt` naming `name`, in the `form`th spelling: all direct, then each part by reference. +/// `None` past the last. +fn selector(doc: &mut Document, form: usize, name: &str) -> Option<(Object, Object)> { + let crypt = Object::name("Crypt"); + let parms = |n: Object| { + let mut d = craft_pdf::cos::Dict::new(); + d.set(b"Name".to_vec(), n); + Object::Dict(d) + }; + let named = parms(Object::name(name)); + Some(match form { + 0 => (crypt, named), + 1 => (crypt, Object::Ref(doc.add(named))), + 2 => (Object::Array(vec![crypt]), Object::Array(vec![Object::Ref(doc.add(named))])), + 3 => (Object::Array(vec![crypt]), Object::Ref(doc.add(Object::Array(vec![named])))), + 4 => (crypt, parms(Object::Ref(doc.add(Object::name(name))))), + 5 => (Object::Ref(doc.add(crypt)), named), + 6 => (Object::Ref(doc.add(Object::Array(vec![crypt]))), Object::Array(vec![named])), + 7 => (Object::Array(vec![Object::Ref(doc.add(crypt))]), Object::Array(vec![named])), + // Behind a filter given by reference: places in the two arrays pair up all the same. + 8 => (Object::Array(vec![Object::Ref(doc.add(Object::name("ASCIIHexDecode"))), crypt]), Object::Array(vec![Object::Null, named])), + // The two entries in different forms, which is how a lone filter is usually spelled. + 9 => (Object::Array(vec![crypt]), named), + 10 => (crypt, Object::Array(vec![named])), + // The filter twice, which names the first's. + 11 => (Object::Array(vec![crypt.clone(), crypt]), Object::Array(vec![named.clone(), named])), + // A reference to a reference. + 12 => { + let name = doc.add(Object::name(name)); + let hop = doc.add(Object::Ref(name)); + (crypt, parms(Object::Ref(hop))) + } + _ => return None, + }) +} + +fn holds(bytes: &[u8], text: &str) -> bool { + String::from_utf8_lossy(bytes).contains(text) +} + +/// An object `/Encrypt` names is written as stored only while it is as stored. A file whose +/// `/Encrypt` names its `/Info` reads that dictionary as stored; once the title is edited, both +/// saves encrypt it rather than writing the new title in the clear. +#[test] +fn an_edited_object_that_encrypt_names_is_encrypted() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + let mut dict = doc.get(doc.trailer().reference(b"Encrypt").unwrap()).as_dict().cloned().unwrap(); + dict.set(b"Junk".to_vec(), Object::Ref(info)); + doc.trailer_mut().set(b"Encrypt".to_vec(), Object::Dict(dict)); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d| d.set(b"Title".to_vec(), PdfString::text("Merger plan"))).unwrap(); + for (save, saved) in [("incremental", write_incremental(&doc, &SaveOptions::default())), ("full", write_full(&doc, &SaveOptions::default()))] { + assert!(!holds(&saved.unwrap(), "Merger plan"), "{save}: the edited title was written in the clear"); + } +} + +/// A stream can select its crypt filter by reference at every step: the chain, an entry of it, the +/// parameters, an entry of those, the name (ISO 32000-2 §7.3.10, §7.4.10). Read as absent, the +/// filter was `/Identity`, so such a stream was saved in the clear and, in a file from elsewhere, +/// its ciphertext read as data. A full save renumbers: what it follows has the document's numbers. +#[test] +fn a_crypt_filter_selected_by_reference_is_used() { + for alg in [Algorithm::Aes128, Algorithm::Aes256] { + for form in 0.. { + let mut doc = Document::open(Arc::new(protect(alg, "", "owner", -1))).unwrap(); + let Some(selected) = selector(&mut doc, form, "StdCF") else { break }; + select(&mut doc, selected); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&saved, "Secret page"), "{alg:?} form {form}: saved encrypted"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "{alg:?} form {form}: read back"); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(!holds(&full, "Secret page"), "{alg:?} form {form}: renumbered, encrypted"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "{alg:?} form {form}: renumbered"); + } + } +} + +/// A reference to a stream that takes `follows` references to read cold, this one included: its +/// nine filters are all read, the last leading to the next stream. +fn spends(doc: &mut Document, follows: usize) -> Object { + let left = follows - 1; + let mut filters: Vec = (0..left.min(8)).map(|i| Object::Ref(doc.add(Object::Int(i as i64)))).collect(); + if left > 8 { + filters.push(spends(doc, left - 8)); + } + let mut dict = craft_pdf::cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(filters)); + Object::Ref(doc.add(Object::Stream(craft_pdf::cos::Stream::from_raw(dict, Vec::new())))) +} + +/// Gives the content stream a `/Type` that takes `follows` references to read, ahead of its filter. +fn spend_first(doc: &mut Document, follows: usize) { + let spent = spends(doc, follows); + let contents = contents_of(doc); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Type".to_vec(), spent); + doc.set(contents, Object::Stream(s)); +} + +/// Past what one read may follow a reference reads as absent, and a crypt filter with no `/Name` is +/// `/Identity`. Together those would read this stream's ciphertext as data, so what can't be told +/// is the document's own filter, as a `/Filter` that can't be read is. +#[test] +fn a_crypt_filter_past_what_a_read_may_follow_is_the_documents_own() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let selected = selector(&mut doc, 1, "StdCF").unwrap(); + select(&mut doc, selected); + spend_first(&mut doc, 40); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&saved, "Secret page"), "saved encrypted"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(!holds(&full, "Secret page"), "renumbered, encrypted"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "renumbered"); +} + +/// A read that takes all it may follow has refused nothing: a crypt filter with no `/Name` after it +/// is still `/Identity`, and the stream is in the clear. +#[test] +fn a_crypt_filter_with_no_name_after_all_a_read_may_follow_is_identity() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + select(&mut doc, (Object::name("Crypt"), Object::Dict(craft_pdf::cos::Dict::new()))); + spend_first(&mut doc, 32); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&saved, "Secret page"), "written as the stream asks"); + assert_eq!(raw_content(&Document::open(Arc::new(saved)).unwrap()), SECRET); +} + +/// `/Identity` selected by reference keeps a stream in the clear in a document that encrypts the +/// rest, in every spelling, on both saves. +#[test] +fn an_identity_crypt_filter_selected_by_reference_leaves_a_stream_alone() { + for form in 0.. { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let Some(selected) = selector(&mut doc, form, "Identity") else { break }; + select(&mut doc, selected); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&saved, "Secret page"), "form {form}: written as the stream asks"); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "form {form}"); + let full = write_full(&back, &SaveOptions::default()).unwrap(); + assert!(holds(&full, "Secret page"), "form {form}: renumbered"); + assert_eq!(raw_content(&Document::open(Arc::new(full)).unwrap()), SECRET, "form {form}: renumbered"); + } +} + +/// New protection drops a crypt filter it doesn't define, whichever way the stream selected it. The +/// chain comes out direct. Parameters in the stream's own dictionary stay, encrypted with it; of an +/// array in an object of its own, which every stream can share, only what decoders read is copied. +#[test] +fn protecting_drops_a_crypt_filter_selected_by_reference() { + let protected = |doc: &mut Document| { + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + write_full(doc, &SaveOptions::default()).unwrap() + }; + for form in 0.. { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let Some(selected) = selector(&mut doc, form, "MyCF") else { break }; + select(&mut doc, selected); + let bytes = protected(&mut doc); + assert!(!holds(&bytes, "Secret page"), "form {form}: encrypted"); + assert!(!holds(&bytes, "MyCF"), "form {form}: no selector left in the stream"); + let back = Document::open(Arc::new(bytes.clone())).unwrap(); + assert_eq!(raw_content(&back), SECRET, "form {form}"); + let contents = back.get(contents_of(&back)); + let filter = contents.as_dict().unwrap().get(b"Filter"); + match form { + 8 => assert!( + matches!(filter, Some(Object::Array(a)) if matches!(a.as_slice(), [Object::Ref(_)])), + "form 8: the other filter stays: {filter:?}" + ), + _ => assert_eq!(filter, None, "form {form}"), + } + assert!(hayro_syntax::Pdf::new(bytes).is_ok(), "form {form}"); + } + for shared in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut named = craft_pdf::cos::Dict::new(); + named.set(b"Name".to_vec(), Object::name("MyCF")); + let mut other = craft_pdf::cos::Dict::new(); + other.set(b"Note".to_vec(), Object::String(PdfString::text("Private note"))); + other.set(b"Predictor".to_vec(), Object::Int(1)); + let parms = Object::Array(vec![Object::Dict(named), Object::Dict(other)]); + let parms = if shared { Object::Ref(doc.add(parms)) } else { parms }; + select(&mut doc, (Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")]), parms)); + let bytes = protected(&mut doc); + assert!(!holds(&bytes, "Private note"), "shared: {shared}"); + let back = Document::open(Arc::new(bytes)).unwrap(); + let contents = back.get(contents_of(&back)); + let kept = contents.as_dict().unwrap().get(b"DecodeParms").unwrap().as_array().unwrap(); + assert_eq!(kept.len(), 1, "one filter left, one entry"); + let kept = kept[0].as_dict().unwrap(); + assert_eq!(kept.get(b"Predictor"), Some(&Object::Int(1)), "shared: {shared}"); + assert_eq!(kept.get(b"Note").is_some(), !shared, "shared: {shared}"); + } +} + +/// Every stream of a file can share one large object as its `/DecodeParms` or `/Filter`, which +/// costs nothing until new protection drops a crypt filter from each. Copied whole, a 1 MB file +/// came out at 1 GB. +#[test] +fn protecting_copies_no_large_object_into_every_stream() { + let pad = Object::Array(vec![Object::Int(1); 100_000]); + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut large = craft_pdf::cos::Dict::new(); + large.set(b"Pad".to_vec(), pad.clone()); + let parms = doc.add(Object::Array(vec![Object::Null, Object::Dict(large)])); + let filter = doc.add(Object::Array(vec![Object::name("Crypt"), Object::Name(vec![b'A'; 100_000]), pad])); + let mut streams = Vec::new(); + for _ in 0..200 { + let mut by_parms = craft_pdf::cos::Dict::new(); + by_parms.set(b"Filter".to_vec(), Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")])); + by_parms.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + streams.push(Object::Ref(doc.add(Object::Stream(craft_pdf::cos::Stream::from_raw(by_parms, b"41>".to_vec()))))); + let mut by_filter = craft_pdf::cos::Dict::new(); + by_filter.set(b"Filter".to_vec(), Object::Ref(filter)); + streams.push(Object::Ref(doc.add(Object::Stream(craft_pdf::cos::Stream::from_raw(by_filter, b"41>".to_vec()))))); + } + doc.update_dict(doc.root().unwrap(), |c| c.set(b"Test".to_vec(), Object::Array(streams))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(bytes.len() < 1 << 20, "{} bytes", bytes.len()); +} + +/// A file can name one large string thousands of times in `/ID`, which holds two. An encrypted file +/// keeps its `/ID`, so each copy would be in the file a save writes. +#[test] +fn a_file_identifier_of_many_references_is_two_strings() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let large = doc.add(Object::String(PdfString { bytes: vec![0xAB; 100_000], hex: true })); + doc.trailer_mut().set(b"ID".to_vec(), Object::Array(vec![Object::Ref(large); 500])); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(bytes.len() < 1 << 20, "{} bytes", bytes.len()); +} + +/// What a file can do with those references: point a stream's selector at the stream itself, or at +/// nothing. They read as absent, so a stream with `/Crypt` is `/Identity` (ISO 32000-2 §7.4.10), +/// and one that names no filter follows the document's. Loading ends either way. +#[test] +fn a_crypt_filter_reference_to_itself_or_nothing_reads_as_absent() { + let named = |n: Object| { + let mut d = craft_pdf::cos::Dict::new(); + d.set(b"Name".to_vec(), n); + Object::Dict(d) + }; + for (what, encrypted) in [ + ("/Filter to itself", true), + ("/Filter nowhere", true), + ("/DecodeParms to itself", false), + ("/DecodeParms nowhere", false), + ("/Name to itself", false), + ("/Name nowhere", false), + ] { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let me = Object::Ref(contents_of(&doc)); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + let crypt = Object::name("Crypt"); + select( + &mut doc, + match what { + "/Filter to itself" => (me, Object::Null), + "/Filter nowhere" => (nowhere, Object::Null), + "/DecodeParms to itself" => (crypt, me), + "/DecodeParms nowhere" => (crypt, nowhere), + "/Name to itself" => (crypt, named(me)), + _ => (crypt, named(nowhere)), + }, + ); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert_eq!(!holds(&saved, "Secret page"), encrypted, "{what}"); + assert_eq!(raw_content(&Document::open(Arc::new(saved)).unwrap()), SECRET, "{what}"); + } +} + +/// A lone parameter dictionary belongs to the first filter, so `/Crypt` behind another filter has +/// none of its own and is `/Identity` (ISO 32000-2 §7.4.1), whatever that dictionary says. +#[test] +fn a_lone_parameter_dictionary_is_the_first_filters() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let mut parms = craft_pdf::cos::Dict::new(); + parms.set(b"Name".to_vec(), Object::name("StdCF")); + let first = Object::Array(vec![Object::name("Crypt"), Object::name("ASCIIHexDecode")]); + let second = Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("Crypt")]); + select(&mut doc, (first, Object::Dict(parms.clone()))); + assert!(!holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "first: it names StdCF"); + select(&mut doc, (second, Object::Dict(parms))); + assert!(holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "second: it names nothing"); +} + +/// A lone parameter dictionary is the first filter's, so dropping a `/Crypt` behind that filter +/// leaves it, and dropping one in front of it takes it away. +#[test] +fn protecting_keeps_the_parameters_of_the_filter_before_a_crypt_filter() { + let protected = |filters: Vec| { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let mut parms = craft_pdf::cos::Dict::new(); + parms.set(b"Name".to_vec(), Object::name("MyCF")); + select(&mut doc, (Object::Array(filters), Object::Dict(parms))); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let back = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + back.get(contents_of(&back)).as_dict().unwrap().contains(b"DecodeParms") + }; + let (hex, crypt) = (Object::name("ASCIIHexDecode"), Object::name("Crypt")); + assert!(protected(vec![hex.clone(), crypt.clone()]), "the first filter's, which stays"); + assert!(!protected(vec![crypt, hex]), "/Crypt's, which goes"); +} + +/// Dropping `/Crypt` takes its entry out of `/DecodeParms`, and an array too short to have one is +/// left as it is, given directly or by reference. +#[test] +fn protecting_leaves_a_parameter_array_too_short_for_the_crypt_filter() { + for by_reference in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let short = Object::Array(vec![Object::Null]); + let parms = if by_reference { Object::Ref(doc.add(short)) } else { short }; + select(&mut doc, (Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("Crypt")]), parms)); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!holds(&bytes, "Secret page"), "by reference: {by_reference}"); + let back = Document::open(Arc::new(bytes)).unwrap(); + assert_eq!(raw_content(&back), SECRET, "by reference: {by_reference}"); + let contents = back.get(contents_of(&back)); + let filter = contents.as_dict().unwrap().get(b"Filter").unwrap().as_array().unwrap(); + assert_eq!(filter.len(), 1, "only /Crypt goes"); + assert_eq!(filter[0].as_name(), Some(b"ASCIIHexDecode".as_slice()), "the other filter stays as it was"); + } +} + +/// A chain of up to [`craft_pdf::filters::MAX_CHAIN`] filters and `/Crypt` is read; a longer one +/// names no crypt filter, so its entries aren't followed for every stream sharing it. +#[test] +fn a_chain_longer_than_any_decoder_takes_names_no_crypt_filter() { + let chain = |others: usize| { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let mut filters = vec![Object::name("ASCIIHexDecode"); others]; + filters.push(Object::name("Crypt")); + let array = doc.add(Object::Array(filters)); + select(&mut doc, (Object::Ref(array), Object::Null)); + holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page") + }; + assert!(chain(craft_pdf::filters::MAX_CHAIN), "the longest chain read: /Identity, in the clear"); + assert!(!chain(craft_pdf::filters::MAX_CHAIN + 1), "one more: the document's filter"); +} + +/// A stream's `/Type` by reference picks its handling like one given directly: metadata left +/// readable under `/EncryptMetadata false`, an embedded file under `/EFF`. +#[test] +fn a_stream_type_given_by_reference_picks_its_crypt_filter() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let catalog = doc.get(doc.root().unwrap()).as_dict().cloned().unwrap(); + let meta = catalog.reference(b"Metadata").unwrap(); + let ty = doc.add(Object::name("Metadata")); + let Object::Stream(mut s) = doc.get(meta).as_ref().clone() else { panic!("no metadata stream") }; + s.dict.set(b"Type".to_vec(), Object::Ref(ty)); + doc.set(meta, Object::Stream(s)); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "o", + permissions: -1, + encrypt_metadata: false, + seed: [5; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(holds(&bytes, "meta"), "search engines can read XMP"); + + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + doc.update_dict(encrypt, |e| e.set(b"EFF".to_vec(), Object::name("Identity"))).unwrap(); + let mut doc = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let contents = contents_of(&doc); + let ty = doc.add(Object::name("EmbeddedFile")); + let Object::Stream(mut s) = doc.get(contents).as_ref().clone() else { panic!("no content stream") }; + s.dict.set(b"Type".to_vec(), Object::Ref(ty)); + doc.set(contents, Object::Stream(s)); + assert!(holds(&write_incremental(&doc, &SaveOptions::default()).unwrap(), "Secret page"), "/EFF /Identity leaves it alone"); +} + +/// What makes a dictionary a signature's can be given by reference, and what is null or refers to +/// nothing is absent: `/Contents` stays as written in each of these. +#[test] +fn a_signature_dictionary_is_known_in_every_form() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let ty = doc.add(Object::name("Sig")); + let range = Object::Array([0, 10, 20, 30].into_iter().map(Object::Int).collect()); + let by_reference = doc.add(range.clone()); + let nowhere = Object::Ref(ObjRef::new(9_999, 0)); + let forms: [(Vec<(&str, Object)>, &str); 5] = [ + (vec![("Type", Object::Ref(ty))], "DEAD"), + (vec![("ByteRange", Object::Ref(by_reference))], "BEEF"), + (vec![("Type", Object::Null), ("ByteRange", range.clone())], "CAFE"), + (vec![("Type", nowhere.clone()), ("ByteRange", range.clone())], "F00D"), + (vec![("Subtype", nowhere), ("ByteRange", range)], "D00D"), + ]; + let mut signatures = Vec::new(); + for (entries, contents) in forms { + let mut sig = craft_pdf::cos::Dict::new(); + for (key, value) in entries { + sig.set(key.as_bytes().to_vec(), value); + } + let bytes = (0..contents.len()).step_by(2).map(|i| u8::from_str_radix(&contents[i..i + 2], 16).unwrap()).collect(); + sig.set(b"Contents".to_vec(), Object::String(PdfString { bytes, hex: true })); + signatures.push(Object::Ref(doc.add(Object::Dict(sig)))); + } + // Nested in another object, it follows nothing, but known directly it is one all the same. + let mut nested = craft_pdf::cos::Dict::new(); + nested.set(b"Type".to_vec(), Object::name("Sig")); + nested.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0xAB, 0xBA], hex: true })); + let mut field = craft_pdf::cos::Dict::new(); + field.set(b"V".to_vec(), Object::Dict(nested)); + signatures.push(Object::Ref(doc.add(Object::Dict(field)))); + // Neither of what makes one: a string /Contents alone is anybody's, so it's encrypted. + let mut other = craft_pdf::cos::Dict::new(); + other.set(b"Contents".to_vec(), Object::String(PdfString { bytes: vec![0xAB, 0xCD], hex: true })); + signatures.push(Object::Ref(doc.add(Object::Dict(other)))); + doc.update_dict(doc.root().unwrap(), |c| c.set(b"Test".to_vec(), Object::Array(signatures))).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [3; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + for contents in ["DEAD", "BEEF", "CAFE", "F00D", "D00D", "ABBA"] { + assert!(holds(&bytes, &format!("<{contents}>")), "{contents}: written as is"); + } + assert!(!holds(&bytes, ""), "without a /ByteRange it is no signature"); +} + +/// What a test needs to write an update of a classic file protected with AES-128. +struct Classic { + bytes: Vec, + root: ObjRef, + pages: ObjRef, + info: ObjRef, + encrypt: ObjRef, + id: Vec, + dict: craft_pdf::cos::Dict, +} + +fn classic_file() -> Classic { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + let encrypt = doc.trailer().reference(b"Encrypt").unwrap(); + let id = doc.trailer().get(b"ID").unwrap().as_array().unwrap()[0].as_string().unwrap().bytes.clone(); + let dict = doc.get(encrypt).as_dict().cloned().unwrap(); + let pages = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Pages").unwrap(); + Classic { root: doc.root().unwrap(), pages, info: doc.trailer().reference(b"Info").unwrap(), bytes, encrypt, id, dict } +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02X}")).collect() +} + +/// `bytes` with an update appended: `objects`, and a trailer of `trailer` entries, in which `@n` +/// stands for the offset of object `n`. An object with nothing in it gets an entry of the +/// cross-reference table that points at nothing. +fn updated(bytes: &[u8], objects: &[(u32, Vec)], trailer: &str) -> Vec { + let at = bytes.windows(9).rposition(|w| w == b"startxref").unwrap(); + let prev: usize = String::from_utf8_lossy(&bytes[at + 9..]).split_whitespace().next().unwrap().parse().unwrap(); + let mut out = bytes.to_vec(); + let mut table = String::from("xref\n"); + let mut trailer = trailer.to_string(); + for (num, body) in objects { + if body.is_empty() { + table.push_str(&format!("{num} 1\n0000000010 00000 n \n")); + continue; + } + trailer = trailer.replace(&format!("@{num}"), &out.len().to_string()); + table.push_str(&format!("{num} 1\n{:010} 00000 n \n", out.len())); + out.extend_from_slice(format!("{num} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(table.as_bytes()); + out.extend_from_slice(format!("trailer\n<< {trailer} /Size 2000 /Prev {prev} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + out +} + +/// The `/Encrypt` dictionary of `file`, with `/O`, `/StmF` and `/StrF` as given. +fn encrypt_with(file: &Classic, o: &str, filter: &str) -> Vec { + let string = |key: &[u8]| hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes); + format!( + "<< /Filter /Standard /V 4 /R 4 /Length 128 /P {} /O {o} /U <{}> /CF << /StdCF << /CFM /AESV2 /Length 16 /AuthEvent /DocOpen >> >> /StmF /{filter} /StrF /{filter} >>", + file.dict.int(b"P").unwrap(), + string(b"U") + ) + .into_bytes() +} + +/// An object stream can have its parameters by reference, and a stream's crypt filter can be +/// selected by a member of it: legal together, though each is rare. What reads the stream first +/// goes through the object stream, and members must come out of it as they do read directly. They +/// didn't, while a read made for a selector wasn't allowed to follow its own: the object stream was +/// kept undecoded, its members lost for good, and a full save wrote nulls without saying so. +#[test] +fn an_object_stream_reached_from_a_selector_keeps_its_members() { + let file = classic_file(); + let parms = "[<< /Name /Identity >> null]"; + let header = format!("902 0 903 {} ", parms.len() + 1); + let members = format!("{header}{parms} << /Keep (yes) >>"); + let (mut rows, mut before) = (vec![1u8], 0u8); + for b in members.bytes() { + rows.push(b.wrapping_sub(before)); + before = b; + } + let coded = craft_pdf::cos::Stream::flate(craft_pdf::cos::Dict::new(), &rows).raw; + let mut object_stream = + format!("<< /Type /ObjStm /N 2 /First {} /Filter /FlateDecode /DecodeParms 911 0 R /Length {} >>\nstream\n", header.len(), coded.len()) + .into_bytes(); + object_stream.extend_from_slice(&coded); + object_stream.extend_from_slice(b"\nendstream"); + let data = [2u8, 0x03, 0x8E, 0, 2, 0x03, 0x8E, 1]; + let mut xref = format!("<< /Type /XRef /Size 2000 /W [1 2 1] /Index [902 2] /Length {} >>\nstream\n", data.len()).into_bytes(); + xref.extend_from_slice(&data); + xref.extend_from_slice(b"\nendstream"); + let string = |key: &[u8]| format!("<{}>", hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes)); + let objects = [ + (file.root.num, format!("<< /Type /Catalog /Pages {} 0 R /A 900 0 R /B 903 0 R >>", file.pages.num).into_bytes()), + (file.encrypt.num, encrypt_with(&file, &string(b"O"), "Identity")), + (900, b"<< /Filter [/Crypt /ASCIIHexDecode] /DecodeParms 902 0 R /Length 1 >>\nstream\n>\nendstream".to_vec()), + (910, object_stream), + (911, format!("<< /Predictor 11 /Columns {} >>", members.len()).into_bytes()), + (912, xref), + ]; + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>] /XRefStm @912", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let keep = + |doc: &Document, r: &Object| doc.resolve(r).as_dict().and_then(|d| d.get(b"Keep")).and_then(Object::as_string).map(|s| s.bytes.clone()); + let member = Object::Ref(ObjRef::new(903, 0)); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + assert_eq!(keep(&doc, &member).as_deref(), Some(b"yes".as_slice()), "read first"); + let doc = Document::open(Arc::new(bytes.clone())).unwrap(); + let _ = doc.get(ObjRef::new(900, 0)); + assert_eq!(keep(&doc, &member).as_deref(), Some(b"yes".as_slice()), "read after the stream selecting from it"); + let saved = write_full(&Document::open(Arc::new(bytes)).unwrap(), &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + let catalog = back.get(back.root().unwrap()); + assert_eq!(keep(&back, catalog.as_dict().unwrap().get(b"B").unwrap()).as_deref(), Some(b"yes".as_slice()), "after a full save"); +} + +/// `walk` goes as deep as an object nests, so following a reference from down there adds that depth +/// again for each object it loads: this file, under 10 KB, overflowed a 2 MiB stack and took the +/// process with it. Only the object itself follows references, and it does so in very little stack. +#[test] +fn nested_dictionaries_load_nothing_on_a_small_stack() { + let file = classic_file(); + let first = 960u32; + let objects: Vec<(u32, Vec)> = (first..first + 40) + .map(|num| { + let open = "[".repeat(120); + let close = "]".repeat(120); + (num, format!("{open}<< /Contents () /Type {} 0 R >>{close}", num + 1).into_bytes()) + }) + .collect(); + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let reading = std::thread::Builder::new().stack_size(1 << 20).spawn(move || { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.get(ObjRef::new(first, 0)).as_array().is_some() + }); + assert_eq!(reading.unwrap().join().ok(), Some(true)); +} + +/// The object itself can follow references, a ladder of them as long as loading may nest, and that +/// fits a small stack too. +#[test] +fn a_ladder_of_signature_checks_fits_a_small_stack() { + let file = classic_file(); + let first = 960u32; + let objects: Vec<(u32, Vec)> = + (first..first + 40).map(|num| (num, format!("<< /Contents () /Type {} 0 R >>", num + 1).into_bytes())).collect(); + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.encrypt.num, id = hex(&file.id)); + let bytes = updated(&file.bytes, &objects, &trailer); + let reading = std::thread::Builder::new().stack_size(1 << 20).spawn(move || { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.get(ObjRef::new(first, 0)).as_dict().is_some() + }); + assert_eq!(reading.unwrap().join().ok(), Some(true)); +} + +/// The strings of `/Encrypt` are never encrypted (ISO 32000-2 §7.6.2), and can be objects of their +/// own. Read as ordinary strings they were decrypted, and a full save encrypted them again under +/// another number, so the file it wrote no longer opened with its own password. +#[test] +fn encrypt_strings_given_by_reference_stay_as_stored() { + let file = classic_file(); + let string = |key: &[u8]| hex(&file.dict.get(key).unwrap().as_string().unwrap().bytes); + let replaced = format!( + "<< /Filter /Standard /V 4 /R 4 /Length 128 /P {} /O 900 0 R /U 901 0 R /CF << /StdCF << /CFM /AESV2 /Length 16 /AuthEvent /DocOpen >> >> /StmF /StdCF /StrF /StdCF >>", + file.dict.int(b"P").unwrap() + ); + let objects = [ + (file.encrypt.num, replaced.into_bytes()), + (900, format!("<{}>", string(b"O")).into_bytes()), + (901, format!("<{}>", string(b"U")).into_bytes()), + ]; + let trailer = + format!("/Root {} 0 R /Info {} 0 R /Encrypt {} 0 R /ID [<{id}> <{id}>]", file.root.num, file.info.num, file.encrypt.num, id = hex(&file.id)); + let source = Document::open(Arc::new(updated(&file.bytes, &objects, &trailer))).unwrap(); + assert_eq!(content(&source), SECRET, "the file reads"); + assert_eq!(title(&source).as_deref(), Some("Quarterly numbers")); + for object_streams in [false, true] { + let saved = write_full(&source, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(content(&back), SECRET, "object streams: {object_streams}"); + assert_eq!(title(&back).as_deref(), Some("Quarterly numbers"), "object streams: {object_streams}"); + } +} + +/// A save keeps what a file is protected with: its `/Encrypt`, which holds the handler and the +/// permissions, its `/ID`, and so its file key, under every algorithm and by either kind of save. +#[test] +fn a_save_keeps_the_handler_the_key_and_the_permissions() { + for alg in ALL { + let doc = Document::open_with_password(Arc::new(protect(alg, "user", "owner", -1 ^ 0b1_0000)), Some("user")).unwrap(); + let held = |d: &Document| { + let encrypt = d.resolve(d.trailer().get(b"Encrypt").unwrap()).as_dict().unwrap().clone(); + (encrypt, d.trailer().get(b"ID").cloned(), d.security().unwrap().file_key().to_vec(), format!("{:?}", d.permissions())) + }; + let before = held(&doc); + let mut edited = doc.clone(); + let _ = edited.add(Object::Int(1)); + for (save, saved) in [ + ("incremental", write_incremental(&edited, &SaveOptions::default()).unwrap()), + ("full", write_full(&doc, &SaveOptions::default()).unwrap()), + ] { + let back = Document::open_with_password(Arc::new(saved), Some("user")).unwrap(); + assert!(held(&back) == before, "{alg:?}: a {save} save"); + assert_eq!(content(&back), SECRET, "{alg:?}: a {save} save"); + } + } +} + +/// `/ID` has to be direct in an encrypted file, but one that isn't has its key made from the same +/// bytes. A full save writes it direct, and an incremental one leaves it as it was. +#[test] +fn a_file_identifier_given_by_reference_is_read() { + let file = classic_file(); + let id = hex(&file.id); + let reference = |entries: &str, objects: &[(u32, Vec)]| { + let trailer = format!("/Root {} 0 R /Encrypt {} 0 R /ID {entries}", file.root.num, file.encrypt.num); + updated(&file.bytes, objects, &trailer) + }; + let whole = reference("900 0 R", &[(900, format!("[<{id}> <{id}>]").into_bytes())]); + let entries = reference("[900 0 R 901 0 R]", &[(900, format!("<{id}>").into_bytes()), (901, format!("<{id}>").into_bytes())]); + for file in [whole, entries] { + let source = Document::open(Arc::new(file)).unwrap(); + assert_eq!(content(&source), SECRET); + let back = Document::open(Arc::new(write_full(&source, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(content(&back), SECRET); + assert!(matches!(back.trailer().get(b"ID"), Some(Object::Array(_))), "written direct"); + let mut edited = source.clone(); + let _ = edited.add(Object::Int(1)); + let back = Document::open(Arc::new(write_incremental(&edited, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(content(&back), SECRET, "an incremental save"); + assert_eq!(back.trailer().get(b"ID"), edited.trailer().get(b"ID"), "and leaves it as it was"); + } +} + +/// A direct `/Encrypt` can refer to objects. A full save dropped those it found only there, and kept +/// its references as they were under the old numbers, so nothing in the file read afterwards. +#[test] +fn a_direct_encrypt_dictionary_keeps_what_it_refers_to() { + for object_streams in [false, true] { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: Algorithm::Aes128, + user_password: "", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let mut dict = doc.get(doc.trailer().reference(b"Encrypt").unwrap()).as_dict().cloned().unwrap(); + let filter = doc.add(Object::name("StdCF")); + let filters = doc.add(dict.get(b"CF").unwrap().clone()); + dict.set(b"StmF".to_vec(), Object::Ref(filter)); + dict.set(b"CF".to_vec(), Object::Ref(filters)); + doc.trailer_mut().set(b"Encrypt".to_vec(), Object::Dict(dict)); + let options = SaveOptions { object_streams, ..SaveOptions::default() }; + let saved = write_full(&doc, &options).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(content(&back), SECRET, "object streams: {object_streams}"); + let again = write_full(&back, &options).unwrap(); + assert_eq!(content(&Document::open(Arc::new(again)).unwrap()), SECRET, "object streams: {object_streams}, saved again"); + } +} + +/// How a stream is treated can be given by reference, and what that loads can be a stream too. A +/// file can make a ladder of those, where each rung reads the rung below, then two trees wider than +/// the object cache, which push that rung out of it, then the rung below again. Each rung would +/// take twice what the one below does, if one read could follow any number of references. +#[test] +fn selectors_that_lead_to_selectors_load_at_once() { + let mut doc = Document::open(Arc::new(protect(Algorithm::Aes128, "", "owner", -1))).unwrap(); + let stream_of = |doc: &mut Document, filters: Vec| { + let mut dict = craft_pdf::cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(filters)); + doc.add(Object::Stream(craft_pdf::cos::Stream::from_raw(dict, Vec::new()))) + }; + let tree = |doc: &mut Document| { + let mut level: Vec = (0..6_561).map(|i| doc.add(Object::Int(i))).collect(); + while level.len() > 1 { + let chunks: Vec> = level.chunks(9).map(|c| c.iter().map(|r| Object::Ref(*r)).collect()).collect(); + level = chunks.into_iter().map(|c| stream_of(doc, c)).collect(); + } + Object::Ref(level[0]) + }; + let (first, second) = (tree(&mut doc), tree(&mut doc)); + let mut top = doc.add(Object::Int(0)); + // Twelve rungs of two, and the trees' five levels, stay inside the 32 that loading may nest. + for _ in 0..12 { + let (a, b) = (stream_of(&mut doc, vec![Object::Ref(top)]), stream_of(&mut doc, vec![Object::Ref(top)])); + top = stream_of(&mut doc, vec![Object::Ref(a), first.clone(), second.clone(), Object::Ref(b)]); + } + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let doc = Document::open(Arc::new(saved)).unwrap(); + let _ = done.send(doc.get(top).as_dict().is_some()); + }); + // With the budget it takes milliseconds, and without it longer than anyone would wait, so a + // margin this wide can't fail on a busy machine. + assert_eq!(finished.recv_timeout(std::time::Duration::from_secs(120)), Ok(true), "loading must not fan out"); +} diff --git a/crates/pdf/tests/cos/file_identifier.rs b/crates/pdf/tests/cos/file_identifier.rs new file mode 100644 index 000000000..551db4ebd --- /dev/null +++ b/crates/pdf/tests/cos/file_identifier.rs @@ -0,0 +1,277 @@ +//! The file identifier (`/ID`, ISO 32000-2 §14.4) of a full save. An ordinary save keeps the +//! identifier the file has and writes the same bytes twice. A rewrite the document asked for (after +//! redaction or sanitizing) gives the file a new pair. An encrypted file keeps its first string, +//! because its file key is derived from it. + +use std::sync::Arc; + +use crate::testkit::Pdf; +use craft_pdf::cos::{Algorithm, Document, NewEncryption, Object, PdfString, SaveOptions, write_full, write_incremental}; + +const ID_HEX: &str = "/ID [<00112233445566778899AABBCCDDEEFF> <00112233445566778899AABBCCDDEEFF>]"; +const ID_BYTES: [u8; 16] = [0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB, 0xCC, 0xDD, 0xEE, 0xFF]; + +/// One page whose content holds a secret, with `entry` added to the trailer (for example an `/ID`). +fn source_with(entry: Option<&str>) -> Vec { + let pdf = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>") + .stream("", "BT /F1 12 Tf 20 100 Td (TOPSECRET) Tj ET") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + match entry { + Some(entry) => pdf.trailer_entry(entry).build(), + None => pdf.build(), + } +} + +/// [`source_with`] with the standard identifier, or none. +fn source(with_id: bool) -> Vec { + source_with(with_id.then_some(ID_HEX)) +} + +/// A one-page file whose 70,000-byte content stream ends in `last`. Two of these built with +/// different last bytes have the same length and differ only past the first 64 KiB. +fn large_source(last: u8) -> Vec { + let mut content = vec![b' '; 70_000]; + content.push(last); + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>") + .stream("", content) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .build() +} + +fn reopen(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// The strings of the trailer's `/ID` (empty when there is none). +fn id_strings(doc: &Document) -> Vec> { + match doc.trailer().get(b"ID") { + Some(Object::Array(a)) => a.iter().map(|s| s.as_string().map(|s| s.bytes.clone()).unwrap_or_default()).collect(), + _ => Vec::new(), + } +} + +#[test] +fn an_ordinary_save_keeps_the_identifier_and_writes_the_same_bytes() { + let doc = reopen(source(true)); + let opts = SaveOptions::default(); + let saved = write_full(&doc, &opts).unwrap(); + assert_eq!(saved, write_full(&doc, &opts).unwrap(), "unchanged inputs produce identical bytes"); + assert_eq!(id_strings(&reopen(saved)), vec![ID_BYTES.to_vec(), ID_BYTES.to_vec()]); + let appended = reopen(write_incremental(&doc, &opts).unwrap()); + assert_eq!(id_strings(&appended), vec![ID_BYTES.to_vec(), ID_BYTES.to_vec()]); +} + +#[test] +fn an_ordinary_save_of_a_file_without_an_identifier_creates_the_same_one_every_time() { + let doc = reopen(source(false)); + let opts = SaveOptions::default(); + let saved = write_full(&doc, &opts).unwrap(); + assert_eq!(saved, write_full(&doc, &opts).unwrap(), "unchanged inputs produce identical bytes"); + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|s| s.len() == 16), "{ids:?}"); +} + +#[test] +fn a_rewrite_the_document_asked_for_gets_a_new_identifier() { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + assert!(doc.full_save_required() && doc.new_id_required()); + let opts = SaveOptions::default(); + for saved in [write_incremental(&doc, &opts).unwrap(), write_full(&doc, &opts).unwrap()] { + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2); + for id in &ids { + assert_eq!(id.len(), 16); + assert_ne!(id.as_slice(), ID_BYTES.as_slice(), "the copy must not claim the source's identifier"); + } + } + // Every save is a new identity, not the same one again. + let first = id_strings(&reopen(write_full(&doc, &opts).unwrap())); + let second = id_strings(&reopen(write_full(&doc, &opts).unwrap())); + assert_ne!(first, second); +} + +#[test] +fn a_rewrite_without_an_identifier_still_gets_one() { + let mut doc = reopen(source(false)); + doc.require_full_save_with_new_id(); + let ids = id_strings(&reopen(write_full(&doc, &SaveOptions::default()).unwrap())); + assert_eq!(ids.len(), 2); + assert!(ids.iter().all(|s| s.len() == 16), "{ids:?}"); +} + +#[test] +fn caller_entropy_makes_the_new_identifier_reproducible() { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + let with = |entropy: u8| SaveOptions { id_entropy: Some([entropy; 16]), ..SaveOptions::default() }; + let first = write_full(&doc, &with(1)).unwrap(); + assert_eq!(first, write_full(&doc, &with(1)).unwrap(), "the same entropy gives the same bytes"); + let other = write_full(&doc, &with(2)).unwrap(); + assert_ne!(id_strings(&reopen(first)), id_strings(&reopen(other)), "different entropy gives a different identifier"); +} + +#[test] +fn an_encrypted_rewrite_keeps_the_first_identifier_string_its_key_depends_on() { + for algorithm in [Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = reopen(source(true)); + doc.set_encryption(&NewEncryption { + algorithm, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + let protected = write_full(&doc, &SaveOptions::default()).unwrap(); + let before = Document::open_with_password(Arc::new(protected), Some("user")).unwrap(); + + let mut doc = before.clone(); + doc.require_full_save_with_new_id(); + let out = write_incremental(&doc, &SaveOptions::default()).unwrap(); + // The output opens with the same password only if its first identifier string is unchanged. + let back = Document::open_with_password(Arc::new(out), Some("user")).unwrap(); + let (old, new) = (id_strings(&before), id_strings(&back)); + assert_eq!(new.len(), 2, "{algorithm:?}"); + assert_eq!(new.first(), old.first(), "{algorithm:?}: the key depends on the first string"); + assert_ne!(new.get(1), old.get(1), "{algorithm:?}: the second string is new"); + } +} + +/// The `/ID` shapes whose key the reader takes as the empty string: none at all, one entry, and a +/// first entry that is not a string. +fn unusable_ids() -> Vec<(&'static str, Option)> { + let empty = Object::String(PdfString::literal(Vec::new())); + vec![ + ("absent", None), + ("one entry", Some(Object::Array(vec![empty.clone()]))), + ("first entry not a string", Some(Object::Array(vec![Object::Int(7), empty]))), + ] +} + +/// An encrypted file whose key is derived from the empty string, with its trailer `/ID` replaced by `id`. +fn protected_with_id(algorithm: Algorithm, id: Option) -> Vec { + let mut doc = reopen(source_with(Some("/ID [<> <>]"))); + doc.set_encryption(&NewEncryption { + algorithm, + user_password: "user", + owner_password: "owner", + permissions: -1, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + match id { + Some(id) => doc.trailer_mut().set(b"ID".to_vec(), id), + None => { + doc.trailer_mut().remove(b"ID"); + } + } + write_full(&doc, &SaveOptions::default()).unwrap() +} + +#[test] +fn an_encrypted_rewrite_without_a_usable_identifier_still_gets_a_new_one() { + // The key is derived from the first string of /ID, and a missing /ID, a one-entry /ID and a + // first entry that is not a string all count as the empty string. The rewrite keeps that + // (empty) first string, so the file still opens with its password, and writes a new second one. + for algorithm in [Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + for (shape, id) in unusable_ids() { + let before = Document::open_with_password(Arc::new(protected_with_id(algorithm, id)), Some("user")).unwrap(); + let mut doc = before.clone(); + doc.require_full_save_with_new_id(); + let out = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open_with_password(Arc::new(out), Some("user")).unwrap(); + let ids = id_strings(&back); + let previous = id_strings(&before); + assert_eq!(ids.len(), 2, "{algorithm:?}, {shape}: the rewrite writes a pair"); + assert!(ids.first().is_some_and(|s| s.is_empty()), "{algorithm:?}, {shape}: the first string is the one the key uses"); + assert_eq!(ids.get(1).map(Vec::len), Some(16), "{algorithm:?}, {shape}"); + assert_ne!(ids.get(1), previous.get(1), "{algorithm:?}, {shape}: the second string is new"); + } + } +} + +/// The standard fixture rewritten for a new identifier, with `id_entropy` of 0x42. +fn fixed_entropy_rewrite() -> Vec { + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + write_full(&doc, &SaveOptions { id_entropy: Some([0x42; 16]), ..SaveOptions::default() }).unwrap() +} + +#[test] +fn a_fixed_entropy_gives_the_same_identifier_every_time() { + // The formula itself is pinned on fixed bytes in the writer's tests. The written file is not the + // same on every platform, so no identifier of a rewrite can be pinned here. + let first = id_strings(&reopen(fixed_entropy_rewrite())); + assert_eq!(first.len(), 2); + assert!(first.iter().all(|s| s.len() == 16), "{first:?}"); + assert_eq!(first, id_strings(&reopen(fixed_entropy_rewrite()))); +} + +#[test] +fn the_identifier_depends_on_the_whole_rewritten_file() { + // Same caller entropy and same length; the two rewrites differ only in one byte of a stream + // that lies past the first 64 KiB of the output. Their identifiers must still differ. + let opts = SaveOptions { id_entropy: Some([9; 16]), ..SaveOptions::default() }; + let rewrite = |last: u8| { + let mut doc = reopen(large_source(last)); + doc.require_full_save_with_new_id(); + write_full(&doc, &opts).unwrap() + }; + let (a, b) = (rewrite(b'A'), rewrite(b'B')); + assert_eq!(a.len(), b.len(), "the two rewrites have the same length"); + let first_difference = a.iter().zip(&b).position(|(x, y)| x != y).unwrap_or(usize::MAX); + assert!(first_difference > 64 * 1024, "the rewrites first differ at byte {first_difference}, inside the first 64 KiB"); + assert_ne!(id_strings(&reopen(a)), id_strings(&reopen(b)), "content past 64 KiB must change the identifier"); +} + +/// Where [`stand_in_for_a_fresh_process`] writes its rewrite. +const ID_PROBE: &str = "CRAFT_PDF_TEST_ID_PROBE"; + +/// The child half of [`separate_fresh_processes_get_different_identifiers`]: it runs alone, in its +/// own process, and writes a default-path rewrite of the standard fixture to the file the parent +/// named. Without the variable it is a plain no-op. +#[test] +fn stand_in_for_a_fresh_process() { + let Some(path) = std::env::var_os(ID_PROBE) else { return }; + let mut doc = reopen(source(true)); + doc.require_full_save_with_new_id(); + std::fs::write(path, write_full(&doc, &SaveOptions::default()).unwrap()).unwrap(); +} + +#[test] +fn separate_fresh_processes_get_different_identifiers() { + // The default identifier is meant to differ between fresh instances of the whole save pipeline, + // not only between two calls in one process (which the counter already separates). On the native + // test target a fresh instance is a fresh process, whose hash keys the operating system seeds; + // the fix under test runs on WebAssembly, where no browser test harness exists — there the + // browser's `crypto.getRandomValues` stands in for this seeding, and this test cannot see it. + let exe = std::env::current_exe().unwrap(); + let probe = |name: &str| { + let path = std::env::temp_dir().join(format!("craft-pdf-id-probe-{}-{name}.pdf", std::process::id())); + let out = std::process::Command::new(&exe) + // The test binary holds every `cos` test module, so the child test is named by its path. + .args(["--exact", "file_identifier::stand_in_for_a_fresh_process", "--nocapture", "--"]) + .env(ID_PROBE, &path) + .output() + .expect("run this test's own binary as a child"); + assert!(out.status.success(), "{name}: the child failed: {}", String::from_utf8_lossy(&out.stderr)); + let saved = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: the child wrote its file: {e}")); + let ids = id_strings(&reopen(saved)); + assert_eq!(ids.len(), 2, "{name}"); + assert!(ids.iter().all(|s| s.len() == 16), "{name}: {ids:?}"); + let _ = std::fs::remove_file(&path); + ids + }; + let (first, second) = (probe("a"), probe("b")); + assert_ne!(first, second, "two fresh processes must not share an identifier"); +} diff --git a/crates/pdf/tests/cos/generations.rs b/crates/pdf/tests/cos/generations.rs new file mode 100644 index 000000000..77e5475d7 --- /dev/null +++ b/crates/pdf/tests/cos/generations.rs @@ -0,0 +1,510 @@ +//! An indirect reference identifies an object number AND its generation (ISO 32000-2 §7.3.10). + +use std::sync::Arc; + +use craft_pdf::cos::{Document, ObjRef, Object, SaveOptions, XrefEntry, write_full, write_incremental}; +use hayro_syntax::object::{Dict as HayroDict, ObjectIdentifier}; + +/// Object 3 was reused: only 3 1 R refers to the current value, never 3 0 R. +fn reused_object() -> Arc> { + object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 1, "42"), + ]) +} + +fn object_file(objects: &[(u32, u16, &str)]) -> Arc> { + object_file_with_entry(objects, None) +} + +fn object_file_with_entry(objects: &[(u32, u16, &str)], changed: Option<(u32, u16, bool)>) -> Arc> { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, generation, value) in objects { + offsets.push((*num, bytes.len(), *generation)); + bytes.extend_from_slice(format!("{num} {generation} obj\n{value}\nendobj\n").as_bytes()); + } + let xref = bytes.len(); + let size = objects.len() + 1; + bytes.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); + for (num, offset, generation) in offsets { + let (generation, free) = changed.filter(|(n, _, _)| *n == num).map_or((generation, false), |(_, g, free)| (g, free)); + let (offset, kind) = if free { (0, 'f') } else { (offset, 'n') }; + bytes.extend_from_slice(format!("{offset:010} {generation:05} {kind} \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size {size} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + Arc::new(bytes) +} + +fn contradictory_entry(free: bool) -> Arc> { + object_file_with_entry( + &[(1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "42")], + Some((3, if free { 2 } else { 0 }, free)), + ) +} + +#[test] +fn contradictory_xref_generation_recovers_the_matching_header_and_records_repair() { + for warm in [false, true] { + let doc = Document::open(contradictory_entry(false)).unwrap(); + if warm { + assert_eq!(*doc.try_get(3).unwrap(), Object::Int(42)); + } + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 1))), Object::Int(42)); + // Preserve the existing xref-matching behavior, but never accept an unrelated generation. + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); + assert!(!doc.xref_entry_is_exact(3), "a contradictory generation is not an exact xref entry"); + assert!( + doc.repair_log() + .iter() + .any(|line| { line.contains("object 3") && line.contains("xref generation 0") && line.contains("header generation 1") }), + "{:?}", + doc.repair_log() + ); + let repairs = doc.repair_log().to_vec(); + for _ in 0..3 { + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + } + assert_eq!(doc.repair_log(), repairs); + + let mut edited = doc.clone(); + edited.free(ObjRef::new(3, 1)); + for generation in [0, 1, 2] { + assert_eq!(*edited.get(ObjRef::new(3, generation)), Object::Null); + } + edited.set(ObjRef::new(3, 2), Object::Int(99)); + assert_eq!(*edited.get(ObjRef::new(3, 1)), Object::Null); + assert_eq!(*edited.get(ObjRef::new(3, 2)), Object::Int(99)); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + } +} + +/// A damaged row may name generation zero while the retained object and catalog use one. +fn contradictory_dictionary() -> Arc> { + object_file_with_entry( + &[(1, 0, "<< /Type /Catalog /Pages 2 0 R /Current 3 1 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "<< /Value 42 >>")], + Some((3, 0, false)), + ) +} + +fn assert_repaired_dictionary(bytes: Vec, value: i32) { + let bytes = Arc::new(bytes); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::>(b"Current").unwrap().get::(b"Value"), Some(value)); + let back = Document::open(bytes).unwrap(); + let catalog = back.get(back.root().unwrap()); + let current = back.resolve(catalog.as_dict().unwrap().get(b"Current").unwrap()); + assert_eq!(current.as_dict().unwrap().int(b"Value"), Some(i64::from(value))); +} + +#[test] +fn repaired_header_references_survive_full_saves_and_dictionary_edits() { + let original = contradictory_dictionary(); + let doc = Document::open(original.clone()).unwrap(); + for object_streams in [false, true] { + assert_repaired_dictionary(write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(), 42); + } + let mut edited = doc.clone(); + edited.update_dict(ObjRef::new(3, 1), |dict| dict.set(b"Value".to_vec(), Object::Int(77))).unwrap(); + assert_eq!(edited.generation(3), 1); + let incremental = write_incremental(&edited, &SaveOptions::default()).unwrap(); + assert!(edited.full_save_required(), "repair-aware dictionary edits normalize on full rewrite"); + assert_repaired_dictionary(incremental, 77); + for object_streams in [false, true] { + assert_repaired_dictionary(write_full(&edited, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(), 77); + } + let mut restored = doc.clone(); + restored.overlay_changes_from(&edited).unwrap(); + assert_repaired_dictionary(write_incremental(&restored, &SaveOptions::default()).unwrap(), 77); +} + +#[test] +fn free_xref_slots_never_revive_retained_object_headers() { + for warm in [false, true] { + let doc = Document::open(contradictory_entry(true)).unwrap(); + assert!(matches!(doc.xref_entry(3), Some(XrefEntry::Free { next_generation: 2 }))); + if warm { + assert_eq!(*doc.try_get(3).unwrap(), Object::Null); + } + for generation in [0, 1, 2] { + assert_eq!(*doc.get(ObjRef::new(3, generation)), Object::Null); + } + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 1))), Object::Null); + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + } + // Control: the retained bytes really are a readable object when its slot is live. + let live = Document::open(contradictory_entry(false)).unwrap(); + assert_eq!(*live.try_get(3).unwrap(), Object::Int(42)); +} + +#[test] +fn incremental_generation_identity_is_visible_to_an_independent_reader() { + let original = reused_object(); + let mut doc = Document::open(original.clone()).unwrap(); + doc.set(ObjRef::new(3, 1), Object::Int(77)); + let bytes = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(bytes.starts_with(&original)); + let oracle = hayro_syntax::Pdf::new(Arc::new(bytes)).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(77)); + assert_eq!(catalog.get::(b"Stale"), None); +} + +#[test] +fn generation_repair_diagnostics_are_bounded_without_disabling_recovery() { + let mut objects = vec![(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")]; + objects.extend((3..303).map(|num| (num, 1, "42"))); + let bytes = object_file(&objects); + let bytes = String::from_utf8(bytes.as_ref().clone()).unwrap().replace("00001 n \n", "00000 n \n"); + for materialized in [false, true] { + let bytes = if materialized { bytes.replace("/Size 303", "/Size 302") } else { bytes.clone() }; + let doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert_eq!(doc.repair_log().len(), 257); + assert!(doc.repair_log().last().unwrap().contains("omitted after 256 objects")); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(302, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(302, 2)), Object::Null); + assert_eq!(doc.repair_log().len(), 257); + } +} + +#[test] +fn references_require_the_current_generation_even_with_a_warm_cache() { + let bytes = reused_object(); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + assert_eq!(oracle.xref().get::(ObjectIdentifier::new(3, 0)), None); + assert_eq!(oracle.xref().get::(ObjectIdentifier::new(3, 1)), Some(42)); + + let doc = Document::open(bytes).unwrap(); + assert!(doc.repair_log().is_empty()); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.try_get(3).unwrap(), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.resolve(&Object::Ref(ObjRef::new(3, 0))), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); +} + +#[test] +fn overlay_generation_changes_do_not_retarget_old_references_or_clones() { + let mut doc = Document::open(reused_object()).unwrap(); + let snapshot = doc.clone(); + let old = ObjRef::new(3, 1); + let new = ObjRef::new(3, 2); + doc.free(old); + assert_eq!(*doc.get(old), Object::Null); + doc.set(new, Object::Int(99)); + assert_eq!(*doc.get(old), Object::Null); + assert_eq!(*doc.get(new), Object::Int(99)); + assert_eq!(*snapshot.get(old), Object::Int(42)); + assert_eq!(*snapshot.get(new), Object::Null); + + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert!(back.repair_log().is_empty()); + assert_eq!(*back.get(old), Object::Null); + assert_eq!(*back.get(new), Object::Int(99)); +} + +#[test] +fn restoring_snapshot_preserves_changes_to_nonzero_generation_objects() { + for generation in [1, 2] { + let mut saved = Document::open(reused_object()).unwrap(); + let mut current = saved.clone(); + let reference = ObjRef::new(3, generation); + current.set(reference, Object::Int(99)); + current.update_dict(current.root().unwrap(), |catalog| catalog.set(b"Current".to_vec(), Object::Ref(reference))).unwrap(); + + saved.overlay_changes_from(¤t).unwrap(); + assert_eq!(*saved.get(reference), Object::Int(99), "generation {generation}"); + let back = Document::open(Arc::new(write_incremental(&saved, &SaveOptions::default()).unwrap())).unwrap(); + assert!(back.repair_log().is_empty()); + assert_eq!(*back.get(reference), Object::Int(99), "generation {generation}"); + let catalog = back.get(back.root().unwrap()); + assert_eq!(*back.resolve(catalog.as_dict().unwrap().get(b"Current").unwrap()), Object::Int(99)); + } +} + +#[test] +fn full_save_does_not_turn_stale_references_into_live_ones() { + let doc = Document::open(reused_object()).unwrap(); + for object_streams in [false, true] { + let saved = write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(); + let saved = Arc::new(saved); + let oracle = hayro_syntax::Pdf::new(saved.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(42)); + assert_eq!(catalog.get::(b"Stale"), None); + let back = Document::open(saved).unwrap(); + assert!(back.repair_log().is_empty()); + let catalog = back.get(back.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + assert_eq!(catalog.get(b"Stale"), Some(&Object::Null)); + assert_eq!(*back.resolve(catalog.get(b"Current").unwrap()), Object::Int(42)); + } +} + +#[test] +fn compressed_objects_only_accept_generation_zero() { + let doc = Document::open(reused_object()).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let current = catalog.as_dict().unwrap().reference(b"Current").unwrap(); + assert_eq!(current.generation, 0); + assert!(matches!(back.xref_entry(current.num), Some(XrefEntry::InStream { .. }))); + assert_eq!(*back.get(current), Object::Int(42)); + assert_eq!(*back.get(ObjRef::new(current.num, 1)), Object::Null); +} + +#[test] +fn indirect_stream_lengths_require_the_referenced_generation() { + for generation in [0, 1] { + // The embedded endstream is data when /Length resolves, but the existing missing- + // length recovery stops there when the reference is stale. + let stream = format!("<< /Length 4 {generation} R >>\nstream\nA\nendstream\nB\nendstream"); + let doc = Document::open(object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 0, &stream), + (4, 1, "13"), + ])) + .unwrap(); + let object = doc.get(ObjRef::new(3, 0)); + let Object::Stream(stream) = &*object else { panic!("expected stream") }; + assert_eq!(&stream.raw[..], if generation == 1 { b"A\nendstream\nB".as_slice() } else { b"A".as_slice() }); + } +} + +/// A newer valid row must not hide damage in a retained older revision. +fn shadowed_generation_disagreement(materialized: bool, shifted: bool) -> Arc> { + let original = contradictory_dictionary(); + let text = std::str::from_utf8(&original).unwrap(); + let previous: usize = text.rsplit("startxref\n").next().unwrap().lines().next().unwrap().parse().unwrap(); + let mut bytes = (*original).clone(); + let offset = bytes.len(); + bytes.extend_from_slice(b"3 1 obj\n<< /Value 77 >>\nendobj\n"); + let xref = bytes.len(); + let size = if materialized { 3 } else { 4 }; + bytes.extend_from_slice( + format!("xref\n3 1\n{offset:010} 00001 n \ntrailer\n<< /Size {size} /Root 1 0 R /Prev {previous} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(), + ); + if shifted { + bytes.splice(..0, b"junk before the header\n".iter().copied()); + } + Arc::new(bytes) +} + +#[test] +fn retained_generation_disagreements_are_reported_in_materialized_mode_too() { + for materialized in [false, true] { + for shifted in [false, true] { + let doc = Document::open(shadowed_generation_disagreement(materialized, shifted)).unwrap(); + assert_eq!(doc.revisions().len(), 2, "the xref chain is retained"); + assert!(!doc.reconstructed()); + assert_eq!(doc.get(ObjRef::new(3, 1)).as_dict().unwrap().int(b"Value"), Some(77)); + assert!( + doc.repair_log().iter().any(|line| line.contains("object 3: xref generation 0 disagrees with object header generation 1")), + "materialized={materialized}, shifted={shifted}: {:?}", + doc.repair_log() + ); + } + } +} + +#[test] +fn repair_aware_edits_keep_both_aliases_and_full_save_normalizes_them() { + let mut doc = Document::open(contradictory_entry(false)).unwrap(); + let snapshot = doc.clone(); + doc.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + for generation in [0, 1] { + assert_eq!(*doc.get(ObjRef::new(3, generation)), Object::Int(77)); + assert_eq!(*snapshot.get(ObjRef::new(3, generation)), Object::Int(42)); + } + let state = doc.read_state(); + assert!(doc.set_existing(ObjRef::new(3, 2), Object::Int(99)).is_err()); + assert!(state.matches(&doc)); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Null); + let mut restored = snapshot.clone(); + restored.overlay_changes_from(&doc).unwrap(); + for changed in [&doc, &restored] { + assert!(changed.full_save_required()); + for bytes in [ + write_incremental(changed, &SaveOptions::default()).unwrap(), + write_full(changed, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(), + ] { + let bytes = Arc::new(bytes); + let oracle = hayro_syntax::Pdf::new(bytes.clone()).unwrap(); + let catalog = oracle.xref().get::>(ObjectIdentifier::new(1, 0)).unwrap(); + assert_eq!(catalog.get::(b"Current"), Some(77)); + assert_eq!(catalog.get::(b"Stale"), Some(77), "both source aliases stay live"); + let back = Document::open(bytes).unwrap(); + assert!(back.repair_log().is_empty(), "normalized output has no generation repairs"); + } + } + // Explicit replacement keeps its narrower, caller-supplied generation contract. + doc.set(ObjRef::new(3, 2), Object::Int(99)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Null); + assert_eq!(*doc.get(ObjRef::new(3, 2)), Object::Int(99)); +} + +#[test] +fn hybrid_generation_diagnostics_observe_rows_before_the_table_hides_them() { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, generation, body) in [(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), (3, 1, "42")] { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{num} {generation} obj\n{body}\nendobj\n").as_bytes()); + } + let hybrid = bytes.len(); + offsets.push(hybrid); + bytes.extend_from_slice(b"4 0 obj\n<< /Type /XRef /Size 5 /W [1 4 2] /Index [3 1] /Length 7 >>\nstream\n"); + bytes.push(1); + bytes.extend_from_slice(&(offsets[2] as u32).to_be_bytes()); + bytes.extend_from_slice(&0u16.to_be_bytes()); + bytes.extend_from_slice(b"\nendstream\nendobj\n"); + let xref = bytes.len(); + bytes.extend_from_slice(b"xref\n0 5\n0000000000 65535 f \n"); + for (i, offset) in offsets.into_iter().enumerate() { + let generation = u16::from(i == 2); + bytes.extend_from_slice(format!("{offset:010} {generation:05} n \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size 5 /Root 1 0 R /XRefStm {hybrid} >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + assert!(!doc.reconstructed()); + assert!(doc.xref_oddities().iter().any(|(_, num)| *num == 3), "the conflicting hybrid row is set aside"); + assert_eq!(*doc.get(ObjRef::new(3, 1)), Object::Int(42)); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null, "the table still determines accepted references"); + assert!(doc.repair_log().iter().any(|line| line.contains("object 3: xref generation 0 disagrees with object header generation 1"))); +} + +#[test] +fn discarded_xref_repair_notes_do_not_survive_object_header_reconstruction() { + let bytes = + String::from_utf8((*contradictory_dictionary()).clone()).unwrap().replace("/Root 1 0 R", "/Root 99 0 R").replace("/Size 4", "/Size 3"); + let doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); + assert!(doc.reconstructed()); + assert!(doc.repair_log().iter().all(|line| !line.contains("xref generation")), "{:?}", doc.repair_log()); +} + +#[test] +fn restoring_equal_values_still_restores_the_accepted_generation_aliases() { + for edited in [false, true] { + for value in [42, 77] { + let mut current = Document::open(contradictory_entry(false)).unwrap(); + if edited { + current.set_existing(ObjRef::new(3, 0), Object::Int(42)).unwrap(); + } + let mut saved = Document::open(object_file(&[ + (1, 0, "<< /Type /Catalog /Pages 2 0 R /Stale 3 0 R /Current 3 1 R >>"), + (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>"), + (3, 0, &value.to_string()), + ])) + .unwrap(); + assert_eq!(*saved.get(ObjRef::new(3, 1)), Object::Null); + saved.overlay_changes_from(¤t).unwrap(); + for generation in [0, 1] { + assert_eq!(*saved.get(ObjRef::new(3, generation)), Object::Int(42), "edited={edited}, value={value}"); + } + assert!(saved.full_save_required()); + } + } +} + +#[test] +fn repaired_page_tree_aliases_keep_one_identity_in_full_saves() { + let source = object_file_with_entry( + &[ + (1, 0, "<< /Type /Catalog /Pages 2 1 R /Stale 2 2 R >>"), + (2, 1, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), + (3, 0, "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 10 10] >>"), + ], + Some((2, 0, false)), + ); + for edited in [false, true] { + let mut doc = Document::open(source.clone()).unwrap(); + if edited { + doc.update_dict(ObjRef::new(2, 0), |dict| dict.set(b"Edited".to_vec(), Object::Bool(true))).unwrap(); + } + let mut restored = Document::open(source.clone()).unwrap(); + restored.overlay_changes_from(&doc).unwrap(); + for current in [&doc, &restored] { + for object_streams in [false, true] { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + let mut saves = vec![write_full(current, &opts).unwrap()]; + if edited { + saves.push(write_incremental(current, &opts).unwrap()); + } + for bytes in saves { + let back = Document::open(Arc::new(bytes)).unwrap(); + let catalog = back.get(back.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + let pages_ref = catalog.reference(b"Pages").unwrap(); + let pages = back.get(pages_ref); + let child_ref = pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap()[0].as_ref().unwrap(); + let child = back.get(child_ref); + assert_eq!(child.as_dict().unwrap().reference(b"Parent"), Some(pages_ref), "edited={edited}, packed={object_streams}"); + assert!(matches!(catalog.get(b"Stale"), Some(Object::Null)), "unrelated generations stay null"); + assert!(back.repair_log().is_empty()); + } + } + } + } +} + +#[test] +fn restoring_explicit_sets_removes_old_generation_aliases() { + for value in [42, 77] { + let mut current = Document::open(contradictory_entry(false)).unwrap(); + current.set(ObjRef::new(3, 0), Object::Int(42)); + for repaired_overlay in [false, true] { + let mut saved = Document::open(contradictory_entry(false)).unwrap(); + if repaired_overlay { + saved.set_existing(ObjRef::new(3, 0), Object::Int(value)).unwrap(); + } + saved.overlay_changes_from(¤t).unwrap(); + assert_eq!(*saved.get(ObjRef::new(3, 0)), Object::Int(42)); + assert_eq!(*saved.get(ObjRef::new(3, 1)), Object::Null, "value={value}, overlay={repaired_overlay}"); + } + } +} + +/// `accepted_generations` lists exactly the generations `get` resolves, for every kind of slot: +/// a clean file slot, a damaged one (row and header disagree), a repair-aware edit, a plain +/// edit, a freed slot, a free row, an object stream member and an unknown number. +#[test] +fn accepted_generations_are_the_generations_get_resolves() { + let agrees = |doc: &Document, num: u32, expected: &[u16]| { + assert_eq!(doc.accepted_generations(num), expected, "object {num}"); + for generation in 0..8 { + let live = !matches!(*doc.get(ObjRef::new(num, generation)), Object::Null); + assert_eq!(live, expected.contains(&generation), "object {num} generation {generation}"); + } + }; + let doc = Document::open(contradictory_entry(false)).unwrap(); + agrees(&doc, 1, &[0]); + agrees(&doc, 3, &[0, 1]); + agrees(&doc, 9, &[]); + let mut repaired = doc.clone(); + repaired.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + agrees(&repaired, 3, &[0, 1]); + let mut narrowed = doc.clone(); + narrowed.set(ObjRef::new(3, 2), Object::Int(99)); + agrees(&narrowed, 3, &[2]); + let mut freed = doc.clone(); + freed.free(ObjRef::new(3, 0)); + agrees(&freed, 3, &[]); + agrees(&Document::open(contradictory_entry(true)).unwrap(), 3, &[]); + let mut doc = + Document::open(object_file(&[(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")])).unwrap(); + let member = doc.add(Object::Dict(Default::default())); + doc.update_dict(ObjRef::new(1, 0), |c| c.set(b"Extra".to_vec(), Object::Ref(member))).unwrap(); + let packed = Document::open(Arc::new(write_full(&doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap(); + assert!(matches!(packed.xref_entry(member.num), Some(XrefEntry::InStream { .. })), "the member is in an object stream"); + agrees(&packed, member.num, &[0]); +} diff --git a/crates/pdf/tests/cos/main.rs b/crates/pdf/tests/cos/main.rs new file mode 100644 index 000000000..f62aa43d1 --- /dev/null +++ b/crates/pdf/tests/cos/main.rs @@ -0,0 +1,17 @@ +//! Tests of the `cos` feature, linked into one test binary. Not built for the web: proptest and +//! the oracle processes are not there. +#![cfg(all(feature = "cos", not(target_arch = "wasm32")))] + +mod bounded_writer; +mod corpus; +mod efopen; +mod encryption; +mod file_identifier; +mod generations; +mod oracle; +mod proptests; +mod serialize_format; +mod shared_streams; +#[path = "../../src/cos/testkit.rs"] +mod testkit; +mod writer; diff --git a/crates/pdf/tests/cos/oracle.rs b/crates/pdf/tests/cos/oracle.rs new file mode 100644 index 000000000..1efbc9981 --- /dev/null +++ b/crates/pdf/tests/cos/oracle.rs @@ -0,0 +1,231 @@ +//! External oracle processes: independent implementations the object layer's output is checked +//! against (from PdfCraft's `pdfcraft-oracle`). +//! +//! **A missing oracle is either a declared skip or a failure, never a silent pass.** +//! [`REQUIRED_VAR`] decides which: set it where the tools are installed, so their absence fails +//! the run; without it, a missing tool prints why and the test returns. +//! +//! Test scaffolding, compiled into this test binary only. +#![allow(dead_code)] + +use std::fmt; +use std::process::Command; + +/// Setting this environment variable to any value turns a missing oracle from a skip into a +/// failure. CI sets it on the runners that install the tools. +/// +/// Any value counts, including `0` — the same convention as `CRAFT_FONTS_REQUIRED`, which this +/// deliberately mirrors. The surprising half of that convention errs toward *requiring* an oracle, +/// which is the safe direction for a gate whose whole purpose is to stop absence passing quietly. +pub const REQUIRED_VAR: &str = "CRAFT_PDF_ORACLES_REQUIRED"; + +/// An external program found on `PATH`, ready to run. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Oracle { + program: &'static str, + version: String, +} + +impl Oracle { + /// A fresh [`Command`] for the program. Callers add their own arguments. + pub fn command(&self) -> Command { + Command::new(self.program) + } + + /// The program name, as it is looked up on `PATH`. + pub fn program(&self) -> &'static str { + self.program + } + + /// The first line the program printed when asked for its version, for test output. Empty when + /// it printed nothing recognisable; presence of the binary is what decides availability, not + /// whether its version could be parsed. + pub fn version(&self) -> &str { + &self.version + } +} + +/// An oracle that could not be used, and whether that is fatal. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Absent { + program: &'static str, + cause: String, + required: bool, +} + +impl Absent { + /// Whether this absence must fail the run rather than skip it, i.e. whether [`REQUIRED_VAR`] + /// was set when the probe ran. + pub fn required(&self) -> bool { + self.required + } + + /// The program that was looked for. + pub fn program(&self) -> &'static str { + self.program + } + + /// How to install it, so a skip notice tells the reader what to do about it. + fn install_hint(program: &str) -> &'static str { + match program { + "qpdf" => "apt install qpdf / brew install qpdf", + "pdftoppm" | "pdftotext" => "apt install poppler-utils / brew install poppler", + _ => "install it and put it on PATH", + } + } +} + +impl fmt::Display for Absent { + /// Reads as an instruction either way: required absences are failures and name the gate that + /// made them one, optional absences name the install command. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.required { + write!( + f, + "oracle `{}` is required but {}: {} is set, so this is a failure rather than a skip ({})", + self.program, + self.cause, + REQUIRED_VAR, + Self::install_hint(self.program), + ) + } else { + write!( + f, + "skipping: oracle `{}` {} ({}); set {} to make this a failure instead", + self.program, + self.cause, + Self::install_hint(self.program), + REQUIRED_VAR, + ) + } + } +} + +/// Look for `program` on `PATH`, asking it for its version to confirm it runs. +/// +/// Presence is decided by whether the program could be started at all, not by its exit status: a +/// tool that runs but reports its version oddly is still an oracle. +pub fn find(program: &'static str, version_arg: &str) -> Result { + let required = std::env::var_os(REQUIRED_VAR).is_some(); + match Command::new(program).arg(version_arg).output() { + Ok(out) => { + // Poppler's tools print their version to stderr, qpdf to stdout. + let text = if out.stdout.is_empty() { &out.stderr } else { &out.stdout }; + let version = String::from_utf8_lossy(text).lines().next().unwrap_or_default().trim().to_owned(); + Ok(Oracle { program, version }) + } + Err(e) => Err(Absent { program, cause: format!("is not on PATH ({e})"), required }), + } +} + +/// qpdf: structural validation (`--check`) and the independent owner-password oracle. +pub fn qpdf() -> Result { + find("qpdf", "--version") +} + +/// Poppler's `pdftoppm`: renders a page to pixels, for comparing against our own raster output. +pub fn pdftoppm() -> Result { + find("pdftoppm", "-v") +} + +/// Poppler's `pdftotext`: the text-extraction oracle (`cargo xtask text-oracle` scores word F1 +/// against it over the corpus). +pub fn pdftotext() -> Result { + find("pdftotext", "-v") +} + +/// Binds an oracle, or returns from the calling test after saying on stderr why it was skipped. +/// +/// When [`REQUIRED_VAR`] is set the absence fails the test instead. The panic is deliberate and is +/// how a test reports failure; it expands into the caller, which is test code, so the workspace's +/// never-panic rule is not weakened — this crate itself contains no panicking path. +/// +/// The macro expands to a `return`, so it belongs in a test function returning `()`. A test that +/// returns `Result` should match on the probe itself. +macro_rules! or_skip { + ($probe:expr) => { + match $probe { + ::core::result::Result::Ok(oracle) => oracle, + ::core::result::Result::Err(absent) => { + let absent: $crate::oracle::Absent = absent; + ::core::assert!(!$crate::oracle::Absent::required(&absent), "{absent}"); + ::std::eprintln!("{absent}"); + return; + } + } + }; +} +pub(crate) use or_skip; + +#[cfg(test)] +mod tests { + use super::*; + + /// The one behaviour the crate exists for: absence is never silent, and which of the two loud + /// forms it takes is decided by the environment variable. + #[test] + fn a_missing_oracle_is_fatal_exactly_when_it_is_required() { + let optional = Absent { program: "qpdf", cause: "is not on PATH".into(), required: false }; + let required = Absent { required: true, ..optional.clone() }; + + assert!(!optional.required()); + assert!(required.required()); + + // Both say what is wrong and how to fix it; only the required one calls itself a failure. + let (opt, req) = (optional.to_string(), required.to_string()); + assert!(opt.contains("skipping"), "{opt}"); + assert!(opt.contains("apt install qpdf"), "{opt}"); + assert!(opt.contains(REQUIRED_VAR), "{opt}"); + assert!(req.contains("required"), "{req}"); + assert!(req.contains("failure"), "{req}"); + assert!(!req.contains("skipping"), "{req}"); + } + + /// A tool that does not exist is reported as absent rather than hanging or panicking, and the + /// reason names the program. + #[test] + fn a_program_that_does_not_exist_is_absent_with_a_reason() { + let Err(absent) = find("pdfcraft-no-such-oracle-binary", "--version") else { + panic!("a nonexistent program must not be reported as present"); + }; + assert_eq!(absent.program(), "pdfcraft-no-such-oracle-binary"); + let text = absent.to_string(); + assert!(text.contains("pdfcraft-no-such-oracle-binary"), "{text}"); + assert!(text.contains("not on PATH"), "{text}"); + } + + /// Hostile and odd program names must not panic, hang or be mistaken for a working oracle. + #[test] + fn hostile_program_names_are_absent_not_a_panic() { + for name in ["", " ", "\0bad", "../../etc/passwd", &"x".repeat(4096)] { + // `find` takes a 'static str; leak the generated ones, which is fine in a test. + let name: &'static str = Box::leak(name.to_owned().into_boxed_str()); + let found = find(name, "--version"); + if let Ok(o) = found { + // Nothing by these names should run; if one somehow does, it must still be sane. + assert_eq!(o.program(), name); + } + } + } + + /// The install hint is specific per tool, so a skip notice is actionable. + #[test] + fn every_known_oracle_has_an_install_hint() { + for p in ["qpdf", "pdftoppm", "pdftotext"] { + let hint = Absent::install_hint(p); + assert!(hint.contains("install"), "{p}: {hint}"); + assert_ne!(hint, Absent::install_hint("something-else"), "{p} needs its own hint"); + } + } + + /// Probing is read-only and repeatable: asking twice gives the same answer. + #[test] + fn probing_is_stable() { + let a = qpdf().map(|o| o.program()); + let b = qpdf().map(|o| o.program()); + assert_eq!(a.is_ok(), b.is_ok()); + if let (Ok(a), Ok(b)) = (a, b) { + assert_eq!(a, b); + } + } +} diff --git a/crates/pdf/tests/cos/proptests.rs b/crates/pdf/tests/cos/proptests.rs new file mode 100644 index 000000000..171b0a23e --- /dev/null +++ b/crates/pdf/tests/cos/proptests.rs @@ -0,0 +1,464 @@ +//! Property tests for the object layer's save fidelity — proptest coverage of two areas raised in +//! pdfcraft #1187. **This does not close pdfcraft #1187.** That issue asks for five Hegel properties; this adds +//! save/reopen and edit-history coverage using `proptest`, which is already a dev-dependency of this +//! crate, so no new dependency is needed to review it. Left for follow-ups: +//! #1187's reference-following comparison, its snapshot/undo state machine, the password property (the +//! one whose pilot found #1185) and its damaged-file byte splices. The object-level serialise → parse +//! round trip it also lists already exists inline in `src/writer.rs`. +//! +//! Covered: +//! +//! 1. **A full save keeps every value it wrote into the catalog**, in both output styles (objects +//! packed into object streams, §7.5.7; the classic cross-reference table otherwise), keeps the +//! catalog's own structure intact (`/Type /Catalog`, and a `/Pages` node whose `/Kids` array and +//! `/Count` agree — the fixture's tree is empty, so this pins shape and agreement, not rendered +//! pages), preserves dictionary +//! key order (which `README.md:75` promises is byte-stable), emits spec-shaped §7.5.4 cross-reference +//! rows, and reopens with nothing to repair. +//! 2. **An incremental save is a strict append**: the original bytes come out untouched at the front +//! (§7.5.6), the file keeps two revisions whose first still points at the base cross-reference and +//! opens standalone, the edit is visible while every existing value stays put in order, and it +//! reopens with nothing to repair. +//! 3. **A write/delete sequence equals a plain map**, checked after every step and again once saved and +//! reopened. A deleted key must stay gone — not return its old value, not read as null either. +//! Deletes are generated against keys that were just written, so the delete path cannot be skipped. +//! +//! ## Scope, stated honestly +//! Generated values are pure data — no streams and no indirect references — so this says nothing about +//! stream payloads or the renumbering a full save legitimately performs. Strings are compared as bytes, +//! so literal ↔ hexadecimal re-emission is not treated as a change (§7.3.6); the *choice* of spelling +//! is therefore not asserted, and `PdfString.hex` is not pinned here. Dictionary keys at the catalog +//! level *are* compared in order, so reversing them fails; nested dictionary order inside a value is +//! sorted away by [`canon`] and is not pinned. Generated reals stay within ±1e6: `write_real` +//! (`src/writer.rs:46`) turns a non-finite real into `0` and an integral real below 1e15 into integer +//! syntax, so [`canon`] applies the writer's documented normalisation to **both** sides rather than +//! forgiving a mismatch afterwards. + +use std::sync::Arc; + +use proptest::prelude::*; +use proptest::test_runner::TestCaseError; + +use craft_pdf::cos::{Dict, Document, Object, PdfString, SaveOptions, write_full, write_incremental}; + +/// The prefix every generated key carries. `Document::new_empty` puts `/Type` and `/Pages` in the +/// catalog; property 1 asserts those two structurally rather than ignoring them. +const KEY: &str = "px"; + +/// Generated keys go into the catalog in this deliberately unsorted sequence. Ascending insertion would make +/// "preserves dictionary key order" and "every dictionary sorted by key" the same observable, so a writer +/// that silently sorted would rewrite every saved byte and stay green. This scramble catches sorting of the +/// keys it names; it does NOT on its own pin where structural entries like `/Type` sit, because `saved` +/// filters to the `px` prefix - that gap is closed separately by [`catalog_keys`] asserting the full +/// sequence. Nested dictionary order is deliberately not pinned (see `canon`). Every permutation of the +/// generated keys is NOT drawn here either: one fixed order per case, chosen to make sorting detectable. +const INSERTION_ORDER: [usize; 8] = [5, 0, 7, 2, 6, 1, 4, 3]; + +/// A value reduced to a comparable spelling, with the writer's documented normalisations applied. +fn canon(o: &Object) -> String { + match o { + Object::Null => "null".into(), + Object::Bool(b) => format!("b{b}"), + Object::Int(i) => format!("i{i}"), + Object::Real(r) => { + if !r.is_finite() { + "i0".into() + } else if r.fract() == 0.0 && r.abs() < 1e15 { + format!("i{}", *r as i64) + } else { + format!("r{r}") + } + } + Object::Name(n) => format!("n{}", String::from_utf8_lossy(n)), + // Bytes, because a literal may be re-emitted as hexadecimal or the reverse; both spellings + // denote the same bytes (§7.3.6). The `hex` flag itself is deliberately not compared — see + // the scope note at the top. + Object::String(s) => format!("s{}", s.bytes.iter().map(|b| format!("{b:02x}")).collect::()), + Object::Array(a) => format!("[{}]", a.iter().map(canon).collect::>().join(",")), + Object::Dict(d) => { + // Sorted inside a value: nested key order is not pinned here. Catalog-level order is, and + // it is asserted in iteration order by `saved`. + let mut pairs: Vec<(String, String)> = d.iter().map(|(k, v)| (String::from_utf8_lossy(k).into_owned(), canon(v))).collect(); + pairs.sort(); + format!("{{{}}}", pairs.iter().map(|(k, v)| format!("{k}=>{v}")).collect::>().join(",")) + } + // Unreachable under the generators below, which build no streams and no references. Kept total + // so this compiles if someone widens them — at which point it should compare decoded bytes and + // resolved targets instead of a constant. + Object::Stream(_) => "stream".into(), + Object::Ref(r) => format!("ref{r}"), + } +} + +fn leaf() -> impl Strategy { + prop_oneof![ + Just(Object::Null), + any::().prop_map(Object::Bool), + (-1_000_000i64..1_000_000i64).prop_map(Object::Int), + (-1e6f64..1e6f64).prop_map(Object::Real), + // Non-finite reals, drawn explicitly. `write_real` (`src/writer.rs:46`) replaces them with + // `0`, and [`canon`] spells both sides that way, so this is what makes the normalisation + // described in the module header actually exercised rather than merely asserted in prose. + prop::sample::select(vec![f64::NAN, f64::INFINITY, f64::NEG_INFINITY]).prop_map(Object::Real), + // Raw bytes, including ones a literal string must escape and the NUL byte. + prop::collection::vec(any::(), 0..12).prop_map(|b| Object::String(PdfString::literal(b))), + "[A-Za-z][A-Za-z0-9]{0,6}".prop_map(|s| Object::name(&s)), + ] +} + +/// Nested values to a bounded depth, so generation cannot recurse without limit. +fn value(depth: usize) -> BoxedStrategy { + if depth == 0 { + return leaf().boxed(); + } + prop_oneof![ + leaf().boxed(), + prop::collection::vec(value(depth - 1), 0..5).prop_map(Object::Array).boxed(), + prop::collection::btree_map("[a-z]{1,4}", value(depth - 1), 0..5) + .prop_map(|m| { + let mut d = Dict::new(); + for (k, v) in m { + d.set(k.into_bytes(), v); + } + Object::Dict(d) + }) + .boxed(), + ] + .boxed() +} + +fn find(hay: &[u8], needle: &[u8]) -> Option { + hay.windows(needle.len()).position(|w| w == needle) +} + +/// Every catalog key, in the dictionary's own order - structural entries included. `saved` filters to +/// `px*`, which is why sorting of generated keys is visible there but `/Type` moving to the end was not; +/// this covers that. Read from memory before a save and from the parsed file afterwards, so the two sides +/// share only `Dict` iteration, not the serialiser under test. +fn catalog_keys(doc: &Document) -> Vec { + let root = match doc.root() { + Some(r) => r, + None => return Vec::new(), + }; + match doc.get(root).as_dict() { + Some(d) => d.iter().map(|(k, _)| String::from_utf8_lossy(k).into_owned()).collect(), + None => Vec::new(), + } +} + +/// The catalog's `px*` entries in **iteration order**, so a reordered save fails. +fn saved(doc: &Document) -> Result, TestCaseError> { + let root = doc.root().ok_or_else(|| TestCaseError::fail("no catalog in the saved file"))?; + let cat = doc.get(root); + let d = match cat.as_dict() { + Some(d) => d, + None => return Err(TestCaseError::fail("catalog is not a dictionary")), + }; + Ok(d.iter().filter(|(k, _)| k.starts_with(KEY.as_bytes())).map(|(k, v)| (String::from_utf8_lossy(k).into_owned(), canon(v))).collect()) +} + +/// The catalog's structure, which a reader needs whether or not it holds our test keys. +fn assert_catalog_structure(doc: &Document, style: &str) -> Result<(), TestCaseError> { + let root = doc.root().ok_or_else(|| TestCaseError::fail(format!("no catalog after the {style} save")))?; + let cat = doc.get(root); + let d = match cat.as_dict() { + Some(d) => d, + None => return Err(TestCaseError::fail(format!("catalog is not a dictionary after the {style} save"))), + }; + prop_assert_eq!(d.name(b"Type"), Some(&b"Catalog"[..]), "catalog /Type changed on the {} save", style); + let pages = match d.get(b"Pages") { + Some(p) => p, + None => return Err(TestCaseError::fail(format!("catalog lost /Pages on the {style} save"))), + }; + let resolved = doc.resolve(pages); + let pd = match resolved.as_dict() { + Some(d) => d, + None => return Err(TestCaseError::fail(format!("/Pages does not resolve to a dictionary on the {style} save"))), + }; + prop_assert_eq!(pd.name(b"Type"), Some(&b"Pages"[..]), "/Pages /Type changed on the {} save", style); + let kids = match pd.get(b"Kids").and_then(Object::as_array) { + Some(k) => k, + None => return Err(TestCaseError::fail(format!("/Pages /Kids is missing or not an array on the {style} save"))), + }; + // /Count has to agree with /Kids. A page tree whose count lies still opens, then misnumbers every + // page downstream, so checking presence alone would not be enough. This fixture's tree is empty, so + // what is pinned here is shape and agreement — not the existence of rendered pages. + prop_assert_eq!(pd.int(b"Count"), Some(kids.len() as i64), "/Pages /Count disagrees with its {} kids on the {} save", kids.len(), style); + Ok(()) +} + +/// §7.5.4: every entry of a classic cross-reference table is exactly 20 bytes, closed by a two-byte +/// end of line. Unpadded offsets still open in lenient readers and corrupt strict ones, so the byte +/// shape is asserted rather than inferred from a successful reopen. +fn assert_classic_xref_rows(bytes: &[u8]) -> Result<(), TestCaseError> { + // Anchored on the file's own startxref (§7.5.3) rather than the first "xref" text in the buffer: + // a generated document name could otherwise point the scan at prose. A startxref that does not land + // on the `xref` keyword means a cross-reference stream, which has no table to check here. + let Some(xoff) = startxref(bytes) else { return Err(TestCaseError::fail("no readable startxref in a classic-table save")) }; + let x = xoff as usize; + if bytes.get(x..x + 4) != Some(&b"xref"[..]) { + return Ok(()); + } + let nl = match bytes[x..].iter().position(|b| *b == b'\n') { + Some(n) => n, + None => return Err(TestCaseError::fail("xref keyword not followed by a newline")), + }; + let rest = &bytes[x + nl + 1..]; + let end = match find(rest, b"trailer") { + Some(e) => e, + None => return Err(TestCaseError::fail("cross-reference table with no trailer")), + }; + let mut rows = 0usize; + let mut expected_rows = 0usize; + for line in rest[..end].split(|b| *b == b'\n') { + if line.is_empty() { + continue; + } + // A CR before the line feed belongs to the two-byte terminator (§7.5.4), not to the keyword: + // stripped here so a CR+LF row is still recognised as a legal row rather than reported (wrongly) + // as "no cross-reference rows found". Producers do emit CR+LF terminators. + // + // What catches what: classification keys off the keyword, so a row whose keyword is garbage never + // reaches the width and length asserts below. Those rows are named by the `else` arm at the end + // of this loop; the row-count compare after it is only a backstop. The length assert is not the + // safety net for malformed keywords, and this comment used to imply that it was. + let fields: Vec<&[u8]> = line + .split(|b| *b == b' ') + .filter(|f| !f.is_empty()) + .map(|f| match f.last() { + Some(b'\r') => &f[..f.len() - 1], + _ => f, + }) + .collect(); + if fields.len() == 2 && fields.iter().all(|f| f.iter().all(u8::is_ascii_digit)) { + // ` ` subsection header. + expected_rows += std::str::from_utf8(fields[1]).ok().and_then(|s| s.parse::().ok()).unwrap_or(0); + continue; + } + if fields.len() == 3 && matches!(fields[2], b"n" | b"f") { + // Classified by keyword, not by width: an unpadded row must still be recognised as a row, so + // the failure names the real crime (wrong width) instead of the misleading "no rows found". + // Both keywords are legal — `n` in use, `f` free (the head of the free chain carries + // generation 65535). §7.5.4 fixes each field's width and the whole entry at 20 bytes. + prop_assert_eq!( + fields[0].len(), + 10, + "a cross-reference offset must be 10 digits (§7.5.4), got {:?} in row {:?}", + String::from_utf8_lossy(fields[0]), + String::from_utf8_lossy(line) + ); + prop_assert_eq!( + fields[1].len(), + 5, + "a cross-reference generation must be 5 digits (§7.5.4), got {:?} in row {:?}", + String::from_utf8_lossy(fields[1]), + String::from_utf8_lossy(line) + ); + // Exactly 20 bytes per entry in the file (§7.5.4). Both legal terminators land on 19 bytes + // for the line as split here: SP+LF, which is what this writer emits today, and CR+LF, whose + // terminator is the pair itself. A 21-byte entry (SP+CRLF, or a doubled CR) fails here. + prop_assert_eq!( + line.len(), + 19, + "a cross-reference row is {} bytes; §7.5.4 wants exactly 20 in the file (SP+LF or CR+LF): {:?}", + line.len(), + String::from_utf8_lossy(line) + ); + rows += 1; + } else { + // Inside the table region, a line is either a subsection header or an entry. Anything else is + // named here with its bytes quoted, instead of surfacing later as a bare row-count mismatch + // ("no cross-reference rows found") that blames the table for one bad line. + prop_assert_eq!( + fields.len(), + 3, + "unrecognised line in the cross-reference table, want ` n|f`: {:?}", + String::from_utf8_lossy(line) + ); + prop_assert!( + matches!(fields[2], b"n" | b"f"), + "illegal cross-reference keyword {:?} in row {:?}", + String::from_utf8_lossy(fields[2]), + String::from_utf8_lossy(line) + ); + } + } + prop_assert!(rows > 0, "no cross-reference rows found in a classic-table save"); + prop_assert_eq!(rows, expected_rows, "the subsection headers promised {} rows, the table holds {}", expected_rows, rows); + Ok(()) +} + +/// The `startxref` value of a file, which the next incremental update must point back at via `/Prev`. +/// The keyword is followed by whitespace, then the offset (§7.5.3), so leading whitespace is skipped. +fn startxref(bytes: &[u8]) -> Option { + let at = find(bytes, b"startxref")?; + let tail = &bytes[at + 9..]; + let digits: String = tail.iter().skip_while(|b| b.is_ascii_whitespace()).take_while(|b| b.is_ascii_digit()).map(|b| char::from(*b)).collect(); + digits.parse().ok() +} + +proptest! { + // 256 cases per property; the suite runs in well under a second, so coverage is free here. + #![proptest_config(ProptestConfig::with_cases(256))] + + /// 1. A full save keeps the values it wrote, their order, the catalog's structure and the shape of + /// its cross-reference table — in both output styles — and reopens with nothing to repair. + #[test] + fn full_save_keeps_the_catalog_intact(map in prop::collection::btree_map(0usize..8, value(3), 1..6)) { + let mut doc = Document::new_empty(); + let root = doc.root().expect("an empty document has a catalog"); + doc.update_dict(root, |d| { + for i in INSERTION_ORDER.into_iter().filter(|&i| map.contains_key(&i)) { + d.set(format!("{KEY}{i}").into_bytes(), map[&i].clone()); + } + }) + .map_err(|e| TestCaseError::fail(format!("cannot write the generated catalog: {e}")))?; + // Insertion order is this scrambled sequence filtered to the drawn keys: what must come back. + let expected: Vec<(String, String)> = INSERTION_ORDER.into_iter().filter(|&i| map.contains_key(&i)).map(|i| (format!("{KEY}{i}"), canon(&map[&i]))).collect(); + + // Full key sequence taken from memory BEFORE any byte is written - /Type and /Pages included. + let order_before = catalog_keys(&doc); + + // Right before any save, or the later assertion would prove nothing about saving. + prop_assert_eq!(saved(&doc)?, expected.clone(), "the in-memory catalog is wrong before saving"); + + for object_streams in [true, false] { + let style: &str = if object_streams { "object-stream" } else { "classic-table" }; + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + let bytes = write_full(&doc, &opts).map_err(|e| TestCaseError::fail(format!("full save failed: {e}")))?; + + if !object_streams { + assert_classic_xref_rows(&bytes).map_err(|e| TestCaseError::fail(format!("{style}: {e}")))?; + } + + let again = Document::open(Arc::new(bytes)).map_err(|e| TestCaseError::fail(format!("our own full save must reopen: {e}")))?; + + prop_assert!(again.repair_log().is_empty(), "a file we wrote must not need repair ({}); log: {:?}", style, again.repair_log()); + // Values *and* order: README.md promises dictionaries keep their key order. + prop_assert_eq!(saved(&again)?, expected.clone(), "the catalog changed on a {} save (values or order)", style); + // Structural position too: /Type and /Pages must not drift to the end or swap places. + prop_assert_eq!(catalog_keys(&again), order_before.clone(), "the catalog's full key sequence changed on a {} save", style); + assert_catalog_structure(&again, style)?; + } + } + + /// 2. An incremental save leaves the original bytes untouched, chains back through `/Prev`, and + /// reopens carrying every intended value plus the edit. The base is built **both** ways - + /// cross-reference stream and classic table - because the two are serialised by different code, and a + /// path that never runs cannot fail here. Expected contents are computed from the values we put in, + /// not read back out of the writer: comparing against our own earlier output would pass a writer that + /// is consistently wrong, which is the one thing a fidelity test must not do. + #[test] + fn incremental_save_appends_without_touching_the_original(map in prop::collection::btree_map(0usize..8, value(2), 1..5), marker in -1_000_000i64..1_000_000i64) { + let mut doc = Document::new_empty(); + let root = doc.root().expect("an empty document has a catalog"); + doc.update_dict(root, |d| { + for i in INSERTION_ORDER.into_iter().filter(|&i| map.contains_key(&i)) { + d.set(format!("{KEY}{i}").into_bytes(), map[&i].clone()); + } + }) + .map_err(|e| TestCaseError::fail(format!("cannot write the generated catalog: {e}")))?; + + // Our own record of what the catalog should hold, derived from the inputs before any byte is + // written. The edit key joins it below; order follows INSERTION_ORDER, not sorted order (§7.5.4). + let intended: Vec<(String, String)> = INSERTION_ORDER.into_iter().filter(|&i| map.contains_key(&i)).map(|i| (format!("{KEY}{i}"), canon(&map[&i]))).collect(); + + for object_streams in [true, false] { + let style: &str = if object_streams { "object-stream" } else { "classic-table" }; + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + + let base = write_full(&doc, &opts).map_err(|e| TestCaseError::fail(format!("{}: full save failed: {e}", style)))?; + let base_xref = startxref(&base).ok_or_else(|| TestCaseError::fail(format!("{}: the base file has no readable startxref", style)))?; + + let mut again = Document::open(Arc::new(base.clone())).map_err(|e| TestCaseError::fail(format!("{}: the saved file must reopen: {e}", style)))?; + // Independent of the writer's own readings: the reopen must show what we intended to write. + prop_assert_eq!(saved(&again)?, intended.clone(), "{}: the base catalog differs from what we put in", style); + + let root2 = again.root().expect("the reopened document has a catalog"); + again.update_dict(root2, |d| d.set(b"pxMark".to_vec(), Object::Int(marker))).map_err(|e| TestCaseError::fail(format!("{}: cannot apply the incremental edit: {e}", style)))?; + + let bytes = write_incremental(&again, &opts).map_err(|e| TestCaseError::fail(format!("{}: incremental save failed: {e}", style)))?; + prop_assert!(bytes.len() > base.len(), "{}: an incremental save must add bytes ({} over a base of {})", style, bytes.len(), base.len()); + prop_assert_eq!(bytes.get(..base.len()), Some(&base[..]), "{}: an incremental save must not rewrite the original bytes", style); + + let out = Document::open(Arc::new(bytes.clone())).map_err(|e| TestCaseError::fail(format!("{}: the incremental file must reopen: {e}", style)))?; + prop_assert!(out.repair_log().is_empty(), "{}: an increment we wrote must not need repair; log: {:?}", style, out.repair_log()); + + // The revision chain in the crate's own terms (Revision { xref_offset, is_stream }): two + // revisions, the first still pointing at the base cross-reference. Without that link no other + // reader can reach the earlier revision's objects, which is the classic incremental corruption. + // /Prev sits in the trailer for a table base and inside the cross-reference stream for a + // stream base, so it is not read here; `revisions` covers both (§7.5.3-7.5.6). + let revs = out.revisions(); + prop_assert_eq!(revs.len(), 2, "{}: an incremental save must leave two revisions: {:?}", style, revs); + prop_assert_eq!(revs[0].xref_offset, base_xref, "{}: the first revision must still point at the base xref {}", style, base_xref); + prop_assert!(revs[1].xref_offset > base_xref, "{}: the second revision must sit after the first: {:?}", style, revs); + let ends = out.revision_ends(); + prop_assert_eq!(ends.last().copied(), Some(bytes.len()), "{}: the last revision must run to the end of the file: {:?}", style, ends); + + // The base revision opens on its own. "Nothing was rewritten" buys the user this: cut the file + // at the append point and the earlier document still reads, with our intended values. + if let Some(end0) = ends.first().copied() { + let before_doc = Document::open(Arc::new(bytes[..end0].to_vec())).map_err(|e| TestCaseError::fail(format!("{}: the base revision must open alone: {e}", style)))?; + prop_assert_eq!(saved(&before_doc)?, intended.clone(), "{}: the base revision lost or changed values when opened without the increment", style); + } + + assert_catalog_structure(&out, style)?; + + let mut expected = intended.clone(); + expected.push(("pxMark".into(), canon(&Object::Int(marker)))); + prop_assert_eq!(saved(&out)?, expected.clone(), "{}: the increment lost a value, reordered the catalog, or dropped the edit", style); + } + } + + /// 3. Writes then deletes against a plain ordered model, checked after every step and again after + /// save and reopen. Deletes always target a key that exists, so the delete path is never empty. + #[test] + fn edit_sequence_equals_a_plain_map(plan in prop::collection::btree_map(0usize..8, value(2), 1..6).prop_flat_map(|sets| { + let keys: Vec = sets.keys().copied().collect(); + (Just(sets), prop::collection::vec(0usize..keys.len(), 1..=keys.len())).prop_map(move |(s, picks)| { + let mut dels: Vec = picks.into_iter().map(|i| keys[i]).collect(); + // Deduplicated: the same key drawn twice would make the second delete target a key that is + // already gone, which is legal PDF but not what this property means to assert. `picks` draws + // 1..=keys.len() over a non-empty key set, so `dels` cannot come back empty — the guarantee + // lives in the generator, not in an assume that could never fire. + dels.sort_unstable(); + dels.dedup(); + (s, dels) + }) + })) { + let (sets, deletes) = plan; + let mut doc = Document::new_empty(); + let root = doc.root().expect("an empty document has a catalog"); + + let mut model: Vec<(String, String)> = Vec::new(); + for (i, o) in &sets { + let key = format!("{KEY}{i}"); + doc.update_dict(root, |d| d.set(key.clone().into_bytes(), o.clone())).map_err(|e| TestCaseError::fail(format!("cannot write {key}: {e}")))?; + model.push((key.clone(), canon(o))); + prop_assert_eq!(saved(&doc)?, model.clone(), "after writing {} the catalog is wrong before saving", key); + } + + for i in &deletes { + let key = format!("{KEY}{i}"); + prop_assert!(model.iter().any(|(k, _)| k == &key), "the generator tried to delete a key that was never written: {}", key); + doc.update_dict(root, |d| { + d.remove(key.as_bytes()); + }) + .map_err(|e| TestCaseError::fail(format!("cannot delete {key}: {e}")))?; + model.retain(|(k, _)| k != &key); + prop_assert_eq!(saved(&doc)?, model.clone(), "after deleting {} it is still reachable", key); + } + + let bytes = write_full(&doc, &SaveOptions::default()).map_err(|e| TestCaseError::fail(format!("full save failed: {e}")))?; + let again = Document::open(Arc::new(bytes)).map_err(|e| TestCaseError::fail(format!("our own save must reopen: {e}")))?; + let got = saved(&again)?; + prop_assert_eq!(got.clone(), model.clone(), "the reopened catalog differs from the edit history (values or order)"); + for i in 0..8usize { + let key = format!("{KEY}{i}"); + if !model.iter().any(|(k, _)| k == &key) { + prop_assert!(!got.iter().any(|(g, _)| *g == key), "a deleted key came back after save+reopen: {}", key); + } + } + } +} diff --git a/crates/pdf/tests/cos/serialize_format.rs b/crates/pdf/tests/cos/serialize_format.rs new file mode 100644 index 000000000..fb2499796 --- /dev/null +++ b/crates/pdf/tests/cos/serialize_format.rs @@ -0,0 +1,116 @@ +//! `serialize` writes integers, references, hexadecimal strings, octal escapes and `#xx` name +//! escapes with its own digit loops. They must give the bytes the standard formatter gives. + +use craft_pdf::cos::{ObjRef, Object, PdfString, serialize}; +use proptest::prelude::*; + +fn bytes_of(o: &Object) -> Vec { + let mut v = Vec::new(); + serialize(o, &mut v); + v +} + +#[test] +fn integer_extremes_and_digit_boundaries() { + let mut cases = vec![0i64, 1, -1, 9, 10, 99, 100, i64::MAX, i64::MIN, i64::MIN + 1, i64::from(i32::MAX), i64::from(i32::MIN)]; + let mut p = 1i64; + while let Some(next) = p.checked_mul(10) { + cases.extend([next - 1, next, next + 1, -next]); + p = next; + } + for i in cases { + assert_eq!(bytes_of(&Object::Int(i)), i.to_string().into_bytes(), "{i}"); + } +} + +#[test] +fn reference_extremes() { + for (n, g) in [(0, 0), (1, 0), (u32::MAX, u16::MAX), (10, 65535), (4_000_000_000, 7)] { + assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } +} + +#[test] +fn every_byte_value_in_hex_strings_octal_escapes_and_names() { + let all: Vec = (0..=255).collect(); + let hex: String = all.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(bytes_of(&Object::String(PdfString { bytes: all.clone(), hex: true })), format!("<{hex}>").into_bytes()); + // A literal string with a few control bytes escapes them in octal. + let lit = bytes_of(&Object::String(PdfString { bytes: b"abcdefghijkl\x01\x7f\xffz".to_vec(), hex: false })); + assert_eq!(lit, b"(abcdefghijkl\\001\\177\\377z)"); + let name = bytes_of(&Object::Name(all.clone())); + let mut want = b"/".to_vec(); + for &b in &all { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + want.push(b); + } else { + want.extend_from_slice(format!("#{b:02X}").as_bytes()); + } + } + assert_eq!(name, want); +} + +#[test] +fn whole_real_at_i32_min_stays_a_real_token() { + // -2^31 is outside the Annex C integer range (±2^31-1), so it keeps its trailing point. + assert_eq!(bytes_of(&Object::Real(f64::from(i32::MIN))), b"-2147483648."); +} + +proptest! { + #[test] + fn integers_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Int(i)), format!("{i}").into_bytes()); + } + + #[test] + fn whole_reals_match_the_formatter(i in -i32::MAX..=i32::MAX) { + prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); + } + + #[test] + fn references_match_the_formatter(n in any::(), g in any::()) { + prop_assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } + + #[test] + fn hex_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + let mut want = String::from("<"); + for b in &bytes { + want.push_str(&format!("{b:02X}")); + } + want.push('>'); + prop_assert_eq!(bytes_of(&Object::String(PdfString { bytes, hex: true })), want.into_bytes()); + } + + #[test] + fn binary_literal_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + // Mostly-binary input is written as hex; mostly-text input with escapes is a literal. + let got = bytes_of(&Object::String(PdfString { bytes: bytes.clone(), hex: false })); + let binary = bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + let mut want = Vec::new(); + if binary * 4 > bytes.len().max(1) { + want.push(b'<'); + for b in &bytes { + want.extend_from_slice(format!("{b:02X}").as_bytes()); + } + want.push(b'>'); + } else { + want.push(b'('); + for &b in &bytes { + match b { + b'(' | b')' | b'\\' => want.extend_from_slice(&[b'\\', b]), + b'\n' => want.extend_from_slice(b"\\n"), + b'\r' => want.extend_from_slice(b"\\r"), + b'\t' => want.extend_from_slice(b"\\t"), + 0x08 => want.extend_from_slice(b"\\b"), + 0x0C => want.extend_from_slice(b"\\f"), + b' '..=b'~' => want.push(b), + _ => want.extend_from_slice(format!("\\{b:03o}").as_bytes()), + } + } + want.push(b')'); + } + prop_assert_eq!(got, want); + } +} diff --git a/crates/pdf/tests/cos/shared_streams.rs b/crates/pdf/tests/cos/shared_streams.rs new file mode 100644 index 000000000..90c2ad29b --- /dev/null +++ b/crates/pdf/tests/cos/shared_streams.rs @@ -0,0 +1,78 @@ +//! Stream data read from a file points into the file's bytes instead of being copied: the +//! document's object cache holds parsed dictionaries, not a second copy of every stream. + +use std::sync::Arc; + +use craft_pdf::cos::{Document, ObjRef, Object, SaveOptions, write_full, write_incremental}; + +const CONTENT: &[u8] = b"0 0 1 rg 10 10 30 20 re f"; + +fn fixture() -> Vec { + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 100 50] >>".to_string(), + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R >>".to_string(), + format!("<< /Length {} >>\nstream\n{}\nendstream", CONTENT.len(), String::from_utf8_lossy(CONTENT)), + ]; + let mut out = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +const CONTENTS: ObjRef = ObjRef { num: 4, generation: 0 }; + +fn stream_bytes(doc: &Document, r: ObjRef) -> (Vec, *const u8) { + match &*doc.get(r) { + Object::Stream(s) => (s.raw.to_vec(), s.raw.as_ptr()), + other => panic!("{other:?}"), + } +} + +#[test] +fn cached_streams_are_the_files_own_bytes() { + let file = Arc::new(fixture()); + let doc = Document::open(file.clone()).unwrap(); + let (data, at) = stream_bytes(&doc, CONTENTS); + assert_eq!(data, CONTENT); + assert!(file.as_ptr_range().contains(&at), "not a copy"); + // A full save reads and caches every object; the cache still holds no copy of stream data. + let full = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(file.as_ptr_range().contains(&stream_bytes(&doc, CONTENTS).1)); + // Saves are unchanged: the stream is written as it was read. + let back = Document::open(Arc::new(full)).unwrap(); + assert_eq!(stream_bytes(&back, ObjRef::new(4, 0)).0, CONTENT); +} + +#[test] +fn edits_within_a_document_keep_pointing_into_its_file() { + let file = Arc::new(fixture()); + let mut doc = Document::open(file.clone()).unwrap(); + doc.update_dict(CONTENTS, |d| d.set(b"Marked".to_vec(), Object::Bool(true))).unwrap(); + assert!(file.as_ptr_range().contains(&stream_bytes(&doc, CONTENTS).1)); + let updated = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(updated)).unwrap(); + assert_eq!(stream_bytes(&back, CONTENTS).0, CONTENT); +} + +#[test] +fn streams_moved_to_another_document_get_their_own_bytes() { + let file = Arc::new(fixture()); + let source = Document::open(file.clone()).unwrap(); + let moved = (*source.get(CONTENTS)).clone(); + let mut target = Document::new_empty(); + let r = target.add(moved); + drop(source); + // The target keeps just the stream's bytes, not the whole source file. + assert_eq!(Arc::strong_count(&file), 1); + assert_eq!(stream_bytes(&target, r).0, CONTENT); +} diff --git a/crates/pdf/tests/cos/writer.rs b/crates/pdf/tests/cos/writer.rs new file mode 100644 index 000000000..52b58f880 --- /dev/null +++ b/crates/pdf/tests/cos/writer.rs @@ -0,0 +1,622 @@ +//! Full saves with object streams and cross-reference streams (M1.9), read back by our parser +//! and by hayro, and incremental saves stacked on top of them. + +use std::sync::Arc; + +use crate::testkit::Pdf; +use craft_pdf::cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full, write_incremental}; + +/// Four pages, a shared font, document info with a non-ASCII title. +fn fixture() -> Vec { + let kids: Vec = (0..4).map(|i| format!("{} 0 R", 4 + 2 * i)).collect(); + let mut pdf = Pdf::new() + .version("1.4") + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj(format!("<< /Type /Pages /Kids [{}] /Count 4 /MediaBox [0 0 200 300] >>", kids.join(" "))) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>"); + for i in 0..4 { + pdf = pdf + .obj(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i)) + .stream("", format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1)); + } + let info = pdf.next_obj(); + pdf.obj("<< /Title (Caf\\351 \\(draft\\)) >>").trailer_entry(&format!("/Info {info} 0 R")).build() +} + +fn title(doc: &Document) -> Vec { + let info = doc.resolve(doc.trailer().get(b"Info").unwrap()); + info.as_dict().unwrap().get(b"Title").unwrap().as_string().unwrap().bytes.clone() +} + +fn hayro_pages(bytes: &[u8]) -> usize { + hayro_syntax::Pdf::new(Arc::new(bytes.to_vec())).expect("hayro opens").pages().len() +} + +#[test] +fn full_save_packs_objects_into_object_streams() { + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let text = String::from_utf8_lossy(&packed); + assert!(packed.starts_with(b"%PDF-1.5"), "object streams need PDF 1.5"); + assert!(text.contains("/ObjStm") && text.contains("/XRef")); + assert!(!text.contains("\nxref\n"), "no classic table"); + assert!(!text.contains("/Type /Catalog") && !text.contains("/Type/Catalog"), "the catalog is compressed"); + + let back = Document::open(Arc::new(packed.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); + assert_eq!(hayro_pages(&packed), 4); + + let classic = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(classic.starts_with(b"%PDF-1.4")); + assert!(String::from_utf8_lossy(&classic).contains("\nxref\n")); + assert_eq!(hayro_pages(&classic), 4); +} + +#[test] +fn incremental_saves_stack_on_an_xref_stream_file() { + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = Arc::new(write_full(&doc, &SaveOptions::default()).unwrap()); + let mut doc = Document::open(packed.clone()).unwrap(); + // Edit an object that lives in an object stream. + let info = doc.trailer().reference(b"Info").unwrap(); + doc.update_dict(info, |d: &mut Dict| { + d.set(b"Title".to_vec(), Object::String(craft_pdf::cos::PdfString { bytes: b"Second".to_vec(), hex: false })) + }) + .unwrap(); + let added = doc.add(Object::Int(42)); + let updated = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(updated.starts_with(&packed), "an incremental save only appends"); + let tail = String::from_utf8_lossy(&updated[packed.len()..]); + assert!(tail.contains("/XRef") && tail.contains("/Prev"), "the update uses an xref stream too"); + + let back = Document::open(Arc::new(updated.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Second"); + assert_eq!(*back.get(ObjRef::new(added.num, 0)), Object::Int(42)); + assert_eq!(back.revisions().len(), 2); + assert_eq!(hayro_pages(&updated), 4); +} + +/// The revision is joined to the original once, at its exact size. A revision of more than a few +/// kilobytes used to outgrow the buffer: the whole original was copied a second time, and the +/// result kept up to as much again unused for as long as it was the document's working bytes. +#[test] +fn incremental_saves_are_allocated_at_their_exact_size() { + let table = Arc::new(fixture()); + let stream = Arc::new(write_full(&Document::open(table.clone()).unwrap(), &SaveOptions::default()).unwrap()); + let no_newline = Arc::new(fixture().trim_ascii_end().to_vec()); + for original in [table, stream, no_newline] { + let mut doc = Document::open(original.clone()).unwrap(); + let data: Vec = (0..100_000u32).map(|i| (i % 251) as u8).collect(); + let added = doc.add(Object::Stream(Stream { dict: Dict::new(), raw: data.clone().into() })); + let updated = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert_eq!(updated.capacity(), updated.len()); + assert!(updated.starts_with(&original), "an incremental save only appends"); + let back = Document::open(Arc::new(updated.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + match &*back.get(added) { + Object::Stream(s) => assert_eq!(*s.raw, data), + other => panic!("{other:?}"), + } + assert_eq!(hayro_pages(&updated), 4); + } +} + +/// A full save grows its buffer as it writes, so it used to end with up to as much again unused, +/// kept alive with the document's working bytes. It now ends at its exact size. +#[test] +fn full_saves_are_allocated_at_their_exact_size() { + let mut doc = Document::open(Arc::new(fixture())).unwrap(); + let data: Vec = (0..100_000u32).map(|i| (i % 251) as u8).collect(); + let added = doc.add(Object::Stream(Stream { dict: Dict::new(), raw: Arc::new(data.clone()).into() })); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(added))).unwrap(); + for object_streams in [true, false] { + let full = write_full(&doc, &SaveOptions { object_streams, ..SaveOptions::default() }).unwrap(); + assert_eq!(full.capacity(), full.len(), "object streams: {object_streams}"); + let back = Document::open(Arc::new(full.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + let extra = back.dict(&Object::Ref(back.root().unwrap())).and_then(|d| d.get(b"Extra").cloned()).unwrap(); + match &*back.resolve(&extra) { + Object::Stream(s) => assert_eq!(*s.raw, data), + other => panic!("{other:?}"), + } + assert_eq!(hayro_pages(&full), 4); + } +} + +/// A one-page classic-table file with no `/ID`, the case that used to append `xref` and then +/// `trailer` with no subsection (#1339). +fn classic_table() -> Vec { + let objects = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Resources << >> /Contents 4 0 R >>", + "<< /Length 0 >>\nstream\n\nendstream", + ]; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{object}\nendobj\n", i + 1).as_bytes()); + } + let xref = bytes.len(); + bytes.extend_from_slice(b"xref\n0 5\n0000000000 65535 f \n"); + for offset in offsets { + bytes.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend_from_slice(format!("trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); + bytes +} + +fn qpdf_check(qpdf: &crate::oracle::Oracle, bytes: &[u8]) -> std::process::Output { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("pdfcraft-xref-{}-{n}.pdf", std::process::id())); + std::fs::write(&path, bytes).unwrap(); + let check = qpdf.command().arg("--check").arg(&path).output().unwrap(); + let _ = std::fs::remove_file(&path); + check +} + +fn page(doc: &Document) -> Dict { + let root = doc.resolve(doc.trailer().get(b"Root").unwrap()); + let pages = doc.resolve(root.as_dict().unwrap().get(b"Pages").unwrap()); + let kids = pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap(); + doc.resolve(kids.first().unwrap()).as_dict().cloned().unwrap() +} + +fn assert_qpdf_clean(bytes: &[u8]) { + let qpdf = crate::oracle::or_skip!(crate::oracle::qpdf()); + let check = qpdf_check(&qpdf, bytes); + let report = format!("{}{}", String::from_utf8_lossy(&check.stdout), String::from_utf8_lossy(&check.stderr)); + assert!(check.status.success() && !report.contains("WARNING"), "{report}"); +} + +#[test] +fn unchanged_incremental_save_of_a_classic_table_is_a_valid_xref() { + let input = classic_table(); + let doc = Document::open(Arc::new(input.clone())).unwrap(); + assert!(!doc.is_modified()); + assert!(doc.repair_log().is_empty()); + assert_eq!(doc.revisions().len(), 1); + assert!(!doc.revisions()[0].is_stream); + + let unchanged = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(unchanged.starts_with(&input)); + let appended = &unchanged[input.len()..]; + assert!(appended.windows(8).any(|w| w == b"xref\n0 0"), "a section with no entries still needs a subsection"); + assert!(!appended.windows(13).any(|w| w == b"xref\ntrailer")); + + let back = Document::open(Arc::new(unchanged.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(back.revisions().len(), 2); + assert_eq!(page(&back).name(b"Type"), Some(b"Page".as_slice())); + assert!(back.trailer().get(b"ID").is_some(), "a file without an identifier still receives one"); + assert_eq!(hayro_pages(&unchanged), 1); + assert_qpdf_clean(&unchanged); + + let mut edited = doc.clone(); + edited.update_dict(ObjRef::new(3, 0), |d| d.set(b"Rotate".to_vec(), Object::Int(90))).unwrap(); + let edited = write_incremental(&edited, &SaveOptions::default()).unwrap(); + let edited_back = Document::open(Arc::new(edited.clone())).unwrap(); + assert!(edited_back.repair_log().is_empty(), "{:?}", edited_back.repair_log()); + assert_eq!(edited_back.get(ObjRef::new(3, 0)).as_dict().unwrap().int(b"Rotate"), Some(90)); + assert_qpdf_clean(&edited); + + let stamped = write_incremental(&doc, &SaveOptions { mod_date: Some("D:20260101000000Z".into()), ..SaveOptions::default() }).unwrap(); + let stamped_back = Document::open(Arc::new(stamped.clone())).unwrap(); + assert!(stamped_back.repair_log().is_empty(), "{:?}", stamped_back.repair_log()); + let info = stamped_back.resolve(stamped_back.trailer().get(b"Info").unwrap()); + assert_eq!( + info.as_dict().unwrap().get(b"ModDate").and_then(|d| d.as_string()).map(|s| s.bytes.as_slice()), + Some(b"D:20260101000000Z".as_slice()) + ); + assert_qpdf_clean(&stamped); +} + +#[test] +fn incremental_save_keeps_a_trailer_only_edit() { + let input = classic_table(); + let mut doc = Document::open(Arc::new(input)).unwrap(); + assert!(!doc.is_modified()); + doc.trailer_mut().set(b"Custom".to_vec(), Object::String(craft_pdf::cos::PdfString::literal(b"keep me".to_vec()))); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(back.trailer().get(b"Custom").and_then(|o| o.as_string()).map(|s| s.bytes.as_slice()), Some(b"keep me".as_slice())); + assert_eq!(hayro_pages(&saved), 1); + assert_qpdf_clean(&saved); +} + +#[test] +fn incremental_save_ignores_a_negative_trailer_size() { + let mut doc = Document::open(Arc::new(classic_table())).unwrap(); + doc.trailer_mut().set(b"Size".to_vec(), Object::Int(-1)); + let next = doc.add(Object::Int(5)).num + 1; + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(saved)).unwrap(); + assert_eq!(back.trailer().int(b"Size"), Some(i64::from(next))); +} + +/// What one edit added and a later one unlinked (a deleted comment, a cleared redaction mark) is +/// not written: the file never held it, and writing it would keep what was removed. Additions the +/// update still refers to, from the trailer or a changed object and through other additions, are. +#[test] +fn incremental_save_leaves_out_additions_nothing_refers_to() { + let mut doc = Document::open(Arc::new(classic_table())).unwrap(); + let text = |s: &str| Object::String(craft_pdf::cos::PdfString::literal(s.as_bytes().to_vec())); + let dict = |entries: &[(&str, Object)]| { + let mut d = Dict::new(); + entries.iter().for_each(|(k, v)| d.set(k.as_bytes().to_vec(), v.clone())); + Object::Dict(d) + }; + let page_obj = ObjRef::new(3, 0); + let leaf = doc.add(text("kept leaf")); + let kept = doc.add(dict(&[("Leaf", Object::Ref(leaf))])); + doc.update_dict(page_obj, |d| d.set(b"PieceInfo".to_vec(), Object::Ref(kept))).unwrap(); + let by_trailer = doc.add(text("kept by the trailer")); + doc.trailer_mut().set(b"Custom".to_vec(), Object::Ref(by_trailer)); + // Linked from the page and unlinked again, as clearing a mark does. + let dropped_leaf = doc.add(text("dropped leaf")); + let dropped = doc.add(dict(&[("Leaf", Object::Ref(dropped_leaf))])); + doc.update_dict(page_obj, |d| d.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(dropped)]))).unwrap(); + doc.update_dict(page_obj, |d| { + d.remove(b"Annots"); + }) + .unwrap(); + // Unlinked additions that refer to each other. + let first = doc.add(Object::Null); + let second = doc.add(dict(&[("Next", Object::Ref(first)), ("Note", text("dropped cycle"))])); + doc.set(first, dict(&[("Next", Object::Ref(second))])); + + // Off by default: a writer may add what only it finds again, by scanning. + let kept_all = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(kept_all.windows(12).any(|w| w == b"dropped leaf")); + let saved = write_incremental(&doc, &SaveOptions { drop_unlinked: true, ..SaveOptions::default() }).unwrap(); + let holds = |s: &str| saved.windows(s.len()).any(|w| w == s.as_bytes()); + assert!(holds("kept leaf") && holds("kept by the trailer")); + assert!(!holds("dropped leaf") && !holds("dropped cycle"), "unlinked additions are written"); + let back = Document::open(Arc::new(saved.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(back.get(leaf).as_string().map(|s| s.bytes.as_slice()), Some(b"kept leaf".as_slice())); + for gone in [dropped, dropped_leaf, first, second] { + assert_eq!(*back.get(gone), Object::Null); + } + assert!(!page(&back).contains(b"Annots")); + assert_qpdf_clean(&saved); + + // Freed beforehand, they are written the same way without the option, and what is added + // after that, nothing referring to it either (a document timestamp), is kept. + let mut freed = doc.clone(); + freed.free_unlinked_additions(); + assert_eq!(write_incremental(&freed, &SaveOptions::default()).unwrap(), saved, "the same objects, freed the same way"); + freed.add(text("added after")); + let stamped = write_incremental(&freed, &SaveOptions::default()).unwrap(); + assert!(stamped.windows(11).any(|w| w == b"added after") && !stamped.windows(12).any(|w| w == b"dropped leaf")); +} + +#[test] +fn qpdf_accepts_object_stream_output() { + let qpdf = crate::oracle::or_skip!(crate::oracle::qpdf()); + let doc = Document::open(Arc::new(fixture())).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let path = std::env::temp_dir().join(format!("pdfcraft-objstm-{}.pdf", std::process::id())); + std::fs::write(&path, &packed).unwrap(); + let check = qpdf.command().arg("--check").arg(&path).output().unwrap(); + let _ = std::fs::remove_file(&path); + let report = String::from_utf8_lossy(&check.stdout); + assert!(check.status.success() && report.contains("No syntax or stream encoding errors"), "{report}"); +} + +#[test] +fn the_catalog_of_an_encrypted_document_stays_outside_object_streams() { + // Acrobat reports an encrypted file as damaged when its catalog is in an object stream, + // before asking for the password (#774). Unencrypted output still packs the catalog. + fn root_is_packed(bytes: &[u8], password: Option<&str>) -> bool { + let doc = Document::open_with_password(Arc::new(bytes.to_vec()), password).unwrap(); + let root = doc.trailer().reference(b"Root").unwrap(); + let direct = format!("\n{} 0 obj", root.num); + !bytes.windows(direct.len()).any(|w| w == direct.as_bytes()) + } + let plain = Document::open(Arc::new(fixture())).unwrap(); + assert!(root_is_packed(&write_full(&plain, &SaveOptions::default()).unwrap(), None), "unencrypted: catalog in an object stream"); + let mut enc = plain.clone(); + enc.set_encryption(&craft_pdf::cos::NewEncryption { + algorithm: craft_pdf::cos::Algorithm::Aes256, + user_password: "user", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let packed = write_full(&enc, &SaveOptions::default()).unwrap(); + assert!(!root_is_packed(&packed, Some("user")), "encrypted: catalog must be a standalone object"); + let back = Document::open_with_password(Arc::new(packed.clone()), Some("user")).expect("reopens"); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); +} + +#[test] +fn objects_referenced_by_the_encrypt_dictionary_stay_outside_object_streams() { + // Encrypt a document, then move its crypt filters into indirect objects, as some producers + // do (pdf.js issue7665). A full save must keep them readable before decryption. + let doc = Document::open(Arc::new(fixture())).unwrap(); + let mut enc = doc.clone(); + enc.set_encryption(&craft_pdf::cos::NewEncryption { + algorithm: craft_pdf::cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: -4, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(write_full(&enc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap()); + let mut doc = Document::open(bytes).unwrap(); + let enc_ref = doc.trailer().reference(b"Encrypt").expect("indirect /Encrypt"); + let cf = doc.get(enc_ref).as_dict().unwrap().get(b"CF").cloned().unwrap(); + let cf_ref = doc.add(cf); + doc.update_dict(enc_ref, |d| d.set(b"CF".to_vec(), Object::Ref(cf_ref))).unwrap(); + // Visit this alias from Root before the encryption walk reaches /CF's other alias. + doc.update_dict(doc.root().unwrap(), |d| d.set(b"CFProbe".to_vec(), Object::Ref(cf_ref))).unwrap(); + let classic = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let saved = Document::open(Arc::new(classic.clone())).unwrap(); + let cf_num = saved.get(saved.root().unwrap()).as_dict().unwrap().reference(b"CFProbe").unwrap().num; + for damaged in [false, true] { + let mut bytes = classic.clone(); + if damaged { + // Equal-length edits preserve the xref offsets and its generation-zero row. + for (from, to) in [(format!("\n{cf_num} 0 obj"), format!("\n{cf_num} 1 obj")), (format!("/CF {cf_num} 0 R"), format!("/CF {cf_num} 1 R"))] + { + let start = bytes.windows(from.len()).position(|w| w == from.as_bytes()).unwrap(); + bytes[start..start + from.len()].copy_from_slice(to.as_bytes()); + } + } + let doc = Document::open(Arc::new(bytes)).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + let back = Document::open(Arc::new(packed.clone())).expect("reopens"); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(title(&back), b"Caf\xe9 (draft)"); + assert_eq!(hayro_pages(&packed), 4); + let probe = back.get(back.root().unwrap()).as_dict().unwrap().reference(b"CFProbe").unwrap(); + let encrypt = back.get(back.trailer().reference(b"Encrypt").unwrap()); + assert_eq!(encrypt.as_dict().unwrap().reference(b"CF"), Some(probe)); + assert!( + matches!(back.xref_entry(probe.num), Some(craft_pdf::cos::XrefEntry::InFile { .. })), + "crypt filters must stay outside object streams" + ); + } +} + +#[test] +fn page_tree_loops_in_object_streams_do_not_crash_readers() { + // pdf.js Pages-tree-refs.pdf: /Kids 3 → 4 → 5 → 3. Written faithfully into object streams, + // it used to overflow hayro-syntax's stack (vendored patch: cycle guard in `resolve_pages`). + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [6 0 R 3 0 R] /Count 2 /MediaBox [0 0 595 842] >>", + "<< /Type /Pages /Kids [4 0 R] /Count 1 >>", + "<< /Type /Pages /Kids [5 0 R] /Count 1 >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R >>", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + let packed = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(String::from_utf8_lossy(&packed).contains("/ObjStm")); + assert_eq!(hayro_pages(&packed), 1); +} + +/// Catalog, empty page tree, and trailer entries a reader must not invent: a direct name, a +/// direct string, and a string object reachable only from the trailer. +fn trailer_fixture() -> Vec { + let mut out = b"%PDF-1.4\n".to_vec(); + let mut offsets = Vec::new(); + for (i, body) in ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(only from the trailer)"].iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(b"xref\n0 4\n0000000000 65535 f \n"); + for offset in offsets { + out.extend_from_slice(format!("{offset:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice( + format!("trailer\n<< /Size 4 /Root 1 0 R /Version /1.7 /Custom (keep me) /Note 3 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes(), + ); + out +} + +fn trailer_custom(doc: &Document) -> Option> { + doc.trailer().get(b"Custom").and_then(Object::as_string).map(|s| s.bytes.clone()) +} + +fn trailer_note(doc: &Document) -> Option> { + doc.trailer().get(b"Note").map(|o| doc.resolve(o)).and_then(|o| o.as_string().map(|s| s.bytes.clone())) +} + +#[test] +fn full_save_preserves_unknown_trailer_data() { + let input = trailer_fixture(); + let doc = Document::open(Arc::new(input)).unwrap(); + assert_eq!(trailer_custom(&doc), Some(b"keep me".to_vec())); + assert_eq!(trailer_note(&doc).as_deref(), Some(b"only from the trailer".as_slice())); + assert_eq!(doc.trailer().get(b"Version").and_then(Object::as_name), Some(&b"1.7"[..])); + assert!(doc.repair_log().is_empty()); + assert!(!doc.full_save_required()); + + let inc = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert_eq!(trailer_custom(&inc), trailer_custom(&doc)); + assert_eq!(trailer_note(&inc), trailer_note(&doc)); + assert_eq!(inc.trailer().get(b"Version"), doc.trailer().get(b"Version")); + assert!(inc.repair_log().is_empty()); + + for object_streams in [true, false] { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + let bytes = write_full(&doc, &opts).unwrap(); + let back = Document::open(Arc::new(bytes)).unwrap(); + assert!(back.root().is_some()); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(trailer_custom(&back), trailer_custom(&doc)); + assert_eq!(trailer_note(&back), trailer_note(&doc)); + assert_eq!(back.trailer().get(b"Version"), doc.trailer().get(b"Version")); + } + + let mut required = doc.clone(); + required.require_full_save(); + let back = Document::open(Arc::new(write_incremental(&required, &SaveOptions::default()).unwrap())).unwrap(); + assert!(back.root().is_some()); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(trailer_custom(&back), trailer_custom(&doc)); + assert_eq!(trailer_note(&back), trailer_note(&doc)); + assert_eq!(back.trailer().get(b"Version"), doc.trailer().get(b"Version")); +} + +/// The `startxref` value at the end of `bytes`. +fn last_startxref(bytes: &[u8]) -> usize { + let at = bytes.windows(9).rposition(|w| w == b"startxref").unwrap(); + String::from_utf8_lossy(&bytes[at + 9..]).split_whitespace().next().unwrap().parse().unwrap() +} + +#[test] +fn incremental_saves_of_a_file_with_bytes_before_its_header_count_from_the_header() { + // Offsets written before the prefix was added count from `%PDF-`, as in the KillerPDF file + // of #2364; the appended section must count the same way as the `/Prev` it points back to. + let prefix = vec![b'#'; 128]; + let as_stream = write_full(&Document::open(Arc::new(fixture())).unwrap(), &SaveOptions::default()).unwrap(); + for (name, body, marker) in [("table", classic_table(), &b"xref"[..]), ("stream", as_stream, &b" 0 obj"[..])] { + let input = [prefix.as_slice(), &body].concat(); + let mut doc = Document::open(Arc::new(input.clone())).unwrap(); + assert!(doc.revisions().last().unwrap().header_relative, "{name}"); + doc.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(craft_pdf::cos::PdfString::literal(b"en".to_vec())))).unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.starts_with(&input), "{name}"); + + let xref = last_startxref(&saved) + prefix.len(); + let section = saved.get(xref..).unwrap_or_default(); + assert!(section.windows(marker.len()).position(|w| w == marker).is_some_and(|p| p < 16), "{name}: startxref misses its section"); + let back = Document::open(Arc::new(saved.clone())).unwrap(); + assert_eq!(back.revisions().len(), doc.revisions().len() + 1, "{name}"); + assert!(back.revisions().iter().all(|r| r.header_relative), "{name}: one convention across the chain"); + let edited = back.get(ObjRef::new(1, 0)); + assert_eq!(edited.as_dict().unwrap().get(b"Lang").and_then(|o| o.as_string()).map(|s| s.bytes.as_slice()), Some(b"en".as_slice())); + assert!(back.xref_entry_is_exact(1), "{name}"); + assert_qpdf_clean(&saved); + } +} + +#[test] +fn incremental_saves_of_a_file_whose_offsets_count_from_its_first_byte_keep_counting_so() { + // The other convention: the same file, but its offsets include the bytes before the header. + let prefix = vec![b'#'; 128]; + let mut input = [prefix.as_slice(), &classic_table()].concat(); + let xref = input.windows(6).rposition(|w| w == b"\nxref\n").unwrap() + 1; + let table: String = String::from_utf8_lossy(&input[xref..]) + .lines() + .map(|line| match line.strip_suffix(" 00000 n ").and_then(|o| o.parse::().ok()) { + Some(o) => format!("{:010} 00000 n \n", o + prefix.len()), + None if line.parse::().is_ok() => format!("{xref}\n"), + None => format!("{line}\n"), + }) + .collect(); + input.truncate(xref); + input.extend_from_slice(table.as_bytes()); + + let mut doc = Document::open(Arc::new(input.clone())).unwrap(); + assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); + assert!(!doc.revisions().last().unwrap().header_relative); + doc.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(craft_pdf::cos::PdfString::literal(b"en".to_vec())))).unwrap(); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.get(last_startxref(&saved)..).unwrap_or_default().starts_with(b"xref")); + let back = Document::open(Arc::new(saved.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_eq!(back.revisions().len(), 2); + assert!(back.xref_entry_is_exact(1)); + // No qpdf check here: qpdf counts every offset from the header in a file with bytes before + // it, so it already reconstructs this input before anything is appended. +} + +/// One page whose catalog carries a PDF 2.0 document part tree (`/DPartRoot`, ISO 32000-2 +/// 14.12) and whose structure tree root lists structure namespaces (`/Namespaces`, 14.7.4), +/// both through indirect objects that nothing else references. +fn pdf2_fixture() -> Vec { + Pdf::new() + .version("2.0") + .obj("<< /Type /Catalog /Pages 2 0 R /DPartRoot 6 0 R /StructTreeRoot 8 0 R >>") + .obj("<< /Type /Pages /Kids [4 0 R] /Count 1 /MediaBox [0 0 200 300] >>") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 5 0 R /Resources << /Font << /F1 3 0 R >> >> >>") + .stream("", "BT /F1 24 Tf 20 150 Td (Part) Tj ET") + .obj("<< /Type /DPartRoot /DPartRootNode 7 0 R /NodeNameList [/Chapter] >>") + .obj("<< /Type /DPart /Parent 6 0 R /Start 4 0 R /End 4 0 R /DPM << /Kind (cover) >> >>") + .obj("<< /Type /StructTreeRoot /K [] /Namespaces [9 0 R] >>") + .obj("<< /Type /Namespace /NS (http://iso.org/pdf2/ssn) /Schema << /Type /Filespec /F (ssn.rnc) >> >>") + .build() +} + +/// What `pdf2_fixture` declares, read back through the catalog. +fn assert_dpart_and_namespaces(doc: &Document) { + let catalog = doc.get(doc.root().unwrap()); + let catalog = catalog.as_dict().unwrap(); + let root = doc.resolve(catalog.get(b"DPartRoot").expect("catalog keeps /DPartRoot")); + let root = root.as_dict().unwrap(); + let node = doc.resolve(root.get(b"DPartRootNode").expect("/DPartRootNode kept")); + let node = node.as_dict().unwrap(); + assert_eq!(node.name(b"Type"), Some(&b"DPart"[..])); + // Both ends of the part are still the one page, found through /Pages (references are + // renumbered by a full save, so they are compared inside this document). + let pages = doc.resolve(catalog.get(b"Pages").unwrap()); + let kids = pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap(); + let page = kids.first().and_then(|k| k.as_ref()).expect("the page is still the first kid"); + assert_eq!(node.reference(b"Start"), Some(page), "the part still starts at its page"); + assert_eq!(node.reference(b"End"), Some(page), "the part still ends at its page"); + assert!(doc.resolve(node.get(b"DPM").unwrap()).as_dict().unwrap().get(b"Kind").is_some()); + let tree = doc.resolve(catalog.get(b"StructTreeRoot").unwrap()); + let namespaces = doc.resolve(tree.as_dict().unwrap().get(b"Namespaces").expect("/Namespaces kept")); + let namespace = doc.resolve(&namespaces.as_array().unwrap()[0]); + let namespace = namespace.as_dict().unwrap(); + assert_eq!(namespace.get(b"NS").unwrap().as_string().unwrap().bytes, b"http://iso.org/pdf2/ssn"); + assert!(namespace.get(b"Schema").is_some(), "/Schema is retained"); +} + +#[test] +fn document_parts_and_structure_namespaces_survive_full_and_incremental_saves() { + let original = Arc::new(pdf2_fixture()); + let doc = Document::open(original.clone()).unwrap(); + assert_dpart_and_namespaces(&doc); + + for object_streams in [true, false] { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + let full = write_full(&doc, &opts).unwrap(); + let back = Document::open(Arc::new(full.clone())).unwrap(); + assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); + assert_dpart_and_namespaces(&back); + // A second generation, rewritten from the rewritten file. + let second = Document::open(Arc::new(write_full(&back, &opts).unwrap())).unwrap(); + assert!(second.repair_log().is_empty(), "{:?}", second.repair_log()); + assert_dpart_and_namespaces(&second); + } + + // An incremental save of an unrelated edit leaves the file's bytes, and so both, alone. + let mut edited = Document::open(original.clone()).unwrap(); + edited.add(Object::Int(1)); + let updated = write_incremental(&edited, &SaveOptions::default()).unwrap(); + assert!(updated.starts_with(&original)); + let reopened = Document::open(Arc::new(updated)).unwrap(); + assert!(reopened.repair_log().is_empty(), "{:?}", reopened.repair_log()); + assert_dpart_and_namespaces(&reopened); +}