diff --git a/Cargo.lock b/Cargo.lock index 2f716ff214..f6a04eb16c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1187,25 +1187,39 @@ dependencies = [ [[package]] name = "craft-cms" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ + "md-5 0.10.6", "rayon", ] +[[package]] +name = "craft-codepage" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" + +[[package]] +name = "craft-csv" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-docpath" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" [[package]] name = "craft-fonts-build" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" [[package]] name = "craft-i18n" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "fixed_decimal", "icu_decimal", @@ -1228,7 +1242,7 @@ dependencies = [ [[package]] name = "craft-keymap" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "egui", "serde", @@ -1238,7 +1252,7 @@ dependencies = [ [[package]] name = "craft-menu" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "craft-keymap", "craft-quit", @@ -1248,24 +1262,51 @@ dependencies = [ "objc2-app-kit 0.3.2", ] +[[package]] +name = "craft-pdf" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" +dependencies = [ + "aes", + "chrono", + "craft-stream-codecs", + "flate2", + "js-sys", + "md-5 0.11.0", + "sha2 0.11.1", + "stringprep", + "thiserror 2.0.21", + "wasm-bindgen", + "zeroize", +] + +[[package]] +name = "craft-prc" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" +dependencies = [ + "flate2", + "thiserror 2.0.21", +] + [[package]] name = "craft-print" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "craft-winprint-gdi", "log", "miniz_oxide 0.9.1", "roxmltree 0.21.2", "tempfile", - "winprint 0.2.1 (git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0)", + "winprint 0.2.1 (git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02)", "winsafe", ] [[package]] name = "craft-quit" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "dispatch2", "egui", @@ -1273,10 +1314,30 @@ dependencies = [ "objc2-app-kit 0.3.2", ] +[[package]] +name = "craft-rtf" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" +dependencies = [ + "craft-codepage", + "encoding_rs", +] + +[[package]] +name = "craft-stream-codecs" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" +dependencies = [ + "fax", + "hayro-ccitt", + "hayro-jbig2", + "thiserror 2.0.21", +] + [[package]] name = "craft-ui" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "craft-fonts-build", "craft-i18n", @@ -1297,7 +1358,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "chrono", "dunce", @@ -1352,7 +1413,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=7bb6c9007#7bb6c90076a0177d01cc4c482e7fd8dc9f31a8bf" +source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991" dependencies = [ "flate2", "thiserror 2.0.21", @@ -2785,7 +2846,6 @@ version = "0.3.0" dependencies = [ "fearless_simd", "hayro-ccitt", - "image", ] [[package]] @@ -3517,7 +3577,7 @@ dependencies = [ "indexmap", "itoa", "log", - "md-5", + "md-5 0.11.0", "nom", "rand", "rangemap", @@ -3534,6 +3594,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest 0.10.7", +] + [[package]] name = "md-5" version = "0.11.0" @@ -4345,6 +4415,7 @@ version = "0.21.1" dependencies = [ "base64 0.23.1", "craft-cms", + "craft-prc", "flate2", "image", "pdfcraft-cos", @@ -4353,7 +4424,6 @@ dependencies = [ "pdfcraft-model", "pdfcraft-organize", "pdfcraft-platform", - "pdfcraft-prc", "pdfcraft-prc-render", "pdfcraft-render", "pdfcraft-xfa", @@ -4398,17 +4468,12 @@ dependencies = [ name = "pdfcraft-cos" version = "0.21.1" dependencies = [ - "chrono", + "craft-pdf", "hayro-syntax", - "js-sys", - "pdfcraft-crypt", - "pdfcraft-filters", "pdfcraft-oracle", "pdfcraft-testkit", "proptest", "serde_json", - "thiserror 2.0.21", - "wasm-bindgen", ] [[package]] @@ -4435,13 +4500,8 @@ dependencies = [ name = "pdfcraft-crypt" version = "0.21.1" dependencies = [ - "aes", - "md-5", + "craft-pdf", "proptest", - "sha2 0.11.1", - "stringprep", - "thiserror 2.0.21", - "zeroize", ] [[package]] @@ -4512,6 +4572,7 @@ dependencies = [ name = "pdfcraft-export" version = "0.21.1" dependencies = [ + "craft-rtf", "craft-zip", "flate2", ] @@ -4520,12 +4581,8 @@ dependencies = [ name = "pdfcraft-filters" version = "0.21.1" dependencies = [ - "fax", - "flate2", - "hayro-ccitt", - "hayro-jbig2", + "craft-pdf", "proptest", - "thiserror 2.0.21", "weezl 0.2.1", ] @@ -4652,21 +4709,12 @@ dependencies = [ "zbus", ] -[[package]] -name = "pdfcraft-prc" -version = "0.21.1" -dependencies = [ - "flate2", - "proptest", - "thiserror 2.0.21", -] - [[package]] name = "pdfcraft-prc-render" version = "0.21.1" dependencies = [ + "craft-prc", "flate2", - "pdfcraft-prc", "thiserror 2.0.21", ] @@ -4881,6 +4929,7 @@ name = "pdfcraft-xfdf" version = "0.21.1" dependencies = [ "base64 0.23.1", + "craft-csv", "pdfcraft-annot", "pdfcraft-cos", "pdfcraft-forms", @@ -8368,7 +8417,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0#bc06a7c48b768eb27a828c7986f6d2868b245ee0" +source = "git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02#72951d7455ebcab569fbd9386c513d3d9f53fb02" dependencies = [ "flate2", "fmt-derive", diff --git a/Cargo.toml b/Cargo.toml index 5532bbef13..697c5d3dd9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,25 +15,28 @@ repository = "https://github.com/storytold/pdfcraft" # Small, revision-bound UI drafts; the existing engine remains the owner of PDF edits/history. edit-session = { path = "vendor/edit-session", version = "=0.1.0-alpha.2" } # Shared frontend components; engines remain independent of egui. -craft-ui = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff", features = ["serde"] } +craft-ui = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", features = ["serde"] } # Quit the macOS way (#1136): ⌘Q asks about unsaved changes, and closing quits through `terminate:`. -craft-quit = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } +craft-quit = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } # One menu definition for the native macOS menu bar and the in-window bar elsewhere. -craft-menu = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } -craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff", default-features = false } +craft-menu = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } +craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", default-features = false } # Interface translations: lookup, fallback, CLDR plurals and language names over the craft-i18n # catalogs that `crates/ui-egui/build.rs` embeds from `CRAFT_I18N_DIR` (storytold/craft-i18n, at # the commit in `craft-i18n.rev`). Already in the tree through craft-ui. -craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } +craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } # Shared system spooler; document composition remains in pdfcraft-print. -craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } +craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } # ICC transforms for app-managed print colour; at craft-print's commit, so one craft-libs checkout. -craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" } -craft-zip = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "7bb6c9007" } +craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } +craft-zip = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } # Whether a path stored in a document may be opened (GoToR link targets). -craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } +craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "e2dc681a3" } # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "e2dc681a3", features = ["egui"] } +craft-pdf = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", features = ["cos", "crypt", "filters"] } +craft-rtf = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } +craft-csv = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" } # internal pdfcraft-geom = { path = "crates/geom", version = "0.21.0" } pdfcraft-filters = { path = "crates/filters", version = "0.21.0" } diff --git a/crates/cos/Cargo.toml b/crates/cos/Cargo.toml index 0bf2e894d2..8bb7d6f017 100644 --- a/crates/cos/Cargo.toml +++ b/crates/cos/Cargo.toml @@ -9,18 +9,7 @@ license.workspace = true workspace = true [dependencies] -pdfcraft-crypt = { workspace = true } -pdfcraft-filters = { workspace = true } -thiserror = { workspace = true } -# PDF date strings (`pdf_date`) go through chrono (no hand-rolled calendar code). -chrono = { workspace = true } - -# On the web, a rewrite's fresh file identifier takes its entropy from the browser -# (`crypto.getRandomValues`). Both crates are already in the dependency tree of the web shell -# (`js-sys` 0.3 / `wasm-bindgen` 0.2); native builds compile neither. -[target.'cfg(target_arch = "wasm32")'.dependencies] -js-sys = "0.3" -wasm-bindgen = "0.2" +craft-pdf = { workspace = true } [dev-dependencies] pdfcraft-oracle = { workspace = true } diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs deleted file mode 100644 index 0067304f66..0000000000 --- a/crates/cos/src/budget.rs +++ /dev/null @@ -1,1187 +0,0 @@ -//! A shared allowance for operations that decode many streams. - -use std::collections::HashSet; - -use pdfcraft_filters::FilterError; - -use crate::{CosError, Document, ObjRef, Object, Stream}; - -/// The steps of decoding an operation over a document's images may take whatever the file's size. -/// A step of JBIG2 decoding ([`hayro_jbig2::Image::decode_within`]) is about a pixel, and takes 6 to -/// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the -/// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to -/// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step -/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A -/// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks -/// that make nothing takes up to 21 ns a byte: about 7 ns a step. -const IMAGE_WORK: u64 = 1 << 34; -/// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a -/// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to -/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. -const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; - -/// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every -/// form XObject it enters, each occurrence counted again. -pub const MAX_PAGE_DECODED: usize = 256 << 20; -/// Form XObjects one page's content walk may enter, each entry counted again. -pub const MAX_PAGE_FORMS: usize = 50_000; - -/// The most pixels an image redaction decodes, to clear the ones under a mark or to store again -/// without the `/JBIG2Globals` it shares with a redacted image (`pdfcraft_redact`). Optimize has -/// JBIG2 images share a dictionary only within it (`pdfcraft_optimize`), so that redaction can -/// store each of them again: one it couldn't would refuse the redaction. -pub const MAX_REDACTED_IMAGE_PIXELS: u64 = 64 << 20; - -/// The steps a sample of a JPEG costs: a few bytes can hold a page of one grey, and decoding, -/// resampling and storing it took about 20 to 45 ns a sample. -const JPEG_WORK: u64 = 2; -/// The steps a byte of a JPEG costs to read: its tables took 24 ns a byte before a size was known, -/// and images can share one, so reading it again for each has to count. -const JPEG_READ_WORK: u64 = 4; - -/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data and -/// whether it stopped at damage, or the error, each with that count. -type Decoder = fn(&Stream, usize) -> Result<(Vec, bool, usize), (FilterError, usize)>; - -/// What one operation over a document's images may still spend decoding them: exporting them, -/// optimizing them, and storing LZW streams with Flate as it does, clearing them for redaction. -/// -/// An image has its own limit, which doesn't bound a file of thousands of images made to reach it: -/// a few hundred bytes of JBIG2 can ask for seconds of decoding, 4 KB of Flate inside Flate for a -/// gigabyte, and an image that shares `/JBIG2Globals` decodes them again. So images share this -/// allowance, whatever they do with what they decode and whether or not it works, and past it they -/// aren't decoded. A caller then treats them as it does an image it can't read, and -/// [`ImageBudget::refused`] says how many there were. -/// -/// An image also pays for its pixels once decoded, as what reads it goes through them one by one -/// however few bytes coded them: blank rows of fax or Flate take almost none. It pays for no more -/// than its data holds, a bit a pixel at least, so an image declaring a size its data doesn't fill -/// costs what a smaller one would, not the run's allowance. A JPEG pays for its samples when -/// it is decoded ([`ImageBudget::pay_jpeg`]), and is otherwise not charged, as JPEG 2000 isn't: -/// their decoders don't run here. -#[derive(Debug)] -pub struct ImageBudget { - work: u64, - refused: usize, -} - -impl ImageBudget { - /// The allowance for an operation on `doc`, which a longer file makes larger. The streams made - /// or changed since it was opened count as the file's bytes do, so a document built in memory - /// (combined, created, or given pages of scans) gets what it would once saved and opened again. - pub fn new(doc: &Document) -> Self { - let length = |n: usize| u64::try_from(n).unwrap_or(u64::MAX); - let edited = doc - .scan_objects_subset(doc.modified_objects()) - .filter_map(|(_, object)| match &*object.ok()? { - Object::Stream(s) => Some(length(s.raw.len())), - _ => None, - }) - .fold(0u64, u64::saturating_add); - let bytes = length(doc.bytes().len()).saturating_add(edited); - Self::with_work(IMAGE_WORK.saturating_add(bytes.saturating_mul(IMAGE_WORK_PER_BYTE))) - } - - /// An allowance of `work` steps of decoding. - pub fn with_work(work: u64) -> Self { - Self { work, refused: 0 } - } - - /// The samples of image `s`, decoded and charged to the allowance: the one way to decode an - /// image outside the renderer, so that an operation over many images can't decode them without - /// one. - /// - /// They go through its filters ([`Stream::decoded`]), or, for a CCITT fax or JBIG2 image (its - /// last filter), through the filters before it and then [`pdfcraft_filters::decode_ccitt_within`] - /// with its [`Document::fax_params`], or [`pdfcraft_filters::decode_jbig2_within`] with its - /// `/JBIG2Globals`, as `CCITTFaxDecode` and `JBIG2Decode` give them (ISO 32000-2 §7.4.6, - /// §7.4.7). Parameters given by reference count, for every filter - /// ([`Document::with_direct_parms`]). A fax image's `/Columns` has to be its `/Width`, and a - /// JBIG2 image's page its `/Width` × `/Height`: PdfCraft's renderer draws the whole page into - /// the image. Up to its `/Height` rows are decoded, whatever a fax image's `/Rows` says; damaged - /// fax data gives fewer, so a caller checks it has the rows it needs. Damaged JBIG2 data, and - /// globals that can't be read (within [`MAX_JBIG2_GLOBALS`](crate::MAX_JBIG2_GLOBALS)), are an - /// error. - /// - /// No stage, the filters before a fax or JBIG2 decoder included, may produce more than `max` - /// bytes, nor more than [`MAX_DECODED`](crate::MAX_DECODED). A JBIG2 image's `/JBIG2Globals` - /// are another stream, shared between images, and are decoded within `MAX_JBIG2_GLOBALS` - /// whatever `max` is. Each filter as it runs, the globals when they have to be read, and the fax - /// or JBIG2 decoder spend from the allowance, and the image then pays for its pixels; with - /// nothing left an image is refused before anything is read, its globals included. - /// - /// ``` - /// # let doc = pdfcraft_cos::Document::new_empty(); - /// # let s = pdfcraft_cos::Stream::from_raw(pdfcraft_cos::Dict::new(), Vec::new()); - /// let samples = pdfcraft_cos::ImageBudget::new(&doc).decode(&doc, &s, 1 << 20); - /// assert_eq!(samples.unwrap(), b""); - /// ``` - /// - /// `Document` has no way of its own to decode an image: - /// - /// ```compile_fail - /// # let doc = pdfcraft_cos::Document::new_empty(); - /// # let s = pdfcraft_cos::Stream::from_raw(pdfcraft_cos::Dict::new(), Vec::new()); - /// let _ = doc.decoded_image(&s); - /// ``` - /// - /// ```compile_fail - /// # let doc = pdfcraft_cos::Document::new_empty(); - /// # let s = pdfcraft_cos::Stream::from_raw(pdfcraft_cos::Dict::new(), Vec::new()); - /// let _ = doc.decoded_image_within(&s, 1 << 20); - /// ``` - pub fn decode(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { - let decoded = doc.decoded_image_spending(s, max, &mut self.work); - self.count_refused(s, &decoded); - self.pay_size(s, decoded?) - } - - /// Image `s` through its filters alone, charged to the allowance: [`Stream::decoded_within`] - /// with its `/DecodeParms` given by reference read too ([`Document::with_direct_parms`]), for an - /// image that has no fax or JBIG2 decoder to go through. - pub fn decode_filters(&mut self, doc: &Document, s: &Stream, max: usize) -> Result, CosError> { - let decoded = doc.with_direct_parms(s).decoded_spending(max, &mut self.work); - self.count_refused(s, &decoded); - let decoded = decoded?; - if s.filters().iter().all(|(f, _)| f.is_decodable()) { self.pay_size(s, decoded) } else { Ok(decoded) } - } - - /// `data` through `chain`, any corruption being an error ([`pdfcraft_filters::decode_within`]), - /// for a stream a run rewrites rather than an image it reads, charged to the allowance. - pub fn decode_chain( - &mut self, - chain: &[(pdfcraft_filters::Filter, pdfcraft_filters::Params)], - data: &[u8], - max: usize, - ) -> Result, FilterError> { - pdfcraft_filters::decode_within(chain, data, max, &mut self.work) - } - - /// Pays `steps` of other work on decoded data, or spends all there is and says so. - pub fn spend(&mut self, steps: u64) -> bool { - let left = self.work.checked_sub(steps); - self.work = left.unwrap_or(0); - left.is_some() - } - - /// Pays for `pixels` a decode goes on to work through, or spends all there is, and the image - /// is refused. - pub fn pay_pixels(&mut self, pixels: u64) -> bool { - let paid = self.spend(pixels); - self.refused += usize::from(!paid); - paid - } - - /// Pays for reading JPEG `data` and decoding its `samples`, as [`ImageBudget::pay_pixels`] does. - /// A caller pays before its tables are parsed, which can be most of the file. - pub fn pay_jpeg(&mut self, samples: u64, data: &[u8]) -> bool { - let reading = (data.len() as u64).saturating_mul(JPEG_READ_WORK); - self.pay_pixels(samples.saturating_mul(JPEG_WORK).saturating_add(reading)) - } - - /// Pays for the pixels of image `s` that its `decoded` data holds, those of its `/Width` and - /// `/Height` but no more than a bit of the data each, unless it is JBIG2, whose decoder pays for - /// its page, or isn't decoded here. What can't be paid for refuses the image. - fn pay_size(&mut self, s: &Stream, decoded: Vec) -> Result, CosError> { - if !charged(s) || matches!(s.filters().last(), Some((pdfcraft_filters::Filter::Jbig2, _))) { - return Ok(decoded); - } - let side = |key: &[u8]| s.dict.int(key).and_then(|v| u64::try_from(v).ok()).unwrap_or(0); - let held = u64::try_from(decoded.len()).unwrap_or(u64::MAX).saturating_mul(8); - if self.pay_pixels(side(b"Width").saturating_mul(side(b"Height")).min(held)) { - Ok(decoded) - } else { - Err(CosError::Filter(FilterError::Spent.to_string())) - } - } - - /// An image that failed with nothing left was refused, unless it has a filter that costs - /// nothing here, which fails whatever is left. - fn count_refused(&mut self, s: &Stream, decoded: &Result, CosError>) { - if decoded.is_err() && self.work == 0 && charged(s) { - self.refused += 1; - } - } - - /// The decodes refused for lack of allowance: an image read twice over is counted twice. - pub fn refused(&self) -> usize { - self.refused - } - - /// Charges `work` steps paid for out of another allowance, by a thread working on its own. - pub fn charge(&mut self, work: u64) { - self.work = self.work.saturating_sub(work); - } - - /// The steps of decoding still allowed. - pub fn work_left(&self) -> u64 { - self.work - } -} - -/// Whether `s` can be decoded here and so charged: general filters, a fax or JBIG2 decoder last at -/// most. Those decoders anywhere else can't run. -fn charged(s: &Stream) -> bool { - use pdfcraft_filters::Filter; - s.readable_filters().is_ok_and(|chain| { - chain.split_last().is_none_or(|((last, _), before)| { - (last.is_decodable() || matches!(last, Filter::Jbig2 | Filter::CcittFax)) && before.iter().all(|(f, _)| f.is_decodable()) - }) - }) -} - -/// What one operation may still decode and enter across all the streams it reads. -/// -/// [`MAX_DECODED`](crate::MAX_DECODED) bounds one decode. It does not bound a page whose -/// `/Contents` names the same compressed stream thousands of times, or a form drawn thousands of -/// times from other forms: each occurrence would get a fresh allowance. A `DecodeBudget` is -/// shared by every decode of one operation, so the total decoded bytes and form entries stay -/// under fixed limits however the file repeats its references. A decode that runs past the -/// remaining allowance spends all of it, so a bomb referenced many times is inflated once. -#[derive(Debug)] -pub struct DecodeBudget { - /// The whole allowance, for errors. - limit: usize, - bytes: usize, - forms: usize, - seen: HashSet, - /// Decodes refused for lack of allowance ([`DecodeBudget::refused`]). - refusals: usize, -} - -impl DecodeBudget { - /// An allowance of `bytes` decoded bytes and `forms` form entries. - pub fn new(bytes: usize, forms: usize) -> Self { - Self { limit: bytes, bytes, forms, seen: HashSet::new(), refusals: 0 } - } - - /// The allowance for one page: [`MAX_PAGE_DECODED`] bytes and [`MAX_PAGE_FORMS`] forms. - pub fn page() -> Self { - Self::new(MAX_PAGE_DECODED, MAX_PAGE_FORMS) - } - - /// Decoded bytes still allowed. - pub fn remaining(&self) -> usize { - self.bytes - } - - /// How many decodes were refused for lack of allowance: one that came once it was spent, or - /// ran past what was left. A decode that spent exactly what was left wasn't refused, so an - /// allowance used up to the byte has none. - pub fn refused(&self) -> usize { - self.refusals - } - - /// Whether the allowance is spent and `stream`, which has data, would be refused, noting the - /// refusal ([`DecodeBudget::refused`]): a caller asks this before resolving the stream's - /// parameters, work the allowance doesn't count. A stream without data still reads. - pub fn refuses(&mut self, stream: &Stream) -> bool { - let refuses = self.bytes == 0 && !stream.raw.is_empty(); - self.refusals += usize::from(refuses); - refuses - } - - /// Whether a decode was refused for lack of allowance or a form entry for lack of visits. - pub fn exhausted(&self) -> bool { - self.bytes == 0 || self.forms == 0 - } - - /// `stream` decoded like [`Stream::decoded`], charged to the allowance. Fails, without - /// decoding, once the allowance is spent. A stream damaged partway gives what decoded before - /// the damage, so code that writes the stream back reads it with - /// [`DecodeBudget::decode_with_damage`] instead. - pub fn decode(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_within_counted).map(|(data, _)| data) - } - - /// Like [`DecodeBudget::decode`], for a stream with a tighter bound of its own: decoding - /// past `max` bytes is an error that charges `max`, the work it took to find out. - pub fn decode_within(&mut self, stream: &Stream, max: usize) -> Result, CosError> { - self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) - } - - /// Like [`DecodeBudget::decode_within`], with every filter stage held to `max`, as - /// [`Stream::decoded_within`] holds it, and the stages together to `max` for each of them, the - /// stream's `/DecodeParms` given by reference read in `doc` through a view of it that shares - /// its data and copies no more of its dictionary than its filters and parameters - /// ([`DecodeBudget::decode_with_damage`] reads it so). A - /// stream so reads here when it reads without an allowance and the allowance can pay for - /// what its stages decode, the output of one stage on top of another's. A stage past `max` - /// is an error that charges what the stages before it decoded and `max`, the work it took to - /// find out; past the stages' total it charges that total, and past what is left it spends - /// all of it, as [`DecodeBudget::decode_within`] does. - pub fn decode_stages_within(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result, CosError> { - // Before the parameters are resolved, work the allowance doesn't count. - if self.refuses(stream) { - return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); - } - let stream = doc.decoding_view(stream); - let total = max.saturating_mul(stream.filters().len().max(1)); - let room = total.min(self.bytes); - match stream.decoded_within_each_counted(max, room) { - Ok((data, _, spent)) => { - self.bytes = self.bytes.saturating_sub(spent); - Ok(data) - } - // A stage stopped at `max`, below what was left of the room. - Err((FilterError::LimitExceeded(_), spent)) if room.saturating_sub(spent) > max => { - self.bytes = self.bytes.saturating_sub(spent.saturating_add(max)); - Err(CosError::Filter(FilterError::LimitExceeded(max).to_string())) - } - // At the stages' total, with more left. - Err((FilterError::LimitExceeded(_), _)) if total < self.bytes => { - self.bytes = self.bytes.saturating_sub(total); - Err(CosError::Filter(FilterError::LimitExceeded(total).to_string())) - } - Err((FilterError::LimitExceeded(_), _)) => { - self.bytes = 0; - self.refusals += 1; - Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())) - } - Err((e, spent)) => { - self.bytes = self.bytes.saturating_sub(spent); - Err(CosError::Filter(e.to_string())) - } - } - } - - /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), with the stream's - /// `/DecodeParms` given by reference read in `doc`, also saying whether the stream is damaged: - /// `true` when a filter stopped at corrupt data, so the bytes are only what decoded before it. - /// A stream written back from them loses everything after the damage, which other readers may - /// still read. - pub fn decode_with_damage(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { - self.charge(&doc.decoding_view(stream), max, Stream::decoded_within_counted) - } - - /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to - /// the allowance, with its `/DecodeParms` given by reference read in `doc`. - pub fn decode_strict(&mut self, doc: &Document, stream: &Stream) -> Result, CosError> { - let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); - self.charge(&doc.decoding_view(stream), usize::MAX, strict).map(|(data, _)| data) - } - - /// Decodes `stream` within `max` and the allowance, charging what every filter stage - /// decoded: a stage's output counts even when a later stage shrinks it, and a corrupt stream - /// pays for the bytes decoded before the error, so referencing a stream again and again - /// spends the allowance by the work each decode does. - fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result<(Vec, bool), CosError> { - if self.refuses(stream) { - return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); - } - if max < self.bytes { - let result = decode(stream, max); - let spent = match &result { - Ok((_, _, spent)) => *spent, - Err((FilterError::LimitExceeded(_), _)) => max, - Err((_, spent)) => *spent, - }; - self.bytes = self.bytes.saturating_sub(spent); - return result.map(|(data, damaged, _)| (data, damaged)).map_err(|(e, _)| CosError::Filter(e.to_string())); - } - match decode(stream, self.bytes) { - Ok((data, damaged, spent)) => { - self.bytes = self.bytes.saturating_sub(spent); - Ok((data, damaged)) - } - Err((FilterError::LimitExceeded(_), _)) => { - self.bytes = 0; - self.refusals += 1; - Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())) - } - Err((e, spent)) => { - self.bytes = self.bytes.saturating_sub(spent); - Err(CosError::Filter(e.to_string())) - } - } - } - - /// Counts one form entry. `false` once [`DecodeBudget::new`]'s `forms` entries are spent: - /// the caller leaves that form out. - pub fn enter_form(&mut self) -> bool { - match self.forms.checked_sub(1) { - Some(left) => { - self.forms = left; - true - } - None => false, - } - } - - /// `true` the first time `r` is passed to this budget, for callers that need each - /// referenced stream only once. - pub fn first_visit(&mut self, r: ObjRef) -> bool { - self.seen.insert(r) - } - - /// Each stream `contents` names, decoded against the allowance, in order, its `/DecodeParms` - /// given by reference read in `doc`. `contents` is a page's `/Contents`: one stream, or an - /// array whose non-stream entries are left out. A stream that fails to decode, or arrives - /// after the allowance is spent, is an `Err` in its place, so the caller chooses between - /// skipping it and giving up. - pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { - self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() - } - - /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway - /// ([`DecodeBudget::decode_with_damage`], which reads `/DecodeParms` given by reference). - pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { - match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode_with_damage(doc, s, usize::MAX)], - Object::Array(list) => list - .iter() - .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode_with_damage(doc, s, usize::MAX)), - _ => None, - }) - .collect(), - _ => Vec::new(), - } - } -} - -#[cfg(test)] -mod image_tests { - use super::*; - use crate::Dict; - - fn jbig2(parms: Option) -> Stream { - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(16)); - dict.set(b"Height".to_vec(), Object::Int(2)); - dict.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); - if let Some(parms) = parms { - dict.set(b"DecodeParms".to_vec(), parms); - } - Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) - } - - /// What a Flate stage pays to read `n` bytes, besides what it makes. - fn read(n: usize) -> u64 { - pdfcraft_filters::FLATE_READ_WORK * n as u64 - } - - /// Each image spends what it took from one allowance, and those past it are refused and - /// counted, whatever their filters. What can't be decoded here costs nothing, and fails - /// without being one of these. - #[test] - fn the_images_of_an_operation_share_its_allowance() { - let doc = Document::new_empty(); - let image = jbig2(None); - let mut budget = ImageBudget::with_work(u64::MAX); - let samples = budget.decode(&doc, &image, 1 << 20).unwrap(); - let cost = u64::MAX - budget.work_left(); - assert!(cost > 0); - let mut budget = ImageBudget::with_work(3 * cost); - for left in [2, 1, 0] { - assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(budget.work_left(), left * cost); - } - assert_eq!(budget.refused(), 0, "the allowance held exactly three"); - for refused in [1, 2] { - assert!(budget.decode(&doc, &image, 1 << 20).is_err()); - assert_eq!(budget.refused(), refused); - } - assert!(budget.decode(&doc, &Stream::flate(Dict::new(), b"samples"), 1 << 20).is_err()); - assert!(budget.decode(&doc, &Stream::from_raw(Dict::new(), b"samples".to_vec()), 1 << 20).is_err()); - assert_eq!(budget.refused(), 4, "other images share it"); - for name in ["NoSuchFilter", "DCTDecode", "JPXDecode"] { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name(name)); - assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); - assert_eq!(budget.refused(), 4, "{name} isn't one of these"); - } - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Int(5)); - assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); - assert_eq!(budget.refused(), 4, "a /Filter that can't be read isn't either"); - } - - /// What a file of images each inflating a megabyte asks for: 4 KB of Flate inside Flate can ask - /// for a gigabyte, and these aren't JBIG2, so without the allowance nothing would stop them. - #[test] - fn filter_bombs_stop_at_the_allowance() { - let doc = Document::new_empty(); - let samples = vec![0; 1 << 20]; - let inner = pdfcraft_filters::encode_flate(&samples); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); - let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); - let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); - let mut budget = ImageBudget::with_work(2 * cost + 1); - for left in [cost + 1, 1] { - assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); - assert_eq!(budget.work_left(), left); - } - assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); - assert_eq!((budget.work_left(), budget.refused()), (0, 1), "the first it can't pay for ends it"); - } - - /// Filters in front of a JBIG2 decoder pay as they run, on top of what the decoder takes, and - /// one that fails has still paid: nothing about the JBIG2 data is known by then, and each image - /// of a file can do this. - #[test] - fn filters_in_front_of_jbig2_pay_whether_or_not_they_work() { - let doc = Document::new_empty(); - let alone = jbig2(None); - let mut budget = ImageBudget::with_work(u64::MAX); - let samples = budget.decode(&doc, &alone, 1 << 20).unwrap(); - let decoding = u64::MAX - budget.work_left(); - let behind = |raw: Vec| { - let mut dict = alone.dict.clone(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("JBIG2Decode")])); - Stream::from_raw(dict, raw) - }; - - let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); - let mut budget = ImageBudget::with_work(u64::MAX); - assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); - - // Past this image's limit: an error, with what finding out took paid and the rest left. - let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); - let mut budget = ImageBudget::with_work(u64::MAX); - assert!(budget.decode(&doc, &bomb, 1000).is_err()); - assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); - assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); - - // A small allowance is a limit too: it stops there, and the image is refused. - let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); - assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); - assert_eq!((budget.work_left(), budget.refused()), (0, 1)); - } - - /// Globals that are read again for every image pay every time, and fail having paid. Kept - /// ones have no filters left to undo, so they pay for being read and nothing more. - #[test] - fn globals_pay_for_their_filters_each_time_they_are_decoded() { - let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; - let raw = pdfcraft_filters::encode_flate(&skipped); - let stream = |doc: &Document, r: ObjRef| match &*doc.get(r) { - Object::Stream(s) => s.clone(), - _ => panic!("not a stream"), - }; - let spent = |doc: &Document, s: &Stream| { - let mut budget = ImageBudget::with_work(u64::MAX); - budget.decode(doc, s, 1 << 20).unwrap(); - u64::MAX - budget.work_left() - }; - let mut doc = Document::new_empty(); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let globals = doc.add(Object::Stream(Stream::from_raw(dict, raw.clone()))); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - let image = doc.add(Object::Stream(jbig2(Some(Object::Dict(parms))))); - let root = doc.root().unwrap(); - doc.update_dict(root, |c| c.set(b"Image".to_vec(), Object::Ref(image))).unwrap(); - - // Added in memory, so not kept: the filters are undone for each read. - let s = stream(&doc, image); - let first = spent(&doc, &s); - assert_eq!(spent(&doc, &s), first, "not kept, so paid for again"); - - // Read from a file, so kept: the second read has no filters to undo. - let file = crate::write_full(&doc, &crate::SaveOptions::default()).unwrap(); - let doc = Document::open(std::sync::Arc::new(file)).unwrap(); - let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); - let s = stream(&doc, image); - let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); - assert_eq!(spent(&doc, &s), first, "the first read undoes them"); - assert_eq!(spent(&doc, &s), first - filters, "kept"); - - // Globals that fail pay too, with the image not refused: these go past their own limit. - let over = pdfcraft_filters::encode_flate(&vec![0; crate::MAX_JBIG2_GLOBALS + 1]); - let mut doc = Document::new_empty(); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let globals = doc.add(Object::Stream(Stream::from_raw(dict, over.clone()))); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - let image = jbig2(Some(Object::Dict(parms))); - let mut budget = ImageBudget::with_work(u64::MAX); - assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); - assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); - assert_eq!(budget.refused(), 0); - } - - /// A fax image pays for what comes before its decoder and for the rows it asks for, and past - /// the allowance it is refused like any other. - #[test] - fn fax_images_share_the_allowance() { - let doc = Document::new_empty(); - let (w, h) = (4096u32, 64u32); - let coded = pdfcraft_filters::encode_g4(&vec![0xFF; 512 * h as usize], w, h).unwrap(); - let mut parms = Dict::new(); - parms.set(b"K".to_vec(), Object::Int(-1)); - parms.set(b"Columns".to_vec(), Object::Int(i64::from(w))); - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(i64::from(w))); - dict.set(b"Height".to_vec(), Object::Int(i64::from(h))); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("CCITTFaxDecode")])); - dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); - let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); - let pixels = u64::from(w) * u64::from(h); - let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; - let mut budget = ImageBudget::with_work(cost); - assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); - assert_eq!(budget.work_left(), 0, "exactly that"); - assert!(budget.decode(&doc, &image, 1 << 20).is_err()); - assert_eq!(budget.refused(), 1); - } - - /// An image pays for its pixels however few bytes hold them, or a blank page of fax or Flate - /// would cost a run a second for a few bytes' worth of allowance; a JBIG2 image doesn't pay - /// twice, and one this doesn't decode pays nothing. Past what is left it isn't read. - #[test] - fn an_image_pays_for_its_pixels() { - let doc = Document::new_empty(); - let blank = |filter: Option<&str>, data: Vec| { - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(4096)); - dict.set(b"Height".to_vec(), Object::Int(1024)); - if let Some(filter) = filter { - dict.set(b"Filter".to_vec(), Object::name(filter)); - } - Stream::from_raw(dict, data) - }; - let rows = vec![0xFFu8; 512 * 1024]; - let flate = blank(Some("FlateDecode"), pdfcraft_filters::encode_flate(&rows)); - let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; - let mut budget = ImageBudget::with_work(cost); - assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); - assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); - // Its filters alone pay the same. - let mut budget = ImageBudget::with_work(cost); - assert_eq!(budget.decode_filters(&doc, &flate, 1 << 24).unwrap(), rows); - assert_eq!(budget.work_left(), 0); - // Short of its pixels it is refused, having paid for its filters. - let mut budget = ImageBudget::with_work(4096 * 1024 - 1); - assert!(budget.decode(&doc, &flate, 1 << 24).unwrap_err().to_string().contains("more image decoding than")); - assert_eq!((budget.work_left(), budget.refused()), (0, 1)); - let mut budget = ImageBudget::with_work(4096 * 1024 - 1); - assert!(budget.decode_filters(&doc, &flate, 1 << 24).is_err()); - assert_eq!((budget.work_left(), budget.refused()), (0, 1)); - // What isn't decoded here pays for nothing, and fails as it would. - let mut budget = ImageBudget::with_work(10); - assert!(budget.decode(&doc, &blank(Some("DCTDecode"), b"data".to_vec()), 1 << 24).is_err()); - assert_eq!((budget.work_left(), budget.refused()), (10, 0)); - // Unfiltered data pays too, and a size that isn't one is no size. - let mut budget = ImageBudget::with_work(u64::MAX); - budget.decode(&doc, &blank(None, rows.clone()), 1 << 24).unwrap(); - assert_eq!(u64::MAX - budget.work_left(), rows.len() as u64 + 4096 * 1024); - let mut odd = Dict::new(); - odd.set(b"Width".to_vec(), Object::Int(-4)); - odd.set(b"Height".to_vec(), Object::Int(9)); - let mut budget = ImageBudget::with_work(u64::MAX); - budget.decode(&doc, &Stream::from_raw(odd, vec![1, 2]), 1 << 24).unwrap(); - assert_eq!(u64::MAX - budget.work_left(), 2); - // A JPEG pays twice over for each sample, and four times for each byte it is read from. - let mut budget = ImageBudget::with_work(30); - assert!(budget.pay_jpeg(9, b"ab")); - assert_eq!((budget.work_left(), budget.refused()), (4, 0)); - assert!(!budget.pay_jpeg(0, b"ab")); - assert_eq!((budget.work_left(), budget.refused()), (0, 1)); - } - - /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays - /// little for it: a scan-like picture stored in under half its size costs no more than half - /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so - /// a letterhead on hundreds of pages has to fit about as often as it did unpriced (here, at - /// most half again the cost). - #[test] - fn an_image_an_encoder_made_pays_little_for_its_reading() { - let doc = Document::new_empty(); - let mut seed = 0x2545_F491_4F6C_DD1Du64; - let samples: Vec = (0..256 * 256usize) - .map(|i| { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - ((i % 256) as u8 / 16) ^ (seed as u8 & 3) - }) - .collect(); - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(256)); - dict.set(b"Height".to_vec(), Object::Int(256)); - let image = Stream::flate(dict, &samples); - assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); - let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; - let mut budget = ImageBudget::with_work(u64::MAX); - budget.decode(&doc, &image, 1 << 20).unwrap(); - let paid = u64::MAX - budget.work_left(); - assert_eq!(paid, unpriced + read(image.raw.len())); - assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); - } - - /// An image pays for the pixels its data holds, not those its size declares: one claiming - /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the - /// images after it are decoded as they would be without it. - #[test] - fn an_image_pays_for_no_more_pixels_than_its_data_holds() { - let doc = Document::new_empty(); - let sized = |side: i64, data: &[u8]| { - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(side)); - dict.set(b"Height".to_vec(), Object::Int(side)); - Stream::flate(dict, data) - }; - let claims = sized(131_072, &[0; 16]); - let image = sized(64, &[7; 64 * 64]); - let cost = |s: &Stream| { - let mut budget = ImageBudget::with_work(u64::MAX); - budget.decode(&doc, s, 1 << 24).unwrap(); - u64::MAX - budget.work_left() - }; - assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); - assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); - for decode in [ImageBudget::decode, ImageBudget::decode_filters] { - let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); - assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); - assert_eq!(decode(&mut budget, &doc, &image, 1 << 24).unwrap(), [7; 64 * 64], "the image after it is decoded"); - assert_eq!((budget.work_left(), budget.refused()), (0, 0)); - } - } - - /// A stream a run rewrites is read strictly and charged, and what the caller does with it after - /// comes out of the same allowance: what can't be paid for spends all of it. - #[test] - fn a_rewritten_stream_shares_the_allowance() { - use pdfcraft_filters::{Filter, FilterError, Params}; - let samples = vec![7u8; 1000]; - let raw = pdfcraft_filters::encode_flate(&samples); - let chain = [(Filter::Flate, Params::default())]; - let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); - let mut budget = ImageBudget::with_work(cost + 10); - assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); - assert_eq!(budget.work_left(), 10); - assert!(budget.spend(4)); - assert_eq!(budget.work_left(), 6); - assert!(budget.spend(6), "exactly what is left"); - assert_eq!(budget.work_left(), 0); - assert!(budget.spend(0)); - let mut budget = ImageBudget::with_work(5); - assert!(!budget.spend(6)); - assert_eq!(budget.work_left(), 0, "spends all of it"); - assert!(matches!(budget.decode_chain(&chain, &raw, 1 << 20), Err(FilterError::Spent))); - let cut = raw.get(..raw.len() / 2).unwrap(); - assert!(matches!(ImageBudget::with_work(u64::MAX).decode_chain(&chain, cut, 1 << 20), Err(FilterError::Corrupt { .. })), "strictly"); - } - - /// An image read through its filters alone reads parameters given by reference, is charged, and - /// is refused like the others. It isn't given to a decoder. - #[test] - fn filters_alone_are_charged_too() { - let mut doc = Document::new_empty(); - let p = pdfcraft_filters::Params { predictor: 12, columns: 4, ..pdfcraft_filters::Params::default() }; - let rows = vec![7u8; 400]; - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(12)); - parms.set(b"Columns".to_vec(), Object::Int(4)); - let parms = doc.add(Object::Dict(parms)); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); - let raw = pdfcraft_filters::encode(&pdfcraft_filters::Filter::Flate, &p, &rows).unwrap(); - let image = Stream::from_raw(dict, raw); - let mut budget = ImageBudget::with_work(u64::MAX); - assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); - let cost = u64::MAX - budget.work_left(); - // The tag byte of each of its 100 rows counts, which the predictor takes off. - assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); - let mut budget = ImageBudget::with_work(cost - 1); - assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); - assert_eq!((budget.work_left(), budget.refused()), (0, 1)); - let mut budget = ImageBudget::with_work(u64::MAX); - assert!(budget.decode_filters(&doc, &jbig2(None), 1 << 20).is_err()); - assert_eq!(budget.work_left(), u64::MAX, "no decoder here, so nothing was read"); - // Nor for a fax image, nor for a decoder that isn't last, however large they say they are. - for chain in [vec!["CCITTFaxDecode"], vec!["CCITTFaxDecode", "FlateDecode"], vec!["JBIG2Decode", "FlateDecode"]] { - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(1000)); - dict.set(b"Height".to_vec(), Object::Int(1000)); - dict.set(b"Filter".to_vec(), Object::Array(chain.iter().map(|f| Object::name(f)).collect())); - let image = Stream::from_raw(dict, b"data".to_vec()); - let mut budget = ImageBudget::with_work(u64::MAX); - assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); - assert_eq!(budget.work_left(), u64::MAX, "{chain:?}"); - } - // Through `decode` a decoder that isn't last pays nothing either; one that is does. - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(1000)); - dict.set(b"Height".to_vec(), Object::Int(1000)); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("JBIG2Decode"), Object::name("FlateDecode")])); - let mut budget = ImageBudget::with_work(u64::MAX); - assert!(budget.decode(&doc, &Stream::from_raw(dict, b"data".to_vec()), 1 << 20).is_err()); - assert_eq!(budget.work_left(), u64::MAX); - } - - /// PDFBOX-3932: an image and soft mask declared `/Predictor 15` that their writer never - /// applied, so their rows carry no PNG filter-type byte. Exported, they were "shorter than - /// their size says"; they are read as the samples they are, and tagged rows as before. - #[test] - fn an_image_whose_rows_carry_no_png_tags_decodes_to_its_samples() { - let doc = Document::new_empty(); - let samples: Vec = (0..36u8).map(|i| 200 - i).collect(); - let image = |height: Option, raw: Vec| { - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(15)); - parms.set(b"Colors".to_vec(), Object::Int(3)); - parms.set(b"Columns".to_vec(), Object::Int(3)); - let mut dict = Dict::new(); - dict.set(b"Width".to_vec(), Object::Int(3)); - if let Some(h) = height { - dict.set(b"Height".to_vec(), Object::Int(h)); - } - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - Stream::from_raw(dict, raw) - }; - let untagged = pdfcraft_filters::encode_flate(&samples); - let mut budget = ImageBudget::with_work(u64::MAX); - assert_eq!(budget.decode(&doc, &image(Some(4), untagged.clone()), 1 << 20).unwrap(), samples); - // Without a height to tell, the predictor is undone, as before. - assert_ne!(budget.decode(&doc, &image(None, untagged.clone()), 1 << 20).ok(), Some(samples.clone())); - assert_ne!(budget.decode(&doc, &image(Some(5), untagged), 1 << 20).ok(), Some(samples.clone())); - let p = pdfcraft_filters::Params { predictor: 15, colors: 3, columns: 3, ..pdfcraft_filters::Params::default() }; - let tagged = pdfcraft_filters::encode(&pdfcraft_filters::Filter::Flate, &p, &samples).unwrap(); - assert_eq!(budget.decode(&doc, &image(Some(4), tagged), 1 << 20).unwrap(), samples); - } - - /// The images of a file share its globals, so an image refused doesn't read them: this one's - /// can't be read, which is how to tell. - #[test] - fn an_image_past_the_allowance_does_not_read_its_globals() { - let mut doc = Document::new_empty(); - let mut unreadable = Dict::new(); - unreadable.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); - let globals = doc.add(Object::Stream(Stream::from_raw(unreadable, b"data".to_vec()))); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - let image = jbig2(Some(Object::Dict(parms))); - let read = ImageBudget::with_work(u64::MAX).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); - assert!(read.contains("globals"), "{read}"); - let refused = ImageBudget::with_work(0).decode(&doc, &image, 1 << 20).unwrap_err().to_string(); - assert!(refused.contains("more image decoding than"), "{refused}"); - } - - /// A longer file gets more, by its length, and an empty one the fixed part. - #[test] - fn a_longer_file_allows_more_decoding() { - assert_eq!(ImageBudget::new(&Document::new_empty()).work_left(), IMAGE_WORK); - let pdf = b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF".to_vec(); - let len = pdf.len() as u64; - let doc = Document::open(std::sync::Arc::new(pdf)).unwrap(); - assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + len * IMAGE_WORK_PER_BYTE); - } - - /// A document made in memory, as Combine, Create and inserted scans make one, gets for the - /// streams it was given what it would get for them once saved: what isn't a stream adds nothing. - #[test] - fn streams_made_in_memory_allow_more_decoding_too() { - let mut doc = Document::new_empty(); - doc.add(Object::Stream(Stream::from_raw(Dict::new(), vec![0; 3000]))); - doc.add(Object::Dict(Dict::new())); - doc.add(Object::Stream(Stream::flate(Dict::new(), &[1; 100]))); - let flate = pdfcraft_filters::encode_flate(&[1; 100]).len() as u64; - assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + (3000 + flate) * IMAGE_WORK_PER_BYTE); - // A stream of an opened file changed in memory counts with the file. - let pdf = b"%PDF-1.7\n1 0 obj << /Type /Catalog >> endobj\ntrailer << /Root 1 0 R >>\n%%EOF".to_vec(); - let len = pdf.len() as u64; - let mut doc = Document::open(std::sync::Arc::new(pdf)).unwrap(); - doc.add(Object::Stream(Stream::from_raw(Dict::new(), vec![0; 500]))); - assert_eq!(ImageBudget::new(&doc).work_left(), IMAGE_WORK + (len + 500) * IMAGE_WORK_PER_BYTE); - } - - // What the constants' comments promise, for a page scanned at 300 dpi: about 1,000 of them - // whatever the file's size, and a page paying for itself with 2 KB of the file. - const PAGE: u64 = 17_000_000; - const _: () = assert!(IMAGE_WORK >= 1_000 * PAGE); - const _: () = assert!(IMAGE_WORK_PER_BYTE * 2_100 >= PAGE); - // And that a stream can be decoded about 8 times at Flate's best, 1,032 to one, and the byte in. - const _: () = assert!(IMAGE_WORK_PER_BYTE >= 7 * 1_033); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::Dict; - - #[test] - fn repeated_references_share_one_allowance() { - let mut doc = Document::new_empty(); - let bomb = doc.add(Object::Stream(Stream::flate(Dict::new(), &vec![b' '; 1 << 20]))); - let contents = Object::Array(vec![Object::Ref(bomb); 64]); - let mut budget = DecodeBudget::new(10 << 20, 1); - let decoded = budget.decode_contents(&doc, &contents); - assert_eq!(decoded.len(), 64); - assert_eq!(decoded.iter().filter(|d| d.is_ok()).count(), 10); - let total: usize = decoded.iter().flatten().map(Vec::len).sum(); - assert_eq!(total, 10 << 20); - assert!(budget.exhausted()); - // A decode past the allowance spends it, so a bomb is inflated at most once more. - let mut budget = DecodeBudget::new((1 << 20) - 1, 1); - assert!(budget.decode_contents(&doc, &contents).iter().all(Result::is_err)); - assert_eq!(budget.remaining(), 0); - let mut budget = DecodeBudget::new((1 << 20) - 1, 1); - let Object::Stream(stream) = &*doc.get(bomb) else { panic!("not a stream") }; - assert!(budget.decode_strict(&doc, stream).is_err()); - assert_eq!(budget.remaining(), 0); - - let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_within(stream, 1000).is_err()); - assert_eq!(budget.remaining(), (10 << 20) - 1000); - assert_eq!(budget.decode_within(stream, 2 << 20).map(|d| d.len()).ok(), Some(1 << 20)); - assert_eq!(budget.remaining(), (9 << 20) - 1000); - - assert!(budget.enter_form()); - assert!(!budget.enter_form()); - assert!(budget.first_visit(bomb)); - assert!(!budget.first_visit(bomb)); - } - - /// A corrupt stream fails a strict decode after decoding part of it: each reference pays - /// for that part, so references to it can't decode without limit. - #[test] - fn failed_strict_decodes_charge_what_they_decoded() { - let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); - let cut = whole.raw.len() / 2; - let corrupt = Stream::from_raw(whole.dict.clone(), whole.raw.get(..cut).unwrap_or_default().to_vec()); - let partial = corrupt.decoded().map(|d| d.len()).unwrap_or(0); - assert!(partial > 0, "a cut flate stream decodes in part"); - let doc = Document::new_empty(); - let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_strict(&doc, &corrupt).is_err()); - assert_eq!(budget.remaining(), (10 << 20) - partial); - let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_within(&corrupt, 1 << 30).is_ok()); - assert_eq!(budget.remaining(), (10 << 20) - partial); - } - - /// A stream damaged partway decodes to what came before the damage, and says so, so that code - /// writing it back can leave it as it is. What every reader takes is no damage: a missing - /// checksum, no filter at all. - #[test] - fn a_stream_damaged_partway_says_so() { - let doc = Document::new_empty(); - let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(&doc, s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); - let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); - let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); - let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); - assert!(partial > 0, "a cut flate stream decodes in part"); - let mut budget = DecodeBudget::new(10 << 20, 1); - assert_eq!(read(&mut budget, &cut, usize::MAX), Some((partial, true))); - assert_eq!(budget.remaining(), (10 << 20) - partial); - // Within a bound of its own, under what is left. - assert_eq!(read(&mut budget, &cut, 4 << 20), Some((partial, true))); - assert_eq!(budget.decode(&cut).map(|d| d.len()).ok(), Some(partial)); - - let mut budget = DecodeBudget::new(10 << 20, 1); - assert_eq!(read(&mut budget, &whole, usize::MAX), Some((1 << 20, false))); - assert_eq!(read(&mut budget, &whole, 4 << 20), Some((1 << 20, false))); - let unchecked = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() - 4).unwrap_or_default().to_vec()); - assert_eq!(read(&mut budget, &unchecked, usize::MAX), Some((1 << 20, false))); - assert_eq!(read(&mut budget, &Stream::from_raw(Dict::new(), b"0 0 m".to_vec()), usize::MAX), Some((5, false))); - // A later stage's damage is the stream's. - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); - let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); - assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); - // So is an earlier stage's, though the stage after it reads all it was given: a stored - // block of whole hex, then a block of the reserved type (RFC 1951 §3.2.3). - let readable = b"4142>"; - let len = u16::try_from(readable.len()).unwrap_or_default(); - let mut raw = vec![0x78, 0x01, 0x00]; - raw.extend_from_slice(&len.to_le_bytes()); - raw.extend_from_slice(&(!len).to_le_bytes()); - raw.extend_from_slice(readable); - raw.push(0x07); - let flate_damaged = Stream::from_raw(hex.dict.clone(), raw); - assert_eq!(budget.decode_with_damage(&doc, &flate_damaged, usize::MAX).ok(), Some((b"AB".to_vec(), true))); - } - - /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a - /// predicted stream decodes as with them direct. Of the stream, decoding copies no more than - /// its filters and their parameters. - #[test] - fn parameters_given_by_reference_are_read() { - let content = b"q 10 0 0 10 0 0 cm /Im Do Q"; - // PNG Up, one column: each byte a row of its own, tagged 2. - let mut rows = Vec::new(); - let mut above = 0u8; - for &b in content { - rows.extend([2, b.wrapping_sub(above)]); - above = b; - } - let raw = Stream::flate(Dict::new(), &rows).raw; - let mut doc = Document::new_empty(); - let direct: Dict = [(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect(); - let by_reference = Object::Ref(doc.add(Object::Dict(direct.clone()))); - let predictor = doc.add(Object::Int(12)); - let inner = Object::Dict([(b"Predictor".to_vec(), Object::Ref(predictor))].into_iter().collect()); - for parms in [Object::Dict(direct), by_reference, inner] { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - dict.set(b"DecodeParms".to_vec(), parms.clone()); - dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); - let s = Stream::from_raw(dict, raw.to_vec()); - assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); - assert_eq!(DecodeBudget::page().decode_strict(&doc, &s).ok(), Some(content.to_vec()), "{parms:?}"); - let r = doc.add(Object::Stream(s.clone())); - let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); - assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); - assert_eq!(DecodeBudget::page().decode_contents(&doc, &Object::Ref(r)), [Ok(content.to_vec())], "{parms:?}"); - let view = doc.decoding_view(&s); - assert!(matches!(view, std::borrow::Cow::Borrowed(_)) || view.dict.len() == 2, "{:?}", view.dict); - } - } - - /// A page's content streams each say whether they are damaged, in their places. - #[test] - fn a_pages_streams_each_say_whether_they_are_damaged() { - let mut doc = Document::new_empty(); - let whole = doc.add(Object::Stream(Stream::flate(Dict::new(), b"0 0 m 1 1 l S"))); - let Object::Stream(s) = &*doc.get(whole) else { panic!("not a stream") }; - let cut = Stream::from_raw(s.dict.clone(), s.raw.get(..s.raw.len() - 6).unwrap_or_default().to_vec()); - let cut = doc.add(Object::Stream(cut)); - let contents = Object::Array(vec![Object::Ref(whole), Object::Null, Object::Ref(cut)]); - let mut budget = DecodeBudget::page(); - let read: Vec<_> = budget.decode_contents_with_damage(&doc, &contents).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); - assert_eq!(read, [Some(false), Some(true)]); - let one: Vec<_> = - budget.decode_contents_with_damage(&doc, &Object::Ref(cut)).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); - assert_eq!(one, [Some(true)]); - assert_eq!(budget.decode_contents(&doc, &contents).iter().filter(|r| r.is_ok()).count(), 2, "read up to the damage"); - } - - /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, - /// gives nothing back, and each reference still pays for what it decoded. - #[test] - fn a_decode_that_gives_nothing_back_still_pays() { - let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; - data.extend(vec![0u8; 0x0601]); - data.push(0x07); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let stream = Stream::from_raw(dict, data); - let mut budget = DecodeBudget::new(10 * 0x0601, 1); - assert!(budget.decode(&stream).unwrap().is_empty()); - assert_eq!(budget.remaining(), 9 * 0x0601); - } - - /// A chain whose later stage shrinks the data pays for every stage's output: a Flate stage - /// that inflates to hex text, which ASCIIHex then halves, charges both. - #[test] - fn decodes_charge_every_stage_of_a_chain() { - let hex: Vec = std::iter::repeat_n(b"ab".as_slice(), 1 << 19).flatten().copied().collect(); - let flate = Stream::flate(Dict::new(), &hex); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); - let chained = Stream::from_raw(dict, flate.raw.to_vec()); - assert_eq!(chained.decoded().map(|d| d.len()).ok(), Some(1 << 19)); - let charged = (1 << 20) + (1 << 19); - let mut budget = DecodeBudget::new(10 << 20, 1); - assert_eq!(budget.decode(&chained).map(|d| d.len()).ok(), Some(1 << 19)); - assert_eq!(budget.remaining(), (10 << 20) - charged); - assert!(budget.decode_within(&chained, 2 << 20).is_ok()); - assert_eq!(budget.remaining(), (10 << 20) - 2 * charged); - assert!(budget.decode_strict(&Document::new_empty(), &chained).is_ok()); - assert_eq!(budget.remaining(), (10 << 20) - 3 * charged); - } - - /// `decode_stages_within` holds every stage to the limit, as a decode without an allowance - /// does, and the stages together to the limit for each of them. A stream behind ASCIIHex and - /// Flate whose stages each stay within it reads, the hex stage's output charged on top; one - /// whose Flate stage would make more stops there, charged what the hex stage made and the - /// limit, the rest of the allowance left; a Flate stage making hex text past the limit leaves - /// the stream unread though the total has room, as `Stream::decoded_within` leaves it; and a - /// stream past what is left spends all of it. - #[test] - fn each_stage_is_held_to_the_limit() { - let chained = |data: &[u8], filters: [&str; 2]| { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(filters.into_iter().map(Object::name).collect())); - Stream::from_raw(dict, data.to_vec()) - }; - let hexed = |data: &[u8]| { - let flate = Stream::flate(Dict::new(), data); - let hex: Vec = flate.raw.iter().flat_map(|b| format!("{b:02X}").into_bytes()).chain(*b">").collect(); - (chained(&hex, ["ASCIIHexDecode", "FlateDecode"]), flate.raw.len()) - }; - let max = 1 << 16; - let (fits, inner) = hexed(&vec![0u8; max]); - let mut budget = DecodeBudget::new(1 << 20, 0); - assert_eq!(budget.decode_stages_within(&Document::new_empty(), &fits, max).map(|d| d.len()).ok(), Some(max)); - assert_eq!(budget.remaining(), (1 << 20) - inner - max); - let (bomb, inner) = hexed(&vec![0u8; 8 * max]); - let mut budget = DecodeBudget::new(1 << 20, 0); - assert!(budget.decode_stages_within(&Document::new_empty(), &bomb, max).is_err()); - assert_eq!(budget.remaining(), (1 << 20) - inner - max, "the Flate stage stopped at the limit"); - // Flate making 1.2 times the limit of hex text, which ASCIIHex halves: the stages together - // make 1.8 times the limit, within twice it, but the Flate stage alone is past it. - let text = "00".repeat(max * 6 / 10).into_bytes(); - let wide = chained(&Stream::flate(Dict::new(), &text).raw, ["FlateDecode", "ASCIIHexDecode"]); - assert!(wide.decoded_within(max).is_err()); - assert!(DecodeBudget::new(1 << 20, 0).decode_stages_within(&Document::new_empty(), &wide, max).is_err()); - let mut budget = DecodeBudget::new(max, 0); - assert!(budget.decode_stages_within(&Document::new_empty(), &fits, max).is_err()); - assert_eq!(budget.remaining(), 0); - } - - /// What `decode_stages_within` charges, and which limit it reports, on each way a read stops: - /// a stage at the stream's limit with more of the room left charges what came before and the - /// limit; at the stages' total, with more of the allowance left, it charges that total; at - /// what is left of the allowance it spends all of it and reports the allowance's limit, as - /// `decode_within` does. Each boundary between them is read on both sides. - #[test] - fn a_stage_past_its_limit_charges_what_came_before_and_the_limit() { - let max = 1 << 12; - let hex = |data: &[u8]| data.iter().flat_map(|b| format!("{b:02X}").into_bytes()).chain(*b">").collect::>(); - let two = |raw: &[u8]| { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("ASCIIHexDecode"), Object::name("FlateDecode")])); - Stream::from_raw(dict, hex(raw)) - }; - let says = |read: Result, CosError>, limit: usize| { - read.err().map(|e| e.to_string()) == Some(CosError::Filter(FilterError::LimitExceeded(limit).to_string()).to_string()) - }; - // Flate data making eight times the limit: the hex stage makes `inner` bytes of it, and - // the Flate stage stops at the limit. - let bomb = Stream::flate(Dict::new(), &vec![0u8; 8 * max]).raw.to_vec(); - let inner = bomb.len(); - // A byte more of the room than the hex stage and the limit: the Flate stage stops at the - // limit, so that is what it charges. - let mut budget = DecodeBudget::new(inner + max + 1, 0); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&bomb), max), max)); - assert_eq!(budget.remaining(), 1); - // Exactly the hex stage and the limit: what was left stopped it as much as the limit did, - // so it is past the allowance, spends it and reports the allowance's limit. - let mut budget = DecodeBudget::new(inner + max, 0); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&bomb), max), inner + max)); - assert_eq!(budget.remaining(), 0); - // The same data padded to exactly the limit: the hex stage makes all of it, and the Flate - // stage, at the limit, reaches the stages' total. With more of the allowance left, that - // total is what it charges and reports. - let mut padded = bomb.clone(); - padded.resize(max, 0); - let mut budget = DecodeBudget::new(10 * max, 0); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), 2 * max)); - assert_eq!(budget.remaining(), 8 * max); - // With exactly the total left (of an allowance of three times the limit), it is past the - // allowance: all of it is spent, and the allowance's limit reported. - let mut budget = DecodeBudget::new(3 * max, 0); - budget.decode(&Stream::from_raw(Dict::new(), vec![0; max])).unwrap(); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), 3 * max)); - assert_eq!(budget.remaining(), 0); - // With less than the total left, the same. - let mut budget = DecodeBudget::new(max, 0); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &two(&padded), max), max)); - assert_eq!(budget.remaining(), 0); - // With nothing left, a stream with data is refused before it is decoded, even one that - // decodes to nothing; an empty one reads. - let mut budget = DecodeBudget::new(0, 0); - assert!(says(budget.decode_stages_within(&Document::new_empty(), &Stream::flate(Dict::new(), b""), max), 0)); - assert_eq!(budget.decode_stages_within(&Document::new_empty(), &Stream::from_raw(Dict::new(), Vec::new()), max).ok(), Some(Vec::new())); - } -} diff --git a/crates/cos/src/bytes.rs b/crates/cos/src/bytes.rs deleted file mode 100644 index f6ad1b8e8a..0000000000 --- a/crates/cos/src/bytes.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Stream data that can share a larger buffer instead of copying out of it. - -use std::ops::{Deref, Range}; -use std::sync::Arc; - -/// A stream's bytes: a buffer of their own, or a range of a larger shared buffer. Streams read -/// from a file point into the file's bytes, which the document keeps anyway, so caching a -/// parsed stream costs its dictionary and not a second copy of its data. -#[derive(Clone, Default)] -pub struct Bytes { - buf: Arc>, - range: Range, -} - -impl Bytes { - /// `range` of `buf`, without copying; `None` when the range is not within `buf`. - pub(crate) fn view(buf: &Arc>, range: Range) -> Option { - (range.start <= range.end && range.end <= buf.len()).then(|| Self { buf: buf.clone(), range }) - } - - /// Whether these bytes keep alive a larger buffer than themselves that isn't `owner`. - pub(crate) fn borrows_other_than(&self, owner: &Arc>) -> bool { - self.range.len() < self.buf.len() && !Arc::ptr_eq(&self.buf, owner) - } - - /// Whether these bytes are (a view into) `buf`, holding no memory of their own. - pub(crate) fn shares(&self, buf: &Arc>) -> bool { - Arc::ptr_eq(&self.buf, buf) - } - - /// The same bytes, in a buffer of their own. - pub(crate) fn detached(&self) -> Self { - Self::from(self.to_vec()) - } -} - -impl From> for Bytes { - fn from(v: Vec) -> Self { - let range = 0..v.len(); - Self { buf: Arc::new(v), range } - } -} - -impl From>> for Bytes { - fn from(buf: Arc>) -> Self { - let range = 0..buf.len(); - Self { buf, range } - } -} - -impl Deref for Bytes { - type Target = [u8]; - - fn deref(&self) -> &[u8] { - // `view` and `From` only build in-bounds ranges; an empty slice keeps this panic-free. - self.buf.get(self.range.clone()).unwrap_or_default() - } -} - -impl AsRef<[u8]> for Bytes { - fn as_ref(&self) -> &[u8] { - self - } -} - -impl PartialEq for Bytes { - fn eq(&self, other: &Self) -> bool { - (Arc::ptr_eq(&self.buf, &other.buf) && self.range == other.range) || **self == **other - } -} - -impl Eq for Bytes {} - -impl std::fmt::Debug for Bytes { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - (**self).fmt(f) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn views_share_their_buffer_and_compare_by_content() { - let file = Arc::new(b"0123456789".to_vec()); - let view = Bytes::view(&file, 2..5).expect("in range"); - assert_eq!(&*view, b"234"); - assert_eq!(view.as_ptr(), file[2..].as_ptr(), "no copy"); - assert_eq!(view, Bytes::from(b"234".to_vec())); - assert!(Bytes::view(&file, 5..11).is_none() && Bytes::view(&file, Range { start: 6, end: 5 }).is_none()); - let other = Arc::new(Vec::new()); - assert!(view.borrows_other_than(&other) && !view.borrows_other_than(&file)); - let own = view.detached(); - assert!(!own.borrows_other_than(&other) && own == view && own.as_ptr() != view.as_ptr()); - assert!(!Bytes::from(file.clone()).borrows_other_than(&other), "a whole buffer is its own"); - } -} diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs deleted file mode 100644 index f3ea35d8c8..0000000000 --- a/crates/cos/src/document.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Document: cross-reference data, lazy object loading, and a copy-on-write overlay of edits. -//! -//! `Document` is cheap to clone (the original bytes, xref index and parse caches are shared), so -//! a clone is a snapshot: undo/redo and background jobs hold clones while the UI edits another. -//! Edits never touch the original bytes; the writer appends them (incremental save) or rewrites -//! the file (full save). - -use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::sync::{Arc, Mutex, Weak}; - -use crate::CosError; -use crate::object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream}; -use crate::parser::{EndstreamIndex, Lexer, is_whitespace, parse_indirect, parse_indirect_shared}; - -mod objects; -mod open; -mod reader; -mod state; -mod xref; - -pub use objects::DECODE_PARMS_KEYS; -pub(crate) use objects::Held; -#[cfg(test)] -pub(crate) use reader::find; -#[cfg(test)] -use reader::rfind; -use state::{ObjStm, Slot, Unreadable}; -pub(crate) use state::{ObjectReader, held_bytes}; -pub use state::{ReadState, SourceIdentity}; -#[cfg(test)] -use state::{SECTION_READS, objstm_member}; -use xref::XrefIndex; -pub use xref::{Revision, XrefEntry}; - -#[derive(Clone)] -pub struct Document { - data: Arc>, - /// Where `endstream` occurs in `data`; see [`EndstreamIndex`]. - endstreams: Arc, - source_identity: Arc<()>, - entries: Arc, - trailer: Dict, - revisions: Arc>, - /// See [`Self::xref_oddities`]. - xref_oddities: Arc>, - /// Where the file's object headers are, as (offset, generation) of the last header of each - /// number. Filled only when the cross-reference data proved inconsistent; see - /// `fill_xref_gaps`. - located: Arc>, - repair_log: Arc>, - /// See [`Self::rebuilt_unlinked_update`]. - unlinked_update: bool, - /// What loading each file object gave, a failure as well: a damaged object is otherwise - /// loaded again on every read, and a member of an object stream that can't be decoded - /// decodes the stream again (`Document::load`). - cache: Arc>, - objstms: Arc>>>, - /// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the - /// source, and is reset wherever they are. - unreadable: Arc>, - /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan - /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). - jbig2_globals: Arc>>>>, - stream_limit: usize, - overlay: BTreeMap, - next_num: u32, - /// Position of the `%PDF-` header (some files have junk before it). - header_offset: usize, - version: String, - /// The authenticated security handler of an encrypted document. - security: Option>, - /// Object number of the `/Encrypt` dictionary (never encrypted itself). - encrypt_num: Option, - /// What that dictionary and `/ID` refer to, which the key comes from, and so is as stored. - encrypt_members: Arc>, - /// Object numbers a scan of the file didn't find: the file doesn't change, so it never will. - unfound: Arc>>, - /// Encryption was added, changed or removed since opening: only a full save can apply it. - encryption_changed: bool, - /// Set by edits that must not leave earlier revisions in the file (redaction). - full_save: bool, - /// The next full save also gives the file a new identifier (`/ID`), see - /// [`Document::require_full_save_with_new_id`]. - new_id: bool, - /// The handler and `/Encrypt` object number that saves use, when encryption changed. - out_security: Option>, - out_encrypt_num: Option, -} - -impl std::fmt::Debug for Document { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Document").field("bytes", &self.data.len()).field("xref", &self.entries).field("edits", &self.overlay.len()).finish() - } -} - -#[cfg(test)] -mod tests; diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs deleted file mode 100644 index 42aef4af0f..0000000000 --- a/crates/cos/src/document/objects.rs +++ /dev/null @@ -1,1121 +0,0 @@ -use std::collections::HashSet; - -use super::reader::find; -use super::state::{ - LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadFrame, ReadState, ScanLengths, Slot, SourceIdentity, loaded_bytes, loading, note_overlay_read, - note_read, objstm_member, overlay_reads, -}; -use super::xref::{Revision, XrefEntry, XrefIndex}; -use super::{ - Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, -}; - -/// The most decoded `/JBIG2Globals` a document keeps for its images to share (see -/// `Document::jbig2_globals`): two streams at [`MAX_JBIG2_GLOBALS`]. -const JBIG2_GLOBALS_HELD: usize = 2 * MAX_JBIG2_GLOBALS; - -impl Document { - pub fn bytes(&self) -> &Arc> { - &self.data - } - - /// Identity of the original bytes, xref and authenticated decode context. Edits and - /// clones preserve it; reopening (including after any save) produces a new identity. - pub fn source_identity(&self) -> SourceIdentity { - SourceIdentity(Arc::downgrade(&self.source_identity)) - } - - /// What the document reads as now (see [`ReadState`]): unlike [`Self::source_identity`], it - /// changes with every edit, so a value derived from the document can be cached against it. - pub fn read_state(&self) -> ReadState { - ReadState { source: self.source_identity(), trailer: self.trailer.clone(), overlay: self.overlay.clone() } - } - - pub fn version(&self) -> &str { - &self.version - } - - /// Set the PDF header version (`"1.4"`, `"1.7"`, `"2.0"`, …) written by a full save. - pub fn set_version(&mut self, version: impl Into) { - self.version = version.into(); - } - - pub fn trailer(&self) -> &Dict { - &self.trailer - } - - pub fn trailer_mut(&mut self) -> &mut Dict { - &mut self.trailer - } - - pub fn revisions(&self) -> &[Revision] { - &self.revisions - } - - /// Position of the `%PDF-` header in the file: 0 unless bytes come before it. - pub fn header_offset(&self) -> usize { - self.header_offset - } - - /// There is no cross-reference chain to append to: the index was rebuilt from object - /// headers because the file's own was unusable (see [`Self::repair_log`]), or the document - /// was built from scratch ([`Self::new_empty`]). A save writes such a document in full, even - /// when nothing was edited: for a repaired file that is what puts the repair on disk. - pub fn reconstructed(&self) -> bool { - self.revisions.is_empty() - } - - /// Where each revision of the file ends (oldest first): the byte after the `%%EOF` (and its - /// end-of-line) that closes its cross-reference section. `data[..end]` is that revision as it - /// was saved. A linearized file's first-page section belongs to the revision it precedes, so - /// ends only ever increase. Empty for a reconstructed file. - pub fn revision_ends(&self) -> Vec { - let d = &self.data[..]; - let mut ends: Vec = Vec::with_capacity(self.revisions.len()); - for r in self.revisions.iter() { - let from = (r.xref_offset as usize).saturating_add(self.header_offset).min(d.len()); - let Some(at) = d[from..].windows(5).position(|w| w == b"%%EOF") else { continue }; - let mut end = from + at + 5; - while end < d.len() && matches!(d[end], b'\r' | b'\n') && end - (from + at + 5) < 2 { - end += 1; - } - match ends.last_mut() { - Some(last) if end <= *last => {} - _ => ends.push(end), - } - } - ends - } - - pub fn repair_log(&self) -> &[String] { - &self.repair_log - } - - /// Whether the file was rebuilt from its object headers because its last update has no `/Prev` - /// and so cut its page tree off the cross-reference chain (ISO 32000-2 §7.5.6, §7.5.8.2). The - /// renderer rebuilds such a file under the same conditions, so the two read the same objects. - pub fn rebuilt_unlinked_update(&self) -> bool { - self.unlinked_update - } - - /// Objects of the file that could not be read, lowest number first, as lines such as - /// "object 11 0 could not be read: syntax error at byte 512: ...". [`Self::get`] reads each - /// as null (ISO 32000-2 §7.3.10), so a document with one opens, but what it held is lost: - /// unlike [`Self::repair_log`], nothing was repaired. Lists the objects read so far, by this - /// document, its clones or a scan of it, that have not since been edited; past 100, a last - /// line says there were more. Opening again, authenticating or rebuilding the index starts - /// the list afresh, as it does the object cache. - pub fn unreadable_objects(&self) -> Vec { - let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() }; - unreadable.lines(|num| self.overlay.contains_key(&num)) - } - - /// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file - /// whose broken object is asked for again and again pays for it once. - fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { - // A poisoned lock only means the object goes unlisted; the read itself still fails. - let Ok(mut unreadable) = self.unreadable.lock() else { return }; - let hidden = |n: u32| self.overlay.contains_key(&n); - if unreadable.wants(num, hidden) { - unreadable.note(num, self.generation(num), &reason(), hidden); - } - } - - /// Why an object listed at `offset` could not be read: what parsing it there, without resolving - /// an indirect `/Length`, found. A best account, not the only one: the reader also tried - /// elsewhere. - fn why_unreadable(&self, offset: usize) -> String { - match parse_indirect_shared(&self.data, offset, &|_| None, &self.endstreams) { - Err(e) => e.to_string(), - Ok(_) => "no readable copy was found at its cross-reference offset or elsewhere in the file".into(), - } - } - - /// Add a line to [`Self::repair_log`]: a leniency applied while reading that the user should - /// know about (fidelity: never silent). - pub fn note_repair(&mut self, line: String) { - let mut log = self.repair_log.as_ref().clone(); - log.push(line); - self.repair_log = Arc::new(log); - } - - /// `true` when there are unsaved edits. Removing an `/Encrypt` dictionary written inline in - /// the trailer (as MuPDF writes it) changes no object, and is an edit all the same. - pub fn is_modified(&self) -> bool { - !self.overlay.is_empty() || self.encryption_changed - } - - /// Object numbers changed since the document was opened (or last saved). - pub fn modified_objects(&self) -> Vec { - self.overlay.keys().copied().collect() - } - - /// Where object `num` is stored in the file (ignoring unsaved edits). - pub fn xref_entry(&self, num: u32) -> Option { - self.entries.lookup(num, &self.data) - } - - /// Cross-reference entries this reader did not take at face value, as (byte offset of the - /// section, object number): an in-use entry at offset 0, a second and different entry for - /// one object in the same section, a reserved cross-reference stream entry type, or a - /// hybrid /XRefStm entry that disagrees with the table. The reader sets such an entry aside - /// and keeps what it already had for the object. Another reader, the page renderer among - /// them, may follow it instead, so the two can then show different objects for one number. - /// Code that compares revisions (signature validation) must not call such an object - /// unchanged. - pub fn xref_oddities(&self) -> &[(u64, u32)] { - &self.xref_oddities - } - - /// Whether the cross-reference entry of `num` locates that object itself: its `num G obj` - /// header at exactly the recorded offset, or the member with that number at the recorded - /// position of its object stream. When it does not, [`Self::get`] still finds the object by - /// searching, but another reader may resolve the number differently. Ignores unsaved edits. - pub fn xref_entry_is_exact(&self, num: u32) -> bool { - match self.xref_entry(num) { - None | Some(XrefEntry::Free { .. }) => true, - Some(XrefEntry::InFile { offset, generation }) => { - let at = |off: usize| matches!(parse_indirect_shared(&self.data, off, &|_| None, &self.endstreams), Ok((id, _)) if id == ObjRef::new(num, generation)); - // Offsets may count from a header that does not start the file. - usize::try_from(offset).is_ok_and(|off| at(off) || (self.header_offset != 0 && at(off.saturating_add(self.header_offset)))) - } - Some(XrefEntry::InStream { stream, index }) => self - .objstm(stream) - .is_ok_and(|stm| usize::try_from(index).ok().and_then(|i| stm.index.get(i)).is_some_and(|m| m.is_some_and(|(n, _)| n == num))), - } - } - - /// Object `num` has an unsaved edit. - pub fn is_edited(&self, num: u32) -> bool { - self.overlay.contains_key(&num) - } - - pub fn root(&self) -> Option { - self.trailer.reference(b"Root") - } - - // ── object access ─────────────────────────────────────────────────────────────────────── - - /// Fetch an object by reference. Undefined references are `Null` (ISO 32000-2 §7.3.10). - /// A live file slot also accepts its actual header generation when a damaged xref row - /// disagrees; the disagreement is recorded in [`Self::repair_log`]. Free slots never do. - pub fn get(&self, r: ObjRef) -> Arc { - self.try_get_reference(r).unwrap_or_else(|_| Arc::new(Object::Null)) - } - - pub(super) fn reference_matches(&self, r: ObjRef) -> bool { - match self.overlay.get(&r.num) { - Some(Slot::Set(generation, _)) => r.generation == *generation, - Some(Slot::RepairedSet(generation, alias, _)) => r.generation == *generation || r.generation == *alias, - Some(Slot::Freed(_)) => false, - None => match self.entries.lookup(r.num, &self.data) { - Some(XrefEntry::InFile { offset, generation }) => { - r.generation == generation || self.indexed_header_reference(r.num, offset) == Some(r) - } - Some(XrefEntry::InStream { .. }) => r.generation == 0, - None | Some(XrefEntry::Free { .. }) => false, - }, - } - } - - /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), - /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged - /// file slot (or a repair-aware edit of one) also the generation its object header carries. - /// Code comparing revisions (signature validation) uses it to tell whether the references - /// that reached an object in one revision still reach the same slot in another. - pub fn accepted_generations(&self, num: u32) -> Vec { - // Mirrors `reference_matches`, which stays allocation-free for every lookup. - let mut generations = match self.overlay.get(&num) { - Some(Slot::Set(generation, _)) => vec![*generation], - Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], - Some(Slot::Freed(_)) => Vec::new(), - None => match self.entries.lookup(num, &self.data) { - Some(XrefEntry::InFile { offset, generation }) => { - std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() - } - Some(XrefEntry::InStream { .. }) => vec![0], - None | Some(XrefEntry::Free { .. }) => Vec::new(), - }, - }; - generations.sort_unstable(); - generations.dedup(); - generations - } - - pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { - if !self.reference_matches(r) { - // An edit of `r.num` can make it match, so a load that read null here read it too; - // with `r.num` edited (freed, or set at another generation) the null is the edit's. - if self.overlay.contains_key(&r.num) { - note_overlay_read(); - } - note_read(r.num, None); - return Ok(Arc::new(Object::Null)); - } - self.try_get(r.num) - } - - /// A repair probe, not a whole-file search: only the indexed header (or the same offset - /// relative to a shifted PDF header) can establish this additional generation. A small - /// window bounds hostile whitespace/tokens; ordinary xref-matching reads are unchanged. - pub(super) fn indexed_header_reference(&self, num: u32, offset: u64) -> Option { - let at = |off: usize| { - let bytes = self.data.get(off..off.saturating_add(128).min(self.data.len()))?; - let mut lexer = Lexer::new(bytes, 0); - lexer.skip_ws(); - let number = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; - lexer.skip_ws(); - let generation = std::str::from_utf8(lexer.token()).ok()?.parse::().ok()?; - (number == num && lexer.eat_keyword(b"obj")).then_some(ObjRef::new(number, generation)) - }; - let off = usize::try_from(offset).ok()?; - at(off).or_else(|| if self.header_offset == 0 { None } else { off.checked_add(self.header_offset).and_then(at) }) - } - - /// Add at most one bounded diagnostic per object. False stops diagnostic probing only; - /// parsers must still read the remaining xref rows needed to build their index. - pub(super) fn inspect_xref_generation_repair(&self, num: u32, entry: XrefEntry, recorded: &mut HashSet, log: &mut Vec) -> bool { - let XrefEntry::InFile { offset, generation } = entry else { return true }; - if recorded.contains(&num) { - return true; - } - let Some(header) = self.indexed_header_reference(num, offset) else { return true }; - if header.generation == generation { - return true; - } - if recorded.len() == 256 { - log.push("additional xref/header generation disagreements omitted after 256 objects".into()); - return false; - } - recorded.insert(num); - log.push(format!("object {num}: xref generation {generation} disagrees with object header generation {}", header.generation)); - true - } - - /// Diagnose once at open without expanding compact ranges or decoding an xref twice. - /// Materialized readers retain bounded notes while they read accepted sections, before - /// newer rows hide older ones. Discarded parse attempts/rebuilt indexes carry no notes. - pub(super) fn note_xref_generation_repairs(&self, log: &mut Vec) { - match &*self.entries { - XrefIndex::Materialized(_, notes) => log.extend(notes.iter().cloned()), - XrefIndex::Compact { revisions, .. } => { - let mut recorded = HashSet::new(); - for revision in revisions { - for section in std::iter::once(&revision.primary).chain(revision.hybrid.iter()) { - for num in section.numbers() { - if let Some(entry) = section.lookup(num, &self.data) - && !self.inspect_xref_generation_repair(num, entry, &mut recorded, log) - { - return; - } - } - } - } - } - } - } - - /// Follow a reference if `o` is one; otherwise return `o` itself. An object can be nothing but - /// a reference to another, which is followed too, up to [`MAX_REFERENCE_HOPS`] of them. - pub fn resolve(&self, o: &Object) -> Arc { - let Object::Ref(r) = o else { return Arc::new(o.clone()) }; - let mut held = self.get(*r); - for _ in 0..MAX_REFERENCE_HOPS { - let Object::Ref(r) = &*held else { break }; - held = self.get(*r); - } - held - } - - /// `s` with the `/DecodeParms` its filters use made direct, so [`Stream::decoded`] reads them: - /// an indirect dictionary resolved, and of an array (itself perhaps indirect), the entries at - /// the filters' places (`/Crypt` counts, ISO 32000-2 §7.4.1) resolved, the rest dropped. Of - /// each dictionary only the entries the decoders read are kept, numbers and flags given by - /// reference (`/Predictor 7 0 R`) resolved; `/JBIG2Globals` stays a reference. A `/Crypt` - /// entry, which no decoder reads ([`Stream::filters`] leaves it out), is null. `s` itself - /// when none of those is indirect, or when the other filters are more than any decoder takes - /// ([`pdfcraft_filters::MAX_CHAIN`]). Only small values are copied, so a file can't have a - /// large object copied once for each key and filter. - pub fn with_direct_parms<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { - let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; - let mut s = s.clone(); - s.dict.set(b"DecodeParms".to_vec(), made); - Cow::Owned(s) - } - - /// [`Document::with_direct_parms`] for decoding alone: where that copies `s`, this makes a - /// stream of `s`'s data, shared, and a dictionary of its `/Filter` and the parameters made - /// direct, all its filters read. A stream named again and again, as a page's `/Contents` may - /// list one, isn't copied whole each time. - pub(crate) fn decoding_view<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { - let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; - let mut dict = Dict::new(); - if let Some(f) = s.dict.get(b"Filter") { - dict.set(b"Filter".to_vec(), f.clone()); - } - dict.set(b"DecodeParms".to_vec(), made); - Cow::Owned(Stream { dict, raw: s.raw.clone() }) - } - - /// The `/DecodeParms` [`Document::with_direct_parms`] gives `s`, or `None` when it keeps `s`. - fn direct_parms(&self, s: &Stream) -> Option { - let p = s.dict.get(b"DecodeParms")?; - let crypt = |f: &Object| matches!(f.as_name(), Some(b"Crypt")); - let filters: Vec = match s.dict.get(b"Filter") { - Some(Object::Array(a)) => a.iter().map(crypt).collect(), - Some(f) => vec![crypt(f)], - None => Vec::new(), - }; - if filters.iter().filter(|c| !**c).count() > pdfcraft_filters::MAX_CHAIN { - return None; - } - let mut changed = matches!(p, Object::Ref(_)); - // Its numbers and flags kept, resolved; anything else is no filter's parameters (null). - let one = |e: &Object, changed: &mut bool| -> Object { - *changed |= matches!(e, Object::Ref(_)); - let Object::Dict(d) = &*self.held(e) else { return Object::Null }; - let mut made = Dict::new(); - for key in DECODE_PARMS_KEYS { - let Some(v) = d.get(key) else { continue }; - *changed |= matches!(v, Object::Ref(_)); - let v = self.held(v); - if matches!(*v, Object::Int(_) | Object::Real(_) | Object::Bool(_)) { - made.set(key.to_vec(), (*v).clone()); - } - } - if let Some(g @ Object::Ref(_)) = d.get(b"JBIG2Globals") { - made.set(b"JBIG2Globals".to_vec(), g.clone()); - } - Object::Dict(made) - }; - let made = match &*self.held(p) { - Object::Array(a) => { - Object::Array(a.iter().zip(&filters).map(|(e, crypt)| if *crypt { Object::Null } else { one(e, &mut changed) }).collect()) - } - _ => one(p, &mut changed), - }; - changed.then_some(made) - } - - /// `o`, or the object it refers to, without copying either. - pub(crate) fn held<'o>(&self, o: &'o Object) -> Held<'o> { - match o { - Object::Ref(_) => Held::Resolved(self.resolve(o)), - o => Held::Direct(o), - } - } - - /// Resolve and return a dictionary (a stream's dictionary counts). - pub fn dict(&self, o: &Object) -> Option { - self.resolve(o).as_dict().cloned() - } - - /// The CCITT parameters of image `s` when its last filter is `CCITTFaxDecode`: its - /// `/DecodeParms` (resolved; the dictionary of a lone filter, or its entry in an array), with the - /// defaults of ISO 32000-2 Table 11 for what that leaves out. `None` for another image, and for - /// one whose `/Filter` isn't a name or an array of names. - pub fn fax_params(&self, s: &Stream) -> Option { - let chain = s.readable_filters().ok()?; - if !matches!(chain.last(), Some((pdfcraft_filters::Filter::CcittFax, _))) { - return None; - } - let parms = self.last_filter_parms(s); - let int = |key: &[u8], default: i64| parms.get(key).and_then(|v| self.resolve(v).as_int()).unwrap_or(default); - let flag = |key: &[u8], default: bool| match parms.get(key).map(|v| self.resolve(v)).as_deref() { - Some(Object::Bool(b)) => *b, - _ => default, - }; - let d = pdfcraft_filters::CcittParams::default(); - Some(pdfcraft_filters::CcittParams { - k: int(b"K", d.k), - columns: int(b"Columns", d.columns), - rows: int(b"Rows", d.rows), - end_of_line: flag(b"EndOfLine", d.end_of_line), - encoded_byte_align: flag(b"EncodedByteAlign", d.encoded_byte_align), - end_of_block: flag(b"EndOfBlock", d.end_of_block), - black_is_1: flag(b"BlackIs1", d.black_is_1), - }) - } - - /// The `/DecodeParms` dictionary of stream `s`'s last filter, resolved: that of a lone filter, - /// or its entry in an array. Empty when there is none. - fn last_filter_parms(&self, s: &Stream) -> Dict { - // Its place in /Filter, /Crypt included, pairs it with its /DecodeParms entry (§7.4.1): the - // last filter but a /Crypt, which `filters` leaves out wherever it stands. - let index = match s.dict.get(b"Filter") { - Some(Object::Array(a)) => a.iter().rposition(|f| f.as_name() != Some(b"Crypt".as_slice())).unwrap_or(0), - _ => 0, - }; - match s.dict.get(b"DecodeParms").map(|p| self.resolve(p)).as_deref() { - Some(Object::Array(a)) => a.get(index).and_then(|p| self.dict(p)), - Some(p) if index == 0 => p.as_dict().cloned(), - _ => None, - } - .unwrap_or_default() - } - - /// [`Document::decoded_image_spending`] with no limit on the work, for tests of what an image - /// decodes to: everything else decodes images through an [`ImageBudget`](crate::ImageBudget). - #[cfg(test)] - pub(crate) fn decoded_image(&self, s: &Stream) -> Result, CosError> { - self.decoded_image_within(s, MAX_DECODED) - } - - /// [`Document::decoded_image`] within `max` bytes. - #[cfg(test)] - pub(crate) fn decoded_image_within(&self, s: &Stream, max: usize) -> Result, CosError> { - let mut unbounded = u64::MAX; - self.decoded_image_spending(s, max, &mut unbounded) - } - - /// The samples of image `s`, decoded within `max` bytes ([`MAX_DECODED`] at most), spending from - /// `work`: see [`ImageBudget::decode`](crate::ImageBudget::decode), the way in from outside this - /// crate, which says what this decodes and how. - pub(crate) fn decoded_image_spending(&self, s: &Stream, max: usize, work: &mut u64) -> Result, CosError> { - let max = max.min(MAX_DECODED); - let s = &*self.with_direct_parms(s); - let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string()); - let chain = s.readable_filters().map_err(filter)?; - // The filters before the image codec, such as ASCII85Decode, are undone first. - let undo = |before: &[(pdfcraft_filters::Filter, pdfcraft_filters::Params)], work: &mut u64| match before { - [] => Ok(Cow::Borrowed(&s.raw[..])), - _ => pdfcraft_filters::decode_tolerant_within(before, &s.raw, max, work).map(Cow::Owned).map_err(filter), - }; - // `/Width` and `/Height` are read as given directly, as every caller reads them first: one - // given by reference has the image refused there, before it gets here. - let height = s.dict.int(b"Height").and_then(|h| u32::try_from(h).ok()); - if let Some(((pdfcraft_filters::Filter::Jbig2, _), before)) = chain.split_last() { - let (Some(columns), Some(height)) = (s.dict.int(b"Width").and_then(|w| u32::try_from(w).ok()), height) else { - return Err(CosError::Filter("a JBIG2 image's /Width or /Height isn't a size".into())); - }; - // Nothing to read for what can't be decoded: the images of a file share its globals. - let globals = if *work == 0 { None } else { self.jbig2_globals(s, work)? }; - let data = if *work == 0 { Cow::Borrowed(&[][..]) } else { undo(before, work)? }; - return pdfcraft_filters::decode_jbig2_within(&data, globals.as_deref().map(Vec::as_slice), columns, height, max, work).map_err(filter); - } - let Some(p) = self.fax_params(s) else { - // The last filter gives the image's rows: rows without the tags of the PNG predictor - // they declare are read as they are (`pdfcraft_filters::decode_image_tolerant_within`). - return match height { - Some(rows) => s.decoded_image_rows_spending(max, rows, work), - None => s.decoded_spending(max, work), - }; - }; - // As many rows as the image has: a /Rows below /Height would cut the picture short, and - // rows past /Height aren't drawn. - let p = pdfcraft_filters::CcittParams { rows: 0, ..p }; - if s.dict.int(b"Width") != Some(p.columns) { - return Err(CosError::Filter("a fax image's /Columns aren't its /Width".into())); - } - let stride = usize::try_from(p.columns).unwrap_or(usize::MAX).div_ceil(8).max(1); - let rows = u32::try_from(max / stride).unwrap_or(u32::MAX).min(height.unwrap_or(0)); - let data = undo(chain.split_last().map_or(&[][..], |(_, before)| before), work)?; - pdfcraft_filters::decode_ccitt_within(&data, &p, rows, work).map_err(filter) - } - - /// What the `/JBIG2Globals` stream of JBIG2 image `s` (in its last filter's `/DecodeParms`) - /// decodes to, within [`MAX_JBIG2_GLOBALS`]; `None` without one (a reference to no object is - /// one to null, ISO 32000-2 §7.3.10, which is an absent entry, §7.3.7). The image is then read - /// without them, as the renderer draws it, so an image that needed them mostly fails to decode - /// and can decode to other pixels. A stream that is there and can't be read is an error, rather - /// than the image decoded without it. - /// - /// The images of a scan share one globals stream, so a stream of the file, not edited since - /// it was read (the same in every clone) and with its `/DecodeParms` direct (by reference they - /// are another object, which can change while this one doesn't), has its filters undone once a - /// document and is kept, up to [`JBIG2_GLOBALS_HELD`] in all; others are read each time. That - /// saves the filters only: the segments it holds are decoded again for every image. Undoing - /// the filters spends from `work`, so globals read for every image pay every time. - fn jbig2_globals(&self, s: &Stream, work: &mut u64) -> Result>>, CosError> { - let Some(entry) = self.last_filter_parms(s).get(b"JBIG2Globals").cloned() else { return Ok(None) }; - let held = match entry { - Object::Ref(r) if !self.overlay.contains_key(&r.num) => Some(r.num), - _ => None, - }; - if let Some(hit) = held.and_then(|num| self.jbig2_globals.lock().ok()?.get(&num).cloned()) { - return Ok(Some(hit)); - } - let Object::Stream(g) = &*self.resolve(&entry) else { return Ok(None) }; - let readable = self.with_direct_parms(g); - let held = held.filter(|_| matches!(readable, Cow::Borrowed(_))); - let mut data = readable.decoded_spending(MAX_JBIG2_GLOBALS, work).map_err(|e| { - let detail = match e { - CosError::Filter(detail) => detail, - e => e.to_string(), - }; - CosError::Filter(format!("the JBIG2 globals can't be read: {detail}")) - })?; - // What is kept counts by its length, so that's all it may hold on to. - data.shrink_to_fit(); - let data = Arc::new(data); - // A poisoned lock only means nothing is kept. - if let (Some(num), Ok(mut kept)) = (held, self.jbig2_globals.lock()) { - let total = kept.values().map(|g| g.len()).fold(data.len(), usize::saturating_add); - if total <= JBIG2_GLOBALS_HELD { - kept.insert(num, data.clone()); - } - } - Ok(Some(data)) - } - - pub fn try_get(&self, num: u32) -> Result, CosError> { - if let Some(slot) = self.overlay.get(&num) { - note_read(num, None); - note_overlay_read(); - return Ok(match slot { - Slot::Set(_, o) | Slot::RepairedSet(_, _, o) => o.clone(), - Slot::Freed(_) => Arc::new(Object::Null), - }); - } - // Clones share the cache: what a load read that this document has edited since (the - // member's object stream, a `/Length`) makes the entry another document's. - if let Some((loaded, reads)) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) - && !reads.touch(&self.overlay) - { - note_read(num, Some(reads)); - return loaded.clone(); - } - note_read(num, None); - // Loading can re-enter (indirect stream `/Length`, object streams); damaged files make - // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. - let _guard = - LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let _scope = crate::security::FollowScope::enter(); - let edits_read = overlay_reads(); - let frame = ReadFrame::enter(); - let loaded = self.load(num, 0).map(Arc::new); - let reads = frame.reads(); - drop(frame); - if let Err(e) = &loaded - && !matches!(e, CosError::Poisoned) - { - self.note_unreadable(num, || e.to_string()); - } - // A failure is kept like an object, or every read of a damaged object loads it again. A - // poisoned lock says nothing about the object. Clones share the cache, so a load that read - // this document's edits (an edited object stream or `/Length`) is not kept. - if overlay_reads() == edits_read && !matches!(loaded, Err(CosError::Poisoned)) { - // The editing cache is bounded like the scan reader's: re-parsing a file-backed view - // is cheap, and returned objects outlive eviction. - self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone(), loaded_bytes(&loaded, &self.data), reads); - self.evict_editor_cache_if_over_budget(num)?; - } - loaded - } - - /// Bounds for the editing cache: roomier than the scan reader's (128 objects / 8 MB) - /// so interactive editing does not thrash, but still capped so touching a large - /// document cannot pin its whole object graph. Only owned (decrypted/copied) bytes - /// count; file-backed views re-parse for free. - /// - /// Worst case per document: input bytes + `EDITOR_CACHE_BYTES` (32 MB) of owned object - /// data + `EDITOR_OBJSTM_BYTES` (32 MB) + unsaved `overlay` edits. Each object is measured - /// once, as it is cached ([`ObjectCache`](super::state::ObjectCache)), so both caps hold - /// after every insertion. - /// Clones share these caches, so one clone's eviction drops another's hot set; - /// that only costs re-parsing, never correctness. Eviction keeps just the newcomer: - /// file-backed views re-parse cheaply and editor working sets stay far below 1024. - pub(super) const EDITOR_CACHE_OBJECTS: usize = 1024; - - pub(super) const EDITOR_CACHE_BYTES: usize = 32 * 1024 * 1024; - - /// Object streams decode to megabytes each, so fewer are kept than plain objects. - const EDITOR_OBJSTM_OBJECTS: usize = 256; - - const EDITOR_OBJSTM_BYTES: usize = 32 * 1024 * 1024; - - fn evict_editor_cache_if_over_budget(&self, fresh_num: u32) -> Result<(), CosError> { - let mut cache = self.cache.lock().map_err(|_| CosError::Poisoned)?; - if cache.len() <= Self::EDITOR_CACHE_OBJECTS && cache.owned() <= Self::EDITOR_CACHE_BYTES { - return Ok(()); - } - // Eviction keeps just the newcomer: file-backed views re-parse cheaply. - cache.retain(|n| *n == fresh_num); - Ok(()) - } - - fn evict_editor_streams_if_over_budget(&self, current: u32) -> Result<(), CosError> { - // The caller inserts `current` first, so retaining it keeps a live entry. - let over = { - let streams = self.objstms.lock().map_err(|_| CosError::Poisoned)?; - let bytes = streams.values().fold(0usize, |bytes, stream| { - bytes - .saturating_add(stream.data.capacity()) - .saturating_add(stream.index.capacity().saturating_mul(std::mem::size_of::>())) - }); - bytes > Self::EDITOR_OBJSTM_BYTES || streams.len() > Self::EDITOR_OBJSTM_OBJECTS - }; - if over { - self.objstms.lock().map_err(|_| CosError::Poisoned)?.retain(|n, _| *n == current); - } - Ok(()) - } - - /// Visit every current object without filling this document's editing cache. - /// - /// Full-document searches (for example, standalone timestamps) need to inspect objects - /// that may never be used again. Keep their parse caches private and release them in - /// batches. Returned objects own their data and remain valid after the next batch. - pub fn scan_objects(&self) -> impl Iterator)> + use<> { - let numbers = self.object_numbers(); - let mut reader = self.object_reader(); - numbers.into_iter().map(move |num| { - let reference = ObjRef::new(num, reader.document.generation(num)); - (reference, reader.get(reference)) - }) - } - - /// Like [`Self::scan_objects`], preserving read errors for callers that must distinguish - /// incomplete discovery from an actual null object. Caches remain private and bounded. - pub fn scan_objects_checked(&self) -> impl Iterator, CosError>)> + use<> { - self.scan_objects_subset(self.object_numbers()) - } - - /// Inspect selected current objects with the same bounded cache as a whole-file scan. - /// Missing/freed numbers yield `Null`; malformed objects return their read error. - pub fn scan_objects_subset(&self, numbers: Vec) -> impl Iterator, CosError>)> + use<> { - let mut reader = self.object_reader(); - numbers.into_iter().map(move |num| { - let reference = ObjRef::new(num, reader.document.generation(num)); - (reference, reader.try_get(reference)) - }) - } - - /// Inspect selected objects in the immutable source, ignoring all unsaved edits. This - /// allows a consumer to check an edit's dependencies without retaining an old document. - pub fn scan_original_objects_subset(&self, numbers: Vec) -> impl Iterator, CosError>)> + use<> { - let mut reader = self.object_reader(); - reader.document.overlay.clear(); - numbers.into_iter().map(move |num| { - let reference = ObjRef::new(num, reader.document.generation(num)); - (reference, reader.try_get(reference)) - }) - } - - /// Opened with a tighter stream limit than editing uses ([`Document::open_with_stream_limit`]). - pub(crate) fn stream_limited(&self) -> bool { - self.stream_limit < crate::object::MAX_DECODED - } - - pub(crate) fn object_reader(&self) -> ObjectReader { - let mut document = self.clone(); - document.cache = Arc::default(); - document.objstms = Arc::default(); - ObjectReader { document, decoded_count: 0 } - } - - fn load(&self, num: u32, depth: u32) -> Result { - if depth > 16 { - return Err(CosError::Syntax { offset: 0, detail: "reference cycle while loading".into() }); - } - match self.entries.lookup(num, &self.data) { - None | Some(XrefEntry::Free { .. }) => Ok(Object::Null), - Some(XrefEntry::InFile { offset, .. }) => { - let resolve = |r: ObjRef| self.stream_length(r); - let off = offset as usize; - let at = |off: usize| match parse_indirect_shared(&self.data, off, &resolve, &self.endstreams) { - Ok((id, o)) if id.num == num => Some(self.decrypted(id, o)), - _ => None, - }; - // Offsets relative to a shifted header, or simply wrong: try both fixes. - at(off) - .or_else(|| if self.header_offset == 0 { None } else { at(off.saturating_add(self.header_offset)) }) - .or_else(|| self.located.get(&num).and_then(|(o, _)| usize::try_from(*o).ok()).and_then(at)) - .or_else(|| self.scan_for(num).map(|(id, o)| self.decrypted(id, o))) - .ok_or_else(|| { - self.note_unreadable(num, || self.why_unreadable(off)); - CosError::MissingObject(num) - }) - } - Some(XrefEntry::InStream { stream, index }) => { - let stm = self.objstm(stream)?; - // The member at the entry's position, else wherever the stream lists `num`. - let at = usize::try_from(index).ok().and_then(|i| stm.index.get(i)).copied().flatten().filter(|(n, _)| *n == num); - let (_, off) = at.or_else(|| stm.index.iter().flatten().find(|(n, _)| *n == num).copied()).ok_or(CosError::MissingObject(num))?; - Lexer::new(&stm.data, off).object() - } - } - } - - pub(super) fn objstm(&self, num: u32) -> Result, CosError> { - // An edited or freed stream is this document's alone: clones share the parsed streams. - let edited = self.overlay.contains_key(&num); - if !edited - && let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) - && !s.reads.touch(&self.overlay) - { - note_read(num, Some(&s.reads)); - return Ok(s.clone()); - } - let frame = ReadFrame::enter(); - // A file entry that cannot place the stream (one that says it is itself compressed - // inside another stream, as in #2367, or points nowhere) is tried by the stream's own - // header, as a plain object with a wrong offset is. An edited or freed stream, in the - // overlay or in a saved revision's table, is what that says it is: the file's copy - // must not come back. - let edits_read = overlay_reads(); - let loaded = match self.try_get(num) { - Ok(o) if matches!(*o, Object::Stream(_)) => o, - _ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => { - return Err(CosError::MissingObject(num)); - } - _ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?), - }; - let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) }; - let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize; - let first = usize::try_from(s.dict.int(b"First").unwrap_or(0).max(0)).ok(); - // Its parameters can be given by reference, and can sit in another object stream. A scan of - // the file's object streams comes here without a read around it. - let _scope = crate::security::FollowScope::enter(); - let readable = crate::security::following(Cow::Borrowed(s), || self.with_direct_parms(s)); - #[cfg(test)] - super::tests::OBJSTM_DECODES.with(|decodes| decodes.set(decodes.get() + 1)); - let data = readable.decoded_within(self.stream_limit)?; - let mut lx = Lexer::new(&data, 0); - let mut index = Vec::with_capacity(n); - for _ in 0..n { - let (Some(Object::Int(onum)), Some(Object::Int(off))) = (lx.object().ok(), lx.object().ok()) else { break }; - index.push(objstm_member(onum, off, first, data.len())); - } - let stm = Arc::new(ObjStm { index, data: Arc::new(data), reads: frame.reads() }); - // A stream whose load or decoding read this document's edits (its `/Length`, its - // `/DecodeParms`) is as much its own as an edited one. - if edited || overlay_reads() != edits_read { - return Ok(stm); - } - // Same bound as the object cache: decoded object streams can be megabytes each. - // Insert first so the retained `num` is actually in the map. - self.objstms.lock().map_err(|_| CosError::Poisoned)?.insert(num, stm.clone()); - self.evict_editor_streams_if_over_budget(num)?; - Ok(stm) - } - - /// An indirect stream `/Length`, read through the loader's guard against cycles. Kept in - /// [`LENGTHS`] only during a load, which clears it when it ends: a scan of the file from - /// [`Self::objstm`] can come here without one. - fn stream_length(&self, r: ObjRef) -> Option { - if self.overlay.contains_key(&r.num) { - note_overlay_read(); - } - let cached = loading(); - // Each reuse counts as a read of the length and of what resolving it read, and as an edit - // read when that went through this document's edits. - if cached && let Some((known, edits, reads)) = LENGTHS.with(|m| m.borrow().get(&r).cloned()) { - if edits { - note_overlay_read(); - } - note_read(r.num, Some(&reads)); - return known; - } - note_read(r.num, None); - let edits_read = overlay_reads(); - let frame = ReadFrame::enter(); - let length = self.try_get_reference(r).ok().and_then(|o| o.as_int()); - if cached { - LENGTHS.with(|m| m.borrow_mut().insert(r, (length, overlay_reads() != edits_read, frame.reads()))); - } - length - } - - /// Last-resort lookup: scan the file for `num G obj`. What isn't there is remembered: each miss - /// reads the whole file, and a file can refer to such an object from everywhere. - fn scan_for(&self, num: u32) -> Option<(ObjRef, Object)> { - if self.unfound.lock().is_ok_and(|unfound| unfound.contains(&num)) { - return None; - } - #[cfg(test)] - super::tests::FILE_SCANS.with(|scans| scans.set(scans.get() + 1)); - let needle = format!("{num} "); - let data = &self.data; - let budget = ScanLengths::enter(); - let resolve = |r: ObjRef| if budget.take() { self.stream_length(r) } else { None }; - let mut found = None; - let mut i = 0; - while let Some(p) = find(data, needle.as_bytes(), i, data.len()) { - i = p + 1; - if p > 0 && !is_whitespace(data[p - 1]) { - continue; - } - // An indirect `/Length` is resolved, as at the object's offset, or a stream that ends - // in a misspelt `endstream` is searched on into a later object's. A header that parses - // as far as the `/Length` starts with `num` itself, after white space. A budget shared - // with the scans this leads to bounds how often the file is read again ([`ScanLengths`]). - if let Ok((id, o)) = parse_indirect_shared(data, p, &resolve, &self.endstreams) - && id.num == num - { - found = Some((id, o)); // keep the last (newest) definition - } - } - if found.is_none() - && let Ok(mut unfound) = self.unfound.lock() - { - unfound.insert(num); - } - found - } - - // ── editing ───────────────────────────────────────────────────────────────────────────── - - /// Stream data still pointing into another document's file would keep that whole file - /// alive for as long as this edit exists: give it a buffer of its own (just its bytes). - fn adopt(&self, obj: Object) -> Object { - match obj { - Object::Stream(mut s) if s.raw.borrows_other_than(&self.data) => { - s.raw = s.raw.detached(); - Object::Stream(s) - } - other => other, - } - } - - /// Replace an object (keeping its generation). - pub fn set(&mut self, r: ObjRef, obj: impl Into) { - let obj = self.adopt(obj.into()); - self.overlay.insert(r.num, Slot::Set(r.generation, Arc::new(obj))); - if r.num >= self.next_num { - self.next_num = r.num.saturating_add(1); - } - } - - /// Replace an accepted existing reference without losing a damaged source slot's aliases. - /// Unlike [`Self::set`], this preserves both xref/header generations when they disagree. - /// Such an edit forces a full rewrite so the output has consistent references, rather than - /// appending a new contradictory xref row. Missing, free and stale references are errors. - pub fn set_existing(&mut self, r: ObjRef, obj: impl Into) -> Result<(), CosError> { - if !self.reference_matches(r) { - return Err(CosError::MissingObject(r.num)); - } - let alias = self.alternate_generation(r); - if let Some(alias) = alias { - let obj = self.adopt(obj.into()); - self.overlay.insert(r.num, Slot::RepairedSet(r.generation, alias, Arc::new(obj))); - } else { - self.set(r, obj); - } - Ok(()) - } - - /// Other accepted generation for a known-live reference to a damaged slot. - pub(super) fn alternate_generation(&self, r: ObjRef) -> Option { - match self.overlay.get(&r.num) { - Some(Slot::RepairedSet(generation, alias, _)) => Some(if r.generation == *generation { *alias } else { *generation }), - Some(_) => None, - None => match self.entries.lookup(r.num, &self.data) { - Some(XrefEntry::InFile { offset, generation }) => self.indexed_header_reference(r.num, offset).and_then(|header| { - (header.generation != generation).then_some(if r.generation == generation { header.generation } else { generation }) - }), - _ => None, - }, - } - } - - /// Add a new indirect object and return its reference. - pub fn add(&mut self, obj: impl Into) -> ObjRef { - let r = ObjRef::new(self.allocate_num(), 0); - let obj = self.adopt(obj.into()); - self.overlay.insert(r.num, Slot::Set(0, Arc::new(obj))); - r - } - - /// Find a never-used number, retaining the high-water mark for incremental /Size. - /// Exhausted hints fall back to gaps without replacing source or overlay objects. - fn allocate_num(&mut self) -> u32 { - let mut num = if self.next_num == u32::MAX { 1 } else { self.next_num.max(1) }; - while self.entries.lookup(num, &self.data).is_some() || self.overlay.contains_key(&num) { - num = if num == u32::MAX - 1 { 1 } else { num + 1 }; - } - self.next_num = self.next_num.max(num.saturating_add(1)); - num - } - - /// Delete an object (its number becomes free with the next generation). - pub fn free(&mut self, r: ObjRef) { - self.overlay.insert(r.num, Slot::Freed(r.generation.saturating_add(1))); - } - - /// Free what was added since the file was read and nothing refers to any more: the objects - /// [`SaveOptions::drop_unlinked`](crate::SaveOptions::drop_unlinked) leaves out of an - /// incremental save, freed as it writes them (the generation stays). For a writer about to - /// add an object that nothing refers to either, which that option would leave out too. - pub fn free_unlinked_additions(&mut self) { - for num in crate::writer::unlinked_additions(self) { - let generation = self.generation(num); - self.overlay.insert(num, Slot::Freed(generation)); - } - } - - /// Record `current`'s object graph as unsaved edits of this document. - /// - /// Used when the working snapshot was restored from an earlier revision and the next - /// incremental save must still append to this file (the last saved revision). Objects - /// that already match are left untouched, except members of an object stream of this file - /// whose stream number `current` frees or reuses: those are written again on their own. - /// Reads go through private readers so neither document's editing cache is filled. - pub fn overlay_changes_from(&mut self, current: &Document) -> Result<(), CosError> { - let saved_nums = self.object_numbers(); - let current_nums = current.object_numbers(); - let mut freed = Vec::new(); - let mut changed = Vec::new(); - { - let mut saved_reader = self.object_reader(); - let mut current_reader = current.object_reader(); - for &num in &saved_nums { - if current_nums.binary_search(&num).is_err() { - freed.push(num); - } - } - for &num in ¤t_nums { - let generation = current.generation(num); - let same = saved_nums.binary_search(&num).is_ok() - && self.generation(num) == generation - && self.alternate_generation(ObjRef::new(num, generation)) == current.alternate_generation(ObjRef::new(num, generation)) - && saved_reader.try_get(ObjRef::new(num, generation))? == current_reader.try_get(ObjRef::new(num, generation))?; - if !same { - changed.push(num); - } - } - } - // An object stream of this file that is freed or replaced takes its members with it, so - // a member that is otherwise unchanged is written again on its own. - let gone = |n: u32| freed.binary_search(&n).is_ok() || changed.binary_search(&n).is_ok(); - let orphaned: Vec = current_nums - .iter() - .copied() - .filter(|&num| changed.binary_search(&num).is_err()) - .filter(|&num| { - let packed_in = match self.xref_entry(num) { - Some(XrefEntry::InStream { stream, .. }) => Some(stream), - _ => None, - }; - packed_in.is_some_and(gone) - }) - .collect(); - changed.extend(orphaned); - for num in freed { - let generation = self.generation(num); - self.free(ObjRef::new(num, generation)); - } - let mut current_reader = current.object_reader(); - for num in changed { - let generation = current.generation(num); - let object = current_reader.try_get(ObjRef::new(num, generation))?; - if let Some(alias) = current.alternate_generation(ObjRef::new(num, generation)) { - let object = self.adopt((*object).clone()); - self.overlay.insert(num, Slot::RepairedSet(generation, alias, Arc::new(object))); - self.next_num = self.next_num.max(num.saturating_add(1)); - } else { - self.set(ObjRef::new(num, generation), (*object).clone()); - } - } - *self.trailer_mut() = current.trailer().clone(); - Ok(()) - } - - /// Edit a dictionary (or stream dictionary) object in place. - pub fn update_dict(&mut self, r: ObjRef, f: impl FnOnce(&mut Dict)) -> Result<(), CosError> { - let mut obj = (*self.get(r)).clone(); - let d = obj.as_dict_mut().ok_or(CosError::NotADictionary(r))?; - f(d); - self.set_existing(r, obj) - } - - /// Current generation of an object number. - pub fn generation(&self, num: u32) -> u16 { - match self.overlay.get(&num) { - Some(Slot::Set(g, _) | Slot::RepairedSet(g, _, _)) => *g, - Some(Slot::Freed(g)) => *g, - None => match self.entries.lookup(num, &self.data) { - Some(XrefEntry::InFile { generation, .. }) => generation, - Some(XrefEntry::Free { next_generation }) => next_generation, - _ => 0, - }, - } - } - - pub(crate) fn overlay_entries(&self) -> impl Iterator>)> + '_ { - self.overlay.iter().map(|(n, s)| match s { - Slot::Set(g, o) | Slot::RepairedSet(g, _, o) => (*n, *g, Some(o.clone())), - Slot::Freed(g) => (*n, *g, None), - }) - } - - pub(crate) fn next_num(&self) -> u32 { - self.next_num - } - - /// All object numbers known (file and overlay), excluding freed ones. - pub fn object_numbers(&self) -> Vec { - let source_numbers = self.entries.object_numbers(&self.data); - let mut numbers = Vec::with_capacity(source_numbers.len().max(self.overlay.len())); - let mut source = source_numbers.into_iter().peekable(); - for (&num, slot) in &self.overlay { - while source.peek().is_some_and(|n| *n < num) { - if let Some(n) = source.next() { - numbers.push(n); - } - } - if source.peek().is_some_and(|n| *n == num) { - source.next(); - } - if matches!(slot, Slot::Set(..) | Slot::RepairedSet(..)) { - numbers.push(num); - } - } - numbers.extend(source); - numbers - } - - /// Rebase this document on bytes written by the writer (after a save): edits become part of - /// the file and the overlay is cleared, so the next incremental save appends only new edits. - pub fn reopen_after_save(&self, bytes: Arc>) -> Result { - Document::open(bytes) - } - - // ── cross-reference parsing ───────────────────────────────────────────────────────────── - - pub(super) fn root_is_catalog(&self) -> bool { - let Some(r) = self.root() else { return false }; - matches!(self.try_get_reference(r).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog") || d.contains(b"Pages")) - } -} - -/// The `/DecodeParms` entries the decoders read (ISO 32000-2 Tables 8, 11 and 13: the Flate and -/// LZW predictors, CCITT, DCT's colour transform): numbers and flags a file may give by -/// reference, which [`Dict::int`] doesn't follow. -pub const DECODE_PARMS_KEYS: [&[u8]; 13] = [ - b"Predictor", - b"Colors", - b"BitsPerComponent", - b"Columns", - b"EarlyChange", - b"K", - b"Rows", - b"EndOfLine", - b"EncodedByteAlign", - b"EndOfBlock", - b"BlackIs1", - b"DamagedRowsBeforeError", - b"ColorTransform", -]; - -/// How many references in a row [`Document::resolve`] follows: enough for any file, and it ends a -/// cycle of them. -const MAX_REFERENCE_HOPS: usize = 32; - -/// An object read in place, or the object a reference names as the document holds it: either way -/// nothing is copied ([`Document::held`]). -pub(crate) enum Held<'o> { - Direct(&'o Object), - Resolved(Arc), -} - -impl std::ops::Deref for Held<'_> { - type Target = Object; - - fn deref(&self) -> &Object { - match self { - Held::Direct(o) => o, - Held::Resolved(o) => o, - } - } -} diff --git a/crates/cos/src/document/open.rs b/crates/cos/src/document/open.rs deleted file mode 100644 index 8caf0b83bc..0000000000 --- a/crates/cos/src/document/open.rs +++ /dev/null @@ -1,503 +0,0 @@ -use super::reader::find; -use super::state::{Credentials, Slot}; -use super::xref::XrefIndex; -use super::{Arc, BTreeMap, CosError, Dict, Document, HashSet, ObjRef, Object}; - -/// More objects than `/Encrypt` and `/ID` refer to in any file, so that one which refers to the page -/// tree, say, doesn't have a whole document read to open it. -const MAX_ENCRYPT_MEMBERS: usize = 64; - -impl Document { - /// A new document with an empty page tree (for split / extract / combine). It has no - /// original bytes, so saving always writes a full file. - pub fn new_empty() -> Self { - let mut doc = Document { - data: Arc::new(Vec::new()), - endstreams: Arc::default(), - source_identity: Arc::new(()), - entries: Arc::new(XrefIndex::empty()), - trailer: Dict::new(), - revisions: Arc::new(Vec::new()), - xref_oddities: Arc::default(), - located: Arc::default(), - repair_log: Arc::new(Vec::new()), - unlinked_update: false, - cache: Arc::default(), - objstms: Arc::default(), - unreadable: Arc::default(), - jbig2_globals: Arc::default(), - stream_limit: crate::object::MAX_DECODED, - overlay: BTreeMap::new(), - next_num: 1, - header_offset: 0, - version: "1.7".into(), - security: None, - encrypt_num: None, - encrypt_members: Arc::default(), - unfound: Arc::default(), - encryption_changed: false, - full_save: false, - new_id: false, - out_security: None, - out_encrypt_num: None, - }; - let mut pages = Dict::new(); - pages.set(b"Type".to_vec(), Object::name("Pages")); - pages.set(b"Kids".to_vec(), Object::Array(Vec::new())); - pages.set(b"Count".to_vec(), Object::Int(0)); - let pages = doc.add(pages); - let mut catalog = Dict::new(); - catalog.set(b"Type".to_vec(), Object::name("Catalog")); - catalog.set(b"Pages".to_vec(), Object::Ref(pages)); - let root = doc.add(catalog); - doc.trailer.set(b"Root".to_vec(), Object::Ref(root)); - doc - } - - /// Parse a document. Damaged cross-reference data is reconstructed; see `repair_log`. - /// Encrypted documents open only if their user password is empty; see `open_with_password`. - pub fn open(data: Arc>) -> Result { - Self::open_with_password(data, None) - } - - /// Parse a document, authenticating encrypted ones with `password` (owner or user; `None` - /// tries the empty password). Fails with `NeedsPassword` / `WrongPassword` as appropriate. - pub fn open_with_password(data: Arc>, password: Option<&str>) -> Result { - Self::open_with_stream_limit(data, password, crate::object::MAX_DECODED) - } - - /// Open with a tighter decompression bound for object and cross-reference streams. - /// Inspection can use this without changing the editing reader's default limit. Such a - /// document is read-only: objects in a larger object stream read as missing, so the - /// writers refuse to save it ([`CosError::ReadOnlyLimit`]). - pub fn open_with_stream_limit(data: Arc>, password: Option<&str>, stream_limit: usize) -> Result { - Self::open_as(data, Credentials::Password(password), stream_limit) - } - - /// Open `data`, an earlier revision of this file (the bytes a signature covers) or another - /// save of it, with this document's credentials: an encrypted one must have the same - /// `/Encrypt` dictionary and `/ID` as this document, whose file key is then reused, so no - /// password is needed or kept; unencrypted data opens as [`open`](Self::open) does. Data - /// encrypted differently fails with `NeedsPassword`. - pub fn open_with_same_credentials(&self, data: Arc>) -> Result { - let credentials = match &self.security { - Some(handler) => Credentials::Handler { handler, id0: self.first_id() }, - None => Credentials::Password(None), - }; - Self::open_as(data, credentials, self.stream_limit) - } - - fn open_as(data: Arc>, credentials: Credentials<'_>, stream_limit: usize) -> Result { - // Viewers accept files whose header is missing or damaged as long as the body looks - // like PDF; so do we (a note goes to the repair log). - let header = find(&data, b"%PDF-", 0, 1024); - let headerless = header.is_none(); - if headerless && find(&data, b" obj", 0, 4096).is_none() { - return Err(CosError::NotPdf); - } - let header_offset = header.unwrap_or(0); - let version = match header { - Some(h) => String::from_utf8_lossy(&data[(h + 5).min(data.len())..(h + 8).min(data.len())]).trim().to_string(), - None => "1.4".to_string(), - }; - let mut doc = Document { - data, - endstreams: Arc::default(), - source_identity: Arc::new(()), - entries: Arc::new(XrefIndex::empty()), - trailer: Dict::new(), - revisions: Arc::new(Vec::new()), - xref_oddities: Arc::default(), - located: Arc::default(), - repair_log: Arc::new(Vec::new()), - unlinked_update: false, - cache: Arc::default(), - objstms: Arc::default(), - unreadable: Arc::default(), - jbig2_globals: Arc::default(), - stream_limit: stream_limit.min(crate::object::MAX_DECODED), - overlay: BTreeMap::new(), - next_num: 1, - header_offset, - version, - security: None, - encrypt_num: None, - encrypt_members: Arc::default(), - unfound: Arc::default(), - encryption_changed: false, - full_save: false, - new_id: false, - out_security: None, - out_encrypt_num: None, - }; - let mut log = Vec::new(); - if headerless { - log.push("the %PDF- header is missing; reading the file as PDF 1.4".into()); - } - let parsed = doc.read_xref_chain(&mut log); - let mut authenticated = false; - // The newest section cut the page tree off from the chain (see `update_cuts_off_the_pages`). - let mut cut_off = false; - let ok = match parsed { - Ok((entries, trailer, revisions, oddities)) => { - doc.entries = Arc::new(entries); - doc.trailer = trailer; - doc.revisions = Arc::new(revisions); - doc.xref_oddities = Arc::new(oddities); - doc.fill_xref_gaps(&mut log); - // Authenticate before checking the catalog: it may sit in an encrypted object - // stream, which reads as garbage until the key is known. - if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() { - doc.authenticate(&enc, &credentials)?; - authenticated = true; - } - let catalog = doc.root_is_catalog(); - cut_off = catalog && doc.update_cuts_off_the_pages(); - catalog && !cut_off - } - Err(e) => { - log.push(format!("cross-reference data unreadable ({e}); rebuilding from object headers")); - false - } - }; - if cut_off { - log.push(UNLINKED_UPDATE.into()); - doc.unlinked_update = true; - } else if !ok && !doc.entries.is_empty() { - log.push("cross-reference data does not lead to a valid catalog; rebuilding from object headers".into()); - } - if !ok { - doc.reconstruct(&mut log)?; - } - // Header generations are plaintext, including in encrypted files. Inspect only the - // bounded headers, not object bodies, and retain the compact xref representation. - doc.note_xref_generation_repairs(&mut log); - doc.repair_log = Arc::new(log); - // Reconstruction may have found a different trailer: (re)authenticate against it. - if (!ok || !authenticated) - && let Some(enc) = doc.trailer.get(b"Encrypt").cloned() - { - doc.authenticate(&enc, &credentials)?; - } - if Arc::make_mut(&mut doc.entries).remove_exhausted() { - Arc::make_mut(&mut doc.repair_log).push("skipped exhausted xref object number 4294967295".into()); - } - let max = doc.entries.max_num(); - let size = u32::try_from(doc.trailer.int(b"Size").unwrap_or(0).max(0)).unwrap_or(u32::MAX); - if size == u32::MAX { - Arc::make_mut(&mut doc.repair_log).push("exhausted trailer /Size; new objects use unused object numbers".into()); - } - doc.next_num = max.saturating_add(1).max(size).max(1); - Ok(doc) - } - - /// Whether the chain of cross-reference sections ends at the newest one (no `/Prev`, or one - /// that isn't an offset) although the file holds earlier sections, and the catalog's `/Pages` - /// is not among the objects that one section reaches. ISO 32000-2 §7.5.6 and §7.5.8.2 require - /// the `/Prev`; without it the earlier revisions, page tree and all, are cut off, so the file - /// is rebuilt from its object headers as a damaged one is (the newest definition of each - /// object wins), and the repair log says why. (A file whose `/Root` itself is missing is - /// rebuilt anyway.) - fn update_cuts_off_the_pages(&self) -> bool { - if self.revisions.len() != 1 { - return false; - } - let Some(root) = self.root() else { return false }; - let Ok(catalog) = self.try_get_reference(root) else { return false }; - let Object::Dict(catalog) = &*catalog else { return false }; - // The page tree first: the scan for an earlier `startxref` reads the whole file. - !catalog.get(b"Pages").is_some_and(|pages| matches!(&*self.resolve(pages), Object::Dict(_))) && earlier_sections(&self.data) - } - - fn authenticate(&mut self, enc: &Object, credentials: &Credentials<'_>) -> Result<(), CosError> { - // Authentication can change how the same input is decoded. - self.source_identity = Arc::new(()); - let (num, dict) = match enc { - Object::Ref(r) => (Some(r.num), self.get(*r).as_dict().cloned()), - Object::Dict(d) => (None, Some(d.clone())), - _ => (None, None), - }; - let Some(dict) = dict else { - // A dangling /Encrypt: viewers treat the file as unencrypted, and so do we. - let mut log = self.repair_log.as_ref().clone(); - log.push("the trailer names an /Encrypt dictionary that does not exist; reading the file as unencrypted".into()); - self.repair_log = Arc::new(log); - self.trailer.remove(b"Encrypt"); - return Ok(()); - }; - let params = crate::security::encrypt_dict(self, &dict); - let id0 = self.first_id(); - // Read here, while nothing is decrypted yet. - let members = self.encryption_members(enc); - let handler = match credentials { - Credentials::Password(password) => match pdfcraft_crypt::SecurityHandler::open(params.clone(), &id0, *password) { - Ok(handler) => Arc::new(handler), - // No password, or the empty one: what only the embedded files need isn't asked. - Err(pdfcraft_crypt::CryptError::WrongPassword) => match self.without_password(&dict, params) { - Some(handler) if password.is_none_or(str::is_empty) => Arc::new(handler), - _ if password.is_none() => return Err(CosError::NeedsPassword), - _ => return Err(CosError::WrongPassword), - }, - Err(other) => return Err(CosError::Security(other.to_string())), - }, - // The same encryption as the document whose key this is: that key opens it. A handler - // without the key opens only what it would open by itself. - Credentials::Handler { handler, id0: known } if *handler.dict() == params && *known == id0 => { - if !handler.has_key() && !crate::security::efopen_only(self, &dict) { - return Err(CosError::NeedsPassword); - } - Arc::clone(handler) - } - Credentials::Handler { .. } => return Err(CosError::NeedsPassword), - }; - self.security = Some(handler); - self.encrypt_num = num; - self.encrypt_members = Arc::new(members); - // Objects read while locating the catalog were read before decryption was possible. - self.cache = Arc::default(); - self.objstms = Arc::default(); - self.unreadable = Arc::default(); - Ok(()) - } - - /// The handler for a document opened without a password that only its embedded files need - /// (see [`crate::security::efopen_only`]). Not a repair: ISO 32000-2 §7.6.6 has such a - /// document open so; [`Document::embedded_files_locked`] tells it. - fn without_password(&self, dict: &Dict, params: pdfcraft_crypt::EncryptDict) -> Option { - crate::security::efopen_only(self, dict).then(|| pdfcraft_crypt::SecurityHandler::without_key(params)).flatten() - } - - /// A document opened without the password its embedded files need takes either password - /// later: with the file key its embedded files decrypt, so what was read as stored is read - /// again. Refused while an edited stream holds data read as stored, which the key would - /// encrypt a second time. - fn unlock_embedded_files(&mut self, password: &str) -> Result<(), CosError> { - let Some(current) = self.security.as_deref() else { return Ok(()) }; - if self.overlay_entries().any(|(_, _, o)| matches!(o.as_deref(), Some(Object::Stream(s)) if self.reads_as_stored(s))) { - return Err(CosError::Security("an embedded file was edited before the password was given: save or undo that edit first".into())); - } - let handler = pdfcraft_crypt::SecurityHandler::open(current.dict().clone(), &self.first_id(), Some(password)).map_err(|e| match e { - pdfcraft_crypt::CryptError::WrongPassword => CosError::WrongPassword, - other => CosError::Security(other.to_string()), - })?; - self.security = Some(Arc::new(handler)); - self.source_identity = Arc::new(()); - self.cache = Arc::default(); - self.objstms = Arc::default(); - Ok(()) - } - - /// Whether the document opened without the password its embedded files need (`/AuthEvent - /// /EFOpen`): their data is held, and saved, as stored, and isn't to be read as the files. - pub fn embedded_files_locked(&self) -> bool { - self.security.as_deref().is_some_and(|h| !h.has_key()) - } - - /// Whether `s`, a stream of this document, holds encrypted data that reads as stored because the - /// document opened without the password its embedded files need (see - /// [`Document::embedded_files_locked`]). Such data isn't the file it encrypts. - pub fn reads_as_stored(&self, s: &crate::Stream) -> bool { - self.security.as_deref().is_some_and(|h| crate::security::stream_reads_as_stored(self, h, s)) - } - - /// The first file identifier, part of the encryption key (empty when there is none). - fn first_id(&self) -> Vec { - let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); - let first = id.as_deref().and_then(Object::as_array).and_then(|a| a.first()).map(|s| self.resolve(s)); - first.as_deref().and_then(Object::as_string).map(|s| s.bytes.clone()).unwrap_or_default() - } - - /// The objects `/Encrypt`, given as `enc`, and `/ID` refer to, the ones they refer to, and so - /// on, up to [`MAX_ENCRYPT_MEMBERS`]. Their strings aren't encrypted (ISO 32000-2 §7.6.2): the - /// key is made from them, so they have to read as stored, and stay so. A number with no object - /// behind it, and a stream, which those dictionaries never hold, isn't one: an object added - /// under that number later, or a page's content a hostile `/Encrypt` names, is encrypted. - fn encryption_members(&self, enc: &Object) -> HashSet { - let mut pending = Vec::new(); - crate::writer::collect_refs(enc, &mut |r| pending.push(r)); - if let Some(id) = self.trailer.get(b"ID") { - crate::writer::collect_refs(id, &mut |r| pending.push(r)); - } - let mut members = HashSet::new(); - while let Some(r) = pending.pop() { - if members.len() >= MAX_ENCRYPT_MEMBERS || members.contains(&r.num) { - continue; - } - let held = self.get(r); - if matches!(&*held, Object::Null | Object::Stream(_)) { - continue; - } - members.insert(r.num); - crate::writer::collect_refs(&held, &mut |x| pending.push(x)); - } - members - } - - /// Whether object `num` is written as it is held, being one of [`Document::encryption_members`], - /// unedited, and the save writing the same `/Encrypt`. What was edited or added is encrypted - /// whatever its number. - pub(crate) fn stored_as_is(&self, num: u32) -> bool { - !self.encryption_changed && !self.is_edited(num) && self.encrypt_members.contains(&num) - } - - /// Lift the restrictions of a document opened with its user password by entering the - /// owner (permissions) password (§7.6.4). Fails with `WrongPassword` for any other - /// password. Does nothing when the document is unencrypted or already opened as owner. - /// Both passwords unlock the same file key, so nothing read so far is decoded again. - pub fn unlock(&mut self, password: &str) -> Result<(), CosError> { - let Some(current) = self.security.as_deref() else { return Ok(()) }; - if !current.has_key() { - return self.unlock_embedded_files(password); - } - if current.auth() == pdfcraft_crypt::Auth::Owner { - return Ok(()); - } - let owner = current.authenticate_owner(&self.first_id(), password).map_err(|e| match e { - pdfcraft_crypt::CryptError::WrongPassword => CosError::WrongPassword, - other => CosError::Security(other.to_string()), - })?; - self.security = Some(Arc::new(owner)); - Ok(()) - } - - /// The security handler, when the document is encrypted. - pub fn security(&self) -> Option<&pdfcraft_crypt::SecurityHandler> { - self.security.as_deref() - } - - /// What the opening password allows (`None` for unencrypted documents: everything). - pub fn permissions(&self) -> Option { - self.security().map(|s| s.permissions()) - } - - /// Make the next save a full rewrite: earlier revisions (which still hold what an edit - /// removed) must not stay in the file. Redaction requires this. - pub fn require_full_save(&mut self) { - self.full_save = true; - } - - /// Like [`Document::require_full_save`], and the rewritten file also gets a new file identifier - /// (`/ID`), so that a redacted or sanitized copy never claims to be its source (ISO 32000-2 - /// §14.4). An encrypted file keeps the first string of its identifier, which its key depends on. - pub fn require_full_save_with_new_id(&mut self) { - self.full_save = true; - self.new_id = true; - } - - /// The next save must rewrite the whole file (see [`Document::require_full_save`]). - /// Repair-aware edits retaining two accepted generations cannot use one incremental slot. - pub fn full_save_required(&self) -> bool { - self.full_save || self.overlay.values().any(|slot| matches!(slot, Slot::RepairedSet(..))) - } - - /// The next full save gives the file a new identifier (see [`Document::require_full_save_with_new_id`]). - pub fn new_id_required(&self) -> bool { - self.new_id - } - - /// Security was set or removed since the document was opened (the next save applies it). - pub fn encryption_changed(&self) -> bool { - self.encryption_changed - } - - /// The handler used to write: the new one after `set_encryption` / `remove_encryption`, - /// otherwise the one the document was opened with. - pub(crate) fn output_security(&self) -> (Option<&pdfcraft_crypt::SecurityHandler>, Option) { - if self.encryption_changed { (self.out_security.as_deref(), self.out_encrypt_num) } else { (self.security.as_deref(), self.encrypt_num) } - } - - /// The security the next save writes: protection applied with `set_encryption`, none after - /// `remove_encryption`, otherwise the security the document was opened with. - pub fn output_handler(&self) -> Option<&pdfcraft_crypt::SecurityHandler> { - self.output_security().0 - } - - /// Protect the document with a password (§7.6.4). Takes effect on the next save, which is - /// always a full rewrite. Returns the handler (authenticated as owner). - pub fn set_encryption(&mut self, params: &pdfcraft_crypt::NewEncryption) -> Result<(), CosError> { - // The file identifier is part of the key; make sure it exists and keep it. - let id = self.trailer.get(b"ID").map(|id| self.resolve(id)); - let id0 = match id.as_deref() { - Some(Object::Array(a)) if a.len() == 2 => self.first_id(), - _ => { - let id = pdfcraft_crypt::file_id(¶ms.seed); - let s = Object::String(crate::PdfString { bytes: id.clone(), hex: true }); - self.trailer.set(b"ID".to_vec(), Object::Array(vec![s.clone(), s])); - id - } - }; - let h = pdfcraft_crypt::create(params, &id0).map_err(|e| CosError::Security(e.to_string()))?; - let d = h.dict(); - let s = |b: &[u8]| Object::String(crate::PdfString { bytes: b.to_vec(), hex: true }); - let mut e = Dict::new(); - e.set(b"Filter".to_vec(), Object::name("Standard")); - e.set(b"V".to_vec(), Object::Int(d.v)); - e.set(b"R".to_vec(), Object::Int(d.r)); - e.set(b"Length".to_vec(), Object::Int(d.length_bits)); - e.set(b"O".to_vec(), s(&d.o)); - e.set(b"U".to_vec(), s(&d.u)); - e.set(b"P".to_vec(), Object::Int(i64::from(d.p))); - if d.v >= 4 { - let mut cf = Dict::new(); - let mut std_cf = Dict::new(); - std_cf.set(b"Type".to_vec(), Object::name("CryptFilter")); - let (cfm, len) = if d.v >= 5 { ("AESV3", 32) } else { ("AESV2", 16) }; - std_cf.set(b"CFM".to_vec(), Object::name(cfm)); - std_cf.set(b"Length".to_vec(), Object::Int(len)); - std_cf.set(b"AuthEvent".to_vec(), Object::name("DocOpen")); - cf.set(b"StdCF".to_vec(), Object::Dict(std_cf)); - e.set(b"CF".to_vec(), Object::Dict(cf)); - e.set(b"StmF".to_vec(), Object::name("StdCF")); - e.set(b"StrF".to_vec(), Object::name("StdCF")); - if !d.encrypt_metadata { - e.set(b"EncryptMetadata".to_vec(), Object::Bool(false)); - } - } - if d.v >= 5 { - e.set(b"OE".to_vec(), s(&d.oe)); - e.set(b"UE".to_vec(), s(&d.ue)); - e.set(b"Perms".to_vec(), s(&d.perms)); - } - if let Some(Object::Ref(old)) = self.trailer.get(b"Encrypt").cloned() { - self.free(old); - } - let r = self.add(e); - self.trailer.set(b"Encrypt".to_vec(), Object::Ref(r)); - // Objects are still read with the original handler; saves use the new one. - self.out_security = Some(Arc::new(h)); - self.out_encrypt_num = Some(r.num); - self.encryption_changed = true; - Ok(()) - } - - /// Remove password protection (requires the document to be open — any authenticated - /// password; callers should require the owner password, §7.6.4). Applied by a full save. - pub fn remove_encryption(&mut self) { - if let Some(Object::Ref(r)) = self.trailer.get(b"Encrypt").cloned() { - self.free(r); - } - self.trailer.remove(b"Encrypt"); - self.out_security = None; - self.out_encrypt_num = None; - self.encryption_changed = true; - } - - /// Decrypt a freshly parsed indirect object if the document is encrypted. - pub(super) fn decrypted(&self, id: ObjRef, o: Object) -> Object { - match &self.security { - Some(h) if Some(id.num) != self.encrypt_num && !self.encrypt_members.contains(&id.num) => { - crate::security::transform(self, h, &o, id.num, id.generation, crate::security::Pass::Decrypt) - } - _ => o, - } - } -} - -/// The repair-log line for a file rebuilt because its last update has no `/Prev`. -const UNLINKED_UPDATE: &str = "the file's last update has no /Prev link to its earlier revisions, so their objects (the page tree among them) are unreachable through the cross-reference chain; rebuilding from object headers"; - -/// Whether `data` has more than one `startxref`: an update was appended to it, or it is -/// linearized (whose first-page section links to the main one). -fn earlier_sections(data: &[u8]) -> bool { - find(data, b"startxref", 0, data.len()).and_then(|at| find(data, b"startxref", at.saturating_add(9), data.len())).is_some() -} diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs deleted file mode 100644 index 3e0b9684c1..0000000000 --- a/crates/cos/src/document/reader.rs +++ /dev/null @@ -1,957 +0,0 @@ -#[cfg(test)] -use super::state::SECTION_READS; -use super::state::Unreadable; -use super::xref::{ - MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData, -}; -use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, HashSet, Lexer, ObjRef, Object, is_whitespace, parse_indirect, parse_indirect_shared}; - -impl Document { - #[allow(clippy::type_complexity)] - pub(super) fn read_xref_chain(&self, log: &mut Vec) -> Result<(XrefIndex, Dict, Vec, Vec<(u64, u32)>), CosError> { - let mut compact_log = Vec::new(); - if let Ok(Some((index, trailer, revisions))) = self.read_compact_xref_chain(&mut compact_log) { - let size = trailer.int(b"Size").unwrap_or(0).max(0); - if size > 0 && size <= i64::from(u32::MAX) && u64::from(index.max_num()) < size as u64 { - log.extend(compact_log); - // The compact reader declines any section with set-aside entries, so there are none here. - return Ok((index, trailer, revisions, Vec::new())); - } - } - let (entries, trailer, revisions, oddities, generation_repairs) = self.read_xref_chain_materialized(log)?; - Ok((XrefIndex::Materialized(Arc::new(entries), Arc::new(generation_repairs)), trailer, revisions, oddities)) - } - - fn read_compact_xref_chain(&self, log: &mut Vec) -> Result)>, CosError> { - let data = &self.data; - let tail_start = data.len().saturating_sub(4096); - let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?; - let mut lx = Lexer::new(data, sx + 9); - let first = match lx.object() { - Ok(Object::Int(n)) if n >= 0 => n as u64, - _ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }), - }; - let mut sections = Vec::new(); - let mut retained_rows = 0usize; - let mut retained_bytes = 0usize; - let mut trailer: Option = None; - let mut revisions = Vec::new(); - let mut visited = HashSet::new(); - let mut next = Some(first); - while let Some(off) = next.take() { - if !visited.insert(off) || visited.len() > 10_000 { - log.push(format!("cross-reference chain loops at offset {off}; stopped")); - break; - } - let ((primary, sect_trailer, is_stream), header_relative) = match self.read_compact_section(off as usize) { - Ok(Some(v)) => (v, false), - Ok(None) => return Ok(None), - Err(e) if self.header_offset > 0 => { - log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header")); - match self.read_compact_section(off as usize + self.header_offset)? { - Some(v) => (v, true), - None => return Ok(None), - } - } - // An unreadable older section loses every object only it locates: the - // materialized reader logs it and the file is rebuilt from object headers (#1656). - Err(e) => return Err(e), - }; - revisions.push(Revision { xref_offset: off, is_stream, header_relative }); - if primary.renumbered { - log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0")); - } - let Some(rows) = retained_rows.checked_add(primary.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) }; - let Some(bytes) = retained_bytes.checked_add(primary.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES) else { - return Ok(None); - }; - retained_rows = rows; - retained_bytes = bytes; - let hybrid = if let Some(x) = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0) { - match self.read_compact_section(x as usize) { - Ok(Some((section, _, _))) => { - let Some(rows) = retained_rows.checked_add(section.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { - return Ok(None); - }; - let Some(bytes) = retained_bytes.checked_add(section.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES) - else { - return Ok(None); - }; - retained_rows = rows; - retained_bytes = bytes; - Some(section) - } - Ok(None) => return Ok(None), - Err(e) => { - log.push(format!("hybrid /XRefStm at {x} unreadable: {e}")); - None - } - } - } else { - None - }; - if let Some(hybrid) = &hybrid { - // A hybrid entry that disagrees with the table is set aside by the materialized reader. - let mut numbers = Vec::new(); - hybrid.extend_numbers(&mut numbers); - for num in numbers { - let Some(entry) = hybrid.lookup(num, &self.data) else { continue }; - if primary.lookup(num, &self.data).is_some_and(|old| !matches!(old, XrefEntry::Free { .. }) && old != entry) { - return Ok(None); - } - } - } - sections.push(XrefRevisionSection { primary, hybrid }); - next = prev_offset(data, §_trailer, off, |num| sections.iter().find_map(|s| s.lookup(num, data)), log); - if trailer.is_none() { - trailer = Some(sect_trailer); - } - } - revisions.reverse(); - let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?; - for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { - trailer.remove(k); - } - let mut max_num = 0; - let mut has_entries = false; - for revision in §ions { - max_num = max_num.max(revision.primary.max_num); - has_entries |= revision.primary.has_entries; - if let Some(hybrid) = &revision.hybrid { - max_num = max_num.max(hybrid.max_num); - has_entries |= hybrid.has_entries; - } - } - Ok(Some((XrefIndex::Compact { revisions: sections, max_num, has_entries }, trailer, revisions))) - } - - fn read_compact_section(&self, off: usize) -> Result, CosError> { - let data = &self.data; - let mut lx = Lexer::new(data, off); - if lx.eat_keyword(b"xref") { - let mut ranges = Vec::new(); - let mut layout_ok = true; - let mut max_num = 0; - let mut has_entries = false; - let mut previous_end = None; - let mut row_count = 0usize; - let mut renumbered = false; - loop { - lx.skip_ws(); - if lx.eat_keyword(b"trailer") { - let t = lx.object()?; - let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?; - if !layout_ok { - return Ok(None); - } - return Ok(Some((XrefSection { data: XrefSectionData::Table, ranges, row_count, max_num, has_entries, renumbered }, d, false))); - } - let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { - return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() }); - }; - if !valid_xref_range(start, count) || count > 10_000_000 { - return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() }); - } - let shifted = subsection_start(data, self.header_offset, lx.pos, start, count); - renumbered |= shifted != start; - let start = shifted; - let Some(next_rows) = usize::try_from(count).ok().and_then(|count| row_count.checked_add(count)) else { return Ok(None) }; - if next_rows > MAX_COMPACT_XREF_ROWS { - return Ok(None); - } - row_count = next_rows; - let mut range_offset = None; - let mut stride = None; - let mut previous_record = None; - for i in 0..count { - lx.skip_ws(); - let record = lx.pos; - if range_offset.is_none() { - range_offset = Some(record); - } - if let Some(previous) = previous_record { - let Some(delta) = record.checked_sub(previous) else { - layout_ok = false; - continue; - }; - if stride.is_some_and(|known| known != delta) { - layout_ok = false; - } - stride.get_or_insert(delta); - } - previous_record = Some(record); - let o = lx.token(); - lx.skip_ws(); - let _generation = lx.token(); - lx.skip_ws(); - let kind = lx.token(); - let num = (start + i) as u32; - let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::().ok()); - match (kind, offset) { - (b"n", Some(offset)) if offset != 0 => { - has_entries = true; - max_num = max_num.max(num); - } - (b"n", Some(_)) => { - // Set aside by the materialized reader, which records it. - if num != 0 { - return Ok(None); - } - } - (b"f", _) => { - has_entries = true; - max_num = max_num.max(num); - } - _ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }), - } - } - if count > 0 { - let range_start = start as u32; - let range_count = count as u32; - let end = u64::from(range_start) + u64::from(range_count); - if previous_end.is_some_and(|previous| u64::from(range_start) < previous) { - layout_ok = false; - } - previous_end = Some(end); - ranges.push(XrefRange { - start: range_start, - count: range_count, - offset: range_offset.unwrap_or(lx.pos), - stride: stride.unwrap_or(1), - }); - } - } - } - let (_, obj) = parse_indirect(data, off, &|_| None)?; - let Object::Stream(stream) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) }; - if stream.dict.name(b"Type") != Some(b"XRef") { - return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() }); - } - let w: Vec = stream - .dict - .get(b"W") - .and_then(Object::as_array) - .map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect()) - .unwrap_or_default(); - if w.len() < 3 { - return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() }); - } - let widths = [w[0], w[1], w[2]]; - let row = widths.iter().sum::(); - if row == 0 { - return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() }); - } - let size = stream.dict.int(b"Size").unwrap_or(0).max(0); - let index: Vec = match stream.dict.get(b"Index").and_then(Object::as_array) { - Some(a) => a.iter().filter_map(Object::as_int).collect(), - None => vec![0, size], - }; - let mut requested_rows = 0usize; - let mut previous_end = None; - for pair in index.chunks(2) { - let [start, count] = pair else { break }; - if !valid_xref_range(*start, *count) { - return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() }); - } - let Ok(count) = usize::try_from(*count) else { return Ok(None) }; - let Some(next) = requested_rows.checked_add(count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) }; - if count > 0 { - let end = (*start as u64).saturating_add(count as u64); - if previous_end.is_some_and(|previous| (*start as u64) < previous) { - return Ok(None); - } - previous_end = Some(end); - } - requested_rows = next; - } - if requested_rows.checked_mul(row).is_none_or(|bytes| bytes > MAX_COMPACT_XREF_BYTES) { - return Ok(None); - } - let raw = stream.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit).min(MAX_COMPACT_XREF_BYTES))?; - if raw.capacity() > MAX_COMPACT_XREF_BYTES { - return Ok(None); - } - let mut ranges = Vec::new(); - let mut row_start = 0usize; - let available_rows = raw.len() / row; - for pair in index.chunks(2) { - let [start, count] = pair else { break }; - let range_start = *start as u32; - let Ok(requested) = usize::try_from(*count) else { return Ok(None) }; - let range_count = requested.min(available_rows.saturating_sub(row_start)); - if range_count > 0 { - let Ok(range_count_u32) = u32::try_from(range_count) else { return Ok(None) }; - let Some(byte_offset) = row_start.checked_mul(row) else { return Ok(None) }; - ranges.push(XrefRange { start: range_start, count: range_count_u32, offset: byte_offset, stride: row }); - } - let Some(next) = row_start.checked_add(range_count) else { return Ok(None) }; - row_start = next; - } - // Rows the materialized reader sets aside (reserved types, offset 0) are left to it. - for range in &ranges { - for i in 0..range.count as usize { - let at = range.offset + i * row; - let Some(r) = raw.get(at..at + row) else { break }; - let field = |from: usize, width: usize, default: u64| -> u64 { - if width == 0 { - return default; - } - r[from..from + width].iter().fold(0u64, |acc, byte| acc << 8 | u64::from(*byte)) - }; - let kind = field(0, widths[0], 1); - let offset = field(widths[0], widths[1], 0); - if kind > 2 || (kind == 1 && offset == 0 && range.start + i as u32 != 0) { - return Ok(None); - } - } - } - let max_num = ranges.iter().filter_map(|range| range.start.checked_add(range.count.saturating_sub(1))).max().unwrap_or(0); - let row_count = row_start; - let section = XrefSection { - data: XrefSectionData::Stream { bytes: Arc::new(raw), widths }, - ranges, - row_count, - max_num, - has_entries: false, - renumbered: false, - }; - let has_entries = section.has_any_entry(); - let section = XrefSection { has_entries, ..section }; - Ok(Some((section, stream.dict.clone(), true))) - } - - /// Read the cross-reference chain from `startxref` into one map of entries, newer sections - /// winning, with the trailer, the revisions oldest first and the set-aside entries. A chain - /// that lists more than [`MAX_XREF_OBJECTS`] rows in all is refused with - /// [`CosError::Syntax`], and the caller rebuilds the file from object headers (#5082). - #[allow(clippy::type_complexity)] - pub(super) fn read_xref_chain_materialized( - &self, - log: &mut Vec, - ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { - self.read_xref_chain_materialized_with_limit(log, MAX_XREF_OBJECTS) - } - - /// [`Self::read_xref_chain_materialized`] refusing a chain past `row_limit` rows in all - /// instead of [`MAX_XREF_OBJECTS`], so tests can reach the limit with a few rows. - #[allow(clippy::type_complexity)] - pub(super) fn read_xref_chain_materialized_with_limit( - &self, - log: &mut Vec, - row_limit: usize, - ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { - let data = &self.data; - let tail_start = data.len().saturating_sub(4096); - let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?; - let mut lx = Lexer::new(data, sx + 9); - let first = match lx.object() { - Ok(Object::Int(n)) if n >= 0 => n as u64, - _ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }), - }; - let mut entries = BTreeMap::new(); - let mut trailer: Option = None; - let mut revisions = Vec::new(); - let mut oddities = Vec::new(); - let mut visited = HashSet::new(); - // Hybrid streams already read, by offset: their entries, or `None` when unreadable. A newer - // section has merged every entry of one into `entries`, so an older section that names it - // again only has its own table compared with them, without decoding the stream again. - // At most MAX_XREF_OBJECTS entries are kept in all; a stream past that is read again. - let mut hybrids: HashMap>> = HashMap::new(); - let (mut hybrid_rows, mut noted) = (0usize, HashSet::new()); - let mut generation_repairs = Vec::new(); - let mut recorded = HashSet::new(); - let mut diagnose = true; - // Every row the sections (and their hybrid streams) list, counted by `read_section` as it - // reads them, before any are merged or deduplicated. - let mut merged_rows = 0usize; - let mut next = Some(first); - while let Some(off) = next.take() { - if !visited.insert(off) || visited.len() > 10_000 { - log.push(format!("cross-reference chain loops at offset {off}; stopped")); - break; - } - // This section's own entries; older sections only fill what newer ones left out. - let mut sect = BTreeMap::new(); - // Object numbers whose entry in this section was set aside ([`Self::xref_oddities`]). - let mut odd = Vec::new(); - let rows_before = merged_rows; - let ((sect_trailer, is_stream), header_relative) = match self.read_section(off as usize, &mut sect, &mut odd, &mut merged_rows, log) { - Ok(v) => (v, false), - Err(e) if self.header_offset > 0 => { - log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header")); - sect.clear(); - odd.clear(); - merged_rows = rows_before; - (self.read_section(off as usize + self.header_offset, &mut sect, &mut odd, &mut merged_rows, log)?, true) - } - Err(e) => { - // An unreadable older section loses every object only it locates (the page - // tree, often): rebuild from object headers, as readers do (#1656). - if !revisions.is_empty() { - log.push(format!("older cross-reference section at {off} is unreadable: {e}")); - } - return Err(e); - } - }; - revisions.push(Revision { xref_offset: off, is_stream, header_relative }); - for (&num, &entry) in § { - if !diagnose { - break; - } - diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); - } - // Hybrid files (§7.5.8.4): the /XRefStm entries supplement the table and replace the - // free entries it gives objects only a cross-reference stream can locate. - let hybrid = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0); - if let Some((x, read)) = hybrid.and_then(|x| hybrids.get(&x).map(|read| (x, read))) { - if noted.insert(x) { - log.push(format!("hybrid /XRefStm at {x} named again by the section at {off}; read once")); - } - if let Some(stm) = read { - for (num, old) in § { - // The table places the object somewhere other than the stream does. - if !matches!(old, XrefEntry::Free { .. }) && stm.get(num).is_some_and(|e| e != old) { - odd.push(*num); - } - } - } - } else if let Some(x) = hybrid { - let mut stm = BTreeMap::new(); - let read = match self.read_section(x as usize, &mut stm, &mut odd, &mut merged_rows, log) { - Ok(_) => { - for (&num, &entry) in &stm { - if !diagnose { - break; - } - diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs); - } - for (&num, &e) in &stm { - match sect.get(&num) { - None | Some(XrefEntry::Free { .. }) => { - sect.insert(num, e); - } - // The table already places the object somewhere else. - Some(old) if *old != e => odd.push(num), - Some(_) => {} - } - } - Some(stm) - } - Err(e) => { - log.push(format!("hybrid /XRefStm at {x} unreadable: {e}")); - None - } - }; - let rows = read.as_ref().map_or(0, BTreeMap::len); - if let Some(total) = hybrid_rows.checked_add(rows).filter(|total| *total <= MAX_XREF_OBJECTS) { - hybrid_rows = total; - hybrids.insert(x, read); - } - } - // Each section's rows are capped, so the chain is too: at most `row_limit` rows - // (MAX_XREF_OBJECTS outside tests) read in all, or the file is rebuilt from object - // headers, as when one section's stream decodes past that. This counts every row - // read, before deduplication, not the map's size: a chain that relists the same - // objects, or a stream whose `/Index` repeats a range (`[0 4 0 4]`), costs as much to - // merge (#5082). Rows set aside are among the rows read, and a row is set aside at - // most twice (in its section and against a hybrid stream), so they stay bounded too. A - // section's rows (its hybrid stream's included) never outnumber the rows the compact - // reader counts for it, and that reader stops at about a quarter of MAX_XREF_OBJECTS: - // no chain within its limit is refused. - if merged_rows > row_limit { - return Err(CosError::Syntax { - offset: usize::try_from(off).unwrap_or(usize::MAX), - detail: format!("cross-reference chain lists more than {row_limit} entries"), - }); - } - oddities.extend(odd.into_iter().map(|num| (off, num))); - for (num, e) in sect { - entries.entry(num).or_insert(e); - } - next = prev_offset(data, §_trailer, off, |num| entries.get(&num).copied(), log); - if trailer.is_none() { - trailer = Some(sect_trailer); - } - } - revisions.reverse(); - let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?; - for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { - trailer.remove(k); - } - Ok((entries, trailer, revisions, oddities, generation_repairs)) - } - - /// Read one section at `off`, adding entries not already known (newer sections win). `odd` - /// collects the object numbers whose entry here is set aside ([`Self::xref_oddities`]), and - /// `counted` grows by every row read, repeated and set-aside rows included (#5082). - fn read_section( - &self, - off: usize, - entries: &mut BTreeMap, - odd: &mut Vec, - counted: &mut usize, - log: &mut Vec, - ) -> Result<(Dict, bool), CosError> { - #[cfg(test)] - SECTION_READS.with(|c| c.set(c.get() + 1)); - let data = &self.data; - let mut lx = Lexer::new(data, off); - if lx.eat_keyword(b"xref") { - loop { - lx.skip_ws(); - if lx.eat_keyword(b"trailer") { - let t = lx.object()?; - let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?; - return Ok((d, false)); - } - let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { - return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() }); - }; - if !valid_xref_range(start, count) || count > 10_000_000 { - return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() }); - } - let shifted = subsection_start(data, self.header_offset, lx.pos, start, count); - if shifted != start { - log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0")); - } - let start = shifted; - for i in 0..count { - *counted = counted.saturating_add(1); - lx.skip_ws(); - let o = lx.token(); - lx.skip_ws(); - let g = lx.token(); - lx.skip_ws(); - let kind = lx.token(); - let num = (start + i) as u32; - let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::().ok()); - let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0).min(u16::MAX as u32) as u16; - let entry = match (kind, offset) { - (b"n", Some(offset)) => XrefEntry::InFile { offset, generation }, - (b"f", _) => XrefEntry::Free { next_generation: generation }, - _ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }), - }; - // Offset 0 "in use" entries are a known producer bug: ignore them. (Object 0 - // is never a real object, so its entry cannot mislead anyone.) - if matches!(entry, XrefEntry::InFile { offset: 0, .. }) { - if num != 0 { - odd.push(num); - } - continue; - } - if *entries.entry(num).or_insert(entry) != entry { - odd.push(num); - } - } - } - } - // Cross-reference stream. - let (_, obj) = parse_indirect(data, off, &|_| None)?; - let Object::Stream(s) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) }; - if s.dict.name(b"Type") != Some(b"XRef") { - return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() }); - } - let w: Vec = s - .dict - .get(b"W") - .and_then(Object::as_array) - .map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect()) - .unwrap_or_default(); - if w.len() < 3 { - return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() }); - } - let size = s.dict.int(b"Size").unwrap_or(0).max(0); - let index: Vec = match s.dict.get(b"Index").and_then(Object::as_array) { - Some(a) => a.iter().filter_map(Object::as_int).collect(), - None => vec![0, size], - }; - let row = w[0] + w[1] + w[2]; - if row == 0 { - return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() }); - } - // One row per object, and a document has at most 8,388,607 indirect objects (the classic - // implementation limit): decoding more than that (about 200 MB at the widest rows) is a - // decompression bomb, not cross-reference data. The section is then reconstructed. - let raw = s.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit))?; - let field = |r: &[u8], from: usize, len: usize, default: u64| -> u64 { - if len == 0 { - return default; - } - r[from..from + len].iter().fold(0u64, |acc, b| acc << 8 | *b as u64) - }; - let mut rows = raw.chunks_exact(row); - for pair in index.chunks(2) { - let [start, count] = pair else { break }; - if !valid_xref_range(*start, *count) { - return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() }); - } - for i in 0..*count { - let Some(r) = rows.next() else { break }; - *counted = counted.saturating_add(1); - let t = field(r, 0, w[0], 1); - let a = field(r, w[0], w[1], 0); - let b = field(r, w[0] + w[1], w[2], 0); - let num = (*start + i) as u32; - let entry = match t { - 0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 }, - 1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 }, - 2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 }, - // Reserved types are treated as null references (§7.5.8.3). - _ => { - odd.push(num); - continue; - } - }; - if matches!(entry, XrefEntry::InFile { offset: 0, .. }) { - if num != 0 { - odd.push(num); - } - continue; - } - if *entries.entry(num).or_insert(entry) != entry { - odd.push(num); - } - } - } - Ok((s.dict.clone(), true)) - } - - /// Cross-reference data that leaves out objects the file contains: a subsection numbered one - /// too low (object 0 in use, the last object without an entry), or an in-use entry at offset 0 - /// that nothing else replaces. (A number no section lists is not evidence: it is a null - /// object, §7.3.10, and valid files have them below an oversized `/Size`.) Viewers rebuild - /// the table and show these objects; reading them as null would drop them from every copy. - /// When one of those shows, the file is scanned once for object headers, and wrong offsets - /// are looked up there (`load`). Only the numbers the damage leaves out take the last - /// header's location: one whose entry was set aside, or the last object of a subsection - /// numbered one too low. Any other unlisted number stays null, and existing - /// entries are never replaced: an object freed by an update stays freed. - pub(super) fn fill_xref_gaps(&mut self, log: &mut Vec) { - let entries = &self.entries; - let shifted = matches!(entries.lookup(0, &self.data), Some(XrefEntry::InFile { .. })); - let set_aside = self.xref_oddities.iter().any(|(_, num)| entries.lookup(*num, &self.data).is_none()); - if !(shifted || set_aside) { - return; - } - let mut located = BTreeMap::new(); - for (num, offset, generation, _) in object_headers(&self.data) { - located.insert(num, (offset, generation)); - } - let mut filled = entries.to_map(&self.data); - // A subsection `0 N` whose rows are those of objects 1 to N leaves out object N, whatever - // newer sections say about its neighbours. - let past_shift: HashSet = if shifted { - self.revisions - .iter() - .filter(|r| !r.is_stream) - .filter_map(|r| usize::try_from(r.xref_offset).ok().map(|o| if r.header_relative { o.saturating_add(self.header_offset) } else { o })) - .filter_map(|at| shifted_subsection_end(&self.data, at)) - .collect() - } else { - HashSet::new() - }; - let odd: HashSet = self.xref_oddities.iter().map(|(_, n)| *n).collect(); - let mut found = 0usize; - for (&num, &(offset, generation)) in &located { - let left_out = past_shift.contains(&num) || odd.contains(&num); - if num != 0 && !filled.contains_key(&num) && left_out { - filled.insert(num, XrefEntry::InFile { offset, generation }); - found += 1; - } - } - if found > 0 { - log.push(format!("the cross-reference data leaves out {found} objects the file contains; located them by their object headers")); - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(filled), Arc::default())); - } - self.located = Arc::new(located); - } - - /// Rebuild the cross-reference index by scanning for `N G obj` headers (§7.5.4 recovery). - pub(super) fn reconstruct(&mut self, log: &mut Vec) -> Result<(), CosError> { - let data = self.data.clone(); - let mut entries: BTreeMap = BTreeMap::new(); - let mut objstms = Vec::new(); - // Offsets of objects that may be cross-reference streams: their dictionaries are the - // trailers of files that have no `trailer` keyword (§7.5.8). - let mut xref_streams = Vec::new(); - let headers = object_headers(&data); - // Where `/XRef` occurs in the file, found in one pass. - let mut xref_marks = Vec::new(); - let mut mark_from = 0; - while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { - xref_marks.push(p); - mark_from = p + 1; - } - for &(num, offset, generation, body) in &headers { - entries.insert(num, XrefEntry::InFile { offset, generation }); - objstms.push(num); - // `/XRef` within the 1024 bytes after the header (the first one that is there). - let first = xref_marks.partition_point(|&p| p < body); - if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) - && let Ok(at) = usize::try_from(offset) - { - xref_streams.push(at); - } - } - if entries.is_empty() { - return Err(CosError::Syntax { offset: 0, detail: "no objects found".into() }); - } - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()), Arc::default())); - self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); - *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); - // Objects inside object streams. - for num in objstms { - if let Ok(o) = self.try_get(num) - && let Object::Stream(s) = &*o - && s.dict.name(b"Type") == Some(b"ObjStm") - && let Ok(stm) = self.objstm(num) - { - for (idx, member) in stm.index.iter().enumerate() { - if let Some((inner, _)) = member { - entries.entry(*inner).or_insert(XrefEntry::InStream { stream: num, index: idx as u32 }); - } - } - } - } - let object_count = entries.len(); - self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries), Arc::default())); - // Failures from the scan stand: the members added change no scanned entry, and a - // `/Length` found only now cannot make a stream unreadable. - self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); - // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a - // usable /Root (last in the file, whichever kind), else the catalog itself. One whose - // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another - // dictionary, such as /Info, wherever it is; an FDF's /Root (no /Type, no /Pages) still - // counts when nothing better is there. - // A file whose only trailers are cross-reference streams (and whose `startxref` is gone - // or points nowhere) keeps its /Info, /ID and /Encrypt this way. - let rank = |at: usize, d: &Dict| { - let root = self.try_get_reference(d.reference(b"Root")?).ok()?; - let Object::Dict(c) = &*root else { return None }; - Some((c.name(b"Type") == Some(b"Catalog") || c.contains(b"Pages"), at)) - }; - let mut trailer: Option<((bool, usize), Dict)> = None; - let mut from = 0; - while let Some(p) = find(&data, b"trailer", from, data.len()) { - from = p + 7; - let mut lx = Lexer::new(&data, from); - if let Ok(Object::Dict(d)) = lx.object() - && let Some(r) = rank(p, &d) - && trailer.as_ref().is_none_or(|(best, _)| r > *best) - { - trailer = Some((r, d)); - } - } - for at in xref_streams { - if trailer.as_ref().is_some_and(|((catalog, p), _)| *catalog && *p > at) { - continue; - } - if let Ok((_, Object::Stream(s))) = parse_indirect_shared(&data, at, &|_| None, &self.endstreams) - && s.dict.name(b"Type") == Some(b"XRef") - && let Some(r) = rank(at, &s.dict) - && trailer.as_ref().is_none_or(|(best, _)| r > *best) - { - trailer = Some((r, s.dict)); - } - } - let trailer = match trailer { - Some((_, t)) => t, - None => { - let catalog = self - .entries - .object_numbers(&self.data) - .into_iter() - .find(|n| matches!(self.try_get(*n).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog"))); - let Some(c) = catalog else { return Err(CosError::Syntax { offset: 0, detail: "no document catalog found".into() }) }; - let mut t = Dict::new(); - t.set(b"Root".to_vec(), Object::Ref(ObjRef::new(c, self.generation(c)))); - t - } - }; - let mut trailer = trailer; - // What a cross-reference stream's dictionary carries for itself goes, as the chain - // readers drop it: a save writes its own. - for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] { - trailer.remove(k); - } - self.trailer = trailer; - self.revisions = Arc::new(Vec::new()); - log.push(format!("rebuilt cross-reference index from {object_count} object headers")); - Ok(()) - } -} - -/// For a classic table at `at` whose first subsection is `0 N` and opens with an in-use row -/// (the rows of objects 1 to N), the number N its rows really end at. -fn shifted_subsection_end(data: &[u8], at: usize) -> Option { - let mut lx = Lexer::new(data, at); - if !lx.eat_keyword(b"xref") { - return None; - } - let (Ok(Object::Int(0)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { return None }; - lx.skip_ws(); - let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); - lx.skip_ws(); - lx.token(); - lx.skip_ws(); - match (offset, lx.token()) { - (Some(offset), b"n") if offset != 0 => u32::try_from(count).ok().filter(|n| *n > 0), - _ => None, - } -} - -/// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position -/// after `obj`). -fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { - let mut headers = Vec::new(); - let mut i = 0; - while i < data.len() { - // Object headers start at a line start (or after whitespace). - let at_start = i == 0 || is_whitespace(data[i - 1]) || data[i - 1] == b'>'; - if at_start && data[i].is_ascii_digit() { - let mut lx = Lexer::new(data, i); - let n = lx.token(); - lx.skip_ws(); - let g = lx.token(); - if !n.is_empty() && !g.is_empty() && n.iter().all(u8::is_ascii_digit) && g.iter().all(u8::is_ascii_digit) && lx.eat_keyword(b"obj") { - let num = std::str::from_utf8(n).ok().and_then(|s| s.parse::().ok()); - let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0); - if let Some(num) = num { - headers.push((num, i as u64, generation, lx.pos)); - } - i = lx.pos; - continue; - } - } - i += 1; - } - headers -} - -/// The offset of the section before the one whose trailer this is (`/Prev`), if any. Some writers -/// give it as a reference, `N 0 R`. When the sections read so far (`lookup`) place that object -/// and it holds an offset (iText's `indirectPrev.pdf`), that is the offset; otherwise, as the -/// tiff2pdf files in JHOVE's PDF-HUL-157 cases need, the sections before are at N. Either way the -/// reference is noted. Any other value that is not an offset is noted and ends the chain. -fn prev_offset(data: &[u8], trailer: &Dict, off: u64, lookup: impl Fn(u32) -> Option, log: &mut Vec) -> Option { - match trailer.get(b"Prev") { - Some(Object::Int(p)) if *p >= 0 => Some(*p as u64), - Some(Object::Ref(r)) => { - let held = match lookup(r.num) { - Some(XrefEntry::InFile { offset, generation }) if generation == r.generation => { - usize::try_from(offset).ok().and_then(|at| parse_indirect(data, at, &|_| None).ok()).and_then(|(at, value)| match value { - Object::Int(p) if at == *r && p >= 0 => Some(p as u64), - _ => None, - }) - } - _ => None, - }; - match held { - Some(p) => { - log.push(format!( - "the trailer at {off} gives /Prev as the reference {} {} R; reading the offset {p} it holds", - r.num, r.generation - )); - Some(p) - } - None => { - log.push(format!( - "the trailer at {off} gives /Prev as the reference {} {} R; reading it as offset {}", - r.num, r.generation, r.num - )); - Some(u64::from(r.num)) - } - } - } - Some(_) => { - log.push(format!("the trailer at {off} has a /Prev that is not an offset; the sections before it are unreachable")); - None - } - None => None, - } -} - -pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option { - let to = to.min(hay.len()); - if from >= to || needle.is_empty() || to - from < needle.len() { - return None; - } - let hay = &hay[from..to]; - // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. - const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; - let (k, rare) = - needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); - let mut i = k; - while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { - let start = i + p - k; - if hay.get(start..start + needle.len()) == Some(needle) { - return Some(start + from); - } - i += p + 1; - } - None -} - -/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which -/// the compiler turns into vector compares; only a block that holds the byte is searched. -fn position_of(hay: &[u8], byte: u8) -> Option { - const BLOCK: usize = 32; - let (blocks, rest) = hay.as_chunks::(); - for (i, block) in blocks.iter().enumerate() { - if block.iter().fold(false, |any, &b| any | (b == byte)) { - return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); - } - } - rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) -} - -pub(super) fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { - if hay.len() < needle.len() { - return None; - } - (from..=hay.len() - needle.len()).rev().find(|&i| &hay[i..i + needle.len()] == needle) -} - -/// The object number a classic xref subsection really starts at. A common producer bug heads the -/// table `1 n` and then writes object 0's free entry (`0000000000 65535 f`) as its first row, so -/// every row would be read one number too high and object 1 as free. Such a subsection is read -/// from 0 when its second row, read that way, locates an object that is indeed numbered 1, at -/// that offset or, as `load` reads offsets, relative to a header shifted by `header_offset`. -/// `rows` is the position just after the subsection header. -fn subsection_start(data: &[u8], header_offset: usize, rows: usize, start: i64, count: i64) -> i64 { - if start != 1 || count < 2 { - return start; - } - let mut lx = Lexer::new(data, rows); - let mut row = || { - lx.skip_ws(); - let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); - lx.skip_ws(); - let generation = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::().ok()); - lx.skip_ws(); - (offset, generation, lx.token() == b"f") - }; - let (Some(0), Some(65535), true) = row() else { return start }; - let (Some(second), _, false) = row() else { return start }; - let object_one = |at: usize| { - if at >= data.len() { - return false; - } - let mut obj = Lexer::new(data, at); - matches!((obj.object(), obj.object()), (Ok(Object::Int(1)), Ok(Object::Int(_)))) && obj.eat_keyword(b"obj") - }; - let Ok(second) = usize::try_from(second) else { return start }; - if object_one(second) || (header_offset != 0 && object_one(second.saturating_add(header_offset))) { 0 } else { start } -} - -/// Object numbers must fit the representation used by references and the xref map. -/// Validate the whole subsection before adding entries, so neither arithmetic overflow nor -/// truncation can turn a damaged range into entries for unrelated objects. -fn valid_xref_range(start: i64, count: i64) -> bool { - u32::try_from(start).is_ok() && count >= 0 && start.checked_add(count).is_some_and(|end| end <= i64::from(u32::MAX) + 1) -} - -/// The most indirect objects a document can have (the classic implementation limit, ISO -/// 32000-1 Annex C): it bounds how much data a cross-reference stream can hold. -const MAX_XREF_OBJECTS: usize = 8_388_607; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs deleted file mode 100644 index 7359ce715a..0000000000 --- a/crates/cos/src/document/state.rs +++ /dev/null @@ -1,504 +0,0 @@ -use super::xref::XrefEntry; -use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, ObjRef, Object, Weak}; - -thread_local! { - /// The objects being loaded on this thread, innermost last (see `Document::try_get`). - static LOADING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; - /// Indirect `/Length` values resolved during the current outermost load: a damaged file - /// retries a parse several ways, and each retry would resolve the same lengths again - /// (exponential in the chain of streams whose lengths point at further streams). Each is - /// kept with whether resolving it read unsaved edits. - pub(super) static LENGTHS: std::cell::RefCell> = std::cell::RefCell::new(HashMap::new()); - /// What the loads running on this thread have read ([`ReadFrame`]): how many frames are - /// open, the object numbers read inside them, and how often that list was full. - static READS: std::cell::RefCell<(u32, Vec, u64)> = const { std::cell::RefCell::new((0, Vec::new(), 0)) }; - /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). - static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; - /// How deep `Document::scan_for` is nested on this thread, and how many indirect `/Length` - /// values the outermost scan and those inside it have resolved ([`ScanLengths`]). - static SCAN_LENGTHS: std::cell::Cell<(u32, u32)> = const { std::cell::Cell::new((0, 0)) }; -} - -/// The indirect `/Length` values a scan of the file for an object may resolve, shared with the -/// scans it leads to. Each header in the file that starts with the object's number is parsed, and -/// a `/Length` that names an object the cross-reference table cannot place scans the file for that -/// one: a damaged file with many such headers, each naming another missing object, would read the -/// whole file once per header. Past the budget a `/Length` is left unresolved, and the stream is -/// read as before resolution, by its `endstream`. -pub(super) struct ScanLengths; - -impl ScanLengths { - /// A real object is defined once per revision, and its `/Length` once per definition. - pub(super) const MAX: u32 = 32; - - /// Marks a scan as running; the outermost one starts the budget afresh. - pub(super) fn enter() -> Self { - SCAN_LENGTHS.with(|c| { - let (depth, used) = c.get(); - c.set((depth.saturating_add(1), if depth == 0 { 0 } else { used })); - }); - ScanLengths - } - - /// Whether one more `/Length` may be resolved, counting it if so. - pub(super) fn take(&self) -> bool { - SCAN_LENGTHS.with(|c| { - let (depth, used) = c.get(); - if used >= Self::MAX { - return false; - } - c.set((depth, used.saturating_add(1))); - true - }) - } -} - -impl Drop for ScanLengths { - fn drop(&mut self) { - SCAN_LENGTHS.with(|c| { - let (depth, used) = c.get(); - c.set((depth.saturating_sub(1), used)); - }); - } -} - -/// A resolved `/Length` in [`LENGTHS`]: its value, whether resolving it read unsaved edits, and -/// what it read. -pub(super) type Length = (Option, bool, Reads); - -/// The other objects a cached load read: an object stream for a member, an indirect `/Length`, -/// and what those read in turn. Clones share the caches, so a document whose edits include one of -/// them reads the object again instead (`Document::try_get`); `None` is a load that read more -/// than [`Reads::MAX`], which such a document always reads again. -#[derive(Clone, Debug, Default)] -pub(super) struct Reads(Option>); - -impl Reads { - /// Objects one entry keeps: a member's stream and a length or two, and what they read. - const MAX: usize = 64; - /// Object numbers the open frames keep between them. - const HELD: usize = 4096; - - /// Whether `overlay` (a document's edits) holds any of these objects. - pub(super) fn touch(&self, overlay: &BTreeMap) -> bool { - !overlay.is_empty() && self.0.as_ref().is_none_or(|nums| nums.iter().any(|n| overlay.contains_key(n))) - } - - fn bytes(&self) -> usize { - self.0.as_ref().map_or(0, |nums| nums.len().saturating_mul(std::mem::size_of::())) - } -} - -/// Counts `num` as read by the loads open on this thread, with what loading it read (`reads`). -pub(super) fn note_read(num: u32, reads: Option<&Reads>) { - READS.with(|r| { - let (depth, held, full) = &mut *r.borrow_mut(); - if *depth == 0 { - return; - } - let more = reads.map_or(Some(&[][..]), |r| r.0.as_deref()); - match more { - Some(more) if held.len().saturating_add(more.len()) < Reads::HELD => { - held.push(num); - held.extend_from_slice(more); - } - _ => *full = full.wrapping_add(1), - } - }); -} - -/// Collects what one load reads, for [`Reads`]; frames nest, and each read counts for every -/// frame open around it. -pub(super) struct ReadFrame { - start: usize, - full: u64, -} - -impl ReadFrame { - pub(super) fn enter() -> Self { - READS.with(|r| { - let (depth, held, full) = &mut *r.borrow_mut(); - *depth = depth.saturating_add(1); - ReadFrame { start: held.len(), full: *full } - }) - } - - /// What was read since the frame opened. - pub(super) fn reads(&self) -> Reads { - READS.with(|r| { - let (_, held, full) = &*r.borrow(); - if *full != self.full { - return Reads(None); - } - let mut nums = held.get(self.start..).unwrap_or_default().to_vec(); - nums.sort_unstable(); - nums.dedup(); - Reads((nums.len() <= Reads::MAX).then(|| Arc::from(nums))) - }) - } -} - -impl Drop for ReadFrame { - fn drop(&mut self) { - READS.with(|r| { - let (depth, held, _) = &mut *r.borrow_mut(); - *depth = depth.saturating_sub(1); - if *depth == 0 { - held.clear(); - } - }); - } -} - -/// Unsaved edits read so far on this thread: a load during which this changes depended on one. -pub(super) fn overlay_reads() -> u64 { - OVERLAY_READS.with(std::cell::Cell::get) -} - -pub(super) fn note_overlay_read() { - OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1))); -} - -/// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream -/// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is -/// bounded as well. -pub(super) struct LoadGuard; - -impl LoadGuard { - const MAX: usize = 32; - - pub(super) fn enter(num: u32) -> Option { - LOADING.with(|l| { - let mut l = l.borrow_mut(); - if l.len() >= Self::MAX || l.contains(&num) { - return None; - } - l.push(num); - Some(LoadGuard) - }) - } -} - -/// Whether an object is being loaded on this thread: [`LENGTHS`] holds only while one is, since -/// it is cleared when the outermost load ends. -pub(super) fn loading() -> bool { - LOADING.with(|l| !l.borrow().is_empty()) -} - -/// What loading a file object gave, kept in the document's cache either way. -pub(super) type Loaded = Result, CosError>; - -/// The bytes a load result holds of its own ([`held_bytes`]); a failure holds none. -pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc>) -> usize { - loaded.as_ref().map_or(0, |o| held_bytes(o, data, 0)) -} - -impl Drop for LoadGuard { - fn drop(&mut self) { - let outermost = LOADING.with(|l| { - let mut l = l.borrow_mut(); - l.pop(); - l.is_empty() - }); - if outermost { - LENGTHS.with(|m| m.borrow_mut().clear()); - } - } -} - -#[derive(Clone, Debug)] -pub(super) struct ObjStm { - /// One slot per member in stream order, since cross-reference entries name a member by its - /// position: (object number, byte offset within `data`), or `None` for a member whose number - /// or offset is unusable ([`objstm_member`]). - pub(super) index: Vec>, - pub(super) data: Arc>, - /// What parsing it read: the stream itself and its `/Length`, `/DecodeParms` and so on. - pub(super) reads: Reads, -} - -/// Where member `num` of an object stream starts: its `offset` past `/First` (`first`), when the -/// number is an object number and the start lies inside the decoded `len` bytes. Checked on every -/// target, so a 32-bit build neither truncates nor wraps a large `/First` or offset (#1815). -pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: usize) -> Option<(u32, usize)> { - let num = u32::try_from(num).ok()?; - let start = first?.checked_add(usize::try_from(offset).ok()?)?; - (start < len).then_some((num, start)) -} - -/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). -/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that -/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole -/// list holds at most `MAX` plus the objects edited in clones. -#[derive(Debug, Default)] -pub(super) struct Unreadable { - pub(super) objects: BTreeMap, - pub(super) more: bool, -} - -impl Unreadable { - pub(super) const MAX: usize = 100; - /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. - const REASON: usize = 160; - - /// Whether `num` is to be recorded: not listed yet and there is room among the entries the - /// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]), - /// so a clone's edits never use up its own allowance. A full list notes that there were more, - /// so a caller asks this before building the reason. - pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool { - if self.objects.contains_key(&num) { - return false; - } - if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX { - self.more = true; - return false; - } - true - } - - /// Record `num` once; the first reason given is kept. - pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) { - if self.wants(num, hidden) { - self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); - } - } - - /// One line per recorded object not `hidden`, then one saying there were more, if there were. - pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { - // Clones share the list and each hides its own edits, so the list may hold more than - // `MAX` entries in all; each document still shows at most `MAX`. - let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num)); - let mut lines: Vec = shown - .by_ref() - .take(Self::MAX) - .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) - .collect(); - if self.more || shown.next().is_some() { - lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); - } - lines - } -} - -/// Where an overlay slot came from. -#[derive(Clone, Debug, PartialEq)] -pub(super) enum Slot { - Set(u16, Arc), - /// Both generations accepted from a damaged live source slot survive a repair-aware edit. - /// A full rewrite is required: a new xref slot cannot encode two generations. - RepairedSet(u16, u16, Arc), - Freed(u16), -} - -/// An immutable input/decode context shared by document clones. Holding this identity -/// retains neither the input bytes nor the parsed document. Weak ownership prevents an -/// expired identity from matching a later allocation at the same address. -#[derive(Clone, Debug)] -pub struct SourceIdentity(pub(super) Weak<()>); - -impl PartialEq for SourceIdentity { - fn eq(&self, other: &Self) -> bool { - self.0.ptr_eq(&other.0) - } -} - -impl Eq for SourceIdentity {} - -/// What a document reads as: its source ([`SourceIdentity`]), its trailer and every object changed -/// since it was opened. A clone, or a snapshot that undo restores, has the same state, and every -/// change makes a different one. Changed objects are compared as allocations, which the state -/// keeps alive, so a later object can't take an address one of them had. -#[derive(Clone)] -pub struct ReadState { - pub(super) source: SourceIdentity, - pub(super) trailer: Dict, - pub(super) overlay: BTreeMap, -} - -impl ReadState { - /// Whether `doc` reads as this state. - pub fn matches(&self, doc: &Document) -> bool { - self.source == doc.source_identity() - && self.trailer == doc.trailer - && self.overlay.len() == doc.overlay.len() - && self.overlay.iter().zip(&doc.overlay).all(|((a, x), (b, y))| { - a == b - && match (x, y) { - (Slot::Set(g, o), Slot::Set(h, p)) => g == h && Arc::ptr_eq(o, p), - (Slot::RepairedSet(g, a, o), Slot::RepairedSet(h, b, p)) => g == h && a == b && Arc::ptr_eq(o, p), - (Slot::Freed(g), Slot::Freed(h)) => g == h, - _ => false, - } - }) - } -} - -impl std::fmt::Debug for ReadState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ReadState").field("changed_objects", &self.overlay.len()).finish_non_exhaustive() - } -} - -/// A temporary reader for whole-document work. Its caches never populate the editor or -/// undo snapshots. Returned objects survive cache eviction; a stream read from the file is a -/// view into the input bytes (it keeps the whole input alive while held, not a copy). -pub(crate) struct ObjectReader { - pub(crate) document: Document, - pub(super) decoded_count: usize, -} - -impl ObjectReader { - pub(super) const OBJECTS: usize = 128; - pub(super) const STREAM_BYTES: usize = 8 * 1024 * 1024; - - pub(crate) fn get(&mut self, reference: ObjRef) -> Arc { - self.try_get(reference).unwrap_or_else(|_| Arc::new(Object::Null)) - } - - pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { - let object = self.document.try_get_reference(reference); - // A missed eviction here only costs a re-parse, so never fail the read on a poisoned - // lock. The editing-cache evictors propagate instead; they guard user edits. - if let Ok(mut streams) = self.document.objstms.lock() { - // Only insertion changes this private cache between reads. Most neighboring - // objects reuse a decoded stream: do not sum the whole store for every object. - if streams.len() != self.decoded_count { - let bytes = streams.values().fold(0usize, |bytes, stream| { - bytes - .saturating_add(stream.data.capacity()) - .saturating_add(stream.index.capacity().saturating_mul(std::mem::size_of::>())) - }); - if bytes > Self::STREAM_BYTES || streams.len() >= Self::OBJECTS { - // Retain one oversized active stream so its next object does not require - // decoding it again. That stream still obeys the document's decode cap. - let current = match self.document.xref_entry(reference.num) { - Some(XrefEntry::InStream { stream, .. }) => Some(stream), - _ => None, - }; - streams.retain(|num, _| Some(*num) == current); - } - self.decoded_count = streams.len(); - } - } - // Parsed objects stay cached (an indirect /Length or a dictionary read again costs a - // lookup, not a parse) until the cache passes its object count or holds more than - // STREAM_BYTES of data of its own. Streams read from the file are views into the - // input bytes, which the document keeps anyway, so they don't count; decrypted - // streams and strings do. Checked after failed loads too: resolving a /Length may - // have cached large objects. - if let Ok(mut cache) = self.document.cache.lock() - && (cache.len() >= Self::OBJECTS || cache.owned() > Self::STREAM_BYTES) - { - cache.clear(); - } - object - } -} - -#[cfg(test)] -thread_local! { - /// Cross-reference sections read on this thread. - pub(super) static SECTION_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// What loading each object gave (a failure as well), by number, each with what it holds ([`held_bytes`]) measured once, when it goes -/// in, and their sum kept beside them. -/// -/// Both caches are bounded by those bytes. Summing them on every read walked every cached object -/// whole each time: a file whose page dictionaries carry kilobytes of private data (InDesign's -/// `/PieceInfo`) took seconds to open, almost all of it re-measuring objects already measured -/// (#5873). -#[derive(Default)] -pub(crate) struct ObjectCache { - entries: HashMap, - owned: usize, -} - -impl ObjectCache { - /// What loading `num` gave, and what that load read. - pub(super) fn get(&self, num: &u32) -> Option<(&Loaded, &Reads)> { - self.entries.get(num).map(|(object, _, reads)| (object, reads)) - } - - #[cfg(test)] - pub(crate) fn contains_key(&self, num: &u32) -> bool { - self.entries.contains_key(num) - } - - /// Keep what loading `num` gave, which holds `owned` bytes ([`loaded_bytes`]), and what it read. - pub(super) fn insert(&mut self, num: u32, object: Loaded, owned: usize, reads: Reads) { - let owned = owned.saturating_add(reads.bytes()); - if let Some((_, before, _)) = self.entries.insert(num, (object, owned, reads)) { - self.owned = self.owned.saturating_sub(before); - } - self.owned = self.owned.saturating_add(owned); - } - - pub(crate) fn len(&self) -> usize { - self.entries.len() - } - - #[cfg(test)] - pub(crate) fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - /// The bytes the cached objects hold between them. - pub(crate) fn owned(&self) -> usize { - self.owned - } - - pub(crate) fn clear(&mut self) { - self.entries.clear(); - self.owned = 0; - } - - pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) { - self.entries.retain(|num, _| keep(num)); - self.owned = self.entries.values().fold(0usize, |n, (_, owned, _)| n.saturating_add(*owned)); - } - - #[cfg(test)] - pub(crate) fn values(&self) -> impl Iterator> { - self.entries.values().filter_map(|(object, _, _)| object.as_ref().ok()) - } -} - -/// Nesting [`held_bytes`] walks: past the parser's own nesting limit, so every parsed object -/// is measured in full. -const OWNED_BYTES_DEPTH: usize = 256; - -/// What keeping `object` parsed costs: string and stream data that is not a view into `data` -/// (the input file), and the parsed tree around it -- an [`Object`] per value and a key per -/// dictionary entry. An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured and -/// counts as too large to keep. -/// -/// The tree counts. Measured by its strings and streams alone, a page dictionary carrying -/// InDesign's `/PieceInfo` -- thousands of small numeric arrays, parsed into tens of thousands of -/// values -- weighed nothing, so the caches kept a thousand of them: a 35 MB file held 1.5 GB -/// (#5873). -pub(crate) fn held_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { - if depth > OWNED_BYTES_DEPTH { - return usize::MAX; - } - let node = std::mem::size_of::(); - // A key, its slot beside the value, and room in the index a large dictionary keeps. - let entry = |key: &[u8]| key.len().saturating_add(std::mem::size_of::<(crate::Name, Object)>()).saturating_add(16); - let dict = - |d: &crate::Dict, start: usize| d.iter().fold(start, |n, (k, v)| n.saturating_add(entry(k)).saturating_add(held_bytes(v, data, depth + 1))); - match object { - Object::String(s) => node.saturating_add(s.bytes.len()), - Object::Name(n) => node.saturating_add(n.len()), - Object::Stream(s) => dict(&s.dict, node.saturating_add(if s.raw.shares(data) { 0 } else { s.raw.len() })), - Object::Array(a) => a.iter().fold(node, |n, v| n.saturating_add(held_bytes(v, data, depth + 1))), - Object::Dict(d) => dict(d, node), - _ => node, - } -} - -/// How an encrypted document is opened: with a password, or with the file key of a document -/// that has the same encryption (an earlier revision of the same file). -pub(super) enum Credentials<'a> { - Password(Option<&'a str>), - Handler { handler: &'a Arc, id0: Vec }, -} diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs deleted file mode 100644 index 814d4ef8eb..0000000000 --- a/crates/cos/src/document/tests.rs +++ /dev/null @@ -1,2709 +0,0 @@ -use super::*; - -/// What [`crate::reachable_objects`] gives is what a full save keeps: not an object nothing refers to, nor one -/// a reference only dangles at, and a ring of references ends. -#[test] -fn what_is_reachable_is_what_a_full_save_keeps() { - let mut doc = Document::new_empty(); - let root = doc.root().unwrap(); - let kept = doc.add(Object::Int(7)); - let dangling = ObjRef::new(4_000, 0); - let list = doc.add(Object::Null); - let ring = doc.add(Object::Array(vec![Object::Ref(list)])); - doc.set(list, Object::Array(vec![Object::Ref(kept), Object::Ref(ring), Object::Ref(dangling)])); - let orphan = doc.add(Object::Int(8)); - doc.update_dict(root, |d| d.set(b"Held".to_vec(), Object::Ref(list))).unwrap(); - let found = crate::reachable_objects(&doc).unwrap(); - assert!([root, kept, list, ring].iter().all(|r| found.contains(&r.num))); - assert!(!found.contains(&orphan.num) && !found.contains(&dangling.num)); - let saved = Document::open(Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap())).unwrap(); - let holds = |n: i64| saved.scan_objects().any(|(_, o)| matches!(*o, Object::Int(v) if v == n)); - assert!(holds(7) && !holds(8)); -} - -std::thread_local! { - /// How many times `Document::scan_for` read the file on this thread. - pub(super) static FILE_SCANS: std::cell::Cell = const { std::cell::Cell::new(0) }; - /// How many times `Document::objstm` decoded an object stream on this thread. - pub(super) static OBJSTM_DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// A scan that doesn't find an object isn't made again: each reads the whole file, and a file can -/// refer to such an object from every object it has. -#[test] -fn what_a_scan_of_the_file_did_not_find_is_remembered() { - let body = b"%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n"; - let mut out = body.to_vec(); - let xref = out.len(); - out.extend_from_slice(b"xref\n0 4\n0000000000 65535 f \n"); - out.extend_from_slice(format!("{:010} 00000 n \n", 9).as_bytes()); - out.extend_from_slice(format!("{:010} 00000 n \n", body.iter().position(|b| *b == b'\n').unwrap() + 49).as_bytes()); - out.extend_from_slice(b"0000000001 00000 n \n"); - out.extend_from_slice(format!("trailer << /Size 4 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); - let doc = Document::open(Arc::new(out)).unwrap(); - FILE_SCANS.with(|scans| scans.set(0)); - for _ in 0..5 { - assert!(doc.try_get(3).is_err()); - } - assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1); - let clone = doc.clone(); - assert!(clone.try_get(3).is_err()); - assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "a clone has the same file"); -} - -/// An object found by a scan of the file, because its cross-reference offset is wrong, has its -/// indirect `/Length` resolved as at the right offset. Unresolved, a stream ending in a misspelt -/// `endstream` (`endstrea`) was searched on into a later object's `endstream`, and took that -/// object's header and data as its own. -#[test] -fn a_scanned_stream_resolves_its_indirect_length() { - let mut out = build( - &[ - "<< /Type /Catalog /Pages 2 0 R >>", - "<< /Type /Pages /Kids [] /Count 0 >>", - "<< /Length 4 0 R >>\nstream\nHELLO\nendstrea", - "5", - "<< /Length 5 >>\nstream\nWORLD\nendstream", - ], - "/Root 1 0 R", - ); - // Point object 3's entry at the catalog, so only the scan finds it. - let at = |needle: &[u8], out: &[u8]| out.windows(needle.len()).position(|w| w == needle).unwrap(); - let catalog = format!("{:010} 00000 n ", at(b"1 0 obj", &out)); - let entry = at(format!("{:010} 00000 n ", at(b"3 0 obj", &out)).as_bytes(), &out); - out.splice(entry..entry + catalog.len(), catalog.bytes()); - let doc = Document::open(Arc::new(out)).unwrap(); - let stream = doc.try_get(3).unwrap(); - let Object::Stream(s) = &*stream else { panic!("stream expected: {stream:?}") }; - assert_eq!(&s.raw[..], b"HELLO"); -} - -/// A scan of the file resolves only so many indirect `/Length` values. Each header that starts -/// with the sought number is parsed, and a `/Length` naming an object the table cannot place scans -/// the file for that one: with a hundred such headers, each naming another misplaced object, one -/// lookup read the whole file a hundred and one times. -#[test] -fn a_scan_of_the_file_resolves_a_bounded_number_of_lengths() { - const HEADERS: u32 = 100; - let mut out = b"%PDF-1.7\n".to_vec(); - let catalog = out.len(); - out.extend_from_slice(b"1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n"); - let pages = out.len(); - out.extend_from_slice(b"2 0 obj << /Type /Pages /Kids [] /Count 0 >> endobj\n"); - // Object 3 is defined only by headers inside this string, each with its own misplaced /Length. - out.extend_from_slice(b"4 0 obj (\n"); - for k in 0..HEADERS { - out.extend_from_slice(format!("3 0 obj << /Length {} 0 R >> stream\nX\nendstream endobj\n", 10 + k).as_bytes()); - } - out.extend_from_slice(b") endobj\n"); - let size = 10 + HEADERS; - let xref = out.len(); - out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); - for num in 1..size { - let at = if num == 2 { pages } else { catalog }; - out.extend_from_slice(format!("{at:010} 00000 n \n").as_bytes()); - } - out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); - let doc = Document::open(Arc::new(out)).unwrap(); - FILE_SCANS.with(|scans| scans.set(0)); - let three = doc.try_get(3).unwrap(); - assert!(matches!(&*three, Object::Stream(_)), "{three:?}"); - let scans = FILE_SCANS.with(std::cell::Cell::get); - assert!(scans <= 1 + super::state::ScanLengths::MAX as usize, "{scans} scans of the file"); -} - -/// An object can be nothing but a reference to another. A chain of them is followed, and one that -/// goes round ends in what isn't a dictionary or a name, without hanging. -#[test] -fn references_to_references_are_followed() { - let mut doc = Document::new_empty(); - let name = doc.add(Object::name("Target")); - let one = doc.add(Object::Ref(name)); - let two = doc.add(Object::Ref(one)); - assert_eq!(doc.resolve(&Object::Ref(two)).as_name(), Some(b"Target".as_slice())); - let a = doc.add(Object::Null); - let b = doc.add(Object::Ref(a)); - doc.set(a, Object::Ref(b)); - assert_eq!(doc.resolve(&Object::Ref(a)).as_name(), None); -} - -/// An object stream's `/DecodeParms` can be given by reference: without them a predictor isn't -/// undone, so its members were lost. -#[test] -fn an_object_stream_with_parameters_by_reference_is_read() { - let members = b"3 0 << /Hidden (yes) >>"; - // PNG Sub, one row. - let mut rows = vec![1u8]; - let mut before = 0u8; - for b in members { - rows.push(b.wrapping_sub(before)); - before = *b; - } - let coded = pdfcraft_filters::encode_flate(&rows); - let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 7]; - let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { - offs[n] = out.len(); - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); - obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); - let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /DecodeParms 6 0 R /Length {} >>\nstream\n", coded.len()).into_bytes(); - stm.extend_from_slice(&coded); - stm.extend_from_slice(b"\nendstream"); - obj(&mut out, 4, &stm); - let data = [2u8, 0, 4, 0]; - let mut xs = format!("<< /Type /XRef /Size 7 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); - xs.extend_from_slice(&data); - xs.extend_from_slice(b"\nendstream"); - obj(&mut out, 5, &xs); - obj(&mut out, 6, format!("<< /Predictor 11 /Columns {} >>", members.len()).as_bytes()); - let table = out.len(); - out.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); - for (n, off) in offs.iter().enumerate().skip(1) { - let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; - out.extend_from_slice(line.as_bytes()); - } - out.extend_from_slice(format!("trailer << /Size 7 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); - let doc = Document::open(Arc::new(out)).unwrap(); - let hidden = doc.get(ObjRef::new(3, 0)); - assert_eq!(hidden.as_dict().and_then(|d| d.get(b"Hidden")).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"yes".to_vec())); -} - -/// Each entry of an `/Encrypt` dictionary, and of a crypt filter in it, reads the same by reference. -/// Read as absent, `/StmF` or `/CFM` was `/Identity`, so a whole file's streams stayed ciphertext. -#[test] -fn encrypt_dictionary_entries_read_the_same_by_reference() { - let string = |s: &str| Object::String(crate::PdfString::text(s)); - let mut filter = Dict::new(); - filter.set(b"CFM".to_vec(), Object::name("AESV2")); - filter.set(b"Length".to_vec(), Object::Int(5)); - let cf = |filter: Dict| { - let mut cf = Dict::new(); - cf.set(b"StdCF".to_vec(), Object::Dict(filter)); - cf.set(b"Other".to_vec(), Object::Dict(Dict::new())); - Object::Dict(cf) - }; - let mut d = Dict::new(); - let entries = [ - ("Filter", Object::name("Standard")), - ("V", Object::Int(4)), - ("R", Object::Int(4)), - ("Length", Object::Int(128)), - ("P", Object::Int(-44)), - ("O", string("owner")), - ("U", string("user")), - ("EncryptMetadata", Object::Bool(false)), - ("CF", cf(filter.clone())), - ("StmF", Object::name("StdCF")), - ("StrF", Object::name("Other")), - ("EFF", Object::name("StdCF")), - ]; - for (key, value) in &entries { - d.set(key.as_bytes().to_vec(), value.clone()); - } - let mut doc = Document::new_empty(); - let direct = crate::security::encrypt_dict(&doc, &d); - assert_eq!((direct.stm_f.as_slice(), direct.str_f.as_slice(), direct.ef_f.as_slice()), (&b"StdCF"[..], &b"Other"[..], &b"StdCF"[..])); - assert_eq!(direct.length_bits, 40, "the filter's own length, in bytes"); - assert!(!direct.encrypt_metadata); - assert_eq!(direct.crypt_filters.iter().filter(|(_, m)| *m == pdfcraft_crypt::Method::Aes128).count(), 1); - for (key, value) in &entries { - let mut one = d.clone(); - let r = doc.add(value.clone()); - one.set(key.as_bytes().to_vec(), Object::Ref(r)); - assert_eq!(crate::security::encrypt_dict(&doc, &one), direct, "/{key}"); - } - for key in ["CFM", "Length"] { - let mut by_reference = filter.clone(); - let r = doc.add(filter.get(key.as_bytes()).unwrap().clone()); - by_reference.set(key.as_bytes().to_vec(), Object::Ref(r)); - let mut one = d.clone(); - one.set(b"CF".to_vec(), cf(by_reference)); - assert_eq!(crate::security::encrypt_dict(&doc, &one), direct, "/{key} in the filter"); - } -} - -/// /DecodeParms given by reference, whole, as array entries or as the numbers inside, are read -/// once made direct: here a Flate predictor, which the stream alone doesn't see. Only the entries the filters use are -/// resolved, and nothing is for a chain longer than a decoder takes. -#[test] -fn decode_parms_given_by_reference_are_read_once_direct() { - let rows = [1u8, 2, 3, 4, 5, 6, 7, 8]; - let params = pdfcraft_filters::Params { predictor: 12, colors: 1, bits_per_component: 8, columns: 4, early_change: 1 }; - let coded = pdfcraft_filters::encode(&pdfcraft_filters::Filter::Flate, ¶ms, &rows).unwrap(); - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(12)); - parms.set(b"Columns".to_vec(), Object::Int(4)); - let mut doc = Document::new_empty(); - let r = doc.add(Object::Dict(parms.clone())); - let array = doc.add(Object::Array(vec![Object::Ref(r)])); - let stream = |filters: usize, p: Object| { - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"); filters])); - d.set(b"DecodeParms".to_vec(), p); - Stream::from_raw(d, coded.clone()) - }; - for p in [Object::Ref(r), Object::Array(vec![Object::Ref(r)]), Object::Ref(array)] { - let s = stream(1, p.clone()); - assert_ne!(s.decoded().unwrap(), rows, "{p:?} read as it is"); - assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "{p:?}"); - } - // A number inside given by reference, which `Dict::int` doesn't follow: in the dictionary - // itself, and in an array's entry. - let twelve = doc.add(Object::Int(12)); - let mut inner = parms.clone(); - inner.set(b"Predictor".to_vec(), Object::Ref(twelve)); - for p in [Object::Dict(inner.clone()), Object::Array(vec![Object::Dict(inner)])] { - let s = stream(1, p.clone()); - assert_ne!(s.decoded().unwrap(), rows, "{p:?} read as it is"); - assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "{p:?}"); - } - // Only numbers and flags are copied: a key naming a big array, or an entry that is no - // dictionary, is left out (a file could name one big object from every key and filter). - let big = doc.add(Object::Array(vec![Object::Int(1); 100_000])); - let mut hostile = Dict::new(); - hostile.set(b"Predictor".to_vec(), Object::Ref(big)); - hostile.set(b"Columns".to_vec(), Object::Ref(big)); - hostile.set(b"Colors".to_vec(), Object::Ref(twelve)); - let entry = doc.add(Object::Dict(hostile)); - let two = stream(2, Object::Array(vec![Object::Ref(entry), Object::Ref(big)])); - let made = doc.with_direct_parms(&two); - let made = made.dict.get(b"DecodeParms").and_then(Object::as_array).unwrap(); - let first = made[0].as_dict().unwrap(); - assert_eq!((first.contains(b"Predictor"), first.contains(b"Columns"), first.int(b"Colors")), (false, false, Some(12))); - assert!(matches!(made[1], Object::Null), "{:?}", made[1]); - let direct = stream(1, Object::Dict(parms)); - assert!(matches!(doc.with_direct_parms(&direct), Cow::Borrowed(_)), "nothing to resolve"); - assert_eq!(direct.decoded().unwrap(), rows); - // Entries past the filters' are left out, unresolved; so is everything past the chain limit. - let many = stream(1, Object::Array(vec![Object::Ref(r); 1000])); - let made = doc.with_direct_parms(&many); - assert_eq!(made.dict.get(b"DecodeParms").and_then(Object::as_array).map(Vec::len), Some(1)); - let long = stream(pdfcraft_filters::MAX_CHAIN + 1, Object::Array(vec![Object::Ref(r); 9])); - assert!(matches!(doc.with_direct_parms(&long), Cow::Borrowed(_))); - // A chain as long as a decoder takes is resolved, and a `/Crypt` in it doesn't count, as - // the decoders leave it out: ASCIIHex as often as the limit allows, then the predictor's - // Flate, its parameters by reference. - let hex = |b: &[u8]| b.iter().flat_map(|x| format!("{x:02X}").into_bytes()).chain(*b">").collect::>(); - let wrapped = (1..pdfcraft_filters::MAX_CHAIN).fold(coded.clone(), |data, _| hex(&data)); - for crypt in [false, true] { - let mut names = vec![Object::name("ASCIIHexDecode"); pdfcraft_filters::MAX_CHAIN - 1]; - names.push(Object::name("FlateDecode")); - if crypt { - names.insert(0, Object::name("Crypt")); - } - let mut entries = vec![Object::Null; names.len() - 1]; - entries.push(Object::Ref(r)); - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::Array(names)); - d.set(b"DecodeParms".to_vec(), Object::Array(entries)); - let s = Stream::from_raw(d, wrapped.clone()); - assert_eq!(doc.with_direct_parms(&s).decoded().unwrap(), rows, "with /Crypt: {crypt}"); - } -} - -/// A CCITT fax image reads through its /DecodeParms, given directly, in an array or by -/// reference (its numbers too), with /BlackIs1, and behind other filters; one whose /Columns -/// aren't its /Width is refused, /Height rows are decoded whatever /Rows says, and other images -/// decode through their filters. -#[test] -fn fax_images_decode_with_their_parameters() { - // 10 × 3: the first five samples of each row white, the rest black. - let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; - let coded = pdfcraft_filters::encode_g4(&bits, 10, 3).unwrap(); - let mut doc = Document::new_empty(); - let mut parms = Dict::new(); - parms.set(b"K".to_vec(), Object::Int(-1)); - parms.set(b"Columns".to_vec(), Object::Int(10)); - let indirect = doc.add(Object::Dict(parms.clone())); - let image = |doc: &Document, parms: Option, width: i64, height: i64| { - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(width)); - d.set(b"Height".to_vec(), Object::Int(height)); - d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); - if let Some(p) = parms { - d.set(b"DecodeParms".to_vec(), p); - } - doc.decoded_image(&Stream::from_raw(d, coded.clone())) - }; - for p in [ - Object::Dict(parms.clone()), - Object::Array(vec![Object::Dict(parms.clone())]), - Object::Ref(indirect), - Object::Array(vec![Object::Ref(indirect)]), - ] { - assert_eq!(image(&doc, Some(p.clone()), 10, 3).unwrap(), bits, "{p:?}"); - } - let mut inverted = parms.clone(); - inverted.set(b"BlackIs1".to_vec(), Object::Bool(true)); - assert_eq!( - image(&doc, Some(Object::Dict(inverted)), 10, 3).unwrap(), - [0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000, 0b0000_0111, 0b1100_0000] - ); - assert_eq!(image(&doc, Some(Object::Dict(parms.clone())), 10, 2).unwrap(), bits[..4], "at most /Height rows"); - // Within a bound: no more rows than it holds. - let mut bounded = Dict::new(); - bounded.set(b"Width".to_vec(), Object::Int(10)); - bounded.set(b"Height".to_vec(), Object::Int(3)); - bounded.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); - bounded.set(b"DecodeParms".to_vec(), Object::Dict(parms.clone())); - assert_eq!(doc.decoded_image_within(&Stream::from_raw(bounded, coded.clone()), 5).unwrap(), bits[..4], "two rows of two bytes in five"); - let mut by_reference = Dict::new(); - by_reference.set(b"K".to_vec(), Object::Ref(doc.add(Object::Int(-1)))); - by_reference.set(b"Columns".to_vec(), Object::Ref(doc.add(Object::Int(10)))); - assert_eq!(image(&doc, Some(Object::Dict(by_reference)), 10, 3).unwrap(), bits, "/K and /Columns by reference"); - let mut understated = parms.clone(); - understated.set(b"Rows".to_vec(), Object::Int(1)); - assert_eq!(image(&doc, Some(Object::Dict(understated.clone())), 10, 3).unwrap(), bits, "/Height rows, whatever /Rows says"); - // More /Rows than /Height (PDFBOX-771: 1,084,975,104), as the renderer reads it too. - let mut overstated = parms.clone(); - overstated.set(b"Rows".to_vec(), Object::Int(1_000_000_000)); - assert_eq!(image(&doc, Some(Object::Dict(overstated)), 10, 3).unwrap(), bits, "/Height rows, not /Rows past it"); - // The parameters as the file gives them, /Rows included, and Table 11's defaults. - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::name("CCITTFaxDecode")); - d.set(b"DecodeParms".to_vec(), Object::Dict(understated)); - let p = doc.fax_params(&Stream::from_raw(d.clone(), Vec::new())).unwrap(); - assert_eq!((p.k, p.columns, p.rows, p.end_of_block, p.black_is_1), (-1, 10, 1, true, false)); - d.set(b"Filter".to_vec(), Object::name("FlateDecode")); - assert!(doc.fax_params(&Stream::from_raw(d, Vec::new())).is_none(), "not a fax image"); - assert!(matches!(image(&doc, Some(Object::Dict(parms.clone())), 12, 3), Err(CosError::Filter(_))), "/Columns other than /Width"); - assert!(matches!(image(&doc, None, 10, 3), Err(CosError::Filter(_))), "the default 1728 /Columns"); - let mut plain = Dict::new(); - plain.set(b"Filter".to_vec(), Object::name("FlateDecode")); - assert_eq!(doc.decoded_image(&Stream::from_raw(plain, pdfcraft_filters::encode_flate(&bits))).unwrap(), bits); - - // Behind other filters, its parameters at its own place in the array (/Crypt counts). - let behind = |filters: &[Object], parms: &[Object], data: Vec| { - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(10)); - d.set(b"Height".to_vec(), Object::Int(3)); - d.set(b"Filter".to_vec(), Object::Array(filters.to_vec())); - d.set(b"DecodeParms".to_vec(), Object::Array(parms.to_vec())); - doc.decoded_image(&Stream::from_raw(d, data)) - }; - let (fax, null, own) = (Object::name("CCITTFaxDecode"), Object::Null, Object::Dict(parms.clone())); - let hex = coded.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); - let flated = pdfcraft_filters::encode_flate(&coded); - let ahx = [Object::name("AHx"), fax.clone()]; - assert_eq!(behind(&ahx, &[null.clone(), Object::Ref(indirect)], hex.clone()).unwrap(), bits, "behind ASCIIHexDecode"); - assert_eq!( - behind(&[Object::name("FlateDecode"), Object::name("CCF")], &[null.clone(), own.clone()], flated.clone()).unwrap(), - bits, - "behind Flate" - ); - assert_eq!(behind(&[Object::name("Crypt"), fax.clone()], &[null.clone(), own.clone()], coded.clone()).unwrap(), bits, "after /Crypt"); - // A /Crypt belongs first (§7.4.10); one after the fax filter leaves it its own entry. - assert_eq!(behind(&[fax.clone(), Object::name("Crypt")], &[own.clone(), null.clone()], coded.clone()).unwrap(), bits, "before /Crypt"); - assert!(behind(&ahx, &[own.clone(), null.clone()], hex).is_err(), "the first entry is the other filter's"); - assert!(behind(&[fax.clone(), Object::name("FlateDecode")], &[own.clone(), null.clone()], flated).is_err(), "CCITT, then Flate"); - assert!(behind(&[Object::Int(5), fax], &[null, own], coded.clone()).is_err(), "a /Filter entry that isn't a name"); -} - -/// Images that share a `/JBIG2Globals` stream of the file decode it once a document: it is kept -/// for the next, in every clone. A globals stream edited since is read afresh (the kept one is the -/// file's), and objects added in memory aren't kept. -#[test] -fn shared_jbig2_globals_are_read_once() { - let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; - let coded = pdfcraft_filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); - let mut doc = Document::new_empty(); - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; - let globals = doc.add(Object::Stream(Stream::from_raw(d, pdfcraft_filters::encode_flate(&skipped)))); - let image = |doc: &mut Document| { - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(10)); - d.set(b"Height".to_vec(), Object::Int(3)); - d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - d.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - doc.add(Object::Stream(Stream::from_raw(d, coded.clone()))) - }; - let (a, b) = (image(&mut doc), image(&mut doc)); - let stream = |doc: &Document, r: ObjRef| match &*doc.get(r) { - Object::Stream(s) => s.clone(), - _ => panic!("not a stream"), - }; - assert_eq!(doc.decoded_image(&stream(&doc, a)).unwrap(), bits); - assert!(doc.jbig2_globals.lock().unwrap().is_empty(), "an object added in memory isn't kept"); - - // The same objects read from a file, the catalog holding on to the images (a full save may - // number them anew). - let root = doc.root().unwrap(); - doc.update_dict(root, |c| c.set(b"Images".to_vec(), Object::Array(vec![Object::Ref(a), Object::Ref(b)]))).unwrap(); - let file = crate::write_full(&doc, &crate::SaveOptions::default()).unwrap(); - let mut doc = Document::open(Arc::new(file)).unwrap(); - let images = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"Images").unwrap().as_array().unwrap().clone(); - let (a, b) = (images[0].as_ref().unwrap(), images[1].as_ref().unwrap()); - let parms = stream(&doc, a).dict.get(b"DecodeParms").unwrap().as_dict().unwrap().clone(); - let globals = parms.reference(b"JBIG2Globals").unwrap(); - let clone = doc.clone(); - assert_eq!(doc.decoded_image(&stream(&doc, a)).unwrap(), bits); - assert_eq!(doc.jbig2_globals.lock().unwrap().len(), 1, "kept"); - let kept = doc.jbig2_globals.lock().unwrap()[&globals.num].clone(); - assert_eq!(clone.decoded_image(&stream(&clone, b)).unwrap(), bits, "the clone's other image"); - let shared = clone.jbig2_globals.lock().unwrap()[&globals.num].clone(); - assert!(Arc::ptr_eq(&kept, &shared), "the clone uses the one kept, not a second decode"); - doc.set(globals, Object::Stream(Stream::from_raw(Dict::new(), b"not JBIG2".to_vec()))); - assert!(doc.decoded_image(&stream(&doc, b)).is_err(), "the edited globals are read afresh"); - assert_eq!(clone.decoded_image(&stream(&clone, b)).unwrap(), bits, "the clone keeps the file's"); -} - -/// A JBIG2 image's size is read as given directly, as every caller reads it first, and an image -/// with neither is an error, not an image of no rows. -#[test] -fn a_jbig2_image_needs_its_size_given_directly() { - let coded = pdfcraft_filters::encode_jbig2_generic(&[0xFF, 0xFF], 8, 2).unwrap(); - let mut doc = Document::new_empty(); - let size = doc.add(Object::Int(2)); - let image = |width: Object, height: Object| { - let mut d = Dict::new(); - d.set(b"Width".to_vec(), width); - d.set(b"Height".to_vec(), height); - d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); - doc.decoded_image(&Stream::from_raw(d, coded.clone())) - }; - assert!(image(Object::Int(8), Object::Int(2)).is_ok()); - assert!(image(Object::Ref(size), Object::Int(2)).is_err(), "/Width by reference"); - assert!(image(Object::Int(8), Object::Ref(size)).is_err(), "/Height by reference"); -} - -/// A document that kept globals of 64 MiB keeps no more, and the images past that still read: -/// three streams of 22 MiB, shared by three images, are two kept and one decoded each time. -#[test] -fn kept_jbig2_globals_are_bounded_in_total() { - let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; - let coded = pdfcraft_filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); - let length = 22 * 1024 * 1024; - let mut skipped = vec![0, 0, 0, 0, 0x3E, 0, 0]; - skipped.extend_from_slice(&u32::try_from(4 + length).unwrap().to_be_bytes()); - skipped.extend_from_slice(&[0, 0, 0, 1]); - skipped.resize(skipped.len() + length, 0); - let mut doc = Document::new_empty(); - let mut images = Vec::new(); - for _ in 0..3 { - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let globals = doc.add(Object::Stream(Stream::from_raw(d, pdfcraft_filters::encode_flate(&skipped)))); - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(10)); - d.set(b"Height".to_vec(), Object::Int(3)); - d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - d.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - images.push(Object::Ref(doc.add(Object::Stream(Stream::from_raw(d, coded.clone()))))); - } - let root = doc.root().unwrap(); - doc.update_dict(root, |c| c.set(b"Images".to_vec(), Object::Array(images))).unwrap(); - let doc = Document::open(Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap())).unwrap(); - let images = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"Images").unwrap().as_array().unwrap().clone(); - for image in &images { - let Object::Stream(s) = &*doc.get(image.as_ref().unwrap()) else { panic!("not a stream") }; - assert_eq!(doc.decoded_image(s).unwrap(), bits, "every image reads"); - } - let kept = doc.jbig2_globals.lock().unwrap(); - assert_eq!(kept.len(), 2, "two fit in 64 MiB, a third doesn't"); - assert!(kept.values().all(|g| g.capacity() == g.len()), "what is kept holds no more than it counts"); -} - -/// Globals whose `/DecodeParms` are given by reference aren't kept: that's another object, so -/// editing it changes what they decode to while the stream itself, the key kept by, stays the same. -#[test] -fn jbig2_globals_with_parameters_by_reference_are_read_afresh() { - let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; - let coded = pdfcraft_filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); - let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; - let params = pdfcraft_filters::Params { predictor: 12, colors: 1, bits_per_component: 8, columns: 15, early_change: 1 }; - let mut doc = Document::new_empty(); - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(12)); - parms.set(b"Columns".to_vec(), Object::Int(15)); - let parms = doc.add(Object::Dict(parms)); - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::name("FlateDecode")); - d.set(b"DecodeParms".to_vec(), Object::Ref(parms)); - let predicted = pdfcraft_filters::encode(&pdfcraft_filters::Filter::Flate, ¶ms, &skipped).unwrap(); - let globals = doc.add(Object::Stream(Stream::from_raw(d, predicted))); - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(10)); - d.set(b"Height".to_vec(), Object::Int(3)); - d.set(b"Filter".to_vec(), Object::name("JBIG2Decode")); - let mut image_parms = Dict::new(); - image_parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - d.set(b"DecodeParms".to_vec(), Object::Dict(image_parms)); - let image = doc.add(Object::Stream(Stream::from_raw(d, coded))); - let root = doc.root().unwrap(); - doc.update_dict(root, |c| c.set(b"Image".to_vec(), Object::Ref(image))).unwrap(); - let mut doc = Document::open(Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap())).unwrap(); - let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); - let stream = |doc: &Document| match &*doc.get(image) { - Object::Stream(s) => s.clone(), - _ => panic!("not a stream"), - }; - assert_eq!(doc.decoded_image(&stream(&doc)).unwrap(), bits); - assert!(doc.jbig2_globals.lock().unwrap().is_empty(), "not kept"); - let globals = stream(&doc).dict.get(b"DecodeParms").unwrap().as_dict().unwrap().reference(b"JBIG2Globals").unwrap(); - let Object::Stream(g) = &*doc.get(globals) else { panic!("not a stream") }; - let parms = g.dict.reference(b"DecodeParms").unwrap(); - doc.set(parms, Object::Dict(Dict::new())); - assert!(doc.decoded_image(&stream(&doc)).is_err(), "without the predictor they read otherwise"); -} - -/// A JBIG2 image reads through its decoder, behind other filters too, with the `/JBIG2Globals` of -/// its own `/DecodeParms` (a dictionary or its place in an array, /Crypt counting, by reference -/// too): globals that can't be read are an error, and ones at another filter's place aren't its -/// own, nor are those of a lone dictionary when other filters come first. Its page has to be its -/// /Width × /Height, and fit the bound. -#[test] -fn jbig2_images_decode_with_their_globals() { - // 10 × 3: the first five samples of each row white, the rest black. - let bits = [0b1111_1000, 0, 0b1111_1000, 0, 0b1111_1000, 0]; - let coded = pdfcraft_filters::encode_jbig2_generic(&bits, 10, 3).unwrap(); - let hex = coded.iter().map(|b| format!("{b:02x}")).chain([">".to_string()]).collect::().into_bytes(); - let mut doc = Document::new_empty(); - // An extension segment (ISO/IEC 14492 7.4.14), which a decoder skips, Flate-encoded; and data - // that isn't JBIG2. - let globals = |doc: &mut Document, data: &[u8]| { - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let globals = doc.add(Object::Stream(Stream::from_raw(d, pdfcraft_filters::encode_flate(data)))); - let mut parms = Dict::new(); - parms.set(b"JBIG2Globals".to_vec(), Object::Ref(globals)); - Object::Dict(parms) - }; - let (good, bad) = (globals(&mut doc, &[0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]), globals(&mut doc, b"not JBIG2")); - let indirect = Object::Ref(doc.add(bad.clone())); - let image = |filters: &[&str], parms: Option, data: &[u8], (width, height): (i64, i64), max: usize| { - let mut d = Dict::new(); - d.set(b"Width".to_vec(), Object::Int(width)); - d.set(b"Height".to_vec(), Object::Int(height)); - d.set(b"Filter".to_vec(), Object::Array(filters.iter().map(|f| Object::name(f)).collect())); - if let Some(p) = parms { - d.set(b"DecodeParms".to_vec(), p); - } - doc.decoded_image_within(&Stream::from_raw(d, data.to_vec()), max) - }; - let jbig2 = ["JBIG2Decode"]; - let full = (10, 3); - assert_eq!(image(&jbig2, None, &coded, full, MAX_DECODED).unwrap(), bits); - assert_eq!(image(&jbig2, Some(good.clone()), &coded, full, MAX_DECODED).unwrap(), bits, "globals it skips"); - for parms in [bad.clone(), Object::Array(vec![bad.clone()]), indirect.clone(), Object::Array(vec![indirect])] { - let e = image(&jbig2, Some(parms.clone()), &coded, full, MAX_DECODED).unwrap_err(); - assert!(e.to_string().contains("JBIG2"), "{parms:?}: {e}"); - } - let behind = ["ASCIIHexDecode", "JBIG2Decode"]; - assert_eq!(image(&behind, None, &hex, full, MAX_DECODED).unwrap(), bits, "behind ASCIIHexDecode"); - assert!(image(&behind, Some(Object::Array(vec![Object::Null, bad.clone()])), &hex, full, MAX_DECODED).is_err(), "its own globals"); - assert_eq!(image(&behind, Some(Object::Array(vec![bad.clone(), Object::Null])), &hex, full, MAX_DECODED).unwrap(), bits, "another's"); - assert_eq!(image(&behind, Some(bad.clone()), &hex, full, MAX_DECODED).unwrap(), bits, "a lone dictionary, the first filter's"); - let after_crypt = ["Crypt", "JBIG2Decode"]; - assert!(image(&after_crypt, Some(Object::Array(vec![Object::Null, bad.clone()])), &coded, full, MAX_DECODED).is_err(), "after /Crypt"); - assert_eq!(image(&after_crypt, Some(Object::Array(vec![bad, Object::Null])), &coded, full, MAX_DECODED).unwrap(), bits, "/Crypt's"); - - for size in [(11, 3), (9, 3), (10, 2), (10, 4)] { - assert!(image(&jbig2, None, &coded, size, MAX_DECODED).is_err(), "a page other than {size:?}"); - } - assert_eq!(image(&jbig2, None, &coded, full, 6).unwrap(), bits, "the page within the bound"); - assert!(image(&jbig2, None, &coded, full, 5).is_err(), "the page past the bound"); -} - -#[test] -fn a_missing_file_id_is_derived_from_the_encryption_seed() { - let mut doc = Document::new_empty(); - let seed = [9u8; 32]; - doc.set_encryption(&pdfcraft_crypt::NewEncryption { - algorithm: pdfcraft_crypt::Algorithm::Aes256, - user_password: "user", - owner_password: "owner", - permissions: -4, - encrypt_metadata: true, - seed, - }) - .unwrap(); - let id = doc.trailer().get(b"ID").and_then(|o| o.as_array()).and_then(|a| a.first()).and_then(|o| o.as_string()).map(|s| s.bytes.clone()); - assert_eq!(id.as_deref(), Some(pdfcraft_crypt::file_id(&seed).as_slice())); - - let mut kept = Document::new_empty(); - let fixed = Object::String(crate::PdfString { bytes: b"0123456789abcdef".to_vec(), hex: true }); - kept.trailer_mut().set(b"ID".to_vec(), Object::Array(vec![fixed.clone(), fixed])); - kept.set_encryption(&pdfcraft_crypt::NewEncryption { - algorithm: pdfcraft_crypt::Algorithm::Aes256, - user_password: "user", - owner_password: "owner", - permissions: -4, - encrypt_metadata: true, - seed, - }) - .unwrap(); - let still = kept.trailer().get(b"ID").and_then(|o| o.as_array()).and_then(|a| a.first()).and_then(|o| o.as_string()).map(|s| s.bytes.clone()); - assert_eq!(still.as_deref(), Some(b"0123456789abcdef".as_slice())); -} - -#[test] -fn a_read_state_matches_a_clone_and_no_changed_document() { - let mut doc = Document::new_empty(); - let a = doc.add(Object::Int(1)); - let state = doc.read_state(); - let snapshot = doc.clone(); - assert!(state.matches(&doc) && state.matches(&snapshot)); - // The same value set again is still a change: another allocation. - doc.set(a, Object::Int(1)); - assert!(!state.matches(&doc) && state.matches(&snapshot)); - let edited = doc.read_state(); - doc.free(a); - assert!(!edited.matches(&doc)); - let mut trailer_changed = snapshot.clone(); - trailer_changed.trailer_mut().set(b"Info".to_vec(), Object::Null); - assert!(!state.matches(&trailer_changed)); - // Another document's state never matches, even with the same changes. - let mut other = Document::new_empty(); - other.add(Object::Int(1)); - assert!(!other.read_state().matches(&snapshot)); -} - -/// Build a classic-xref file from object bodies (object i+1 = bodies[i]). -pub(crate) fn build(bodies: &[&str], trailer: &str) -> Vec { - let mut out = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for (i, b) in bodies.iter().enumerate() { - offsets.push(out.len()); - out.extend_from_slice(format!("{} 0 obj\n{b}\nendobj\n", i + 1).as_bytes()); - } - let xref = out.len(); - out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", bodies.len() + 1).as_bytes()); - for o in offsets { - out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); - } - out.extend_from_slice(format!("trailer\n<< /Size {} {trailer} >>\nstartxref\n{xref}\n%%EOF\n", bodies.len() + 1).as_bytes()); - out -} - -fn assert_compact_matches_materialized(doc: &Document) { - assert!(matches!(doc.entries.as_ref(), XrefIndex::Compact { .. }), "valid xrefs should not retain a per-entry map"); - let (entries, _, revisions, _, _) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); - assert_eq!(revisions.as_slice(), doc.revisions()); - let max = entries.keys().next_back().copied().unwrap_or(0); - assert_eq!(doc.entries.max_num(), max); - for num in 0..=max { - assert_eq!(doc.xref_entry(num), entries.get(&num).copied(), "object {num}"); - } - let expected: Vec<_> = entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect(); - assert_eq!(doc.object_numbers(), expected); -} - -#[test] -fn compact_xref_tables_streams_and_revisions_match_the_materialized_reader() { - let classic = Document::open(Arc::new(build( - &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 9 >>"], - "/Root 1 0 R", - ))) - .unwrap(); - assert_compact_matches_materialized(&classic); - - let mut sparse = b"%PDF-1.7\n".to_vec(); - let root_offset = sparse.len(); - sparse.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 5 0 R >>\nendobj\n"); - let pages_offset = sparse.len(); - sparse.extend_from_slice(b"5 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); - let xref = sparse.len(); - sparse.extend_from_slice( - format!( - "xref\n0 1\n0000000000 65535 f \n1 1\n{root_offset:010} 00000 n \n5 1\n{pages_offset:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n" - ) - .as_bytes(), - ); - let sparse = Document::open(Arc::new(sparse)).unwrap(); - assert_eq!(sparse.object_numbers(), vec![1, 5]); - assert_compact_matches_materialized(&sparse); - - let mut sparse_stream = b"%PDF-1.7\n".to_vec(); - let mut offsets = [0u32; 8]; - offsets[1] = sparse_stream.len() as u32; - sparse_stream.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 5 0 R >>\nendobj\n"); - offsets[5] = sparse_stream.len() as u32; - sparse_stream.extend_from_slice(b"5 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); - offsets[6] = sparse_stream.len() as u32; - sparse_stream.extend_from_slice(b"6 0 obj\n<< /Unused true >>\nendobj\n"); - offsets[7] = sparse_stream.len() as u32; - let xref_row = |kind: u8, offset: u32, generation: u16| { - let mut row = Vec::with_capacity(7); - row.push(kind); - row.extend_from_slice(&offset.to_be_bytes()); - row.extend_from_slice(&generation.to_be_bytes()); - row - }; - let mut rows = Vec::new(); - rows.extend(xref_row(0, 0, u16::MAX)); - rows.extend(xref_row(1, offsets[1], 0)); - rows.extend(xref_row(1, offsets[5], 0)); - rows.extend(xref_row(1, offsets[6], 0)); - rows.extend(xref_row(1, offsets[7], 0)); - sparse_stream.extend_from_slice( - format!("7 0 obj\n<< /Type /XRef /Size 8 /Root 1 0 R /W [1 4 2] /Index [0 2 5 3] /Length {} >>\nstream\n", rows.len()).as_bytes(), - ); - sparse_stream.extend_from_slice(&rows); - sparse_stream.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{}\n%%EOF\n", offsets[7]).as_bytes()); - let sparse_stream = Document::open(Arc::new(sparse_stream)).unwrap(); - assert_eq!(sparse_stream.object_numbers(), vec![1, 5, 6, 7]); - assert_compact_matches_materialized(&sparse_stream); - - let stream_bytes = crate::write_full(&classic, &crate::SaveOptions::default()).unwrap(); - let stream = Document::open(Arc::new(stream_bytes)).unwrap(); - assert!(stream.revisions().last().is_some_and(|revision| revision.is_stream)); - assert_compact_matches_materialized(&stream); - - let mut edited = classic.clone(); - edited.update_dict(ObjRef::new(1, 0), |dict| dict.set(b"Lang".to_vec(), Object::String(crate::PdfString::literal("en")))).unwrap(); - edited.free(ObjRef::new(3, 0)); - let incremental = crate::write_incremental(&edited, &crate::SaveOptions::default()).unwrap(); - let updated = Document::open(Arc::new(incremental)).unwrap(); - assert_eq!(updated.revisions().len(), 2); - assert_compact_matches_materialized(&updated); -} - -#[test] -fn malformed_size_uses_the_materialized_compatibility_path() { - let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 9 >>"], "/Root 1 0 R"); - let marker = b"/Size 4"; - let at = bytes.windows(marker.len()).position(|window| window == marker).unwrap(); - bytes[at..at + marker.len()].copy_from_slice(b"/Size 2"); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(matches!(doc.entries.as_ref(), XrefIndex::Materialized(..))); - assert_eq!(doc.next_num, 4); - assert_eq!(doc.object_numbers(), vec![1, 2, 3]); -} - -/// Entries the reader sets aside are remembered, so a caller comparing revisions can tell -/// that another reader may resolve the object differently. -#[test] -fn cross_reference_entries_that_are_set_aside_are_recorded() { - let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); - let plain = Document::open(Arc::new(base.clone())).unwrap(); - assert!(plain.xref_oddities().is_empty()); - assert!((1..=3).all(|n| plain.xref_entry_is_exact(n))); - let prev = String::from_utf8_lossy(&base).rfind("xref\n0 4").unwrap(); - let third = plain.xref_entry(3).unwrap(); - - // A later section: object 3 "in use" at offset 0, object 2 listed twice with different - // offsets, and object 0 at offset 0 (the usual producer slip, which names no object). - let mut bytes = base.clone(); - let section = bytes.len(); - bytes.extend_from_slice( - format!("xref\n0 1\n0000000000 65535 n \n3 1\n0000000000 00000 n \n2 1\n0000000058 00000 n \n2 1\n0000000009 00000 n \ntrailer\n<< /Size 4 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.xref_oddities(), [(section as u64, 3), (section as u64, 2)]); - // What the reader resolves is what it resolved before: the first entry it kept. - assert_eq!(doc.xref_entry(3), Some(third)); - assert_eq!(doc.xref_entry(2), Some(XrefEntry::InFile { offset: 58, generation: 0 })); -} - -/// An entry that misses its object still resolves (by searching) but is not exact. -#[test] -fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() { - let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); - let text = String::from_utf8_lossy(&base).into_owned(); - let at = text.find("3 0 obj").unwrap(); - let bytes = text.replacen(&format!("{at:010} 00000 n"), &format!("{:010} 00000 n", at + 2), 1).into_bytes(); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.get(ObjRef { num: 3, generation: 0 }).as_string().map(|s| s.bytes.clone()), Some(b"three".to_vec())); - assert!(!doc.xref_entry_is_exact(3)); - assert!(doc.xref_entry_is_exact(1) && doc.xref_entry_is_exact(2)); -} - -/// An object that can't be read at its entry or anywhere else is searched for in the whole file -/// once: every later read, in a clone too, gives the same error without searching again. An -/// edit of that number still answers for it. -#[test] -fn an_unreadable_object_is_searched_for_once() { - let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"]; - let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap(); - FILE_SCANS.with(|scans| scans.set(0)); - let first = doc.try_get(3); - assert!(first.is_err(), "{first:?}"); - for _ in 0..10 { - assert_eq!(doc.try_get(3), first); - assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); - } - let mut edited = doc.clone(); - assert_eq!(edited.try_get(3), first); - assert_eq!(FILE_SCANS.with(std::cell::Cell::get), 1, "searches for an object that isn't there"); - let report = doc.unreadable_objects(); - assert_eq!(report.len(), 1, "cached failures must stay listed exactly once: {report:?}"); - assert_eq!(edited.unreadable_objects(), report, "clones retain the cached failure diagnostic"); - edited.set(ObjRef::new(3, 0), Object::Int(7)); - assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Int(7))); - assert_eq!(doc.try_get(3), first, "the edit is the clone's alone"); - edited.free(ObjRef::new(3, 0)); - assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); -} - -/// A member of an object stream that can't be decoded (here, one that decodes past the document's -/// stream limit) is decoded once: every later read, in a clone too, gives the same error without -/// decoding the stream again. The stream itself loads, so no search of the file is involved. -#[test] -fn a_member_of_an_undecodable_object_stream_is_decoded_once() { - let file = object_stream_file("3 0 ", &format!("<< /Hidden ({}) >>", "x".repeat(4096)), 0); - let doc = Document::open_with_stream_limit(Arc::new(file), None, 1024).unwrap(); - OBJSTM_DECODES.with(|decodes| decodes.set(0)); - let first = doc.try_get(3); - assert!(matches!(first, Err(CosError::Filter(_))), "{first:?}"); - for _ in 0..10 { - assert_eq!(doc.try_get(3), first); - } - assert_eq!(doc.clone().try_get(3), first); - assert_eq!(OBJSTM_DECODES.with(std::cell::Cell::get), 1, "decodes an object stream that can't be decoded"); -} - -/// Clones share the cache, and an edit is one clone's alone: a member of an object stream that -/// one clone freed or replaced still reads, in the others, from the file's stream. -#[test] -fn an_edited_object_stream_leaves_other_clones_members_alone() { - let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); - let doc = Document::open(file.clone()).unwrap(); - let mut freed = doc.clone(); - freed.free(ObjRef::new(4, 0)); - assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4))); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the free reached another clone"); - let doc = Document::open(file).unwrap(); - let mut replaced = doc.clone(); - let mut dict = Dict::new(); - dict.set(b"Type".to_vec(), Object::name("ObjStm")); - dict.set(b"N".to_vec(), Object::Int(1)); - dict.set(b"First".to_vec(), Object::Int(4)); - replaced.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec())); - assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no")); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); -} - -/// A member another clone read from the file's object stream is not what a clone that replaced -/// the stream reads, nor is one the document read itself before replacing it. -#[test] -fn a_replaced_object_stream_hides_the_members_read_before() { - let replacement = || { - let mut dict = Dict::new(); - dict.set(b"Type".to_vec(), Object::name("ObjStm")); - dict.set(b"N".to_vec(), Object::Int(1)); - dict.set(b"First".to_vec(), Object::Int(4)); - crate::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec()) - }; - let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); - let doc = Document::open(file.clone()).unwrap(); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); - let mut replaced = doc.clone(); - replaced.set(ObjRef::new(4, 0), replacement()); - assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no"), "another clone's read of the file's stream"); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); - let mut doc = Document::open(file.clone()).unwrap(); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); - doc.set(ObjRef::new(4, 0), replacement()); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("no"), "the document's own read before the edit"); - let doc = Document::open(file).unwrap(); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); - let mut freed = doc.clone(); - freed.free(ObjRef::new(4, 0)); - assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4)), "another clone's read before the free"); -} - -/// An object stream another clone framed by the file's indirect `/Length` is not what a clone -/// that edited that length reads. -#[test] -fn an_edited_object_stream_length_hides_the_members_read_before() { - let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); - let doc = Document::open(file).unwrap(); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here")); - let mut edited = doc.clone(); - edited.set(ObjRef::new(6, 0), Object::Int("3 0 << /Hidden (cut".len() as i64)); - assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "another clone's framing of the stream"); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); -} - -/// A stream another clone read by the file's indirect `/Length` is not what a clone that edited -/// that length reads. -#[test] -fn an_edited_stream_length_hides_the_stream_read_before() { - let bodies = [ - "<< /Type /Catalog /Pages 2 0 R >>", - "<< /Type /Pages /Kids [] /Count 0 >>", - "<< /Length 4 0 R >>\nstream\nshort\nendstream long\nendstream", - "20", - ]; - let mut out = b"%PDF-1.4\n".to_vec(); - let mut table = String::from("xref\n0 5\n0000000000 65535 f \n"); - for (n, body) in bodies.iter().enumerate() { - table.push_str(&format!("{:010} 00000 n \n", out.len())); - out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", n + 1).as_bytes()); - } - let start = out.len(); - out.extend_from_slice(format!("{table}trailer\n<< /Size 5 /Root 1 0 R >>\nstartxref\n{start}\n%%EOF\n").as_bytes()); - let data = |doc: &Document| match &*doc.get(ObjRef::new(3, 0)) { - Object::Stream(s) => s.raw.to_vec(), - other => panic!("{other:?}"), - }; - let doc = Document::open(Arc::new(out)).unwrap(); - assert_eq!(data(&doc), b"short\nendstream long"); - let mut edited = doc.clone(); - edited.set(ObjRef::new(4, 0), Object::Int(5)); - assert_eq!(data(&edited), b"short", "another clone's read by the file's length"); - assert_eq!(data(&doc), b"short\nendstream long", "the edited length reached another clone"); -} - -/// A file whose object 3 sits in an object stream PNG-predicted by `/DecodeParms 6 1 R`, a -/// generation the file doesn't have, and the parameters that make it decode. -fn predicted_object_stream() -> (Vec, Dict) { - let members = b"3 0 << /Hidden (yes) >>"; - // PNG Sub, one row. - let mut rows = vec![1u8]; - let mut before = 0u8; - for b in members { - rows.push(b.wrapping_sub(before)); - before = *b; - } - let coded = pdfcraft_filters::encode_flate(&rows); - let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 7]; - let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { - offs[n] = out.len(); - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); - obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); - let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Filter /FlateDecode /DecodeParms 6 1 R /Length {} >>\nstream\n", coded.len()).into_bytes(); - stm.extend_from_slice(&coded); - stm.extend_from_slice(b"\nendstream"); - obj(&mut out, 4, &stm); - let data = [2u8, 0, 4, 0]; - let mut xs = format!("<< /Type /XRef /Size 7 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); - xs.extend_from_slice(&data); - xs.extend_from_slice(b"\nendstream"); - obj(&mut out, 5, &xs); - obj(&mut out, 6, b"null"); - let table = out.len(); - out.extend_from_slice(b"xref\n0 7\n0000000000 65535 f \n"); - for (n, off) in offs.iter().enumerate().skip(1) { - let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; - out.extend_from_slice(line.as_bytes()); - } - out.extend_from_slice(format!("trailer << /Size 7 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(11)); - parms.set(b"Columns".to_vec(), Object::Int(members.len() as i64)); - (out, parms) -} - -fn hidden(doc: &Document) -> Option> { - doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.get(b"Hidden")).and_then(Object::as_string).map(|s| s.bytes.clone()) -} - -/// An object stream whose `/DecodeParms` names a generation the file doesn't have decodes without -/// them; a clone that sets that generation decodes with them, not by the stream another clone read. -#[test] -fn parameters_an_edit_makes_match_hide_the_object_stream_read_before() { - let (file, parms) = predicted_object_stream(); - let doc = Document::open(Arc::new(file)).unwrap(); - assert_eq!(hidden(&doc), None, "no object 6 1: no predictor undone"); - let mut edited = doc.clone(); - edited.set(ObjRef::new(6, 1), Object::Dict(parms)); - assert_eq!(hidden(&edited), Some(b"yes".to_vec()), "another clone's stream, decoded without them"); -} - -/// The other way round: an object stream a clone decoded by its edited parameters stays that -/// clone's, so the document it was cloned from still decodes by the file's. -#[test] -fn an_object_stream_decoded_by_edited_parameters_is_not_shared() { - let (file, parms) = predicted_object_stream(); - let doc = Document::open(Arc::new(file)).unwrap(); - let mut edited = doc.clone(); - edited.set(ObjRef::new(6, 1), Object::Dict(parms)); - assert_eq!(hidden(&edited), Some(b"yes".to_vec())); - assert_eq!(hidden(&doc), None, "the clone's decoding, read from the shared streams"); -} - -/// What a load read is kept only up to a bound; past it the load counts as having read every -/// object, so a document with any edit reads it again. -#[test] -fn a_load_that_read_too_much_counts_as_reading_everything() { - use super::state::{ReadFrame, note_read}; - let mut overlay = BTreeMap::new(); - overlay.insert(999_999, Slot::Freed(1)); - let frame = ReadFrame::enter(); - for n in 0..8 { - note_read(n, None); - } - assert!(!frame.reads().touch(&overlay), "a few reads, none edited"); - assert!(!frame.reads().touch(&BTreeMap::new()), "nothing is edited"); - for n in 8..100 { - note_read(n, None); - } - assert!(frame.reads().touch(&overlay), "more objects than an entry keeps"); - drop(frame); - let outer = ReadFrame::enter(); - let inner = ReadFrame::enter(); - for n in 0..5_000 { - note_read(n % 4, None); - } - assert!(inner.reads().touch(&overlay), "more reads than the thread keeps"); - drop(inner); - assert!(outer.reads().touch(&overlay), "an outer frame missed them too"); -} - -/// A member that failed to load from the file's object stream is kept as a failure, but replacing -/// or freeing that stream forgets it: the member then reads from the edit. -#[test] -fn an_edited_object_stream_forgets_its_members_failures() { - let file = Arc::new(object_stream_file("3 0 ", ") broken", 0)); - let mut doc = Document::open(file.clone()).unwrap(); - assert!(doc.try_get(3).is_err(), "the file's member does not parse"); - let mut dict = Dict::new(); - dict.set(b"Type".to_vec(), Object::name("ObjStm")); - dict.set(b"N".to_vec(), Object::Int(1)); - dict.set(b"First".to_vec(), Object::Int(4)); - doc.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (fixed) >>".to_vec())); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("fixed"), "the cached failure outlived the edit"); - assert!(doc.try_get(3).is_ok()); - let mut doc = Document::open(file).unwrap(); - assert!(matches!(doc.try_get(3), Err(CosError::Syntax { .. })), "the file's member does not parse"); - doc.free(ObjRef::new(4, 0)); - assert_eq!(doc.try_get(3), Err(CosError::MissingObject(4)), "the freed stream's member"); -} - -/// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: -/// the others still read its members within the file's length. -#[test] -fn an_edited_object_stream_length_leaves_other_clones_members_alone() { - let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); - let doc = Document::open(file).unwrap(); - let mut edited = doc.clone(); - // Ends the stream at the `endstream` inside the member's string. - let cut = "3 0 << /Hidden (cut".len(); - edited.set(ObjRef::new(6, 0), Object::Int(cut as i64)); - assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); -} - -/// A `/Length` read through one clone's edits and reused within the same load still keeps that -/// load to the clone. Here the file's offsets are header-relative behind junk, so object 10's -/// offset first lands on object 12, whose length (object 8, in the edited object stream 9) is -/// read; object 10's own length then loads object stream 4, framed by object 8 too. -#[test] -fn a_reused_edited_length_leaves_other_clones_members_alone() { - let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 13]; - let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { - offs[n] = out.len(); - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - let stream = |dict: &str, data: &str| format!("<< {dict} >>\nstream\n{data}\nendstream").into_bytes(); - obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); - obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); - let inner = "11 0 3 2 5 << /Hidden (cut endstream here) >>"; - obj(&mut out, 4, &stream("/Type /ObjStm /N 2 /First 9 /Length 8 0 R", inner)); - let lengths = format!("8 0 {}", inner.len()); - obj(&mut out, 9, &stream(&format!("/Type /ObjStm /N 1 /First 4 /Length {}", lengths.len()), &lengths)); - obj(&mut out, 12, &stream("/Length 8 0 R", &"x".repeat(40))); - obj(&mut out, 10, &stream("/Length 11 0 R", "PDATA")); - // A cross-reference stream: a hybrid file's /XRefStm is not looked for behind junk. - let compressed = [(3, 4, 1), (8, 9, 0), (11, 4, 0)]; - let mut data = Vec::new(); - for (n, off) in offs.iter().enumerate() { - let row = match compressed.iter().find(|c| c.0 == n) { - Some(&(_, stream, index)) => [2, 0, stream, index], - None if *off == 0 => [0, 0, 0, 0], - None => { - let [.., hi, lo] = u32::try_from(*off).unwrap().to_be_bytes(); - [1, hi, lo, 0] - } - }; - data.extend_from_slice(&row); - } - let table = out.len(); - let dict = format!("<< /Type /XRef /Size 14 /W [1 2 1] /Index [0 13] /Root 1 0 R /Length {} >>", data.len()); - out.extend_from_slice(format!("13 0 obj\n{dict}\nstream\n").as_bytes()); - out.extend_from_slice(&data); - out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{table}\n%%EOF\n").as_bytes()); - let mut file = vec![b'J'; offs[10] - offs[12] - 1]; - file.push(b'\n'); - file.extend_from_slice(&out); - let doc = Document::open(Arc::new(file)).unwrap(); - let mut edited = doc.clone(); - let cut = "11 0 3 2 5 << /Hidden (cut".len(); - let mut dict = Dict::new(); - dict.set(b"Type".to_vec(), Object::name("ObjStm")); - dict.set(b"N".to_vec(), Object::Int(1)); - dict.set(b"First".to_vec(), Object::Int(4)); - edited.set(ObjRef::new(9, 0), crate::Stream::from_raw(dict, format!("8 0 {cut}").into_bytes())); - assert!(matches!(&*edited.get(ObjRef::new(10, 0)), Object::Stream(s) if &s.raw[..] == b"PDATA")); - assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); -} - -/// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered -/// `1 n` instead of `0 n` (#2676): every row is one number off. -fn misnumbered_table(start: u32) -> Vec { - let bodies = ["<< /Marker (first object) >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Type /Catalog /Pages 2 0 R >>"]; - let mut out = b"%PDF-1.7 -" - .to_vec(); - let mut offsets = Vec::new(); - for (i, body) in bodies.iter().enumerate() { - offsets.push(out.len()); - out.extend_from_slice( - format!( - "{} 0 obj -{body} -endobj -", - i + 1 - ) - .as_bytes(), - ); - } - let xref = out.len(); - out.extend_from_slice(format!("xref\n{start} 4\n0000000000 65535 f \n").as_bytes()); - for o in offsets { - out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); - } - out.extend_from_slice(format!("trailer\n<< /Size 4 /Root 3 0 R >>\nstartxref\n{xref}\n%%EOF\n").as_bytes()); - out -} - -#[test] -fn a_table_numbered_from_1_that_begins_with_object_0_is_read_from_0() { - let doc = Document::open(Arc::new(misnumbered_table(1))).unwrap(); - let first = doc.get(ObjRef::new(1, 0)).as_dict().cloned().expect("object 1 is not free"); - assert!(first.contains(b"Marker")); - assert_eq!(doc.root(), Some(ObjRef::new(3, 0))); - assert!(doc.repair_log().iter().any(|l| l.contains("numbered from 1")), "{:?}", doc.repair_log()); - // A correct table reads the same and logs nothing. - let fine = Document::open(Arc::new(misnumbered_table(0))).unwrap(); - assert!(fine.get(ObjRef::new(1, 0)).as_dict().is_some_and(|d| d.contains(b"Marker"))); - assert!(!fine.repair_log().iter().any(|l| l.contains("numbered from 1"))); -} - -#[test] -fn a_table_that_really_starts_at_1_keeps_its_numbers() { - // Object 1 deleted for good (generation 65535) in a subsection that does start at 1: its - // second row locates object 2, so nothing is shifted. - let mut bytes = b"%PDF-1.7\n".to_vec(); - let pages = bytes.len(); - bytes.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); - let catalog = bytes.len(); - bytes.extend_from_slice(b"3 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); - let xref = bytes.len(); - bytes.extend_from_slice( - format!( - "xref\n0 1\n0000000000 65535 f \n1 3\n0000000000 65535 f \n{pages:010} 00000 n \n{catalog:010} 00000 n \ntrailer\n<< /Size 4 /Root 3 0 R >>\nstartxref\n{xref}\n%%EOF\n" - ) - .as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(doc.get(ObjRef::new(2, 0)).as_dict().is_some_and(|d| d.name(b"Type") == Some(b"Pages"))); - assert!(doc.get(ObjRef::new(1, 0)).as_dict().is_none(), "object 1 stays free"); - assert!(!doc.repair_log().iter().any(|l| l.contains("numbered from 1"))); -} - -/// A one-page scan: the page draws image 5 with content stream 4. -fn scanned_page() -> Vec { - build( - &[ - "<< /Type /Catalog /Pages 2 0 R >>", - "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 10 10] /Resources << /XObject << /Im0 5 0 R >> >> /Contents 4 0 R >>", - "<< /Length 21 >>\nstream\n10 0 0 10 0 0 cm /Im0 Do\nendstream", - "<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>\nstream\nA\nendstream", - ], - "/Root 1 0 R", - ) -} - -/// `bytes` with its cross-reference table replaced by `table` (the lines after `xref`). -fn with_table(bytes: &[u8], table: &[String], size: usize) -> Vec { - let text = String::from_utf8_lossy(bytes).into_owned(); - let at = text.rfind("\nxref\n").unwrap() + 1; - let mut out = bytes[..at].to_vec(); - out.extend_from_slice(format!("xref\n{}\ntrailer\n<< /Size {size} /Root 1 0 R >>\nstartxref\n{at}\n%%EOF\n", table.join("\n")).as_bytes()); - out -} - -/// The table's entry lines, object 0 first. -fn table_entries(bytes: &[u8]) -> Vec { - let text = String::from_utf8_lossy(bytes).into_owned(); - let at = text.rfind("\nxref\n").unwrap() + 1; - text[at..].lines().skip(2).take_while(|l| !l.starts_with("trailer")).map(str::to_string).collect() -} - -/// The classic off-by-one: the subsection says it starts at 0 but lists objects 1 to 5, so -/// the last object has no entry. Viewers rebuild the table and show the page; a reader that -/// took the missing entry for a null object dropped the image when pages were copied (#1628). -#[test] -fn an_off_by_one_xref_subsection_still_finds_the_last_object() { - let base = scanned_page(); - let entries = table_entries(&base); - let mut table = vec!["0 5".to_string()]; - table.extend(entries[1..6].iter().cloned()); - let doc = Document::open(Arc::new(with_table(&base, &table, 5))).unwrap(); - let image = doc.get(ObjRef::new(5, 0)); - assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "{image:?}"); - assert!(matches!(&*image, Object::Stream(_))); - assert!(matches!(&*doc.get(ObjRef::new(4, 0)), Object::Stream(s) if &s.raw[..] == b"10 0 0 10 0 0 cm /Im0 Do")); - assert!(doc.repair_log().iter().any(|l| l.contains("object headers")), "{:?}", doc.repair_log()); -} - -/// An "in use" entry at offset 0 says nothing about where the object is: the object is -/// found by its header, and the repair is recorded. -#[test] -fn an_object_listed_at_offset_zero_is_found_by_its_header() { - let base = scanned_page(); - for num in [4usize, 5] { - let mut table = vec!["0 6".to_string()]; - table.extend(table_entries(&base)); - table[num + 1] = "0000000000 00000 n ".into(); - let doc = Document::open(Arc::new(with_table(&base, &table, 6))).unwrap(); - assert!(matches!(&*doc.get(ObjRef::new(num as u32, 0)), Object::Stream(_)), "object {num}"); - assert!(doc.repair_log().iter().any(|l| l.contains("object headers")), "object {num}: {:?}", doc.repair_log()); - } -} - -/// A valid table that simply does not list every number below `/Size` is not repaired: an -/// unlisted number is a null object (§7.3.10), even when an old body with that number is -/// still in the file. -#[test] -fn a_consistent_table_with_unlisted_numbers_is_taken_as_it_is() { - let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); - let at = String::from_utf8_lossy(&base).rfind("\nxref\n").unwrap() + 1; - let mut bytes = base[..at].to_vec(); - bytes.extend_from_slice(b"4 0 obj\n(stale)\nendobj\n"); - let mut table = vec!["0 4".to_string()]; - table.extend(table_entries(&base)); - let doc = Document::open(Arc::new(with_table(&[&bytes[..], &base[at..]].concat(), &table, 6))).unwrap(); - assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); - assert_eq!(*doc.get(ObjRef::new(4, 0)), Object::Null); - assert!(matches!(&*doc.get(ObjRef::new(3, 0)), Object::String(_))); -} - -/// Filling the gaps of a damaged table never brings back an object that an incremental -/// update freed: its body is still in the file, but the newer section says it is gone. -#[test] -fn an_object_freed_by_an_update_stays_freed_when_the_table_is_repaired() { - let base = scanned_page(); - // The original table lists the content stream at offset 0, so the gaps get filled. - let mut table = vec!["0 6".to_string()]; - table.extend(table_entries(&base)); - table[5] = "0000000000 00000 n ".into(); - let mut bytes = with_table(&base, &table, 6); - let prev = String::from_utf8_lossy(&bytes).rfind("\nxref\n").unwrap() + 1; - let section = bytes.len(); - bytes.extend_from_slice( - format!("xref\n5 1\n0000000000 00001 f \ntrailer\n<< /Size 6 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(matches!(&*doc.get(ObjRef::new(4, 0)), Object::Stream(_)), "the gap is filled"); - assert_eq!(*doc.get(ObjRef::new(5, 0)), Object::Null, "the freed image stays freed"); - assert_eq!(doc.xref_entry(5), Some(XrefEntry::Free { next_generation: 1 })); -} - -/// Repairing a damaged table fills only the numbers the damage leaves out: an old body of a -/// number no section lists, under an oversized `/Size`, stays a null object. -#[test] -fn a_repaired_table_leaves_an_unlisted_number_null() { - let base = scanned_page(); - let at = String::from_utf8_lossy(&base).rfind("\nxref\n").unwrap() + 1; - let entries = table_entries(&base); - let mut offset_zero = vec!["0 6".to_string()]; - offset_zero.extend(entries.iter().cloned()); - offset_zero[5] = "0000000000 00000 n ".into(); - let mut off_by_one = vec!["0 5".to_string()]; - off_by_one.extend(entries[1..6].iter().cloned()); - for (name, table, found) in [("offset 0", offset_zero, 4), ("off by one", off_by_one, 5)] { - let bytes = [&base[..at], b"7 0 obj\n(stale)\nendobj\n", &base[at..]].concat(); - let doc = Document::open(Arc::new(with_table(&bytes, &table, 9))).unwrap(); - assert!(matches!(&*doc.get(ObjRef::new(found, 0)), Object::Stream(_)), "{name}: the gap is filled"); - assert_eq!(*doc.get(ObjRef::new(7, 0)), Object::Null, "{name}: {:?}", doc.repair_log()); - assert_eq!(doc.xref_entry(7), None, "{name}"); - } -} - -/// The last object of a subsection numbered one too low is filled from that subsection, -/// whatever later happens to the row before it: an incremental save that rewrites object 4, -/// or object 4's own row at offset 0. -#[test] -fn the_last_object_of_a_shifted_subsection_is_filled_whatever_its_neighbour_says() { - let base = scanned_page(); - let entries = table_entries(&base); - let mut off_by_one = vec!["0 5".to_string()]; - off_by_one.extend(entries[1..6].iter().cloned()); - let shifted = with_table(&base, &off_by_one, 5); - let prev = String::from_utf8_lossy(&shifted).rfind("\nxref\n").unwrap() + 1; - let mut updated = shifted.clone(); - let body = updated.len(); - updated.extend_from_slice(b"4 0 obj\n<< /Length 21 >>\nstream\n10 0 0 10 0 0 cm /Im0 Do\nendstream\nendobj\n"); - let section = updated.len(); - updated.extend_from_slice( - format!("xref\n4 1\n{body:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R /Prev {prev} >>\nstartxref\n{section}\n%%EOF\n").as_bytes(), - ); - let mut neighbour_at_zero = off_by_one.clone(); - neighbour_at_zero[5] = "0000000000 00000 n ".into(); - for (name, bytes) in [("updated", updated), ("row 4 at offset 0", with_table(&base, &neighbour_at_zero, 5))] { - let doc = Document::open(Arc::new(bytes)).unwrap(); - let image = doc.get(ObjRef::new(5, 0)); - assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "{name}: {:?}", doc.repair_log()); - } -} - -/// The image the first page draws as `/Im1`, followed through the page tree. -fn first_page_image(doc: &Document) -> Arc { - let catalog = doc.get(doc.root().unwrap()); - let pages = doc.resolve(catalog.as_dict().unwrap().get(b"Pages").unwrap()); - let page = doc.resolve(&pages.as_dict().unwrap().get(b"Kids").unwrap().as_array().unwrap()[0]); - let resources = doc.resolve(page.as_dict().unwrap().get(b"Resources").unwrap()); - let xobjects = doc.resolve(resources.as_dict().unwrap().get(b"XObject").unwrap()); - doc.resolve(xobjects.as_dict().unwrap().get(b"Im1").unwrap()) -} - -/// The opposite off-by-one (#1628): the subsection says it starts at 1, but its first entry -/// is object 0's free-list head, so every entry lands one number too high and object 1, the -/// image, reads as free. The offset the table gives object 2 holds `1 0 obj`, so the -/// subsection is read from 0, and the image survives a full save. -#[test] -fn a_subsection_from_one_that_opens_with_the_free_list_head_is_read_from_zero() { - let base = build( - &[ - "<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceRGB /BitsPerComponent 8 /Length 12 >>\nstream\nABCDEFGHIJKL\nendstream", - "<< /Length 29 >>\nstream\nq 72 0 0 72 0 0 cm /Im1 Do Q\nendstream", - "<< /Type /Page /Parent 4 0 R /MediaBox [0 0 72 72] /Resources << /XObject << /Im1 1 0 R >> >> /Contents 2 0 R >>", - "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", - "<< /Type /Catalog /Pages 4 0 R >>", - "<< /Subject (synthetic) >>", - ], - "/Root 5 0 R /Info 6 0 R", - ); - // /Size 7 is one short of the shifted numbers; /Size 8 covers them all. Junk before the - // header moves every object past the offsets the table gives; it is not a comment, which - // the lexer would skip on its way to `1 0 obj` anyway. - for (size, junk) in [(7, ""), (8, ""), (7, "Content-Type: application/pdf\n\n")] { - let text = String::from_utf8_lossy(&base).replace("\nxref\n0 7\n", "\nxref\n1 7\n").replace("/Size 7 ", &format!("/Size {size} ")); - let doc = Document::open(Arc::new(format!("{junk}{text}").into_bytes())).unwrap(); - let image = first_page_image(&doc); - assert_eq!(image.as_dict().and_then(|d| d.name(b"Subtype")), Some(&b"Image"[..]), "/Size {size} {junk:?}: {image:?}"); - assert_eq!(doc.xref_entry(0), Some(XrefEntry::Free { next_generation: 65535 }), "/Size {size} {junk:?}"); - assert!(doc.repair_log().iter().any(|l| l.contains("from 0")), "/Size {size} {junk:?}: {:?}", doc.repair_log()); - let saved = Document::open(Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap())).unwrap(); - let image = first_page_image(&saved); - assert!(matches!(&*image, Object::Stream(s) if &s.raw[..] == b"ABCDEFGHIJKL"), "/Size {size} {junk:?}: {image:?}"); - } -} - -/// Object 1 really freed: the subsection from 1 opens with a free entry, but the next entry's -/// offset holds object 2, not object 1. The table is taken as it is, and the old body of -/// object 1 still in the file stays freed. -#[test] -fn a_subsection_from_one_whose_next_offset_holds_object_two_is_taken_as_it_is() { - let base = build(&["(freed)", "<< /Type /Catalog /Pages 3 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 2 0 R"); - let entries = table_entries(&base); - let mut table = vec!["1 3".to_string(), "0000000000 65535 f ".to_string()]; - table.extend(entries[2..].iter().cloned()); - let text = String::from_utf8_lossy(&base).into_owned(); - let at = text.rfind("\nxref\n").unwrap() + 1; - for junk in ["", "Content-Type: application/pdf\n\n"] { - let mut bytes = junk.as_bytes().to_vec(); - bytes.extend_from_slice(&base[..at]); - bytes.extend_from_slice(format!("xref\n{}\ntrailer\n<< /Size 4 /Root 2 0 R >>\nstartxref\n{at}\n%%EOF\n", table.join("\n")).as_bytes()); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.xref_entry(1), Some(XrefEntry::Free { next_generation: 65535 }), "{junk:?}"); - assert_eq!(*doc.get(ObjRef::new(1, 0)), Object::Null, "{junk:?}: the freed object stays freed"); - assert!(matches!(&*doc.get(ObjRef::new(2, 0)), Object::Dict(d) if d.name(b"Type") == Some(&b"Catalog"[..])), "{junk:?}"); - assert!(!doc.repair_log().iter().any(|l| l.contains("from 0")), "{junk:?}: {:?}", doc.repair_log()); - assert!(!junk.is_empty() || doc.repair_log().is_empty(), "{:?}", doc.repair_log()); - } -} - -#[test] -fn opening_an_exhausted_classic_xref_repairs_and_allocates() { - // The extra in-use row need not be loaded to exhaust the next object number. - let bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\nxref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n0000000058 00000 n \n4294967295 1\n0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\nstartxref\n110\n%%EOF\n"; - assert_eq!(bytes.len(), 266); - let mut doc = Document::open(Arc::new(bytes.to_vec())).unwrap(); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - assert!(doc.repair_log().iter().any(|line| line.contains("exhausted xref"))); - assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); -} - -#[test] -fn opening_an_exhausted_xref_stream_repairs_and_allocates() { - let mut bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n".to_vec(); - let offset = bytes.len(); - bytes.extend_from_slice(b"3 0 obj\n<< /Type /XRef /Size 4 /Root 1 0 R /W [1 4 2] /Index [0 3 4294967295 1] /Length 28 >>\nstream\n"); - for (kind, position, generation) in [(0_u8, 0_u32, 65535_u16), (1, 9, 0), (1, 58, 0), (1, 58, 0)] { - bytes.push(kind); - bytes.extend_from_slice(&position.to_be_bytes()); - bytes.extend_from_slice(&generation.to_be_bytes()); - } - bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{offset}\n%%EOF\n").as_bytes()); - let mut doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - assert!(doc.repair_log().iter().any(|line| line.contains("exhausted xref"))); - assert_eq!(doc.add(Object::Null), ObjRef::new(4, 0)); -} - -#[test] -fn exhausted_size_and_high_edits_allocate_without_overwriting_objects() { - let ordinary = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - for size in ["4294967295", "4294967296", "9223372036854775807"] { - let bytes = String::from_utf8(ordinary.clone()).unwrap().replace("/Size 3", &format!("/Size {size}")); - let mut doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); - assert!(doc.repair_log().iter().any(|line| line.contains("exhausted trailer"))); - assert_eq!(doc.add(Object::Int(42)), ObjRef::new(3, 0)); - doc.set(ObjRef::new(u32::MAX, 0), Object::Int(99)); - doc.set(ObjRef::new(u32::MAX - 1, 0), Object::Int(98)); - assert_eq!(doc.add(Object::Int(43)), ObjRef::new(4, 0)); - assert_eq!(doc.get(ObjRef::new(3, 0)).as_int(), Some(42)); - assert_eq!(doc.get(ObjRef::new(u32::MAX, 0)).as_int(), Some(99)); - assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); - } - let mut doc = Document::new_empty(); - doc.set(ObjRef::new(u32::MAX - 2, 0), Object::Null); - assert_eq!(doc.add(Object::Null), ObjRef::new(u32::MAX - 1, 0)); - assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); - assert_eq!(doc.add(Object::Null), ObjRef::new(4, 0)); -} - -#[test] -fn exhausted_xref_controls_preserve_ordinary_ignored_and_repaired_entries() { - let ordinary = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - for (extra, repaired) in - [("", false), ("4294967295 1\n0000000000 00000 n \n", false), ("4294967295 2\n0000000058 00000 n \n0000000058 00000 n \n", true)] - { - let bytes = String::from_utf8(ordinary.clone()).unwrap().replacen("trailer\n", &format!("{extra}trailer\n"), 1); - let mut doc = Document::open(Arc::new(bytes.into_bytes())).unwrap(); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); - assert_eq!(!doc.repair_log().is_empty(), repaired); - if repaired { - assert!(doc.repair_log().iter().any(|line| line.contains("xref") && line.contains("range"))); - } - assert_eq!(doc.add(Object::Null), ObjRef::new(3, 0)); - } -} - -#[test] -fn invalid_xref_object_ranges_are_reconstructed() { - // Original in-memory PDF: an intact catalog followed by a damaged xref section. - for stream in [false, true] { - for (start, count) in [(i64::MAX, 2), (-1, 2), (4_294_967_296, 1), (4_294_967_295, 2)] { - let mut bytes = - b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n".to_vec(); - let offset = bytes.len(); - if stream { - bytes.extend_from_slice( - format!("3 0 obj\n<< /Type /XRef /Root 1 0 R /Size 4 /W [1 1 1] /Index [{start} {count}] /Length 6 >>\nstream\n").as_bytes(), - ); - bytes.extend_from_slice(&[0, 0, 0, 0, 0, 0]); - bytes.extend_from_slice(b"\nendstream\nendobj\n"); - } else { - bytes.extend_from_slice( - format!("xref\n{start} {count}\n0000000000 65535 f \n0000000000 65535 f \ntrailer\n<< /Root 1 0 R /Size 3 >>\n").as_bytes(), - ); - } - bytes.extend_from_slice(format!("startxref\n{offset}\n%%EOF\n").as_bytes()); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!( - doc.repair_log().iter().any(|line| line.contains("xref") && line.contains("range")), - "stream={stream}, start={start}, count={count}: {:?}", - doc.repair_log() - ); - assert_eq!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().name(b"Type"), Some(b"Catalog".as_slice())); - } - } -} - -#[test] -fn objects_only_an_unreadable_older_section_locates_are_found_by_scanning() { - // The newest section lists the catalog only; its /Prev points past the end of the file, - // so the page tree is reachable only by scanning object headers, as readers do (#1656). - let mut bytes = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for (num, body) in - [(1, "<< /Type /Catalog /Pages 2 0 R >>"), (2, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>"), (3, "<< /Type /Page /Parent 2 0 R >>")] - { - offsets.push(bytes.len()); - bytes.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); - } - let xref = bytes.len(); - bytes.extend_from_slice( - format!( - "xref\n0 2\n0000000000 65535 f \n{:010} 00000 n \ntrailer\n<< /Root 1 0 R /Size 4 /Prev 999999 >>\nstartxref\n{xref}\n%%EOF\n", - offsets[0] - ) - .as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.get(ObjRef::new(2, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(b"Pages".as_slice())); - assert_eq!(doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(b"Page".as_slice())); - assert!(doc.repair_log().iter().any(|l| l.contains("999999")), "{:?}", doc.repair_log()); -} - -#[test] -fn self_referential_length_does_not_overflow() { - let bytes = build( - &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Length 3 0 R >>\nstream\nabc\nendstream"], - "/Root 1 0 R", - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - // Either recovered via the endstream search or reported — but never a stack overflow. - let o = doc.get(ObjRef::new(3, 0)); - if let Object::Stream(s) = o.as_ref() { - assert_eq!(&s.raw[..], b"abc"); - } -} - -#[test] -fn mutually_referential_lengths_do_not_overflow() { - let bytes = build( - &[ - "<< /Type /Catalog /Pages 2 0 R >>", - "<< /Type /Pages /Kids [] /Count 0 >>", - "<< /Length 4 0 R >>\nstream\nabc\nendstream", - "<< /Length 3 0 R >>\nstream\nxyz\nendstream", - ], - "/Root 1 0 R", - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - let _ = doc.get(ObjRef::new(3, 0)); - let _ = doc.get(ObjRef::new(4, 0)); -} - -#[test] -fn reads_classic_revision_and_edits_are_copy_on_write() { - let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.revisions().len(), 1); - assert!(doc.repair_log().is_empty()); - let mut edited = doc.clone(); - edited.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(crate::PdfString::literal("en")))).unwrap(); - assert!(edited.is_modified() && !doc.is_modified(), "clone is an independent snapshot"); - assert!(doc.get(ObjRef::new(1, 0)).as_dict().unwrap().get(b"Lang").is_none()); -} - -#[test] -fn object_numbers_merge_source_and_overlay_in_order() { - let mut doc = Document::new_empty(); - doc.overlay.clear(); - doc.entries = Arc::new(XrefIndex::Materialized( - Arc::new(BTreeMap::from([ - (0, XrefEntry::Free { next_generation: 65535 }), - (2, XrefEntry::InFile { offset: 10, generation: 7 }), - (4, XrefEntry::Free { next_generation: 9 }), - (6, XrefEntry::InStream { stream: 80, index: 1 }), - (8, XrefEntry::InFile { offset: 20, generation: 5 }), - (10, XrefEntry::InFile { offset: 30, generation: 3 }), - (u32::MAX, XrefEntry::InFile { offset: 40, generation: 1 }), - ])), - Arc::default(), - )); - doc.set(ObjRef::new(1, 2), Object::Int(1)); // before the first live source slot - doc.set(ObjRef::new(2, 8), Object::Int(2)); // override, without duplication - doc.set(ObjRef::new(4, 9), Object::Int(4)); // reuse a source free slot - doc.free(ObjRef::new(6, 0)); // remove a compressed source object - doc.free(ObjRef::new(7, 2)); // free a number absent from the source - doc.free(ObjRef::new(10, 3)); - doc.set(ObjRef::new(11, 4), Object::Int(11)); - assert_eq!(doc.object_numbers(), vec![1, 2, 4, 8, 11, u32::MAX]); - assert_eq!(doc.generation(2), 8); - assert_eq!(doc.generation(4), 9); - assert_eq!(doc.generation(6), 1); - assert_eq!(doc.generation(8), 5); - assert_eq!(doc.generation(10), 4); - doc.overlay.clear(); - assert_eq!(doc.object_numbers(), vec![2, 6, 8, 10, u32::MAX]); - doc.entries = Arc::new(XrefIndex::empty()); - assert!(doc.object_numbers().is_empty()); - doc.set(ObjRef::new(5, 7), Object::Int(5)); - assert_eq!(doc.object_numbers(), vec![5]); -} - -#[test] -fn full_object_scans_preserve_edits_without_retaining_the_scan() { - let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>".to_owned(), "<< /Type /Pages /Kids [] /Count 0 >>".to_owned()]; - bodies.extend((0..400).map(|i| format!("<< /Value {i} >>"))); - let refs: Vec<_> = bodies.iter().map(String::as_str).collect(); - let mut doc = Document::open(Arc::new(build(&refs, "/Root 1 0 R"))).unwrap(); - doc.set(ObjRef::new(3, 0), Object::Int(900)); - doc.free(ObjRef::new(4, 0)); - let added = doc.add(Object::Int(901)); - let cached = doc.cache.lock().unwrap().len(); - let held: Vec<_> = doc.scan_objects().collect(); - assert_eq!(held.len(), 402); - assert_eq!(doc.cache.lock().unwrap().len(), cached, "scanning must not populate the editor cache"); - assert!(doc.objstms.lock().unwrap().is_empty()); - assert!(!held.iter().any(|(r, _)| r.num == 4)); - assert_eq!(held.iter().find(|(r, _)| r.num == 3).unwrap().1.as_int(), Some(900)); - assert_eq!(held.iter().find(|(r, _)| *r == added).unwrap().1.as_int(), Some(901)); - assert_eq!(held.iter().find(|(r, _)| r.num == 400).unwrap().1.as_dict().unwrap().int(b"Value"), Some(397)); - // Previously returned objects survive cache eviction and dropping the source. - drop(doc); - assert_eq!(held.iter().find(|(r, _)| r.num == 128).unwrap().1.as_dict().unwrap().int(b"Value"), Some(125)); -} - -#[test] -fn temporary_reader_keeps_file_views_and_evicts_data_of_its_own() { - // A stream read from the file is a view into the input: caching it costs no copy, so - // even a large one stays cached (its /Length object too) instead of being re-parsed. - let body = "x".repeat(ObjectReader::STREAM_BYTES + 1); - let stream = format!("<< /Length 4 0 R >>\nstream\n{body}\nendstream"); - let length = body.len().to_string(); - let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", &stream, &length], "/Root 1 0 R"); - let doc = Document::open(Arc::new(bytes)).unwrap(); - let mut reader = doc.object_reader(); - let read = reader.get(ObjRef::new(3, 0)); - let Object::Stream(s) = &*read else { panic!("stream") }; - assert!(s.raw.shares(&reader.document.data), "no copy of the file's bytes"); - assert!(reader.document.cache.lock().unwrap().contains_key(&3), "a view costs nothing to keep"); - // Data of its own (decrypted streams, strings) counts: past the budget the cache is - // cleared after the next read, including a failed one. - let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - let size = held_bytes(&owned, &reader.document.data, 0); - reader.document.cache.lock().unwrap().insert(3, Ok(Arc::new(owned)), size, Default::default()); - assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); - assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); -} - -#[test] -fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { - let mut source = Document::new_empty(); - let objects: Vec<_> = (0..260) - .map(|i| { - let mut value = vec![b'x'; 48 * 1024]; - value[..4].copy_from_slice(&(i as u32).to_le_bytes()); - Object::Ref(source.add(Object::String(crate::PdfString::literal(value)))) - }) - .collect(); - source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); - let bytes = Arc::new(crate::write_full(&source, &crate::SaveOptions::default()).unwrap()); - let doc = Document::open(bytes).unwrap(); - let root = doc.get(doc.root().unwrap()); - let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap(); - let snapshot = doc.clone(); - let cached = doc.cache.lock().unwrap().len(); - let streams = doc.objstms.lock().unwrap().len(); - let mut reader = doc.object_reader(); - let mut held = None; - for (i, reference) in references.iter().enumerate() { - let Object::Ref(reference) = reference else { panic!("indirect object") }; - let object = reader.get(*reference); - assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); - if i == 0 { - held = Some(object); - } - assert!(reader.document.cache.lock().unwrap().len() < ObjectReader::OBJECTS); - let decoded = reader.document.objstms.lock().unwrap(); - assert!(decoded.len() < ObjectReader::OBJECTS); - let bytes: usize = decoded.values().map(|s| s.data.capacity() + s.index.capacity() * std::mem::size_of::>()).sum(); - assert!(bytes <= ObjectReader::STREAM_BYTES || decoded.len() == 1, "only one oversized stream may remain"); - } - drop(reader); - assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes(), "owned results survive evictions"); - for object_streams in [false, true] { - let saved = crate::write_full(&doc, &crate::SaveOptions { object_streams, ..Default::default() }).unwrap(); - assert!(Document::open(Arc::new(saved)).unwrap().repair_log().is_empty()); - assert_eq!(doc.cache.lock().unwrap().len(), cached, "full saves must not fill the editor cache"); - assert_eq!(doc.objstms.lock().unwrap().len(), streams); - assert!(Arc::ptr_eq(&doc.cache, &snapshot.cache), "do not detach or clear caller snapshots"); - } -} - -#[test] -fn streams_without_length_share_one_endstream_search() { - let n = 2_000; - let mut file = b"%PDF-1.7\n".to_vec(); - for k in 1..=n { - file.extend_from_slice(format!("{k} 0 obj\n<< >>\nstream\nxx\n").as_bytes()); - } - file.extend_from_slice(b"endstream\nendobj\n"); - file.extend_from_slice( - format!("{0} 0 obj\n<< /Type /Catalog /Pages {1} 0 R >>\nendobj\n{1} 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n", n + 1, n + 2) - .as_bytes(), - ); - file.extend_from_slice(format!("trailer\n<< /Root {} 0 R /Size {} >>\n%%EOF\n", n + 1, n + 3).as_bytes()); - let len = file.len(); - crate::parser::ENDSTREAM_SCAN.with(|c| c.set(0)); - let doc = Document::open(Arc::new(file)).unwrap(); - for k in [1, n / 2, n] { - assert!(matches!(doc.try_get(k).as_deref(), Ok(Object::Stream(_))), "object {k}"); - } - let scanned = crate::parser::ENDSTREAM_SCAN.with(|c| c.get()); - assert!(scanned <= 4 * len, "{scanned} bytes searched in a {len}-byte file"); -} - -#[test] -fn editor_cache_counts_deeply_nested_strings() { - let mut nested = Object::String(crate::PdfString::literal(vec![b'x'; 1 << 20])); - for _ in 0..8 { - nested = Object::Array(vec![nested]); - } - let mut source = Document::new_empty(); - let objects: Vec<_> = (0..48).map(|_| Object::Ref(source.add(nested.clone()))).collect(); - source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); - let bytes = Arc::new(crate::write_full(&source, &crate::SaveOptions::default()).unwrap()); - let doc = Document::open(bytes).unwrap(); - let held = held_bytes(&nested, &doc.data, 0); - assert!((1 << 20..(1 << 20) + 4096).contains(&held), "the string and a little tree: {held}"); - let root = doc.get(doc.root().unwrap()); - for reference in root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap() { - let Object::Ref(reference) = reference else { panic!("indirect object") }; - drop(doc.get(*reference)); - let cache = doc.cache.lock().unwrap(); - let owned = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); - let strings = cache.values().filter(|o| o.as_array().is_some()).count(); - assert!(strings << 20 <= Document::EDITOR_CACHE_BYTES + (1 << 20), "{strings} MiB strings cached, {owned} bytes counted"); - } -} - -#[test] -fn editor_cache_stays_bounded() { - let mut source = Document::new_empty(); - let objects: Vec<_> = (0..(Document::EDITOR_CACHE_OBJECTS + 64)) - .map(|i| { - let mut value = vec![b'x'; 1024]; - value[..4].copy_from_slice(&(i as u32).to_le_bytes()); - Object::Ref(source.add(Object::String(crate::PdfString::literal(value)))) - }) - .collect(); - source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); - let bytes = Arc::new(crate::write_full(&source, &crate::SaveOptions::default()).unwrap()); - let doc = Document::open(bytes).unwrap(); - let root = doc.get(doc.root().unwrap()); - let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap().clone(); - let mut held = None; - for (i, reference) in references.iter().enumerate() { - let Object::Ref(reference) = reference else { panic!("indirect object") }; - let object = doc.get(*reference); - assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); - if i == 0 { - held = Some(object); - } - assert!(doc.cache.lock().unwrap().len() <= Document::EDITOR_CACHE_OBJECTS, "editing must not pin the whole object graph"); - // Eviction keeps the newcomer. - assert!(doc.cache.lock().unwrap().contains_key(&reference.num), "eviction must keep the newcomer"); - } - // Returned objects survive eviction, through both handles of the shared cache. - assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); - let snapshot = doc.clone(); - let first = references[0].as_ref().unwrap(); - assert_eq!(&snapshot.get(first).as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); -} - -#[test] -fn editor_cache_byte_bound_fires() { - // 40 × 1 MB strings: far below the count cap, above the 32 MB byte budget, which is - // checked after every insertion. - let mut source = Document::new_empty(); - let objects: Vec<_> = (0..40) - .map(|i| { - let mut value = vec![b'y'; 1024 * 1024]; - value[..4].copy_from_slice(&(i as u32).to_le_bytes()); - Object::Ref(source.add(Object::String(crate::PdfString::literal(value)))) - }) - .collect(); - source.update_dict(source.root().unwrap(), |d| d.set(b"Extension".to_vec(), Object::Array(objects))).unwrap(); - let bytes = Arc::new(crate::write_full(&source, &crate::SaveOptions::default()).unwrap()); - let doc = Document::open(bytes).unwrap(); - let root = doc.get(doc.root().unwrap()); - let references = root.as_dict().unwrap().get(b"Extension").unwrap().as_array().unwrap().clone(); - let mut held = None; - for (i, reference) in references.iter().enumerate() { - let Object::Ref(reference) = reference else { panic!("indirect object") }; - let object = doc.get(*reference); - assert_eq!(&object.as_string().unwrap().bytes[..4], &(i as u32).to_le_bytes()); - if i == 0 { - held = Some(object); - } - // Eviction keeps the newcomer. - assert!(doc.cache.lock().unwrap().contains_key(&reference.num), "eviction must keep the newcomer"); - } - assert!(doc.cache.lock().unwrap().len() < references.len(), "the byte budget must evict even below the count cap"); - let last = references[references.len() - 1].as_ref().unwrap(); - assert!(doc.cache.lock().unwrap().contains_key(&last.num), "eviction must keep the newcomer"); - assert_eq!(&held.unwrap().as_string().unwrap().bytes[..4], &0u32.to_le_bytes()); -} - -#[test] -fn a_stream_limited_document_cannot_be_saved() { - let bytes = Arc::new(crate::write_full(&Document::new_empty(), &crate::SaveOptions::default()).unwrap()); - let doc = Document::open_with_stream_limit(bytes.clone(), None, 1 << 20).unwrap(); - for saved in [crate::write_full(&doc, &crate::SaveOptions::default()), crate::write_incremental(&doc, &crate::SaveOptions::default())] { - assert!(matches!(saved, Err(CosError::ReadOnlyLimit)), "{saved:?}"); - } - let doc = Document::open(bytes).unwrap(); - assert!(crate::write_full(&doc, &crate::SaveOptions::default()).is_ok()); -} - -#[test] -fn inspection_stream_limit_applies_before_loading_compressed_objects() { - let mut doc = Document::new_empty(); - let large = doc.add(Object::String(crate::PdfString::literal(vec![b'x'; 4096]))); - doc.update_dict(doc.root().unwrap(), |catalog| catalog.set(b"Extension".to_vec(), Object::Ref(large))).unwrap(); - let bytes = Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap()); - assert!(Document::open(bytes.clone()).is_ok()); - assert!(Document::open_with_stream_limit(bytes, None, 512).is_err()); -} - -/// A /Prev chain that loops back on itself stops at the loop (and says so). -#[test] -fn looping_revision_chains_stop() { - let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - // An update whose /Prev points at itself. - let at = bytes.len(); - bytes.extend_from_slice( - format!("xref\n0 1\n0000000000 65535 f \ntrailer << /Size 3 /Root 1 0 R /Prev {at} >>\nstartxref\n{at}\n%%EOF\n").as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - assert!(doc.repair_log().iter().any(|l| l.contains("loops")), "{:?}", doc.repair_log()); -} - -/// A hybrid-reference file: the classic table leaves object 3 out; the `/XRefStm` stream -/// says it lives in object stream 4. -#[test] -fn hybrid_reference_files_read_the_xref_stream_too() { - let doc = Document::open(Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0))).unwrap(); - assert!(doc.repair_log().is_empty(), "{:?}", doc.repair_log()); - assert_compact_matches_materialized(&doc); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); -} - -/// A hybrid-reference file whose classic table leaves object 3 out, and whose `/XRefStm` -/// stream puts it at position `member` of object stream 4, which holds `header` (the number -/// and offset pairs) and then `members`. -fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { - object_stream_file_with(header, members, member, false) -} - -/// [`object_stream_file`], with the stream's `/Length` in object 6 when `indirect_length`. -fn object_stream_file_with(header: &str, members: &str, member: u8, indirect_length: bool) -> Vec { - let mut out = b"%PDF-1.5\n".to_vec(); - let size = if indirect_length { 7 } else { 6 }; - let mut offs = vec![0usize; size]; - let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { - offs[n] = out.len(); - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); - obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); - let count = header.split_whitespace().count() / 2; - let inner = format!("{header}{members}"); - let length = if indirect_length { "6 0 R".to_string() } else { inner.len().to_string() }; - let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {length} >>\nstream\n", header.len()).into_bytes(); - stm.extend_from_slice(inner.as_bytes()); - stm.extend_from_slice(b"\nendstream"); - obj(&mut out, 4, &stm); - let data = [2u8, 0, 4, member]; - let mut xs = format!("<< /Type /XRef /Size 6 /W [1 2 1] /Index [3 1] /Length {} >>\nstream\n", data.len()).into_bytes(); - xs.extend_from_slice(&data); - xs.extend_from_slice(b"\nendstream"); - obj(&mut out, 5, &xs); - if indirect_length { - obj(&mut out, 6, inner.len().to_string().as_bytes()); - } - let table = out.len(); - out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); - for (n, off) in offs.iter().enumerate().skip(1) { - let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; - out.extend_from_slice(line.as_bytes()); - } - out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); - out -} - -/// Many revisions naming one hybrid /XRefStm read it once. -#[test] -fn a_hybrid_stream_named_by_many_revisions_is_read_once() { - let mut out = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); - let stream = find(&out, b"5 0 obj", 0, out.len()).unwrap(); - let mut prev = rfind(&out, b"\nxref\n", 0).unwrap() + 1; - let revisions = 200; - for _ in 0..revisions { - let at = out.len(); - out.extend_from_slice(format!("xref\ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {prev} >>\nstartxref\n{at}\n%%EOF\n").as_bytes()); - prev = at; - } - let doc = Document::open(Arc::new(out)).unwrap(); - SECTION_READS.with(|c| c.set(0)); - let (entries, ..) = doc.read_xref_chain_materialized(&mut Vec::new()).unwrap(); - let reads = SECTION_READS.with(|c| c.get()); - assert!(reads <= revisions + 2, "{reads} sections read for {} revisions", revisions + 1); - assert!(matches!(entries.get(&3), Some(XrefEntry::InStream { stream: 4, .. })), "{:?}", entries.get(&3)); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); - assert_compact_matches_materialized(&doc); -} - -/// An older section that names an /XRefStm a newer section already read still has its own -/// table compared with that stream, so an entry it sets aside is recorded for it. -#[test] -fn a_repeated_hybrid_stream_records_each_sections_conflicts() { - let mut out = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); - let stream = find(&out, b"5 0 obj", 0, out.len()).unwrap(); - let catalog = find(&out, b"1 0 obj", 0, out.len()).unwrap(); - let base = rfind(&out, b"\nxref\n", 0).unwrap() + 1; - // This section's table places object 3 at the catalog; the stream puts it in object stream 4. - let older = out.len(); - out.extend_from_slice( - format!("xref\n3 1\n{catalog:010} 00000 n \ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {base} >>\nstartxref\n{older}\n%%EOF\n") - .as_bytes(), - ); - let newer = out.len(); - out.extend_from_slice(format!("xref\ntrailer << /Size 6 /Root 1 0 R /XRefStm {stream} /Prev {older} >>\nstartxref\n{newer}\n%%EOF\n").as_bytes()); - let doc = Document::open(Arc::new(out)).unwrap(); - assert_eq!(doc.xref_oddities(), [(older as u64, 3)]); - assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes")); -} - -fn text_of(doc: &Document, num: u32, key: &[u8]) -> Option { - doc.get(ObjRef::new(num, 0)).as_dict().and_then(|d| d.get(key).and_then(|v| v.as_string().map(|s| s.to_text()))) -} - -/// An object stream member numbered 2^32 + 3 is not object 3 (#1815): the number used to be -/// cast to `u32`, which dropped the high bits. -#[test] -fn object_stream_numbers_beyond_u32_are_not_truncated() { - let control = Document::open(Arc::new(object_stream_file("3 0 ", "<< /Member (yes) >>", 0))).unwrap(); - assert_eq!(text_of(&control, 3, b"Member").as_deref(), Some("yes")); - let doc = Document::open(Arc::new(object_stream_file("4294967299 0 ", "<< /Member (yes) >>", 0))).unwrap(); - assert!(matches!(*doc.get(ObjRef::new(3, 0)), Object::Null), "{:?}", doc.get(ObjRef::new(3, 0))); - assert!(!doc.xref_entry_is_exact(3)); -} - -/// Member starts are checked rather than cast, so they cannot wrap on any target (#1815). -#[test] -fn object_stream_member_starts_are_checked() { - assert_eq!(objstm_member(3, 4, Some(10), 20), Some((3, 14))); - assert_eq!(objstm_member(3, 9, Some(10), 20), Some((3, 19)), "the last byte"); - assert_eq!(objstm_member(3, 10, Some(10), 20), None, "at the end of the data"); - assert_eq!(objstm_member(3, 1, Some(usize::MAX), 20), None, "/First plus the offset overflows"); - assert_eq!(objstm_member(3, i64::MAX, Some(10), 20), None); - assert_eq!(objstm_member(3, -1, Some(10), 20), None); - assert_eq!(objstm_member(3, 0, None, 20), None, "/First does not fit in usize"); - assert_eq!(objstm_member(i64::from(u32::MAX), 0, Some(0), 1), Some((u32::MAX, 0))); - assert_eq!(objstm_member(i64::from(u32::MAX) + 1, 0, Some(0), 1), None); - assert_eq!(objstm_member(-1, 0, Some(0), 1), None); -} - -/// A member at a negative offset is unusable, not the stream's first member (#1815): the -/// offset used to be clamped to zero, which read the object at `/First`. -#[test] -fn object_stream_negative_offsets_are_refused() { - let doc = Document::open(Arc::new(object_stream_file("7 0 3 -1 ", "<< /First (yes) >>", 1))).unwrap(); - assert!(matches!(*doc.get(ObjRef::new(3, 0)), Object::Null), "{:?}", doc.get(ObjRef::new(3, 0))); - let control = Document::open(Arc::new(object_stream_file("7 0 3 19 ", "<< /First (yes) >> << /Second (yes) >>", 1))).unwrap(); - assert_eq!(text_of(&control, 3, b"Second").as_deref(), Some("yes")); -} - -#[test] -fn revision_ends_split_incremental_updates() { - let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - let first = bytes.len(); - let doc = Document::open(Arc::new(bytes.clone())).unwrap(); - assert_eq!(doc.revision_ends(), vec![first]); - let mut edited = doc.clone(); - edited.update_dict(ObjRef::new(1, 0), |d| d.set(b"Lang".to_vec(), Object::String(crate::PdfString::literal("en")))).unwrap(); - bytes = crate::write_incremental(&edited, &crate::SaveOptions::default()).unwrap(); - let doc = Document::open(Arc::new(bytes.clone())).unwrap(); - let ends = doc.revision_ends(); - assert_eq!(ends.len(), 2, "{ends:?}"); - assert!(ends[0] >= first && ends[0] < bytes.len() && ends[1] == bytes.len(), "{ends:?} of {}", bytes.len()); - // The first revision opens on its own, without the update. - let old = Document::open(Arc::new(bytes[..ends[0]].to_vec())).unwrap(); - assert!(old.get(ObjRef::new(1, 0)).as_dict().unwrap().get(b"Lang").is_none()); -} - -/// qpdf's `xref-compressed-in-compressed.pdf` in miniature: object 3 is an object stream -/// holding object 4, and the cross-reference stream says object 3 itself lives in stream 99. -fn compressed_in_compressed_fixture() -> Document { - let mut out = b"%PDF-1.5\n".to_vec(); - let at = |out: &mut Vec, num: u32, body: &str| { - let off = out.len(); - out.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); - off as u16 - }; - let o1 = at(&mut out, 1, "<< /Type /Catalog /Pages 2 0 R /Outlines 4 0 R >>"); - let o2 = at(&mut out, 2, "<< /Type /Pages /Kids [] /Count 0 >>"); - let inner = "<< /Type /Outlines /Count 0 >>"; - at(&mut out, 3, &format!("<< /Type /ObjStm /N 1 /First 4 /Length {} >>\nstream\n4 0 {inner}\nendstream", 4 + inner.len())); - // /W [1 2 1]: object 3 is said to live in stream 99 (which does not exist), 4 in stream 3. - let rows: Vec<[u8; 4]> = - vec![[0, 0, 0, 255], [1, 0, o1 as u8, 0], [1, 0, o2 as u8, 0], [2, 0, 99, 0], [2, 0, 3, 0], [1, (out.len() >> 8) as u8, out.len() as u8, 0]]; - let rows: Vec = rows.concat(); - let xref = out.len(); - out.extend_from_slice( - format!("5 0 obj\n<< /Type /XRef /W [1 2 1] /Index [0 6] /Size 6 /Root 1 0 R /Length {} >>\nstream\n", rows.len()).as_bytes(), - ); - out.extend_from_slice(&rows); - out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{xref}\n%%EOF\n").as_bytes()); - Document::open(Arc::new(out)).unwrap() -} - -/// An object stream whose own cross-reference entry says it is compressed inside another -/// stream (which the format forbids, and which does not exist here: qpdf's -/// `xref-compressed-in-compressed.pdf`, #2367) could not be loaded, so every object in it -/// read as null and a full save wrote `/Outlines null`. The stream is now found by its own -/// header, as a plain object with a wrong offset is. -#[test] -fn an_object_stream_marked_as_compressed_is_found_by_its_header() { - let doc = compressed_in_compressed_fixture(); - assert_eq!(doc.revisions().len(), 1, "the chain itself reads: {:?}", doc.repair_log()); - let outlines = doc.get(ObjRef::new(4, 0)); - assert_eq!(outlines.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Outlines"[..]), "{outlines:?}"); - let saved = crate::write_full(&doc, &crate::SaveOptions::default()).unwrap(); - let back = Document::open(Arc::new(saved)).unwrap(); - let catalog = back.get(back.root().unwrap()); - let outlines = back.resolve(catalog.as_dict().unwrap().get(b"Outlines").unwrap()); - assert_eq!(outlines.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Outlines"[..]), "the full save keeps it: {outlines:?}"); -} - -/// The header fallback is for file entries only: once the stream is freed (or replaced) in -/// the overlay, its objects must not come back from the file's copy of it. -#[test] -fn a_freed_object_stream_is_not_found_by_its_header() { - let doc = compressed_in_compressed_fixture(); - assert!(doc.get(ObjRef::new(4, 0)).as_dict().is_some()); - let mut edited = doc.clone(); - edited.free(ObjRef::new(3, 0)); - let fresh = edited.object_reader().document; - assert!(matches!(*fresh.get(ObjRef::new(4, 0)), Object::Null), "{:?}", fresh.get(ObjRef::new(4, 0))); - let mut replaced = doc.clone(); - replaced.set(ObjRef::new(3, 0), Object::Int(0)); - let fresh = replaced.object_reader().document; - assert!(matches!(*fresh.get(ObjRef::new(4, 0)), Object::Null), "{:?}", fresh.get(ObjRef::new(4, 0))); -} - -/// The same once the free is saved: the revision's table says the stream is free, and -/// neither reading the file back nor rewriting it in full brings its objects back. -#[test] -fn a_container_freed_in_a_saved_revision_is_not_found_by_its_header() { - let mut edited = compressed_in_compressed_fixture(); - edited.free(ObjRef::new(3, 0)); - let incremental = crate::write_incremental(&edited, &crate::SaveOptions::default()).unwrap(); - let reopened = Document::open(Arc::new(incremental)).unwrap(); - assert_eq!(reopened.revisions().len(), 2); - assert!(matches!(reopened.xref_entry(3), Some(XrefEntry::Free { .. }))); - assert!(matches!(*reopened.get(ObjRef::new(4, 0)), Object::Null), "{:?}", reopened.get(ObjRef::new(4, 0))); - for object_streams in [true, false] { - let opts = crate::SaveOptions { object_streams, ..Default::default() }; - let full = crate::write_full(&reopened, &opts).unwrap(); - let back = Document::open(Arc::new(full)).unwrap(); - let catalog = back.get(back.root().unwrap()); - let outlines = catalog.as_dict().unwrap().get(b"Outlines").map(|o| back.resolve(o)); - assert!(outlines.as_ref().is_none_or(|o| matches!(**o, Object::Null)), "the full save keeps it out: {outlines:?}"); - } -} - -/// A trailer whose /Prev is written as the reference `N 0 R` (tiff2pdf did this; the files -/// are JHOVE's PDF-HUL-157 cases, #2366) used to end the cross-reference chain silently, so -/// every object of the earlier sections read as null: a four-page file showed one page, and -/// a full save wrote the three missing kids as null. The offset is read from the reference. -#[test] -fn a_prev_written_as_a_reference_still_reaches_the_earlier_sections() { - let base = - build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R >>"], "/Root 1 0 R"); - let prev = String::from_utf8_lossy(&base).rfind("xref\n0 4").unwrap(); - let update = |prev_entry: &str| { - let mut bytes = base.clone(); - let at = bytes.len(); - bytes.extend_from_slice(b"4 0 obj\n<< /Title (Update) >>\nendobj\n"); - let xref = bytes.len(); - bytes.extend_from_slice( - format!("xref\n4 1\n{at:010} 00000 n \ntrailer\n<< /Size 5 /Root 1 0 R /Info 4 0 R /Prev {prev_entry} >>\nstartxref\n{xref}\n%%EOF\n") - .as_bytes(), - ); - Document::open(Arc::new(bytes)).unwrap() - }; - let good = update(&prev.to_string()); - let bad = update(&format!("{prev} 0 R")); - assert!(good.repair_log().is_empty(), "{:?}", good.repair_log()); - assert_eq!(bad.revisions().len(), 2, "{:?}", bad.repair_log()); - assert_eq!(bad.repair_log().len(), 1, "a note about the /Prev: {:?}", bad.repair_log()); - assert_eq!(bad.xref_entry(3), good.xref_entry(3), "the page of the first section is reached"); - assert_eq!(bad.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); - assert_eq!( - bad.get(ObjRef::new(4, 0)).as_dict().and_then(|d| d.get(b"Title").and_then(|t| t.as_string().map(|s| s.to_text()))), - Some("Update".into()) - ); -} - -/// iText's `indirectPrev.pdf` in miniature: the update's trailer gives `/Prev 4 0 R`, and -/// object 4, listed in that update, holds the offset of the first section. Read as offset 4 -/// (inside the header), the chain broke and the file was rebuilt without its revisions. -#[test] -fn a_prev_reference_to_an_object_holding_the_offset_reads_that_offset() { - let mut bytes = - build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R >>"], "/Root 1 0 R"); - let prev = String::from_utf8_lossy(&bytes).rfind("xref\n0 4").unwrap(); - let mut at = Vec::new(); - for (num, body) in - [(2, "<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>".to_string()), (4, prev.to_string()), (5, "<< /Type /Page /Parent 2 0 R >>".into())] - { - at.push(bytes.len()); - bytes.extend_from_slice(format!("{num} 0 obj\n{body}\nendobj\n").as_bytes()); - } - let xref = bytes.len(); - bytes.extend_from_slice( - format!( - "xref\n2 1\n{:010} 00000 n \n4 2\n{:010} 00000 n \n{:010} 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R /Prev 4 0 R >>\nstartxref\n{xref}\n%%EOF\n", - at[0], at[1], at[2] - ) - .as_bytes(), - ); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.revisions().len(), 2, "{:?}", doc.repair_log()); - assert_eq!(doc.repair_log().len(), 1, "a note about the /Prev, and no rebuild: {:?}", doc.repair_log()); - assert!(doc.repair_log()[0].contains(&format!("the offset {prev}")), "{:?}", doc.repair_log()); - assert_eq!(doc.get(ObjRef::new(3, 0)).as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); - assert_compact_matches_materialized(&doc); -} - -/// A file whose only trailers are cross-reference stream dictionaries, with its `startxref` -/// out of reach (here a second file appended after it, cut off before its own -/// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used -/// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after -/// editing one Info field dropped every other. -#[test] -fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { - let table = build( - &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Kept) /Creator (Me) >>"], - "/Root 1 0 R /Info 3 0 R", - ); - let packed = crate::write_full(&Document::open(Arc::new(table)).unwrap(), &crate::SaveOptions::default()).unwrap(); - let text = String::from_utf8_lossy(&packed); - assert!(text.contains("/XRef") && !text.contains("trailer"), "a cross-reference stream is the only trailer"); - let mut bytes = packed.clone(); - let appended = format!("%PDF-1.3\n90 0 obj\n<< /Length 5000 >>\nstream\n{}\nendstream\nendobj\n", "x".repeat(5000)); - bytes.extend_from_slice(appended.as_bytes()); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(doc.revisions().is_empty(), "rebuilt: {:?}", doc.repair_log()); - let info = doc.trailer().reference(b"Info").expect("the cross-reference stream's /Info"); - let field = |d: &Document, key: &[u8]| d.get(info).as_dict().unwrap().get(key).unwrap().as_string().unwrap().bytes.clone(); - assert_eq!(field(&doc, b"Title"), b"Kept"); - assert!(doc.trailer().get(b"ID").is_some(), "and its /ID"); - let mut edited = doc.clone(); - edited.update_dict(info, |d| d.set(b"Subject".to_vec(), Object::String(crate::PdfString::literal("new")))).unwrap(); - let saved = crate::write_full(&edited, &crate::SaveOptions::default()).unwrap(); - let back = Document::open(Arc::new(saved)).unwrap(); - assert!(back.repair_log().is_empty(), "{:?}", back.repair_log()); - let info = back.trailer().reference(b"Info").expect("the full save keeps /Info"); - let field = |key: &[u8]| back.get(info).as_dict().unwrap().get(key).unwrap().as_string().unwrap().bytes.clone(); - assert_eq!((field(b"Title"), field(b"Creator"), field(b"Subject")), (b"Kept".to_vec(), b"Me".to_vec(), b"new".to_vec())); -} - -/// A rebuild prefers a trailer whose /Root is the catalog (as `root_is_catalog` judges it): a -/// later cross-reference stream whose /Root points at the Info dictionary does not replace an -/// earlier good trailer, or a full save would write the Info dictionary as /Root. -#[test] -fn reconstruction_skips_a_trailer_whose_root_is_not_the_catalog() { - let mut bytes = - build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Kept) >>"], "/Root 1 0 R /Info 3 0 R"); - bytes.extend_from_slice(b"4 0 obj\n<< /Type /XRef /Size 5 /W [1 1 1] /Root 3 0 R /Length 3 >>\nstream\nabc\nendstream\nendobj\n"); - let appended = format!("%PDF-1.3\n90 0 obj\n<< /Length 5000 >>\nstream\n{}\nendstream\nendobj\n", "x".repeat(5000)); - bytes.extend_from_slice(appended.as_bytes()); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(doc.repair_log().iter().any(|l| l.contains("rebuilt")), "{:?}", doc.repair_log()); - assert_eq!(doc.trailer().reference(b"Root").map(|r| r.num), Some(1), "the catalog, not the Info dictionary"); - assert_eq!(doc.trailer().reference(b"Info").map(|r| r.num), Some(3)); -} - -#[test] -fn damaged_xref_is_reconstructed() { - let mut bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"], "/Root 1 0 R"); - let n = bytes.len(); - bytes[n - 12..n - 6].copy_from_slice(b"999999"); // point startxref into nowhere - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert!(doc.revisions().is_empty()); - assert!(!doc.repair_log().is_empty()); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); -} - -#[test] -fn a_decompression_bomb_in_an_xref_stream_is_refused_and_the_file_reconstructed() { - // 32 MiB of rows behind two FlateDecode filters (a few hundred bytes in the file) for - // /W [1 1 1]: at 3 bytes a row that is more than 8,388,607 objects' worth, so it is not - // decoded (in full: the limit applies while inflating) and the objects are found by - // reconstruction instead. - let bomb = pdfcraft_filters::encode_flate(&pdfcraft_filters::encode_flate(&vec![0u8; 32 << 20])); - let mut bytes = b"%PDF-1.7\n".to_vec(); - let o1 = bytes.len(); - bytes.extend_from_slice(b"1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"); - let o2 = bytes.len(); - bytes.extend_from_slice(b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"); - let x = bytes.len(); - bytes.extend_from_slice( - format!("3 0 obj\n<< /Type /XRef /Size 4 /W [1 1 1] /Root 1 0 R /Filter [/FlateDecode /FlateDecode] /Length {} >>\nstream\n", bomb.len()) - .as_bytes(), - ); - bytes.extend_from_slice(&bomb); - bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); - assert!(o1 < o2 && o2 < x); - let doc = Document::open(Arc::new(bytes)).unwrap(); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - assert!(doc.repair_log().iter().any(|l| l.contains("limit")), "{:?}", doc.repair_log()); -} - -/// A chain of cross-reference streams whose sections each stay well under the per-section -/// limit but together list more than 8,388,607 rows (#5082). Too many rows for the compact -/// reader, the chain was merged whole; it is now refused and the file rebuilt from object -/// headers. The bulk rows are set aside (in use at offset 0), so the test holds a few bytes a row. -#[test] -fn a_cross_reference_chain_past_the_total_cap_is_refused_and_the_file_reconstructed() { - const BULK: usize = 4_200_000; - let mut bytes = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for body in - ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>"] - { - offsets.push(bytes.len()); - bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); - } - // Objects 4 and 5: /W [1 0 0] rows of type 1 at offset 0, each a set-aside entry. - let bulk = pdfcraft_filters::encode_flate(&vec![1u8; BULK]); - let size = 16 + 2 * BULK; - let mut prev = None; - for (num, start) in [(4, 16 + BULK), (5, 16)] { - offsets.push(bytes.len()); - let prev_key = prev.map(|p| format!(" /Prev {p}")).unwrap_or_default(); - bytes.extend_from_slice( - format!( - "{num} 0 obj\n<< /Type /XRef /Size {size} /W [1 0 0] /Index [{start} {BULK}]{prev_key} /Filter /FlateDecode /Length {} >>\nstream\n", - bulk.len() - ) - .as_bytes(), - ); - bytes.extend_from_slice(&bulk); - bytes.extend_from_slice(b"\nendstream\nendobj\n"); - prev = offsets.last().copied(); - } - // Object 6, the newest section: objects 0 to 6 at their offsets. - let x = bytes.len(); - offsets.push(x); - let mut rows = vec![0u8, 0, 0, 0, 0, 255]; - for o in &offsets { - rows.push(1); - rows.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); - rows.push(0); - } - bytes.extend_from_slice( - format!("6 0 obj\n<< /Type /XRef /Size {size} /Root 1 0 R /W [1 4 1] /Index [0 7] /Prev {} /Length {} >>\nstream\n", offsets[4], rows.len()) - .as_bytes(), - ); - bytes.extend_from_slice(&rows); - bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); - - let doc = Document::open(Arc::new(bytes)).unwrap(); - let log = doc.repair_log(); - assert!( - log.iter().any(|l| l.contains("cross-reference chain lists more than 8388607 entries") && l.contains("rebuilding from object headers")), - "{log:?}" - ); - assert!(log.iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{log:?}"); - assert!(doc.revisions().is_empty()); - assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); - let page = doc.try_get(3).unwrap(); - assert_eq!(page.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); - // The chain read itself is refused at the second bulk section, which each decode within its own limit. - let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized(&mut Vec::new()) else { panic!("the chain must be refused") }; - assert_eq!(offset, offsets[3]); - assert!(detail.contains("more than 8388607 entries"), "{detail}"); -} - -/// Three classic tables listing 2, 1 and 1 rows, newest first: objects 1 and 2, then object 3 -/// (or object 1 again when `overlap`), then object 4. -fn three_table_chain(overlap: bool) -> (Vec, usize) { - let mut bytes = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for (num, body) in - ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 3 >>", "<< /Value 4 >>"].iter().enumerate() - { - offsets.push(bytes.len()); - bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", num + 1).as_bytes()); - } - let older = bytes.len(); - bytes.extend_from_slice(format!("xref\n4 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R >>\n", offsets[3]).as_bytes()); - let middle = bytes.len(); - let middle_num = if overlap { 1 } else { 3 }; - bytes.extend_from_slice( - format!("xref\n{middle_num} 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {older} >>\n", offsets[middle_num - 1]).as_bytes(), - ); - let newer = bytes.len(); - bytes.extend_from_slice( - format!( - "xref\n1 1\n{:010} 00000 n \n2 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {middle} >>\nstartxref\n{newer}\n%%EOF\n", - offsets[0], offsets[1], - ) - .as_bytes(), - ); - (bytes, older) -} - -/// The chain's row limit counts every row merged (#5082): four rows over three sections pass a -/// limit of 4 and are refused at the oldest section by a limit of 3, both when the sections -/// list different objects and when one relists an object a newer section has (a map of three -/// entries, but four rows merged). -#[test] -fn the_chain_row_limit_counts_disjoint_and_relisted_rows() { - for overlap in [false, true] { - let (bytes, older) = three_table_chain(overlap); - let doc = Document::open(Arc::new(bytes)).unwrap(); - let (entries, ..) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 4).unwrap(); - assert_eq!(entries.len(), if overlap { 3 } else { 4 }); - let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 3) else { - panic!("four rows must be refused by a limit of 3 (overlap: {overlap})") - }; - assert_eq!(offset, older); - assert!(detail.contains("more than 3 entries"), "{detail}"); - } -} - -/// Rows set aside count against the chain's row limit (#5082): a hybrid file whose table puts -/// object 3 at an offset while its /XRefStm puts it in an object stream merges six rows and sets -/// one aside, so a limit of 7 passes and a limit of 6 refuses it. -#[test] -fn the_chain_row_limit_counts_hybrid_rows_set_aside() { - let mut bytes = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); - // Object 3's row in the table (`xref\n0 6\n`, 20-byte rows): free, now in use at offset 9. - let row = find(&bytes, b"xref\n0 6\n", 0, bytes.len()).unwrap() + 9 + 3 * 20; - assert_eq!(&bytes[row..row + 20], b"0000000000 65535 f \n"); - bytes[row..row + 20].copy_from_slice(b"0000000009 00000 n \n"); - let doc = Document::open(Arc::new(bytes)).unwrap(); - let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7).unwrap(); - assert_eq!(entries.len(), 6); - assert_eq!(oddities.iter().map(|(_, num)| *num).collect::>(), vec![3]); - let Err(CosError::Syntax { detail, .. }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 6) else { - panic!("six rows and one set aside must be refused by a limit of 6") - }; - assert!(detail.contains("more than 6 entries"), "{detail}"); -} - -/// A cross-reference stream whose `/Index [0 4 0 4]` lists objects 0 to 3 twice, the same rows -/// both times: eight rows read, four entries (#5082). The chain's row limit counts the rows -/// read, before deduplication, so a limit of 8 passes and a limit of 7 refuses it. -#[test] -fn the_chain_row_limit_counts_rows_an_xref_stream_index_repeats() { - let mut bytes = b"%PDF-1.7\n".to_vec(); - let mut offsets = Vec::new(); - for body in ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"] { - offsets.push(bytes.len()); - bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); - } - let x = bytes.len(); - offsets.push(x); - let mut once = vec![0u8, 0, 0, 0, 0, 255]; - for o in &offsets { - once.push(1); - once.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); - once.push(0); - } - let rows = [once.clone(), once].concat(); - bytes.extend_from_slice( - format!("3 0 obj\n<< /Type /XRef /Size 4 /Root 1 0 R /W [1 4 1] /Index [0 4 0 4] /Length {} >>\nstream\n", rows.len()).as_bytes(), - ); - bytes.extend_from_slice(&rows); - bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); - let doc = Document::open(Arc::new(bytes)).unwrap(); - let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 8).unwrap(); - assert_eq!(entries.len(), 4); - assert!(oddities.is_empty(), "{oddities:?}"); - let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7) else { - panic!("eight rows listing four objects must be refused by a limit of 7") - }; - assert_eq!(offset, x); - assert!(detail.contains("more than 7 entries"), "{detail}"); -} - -#[test] -fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { - // From the nightly fuzz job: `6 0 obj << /Length 6 0 R >> stream` with no `endstream` - // after it. Every parse attempt failed, and each attempt resolved the length by loading - // object 6 again, retrying the same ways down to the nesting limit: about 2^32 parses. - // Objects being loaded are now a seen-set, lengths are memoised, and the second parse - // only runs when the header is shifted. - let mut bytes = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog >>\nendobj\n".to_vec(); - bytes.extend_from_slice(b"6 0 obj\n<< /Length 6 0 R >>\nstream\nno end marker follows\n"); - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - if let Ok(doc) = Document::open(Arc::new(bytes)) { - let _ = doc.try_get(6); - } - let _ = tx.send(()); - }); - rx.recv_timeout(std::time::Duration::from_secs(20)).expect("loading must not retry exponentially"); -} - -/// A file whose object 3 (its `/Info`) can't be parsed: the rest reads. -fn with_unreadable_info() -> Vec { - build( - &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Author ) >>"], - "/Root 1 0 R /Info 3 0 R", - ) -} - -/// An object that can't be parsed reads as null and is listed once, by number and generation, -/// with why. Clones and scans share the list, an edit takes the object off it (in that document -/// only), and a sound file lists nothing. -#[test] -fn unreadable_objects_are_listed_with_the_reason() { - let doc = Document::open(Arc::new(with_unreadable_info())).unwrap(); - assert!(doc.unreadable_objects().is_empty(), "nothing has read object 3 yet"); - assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); - assert!(doc.try_get(3).is_err()); - let lines = doc.unreadable_objects(); - assert_eq!(lines.len(), 1, "read twice, listed once: {lines:?}"); - assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); - let scanned = doc.clone(); - assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); - assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); - let unread = Document::open(Arc::new(with_unreadable_info())).unwrap(); - let scanned = unread.clone(); - assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); - assert_eq!(unread.unreadable_objects(), lines, "a clone's scan reaches the original"); - let mut edited = doc.clone(); - edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); - assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); - assert_eq!(doc.unreadable_objects(), lines); - let sound = - build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) >>"], "/Root 1 0 R /Info 3 0 R"); - let sound = Document::open(Arc::new(sound)).unwrap(); - assert_eq!(sound.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 0); - assert!(sound.unreadable_objects().is_empty()); -} - -/// A file rebuilt from its object headers still lists an object the scan could not read. -#[test] -fn a_rebuilt_file_lists_the_objects_its_scan_could_not_read() { - let text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n3 0 obj\n<< /A ) >>\nendobj\n%%EOF\n"; - let doc = Document::open(Arc::new(text.as_bytes().to_vec())).unwrap(); - assert!(doc.repair_log().iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{:?}", doc.repair_log()); - let lines = doc.unreadable_objects(); - assert_eq!(lines.len(), 1, "{lines:?}"); - assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); -} - -/// The list keeps at most 100 objects and then says there were more; an object whose -/// cross-reference offset holds no object is listed with that as its reason. -#[test] -fn the_unreadable_object_list_is_bounded() { - let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; - bodies.extend(std::iter::repeat_n("<< /A ) >>", 120)); - let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); - assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 120); - let lines = doc.unreadable_objects(); - assert_eq!(lines.len(), 101, "{:?}", lines.last()); - assert!(lines[0].starts_with("object 3 0 could not be read"), "{}", lines[0]); - assert_eq!(lines[100], "more objects could not be read; only the first 100 are listed"); - - let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); - let text = String::from_utf8_lossy(&base).into_owned(); - let gone = text.replacen("3 0 obj\n(three)", "9 0 obj\n(three)", 1); - let doc = Document::open(Arc::new(gone.into_bytes())).unwrap(); - assert!(doc.try_get(3).is_err()); - assert_eq!( - doc.unreadable_objects(), - ["object 3 0 could not be read: no readable copy was found at its cross-reference offset or elsewhere in the file"] - ); -} - -/// The unreadable list is shared by clones, and each hides what it has edited. A clone that edited -/// every listed object still lists its own failures: hidden entries don't use up its allowance. -#[test] -fn objects_a_clone_edited_do_not_use_up_its_unreadable_allowance() { - let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; - bodies.extend(std::iter::repeat_n("<< /A ) >>", 110)); - let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); - for num in 3..103 { - assert!(doc.try_get(num).is_err()); - } - assert_eq!(doc.unreadable_objects().len(), 100); - let mut edited = doc.clone(); - for num in 3..103 { - edited.set(ObjRef::new(num, 0), Object::Null); - } - for num in 103..113 { - assert!(edited.try_get(num).is_err()); - } - let lines = edited.unreadable_objects(); - assert_eq!(lines.len(), 10, "{lines:?}"); - assert!(lines[0].starts_with("object 103 0 could not be read"), "{}", lines[0]); - let original = doc.unreadable_objects(); - assert_eq!(original.len(), 101, "the original still shows at most 100, then says there were more"); - assert_eq!(original[100], "more objects could not be read; only the first 100 are listed"); -} - -#[test] -fn a_restored_revision_keeps_what_the_saved_file_held_in_object_streams() { - // The earlier revision: an empty document read back from a file whose catalog and pages - // sit in an object stream. - let earlier = Document::open(Arc::new(crate::write_full(&Document::new_empty(), &crate::SaveOptions::default()).unwrap())).unwrap(); - // The saved revision adds an object the catalog links to, so a full save numbers its objects - // anew; an undo then returns to the earlier graph. - let mut later = earlier.clone(); - let extra = later.add(Object::Int(7)); - let root = later.root().unwrap(); - later.update_dict(root, |c| c.set(b"Extra".to_vec(), Object::Ref(extra))).unwrap(); - let saved = crate::write_full(&later, &crate::SaveOptions::default()).unwrap(); - let mut base = Document::open(Arc::new(saved)).unwrap(); - assert!( - base.object_numbers().iter().any(|&n| matches!(base.xref_entry(n), Some(XrefEntry::InStream { .. }))), - "the saved file holds objects in a stream" - ); - base.overlay_changes_from(&earlier).unwrap(); - let file = crate::write_incremental(&base, &crate::SaveOptions::default()).unwrap(); - let reread = Document::open(Arc::new(file)).unwrap(); - let catalog = reread.get(reread.root().unwrap()).as_dict().unwrap().clone(); - assert!(catalog.get(b"Extra").is_none(), "the restored catalog"); - let pages = reread.resolve(catalog.get(b"Pages").unwrap()); - assert_eq!(pages.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Pages"[..]), "the pages survive: {pages:?}"); -} - -/// The cache keeps the sum of what its objects hold as they come and go, so neither reader has -/// to measure the cache again on every read (#5873). -#[test] -fn the_object_cache_keeps_its_total_as_objects_come_and_go() { - let data = Arc::new(Vec::new()); - let one = Arc::new(Object::String(crate::PdfString::literal(vec![b'x'; 100]))); - let two = Arc::new(Object::Array(vec![Object::Int(1); 10])); - let (a, b) = (held_bytes(&one, &data, 0), held_bytes(&two, &data, 0)); - let mut cache = super::state::ObjectCache::default(); - cache.insert(1, Ok(one.clone()), a, Default::default()); - cache.insert(2, Ok(two), b, Default::default()); - assert_eq!(cache.owned(), a + b); - cache.insert(2, Ok(one), a, Default::default()); - assert_eq!(cache.owned(), a + a, "a replaced object no longer counts"); - cache.retain(|n| *n == 1); - assert_eq!((cache.len(), cache.owned()), (1, a)); - cache.clear(); - assert_eq!((cache.len(), cache.owned()), (0, 0)); -} - -/// A dictionary of numbers owns no string or stream bytes, but parsed it is tens of thousands -/// of values: InDesign's per-page `/PieceInfo`. Counting only strings and streams, the editing -/// cache kept a thousand such pages and a 35 MB book held 1.5 GB (#5873). What the parsed tree -/// holds counts now, so the byte budget bounds them. -#[test] -fn the_editor_cache_charges_a_dictionary_for_the_values_it_holds() { - let entries: String = (0..2000).map(|i| format!("/{i}[{i} 141 3 116.22 -375.993 1.0]")).collect(); - let dict = format!("<< /PieceInfo << /InDesign << /Map << {entries} >> >> >> >>"); - let count = 300; - let refs: String = (0..count).map(|i| format!("{} 0 R ", i + 2)).collect(); - let mut bodies = vec![format!("<< /Type /Catalog /Extension [{refs}] >>")]; - bodies.extend((0..count).map(|_| dict.clone())); - let bodies: Vec<&str> = bodies.iter().map(String::as_str).collect(); - let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); - let one = doc.get(crate::ObjRef::new(2, 0)); - let held = held_bytes(&one, &doc.data, 0); - assert!(held > 2000 * std::mem::size_of::(), "the parsed tree counts: {held} bytes"); - for i in 0..count { - drop(doc.get(crate::ObjRef::new(i as u32 + 2, 0))); - } - let cache = doc.cache.lock().unwrap(); - assert!(cache.owned() <= Document::EDITOR_CACHE_BYTES, "{} bytes held", cache.owned()); - assert!(cache.len() < count, "the byte budget evicted some, below the count cap: {} kept", cache.len()); - let sum = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); - assert_eq!(cache.owned(), sum, "the kept total is the cache's own"); -} - -/// The width (`/MediaBox` x1) of each of `pages`, in order. -fn page_widths(pages: &[Dict]) -> Vec> { - pages.iter().map(|p| p.get(b"MediaBox").and_then(Object::as_array).and_then(|b| b.get(2)).and_then(Object::as_int)).collect() -} - -/// The pages of `doc` in page-tree order, as the dictionaries the document holds. -fn page_tree_pages(doc: &Document) -> Vec { - let catalog = doc.get(doc.root().expect("root")); - let pages = doc.resolve(catalog.as_dict().and_then(|c| c.get(b"Pages")).expect("/Pages")); - let kids = pages.as_dict().and_then(|p| p.get(b"Kids")).and_then(Object::as_array).cloned().unwrap_or_default(); - kids.iter().filter_map(|k| doc.resolve(k).as_dict().cloned()).collect() -} - -/// ISO 32000-2 §7.5.6 and §7.5.8.2: an update reaches the revisions before it through `/Prev`. -/// One without it cuts them off, page tree and all. The file is rebuilt from its object headers -/// as a damaged one is, so its page tree is found, the newest definition of each object wins, and -/// the repair log says why; a save rewrites it in full, with every page. -#[test] -fn an_update_without_prev_is_rebuilt_with_its_page_tree_and_a_repair_note() { - let doc = Document::open(Arc::new(pdfcraft_testkit::update_without_prev(false))).expect("the unlinked file opens"); - let pages = page_tree_pages(&doc); - assert_eq!(pages.len(), 3); - assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page wins: {pages:?}"); - assert!(doc.reconstructed() && doc.rebuilt_unlinked_update()); - assert!(doc.repair_log().iter().any(|l| l.contains("no /Prev")), "{:?}", doc.repair_log()); - let catalog = doc.get(doc.root().expect("root")); - assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog wins"); - - let saved = crate::write_incremental(&doc, &crate::SaveOptions::default()).expect("saves"); - let reopened = Document::open(Arc::new(saved)).expect("the saved file opens"); - let pages = page_tree_pages(&reopened); - assert_eq!(page_widths(&pages), [Some(200), Some(300), Some(400)], "every page is saved, in order"); - assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page is saved: {pages:?}"); - let catalog = reopened.get(reopened.root().expect("root")); - assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog is saved"); - assert!(reopened.repair_log().is_empty(), "{:?}", reopened.repair_log()); - - // The same file with `/Prev` reads through its chain, as before. - let linked = Document::open(Arc::new(pdfcraft_testkit::update_without_prev(true))).expect("the linked file opens"); - assert_eq!(page_tree_pages(&linked).len(), 3); - assert!(!linked.reconstructed() && !linked.rebuilt_unlinked_update()); - assert_eq!(linked.revisions().len(), 2); - assert!(linked.repair_log().is_empty(), "{:?}", linked.repair_log()); -} - -/// A hybrid update (ISO 32000-2 §7.5.8.4) without `/Prev`: its `/XRefStm` stream belongs to the -/// update, so the earlier revision is cut off just the same, and the file is rebuilt. -#[test] -fn a_hybrid_update_without_prev_is_rebuilt_too() { - let doc = Document::open(Arc::new(pdfcraft_testkit::hybrid_update_without_prev(false))).expect("the unlinked file opens"); - let pages = page_tree_pages(&doc); - assert_eq!(pages.len(), 3); - assert!(pages.first().is_some_and(|p| p.contains(b"Annots")), "the update's page wins: {pages:?}"); - assert!(doc.rebuilt_unlinked_update(), "{:?}", doc.repair_log()); - assert!(doc.repair_log().iter().any(|l| l.contains("no /Prev")), "{:?}", doc.repair_log()); - let catalog = doc.get(doc.root().expect("root")); - assert!(catalog.as_dict().is_some_and(|c| c.contains(b"Lang")), "the update's catalog wins"); - let linked = Document::open(Arc::new(pdfcraft_testkit::hybrid_update_without_prev(true))).expect("the linked file opens"); - assert_eq!(page_tree_pages(&linked).len(), 3); - assert!(!linked.rebuilt_unlinked_update() && linked.repair_log().is_empty(), "{:?}", linked.repair_log()); - assert_eq!(linked.revisions().len(), 2); -} - -#[test] -fn find_agrees_with_a_plain_window_search() { - let naive = |hay: &[u8], needle: &[u8], from: usize, to: usize| { - if from >= to || needle.is_empty() || to - from < needle.len() { - return None; - } - hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) - }; - let mut seed = 0x9E37_79B9_7F4A_7C15u64; - let mut next = move || { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - seed - }; - let alphabet = b"/XRef t\n<>0"; - for _ in 0..20_000 { - let hay: Vec = (0..next() % 200).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); - let needles: [&[u8]; 5] = [b"/XRef", b"trailer", b"R", b"ff", b"<<"]; - let needle = needles[(next() % 5) as usize]; - let from = (next() % (hay.len() as u64 + 1)) as usize; - let to = from + (next() % (hay.len() as u64 + 1 - from as u64)) as usize; - assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}"); - } -} diff --git a/crates/cos/src/document/xref.rs b/crates/cos/src/document/xref.rs deleted file mode 100644 index 771df28b80..0000000000 --- a/crates/cos/src/document/xref.rs +++ /dev/null @@ -1,275 +0,0 @@ -use super::{Arc, BTreeMap, Lexer}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum XrefEntry { - Free { next_generation: u16 }, - InFile { offset: u64, generation: u16 }, - InStream { stream: u32, index: u32 }, -} - -#[derive(Clone, Copy, Debug)] -pub(super) struct XrefRange { - pub(super) start: u32, - pub(super) count: u32, - pub(super) offset: usize, - pub(super) stride: usize, -} - -#[derive(Clone, Debug)] -pub(super) enum XrefSectionData { - Table, - Stream { bytes: Arc>, widths: [usize; 3] }, -} - -#[derive(Clone, Debug)] -pub(super) struct XrefSection { - pub(super) data: XrefSectionData, - pub(super) ranges: Vec, - pub(super) row_count: usize, - pub(super) max_num: u32, - pub(super) has_entries: bool, - /// A table subsection was read one number lower than it says ([`subsection_start`]). - pub(super) renumbered: bool, -} - -#[derive(Clone, Debug)] -pub(super) struct XrefRevisionSection { - pub(super) primary: XrefSection, - pub(super) hybrid: Option, -} - -#[derive(Clone, Debug)] -pub(super) enum XrefIndex { - Compact { revisions: Vec, max_num: u32, has_entries: bool }, - Materialized(Arc>, Arc>), -} - -pub(super) const MAX_COMPACT_XREF_ROWS: usize = 2_000_000; - -pub(super) const MAX_COMPACT_XREF_BYTES: usize = 64 * 1024 * 1024; - -impl XrefSection { - pub(super) fn has_any_entry(&self) -> bool { - match &self.data { - XrefSectionData::Table => self.has_entries, - XrefSectionData::Stream { bytes, widths } => { - let row_width = widths.iter().sum::(); - if row_width == 0 { - return false; - } - for range in &self.ranges { - let available = bytes.len().saturating_sub(range.offset) / row_width; - for i in 0..available.min(range.count as usize) { - let Some(offset) = i.checked_mul(row_width).and_then(|delta| range.offset.checked_add(delta)) else { continue }; - let Some(row) = bytes.get(offset..offset.saturating_add(row_width)) else { continue }; - let read_field = |from: usize, width: usize, default: u64| -> Option { - if width == 0 { - return Some(default); - } - let end = from.checked_add(width)?; - Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte))) - }; - let Some(kind) = read_field(0, widths[0], 1) else { continue }; - if kind == 0 || kind == 2 { - return true; - } - if kind == 1 && read_field(widths[0], widths[1], 0).is_some_and(|offset| offset != 0) { - return true; - } - } - } - false - } - } - } - - pub(super) fn retained_bytes(&self) -> usize { - match &self.data { - XrefSectionData::Table => 0, - XrefSectionData::Stream { bytes, .. } => bytes.capacity(), - } - } - - pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { - let after = self.ranges.partition_point(|range| range.start <= num); - let range = self.ranges.get(after.checked_sub(1)?)?; - let end = u64::from(range.start).checked_add(u64::from(range.count))?; - if u64::from(num) >= end { - return None; - } - let row = usize::try_from(num.checked_sub(range.start)?).ok()?; - let offset = range.offset.checked_add(row.checked_mul(range.stride)?)?; - match &self.data { - XrefSectionData::Table => { - let mut lexer = Lexer::new(file, offset); - lexer.skip_ws(); - let object_offset = lexer.token(); - lexer.skip_ws(); - let generation = lexer.token(); - lexer.skip_ws(); - let kind = lexer.token(); - match kind { - b"n" => { - let object_offset = std::str::from_utf8(object_offset).ok()?.parse::().ok()?; - if object_offset == 0 { - return None; - } - let generation = std::str::from_utf8(generation).ok()?.parse::().unwrap_or(0).min(u16::MAX as u32) as u16; - Some(XrefEntry::InFile { offset: object_offset, generation }) - } - b"f" => { - let generation = std::str::from_utf8(generation).ok()?.parse::().unwrap_or(0).min(u16::MAX as u32) as u16; - Some(XrefEntry::Free { next_generation: generation }) - } - _ => None, - } - } - XrefSectionData::Stream { bytes, widths } => { - let row_width = widths.iter().sum::(); - let end = offset.checked_add(row_width)?; - let row = bytes.get(offset..end)?; - let field = |from: usize, width: usize, default: u64| -> Option { - if width == 0 { - return Some(default); - } - let end = from.checked_add(width)?; - Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte))) - }; - let kind = field(0, widths[0], 1)?; - let a = field(widths[0], widths[1], 0)?; - let b = field(widths[0] + widths[1], widths[2], 0)?; - let entry = match kind { - 0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 }, - 1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 }, - 2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 }, - _ => return None, - }; - (!matches!(entry, XrefEntry::InFile { offset: 0, .. })).then_some(entry) - } - } - } - - pub(super) fn numbers(&self) -> impl Iterator + '_ { - self.ranges.iter().flat_map(|range| { - let available = match &self.data { - XrefSectionData::Table => range.count as usize, - XrefSectionData::Stream { bytes, widths } => { - let row_width = widths.iter().sum::(); - bytes.len().saturating_sub(range.offset).checked_div(row_width).unwrap_or(0) - } - }; - (0..available.min(range.count as usize)).filter_map(move |i| u32::try_from(i).ok().and_then(|i| range.start.checked_add(i))) - }) - } - - pub(super) fn extend_numbers(&self, numbers: &mut Vec) { - numbers.extend(self.numbers()); - } -} - -impl XrefRevisionSection { - pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { - let primary = self.primary.lookup(num, file); - let hybrid = self.hybrid.as_ref().and_then(|section| section.lookup(num, file)); - match primary { - None | Some(XrefEntry::Free { .. }) => hybrid.or(primary), - entry => entry, - } - } - - fn extend_numbers(&self, numbers: &mut Vec) { - self.primary.extend_numbers(numbers); - if let Some(hybrid) = &self.hybrid { - hybrid.extend_numbers(numbers); - } - } -} - -impl XrefIndex { - pub(super) fn empty() -> Self { - Self::Materialized(Arc::new(BTreeMap::new()), Arc::default()) - } - - pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option { - match self { - Self::Compact { revisions, .. } => revisions.iter().find_map(|revision| revision.lookup(num, file)), - Self::Materialized(entries, _) => entries.get(&num).copied(), - } - } - - pub(super) fn is_empty(&self) -> bool { - match self { - Self::Compact { has_entries, .. } => !has_entries, - Self::Materialized(entries, _) => entries.is_empty(), - } - } - - /// Drop an entry for object number 4294967295, which no incremental `/Size` can cover. - /// Only a materialized index can hold one: the compact reader requires `max_num < /Size`. - pub(super) fn remove_exhausted(&mut self) -> bool { - match self { - Self::Compact { .. } => false, - Self::Materialized(entries, _) => Arc::make_mut(entries).remove(&u32::MAX).is_some(), - } - } - - /// Every entry, newest section first, as the materialized reader keeps them. - pub(super) fn to_map(&self, file: &[u8]) -> BTreeMap { - match self { - Self::Materialized(entries, _) => entries.as_ref().clone(), - Self::Compact { revisions, .. } => { - let mut numbers = Vec::new(); - for revision in revisions { - revision.extend_numbers(&mut numbers); - } - numbers.into_iter().filter_map(|num| self.lookup(num, file).map(|entry| (num, entry))).collect() - } - } - } - - pub(super) fn max_num(&self) -> u32 { - match self { - Self::Compact { max_num, .. } => *max_num, - Self::Materialized(entries, _) => entries.keys().next_back().copied().unwrap_or(0), - } - } - - pub(super) fn object_numbers(&self, file: &[u8]) -> Vec { - match self { - Self::Materialized(entries, _) => { - entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect() - } - Self::Compact { revisions, .. } => { - if let [revision] = revisions.as_slice() - && revision.hybrid.is_none() - { - let mut numbers = Vec::with_capacity(revision.primary.row_count); - revision.primary.extend_numbers(&mut numbers); - numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. }))); - return numbers; - } - let mut numbers = Vec::new(); - for revision in revisions { - revision.extend_numbers(&mut numbers); - } - numbers.sort_unstable(); - numbers.dedup(); - numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. }))); - numbers - } - } - } -} - -/// One cross-reference section of the file (one per save; the first is the original). -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct Revision { - /// Offset of the xref section (`startxref` value). - pub xref_offset: u64, - /// `true` if the section is a cross-reference stream. - pub is_stream: bool, - /// `true` if `xref_offset` counts from the `%PDF-` header rather than from the start of the - /// file. The two differ only in a file with bytes before its header, and an update appended - /// to such a file must count the same way as the section it points back to. - pub header_relative: bool, -} diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 5de573294d..b0abf693e3 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -1,57 +1,2 @@ -//! pdfcraft-cos — the PDF object layer (L1, standalone with `pdfcraft-filters`). -//! -//! Parses the COS object graph lazily and tolerantly, keeps edits in a copy-on-write overlay -//! (cheap snapshots for undo), and writes documents back either incrementally (original bytes -//! untouched) or as a full, garbage-collected rewrite. The PDF object graph *is* PdfCraft's -//! document model (plan/architecture.md §5, ADR-0010). -//! -//! Done: every cross-reference form (tables, streams, hybrid `/XRefStm`), object streams, repair -//! by scanning, encryption on load and save (R2–R6, crypt filters, `/EFF`), incremental writing, -//! and full saves that pack objects into compressed object streams with a cross-reference stream. -//! Not yet: streaming very large files within a memory budget (`ByteSource`, -//! `core.lazy-loading`), and linearized output (`core.linearization`, M11). See the README. - -#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] - -mod budget; -mod bytes; -mod document; -mod object; -pub mod page_labels; -mod parser; -mod security; -mod writer; - -pub use budget::{DecodeBudget, ImageBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS, MAX_REDACTED_IMAGE_PIXELS}; -pub use bytes::Bytes; -pub use document::{DECODE_PARMS_KEYS, Document, ReadState, Revision, SourceIdentity, XrefEntry}; -pub use object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, Name, ObjRef, Object, PdfString, Stream, TextChars}; -pub use parser::{Lexer, is_whitespace, parse_indirect}; -pub use pdfcraft_crypt::{ - Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, -}; -pub use writer::{SaveOptions, civil_from_days, new_file_id, pdf_date, reachable_objects, serialize, write_full, write_incremental}; - -#[derive(Debug, thiserror::Error, Clone, PartialEq)] -pub enum CosError { - #[error("not a PDF file (no %PDF- header)")] - NotPdf, - #[error("syntax error at byte {offset}: {detail}")] - Syntax { offset: usize, detail: String }, - #[error("object {0} is missing")] - MissingObject(u32), - #[error("object {0} is not a dictionary")] - NotADictionary(ObjRef), - #[error("stream data could not be decoded: {0}")] - Filter(String), - #[error("the document is protected by a password")] - NeedsPassword, - #[error("the password is incorrect")] - WrongPassword, - #[error("unsupported security: {0}")] - Security(String), - #[error("internal lock poisoned")] - Poisoned, - #[error("this document was opened read-only with a decode limit and cannot be saved")] - ReadOnlyLimit, -} +//! PDF object layer (`craft-pdf::cos`). +pub use craft_pdf::cos::*; diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs deleted file mode 100644 index 5d121dc41e..0000000000 --- a/crates/cos/src/object.rs +++ /dev/null @@ -1,834 +0,0 @@ -//! The PDF object model (ISO 32000-2 §7.3). -//! -//! Dictionaries keep their key order so rewritten objects stay recognisable in diffs and -//! byte-stable when re-serialized. Streams keep their *encoded* bytes; decoding is on demand. - -use pdfcraft_filters::{Filter, Params}; - -use crate::{Bytes, CosError}; - -/// An indirect reference: object number and generation. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub struct ObjRef { - pub num: u32, - pub generation: u16, -} - -impl ObjRef { - pub const fn new(num: u32, generation: u16) -> Self { - Self { num, generation } - } -} - -impl std::fmt::Display for ObjRef { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} {} R", self.num, self.generation) - } -} - -/// A string object. `hex` records how it was written, so it is re-emitted the same way. -#[derive(Clone, Debug, PartialEq)] -pub struct PdfString { - pub bytes: Vec, - pub hex: bool, -} - -impl PdfString { - pub fn literal(bytes: impl Into>) -> Self { - Self { bytes: bytes.into(), hex: false } - } - - /// A text string (ISO 32000-2 §7.9.2.2): PDFDocEncoding when possible, else UTF-16BE with BOM. - pub fn text(s: &str) -> Self { - if s.chars().all(|c| (' '..='~').contains(&c) || c == '\n' || c == '\r' || c == '\t') { - return Self::literal(s.as_bytes().to_vec()); - } - let mut bytes = vec![0xFE, 0xFF]; - for u in s.encode_utf16() { - bytes.extend_from_slice(&u.to_be_bytes()); - } - Self { bytes, hex: false } - } - - /// Decode a text string: UTF-16BE (BOM), UTF-8 (BOM, PDF 2.0) or PDFDocEncoding. - /// - /// Lenient beyond the spec, as readers are: invalid sequences after a BOM decode lossily - /// instead of dropping the string, a BOM-less string whose bytes are valid UTF-8 with - /// non-ASCII bytes (written by some producers) is read as UTF-8 rather than as - /// PDFDocEncoding mojibake, and stray leading U+FEFF (doubled BOMs) are removed. - pub fn to_text(&self) -> String { - self.text_chars().collect() - } - - /// The characters [`PdfString::to_text`] decodes, one at a time, without building a string. - pub fn text_chars(&self) -> TextChars<'_> { - let decode = match self.bytes.as_slice() { - [0xFE, 0xFF, rest @ ..] => { - let units: fn(&[u8; 2]) -> u16 = |c| u16::from_be_bytes(*c); - Decode::Utf16(char::decode_utf16(rest.as_chunks::<2>().0.iter().map(units))) - } - [0xEF, 0xBB, 0xBF, rest @ ..] => Decode::Utf8Lossy { chunks: rest.utf8_chunks(), valid: "".chars(), replace: false }, - b => match std::str::from_utf8(b) { - // Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch. - Ok(utf8) if !b.is_ascii() => Decode::Utf8(utf8.chars()), - _ => Decode::PdfDoc(b.iter()), - }, - }; - TextChars { decode, leading: true } - } - - /// Whether the string decodes to `s` ([`PdfString::to_text`]), compared without allocating. - pub fn text_eq(&self, s: &str) -> bool { - self.text_chars().eq(s.chars()) - } -} - -/// The characters of a text string: [`PdfString::text_chars`]. -pub struct TextChars<'a> { - decode: Decode<'a>, - /// Still at the start, where stray U+FEFF (doubled BOMs) are dropped. - leading: bool, -} - -/// The UTF-16BE code units after a BOM (an odd last byte is left out). -type Utf16Units<'a> = std::iter::Map, fn(&[u8; 2]) -> u16>; - -enum Decode<'a> { - Utf16(std::char::DecodeUtf16>), - /// After a UTF-8 BOM: each invalid sequence is one U+FFFD, as `String::from_utf8_lossy` has it. - Utf8Lossy { - chunks: std::str::Utf8Chunks<'a>, - valid: std::str::Chars<'a>, - replace: bool, - }, - Utf8(std::str::Chars<'a>), - PdfDoc(std::slice::Iter<'a, u8>), -} - -impl Iterator for Decode<'_> { - type Item = char; - - fn next(&mut self) -> Option { - match self { - Decode::Utf16(units) => units.next().map(|u| u.unwrap_or(char::REPLACEMENT_CHARACTER)), - Decode::Utf8Lossy { chunks, valid, replace } => loop { - if let Some(c) = valid.next() { - return Some(c); - } - if *replace { - *replace = false; - return Some(char::REPLACEMENT_CHARACTER); - } - let chunk = chunks.next()?; - *valid = chunk.valid().chars(); - *replace = !chunk.invalid().is_empty(); - }, - Decode::Utf8(chars) => chars.next(), - Decode::PdfDoc(bytes) => bytes.next().map(|&c| pdfcraft_crypt::pdfdoc_char(c)), - } - } -} - -impl Iterator for TextChars<'_> { - type Item = char; - - fn next(&mut self) -> Option { - loop { - let c = self.decode.next()?; - if self.leading && c == '\u{FEFF}' { - continue; - } - self.leading = false; - return Some(c); - } - } -} - -/// A name object (without the leading `/`, with `#xx` escapes already decoded). -pub type Name = Vec; - -#[cfg(test)] -thread_local! { - /// Key comparisons and index lookups made by [`Dict`] on this thread. - static KEY_COMPARES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// Entries from which a [`Dict`] keeps a key index. Smaller dictionaries (nearly all of them) -/// search their entries; a parsed file controls the key count, and a linear search per -/// inserted key would make a dictionary of many distinct keys quadratic to build. -const INDEX_FROM: usize = 32; - -/// An ordered dictionary. -#[derive(Clone, Default)] -pub struct Dict { - entries: Vec<(Name, Object)>, - /// Key to position in `entries`, kept once the dictionary reaches [`INDEX_FROM`] entries. - /// Boxed so that every `Dict`, and so every `Object`, grows by one pointer, not by a map. - #[allow(clippy::box_collection)] - index: Option>>, -} - -impl PartialEq for Dict { - fn eq(&self, other: &Self) -> bool { - self.entries == other.entries - } -} - -impl std::fmt::Debug for Dict { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Dict").field("entries", &self.entries).finish() - } -} - -impl Dict { - pub fn new() -> Self { - Self::default() - } - - fn position(&self, key: &[u8]) -> Option { - if let Some(index) = &self.index { - #[cfg(test)] - KEY_COMPARES.with(|c| c.set(c.get() + 1)); - return index.get(key).copied(); - } - self.entries.iter().position(|(k, _)| { - #[cfg(test)] - KEY_COMPARES.with(|c| c.set(c.get() + 1)); - k == key - }) - } - - pub fn get(&self, key: &[u8]) -> Option<&Object> { - self.position(key).and_then(|i| self.entries.get(i)).map(|(_, v)| v) - } - - pub fn get_mut(&mut self, key: &[u8]) -> Option<&mut Object> { - let i = self.position(key)?; - self.entries.get_mut(i).map(|(_, v)| v) - } - - /// Insert or replace, keeping the position of an existing key. - pub fn set(&mut self, key: impl Into, value: impl Into) { - let key = key.into(); - let value = value.into(); - match self.position(&key).and_then(|i| self.entries.get_mut(i)) { - Some((_, v)) => *v = value, - None => { - let at = self.entries.len(); - if let Some(index) = &mut self.index { - index.insert(key.clone(), at); - } else if at + 1 >= INDEX_FROM { - let index = self.entries.iter().enumerate().map(|(i, (k, _))| (k.clone(), i)).chain([(key.clone(), at)]).collect(); - self.index = Some(Box::new(index)); - } - self.entries.push((key, value)); - } - } - } - - pub fn remove(&mut self, key: &[u8]) -> Option { - let i = self.position(key)?; - if let Some(index) = &mut self.index { - index.remove(key); - index.values_mut().filter(|at| **at > i).for_each(|at| *at -= 1); - } - Some(self.entries.remove(i).1) - } - - pub fn contains(&self, key: &[u8]) -> bool { - self.get(key).is_some() - } - - pub fn iter(&self) -> impl Iterator { - self.entries.iter().map(|(k, v)| (k, v)) - } - - pub fn len(&self) -> usize { - self.entries.len() - } - - pub fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - pub fn name(&self, key: &[u8]) -> Option<&[u8]> { - self.get(key).and_then(Object::as_name) - } - - pub fn int(&self, key: &[u8]) -> Option { - self.get(key).and_then(Object::as_int) - } - - pub fn reference(&self, key: &[u8]) -> Option { - self.get(key).and_then(Object::as_ref) - } -} - -impl FromIterator<(Name, Object)> for Dict { - fn from_iter>(iter: T) -> Self { - let mut d = Dict::new(); - for (k, v) in iter { - d.set(k, v); - } - d - } -} - -/// A stream: dictionary plus its data *as stored* (still filtered). -#[derive(Clone, Debug, PartialEq)] -pub struct Stream { - pub dict: Dict, - pub raw: Bytes, -} - -/// Upper bound for a single decoded stream (decompression-bomb defence). -pub const MAX_DECODED: usize = 1 << 30; - -/// The most an image's `/JBIG2Globals` stream may decode to: the symbol and pattern dictionaries an -/// encoder shares between pages, themselves coded, so far less than any stream may decode to -/// ([`MAX_DECODED`]). They are decoded before the image's JBIG2 page is read, so the page's size -/// can't bound them: they have a limit of their own. -pub const MAX_JBIG2_GLOBALS: usize = 32 << 20; - -impl Stream { - /// A new stream from already-encoded bytes (`/Length` is set at write time). - pub fn from_raw(dict: Dict, raw: Vec) -> Self { - Self { dict, raw: raw.into() } - } - - /// A new stream holding `data` compressed with Flate. - pub fn flate(mut dict: Dict, data: &[u8]) -> Self { - dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); - dict.remove(b"DecodeParms"); - Self { dict, raw: pdfcraft_filters::encode_flate(data).into() } - } - - /// The filter chain declared in the dictionary. - pub fn filters(&self) -> Vec<(Filter, Params)> { - let names: Vec<&[u8]> = match self.dict.get(b"Filter") { - Some(Object::Name(n)) => vec![n], - Some(Object::Array(a)) => a.iter().filter_map(Object::as_name).collect(), - _ => Vec::new(), - }; - let parms: Vec> = match self.dict.get(b"DecodeParms") { - Some(Object::Dict(d)) => vec![Some(d)], - Some(Object::Array(a)) => a.iter().map(Object::as_dict).collect(), - _ => Vec::new(), - }; - names - .into_iter() - .enumerate() - // `/Crypt` is applied by the security handler when the object is loaded (§7.4.10). - .filter(|(_, n)| *n != b"Crypt") - .map(|(i, n)| { - let p = parms.get(i).copied().flatten(); - let get = |k: &[u8], d: i64| p.and_then(|p| p.int(k)).unwrap_or(d); - let params = Params { - predictor: get(b"Predictor", 1), - colors: get(b"Colors", 1), - bits_per_component: get(b"BitsPerComponent", 8), - columns: get(b"Columns", 1), - early_change: get(b"EarlyChange", 1), - }; - (Filter::from_name(n), params) - }) - .collect() - } - - /// [`Stream::filters`], or an error when `/Filter` is present but is not a name or an - /// array of names (an indirect reference, say): the data is not unfiltered, so it must not - /// read as decoded. - pub(crate) fn readable_filters(&self) -> Result, pdfcraft_filters::FilterError> { - let readable = match self.dict.get(b"Filter") { - None | Some(Object::Null | Object::Name(_)) => true, - Some(Object::Array(a)) => a.iter().all(|f| f.as_name().is_some()), - Some(_) => false, - }; - if !readable { - return Err(pdfcraft_filters::FilterError::Unsupported("/Filter is not a name or an array of names".into())); - } - Ok(self.filters()) - } - - /// Decoded data, tolerating truncated or corrupt encodings the way viewers do (the data - /// decoded before the damage is returned). Decoding past - /// [`crate::budget::MAX_PAGE_DECODED`] (256 MiB), the most the renderer reads from one - /// stream, is an error. Streams with image codecs fail with - /// `CosError::Filter` (keep them encoded). Use `decoded_strict` to detect damage. - pub fn decoded(&self) -> Result, CosError> { - self.decoded_within(crate::budget::MAX_PAGE_DECODED) - } - - /// Like [`Stream::decoded`], for a stream whose decoded size has a tighter bound than - /// [`MAX_DECODED`]: decoding past `max` bytes is an error. - pub fn decoded_within(&self, max: usize) -> Result, CosError> { - let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant(chain, data, max).map(|(v, _)| (v, 0)).map_err(|e| (e, 0)) - }; - self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) - } - - /// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept - /// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter - /// stages together, and the result gives whether the data stopped at damage (it is then what - /// decoded before it) and the bytes the stages decoded (the work done, on success and on - /// failure). - pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, bool, usize), (pdfcraft_filters::FilterError, usize)> { - let mut damaged = false; - let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| { - damaged = partial; - (v, spent) - }) - }; - self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) - } - - /// [`Stream::decoded_within_counted`] with every filter stage also held to `each`, as - /// [`Stream::decoded_within`] holds it, while `max` bounds the stages together - /// ([`pdfcraft_filters::decode_tolerant_counted_each`]). Without filters the data is held to - /// both. - pub(crate) fn decoded_within_each_counted( - &self, - each: usize, - max: usize, - ) -> Result<(Vec, bool, usize), (pdfcraft_filters::FilterError, usize)> { - let mut damaged = false; - let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant_counted_each(chain, data, each.min(MAX_DECODED), max).map(|(v, partial, spent)| { - damaged = partial; - (v, spent) - }) - }; - let max = if self.readable_filters().is_ok_and(|chain| chain.is_empty()) { max.min(each) } else { max }; - self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) - } - - /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a - /// document's images ([`pdfcraft_filters::decode_tolerant_within`]). - pub(crate) fn decoded_spending(&self, max: usize, work: &mut u64) -> Result, CosError> { - let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string()); - let chain = self.readable_filters().map_err(filter)?; - pdfcraft_filters::decode_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), work).map_err(filter) - } - - /// [`Stream::decoded_spending`] for the samples of an image `rows` high (its `/Height`): - /// [`pdfcraft_filters::decode_image_tolerant_within`]. - pub(crate) fn decoded_image_rows_spending(&self, max: usize, rows: u32, work: &mut u64) -> Result, CosError> { - let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string()); - let chain = self.readable_filters().map_err(filter)?; - pdfcraft_filters::decode_image_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), rows, work).map_err(filter) - } - - /// The stream decoded through its filters by `decode`, or copied when it has none, within - /// `max` and [`MAX_DECODED`]. - fn decode_chain( - &self, - max: usize, - decode: impl FnOnce(&[(Filter, Params)], &[u8], usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)>, - ) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { - let max = max.min(MAX_DECODED); - let chain = self.readable_filters().map_err(|e| (e, 0))?; - if chain.is_empty() { - return self - .raw_within_typed(max) - .map(|v| { - let n = v.len(); - (v, n) - }) - .map_err(|e| (e, 0)); - } - decode(&chain, &self.raw, max) - } - - /// Decoded data, bounded by [`MAX_DECODED`]; any corruption is an error. - pub fn decoded_strict(&self) -> Result, CosError> { - self.decoded_strict_within(MAX_DECODED) - } - - /// Like [`Stream::decoded_strict`], bounded by `max` (and [`MAX_DECODED`]): for data that - /// will be copied elsewhere, where a damaged stream must not pass for a shorter one. - pub fn decoded_strict_within(&self, max: usize) -> Result, CosError> { - let decode = |chain: &[(Filter, Params)], data: &[u8], max| pdfcraft_filters::decode(chain, data, max).map(|v| (v, 0)).map_err(|e| (e, 0)); - self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) - } - - /// [`Stream::decoded_strict`] bounded by `max` and counted like - /// [`Stream::decoded_within_counted`]. - pub(crate) fn decoded_strict_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { - self.decode_chain(max, pdfcraft_filters::decode_counted) - } - - /// Data after the first filter alone, bounded by `max`: the rest of the chain stays encoded. - /// Any corruption in that first filter is an error. - pub fn decoded_first_filter_strict(&self, max: usize) -> Result, CosError> { - let mut chain = self.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?; - chain.truncate(1); - if chain.is_empty() { - return self.raw_within(max); - } - pdfcraft_filters::decode(&chain, &self.raw, max.min(MAX_DECODED)).map_err(|e| CosError::Filter(e.to_string())) - } - - fn raw_within(&self, max: usize) -> Result, CosError> { - self.raw_within_typed(max).map_err(|e| CosError::Filter(e.to_string())) - } - - fn raw_within_typed(&self, max: usize) -> Result, pdfcraft_filters::FilterError> { - let max = max.min(MAX_DECODED); - // An empty decoder chain still produces an owned output buffer: check its limit - // before cloning, including streams whose /Crypt filter was applied on load. - if self.raw.len() > max { - return Err(pdfcraft_filters::FilterError::LimitExceeded(max)); - } - Ok(self.raw.to_vec()) - } -} - -#[derive(Clone, Debug, PartialEq)] -pub enum Object { - Null, - Bool(bool), - Int(i64), - Real(f64), - String(PdfString), - Name(Name), - Array(Vec), - Dict(Dict), - Stream(Stream), - Ref(ObjRef), -} - -impl Object { - pub fn name(n: &str) -> Self { - Object::Name(n.as_bytes().to_vec()) - } - - pub fn as_int(&self) -> Option { - match self { - Object::Int(i) => Some(*i), - Object::Real(r) if r.fract() == 0.0 && r.is_finite() => Some(*r as i64), - _ => None, - } - } - - pub fn as_f64(&self) -> Option { - match self { - Object::Int(i) => Some(*i as f64), - Object::Real(r) => Some(*r), - _ => None, - } - } - - pub fn as_name(&self) -> Option<&[u8]> { - match self { - Object::Name(n) => Some(n), - _ => None, - } - } - - pub fn as_ref(&self) -> Option { - match self { - Object::Ref(r) => Some(*r), - _ => None, - } - } - - pub fn as_dict(&self) -> Option<&Dict> { - match self { - Object::Dict(d) => Some(d), - Object::Stream(s) => Some(&s.dict), - _ => None, - } - } - - pub fn as_dict_mut(&mut self) -> Option<&mut Dict> { - match self { - Object::Dict(d) => Some(d), - Object::Stream(s) => Some(&mut s.dict), - _ => None, - } - } - - pub fn as_array(&self) -> Option<&Vec> { - match self { - Object::Array(a) => Some(a), - _ => None, - } - } - - pub fn as_string(&self) -> Option<&PdfString> { - match self { - Object::String(s) => Some(s), - _ => None, - } - } -} - -impl From for Object { - fn from(v: i64) -> Self { - Object::Int(v) - } -} -impl From for Object { - fn from(v: i32) -> Self { - Object::Int(v as i64) - } -} -impl From for Object { - fn from(v: f64) -> Self { - Object::Real(v) - } -} -impl From for Object { - fn from(v: bool) -> Self { - Object::Bool(v) - } -} -impl From for Object { - fn from(v: ObjRef) -> Self { - Object::Ref(v) - } -} -impl From for Object { - fn from(v: Dict) -> Self { - Object::Dict(v) - } -} -impl From> for Object { - fn from(v: Vec) -> Self { - Object::Array(v) - } -} -impl From for Object { - fn from(v: PdfString) -> Self { - Object::String(v) - } -} -impl From for Object { - fn from(v: Stream) -> Self { - Object::Stream(v) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_filter_that_is_not_a_direct_name_is_not_decoded_as_absent() { - let stream = |filter: Object| { - let mut d = Dict::new(); - d.set(b"Filter".to_vec(), filter); - Stream::from_raw(d, b"48 69>".to_vec()) - }; - let hex = Object::Name(b"ASCIIHexDecode".to_vec()); - assert_eq!(stream(hex.clone()).decoded_strict().ok(), Some(b"Hi".to_vec())); - assert_eq!(stream(Object::Array(vec![hex.clone()])).decoded().ok(), Some(b"Hi".to_vec())); - assert_eq!(stream(Object::Null).decoded_strict().ok(), Some(b"48 69>".to_vec())); - for filter in [ - Object::Ref(ObjRef::new(5, 0)), - Object::Array(vec![Object::Ref(ObjRef::new(5, 0))]), - Object::Int(3), - Object::Array(vec![hex, Object::Int(1)]), - ] { - let s = stream(filter.clone()); - assert!(s.decoded_strict().is_err(), "{filter:?}"); - assert!(s.decoded().is_err(), "{filter:?}"); - assert!(s.decoded_first_filter_strict(1 << 20).is_err(), "{filter:?}"); - } - } - - #[test] - fn parsing_many_distinct_keys_is_not_quadratic() { - let n = 20_000; - let body: String = (0..n).map(|i| format!("/K{i} {i} ")).collect(); - let text = format!("<< {body}/K7 true >>"); - KEY_COMPARES.with(|c| c.set(0)); - let Ok(Object::Dict(d)) = crate::parser::Lexer::new(text.as_bytes(), 0).object() else { panic!("not a dict") }; - let compares = KEY_COMPARES.with(|c| c.get()); - assert!(compares < 8 * n, "{compares} key comparisons for {n} keys"); - // Order and replace-in-place semantics stay those of a plain ordered dictionary. - assert_eq!(d.len(), n); - assert_eq!(d.iter().next().map(|(k, _)| k.as_slice()), Some(&b"K0"[..])); - assert_eq!(d.get(b"K7"), Some(&Object::Bool(true))); - assert_eq!(d.iter().nth(7).map(|(_, v)| v), Some(&Object::Bool(true))); - let mut d = d; - assert_eq!(d.remove(b"K3"), Some(Object::Int(3))); - assert_eq!(d.get(b"K4"), Some(&Object::Int(4))); - assert_eq!(d.iter().nth(3).map(|(k, _)| k.as_slice()), Some(&b"K4"[..])); - d.set(b"K3".to_vec(), Object::Null); - assert_eq!(d.iter().last().map(|(k, _)| k.as_slice()), Some(&b"K3"[..])); - assert!(d.get(b"K20000").is_none()); - } - - #[test] - fn text_strings_round_trip() { - for s in ["Plain ASCII", "Café — naïve", "日本語", "emoji 📄"] { - assert_eq!(PdfString::text(s).to_text(), s); - } - assert!(!PdfString::text("abc").bytes.starts_with(&[0xFE, 0xFF])); - } - - #[test] - fn text_string_encodings() { - // UTF-16BE with BOM. - let mut utf16 = vec![0xFE, 0xFF]; - utf16.extend("第一章 概述".encode_utf16().flat_map(u16::to_be_bytes)); - assert_eq!(PdfString::literal(utf16).to_text(), "第一章 概述"); - // UTF-8 with BOM (PDF 2.0): the BOM is not part of the text. - let mut utf8 = vec![0xEF, 0xBB, 0xBF]; - utf8.extend_from_slice("第一章".as_bytes()); - assert_eq!(PdfString::literal(utf8).to_text(), "第一章"); - // Invalid UTF-8 after the BOM decodes lossily instead of vanishing. - assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, b'A', 0xFF, b'B']).to_text(), "A\u{FFFD}B"); - // BOM-less raw UTF-8 from producers that skip the BOM. - assert_eq!(PdfString::literal("目录".as_bytes()).to_text(), "目录"); - // PDFDocEncoding Latin-1 (not valid UTF-8) stays PDFDocEncoding. - assert_eq!(PdfString::literal(vec![b'C', b'a', b'f', 0xE9]).to_text(), "Café"); - // Doubled BOMs leave no U+FEFF behind. - assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A']).to_text(), "A"); - assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF, b'A']).to_text(), "A"); - // Odd trailing byte and empty strings never panic. - assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0x00, b'A', 0x00]).to_text(), "A"); - assert_eq!(PdfString::literal(vec![0xFE, 0xFF]).to_text(), ""); - assert_eq!(PdfString::literal(Vec::new()).to_text(), ""); - } - - #[test] - fn text_eq_compares_what_to_text_decodes() { - let utf16 = |s: &str| [0xFE, 0xFF].into_iter().chain(s.encode_utf16().flat_map(u16::to_be_bytes)).collect::>(); - // Each invalid UTF-8 sequence is one U+FFFD, as String::from_utf8_lossy has it. - let lossy = [0xEF, 0xBB, 0xBF, 0xC3, 0x28, b'x', 0xF0, 0x9F, 0x92, 0xFF]; - assert_eq!(PdfString::literal(lossy.to_vec()).to_text(), String::from_utf8_lossy(&lossy[3..])); - let cases: Vec> = vec![ - Vec::new(), - b"Plain".to_vec(), - vec![b'C', b'a', b'f', 0xE9, 0x18], - "目录".as_bytes().to_vec(), - utf16("第一章"), - vec![0xFE, 0xFF, 0xD8, 0x00, 0x00, b'A'], - vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A', 0xFE, 0xFF], - lossy.to_vec(), - ]; - for bytes in cases { - let s = PdfString::literal(bytes); - let text = s.to_text(); - assert!(s.text_eq(&text), "{text:?}"); - assert!(!s.text_eq(&format!("{text}x")), "{text:?}"); - assert!(text.is_empty() || !s.text_eq(&text[..text.len() - text.chars().last().map_or(0, char::len_utf8)]), "{text:?}"); - } - // A U+FEFF after the first character is text and stays. - assert!(PdfString::literal(utf16("A\u{FEFF}")).text_eq("A\u{FEFF}")); - } - - #[test] - fn pdfdoc_encoding_high_range() { - assert_eq!(PdfString::literal(vec![0x84, 0x92, 0xA0, b'A']).to_text(), "—™€A"); - } - - #[test] - fn dict_preserves_order_and_replaces_in_place() { - let mut d = Dict::new(); - d.set(b"B".to_vec(), 1); - d.set(b"A".to_vec(), 2); - d.set(b"B".to_vec(), 3); - let keys: Vec<_> = d.iter().map(|(k, _)| k.clone()).collect(); - assert_eq!(keys, vec![b"B".to_vec(), b"A".to_vec()]); - assert_eq!(d.int(b"B"), Some(3)); - } - - #[test] - fn raw_streams_refuse_decoding_past_a_caller_limit() { - let stream = Stream::from_raw(Dict::new(), b"abc".to_vec()); - for max in [0, 2] { - assert!(matches!(stream.decoded_within(max), Err(CosError::Filter(_)))); - } - } - - #[test] - fn strict_decoding_is_bounded_by_the_callers_remaining_allowance() { - let stream = Stream::flate(Dict::new(), &vec![0u8; 1 << 20]); - assert!(matches!(stream.decoded_strict_within(1000), Err(CosError::Filter(_)))); - assert_eq!(stream.decoded_strict_within(1 << 20).map(|v| v.len()).ok(), Some(1 << 20)); - } - - #[test] - fn empty_filter_arrays_refuse_decoding_past_a_caller_limit() { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(Vec::new())); - let stream = Stream::from_raw(dict, b"abc".to_vec()); - assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); - } - - #[test] - fn crypt_only_streams_refuse_decoding_past_a_caller_limit() { - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("Crypt")); - let mut parms = Dict::new(); - parms.set(b"Name".to_vec(), Object::name("Identity")); - dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - let stream = Stream::from_raw(dict, b"abc".to_vec()); - assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); - } - - #[test] - fn identity_streams_preserve_bytes_within_a_caller_limit() { - let mut array = Dict::new(); - array.set(b"Filter".to_vec(), Object::Array(Vec::new())); - let mut crypt = Dict::new(); - crypt.set(b"Filter".to_vec(), Object::name("Crypt")); - for dict in [Dict::new(), array, crypt] { - let stream = Stream::from_raw(dict.clone(), b"abc".to_vec()); - assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); - assert_eq!(stream.decoded_within(usize::MAX).unwrap(), b"abc"); - assert_eq!(stream.decoded().unwrap(), b"abc"); - assert_eq!(stream.decoded_strict().unwrap(), b"abc"); - assert!(Stream::from_raw(dict, Vec::new()).decoded_within(0).unwrap().is_empty()); - } - } - - #[test] - fn filtered_streams_preserve_the_caller_limit() { - let stream = Stream::flate(Dict::new(), b"abc"); - assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_)))); - assert_eq!(stream.decoded_within(3).unwrap(), b"abc"); - assert_eq!(stream.decoded_strict().unwrap(), b"abc"); - } - - #[test] - fn raw_flate_fallback_streams_preserve_the_caller_limit() { - // Raw deflate with a stored byte and a final empty block. 78 01 also - // passes the zlib header check, so decoding must retry the whole input. - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); - let stream = Stream::from_raw(dict, vec![0x78, 0x01, 0x00, 0xfe, 0xff, b'A', 0x01, 0x00, 0x00, 0xff, 0xff]); - let result = stream.decoded_within(0); - assert!(matches!(&result, Err(CosError::Filter(detail)) if detail.contains("exceeds the limit of 0 bytes")), "{result:?}"); - assert_eq!(stream.decoded_within(1).unwrap(), b"A"); - assert_eq!(stream.decoded_strict().unwrap(), b"A"); - } - - #[test] - fn decoded_stops_at_renderer_cap_of_256_mib() { - let data = vec![0u8; (256 << 20) + 1]; - let stream = Stream::flate(Dict::new(), &data); - assert!(matches!(stream.decoded(), Err(CosError::Filter(_)))); - } - - #[test] - fn first_filter_decodes_only_the_outer_wrapper() { - let data = "0 0 m 100 100 l S\n".repeat(64).into_bytes(); - let inner = Stream::flate(Dict::new(), &data).raw.to_vec(); - for (outer, name, corrupt) in [(Filter::Ascii85, "ASCII85Decode", &b"{~>"[..]), (Filter::AsciiHex, "ASCIIHexDecode", &b"G0>"[..])] { - let encoded = pdfcraft_filters::encode(&outer, &Params::default(), &inner).unwrap(); - let mut dict = Dict::new(); - dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name(name), Object::name("FlateDecode")])); - let stream = Stream::from_raw(dict.clone(), encoded); - assert_eq!(stream.decoded_first_filter_strict(usize::MAX).unwrap(), inner); - assert_eq!(stream.decoded_strict().unwrap(), data); - assert!(matches!(stream.decoded_first_filter_strict(inner.len() - 1), Err(CosError::Filter(_)))); - assert!(matches!(Stream::from_raw(dict, corrupt.to_vec()).decoded_first_filter_strict(usize::MAX), Err(CosError::Filter(_)))); - } - } -} diff --git a/crates/cos/src/page_labels.rs b/crates/cos/src/page_labels.rs deleted file mode 100644 index 557929f8bf..0000000000 --- a/crates/cos/src/page_labels.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! Bounded numbering for page labels (ISO 32000-2 §12.4.2). -//! Pure helpers shared by inspection and editing; no object graph or renderer dependency. - -/// A complete custom label, including its prefix, may contain this many UTF-8 bytes. -pub const MAX_LABEL_BYTES: usize = 1024; -/// Raw prefixes are bounded before decoding; this admits a full UTF-16BE ASCII label. -pub const MAX_PREFIX_BYTES: usize = 2 * MAX_LABEL_BYTES + 2; -/// Sum of the UTF-8 bytes of custom labels for one document. -pub const MAX_LABEL_TOTAL_BYTES: usize = 4 << 20; -/// Sum of the UTF-8 bytes of the decoded prefixes of one `/PageLabels` tree. Each range decodes -/// its own prefix, and ranges may share one prefix object, so without this a small file could ask -/// for a copy per range: [`MAX_LABEL_TREE_WORK`] ranges naming one 1,024-byte prefix would hold -/// about 200 MiB. Real documents stay far below it. -pub const MAX_PREFIX_TOTAL_BYTES: usize = 4 << 20; -/// The most number-tree entries (nodes, kids and range pairs) read from one `/PageLabels` -/// tree. Scaled for real documents: one range per page on a ten-thousand-page PDF happens -/// (prefix labels; #307's file has 9,156 pages), so the cap is twenty times that while still -/// bounding hostile trees. -pub const MAX_LABEL_TREE_WORK: usize = 200_000; -/// Maximum nesting of a label number tree. -pub const MAX_LABEL_TREE_DEPTH: usize = 32; - -/// Lowercase Roman numerals, or `None` if the output exceeds the remaining label budget. -/// The budget is capped at [`MAX_LABEL_BYTES`], including when `n` is hostile. -pub fn roman(mut n: u64, max_bytes: usize) -> Option { - const PARTS: [(u64, &str); 13] = [ - (1000, "m"), - (900, "cm"), - (500, "d"), - (400, "cd"), - (100, "c"), - (90, "xc"), - (50, "l"), - (40, "xl"), - (10, "x"), - (9, "ix"), - (5, "v"), - (4, "iv"), - (1, "i"), - ]; - let max_bytes = max_bytes.min(MAX_LABEL_BYTES); - let mut out = String::new(); - for (value, part) in PARTS { - let count = usize::try_from(n / value).ok()?; - let bytes = count.checked_mul(part.len())?; - if bytes > max_bytes.saturating_sub(out.len()) { - return None; - } - // The quotient is checked against the tiny byte budget before repeating. - out.push_str(&part.repeat(count)); - n %= value; - } - Some(out) -} - -/// a..z, aa..zz, aaa.. (the repeated-letter numbering specified by PDF). -/// Returns `None` instead of truncating a label that exceeds the remaining byte budget. -pub fn alpha(n: u64, max_bytes: usize) -> Option { - let n = n.max(1) - 1; - let count = usize::try_from(n / 26 + 1).ok()?; - if count > max_bytes.min(MAX_LABEL_BYTES) { - return None; - } - let letter = char::from(b'a' + (n % 26) as u8); - Some(std::iter::repeat_n(letter, count).collect()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn bounded_page_labels_follow_pdf_numbering() { - assert_eq!(roman(4, MAX_LABEL_BYTES).as_deref(), Some("iv")); - assert_eq!(roman(1994, MAX_LABEL_BYTES).as_deref(), Some("mcmxciv")); - for (n, expected) in [(0, "a"), (1, "a"), (26, "z"), (27, "aa"), (52, "zz"), (53, "aaa")] { - assert_eq!(alpha(n, MAX_LABEL_BYTES).as_deref(), Some(expected)); - } - } - - #[test] - fn bounded_page_labels_check_lengths_before_repeating() { - assert_eq!(roman(1_024_000, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES); - assert!(roman(1_024_001, MAX_LABEL_BYTES).is_none()); - assert_eq!(alpha(26 * 1024, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES); - assert!(alpha(26 * 1024 + 1, MAX_LABEL_BYTES).is_none()); - assert_eq!(roman(4, 2).as_deref(), Some("iv")); - assert!(roman(4, 1).is_none()); - assert_eq!(alpha(27, 2).as_deref(), Some("aa")); - assert!(alpha(27, 1).is_none()); - for max in [0, MAX_LABEL_BYTES, usize::MAX] { - assert!(roman(u64::MAX, max).is_none()); - assert!(alpha(u64::MAX, max).is_none()); - } - } -} diff --git a/crates/cos/src/parser.rs b/crates/cos/src/parser.rs deleted file mode 100644 index 49793d950c..0000000000 --- a/crates/cos/src/parser.rs +++ /dev/null @@ -1,886 +0,0 @@ -//! Lexer and object parser (ISO 32000-2 §7.2–7.3), tolerant of common real-world damage. -//! -//! The parser works on a byte slice and a position; it never allocates for skipped data and -//! never panics on malformed input (every index is bounds-checked). Depth is limited to defend -//! against pathological nesting. - -use std::sync::Arc; - -use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; -use crate::{Bytes, CosError}; - -const MAX_DEPTH: usize = 128; -/// Objects one top-level [`Lexer::object`] call may build. Real arrays (page trees, parent -/// trees, widths) stay far below; a decompressed object stream of `0 0 0 ...` does not. -const MAX_NODES: usize = 1 << 22; - -/// One of the six PDF whitespace characters (ISO 32000-2 §7.2.3, Table 1). -pub fn is_whitespace(b: u8) -> bool { - matches!(b, b'\0' | b'\t' | b'\n' | b'\x0C' | b'\r' | b' ') -} - -pub(crate) fn is_delimiter(b: u8) -> bool { - matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%') -} - -fn is_regular(b: u8) -> bool { - !is_whitespace(b) && !is_delimiter(b) -} - -/// A cursor over PDF bytes. -pub struct Lexer<'a> { - pub data: &'a [u8], - pub pos: usize, - /// Objects built by the current top-level [`Lexer::object`] call. - nodes: usize, -} - -impl<'a> Lexer<'a> { - pub fn new(data: &'a [u8], pos: usize) -> Self { - Self { data, pos: pos.min(data.len()), nodes: 0 } - } - - fn peek(&self) -> Option { - self.data.get(self.pos).copied() - } - - /// Skip whitespace and comments. - pub fn skip_ws(&mut self) { - while let Some(b) = self.peek() { - if is_whitespace(b) { - self.pos += 1; - } else if b == b'%' { - while let Some(c) = self.peek() { - if c == b'\n' || c == b'\r' { - break; - } - self.pos += 1; - } - } else { - break; - } - } - } - - /// Read a run of regular characters (a keyword or number). - pub fn token(&mut self) -> &'a [u8] { - let start = self.pos.min(self.data.len()); - while self.peek().is_some_and(is_regular) { - self.pos += 1; - } - &self.data[start..self.pos] - } - - /// `true` (and consumes it) if the next token is exactly `kw`. - pub fn eat_keyword(&mut self, kw: &[u8]) -> bool { - self.skip_ws(); - let end = self.pos + kw.len(); - if self.data.get(self.pos..end) == Some(kw) && self.data.get(end).is_none_or(|b| !is_regular(*b)) { - self.pos = end; - true - } else { - false - } - } - - fn err(&self, what: &str) -> CosError { - CosError::Syntax { offset: self.pos, detail: what.to_string() } - } - - /// Parse one direct object. References `N G R` are recognised. - pub fn object(&mut self) -> Result { - self.nodes = 0; - self.object_depth(0) - } - - fn over_budget(&self) -> bool { - self.nodes > MAX_NODES - } - - fn object_depth(&mut self, depth: usize) -> Result { - if depth > MAX_DEPTH { - return Err(self.err("nesting too deep")); - } - self.nodes = self.nodes.saturating_add(1); - if self.over_budget() { - return Err(self.err("object too large")); - } - self.skip_ws(); - let Some(b) = self.peek() else { return Err(self.err("unexpected end of data")) }; - match b { - b'/' => { - self.pos += 1; - Ok(Object::Name(self.name_body())) - } - b'(' => { - self.pos += 1; - Ok(Object::String(PdfString { bytes: self.literal_string()?, hex: false })) - } - b'<' if self.data.get(self.pos + 1) == Some(&b'<') => { - self.pos += 2; - Ok(Object::Dict(self.dict_body(depth)?)) - } - b'<' => { - self.pos += 1; - Ok(Object::String(PdfString { bytes: self.hex_string()?, hex: true })) - } - b'[' => { - self.pos += 1; - let mut items = Vec::new(); - loop { - self.skip_ws(); - match self.peek() { - None => return Err(self.err("unterminated array")), - Some(b']') => { - self.pos += 1; - break; - } - _ => { - let before = self.pos; - match self.object_depth(depth + 1) { - Ok(o) => items.push(o), - Err(e) if self.over_budget() => return Err(e), - // Tolerate junk inside arrays by skipping one byte. - Err(_) if self.pos == before => self.pos += 1, - Err(e) => return Err(e), - } - } - } - } - Ok(Object::Array(items)) - } - b'+' | b'-' | b'.' | b'0'..=b'9' => self.number_or_ref(), - _ => { - let start = self.pos; - let tok = self.token(); - match tok { - b"true" => Ok(Object::Bool(true)), - b"false" => Ok(Object::Bool(false)), - b"null" => Ok(Object::Null), - b"" => { - self.pos = start + 1; - Err(CosError::Syntax { offset: start, detail: format!("unexpected byte 0x{b:02x}") }) - } - other => Err(CosError::Syntax { offset: start, detail: format!("unexpected keyword {:?}", String::from_utf8_lossy(other)) }), - } - } - } - } - - fn name_body(&mut self) -> Vec { - let mut out = Vec::new(); - while let Some(b) = self.peek() { - if !is_regular(b) { - break; - } - self.pos += 1; - if b == b'#' - && let (Some(h), Some(l)) = (self.data.get(self.pos).and_then(|c| hexval(*c)), self.data.get(self.pos + 1).and_then(|c| hexval(*c))) - { - out.push(h << 4 | l); - self.pos += 2; - continue; - } - out.push(b); - } - out - } - - fn literal_string(&mut self) -> Result, CosError> { - let mut out = Vec::new(); - let mut depth = 1usize; - while let Some(b) = self.peek() { - self.pos += 1; - match b { - b'(' => { - depth += 1; - out.push(b); - } - b')' => { - depth -= 1; - if depth == 0 { - return Ok(out); - } - out.push(b); - } - b'\\' => { - let Some(e) = self.peek() else { break }; - self.pos += 1; - match e { - b'n' => out.push(b'\n'), - b'r' => out.push(b'\r'), - b't' => out.push(b'\t'), - b'b' => out.push(8), - b'f' => out.push(12), - b'(' | b')' | b'\\' => out.push(e), - b'\r' => { - if self.peek() == Some(b'\n') { - self.pos += 1; - } - } - b'\n' => {} - b'0'..=b'7' => { - let mut v = (e - b'0') as u32; - for _ in 0..2 { - match self.peek() { - Some(d @ b'0'..=b'7') => { - v = v * 8 + (d - b'0') as u32; - self.pos += 1; - } - _ => break, - } - } - out.push((v & 0xFF) as u8); - } - other => out.push(other), - } - } - // End-of-line markers inside strings are normalised to \n (§7.3.4.2). - b'\r' => { - if self.peek() == Some(b'\n') { - self.pos += 1; - } - out.push(b'\n'); - } - other => out.push(other), - } - } - Err(self.err("unterminated string")) - } - - fn hex_string(&mut self) -> Result, CosError> { - let mut out = Vec::new(); - let mut hi: Option = None; - while let Some(b) = self.peek() { - self.pos += 1; - if b == b'>' { - if let Some(h) = hi { - out.push(h << 4); - } - return Ok(out); - } - if is_whitespace(b) { - continue; - } - let Some(v) = hexval(b) else { return Err(self.err("invalid hex digit")) }; - match hi.take() { - None => hi = Some(v), - Some(h) => out.push(h << 4 | v), - } - } - Err(self.err("unterminated hex string")) - } - - fn dict_body(&mut self, depth: usize) -> Result { - let mut d = Dict::new(); - loop { - self.skip_ws(); - match self.peek() { - None => return Err(self.err("unterminated dictionary")), - Some(b'>') if self.data.get(self.pos + 1) == Some(&b'>') => { - self.pos += 2; - return Ok(d); - } - Some(b'/') => { - self.pos += 1; - let key = self.name_body(); - self.skip_ws(); - // A key directly followed by `>>` has no value: treat as null. (A following - // `/` is a name *value*, so it cannot signal a missing value.) - if self.peek() == Some(b'>') && self.data.get(self.pos + 1) == Some(&b'>') { - d.set(key, Object::Null); - continue; - } - let value = self.object_depth(depth + 1)?; - d.set(key, value); - } - Some(_) => { - // Tolerate garbage between entries (seen in damaged files): skip a token. - let before = self.pos; - let _ = self.object_depth(depth + 1); - if self.over_budget() { - return Err(self.err("object too large")); - } - if self.pos == before { - self.pos += 1; - } - } - } - } - } - - fn number_or_ref(&mut self) -> Result { - let start = self.pos; - let first = self.number()?; - // Look ahead for "G R" to form a reference. - if let Object::Int(n) = first - && n >= 0 - { - let save = self.pos; - self.skip_ws(); - if self.peek().is_some_and(|b| b.is_ascii_digit()) { - let gstart = self.pos; - let g = self.token(); - if g.iter().all(u8::is_ascii_digit) && self.eat_keyword(b"R") { - let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::().ok()).unwrap_or(0); - if n <= u32::MAX as i64 { - return Ok(Object::Ref(ObjRef::new(n as u32, generation.min(u16::MAX as u32) as u16))); - } - } - let _ = gstart; - } - self.pos = save; - } - let _ = start; - Ok(first) - } - - fn number(&mut self) -> Result { - let start = self.pos; - let tok = self.token(); - let s = std::str::from_utf8(tok).map_err(|_| self.err("bad number"))?; - if s.contains('.') { - // Tolerate forms like "--5", "5-" or "1.2.3" by parsing the longest valid prefix. - // Only a doubled sign needs a cleaned copy: a page of InDesign `/PieceInfo` is - // thousands of plain reals, and building a copy and a substring search for each - // was a large part of reading one (#5873). - let t = s.trim_start_matches('+'); - let cleaned = if t.as_bytes().windows(2).any(|w| w == b"--") { std::borrow::Cow::Owned(t.replacen("--", "-", 1)) } else { t.into() }; - let v = cleaned.get(..real_prefix_len(cleaned.as_bytes())).and_then(|p| parse_counted::(p).ok()); - return Ok(Object::Real(v.unwrap_or(0.0))); - } - // Tolerate a doubled sign ("--280") like the real branch does. - let t = s.trim_start_matches('+'); - let cleaned = t.strip_prefix('-').filter(|r| r.starts_with('-')).unwrap_or(t); - match cleaned.parse::() { - Ok(v) => Ok(Object::Int(v)), - Err(_) => { - // Out-of-range integers are treated as reals (§7.3.3 allows implementation limits). - cleaned.parse::().map(Object::Real).map_err(|_| CosError::Syntax { offset: start, detail: format!("bad number {s:?}") }) - } - } - } -} - -/// Length of the longest prefix of `s` that reads as a decimal number: a sign, digits with at -/// most one `.`, and an exponent with digits. Zero when that prefix has no digit. Found in one -/// pass so a long damaged token is converted once, not once per shorter prefix. -fn real_prefix_len(s: &[u8]) -> usize { - let digits = |from: usize| s.get(from..).map_or(0, |r| r.iter().take_while(|b| b.is_ascii_digit()).count()); - let mut at = usize::from(matches!(s.first(), Some(b'+' | b'-'))); - let int = digits(at); - at += int; - let mut frac = 0; - if s.get(at) == Some(&b'.') { - frac = digits(at + 1); - at += 1 + frac; - } - if int + frac == 0 { - return 0; - } - if matches!(s.get(at), Some(b'e' | b'E')) { - let sign = usize::from(matches!(s.get(at + 1), Some(b'+' | b'-'))); - let exp = digits(at + 1 + sign); - if exp > 0 { - at += 1 + sign + exp; - } - } - at -} - -#[cfg(test)] -thread_local! { - /// Bytes searched for `endstream` on this thread. - pub(crate) static ENDSTREAM_SCAN: std::cell::Cell = const { std::cell::Cell::new(0) }; - /// Bytes handed to number conversion on this thread. - static NUMBER_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -fn parse_counted(s: &str) -> Result { - #[cfg(test)] - NUMBER_BYTES.with(|c| c.set(c.get() + s.len())); - s.parse() -} - -fn hexval(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - -/// Parse an indirect object at `offset`: `N G obj [stream…endstream] endobj`. -/// `resolve_length` resolves an indirect `/Length`. Returns the object and its (num, gen). -pub fn parse_indirect(data: &[u8], offset: usize, resolve_length: &dyn Fn(ObjRef) -> Option) -> Result<(ObjRef, Object), CosError> { - parse_indirect_in(data, None, offset, resolve_length, None) -} - -/// [`parse_indirect`] for a document's own bytes: stream data points into `data` instead of -/// being copied out of it. -/// `endstreams` is the document's [`EndstreamIndex`] for `data`. -pub(crate) fn parse_indirect_shared( - data: &Arc>, - offset: usize, - resolve_length: &dyn Fn(ObjRef) -> Option, - endstreams: &EndstreamIndex, -) -> Result<(ObjRef, Object), CosError> { - parse_indirect_in(data, Some(data), offset, resolve_length, Some(endstreams)) -} - -/// Where `endstream` occurs in a document's bytes, found once, the first time a stream's -/// /Length is wrong or missing. Searching the rest of the file for each such stream instead -/// makes many of them ending at one late `endstream` quadratic to read. -pub(crate) type EndstreamIndex = std::sync::OnceLock>; - -fn endstream_offsets(data: &[u8]) -> Vec { - #[cfg(test)] - ENDSTREAM_SCAN.with(|c| c.set(c.get() + data.len())); - data.windows(9).enumerate().filter(|(i, w)| *w == b"endstream" && endstream_at(data, *i)).map(|(i, _)| i).collect() -} - -fn parse_indirect_in( - data: &[u8], - shared: Option<&Arc>>, - offset: usize, - resolve_length: &dyn Fn(ObjRef) -> Option, - endstreams: Option<&EndstreamIndex>, -) -> Result<(ObjRef, Object), CosError> { - let mut lx = Lexer::new(data, offset); - lx.skip_ws(); - let num_tok = lx.token(); - lx.skip_ws(); - let gen_tok = lx.token(); - let parse_u = |t: &[u8]| std::str::from_utf8(t).ok().and_then(|s| s.parse::().ok()); - let (Some(num), Some(generation)) = (parse_u(num_tok), parse_u(gen_tok)) else { - return Err(CosError::Syntax { offset, detail: "expected object header".into() }); - }; - if !lx.eat_keyword(b"obj") { - return Err(CosError::Syntax { offset: lx.pos, detail: "expected 'obj'".into() }); - } - let id = ObjRef::new(num.min(u32::MAX as u64) as u32, generation.min(u16::MAX as u64) as u16); - let obj = match lx.object() { - Ok(o) => o, - // "N G obj endobj" (empty) is treated as null. - Err(_) if lx.eat_keyword(b"endobj") => return Ok((id, Object::Null)), - Err(e) => return Err(e), - }; - lx.skip_ws(); - if let Object::Dict(dict) = &obj - && lx.eat_keyword(b"stream") - { - // The keyword is followed by CRLF or LF (a lone CR is tolerated). Microsoft Reporting - // Services writes a space first (`stream \r\n`); spaces and tabs count only when a line - // break follows, so they are never taken from the data. - let mut eol = lx.pos; - while matches!(data.get(eol), Some(b' ' | b'\t')) { - eol += 1; - } - if matches!(data.get(eol), Some(b'\r' | b'\n')) { - lx.pos = eol; - } - match (data.get(lx.pos), data.get(lx.pos + 1)) { - (Some(b'\r'), Some(b'\n')) => lx.pos += 2, - (Some(b'\n'), _) | (Some(b'\r'), _) => lx.pos += 1, - _ => {} - } - let start = lx.pos; - let declared = match dict.get(b"Length") { - Some(Object::Int(n)) => Some(*n), - Some(Object::Ref(r)) => resolve_length(*r), - _ => None, - }; - // A /Length that fits in the file (not negative, and no wrap on a 32-bit target). - let fits = declared.and_then(|l| usize::try_from(l).ok()).filter(|l| start.checked_add(*l).is_some_and(|end| end <= data.len())); - // What follows the /Length bytes, white space aside. - let after = |len: usize| { - let mut after = Lexer::new(data, start.saturating_add(len)); - after.skip_ws(); - data.get(after.pos..).unwrap_or_default() - }; - let len = match fits { - // The /Length ends at `endstream`, or where it would be: misspelt (`endstrea`), or - // left out before `endobj` or the end of the file. The stream is the bytes its /Length - // gives, "how many bytes of the PDF file are used for the stream's data" (ISO 32000-2 - // §7.3.8.2), as other readers take it. A producer that wrote `endstrea` after every - // stream lost every font and every page's content, and searching on would run into a - // later object's `endstream`. - Some(l) if at_stream_end(after(l)) => l, - // Wrong or missing /Length: search for "endstream" (§7.3.8.1 recovery). With none - // left in the file, a /Length that fits is still the stream's extent. - _ => match find_endstream(data, start, endstreams) { - Some(end) => end - start, - None => fits.ok_or_else(|| CosError::Syntax { offset: start, detail: "stream without endstream".into() })?, - }, - }; - let range = start..start + len; - let raw = match shared { - Some(buf) => Bytes::view(buf, range), - None => data.get(range).map(|r| Bytes::from(r.to_vec())), - } - .ok_or_else(|| CosError::Syntax { offset: start, detail: "stream data past the end of the file".into() })?; - let mut dict = dict.clone(); - dict.set(b"Length".to_vec(), Object::Int(raw.len() as i64)); - return Ok((id, Object::Stream(Stream { dict, raw }))); - } - Ok((id, obj)) -} - -/// Whether `rest`, what follows a stream's /Length bytes with white space skipped, is where the -/// stream ends. A whole `endstream` token is: a token runs on through regular characters (ISO -/// 32000-2 §7.2.3), so payload such as `endstreamer` is not one. A misspelt `endstream` -/// (`endstrea`), an `endobj` with the `endstream` left out, or both, end it only where the object -/// ends as well, before the end of the data, another object's `N G obj` header, or `xref`, -/// `trailer` or `startxref`: payload such as `endobj more` or `endobject`, where a wrong /Length -/// stops, is not, and the search for `endstream` runs instead. So does anything else but the end -/// of the data. -fn at_stream_end(rest: &[u8]) -> bool { - let mut lx = Lexer::new(rest, 0); - if lx.eat_keyword(b"endstream") { - return true; - } - let misspelt = lx.eat_keyword(b"endstrea"); - let endobj = lx.eat_keyword(b"endobj"); - lx.skip_ws(); - if lx.pos >= rest.len() { - return true; - } - if !(misspelt || endobj) { - return false; - } - if [&b"xref"[..], b"trailer", b"startxref"].iter().any(|kw| Lexer::new(rest, lx.pos).eat_keyword(kw)) { - return true; - } - let number = |lx: &mut Lexer| { - lx.skip_ws(); - let t = lx.token(); - !t.is_empty() && t.iter().all(u8::is_ascii_digit) - }; - number(&mut lx) && number(&mut lx) && lx.eat_keyword(b"obj") -} - -/// Whether a whole `endstream` token starts at `at` in `data`: one followed by a regular -/// character (`endstreamer`) is payload (ISO 32000-2 §7.2.3). -fn endstream_at(data: &[u8], at: usize) -> bool { - data.get(at..).is_some_and(|d| d.starts_with(b"endstream")) && !data.get(at.saturating_add(9)).copied().is_some_and(is_regular) -} - -/// Offset where stream data ends (before the EOL preceding "endstream"). -fn find_endstream(data: &[u8], from: usize, endstreams: Option<&EndstreamIndex>) -> Option { - let at = match endstreams { - Some(index) => { - let index = index.get_or_init(|| endstream_offsets(data)); - index.get(index.partition_point(|&p| p < from)).copied()? - } - None => { - let hay = data.get(from..)?; - let found = hay.windows(9).enumerate().position(|(i, w)| w == b"endstream" && endstream_at(hay, i)); - #[cfg(test)] - ENDSTREAM_SCAN.with(|c| c.set(c.get() + found.map_or(hay.len(), |p| p + 9))); - found? + from - } - }; - let mut end = at; - if end > from && data[end - 1] == b'\n' { - end -= 1; - } - if end > from && data[end - 1] == b'\r' { - end -= 1; - } - Some(end) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn numbers_with_multibyte_garbage_do_not_panic() { - // From `cargo xtask fuzz`: "2 0 obj\n.\u{2014}" (an em dash after a dot). - let doc = crate::Document::open(std::sync::Arc::new("2 0 obj\n.\u{2014}\n".as_bytes().to_vec())); - assert!(doc.is_err()); - for t in [".\u{2014}", "1.\u{e9}5", "-.\u{1F600}", "+.5\u{2014}"] { - let _ = Lexer::new(t.as_bytes(), 0).object(); - } - assert_eq!(Lexer::new("1.5\u{2014}".as_bytes(), 0).object().unwrap(), Object::Real(1.5)); - } - - #[test] - fn a_damaged_real_is_converted_once() { - let token = format!("1.{}{}", "5".repeat(2_000), "x".repeat(20_000)); - NUMBER_BYTES.with(|c| c.set(0)); - let Ok(Object::Real(v)) = Lexer::new(token.as_bytes(), 0).object() else { panic!("not a real") }; - let converted = NUMBER_BYTES.with(|c| c.get()); - assert!(converted <= token.len(), "{converted} bytes converted for a {}-byte token", token.len()); - assert!((v - 1.5555).abs() < 1e-3); - // The longest numeric prefix wins, as before. - for (t, want) in - [("1.2.3", 1.2), ("--5.", -5.0), ("+.5x", 0.5), ("5.-", 5.0), ("1.5e3", 1500.0), ("1.5e+", 1.5), ("-.x", 0.0), (".", 0.0), ("-inf.", 0.0)] - { - assert_eq!(Lexer::new(t.as_bytes(), 0).object().ok(), Some(Object::Real(want)), "{t}"); - } - } - - fn obj(s: &str) -> Object { - Lexer::new(s.as_bytes(), 0).object().unwrap_or_else(|e| panic!("{s}: {e}")) - } - - #[test] - fn doubled_sign_integer_reads_like_the_real_form() { - // Some writers emit a TJ kerning as "--280"; the real form "--280.0" is already read. - assert_eq!(obj("--280.0"), Object::Real(-280.0)); - assert_eq!(obj("--280"), Object::Int(-280)); - match obj("[(Hello) --280 (World)]") { - Object::Array(items) => { - assert_eq!(items.len(), 3); - assert_eq!(items[1], Object::Int(-280)); - } - other => panic!("not an array: {other:?}"), - } - } - - #[test] - fn scalars() { - assert_eq!(obj("true"), Object::Bool(true)); - assert_eq!(obj("null"), Object::Null); - assert_eq!(obj(" -17 "), Object::Int(-17)); - assert_eq!(obj("+42"), Object::Int(42)); - assert_eq!(obj("-.5"), Object::Real(-0.5)); - assert_eq!(obj("4."), Object::Real(4.0)); - assert_eq!(obj("99999999999999999999"), Object::Real(1e20)); - assert_eq!(obj("12 0 R"), Object::Ref(ObjRef::new(12, 0))); - assert_eq!(obj("12 0 obj").as_int(), Some(12), "12 followed by 0 obj is not a reference"); - } - - #[test] - fn names_decode_hex_escapes() { - assert_eq!(obj("/A#20B"), Object::Name(b"A B".to_vec())); - assert_eq!(obj("/Adobe#"), Object::Name(b"Adobe#".to_vec())); - assert_eq!(obj("/"), Object::Name(Vec::new())); - } - - #[test] - fn literal_strings() { - let s = |x: &str| obj(x).as_string().expect("string").bytes.clone(); - assert_eq!(s("(a (nested) b)"), b"a (nested) b"); - assert_eq!(s(r"(\n\t\\\(\))"), b"\n\t\\()"); - assert_eq!(s(r"(\101\1011)"), b"AA1"); - assert_eq!(s("(line\\\ncontinued)"), b"linecontinued"); - assert_eq!(s("(cr\r\nlf)"), b"cr\nlf"); - } - - #[test] - fn hex_strings() { - let s = obj("<48 65 6C6c6f7>"); - assert_eq!(s.as_string().unwrap().bytes, b"Hellop"); - assert!(s.as_string().unwrap().hex); - } - - #[test] - fn arrays_and_dicts() { - let o = obj("<< /Type /Page /Kids [1 0 R 2 0 R] /Box [0 0 612.5 792] /Nested << /A true >> /Empty >>"); - let d = o.as_dict().unwrap(); - assert_eq!(d.name(b"Type"), Some(&b"Page"[..])); - assert_eq!(d.get(b"Kids").unwrap().as_array().unwrap().len(), 2); - assert_eq!(d.get(b"Box").unwrap().as_array().unwrap()[2], Object::Real(612.5)); - assert_eq!(d.get(b"Empty"), Some(&Object::Null)); - } - - #[test] - fn comments_are_whitespace() { - assert_eq!(obj("% hello\n [1 % two\n 3]"), Object::Array(vec![Object::Int(1), Object::Int(3)])); - } - - #[test] - fn indirect_stream_with_wrong_length_recovers() { - let data = b"7 0 obj\n<< /Length 999 >>\nstream\nHELLO\nendstream\nendobj\n"; - let (id, o) = parse_indirect(data, 0, &|_| None).unwrap(); - assert_eq!(id, ObjRef::new(7, 0)); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO"); - assert_eq!(s.dict.int(b"Length"), Some(5)); - } - - /// With no `endstream` left in the file, as a producer that writes `endstrea` leaves every - /// stream, a stream is the bytes its /Length gives, "how many bytes of the PDF file are used - /// for the stream's data" (ISO 32000-2 §7.3.8.2): it was an error, and the object null, so - /// such a file lost every font and page content. A /Length past the file, or none, is still - /// no stream. - #[test] - fn indirect_stream_without_endstream_is_its_length() { - for end in [&b"\r\nendstrea\rendobj\r\n"[..], b"\nendobj\n", b""] { - let mut data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO".to_vec(); - data.extend_from_slice(end); - let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO", "{end:?}"); - } - assert!(parse_indirect(b"7 0 obj\n<< /Length 6 >>\nstream\nHELLO", 0, &|_| None).is_err(), "past the file"); - assert!(parse_indirect(b"7 0 obj\n<< >>\nstream\nHELLO\nendstrea\nendobj\n", 0, &|_| None).is_err(), "no /Length"); - } - - /// A stream ending in `endstrea` is its /Length even when a later object does have an - /// `endstream`: the search for one would otherwise run on into that object and take its - /// header and data as this stream's. - #[test] - fn a_stream_without_endstream_is_not_read_into_a_later_one() { - let data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO\r\nendstrea\rendobj\r\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n"; - let second = data.windows(7).position(|w| w == b"8 0 obj").unwrap(); - for (offset, want) in [(0, &b"HELLO"[..]), (second, b"WORLD")] { - let (_, o) = parse_indirect(data, offset, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected at {offset}") }; - assert_eq!(&s.raw[..], want, "at {offset}"); - } - // Through the loader's shared index of `endstream` offsets as well. - let index = EndstreamIndex::new(); - let (_, o) = parse_indirect_in(data, None, 0, &|_| None, Some(&index)).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO"); - } - - /// The same for a stream whose `endstream` is left out before `endobj`: the later object's - /// `endstream` is not this stream's end. - #[test] - fn a_stream_ending_at_endobj_is_not_read_into_a_later_one() { - let data = b"7 0 obj\n<< /Length 5 >>\nstream\nHELLO\nendobj\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n"; - let (_, o) = parse_indirect(data, 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO"); - } - - /// A wrong /Length that stops before payload bytes beginning like an end marker (`endobject`, - /// `endstreak`, `endstreamer`) is not taken for the stream's end: a token runs on through - /// regular characters (ISO 32000-2 §7.2.3), so the search for `endstream` finds the whole - /// payload, and skips `endstreamer` too, by itself and through the loader's shared index. - #[test] - fn a_length_ending_inside_a_longer_word_is_not_an_end_marker() { - for word in ["endobject", "endstreak", "endstreamer"] { - let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {word} more\nendstream\nendobj\n"); - let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}"); - let index = EndstreamIndex::new(); - let (_, o) = parse_indirect_in(pdf.as_bytes(), None, 0, &|_| None, Some(&index)).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}, indexed"); - } - // Whole markers still end it, followed by white space, a delimiter or the end of the data. - for tail in ["endobj\n", "endstrea%", "endobj"] { - let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{tail}"); - let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO", "{tail:?}"); - } - } - - /// A whole `endobj` or `endstrea` token where a wrong /Length stops is payload, not the - /// stream's end, unless the object ends there too: what follows is the end of the data, - /// another object's header, or `xref`, `trailer` or `startxref`. Otherwise the search for - /// `endstream` finds the whole payload. - #[test] - fn an_end_marker_inside_the_payload_does_not_end_the_stream() { - for payload in ["endobj more", "endstrea more", "endstrea endobj more", "endobj 12 more"] { - let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {payload}\nendstream\nendobj\n"); - let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(s.raw[..], *format!("HELLO {payload}").as_bytes(), "{payload}"); - } - // Where the object does end, the /Length is the stream, also with an `endstream` later. - for next in ["8 0 obj\n<< >>", "%c\n12 0 obj", "xref", "trailer", "startxref"] { - for marker in ["endobj", "endstrea\nendobj", "endstrea"] { - let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{marker}\n{next}\nstream\nWORLD\nendstream\nendobj\n"); - let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO", "{marker:?} then {next:?}"); - } - } - } - /// Without an `endstream`, the /Length is trusted as written when it fits, also when it is - /// wrong, and an indirect one is resolved first; one that doesn't fit, a negative one and one - /// too large for any file are errors rather than wrapped into a short read. - #[test] - fn a_stream_without_endstream_takes_its_length_only_when_it_fits() { - let stream = |length: &str, tail: &str| format!("7 0 obj\n<< /Length {length} >>\nstream\nHELLO{tail}"); - let raw = |pdf: &str, lookup: &dyn Fn(ObjRef) -> Option| match parse_indirect(pdf.as_bytes(), 0, lookup) { - Ok((_, Object::Stream(s))) => Ok(s.raw.to_vec()), - Ok((_, other)) => panic!("{pdf:?} is {other:?}"), - Err(e) => Err(e), - }; - let none = |_: ObjRef| None; - // Wrong but fitting: the bytes it gives, not the bytes up to `endstrea`. - assert_eq!(raw(&stream("3", "\nendstrea\nendobj\n"), &none).unwrap(), b"HEL"); - // Exactly to the end of the file, and nothing at all. - assert_eq!(raw(&stream("5", ""), &none).unwrap(), b"HELLO"); - assert_eq!(raw(&stream("0", "\nendstrea\nendobj\n"), &none).unwrap(), b""); - // Indirect, resolved through the lookup. - let lookup = |r: ObjRef| (r == ObjRef::new(9, 0)).then_some(4); - assert_eq!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &lookup).unwrap(), b"HELL"); - assert!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &none).is_err(), "an unresolved /Length"); - // One past the end, negative, the largest integer and one beyond it. - for length in ["6", "-1", "-9223372036854775808", "9223372036854775807", "18446744073709551621"] { - assert!(raw(&stream(length, ""), &none).is_err(), "/Length {length}"); - } - } - - #[test] - fn indirect_stream_with_a_space_before_the_line_break() { - // `stream \r\n` as Microsoft Reporting Services writes it: the data starts after the - // line break, so the /Length is still right and the data keeps no leading space. - for eol in [&b" \r\n"[..], b" \n", b"\t \r"] { - let mut data = b"7 0 obj\n<< /Length 5 >>\nstream".to_vec(); - data.extend_from_slice(eol); - data.extend_from_slice(b"HELLO\r\nendstream\nendobj\n"); - let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b"HELLO", "{eol:?}"); - } - // Spaces with no line break after them are data, not a keyword terminator. - let data = b"7 0 obj\n<< /Length 7 >>\nstream HELLO\nendstream\nendobj\n"; - let (_, o) = parse_indirect(data, 0, &|_| None).unwrap(); - let Object::Stream(s) = o else { panic!("stream expected") }; - assert_eq!(&s.raw[..], b" HELLO"); - } - - #[test] - fn indirect_length_is_resolved() { - let data = b"7 0 obj\n<< /Length 8 0 R >>\nstream\r\nAB\r\nendstream\nendobj\n"; - let (_, o) = parse_indirect(data, 0, &|r| (r.num == 8).then_some(2)).unwrap(); - let Object::Stream(s) = o else { panic!() }; - assert_eq!(&s.raw[..], b"AB"); - } - - #[test] - fn garbage_never_panics() { - for input in ["<<", "[", "(", "<", "<< /A", "[1 2", ")", "]", ">>", "<<>>>", "1 0", "\0\0", "/A#", "", "((("] { - let _ = Lexer::new(input.as_bytes(), 0).object(); - } - let deep = "[".repeat(10_000); - assert!(Lexer::new(deep.as_bytes(), 0).object().is_err()); - } - - #[test] - fn one_object_has_a_node_budget() { - // A wide array (from a tiny compressed object stream) must not grow without bound. - let wide = |n: usize| format!("[{}]", "0 ".repeat(n)); - let at_cap = Lexer::new(wide(MAX_NODES - 1).as_bytes(), 0).object(); - assert!(matches!(at_cap, Ok(Object::Array(ref items)) if items.len() == MAX_NODES - 1)); - assert!(Lexer::new(wide(MAX_NODES).as_bytes(), 0).object().is_err()); - // The same inside a dictionary value and behind tolerated junk. - assert!(Lexer::new(format!("<< /A {} >>", wide(MAX_NODES)).as_bytes(), 0).object().is_err()); - assert!(Lexer::new(format!("[[{}] ]", "0 ".repeat(MAX_NODES)).as_bytes(), 0).object().is_err()); - // The budget is per top-level object: a lexer reused for many operands keeps working. - let two = wide(MAX_NODES - 1).repeat(2); - let mut lx = Lexer::new(two.as_bytes(), 0); - assert!(lx.object().is_ok() && lx.object().is_ok()); - } - - /// A plain real is read without a cleaned copy of its token; the forms that need one read - /// as they did (#5873). - #[test] - fn reals_read_the_same_with_and_without_a_doubled_sign() { - for (token, want) in [("1.5", 1.5), ("-375.993", -375.993), ("+2.25", 2.25), ("--5.5", -5.5), ("1.2.3", 1.2), ("-.5", -0.5), ("0.0", 0.0)] { - assert_eq!(Lexer::new(token.as_bytes(), 0).object().unwrap(), Object::Real(want), "{token}"); - } - } -} diff --git a/crates/cos/src/security.rs b/crates/cos/src/security.rs deleted file mode 100644 index 28813b0ac9..0000000000 --- a/crates/cos/src/security.rs +++ /dev/null @@ -1,376 +0,0 @@ -//! Bridging the COS object model and `pdfcraft-crypt`: reading `/Encrypt` dictionaries and -//! decrypting / encrypting the strings and streams of an object (§7.6.2). -//! -//! Not encrypted, per the spec: the `/Encrypt` dictionary itself, cross-reference streams, the -//! trailer `/ID`, objects inside object streams (the object stream is encrypted as a whole), -//! the `/Contents` of signature dictionaries, and metadata streams when `/EncryptMetadata` is -//! false. Streams naming a crypt filter (`/Crypt` in `/Filter`) use that filter; embedded files -//! use `/EFF`. - -use std::cell::Cell; - -use pdfcraft_crypt::{EncryptDict, Method, SecurityHandler, StreamKind}; -use pdfcraft_filters::MAX_CHAIN; - -use crate::document::Held; -use crate::{Dict, Document, Object, PdfString, Stream}; - -/// How many references one read or write of an object may follow, with all that it loads on the -/// way. How an object is treated can be given by reference, and what that loads can do the same: a -/// file could make one read a tree of them, each object read again once the cache has let it go. -/// An object takes a few of these, and two dozen if every part of it is given by reference; past -/// this they read as absent. The parameters of an object stream, read together, count as one. -const MAX_FOLLOWED: u32 = 32; - -std::thread_local! { - /// How many [`FollowScope`]s this thread is inside. - static SCOPES: Cell = const { Cell::new(0) }; - /// The references followed in the outermost of them. - static FOLLOWED: Cell = const { Cell::new(0) }; - /// The references refused in the outermost of them, past [`MAX_FOLLOWED`]. - static REFUSED: Cell = const { Cell::new(0) }; -} - -/// One read or write of an object, with everything it loads on the way: what [`MAX_FOLLOWED`] counts. -pub(crate) struct FollowScope; - -impl FollowScope { - pub(crate) fn enter() -> Self { - if SCOPES.replace(SCOPES.get().saturating_add(1)) == 0 { - FOLLOWED.set(0); - REFUSED.set(0); - } - FollowScope - } -} - -impl Drop for FollowScope { - fn drop(&mut self) { - SCOPES.set(SCOPES.get().saturating_sub(1)); - } -} - -/// What `f` gives, counted against [`MAX_FOLLOWED`], or `spent` past it. Called inside a -/// [`FollowScope`], which is what starts the count again. -pub(crate) fn following(spent: T, f: impl FnOnce() -> T) -> T { - if FOLLOWED.get() >= MAX_FOLLOWED { - REFUSED.set(REFUSED.get().saturating_add(1)); - return spent; - } - FOLLOWED.set(FOLLOWED.get() + 1); - f() -} - -/// `o`, or the object it refers to, for what decides how a stream or dictionary is treated. Past -/// [`MAX_FOLLOWED`] a reference reads as absent. -fn followed<'o>(doc: &Document, o: &'o Object) -> Held<'o> { - match o { - Object::Ref(_) => following(Held::Direct(&Object::Null), || doc.held(o)), - o => Held::Direct(o), - } -} - -fn bytes_of(doc: &Document, d: &Dict, key: &[u8]) -> Vec { - d.get(key).map(|v| doc.resolve(v)).and_then(|v| v.as_string().map(|s| s.bytes.clone())).unwrap_or_default() -} - -/// Read the handler parameters from an `/Encrypt` dictionary. -pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict { - let int = |k: &[u8], default: i64| d.get(k).map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(default); - let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec)); - let mut crypt_filters = Vec::new(); - if let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) { - for (k, v) in cf.iter() { - let Some(f) = doc.resolve(v).as_dict().cloned() else { continue }; - let method = match name(&f, b"CFM").as_deref() { - Some(b"V2") => Method::Rc4, - Some(b"AESV2") => Method::Aes128, - Some(b"AESV3") => Method::Aes256, - _ => Method::Identity, - }; - crypt_filters.push((k.clone(), method)); - } - } - let v = int(b"V", 0); - // V4: the key length comes from the standard crypt filter's /Length (bytes, or bits in - // some files); /Length on the dictionary is often a stale 40. A length that isn't - // positive means nothing and reads as if absent. - let cf_len = d - .get(b"CF") - .map(|c| doc.resolve(c)) - .and_then(|c| c.as_dict().and_then(|c| c.get(b"StdCF").map(|f| doc.resolve(f)))) - .and_then(|f| f.as_dict().and_then(|f| f.get(b"Length").map(|n| doc.resolve(n)))) - .and_then(|n| n.as_int()) - .filter(|n| *n > 0); - let length_bits = match (v, cf_len) { - (5.., _) => 256, - (4, Some(n)) if n <= 32 => n * 8, - (4, Some(n)) => n, - (4, None) => 128, - _ => int(b"Length", 40), - }; - EncryptDict { - filter: name(d, b"Filter").unwrap_or_default(), - v, - r: int(b"R", 0), - length_bits, - o: bytes_of(doc, d, b"O"), - u: bytes_of(doc, d, b"U"), - oe: bytes_of(doc, d, b"OE"), - ue: bytes_of(doc, d, b"UE"), - perms: bytes_of(doc, d, b"Perms"), - p: int(b"P", -1) as i32, - encrypt_metadata: !matches!(d.get(b"EncryptMetadata").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(false))), - crypt_filters, - stm_f: name(d, b"StmF").unwrap_or_else(|| b"Identity".to_vec()), - str_f: name(d, b"StrF").unwrap_or_else(|| b"Identity".to_vec()), - ef_f: name(d, b"EFF").unwrap_or_default(), - } -} - -/// Whether `d`, an `/Encrypt` dictionary, has crypt filters that encrypt (`/CFM` `V2`, `AESV2` or -/// `AESV3`), each authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO -/// 32000-2 §7.6.6, Table 25): what a document asks for its embedded files alone, which it opens -/// without (§7.6.6: "EFOpen Authorization shall be required when accessing embedded files"). The -/// files use them through `/EFF` or through a `/Crypt` filter of their own. Its strings and streams -/// have to be in the clear too, which [`SecurityHandler::without_key`] checks. -pub(crate) fn efopen_only(doc: &Document, d: &Dict) -> bool { - let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec)); - let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) else { return false }; - let mut encrypting = Vec::new(); - for (k, v) in cf.iter() { - let Some(f) = doc.resolve(v).as_dict().cloned() else { continue }; - if !matches!(name(&f, b"CFM").as_deref(), Some(b"V2" | b"AESV2" | b"AESV3")) { - continue; - } - if name(&f, b"AuthEvent").as_deref() != Some(b"EFOpen") { - return false; - } - encrypting.push(k.clone()); - } - !encrypting.is_empty() -} - -/// Whether `s`, read from a document whose handler has no file key, holds data that handler would -/// decrypt, which therefore reads as stored: still encrypted. -pub(crate) fn stream_reads_as_stored(doc: &Document, h: &SecurityHandler, s: &Stream) -> bool { - if h.has_key() { - return false; - } - let _scope = FollowScope::enter(); - let named = named_crypt_filter(doc, &s.dict); - let ty = s.dict.get(b"Type").map(|t| followed(doc, t)); - let kind = match (&named, ty.as_deref().and_then(Object::as_name)) { - (Some(n), _) => StreamKind::Named(n), - (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, - _ => StreamKind::Normal, - }; - h.decrypts(kind) -} - -/// Whether `f`, an entry of a `/Filter` chain or its only filter, is `/Crypt`. -fn is_crypt(doc: &Document, f: &Object) -> bool { - followed(doc, f).as_name() == Some(b"Crypt") -} - -/// The crypt filter a stream names in its `/Filter` chain (`/Crypt` + `/DecodeParms /Name`), each -/// given directly or by reference, in an array or not. A chain longer than any decoder takes names -/// none: nothing can read that stream anyway, and every stream of a file can share one long array. -/// A `/Name` that reads as absent means `/Identity`, unless a reference on the way to it was refused -/// (see [`MAX_FOLLOWED`]): that says nothing about the stream, so it names none, and the document's -/// own filter is used. -fn named_crypt_filter(doc: &Document, dict: &Dict) -> Option> { - let refused = REFUSED.get(); - let filter = followed(doc, dict.get(b"Filter")?); - let i = match &*filter { - Object::Array(a) if a.len() <= MAX_CHAIN + 1 => a.iter().position(|f| is_crypt(doc, f))?, - f if is_crypt(doc, f) => 0, - _ => return None, - }; - let parms = dict.get(b"DecodeParms").map(|p| followed(doc, p)); - let entry = match parms.as_deref() { - Some(Object::Array(a)) => a.get(i).map(|p| followed(doc, p)), - Some(p) if i == 0 => Some(Held::Direct(p)), - _ => None, - }; - let name = entry.as_deref().and_then(Object::as_dict).and_then(|p| p.get(b"Name")).map(|n| followed(doc, n)); - match name.as_deref().and_then(Object::as_name) { - Some(name) => Some(name.to_vec()), - // /Identity here would read ciphertext as data, and write the stream in the clear. - None if REFUSED.get() > refused => None, - None => Some(b"Identity".to_vec()), - } -} - -/// A signature dictionary (field signature, document timestamp or usage-rights signature), -/// whose `/Contents` is never encrypted (ISO 32000-2 §7.6.2, §12.8.1): the signer writes a -/// placeholder, hashes the file around it and patches the value in place, so encrypting it -/// would make signing impossible. Identified as other readers do: by `/Type`, or by a string -/// `/Contents` alongside an array `/ByteRange` (`/Type` is optional). A signature dictionary -/// has no `/Subtype`; a dictionary with one (an annotation, whose `/Contents` is the comment -/// text) or with another `/Type` is never exempt, whatever else it carries. -/// With `doc` its entries are read through references; a dictionary nested in another object -/// has none (see [`transform`]). -fn is_signature_dict(doc: Option<&Document>, d: &Dict) -> bool { - // All this decides is what happens to such a /Contents. - if !matches!(d.get(b"Contents"), Some(Object::String(_))) { - return false; - } - // A null entry, or one that refers to nothing, is an absent one (ISO 32000-2 §7.3.7, §7.3.10). - let entry = |key: &[u8]| { - let value = d.get(key)?; - let held = doc.map_or(Held::Direct(value), |doc| followed(doc, value)); - (!matches!(*held, Object::Null)).then_some(held) - }; - if entry(b"Subtype").is_some() { - return false; - } - match entry(b"Type") { - Some(t) => matches!(t.as_name(), Some(b"Sig" | b"DocTimeStamp")), - None => matches!(entry(b"ByteRange").as_deref(), Some(Object::Array(_))), - } -} - -/// Longer than any filter's name. A name can be of any length, and a chain every stream shares can -/// hold a long one. -const MAX_FILTER_NAME: usize = 127; - -/// The dictionary of `source` without the `/Crypt` entries of its filter chain and their decode -/// parameters. The two come out direct when given by reference, which every stream of a file can -/// share, so of what was shared only small values are copied: filter names, and the parameters -/// decoders read ([`Document::with_direct_parms`]). The chain is short, as [`named_crypt_filter`] -/// found a filter in it. -fn drop_crypt_filter(doc: &Document, source: &Stream) -> Dict { - let entry = |f: &Object| match f { - f if is_crypt(doc, f) => Object::name("Crypt"), - Object::Name(n) if n.len() <= MAX_FILTER_NAME => f.clone(), - Object::Ref(_) => f.clone(), - _ => Object::Null, - }; - let filter = source.dict.get(b"Filter").map(|f| followed(doc, f)); - let chain: Vec = match filter.as_deref() { - Some(Object::Array(a)) => a.iter().map(entry).collect(), - Some(f) => vec![entry(f)], - None => Vec::new(), - }; - let crypt: Vec = chain.iter().map(|f| f.as_name() == Some(b"Crypt")).collect(); - let mut chained = Stream::from_raw(source.dict.clone(), Vec::new()); - chained.dict.set(b"Filter".to_vec(), Object::Array(chain.clone())); - // A lone parameter dictionary belongs to the first filter (as `named_crypt_filter` reads it). - let parms = match doc.with_direct_parms(&chained).dict.get(b"DecodeParms") { - Some(Object::Array(p)) => Some(Object::Array(p.iter().zip(&crypt).filter(|(_, c)| !**c).map(|(p, _)| p.clone()).collect())), - Some(Object::Dict(_)) if crypt.first() == Some(&true) => None, - _ => source.dict.get(b"DecodeParms").cloned(), - }; - let mut dict = source.dict.clone(); - match parms { - Some(p) => dict.set(b"DecodeParms".to_vec(), p), - None => { - dict.remove(b"DecodeParms"); - } - } - let filters: Vec = chain.into_iter().zip(&crypt).filter(|(_, c)| !**c).map(|(f, _)| f).collect(); - if filters.is_empty() { - dict.remove(b"Filter"); - if matches!(dict.get(b"DecodeParms"), Some(Object::Array(p)) if p.is_empty()) { - dict.remove(b"DecodeParms"); - } - } else { - dict.set(b"Filter".to_vec(), Object::Array(filters)); - } - dict -} - -/// What [`transform`] does to an object. -#[derive(Clone, Copy, PartialEq, Eq)] -pub(crate) enum Pass { - /// Decrypt, with the handler the document was opened with. - Decrypt, - /// Encrypt with the handler the document was opened with: streams keep their crypt filters. - Encrypt, - /// Encrypt with protection applied by `set_encryption`. A stream's crypt filter came from - /// the source's `/Encrypt`; unless the new one defines that name it is dropped, so the - /// stream follows the new policy instead of reading as `/Identity` and staying in clear. - EncryptReplacing, -} - -/// Decrypt or encrypt every string and stream in an indirect object, except a signature -/// dictionary's `/Contents` (see [`is_signature_dict`]). -/// -/// `o` is as `doc` holds it: what a stream's dictionary gives by reference is read from `doc`, so -/// its references have to be the document's, not a save's renumbered ones. -pub(crate) fn transform(doc: &Document, h: &SecurityHandler, o: &Object, num: u32, generation: u16, pass: Pass) -> Object { - let _scope = FollowScope::enter(); - let decrypt = pass == Pass::Decrypt; - let strings = |s: &PdfString| { - let bytes = if decrypt { h.decrypt_string(num, generation, &s.bytes) } else { h.encrypt_string(num, generation, &s.bytes) }; - PdfString { bytes, hex: s.hex || !decrypt } - }; - // References are followed here only, for the object itself, never for what is nested in it. - // `walk` goes deep, and a load from down there adds that depth again for each object loaded, - // which overflowed the stack on a 10 KB file. A stream can't be nested, and a dictionary that - // is reads as it says directly. - match o { - Object::Stream(s) => stream(doc, s, &strings, h, num, generation, pass), - Object::Dict(d) if is_signature_dict(Some(doc), d) => signature_dict(d, &strings), - o => walk(o, &strings), - } -} - -/// `d`, a signature dictionary, with the strings of everything but its `/Contents` transformed. -fn signature_dict(d: &Dict, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { - Object::Dict( - d.iter() - .map(|(k, v)| match (k.as_slice(), v) { - (b"Contents", Object::String(_)) => (k.clone(), v.clone()), - _ => (k.clone(), walk(v, strings)), - }) - .collect(), - ) -} - -fn walk(o: &Object, strings: &dyn Fn(&PdfString) -> PdfString) -> Object { - match o { - Object::String(s) => Object::String(strings(s)), - Object::Array(a) => Object::Array(a.iter().map(|x| walk(x, strings)).collect()), - Object::Dict(d) if is_signature_dict(None, d) => signature_dict(d, strings), - Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect()), - other => other.clone(), - } -} - -fn stream( - doc: &Document, - s: &Stream, - strings: &dyn Fn(&PdfString) -> PdfString, - h: &SecurityHandler, - num: u32, - generation: u16, - pass: Pass, -) -> Object { - let decrypt = pass == Pass::Decrypt; - let walked = |d: &Dict| -> Dict { d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect() }; - let ty = s.dict.get(b"Type").map(|t| followed(doc, t)); - let ty = ty.as_deref().and_then(Object::as_name); - // Cross-reference streams are never encrypted (§7.5.8), but only reading may trust the - // label: a stream a producer calls /XRef stays plaintext if writing believes it, and the - // writer builds its own cross-reference streams outside this transform. - if (decrypt && ty == Some(b"XRef")) || (ty == Some(b"Metadata") && !h.encrypts_metadata()) { - return Object::Stream(Stream { dict: walked(&s.dict), raw: s.raw.clone() }); - } - let mut named = named_crypt_filter(doc, &s.dict); - // Before the walk, which then encrypts what is in the dictionary this leaves. - let foreign = pass == Pass::EncryptReplacing && named.as_deref().is_some_and(|n| !h.dict().crypt_filters.iter().any(|(f, _)| f == n)); - let dropped = foreign.then(|| drop_crypt_filter(doc, s)); - if dropped.is_some() { - named = None; - } - let dict = walked(dropped.as_ref().unwrap_or(&s.dict)); - let kind = match (&named, ty) { - (Some(n), _) => StreamKind::Named(n), - (None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile, - _ => StreamKind::Normal, - }; - let raw = if decrypt { h.decrypt_stream(num, generation, &s.raw, kind) } else { h.encrypt_stream(num, generation, &s.raw, kind) }; - Object::Stream(Stream { dict, raw: raw.into() }) -} diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs deleted file mode 100644 index 56731f7e39..0000000000 --- a/crates/cos/src/writer.rs +++ /dev/null @@ -1,1211 +0,0 @@ -//! Serialization and saving (ISO 32000-2 §7.5). -//! -//! * `write_incremental` appends the edited objects, a cross-reference section in the same style -//! as the file's last one (table or stream) and a trailer with `/Prev`. The original bytes are -//! never modified, which keeps digital signatures over earlier revisions valid. With -//! `SaveOptions::drop_unlinked`, objects added since the file was read that nothing refers to -//! any more are left out (written as free). -//! * `write_full` writes only objects reachable from the trailer, renumbered densely, with a -//! cross-reference table or stream — the "Save As" / garbage-collecting path. Unknown trailer -//! entries are copied, including objects reachable only from them. - -use std::borrow::Cow; -use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; -use std::io::Write; -use std::sync::Arc; - -use crate::CosError; -use crate::document::{Document, ObjectReader, XrefEntry, held_bytes}; -use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; - -/// Serialize one object. Deterministic: same object → same bytes. -pub fn serialize(o: &Object, out: &mut Vec) { - match o { - Object::Null => out.extend_from_slice(b"null"), - Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), - Object::Int(i) => write_int(*i, out), - Object::Real(r) => write_real(*r, out), - Object::String(s) => write_string(s, out), - Object::Name(n) => write_name(n, out), - Object::Array(a) => { - out.push(b'['); - for (i, x) in a.iter().enumerate() { - if i > 0 { - out.push(b' '); - } - serialize(x, out); - } - out.push(b']'); - } - Object::Dict(d) => write_dict(d, out), - Object::Stream(s) => write_stream(s, out), - Object::Ref(r) => { - write_digits(u64::from(r.num), out); - out.push(b' '); - write_digits(u64::from(r.generation), out); - out.extend_from_slice(b" R"); - } - } -} - -/// Upper-case hexadecimal digits. -const HEX: &[u8; 16] = b"0123456789ABCDEF"; - -/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. -fn write_digits(mut v: u64, out: &mut Vec) { - // `u64::MAX` has 20 digits, so `at` never goes below 0. - let mut buf = [0u8; 20]; - let mut at = buf.len(); - loop { - at -= 1; - buf[at] = b'0' + (v % 10) as u8; - v /= 10; - if v == 0 { - break; - } - } - out.extend_from_slice(&buf[at..]); -} - -/// A signed integer in decimal, as `write!(out, "{i}")` writes it. -fn write_int(i: i64, out: &mut Vec) { - if i < 0 { - out.push(b'-'); - } - write_digits(i.unsigned_abs(), out); -} - -/// `b` as two upper-case hexadecimal digits. -fn hex_pair(b: u8) -> [u8; 2] { - [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] -} - -fn write_real(r: f64, out: &mut Vec) { - if !r.is_finite() { - out.push(b'0'); - return; - } - // A whole number is written as an integer only within the integer range Annex C gives - // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. - if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { - write_int(i64::from(r as i32), out); - return; - } - // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the - // shortest string that parses back to the same value, never in exponent form, so values - // round-trip exactly (rounding to a fixed number of places visibly shifted shading colours). - let s = format!("{r}"); - out.extend_from_slice(if s == "-0" { b"0" } else { s.as_bytes() }); - // A whole number past that range prints without a point; the trailing `.` keeps it a real. - if !s.contains('.') && s != "-0" { - out.push(b'.'); - } -} - -fn write_name(n: &[u8], out: &mut Vec) { - out.push(b'/'); - for &b in n { - let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); - if regular { - out.push(b); - } else { - out.push(b'#'); - out.extend_from_slice(&hex_pair(b)); - } - } -} - -fn write_string(s: &PdfString, out: &mut Vec) { - let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); - if s.hex || binary * 4 > s.bytes.len().max(1) { - out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); - out.push(b'<'); - for &b in &s.bytes { - out.extend_from_slice(&hex_pair(b)); - } - out.push(b'>'); - return; - } - out.push(b'('); - for &b in &s.bytes { - match b { - b'(' | b')' | b'\\' => { - out.push(b'\\'); - out.push(b); - } - b'\n' => out.extend_from_slice(b"\\n"), - b'\r' => out.extend_from_slice(b"\\r"), - b'\t' => out.extend_from_slice(b"\\t"), - 0x08 => out.extend_from_slice(b"\\b"), - 0x0C => out.extend_from_slice(b"\\f"), - b' '..=b'~' => out.push(b), - _ => { - out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); - } - } - } - out.push(b')'); -} - -fn write_dict(d: &Dict, out: &mut Vec) { - out.extend_from_slice(b"<<"); - for (k, v) in d.iter() { - write_name(k, out); - out.push(b' '); - serialize(v, out); - } - out.extend_from_slice(b">>"); -} - -fn write_stream(s: &Stream, out: &mut Vec) { - let mut d = s.dict.clone(); - d.set(b"Length".to_vec(), Object::Int(s.raw.len() as i64)); - write_dict(&d, out); - out.extend_from_slice(b"\nstream\n"); - out.extend_from_slice(&s.raw); - out.extend_from_slice(b"\nendstream"); -} - -/// A document opened without the password its embedded files need holds their data as stored and -/// can't encrypt: a save keeps its `/Encrypt` as it is, and writes no edited or added stream that -/// would have to be encrypted (an attached file put in, say), which would be written in the clear. -fn keyless_check(doc: &Document) -> Result<(), CosError> { - if !doc.embedded_files_locked() { - return Ok(()); - } - let refuse = - |what: &str| Err(CosError::Security(format!("{what} without the password of the document's embedded files: open it with the password"))); - if doc.encryption_changed() || doc.trailer().get(b"Encrypt").is_none() { - return refuse("the document's security can't change"); - } - if doc.overlay_entries().any(|(_, _, o)| matches!(o.as_deref(), Some(Object::Stream(s)) if doc.reads_as_stored(s))) { - return refuse("an edited or added embedded file can't be encrypted"); - } - Ok(()) -} - -/// The object as written: encrypted with its (output) number when the document is encrypted, -/// except the `/Encrypt` dictionary itself (`source_num` is its number in the source document). -/// `o` has the document's numbers in its references ([`crate::security::transform`]). -fn prepared<'a>(doc: &Document, source_num: u32, num: u32, generation: u16, o: &'a Object) -> Cow<'a, Object> { - match doc.output_security() { - (Some(h), encrypt_num) if Some(source_num) != encrypt_num && !doc.stored_as_is(source_num) => { - let pass = if doc.encryption_changed() { crate::security::Pass::EncryptReplacing } else { crate::security::Pass::Encrypt }; - Cow::Owned(crate::security::transform(doc, h, o, num, generation, pass)) - } - _ => Cow::Borrowed(o), - } -} - -fn write_indirect(num: u32, generation: u16, o: &Object, out: &mut Vec) { - let _ = writeln!(out, "{num} {generation} obj"); - serialize(o, out); - out.extend_from_slice(b"\nendobj\n"); -} - -/// Options for saving. -#[derive(Clone, Debug)] -pub struct SaveOptions { - /// Replace or add `/Info /ModDate` with this PDF date string (e.g. from `pdf_date`). - pub mod_date: Option, - /// Seed for a new file identifier when the document has none (deterministic in tests). - pub id_seed: u64, - /// Full saves: pack non-stream objects into compressed object streams with a cross-reference - /// stream (PDF 1.5+, §7.5.7–7.5.8). Off writes a classic table that any reader accepts. - pub object_streams: bool, - /// Incremental saves: leave out what was added since the file was read and nothing refers to - /// any more, such as a comment added and deleted again or a redaction mark added and cleared - /// (written as free). Off by default, because a writer may add an object only it finds again - /// by scanning, as a standalone document timestamp is; such a writer frees the leftovers - /// before adding it, with [`Document::free_unlinked_additions`]. Full saves always leave - /// these out. - pub drop_unlinked: bool, - /// Entropy for the file identifier of a rewrite that asks for a new one (see - /// [`Document::require_full_save_with_new_id`]). Given, the identifier depends only on this and - /// on the written bytes, so the same save gives the same file; tests use it to pin output. - /// Without it, fresh entropy from the host (the operating system on native targets, the - /// browser on WebAssembly) is mixed in. - pub id_entropy: Option<[u8; 16]>, -} - -impl Default for SaveOptions { - fn default() -> Self { - Self { mod_date: None, id_seed: 0x5052_494E_5443_5241, object_streams: true, drop_unlinked: false, id_entropy: None } - } -} - -/// Objects per object stream (Acrobat and qpdf use 100–200). -const OBJSTM_SIZE: usize = 100; -/// Keep a batch's serialized body small as well as limiting its object count. One object -/// may exceed this target; it is emitted in its own object stream without retaining a batch. -const OBJSTM_BYTES: usize = 8 * 1024 * 1024; - -/// Where an object lives, for the cross-reference section. -#[derive(Clone, Copy, Debug, PartialEq)] -enum Row { - /// Free, with the generation the number gets when reused. - Free(u16), - /// At a byte offset. - InFile(u64, u16), - /// Inside object stream `stream`, at position `index`. - InStream(u32, u32), -} - -/// Append the document's edits to its original bytes (incremental update, §7.5.6). -/// -/// A document that had to be reconstructed (no usable cross-reference chain) cannot be updated -/// incrementally — other readers would follow the broken chain — so it is rewritten in full, -/// which also repairs the file (what users expect after "the file was damaged and repaired"). -pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - keyless_check(doc)?; - // Reconstructed files have no chain to append to; added or removed encryption must - // rewrite every object; redaction must not leave the old revision behind. All need a full save. - if doc.reconstructed() || doc.encryption_changed() || doc.full_save_required() { - return write_full(doc, opts); - } - let mut doc = doc.clone(); - stamp_mod_date(&mut doc, opts); - let original = doc.bytes().clone(); - // The revision is written on its own and joined to the original at the end, so the result is - // allocated once at its exact size: growing a buffer that holds a large original past a - // guessed capacity would copy the whole file again and keep up to as much again unused. - let newline = !matches!(original.last(), Some(b'\n' | b'\r')); - // A file with bytes before its `%PDF-` header may count its offsets from the header. The new - // section then has to as well: readers take `/Prev` and `startxref` in one convention, and a - // mix sends them into reconstruction (#2364). - let shift = if doc.revisions().last().is_some_and(|r| r.header_relative) { doc.header_offset() } else { 0 }; - let base = (original.len() + usize::from(newline)).saturating_sub(shift) as u64; - let mut out = Vec::new(); - let mut offsets: BTreeMap = BTreeMap::new(); // num → (offset, gen, in use) - let unlinked = if opts.drop_unlinked { unlinked_additions(&doc) } else { HashSet::new() }; - for (num, generation, obj) in doc.overlay_entries() { - match obj { - // Written as free, so `/Size` still covers every number this document handed out. The - // generation stays: the number was never in use, and an undo can bring it back as is. - Some(_) if unlinked.contains(&num) => { - offsets.insert(num, (0, generation, false)); - } - Some(o) => { - offsets.insert(num, (base + out.len() as u64, generation, true)); - write_indirect(num, generation, &prepared(&doc, num, num, generation, &o), &mut out); - } - None => { - offsets.insert(num, (0, generation, false)); - } - } - } - let prev = doc.revisions().last().map(|r| r.xref_offset); - let as_stream = doc.revisions().last().is_some_and(|r| r.is_stream); - let mut trailer = doc.trailer().clone(); - if doc.output_security().0.is_none() { - ensure_id(&mut trailer, opts, &original); // an encrypted file's ID is part of its key - } - if let Some(p) = prev { - trailer.set(b"Prev".to_vec(), Object::Int(p as i64)); - } - let size = doc.next_num().max(trailer.int(b"Size").map_or(0, |n| n.clamp(0, i64::from(u32::MAX)) as u32)); - if as_stream { - let rows = offsets.iter().map(|(n, (o, g, used))| (*n, if *used { Row::InFile(*o, *g) } else { Row::Free(*g) })).collect(); - write_xref_stream(&mut out, base, &mut trailer, rows, size); - } else { - trailer.set(b"Size".to_vec(), Object::Int(size as i64)); - let xref_at = base + out.len() as u64; - write_xref_table(&mut out, &offsets, prev.is_none()); - out.extend_from_slice(b"trailer\n"); - write_dict(&trailer, &mut out); - let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); - } - let mut file = Vec::with_capacity(original.len() + usize::from(newline) + out.len()); - file.extend_from_slice(&original); - if newline { - file.push(b'\n'); - } - file.extend_from_slice(&out); - Ok(file) -} - -/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and -/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets -/// (aliases of one slot included) and, in visit order, whether it is a stream and the parsed -/// object where the budget allowed keeping it. -type SaveOrder = (HashMap, Vec<(ObjRef, bool)>, Vec>>); - -/// Number the objects reachable from the trailer. `retain` keeps parsed objects for the write -/// pass, up to the budget. A caller that needs only the numbers passes `false`, so the traversal -/// holds no parsed objects beyond the current one. -fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict, retain: bool) -> Result { - // Breadth-first walk from the trailer, assigning new numbers in visit order. - let mut map: HashMap = HashMap::new(); - // Record the type while traversing so classifying packed objects needs no second parse. - let mut order: Vec<(ObjRef, bool)> = Vec::new(); - // The parsed object of each entry in `order`, kept for the write pass while the budget - // allows it. Past the budget the write pass reads the object again. - let mut kept: Vec>> = Vec::new(); - let mut kept_budget = KeptObjects::default(); - let mut queue: VecDeque = VecDeque::new(); - let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); - for key in [&b"Root"[..], b"Info", b"Encrypt"] { - if let Some(r) = trailer_in.reference(key) { - queue.push_back(r); - } - } - // A direct /Encrypt can refer to objects, /CF or /StmF, say. - if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { - visit_refs(encrypt, &mut queue); - } - // Unknown trailer entries are document data. Objects they alone reference must be written - // too, or a full rewrite would keep the key and drop its value. - for (key, value) in trailer_in.iter() { - if !trailer_key_rewritten(key) { - collect_refs(value, &mut |r| queue.push_back(r)); - } - } - while let Some(r) = queue.pop_front() { - if map.contains_key(&r) { - continue; - } - let o = reader.get(r); - if matches!(*o, Object::Null) { - continue; // dangling reference: written as null by renumbering below - } - // Accepted xref/header aliases denote one slot, including page-tree cycles. - // Check for null first: an unrelated or freed generation must remain dangling. - let canonical = ObjRef::new(r.num, reader.document.generation(r.num)); - if let Some(&num) = map.get(&canonical) { - map.insert(r, num); - continue; - } - let num = output_number(order.len())?; - map.insert(r, num); - map.insert(canonical, num); - order.push((canonical, matches!(*o, Object::Stream(_)))); - visit_refs(&o, &mut queue); - kept.push((retain && kept_budget.admit(&o, reader.document.bytes())).then_some(o)); - } - Ok((map, order, kept)) -} - -/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] -/// reaches from the trailer, object streams included. For a check that something removed from a -/// document cannot reach the saved file through a reference nobody looked for. -pub fn reachable_objects(doc: &Document) -> Result, CosError> { - let mut reader = doc.object_reader(); - let trailer_in = reader.document.trailer().clone(); - let (_, order, _) = full_save_order(&mut reader, &trailer_in, false)?; - Ok(order.into_iter().map(|(r, _)| r.num).collect()) -} - -/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. -pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - keyless_check(doc)?; - // Renumbering breaks a key that mixes in the object number. - if doc.security().is_some_and(|h| !h.has_key() && !h.survives_renumbering()) { - return Err(CosError::Security( - "the embedded files are encrypted and the document was opened without their password: open it with the password to save it in full" - .into(), - )); - } - // A full traversal must not fill the editor's shared caches (which undo snapshots also - // own). Parsed objects and decoded object streams are temporary working data here. - let mut reader = doc.object_reader(); - stamp_mod_date(&mut reader.document, opts); - let trailer_in = reader.document.trailer().clone(); - let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in, true)?; - let version = reader.document.version().chars().take(3).collect::(); - let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; - let mut out = Vec::new(); - let _ = writeln!(out, "%PDF-{version}"); - // A binary comment (raw high bytes) marks the file as binary for transfer tools. - out.extend_from_slice(&[b'%', 0xE2, 0xE3, 0xCF, 0xD3, b'\n']); - // The /Encrypt dictionary and everything it references (e.g. indirect /CF crypt-filter - // dictionaries) are needed before anything can be decrypted: never inside object streams. - let mut encryption_objects: std::collections::HashSet = std::collections::HashSet::new(); - let mut pending: Vec = Vec::new(); - if let Some(encrypt) = trailer_in.get(b"Encrypt") { - collect_refs(encrypt, &mut |r| pending.push(r)); - } - while let Some(r) = pending.pop() { - if let Some(&num) = map.get(&r) - && encryption_objects.insert(num) - { - collect_refs(&reader.get(r), &mut |x| pending.push(x)); - } - } - // Acrobat refuses an encrypted file whose catalog sits in an object stream ("the file is - // damaged and could not be repaired", before it even asks for the password), while other - // readers accept it. qpdf keeps the catalog out of object streams too. Verified with - // Acrobat DC on macOS (#774). - let root_outside = - if reader.document.output_security().0.is_some() { trailer_in.reference(b"Root").and_then(|r| map.get(&r)).copied() } else { None }; - let mut rows: BTreeMap = BTreeMap::new(); - // Keep only references, not renamed copies of the entire object graph. Standalone - // objects still precede object streams, preserving the established output order. - let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); - for (i, (r, is_stream)) in order.iter().enumerate() { - let num = output_number(i)?; - let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(&num) && root_outside != Some(num); - if in_stream { - packed.push((num, *r, kept[i].take())); - } else { - let o = written_object(&mut reader, *r, *is_stream, kept[i].take())?; - let o = renumber(&prepared(&reader.document, r.num, num, 0, &o), &map); - rows.insert(num, Row::InFile(out.len() as u64, 0)); - write_indirect(num, 0, &o, &mut out); - } - } - // Object streams take the numbers after the last object. Their contents are not encrypted - // individually: the whole stream is (§7.5.7, §7.6.2). - let mut next = output_number(order.len())?; - let mut batch = ObjectStreamBatch::default(); - for (num, reference, parsed) in packed { - let mut encoded = Vec::new(); - serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); - if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { - batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; - } - let _ = write!(batch.header, "{} {} ", num, batch.body.len()); - batch.body.extend_from_slice(&encoded); - batch.body.push(b'\n'); - drop(encoded); - rows.insert(num, Row::InStream(next, batch.count as u32)); - batch.count += 1; - if batch.count == OBJSTM_SIZE || batch.body.len() >= OBJSTM_BYTES { - batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; - } - } - batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; - let mut trailer = full_save_trailer(&reader.document, &trailer_in, &map, opts, &out); - if opts.object_streams { - write_xref_stream(&mut out, 0, &mut trailer, rows, next); - } else { - trailer.set(b"Size".to_vec(), Object::Int(next as i64)); - let offsets: BTreeMap = - rows.iter().filter_map(|(n, r)| if let Row::InFile(o, g) = r { Some((*n, (*o, *g, true))) } else { None }).collect(); - let xref_at = out.len(); - write_xref_table(&mut out, &offsets, true); - out.extend_from_slice(b"trailer\n"); - write_dict(&trailer, &mut out); - let _ = write!(out, "\nstartxref\n{xref_at}\n%%EOF\n"); - } - // The size isn't known until the end, and the buffer grew by doubling. The result is often - // kept as a document's working bytes for as long as it's open: drop the unused capacity. - out.shrink_to_fit(); - Ok(out) -} - -/// What `write_full` keeps of the objects it parsed while numbering them, measured with -/// [`held_bytes`] (the parsed tree and any data that is not a view into the file). Beyond the -/// budget objects are read again when written, which bounds the memory of a save of a huge -/// document. -#[derive(Default)] -struct KeptObjects { - bytes: usize, -} - -impl KeptObjects { - const BYTES: usize = 64 * 1024 * 1024; - - /// Whether `object` still fits the budget; if so it is counted against it. - fn admit(&mut self, object: &Object, data: &Arc>) -> bool { - let bytes = self.bytes.saturating_add(held_bytes(object, data, 0)); - if bytes > Self::BYTES { - return false; - } - self.bytes = bytes; - true - } -} - -/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read -/// again. -fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { - match kept { - Some(o) => Ok(o), - None => reread(reader, r, is_stream), - } -} - -/// Assemble the full rewrite's trailer and apply its identifier policy. -fn full_save_trailer(doc: &Document, trailer_in: &Dict, map: &HashMap, opts: &SaveOptions, out: &[u8]) -> Dict { - let mut trailer = Dict::new(); - for key in [&b"Root"[..], b"Info", b"Encrypt"] { - if let Some(r) = trailer_in.reference(key).and_then(|r| map.get(&r)) { - trailer.set(key.to_vec(), Object::Ref(ObjRef::new(*r, 0))); - } - } - if let Some(encrypt @ Object::Dict(_)) = trailer_in.get(b"Encrypt") { - trailer.set(b"Encrypt".to_vec(), renumber(encrypt, map)); - } - if let Some(id) = trailer_in.get(b"ID") { - trailer.set(b"ID".to_vec(), direct_id(doc, id)); - } - for (key, value) in trailer_in.iter() { - if !trailer_key_rewritten(key) { - trailer.set(key.clone(), renumber(value, map)); - } - } - if doc.output_security().0.is_none() { - ensure_id(&mut trailer, opts, doc.bytes()); - } - if doc.new_id_required() { - regenerate_id(&mut trailer, opts, out, doc.output_security().0.is_some()); - } - trailer -} - -/// Object `r` read again for writing. The traversal read it once and only kept its number, so -/// a second read that fails, finds nothing or finds a different kind of object must fail the -/// save rather than write `null` in its place. -fn reread(reader: &mut ObjectReader, r: ObjRef, is_stream: bool) -> Result, CosError> { - let o = reader.try_get(r)?; - if matches!(*o, Object::Null) || matches!(*o, Object::Stream(_)) != is_stream { - return Err(CosError::Syntax { offset: 0, detail: format!("object {} read differently while saving; nothing was written", r.num) }); - } - Ok(o) -} - -#[derive(Default)] -struct ObjectStreamBatch { - header: Vec, - body: Vec, - count: usize, -} - -impl ObjectStreamBatch { - fn flush(&mut self, doc: &Document, next: &mut u32, rows: &mut BTreeMap, out: &mut Vec) -> Result<(), CosError> { - if self.count == 0 { - return Ok(()); - } - let following = next.checked_add(1).ok_or_else(|| CosError::Syntax { offset: 0, detail: "too many objects to save".into() })?; - // Release capacity after each batch too: an unusually large object must not leave - // a large scratch buffer alive for the rest of the save. - let Self { mut header, body, count } = std::mem::take(self); - let first = header.len(); - header.extend_from_slice(&body); - drop(body); - let mut d = Dict::new(); - d.set(b"Type".to_vec(), Object::name("ObjStm")); - d.set(b"N".to_vec(), Object::Int(count as i64)); - d.set(b"First".to_vec(), Object::Int(first as i64)); - let stm = Object::Stream(Stream::flate(d, &header)); - rows.insert(*next, Row::InFile(out.len() as u64, 0)); - write_indirect(*next, 0, &prepared(doc, u32::MAX, *next, 0, &stm), out); - *next = following; - Ok(()) - } -} - -/// `/ID` for the trailer, direct, as it has to be with `/Encrypt`: nothing else in the file holds -/// what it refers to now. Given by reference it is two entries at most, however many references a -/// file gave, so that one large string isn't copied for each. An entry that isn't a string stays in -/// its place as an empty one, which is how the key read it. -fn direct_id(doc: &Document, id: &Object) -> Object { - if matches!(id, Object::Array(a) if !a.iter().any(|s| matches!(s, Object::Ref(_)))) { - return id.clone(); - } - let entry = |s: &Object| match &*doc.resolve(s) { - s @ Object::String(_) => s.clone(), - _ => Object::String(crate::PdfString { bytes: Vec::new(), hex: true }), - }; - let held = doc.resolve(id); - Object::Array(held.as_array().into_iter().flatten().take(2).map(entry).collect()) -} - -/// Trailer keys a full rewrite generates itself. Everything else is copied, with references -/// renumbered, so a save does not drop metadata the reader accepted. -fn trailer_key_rewritten(key: &[u8]) -> bool { - matches!( - key, - b"Size" | b"Prev" | b"XRefStm" | b"Type" | b"W" | b"Index" | b"Filter" | b"DecodeParms" | b"Length" | b"Root" | b"Info" | b"Encrypt" | b"ID" - ) -} - -fn output_number(index: usize) -> Result { - u32::try_from(index).ok().and_then(|n| n.checked_add(1)).ok_or_else(|| CosError::Syntax { offset: 0, detail: "too many objects to save".into() }) -} - -/// Objects added since the file was read (their numbers are not in use in it) that nothing an -/// incremental update writes refers to: what one edit added and a later one unlinked, such as a -/// deleted comment or a cleared redaction mark. The file never held them, so leaving them out -/// loses nothing, while writing them would keep what the user removed. Only the update is walked: -/// from the trailer and the file's changed objects through additions. An unchanged object of the -/// file is not followed: a reference it holds to such a number already reads as null in the file -/// (a damaged file's dangling reference), and it still does. -pub(crate) fn unlinked_additions(doc: &Document) -> HashSet { - let added = |num: u32| matches!(doc.xref_entry(num), None | Some(XrefEntry::Free { .. })); - let mut pending: HashMap> = - doc.overlay_entries().filter_map(|(num, _, o)| Some((num, o?))).filter(|(num, _)| added(*num)).collect(); - if pending.is_empty() { - return HashSet::new(); - } - let mut stack: Vec = Vec::new(); - doc.trailer().iter().for_each(|(_, v)| collect_refs(v, &mut |r| stack.push(r))); - for (num, _, o) in doc.overlay_entries() { - if let Some(o) = o - && !added(num) - { - collect_refs(&o, &mut |r| stack.push(r)); - } - } - while let Some(r) = stack.pop() { - if let Some(o) = pending.remove(&r.num) { - collect_refs(&o, &mut |r| stack.push(r)); - } - } - pending.into_keys().collect() -} - -pub(crate) fn collect_refs(o: &Object, f: &mut impl FnMut(ObjRef)) { - match o { - Object::Ref(r) => f(*r), - Object::Array(a) => a.iter().for_each(|x| collect_refs(x, f)), - Object::Dict(d) => d.iter().for_each(|(_, v)| collect_refs(v, f)), - Object::Stream(s) => s.dict.iter().for_each(|(_, v)| collect_refs(v, f)), - _ => {} - } -} - -fn renumber(o: &Object, map: &HashMap) -> Object { - match o { - Object::Ref(r) => map.get(r).map(|n| Object::Ref(ObjRef::new(*n, 0))).unwrap_or(Object::Null), - Object::Array(a) => Object::Array(a.iter().map(|x| renumber(x, map)).collect()), - Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), renumber(v, map))).collect()), - Object::Stream(s) => Object::Stream(Stream { dict: s.dict.iter().map(|(k, v)| (k.clone(), renumber(v, map))).collect(), raw: s.raw.clone() }), - other => other.clone(), - } -} - -/// Classic xref table with one subsection per run of consecutive numbers (§7.5.4). -fn write_xref_table(out: &mut Vec, offsets: &BTreeMap, include_zero: bool) { - out.extend_from_slice(b"xref\n"); - let mut all: Vec<(u32, (u64, u16, bool))> = offsets.iter().map(|(k, v)| (*k, *v)).collect(); - if include_zero { - all.insert(0, (0, (0, 65535, false))); - } - // A section is one or more subsections. `xref` followed immediately by `trailer` is not a - // table (qpdf: "xref syntax invalid") and is what an incremental save of a classic-table - // file used to append when nothing was edited. A zero-length subsection names no objects, - // so the free list and every in-use entry stay as they were, while a trailer-only change - // (a missing `/ID`, or a caller edit of the trailer) is still written. - if all.is_empty() { - let _ = writeln!(out, "0 0"); - return; - } - let mut i = 0; - while i < all.len() { - let start = all[i].0; - let mut j = i; - while j + 1 < all.len() && all[j + 1].0 == all[j].0 + 1 { - j += 1; - } - let _ = writeln!(out, "{} {}", start, j - i + 1); - for (_, (off, generation, used)) in &all[i..=j] { - let _ = writeln!(out, "{:010} {:05} {} ", off, generation, if *used { 'n' } else { 'f' }); - } - i = j + 1; - } -} - -/// A cross-reference stream section (§7.5.8) that takes object number `num`, ending the file. -/// `out` holds the file from byte offset `base` on. Rows are Flate-compressed with the PNG Up -/// predictor, and field widths fit the largest value. -fn write_xref_stream(out: &mut Vec, base: u64, trailer: &mut Dict, mut rows: BTreeMap, num: u32) { - let xref_at = base + out.len() as u64; - rows.insert(num, Row::InFile(xref_at, 0)); - if trailer.get(b"Prev").is_none() { - rows.entry(0).or_insert(Row::Free(65535)); - } - let fields = |r: &Row| match *r { - Row::Free(g) => (0u64, 0u64, u64::from(g)), - Row::InFile(o, g) => (1, o, u64::from(g)), - Row::InStream(s, i) => (2, u64::from(s), u64::from(i)), - }; - let width = |v: u64| ((64 - v.leading_zeros()).div_ceil(8)).max(1) as usize; - let (w2, w3) = rows.values().map(fields).fold((1, 1), |(a, b), (_, x, y)| (width(x).max(a), width(y).max(b))); - let cols = 1 + w2 + w3; - let mut index = Vec::new(); - let mut numbers = rows.keys().copied().peekable(); - while let Some(start) = numbers.next() { - let mut last = start; - let mut count = 1i64; - while numbers.peek().copied().is_some_and(|next| last.checked_add(1) == Some(next)) { - if let Some(next) = numbers.next() { - last = next; - count += 1; - } - } - index.push(Object::Int(i64::from(start))); - index.push(Object::Int(count)); - } - let mut data = Vec::with_capacity(rows.len().saturating_mul(cols + 1)); - // Type is one byte, and each remaining field is at most one u64. Reuse the - // predictor row on the stack instead of allocating a Vec for every xref entry. - let mut prev = [0u8; 17]; - for r in rows.values() { - let (t, a, b) = fields(r); - let a = a.to_be_bytes(); - let b = b.to_be_bytes(); - let fields = std::iter::once(t as u8).chain(a.into_iter().skip(8 - w2)).chain(b.into_iter().skip(8 - w3)); - data.push(2); // PNG "Up" - for (value, previous) in fields.zip(&mut prev) { - data.push(value.wrapping_sub(*previous)); - *previous = value; - } - } - let mut d = trailer.clone(); - d.set(b"Type".to_vec(), Object::name("XRef")); - d.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); - d.set(b"W".to_vec(), Object::Array(vec![Object::Int(1), Object::Int(w2 as i64), Object::Int(w3 as i64)])); - d.set(b"Index".to_vec(), Object::Array(index)); - let mut stream = Stream::flate(d, &data); - let mut parms = Dict::new(); - parms.set(b"Predictor".to_vec(), Object::Int(12)); - parms.set(b"Columns".to_vec(), Object::Int(cols as i64)); - stream.dict.set(b"DecodeParms".to_vec(), Object::Dict(parms)); - write_indirect(num, 0, &Object::Stream(stream), out); - let _ = write!(out, "startxref\n{xref_at}\n%%EOF\n"); - trailer.set(b"Size".to_vec(), Object::Int(num as i64 + 1)); -} - -fn stamp_mod_date(doc: &mut Document, opts: &SaveOptions) { - let Some(date) = &opts.mod_date else { return }; - let date = Object::String(PdfString::literal(date.as_bytes().to_vec())); - match doc.trailer().get(b"Info").cloned() { - Some(Object::Ref(r)) => { - let _ = doc.update_dict(r, |d| d.set(b"ModDate".to_vec(), date.clone())); - } - _ => { - let mut info = Dict::new(); - info.set(b"ModDate".to_vec(), date); - let r = doc.add(info); - doc.trailer_mut().set(b"Info".to_vec(), Object::Ref(r)); - } - } -} - -/// Keep any existing two-element `/ID` array as it is, even a malformed one such as `[<> <>]` or `[1 2]` (an ordinary -/// save preserves what it doesn't understand; Save as PDF/A replaces or completes it), or create one with -/// [`new_file_id`] when there is none or it isn't a two-element array. -fn ensure_id(trailer: &mut Dict, opts: &SaveOptions, content: &[u8]) { - if matches!(trailer.get(b"ID"), Some(Object::Array(a)) if a.len() == 2) { - return; - } - trailer.set(b"ID".to_vec(), new_file_id(content, opts.id_seed)); -} - -/// A new file identifier for a file whose bytes are `content` (ISO 32000-1 14.4): an array of -/// two equal 16-byte strings, the same for the same content and `seed`. -pub fn new_file_id(content: &[u8], seed: u64) -> Object { - // FNV-1a over a sample of the content and the seed: stable, dependency-free. - let mut h: u64 = 0xcbf2_9ce4_8422_2325 ^ seed; - for b in content.iter().step_by((content.len() / 4096).max(1)).chain(content.iter().rev().take(64)) { - h ^= *b as u64; - h = h.wrapping_mul(0x0100_0000_01b3); - } - let mut id = Vec::with_capacity(16); - id.extend_from_slice(&h.to_be_bytes()); - id.extend_from_slice(&h.rotate_left(29).wrapping_mul(0x9E37_79B9_7F4A_7C15).to_be_bytes()); - let s = Object::String(PdfString { bytes: id, hex: true }); - Object::Array(vec![s.clone(), s]) -} - -/// Give a rewritten file a new identifier (§14.4), so that a redacted or sanitized copy does not -/// claim to be its source. An encrypted file keeps the first string, which its key is derived from, -/// and gets a new second one. When there is no such string (no `/ID`, one entry, or a first entry -/// that is not a string) the reader takes the empty string as the key input (`Document::first_id`), -/// so the empty string is written first and the key stays valid. -fn regenerate_id(trailer: &mut Dict, opts: &SaveOptions, written: &[u8], encrypted: bool) { - let fresh = Object::String(PdfString { bytes: new_id_bytes(opts, written), hex: true }); - let pair = if encrypted { - let first = match trailer.get(b"ID") { - Some(Object::Array(a)) => match a.first() { - Some(Object::String(s)) => Object::String(s.clone()), - _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), - }, - _ => Object::String(PdfString { bytes: Vec::new(), hex: true }), - }; - vec![first, fresh] - } else { - vec![fresh.clone(), fresh] - }; - trailer.set(b"ID".to_vec(), Object::Array(pair)); -} - -/// FNV-1a over 64 bits, with its published offset basis and prime. The identifier is built from it -/// rather than from `std`'s `DefaultHasher`, whose algorithm the standard library does not specify. -const FNV_OFFSET: u64 = 0xcbf2_9ce4_8422_2325; -const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; - -fn fnv1a(state: u64, bytes: &[u8]) -> u64 { - bytes.iter().fold(state, |h, b| (h ^ u64::from(*b)).wrapping_mul(FNV_PRIME)) -} - -/// Sixteen bytes for a file identifier, one 8-byte half per lane. The bytes are a pure function of -/// `SaveOptions::id_entropy`, `id_seed` and the written file: the hash is FNV-1a (above) and every -/// integer is written as a fixed-width big-endian value, so the same input gives the same identifier -/// on every platform and toolchain. Without entropy only the salt changes (see `id_salt`). FNV is -/// not collision-resistant against chosen content; the identifier only has to be unique, not secret. -/// Every written byte is hashed, once, in a single traversal of the output. -fn new_id_bytes(opts: &SaveOptions, written: &[u8]) -> Vec { - let salt = id_salt(opts); - let length = u64::try_from(written.len()).unwrap_or(u64::MAX); - let mut halves = [0u8, 1].map(|lane| { - let mut h = fnv1a(FNV_OFFSET, &salt.to_be_bytes()); - h = fnv1a(h, &[lane]); - h = fnv1a(h, &opts.id_seed.to_be_bytes()); - fnv1a(h, &length.to_be_bytes()) - }); - for chunk in written.chunks(4096) { - for h in &mut halves { - *h = fnv1a(*h, chunk); - } - } - halves.iter().flat_map(|h| h.to_be_bytes()).collect() -} - -/// The salt of an identifier. With caller entropy it is a fixed hash of that entropy (deterministic, -/// for tests and callers who pin output). Without it the salt is fresh: on WebAssembly from the -/// browser ([`browser_salt`], so two fresh web apps that write the same bytes cannot share an -/// identifier), and elsewhere from the standard library's random hash keys — seeded by the operating -/// system, so separate processes differ — mixed with a process-wide counter, which separates -/// successive calls in one process. -fn id_salt(opts: &SaveOptions) -> u64 { - use std::hash::BuildHasher; - use std::sync::atomic::{AtomicU64, Ordering}; - static COUNTER: AtomicU64 = AtomicU64::new(0); - match &opts.id_entropy { - Some(entropy) => fnv1a(FNV_OFFSET, entropy), - None => { - #[cfg(target_arch = "wasm32")] - if let Some(salt) = browser_salt() { - return salt; - } - std::collections::hash_map::RandomState::new().hash_one(COUNTER.fetch_add(1, Ordering::Relaxed)) - } - } -} - -/// One byte from one `Math.random` draw: the draw (in [0, 1)) scaled to [0, 256), integer part -/// taken. The quotient is in [0, 256), so a saturating cast never wraps; a NaN or infinite draw -/// (which a conforming engine never returns) folds in as 0. No input can panic here. -#[cfg(any(target_arch = "wasm32", test))] -fn draw_byte(random: f64) -> u8 { - (random * 256.0).floor().rem_euclid(256.0) as u8 -} - -/// The `Math.random` fallback's mixing: one salt byte per draw (see [`draw_byte`]), folded through -/// FNV-1a exactly like the Web Crypto bytes, so all sixteen bytes of the fold carry entropy from -/// the draws. Pure and platform-independent, so the native test target can exercise it; on wasm -/// each draw comes from `Math.random`. -#[cfg(any(target_arch = "wasm32", test))] -fn salt_from_draws(draws: &[f64; 16]) -> u64 { - let mut bytes = [0u8; 16]; - for (byte, draw) in bytes.iter_mut().zip(draws) { - *byte = draw_byte(*draw); - } - fnv1a(FNV_OFFSET, &bytes) -} - -/// Fresh salt entropy from the browser, on WebAssembly. `crypto.getRandomValues` (Web Crypto, the -/// browser's secure generator, available in windows and workers alike) gives sixteen bytes, folded -/// into one salt. When that API is missing or refuses the call, `Math.random` is mixed instead -/// ([`salt_from_draws`]: sixteen draws, so the salt's full width carries entropy) — browsers seed -/// it afresh for every document, so a fresh instance still cannot reproduce another's salt. Only -/// where neither is reachable does the caller fall back to its `RandomState` salt, whose keys are -/// fixed at compile time on `wasm32-unknown-unknown` — the state before browser entropy was wired -/// in. -#[cfg(target_arch = "wasm32")] -fn browser_salt() -> Option { - use wasm_bindgen::JsCast; - let crypto = js_sys::Reflect::get(&js_sys::global(), &wasm_bindgen::JsValue::from_str("crypto")) - .ok() - .filter(|crypto| !crypto.is_undefined() && !crypto.is_null()); - if let Some(crypto) = crypto { - let fill = js_sys::Reflect::get(&crypto, &wasm_bindgen::JsValue::from_str("getRandomValues")) - .ok() - .and_then(|fill| fill.dyn_into::().ok()); - if let Some(fill) = fill { - let bytes = js_sys::Uint8Array::new_with_length(16); - if fill.call1(&crypto, &bytes).is_ok() { - return Some(fnv1a(FNV_OFFSET, &bytes.to_vec())); - } - } - } - // Web Crypto is missing or refused the call: degrade to `Math.random` rather than giving up. - let mut draws = [0.0; 16]; - for draw in &mut draws { - *draw = js_sys::Math::random(); - } - Some(salt_from_draws(&draws)) -} - -/// A PDF date string for "now" given seconds since the Unix epoch (UTC). -pub fn pdf_date(unix_secs: i64) -> String { - let (days, secs) = (unix_secs.div_euclid(86_400), unix_secs.rem_euclid(86_400)); - let (y, m, d) = civil_from_days(days); - // `{y:04}` rather than `%Y`, which writes a `+` before years past 9999 (and a PDF date is a - // bare four-plus-digit year, never signed). No year clamp — the old code wrote `y` as-is. - format!("D:{y:04}{m:02}{d:02}{:02}{:02}{:02}Z", secs / 3600, secs / 60 % 60, secs % 60) -} - -/// Days since 1970-01-01 → proleptic Gregorian (year, month, day). Total for any `i64` count: -/// dates past `NaiveDate`'s ±262143-year range keep their proleptic Gregorian year instead of -/// falling back to the epoch. -/// -/// The Gregorian calendar repeats every 400 years (146_097 days), so chrono converts the day -/// within one cycle and whole cycles go back into the year (an `i128` so the extremes can't -/// overflow), keeping chrono as the only calendar logic. 1970-01-01 is `num_days_from_ce` 719_163. -pub fn civil_from_days(days: i64) -> (i64, u32, u32) { - use chrono::Datelike; - let (cycles, rest) = (days.div_euclid(146_097), days.rem_euclid(146_097)); - // `rest + 719_163` is 719_163..865_259, always a valid CE day. - match i32::try_from(rest + 719_163).ok().and_then(chrono::NaiveDate::from_num_days_from_ce_opt) { - Some(d) => { - let y = i128::from(d.year()) + i128::from(cycles) * 400; - (i64::try_from(y).unwrap_or(if y < 0 { i64::MIN } else { i64::MAX }), d.month(), d.day()) - } - None => (1970, 1, 1), // unreachable - } -} - -#[cfg(test)] -mod tests { - /// `/ID` by reference comes out direct, each entry where it was: the key is made from the first. - #[test] - fn a_file_identifier_keeps_its_entries_in_their_places() { - let doc = Document::new_empty(); - let string = |bytes: &[u8]| Object::String(crate::PdfString { bytes: bytes.to_vec(), hex: true }); - let nowhere = Object::Ref(ObjRef::new(9_999, 0)); - assert_eq!(super::direct_id(&doc, &Object::Array(vec![nowhere, string(b"ab")])), Object::Array(vec![string(b""), string(b"ab")])); - } - - use proptest::prelude::*; - - #[test] - fn the_same_entropy_and_bytes_give_the_same_identifier_on_every_platform() { - // Computed from the formula outside this crate: fixed-width big-endian fields and FNV-1a. - let opts = super::SaveOptions { id_entropy: Some([0x42; 16]), id_seed: 7, ..super::SaveOptions::default() }; - let want = [0x96, 0xf8, 0xcb, 0x7d, 0x5b, 0xb0, 0x09, 0xc8, 0x92, 0xc9, 0xb7, 0x51, 0xdf, 0xe6, 0xab, 0xd1]; - assert_eq!(super::new_id_bytes(&opts, b"%PDF-1.7\n"), want); - } - - #[test] - fn fnv1a_is_the_published_64_bit_algorithm() { - // Published FNV-1a (64 bit) test vectors, so the identifier can be reproduced outside this crate. - assert_eq!(super::fnv1a(super::FNV_OFFSET, b""), 0xcbf2_9ce4_8422_2325); - assert_eq!(super::fnv1a(super::FNV_OFFSET, b"a"), 0xaf63_dc4c_8601_ec8c); - assert_eq!(super::fnv1a(super::FNV_OFFSET, b"foobar"), 0x8594_4171_f739_67e8); - } - - #[test] - fn math_random_draws_fill_the_whole_salt_width() { - // Draw i = (i + 1/2)/256 scales to the integer i, so the fold's byte i proves draw i landed - // there: all sixteen bytes carry their own draw's entropy (the fallback once shifted four - // draws into one word, leaving the other four bytes of the fold constant). - let draws: [f64; 16] = std::array::from_fn(|i| (i as f64 + 0.5) / 256.0); - let bytes: Vec = (0..16).collect(); - assert_eq!(super::salt_from_draws(&draws), super::fnv1a(super::FNV_OFFSET, &bytes)); - } - - #[test] - fn every_math_random_draw_reaches_the_salt() { - let base = [0.5; 16]; - let baseline = super::salt_from_draws(&base); - for i in 0..16 { - let mut draws = base; - draws[i] = 0.75; - assert_ne!(super::salt_from_draws(&draws), baseline, "draw {i} did not reach the salt"); - } - } - - #[test] - fn a_math_random_draw_outside_its_range_still_mixes_without_panicking() { - // A conforming `Math.random` stays in [0, 1), but the mixing must not panic outside it. - assert_eq!(super::draw_byte(0.0), 0); - assert_eq!(super::draw_byte(0.5), 128); - assert_eq!(super::draw_byte(255.5 / 256.0), 255); - assert_eq!(super::draw_byte(-0.5 / 256.0), 255); - assert_eq!(super::draw_byte(-1.0), 0); - assert_eq!(super::draw_byte(2.0), 0); - assert_eq!(super::draw_byte(f64::NAN), 0); - assert_eq!(super::draw_byte(f64::INFINITY), 0); - } - - use super::*; - use crate::Lexer; - - fn roundtrip(o: &Object) -> Object { - let mut out = Vec::new(); - serialize(o, &mut out); - Lexer::new(&out, 0).object().expect("parses") - } - - #[test] - fn kept_objects_stop_at_their_byte_budget() { - let data = Arc::new(Vec::new()); - let string = Object::String(PdfString::literal(vec![b'x'; 1000])); - let array = Object::Array(vec![Object::Int(1), Object::Null]); - let mut kept = KeptObjects::default(); - assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); - assert_eq!(kept.bytes, held_bytes(&string, &data, 0) + held_bytes(&array, &data, 0)); - let mut full = KeptObjects { bytes: KeptObjects::BYTES - held_bytes(&string, &data, 0) + 1 }; - let before = full.bytes; - assert!(!full.admit(&string, &data), "one byte over the budget"); - assert_eq!(full.bytes, before, "a refused object is not counted"); - let mut deep = Object::Null; - for _ in 0..300 { - deep = Object::Array(vec![deep]); - } - assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); - } - - #[test] - fn kept_objects_count_names_and_dictionary_keys() { - let data = Arc::new(Vec::new()); - let name = Object::Name(vec![b'n'; KeptObjects::BYTES + 1]); - let mut kept = KeptObjects::default(); - assert!(!kept.admit(&name, &data), "a name over the byte budget is not kept"); - assert_eq!(kept.bytes, 0); - let mut dict = Dict::new(); - dict.set(vec![b'k'; KeptObjects::BYTES + 1], Object::Null); - assert!(!kept.admit(&Object::Dict(dict), &data), "nor a dictionary key"); - assert_eq!(kept.bytes, 0); - } - - #[test] - fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { - let mut doc = Document::new_empty(); - let dict = doc.add(Object::Dict(Dict::new())); - let mut reader = doc.object_reader(); - assert!(reread(&mut reader, dict, false).is_ok()); - assert!(reread(&mut reader, dict, true).is_err(), "a dictionary where a stream was traversed"); - assert!(reread(&mut reader, ObjRef::new(9999, 0), false).is_err(), "nothing where an object was traversed"); - } - - #[test] - fn xref_predictor_rows_preserve_variable_width_fields_and_sparse_runs() { - for offset in [0, 255, 256, 65535, 65536, u32::MAX as u64, u32::MAX as u64 + 1, u64::MAX] { - for index in [0, 255, 256, u32::MAX] { - for incremental in [false, true] { - let mut trailer = Dict::new(); - if incremental { - trailer.set(b"Prev".to_vec(), Object::Int(17)); - } - let rows = BTreeMap::from([(1, Row::InFile(offset, 32768)), (2, Row::Free(65535)), (4, Row::InStream(65536, index))]); - let mut bytes = Vec::new(); - write_xref_stream(&mut bytes, 0, &mut trailer, rows, 5); - let (_, object) = crate::parser::parse_indirect(&bytes, 0, &|_| None).unwrap(); - let Object::Stream(stream) = object else { panic!("xref stream") }; - let widths: Vec<_> = stream.dict.get(b"W").unwrap().as_array().unwrap().iter().map(|o| o.as_int().unwrap() as usize).collect(); - assert_eq!(widths[0], 1); - assert!((3..=8).contains(&widths[1])); - assert!((2..=4).contains(&widths[2])); - let runs: Vec<_> = stream.dict.get(b"Index").unwrap().as_array().unwrap().iter().map(|o| o.as_int().unwrap()).collect(); - assert_eq!(runs, if incremental { vec![1, 2, 4, 2] } else { vec![0, 3, 4, 2] }); - let mut expected = vec![(1, offset, 32768), (0, 0, 65535), (2, 65536, u64::from(index)), (1, 0, 0)]; - if !incremental { - expected.insert(0, (0, 0, 65535)); - } - let decoded = stream.decoded().unwrap(); - let columns: usize = widths.iter().sum(); - assert_eq!(decoded.len(), columns * expected.len()); - for (row, (kind, first, second)) in decoded.chunks_exact(columns).zip(expected) { - let values: Vec<_> = [0..1, 1..1 + widths[1], 1 + widths[1]..columns] - .into_iter() - .map(|range| row[range].iter().fold(0u64, |value, byte| value * 256 + u64::from(*byte))) - .collect(); - assert_eq!(values, vec![kind, first, second]); - } - } - } - } - } - - #[test] - fn pdf_dates_are_utc_and_byte_identical() { - // `D:YYYYMMDDHHmmSSZ` (ISO 32000-2 §7.9.4), the same strings the hand-rolled code wrote. - assert_eq!(pdf_date(0), "D:19700101000000Z"); - assert_eq!(pdf_date(1_790_000_000), "D:20260921141320Z"); - // Leap days, including the 400-year rule, and the last second of a year. - assert_eq!(pdf_date(1_709_164_800), "D:20240229000000Z"); - assert_eq!(pdf_date(951_782_400), "D:20000229000000Z"); - assert_eq!(pdf_date(1_767_225_599), "D:20251231235959Z"); - // Before the epoch (seconds into a day are counted forward from midnight UTC). - assert_eq!(pdf_date(-1), "D:19691231235959Z"); - // Years past 9999 keep the plain zero-padded year (no `+`), as the hand-rolled code did. - assert_eq!(pdf_date(253_402_300_800), "D:100000101000000Z"); - // Past chrono's ±262143-year range the proleptic Gregorian date is kept, not the epoch. - // Pinned to the old hand-rolled output (which had no year clamp — a negative year keeps - // its sign, exactly as before). - assert_eq!(pdf_date(10_000_000_000_000), "D:3188570520174640Z"); - assert_eq!(pdf_date(-10_000_000_000_000), "D:-3149180813061320Z"); - assert_eq!(crate::civil_from_days(0), (1970, 1, 1)); - assert_eq!(crate::civil_from_days(-1), (1969, 12, 31)); - assert_eq!(crate::civil_from_days(11_016), (2000, 2, 29)); - assert_eq!(crate::civil_from_days(19_782), (2024, 2, 29)); - assert_eq!(crate::civil_from_days(i64::MAX), (25_252_734_927_768_524, 7, 27)); - assert_eq!(crate::civil_from_days(i64::MIN), (-25_252_734_927_764_585, 6, 7)); - } - - #[test] - fn reals_keep_full_precision_without_exponents() { - for r in [0.1, -0.000_000_123_456_789, 1e-12, 123_456.789_012_345, 1.0 / 3.0, -2.5e7 + 0.25] { - let mut out = Vec::new(); - serialize(&Object::Real(r), &mut out); - assert!(!out.contains(&b'e') && !out.contains(&b'E'), "{}", String::from_utf8_lossy(&out)); - assert_eq!(roundtrip(&Object::Real(r)), Object::Real(r)); - } - } - - #[test] - fn whole_reals_past_the_integer_limit_stay_reals() { - for (r, text) in [ - (2_147_483_647.0, "2147483647"), - (-2_147_483_647.0, "-2147483647"), - (2_147_483_648.0, "2147483648."), - (4e9, "4000000000."), - (-4e9, "-4000000000."), - (1e30, "1000000000000000000000000000000."), - (595.0, "595"), - (-0.0, "0"), - ] { - let mut out = Vec::new(); - serialize(&Object::Real(r), &mut out); - assert_eq!(String::from_utf8_lossy(&out), text, "{r}"); - let back = roundtrip(&Object::Real(r)); - if r.abs() > f64::from(i32::MAX) { - assert_eq!(back, Object::Real(r), "{r} must read back as a real"); - } - } - } - - proptest! { - #[test] - fn any_finite_real_round_trips(r in proptest::num::f64::NORMAL | proptest::num::f64::SUBNORMAL | proptest::num::f64::ZERO) { - let back = roundtrip(&Object::Real(r)); - match back { - Object::Real(b) => prop_assert_eq!(b, r), - Object::Int(i) => { - prop_assert!(i.unsigned_abs() <= i32::MAX as u64, "{r} came back as integer {i}"); - prop_assert_eq!(i as f64, r); - } - other => prop_assert!(false, "unexpected {other:?}"), - } - } - - #[test] - fn strings_and_names_round_trip(bytes in proptest::collection::vec(any::(), 0..64)) { - let s = Object::String(PdfString { bytes: bytes.clone(), hex: false }); - prop_assert_eq!(roundtrip(&s).as_string().map(|s| s.bytes.clone()), Some(bytes.clone())); - if !bytes.is_empty() && !bytes.contains(&0) { - prop_assert_eq!(roundtrip(&Object::Name(bytes.clone())), Object::Name(bytes)); - } - } - } -} diff --git a/crates/crypt/Cargo.toml b/crates/crypt/Cargo.toml index dcc7d2b935..27f6099b7f 100644 --- a/crates/crypt/Cargo.toml +++ b/crates/crypt/Cargo.toml @@ -11,12 +11,7 @@ repository.workspace = true workspace = true [dependencies] -aes = "0.9" -md-5 = "0.11" -sha2 = "0.11" -stringprep = "0.1" -thiserror.workspace = true -zeroize.workspace = true +craft-pdf = { workspace = true } [dev-dependencies] proptest.workspace = true diff --git a/crates/crypt/src/aes_cbc.rs b/crates/crypt/src/aes_cbc.rs deleted file mode 100644 index 5d958f4f32..0000000000 --- a/crates/crypt/src/aes_cbc.rs +++ /dev/null @@ -1,89 +0,0 @@ -//! AES-CBC (128 or 256-bit key, chosen by key length) with optional PKCS#5 padding. -//! -//! Decryption is tolerant, as viewers are: a trailing partial block is ignored, and invalid -//! padding leaves the data unpadded rather than failing. - -use aes::cipher::{Array, BlockCipherDecrypt, BlockCipherEncrypt, KeyInit}; -use aes::{Aes128, Aes256}; - -enum Cipher { - A128(Box), - A256(Box), -} - -impl Cipher { - fn new(key: &[u8]) -> Self { - if let Some(k) = key.first_chunk::<32>() { - Cipher::A256(Box::new(Aes256::new(&Array::from(*k)))) - } else { - let mut k = [0u8; 16]; - let n = key.len().min(16); - k[..n].copy_from_slice(&key[..n]); - Cipher::A128(Box::new(Aes128::new(&Array::from(k)))) - } - } - - fn encrypt(&self, block: &mut [u8; 16]) { - let mut b = Array::from(*block); - match self { - Cipher::A128(c) => c.encrypt_block(&mut b), - Cipher::A256(c) => c.encrypt_block(&mut b), - } - *block = b.into(); - } - - fn decrypt(&self, block: &mut [u8; 16]) { - let mut b = Array::from(*block); - match self { - Cipher::A128(c) => c.decrypt_block(&mut b), - Cipher::A256(c) => c.decrypt_block(&mut b), - } - *block = b.into(); - } -} - -/// Encrypt; with `pad`, PKCS#5 padding is added (otherwise `data` must be whole blocks, and a -/// trailing partial block is zero-padded). -pub fn aes_cbc_encrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec { - let c = Cipher::new(key); - let mut buf = data.to_vec(); - if pad { - let n = 16 - buf.len() % 16; - buf.extend(std::iter::repeat_n(n as u8, n)); - } else if !buf.len().is_multiple_of(16) { - buf.resize(buf.len().div_ceil(16) * 16, 0); - } - let mut prev = *iv; - for b in buf.as_chunks_mut::<16>().0 { - for (x, p) in b.iter_mut().zip(prev) { - *x ^= p; - } - c.encrypt(b); - prev = *b; - } - buf -} - -/// Decrypt; with `pad`, PKCS#5 padding is removed when valid. -pub fn aes_cbc_decrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec { - let c = Cipher::new(key); - let blocks = data.as_chunks::<16>().0; - let mut out = Vec::with_capacity(blocks.len() * 16); - let mut prev = *iv; - for &cipher in blocks { - let mut b = cipher; - c.decrypt(&mut b); - for (x, p) in b.iter_mut().zip(prev) { - *x ^= p; - } - out.extend_from_slice(&b); - prev = cipher; - } - if pad && let Some(&n) = out.last() { - let n = n as usize; - if (1..=16).contains(&n) && n <= out.len() && out[out.len() - n..].iter().all(|b| *b as usize == n) { - out.truncate(out.len() - n); - } - } - out -} diff --git a/crates/crypt/src/lib.rs b/crates/crypt/src/lib.rs index 31c4963ae5..a2fee36530 100644 --- a/crates/crypt/src/lib.rs +++ b/crates/crypt/src/lib.rs @@ -1,797 +1,2 @@ -//! pdfcraft-crypt — the PDF standard security handler (ISO 32000-2 §7.6), standalone (L0). -//! -//! Supports every revision of the standard handler: -//! - R2 (40-bit RC4), R3 (RC4 up to 128-bit), R4 (crypt filters: RC4 or AES-128); -//! - R5 (the deprecated AES-256 extension) and R6 (AES-256, PDF 2.0). -//! -//! It authenticates with the user or owner password (owner first, as viewers do) and derives -//! per-object keys. It decrypts and encrypts strings and streams, and honours crypt filters, -//! including `Identity` and embedded-file-only encryption. It can also *create* encryption -//! dictionaries, which security settings (M8) and our own test fixtures use. -//! -//! The crate knows nothing about the COS object model. Callers pass in the values of the -//! `/Encrypt` dictionary and the first element of the trailer `/ID`. - -#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] - -mod aes_cbc; -mod rc4; - -use md5::{Digest as _, Md5}; -use sha2::{Sha256, Sha384, Sha512}; -use zeroize::Zeroizing; - -pub use aes_cbc::{aes_cbc_decrypt, aes_cbc_encrypt}; -pub use rc4::rc4; - -#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] -pub enum CryptError { - #[error("the password is incorrect")] - WrongPassword, - #[error("unsupported security handler: {0}")] - Unsupported(String), - #[error("malformed encryption dictionary: {0}")] - Malformed(String), -} - -/// 32-byte padding string used by revisions 2–4 (§7.6.4.3.2, step a). -pub const PADDING: [u8; 32] = [ - 0x28, 0xBF, 0x4E, 0x5E, 0x4E, 0x75, 0x8A, 0x41, 0x64, 0x00, 0x4E, 0x56, 0xFF, 0xFA, 0x01, 0x08, 0x2E, 0x2E, 0x00, 0xB6, 0xD0, 0x68, 0x3E, 0x80, - 0x2F, 0x0C, 0xA9, 0xFE, 0x64, 0x53, 0x69, 0x7A, -]; - -/// How a crypt filter encrypts (`/CFM`). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Method { - /// `None` / the `Identity` filter: data is not encrypted. - Identity, - /// `V2`: RC4. - Rc4, - /// `AESV2`: AES-128-CBC. - Aes128, - /// `AESV3`: AES-256-CBC. - Aes256, -} - -/// The values of an `/Encrypt` dictionary that the handler needs. -#[derive(Clone, Debug, PartialEq, Default)] -pub struct EncryptDict { - /// `/Filter` (must be `Standard`). - pub filter: Vec, - pub v: i64, - pub r: i64, - /// Key length in bits (`/Length`; 40 when absent). - pub length_bits: i64, - pub o: Vec, - pub u: Vec, - pub oe: Vec, - pub ue: Vec, - pub perms: Vec, - pub p: i32, - pub encrypt_metadata: bool, - /// Crypt filters (`/CF`): name → (method, key length in bytes if given). - pub crypt_filters: Vec<(Vec, Method)>, - pub stm_f: Vec, - pub str_f: Vec, - pub ef_f: Vec, -} - -/// Which password authenticated the document. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Auth { - Owner, - User, -} - -/// Permission flags from `/P`, read the way the handler's revision reads them (ISO 32000-1:2008, -/// Table 22). Owners have every permission. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Permissions { - pub bits: i32, - pub owner: bool, - /// The `/R` value of the handler that read `bits`. - pub revision: i64, -} - -impl Permissions { - fn bit(&self, n: u32) -> bool { - self.owner || (self.bits >> (n - 1)) & 1 == 1 - } - /// Revision 3 and later read the full flag list of Table 22; revision 2 has its own. - fn is_revision_3_or_later(&self) -> bool { - self.revision >= 3 - } - pub fn print(&self) -> bool { - self.bit(3) - } - /// Modify contents other than the operations below. - pub fn modify(&self) -> bool { - self.bit(4) - } - pub fn copy(&self) -> bool { - self.bit(5) - } - pub fn annotate(&self) -> bool { - self.bit(6) - } - pub fn fill_forms(&self) -> bool { - if self.is_revision_3_or_later() { self.bit(9) || self.bit(6) } else { self.bit(6) } - } - /// Revision 2 covers accessibility with the copy flag (bit 5); revision 3 and later with bit 10. - pub fn extract_for_accessibility(&self) -> bool { - if self.is_revision_3_or_later() { self.bit(10) } else { self.bit(5) } - } - /// Insert, rotate or delete pages; create bookmarks and thumbnails. Revision 3 and later grant - /// it by bit 11 alone. Table 22 defines bit 11 for those revisions only, so at revision 2 the - /// modify flag (bit 4) covers it. - pub fn assemble(&self) -> bool { - if self.is_revision_3_or_later() { self.bit(11) } else { self.bit(4) } - } - /// Take pages out into another document (Extract, Split, a source of Combine or Insert - /// Pages): [`Self::assemble`] and copying (bit 5), since the pages carry their text and - /// graphics into a new file without the document's protection (#5081). "Any except - /// extracting pages" sets bit 4 but not bit 11, so at revision 3 and later it grants neither. - pub fn extract_pages(&self) -> bool { - self.assemble() && self.copy() - } - /// Revision 3 and later need bit 3 as well as bit 12. Table 22 defines bit 12 for those - /// revisions only: at revision 2 the print flag (bit 3) is all there is. - pub fn print_high_quality(&self) -> bool { - if self.is_revision_3_or_later() { self.print() && self.bit(12) } else { self.print() } - } -} - -/// What kind of stream is being decrypted (selects the crypt filter). -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum StreamKind<'a> { - Normal, - /// An embedded file stream (uses `/EFF`). - EmbeddedFile, - /// The stream names its own crypt filter (`/Filter [/Crypt …] /DecodeParms << /Name … >>`). - Named(&'a [u8]), -} - -/// An authenticated security handler for one document. -#[derive(Clone, Debug)] -pub struct SecurityHandler { - dict: EncryptDict, - key: Zeroizing>, - auth: Auth, - recovered_user: Option>>, - methods: Methods, -} - -/// The crypt filter methods of a handler, resolved once: a document controls both the number of -/// filters and the number of strings, so a search of `/CF` per string would multiply them. -#[derive(Clone, Debug)] -struct Methods { - string: Method, - stream: Method, - embedded_file: Method, - named: std::collections::HashMap, Method>, -} - -impl Methods { - fn new(dict: &EncryptDict) -> Self { - let mut named = std::collections::HashMap::with_capacity(dict.crypt_filters.len()); - for (name, method) in &dict.crypt_filters { - #[cfg(test)] - tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1)); - // The first of two filters with one name wins, as a search would find it. - named.entry(name.clone()).or_insert(*method); - } - let mut methods = Methods { string: Method::Identity, stream: Method::Identity, embedded_file: Method::Identity, named }; - methods.string = methods.get(&dict.str_f); - methods.stream = methods.get(&dict.stm_f); - methods.embedded_file = methods.get(if dict.ef_f.is_empty() { &dict.stm_f } else { &dict.ef_f }); - methods - } - - fn get(&self, name: &[u8]) -> Method { - if name == b"Identity" || name.is_empty() { - return Method::Identity; - } - #[cfg(test)] - tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1)); - self.named.get(name).copied().unwrap_or(Method::Identity) - } -} - -impl SecurityHandler { - fn new(dict: EncryptDict, key: Zeroizing>, auth: Auth, recovered_user: Option>>) -> Self { - let methods = Methods::new(&dict); - Self { dict, key, auth, recovered_user, methods } - } - - /// Authenticate with `password` (tried as owner password, then user password). `None` - /// tries the empty password, which opens documents that only have an owner password. - pub fn open(dict: EncryptDict, id0: &[u8], password: Option<&str>) -> Result { - if !dict.filter.is_empty() && dict.filter != b"Standard" { - return Err(CryptError::Unsupported(format!("/Filter /{}", String::from_utf8_lossy(&dict.filter)))); - } - let pw = password.unwrap_or(""); - let mut recovered_user = None; - let (key, auth) = match dict.r { - 2..=4 => { - let mut found = None; - for legacy in legacy_candidates(pw) { - if let Some((k, user)) = owner_key_r4(&dict, id0, &legacy) { - recovered_user = Some(user); - found = Some((k, Auth::Owner)); - } else if let Some(k) = user_key_r4(&dict, id0, &legacy) { - found = Some((k, Auth::User)); - } - if found.is_some() { - break; - } - } - found.ok_or(CryptError::WrongPassword)? - } - 5 | 6 => { - let pw = sasl_password(pw); - if let Some(k) = key_r6(&dict, &pw, true)? { - (k, Auth::Owner) - } else if let Some(k) = key_r6(&dict, &pw, false)? { - (k, Auth::User) - } else { - return Err(CryptError::WrongPassword); - } - } - r => return Err(CryptError::Unsupported(format!("revision {r}"))), - }; - Ok(Self::new(dict, key, auth, recovered_user)) - } - - /// A handler with no file key, for a document opened without a password whose strings and - /// streams aren't encrypted and whose encrypted crypt filters are all ones the caller found - /// authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO 32000-2 §7.6.6, - /// Table 25). Data such a filter encrypts is left as stored, read and written alike, so it - /// keeps its encrypted bytes; [`SecurityHandler::decrypts`] tells which. `None` when strings or - /// streams are encrypted, which needs the key, or the handler is older than crypt filters (V4). - pub fn without_key(dict: EncryptDict) -> Option { - if dict.v < 4 { - return None; - } - let methods = Methods::new(&dict); - if methods.string != Method::Identity || methods.stream != Method::Identity { - return None; - } - Some(Self { dict, key: Zeroizing::new(Vec::new()), auth: Auth::User, recovered_user: None, methods }) - } - - /// Whether the handler has the file key: false for [`SecurityHandler::without_key`]. - pub fn has_key(&self) -> bool { - !self.key.is_empty() - } - - /// Whether data of `kind` is encrypted, so that reading it needs the file key. - pub fn decrypts(&self, kind: StreamKind) -> bool { - self.method(Some(kind)) != Method::Identity - } - - /// Whether data left as stored by a handler without the key still decrypts after its object - /// is renumbered: only AES-256 doesn't mix the object number into the key (Algorithm 1.A). - pub fn survives_renumbering(&self) -> bool { - [self.methods.string, self.methods.stream, self.methods.embedded_file] - .into_iter() - .chain(self.methods.named.values().copied()) - .all(|m| matches!(m, Method::Identity | Method::Aes256)) - } - - /// Upgrade to owner access with the owner (permissions) password, as when a restricted - /// document is unlocked after opening. The user password, or any other, is `WrongPassword`. - /// Both passwords unlock the same file key, so what was decrypted so far stays valid; a - /// different key is refused too. - pub fn authenticate_owner(&self, id0: &[u8], password: &str) -> Result { - let owner = Self::open(self.dict.clone(), id0, Some(password))?; - if owner.auth != Auth::Owner || owner.key != self.key { - return Err(CryptError::WrongPassword); - } - Ok(owner) - } - - pub fn auth(&self) -> Auth { - self.auth - } - - /// For R2–R4 documents opened with the owner password: the user password, which owner - /// authentication recovers (Algorithm 7). Lets components that only accept user passwords - /// open the document too. - pub fn recovered_user_password(&self) -> Option>> { - self.recovered_user.clone() - } - - /// For R5/R6, `/P` is stored in clear and again in `/Perms`, encrypted with the file key - /// (Algorithm 13). Where the two differ, a permission counts only when both grant it, so an - /// edited `/P` grants nothing extra. Without a readable `/Perms`, `/P` applies as before. - pub fn permissions(&self) -> Permissions { - let bits = self.perms_p().map_or(self.dict.p, |p| self.dict.p & p); - Permissions { bits, owner: self.auth == Auth::Owner, revision: self.dict.r } - } - - /// `/P` as `/Perms` holds it: AES-256 ECB of one block (= CBC with a zero IV), marked `adb`. - fn perms_p(&self) -> Option { - if !matches!(self.dict.r, 5 | 6) || !self.has_key() { - return None; - } - let plain = Zeroizing::new(aes_cbc_decrypt(&self.key, &[0; 16], self.dict.perms.first_chunk::<16>()?, false)); - let p = i32::from_le_bytes(*plain.first_chunk::<4>()?); - (plain.get(9..12) == Some(&b"adb"[..])).then_some(p) - } - - pub fn dict(&self) -> &EncryptDict { - &self.dict - } - - /// The file encryption key (for tests and diagnostics). - pub fn file_key(&self) -> &[u8] { - &self.key - } - - /// Whether `/Metadata` streams are encrypted. - pub fn encrypts_metadata(&self) -> bool { - self.dict.encrypt_metadata - } - - /// The method for strings or a kind of stream. - fn method(&self, stream: Option) -> Method { - if self.dict.v < 4 { - return Method::Rc4; - } - match stream { - None => self.methods.string, - Some(StreamKind::Normal) => self.methods.stream, - Some(StreamKind::EmbeddedFile) => self.methods.embedded_file, - Some(StreamKind::Named(n)) => self.methods.get(n), - } - } - - /// Algorithm 1 (§7.6.3.2): the key for one object. AES-256 uses the file key directly - /// (Algorithm 1.A, §7.6.3.3). - /// - /// For an RC4 crypt filter under `/V 4`, a file key shorter than 16 bytes is first extended - /// to 16 bytes with zeros. The standard doesn't say so (Algorithm 1 takes the n-byte key as - /// it is), but that is how Acrobat writes and reads such files: a `/V 4` RC4 crypt filter - /// with a 40- or 48-bit key (pdf.js issue 19484). The renderer's `hayro-syntax` reads them - /// the same way, so a page it shows is one this crate can read and save. AES-128 keys are - /// left as they are: `AESV2` takes a 128-bit key (Table 25), and Acrobat's extension is only - /// seen with RC4. Every key PdfCraft writes under `/V 4` is 16 bytes already. - fn object_key(&self, method: Method, num: u32, generation: u16) -> Zeroizing> { - if method == Method::Aes256 { - return self.key.clone(); - } - let mut key = self.key.clone(); - if method == Method::Rc4 && self.dict.v == 4 && key.len() < 16 { - key.resize(16, 0); - } - let mut h = Md5::new(); - h.update(&key); - h.update(&num.to_le_bytes()[..3]); - h.update(generation.to_le_bytes()); - if method == Method::Aes128 { - h.update(b"sAlT"); - } - let mut digest = Zeroizing::new(h.finalize().to_vec()); - digest.truncate((key.len() + 5).min(16)); - digest - } - - fn apply(&self, method: Method, num: u32, generation: u16, data: &[u8], encrypt: bool) -> Vec { - // Without the key, encrypted data stays as stored (see `without_key`). - if !self.has_key() { - return data.to_vec(); - } - match method { - Method::Identity => data.to_vec(), - Method::Rc4 => rc4(&self.object_key(method, num, generation), data), - Method::Aes128 | Method::Aes256 => { - let key = self.object_key(method, num, generation); - if encrypt { - // Deterministic IV derived from the key, object and content: saves are - // reproducible and no IV repeats for different plaintexts. - let mut h = Sha256::new(); - h.update(&key); - h.update(num.to_le_bytes()); - h.update(generation.to_le_bytes()); - h.update(data); - let digest = h.finalize(); - let mut iv = [0u8; 16]; - for (d, s) in iv.iter_mut().zip(digest.iter()) { - *d = *s; - } - let mut out = iv.to_vec(); - out.extend(aes_cbc_encrypt(&key, &iv, data, true)); - out - } else { - let Some((iv, body)) = data.split_first_chunk::<16>() else { - return Vec::new(); // only (part of) an IV: empty plaintext - }; - aes_cbc_decrypt(&key, iv, body, true) - } - } - } - } - - pub fn decrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec { - self.apply(self.method(None), num, generation, data, false) - } - - pub fn encrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec { - self.apply(self.method(None), num, generation, data, true) - } - - pub fn decrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec { - self.apply(self.method(Some(kind)), num, generation, data, false) - } - - pub fn encrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec { - self.apply(self.method(Some(kind)), num, generation, data, true) - } - - /// Whether strings are encrypted at all (false with `/StrF /Identity`). - pub fn encrypts_strings(&self) -> bool { - self.method(None) != Method::Identity - } -} - -// ── Passwords ────────────────────────────────────────────────────────────────────────────────── - -/// PDFDocEncoding (ISO 32000-2 Annex D) where it differs from Latin-1: bytes 0x18–0x1F and -/// 0x80–0xA0 (0x9F is undefined). Here rather than in pdfcraft-cos, which decodes text strings -/// with it, because R2–R4 passwords need it too and cos depends on this crate. -const PDFDOC_LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜']; -const PDFDOC_HIGH: [char; 33] = [ - '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł', 'œ', - 'š', 'ž', '\u{FFFD}', '€', -]; - -/// PDFDocEncoding → Unicode. -pub fn pdfdoc_char(c: u8) -> char { - match c { - 0x18..=0x1F => PDFDOC_LOW[usize::from(c - 0x18)], - 0x80..=0xA0 => PDFDOC_HIGH[usize::from(c - 0x80)], - _ => char::from(c), - } -} - -/// Unicode → PDFDocEncoding, `None` for a character it lacks (ğ, ş, İ, Cyrillic, CJK, …). -pub fn pdfdoc_byte(c: char) -> Option { - let at = |table: &[char], base: u8| table.iter().position(|&t| t == c && t != '\u{FFFD}').and_then(|i| u8::try_from(i).ok()).map(|i| base + i); - at(&PDFDOC_LOW, 0x18).or_else(|| at(&PDFDOC_HIGH, 0x80)).or_else(|| match u32::from(c) { - 0x18..=0x1F | 0x80..=0xA0 => None, - n => u8::try_from(n).ok(), - }) -} - -/// Revisions 2–4: the password in PDFDocEncoding (§7.6.4.3.2, Algorithm 2), at most 32 bytes. -/// Characters PDFDocEncoding lacks are dropped. -fn legacy_password(pw: &str) -> Zeroizing> { - Zeroizing::new(pw.chars().filter_map(pdfdoc_byte).take(32).collect()) -} - -/// The bytes an R2–R4 password may have been written as: PDFDocEncoding, as the specification -/// asks; and for a password with characters PDFDocEncoding lacks (ğ, ş, İ, …), which it gives -/// no bytes, also its UTF-8 bytes, which qpdf and other tools write for such passwords. -fn legacy_candidates(pw: &str) -> Vec>> { - let mut out = vec![legacy_password(pw)]; - if pw.chars().any(|c| pdfdoc_byte(c).is_none()) { - out.push(Zeroizing::new(pw.bytes().take(32).collect())); - } - out -} - -/// Revisions 5–6: SASLprep (RFC 4013), UTF-8, at most 127 bytes. -fn sasl_password(pw: &str) -> Zeroizing> { - let prepped = Zeroizing::new(stringprep::saslprep(pw).map(|c| c.into_owned()).unwrap_or_else(|_| pw.to_string())); - let mut bytes = Zeroizing::new(prepped.as_bytes().to_vec()); - bytes.truncate(127); - bytes -} - -fn pad(pw: &[u8]) -> Zeroizing<[u8; 32]> { - let mut out = Zeroizing::new(PADDING); - let n = pw.len().min(32); - out[..n].copy_from_slice(&pw[..n]); - out[n..].copy_from_slice(&PADDING[..32 - n]); - out -} - -fn key_len(d: &EncryptDict) -> usize { - if d.r == 2 { 5 } else { (d.length_bits.clamp(40, 128) / 8) as usize } -} - -// ── Revisions 2–4 ────────────────────────────────────────────────────────────────────────────── - -/// Algorithm 2: file key from a (user) password. -fn file_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Zeroizing> { - let n = key_len(d); - let mut h = Md5::new(); - h.update(*pad(pw)); - h.update(&d.o[..d.o.len().min(32)]); - h.update(d.p.to_le_bytes()); - h.update(id0); - if d.r >= 4 && !d.encrypt_metadata { - h.update([0xFF; 4]); - } - let mut digest = Zeroizing::new(h.finalize().to_vec()); - if d.r >= 3 { - for _ in 0..50 { - digest = Zeroizing::new(Md5::digest(&digest[..n]).to_vec()); - } - } - digest.truncate(n); - digest -} - -/// Algorithms 4/5: the `/U` value a key produces. -fn u_value_r4(d: &EncryptDict, id0: &[u8], key: &[u8]) -> Vec { - if d.r == 2 { - return rc4(key, &PADDING); - } - let mut h = Md5::new(); - h.update(PADDING); - h.update(id0); - let mut x = rc4(key, &h.finalize()); - for i in 1..=19u8 { - let k = Zeroizing::new(key.iter().map(|b| b ^ i).collect::>()); - x = rc4(&k, &x); - } - x.extend_from_slice(&[0u8; 16]); - x -} - -/// Algorithm 6: authenticate a user password; returns the file key. -/// -/// Some producers ignore `/EncryptMetadata false` when deriving the key (they omit the -/// `FF FF FF FF` of Algorithm 2, step f). Viewers open those files, so if the strict derivation -/// fails we retry the other way. -fn user_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option>> { - let check = |d: &EncryptDict| { - let key = file_key_r4(d, id0, pw); - let u = u_value_r4(d, id0, &key); - let n = if d.r == 2 { 32 } else { 16 }; - (d.u.len() >= n && u[..n] == d.u[..n]).then_some(key) - }; - check(d).or_else(|| (d.r >= 4 && !d.encrypt_metadata).then(|| check(&EncryptDict { encrypt_metadata: true, ..d.clone() })).flatten()) -} - -/// The RC4 key derived from an owner password (Algorithm 3, steps a–d). -fn owner_rc4_key(d: &EncryptDict, owner_pw: &[u8]) -> Zeroizing> { - let n = key_len(d); - let mut digest = Zeroizing::new(Md5::digest(*pad(owner_pw)).to_vec()); - if d.r >= 3 { - for _ in 0..50 { - digest = Zeroizing::new(Md5::digest(&*digest).to_vec()); - } - } - digest.truncate(n); - digest -} - -type SecretBytes = Zeroizing>; - -/// Algorithm 7: authenticate an owner password (recover the user password from `/O`). -/// Returns the file key and the recovered user password (padding removed). -fn owner_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option<(SecretBytes, SecretBytes)> { - let rk = owner_rc4_key(d, pw); - let mut user = Zeroizing::new(d.o[..d.o.len().min(32)].to_vec()); - if d.r == 2 { - user = Zeroizing::new(rc4(&rk, &user)); - } else { - for i in (0..=19u8).rev() { - let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::>()); - user = Zeroizing::new(rc4(&k, &user)); - } - } - let key = user_key_r4(d, id0, &user)?; - // The recovered value is the padded password: strip the padding suffix. - let len = (0..=32).find(|&i| user.len() >= 32 && user[i..32] == PADDING[..32 - i]).unwrap_or(user.len().min(32)); - user.truncate(len); - Some((key, user)) -} - -// ── Revisions 5–6 ────────────────────────────────────────────────────────────────────────────── - -/// The hash of a password for R5 (SHA-256) or R6 (Algorithm 2.B). -fn hash_r6(r: i64, pw: &[u8], salt: &[u8], udata: &[u8]) -> Zeroizing> { - let mut k = Zeroizing::new({ - let mut h = Sha256::new(); - h.update(pw); - h.update(salt); - h.update(udata); - h.finalize().to_vec() - }); - if r == 5 { - return k; - } - let mut round = 0u32; - loop { - let mut k1 = Zeroizing::new(Vec::with_capacity(64 * (pw.len() + k.len() + udata.len()))); - for _ in 0..64 { - k1.extend_from_slice(pw); - k1.extend_from_slice(&k); - k1.extend_from_slice(udata); - } - // `k` is always a SHA-2 digest (32, 48 or 64 bytes): key = its first 16 bytes, IV = the next 16. - let Some((key, rest)) = k.split_first_chunk::<16>() else { break }; - let Some(iv) = rest.first_chunk::<16>() else { break }; - let e = Zeroizing::new(aes_cbc_encrypt(key, iv, &k1, false)); - let m = e.iter().take(16).map(|b| u32::from(*b)).sum::() % 3; - k = Zeroizing::new(match m { - 0 => Sha256::digest(&*e).to_vec(), - 1 => Sha384::digest(&*e).to_vec(), - _ => Sha512::digest(&*e).to_vec(), - }); - round += 1; - if round >= 64 && e.last().is_none_or(|b| u32::from(*b) <= round - 32) { - break; - } - } - k.truncate(32); - k -} - -/// Algorithm 2.A: authenticate as owner (`owner = true`) or user; returns the file key. -fn key_r6(d: &EncryptDict, pw: &[u8], owner: bool) -> Result>>, CryptError> { - if d.u.len() < 48 || d.o.len() < 48 { - return Err(CryptError::Malformed("/O and /U must be 48 bytes for R5/R6".into())); - } - let (hashed, encrypted, udata) = if owner { (&d.o[..48], &d.oe, &d.u[..48]) } else { (&d.u[..48], &d.ue, &[][..]) }; - if *hash_r6(d.r, pw, &hashed[32..40], udata) != hashed[..32] { - return Ok(None); - } - if encrypted.len() < 32 { - return Err(CryptError::Malformed("/OE or /UE must be 32 bytes".into())); - } - let k = hash_r6(d.r, pw, &hashed[40..48], udata); - Ok(Some(Zeroizing::new(aes_cbc_decrypt(&k, &[0; 16], &encrypted[..32], false)))) -} - -// ── Creating encryption ──────────────────────────────────────────────────────────────────────── - -/// Which encryption to create. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum Algorithm { - /// R2, 40-bit RC4 (legacy; for tests and compatibility only). - Rc4_40, - /// R3, 128-bit RC4. - Rc4_128, - /// R4, AES-128 crypt filters. - Aes128, - /// R6, AES-256 (PDF 2.0; the default for new documents). - Aes256, -} - -/// A password as [`create`] prepares it: for R2–R4 its PDFDocEncoding (characters it lacks -/// dropped), at most 32 bytes; for R6 its SASLprep (RFC 4013: soft hyphens and the like removed, -/// other spaces made plain, then NFKC; the password as given if SASLprep refuses it), in UTF-8, -/// at most 127 bytes. -fn prepared_password(algorithm: Algorithm, password: &str) -> Zeroizing> { - match algorithm { - Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => legacy_password(password), - Algorithm::Aes256 => sasl_password(password), - } -} - -/// The bytes new encryption with `algorithm` derives its keys from for `password`: for R2–R4 its -/// PDFDocEncoding cut or padded to exactly 32 bytes (Algorithm 2); for R6 its SASLprep (the -/// password as given if SASLprep refuses it), at most 127 bytes of UTF-8. Two passwords with the -/// same bytes are the same password to the file, and a password with the bytes of `""` opens it -/// without asking, so callers can refuse either before encrypting. Different bytes can still give -/// the same key (RC4-40 keeps 5 bytes of a hash). `create` still uses the user password for an -/// empty R2–R4 owner password. -pub fn effective_password(algorithm: Algorithm, password: &str) -> Vec { - let prepared = prepared_password(algorithm, password); - match algorithm { - Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => pad(&prepared).to_vec(), - Algorithm::Aes256 => prepared.to_vec(), - } -} - -/// Parameters for new encryption. -#[derive(Clone, Debug)] -pub struct NewEncryption<'a> { - pub algorithm: Algorithm, - pub user_password: &'a str, - pub owner_password: &'a str, - pub permissions: i32, - pub encrypt_metadata: bool, - /// 32 bytes of entropy for keys and salts (the caller supplies randomness, so this crate - /// needs no OS access and tests are deterministic). - pub seed: [u8; 32], -} - -/// Expand `seed` into `n` bytes (SHA-256 in counter mode). -fn expand(seed: &[u8; 32], label: &[u8], n: usize) -> Vec { - let mut out = Vec::new(); - let mut counter = 0u32; - while out.len() < n { - let mut h = Sha256::new(); - h.update(seed); - h.update(label); - h.update(counter.to_le_bytes()); - out.extend_from_slice(&h.finalize()); - counter = counter.saturating_add(1); - } - out.truncate(n); - out -} - -/// Sixteen bytes of file identifier from `seed` (SHA-256 in counter mode, label `id`). -/// -/// `/ID` is written in the clear. Deriving it the same way as the file key keeps that public -/// value from being an unkeyed hash of the seed. -pub fn file_id(seed: &[u8; 32]) -> Vec { - expand(seed, b"id", 16) -} - -/// Build the `/Encrypt` values and an authenticated (owner) handler for a new encryption. -pub fn create(params: &NewEncryption, id0: &[u8]) -> Result { - // Bits 7–8 and 13–32 must be 1; bits 1–2 must be 0 (§7.6.4.2). - let p = (params.permissions | !0xFFF | 0xC0) & !0x3; - let mut d = EncryptDict { filter: b"Standard".to_vec(), p, encrypt_metadata: params.encrypt_metadata, ..Default::default() }; - let user = prepared_password(params.algorithm, params.user_password); - let owner = prepared_password(params.algorithm, params.owner_password); - match params.algorithm { - Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => { - (d.v, d.r, d.length_bits) = match params.algorithm { - Algorithm::Rc4_40 => (1, 2, 40), - Algorithm::Rc4_128 => (2, 3, 128), - _ => (4, 4, 128), - }; - if params.algorithm == Algorithm::Aes128 { - d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)]; - d.stm_f = b"StdCF".to_vec(); - d.str_f = b"StdCF".to_vec(); - } else { - d.encrypt_metadata = true; // V < 4 always encrypts metadata - } - let owner = if params.owner_password.is_empty() { user.clone() } else { owner }; - // Algorithm 3: /O - let rk = owner_rc4_key(&d, &owner); - let mut o = rc4(&rk, &*pad(&user)); - if d.r >= 3 { - for i in 1..=19u8 { - let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::>()); - o = rc4(&k, &o); - } - } - d.o = o; - let key = file_key_r4(&d, id0, &user); - d.u = u_value_r4(&d, id0, &key); - Ok(SecurityHandler::new(d, key, Auth::Owner, Some(user))) - } - Algorithm::Aes256 => { - (d.v, d.r, d.length_bits) = (5, 6, 256); - d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes256)]; - d.stm_f = b"StdCF".to_vec(); - d.str_f = b"StdCF".to_vec(); - let key = Zeroizing::new(expand(¶ms.seed, b"file key", 32)); - let salts = expand(¶ms.seed, b"salts", 32); - // Algorithm 8: /U, /UE - let mut u = hash_r6(6, &user, &salts[0..8], &[]).to_vec(); - u.extend_from_slice(&salts[0..16]); - let uk = hash_r6(6, &user, &salts[8..16], &[]); - d.ue = aes_cbc_encrypt(&uk, &[0; 16], &key, false); - // Algorithm 9: /O, /OE - let mut o = hash_r6(6, &owner, &salts[16..24], &u).to_vec(); - o.extend_from_slice(&salts[16..32]); - let ok = hash_r6(6, &owner, &salts[24..32], &u); - d.oe = aes_cbc_encrypt(&ok, &[0; 16], &key, false); - d.u = u; - d.o = o; - // Algorithm 10: /Perms - let mut perms = [0u8; 16]; - perms[..4].copy_from_slice(&p.to_le_bytes()); - perms[4..8].copy_from_slice(&[0xFF; 4]); - perms[8] = if d.encrypt_metadata { b'T' } else { b'F' }; - perms[9..12].copy_from_slice(b"adb"); - perms[12..16].copy_from_slice(&expand(¶ms.seed, b"perms", 4)); - d.perms = aes_cbc_encrypt(&key, &[0; 16], &perms, false); - Ok(SecurityHandler::new(d, key, Auth::Owner, None)) - } - } -} - -#[cfg(test)] -mod tests; +//! PDF standard security handler (`craft-pdf::crypt`). +pub use craft_pdf::crypt::*; diff --git a/crates/crypt/src/rc4.rs b/crates/crypt/src/rc4.rs deleted file mode 100644 index 952a8394d5..0000000000 --- a/crates/crypt/src/rc4.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! RC4 (used by PDF security handler revisions 2–4). Encryption and decryption are the same. - -pub fn rc4(key: &[u8], data: &[u8]) -> Vec { - if key.is_empty() { - return data.to_vec(); - } - let mut s: [u8; 256] = std::array::from_fn(|i| i as u8); - let mut j: u8 = 0; - for i in 0..256 { - j = j.wrapping_add(s[i]).wrapping_add(key[i % key.len()]); - s.swap(i, j as usize); - } - let (mut i, mut j) = (0u8, 0u8); - data.iter() - .map(|b| { - i = i.wrapping_add(1); - j = j.wrapping_add(s[i as usize]); - s.swap(i as usize, j as usize); - b ^ s[s[i as usize].wrapping_add(s[j as usize]) as usize] - }) - .collect() -} diff --git a/crates/crypt/src/tests.rs b/crates/crypt/src/tests.rs deleted file mode 100644 index 39853f33ad..0000000000 --- a/crates/crypt/src/tests.rs +++ /dev/null @@ -1,447 +0,0 @@ -use super::*; - -fn hex(s: &str) -> Vec { - (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() -} - -#[test] -fn file_id_is_a_one_way_16_byte_identifier() { - let seed = [7u8; 32]; - let id = file_id(&seed); - assert_eq!(id.len(), 16); - assert_eq!(id, file_id(&seed)); - assert_ne!(id, seed[..16]); - assert_ne!(id, file_id(&[8u8; 32])); -} - -#[test] -fn rc4_reference_vectors() { - assert_eq!(rc4(b"Key", b"Plaintext"), hex("bbf316e8d940af0ad3")); - assert_eq!(rc4(b"Wiki", b"pedia"), hex("1021bf0420")); - assert_eq!(rc4(b"Secret", b"Attack at dawn"), hex("45a01f645fc35b383552544b9bf5")); -} - -#[test] -fn aes_cbc_nist_sp800_38a_vectors() { - let iv: [u8; 16] = hex("000102030405060708090a0b0c0d0e0f").try_into().unwrap(); - let plain = hex("6bc1bee22e409f96e93d7e117393172aae2d8a571e03ac9c9eb76fac45af8e51"); - let k128 = hex("2b7e151628aed2a6abf7158809cf4f3c"); - let c128 = aes_cbc_encrypt(&k128, &iv, &plain, false); - assert_eq!(c128, hex("7649abac8119b246cee98e9b12e9197d5086cb9b507219ee95db113a917678b2")); - assert_eq!(aes_cbc_decrypt(&k128, &iv, &c128, false), plain); - let k256 = hex("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"); - let c256 = aes_cbc_encrypt(&k256, &iv, &plain, false); - assert_eq!(c256, hex("f58c4c04d6e5f1ba779eabfb5f7bfbd69cfc4e967edb808d679f777bc6702c7d")); - assert_eq!(aes_cbc_decrypt(&k256, &iv, &c256, false), plain); -} - -#[test] -fn aes_padding_is_added_and_tolerantly_removed() { - let key = [7u8; 16]; - let iv = [1u8; 16]; - for n in [0, 1, 15, 16, 17, 100] { - let data: Vec = (0..n as u8).collect(); - let c = aes_cbc_encrypt(&key, &iv, &data, true); - assert_eq!(c.len() % 16, 0); - assert_eq!(aes_cbc_decrypt(&key, &iv, &c, true), data); - } - // Garbage (bad padding, partial block) decrypts without panicking. - let _ = aes_cbc_decrypt(&key, &iv, &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], true); -} - -const ID: &[u8] = b"0123456789abcdef"; - -fn new(alg: Algorithm, user: &str, owner: &str, perms: i32) -> SecurityHandler { - create( - &NewEncryption { algorithm: alg, user_password: user, owner_password: owner, permissions: perms, encrypt_metadata: true, seed: [42; 32] }, - ID, - ) - .unwrap() -} - -const ALL: [Algorithm; 4] = [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256]; - -#[test] -fn created_encryption_authenticates_owner_and_user() { - for alg in ALL { - let made = new(alg, "user pw", "owner pw", 0b0100); // print only - let d = made.dict().clone(); - let owner = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap(); - assert_eq!(owner.auth(), Auth::Owner, "{alg:?}"); - assert_eq!(owner.file_key(), made.file_key(), "{alg:?}"); - assert!(owner.permissions().modify(), "owners can do everything"); - let user = SecurityHandler::open(d.clone(), ID, Some("user pw")).unwrap(); - assert_eq!(user.auth(), Auth::User, "{alg:?}"); - assert_eq!(user.file_key(), made.file_key(), "{alg:?}"); - assert!(user.permissions().print() && !user.permissions().modify() && !user.permissions().copy(), "{alg:?}"); - assert_eq!(SecurityHandler::open(d.clone(), ID, Some("nope")).unwrap_err(), CryptError::WrongPassword, "{alg:?}"); - assert_eq!(SecurityHandler::open(d, ID, None).unwrap_err(), CryptError::WrongPassword, "{alg:?}"); - } -} - -#[test] -fn the_owner_password_upgrades_a_user_session() { - for alg in ALL { - let d = new(alg, "", "secret", 0b0100).dict().clone(); - let user = SecurityHandler::open(d, ID, None).unwrap(); - assert!(!user.permissions().modify(), "{alg:?}"); - // The (empty) user password and wrong ones don't unlock anything. - assert_eq!(user.authenticate_owner(ID, "").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); - assert_eq!(user.authenticate_owner(ID, "nope").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); - let owner = user.authenticate_owner(ID, "secret").unwrap(); - assert_eq!(owner.auth(), Auth::Owner, "{alg:?}"); - assert_eq!(owner.file_key(), user.file_key(), "{alg:?}"); - assert!(owner.permissions().modify(), "{alg:?}"); - } - // A separate open password is not the permissions password. - for alg in ALL { - let d = new(alg, "open", "perms", 0b0100).dict().clone(); - let user = SecurityHandler::open(d, ID, Some("open")).unwrap(); - assert_eq!(user.authenticate_owner(ID, "open").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); - assert_eq!(user.authenticate_owner(ID, "perms").unwrap().auth(), Auth::Owner, "{alg:?}"); - } -} - -#[test] -fn empty_user_password_opens_without_asking() { - for alg in ALL { - let d = new(alg, "", "secret", -1).dict().clone(); - let h = SecurityHandler::open(d, ID, None).unwrap(); - assert_eq!(h.auth(), Auth::User, "{alg:?}"); - } -} - -#[test] -fn strings_and_streams_round_trip_and_are_scrambled() { - for alg in ALL { - let h = new(alg, "", "o", -1); - let text = b"Confidential quarterly numbers".to_vec(); - let enc = h.encrypt_string(12, 0, &text); - assert_ne!(enc, text, "{alg:?}"); - assert_eq!(h.decrypt_string(12, 0, &enc), text, "{alg:?}"); - if alg != Algorithm::Aes256 { - // R2–R4 derive a key per object; R6 uses the file key everywhere (§7.6.3.3). - assert_ne!(h.decrypt_string(13, 0, &enc), text, "object keys differ per object ({alg:?})"); - } - let data: Vec = (0..1000u32).map(|i| (i * 7) as u8).collect(); - let enc = h.encrypt_stream(5, 0, &data, StreamKind::Normal); - assert_eq!(h.decrypt_stream(5, 0, &enc, StreamKind::Normal), data, "{alg:?}"); - // Deterministic: the same object encrypts to the same bytes (reproducible saves). - assert_eq!(h.encrypt_stream(5, 0, &data, StreamKind::Normal), enc); - } -} - -thread_local! { - /// Crypt filter names compared or looked up, on this thread. - pub(crate) static FILTER_COMPARES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -#[test] -fn crypt_filters_are_looked_up_once_per_handler() { - let plain = new(Algorithm::Aes128, "", "o", -1); - let mut d = plain.dict().clone(); - let n = 5_000; - let mut filters: Vec<_> = (0..n).map(|i| (format!("F{i}").into_bytes(), Method::Identity)).collect(); - filters.append(&mut d.crypt_filters); - d.crypt_filters = filters; - FILTER_COMPARES.with(|c| c.set(0)); - let h = SecurityHandler::open(d, ID, None).unwrap(); - let strings = 1_000; - let enc = plain.encrypt_string(7, 0, b"secret"); - for _ in 0..strings { - assert_eq!(h.decrypt_string(7, 0, &enc), b"secret"); - assert_eq!(h.decrypt_stream(7, 0, &plain.encrypt_stream(7, 0, b"s", StreamKind::Normal), StreamKind::Normal), b"s"); - } - assert_eq!(h.decrypt_stream(7, 0, b"x", StreamKind::Named(b"F3")), b"x"); - let compares = FILTER_COMPARES.with(|c| c.get()); - assert!(compares <= 2 * n + 3 * strings, "{compares} filter name comparisons"); -} - -#[test] -fn identity_crypt_filters_leave_data_alone() { - let mut d = new(Algorithm::Aes256, "", "o", -1).dict().clone(); - // Embedded-file-only encryption: strings and normal streams are not encrypted. - d.str_f = b"Identity".to_vec(); - d.stm_f = b"Identity".to_vec(); - d.ef_f = b"StdCF".to_vec(); - let h = SecurityHandler::open(d, ID, None).unwrap(); - assert!(!h.encrypts_strings()); - assert_eq!(h.decrypt_stream(3, 0, b"plain", StreamKind::Normal), b"plain"); - let enc = h.encrypt_stream(3, 0, b"attachment", StreamKind::EmbeddedFile); - assert_ne!(enc, b"attachment"); - assert_eq!(h.decrypt_stream(3, 0, &enc, StreamKind::EmbeddedFile), b"attachment"); - assert_eq!(h.decrypt_stream(3, 0, b"x", StreamKind::Named(b"Identity")), b"x"); -} - -#[test] -fn r6_passwords_are_saslprepped() { - // U+00AD (soft hyphen) is mapped to nothing by SASLprep; "ª" normalises to "a" (NFKC). - let d = new(Algorithm::Aes256, "SªSL\u{AD}prep", "o", -1).dict().clone(); - assert!(SecurityHandler::open(d.clone(), ID, Some("SaSLprep")).is_ok()); - assert!(SecurityHandler::open(d, ID, Some("SaSL-prep")).is_err()); -} - -#[test] -fn effective_passwords_are_the_bytes_the_keys_are_made_from() { - // AES-256: SASLprep drops soft hyphens, makes other spaces plain and folds compatibility - // characters (NFKC); at most 127 bytes of UTF-8. - let r6 = |pw: &str| effective_password(Algorithm::Aes256, pw); - assert_eq!(r6("\u{AD}"), r6(""), "a soft hyphen alone is no password"); - assert_eq!(r6("\u{FF21}"), r6("A"), "fullwidth A"); - assert_eq!(r6("\u{FB01}"), r6("fi"), "the fi ligature"); - assert_eq!(r6("a\u{3000}b"), b"a b", "an ideographic space"); - assert_ne!(r6("A"), r6("a"), "case matters"); - assert_eq!(r6(&"é".repeat(100)).len(), 127); - // RC4 and AES-128: PDFDocEncoding, padded or cut to exactly 32 bytes. - let (a31, a32) = ("a".repeat(31), "a".repeat(32)); - for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { - assert_eq!(effective_password(alg, &format!("{a32}1")), a32.as_bytes(), "{alg:?}"); - assert_eq!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("1{a32}2")), "{alg:?}"); - assert_ne!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("2{a32}")), "{alg:?}"); - assert_eq!(effective_password(alg, &a31), effective_password(alg, &format!("{a31}(")), "{alg:?}: the padding starts with '('"); - assert_eq!(effective_password(alg, ""), PADDING, "{alg:?}"); - assert_eq!(effective_password(alg, "ılık€")[..5], [0x9A, b'l', 0x9A, b'k', 0xA0], "{alg:?}"); - } -} - -#[test] -fn passwords_with_the_same_effective_bytes_are_one_password() { - // A file protected with "\u{FF21}" (fullwidth A) opens with "A". - let d = new(Algorithm::Aes256, "\u{FF21}", "owner", 0b0100).dict().clone(); - assert_eq!(SecurityHandler::open(d.clone(), ID, Some("A")).unwrap().auth(), Auth::User); - assert_eq!(SecurityHandler::open(d, ID, Some("a")).unwrap_err(), CryptError::WrongPassword); - // A soft hyphen is no open password at all. - let d = new(Algorithm::Aes256, "\u{AD}", "owner", 0b0100).dict().clone(); - assert_eq!(SecurityHandler::open(d, ID, None).unwrap().auth(), Auth::User); - // Open and permissions passwords with the same bytes: the open password gives owner rights. - let d = new(Algorithm::Aes256, "\u{FB01}", "fi", 0b0100).dict().clone(); - assert_eq!(effective_password(Algorithm::Aes256, "\u{FB01}"), effective_password(Algorithm::Aes256, "fi")); - assert_eq!(SecurityHandler::open(d, ID, Some("\u{FB01}")).unwrap().auth(), Auth::Owner); - let a31 = "a".repeat(31); - let pairs = [(format!("{a31}a1"), format!("{a31}a2")), (a31.clone(), format!("{a31}("))]; - for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { - for (open, perms) in &pairs { - assert_eq!(effective_password(alg, open), effective_password(alg, perms), "{alg:?}"); - let d = new(alg, open, perms, 0b0100).dict().clone(); - assert_eq!(SecurityHandler::open(d, ID, Some(open)).unwrap().auth(), Auth::Owner, "{alg:?} {open:?}"); - } - } -} - -#[test] -fn unsupported_handlers_are_reported() { - let mut d = new(Algorithm::Aes128, "", "o", -1).dict().clone(); - d.filter = b"Adobe.PubSec".to_vec(); - assert!(matches!(SecurityHandler::open(d.clone(), ID, None), Err(CryptError::Unsupported(_)))); - d.filter = b"Standard".to_vec(); - d.r = 7; - assert!(matches!(SecurityHandler::open(d, ID, None), Err(CryptError::Unsupported(_)))); -} - -#[test] -fn permission_bits_follow_table_22() { - let p = Permissions { bits: -3904, owner: false, revision: 3 }; // 0xFFFFF0C0: only reserved bits set - assert!(!p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.assemble()); - let p = Permissions { bits: -1, owner: false, revision: 3 }; - assert!(p.print() && p.modify() && p.copy() && p.annotate() && p.assemble() && p.fill_forms()); - let p = Permissions { bits: 0, owner: true, revision: 3 }; - assert!(p.modify(), "owners ignore /P"); -} - -/// Page extraction (#5081) is assembly as the revision reads it, and copying (bit 5). -#[test] -fn page_extraction_needs_assembly_and_copying() { - let bit = |n: u32| 1i32 << (n - 1); - let user = |bits: i32, revision: i64| Permissions { bits, owner: false, revision }; - // "Any except extracting pages" (bits 4, 6, 9; copying off) at revision 3 and later. - assert!(!user(bit(4) | bit(6) | bit(9), 4).extract_pages()); - // Pages may change in place, but copying is withheld. - assert!(user(bit(11), 6).assemble() && !user(bit(11), 6).extract_pages()); - assert!(user(bit(11) | bit(5), 6).extract_pages()); - // Revision 2: bit 4 covers assembly, and its reserved bit 11 grants nothing. - assert!(user(bit(4), 2).assemble() && !user(bit(4) | bit(11), 2).extract_pages(), "copying withheld"); - assert!(user(bit(4) | bit(5), 2).extract_pages()); - assert!(!user(bit(5) | bit(11), 2).extract_pages(), "no assembly without bit 4 at revision 2"); - assert!(Permissions { bits: 0, owner: true, revision: 2 }.extract_pages(), "owners ignore /P"); -} - -#[test] -fn perms_limits_what_an_edited_p_grants() { - let made = new(Algorithm::Aes256, "user pw", "owner pw", 0b0100); // print only - let open = |d: &EncryptDict, pw| SecurityHandler::open(d.clone(), ID, Some(pw)).unwrap().permissions(); - let mut d = made.dict().clone(); - assert_eq!(open(&d, "user pw").bits, d.p, "/P and /Perms agree"); - d.p = -4; // /P edited to allow everything, /Perms as written - let p = open(&d, "user pw"); - assert!(p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.fill_forms() && !p.assemble()); - assert!(open(&d, "owner pw").modify(), "owners ignore /P"); - d.p = -3904; // a stricter /P still applies - assert!(!open(&d, "user pw").print()); - // No /Perms, or one that does not decrypt to an `adb` block: /P as before. - for perms in [Vec::new(), vec![0; 16], made.dict().perms[..15].to_vec()] { - let d = EncryptDict { p: -4, perms, ..made.dict().clone() }; - assert!(open(&d, "user pw").modify()); - } - // R2 to R4 have no /Perms: /P is part of their file key, so an edited /P does not open. - let d = EncryptDict { p: -4, ..new(Algorithm::Aes128, "user pw", "owner pw", 0b0100).dict().clone() }; - assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap_err(), CryptError::WrongPassword); -} - -proptest::proptest! { - #[test] - fn malformed_dictionaries_never_panic(o in proptest::collection::vec(proptest::num::u8::ANY, 0..60), - u in proptest::collection::vec(proptest::num::u8::ANY, 0..60), - r in 0i64..9, len in -10i64..300) { - let d = EncryptDict { filter: b"Standard".to_vec(), v: 4, r, length_bits: len, o, u, p: -1, encrypt_metadata: true, ..Default::default() }; - let _ = SecurityHandler::open(d, ID, Some("x")); - } -} - -/// A `/V 4` RC4 crypt filter with a 40- or 48-bit key, as Acrobat writes it (pdf.js issue 19484): -/// the file key is extended to 16 bytes with zeros before Algorithm 1, so each object's key is -/// all 16 bytes of MD5(key, zeros, object, generation). Everything else keeps Algorithm 1 as the -/// standard has it, n + 5 bytes (at most 16) of MD5(key, object, generation[, "sAlT"]): a `/V 2` -/// file with the same short key, a full 128-bit `/V 4` RC4 key, and AESV2, short (88 bits) or -/// full. Each object key is computed here by hand; data written with it must decrypt. -#[test] -fn a_v4_rc4_key_shorter_than_16_bytes_is_extended_with_zeros() { - let text = b"Page tree in an object stream".to_vec(); - let cases = - [(4, 40, Method::Rc4), (4, 48, Method::Rc4), (2, 40, Method::Rc4), (4, 128, Method::Rc4), (4, 88, Method::Aes128), (4, 128, Method::Aes128)]; - for (v, bits, method) in cases { - let mut d = EncryptDict { - filter: b"Standard".to_vec(), - v, - r: if v == 4 { 4 } else { 3 }, - length_bits: bits, - o: vec![7; 32], - p: -3392, - encrypt_metadata: v != 4, - ..Default::default() - }; - if v == 4 { - d.crypt_filters = vec![(b"StdCF".to_vec(), method)]; - d.stm_f = b"StdCF".to_vec(); - d.str_f = b"StdCF".to_vec(); - } - let key = file_key_r4(&d, ID, b""); - assert_eq!(key.len(), (bits / 8) as usize); - d.u = u_value_r4(&d, ID, &key); - let h = SecurityHandler::open(d, ID, None).unwrap(); - let (num, generation) = (13u32, 0u16); - let extended = v == 4 && method == Method::Rc4 && key.len() < 16; - let mut md5 = Md5::new(); - md5.update(&key); - if extended { - md5.update(vec![0u8; 16 - key.len()]); - } - md5.update(&num.to_le_bytes()[..3]); - md5.update(generation.to_le_bytes()); - if method == Method::Aes128 { - md5.update(b"sAlT"); - } - let digest = md5.finalize(); - let n = if extended { 16 } else { key.len() }; - let object_key = digest[..(n + 5).min(16)].to_vec(); - let case = format!("V{v}, {bits}-bit key, {method:?}"); - let written = if method == Method::Rc4 { - rc4(&object_key, &text) - } else { - let iv = [3u8; 16]; - [iv.to_vec(), aes_cbc_encrypt(&object_key, &iv, &text, true)].concat() - }; - assert_eq!(h.decrypt_string(num, generation, &written), text, "{case}"); - assert_eq!(h.decrypt_stream(num, generation, &written, StreamKind::Normal), text, "{case}"); - let saved = h.encrypt_stream(num, generation, &text, StreamKind::Normal); - if method == Method::Rc4 { - assert_eq!(saved, written, "saved as it was read: {case}"); - } - assert_eq!(h.decrypt_stream(num, generation, &saved, StreamKind::Normal), text, "{case}"); - } -} - -#[test] -fn owner_authentication_recovers_the_user_password() { - for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { - let d = new(alg, "user pw", "owner pw", -1).dict().clone(); - let h = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap(); - let _: &Zeroizing> = &h.key; - let _: &Option>> = &h.recovered_user; - assert_eq!(h.recovered_user_password().as_deref().map(Vec::as_slice), Some(&b"user pw"[..]), "{alg:?}"); - assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap().recovered_user_password(), None); - } - let d = new(Algorithm::Rc4_128, "", "owner", -1).dict().clone(); - assert_eq!(SecurityHandler::open(d, ID, Some("owner")).unwrap().recovered_user_password().as_deref().map(Vec::as_slice), Some(&b""[..])); -} - -/// /O, /U and the first /ID of files qpdf 12.4.2 wrote (`--static-id`), with the file keys -/// `qpdf --show-encryption-key` reports. -fn qpdf_file(r: i64, o: &str, u: &str) -> EncryptDict { - let mut d = EncryptDict { - filter: b"Standard".to_vec(), - v: if r == 4 { 4 } else { 2 }, - r, - length_bits: 128, - o: hex(o), - u: hex(u), - p: -4, - encrypt_metadata: true, - ..Default::default() - }; - if r == 4 { - d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)]; - d.stm_f = b"StdCF".to_vec(); - d.str_f = b"StdCF".to_vec(); - } - d -} - -const QPDF_ID: &str = "56b57383aafa8314f3f98222995f5604"; - -#[test] -fn legacy_passwords_are_pdfdoc_encoded_with_a_utf8_fallback() { - let id = hex(QPDF_ID); - // R3 RC4: user "ılık€" in PDFDocEncoding (ı = 0x9A, € = 0xA0); owner "Öğretmen", which - // PDFDocEncoding can't hold, so qpdf wrote its UTF-8 bytes. - let r3 = qpdf_file( - 3, - "d519aa8120ce8c54812b45e5391916fed167b1c84961d90c4ee4c182ad83534b", - "4985c03ddfa3eeb8a90c343512913ec40021446990b9e4114071a4d9104984c1", - ); - let user = SecurityHandler::open(r3.clone(), &id, Some("ılık€")).unwrap(); - assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("3564ffcea186122595ad55ee157da71f"))); - let owner = SecurityHandler::open(r3.clone(), &id, Some("Öğretmen")).unwrap(); - assert_eq!((owner.auth(), owner.file_key().to_vec()), (Auth::Owner, hex("3564ffcea186122595ad55ee157da71f"))); - assert_eq!(owner.recovered_user_password(), Some(Zeroizing::new(vec![0x9A, b'l', 0x9A, b'k', 0xA0]))); - // Dropping ı and € (the old Latin-1 conversion) or a near miss doesn't open it. - for wrong in ["lk", "ilik€", "ılık", "Ogretmen"] { - assert_eq!(SecurityHandler::open(r3.clone(), &id, Some(wrong)).unwrap_err(), CryptError::WrongPassword, "{wrong}"); - } - // R4 AES-128: user "şifre" (UTF-8 bytes), owner "owner". - let r4 = qpdf_file( - 4, - "bb4f8f4bd2236569e765906caf64e4429a4c20d6e996fdef963e9b5080f9e083", - "4a965fb91882772cd09ce8bbff2c33270021446990b9e4114071a4d9104984c1", - ); - let user = SecurityHandler::open(r4.clone(), &id, Some("şifre")).unwrap(); - assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("9f19d465cfd8e525d32aa0bdf10acd06"))); - assert_eq!(SecurityHandler::open(r4.clone(), &id, Some("owner")).unwrap().auth(), Auth::Owner); - assert_eq!(SecurityHandler::open(r4, &id, Some("ifre")).unwrap_err(), CryptError::WrongPassword); -} - -#[test] -fn pdfdoc_encoding_round_trips() { - for b in 0..=255u8 { - if b != 0x9F { - assert_eq!(pdfdoc_byte(pdfdoc_char(b)), Some(b), "{b:#04x}"); - } - } - assert_eq!(pdfdoc_byte('ı'), Some(0x9A)); - assert_eq!(pdfdoc_byte('€'), Some(0xA0)); - assert_eq!(pdfdoc_byte('ç'), Some(0xE7)); - for c in ['ğ', 'ş', 'İ', '\u{A0}', '\u{85}', 'Ж', '日'] { - assert_eq!(pdfdoc_byte(c), None, "{c}"); - } - assert_eq!(legacy_candidates("Çok"), [Zeroizing::new(vec![0xC7, b'o', b'k'])]); - assert_eq!(legacy_candidates("şifre"), [Zeroizing::new(b"ifre".to_vec()), Zeroizing::new("şifre".as_bytes().to_vec())]); -} diff --git a/crates/export/Cargo.toml b/crates/export/Cargo.toml index dc0998c036..155b8a25af 100644 --- a/crates/export/Cargo.toml +++ b/crates/export/Cargo.toml @@ -11,6 +11,7 @@ repository.workspace = true workspace = true [dependencies] +craft-rtf.workspace = true craft-zip.workspace = true [dev-dependencies] diff --git a/crates/export/src/as_rtf.rs b/crates/export/src/as_rtf.rs index 61585d82fb..6e9e98d07e 100644 --- a/crates/export/src/as_rtf.rs +++ b/crates/export/src/as_rtf.rs @@ -4,24 +4,8 @@ use super::layout::{Block, Item, Page, count_lookup_steps, items}; use std::collections::HashMap; fn rtf_text(s: &str) -> String { - let mut o = String::new(); - for c in s.chars().filter(|c| allowed(*c)) { - match c { - '\\' | '{' | '}' => { - o.push('\\'); - o.push(c); - } - c if (c as u32) < 128 => o.push(c), - c => { - // \uN with a ? fallback; UTF-16 units as signed 16-bit numbers. - let mut buf = [0u16; 2]; - for u in c.encode_utf16(&mut buf) { - o.push_str(&format!("\\u{}?", *u as i16)); - } - } - } - } - o + let cleaned: String = s.chars().filter(|c| allowed(*c)).collect(); + craft_rtf::esc(&cleaned) } /// Rich Text Format: paragraphs with their sizes and bold/italic, real table rows, page breaks diff --git a/crates/filters/Cargo.toml b/crates/filters/Cargo.toml index bfb930fe5f..39342ece97 100644 --- a/crates/filters/Cargo.toml +++ b/crates/filters/Cargo.toml @@ -11,12 +11,7 @@ repository.workspace = true workspace = true [dependencies] -fax.workspace = true -# CCITT fax and JBIG2 decoding (`decode_ccitt`, `decode_jbig2`); already in the tree through hayro. -hayro-ccitt.workspace = true -hayro-jbig2.workspace = true -flate2.workspace = true -thiserror.workspace = true +craft-pdf = { workspace = true } [dev-dependencies] proptest.workspace = true diff --git a/crates/filters/src/ascii85.rs b/crates/filters/src/ascii85.rs deleted file mode 100644 index ff3b392198..0000000000 --- a/crates/filters/src/ascii85.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! ASCII85Decode (ISO 32000-2 §7.4.3): groups of five characters `!`..`u` encode four -//! bytes base-85, `z` stands for four zero bytes, `~>` is EOD, white space is ignored and -//! a final partial group of n characters (2–5) yields n − 1 bytes. A leading `<~` and a -//! missing EOD are tolerated. - -use crate::{Failure, Step, is_pdf_whitespace}; - -const NAME: &str = "ASCII85Decode"; - -fn push(out: &mut Vec, bytes: &[u8], max: usize) -> Result<(), Failure> { - if out.len() + bytes.len() > max { - return Err(Failure::limit(max)); - } - out.extend_from_slice(bytes); - Ok(()) -} - -pub(crate) fn decode(data: &[u8], max: usize) -> Step { - let start = data.iter().position(|&b| !is_pdf_whitespace(b)).unwrap_or(data.len()); - let body = data[start..].strip_prefix(b"<~").unwrap_or(&data[start..]); - let mut out = Vec::with_capacity((body.len() / 5 * 4 + 4).min(max)); - let mut acc: u64 = 0; - let mut n = 0usize; - for (i, &b) in body.iter().enumerate() { - match b { - b'!'..=b'u' => { - acc = acc * 85 + u64::from(b - b'!'); - n += 1; - if n == 5 { - let Ok(v) = u32::try_from(acc) else { - return Err(Failure::corrupt(NAME, format!("group ending at {i} exceeds 2^32"), out)); - }; - push(&mut out, &v.to_be_bytes(), max)?; - acc = 0; - n = 0; - } - } - b'z' if n == 0 => push(&mut out, &[0; 4], max)?, - b'z' => return Err(Failure::corrupt(NAME, format!("'z' inside a group at {i}"), out)), - // `~` starts the EOD marker; tolerate it without the following `>`. - b'~' => break, - b if is_pdf_whitespace(b) => {} - b => return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at {i}"), out)), - } - } - match n { - 0 => {} - 1 => return Err(Failure::corrupt(NAME, "final group has a single character", out)), - _ => { - // Pad with `u` (84) to a full group and keep n − 1 bytes. - for _ in n..5 { - acc = acc * 85 + 84; - } - let Ok(v) = u32::try_from(acc) else { - return Err(Failure::corrupt(NAME, "final group exceeds 2^32", out)); - }; - push(&mut out, &v.to_be_bytes()[..n - 1], max)?; - } - } - Ok(out) -} - -/// Encodes with `z` for zero groups, a newline every 75 characters, and a `~>` EOD. -pub(crate) fn encode(data: &[u8]) -> Vec { - let mut out = Vec::with_capacity(data.len() * 5 / 4 + data.len() / 60 + 8); - let mut col = 0; - let mut put = |out: &mut Vec, s: &[u8]| { - for &c in s { - if col == 75 { - out.push(b'\n'); - col = 0; - } - out.push(c); - col += 1; - } - }; - for chunk in data.chunks(4) { - let mut word = [0u8; 4]; - word[..chunk.len()].copy_from_slice(chunk); - let mut v = u32::from_be_bytes(word); - if v == 0 && chunk.len() == 4 { - put(&mut out, b"z"); - continue; - } - let mut digits = [0u8; 5]; - for d in digits.iter_mut().rev() { - *d = (v % 85) as u8 + b'!'; - v /= 85; - } - put(&mut out, &digits[..chunk.len() + 1]); - } - out.extend_from_slice(b"~>"); - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - - fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1000).map_err(|f| f.error) - } - - #[test] - fn vectors() { - // "Man " = 0x4D616E20 = 24·85⁴ + 73·85³ + 80·85² + 78·85 + 61 -> "9jqo^". - assert_eq!(dec(b"9jqo^~>").unwrap(), b"Man "); - assert_eq!(encode(b"Man "), b"9jqo^~>"); - // "M" padded to 0x4D000000 -> "9`" (24, 63); decoding pads with 'u'. - assert_eq!(encode(b"M"), b"9`~>"); - assert_eq!(dec(b"9`~>").unwrap(), b"M"); - assert_eq!(dec(b"z~>").unwrap(), [0; 4]); - assert_eq!(encode(&[0; 4]), b"z~>"); - // A partial zero group is not `z`. - assert_eq!(encode(&[0; 3]), b"!!!!~>"); - assert_eq!(dec(b"!!!!~>").unwrap(), [0; 3]); - assert_eq!(dec(b"s8W-!~>").unwrap(), [0xff; 4]); - } - - #[test] - fn syntax_tolerance() { - assert_eq!(dec(b"<~9jqo^~>").unwrap(), b"Man "); - assert_eq!(dec(b" \n<~9j\r\nq o\t^z~>").unwrap(), b"Man \0\0\0\0"); - assert_eq!(dec(b"9jqo^").unwrap(), b"Man "); // missing EOD - assert_eq!(dec(b"9jqo^~").unwrap(), b"Man "); // bare ~ - assert_eq!(dec(b"9jqo^~>garbage{").unwrap(), b"Man "); - assert_eq!(dec(b"~>").unwrap(), b""); - assert_eq!(dec(b"").unwrap(), b""); - } - - #[test] - fn corrupt() { - assert!(matches!(dec(b"9jz~>"), Err(FilterError::Corrupt { .. }))); - assert!(matches!(dec(b"9jqo^9~>"), Err(FilterError::Corrupt { .. }))); - assert!(matches!(dec(b"uuuuu~>"), Err(FilterError::Corrupt { .. }))); // > 2^32 - 1 - assert!(matches!(dec(b"9jqo^v~>"), Err(FilterError::Corrupt { .. }))); - let f = decode(b"9jqo^{", 100).err().unwrap(); - assert_eq!(f.partial, b"Man "); - } - - #[test] - fn limit() { - assert!(matches!(decode(b"zz", 7).map_err(|f| f.error), Err(FilterError::LimitExceeded(7)))); - assert!(matches!(decode(b"z9`", 4).map_err(|f| f.error), Err(FilterError::LimitExceeded(4)))); - assert_eq!(decode(b"zz", 8).ok().unwrap().len(), 8); - } - - #[test] - fn line_wrapping() { - let data: Vec = (0..=255u8).collect(); - let e = encode(&data); - assert!(e.split(|&b| b == b'\n').all(|l| l.len() <= 77)); - assert_eq!(dec(&e).unwrap_or_default().len(), 256); - } -} diff --git a/crates/filters/src/asciihex.rs b/crates/filters/src/asciihex.rs deleted file mode 100644 index 5ba0574e8c..0000000000 --- a/crates/filters/src/asciihex.rs +++ /dev/null @@ -1,106 +0,0 @@ -//! ASCIIHexDecode (ISO 32000-2 §7.4.2): pairs of hex digits, white space ignored, -//! `>` is EOD, an odd final digit is padded with 0. A missing EOD is tolerated. - -use crate::{Failure, Step, is_pdf_whitespace}; - -const NAME: &str = "ASCIIHexDecode"; - -fn nibble(b: u8) -> Option { - match b { - b'0'..=b'9' => Some(b - b'0'), - b'a'..=b'f' => Some(b - b'a' + 10), - b'A'..=b'F' => Some(b - b'A' + 10), - _ => None, - } -} - -pub(crate) fn decode(data: &[u8], max: usize) -> Step { - let mut out = Vec::with_capacity((data.len() / 2).min(max)); - let mut hi: Option = None; - for (i, &b) in data.iter().enumerate() { - if is_pdf_whitespace(b) { - continue; - } - if b == b'>' { - break; - } - let Some(n) = nibble(b) else { - return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at offset {i}"), out)); - }; - match hi.take() { - None => hi = Some(n), - Some(h) => { - if out.len() == max { - return Err(Failure::limit(max)); - } - out.push(h << 4 | n); - } - } - } - if let Some(h) = hi { - if out.len() == max { - return Err(Failure::limit(max)); - } - out.push(h << 4); - } - Ok(out) -} - -/// Upper-case hex, a newline every 64 digits, terminated by `>`. -pub(crate) fn encode(data: &[u8]) -> Vec { - const HEX: &[u8; 16] = b"0123456789ABCDEF"; - let mut out = Vec::with_capacity(data.len() * 2 + data.len() / 32 + 1); - for (i, &b) in data.iter().enumerate() { - if i > 0 && i % 32 == 0 { - out.push(b'\n'); - } - out.push(HEX[usize::from(b >> 4)]); - out.push(HEX[usize::from(b & 15)]); - } - out.push(b'>'); - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - - fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1000).map_err(|f| f.error) - } - - #[test] - fn vectors() { - assert_eq!(dec(b"48656c6C6F>").unwrap(), b"Hello"); - assert_eq!(dec(b" 4 8\n6\r5\t6\x0c c\x006c 6f >").unwrap(), b"Hello"); - assert_eq!(dec(b"901FA>").unwrap(), [0x90, 0x1f, 0xa0]); - assert_eq!(dec(b"901FA").unwrap(), [0x90, 0x1f, 0xa0]); // missing EOD - assert_eq!(dec(b"41>zz garbage after EOD").unwrap(), b"A"); - assert_eq!(dec(b">").unwrap(), b""); - assert_eq!(dec(b"").unwrap(), b""); - } - - #[test] - fn invalid_char() { - let f = decode(b"4142G43>", 1000).err().unwrap(); - assert!(matches!(f.error, FilterError::Corrupt { .. })); - assert_eq!(f.partial, b"AB"); - } - - #[test] - fn limit() { - assert!(matches!(decode(b"414243", 2).map_err(|f| f.error), Err(FilterError::LimitExceeded(2)))); - assert!(matches!(decode(b"41424", 2).map_err(|f| f.error), Err(FilterError::LimitExceeded(2)))); - assert_eq!(decode(b"4142", 2).ok().unwrap(), b"AB"); - } - - #[test] - fn encoding() { - assert_eq!(encode(b"Hello"), b"48656C6C6F>"); - assert_eq!(encode(b""), b">"); - let long = encode(&[0xab; 40]); - assert_eq!(long[64], b'\n'); - assert_eq!(dec(&long).unwrap(), [0xab; 40]); - } -} diff --git a/crates/filters/src/ccitt.rs b/crates/filters/src/ccitt.rs deleted file mode 100644 index 2a038568ee..0000000000 --- a/crates/filters/src/ccitt.rs +++ /dev/null @@ -1,547 +0,0 @@ -//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`, -//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the -//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give. - -use fax::encoder::Encoder; -use fax::{Color, VecWriter}; - -use crate::FilterError; - -/// The widest row [`encode_g4`] encodes. The encoder counts the positions in a row in 16 bits -/// and adds two of them, so a row stays within half that range. -pub const MAX_G4_COLUMNS: u32 = 32_767; - -/// `rows` rows of `columns` 1-bit samples, most significant bit first, each row starting on a new -/// byte (as a PDF image's samples are packed), encoded as CCITT Group 4 with an end-of-block mark. -/// -/// A 0 bit is coded black and a 1 bit white, so the stream decodes to the same bits with -/// `/BlackIs1 false`, the default: an image keeps its colour space and decode array. Bytes after -/// the last row are ignored, as are the bits that pad a row to a whole byte. -/// -/// Errors: no rows or columns, or rows wider than [`MAX_G4_COLUMNS`] ([`FilterError::Unsupported`]); -/// fewer bytes than the rows need ([`FilterError::Corrupt`]). -pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result, FilterError> { - let width = match u16::try_from(columns) { - Ok(w) if w > 0 && columns <= MAX_G4_COLUMNS => w, - _ => return Err(FilterError::Unsupported(format!("CCITT Group 4 encoding of rows {columns} samples wide"))), - }; - if rows == 0 { - return Err(FilterError::Unsupported("CCITT Group 4 encoding of an image without rows".into())); - } - let stride = usize::from(width).div_ceil(8); - let samples = usize::try_from(rows).ok().and_then(|r| r.checked_mul(stride)).and_then(|n| data.get(..n)).ok_or_else(|| FilterError::Corrupt { - filter: "CCITTFaxDecode", - detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()), - })?; - let mut encoder = Encoder::new(VecWriter::new()); - for row in samples.chunks_exact(stride) { - let pels = (0..usize::from(width)).map(|x| { - let white = row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); - if white { Color::White } else { Color::Black } - }); - // `VecWriter` writes to memory: its error type has no values. - if let Err(never) = encoder.encode_line(pels, width) { - match never {} - } - } - match encoder.finish() { - Ok(writer) => Ok(writer.finish()), - Err(never) => match never {}, - } -} - -/// The widest row [`decode_ccitt_within`] decodes: its row buffer stays within 128 KiB. -pub const MAX_CCITT_COLUMNS: u32 = 1 << 20; - -/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the -/// table's defaults. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct CcittParams { - /// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed. - pub k: i64, - pub columns: i64, - /// `/Rows`: 0 when not given. - pub rows: i64, - pub end_of_line: bool, - pub encoded_byte_align: bool, - pub end_of_block: bool, - pub black_is_1: bool, -} - -impl Default for CcittParams { - fn default() -> Self { - CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false } - } -} - -/// What [`decode_ccitt_within`] decodes once it is paid for. -fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result, FilterError> { - let columns = u32::try_from(p.columns) - .ok() - .filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c)) - .ok_or_else(|| FilterError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?; - let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); - if rows == 0 { - return Ok(Vec::new()); - } - let settings = hayro_ccitt::DecodeSettings { - columns, - rows, - end_of_block: p.end_of_block, - end_of_line: p.end_of_line, - // Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter - // always accepts them), with the fill starting each code on a byte or ending it on one (T.4's - // fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without - // codes, the rows are read on bytes. A reading that comes up short isn't tried the other - // way: data cut short would then read as other rows. - rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)), - encoding: match p.k { - ..0 => hayro_ccitt::EncodingMode::Group4, - 0 => hayro_ccitt::EncodingMode::Group3_1D, - k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) }, - }, - invert_black: false, - }; - // u32 to usize: wasm32 and up have at least 32-bit usize. - let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1); - // The rows decoded before damaged data are kept. - let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)); - Ok(out.out) -} - -/// The steps of reading a byte of fax code ([`decode_ccitt_within`]): noise takes 114 ns a byte of -/// rows and about two bytes of code to a byte of rows, so that and a step a byte of rows is about -/// what a step of JBIG2 decoding takes. -const CODE_WORK: u64 = 4; - -/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most -/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`). -/// -/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by -/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a -/// caller that needs a number of rows checks it got them. -/// -/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes, -/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4 -/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever -/// `end_of_line` says. Group 4 is read as its parameters say. Data with codes but none before the -/// first row is read on bytes, as PdfCraft's renderer draws it; with `k` 0 and the fill ending each -/// code on a byte, that may be other rows than were coded. -/// -/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`FilterError::Unsupported`]). -/// -/// Decoding spends from `work`, what decoding may still cost across a run of images, and there is -/// no way in without it: a -/// byte of rows a step, whether or not the data holds them, and [`CODE_WORK`] a byte of `data`. -/// A blank row takes a bit of code, so the rows are what a few bytes can ask for. It is all paid -/// before anything is decoded, and what `work` can't pay for spends all of it. -pub fn decode_ccitt_within(data: &[u8], p: &CcittParams, max_rows: u32, work: &mut u64) -> Result, FilterError> { - let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); - let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8); - crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?; - decode_ccitt(data, p, max_rows) -} - -/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill -/// included. T.4 sends one before the first row when it sends them at all, and no other code starts -/// with more than eight zeros. -fn starts_with_end_of_line(data: &[u8]) -> bool { - let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false }; - i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11 -} - -/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at -/// most `rows` rows. Pixels past a row's end and rows past the last are dropped. The JBIG2 decoder -/// ([`crate::decode_jbig2_within`]) fills them too. -pub(crate) struct Rows { - pub(crate) out: Vec, - row: Vec, - x: usize, - columns: usize, - rows: usize, - done: usize, - black_is_1: bool, -} - -impl Rows { - pub(crate) fn new(columns: usize, rows: usize, black_is_1: bool) -> Self { - Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 } - } - - /// `count` pixels, white or black. - pub(crate) fn push(&mut self, white: bool, count: usize) { - let end = self.x.saturating_add(count); - if white != self.black_is_1 { - for x in self.x..end.min(self.columns) { - if let Some(b) = self.row.get_mut(x / 8) { - *b |= 0x80 >> (x % 8); - } - } - } - self.x = end; - } - - /// The row filled so far kept, if fewer than `rows` are, and the next one begun. - pub(crate) fn end_row(&mut self) { - if self.done < self.rows { - self.out.extend_from_slice(&self.row); - self.done += 1; - } - self.row.fill(0); - self.x = 0; - } -} - -impl hayro_ccitt::Decoder for Rows { - fn push_pixel(&mut self, white: bool) { - self.push(white, 1); - } - fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { - self.push(white, (chunks as usize).saturating_mul(8)); - } - fn next_line(&mut self) { - self.end_row(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A run pays for the rows an image asks for and the code it reads, before any of it is - /// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number. - #[test] - fn a_run_of_images_pays_for_the_rows_it_asks_for() { - let (w, h) = (4096u32, 64u32); - let stride = (w as usize).div_ceil(8); - let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap(); - let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() }; - let whole = decode_ccitt(&data, &p, h).unwrap(); - let cost = (whole.len() + 4 * data.len()) as u64; - let mut work = cost; - assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough"); - assert_eq!(work, 0); - let mut work = cost + 7; - decode_ccitt_within(&data, &p, h, &mut work).unwrap(); - assert_eq!(work, 7, "only what it took"); - let mut work = cost - 1; - let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err(); - assert!(matches!(refused, FilterError::Spent), "a step short: {refused}"); - assert_eq!(work, 0, "spends all that was left"); - // Nothing left: not read at all, so bad columns aren't reported as bad columns. - let bad = CcittParams { columns: 0, ..p }; - assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| matches!(e, FilterError::Spent))); - // No code at all, for the same rows. - let mut work = u64::MAX; - decode_ccitt_within(&[], &p, h, &mut work).unwrap(); - assert_eq!(u64::MAX - work, whole.len() as u64); - // `/Rows` below what is asked for, and no rows. - let mut work = u64::MAX; - decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap(); - assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64); - let mut work = u64::MAX; - decode_ccitt_within(&data, &p, 0, &mut work).unwrap(); - assert_eq!(u64::MAX - work, 4 * data.len() as u64); - } - - /// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black. - pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec> { - struct Rows(Vec>, Vec, usize); - impl hayro_ccitt::Decoder for Rows { - fn push_pixel(&mut self, white: bool) { - self.1.push(white); - } - fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { - self.1.extend(std::iter::repeat_n(white, chunks as usize * 8)); - } - fn next_line(&mut self) { - let mut row = std::mem::take(&mut self.1); - row.truncate(self.2); - self.0.push(row); - } - } - let settings = hayro_ccitt::DecodeSettings { - columns, - rows, - end_of_block: true, - end_of_line: false, - rows_are_byte_aligned: false, - encoding: hayro_ccitt::EncodingMode::Group4, - invert_black: false, - }; - let mut out = Rows(Vec::new(), Vec::new(), columns as usize); - hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap(); - out.0 - } - - /// The rows of packed samples `data`, as 1 for a 1 bit. - pub(crate) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { - let stride = (columns as usize).div_ceil(8); - (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() - } - - #[test] - fn a_page_of_text_like_marks_round_trips_and_shrinks() { - let (w, h) = (1700u32, 220u32); - let stride = (w as usize).div_ceil(8); - let mut data = vec![0xFF; stride * h as usize]; - // Short black strokes in lines, like glyphs on a white page. - for y in (20..h as usize).step_by(40) { - for dy in 0..12 { - for x in (100..1600).step_by(9) { - if (x / 9 + dy) % 5 != 0 { - data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); - } - } - } - } - let g4 = encode_g4(&data, w, h).unwrap(); - assert_eq!(decode(&g4, w, h), unpack(&data, w, h)); - assert!(g4.len() * 4 < data.len(), "{} bytes from {}", g4.len(), data.len()); - } - - #[test] - fn padding_bits_and_trailing_bytes_are_ignored() { - // 3 columns: the last 5 bits of each byte pad the row; set them to anything. - let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; - let g4 = encode_g4(&data, 3, 3).unwrap(); - assert_eq!(decode(&g4, 3, 3), [[true, false, true], [false, true, false], [true, true, true]]); - } - - #[test] - fn what_it_cannot_encode_is_an_error() { - assert!(matches!(encode_g4(&[0; 8], 0, 1), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode_g4(&[0; 8], 8, 0), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode_g4(&vec![0; 4097], MAX_G4_COLUMNS + 1, 1), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode_g4(&[0; 3], 16, 2), Err(FilterError::Corrupt { .. })), "two rows of 16 need 4 bytes"); - assert!(matches!(encode_g4(&[], 1, u32::MAX), Err(FilterError::Corrupt { .. }))); - let widest = vec![0x55; (MAX_G4_COLUMNS as usize).div_ceil(8) * 2]; - let g4 = encode_g4(&widest, MAX_G4_COLUMNS, 2).unwrap(); - assert_eq!(decode(&g4, MAX_G4_COLUMNS, 2), unpack(&widest, MAX_G4_COLUMNS, 2)); - } - - /// Rows of `columns` samples built from `runs`: alternating white and black runs one longer - /// than those values (so never empty, even when shrinking makes them all 0), repeated until the - /// rows are full; or, with `noise`, the bytes of `runs` as they are. - fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { - let stride = (columns as usize).div_ceil(8); - let mut data = vec![0u8; stride * rows as usize]; - if runs.is_empty() { - return data; - } - if noise { - data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); - return data; - } - let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); - for y in 0..rows as usize { - for x in 0..columns as usize { - if left == 0 { - white = !white; - left = 1 + usize::from(runs[next % runs.len()]); - next += 1; - } - if white { - data[y * stride + x / 8] |= 0x80 >> (x % 8); - } - left -= 1; - } - } - data - } - - fn g4(columns: u32) -> CcittParams { - CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() } - } - - #[test] - fn decoding_gives_the_rows_back() { - // 3 columns, padded with anything: the decoded padding is 0. - let data = [0b1010_1111, 0b0101_0000, 0b1110_1010]; - let coded = encode_g4(&data, 3, 3).unwrap(); - assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]); - // /BlackIs1 gives black as 1. - let p = CcittParams { black_is_1: true, ..g4(3) }; - assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]); - // At most /Rows rows, and at most max_rows. - assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2); - assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1); - assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows"); - assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]); - // Damaged data gives the rows before the damage; none here. - assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3); - } - - #[test] - fn what_it_cannot_decode_is_an_error() { - for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] { - let p = CcittParams { columns, ..CcittParams::default() }; - assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(FilterError::Unsupported(_))), "{columns}"); - } - assert_eq!(CcittParams::default().columns, 1728, "Table 11"); - } - - /// Group 3 data: rows of run lengths below 64, white first, coded one-dimensionally with T.4's - /// terminating codes (ITU-T T.4 Tables 2 and 3). Each row starts with an end-of-line code with - /// `eol`, then with `tag` a 1 bit (the next row one-dimensional, as `k` above 0 has it). Fill - /// bits (0s) go where `fill` says. - fn group_3(rows: &[Vec], eol: bool, tag: bool, fill: Fill) -> Vec { - // The codes for runs 0 to 63, in order. - const WHITE: &str = concat!( - "00110101 000111 0111 1000 1011 1100 1110 1111 10011 10100 00111 01000 001000 000011 110100 110101 101010 101011 ", - "0100111 0001100 0001000 0010111 0000011 0000100 0101000 0101011 0010011 0100100 0011000 00000010 00000011 00011010 ", - "00011011 00010010 00010011 00010100 00010101 00010110 00010111 00101000 00101001 00101010 00101011 00101100 00101101 ", - "00000100 00000101 00001010 00001011 01010010 01010011 01010100 01010101 00100100 00100101 01011000 01011001 01011010 ", - "01011011 01001010 01001011 00110010 00110011 00110100", - ); - const BLACK: &str = concat!( - "0000110111 010 11 10 011 0011 0010 00011 000101 000100 0000100 0000101 0000111 00000100 00000111 000011000 0000010111 ", - "0000011000 0000001000 00001100111 00001101000 00001101100 00000110111 00000101000 00000010111 00000011000 ", - "000011001010 000011001011 000011001100 000011001101 000001101000 000001101001 000001101010 000001101011 000011010010 ", - "000011010011 000011010100 000011010101 000011010110 000011010111 000001101100 000001101101 000011011010 000011011011 ", - "000001010100 000001010101 000001010110 000001010111 000001100100 000001100101 000001010010 000001010011 000000100100 ", - "000000110111 000000111000 000000100111 000000101000 000001011000 000001011001 000000101011 000000101100 000001011010 ", - "000001100110 000001100111", - ); - let code = |white: bool, run: u32| (if white { WHITE } else { BLACK }).split(' ').nth(run as usize).expect("a run below 64"); - let mut bits = String::new(); - for row in rows { - // The end-of-line code is 12 bits. - let ends_code = if fill == Fill::CodeEnd && eol { 12 } else { 0 }; - while fill != Fill::None && !(bits.len() + ends_code).is_multiple_of(8) { - bits.push('0'); - } - if eol { - bits.push_str("000000000001"); - } - if tag { - bits.push('1'); - } - row.iter().enumerate().for_each(|(i, run)| bits.push_str(code(i % 2 == 0, *run))); - } - bits.as_bytes().chunks(8).map(|c| c.iter().enumerate().fold(0u8, |b, (i, d)| b | (u8::from(*d == b'1') << (7 - i)))).collect() - } - - /// Where [`group_3`] puts fill bits: none; so each row, its end-of-line code first, starts on a - /// byte; or so each end-of-line code ends on one (T.4's fill, as TIFF writes it). - #[derive(Clone, Copy, Debug, PartialEq)] - enum Fill { - None, - RowStart, - CodeEnd, - } - - /// `rows` text-like rows of `columns` samples as runs, white first, from `seed`: long white runs - /// and short black ones; a row may start black. - fn text_like(seed: u64, columns: u32, rows: u32) -> Vec> { - let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; - let mut random = |n: u64| { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - (seed % n) as u32 - }; - (0..rows) - .map(|_| { - let (mut runs, mut left) = (Vec::new(), columns); - while left > 0 { - let run = match runs.len() % 2 { - 0 if runs.is_empty() => random(31), - 0 => 1 + random(30), - _ => 1 + random(5), - }; - runs.push(run.min(left)); - left -= run.min(left); - } - runs - }) - .collect() - } - - /// Rows of runs, white first, packed as `decode_ccitt` gives them (white 1). - fn pack(runs: &[Vec], columns: u32) -> Vec { - let stride = (columns as usize).div_ceil(8); - let mut out = vec![0u8; stride * runs.len()]; - for (y, row) in runs.iter().enumerate() { - let mut x = 0; - for (i, run) in row.iter().enumerate() { - if i % 2 == 0 { - (x..x + *run as usize).for_each(|x| out[y * stride + x / 8] |= 0x80 >> (x % 8)); - } - x += *run as usize; - } - } - out - } - - /// Group 3 rows on bytes in each layout, over many pictures, as how a misreading comes out - /// depends on the picture: no codes (as TIFF's Modified Huffman has it), and `end_of_line` - /// without them; codes, each starting on a byte or ending on one, `end_of_line` or not. Each - /// decodes to its picture, and a byte short to fewer rows, not to other rows read another way. - /// Group 4 rows on bytes keep their aligned read, `end_of_line` or not. - #[test] - fn group_3_rows_on_bytes_read_by_the_codes_in_the_data() { - for seed in 0..32u64 { - let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20); - let runs = text_like(seed, columns, rows); - let picture = pack(&runs, columns); - for k in [0, 1] { - for (codes, fill, end_of_line) in [ - (false, Fill::RowStart, false), - (false, Fill::RowStart, true), - (true, Fill::RowStart, false), - (true, Fill::RowStart, true), - (true, Fill::CodeEnd, false), - (true, Fill::CodeEnd, true), - ] { - let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align: true, ..CcittParams::default() }; - let mut data = group_3(&runs, codes, k > 0, fill); - let case = format!("seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, end_of_line {end_of_line}"); - assert_eq!(decode_ccitt(&data, &p, rows).unwrap(), picture, "{case}"); - data.pop(); - assert!(decode_ccitt(&data, &p, rows).unwrap().len() < picture.len(), "{case}, a byte short"); - } - } - } - // White rows, each the one code 1 (vertical, no change) and its fill. - for end_of_line in [false, true] { - let p = CcittParams { k: -1, columns: 64, end_of_line, encoded_byte_align: true, ..CcittParams::default() }; - assert_eq!(decode_ccitt(&[0x80; 16], &p, 16).unwrap(), [0xFF; 8 * 16], "Group 4, end_of_line {end_of_line}"); - } - } - - proptest::proptest! { - /// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder - /// and through `decode_ccitt`. - #[test] - fn bitmaps_round_trip( - columns in 1u32..400, - rows in 1u32..24, - runs in proptest::collection::vec(0u8..=255, 1..48), - noise in proptest::bool::ANY, - ) { - let data = bitmap(columns, rows, &runs, noise); - let coded = encode_g4(&data, columns, rows).unwrap(); - proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows)); - let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap(); - proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows)); - } - - /// Whatever it is given, it returns. - #[test] - fn any_input_returns( - data in proptest::collection::vec(0u8..=255, 0..600), - columns in 0u32..40_000, - rows in 0u32..80, - k in -2i64..3, - end_of_line in proptest::bool::ANY, - encoded_byte_align in proptest::bool::ANY, - ) { - let _ = encode_g4(&data, columns, rows); - let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align, ..CcittParams::default() }; - if let Ok(out) = decode_ccitt(&data, &p, rows) { - proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize); - } - } - } -} diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs deleted file mode 100644 index ab7c501488..0000000000 --- a/crates/filters/src/flate.rs +++ /dev/null @@ -1,304 +0,0 @@ -//! FlateDecode (ISO 32000-2 §7.4.4): zlib (RFC 1950) wrapping deflate (RFC 1951). -//! -//! Real-world quirks handled: -//! - the Adler-32 trailer is not required (some producers omit or corrupt it), because the -//! deflate stream is inflated raw after the 2-byte zlib header; -//! - streams without a valid zlib header are inflated as raw deflate; -//! - data after the end of the deflate stream is ignored. -//! -//! A back-reference to before the start of the output points at nothing, and is damage, as zlib -//! has it: read strictly the stream is refused, and read tolerantly what came before it is kept. -//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) So is a dynamic -//! block whose code has no end-of-block symbol, and nothing of that block is kept. - -use std::io::Write; - -use flate2::{Compression, Decompress, FlushDecompress, Status}; - -use crate::{Failure, Step}; - -const NAME: &str = "FlateDecode"; - -/// Checks the RFC 1950 header: CM = 8, CINFO ≤ 7 and the FCHECK multiple-of-31 rule. -fn zlib_header(data: &[u8]) -> Option<(u8, u8)> { - let (&cmf, &flg) = (data.first()?, data.get(1)?); - let ok = cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(cmf) << 8 | u16::from(flg)) % 31 == 0; - ok.then_some((cmf, flg)) -} - -/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked -/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has -/// left for it once it has paid for the second reading. `None`, that reading can't be paid for, -/// and is [`FilterError::Spent`](crate::FilterError::Spent). -pub(crate) fn decode(data: &[u8], max: usize, again: Option) -> (Step, bool) { - match zlib_header(data) { - Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false), - Some(_) => match inflate_raw(&data[2..], max) { - // A header that only looked valid: retry the whole input as raw deflate. - Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => { - let Some(again) = again else { - return (Err(Failure { error: crate::FilterError::Spent, partial: Vec::new(), discarded: 0 }), false); - }; - let step = match inflate_raw(data, again) { - // Keep the original corruption diagnostic only if the retry is corrupt - // too. A limit error must remain fatal even for tolerant decoding. What the - // retry decoded isn't kept, but a caller charging for work has to hear of it. - Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), - retry => retry, - }; - (step, true) - } - r => (r, false), - }, - None => (inflate_raw(data, max), false), - } -} - -#[cfg(test)] -thread_local! { - /// Bytes inflating gave back on this thread, its partial output included. - pub(crate) static INFLATED: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -fn inflate_raw(input: &[u8], max: usize) -> Step { - let step = inflate(input, max); - #[cfg(test)] - INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len))); - step -} - -fn inflate(input: &[u8], max: usize) -> Step { - let mut d = Decompress::new(false); - let mut out: Vec = Vec::new(); - // One byte beyond the limit lets us tell "exactly max" from "more than max". - let cap = max.saturating_add(1); - loop { - if out.len() == out.capacity() { - if out.len() >= cap { - return Err(Failure::limit(max)); - } - let grow = out.capacity().max(input.len().saturating_mul(2)).max(4096).min(cap - out.len()); - out.reserve_exact(grow); - } - let consumed = d.total_in() as usize; - let produced = out.len(); - let res = d.decompress_vec(&input[consumed.min(input.len())..], &mut out, FlushDecompress::None); - if out.len() > max { - return Err(Failure::limit(max)); - } - match res { - Err(e) => return Err(Failure::corrupt(NAME, e.to_string(), out)), - Ok(Status::StreamEnd) => return Ok(out), - Ok(Status::Ok | Status::BufError) => { - let progressed = d.total_in() as usize != consumed || out.len() != produced; - let room_left = out.len() < out.capacity(); - if room_left && (d.total_in() as usize >= input.len() || !progressed) { - return Err(Failure::corrupt(NAME, "unexpected end of deflate data", out)); - } - } - } - } -} - -/// zlib at level 6. -// The documented never-crash exception: writing into a `Vec` cannot fail. -#[allow(clippy::expect_used)] -pub(crate) fn encode(data: &[u8]) -> Vec { - let mut e = flate2::write::ZlibEncoder::new(Vec::with_capacity(data.len() / 2 + 64), Compression::new(6)); - e.write_all(data).expect("in-memory write"); - e.finish().expect("in-memory write") -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - - fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error) - } - - #[test] - fn known_zlib_vector() { - // zlib("a"): header 78 9C, fixed-Huffman block, Adler-32 0x00620062. - assert_eq!(dec(&[0x78, 0x9c, 0x4b, 0x04, 0x00, 0x00, 0x62, 0x00, 0x62]).unwrap(), b"a"); - } - - #[test] - fn hand_built_stored_block() { - // RFC 1951 §3.2.4 stored block: BFINAL=1 BTYPE=00, LEN=5, NLEN=!5, "hello", - // then Adler-32("hello") = 0x062C0215. - let v = [0x78, 0x01, 0x01, 0x05, 0x00, 0xfa, 0xff, b'h', b'e', b'l', b'l', b'o', 0x06, 0x2c, 0x02, 0x15]; - assert_eq!(dec(&v).unwrap(), b"hello"); - // Missing Adler-32 is tolerated. - assert_eq!(dec(&v[..12]).unwrap(), b"hello"); - // Wrong Adler-32 is tolerated (as in other readers). - let mut bad = v; - bad[15] ^= 0xff; - assert_eq!(dec(&bad).unwrap(), b"hello"); - // Trailing garbage after the stream is ignored. - let mut trailing = v.to_vec(); - trailing.extend_from_slice(b"\r\nendstream"); - assert_eq!(dec(&trailing).unwrap(), b"hello"); - } - - #[test] - fn raw_deflate_fallback() { - let data = b"raw deflate without any zlib header, repeated repeated repeated".repeat(4); - let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::new(6)); - e.write_all(&data).unwrap(); - let raw = e.finish().unwrap(); - assert!(zlib_header(&raw).is_none()); - assert_eq!(dec(&raw).unwrap(), data); - } - - #[test] - fn round_trip_and_missing_checksum() { - let data: Vec = (0..50_000u32).map(|i| (i * i % 251) as u8).collect(); - let z = encode(&data); - assert_eq!(&z[..2], &[0x78, 0x9c]); - assert_eq!(dec(&z).unwrap(), data); - assert_eq!(dec(&z[..z.len() - 4]).unwrap(), data); - assert_eq!(dec(&z[..z.len() - 2]).unwrap(), data); - } - - #[test] - fn empty_input() { - assert_eq!(dec(&encode(b"")).unwrap(), b""); - assert!(matches!(dec(b""), Err(FilterError::Corrupt { .. }))); - } - - #[test] - fn corrupt_and_truncated() { - let data: Vec = (0..10_000u32).map(|i| (i % 13) as u8).collect(); - let z = encode(&data); - match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 { - Err(f) => { - assert!(matches!(f.error, FilterError::Corrupt { .. })); - assert_eq!(&data[..f.partial.len()], &f.partial[..]); - } - Ok(_) => panic!("truncated stream decoded"), - } - // BTYPE = 11 is reserved (RFC 1951 §3.2.3). - assert!(matches!(dec(&[0x78, 0x9c, 0x07, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); - // Preset dictionary. - assert!(matches!(dec(&[0x78, 0x20, 0, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); - // One whose header is followed by data that inflates: refused all the same. - assert!(matches!(dec(&[0x78, 0xBB, 0x4b, 0x04, 0x00]), Err(FilterError::Corrupt { .. }))); - } - - #[test] - fn bomb_is_bounded() { - let z = encode(&vec![0u8; 16 << 20]); - assert!(z.len() < 32 * 1024); - assert!(matches!(dec_limit(&z, 1 << 20), Err(FilterError::LimitExceeded(n)) if n == 1 << 20)); - assert_eq!(dec_limit(&z, 16 << 20).unwrap().len(), 16 << 20); - assert!(matches!(dec_limit(&z, (16 << 20) - 1), Err(FilterError::LimitExceeded(_)))); - assert!(matches!(dec_limit(&z, 0), Err(FilterError::LimitExceeded(0)))); - } - - fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { - decode(d, max, Some(max)).0.map_err(|f| f.error) - } - - /// Deflate's bits: header fields and extra bits least significant first, Huffman codes most - /// significant first (RFC 1951 §3.1.1). - #[derive(Default)] - struct Bits { - out: Vec, - acc: u32, - n: u32, - } - - impl Bits { - fn put(&mut self, value: u32, len: u32) { - for i in 0..len { - self.acc |= ((value >> i) & 1) << self.n; - self.n += 1; - if self.n == 8 { - self.out.push(self.acc as u8); - (self.acc, self.n) = (0, 0); - } - } - } - - fn code(&mut self, code: u32, len: u32) { - for i in (0..len).rev() { - self.put((code >> i) & 1, 1); - } - } - - fn finish(mut self) -> Vec { - if self.n > 0 { - self.out.push(self.acc as u8); - } - self.out - } - } - - /// A dynamic block whose literal and length code has no end-of-block symbol can't end (RFC 1951 - /// §3.2.5): it is refused, and nothing of it is kept. - #[test] - fn a_dynamic_code_without_an_end_is_damage() { - let mut bits = Bits::default(); - bits.put(1, 1); // the final block, - bits.put(2, 2); // dynamic Huffman codes (§3.2.7) - bits.put(0, 5); // 257 literal and length codes - bits.put(0, 5); // 1 distance code - bits.put(14, 4); // 18 code length codes, in their order up to the one for 1 - for symbol in [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1] { - bits.put(u32::from(symbol == 18 || symbol == 1), 3); - } - // Code length codes: 1 is 0, 18 is 1. 'A' and 'B' get a bit each, 256 none. - let zeros = |bits: &mut Bits, n: u32| { - bits.code(1, 1); - bits.put(n - 11, 7); - }; - zeros(&mut bits, 65); - bits.code(0, 1); - bits.code(0, 1); - zeros(&mut bits, 138); - zeros(&mut bits, 52); - bits.code(0, 1); // the distance code: a bit - for _ in 0..4 { - bits.code(0, 1); // 'A' - } - let data = bits.finish(); - assert!(zlib_header(&data).is_none(), "read as raw deflate"); - match decode(&data, 1 << 20, Some(1 << 20)).0 { - Err(f) => { - assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); - assert_eq!(f.partial, b""); - } - Ok(out) => panic!("read as {out:?}"), - } - } - - /// A back-reference to before the start of the output points at nothing: read strictly the - /// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide, - /// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which - /// backend inflates (the price of reading Flate, `FLATE_READ_WORK`, is set for zlib-rs). - #[test] - fn a_copy_from_before_the_start_is_damage() { - let literal = |bits: &mut Bits, b: u8| bits.code(0x30 + u32::from(b), 8); - let mut bits = Bits::default(); - bits.put(1, 1); // the final block, - bits.put(1, 2); // fixed Huffman codes (§3.2.6) - literal(&mut bits, b'A'); - literal(&mut bits, b'B'); - bits.code(1, 7); // length 3 - bits.code(4, 5); // distance 5 (code 4 and an extra bit of 0), three bytes before the start - bits.put(0, 1); - literal(&mut bits, b'C'); - bits.code(0, 7); // end of block - let data = bits.finish(); - assert!(zlib_header(&data).is_none(), "read as raw deflate"); - match decode(&data, 1 << 20, Some(1 << 20)).0 { - Err(f) => { - assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); - assert_eq!(f.partial, b"AB"); - } - Ok(out) => panic!("read as {out:?}"), - } - } -} diff --git a/crates/filters/src/jbig2.rs b/crates/filters/src/jbig2.rs deleted file mode 100644 index 3d3c7b81a0..0000000000 --- a/crates/filters/src/jbig2.rs +++ /dev/null @@ -1,467 +0,0 @@ -//! JBIG2 encoding (ITU-T T.88 | ISO/IEC 14492), for `JBIG2Decode` (ISO 32000-2 §7.4.7), lossless: -//! a generic region, template 0 with its nominal adaptive pixels, typical prediction on, MQ -//! arithmetic coding ([`crate::mq`]); or the image's marks as symbols ([`symbols`]), which a group -//! of images may share through `/JBIG2Globals`. Written from the standard's 6.2.5 (coding a generic -//! region), 7.2 (segment headers), 7.4.1, 7.4.6 and 7.4.8 (the segments) and D.3 (the embedded -//! organisation PDF uses). Decoding is in [`decode`]. - -mod decode; -mod symbols; - -pub use decode::{MAX_JBIG2_PIXELS, decode_jbig2_within}; - -use crate::FilterError; -use crate::mq::{Context, MqEncoder}; - -/// The widest row, and the most rows, [`encode_jbig2_generic`] encodes: what `hayro-jbig2`, the -/// decoder behind `JBIG2Decode` here, takes on a side. JBIG2 allows 32-bit sizes, but past this -/// nothing here could read the stream, and the optimizer, which doesn't read back what it stores, -/// replaced such an image with one that draws nothing. -pub const MAX_JBIG2_COLUMNS: u32 = 65_535; - -/// Template 0's pixels (Figure 3), as (x, y) offsets from the pixel being coded, each a bit of the -/// context in this order: the four on the current row, the five fixed ones on the row above, the -/// three on the row above that, then the adaptive pixels A1–A4 at their nominal places (Table 5). -const TEMPLATE: [(i32, i32); 16] = [ - (-1, 0), - (-2, 0), - (-3, 0), - (-4, 0), - (-2, -1), - (-1, -1), - (0, -1), - (1, -1), - (2, -1), - (-1, -2), - (0, -2), - (1, -2), - (3, -1), - (-3, -1), - (2, -2), - (-2, -2), -]; - -/// The context that codes SLTP when template 0 is used (Figure 8): those pixel values, in -/// [`TEMPLATE`]'s order. Figure 8 reads, by row from x = -4: `0 1 0 1` on the current row, -/// `0 1 1 0 0 1 0` (x = -3 to 3) above it, `1 0 0 1 1` (x = -2 to 2) above that. -const SLTP: [u8; 16] = [1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1]; - -/// Columns of padding either side of a row, so every template pixel of a row's pixels is in it. -const PAD: usize = 4; - -/// `rows` rows of `columns` 1-bit PDF samples as lossless JBIG2, the smaller of two codings: -/// [`encode_jbig2_generic`]'s, or one that stores each distinct mark of the image once, as a -/// symbol, and places its copies (see [`symbols`]). Either way the stream is in the embedded -/// organisation and decodes to the same bits without `/JBIG2Globals`. -/// -/// Errors: as [`encode_jbig2_generic`]. -pub fn encode_jbig2(data: &[u8], columns: u32, rows: u32) -> Result, FilterError> { - let generic = encode_jbig2_generic(data, columns, rows)?; - let samples = rows_to_encode(data, columns, rows).ok(); - Ok(match samples.and_then(|s| symbols::encode(s, columns, rows)) { - Some(coded) if coded.len() < generic.len() => coded, - _ => generic, - }) -} - -/// `rows` rows of `columns` 1-bit PDF samples (most significant bit first, each row starting on -/// a new byte), as a JBIG2 stream in the embedded organisation: a page information segment and an -/// immediate lossless generic region segment, which `JBIG2Decode` decodes to the same bits -/// without `/JBIG2Globals`. -/// -/// A 0 sample is coded as a JBIG2 1 (black), since `JBIG2Decode` gives black as 0, as a grey image's -/// samples have it. Bytes after the last row are ignored, as are the bits that pad a row to a whole -/// byte. -/// -/// Errors: no rows or columns, more of either than [`MAX_JBIG2_COLUMNS`], or more samples than -/// [`MAX_JBIG2_PIXELS`], what the decoder takes ([`FilterError::Unsupported`]); fewer bytes than the -/// rows need ([`FilterError::Corrupt`]). -pub fn encode_jbig2_generic(data: &[u8], columns: u32, rows: u32) -> Result, FilterError> { - let samples = rows_to_encode(data, columns, rows)?; - let width = columns as usize; - let stride = width.div_ceil(8); - let mut contexts: Vec = vec![0; 1 << 16]; - let mut mq = MqEncoder::new(); - let height = samples.len() / stride; - code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { - let packed = samples.get(y * stride..).unwrap_or_default(); - for (x, pixel) in row.iter_mut().enumerate() { - let sample = packed.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); - *pixel = u8::from(!sample); - } - }); - let region = mq.finish(); - let mut out = Vec::with_capacity(region.len() + 64); - segment(&mut out, 0, 48, &[], false, &page_information(columns, rows)); - segment(&mut out, 1, 39, &[], false, &generic_region(columns, rows, 0, 0, ®ion)); - Ok(out) -} - -/// The samples of `rows` rows of `columns` 1-bit samples in `data`, when the encoders here take -/// that size; the errors are [`encode_jbig2_generic`]'s. -fn rows_to_encode(data: &[u8], columns: u32, rows: u32) -> Result<&[u8], FilterError> { - let too_large = columns > MAX_JBIG2_COLUMNS || rows > MAX_JBIG2_COLUMNS || u64::from(columns) * u64::from(rows) > MAX_JBIG2_PIXELS; - if columns == 0 || rows == 0 || too_large { - return Err(FilterError::Unsupported(format!("JBIG2 encoding of {columns} × {rows} samples"))); - } - let stride = (columns as usize).div_ceil(8); - usize::try_from(rows) - .ok() - .and_then(|r| r.checked_mul(stride)) - .and_then(|n| data.get(..n)) - .ok_or_else(|| FilterError::Corrupt { filter: "JBIG2Decode", detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()) }) -} - -/// A group of images coded by [`encode_jbig2_shared`]: the data of the `/JBIG2Globals` stream -/// they share, and each image's own stream, in the order given, or `None` for one that shares -/// nothing with the others. -#[derive(Debug)] -pub struct SharedJbig2 { - pub globals: Vec, - pub images: Vec>>, -} - -/// Images, each `rows` rows of `columns` 1-bit PDF samples, as lossless JBIG2 sharing one symbol -/// dictionary: the marks found in more than one of them are stored once, in a symbol dictionary -/// segment not associated with a page, which is the `/JBIG2Globals` stream's data, and each image's -/// stream, in the embedded organisation, places them from there, with its other marks coded as -/// [`encode_jbig2`]'s symbol coding codes them (see [`symbols`]). With the globals, each decodes to -/// its own bits. An image that holds none of those marks, or that symbol coding doesn't take, has -/// no stream here; it is stored alone. `None` when the images share fewer than two marks. -/// -/// What each image's stream saves is left for the caller to weigh against what the globals take, -/// which the images that use them share. -/// -/// Errors: as [`encode_jbig2_generic`], for the first image it would refuse. -pub fn encode_jbig2_shared(images: &[(&[u8], u32, u32)]) -> Result, FilterError> { - let images = - images.iter().map(|(data, columns, rows)| Ok((rows_to_encode(data, *columns, *rows)?, *columns, *rows))).collect::, _>>()?; - Ok(symbols::encode_shared(&images).map(|(globals, images)| SharedJbig2 { globals, images })) -} - -/// A page information segment's data (7.4.8): the page's size, resolution unknown, eventually -/// lossless, default pixel 0 (white), OR combination, not striped. -fn page_information(columns: u32, rows: u32) -> Vec { - let mut page = Vec::with_capacity(19); - page.extend_from_slice(&columns.to_be_bytes()); - page.extend_from_slice(&rows.to_be_bytes()); - page.extend_from_slice(&[0; 8]); - page.push(0x01); - page.extend_from_slice(&[0, 0]); - page -} - -/// A region segment information field (7.4.1): a `width` × `height` region at (`x`, `y`) on the -/// page, combined with it by OR. -fn region_information(out: &mut Vec, width: u32, height: u32, x: u32, y: u32) { - for v in [width, height, x, y] { - out.extend_from_slice(&v.to_be_bytes()); - } - out.push(0); -} - -/// An immediate lossless generic region segment's data (7.4.6): its region information, flags -/// (MMR 0, template 0, TPGDON 1), the AT pixels at their nominal places, then `coded`. -fn generic_region(width: u32, height: u32, x: u32, y: u32, coded: &[u8]) -> Vec { - let mut data = Vec::with_capacity(26 + coded.len()); - region_information(&mut data, width, height, x, y); - data.push(0x08); - at_pixels(&mut data); - data.extend_from_slice(coded); - data -} - -/// Template 0's four AT pixels at their nominal places, as AT flags (7.4.6.3, 7.4.2.1.2). -fn at_pixels(out: &mut Vec) { - for (x, y) in &TEMPLATE[12..] { - out.push(*x as i8 as u8); - out.push(*y as i8 as u8); - } -} - -/// A segment (7.2) of page 1: its header (number, type, the segments it refers to, the page, data -/// length), then its data. `retained`: a later segment refers to this one. `refers_to` holds each -/// referred-to segment's number and whether a segment after this one refers to it too (its -/// retain bit, 7.2.4). Segment numbers here stay below 256, so each referred-to one takes a byte -/// (7.2.5); a segment refers to at most four, so their count and retain bits take one byte too. -fn segment(out: &mut Vec, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { - segment_of(out, 1, number, kind, refers_to, retained, data); -} - -/// [`segment`], of page `page`: 0 for a segment associated with no page (7.2.6), such as the -/// global ones `/JBIG2Globals` holds, which a page's segments may refer to. -fn segment_of(out: &mut Vec, page: u8, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { - out.extend_from_slice(&number.to_be_bytes()); - out.push(kind); - // At most four, so the count fits its three bits. - let refers_to = refers_to.get(..refers_to.len().min(4)).unwrap_or_default(); - let kept = refers_to.iter().enumerate().fold(0u8, |bits, (k, (_, kept))| bits | u8::from(*kept) << (k + 1)); - out.push((refers_to.len() as u8) << 5 | kept | u8::from(retained)); - out.extend(refers_to.iter().map(|(n, _)| *n)); - out.push(page); - // A segment's data never comes near 4 GiB here: an image is at most 2^28 samples. - out.extend_from_slice(&u32::try_from(data.len()).unwrap_or(u32::MAX).to_be_bytes()); - out.extend_from_slice(data); -} - -/// Codes a `width` × `height` bitmap as a generic region (6.2.5.7) with template 0 and the nominal -/// AT pixels, its decisions in `contexts` (2^16 of them), with typical prediction when `typical` -/// (TPGDON). `fill(y, row)` sets row `y`'s `width` pixels, 1 for black. Pixels outside the bitmap -/// count as 0 (white), as the decoder takes them. -fn code_generic(mq: &mut MqEncoder, contexts: &mut [Context], width: usize, height: usize, typical: bool, mut fill: impl FnMut(usize, &mut [u8])) { - let sltp = SLTP.iter().enumerate().fold(0usize, |cx, (k, v)| cx | usize::from(*v) << k); - // The two rows above the current one and the current one, as JBIG2 pixels (1 black), padded. - let blank = vec![0u8; width + 2 * PAD]; - let (mut above2, mut above1) = (blank.clone(), blank.clone()); - let mut row = blank; - let mut ltp = false; - for y in 0..height { - if let Some(pixels) = row.get_mut(PAD..PAD + width) { - fill(y, pixels); - } - if typical { - // Typical prediction (6.2.5.5): a row the same as the one above is coded as one decision. - let same = row == above1; - if let Some(cx) = contexts.get_mut(sltp) { - mq.encode(cx, same != ltp); - } - ltp = same; - } - if !ltp { - for x in 0..width { - let at = |dx: i32, dy: i32| { - let line = match dy { - 0 => &row, - -1 => &above1, - _ => &above2, - }; - let i = (PAD + x).checked_add_signed(dx as isize); - i.and_then(|i| line.get(i)).copied().unwrap_or(0) - }; - let cx = TEMPLATE.iter().enumerate().fold(0usize, |cx, (k, (dx, dy))| cx | usize::from(at(*dx, *dy)) << k); - if let Some(state) = contexts.get_mut(cx) { - mq.encode(state, row.get(PAD + x) == Some(&1)); - } - } - } - std::mem::swap(&mut above2, &mut above1); - std::mem::swap(&mut above1, &mut row); - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The rows an independent decoder (`hayro-jbig2`) reads from `data`, as PDF samples: true - /// for white (a JBIG2 0). - pub(super) fn decode(data: &[u8]) -> Vec> { - decode_with(data, None) - } - - /// [`decode`], with `/JBIG2Globals`. - pub(super) fn decode_with(data: &[u8], globals: Option<&[u8]>) -> Vec> { - struct Rows(Vec>, Vec, usize); - impl hayro_jbig2::Decoder for Rows { - fn push_pixel(&mut self, black: bool) { - self.1.push(!black); - } - fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { - self.1.extend(std::iter::repeat_n(!black, chunks as usize * 8)); - } - fn next_line(&mut self) { - let mut row = std::mem::take(&mut self.1); - row.truncate(self.2); - self.0.push(row); - } - } - let image = hayro_jbig2::Image::new_embedded(data, globals).unwrap(); - let mut rows = Rows(Vec::new(), Vec::new(), image.width() as usize); - image.decode(&mut rows).unwrap(); - rows.0 - } - - /// The rows of packed samples `data`, true for a 1 bit. - pub(super) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { - let stride = (columns as usize).div_ceil(8); - (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() - } - - /// Rows of `columns` samples from `runs`: alternating white and black runs one longer than - /// those values (never empty, even when shrinking makes them all 0), until the rows are full; - /// or, with `noise`, the bytes of `runs` as they are. - fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { - let stride = (columns as usize).div_ceil(8); - let mut data = vec![0u8; stride * rows as usize]; - if runs.is_empty() { - return data; - } - if noise { - data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); - return data; - } - let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); - for y in 0..rows as usize { - for x in 0..columns as usize { - if left == 0 { - white = !white; - left = 1 + usize::from(runs[next % runs.len()]); - next += 1; - } - if white { - data[y * stride + x / 8] |= 0x80 >> (x % 8); - } - left -= 1; - } - } - data - } - - #[test] - fn a_page_of_text_like_marks_round_trips_and_shrinks() { - let (w, h) = (1700u32, 220u32); - let stride = (w as usize).div_ceil(8); - let mut data = vec![0xFF; stride * h as usize]; - for y in (20..h as usize).step_by(40) { - for dy in 0..12 { - for x in (100..1600).step_by(9) { - if (x / 9 + dy) % 5 != 0 { - data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); - } - } - } - } - let jbig2 = encode_jbig2_generic(&data, w, h).unwrap(); - assert_eq!(decode(&jbig2), unpack(&data, w, h)); - let g4 = crate::encode_g4(&data, w, h).unwrap(); - assert!(jbig2.len() < g4.len(), "JBIG2 {} bytes, Group 4 {}", jbig2.len(), g4.len()); - } - - #[test] - fn edges_padding_and_typical_rows_round_trip() { - // 3 columns: the last 5 bits of each byte pad the row; set them to anything. - let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; - assert_eq!(decode(&encode_jbig2_generic(&data, 3, 3).unwrap()), [[true, false, true], [false, true, false], [true, true, true]]); - for (w, h, fill) in [(1, 1, 0x00), (1, 1, 0xFF), (13, 9, 0x00), (13, 9, 0xFF), (64, 40, 0x5A)] { - let data = vec![fill; (w as usize).div_ceil(8) * h as usize]; - assert_eq!(decode(&encode_jbig2_generic(&data, w, h).unwrap()), unpack(&data, w, h), "{w} × {h} of {fill:#04x}"); - } - } - - /// Packed samples from JBIG2 pixels (1 black): a 0 sample for each 1. - pub(super) fn samples(pixels: &[Vec]) -> Vec { - let width = pixels.first().map_or(0, Vec::len); - let stride = width.div_ceil(8); - let mut data = vec![0xFF; stride * pixels.len()]; - for (y, row) in pixels.iter().enumerate() { - for (x, p) in row.iter().enumerate() { - if *p == 1 { - data[y * stride + x / 8] &= !(0x80 >> (x % 8)); - } - } - } - data - } - - /// A row the same as the one above is typical (6.2.5.5): one decision, not a row of pixels. - #[test] - fn a_repeated_row_is_one_decision() { - let row: Vec = (0..800u32).map(|x| u8::from((x * 7919 + x / 3) % 5 < 2)).collect(); - let pixels = vec![row; 1000]; - let data = samples(&pixels); - crate::mq::DECISIONS.with(|n| n.set(0)); - let jbig2 = encode_jbig2_generic(&data, 800, 1000).unwrap(); - // The first row's 800 pixels, and one typical-row decision for each of the 1000 rows. - assert_eq!(crate::mq::DECISIONS.with(|n| n.get()), 800 + 1000); - assert_eq!(decode(&jbig2), unpack(&data, 800, 1000)); - } - - /// SLTP is coded in the context Figure 8 shows, which ordinary pixels can share: an image - /// tiled with that very neighbourhood decodes only if both coders use the same context. - #[test] - fn the_typical_row_flag_shares_figure_8s_context() { - // Figure 8, around a pixel at x = 4 of an 8-column tile: two rows up x = 2..6, one row up - // x = 1..7, the pixel's own row x = 0..3. - let tile = [[0, 0, 1, 0, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 1, 0], [0, 1, 0, 1, 1, 0, 0, 0]]; - let pixels: Vec> = (0..90).map(|y| (0..400).map(|x| tile[y % 3][x % 8]).collect()).collect(); - let data = samples(&pixels); - assert_eq!(decode(&encode_jbig2_generic(&data, 400, 90).unwrap()), unpack(&data, 400, 90)); - } - - #[test] - fn what_it_cannot_encode_is_an_error() { - assert!(matches!(encode_jbig2_generic(&[0; 8], 0, 1), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode_jbig2_generic(&[0; 8], 8, 0), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode_jbig2_generic(&[0; 3], 16, 2), Err(FilterError::Corrupt { .. })), "two rows of 16 need 4 bytes"); - assert!(matches!(encode_jbig2_generic(&[], 1, 2), Err(FilterError::Corrupt { .. }))); - } - - /// The encoder stops where the decoder does, on each side and in total: past that it would - /// write a stream nothing here can read. 16,384 squared is exactly the bound on samples. - /// `encode_jbig2_shared` refuses what `encode_jbig2_generic` refuses: an image taller than - /// the decoder takes is `Unsupported`, one at the bound isn't refused. (Its one tall mark is - /// too large to be a symbol, so nothing is shared: `Ok(None)`.) - #[test] - fn shared_coding_takes_only_what_the_decoder_takes() { - let past = vec![0x7F; MAX_JBIG2_COLUMNS as usize + 1]; - let tall = (&past[..], 1, MAX_JBIG2_COLUMNS + 1); - assert!(matches!(encode_jbig2_shared(&[tall, tall]), Err(FilterError::Unsupported(_))), "too tall"); - let at = (&past[..MAX_JBIG2_COLUMNS as usize], 1, MAX_JBIG2_COLUMNS); - assert!(matches!(encode_jbig2_shared(&[at, at]), Ok(None)), "at the bound"); - } - - #[test] - fn it_encodes_only_what_the_decoder_takes() { - let unsupported = |columns, rows| matches!(encode_jbig2_generic(&[], columns, rows), Err(FilterError::Unsupported(_))); - assert!(unsupported(MAX_JBIG2_COLUMNS + 1, 1), "too wide"); - assert!(unsupported(1, MAX_JBIG2_COLUMNS + 1), "too tall"); - assert!(unsupported(16_385, 16_384), "too many samples, each side within the bound"); - assert!(unsupported(u32::MAX, u32::MAX), "the product doesn't overflow"); - // At the bounds what it writes reads back, through the decoder that has to read it. - let wide = vec![0x0F; (MAX_JBIG2_COLUMNS as usize).div_ceil(8) * 4]; - assert_eq!(decode(&encode_jbig2_generic(&wide, MAX_JBIG2_COLUMNS, 4).unwrap()), unpack(&wide, MAX_JBIG2_COLUMNS, 4)); - let tall = vec![0x7F; MAX_JBIG2_COLUMNS as usize]; - assert_eq!(decode(&encode_jbig2_generic(&tall, 1, MAX_JBIG2_COLUMNS).unwrap()), unpack(&tall, 1, MAX_JBIG2_COLUMNS)); - struct Rows(u32, u64); - impl hayro_jbig2::Decoder for Rows { - fn push_pixel(&mut self, black: bool) { - self.1 += u64::from(black); - } - fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { - self.1 += u64::from(black) * u64::from(chunks) * 8; - } - fn next_line(&mut self) { - self.0 += 1; - } - } - // One black pixel, so a page that came back blank wouldn't pass. - let mut largest = vec![0xFF; 2_048 * 16_384]; - largest[0] = 0x7F; - let largest = encode_jbig2_generic(&largest, 16_384, 16_384).unwrap(); - let mut rows = Rows(0, 0); - hayro_jbig2::Image::new_embedded(&largest, None).unwrap().decode(&mut rows).unwrap(); - assert_eq!(rows.0, 16_384); - assert_eq!(rows.1, 1); - } - - proptest::proptest! { - /// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder. - #[test] - fn bitmaps_round_trip( - columns in 1u32..300, - rows in 1u32..24, - runs in proptest::collection::vec(0u8..=255, 1..48), - noise in proptest::bool::ANY, - ) { - let data = bitmap(columns, rows, &runs, noise); - let jbig2 = encode_jbig2_generic(&data, columns, rows).unwrap(); - proptest::prop_assert_eq!(decode(&jbig2), unpack(&data, columns, rows)); - } - - /// Whatever it is given, it returns. - #[test] - fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { - let _ = encode_jbig2_generic(&data, columns, rows); - } - } -} diff --git a/crates/filters/src/jbig2/decode.rs b/crates/filters/src/jbig2/decode.rs deleted file mode 100644 index 26ed16b198..0000000000 --- a/crates/filters/src/jbig2/decode.rs +++ /dev/null @@ -1,607 +0,0 @@ -//! JBIG2 decoding for `JBIG2Decode` (ISO 32000-2 §7.4.7), through `hayro-jbig2`. - -use crate::FilterError; -use crate::ccitt::Rows; - -/// The largest JBIG2 page [`decode_jbig2_within`] decodes, in pixels. The decoder decodes a page whole, -/// and takes pages of up to this many (20,000 × 10,000 decodes). It is also the most -/// [`encode_jbig2_generic`](crate::encode_jbig2_generic) encodes, so what it stores reads back. -pub const MAX_JBIG2_PIXELS: u64 = 1 << 28; - -/// A `JBIG2Decode` stream's `data`, with what its `/JBIG2Globals` stream decodes to, decoded as the -/// filter gives it: `rows` rows of `columns` 1-bit samples, most significant bit first, each -/// starting on a new byte, with black as 0 (JBIG2 codes black as 1). -/// -/// The JBIG2 page has to be the image's size, `columns` × `rows`: PdfCraft's renderer draws the -/// whole page into the image, so the rows of a page of another height wouldn't be the image's. The -/// decoder decodes the page whole, so it is checked before it is decoded, and has to fit in `max` -/// bytes as such rows and have at most [`MAX_JBIG2_PIXELS`]. Regions inside it are bounded by the -/// decoder alone: a region may claim more pixels than its page, and is decoded before it is cut to -/// the page. With no rows asked for, nothing is decoded. -/// -/// Errors: `data` or `globals` that don't decode, or a page of another size -/// ([`FilterError::Corrupt`]); a page past `max` ([`FilterError::LimitExceeded`]) or -/// [`MAX_JBIG2_PIXELS`] ([`FilterError::Unsupported`], or `Corrupt` from the decoder, which has the -/// same cap and can refuse the page first). -/// -/// Decoding spends from `work`: what decoding may still cost across a run of images, in the -/// decoder's count of steps. There is no way in without it, so a run of images can't decode them -/// without sharing an allowance. Reading `data` and `globals` is charged first -/// ([`hayro_jbig2::charge_reading`]), so an image that turns out not to decode has paid for that, -/// then what decoding takes ([`hayro_jbig2::Image::decode_within`]). An image takes at most what -/// one decode may on top of its reading and its page, whatever `work` allows, and one that `work` -/// can't pay for spends all of it. With nothing left an image isn't read at all, so the images after -/// that one cost nothing but this check, however large their globals. -pub fn decode_jbig2_within(data: &[u8], globals: Option<&[u8]>, columns: u32, rows: u32, max: usize, work: &mut u64) -> Result, FilterError> { - if rows == 0 { - return Ok(Vec::new()); - } - if *work == 0 { - return Err(FilterError::Spent); - } - hayro_jbig2::charge_reading(work, data.len().saturating_add(globals.map_or(0, <[u8]>::len))).map_err(|_| FilterError::Spent)?; - let corrupt = |detail: String| FilterError::Corrupt { filter: "JBIG2Decode", detail }; - let image = hayro_jbig2::Image::new_embedded(data, globals).map_err(|e| corrupt(e.to_string()))?; - let (width, height) = (image.width(), image.height()); - if (width, height) != (columns, rows) { - return Err(corrupt(format!("a page of {width} × {height} pixels for an image of {columns} × {rows}"))); - } - if u64::from(width) * u64::from(height) > MAX_JBIG2_PIXELS { - return Err(FilterError::Unsupported(format!("JBIG2 pages of {width} × {height} pixels"))); - } - // u32 to usize: wasm32 and up have at least 32-bit usize. - let (width, height) = (width as usize, height as usize); - if width.div_ceil(8).checked_mul(height).is_none_or(|bytes| bytes > max) { - return Err(FilterError::LimitExceeded(max)); - } - let mut out = Rows::new(width, height, false); - // Stopped by what was left, it isn't a damaged image, whatever the decoder calls it. - image.decode_within(&mut out, work).map_err(|e| if *work == 0 { FilterError::Spent } else { corrupt(e.to_string()) })?; - Ok(out.out) -} - -/// JBIG2 codes black as 1, which `JBIG2Decode` gives as 0. -impl hayro_jbig2::Decoder for Rows { - fn push_pixel(&mut self, black: bool) { - self.push(!black, 1); - } - fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { - self.push(!black, (chunks as usize).saturating_mul(8)); - } - fn next_line(&mut self) { - self.end_row(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{encode_jbig2, encode_jbig2_generic}; - - /// [`decode_jbig2_within`] with no limit on the work. - fn decode_jbig2(data: &[u8], globals: Option<&[u8]>, columns: u32, rows: u32, max: usize) -> Result, FilterError> { - let mut unbounded = u64::MAX; - decode_jbig2_within(data, globals, columns, rows, max, &mut unbounded) - } - - /// `rows` rows of `columns` samples of noise, the bits that pad each row to a byte 0. - fn noise(columns: u32, rows: u32) -> Vec { - let stride = (columns as usize).div_ceil(8); - let mut seed = 0x2545_F491_4F6C_DD1Du64; - let mut data: Vec = (0..stride * rows as usize) - .map(|_| { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - seed as u8 - }) - .collect(); - let pad = (stride * 8 - columns as usize) as u32; - data.chunks_mut(stride).for_each(|row| *row.last_mut().unwrap() &= 0xFFu8.checked_shl(pad).unwrap_or(0)); - data - } - - /// The same stream with its page information (the first segment's data, after its 11-byte - /// header: width, then height) claiming a page of `width` × `height`. - fn page_of(data: &[u8], width: u32, height: u32) -> Vec { - let mut data = data.to_vec(); - data[11..15].copy_from_slice(&width.to_be_bytes()); - data[15..19].copy_from_slice(&height.to_be_bytes()); - data - } - - /// The JBIG2 file header's ID string (ISO 14492 D.4.1). - const FILE_HEADER_ID: [u8; 8] = [0x97, 0x4A, 0x42, 0x32, 0x0D, 0x0A, 0x1A, 0x0A]; - - /// `embedded`, segments as [`encode_jbig2_generic`] writes them (11-byte headers: no references, - /// a one-byte page association), behind a file header with `flags` and the page count `pages`, - /// in the organization `flags` declares (D.4.2, bit 0): 1 sequential, each segment's header - /// before its data (D.1); 0 random access, every header first, the last an end of file segment - /// numbered 2, which tells where they end, then every segment's data in the same order (D.2). - fn with_file_header(embedded: &[u8], flags: u8, pages: Option) -> Vec { - let mut data = FILE_HEADER_ID.to_vec(); - data.push(flags); - data.extend(pages.map(u32::to_be_bytes).into_iter().flatten()); - if flags & 0x01 != 0 { - data.extend_from_slice(embedded); - return data; - } - let (mut headers, mut bodies, mut rest) = (Vec::new(), Vec::new(), embedded); - while !rest.is_empty() { - let (header, after) = rest.split_at(11); - assert_eq!(header[5], 0, "no references"); - let (body, after) = after.split_at(u32::from_be_bytes(header[7..11].try_into().unwrap()) as usize); - headers.extend_from_slice(header); - bodies.extend_from_slice(body); - rest = after; - } - headers.extend_from_slice(&[0, 0, 0, 2, 51, 0, 1, 0, 0, 0, 0]); - data.extend(headers); - data.extend(bodies); - data - } - - /// A stream that starts with a JBIG2 file header (ISO 14492 D.4), which some producers write - /// into `JBIG2Decode` data although ISO 32000-1 7.4.7 has the embedded organization there, - /// decodes past it as Acrobat and PDFBox draw it (hayro-jbig2 patch 17), in either organization - /// the header declares, with the page count and without it. "Bit 0: File organisation type. If - /// this bit is 0, the file uses the random-access organisation. If this bit is 1, the file uses - /// the sequential organisation." (D.4.2) A header with its reserved flag bits set is an error, - /// not a panic. - #[test] - fn a_stream_with_a_jbig2_file_header_decodes_past_it() { - let (w, h) = (203u32, 41u32); - let bits = noise(w, h); - let max = bits.len(); - let embedded = encode_jbig2_generic(&bits, w, h).unwrap(); - let decode = |flags: u8, pages: Option| decode_jbig2(&with_file_header(&embedded, flags, pages), None, w, h, max); - // Sequential organization (bit 0 is 1), one page. - assert_eq!(decode(0x01, Some(1)).unwrap(), bits, "sequential, page count given"); - // Sequential, number of pages unknown (bit 1 is 1): no count follows (D.4.3). - assert_eq!(decode(0x03, None).unwrap(), bits, "sequential, page count unknown"); - // Random-access organization (bit 0 is 0): the segment headers, then their data. - assert_ne!(with_file_header(&embedded, 0x00, Some(1))[13..], embedded[..], "laid out otherwise"); - assert_eq!(decode(0x00, Some(1)).unwrap(), bits, "random access, page count given"); - assert_eq!(decode(0x02, None).unwrap(), bits, "random access, page count unknown"); - assert!(decode(0x11, Some(1)).is_err(), "reserved bits"); - } - - /// A file header cut short, or the segments after it, is an error and not a panic: the ID string - /// alone or in part, a header whose flags promise a page count with fewer than its four bytes, - /// a header with no segments after it, and segments cut anywhere in either organization. No - /// prefix of a whole stream decodes to its samples. - #[test] - fn a_truncated_jbig2_file_header_is_an_error() { - let (w, h) = (24u32, 8u32); - let bits = noise(w, h); - let max = bits.len(); - let embedded = encode_jbig2_generic(&bits, w, h).unwrap(); - let decode = |data: &[u8]| decode_jbig2(data, None, w, h, max); - assert!(decode(&FILE_HEADER_ID[..5]).is_err(), "part of the ID string"); - assert!(decode(&FILE_HEADER_ID).is_err(), "the ID string, no flags"); - for count in 0..4 { - let mut data = FILE_HEADER_ID.to_vec(); - data.push(0x01); - data.extend_from_slice(&1u32.to_be_bytes()[..count]); - assert!(decode(&data).is_err(), "{count} bytes of the page count"); - } - for (flags, pages) in [(0x01, Some(1)), (0x03, None), (0x00, Some(1)), (0x02, None)] { - let whole = with_file_header(&embedded, flags, pages); - let header = 9 + if pages.is_some() { 4 } else { 0 }; - assert!(decode(&whole[..header]).is_err(), "flags {flags:#04x}: the header alone"); - assert!(decode(&whole[..whole.len() - 1]).is_err(), "flags {flags:#04x}: the last byte cut"); - for end in 0..whole.len() { - assert_ne!(decode(&whole[..end]).ok(), Some(bits.clone()), "flags {flags:#04x}: {end} bytes"); - } - assert_eq!(decode(&whole).unwrap(), bits, "flags {flags:#04x}: whole"); - } - } - - /// The segments after a file header count toward hayro-jbig2's cap on the segments of a stream - /// together with its `/JBIG2Globals`, in either organization (patches 8 and 17): globals of - /// extension segments, which decode to nothing, fill the cap to the image's segments, and one - /// more is refused. - #[test] - fn segments_after_a_jbig2_file_header_count_with_the_globals() { - /// hayro-jbig2's `MAX_SEGMENTS`, the most segments of a stream with its globals. - const MAX_SEGMENTS: u32 = 1 << 20; - let (w, h) = (24u32, 8u32); - let bits = noise(w, h); - let max = bits.len(); - let embedded = encode_jbig2_generic(&bits, w, h).unwrap(); - // Numbered past the image's segments (0 to 2), of no page, with no data. - let globals = |count: u32| -> Vec { - (16..16 + count) - .flat_map(|number| { - let mut segment = number.to_be_bytes().to_vec(); - segment.extend_from_slice(&[62, 0, 0, 0, 0, 0, 0]); - segment - }) - .collect() - }; - // The image's page information and region, and in random access its end of file segment. - for (flags, organization, image) in [(0x01, "sequential", 2), (0x00, "random access", 3)] { - let data = with_file_header(&embedded, flags, Some(1)); - let (at_cap, past_cap) = (globals(MAX_SEGMENTS - image), globals(MAX_SEGMENTS - image + 1)); - assert_eq!(decode_jbig2(&data, Some(&at_cap), w, h, max).unwrap(), bits, "{organization}: at the cap"); - let refused = decode_jbig2(&data, Some(&past_cap), w, h, max).unwrap_err(); - assert!(refused.to_string().contains("too many segments"), "{organization}: {refused}"); - } - } - - /// Either coding decodes to its samples, black as 0, the bits that pad each row 0: a generic - /// region of noise, and symbols for letters repeated across the page. With no rows asked for, - /// nothing is decoded. - #[test] - fn both_codings_decode_to_their_samples() { - let (w, h) = (203u32, 41u32); - let bits = noise(w, h); - let generic = encode_jbig2_generic(&bits, w, h).unwrap(); - assert_eq!(decode_jbig2(&generic, None, w, h, usize::MAX).unwrap(), bits); - assert_eq!(decode_jbig2(b"not JBIG2", None, w, 0, usize::MAX).unwrap(), b"", "no rows asked for"); - - // Four lines of three letters, black on white. - const LETTERS: [[&str; 7]; 3] = [ - ["#####", "#....", "####.", "#....", "#....", "#....", "#####"], - [".###.", "#...#", "#....", "#....", "#....", "#...#", ".###."], - ["#...#", "#...#", "#####", "#...#", "#...#", "#...#", "#...#"], - ]; - let (w, h) = (400u32, 40u32); - let stride = (w as usize).div_ceil(8); - let mut marks = vec![0xFFu8; stride * h as usize]; - for (line, top) in [2usize, 12, 22, 32].into_iter().enumerate() { - for (k, x) in (3..w as usize - 6).step_by(7).enumerate() { - for (dy, row) in LETTERS[(k + line) % 3].iter().enumerate() { - for (dx, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'#') { - marks[(top + dy) * stride + (x + dx) / 8] &= !(0x80 >> ((x + dx) % 8)); - } - } - } - } - let symbols = encode_jbig2(&marks, w, h).unwrap(); - assert!(symbols.len() < encode_jbig2_generic(&marks, w, h).unwrap().len(), "symbol coded"); - assert_eq!(decode_jbig2(&symbols, None, w, h, usize::MAX).unwrap(), marks); - } - - /// A region's location is unsigned (ISO/IEC 14492 5.4.4), so one past `i32::MAX` is off the - /// page. The decoder took it for a signed one: a region at 0xFFFFFFFC came back onto the page - /// four pixels left of it, and a refinement anywhere past `i32::MAX` was refused, and the page - /// with it. Each stream is a page of 16 × 12 covered by a generic region, then a region of 8 × 6 - /// of one kind past `i32::MAX`, coded by the standard's procedures, and decodes to that page - /// alone; moved onto the page, the region draws. Vendored hayro-jbig2 patch 14. - #[test] - fn a_region_located_past_i32_max_is_off_the_page() { - const PAGE: &str = "0000000030000100000013000000100000000c00000000000000004200000000000126000100000035000000100000000c0000000000000000040003fffdff02fefefed906aa2cb6ee4ccb4bc975d02669ca7a9c14c20c638f5a5e1bffac"; - let cases = [ - // At 0xFFFFFFFC, 1. - ("generic", "00000002260001000000220000000800000006fffffffc00000001000003fffdff02fefefe17c876ae6f77ffac"), - // At 1, 0xFFFFFFFE, with its symbol dictionary. - ( - "text", - "000000020000010000001e000003fffdff02fefefe00000001000000014fdda14149cec477ed5fffac00000003062002010000001b000000080000000600000001fffffffe00001000000001a943ffac", - ), - // At 0xFFFFFFFC, 1, with its pattern dictionary. - ( - "halftone", - "000000021000010000000a000101000000011fffac00000003162002010000002e0000000800000006fffffffc0000000100000000000800000006000000000000000001000000d09301774485ffac", - ), - // At 0xFFFFFFFC, 1, reading the page there (7.4.7.4): 0s. - ("refinement of the page", "000000022a00010000001e0000000800000006fffffffc000000010400ffffffff0bd98f188428ffac"), - // Of an intermediate region at the same place (7.4.7.5), 1, 0xFFFFFFFE. - ( - "refinement of a region", - "0000000224000100000023000000080000000600000001fffffffe000003fffdff02fefefe3d81322da16b7fffac000000032a2002010000001e000000080000000600000001fffffffe0000ffffffffe29946b82229ffac", - ), - // Of an intermediate refinement of an intermediate region, all at 0xFFFFFFFC, 1. - ( - "refinement of a refinement", - "00000002240001000000230000000800000006fffffffc00000001000003fffdff02fefefecdfd921ec8c47fffac00000003282002010000001e0000000800000006fffffffc000000010000ffffffff903cdb447a47ffac000000042a2003010000001e0000000800000006fffffffc000000010000ffffffffdcd6078cab86ffac", - ), - ]; - let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::>(); - let decode = |segments: &str| decode_jbig2(&[hex(PAGE), hex(segments)].concat(), None, 16, 12, usize::MAX).unwrap(); - let page = decode(""); - assert!(page.iter().any(|&byte| byte != 0xFF), "the page has ink"); - for (kind, segments) in cases { - assert_eq!(decode(segments), page, "{kind}"); - // Moved onto the page, to 4, 1 or 1, 2, the same regions draw: they decode, and are - // left out for where they are. - let (off, on) = [("fffffffc00000001", "0000000400000001"), ("00000001fffffffe", "0000000100000002")] - .into_iter() - .find(|(off, _)| segments.contains(off)) - .unwrap(); - assert_ne!(decode(&segments.replace(off, on)), page, "{kind} on the page"); - } - - // A refinement at 2, 0 of an intermediate region at 0x80000000, 0. A refinement of a - // region refines it where it is, "GRREFERENCEDX 0, GRREFERENCEDY 0" (Table 35), and the two - // should be at one place (7.4.7.5). Where they aren't, it decodes as it does where they are. - const ELSEWHERE: &str = "000000022400010000002300000008000000068000000000000000000003fffdff02fefefe1bca3149568cdfffac000000032a2002010000001e000000080000000600000002000000000000ffffffff784a909cc761ffac"; - let refined = decode(ELSEWHERE); - assert_eq!(refined, decode(&ELSEWHERE.replace("8000000000000000", "0000000200000000")), "the region at 2, 0 too"); - assert_ne!(refined, page, "drawn"); - } - - /// A refinement of the page refines "the current contents of the page buffer, restricted to the - /// area of the page buffer specified by this segment's region segment information field" - /// (ISO/IEC 14492 7.4.7.4), at an offset of 0 (Table 35), and reads 0 outside that area - /// (6.3.5.2). The decoder read the page around the area too, and refused a refinement wider or - /// taller than the page. A refinement of another region refines that region, not the page. And - /// a reference 32 or more left of a refinement (GRREFERENCEDX below −31, as a symbol refined - /// with an RDX below −31 has, Table 18) is read from where each row starts in it: each row's - /// first context read 0 for the reference column it starts at, and from 33 left for the one - /// before it too. Each stream is coded by the standard's procedures and decodes to its - /// picture, `#` for black. Vendored hayro-jbig2 patch 16. - #[test] - fn a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is() { - let cases: [(&str, &str, &[&str]); 5] = [ - ( - "a 48 × 6 page a generic region has inked, refined at 33: ink on both sides of the area, template 0", - "000000003000010000001300000030000000060000000000000000420000000000012600010000004100000030000000060000000000000000040003fffdff02fefefe331433558a5c5056cf412003ec9a38164df39bd4463b9331502c4fb1c267ab77b665aa5962ffac000000022a0001000000210000000c0000000600000021000000000400ffffffff9a6c35bbeee221f687ffac", - &[ - ".#..##.######..#.#..#####.##..#........##.##..##", - ".#..####.####.#...###.##.##.#.#.....#.#.####.###", - "##.#......#..#....#..#.###.#..#.#####.###.######", - "......#....###.#.##..#.##........#.#.#...#####..", - "#.##..###.#.#.##.....#...#.#..#.##.#..#...#...#.", - ".###...#..#..#..##..#.#..#####.#.##.......###.##", - ], - ), - ( - "a 24 × 8 inked page refined at 5, 1, template 1", - "000000003000010000001300000018000000080000000000000000420000000000012600010000003500000018000000080000000000000000040003fffdff02fefefeb1e133cb8bcbb8510e81d219bb880dfb40a311dc5c298819efffac000000022a00010000001b0000000a0000000500000005000000010401e55c0078dcd30fffac", - &[ - "#.##.##...##....##.##..#", - "#.##.##...#.####..##..#.", - "##..##.##...#...###..#.#", - ".....#.#.##.#...###....#", - "..#.#...#.#...#.#...##.#", - "....######...#.####....#", - "..######..##.#.#..##.#..", - "...#..##.####.#..##...#.", - ], - ), - ( - "a 16 × 5 inked page refined by a region 4 wider than it, template 0", - "000000003000010000001300000010000000050000000000000000420000000000012600010000002600000010000000050000000000000000040003fffdff02fefefe22e5e253a429cbf2c701ffac000000022a000100000023000000140000000500000000000000000400ffffffff3afa516532bbe3d481926fffac", - &[".#.#.#.#..#.#.#.", "##.####..###...#", "..####.#..####.#", ".###.#.....#..#.", "#.###..#########"], - ), - ( - "an intermediate region at 4, 1 on an inked 20 × 8 page, refined there (OR): the region, not the page, template 0", - "000000003000010000001300000014000000080000000000000000420000000000012600010000003000000014000000080000000000000000040003fffdff02fefefeb91e434e1726ca89ed06c2bffb0fe5fa7fccc80fffac00000002240001000000230000000a000000050000000400000001000003fffdff02fefefece7a9826ba81bfffac000000032a2002010000001e0000000a0000000500000004000000010000ffffffffd046ee6f6190ffac", - &[ - "#.#.#....#.#...#....", - "##..#.#.#.##.....##.", - "##.#.#.#.#.###.##...", - "###.##.####..###.#.#", - "...#.###..###.#.#..#", - "#...#######.#...####", - "..##.##...#..##..#.#", - "#...##...#.##....#..", - ], - ), - ( - "a symbol refined in a symbol dictionary from a 40 × 6 one, RDX −33: 33 left, template 1", - "0000000030000100000013000000070000000600000000000000004200000000000100000100000036000003fffdff02fefefe00000001000000014f913296209e2187f37bddb8589c8beb7d70109de7124ebe47d04914ef7e655c3c4cffac00000002002001010000001f100203fffdff02fefefe00000001000000014fe451a27bddf49f7ef7e7ffac00000003062002010000001b0000000700000006000000000000000004001000000001a943ffac", - &[".####.#", ".##..##", "......#", "...#..#", "#.#.###", "....##."], - ), - ]; - let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::>(); - for (form, stream, picture) in cases { - let (w, h) = (picture[0].len(), picture.len()); - // As the filter gives it: black as 0, each row padded to a byte with 0s. - let mut rows = vec![0u8; w.div_ceil(8) * h]; - for (y, row) in picture.iter().enumerate() { - for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') { - rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8); - } - } - assert_eq!(decode_jbig2(&hex(stream), None, w as u32, h as u32, usize::MAX).unwrap(), rows, "{form}"); - } - } - - /// What a page costs is the same however much a run has left. Given exactly that it decodes, - /// and less stops it: spending all that's left, and what comes after isn't read. A page that - /// fails still spends what it took. - #[test] - fn a_run_of_images_spends_its_allowance_down_to_the_step() { - let (w, h) = (203u32, 41u32); - let data = encode_jbig2_generic(&noise(w, h), w, h).unwrap(); - let whole = decode_jbig2(&data, None, w, h, usize::MAX).unwrap(); - let mut work = u64::MAX; - assert_eq!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap(), whole); - let cost = u64::MAX - work; - assert!(cost >= u64::from(w) * u64::from(h), "a pixel at a time: {cost}"); - let mut work = cost; - assert_eq!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap(), whole, "exactly enough"); - assert_eq!(work, 0); - let mut work = cost + 7; - decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap(); - assert_eq!(work, 7, "only what it took"); - let mut work = cost - 1; - assert!(matches!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work), Err(FilterError::Spent)), "a step short"); - assert_eq!(work, 0, "spends all that was left"); - // Nothing left: not read at all, so this garbage isn't reported as garbage. - assert!(matches!(decode_jbig2_within(b"garbage", None, w, h, usize::MAX, &mut work), Err(FilterError::Spent))); - // Too little even for its page. - let mut work = 3; - assert!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).is_err()); - assert_eq!(work, 0); - // A region that can't be read after the one that can: the page fails having spent the - // first, and the segment it fails on, and that isn't being out of allowance. - let bad_region = [0, 0, 0, 9, 38, 0, 1, 0, 0, 0, 1, 0xFF]; - let mut work = u64::MAX; - assert!(decode_jbig2_within(&[&data[..], &bad_region].concat(), None, w, h, usize::MAX, &mut work).is_err()); - assert_eq!(u64::MAX - work, cost + 1 + 4 * bad_region.len() as u64, "the first region, a segment more, and reading it"); - // Not a stream at all: it pays for being read, and globals count with it. - let mut work = u64::MAX; - assert!(matches!(decode_jbig2_within(b"garbage", Some(b"globals"), w, h, usize::MAX, &mut work), Err(FilterError::Corrupt { .. }))); - assert_eq!(u64::MAX - work, 4 * 14); - } - - /// A refinement whose reference is offset right of it (GRREFERENCEDX above 0, as a refined text - /// instance has from a wider bitmap, ISO/IEC 14492 Table 12, and a refined symbol from an RDX - /// above 0, Table 18) reads the reference from its first column. Left of the reference the - /// decoder emptied what it had read of it, and at its first column didn't read it again, so its - /// first 32 columns read as 0: such a refinement decoded to other pixels than were coded, or - /// failed. Each stream is a refinement coded by the standard's procedures on a page of its own - /// size, and decodes to its picture, `#` for black. Vendored hayro-jbig2 patch 15. - #[test] - fn a_refinement_reads_a_reference_right_of_it_from_its_first_column() { - let cases: [(&str, &str, &[&str]); 4] = [ - ( - "a text instance of an 8 × 6 symbol refined 6 wider: 3 right, template 0", - "00000000300001000000130000000e000000060000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fdd3dba48bdddde35ffac00000002062001010000002c0000000e000000060000000000000000040012ffffffff00000001a96b4a6b78b7fc094fadde1b3e881fffac", - &[".#.#.#....#...", "##.###....##.#", ".###...###..#.", "#.......#...#.", "#.####.#...#..", "#.##...###.#.."], - ), - ( - "refined 2 wider and 2 taller, RDX and RDY 1: 2 right, 2 down, template 1", - "00000000300001000000130000000a000000080000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fdd9c52d3ea827a43ffac0000000206200101000000280000000a00000008000000000000000004801200000001a96c00cdac6dd5f6f848ce872411cfffac", - &["####.##...", "####..####", "####..##..", "...##...#.", "#......#..", "..#.....#.", "##....####", ".#........"], - ), - ( - "a symbol refined in a symbol dictionary, RDX 5, RDY −1, template 0 with its AT pixels moved", - "00000000300001000000130000000c000000060000000000000000420000000000010000010000001e000003fffdff02fefefe00000001000000014fdda13efc4029efc95fffac000000020020010100000027000203fffdff02fefefefeff010100000001000000014fc98c352dd84a906f120ffb49793fffac00000003062002010000001b0000000c00000006000000000000000004001000000001a943ffac", - &["#.#.##.#.###", "..#.#.....##", "##..#...#...", "..##....##..", "##.#.##.####", ".#..#####..#"], - ), - ( - "a text instance of a 40 × 5 symbol refined 2 wider, RDX 1: 2 right, across a word, template 0", - "00000000300001000000130000002a0000000500000000000000004200000000000100000100000031000003fffdff02fefefe00000001000000015240f590d953ae3f4069dede672090d4d61b377906d3f41aafa4f77107ffac00000002062001010000003c0000002a000000050000000000000000040012ffffffff00000001a96c0b9386d4b140486bdc3673dbd031490e468c7f7813d19068ab04a79cbfffac", - &[ - "##.#.#..##.##.#####.#....#.#.#..#..######.", - ".#.#..###....#..##.#..#.####.#..#.###...##", - ".#.#......#....#####....#...###...#####...", - "#...##.##.......#...###.###...##..##.###.#", - "..#..###..###..#...#.......##.#........##.", - ], - ), - ]; - let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::>(); - for (form, stream, picture) in cases { - let (w, h) = (picture[0].len(), picture.len()); - // As the filter gives it: black as 0, each row padded to a byte with 0s. - let mut rows = vec![0u8; w.div_ceil(8) * h]; - for (y, row) in picture.iter().enumerate() { - for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') { - rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8); - } - } - assert_eq!(decode_jbig2(&hex(stream), None, w as u32, h as u32, usize::MAX).unwrap(), rows, "{form}"); - } - } - - /// The page has to be the image's size, and is bounded before it is decoded, being decoded - /// whole: within `max` as rows of one bit, and within [`MAX_JBIG2_PIXELS`]. Globals are given - /// to the decoder: ones it skips decode, ones that aren't JBIG2 are an error, as is damaged - /// data, which never panics. - #[test] - fn what_it_cannot_decode_is_an_error() { - let (w, h) = (203u32, 41u32); - let data = encode_jbig2_generic(&noise(w, h), w, h).unwrap(); - for (columns, rows) in [(w - 1, h), (w + 1, h), (w, h - 1), (w, h + 1)] { - let e = decode_jbig2(&data, None, columns, rows, usize::MAX).unwrap_err(); - assert!(matches!(e, FilterError::Corrupt { .. }), "{columns} × {rows}: {e}"); - } - let page = 26 * h as usize; - assert!(matches!(decode_jbig2(&data, None, w, h, page - 1), Err(FilterError::LimitExceeded(m)) if m == page - 1), "a byte short"); - assert_eq!(decode_jbig2(&data, None, w, h, page).unwrap().len(), page, "the page within max"); - // 16,384 × 16,384 is 2^28 pixels, which decodes; a row more is past the cap, here or in the - // decoder, which has the same one and can refuse it first. - let large = |rows| decode_jbig2(&page_of(&data, 16_384, rows), None, 16_384, rows, usize::MAX); - assert!(matches!(large(16_385), Err(FilterError::Unsupported(_) | FilterError::Corrupt { .. })), "{:?}", large(16_385)); - assert!(large(16_384).is_ok(), "{:?}", large(16_384).map(|page| page.len())); - - // An extension segment (ISO/IEC 14492 7.4.14), which a decoder skips: its 11-byte header - // gives the length of the data after it. - let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1]; - let generic = decode_jbig2(&data, None, w, h, usize::MAX).unwrap(); - assert_eq!(decode_jbig2(&data, Some(&skipped), w, h, usize::MAX).unwrap(), generic); - assert!(matches!(decode_jbig2(&data, Some(b"not JBIG2"), w, h, usize::MAX), Err(FilterError::Corrupt { .. }))); - for end in 0..data.len() { - let _ = decode_jbig2(&data[..end], None, w, h, usize::MAX); - } - } - - /// Pixels past a bitmap's edge are 0 (ISO/IEC 14492 6.3.5.2). A page whose default pixel is 1, - /// and an intermediate text or halftone region whose default pixel is 1, kept 1s past their - /// width in their last word of each row, and a refinement, which reads its reference a word at - /// a time, read them as pixels: of rows not a multiple of 32 wide, it decoded to other pixels - /// than were coded. Each stream is a refinement coded by the standard's procedures from a - /// reference read as 0 past its edge, and decodes to the picture given, `#` for black. - /// Vendored hayro-jbig2 patch 13. - #[test] - fn a_refinement_reads_nothing_past_its_references_width() { - // A page of 7 × 6, then of 33 × 6, whose default pixel is 1, refined whole. - const PAGE_7: &str = "000000003000010000001300000007000000060000000000000000460000000000012a00010000001e000000070000000600000000000000000400ffffffffe42d7e889ebcffac"; - const PAGE_33: &str = "000000003000010000001300000021000000060000000000000000460000000000012a000100000034000000210000000600000000000000000400ffffffff314a8d598037eb3f915ab3d526ccae915ad6a6a7b814111b124ae60fffac"; - // An intermediate text region of 31 × 6 whose default pixel is 1, then one of 7 × 6 on a - // page of 40, each refined into the page. - const TEXT_31: &str = "00000000300001000000130000001f0000000600000000000000004200000000000100000100000030000003fffdff02fefefe00000001000000014f96dceb76c7f1ff585971969cd9f88e538261a6f713be68d77e8e1fffac00000002042001010000001b0000001f00000006000000000000000000029000000001a943ffac000000032a2002010000002f0000001f0000000600000000000000000400ffffffff9ac311688adbc8f570a2f96aef93d332e3b1f4b27b704bffac"; - const TEXT_7: &str = "000000003000010000001300000028000000060000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fe1f79f4a95c5b77fffac00000002042001010000001b0000000700000006000000000000000000029000000001a943ffac000000032a2002010000001e000000070000000600000000000000000400ffffffffa71dae12bd7fffac"; - // An intermediate halftone region of 33 × 6 whose default pixel is 1, refined into the page. - const HALFTONE_33: &str = "000000003000010000001300000021000000060000000000000000420000000000011000010000000a000101000000011fffac00000002142001010000004100000021000000060000000000000000009000000021000000060000000000000000010000004ea6618f551d48ff24c2c5c98124d449f2adea6fca76e4297fffac000000032a20020100000032000000210000000600000000000000000400ffffffffe87db809393a3251e565e7a749e6470ab37f7aef186ceaf5167bffac"; - let cases: [(&str, &str, [&str; 6]); 5] = [ - ("page 7", PAGE_7, ["###.#.#", ".......", "#.####.", "######.", "#.#...#", ".#...##"]), - ( - "page 33", - PAGE_33, - [ - ".#..##........###..####....#.##.#", - ".#.###.###.##.##..#..#...#.....##", - ".....##..#....#.##.######.###..#.", - "##...#..##.###..###.##..#.#.#.#.#", - ".##.#........##.#.#.....###......", - "...#.#####.#..#.###......###..###", - ], - ), - ( - "text 31", - TEXT_31, - [ - "....##....###......#.##.#.#....", - ".#.....##.......##..###.#...#..", - ".#.#.#.####...#####.####...#.#.", - "......#...#...#.##...##...#..#.", - ".#..##.###.#.###..####...#...#.", - "#...#...#..#.#....#.##.#.....#.", - ], - ), - ( - "text 7 on a page of 40", - TEXT_7, - [ - "......#.................................", - ".....##.................................", - "##...##.................................", - "..#.....................................", - "##......................................", - "..#.....................................", - ], - ), - ( - "halftone 33", - HALFTONE_33, - [ - "##...#.###.#.###.....#.####..####", - "##.##.#...#.##.##.##.#...#.#.###.", - "###..#.#....##.###...#..##.##..##", - "##..#...###...##..#.####.###.#...", - "####.###.###.#..#..#.#.####...###", - "#.#..#........#....#.#.##.##..#.#", - ], - ), - ]; - let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::>(); - for (form, stream, picture) in cases { - let w = picture[0].len(); - // As the filter gives it: black as 0, each row padded to a byte with 0s. - let mut rows = vec![0u8; w.div_ceil(8) * picture.len()]; - for (y, row) in picture.iter().enumerate() { - for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') { - rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8); - } - } - assert_eq!(decode_jbig2(&hex(stream), None, w as u32, 6, usize::MAX).unwrap(), rows, "{form}"); - } - } -} diff --git a/crates/filters/src/jbig2/symbols.rs b/crates/filters/src/jbig2/symbols.rs deleted file mode 100644 index 6e9a8bf952..0000000000 --- a/crates/filters/src/jbig2/symbols.rs +++ /dev/null @@ -1,1329 +0,0 @@ -//! JBIG2 symbol coding, lossless: each distinct mark of an image (a group of black pixels that -//! touch, side by side or at a corner) is stored once, as a symbol in a symbol dictionary, and -//! text regions place a copy of it wherever it occurs. Marks too large to be symbols (rules, -//! frames, pictures) are left to a generic region beside them, and so, in [`encode`]'s second -//! plan, are marks that occur only once. Only exact copies share a symbol, so the image decodes to the -//! same bits. Written from the committee draft's 6.4 (the text region decoding procedure), 6.5 -//! (the symbol dictionary decoding procedure), 7.4.2 and 7.4.3 (their segments) and Annex A (the -//! arithmetic integer coders). - -use std::collections::HashMap; -use std::hash::{DefaultHasher, Hash, Hasher}; -use std::ops::Range; - -use super::{at_pixels, code_generic, generic_region, page_information, region_information, segment, segment_of}; -use crate::mq::{Context, MqEncoder}; - -/// The longest side of a mark stored as a symbol. A larger one is left to the generic region. -const MAX_SYMBOL_SIDE: u32 = 256; - -/// The most black runs looked at. Past it (halftones, noise) the image is left to generic coding, -/// which suits it better anyway; this bounds the memory the marks take. -const MAX_RUNS: usize = 1 << 20; - -/// The most distinct marks sorted into symbols, and the most pixels their bitmaps may hold -/// together: past either the image is left to generic coding. These bound the memory they take. -const MAX_SYMBOLS: usize = 1 << 16; -const MAX_SYMBOL_PIXELS: usize = 1 << 24; - -/// The most symbols one plan codes. The decoder PdfCraft renders JBIG2 with (`hayro-jbig2`) -/// refuses a dictionary whose new and exported symbols pass 65,535 together, and a plan exports -/// every symbol it codes, so it refuses 32,768 or more; others may too, so a plan with more isn't -/// made. -const MAX_CODED_SYMBOLS: usize = (1 << 15) - 1; - -/// The most symbol instances one text region places. The decoder PdfCraft renders JBIG2 with -/// refuses a text region of more than 10,000 (a page it then draws blank), so a page with more is -/// placed by several regions, each over a band of rows, all referring to the one dictionary. -const MAX_REGION_INSTANCES: usize = 10_000; - -/// The most symbols a group's shared dictionary holds ([`encode_shared`]): half of what a page may -/// place, so each page keeps room for its own. -const MAX_SHARED_SYMBOLS: usize = MAX_CODED_SYMBOLS / 2; - -// Segment numbers stay at most 256, so a referred-to one takes a byte (7.2.5): the shared -// dictionary, the page, its own dictionary, a text region for each MAX_REGION_INSTANCES of at most -// MAX_RUNS marks, and the generic region. -const _: () = assert!(MAX_RUNS.div_ceil(MAX_REGION_INSTANCES) + 4 <= 256); - -/// The integer coders' ranges (Table A.1): how many 1 bits pick the range, how many bits give the -/// offset into it, and its first value. The last range has no 0 after its prefix. -const RANGES: [(u32, u32, u32); 6] = [(0, 2, 0), (1, 4, 4), (2, 6, 20), (3, 8, 84), (4, 12, 340), (5, 32, 4436)]; - -/// A run of black pixels on row `y`, from `x0` up to but not including `x1`. -#[derive(Clone, Copy, Debug)] -struct Run { - y: u32, - x0: u32, - x1: u32, -} - -/// A mark: its box (`x1` and `y1` just past it) and its runs, as a range of the run order -/// [`marks`] gives. -#[derive(Debug)] -struct Mark { - x0: u32, - x1: u32, - y0: u32, - y1: u32, - runs: Range, -} - -/// A distinct mark's bitmap, a byte a pixel (1 black), and how many marks are exact copies of it. -#[derive(Debug)] -struct Symbol { - width: u32, - height: u32, - pixels: Vec, - uses: u32, -} - -/// An image's marks, sorted out: its black runs, its marks with the run order they index, the -/// distinct marks of at most [`MAX_SYMBOL_SIDE`] a side as symbols, each such mark as (its -/// symbol, the mark), and the marks too large to be symbols. -struct Page { - runs: Vec, - marks: Vec, - order: Vec, - symbols: Vec, - copies: Vec<(u32, u32)>, - large: Vec, -} - -/// A symbol instance (6.4.5): the symbol, and where its bottom-left pixel goes on the page, as S -/// (the column) and T (the row). `width` and `height` are the symbol's. -#[derive(Clone, Copy, Debug)] -struct Instance { - id: u32, - s: u32, - t: u32, - width: u32, - height: u32, -} - -/// A group of images coded by [`encode_shared`]: the globals' data, and each image's stream. -type SharedCoding = (Vec, Vec>>); - -/// The symbols of a group's shared dictionary, as one image's [`plan`] places them: the ID each of -/// the image's symbols has there, if it is there, and how many symbols that dictionary exports. -struct Shared { - ids: Vec>, - count: u32, -} - -/// `rows` rows of `columns` 1-bit PDF samples (black 0), symbol coded: the smaller of two plans, -/// every mark of at most [`MAX_SYMBOL_SIDE`] a side as a symbol, or only those with a copy. -/// `None` when neither plan has from two to [`MAX_CODED_SYMBOLS`] symbols, or the image is too -/// large or too busy to look at. -pub(super) fn encode(samples: &[u8], columns: u32, rows: u32) -> Option> { - let page = page(samples, columns)?; - [1, 2].into_iter().filter_map(|copies| plan(&page, columns, rows, copies, None, None)).min_by_key(Vec::len) -} - -/// Images, each as [`encode`] takes one, symbol coded sharing one dictionary: the globals' data, a -/// symbol dictionary segment associated with no page (number 0) holding the distinct marks found -/// in more than one image, and each image's stream placing them from there, the smaller of -/// [`encode`]'s two plans for its other marks. The group's marks are held within the limits one -/// image's are ([`MAX_SYMBOLS`], [`MAX_SYMBOL_PIXELS`]): past them, later marks aren't shared. Past -/// [`MAX_SHARED_SYMBOLS`] shared marks, those in the most images are kept. -/// -/// An image too large or too busy to look at, or that places none of the shared marks, gets `None`; -/// `None` overall when fewer than two marks are shared. -pub(super) fn encode_shared(images: &[(&[u8], u32, u32)]) -> Option { - // The group's distinct marks, each with how many images hold it (as its uses), looked at an - // image at a time. - let mut table: Vec = Vec::new(); - let mut by_hash: HashMap> = HashMap::new(); - let mut held = 0usize; - for (samples, columns, _) in images { - let Some(page) = page(samples, *columns) else { continue }; - // An image's symbols are distinct, so each counts an image once. - for symbol in page.symbols { - match find(&table, &by_hash, &symbol) { - Some(t) => { - if let Some(s) = table.get_mut(t as usize) { - s.uses = s.uses.saturating_add(1); - } - } - None => { - let Some(more) = held.checked_add(symbol.pixels.len()).filter(|h| *h <= MAX_SYMBOL_PIXELS && table.len() < MAX_SYMBOLS) else { - continue; - }; - held = more; - let t = u32::try_from(table.len()).ok()?; - by_hash.entry(fingerprint(&symbol)).or_default().push(t); - table.push(Symbol { uses: 1, ..symbol }); - } - } - } - } - let uses = |t: &u32| table.get(*t as usize).map_or(0, |s| s.uses); - let mut shared: Vec = (0..u32::try_from(table.len()).ok()?).filter(|t| uses(t) >= 2).collect(); - if shared.len() < 2 { - return None; - } - shared.sort_by_key(|t| std::cmp::Reverse(uses(t))); - shared.truncate(MAX_SHARED_SYMBOLS); - // The dictionary's order, as `plan` keeps it. - shared.sort_by_key(|t| table.get(*t as usize).map(|s| (s.height, s.width))); - let mut global = vec![None; table.len()]; - for (k, t) in shared.iter().enumerate() { - if let Some(slot) = global.get_mut(*t as usize) { - *slot = u32::try_from(k).ok(); - } - } - let symbols: Vec<&Symbol> = shared.iter().filter_map(|t| table.get(*t as usize)).collect(); - let mut globals = Vec::new(); - segment_of(&mut globals, 0, 0, 0, &[], true, &symbol_dictionary(&symbols)?); - let count = u32::try_from(symbols.len()).ok()?; - let streams = images - .iter() - .map(|(samples, columns, rows)| { - let page = page(samples, *columns)?; - let ids = page.symbols.iter().map(|s| find(&table, &by_hash, s).and_then(|t| global.get(t as usize).copied().flatten())).collect(); - let shared = Shared { ids, count }; - [1, 2].into_iter().filter_map(|copies| plan(&page, *columns, *rows, copies, None, Some(&shared))).min_by_key(Vec::len) - }) - .collect(); - Some((globals, streams)) -} - -/// What sorts a symbol's bitmap ([`symbols`] and [`encode_shared`] look exact copies up by it). -fn fingerprint(s: &Symbol) -> u64 { - let mut hasher = DefaultHasher::new(); - (s.width, s.height, &s.pixels).hash(&mut hasher); - hasher.finish() -} - -/// The index in `table` of the symbol with `s`'s bitmap, looked up by its [`fingerprint`]. -fn find(table: &[Symbol], by_hash: &HashMap>, s: &Symbol) -> Option { - by_hash - .get(&fingerprint(s))? - .iter() - .copied() - .find(|t| table.get(*t as usize).is_some_and(|t| (t.width, t.height) == (s.width, s.height) && t.pixels == s.pixels)) -} - -/// The image's marks and symbols; `None` past the limits above. -fn page(samples: &[u8], columns: u32) -> Option { - let width = usize::try_from(columns).ok()?; - let stride = width.div_ceil(8); - if stride == 0 { - return None; - } - let runs = black_runs(samples, width, stride)?; - let (marks, order) = marks(&runs); - let mut page = Page { runs, marks, order, symbols: Vec::new(), copies: Vec::new(), large: Vec::new() }; - symbols(&mut page)?; - Some(page) -} - -/// The black runs of each row (0 samples), row by row and left to right; `None` past [`MAX_RUNS`]. -fn black_runs(samples: &[u8], width: usize, stride: usize) -> Option> { - let black = |row: &[u8], x: usize| row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) == 0); - let mut runs = Vec::new(); - for (y, row) in samples.chunks_exact(stride).enumerate() { - let y = u32::try_from(y).ok()?; - let mut x = 0; - while x < width { - // Whole white bytes at once. - if x % 8 == 0 && row.get(x / 8) == Some(&0xFF) { - x += 8; - continue; - } - if !black(row, x) { - x += 1; - continue; - } - let start = x; - while x < width && black(row, x) { - x += 1; - } - if runs.len() == MAX_RUNS { - return None; - } - runs.push(Run { y, x0: u32::try_from(start).ok()?, x1: u32::try_from(x).ok()? }); - } - } - Some(runs) -} - -/// The marks the runs make, in the order their first runs come (so from the top), and the run -/// order: the runs' indexes, mark by mark, each mark's in row order. -fn marks(runs: &[Run]) -> (Vec, Vec) { - // Union–find over the runs; a set's root is its first run, as the larger root always joins - // the smaller. Runs on adjacent rows join when they overlap, counting a corner. - let mut parent: Vec = (0..runs.len()).map(|i| i as u32).collect(); - let (mut above, mut above_at, mut above_y): (&[Run], usize, Option) = (&[], 0, None); - let mut at = 0; - for row in runs.chunk_by(|a, b| a.y == b.y) { - let y = row.first().map(|r| r.y); - if above_y.and_then(|a| a.checked_add(1)) == y { - let mut i = 0; - for (j, b) in row.iter().enumerate() { - while above.get(i).is_some_and(|a| a.x1 < b.x0) { - i += 1; - } - let mut k = i; - while above.get(k).is_some_and(|a| a.x0 <= b.x1) { - union(&mut parent, above_at + k, at + j); - k += 1; - } - } - } - (above, above_at, above_y) = (row, at, y); - at += row.len(); - } - let mut mark_of = vec![0u32; runs.len()]; - let mut marks: Vec = Vec::new(); - let mut counts: Vec = Vec::new(); - for (i, r) in runs.iter().enumerate() { - let root = find_root(&mut parent, i); - let m = if root == i { - marks.push(Mark { x0: r.x0, x1: r.x1, y0: r.y, y1: r.y.saturating_add(1), runs: 0..0 }); - counts.push(0); - marks.len() - 1 - } else { - mark_of.get(root).map_or(0, |m| *m as usize) - }; - if let Some(slot) = mark_of.get_mut(i) { - *slot = m as u32; - } - if let (Some(mark), Some(count)) = (marks.get_mut(m), counts.get_mut(m)) { - mark.x0 = mark.x0.min(r.x0); - mark.x1 = mark.x1.max(r.x1); - mark.y1 = mark.y1.max(r.y.saturating_add(1)); - *count += 1; - } - } - let mut next = 0; - for (mark, count) in marks.iter_mut().zip(&counts) { - mark.runs = next..next + count; - next += count; - } - let mut order = vec![0u32; runs.len()]; - let mut fill: Vec = marks.iter().map(|m| m.runs.start).collect(); - for (i, m) in mark_of.iter().enumerate() { - if let Some(n) = fill.get_mut(*m as usize) { - if let Some(slot) = order.get_mut(*n) { - *slot = i as u32; - } - *n += 1; - } - } - (marks, order) -} - -/// The root of `i`'s set, halving the path to it as it goes. Parents are never above their -/// children, so the walk ends. -fn find_root(parent: &mut [u32], mut i: usize) -> usize { - loop { - let Some(&p) = parent.get(i) else { return i }; - let p = p as usize; - if p == i { - return i; - } - let grandparent = parent.get(p).copied().unwrap_or(p as u32); - if let Some(slot) = parent.get_mut(i) { - *slot = grandparent; - } - i = grandparent as usize; - } -} - -fn union(parent: &mut [u32], a: usize, b: usize) { - let (a, b) = (find_root(parent, a), find_root(parent, b)); - if let Some(slot) = parent.get_mut(a.max(b)) { - *slot = a.min(b) as u32; - } -} - -/// Each mark's own pixels in its box: those of its runs, not of other marks the box overlaps. -fn pixels(mark: &Mark, runs: &[Run], order: &[u32]) -> Vec { - // A mark's box holds its runs, so these don't go below 0; a symbol is at most 256 a side. - let width = mark.x1.saturating_sub(mark.x0) as usize; - let height = mark.y1.saturating_sub(mark.y0) as usize; - let mut pixels = vec![0u8; width * height]; - for r in order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| runs.get(*i as usize)) { - let row = r.y.saturating_sub(mark.y0) as usize * width; - let from = row + r.x0.saturating_sub(mark.x0) as usize; - let to = row + r.x1.saturating_sub(mark.x0) as usize; - pixels.get_mut(from..to).unwrap_or_default().fill(1); - } - pixels -} - -/// Sorts the marks into symbols: exact copies share one. `None` past [`MAX_SYMBOLS`] or -/// [`MAX_SYMBOL_PIXELS`]. -fn symbols(page: &mut Page) -> Option<()> { - let mut by_hash: HashMap> = HashMap::new(); - let mut kept = 0usize; - for (m, mark) in page.marks.iter().enumerate() { - let m = u32::try_from(m).ok()?; - let (width, height) = (mark.x1.saturating_sub(mark.x0), mark.y1.saturating_sub(mark.y0)); - if width > MAX_SYMBOL_SIDE || height > MAX_SYMBOL_SIDE { - page.large.push(m); - continue; - } - let symbol = Symbol { width, height, pixels: pixels(mark, &page.runs, &page.order), uses: 0 }; - let symbols = &mut page.symbols; - let id = match find(symbols, &by_hash, &symbol) { - Some(id) => id, - None => { - kept = kept.checked_add(symbol.pixels.len())?; - if kept > MAX_SYMBOL_PIXELS || symbols.len() == MAX_SYMBOLS { - return None; - } - let id = u32::try_from(symbols.len()).ok()?; - by_hash.entry(fingerprint(&symbol)).or_default().push(id); - symbols.push(symbol); - id - } - }; - if let Some(s) = symbols.get_mut(id as usize) { - s.uses = s.uses.saturating_add(1); - } - page.copies.push((id, m)); - } - Some(()) -} - -/// One plan's stream: the symbols with at least `copies` uses in a symbol dictionary, text -/// regions placing them (in strips of 2^`log_strips` rows, or whichever size codes smallest), and -/// the other marks in a generic region. `None` with fewer than two such symbols (their IDs would -/// take no bits at all, SBSYMCODELEN 0, a corner of the standard left alone here) or more than -/// [`MAX_CODED_SYMBOLS`]. -/// -/// With `shared`, the symbols a group's dictionary holds are placed from there, whatever their uses -/// here, and the page's own dictionary holds only the others (none, when it has none), as -/// [`encode_shared`] writes them; `None` too for a page that places none of the shared ones. -fn plan(page: &Page, columns: u32, rows: u32, copies: u32, log_strips: Option, shared: Option<&Shared>) -> Option> { - let symbol = |s: u32| page.symbols.get(s as usize); - let global = |s: u32| shared.and_then(|g| g.ids.get(s as usize).copied().flatten()); - let globals = shared.map_or(0, |g| g.count); - // The dictionary's order (6.5.5): height classes from the shortest, each narrowest first. - let mut kept: Vec = - (0..page.symbols.len() as u32).filter(|s| global(*s).is_none() && symbol(*s).is_some_and(|s| s.uses >= copies)).collect(); - let count = usize::try_from(globals).ok()?.checked_add(kept.len())?; - if !(2..=MAX_CODED_SYMBOLS).contains(&count) { - return None; - } - kept.sort_by_key(|s| symbol(*s).map(|s| (s.height, s.width))); - // The shared symbols' IDs come first, as their dictionary is referred to first (Table 31's - // SBSYMS). - let mut id: Vec> = (0..page.symbols.len() as u32).map(global).collect(); - for (k, s) in kept.iter().enumerate() { - if let Some(slot) = id.get_mut(*s as usize) { - *slot = u32::try_from(k).ok().and_then(|k| k.checked_add(globals)); - } - } - let mut instances = Vec::new(); - let mut left = page.large.clone(); - for (s, m) in &page.copies { - match (id.get(*s as usize).copied().flatten(), page.marks.get(*m as usize), symbol(*s)) { - (Some(id), Some(mark), Some(symbol)) => { - instances.push(Instance { id, s: mark.x0, t: mark.y1.saturating_sub(1), width: symbol.width, height: symbol.height }) - } - _ => left.push(*m), - } - } - if shared.is_some() && !instances.iter().any(|i| i.id < globals) { - return None; - } - let kept: Vec<&Symbol> = kept.iter().filter_map(|s| symbol(*s)).collect(); - let dictionary = if kept.is_empty() { None } else { Some(symbol_dictionary(&kept)?) }; - let count = u32::try_from(count).ok()?; - // Bands of at most MAX_REGION_INSTANCES, from the top. - instances.sort_by_key(|i| (i.t, i.s)); - let mut regions = Vec::new(); - for band in instances.chunks_mut(MAX_REGION_INSTANCES) { - let top = band.iter().map(|i| i.t.saturating_add(1).saturating_sub(i.height)).min()?; - let height = band.iter().map(|i| i.t).max()?.checked_sub(top)?.checked_add(1)?; - regions.push(match log_strips { - Some(l) => text_region(band, count, columns, top, height, l)?, - None => (0..4).filter_map(|l| text_region(band, count, columns, top, height, l)).min_by_key(Vec::len)?, - }); - } - let mut out = Vec::new(); - // The shared dictionary is segment 0, in the globals, and a segment refers only to lower - // numbers (7.2.5): the page's own follow it. - let mut number = u32::from(shared.is_some()); - segment(&mut out, number, 48, &[], false, &page_information(columns, rows)); - number += 1; - let mut dictionaries: Vec = shared.map(|_| 0).into_iter().collect(); - if let Some(dictionary) = &dictionary { - segment(&mut out, number, 0, &[], true, dictionary); - dictionaries.push(u8::try_from(number).ok()?); - number += 1; - } - for (k, text) in regions.iter().enumerate() { - // The dictionaries stay retained for the next region, until the last (7.2.4). - let refers_to: Vec<(u8, bool)> = dictionaries.iter().map(|n| (*n, k + 1 < regions.len())).collect(); - segment(&mut out, number, 7, &refers_to, false, text); - number += 1; - } - if !left.is_empty() { - segment(&mut out, number, 39, &[], false, &leftover_region(&left, page)?); - } - Some(out) -} - -/// A symbol dictionary segment's data (7.4.2) defining `symbols` (in dictionary order) and -/// exporting them all: arithmetic coded, no refinement or aggregation, each bitmap a generic -/// region with template 0, its nominal AT pixels and no typical prediction (Table 16), the -/// bitmaps sharing one set of contexts. -fn symbol_dictionary(symbols: &[&Symbol]) -> Option> { - let count = u32::try_from(symbols.len()).ok()?; - let mut data = vec![0, 0]; - at_pixels(&mut data); - data.extend_from_slice(&count.to_be_bytes()); - data.extend_from_slice(&count.to_be_bytes()); - let mut mq = MqEncoder::new(); - let mut contexts: Vec = vec![0; 1 << 16]; - let (mut iadh, mut iadw, mut iaex) = (IntCoder::new(), IntCoder::new(), IntCoder::new()); - let mut height = 0; - for class in symbols.chunk_by(|a, b| a.height == b.height) { - let h = class.first()?.height; - iadh.encode(&mut mq, Some(int(i64::from(h) - i64::from(height))?)); - height = h; - let mut width = 0; - for s in class { - iadw.encode(&mut mq, Some(int(i64::from(s.width) - i64::from(width))?)); - width = s.width; - let w = s.width as usize; - code_generic(&mut mq, &mut contexts, w, s.height as usize, false, |y, row| { - let line = s.pixels.get(y * w..).unwrap_or_default(); - row.iter_mut().zip(line).for_each(|(p, v)| *p = *v); - }); - } - iadw.encode(&mut mq, None); - } - // The export flags (6.5.10): a run of no symbols not exported, then a run of all of them. - iaex.encode(&mut mq, Some(0)); - iaex.encode(&mut mq, Some(int(i64::from(count))?)); - data.extend_from_slice(&mq.finish()); - Some(data) -} - -/// An immediate lossless text region segment's data (7.4.3) placing `instances` in the band of -/// `height` rows from row `top` (which holds them): arithmetic coded, no refinement, strips of -/// 2^`log_strips` rows, each symbol's bottom-left pixel at (S, T) (REFCORNER BOTTOMLEFT, so a -/// line's letters share their T), not transposed, OR, default pixel 0, SBDSOFFSET 0. Sorts -/// `instances` into strips, each left to right. -fn text_region(instances: &mut [Instance], symbols: u32, columns: u32, top: u32, height: u32, log_strips: u32) -> Option> { - // LOGSBSTRIPS has two bits: strips of 1, 2, 4 or 8 rows. - if log_strips > 3 { - return None; - } - let strips = 1i64 << log_strips; - // T counts from the band's top row. - let t = |i: &Instance| i.t.saturating_sub(top); - instances.sort_by_key(|i| (t(i) >> log_strips, i.s)); - let mut data = Vec::new(); - region_information(&mut data, columns, height, 0, top); - data.extend_from_slice(&((log_strips as u16) << 2).to_be_bytes()); - data.extend_from_slice(&u32::try_from(instances.len()).ok()?.to_be_bytes()); - // SBSYMCODELEN (Table 31): ⌈log2 SBNUMSYMS⌉. - let length = symbols.checked_next_power_of_two()?.trailing_zeros(); - let mut ids: Vec = vec![0; 1 << length]; - let (mut iadt, mut iafs, mut iads, mut iait) = (IntCoder::new(), IntCoder::new(), IntCoder::new(), IntCoder::new()); - let mut mq = MqEncoder::new(); - // The initial STRIPT (6.4.5 step 2) is 0, so each strip's delta T is its first row over the - // strip size, from the strip before. - iadt.encode(&mut mq, Some(0)); - let (mut strip_t, mut first_s) = (0i64, 0i64); - for strip in instances.chunk_by(|a, b| t(a) >> log_strips == t(b) >> log_strips) { - let first = i64::from(t(strip.first()?) >> log_strips << log_strips); - iadt.encode(&mut mq, Some(int((first - strip_t) / strips)?)); - strip_t = first; - let mut cur_s = 0; - for (k, instance) in strip.iter().enumerate() { - let s = i64::from(instance.s); - if k == 0 { - iafs.encode(&mut mq, Some(int(s - first_s)?)); - first_s = s; - } else { - iads.encode(&mut mq, Some(int(s - cur_s)?)); - } - if strips > 1 { - iait.encode(&mut mq, Some(int(i64::from(t(instance)) - strip_t)?)); - } - encode_id(&mut mq, &mut ids, length, instance.id); - // The next S counts from this symbol's last column (6.4.5 step 3c x). - cur_s = s + i64::from(instance.width) - 1; - } - iads.encode(&mut mq, None); - } - data.extend_from_slice(&mq.finish()); - Some(data) -} - -/// A generic region segment's data holding the marks `left`, over their box. -fn leftover_region(left: &[u32], page: &Page) -> Option> { - let left: Vec<&Mark> = left.iter().filter_map(|m| page.marks.get(*m as usize)).collect(); - let x0 = left.iter().map(|m| m.x0).min()?; - let y0 = left.iter().map(|m| m.y0).min()?; - let x1 = left.iter().map(|m| m.x1).max()?; - let y1 = left.iter().map(|m| m.y1).max()?; - let (width, height) = (x1.saturating_sub(x0) as usize, y1.saturating_sub(y0) as usize); - let stride = width.div_ceil(8); - // Packed, 1 for black: no larger than the image's own samples. - let mut packed = vec![0u8; stride.checked_mul(height)?]; - for mark in &left { - for r in page.order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| page.runs.get(*i as usize)) { - let row = r.y.saturating_sub(y0) as usize * stride; - for x in r.x0.saturating_sub(x0) as usize..r.x1.saturating_sub(x0) as usize { - if let Some(b) = packed.get_mut(row + x / 8) { - *b |= 0x80 >> (x % 8); - } - } - } - } - let mut mq = MqEncoder::new(); - let mut contexts: Vec = vec![0; 1 << 16]; - code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { - let line = packed.get(y * stride..).unwrap_or_default(); - for (x, p) in row.iter_mut().enumerate() { - *p = u8::from(line.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0)); - } - }); - Some(generic_region(u32::try_from(width).ok()?, u32::try_from(height).ok()?, x0, y0, &mq.finish())) -} - -fn int(v: i64) -> Option { - i32::try_from(v).ok() -} - -/// An arithmetic integer coder other than IAID (A.2): its 512 contexts, indexed by PREV. -struct IntCoder([Context; 512]); - -impl IntCoder { - fn new() -> Self { - IntCoder([0; 512]) - } - - /// Codes `value`, or OOB for `None`. - fn encode(&mut self, mq: &mut MqEncoder, value: Option) { - for (cx, d) in decisions(value) { - if let Some(state) = self.0.get_mut(cx) { - mq.encode(state, d); - } - } - } -} - -/// The decisions that code `value`, or OOB for `None`, with their contexts (A.2, Table A.1): the -/// sign, the prefix that picks the range, then the offset into it, most significant bit first; -/// OOB is a negative 0. Each decision's context is PREV: a leading 1 and the decisions before it, -/// at most the last eight once there are more. -fn decisions(value: Option) -> Vec<(usize, bool)> { - let (negative, magnitude) = value.map_or((true, 0), |v| (v < 0, v.unsigned_abs())); - let (ones, bits, first) = - RANGES.into_iter().find(|(_, bits, first)| *bits == 32 || u64::from(magnitude) < u64::from(*first) + (1 << bits)).unwrap_or(RANGES[5]); - let offset = magnitude - first; - let mut d = vec![negative]; - d.extend(std::iter::repeat_n(true, ones as usize)); - if ones < 5 { - d.push(false); - } - d.extend((0..bits).rev().map(|i| offset >> i & 1 == 1)); - let mut prev = 1usize; - d.into_iter() - .map(|bit| { - let cx = prev; - prev = if prev < 256 { prev << 1 | usize::from(bit) } else { ((prev << 1 | usize::from(bit)) & 511) | 256 }; - (cx, bit) - }) - .collect() -} - -/// Codes symbol ID `id` in `length` bits, most significant first, through IAID (A.3): each -/// decision's context is a leading 1 and all the bits before it. -fn encode_id(mq: &mut MqEncoder, contexts: &mut [Context], length: u32, id: u32) { - let mut prev = 1usize; - for i in (0..length).rev() { - let d = id >> i & 1 == 1; - if let Some(state) = contexts.get_mut(prev) { - mq.encode(state, d); - } - prev = prev << 1 | usize::from(d); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - use crate::jbig2::tests::{decode, decode_with, samples, unpack}; - use crate::jbig2::{encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; - - type Bitmap = Vec>; - - /// A small linear congruential generator, so the pictures are the same every run. - struct Lcg(u64); - - impl Lcg { - fn below(&mut self, n: usize) -> usize { - self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); - (self.0 >> 33) as usize % n.max(1) - } - } - - /// `count` different letter-like shapes: three strokes two pixels thick in a box of 6 to 18 - /// pixels a side, each after the first starting on one before it, so a shape is one mark; - /// trimmed to its pixels. - fn glyphs(count: usize, seed: u64) -> Vec { - let mut r = Lcg(seed); - let mut out: Vec = Vec::new(); - while out.len() < count { - let (w, h) = (6 + r.below(13), 6 + r.below(13)); - let mut g = vec![vec![0u8; w]; h]; - let (mut x, mut y) = (r.below(w), r.below(h)); - for _ in 0..3 { - let (dx, dy) = [(1, 0), (0, 1), (1, 1)][r.below(3)]; - let mut stroke = Vec::new(); - while x < w && y < h { - g[y][x] = 1; - g[y][(x + 1).min(w - 1)] = 1; - stroke.push((x, y)); - x += dx; - y += dy; - } - (x, y) = stroke[r.below(stroke.len())]; - } - let rows: Vec = (0..h).filter(|y| g[*y].contains(&1)).collect(); - let columns: Vec = (0..w).filter(|x| g.iter().any(|row| row[*x] == 1)).collect(); - let g: Bitmap = g[rows[0]..=rows[rows.len() - 1]].iter().map(|row| row[columns[0]..=columns[columns.len() - 1]].to_vec()).collect(); - if !out.contains(&g) { - out.push(g); - } - } - out - } - - /// Draws `glyph` with its top-left pixel at (`x`, `y`), clipped to the page, by OR. - fn draw(page: &mut Bitmap, glyph: &Bitmap, x: i64, y: i64) { - for (gy, row) in glyph.iter().enumerate() { - for (gx, p) in row.iter().enumerate() { - let (px, py) = (x + gx as i64, y + gy as i64); - if *p == 1 - && px >= 0 - && py >= 0 - && let Some(cell) = page.get_mut(py as usize).and_then(|r| r.get_mut(px as usize)) - { - *cell = 1; - } - } - } - } - - /// Lines of text: glyphs side by side, one to four pixels apart, each line's glyphs on its - /// baseline, the lines 30 rows apart; the first glyphs are the most common, as letters are. - fn text(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64) -> Bitmap { - let mut page = vec![vec![0u8; columns]; rows]; - let mut r = Lcg(seed); - for baseline in (24..rows).step_by(30) { - let mut x = 4 + r.below(8); - loop { - let common = r.below(glyphs.len()) + 1; - let g = &glyphs[r.below(common)]; - if x + g[0].len() > columns { - break; - } - draw(&mut page, g, x as i64, (baseline + 1 - g.len()) as i64); - x += g[0].len() + 1 + r.below(4); - } - } - page - } - - /// `n` glyphs anywhere, some over the edges, some over each other. - fn scatter(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64, n: usize) -> Bitmap { - let mut page = vec![vec![0u8; columns]; rows]; - let mut r = Lcg(seed); - for _ in 0..n { - let g = &glyphs[r.below(glyphs.len())]; - draw(&mut page, g, r.below(columns + 12) as i64 - 6, r.below(rows + 12) as i64 - 6); - } - page - } - - /// A segment's number and type, whether it is retained, and the segments it refers to, each - /// with its retain bit. - type Segment = (u32, u8, bool, Vec<(u8, bool)>); - - /// The segments of a stream in the embedded organisation, from their headers (7.2). - fn segments(data: &[u8]) -> Vec { - let (mut segments, mut at) = (Vec::new(), 0); - while at < data.len() { - let flags = data[at + 5]; - let refers = usize::from(flags >> 5); - let length_at = at + 6 + refers + 1; - let length = u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; - let referred = data[at + 6..at + 6 + refers].iter().enumerate().map(|(k, n)| (*n, flags >> (k + 1) & 1 == 1)).collect(); - let number = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()); - segments.push((number, data[at + 4] & 0x3F, flags & 1 == 1, referred)); - at = length_at + 4 + length; - } - segments - } - - fn segment_types(data: &[u8]) -> Vec { - segments(data).into_iter().map(|(_, kind, ..)| kind).collect() - } - - /// Every plan, at every strip size, decodes to the picture; returns how many plans there were. - fn every_plan_round_trips(picture: &Bitmap) -> usize { - let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); - let data = samples(picture); - let page = page(&data, columns).unwrap(); - let mut plans = 0; - for copies in [1, 2] { - for log_strips in 0..4 { - if let Some(coded) = plan(&page, columns, rows, copies, Some(log_strips), None) { - assert_eq!(decode(&coded), unpack(&data, columns, rows), "{copies} copies, strips of {}", 1 << log_strips); - plans += 1; - } - } - } - plans - } - - /// Annex A.2's example (12 through IADW: the decisions 0101000, in contexts 1, 10, 101, …) - /// and Table A.1's codes. - #[test] - fn integers_are_coded_as_annex_a_says() { - assert_eq!(decisions(Some(12)), [(1, false), (2, true), (5, false), (10, true), (21, false), (42, false), (84, false)]); - let bits = |v| decisions(v).iter().map(|(_, d)| u8::from(*d)).collect::>(); - assert_eq!(bits(None), [1, 0, 0, 0], "OOB"); - assert_eq!(bits(Some(0)), [0, 0, 0, 0]); - assert_eq!(bits(Some(-1)), [1, 0, 0, 1]); - assert_eq!(bits(Some(-3)), [1, 0, 1, 1]); - assert_eq!(bits(Some(4)), [0, 1, 0, 0, 0, 0, 0]); - assert_eq!(bits(Some(-19)), [1, 1, 0, 1, 1, 1, 1]); - assert_eq!(bits(Some(83)), [0, 1, 1, 0, 1, 1, 1, 1, 1, 1]); - assert_eq!(bits(Some(4435))[..6], [0, 1, 1, 1, 1, 0]); - let big = bits(Some(4436)); - assert_eq!((big.len(), &big[..6]), (6 + 32, &[0, 1, 1, 1, 1, 1][..])); - assert!(big[6..].iter().all(|b| *b == 0)); - // PREV keeps a leading 1 and the last eight decisions, so it stays below 512, also when - // those eight are all 0 (PREV exactly 256), as in 4436's offset of 0. - for value in [Some(i32::MIN), Some(4436), Some(-4436), Some(4436 + 0x100), Some(i32::MAX), None] { - let contexts: Vec = decisions(value).iter().map(|(cx, _)| *cx).collect(); - assert!(contexts.iter().take(8).all(|cx| *cx < 256) && contexts.iter().skip(8).all(|cx| (256..512).contains(cx)), "{value:?}"); - } - assert!(decisions(Some(4436)).iter().any(|(cx, _)| *cx == 256)); - } - - /// Strips and first columns thousands of pixels apart need Annex A's 32-bit range: a mark at - /// the far right of a 12,000-pixel page, then one at the left below it (S back by 11,000), and - /// lines 5,000 rows apart. Each shape occurs twice, so both plans are made. - #[test] - fn coordinates_past_4435_round_trip() { - let shapes = glyphs(3, 4); - let mut picture = vec![vec![0u8; 12_000]; 40]; - for (k, x) in [11_000, 0, 5_000, 11_980].into_iter().enumerate() { - draw(&mut picture, &shapes[k % 2], x, 2 + 9 * k as i64); - } - assert_eq!(every_plan_round_trips(&picture), 8); - let mut tall = vec![vec![0u8; 30]; 10_050]; - for (k, y) in [0, 5_000, 10_030].into_iter().enumerate() { - draw(&mut tall, &shapes[k % 3], 3, y); - draw(&mut tall, &shapes[(k + 1) % 3], 15, y); - } - assert_eq!(every_plan_round_trips(&tall), 8); - } - - /// The padding bits of a row whose width isn't a multiple of 8 are ignored whatever they are: - /// black (0) padding, as many scans have, neither joins the marks nor stops symbol coding. - #[test] - fn black_padding_bits_are_ignored() { - for columns in [13, 241, 901] { - let picture = text(columns, 120, &glyphs(12, 6), 8); - let mut data = samples(&picture); - let stride = columns.div_ceil(8); - for row in data.chunks_mut(stride) { - row[stride - 1] &= 0xFF << (8 * stride - columns); - } - let (c, r) = (columns as u32, 120); - let page = page(&data, c).unwrap(); - assert!(page.marks.iter().all(|m| m.x1 as usize <= columns), "{columns} columns"); - for copies in [1, 2] { - if let Some(coded) = plan(&page, c, r, copies, None, None) { - assert_eq!(decode(&coded), unpack(&data, c, r), "{columns} columns"); - } - } - if columns > 13 { - assert_eq!(segment_types(&encode_jbig2(&data, c, r).unwrap())[1], 0, "{columns} columns: symbol coded"); - } - } - } - - /// Pixels touching only at a corner, down to the right or down to the left, are one mark - /// (8-connected); side by side with a white column between, two. A mark of 256 pixels a side - /// is a symbol; one 257 wide, or 257 tall, is left to the generic region. The diagonal and the - /// dot occur twice, so both plans are made. - #[test] - fn marks_join_at_corners_and_large_ones_are_left_out() { - let mut picture = vec![vec![0u8; 300]; 300]; - for i in 0..6 { - picture[10 + i][10 + i] = 1; - picture[10 + i][100 + i] = 1; - picture[10 + i][60 - i] = 1; - } - for row in &mut picture[0..257] { - row[280] = 1; - } - picture[30][10] = 1; - picture[30][12] = 1; - picture[40][0..256].fill(1); - picture[60][0..257].fill(1); - for row in &mut picture[44..300] { - row[290] = 1; - } - let data = samples(&picture); - let page = page(&data, 300).unwrap(); - let boxes: Vec<(u32, u32, u32, u32)> = page.marks.iter().map(|m| (m.x0, m.y0, m.x1 - m.x0, m.y1 - m.y0)).collect(); - assert_eq!( - boxes, - [ - (280, 0, 1, 257), - (10, 10, 6, 6), - (55, 10, 6, 6), - (100, 10, 6, 6), - (10, 30, 1, 1), - (12, 30, 1, 1), - (0, 40, 256, 1), - (290, 44, 1, 256), - (0, 60, 257, 1) - ] - ); - let large: Vec<(u32, u32)> = page.large.iter().map(|m| (page.marks[*m as usize].x0, page.marks[*m as usize].y0)).collect(); - assert_eq!(large, [(280, 0), (0, 60)]); - assert_eq!(every_plan_round_trips(&picture), 8); - } - - /// Past 2^20 black runs the image is left to generic coding; at the cap it is still looked at. - #[test] - fn busy_images_are_left_to_generic_coding() { - // 0x55: four black pixels a byte, each its own run, under the same ones on the next row. - let columns = 1 << 20; - let row = vec![0x55u8; columns / 8]; - assert!(page(&row.repeat(2), columns as u32).is_some(), "2^20 runs"); - assert!(page(&row.repeat(3), columns as u32).is_none(), "1.5 × 2^20 runs"); - } - - /// A plan codes at most 32,767 symbols, which an independent decoder reads back; with more, - /// the second plan can still code the repeated ones. Past 65,536 distinct marks, or 2^24 - /// pixels of them, the image is left to generic coding; at either cap it is still looked at. - #[test] - fn too_many_symbols_are_left_to_generic_coding() { - // `n` distinct marks of 17 × 3 (full top and bottom rows joined by a code of 1 to n, in - // binary, on the row between), then `again` more with the codes 1 and 2 in turn. - let codes = |n: usize, again: usize| { - let (columns, rows) = (256 * 18, (n + again).div_ceil(256) * 4); - let mut picture = vec![vec![0u8; columns]; rows]; - for k in 0..n + again { - let (x, y) = (k % 256 * 18, k / 256 * 4); - let code = if k < n { k + 1 } else { (k - n) % 2 + 1 }; - picture[y][x..x + 17].fill(1); - picture[y + 2][x..x + 17].fill(1); - for b in (0..17).filter(|b| code >> b & 1 == 1) { - picture[y + 1][x + b] = 1; - } - } - (samples(&picture), columns as u32, rows as u32) - }; - let (data, columns, rows) = codes(MAX_CODED_SYMBOLS, 0); - let at_cap = page(&data, columns).unwrap(); - assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); - // One more distinct mark, each once: neither plan is made. - let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 0); - let past = page(&data, columns).unwrap(); - assert!(plan(&past, columns, rows, 1, None, None).is_none() && plan(&past, columns, rows, 2, None, None).is_none()); - assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); - // With two of them repeated, the second plan codes those two. - let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 200); - let past = page(&data, columns).unwrap(); - assert!(plan(&past, columns, rows, 1, None, None).is_none()); - assert_eq!(decode(&plan(&past, columns, rows, 2, None, None).unwrap()), unpack(&data, columns, rows)); - // Above a page of text whose letters repeat, that second plan is the one `encode_jbig2` - // keeps: the first, of every mark, would code too many symbols. - let mut picture: Bitmap = vec![vec![0u8; 256 * 18]; 512]; - for k in 0..=MAX_CODED_SYMBOLS { - let (x, y) = (k % 256 * 18, k / 256 * 4); - picture[y][x..x + 17].fill(1); - picture[y + 2][x..x + 17].fill(1); - for b in (0..17).filter(|b| (k + 1) >> b & 1 == 1) { - picture[y + 1][x + b] = 1; - } - } - picture.extend(text(256 * 18, 600, &glyphs(40, 7), 11)); - let data = samples(&picture); - let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); - let coded = encode_jbig2(&data, columns, rows).unwrap(); - assert_eq!(segment_types(&coded)[1], 0, "symbol coded"); - assert_eq!(decode(&coded), unpack(&data, columns, rows)); - let (data, columns, _) = codes(MAX_SYMBOLS, 0); - assert_eq!(page(&data, columns).map(|p| p.symbols.len()), Some(MAX_SYMBOLS)); - let (data, columns, _) = codes(MAX_SYMBOLS + 1, 0); - assert!(page(&data, columns).is_none()); - // `n` black squares of 256 × 256, each with a white pixel in a different place: 2^16 - // pixels a symbol, so 256 of them are exactly the cap. - let squares = |n: usize| { - let (columns, rows) = (16 * 258, n.div_ceil(16) * 258); - let mut picture = vec![vec![0u8; columns]; rows]; - for k in 0..n { - let (x, y) = (k % 16 * 258, k / 16 * 258); - for row in &mut picture[y..y + 256] { - row[x..x + 256].fill(1); - } - picture[y + 1 + k / 254][x + 1 + k % 254] = 0; - } - (samples(&picture), columns as u32, rows as u32) - }; - let (data, columns, rows) = squares(256); - let at_cap = page(&data, columns).unwrap(); - assert_eq!(at_cap.symbols.len(), 256); - assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); - let (data, columns, _) = squares(257); - assert!(page(&data, columns).is_none()); - } - - /// More than 10,000 marks are placed by several text regions, each of at most 10,000 (more - /// than the decoder PdfCraft renders with accepts in one), over bands of rows, all referring to - /// the dictionary, which each region but the last keeps retained; the generic region comes - /// after them, every segment numbered one more than the one before. - #[test] - fn more_than_10000_marks_are_placed_by_several_regions() { - // 25,000 copies of two 17 × 3 shapes, 256 a line, the lines 5 rows apart, and a bar too - // tall to be a symbol beside them. - let mut picture = vec![vec![0u8; 256 * 18 + 100]; 25_000usize.div_ceil(256) * 5]; - for k in 0..25_000 { - let (x, y) = (k % 256 * 18, k / 256 * 5); - picture[y][x..x + 17].fill(1); - picture[y + 2][x..x + 17].fill(1); - picture[y + 1][x + k % 2] = 1; - } - for row in &mut picture[0..300] { - row[256 * 18 + 50] = 1; - } - let data = samples(&picture); - let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); - let coded = encode_jbig2(&data, columns, rows).unwrap(); - let region = |number, kept| (number, 7, false, vec![(1, kept)]); - assert_eq!( - segments(&coded), - [(0, 48, false, vec![]), (1, 0, true, vec![]), region(2, true), region(3, true), region(4, false), (5, 39, false, vec![])] - ); - assert_eq!(decode(&coded), unpack(&data, columns, rows)); - assert_eq!(every_plan_round_trips(&picture), 8); - } - - /// A page of text whose letters repeat exactly, as rendered text does, keeps each letter once: - /// a symbol dictionary and a text region, much smaller than the generic coding. - #[test] - fn repeated_marks_are_stored_once() { - let picture = text(1700, 400, &glyphs(40, 7), 11); - let data = samples(&picture); - let coded = encode_jbig2(&data, 1700, 400).unwrap(); - let generic = encode_jbig2_generic(&data, 1700, 400).unwrap(); - // The dictionary is retained for the text region, which refers to it (7.2.4, 7.2.5). - assert_eq!(segments(&coded), [(0, 48, false, vec![]), (1, 0, true, vec![]), (2, 7, false, vec![(1, false)])]); - assert_eq!(decode(&coded), unpack(&data, 1700, 400)); - assert!(coded.len() * 2 < generic.len(), "symbols {} bytes, generic {}", coded.len(), generic.len()); - assert_eq!(every_plan_round_trips(&picture), 8); - } - - /// Marks too large to be symbols (a frame, a rule) go to a generic region over their box, and - /// with the second plan so do marks that occur once; the page still decodes to its bits. - #[test] - fn large_and_lone_marks_are_left_to_a_generic_region() { - let mut picture = text(900, 300, &glyphs(25, 3), 5); - picture[10][100..800].fill(1); - picture[290][100..800].fill(1); - for row in &mut picture[10..=290] { - row[100] = 1; - row[799] = 1; - } - // A lone blob, unlike any letter. - for (y, row) in picture[150..170].iter_mut().enumerate() { - row[850 + y % 7] = 1; - } - let data = samples(&picture); - let coded = encode_jbig2(&data, 900, 300).unwrap(); - assert_eq!(segment_types(&coded), [48, 0, 7, 39]); - assert_eq!(decode(&coded), unpack(&data, 900, 300)); - assert_eq!(every_plan_round_trips(&picture), 8); - } - - /// Marks on every edge, marks touching only at a corner (one mark), a mark inside another's - /// box, and marks over each other's boxes. - #[test] - fn marks_at_the_edges_and_corners_round_trip() { - let dot = vec![vec![1u8, 1], vec![1, 1]]; - let ell = vec![vec![1u8, 0, 0], vec![1, 0, 0], vec![1, 1, 1]]; - let mut picture = vec![vec![0u8; 41]; 23]; - for (x, y) in [(0, 0), (39, 0), (0, 21), (39, 21), (20, 0), (0, 10), (39, 10), (20, 21)] { - draw(&mut picture, &dot, x, y); - } - for (x, y) in [(10, 5), (28, 14), (12, 14)] { - draw(&mut picture, &ell, x, y); - } - // Corners touching: one mark (`marks_join_at_corners_and_large_ones_are_left_out` checks it). - picture[3][6] = 1; - picture[4][7] = 1; - picture[5][8] = 1; - // A ring with a dot inside, and an ell over the ring's box. - for i in 0..7 { - picture[8][20 + i] = 1; - picture[14][20 + i] = 1; - picture[8 + i][20] = 1; - picture[8 + i][26] = 1; - } - draw(&mut picture, &dot, 22, 10); - draw(&mut picture, &ell, 25, 15); - assert_eq!(every_plan_round_trips(&picture), 8); - } - - /// One shape, however often, would need symbol IDs of no bits: that image is coded as a generic - /// region, as are a blank page and pure noise. - #[test] - fn one_shape_blank_pages_and_noise_are_coded_generically() { - let letter = &glyphs(1, 9)[..]; - let mut r = Lcg(1); - let noise: Bitmap = (0..60).map(|_| (0..200).map(|_| u8::from(r.below(2) == 1)).collect()).collect(); - for picture in [text(600, 200, letter, 2), vec![vec![0u8; 300]; 40], noise] { - let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); - let data = samples(&picture); - assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); - } - } - - /// Each segment's page association (7.2.6): the byte after the numbers it refers to. - fn associations(data: &[u8]) -> Vec { - let (mut pages, mut at) = (Vec::new(), 0); - while at < data.len() { - let refers = usize::from(data[at + 5] >> 5); - pages.push(data[at + 6 + refers]); - let length_at = at + 6 + refers + 1; - at = length_at + 4 + u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; - } - pages - } - - /// `pictures`, `columns` × `rows` each, coded sharing a dictionary; each image's stream, which - /// has to be there, decodes with the globals to its picture. - fn shared_round_trip(pictures: &[Bitmap]) -> (Vec, Vec>) { - let (columns, rows) = (pictures[0][0].len() as u32, pictures[0].len() as u32); - let data: Vec> = pictures.iter().map(|p| samples(p)).collect(); - let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], columns, rows)).collect(); - let shared = encode_jbig2_shared(&images).unwrap().unwrap(); - let (globals, coded) = (shared.globals, shared.images); - let coded: Vec> = coded.into_iter().map(Option::unwrap).collect(); - for (d, c) in data.iter().zip(&coded) { - assert_eq!(decode_with(c, Some(&globals)), unpack(d, columns, rows)); - } - (globals, coded) - } - - /// `font` with a stem 20 pixels tall down each glyph's left side: a font sharing no glyph with - /// any [`glyphs`] gives, whose glyphs are at most 18 pixels tall. - fn stemmed(font: &[Bitmap]) -> Vec { - font.iter() - .map(|g| { - (0..g.len().max(20)).map(|y| std::iter::once(1).chain(g.get(y).cloned().unwrap_or_else(|| vec![0; g[0].len()])).collect()).collect() - }) - .collect() - } - - /// Mark `k` of a set of distinct ones: a stroke 33 pixels long with a pixel above every other - /// column for each 1 bit of `k`, below 2^17. - fn comb(k: usize) -> Bitmap { - let mut g = vec![vec![0u8; 33]; 2]; - g[1].fill(1); - for bit in 0..17 { - g[0][2 * bit] = u8::from(k >> bit & 1 == 1); - } - g - } - - /// The combs `ks`, 120 a row, the rows four pixels apart. - fn combs(ks: impl ExactSizeIterator) -> Bitmap { - let mut page = vec![vec![0u8; 120 * 36]; ks.len().div_ceil(120) * 4]; - for (i, k) in ks.enumerate() { - draw(&mut page, &comb(k), (i % 120 * 36) as i64, (i / 120 * 4) as i64); - } - page - } - - /// The symbols a symbol dictionary segment exports (SDNUMEXSYMS, 7.4.2.1.4), in a stream - /// that is that segment alone: after its header, flags and adaptive pixels. - fn exported(globals: &[u8]) -> usize { - u32::from_be_bytes(globals[21..25].try_into().unwrap()) as usize - } - - /// Pages in one font share its letters: one symbol dictionary, segment 0, associated with no - /// page and retained (7.2.4, 7.2.6), which each page's text region refers to first, its own - /// segments numbered after it (7.2.5). Together they take less than they take alone. - #[test] - fn pages_in_one_font_share_one_dictionary() { - let font = glyphs(40, 7); - let pictures: Vec = (0..4).map(|seed| text(1200, 300, &font, 20 + seed)).collect(); - let (globals, coded) = shared_round_trip(&pictures); - assert_eq!(segments(&globals), [(0, 0, true, vec![])]); - assert_eq!(associations(&globals), [0]); - let (mut shared, mut alone) = (globals.len(), 0); - for (picture, c) in pictures.iter().zip(&coded) { - let segments = segments(c); - assert_eq!(segments.first().map(|s| s.0), Some(1), "the page's own segments follow"); - assert!(segments.iter().filter(|s| s.1 == 7).all(|s| s.3.first() == Some(&(0, false))), "{segments:?}"); - assert!(associations(c).iter().all(|page| *page == 1)); - shared += c.len(); - alone += encode_jbig2(&samples(picture), 1200, 300).unwrap().len(); - } - assert!(shared < alone, "shared {shared} bytes, alone {alone}"); - } - - /// Two pages alike place every mark from the shared dictionary and have none of their own; a - /// page with letters of its own as well keeps them in its own dictionary, referred to after the - /// shared one (its symbols' IDs follow), and the page with nothing shared has no stream. - #[test] - fn what_a_page_has_alone_stays_its_own() { - let (font, other) = (glyphs(30, 7), stemmed(&glyphs(20, 99))); - let same = text(900, 240, &font, 3); - let mut more = text(900, 240, &font, 4); - more.extend(text(900, 120, &other, 5)); - let (globals, coded) = shared_round_trip(&[same.clone(), same.clone()]); - assert!(!globals.is_empty()); - assert!(coded.iter().all(|c| segment_types(c) == [48, 7]), "{:?}", coded.iter().map(|c| segment_types(c)).collect::>()); - let with_more = |p: &Bitmap| p.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 900], 120)).collect::(); - let (_, coded) = shared_round_trip(&[with_more(&same), more]); - assert_eq!(segment_types(&coded[1])[..3], [48, 0, 7]); - assert_eq!(segments(&coded[1])[2].3, [(0, false), (2, false)]); - let alone = text(900, 360, &other, 6); - let data: Vec> = [&same, &same, &alone].iter().map(|p| samples(p)).collect(); - let images: Vec<(&[u8], u32, u32)> = vec![(&data[0], 900, 240), (&data[1], 900, 240), (&data[2], 900, 360)]; - let (_, coded) = encode_shared(&images).unwrap(); - assert!(coded[0].is_some() && coded[1].is_some() && coded[2].is_none()); - } - - /// Images that share fewer than two marks share nothing; two are enough. - #[test] - fn images_with_nothing_in_common_share_nothing() { - let font = glyphs(30, 7); - let (a, b) = (samples(&text(600, 200, &font, 1)), samples(&text(600, 200, &stemmed(&font), 1))); - assert!(encode_shared(&[(&a, 600, 200), (&b, 600, 200)]).is_none()); - assert!(encode_shared(&[(&a, 600, 200)]).is_none(), "one image"); - assert!(encode_shared(&[]).is_none()); - // The two commonest letters of the first page, below the last line of the second. - let mut two = text(600, 200, &stemmed(&font), 1); - draw(&mut two, &font[0], 10, 180); - draw(&mut two, &font[1], 100, 180); - let two = samples(&two); - let (globals, coded) = encode_shared(&[(&a, 600, 200), (&two, 600, 200)]).unwrap(); - assert_eq!(exported(&globals), 2); - assert_eq!(decode_with(coded[1].as_ref().unwrap(), Some(&globals)), unpack(&two, 600, 200)); - } - - /// The images of a group are checked as `encode_jbig2_generic` checks one, and an image it - /// would refuse refuses the group. - #[test] - fn a_group_with_an_image_the_encoder_refuses_is_refused() { - let page = samples(&text(600, 200, &glyphs(30, 7), 1)); - assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page[..10], 600, 200)]), Err(FilterError::Corrupt { .. }))); - assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 0, 200)]), Err(FilterError::Unsupported(_)))); - assert!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 600, 200)]).unwrap().is_some()); - } - - /// A group holds its distinct marks within what one image's are held to: past 65,536 a mark - /// first seen later isn't held, so isn't shared, while those held count every image they're - /// in. Ten marks of the first image are in two images after it, with 464 first seen there. - #[test] - fn past_the_marks_a_group_holds_later_ones_arent_shared() { - let (a, b) = (samples(&combs(0..40_000)), samples(&combs(40_000..65_536))); - let late = combs((0..10).chain(65_536..66_000).collect::>().into_iter()); - let (rows, late) = (late.len() as u32, samples(&late)); - let (columns, a_rows, b_rows) = (120 * 36, 40_000u32.div_ceil(120) * 4, 25_536u32.div_ceil(120) * 4); - let images = [(&a[..], columns, a_rows), (&b[..], columns, b_rows), (&late[..], columns, rows), (&late[..], columns, rows)]; - let (globals, coded) = encode_shared(&images).unwrap(); - assert_eq!(exported(&globals), 10); - for c in &coded[2..] { - assert_eq!(decode_with(c.as_ref().unwrap(), Some(&globals)), unpack(&late, columns, rows)); - } - } - - /// Past what one dictionary shares, it keeps the marks the most images hold: 20,000 shared, - /// the first 10,000 in all three images and the rest in two. The third image then places all - /// its marks from it, and the other two code the rest themselves (in a dictionary of their own - /// or a generic region, whichever is smaller). - #[test] - fn past_the_shared_limit_the_most_widely_held_marks_are_shared() { - let (both, first) = (combs(0..20_000), combs(0..10_000)); - let rows = both.len(); - let first: Bitmap = first.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 120 * 36], rows - first.len())).collect(); - let (globals, coded) = shared_round_trip(&[both.clone(), both, first]); - assert_eq!(exported(&globals), MAX_SHARED_SYMBOLS); - assert_eq!(segment_types(&coded[2]), [48, 7]); - assert!(segment_types(&coded[0]).iter().any(|t| *t == 0 || *t == 39), "{:?}", segment_types(&coded[0])); - } - - proptest::proptest! { - /// Glyphs scattered over pages of one font: whatever is shared, every page with a stream - /// decodes with the globals to its picture. - #[test] - fn shared_groups_round_trip( - columns in 1usize..200, - rows in 1usize..100, - shapes in 2usize..8, - n in 0usize..60, - pages in 2usize..5, - seed in proptest::num::u64::ANY, - ) { - let font = glyphs(shapes, seed); - let data: Vec> = (0..pages).map(|p| samples(&scatter(columns, rows, &font, seed ^ p as u64, n))).collect(); - let (c, r) = (columns as u32, rows as u32); - let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], c, r)).collect(); - if let Some((globals, coded)) = encode_shared(&images) { - for (d, coded) in data.iter().zip(coded) { - if let Some(coded) = coded { - proptest::prop_assert_eq!(decode_with(&coded, Some(&globals)), unpack(d, c, r)); - } - } - } - } - - /// Glyphs anywhere, over the edges and each other, with stray pixels: every plan decodes to - /// the picture, and `encode_jbig2` is never larger than the generic coding. - #[test] - fn scattered_marks_round_trip( - columns in 1usize..240, - rows in 1usize..120, - shapes in 2usize..10, - n in 0usize..80, - seed in proptest::num::u64::ANY, - stray in proptest::collection::vec((0usize..240, 0usize..120), 0..20), - ) { - let mut picture = scatter(columns, rows, &glyphs(shapes, seed), seed, n); - for (x, y) in stray { - if let Some(cell) = picture.get_mut(y).and_then(|r| r.get_mut(x)) { - *cell ^= 1; - } - } - every_plan_round_trips(&picture); - let data = samples(&picture); - let (c, r) = (columns as u32, rows as u32); - let coded = encode_jbig2(&data, c, r).unwrap(); - proptest::prop_assert!(coded.len() <= encode_jbig2_generic(&data, c, r).unwrap().len()); - proptest::prop_assert_eq!(decode(&coded), unpack(&data, c, r)); - } - - /// Whatever it is given, it returns. - #[test] - fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { - let _ = encode_jbig2(&data, columns, rows); - let _ = encode(&data, columns, rows); - let _ = encode_shared(&[(&data, columns, rows), (&data, columns, rows.saturating_sub(1))]); - } - } -} diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 4bee6b451c..223a17b903 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -1,958 +1,2 @@ -//! PDF stream filters (ISO 32000-2 §7.4). -//! -//! Layer L0, standalone: this crate depends on no other PdfCraft crate. -//! -//! Decoding covers the general-purpose filters (`FlateDecode`, `LZWDecode` with -//! TIFF/PNG predictors, `ASCIIHexDecode`, `ASCII85Decode`, `RunLengthDecode`). -//! Image codecs (`DCTDecode`, `JPXDecode`, `JBIG2Decode`, `CCITTFaxDecode`), -//! `Crypt` and unknown filters are reported as [`FilterError::Unsupported`] so -//! callers keep such data encoded. CCITT fax and JBIG2 images decode on their own -//! ([`decode_ccitt_within`], [`decode_jbig2_within`]), each spending from an allowance its caller -//! shares across a run of images, as `pdfcraft_cos::ImageBudget::decode` does after the filters -//! before them. Neither decodes without one: -//! -//! ```compile_fail -//! let _ = pdfcraft_filters::decode_jbig2(&[], None, 1, 1, 1 << 20); -//! ``` -//! -//! ```compile_fail -//! let _ = pdfcraft_filters::decode_ccitt(&[], &pdfcraft_filters::CcittParams::default(), 1); -//! ``` -//! -//! ``` -//! let mut work = 1 << 20; -//! assert!(pdfcraft_filters::decode_jbig2_within(&[], None, 1, 1, 1 << 20, &mut work).is_err()); -//! let p = pdfcraft_filters::CcittParams { columns: 8, ..Default::default() }; -//! assert!(pdfcraft_filters::decode_ccitt_within(&[], &p, 1, &mut work).is_ok()); -//! ``` -//! -//! 1-bit samples can be encoded as CCITT Group 4 ([`encode_g4`]) or lossless JBIG2 -//! ([`encode_jbig2`]), a group of images sharing their marks through `/JBIG2Globals` -//! ([`encode_jbig2_shared`]). -//! -//! Every decoder is bounded by a caller-supplied `max_output` (decompression-bomb -//! defence) and never panics on malformed input. -#![forbid(unsafe_code)] -#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] - -mod ascii85; -mod asciihex; -mod ccitt; -mod flate; -mod jbig2; -mod lzw; -mod mq; -mod predictor; -mod runlength; - -use std::borrow::Cow; - -pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt_within, encode_g4}; -pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, SharedJbig2, decode_jbig2_within, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; - -/// A PDF stream filter, identified by its `/Filter` name. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -pub enum Filter { - Flate, - Lzw, - AsciiHex, - Ascii85, - RunLength, - Dct, - Jpx, - Jbig2, - CcittFax, - Crypt, - /// Any other name (kept verbatim, lossily converted to UTF-8). - Unknown(String), -} - -impl Filter { - /// Parses a filter name. Accepts the full names (ISO 32000-2 Table 6) and the - /// inline-image abbreviations (Table 92): `Fl`, `LZW`, `AHx`, `A85`, `RL`, `DCT`, `CCF`. - /// A leading `/` is ignored. - pub fn from_name(name: &[u8]) -> Filter { - let n = name.strip_prefix(b"/").unwrap_or(name); - match n { - b"FlateDecode" | b"Fl" => Filter::Flate, - b"LZWDecode" | b"LZW" => Filter::Lzw, - b"ASCIIHexDecode" | b"AHx" => Filter::AsciiHex, - b"ASCII85Decode" | b"A85" => Filter::Ascii85, - b"RunLengthDecode" | b"RL" => Filter::RunLength, - b"DCTDecode" | b"DCT" => Filter::Dct, - b"JPXDecode" => Filter::Jpx, - b"JBIG2Decode" => Filter::Jbig2, - b"CCITTFaxDecode" | b"CCF" => Filter::CcittFax, - b"Crypt" => Filter::Crypt, - other => Filter::Unknown(String::from_utf8_lossy(other).into_owned()), - } - } - - /// The full PDF name (without the leading `/`). - pub fn name(&self) -> &str { - match self { - Filter::Flate => "FlateDecode", - Filter::Lzw => "LZWDecode", - Filter::AsciiHex => "ASCIIHexDecode", - Filter::Ascii85 => "ASCII85Decode", - Filter::RunLength => "RunLengthDecode", - Filter::Dct => "DCTDecode", - Filter::Jpx => "JPXDecode", - Filter::Jbig2 => "JBIG2Decode", - Filter::CcittFax => "CCITTFaxDecode", - Filter::Crypt => "Crypt", - Filter::Unknown(s) => s, - } - } - - /// True for the image codecs (DCT, JPX, JBIG2, CCITT fax). - pub fn is_image_codec(&self) -> bool { - matches!(self, Filter::Dct | Filter::Jpx | Filter::Jbig2 | Filter::CcittFax) - } - - /// True for the general-purpose filters, which a chain can hold ([`decode`]). - pub fn is_decodable(&self) -> bool { - matches!(self, Filter::Flate | Filter::Lzw | Filter::AsciiHex | Filter::Ascii85 | Filter::RunLength) - } -} - -/// `/DecodeParms` values relevant to the general-purpose filters (ISO 32000-2 Tables 8–9). -#[derive(Clone, Debug, PartialEq)] -pub struct Params { - /// 1 = none, 2 = TIFF predictor 2, 10–15 = PNG predictors. - pub predictor: i64, - pub colors: i64, - pub bits_per_component: i64, - pub columns: i64, - /// LZW only: 1 (default) switches code width one code early, 0 does not. - pub early_change: i64, -} - -impl Default for Params { - fn default() -> Self { - Params { predictor: 1, colors: 1, bits_per_component: 8, columns: 1, early_change: 1 } - } -} - -/// Errors from decoding or encoding. -#[derive(Debug, thiserror::Error)] -pub enum FilterError { - #[error("{filter}: corrupt data: {detail}")] - Corrupt { filter: &'static str, detail: String }, - #[error("decoded output exceeds the limit of {0} bytes")] - LimitExceeded(usize), - #[error("unsupported filter: {0}")] - Unsupported(String), - /// A run of images has nothing left to spend ([`decode_tolerant_within`]). - #[error("more image decoding than one operation may do")] - Spent, -} - -/// A decoder failure carrying whatever was decoded before the error. -pub(crate) struct Failure { - pub(crate) error: FilterError, - pub(crate) partial: Vec, - /// Bytes decoded on the way that `partial` doesn't hold: they took as long. - pub(crate) discarded: usize, -} - -impl Failure { - pub(crate) fn corrupt(filter: &'static str, detail: impl Into, partial: Vec) -> Self { - Failure { error: FilterError::Corrupt { filter, detail: detail.into() }, partial, discarded: 0 } - } - - pub(crate) fn limit(max: usize) -> Self { - Failure { error: FilterError::LimitExceeded(max), partial: Vec::new(), discarded: 0 } - } -} - -pub(crate) type Step = Result, Failure>; - -/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends -/// all of it. -pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> { - match work.checked_sub(steps) { - Some(left) => { - *work = left; - Ok(()) - } - None => { - *work = 0; - Err(FilterError::Spent) - } - } -} - -/// Decodes `data` through `chain` (first element applied first). -/// -/// Image codecs, `Crypt` and unknown filters anywhere in the chain give -/// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than -/// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer. -pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result, FilterError> { - run(chain, data, max_output, None, true, false, None).map(|(v, _, _)| v).map_err(|(e, _)| e) -} - -/// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages -/// decode together, each stage getting what the earlier ones left, and the result gives those -/// bytes, on success and on failure. A stage's output counts even when a later stage or its -/// predictor shrinks it. -pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), true, false, None).map(|(v, _, spent)| (v, spent)) -} - -/// Like [`decode`], but on corrupt data returns what was decoded before the error and -/// reports `partial = true` (the behaviour viewers expect). The remaining filters of the -/// chain still run on the partial data. Limit and unsupported errors are still errors. -pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool), FilterError> { - run(chain, data, max_output, None, false, false, None).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) -} - -/// [`decode_tolerant`] bounded and counted as [`decode_counted`] is. -pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), false, false, None) -} - -/// [`decode_tolerant_counted`] with the two bounds apart: `max_stage` bounds every stage, as -/// [`decode_tolerant`]'s bound does, and `max_output` the bytes all stages decode together. A -/// stage stopped by either gives [`FilterError::LimitExceeded`] and the bytes decoded before it. -pub fn decode_tolerant_counted_each( - chain: &[(Filter, Params)], - data: &[u8], - max_stage: usize, - max_output: usize, -) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_stage, Some(max_output), false, false, None) -} - -/// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images: -/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each -/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each -/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for -/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of -/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all -/// costs nothing, whatever is left, so what an image's filters are is known from the error. -/// -/// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`. -pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { - spending(chain, data, max_output, work, false, None) -} - -/// [`decode_tolerant_within`] for the samples of an image `rows` high (its `/Height`), which the -/// last stage gives. When that stage is Flate with a PNG predictor and gives exactly `rows` rows -/// without the filter-type byte each predicted row starts with (ISO 32000-2 §7.4.4.4), its writer -/// declared the predictor without applying it: the output is taken as the samples, as other -/// readers take it, and reported as the damage it is, like any other corruption tolerated here. -/// Any other output has its predictor undone as [`decode_tolerant_within`] does. -pub fn decode_image_tolerant_within( - chain: &[(Filter, Params)], - data: &[u8], - max_output: usize, - rows: u32, - work: &mut u64, -) -> Result, FilterError> { - spending(chain, data, max_output, work, false, usize::try_from(rows).ok()) -} - -/// [`decode`], spending from `work` as [`decode_tolerant_within`] does: corrupt data is an error, -/// having paid for what was decoded before it. -pub fn decode_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { - spending(chain, data, max_output, work, true, None) -} - -fn spending( - chain: &[(Filter, Params)], - data: &[u8], - max_output: usize, - work: &mut u64, - strict: bool, - rows: Option, -) -> Result, FilterError> { - decodable(chain)?; - pay(work, data.len() as u64)?; - if chain.is_empty() { - return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) }; - } - let left = usize::try_from(*work).unwrap_or(usize::MAX); - match run(chain, data, max_output, Some(left), strict, true, rows) { - Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out), - // The stage stopped at the lower of the two, which took as much to find out. - Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => { - pay(work, (decoded as u64).saturating_add(max_output as u64))?; - Err(FilterError::LimitExceeded(max_output)) - } - // Past what is left, a stage stops, or a Flate stage doesn't start. - Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => { - *work = 0; - Err(FilterError::Spent) - } - Err((e, decoded)) => pay(work, decoded as u64).and(Err(e)), - } -} - -/// Longest filter chain decoded. Real streams use one to three filters; every stage may produce -/// up to the output limit, so the chain length multiplies the work. -pub const MAX_CHAIN: usize = 8; - -/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], -/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes. -/// Inflating does work for every block it reads, whether or not the block makes anything: an empty -/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs -/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read -/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the -/// worst per byte read comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. -/// (miniz_oxide 0.9.1, flate2's backend here before, took about 1.7 µs for every fixed-Huffman -/// block, empty or not: 1,383 ns a byte read on empty ones.) A stage's start isn't priced, about -/// 0.25 µs: every way to repeat one takes bytes of the file, so what starts cost is bounded by the -/// file's length. Data a real encoder makes took 0.2 to 2.3 ns a byte made, its reading included, so -/// an image pays twice its stored length more than it would unpriced. -pub const FLATE_READ_WORK: u64 = 2; - -/// Whether a chain is one [`run`] takes, and if not, the error. -fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { - if chain.len() > MAX_CHAIN { - return Err(FilterError::Unsupported(format!("a chain of {} filters", chain.len()))); - } - match chain.iter().find(|(f, _)| !f.is_decodable()) { - Some((f, _)) => Err(FilterError::Unsupported(f.name().to_owned())), - None => Ok(()), - } -} - -/// The decoded data, whether it is partial and the bytes all stages decoded, or the error and -/// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. With -/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it -/// reads, before it runs, so a stage `total` can't pay for doesn't run -/// ([`FilterError::Spent`]). `rows` is the height of the image whose samples the last stage -/// produces, if it does ([`decode_image_tolerant_within`]). -fn run( - chain: &[(Filter, Params)], - data: &[u8], - max: usize, - total: Option, - strict: bool, - priced: bool, - rows: Option, -) -> Result<(Vec, bool, usize), (FilterError, usize)> { - decodable(chain).map_err(|e| (e, 0))?; - let mut cur: Cow<[u8]> = Cow::Borrowed(data); - let mut partial = false; - let mut spent = 0usize; - let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX); - for (i, (filter, params)) in chain.iter().enumerate() { - let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 }; - let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent)); - if read > room { - return Err((FilterError::Spent, spent)); - } - spent = spent.saturating_add(read); - #[cfg(test)] - STAGE_BYTES.with(|c| c.set(c.get() + cur.len())); - let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent))); - // A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and - // makes no more than is left once it has been. - let rows = rows.filter(|_| i + 1 == chain.len()); - let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left)); - let (step, work, again) = decode_one(filter, params, &cur, limit, again, rows); - if again { - spent = spent.saturating_add(read); - } - spent = spent.saturating_add(work); - cur = Cow::Owned(match step { - Ok(v) => v, - Err(Failure { error: FilterError::Corrupt { .. }, partial: out, .. }) if !strict => { - partial = true; - out - } - Err(Failure { error: FilterError::LimitExceeded(_), .. }) => return Err((FilterError::LimitExceeded(max), spent)), - Err(f) => return Err((f.error, spent)), - }); - } - Ok((cur.into_owned(), partial, spent)) -} - -/// One stage's result, the bytes its decoder produced (before any predictor), and whether it -/// read its input twice, which it does only within `again` ([`flate::decode`]); `rows` as for -/// [`run`]. -fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option, rows: Option) -> (Step, usize, bool) { - let once = |(step, n): (Step, usize)| (step, n, false); - match filter { - Filter::Flate => { - let (step, twice) = flate::decode(data, max, again); - let (step, n) = with_predictor(step, params, "FlateDecode", rows); - (step, n, twice) - } - // LZW rows are predicted whatever their length, as the renderer reads them. - Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode", None)), - Filter::AsciiHex => once(counted(asciihex::decode(data, max))), - Filter::Ascii85 => once(counted(ascii85::decode(data, max))), - Filter::RunLength => once(counted(runlength::decode(data, max))), - other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false), - } -} - -/// A step with the bytes it produced: its output, or its partial output and what it threw away. -fn counted(step: Step) -> (Step, usize) { - let n = match &step { - Ok(v) => v.len(), - Err(f) => f.partial.len().saturating_add(f.discarded), - }; - (step, n) -} - -/// Applies the predictor to a decoder's result, including to partial output on failure. The -/// count is the decoder's output: a PNG predictor drops a tag byte per row, but the decoder -/// produced it. `rows` (an image's height, for its last stage) applies only to data that decoded -/// whole: data cut short by damage has its predictor undone as it always had. -fn with_predictor(step: Step, params: &Params, name: &'static str, rows: Option) -> (Step, usize) { - let (step, n) = counted(step); - let step = match step { - Ok(v) => predictor::decode(params, v, name, rows), - Err(Failure { error, partial, discarded }) => { - let partial = match predictor::decode(params, partial, name, None) { - Ok(v) => v, - Err(f) => f.partial, - }; - Err(Failure { error, partial, discarded }) - } - }; - (step, n) -} - -/// Encodes `data` with `filter`. Supported: Flate (zlib, level 6), LZW, ASCIIHex, -/// ASCII85, RunLength. For Flate and LZW, `params.predictor >= 2` applies the -/// predictor first (TIFF 2, PNG 10–15; 15 chooses the per-row optimum). -pub fn encode(filter: &Filter, params: &Params, data: &[u8]) -> Result, FilterError> { - match filter { - Filter::Flate => Ok(flate::encode(&predictor::encode(params, data)?)), - Filter::Lzw => Ok(lzw::encode(&predictor::encode(params, data)?, params.early_change != 0)), - Filter::AsciiHex => Ok(asciihex::encode(data)), - Filter::Ascii85 => Ok(ascii85::encode(data)), - Filter::RunLength => Ok(runlength::encode(data)), - other => Err(FilterError::Unsupported(format!("encoding with {}", other.name()))), - } -} - -/// zlib (RFC 1950) at compression level 6. -pub fn encode_flate(data: &[u8]) -> Vec { - flate::encode(data) -} - -/// PDF white-space characters (ISO 32000-2 Table 1). -pub(crate) fn is_pdf_whitespace(b: u8) -> bool { - matches!(b, 0 | 9 | 10 | 12 | 13 | 32) -} - -#[cfg(test)] -thread_local! { - /// Bytes handed to decoding stages on this thread. - static STAGE_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -#[cfg(test)] -mod tests { - #[test] - fn a_long_filter_chain_is_refused_before_decoding() { - // `FF FF` run-length decodes to itself, so every stage of the chain repeats the work. - let data = vec![0xFF; 4096]; - let chain = |n: usize| vec![(super::Filter::RunLength, super::Params::default()); n]; - super::STAGE_BYTES.with(|c| c.set(0)); - let long = super::decode(&chain(1000), &data, 1 << 20); - let read = super::STAGE_BYTES.with(|c| c.get()); - assert!(read <= 16 * data.len(), "{read} bytes decoded through the chain"); - assert!(matches!(long, Err(super::FilterError::Unsupported(_))), "{long:?}"); - assert_eq!(super::decode(&chain(super::MAX_CHAIN), &data, 1 << 20).ok(), Some(data)); - } - - /// [`decode_tolerant_counted_each`] holds every stage to its own bound as well as the stages - /// to their total: a Flate stage that would make more than that bound stops there, though - /// the total has room, and the stage after it gets nothing; within both, the chain reads - /// whole, one stage's output charged on top of another's. - #[test] - fn each_stage_is_held_to_its_own_bound_within_the_total() { - let hex = "00".repeat(1 << 16).into_bytes(); - let data = super::flate::encode(&hex); - let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; - // The total alone lets the Flate stage make all of the hex text, and the hex stage read it. - STAGE_BYTES.with(|c| c.set(0)); - let whole = decode_tolerant_counted(&chain, &data, 3 << 16).map(|(v, _, _)| v); - assert_eq!(whole.ok(), Some(vec![0u8; 1 << 16])); - assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len() + hex.len()); - // Each stage held to 64 KiB, the Flate stage stops there and the hex stage gets nothing. - STAGE_BYTES.with(|c| c.set(0)); - let held = decode_tolerant_counted_each(&chain, &data, 1 << 16, 3 << 16); - assert!(matches!(held, Err((FilterError::LimitExceeded(m), 0)) if m == 1 << 16), "{held:?}"); - assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len()); - // Within both, it reads whole. - let (out, _, spent) = decode_tolerant_counted_each(&chain, &data, 2 << 16, 3 << 16).unwrap(); - assert_eq!((out.len(), spent), (1 << 16, hex.len() + (1 << 16))); - } - - use super::*; - - #[test] - fn names_and_abbreviations() { - for (full, abbr, f) in [ - (&b"FlateDecode"[..], &b"Fl"[..], Filter::Flate), - (b"LZWDecode", b"LZW", Filter::Lzw), - (b"ASCIIHexDecode", b"AHx", Filter::AsciiHex), - (b"ASCII85Decode", b"A85", Filter::Ascii85), - (b"RunLengthDecode", b"RL", Filter::RunLength), - (b"DCTDecode", b"DCT", Filter::Dct), - (b"CCITTFaxDecode", b"CCF", Filter::CcittFax), - ] { - assert_eq!(Filter::from_name(full), f); - assert_eq!(Filter::from_name(abbr), f); - assert_eq!(f.name().as_bytes(), full); - } - assert_eq!(Filter::from_name(b"/JPXDecode"), Filter::Jpx); - assert_eq!(Filter::from_name(b"JBIG2Decode"), Filter::Jbig2); - assert_eq!(Filter::from_name(b"Crypt"), Filter::Crypt); - assert_eq!(Filter::from_name(b"Foo"), Filter::Unknown("Foo".into())); - assert_eq!(Filter::Unknown("Foo".into()).name(), "Foo"); - assert!(Filter::Dct.is_image_codec() && Filter::CcittFax.is_image_codec()); - assert!(!Filter::Flate.is_image_codec() && !Filter::Crypt.is_image_codec()); - } - - #[test] - fn default_params() { - let p = Params::default(); - assert_eq!((p.predictor, p.colors, p.bits_per_component, p.columns, p.early_change), (1, 1, 8, 1, 1)); - } - - #[test] - fn unsupported_filters() { - for f in [Filter::Dct, Filter::Jpx, Filter::Jbig2, Filter::CcittFax, Filter::Crypt, Filter::Unknown("X".into())] { - let chain = [(Filter::AsciiHex, Params::default()), (f.clone(), Params::default())]; - assert!(matches!(decode(&chain, b"00>", 100), Err(FilterError::Unsupported(_)))); - assert!(matches!(decode_tolerant(&chain, b"00>", 100), Err(FilterError::Unsupported(_)))); - assert!(matches!(encode(&f, &Params::default(), b"x"), Err(FilterError::Unsupported(_)))); - } - } - - #[test] - fn empty_chain_is_identity() { - assert_eq!(decode(&[], b"abc", 1).unwrap(), b"abc"); - } - - #[test] - fn chain_ascii85_flate() { - let data = b"Hello, chained filters! Hello, chained filters!".repeat(10); - let z = encode_flate(&data); - let a = encode(&Filter::Ascii85, &Params::default(), &z).unwrap(); - let chain = [(Filter::Ascii85, Params::default()), (Filter::Flate, Params::default())]; - assert_eq!(decode(&chain, &a, 1 << 20).unwrap(), data); - } - - #[test] - fn chain_three_filters_with_predictor() { - let p = Params { predictor: 12, colors: 3, columns: 5, ..Params::default() }; - let data: Vec = (0..300u32).map(|i| (i * 7 % 251) as u8).collect(); - let lzw = encode(&Filter::Lzw, &p, &data).unwrap(); - let rl = encode(&Filter::RunLength, &Params::default(), &lzw).unwrap(); - let hex = encode(&Filter::AsciiHex, &Params::default(), &rl).unwrap(); - let chain = [(Filter::AsciiHex, Params::default()), (Filter::RunLength, Params::default()), (Filter::Lzw, p)]; - assert_eq!(decode(&chain, &hex, 1 << 20).unwrap(), data); - } - - #[test] - fn limit_applies_to_intermediate_buffers() { - let hex = encode(&Filter::AsciiHex, &Params::default(), &[7u8; 100]).unwrap(); - let chain = [(Filter::AsciiHex, Params::default())]; - assert!(matches!(decode(&chain, &hex, 99), Err(FilterError::LimitExceeded(99)))); - assert_eq!(decode(&chain, &hex, 100).unwrap().len(), 100); - } - - #[test] - fn raw_flate_fallback_preserves_output_limits() { - // RFC 1951 stored blocks: the first is nonfinal with LEN=257 and - // NLEN=!257, followed by a final empty block. Its ignored alignment - // bits make the first two bytes also look like a valid zlib header. - let data = vec![b'A'; 257]; - let mut raw = vec![0x78, 0x01, 0x01, 0xfe, 0xfe]; - raw.extend_from_slice(&data); - raw.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]); - let chain = [(Filter::Flate, Params::default())]; - for max in [0, 1, 128, 256] { - assert!(matches!(decode(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max)); - assert!(matches!(decode_tolerant(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max)); - } - for max in [257, 258] { - assert_eq!(decode(&chain, &raw, max).unwrap(), data); - assert_eq!(decode_tolerant(&chain, &raw, max).unwrap(), (data.clone(), false)); - } - } - - #[test] - fn tolerant_flate_returns_partial() { - let data: Vec = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect(); - let mut z = encode_flate(&data); - let cut = z.len() / 2; - z.truncate(cut); - let chain = [(Filter::Flate, Params::default())]; - assert!(matches!(decode(&chain, &z, 1 << 20), Err(FilterError::Corrupt { .. }))); - let (out, partial) = decode_tolerant(&chain, &z, 1 << 20).unwrap(); - assert!(partial); - assert!(!out.is_empty() && out.len() < data.len()); - assert_eq!(&data[..out.len()], &out[..]); - // Complete data is not partial. - let (out, partial) = decode_tolerant(&chain, &encode_flate(&data), 1 << 20).unwrap(); - assert!(!partial); - assert_eq!(out, data); - } - - #[test] - fn tolerant_partial_flows_through_predictor() { - let p = Params { predictor: 12, columns: 4, ..Params::default() }; - let data: Vec = (0..4000u32).map(|i| (i * 31 % 256) as u8).collect(); - let mut z = encode(&Filter::Flate, &p, &data).unwrap(); - z.truncate(z.len() - 20); - let (out, partial) = decode_tolerant(&[(Filter::Flate, p)], &z, 1 << 20).unwrap(); - assert!(partial); - assert_eq!(&data[..out.len()], &out[..]); - } - - /// A counted decode bounds what all stages decode together: Flate inflating 1 MiB of hex - /// that ASCIIHex halves is 1.5 MiB of work, past a 1.25 MiB allowance. A plain decode keeps - /// the limit per stage. - #[test] - fn a_counted_decode_bounds_all_stages_together() { - let hex: Vec = std::iter::repeat_n(b"ab".as_slice(), 1 << 19).flatten().copied().collect(); - let data = encode_flate(&hex); - let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; - let max = (1 << 20) + (1 << 18); - assert_eq!(decode(&chain, &data, max).map(|d| d.len()).ok(), Some(1 << 19)); - assert!(matches!(decode_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max)); - assert_eq!(decode_counted(&chain, &data, 2 << 20).map(|(_, spent)| spent).ok(), Some((1 << 20) + (1 << 19))); - let tolerant = decode_tolerant_counted(&chain, &data, 2 << 20).map(|(out, partial, spent)| (out.len(), partial, spent)); - assert_eq!(tolerant.ok(), Some((1 << 19, false, (1 << 20) + (1 << 19)))); - assert!(matches!(decode_tolerant_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max)); - } - - /// A PNG predictor drops a tag byte per row, but Flate produced it: the count includes it, - /// also when the predictor fails on a bad tag. - #[test] - fn a_counted_decode_counts_output_before_the_predictor() { - let params = Params { predictor: 12, ..Params::default() }; - let rows = vec![7u8; 1 << 16]; - let data = encode(&Filter::Flate, ¶ms, &rows).unwrap(); - let chain = [(Filter::Flate, params.clone())]; - let (out, spent) = decode_counted(&chain, &data, usize::MAX).unwrap(); - assert!(out == rows); - assert_eq!(spent, 2 << 16); - let bad: Vec = std::iter::repeat_n([0u8, 7], 1 << 15).flatten().chain([9, 7]).collect(); - let (e, spent) = decode_counted(&chain, &encode_flate(&bad), usize::MAX).unwrap_err(); - assert!(matches!(e, FilterError::Corrupt { .. }), "{e}"); - assert_eq!(spent, bad.len()); - } - - fn refused(e: &FilterError) -> bool { - matches!(e, FilterError::Spent) - } - - /// A chain pays for what it reads and what each stage decodes, down to the byte: Flate inflates - /// hex that ASCIIHex halves, so a stage counts when a later one shrinks it. - #[test] - fn a_chain_spends_down_to_the_byte() { - let hex = b"ab".repeat(1 << 10); - let data = encode_flate(&hex); - let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; - let whole = decode(&chain, &data, 1 << 20).unwrap(); - let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64; - let mut work = u64::MAX; - assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole); - assert_eq!(u64::MAX - work, cost); - let mut work = cost; - assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole, "exactly enough"); - assert_eq!(work, 0); - let mut work = cost + 7; - decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(); - assert_eq!(work, 7, "only what it took"); - let mut work = cost - 1; - assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()), "a step short"); - assert_eq!(work, 0, "spends all that was left"); - // Each stage has `max_output` to itself, as in a plain decode: together they are past it. - let mut work = u64::MAX; - assert_eq!(decode_tolerant_within(&chain, &data, hex.len(), &mut work).unwrap(), whole); - } - - /// No stage runs past what is left: the first stage of this bomb stops there, so the second - /// isn't given anything to read, and with too little to read it nothing runs at all. - #[test] - fn a_chain_stops_at_what_is_left() { - let inner = encode_flate(&vec![0; 1 << 20]); - let bomb = encode_flate(&inner); - let chain = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; - let read = |work: u64| { - super::STAGE_BYTES.with(|c| c.set(0)); - let mut work = work; - assert!(refused(&decode_tolerant_within(&chain, &bomb, usize::MAX, &mut work).unwrap_err())); - assert_eq!(work, 0); - super::STAGE_BYTES.with(|c| c.get()) - }; - let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64; - assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops"); - assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read"); - assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs"); - assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read"); - assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read"); - } - - /// Past `max_output` the chain has failed, and hasn't run out of anything: it pays what it took - /// to find out, and what is left stays. Whichever limit a stage stops at is the one reported. - #[test] - fn a_stage_past_its_limit_pays_for_finding_out() { - let data = encode_flate(&vec![0; 1 << 16]); - let chain = [(Filter::Flate, Params::default())]; - let mut work = u64::MAX; - assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); - let read = |n: usize| FLATE_READ_WORK * n as u64; - assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000); - // The later stage of two, which pays for the first as well. - let bomb = encode_flate(&data); - let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; - let mut work = u64::MAX; - assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_)))); - assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len())); - // Both limits at once: a run that can't go on is what a caller needs to hear. - let mut work = data.len() as u64 + read(data.len()) + 1000; - assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err())); - assert_eq!(work, 0); - let mut work = data.len() as u64 + read(data.len()) + 1001; - assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); - assert_eq!(work, 1, "the image's limit, with a step to spare"); - } - - /// What a damaged stream decoded is paid for, however the decode ends. - #[test] - fn a_chain_that_fails_has_paid_for_what_it_did() { - let data: Vec = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect(); - let mut cut = encode_flate(&data); - cut.truncate(cut.len() / 2); - let chain = [(Filter::Flate, Params::default())]; - let (partial, _) = decode_tolerant(&chain, &cut, 1 << 20).unwrap(); - assert!(!partial.is_empty()); - let mut work = u64::MAX; - assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial); - let read = |n: usize| FLATE_READ_WORK * n as u64; - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); - // Read strictly it is an error, for the same work. - let mut work = u64::MAX; - assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. }))); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); - let whole = encode_flate(&data); - let mut work = u64::MAX; - assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data); - assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len())); - // A predictor that comes to nothing, after all of this was inflated for it. - let p = Params { predictor: 99, ..Params::default() }; - let data = encode_flate(&vec![0; 1 << 16]); - let mut work = u64::MAX; - assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16)); - } - - /// A stream that reads as zlib only until it fails, and as raw deflate until that fails too, - /// gives back no data, and has still decoded everything the retry did. - #[test] - fn a_retry_that_fails_pays_for_what_it_decoded() { - // Stored as raw deflate: this many bytes, then a block of a reserved type. As zlib: the - // same reserved type at once, so the first reading fails with nothing. - let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; - data.extend(vec![0u8; 0x0601]); - data.push(0x07); - let chain = [(Filter::Flate, Params::default())]; - let (out, partial) = decode_tolerant(&chain, &data, 1 << 20).unwrap(); - assert!(out.is_empty() && partial, "what is read stays as it was"); - let (_, _, spent) = decode_tolerant_counted(&chain, &data, 1 << 20).unwrap(); - assert_eq!(spent, 0x0601); - let (e, spent) = decode_counted(&chain, &data, 1 << 20).unwrap_err(); - assert!(matches!(e, FilterError::Corrupt { .. }), "{e}"); - assert_eq!(spent, 0x0601); - let mut work = u64::MAX; - assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty()); - // Read twice: as zlib, then as raw deflate. - assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64); - // Through a predictor, which would otherwise lose the count. - let p = Params { predictor: 12, columns: 4, ..Params::default() }; - let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap(); - assert_eq!(spent, 0x0601); - } - - /// Inflating does work for every block it reads, whether or not the block makes anything: empty - /// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them - /// is paid for at Flate's price before any is read. - #[test] - fn a_flate_stage_pays_for_what_it_reads() { - // Four empty blocks in five bytes, as raw deflate, then an empty final block. - let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12); - hollow.extend([0x03, 0x00]); - let chain = [(Filter::Flate, Params::default())]; - assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b""); - let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64; - let mut work = u64::MAX; - assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b""); - assert_eq!(u64::MAX - work, cost); - let mut work = cost; - assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough"); - assert_eq!(work, 0); - super::STAGE_BYTES.with(|c| c.set(0)); - let mut work = cost - 1; - assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err())); - assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated"); - // Counted for a byte allowance, as a page's content is, reading costs nothing. - assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0)); - } - - /// A zlib header that only looked valid is read again as raw deflate, and the second reading - /// is paid for as the first was. One that can't be paid for is refused, not read as damaged. - #[test] - fn a_second_reading_is_paid_for() { - // As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many - // zero bytes, then an empty final block. - let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; - data.extend(vec![0u8; 0x0601]); - data.extend([0x03, 0x00]); - let chain = [(Filter::Flate, Params::default())]; - assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]); - let read = FLATE_READ_WORK * data.len() as u64; - let mut work = (data.len() + 0x0601) as u64 + 2 * read; - assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]); - assert_eq!(work, 0, "exactly enough"); - // Enough to read it once and not twice: it isn't read again. - super::flate::INFLATED.with(|c| c.set(0)); - let mut work = data.len() as u64 + 2 * read - 1; - assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); - assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0)); - // Enough to read it twice, and a byte short of what it makes: the second reading makes no - // more than is left once it has been paid for. - super::flate::INFLATED.with(|c| c.set(0)); - let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1; - assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); - assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short"); - } - - /// With nothing left nothing is read. What can't be decoded at all never costs anything, and - /// says so whatever is left. Without filters, what is read is the data. - #[test] - fn what_a_chain_is_shows_whatever_is_left() { - let flate = [(Filter::Flate, Params::default())]; - let mut work = 0; - assert!(refused(&decode_tolerant_within(&flate, b"garbage", 1 << 20, &mut work).unwrap_err())); - let long = vec![(Filter::Flate, Params::default()); MAX_CHAIN + 1]; - for chain in [vec![(Filter::Dct, Params::default())], vec![(Filter::Jbig2, Params::default())], long] { - for left in [0, 100] { - let mut work = left; - let e = decode_tolerant_within(&chain, b"data", 1 << 20, &mut work).unwrap_err(); - assert!(matches!(e, FilterError::Unsupported(_)) && !refused(&e), "{e}"); - assert_eq!(work, left); - } - } - let mut work = 100; - assert_eq!(decode_tolerant_within(&[], b"data", 4, &mut work).unwrap(), b"data"); - assert_eq!(work, 96); - assert!(matches!(decode_tolerant_within(&[], b"data", 3, &mut work), Err(FilterError::LimitExceeded(3)))); - assert_eq!(work, 92); - let mut work = 3; - assert!(refused(&decode_tolerant_within(&[], b"data", 4, &mut work).unwrap_err())); - assert_eq!(work, 0); - } - - /// Four rows of three RGB pixels, every sample above 4 (none reads as a PNG filter type). - fn rgb_rows() -> (Params, Vec) { - let p = Params { predictor: 15, colors: 3, columns: 3, ..Params::default() }; - (p, (0..36u8).map(|i| 200 - i).collect()) - } - - #[test] - fn an_image_whose_rows_carry_no_png_tags_is_read_as_its_samples_and_reported() { - let (p, samples) = rgb_rows(); - let chain = [(Filter::Flate, p.clone())]; - let data = encode_flate(&samples); - let mut work = u64::MAX; - assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut work).unwrap(), samples); - // The repair is reported as the damage it is: tolerant decoding says so, and strict refuses. - let (out, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap(); - assert_eq!((out, partial), (samples.clone(), true)); - let strict = run(&chain, &data, 1 << 20, None, true, false, Some(4)).map(|_| ()).map_err(|(e, _)| e.to_string()); - assert!(strict.as_ref().is_err_and(|e| e.contains("carry no filter-type bytes")), "{strict:?}"); - // The same data that isn't known to be an image's rows still goes through the predictor. - let mut work = u64::MAX; - assert_ne!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), samples); - // LZW is read as before, as the renderer reads it. - let lzw = [(Filter::Lzw, p)]; - let data = encode(&Filter::Lzw, &Params::default(), &samples).unwrap(); - let mut plain = u64::MAX; - assert_eq!( - decode_image_tolerant_within(&lzw, &data, 1 << 20, 4, &mut work).ok(), - decode_tolerant_within(&lzw, &data, 1 << 20, &mut plain).ok() - ); - } - - #[test] - fn an_image_whose_rows_carry_png_tags_has_its_predictor_undone() { - let (p, samples) = rgb_rows(); - let chain = [(Filter::Flate, p.clone())]; - let data = encode(&Filter::Flate, &p, &samples).unwrap(); - let (mut a, mut b) = (u64::MAX, u64::MAX); - assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).unwrap(), samples); - assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut b).unwrap(), samples); - assert_eq!(a, b, "the same work"); - let (_, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap(); - assert!(!partial); - } - - #[test] - fn image_data_of_neither_length_decodes_as_it_did() { - let (p, _) = rgb_rows(); - let chain = [(Filter::Flate, p.clone())]; - for len in [0, 1, 9, 27, 35, 37, 39, 40, 41, 72] { - let data = encode_flate(&(0..len).map(|i| (i * 7 % 5) as u8).collect::>()); - let (mut a, mut b) = (u64::MAX, u64::MAX); - let image = decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).map_err(|e| e.to_string()); - let plain = decode_tolerant_within(&chain, &data, 1 << 20, &mut b).map_err(|e| e.to_string()); - assert_eq!(image, plain, "{len} bytes"); - } - // Rows of the untagged length for another height, or no height. - let samples = encode_flate(&[9; 36]); - for rows in [0, 3, 5, u32::MAX] { - let (mut a, mut b) = (u64::MAX, u64::MAX); - assert_eq!( - decode_image_tolerant_within(&chain, &samples, 1 << 20, rows, &mut a).ok(), - decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok(), - "{rows} rows" - ); - } - // Only the last stage produces the rows: an earlier one is predicted as before. - let hex: Vec = [9u8; 36].iter().flat_map(|b| format!("{b:02x}").into_bytes()).collect(); - let earlier = [(Filter::Flate, p.clone()), (Filter::AsciiHex, Params::default())]; - let data = encode_flate(&hex[..36]); - let (mut a, mut b) = (u64::MAX, u64::MAX); - assert_eq!( - decode_image_tolerant_within(&earlier, &data, 1 << 20, 4, &mut a).ok(), - decode_tolerant_within(&earlier, &data, 1 << 20, &mut b).ok() - ); - // TIFF predictor 2 and no predictor are left alone. - for predictor in [1, 2] { - let chain = [(Filter::Flate, Params { predictor, ..p.clone() })]; - let (mut a, mut b) = (u64::MAX, u64::MAX); - assert_eq!( - decode_image_tolerant_within(&chain, &samples, 1 << 20, 4, &mut a).ok(), - decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok() - ); - } - } - - #[test] - fn image_rows_of_no_bytes_or_past_memory_do_not_panic() { - for (colors, bits_per_component, columns) in [(0, 8, 3), (3, 8, 0), (3, 8, -1), (255, 16, i64::MAX), (1, 1, i64::MAX), (3, 3, 3)] { - let chain = [(Filter::Flate, Params { predictor: 15, colors, bits_per_component, columns, early_change: 1 })]; - for len in [0, 1, 36] { - let data = encode_flate(&vec![7; len]); - for rows in [0, 1, 4, u32::MAX] { - let mut work = u64::MAX; - let _ = decode_image_tolerant_within(&chain, &data, 1 << 20, rows, &mut work); - } - } - } - } -} +//! PDF stream filters (`craft-pdf::filters`). +pub use craft_pdf::filters::*; diff --git a/crates/filters/src/lzw.rs b/crates/filters/src/lzw.rs deleted file mode 100644 index 9653ac2fbe..0000000000 --- a/crates/filters/src/lzw.rs +++ /dev/null @@ -1,314 +0,0 @@ -//! LZWDecode (ISO 32000-2 §7.4.4.2): variable-width codes of 9–12 bits, MSB first, -//! 256 = clear-table, 257 = EOD, first free code 258. With `EarlyChange` 1 (the -//! default) the code width grows one code early. -//! -//! Tolerated: data ending without EOD, a missing initial clear-table code, a table that -//! fills up without a clear-table code (entries are then no longer added). - -use std::collections::HashMap; - -use crate::{Failure, Step}; - -const NAME: &str = "LZWDecode"; -const CLEAR: u16 = 256; -const EOD: u16 = 257; -const FIRST: usize = 258; -const TABLE: usize = 4096; -/// The encoder emits clear-table when its next free code reaches this value. -const ENCODER_RESET_AT: usize = TABLE; - -/// The code width a decoder uses when its next free code is `next`. -fn width(next: usize, early: bool) -> u32 { - match next + usize::from(early) { - 0..512 => 9, - 512..1024 => 10, - 1024..2048 => 11, - _ => 12, - } -} - -struct BitReader<'a> { - data: &'a [u8], - pos: usize, - acc: u32, - bits: u32, -} - -impl BitReader<'_> { - fn read(&mut self, n: u32) -> Option { - while self.bits < n { - let &b = self.data.get(self.pos)?; - self.pos += 1; - self.acc = (self.acc << 8) | u32::from(b); - self.bits += 8; - } - self.bits -= n; - let v = (self.acc >> self.bits) & ((1 << n) - 1); - self.acc &= (1 << self.bits) - 1; - Some(v as u16) - } -} - -pub(crate) fn decode(data: &[u8], early: bool, max: usize) -> Step { - // Table entry = (prefix code, last byte, first byte, length). - let mut prefix = vec![0u16; TABLE]; - let mut last = vec![0u8; TABLE]; - let mut first = vec![0u8; TABLE]; - let mut len = vec![0u16; TABLE]; - for i in 0..256 { - last[i] = i as u8; - first[i] = i as u8; - len[i] = 1; - } - let mut next = FIRST; - let mut w = 9; - let mut prev: Option = None; - let mut r = BitReader { data, pos: 0, acc: 0, bits: 0 }; - let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); - - while let Some(code) = r.read(w) { - match code { - CLEAR => { - next = FIRST; - w = 9; - prev = None; - continue; - } - EOD => break, - _ => {} - } - let code = usize::from(code); - if let Some(p) = prev { - let head = if code < next { - first[code] - } else if code == next && next < TABLE { - first[p] - } else { - return Err(Failure::corrupt(NAME, format!("code {code} not in table (next {next})"), out)); - }; - if next < TABLE { - prefix[next] = p as u16; - last[next] = head; - first[next] = first[p]; - len[next] = len[p] + 1; - next += 1; - } - } else if code >= 256 { - return Err(Failure::corrupt(NAME, format!("code {code} before any table entry"), out)); - } - // Emit the string for `code`, walking the prefix chain backwards. - let n = usize::from(len[code]); - if out.len() + n > max { - return Err(Failure::limit(max)); - } - let start = out.len(); - out.resize(start + n, 0); - let mut c = code; - for slot in out[start..].iter_mut().rev() { - *slot = last[c]; - c = usize::from(prefix[c]); - } - prev = Some(code); - w = width(next, early); - } - Ok(out) -} - -struct BitWriter { - out: Vec, - acc: u32, - bits: u32, -} - -impl BitWriter { - fn write(&mut self, code: u16, n: u32) { - self.acc = (self.acc << n) | u32::from(code); - self.bits += n; - while self.bits >= 8 { - self.bits -= 8; - self.out.push((self.acc >> self.bits) as u8); - } - self.acc &= (1 << self.bits) - 1; - } - - fn finish(mut self) -> Vec { - if self.bits > 0 { - self.out.push((self.acc << (8 - self.bits)) as u8); - } - self.out - } -} - -/// Greedy LZW with a leading clear-table code and a trailing EOD. -pub(crate) fn encode(data: &[u8], early: bool) -> Vec { - let mut wr = BitWriter { out: Vec::with_capacity(data.len() / 2 + 8), acc: 0, bits: 0 }; - let mut dict: HashMap<(u16, u8), u16> = HashMap::new(); - // `next` is the encoder's next free code; `dnext` mirrors the decoder's, which lags - // one entry behind (the decoder adds an entry only once it sees the following code). - let mut next = FIRST; - let mut dnext = FIRST; - let mut codes_since_clear = 0usize; - wr.write(CLEAR, 9); - let emit = |wr: &mut BitWriter, code: u16, dnext: &mut usize, since: &mut usize| { - wr.write(code, width(*dnext, early)); - if *since > 0 && *dnext < TABLE { - *dnext += 1; - } - *since += 1; - }; - let Some((&b0, rest)) = data.split_first() else { - wr.write(EOD, 9); - return wr.finish(); - }; - let mut cur = u16::from(b0); - for &b in rest { - if let Some(&c) = dict.get(&(cur, b)) { - cur = c; - continue; - } - emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); - dict.insert((cur, b), next as u16); - next += 1; - if next == ENCODER_RESET_AT { - wr.write(CLEAR, width(dnext, early)); - dict.clear(); - next = FIRST; - dnext = FIRST; - codes_since_clear = 0; - } - cur = u16::from(b); - } - emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); - wr.write(EOD, width(dnext, early)); - wr.finish() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - - fn dec(d: &[u8], early: bool) -> Result, FilterError> { - decode(d, early, 1 << 26).map_err(|f| f.error) - } - - /// ISO 32000-2 §7.4.4.2 example: codes 256 45 258 258 65 259 66 257. - const SPEC_IN: [u8; 10] = [45, 45, 45, 45, 45, 65, 45, 45, 45, 66]; - const SPEC_OUT: [u8; 9] = [0x80, 0x0B, 0x60, 0x50, 0x22, 0x0C, 0x0C, 0x85, 0x01]; - - #[test] - fn spec_example() { - assert_eq!(dec(&SPEC_OUT, true).unwrap(), SPEC_IN); - assert_eq!(encode(&SPEC_IN, true), SPEC_OUT); - // Below 511 codes the early-change setting makes no difference. - assert_eq!(dec(&SPEC_OUT, false).unwrap(), SPEC_IN); - } - - #[test] - fn missing_eod_and_clear() { - // Drop the EOD (last 9 bits span the final 2 bytes; keep bits for 66 only). - let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; - for c in [45u16, 258, 258, 65, 259, 66] { - w.write(c, 9); - } - assert_eq!(dec(&w.finish(), true).unwrap(), SPEC_IN); - } - - #[test] - fn invalid_codes() { - let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; - for c in [256u16, 65, 300] { - w.write(c, 9); - } - let f = decode(&w.finish(), true, 1000).err().unwrap(); - assert!(matches!(f.error, FilterError::Corrupt { .. })); - assert_eq!(f.partial, b"A"); - let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; - w.write(258, 9); - assert!(matches!(dec(&w.finish(), true), Err(FilterError::Corrupt { .. }))); - } - - #[test] - fn widths() { - assert_eq!((width(510, true), width(511, true), width(511, false), width(512, false)), (9, 10, 9, 10)); - assert_eq!((width(1022, true), width(1023, true), width(2047, true), width(4095, false)), (10, 11, 12, 12)); - } - - fn noisy(n: usize, seed: u32) -> Vec { - let mut s = seed; - (0..n) - .map(|_| { - s = s.wrapping_mul(1_103_515_245).wrapping_add(12_345); - ((s >> 16) % 7) as u8 * 31 - }) - .collect() - } - - #[test] - fn all_widths_and_clears_round_trip() { - for early in [false, true] { - for data in [noisy(200_000, 1), (0..=255u8).cycle().take(70_000).collect(), vec![9u8; 100_000]] { - let e = encode(&data, early); - assert_eq!(dec(&e, early).unwrap(), data); - } - } - } - - #[test] - fn cross_check_with_weezl() { - use weezl::BitOrder::Msb; - for data in [noisy(150_000, 7), SPEC_IN.to_vec(), vec![]] { - // early change 1 is the TIFF size switch; 0 is the plain variant. - let tiff = weezl::encode::Encoder::with_tiff_size_switch(Msb, 8).encode(&data).unwrap(); - assert_eq!(dec(&tiff, true).unwrap(), data); - let plain = weezl::encode::Encoder::new(Msb, 8).encode(&data).unwrap(); - assert_eq!(dec(&plain, false).unwrap(), data); - let ours = encode(&data, true); - assert_eq!(weezl::decode::Decoder::with_tiff_size_switch(Msb, 8).decode(&ours).unwrap(), data); - let ours = encode(&data, false); - assert_eq!(weezl::decode::Decoder::new(Msb, 8).decode(&ours).unwrap(), data); - } - } - - #[test] - fn full_table_without_clear() { - // Hand-roll a stream that fills the table and keeps going at 12 bits. - let data = noisy(40_000, 3); - let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; - let mut dict: HashMap, u16> = (0..=255u8).map(|b| (vec![b], u16::from(b))).collect(); - let (mut next, mut dnext, mut since) = (FIRST, FIRST, 0usize); - let mut cur: Vec = vec![]; - let mut put = |w: &mut BitWriter, code: u16, dnext: &mut usize| { - w.write(code, width(*dnext, true)); - if since > 0 && *dnext < TABLE { - *dnext += 1; - } - since += 1; - }; - w.write(CLEAR, 9); - for &b in &data { - let mut ext = cur.clone(); - ext.push(b); - if dict.contains_key(&ext) { - cur = ext; - continue; - } - put(&mut w, dict[&cur], &mut dnext); - if next < TABLE { - dict.insert(ext, next as u16); - next += 1; - } - cur = vec![b]; - } - put(&mut w, dict[&cur], &mut dnext); - w.write(EOD, width(dnext, true)); - assert_eq!(dec(&w.finish(), true).unwrap(), data); - } - - #[test] - fn limit() { - let e = encode(&vec![0u8; 100_000], true); - assert!(matches!(decode(&e, true, 50_000).map_err(|f| f.error), Err(FilterError::LimitExceeded(50_000)))); - assert_eq!(decode(&e, true, 100_000).ok().unwrap().len(), 100_000); - } -} diff --git a/crates/filters/src/mq.rs b/crates/filters/src/mq.rs deleted file mode 100644 index 04dda2bd9a..0000000000 --- a/crates/filters/src/mq.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! The MQ arithmetic encoder of JBIG2 (ITU-T T.88 | ISO/IEC 14492, Annex E), written from the -//! procedures in its Figures E.2–E.12 and the probability estimates of Table E.1. - -/// Table E.1: for each index, the LPS probability estimate Qe, the next index after an MPS and -/// after an LPS renormalisation, and whether an LPS switches the MPS sense. -const STATES: [(u32, u8, u8, bool); 47] = [ - (0x5601, 1, 1, true), - (0x3401, 2, 6, false), - (0x1801, 3, 9, false), - (0x0AC1, 4, 12, false), - (0x0521, 5, 29, false), - (0x0221, 38, 33, false), - (0x5601, 7, 6, true), - (0x5401, 8, 14, false), - (0x4801, 9, 14, false), - (0x3801, 10, 14, false), - (0x3001, 11, 17, false), - (0x2401, 12, 18, false), - (0x1C01, 13, 20, false), - (0x1601, 29, 21, false), - (0x5601, 15, 14, true), - (0x5401, 16, 14, false), - (0x5101, 17, 15, false), - (0x4801, 18, 16, false), - (0x3801, 19, 17, false), - (0x3401, 20, 18, false), - (0x3001, 21, 19, false), - (0x2801, 22, 19, false), - (0x2401, 23, 20, false), - (0x2201, 24, 21, false), - (0x1C01, 25, 22, false), - (0x1801, 26, 23, false), - (0x1601, 27, 24, false), - (0x1401, 28, 25, false), - (0x1201, 29, 26, false), - (0x1101, 30, 27, false), - (0x0AC1, 31, 28, false), - (0x09C1, 32, 29, false), - (0x08A1, 33, 30, false), - (0x0521, 34, 31, false), - (0x0441, 35, 32, false), - (0x02A1, 36, 33, false), - (0x0221, 37, 34, false), - (0x0141, 38, 35, false), - (0x0111, 39, 36, false), - (0x0085, 40, 37, false), - (0x0049, 41, 38, false), - (0x0025, 42, 39, false), - (0x0015, 43, 40, false), - (0x0009, 44, 41, false), - (0x0005, 45, 42, false), - (0x0001, 45, 43, false), - (0x5601, 46, 46, false), -]; - -#[cfg(test)] -thread_local! { - /// Decisions coded on this thread. - pub(crate) static DECISIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; -} - -/// A context's adaptive state: the index into [`STATES`] (low 7 bits) and the MPS sense (bit 7). -/// 0, the initial value, is index 0 with MPS 0 (E.2.8). -pub(crate) type Context = u8; - -/// The encoder's registers (E.2.1) and the bytes written so far. -pub(crate) struct MqEncoder { - a: u32, - c: u32, - ct: u32, - /// The output, after the byte before the first one (B at INITENC), which is 0x00 and is - /// dropped by [`MqEncoder::finish`]: a carry can't reach it before the first byte is out. - out: Vec, -} - -impl MqEncoder { - /// INITENC (Figure E.10). The byte before the first is 0x00, not 0xFF, so CT starts at 12. - pub(crate) fn new() -> Self { - MqEncoder { a: 0x8000, c: 0, ct: 12, out: vec![0] } - } - - /// The byte B, the last one written (or the one before the first). - fn b(&self) -> u8 { - self.out.last().copied().unwrap_or(0) - } - - /// ENCODE (Figures E.3–E.7): code decision `d` in context `cx`, updating its state. - pub(crate) fn encode(&mut self, cx: &mut Context, d: bool) { - #[cfg(test)] - DECISIONS.with(|n| n.set(n.get() + 1)); - let (index, mps) = (usize::from(*cx & 0x7F), *cx >> 7 == 1); - let Some(&(qe, nmps, nlps, switch)) = STATES.get(index) else { return }; - self.a -= qe; - if d == mps { - // CODEMPS. - if self.a & 0x8000 != 0 { - self.c += qe; - return; - } - if self.a < qe { - self.a = qe; - } else { - self.c += qe; - } - *cx = nmps | (*cx & 0x80); - } else { - // CODELPS. - if self.a < qe { - self.c += qe; - } else { - self.a = qe; - } - let mps = mps != switch; - *cx = nlps | (u8::from(mps) << 7); - } - self.renorm(); - } - - /// RENORME (Figure E.8). - fn renorm(&mut self) { - loop { - self.a <<= 1; - self.c <<= 1; - self.ct -= 1; - if self.ct == 0 { - self.byte_out(); - } - if self.a & 0x8000 != 0 { - break; - } - } - } - - /// BYTEOUT (Figure E.9), with bit stuffing after 0xFF and carry into the byte before. - fn byte_out(&mut self) { - if self.b() == 0xFF { - self.stuffed(); - } else if self.c < 0x800_0000 { - self.plain(); - } else { - let b = self.b().wrapping_add(1); - if let Some(last) = self.out.last_mut() { - *last = b; - } - if b == 0xFF { - self.c &= 0x7FF_FFFF; - self.stuffed(); - } else { - self.plain(); - } - } - } - - fn plain(&mut self) { - self.out.push(((self.c >> 19) & 0xFF) as u8); - self.c &= 0x7_FFFF; - self.ct = 8; - } - - fn stuffed(&mut self) { - self.out.push(((self.c >> 20) & 0xFF) as u8); - self.c &= 0xF_FFFF; - self.ct = 7; - } - - /// FLUSH (Figures E.11 and E.12): the final bytes, then the 0xFF 0xAC marker. - pub(crate) fn finish(mut self) -> Vec { - let top = self.c + self.a; - self.c |= 0xFFFF; - if self.c >= top { - self.c -= 0x8000; - } - self.c <<= self.ct; - self.byte_out(); - self.c <<= self.ct; - self.byte_out(); - if self.b() != 0xFF { - self.out.push(0xFF); - } - self.out.push(0xAC); - self.out.split_off(1) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Annex H.2's test sequence: 256 decisions in one context give these 30 bytes. - #[test] - fn the_annex_h_test_sequence_encodes_to_its_bytes() { - let decisions = [ - 0x00, 0x02, 0x00, 0x51, 0x00, 0x00, 0x00, 0xC0, 0x03, 0x52, 0x87, 0x2A, 0xAA, 0xAA, 0xAA, 0xAA, 0x82, 0xC0, 0x20, 0x00, 0xFC, 0xD7, 0x9E, - 0xF6, 0xBF, 0x7F, 0xED, 0x90, 0x4F, 0x46, 0xA3, 0xBF, - ]; - let mut mq = MqEncoder::new(); - let mut cx = 0; - for byte in decisions { - for bit in (0..8).rev() { - mq.encode(&mut cx, byte >> bit & 1 == 1); - } - } - let expected = [ - 0x84, 0xC7, 0x3B, 0xFC, 0xE1, 0xA1, 0x43, 0x04, 0x02, 0x20, 0x00, 0x00, 0x41, 0x0D, 0xBB, 0x86, 0xF4, 0x31, 0x7F, 0xFF, 0x88, 0xFF, 0x37, - 0x47, 0x1A, 0xDB, 0x6A, 0xDF, 0xFF, 0xAC, - ]; - assert_eq!(mq.finish(), expected); - } -} diff --git a/crates/filters/src/predictor.rs b/crates/filters/src/predictor.rs deleted file mode 100644 index bdf3c8c5c5..0000000000 --- a/crates/filters/src/predictor.rs +++ /dev/null @@ -1,356 +0,0 @@ -//! Predictor functions for FlateDecode and LZWDecode (ISO 32000-2 §7.4.4.4). -//! -//! - Predictor 2: TIFF 6.0 horizontal differencing (Section 14), per component, for -//! 1, 2, 4, 8 and 16 bits per component. -//! - Predictors 10–15: PNG filters (RFC 2083 §6): each row carries its own filter-type -//! byte (0 None, 1 Sub, 2 Up, 3 Average, 4 Paeth), whatever the predictor value is. -//! -//! A final partial row is decoded as far as its data goes. -//! -//! The samples of an image whose rows carry no filter-type byte, although a PNG predictor is -//! declared, are taken as they are and reported as damage ([`decode`]). - -use crate::{Failure, FilterError, Params, Step}; - -struct Layout { - bpc: usize, - colors: usize, - /// Bytes per complete pixel, at least 1 (the PNG `bpp`). - bpp: usize, - /// Bytes per row (without the PNG tag byte). - row: usize, - /// Samples per row (colors × columns). - samples: usize, -} - -fn layout(p: &Params) -> Result { - let bpc = match p.bits_per_component { - b @ (1 | 2 | 4 | 8 | 16) => b as usize, - b => return Err(format!("invalid BitsPerComponent {b}")), - }; - if !(1..=255).contains(&p.colors) { - return Err(format!("invalid Colors {}", p.colors)); - } - if p.columns < 1 { - return Err(format!("invalid Columns {}", p.columns)); - } - let colors = p.colors as usize; - let samples = usize::try_from(p.columns).ok().and_then(|c| c.checked_mul(colors)).ok_or_else(|| format!("Columns {} too large", p.columns))?; - let bits = samples.checked_mul(bpc).ok_or_else(|| format!("Columns {} too large", p.columns))?; - Ok(Layout { bpc, colors, bpp: (colors * bpc).div_ceil(8), row: bits.div_ceil(8), samples }) -} - -/// Undoes the predictor named by `p.predictor` (1 = identity). `rows` is the height of the image -/// `data` is the samples of, if it is: with a PNG predictor, data of exactly `rows` rows without -/// their filter-type bytes ([`untagged`]) is given back as it is, as a [`Failure`] whose partial -/// output is all of it, so that tolerant decoding takes the samples and reports the repair, and -/// strict decoding refuses them. -pub(crate) fn decode(p: &Params, data: Vec, name: &'static str, rows: Option) -> Step { - match p.predictor { - 1 => Ok(data), - 2 | 10..=15 => { - let l = layout(p).map_err(|e| Failure::corrupt(name, e, Vec::new()))?; - if p.predictor == 2 { - return Ok(tiff_decode(&l, data)); - } - if let Some(rows) = rows.filter(|&rows| untagged(&l, data.len(), rows)) { - let detail = - format!("PNG predictor {} declared, but its {rows} rows carry no filter-type bytes: read as unpredicted samples", p.predictor); - return Err(Failure::corrupt(name, detail, data)); - } - png_decode(&l, &data, name) - } - other => Err(Failure::corrupt(name, format!("invalid Predictor {other}"), Vec::new())), - } -} - -/// Whether `len` bytes are exactly `rows` rows of `l` without the filter-type byte that starts each -/// PNG-predicted row (ISO 32000-2 §7.4.4.4), where predicted data would be `rows` × (row + 1): a -/// writer that declared the predictor and didn't apply it. Read with the predictor, every row's -/// first sample was taken for a filter type and the image came out a row short and shifted. -/// PDFium, MuPDF and Poppler read such data as the samples it is. -fn untagged(l: &Layout, len: usize, rows: usize) -> bool { - rows > 0 && l.row > 0 && rows.checked_mul(l.row) == Some(len) -} - -/// Applies the predictor named by `p.predictor` (1 = identity; 15 = per-row optimum). -pub(crate) fn encode(p: &Params, data: &[u8]) -> Result, FilterError> { - match p.predictor { - 1 => Ok(data.to_vec()), - 2 | 10..=15 => { - let l = layout(p).map_err(FilterError::Unsupported)?; - Ok(if p.predictor == 2 { tiff_encode(&l, data) } else { png_encode(&l, data, p.predictor) }) - } - other => Err(FilterError::Unsupported(format!("Predictor {other}"))), - } -} - -// ---- TIFF predictor 2 ---- - -fn get(row: &[u8], i: usize, bpc: usize) -> u16 { - match bpc { - 16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]), - 8 => u16::from(row[i]), - _ => { - let bit = i * bpc; - let shift = 8 - bpc - bit % 8; - u16::from(row[bit / 8] >> shift) & ((1 << bpc) - 1) - } - } -} - -fn set(row: &mut [u8], i: usize, bpc: usize, v: u16) { - match bpc { - 16 => row[2 * i..2 * i + 2].copy_from_slice(&v.to_be_bytes()), - 8 => row[i] = v as u8, - _ => { - let bit = i * bpc; - let shift = 8 - bpc - bit % 8; - let mask = (((1u16 << bpc) - 1) << shift) as u8; - row[bit / 8] = (row[bit / 8] & !mask) | (((v << shift) as u8) & mask); - } - } -} - -/// Complete samples available in a (possibly partial) row. -fn samples_in(l: &Layout, row_len: usize) -> usize { - (row_len * 8 / l.bpc).min(l.samples) -} - -fn tiff_decode(l: &Layout, mut data: Vec) -> Vec { - let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; - for row in data.chunks_mut(l.row) { - for i in l.colors..samples_in(l, row.len()) { - let v = get(row, i, l.bpc).wrapping_add(get(row, i - l.colors, l.bpc)) & mask; - set(row, i, l.bpc, v); - } - } - data -} - -fn tiff_encode(l: &Layout, data: &[u8]) -> Vec { - let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; - let mut out = data.to_vec(); - for row in out.chunks_mut(l.row) { - // Descending, so the left neighbour still holds its original value. - for i in (l.colors..samples_in(l, row.len())).rev() { - let v = get(row, i, l.bpc).wrapping_sub(get(row, i - l.colors, l.bpc)) & mask; - set(row, i, l.bpc, v); - } - } - out -} - -// ---- PNG predictors ---- - -fn paeth(a: u8, b: u8, c: u8) -> u8 { - let p = i16::from(a) + i16::from(b) - i16::from(c); - let (pa, pb, pc) = ((p - i16::from(a)).abs(), (p - i16::from(b)).abs(), (p - i16::from(c)).abs()); - if pa <= pb && pa <= pc { - a - } else if pb <= pc { - b - } else { - c - } -} - -/// The PNG predictor value for byte `i` given the reconstructed current row `cur` -/// (bytes before `i`) and the prior row `up`. -fn predict(tag: u8, cur: &[u8], up: &[u8], i: usize, bpp: usize) -> u8 { - let a = if i >= bpp { cur[i - bpp] } else { 0 }; - let b = up[i]; - let c = if i >= bpp { up[i - bpp] } else { 0 }; - match tag { - 1 => a, - 2 => b, - 3 => ((u16::from(a) + u16::from(b)) / 2) as u8, - 4 => paeth(a, b, c), - _ => 0, - } -} - -fn png_decode(l: &Layout, data: &[u8], name: &'static str) -> Step { - let mut out = Vec::with_capacity(data.len()); - // The prior row: never longer than the input, even for absurd Columns values. - let mut up = vec![0u8; l.row.min(data.len())]; - for chunk in data.chunks(l.row.saturating_add(1)) { - let (tag, src) = (chunk[0], &chunk[1..]); - if tag > 4 { - return Err(Failure::corrupt(name, format!("invalid PNG filter type {tag}"), out)); - } - let start = out.len(); - for (i, &x) in src.iter().enumerate() { - let pred = predict(tag, &out[start..], &up, i, l.bpp); - out.push(x.wrapping_add(pred)); - } - up[..src.len()].copy_from_slice(&out[start..]); - } - Ok(out) -} - -fn png_filter(tag: u8, cur: &[u8], up: &[u8], bpp: usize, dst: &mut Vec) { - dst.clear(); - dst.push(tag); - dst.extend(cur.iter().enumerate().map(|(i, &x)| x.wrapping_sub(predict(tag, cur, up, i, bpp)))); -} - -/// Sum of residuals as signed bytes: the usual heuristic for choosing a PNG filter. -fn cost(filtered: &[u8]) -> u64 { - filtered[1..].iter().map(|&b| u64::from((b as i8).unsigned_abs())).sum() -} - -fn png_encode(l: &Layout, data: &[u8], predictor: i64) -> Vec { - let rows = data.len().div_ceil(l.row.max(1)); - let mut out = Vec::with_capacity(data.len() + rows); - let zeros = vec![0u8; l.row.min(data.len())]; - let (mut best, mut trial) = (Vec::new(), Vec::new()); - let mut up: &[u8] = &zeros; - for cur in data.chunks(l.row) { - let up_row = &up[..cur.len()]; - if predictor == 15 { - png_filter(0, cur, up_row, l.bpp, &mut best); - let mut best_cost = cost(&best); - for tag in 1..=4 { - png_filter(tag, cur, up_row, l.bpp, &mut trial); - let c = cost(&trial); - if c < best_cost { - best_cost = c; - std::mem::swap(&mut best, &mut trial); - } - } - } else { - png_filter((predictor - 10) as u8, cur, up_row, l.bpp, &mut best); - } - out.extend_from_slice(&best); - up = cur; - } - out -} - -#[cfg(test)] -mod tests { - use super::*; - - fn p(predictor: i64, colors: i64, bpc: i64, columns: i64) -> Params { - Params { predictor, colors, bits_per_component: bpc, columns, early_change: 1 } - } - - fn dec(params: &Params, data: &[u8]) -> Result, FilterError> { - decode(params, data.to_vec(), "FlateDecode", None).map_err(|f| f.error) - } - - #[test] - fn png_hand_vectors() { - // 3 columns, 1 color, 8 bpc. Rows: None, Sub, Up, Average, Paeth. - let enc = [ - 0, 10, 20, 30, // None -> 10 20 30 - 1, 5, 1, 1, // Sub -> 5 6 7 - 2, 1, 1, 1, // Up -> 6 7 8 - 3, 10, 10, 10, // Avg -> 10+3=13, 10+(13+7)/2=20, 10+(20+8)/2=24 - 4, 1, 1, - 1, // Paeth: a=0,b=13,c=0 -> p=13 -> b: 14; a=14,b=20,c=13: p=21 -> pa=7,pb=1,pc=8 -> b: 21; - // a=21,b=24,c=20: p=25 -> pa=4,pb=1,pc=5 -> b: 25 - ]; - let out = dec(&p(15, 1, 8, 3), &enc).unwrap(); - assert_eq!(out, [10, 20, 30, 5, 6, 7, 6, 7, 8, 13, 20, 24, 14, 21, 25]); - } - - #[test] - fn paeth_tie_breaks() { - // RFC 2083 order: a, then b, then c. - assert_eq!(paeth(1, 1, 1), 1); - assert_eq!(paeth(10, 20, 15), 15); // p=15: pa=5 pb=5 pc=0 -> c - assert_eq!(paeth(20, 10, 10), 20); // p=20: pa=0 -> a - assert_eq!(paeth(5, 9, 7), 7); // p=7: pa=2 pb=2 pc=0 -> c - } - - #[test] - fn png_multibyte_pixels() { - // colors=3 bpc=16 -> bpp=6; 1 column; Sub on the first pixel uses a=0 throughout. - let enc = [1, 1, 2, 3, 4, 5, 6, 2, 1, 1, 1, 1, 1, 1]; - let out = dec(&p(11, 3, 16, 1), &enc).unwrap(); - assert_eq!(out, [1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7]); - // colors=3 bpc=8, 2 columns: Sub looks 3 bytes back. - let enc = [1, 1, 2, 3, 1, 1, 1]; - assert_eq!(dec(&p(11, 3, 8, 2), &enc).unwrap(), [1, 2, 3, 2, 3, 4]); - } - - #[test] - fn png_partial_final_row() { - let enc = [0, 1, 2, 3, 4, 2, 1, 1]; - assert_eq!(dec(&p(12, 1, 8, 4), &enc).unwrap(), [1, 2, 3, 4, 2, 3]); - // A lone tag byte at the end yields nothing. - assert_eq!(dec(&p(12, 1, 8, 4), &[0, 1, 2, 3, 4, 2]).unwrap(), [1, 2, 3, 4]); - } - - #[test] - fn png_invalid_tag() { - let r = decode(&p(10, 1, 8, 2), vec![0, 1, 2, 5, 1, 1], "FlateDecode", None); - let f = r.err().unwrap(); - assert!(matches!(f.error, FilterError::Corrupt { .. })); - assert_eq!(f.partial, [1, 2]); - } - - #[test] - fn tiff_8bit() { - // 2 colors, 3 columns: differences per component. - let enc = [10, 20, 1, 2, 3, 4, /* row 2 */ 5, 5, 250, 10, 1, 1]; - assert_eq!(dec(&p(2, 2, 8, 3), &enc).unwrap(), [10, 20, 11, 22, 14, 26, 5, 5, 255, 15, 0, 16]); - } - - #[test] - fn tiff_16bit() { - let enc = [0x01, 0x00, 0xff, 0xff, 0x00, 0x02]; - assert_eq!(dec(&p(2, 1, 16, 3), &enc).unwrap(), [0x01, 0x00, 0x00, 0xff, 0x01, 0x01]); - } - - #[test] - fn tiff_sub_byte() { - // 1 bpc, 1 color, 10 columns: bits 1 0 0 1 1 0 0 0 | 1 1 (pad 000000) - // Cumulative XOR: 1 1 1 0 1 1 1 1 | 0 1 - let enc = [0b1001_1000, 0b1100_0000]; - assert_eq!(dec(&p(2, 1, 1, 10), &enc).unwrap(), [0b1110_1111, 0b0100_0000]); - // 4 bpc, 1 color, 3 columns: samples 5, 15, 3 -> 5, 4, 7 (mod 16). - assert_eq!(dec(&p(2, 1, 4, 3), &[0x5f, 0x30]).unwrap(), [0x54, 0x70]); - // 2 bpc, 2 colors, 2 columns: samples 1 2 3 3 -> 1 2 0 1. - assert_eq!(dec(&p(2, 2, 2, 2), &[0b0110_1111]).unwrap(), [0b0110_0001]); - // Padding bits are left alone. - assert_eq!(dec(&p(2, 1, 4, 1), &[0x5f]).unwrap(), [0x5f]); - } - - #[test] - fn tiff_partial_rows() { - // 16-bit with an odd trailing byte: the lone byte is left as is. - assert_eq!(dec(&p(2, 1, 16, 4), &[0, 1, 0, 1, 9]).unwrap(), [0, 1, 0, 2, 9]); - assert_eq!(dec(&p(2, 1, 8, 4), &[1, 1, 1, 1, 2, 2]).unwrap(), [1, 2, 3, 4, 2, 4]); - } - - #[test] - fn invalid_params() { - for bad in [p(2, 1, 3, 1), p(12, 0, 8, 1), p(12, 1, 8, 0), p(3, 1, 8, 1), p(12, 1, 8, i64::MAX), p(2, 300, 8, 1)] { - assert!(matches!(dec(&bad, &[0, 1, 2]), Err(FilterError::Corrupt { .. })), "{bad:?}"); - assert!(encode(&bad, &[1, 2]).is_err()); - } - // Huge but representable Columns does not allocate a huge row. - assert_eq!(dec(&p(12, 1, 8, 1 << 40), &[2, 7, 8]).unwrap(), [7, 8]); - } - - #[test] - fn encode_choices() { - let data = [1u8, 2, 3, 4, 5, 6]; - let up = encode(&p(12, 1, 8, 3), &data).unwrap(); - assert_eq!(up, [2, 1, 2, 3, 2, 3, 3, 3]); - let paeth = encode(&p(14, 1, 8, 3), &data).unwrap(); - assert_eq!(paeth[0], 4); - assert_eq!(paeth[4], 4); - // Per-row optimum picks Sub for a ramp. - let ramp: Vec = (0..64).collect(); - let opt = encode(&p(15, 1, 8, 64), &ramp).unwrap(); - assert_eq!(opt[0], 1); - for pred in [10, 11, 12, 13, 14, 15] { - let e = encode(&p(pred, 1, 8, 3), &data).unwrap(); - assert_eq!(dec(&p(pred, 1, 8, 3), &e).unwrap(), data); - } - } -} diff --git a/crates/filters/src/runlength.rs b/crates/filters/src/runlength.rs deleted file mode 100644 index e77b619f97..0000000000 --- a/crates/filters/src/runlength.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! RunLengthDecode (ISO 32000-2 §7.4.5): a length byte L of 0–127 copies the next -//! L + 1 bytes, 129–255 repeats the next byte 257 − L times, 128 is EOD. -//! A missing EOD is tolerated; a truncated run is corrupt. - -use crate::{Failure, Step}; - -const NAME: &str = "RunLengthDecode"; - -pub(crate) fn decode(data: &[u8], max: usize) -> Step { - let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); - let mut i = 0; - while let Some(&l) = data.get(i) { - i += 1; - match l { - 128 => break, - 0..=127 => { - let n = usize::from(l) + 1; - let avail = n.min(data.len() - i); - if out.len() + avail > max { - return Err(Failure::limit(max)); - } - out.extend_from_slice(&data[i..i + avail]); - i += avail; - if avail < n { - return Err(Failure::corrupt(NAME, "truncated literal run", out)); - } - } - _ => { - let Some(&b) = data.get(i) else { - return Err(Failure::corrupt(NAME, "truncated repeat run", out)); - }; - i += 1; - let n = 257 - usize::from(l); - if out.len() + n > max { - return Err(Failure::limit(max)); - } - out.resize(out.len() + n, b); - } - } - } - Ok(out) -} - -pub(crate) fn encode(data: &[u8]) -> Vec { - let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2); - let n = data.len(); - let mut i = 0; - while i < n { - let mut run = 1; - while i + run < n && run < 128 && data[i + run] == data[i] { - run += 1; - } - if run >= 2 { - out.push((257 - run) as u8); - out.push(data[i]); - i += run; - continue; - } - // Literal: extend until a repeat begins or 128 bytes. - let start = i; - let mut j = i + 1; - while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) { - j += 1; - } - out.push((j - start - 1) as u8); - out.extend_from_slice(&data[start..j]); - i = j; - } - out.push(128); - out -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::FilterError; - - fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 10_000).map_err(|f| f.error) - } - - #[test] - fn vectors() { - assert_eq!(dec(&[2, b'a', b'b', b'c', 254, b'x', 128]).unwrap(), b"abcxxx"); - assert_eq!(dec(&[129, 7, 128]).unwrap(), [7; 128]); - assert!(dec(&[127]).is_err()); - assert_eq!(dec(&[0, b'q']).unwrap(), b"q"); // missing EOD - assert_eq!(dec(&[128, 0, b'q']).unwrap(), b""); // data after EOD ignored - assert_eq!(dec(&[]).unwrap(), b""); - } - - #[test] - fn truncated() { - let f = decode(&[4, b'a', b'b'], 100).err().unwrap(); - assert!(matches!(f.error, FilterError::Corrupt { .. })); - assert_eq!(f.partial, b"ab"); - assert!(matches!(dec(&[0, b'a', 200]), Err(FilterError::Corrupt { .. }))); - } - - #[test] - fn limit() { - assert!(matches!(decode(&[129, 0, 129, 0], 200).map_err(|f| f.error), Err(FilterError::LimitExceeded(200)))); - assert!(matches!(decode(&[3, 1, 2, 3, 4], 3).map_err(|f| f.error), Err(FilterError::LimitExceeded(3)))); - assert_eq!(decode(&[129, 0, 129, 0], 256).ok().unwrap().len(), 256); - } - - #[test] - fn encoding() { - assert_eq!(encode(b"abcxxx"), [2, b'a', b'b', b'c', 254, b'x', 128]); - assert_eq!(encode(b""), [128]); - assert_eq!(encode(&[5; 300]), [129, 5, 129, 5, 213, 5, 128]); - let lit: Vec = (0..200u8).collect(); - let e = encode(&lit); - assert_eq!((e[0], e[129]), (127, 71)); - assert_eq!(dec(&e).unwrap(), lit); - } -} diff --git a/crates/xfdf/Cargo.toml b/crates/xfdf/Cargo.toml index d82e75394e..b700b3d436 100644 --- a/crates/xfdf/Cargo.toml +++ b/crates/xfdf/Cargo.toml @@ -13,6 +13,7 @@ pdfcraft-cos = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-forms = { workspace = true } pdfcraft-model = { workspace = true } +craft-csv = { workspace = true } roxmltree = "0.21" base64 = "0.23" thiserror = { workspace = true } diff --git a/crates/xfdf/src/lib.rs b/crates/xfdf/src/lib.rs index 984d1cdbe5..bd03f532b1 100644 --- a/crates/xfdf/src/lib.rs +++ b/crates/xfdf/src/lib.rs @@ -123,7 +123,13 @@ pub fn export_data(doc: &Document, format: Format) -> String { // A formula-like cell gets a leading `'`, as in `merge_csv`; import takes it off again. let q = |s: &str| { let s = if csv_guarded(s) { format!("'{s}") } else { s.to_string() }; - if s.contains([',', '"', '\n', '\r', '\t']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s } + if s.contains('\t') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + let mut out = String::new(); + craft_csv::push_field(&mut out, &s, ','); + out + } }; format!( "{}\n{}\n", @@ -202,7 +208,13 @@ pub fn merge_csv(files: &[Vec<(String, Vec)>]) -> String { let literal = |s: &str| if runs_as_formula(s) { format!("'{s}") } else { s.to_string() }; let q = |s: &str| { let s = literal(s); - if s.contains([',', '"', '\n', '\r', '\t']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s } + if s.contains('\t') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + let mut out = String::new(); + craft_csv::push_field(&mut out, &s, ','); + out + } }; let mut out = columns.iter().map(|c| q(c)).collect::>().join(","); out.push('\n');