crypt, filters, cos, export, xfdf: migrate to craft-pdf, craft-rtf, craft-csv

This commit is contained in:
Dylan M. Taylor
2026-11-14 04:02:21 +00:00
parent 524d9f6b64
commit ed18238d74
40 changed files with 132 additions and 17834 deletions
Generated
+92 -43
View File
@@ -1187,25 +1187,39 @@ dependencies = [
[[package]]
name = "craft-cms"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"md-5 0.10.6",
"rayon",
]
[[package]]
name = "craft-codepage"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
[[package]]
name = "craft-csv"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-docpath"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
[[package]]
name = "craft-fonts-build"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
[[package]]
name = "craft-i18n"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"fixed_decimal",
"icu_decimal",
@@ -1228,7 +1242,7 @@ dependencies = [
[[package]]
name = "craft-keymap"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"egui",
"serde",
@@ -1238,7 +1252,7 @@ dependencies = [
[[package]]
name = "craft-menu"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"craft-keymap",
"craft-quit",
@@ -1248,24 +1262,51 @@ dependencies = [
"objc2-app-kit 0.3.2",
]
[[package]]
name = "craft-pdf"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"aes",
"chrono",
"craft-stream-codecs",
"flate2",
"js-sys",
"md-5 0.11.0",
"sha2 0.11.1",
"stringprep",
"thiserror 2.0.21",
"wasm-bindgen",
"zeroize",
]
[[package]]
name = "craft-prc"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"flate2",
"thiserror 2.0.21",
]
[[package]]
name = "craft-print"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=60ede99985ef1e5df87a2aca66ed538eb23dec99#60ede99985ef1e5df87a2aca66ed538eb23dec99"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"craft-winprint-gdi",
"log",
"miniz_oxide 0.9.1",
"roxmltree 0.21.2",
"tempfile",
"winprint 0.2.1 (git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0)",
"winprint 0.2.1 (git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02)",
"winsafe",
]
[[package]]
name = "craft-quit"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"dispatch2",
"egui",
@@ -1273,10 +1314,30 @@ dependencies = [
"objc2-app-kit 0.3.2",
]
[[package]]
name = "craft-rtf"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"craft-codepage",
"encoding_rs",
]
[[package]]
name = "craft-stream-codecs"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"fax",
"hayro-ccitt",
"hayro-jbig2",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ui"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=787a7a0f972fd1d7a7618fdbeca3a5267a0407ff#787a7a0f972fd1d7a7618fdbeca3a5267a0407ff"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"craft-fonts-build",
"craft-i18n",
@@ -1297,7 +1358,7 @@ dependencies = [
[[package]]
name = "craft-update"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"chrono",
"dunce",
@@ -1352,7 +1413,7 @@ dependencies = [
[[package]]
name = "craft-zip"
version = "0.1.0"
source = "git+https://github.com/storytold/craft-libs?rev=7bb6c9007#7bb6c90076a0177d01cc4c482e7fd8dc9f31a8bf"
source = "git+https://github.com/storytold/craft-libs?rev=e2dc681a3#e2dc681a3ad8a47038da2f11ddd2c5e1c0454991"
dependencies = [
"flate2",
"thiserror 2.0.21",
@@ -2785,7 +2846,6 @@ version = "0.3.0"
dependencies = [
"fearless_simd",
"hayro-ccitt",
"image",
]
[[package]]
@@ -3517,7 +3577,7 @@ dependencies = [
"indexmap",
"itoa",
"log",
"md-5",
"md-5 0.11.0",
"nom",
"rand",
"rangemap",
@@ -3534,6 +3594,16 @@ version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4050469837a6ff301cd14c1f8f24f88549e6d548f24f64e2148eb0f72cebc51f"
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest 0.10.7",
]
[[package]]
name = "md-5"
version = "0.11.0"
@@ -4345,6 +4415,7 @@ version = "0.21.1"
dependencies = [
"base64 0.23.1",
"craft-cms",
"craft-prc",
"flate2",
"image",
"pdfcraft-cos",
@@ -4353,7 +4424,6 @@ dependencies = [
"pdfcraft-model",
"pdfcraft-organize",
"pdfcraft-platform",
"pdfcraft-prc",
"pdfcraft-prc-render",
"pdfcraft-render",
"pdfcraft-xfa",
@@ -4398,17 +4468,12 @@ dependencies = [
name = "pdfcraft-cos"
version = "0.21.1"
dependencies = [
"chrono",
"craft-pdf",
"hayro-syntax",
"js-sys",
"pdfcraft-crypt",
"pdfcraft-filters",
"pdfcraft-oracle",
"pdfcraft-testkit",
"proptest",
"serde_json",
"thiserror 2.0.21",
"wasm-bindgen",
]
[[package]]
@@ -4435,13 +4500,8 @@ dependencies = [
name = "pdfcraft-crypt"
version = "0.21.1"
dependencies = [
"aes",
"md-5",
"craft-pdf",
"proptest",
"sha2 0.11.1",
"stringprep",
"thiserror 2.0.21",
"zeroize",
]
[[package]]
@@ -4512,6 +4572,7 @@ dependencies = [
name = "pdfcraft-export"
version = "0.21.1"
dependencies = [
"craft-rtf",
"craft-zip",
"flate2",
]
@@ -4520,12 +4581,8 @@ dependencies = [
name = "pdfcraft-filters"
version = "0.21.1"
dependencies = [
"fax",
"flate2",
"hayro-ccitt",
"hayro-jbig2",
"craft-pdf",
"proptest",
"thiserror 2.0.21",
"weezl 0.2.1",
]
@@ -4652,21 +4709,12 @@ dependencies = [
"zbus",
]
[[package]]
name = "pdfcraft-prc"
version = "0.21.1"
dependencies = [
"flate2",
"proptest",
"thiserror 2.0.21",
]
[[package]]
name = "pdfcraft-prc-render"
version = "0.21.1"
dependencies = [
"craft-prc",
"flate2",
"pdfcraft-prc",
"thiserror 2.0.21",
]
@@ -4881,6 +4929,7 @@ name = "pdfcraft-xfdf"
version = "0.21.1"
dependencies = [
"base64 0.23.1",
"craft-csv",
"pdfcraft-annot",
"pdfcraft-cos",
"pdfcraft-forms",
@@ -8368,7 +8417,7 @@ dependencies = [
[[package]]
name = "winprint"
version = "0.2.1"
source = "git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0#bc06a7c48b768eb27a828c7986f6d2868b245ee0"
source = "git+https://github.com/storytold/winprint.rs?rev=72951d7455ebcab569fbd9386c513d3d9f53fb02#72951d7455ebcab569fbd9386c513d3d9f53fb02"
dependencies = [
"flate2",
"fmt-derive",
+13 -10
View File
@@ -15,25 +15,28 @@ repository = "https://github.com/storytold/pdfcraft"
# Small, revision-bound UI drafts; the existing engine remains the owner of PDF edits/history.
edit-session = { path = "vendor/edit-session", version = "=0.1.0-alpha.2" }
# Shared frontend components; engines remain independent of egui.
craft-ui = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff", features = ["serde"] }
craft-ui = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", features = ["serde"] }
# Quit the macOS way (#1136): ⌘Q asks about unsaved changes, and closing quits through `terminate:`.
craft-quit = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" }
craft-quit = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
# One menu definition for the native macOS menu bar and the in-window bar elsewhere.
craft-menu = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" }
craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff", default-features = false }
craft-menu = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
craft-keymap = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", default-features = false }
# Interface translations: lookup, fallback, CLDR plurals and language names over the craft-i18n
# catalogs that `crates/ui-egui/build.rs` embeds from `CRAFT_I18N_DIR` (storytold/craft-i18n, at
# the commit in `craft-i18n.rev`). Already in the tree through craft-ui.
craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" }
craft-i18n = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
# Shared system spooler; document composition remains in pdfcraft-print.
craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" }
craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
# ICC transforms for app-managed print colour; at craft-print's commit, so one craft-libs checkout.
craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "60ede99985ef1e5df87a2aca66ed538eb23dec99" }
craft-zip = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "7bb6c9007" }
craft-cms = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
craft-zip = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
# Whether a path stored in a document may be opened (GoToR link targets).
craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" }
craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "e2dc681a3" }
# craft-update: app-specific pretend versions apply to startup and manual checks.
craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", features = ["egui"] }
craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "e2dc681a3", features = ["egui"] }
craft-pdf = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3", features = ["cos", "crypt", "filters"] }
craft-rtf = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
craft-csv = { git = "https://github.com/storytold/craft-libs", version = "0.1.0", rev = "e2dc681a3" }
# internal
pdfcraft-geom = { path = "crates/geom", version = "0.21.0" }
pdfcraft-filters = { path = "crates/filters", version = "0.21.0" }
+1 -12
View File
@@ -9,18 +9,7 @@ license.workspace = true
workspace = true
[dependencies]
pdfcraft-crypt = { workspace = true }
pdfcraft-filters = { workspace = true }
thiserror = { workspace = true }
# PDF date strings (`pdf_date`) go through chrono (no hand-rolled calendar code).
chrono = { workspace = true }
# On the web, a rewrite's fresh file identifier takes its entropy from the browser
# (`crypto.getRandomValues`). Both crates are already in the dependency tree of the web shell
# (`js-sys` 0.3 / `wasm-bindgen` 0.2); native builds compile neither.
[target.'cfg(target_arch = "wasm32")'.dependencies]
js-sys = "0.3"
wasm-bindgen = "0.2"
craft-pdf = { workspace = true }
[dev-dependencies]
pdfcraft-oracle = { workspace = true }
File diff suppressed because it is too large Load Diff
-98
View File
@@ -1,98 +0,0 @@
//! Stream data that can share a larger buffer instead of copying out of it.
use std::ops::{Deref, Range};
use std::sync::Arc;
/// A stream's bytes: a buffer of their own, or a range of a larger shared buffer. Streams read
/// from a file point into the file's bytes, which the document keeps anyway, so caching a
/// parsed stream costs its dictionary and not a second copy of its data.
#[derive(Clone, Default)]
pub struct Bytes {
buf: Arc<Vec<u8>>,
range: Range<usize>,
}
impl Bytes {
/// `range` of `buf`, without copying; `None` when the range is not within `buf`.
pub(crate) fn view(buf: &Arc<Vec<u8>>, range: Range<usize>) -> Option<Self> {
(range.start <= range.end && range.end <= buf.len()).then(|| Self { buf: buf.clone(), range })
}
/// Whether these bytes keep alive a larger buffer than themselves that isn't `owner`.
pub(crate) fn borrows_other_than(&self, owner: &Arc<Vec<u8>>) -> bool {
self.range.len() < self.buf.len() && !Arc::ptr_eq(&self.buf, owner)
}
/// Whether these bytes are (a view into) `buf`, holding no memory of their own.
pub(crate) fn shares(&self, buf: &Arc<Vec<u8>>) -> bool {
Arc::ptr_eq(&self.buf, buf)
}
/// The same bytes, in a buffer of their own.
pub(crate) fn detached(&self) -> Self {
Self::from(self.to_vec())
}
}
impl From<Vec<u8>> for Bytes {
fn from(v: Vec<u8>) -> Self {
let range = 0..v.len();
Self { buf: Arc::new(v), range }
}
}
impl From<Arc<Vec<u8>>> for Bytes {
fn from(buf: Arc<Vec<u8>>) -> Self {
let range = 0..buf.len();
Self { buf, range }
}
}
impl Deref for Bytes {
type Target = [u8];
fn deref(&self) -> &[u8] {
// `view` and `From` only build in-bounds ranges; an empty slice keeps this panic-free.
self.buf.get(self.range.clone()).unwrap_or_default()
}
}
impl AsRef<[u8]> for Bytes {
fn as_ref(&self) -> &[u8] {
self
}
}
impl PartialEq for Bytes {
fn eq(&self, other: &Self) -> bool {
(Arc::ptr_eq(&self.buf, &other.buf) && self.range == other.range) || **self == **other
}
}
impl Eq for Bytes {}
impl std::fmt::Debug for Bytes {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
(**self).fmt(f)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn views_share_their_buffer_and_compare_by_content() {
let file = Arc::new(b"0123456789".to_vec());
let view = Bytes::view(&file, 2..5).expect("in range");
assert_eq!(&*view, b"234");
assert_eq!(view.as_ptr(), file[2..].as_ptr(), "no copy");
assert_eq!(view, Bytes::from(b"234".to_vec()));
assert!(Bytes::view(&file, 5..11).is_none() && Bytes::view(&file, Range { start: 6, end: 5 }).is_none());
let other = Arc::new(Vec::new());
assert!(view.borrows_other_than(&other) && !view.borrows_other_than(&file));
let own = view.detached();
assert!(!own.borrows_other_than(&other) && own == view && own.as_ptr() != view.as_ptr());
assert!(!Bytes::from(file.clone()).borrows_other_than(&other), "a whole buffer is its own");
}
}
-98
View File
@@ -1,98 +0,0 @@
//! Document: cross-reference data, lazy object loading, and a copy-on-write overlay of edits.
//!
//! `Document` is cheap to clone (the original bytes, xref index and parse caches are shared), so
//! a clone is a snapshot: undo/redo and background jobs hold clones while the UI edits another.
//! Edits never touch the original bytes; the writer appends them (incremental save) or rewrites
//! the file (full save).
use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::sync::{Arc, Mutex, Weak};
use crate::CosError;
use crate::object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream};
use crate::parser::{EndstreamIndex, Lexer, is_whitespace, parse_indirect, parse_indirect_shared};
mod objects;
mod open;
mod reader;
mod state;
mod xref;
pub use objects::DECODE_PARMS_KEYS;
pub(crate) use objects::Held;
#[cfg(test)]
pub(crate) use reader::find;
#[cfg(test)]
use reader::rfind;
use state::{ObjStm, Slot, Unreadable};
pub(crate) use state::{ObjectReader, held_bytes};
pub use state::{ReadState, SourceIdentity};
#[cfg(test)]
use state::{SECTION_READS, objstm_member};
use xref::XrefIndex;
pub use xref::{Revision, XrefEntry};
#[derive(Clone)]
pub struct Document {
data: Arc<Vec<u8>>,
/// Where `endstream` occurs in `data`; see [`EndstreamIndex`].
endstreams: Arc<EndstreamIndex>,
source_identity: Arc<()>,
entries: Arc<XrefIndex>,
trailer: Dict,
revisions: Arc<Vec<Revision>>,
/// See [`Self::xref_oddities`].
xref_oddities: Arc<Vec<(u64, u32)>>,
/// Where the file's object headers are, as (offset, generation) of the last header of each
/// number. Filled only when the cross-reference data proved inconsistent; see
/// `fill_xref_gaps`.
located: Arc<BTreeMap<u32, (u64, u16)>>,
repair_log: Arc<Vec<String>>,
/// See [`Self::rebuilt_unlinked_update`].
unlinked_update: bool,
/// What loading each file object gave, a failure as well: a damaged object is otherwise
/// loaded again on every read, and a member of an object stream that can't be decoded
/// decodes the stream again (`Document::load`).
cache: Arc<Mutex<state::ObjectCache>>,
objstms: Arc<Mutex<HashMap<u32, Arc<ObjStm>>>>,
/// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the
/// source, and is reset wherever they are.
unreadable: Arc<Mutex<Unreadable>>,
/// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan
/// share them, and each would otherwise decode them again (see `Document::jbig2_globals`).
jbig2_globals: Arc<Mutex<HashMap<u32, Arc<Vec<u8>>>>>,
stream_limit: usize,
overlay: BTreeMap<u32, Slot>,
next_num: u32,
/// Position of the `%PDF-` header (some files have junk before it).
header_offset: usize,
version: String,
/// The authenticated security handler of an encrypted document.
security: Option<Arc<pdfcraft_crypt::SecurityHandler>>,
/// Object number of the `/Encrypt` dictionary (never encrypted itself).
encrypt_num: Option<u32>,
/// What that dictionary and `/ID` refer to, which the key comes from, and so is as stored.
encrypt_members: Arc<HashSet<u32>>,
/// Object numbers a scan of the file didn't find: the file doesn't change, so it never will.
unfound: Arc<Mutex<HashSet<u32>>>,
/// Encryption was added, changed or removed since opening: only a full save can apply it.
encryption_changed: bool,
/// Set by edits that must not leave earlier revisions in the file (redaction).
full_save: bool,
/// The next full save also gives the file a new identifier (`/ID`), see
/// [`Document::require_full_save_with_new_id`].
new_id: bool,
/// The handler and `/Encrypt` object number that saves use, when encryption changed.
out_security: Option<Arc<pdfcraft_crypt::SecurityHandler>>,
out_encrypt_num: Option<u32>,
}
impl std::fmt::Debug for Document {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Document").field("bytes", &self.data.len()).field("xref", &self.entries).field("edits", &self.overlay.len()).finish()
}
}
#[cfg(test)]
mod tests;
File diff suppressed because it is too large Load Diff
-503
View File
@@ -1,503 +0,0 @@
use super::reader::find;
use super::state::{Credentials, Slot};
use super::xref::XrefIndex;
use super::{Arc, BTreeMap, CosError, Dict, Document, HashSet, ObjRef, Object};
/// More objects than `/Encrypt` and `/ID` refer to in any file, so that one which refers to the page
/// tree, say, doesn't have a whole document read to open it.
const MAX_ENCRYPT_MEMBERS: usize = 64;
impl Document {
/// A new document with an empty page tree (for split / extract / combine). It has no
/// original bytes, so saving always writes a full file.
pub fn new_empty() -> Self {
let mut doc = Document {
data: Arc::new(Vec::new()),
endstreams: Arc::default(),
source_identity: Arc::new(()),
entries: Arc::new(XrefIndex::empty()),
trailer: Dict::new(),
revisions: Arc::new(Vec::new()),
xref_oddities: Arc::default(),
located: Arc::default(),
repair_log: Arc::new(Vec::new()),
unlinked_update: false,
cache: Arc::default(),
objstms: Arc::default(),
unreadable: Arc::default(),
jbig2_globals: Arc::default(),
stream_limit: crate::object::MAX_DECODED,
overlay: BTreeMap::new(),
next_num: 1,
header_offset: 0,
version: "1.7".into(),
security: None,
encrypt_num: None,
encrypt_members: Arc::default(),
unfound: Arc::default(),
encryption_changed: false,
full_save: false,
new_id: false,
out_security: None,
out_encrypt_num: None,
};
let mut pages = Dict::new();
pages.set(b"Type".to_vec(), Object::name("Pages"));
pages.set(b"Kids".to_vec(), Object::Array(Vec::new()));
pages.set(b"Count".to_vec(), Object::Int(0));
let pages = doc.add(pages);
let mut catalog = Dict::new();
catalog.set(b"Type".to_vec(), Object::name("Catalog"));
catalog.set(b"Pages".to_vec(), Object::Ref(pages));
let root = doc.add(catalog);
doc.trailer.set(b"Root".to_vec(), Object::Ref(root));
doc
}
/// Parse a document. Damaged cross-reference data is reconstructed; see `repair_log`.
/// Encrypted documents open only if their user password is empty; see `open_with_password`.
pub fn open(data: Arc<Vec<u8>>) -> Result<Self, CosError> {
Self::open_with_password(data, None)
}
/// Parse a document, authenticating encrypted ones with `password` (owner or user; `None`
/// tries the empty password). Fails with `NeedsPassword` / `WrongPassword` as appropriate.
pub fn open_with_password(data: Arc<Vec<u8>>, password: Option<&str>) -> Result<Self, CosError> {
Self::open_with_stream_limit(data, password, crate::object::MAX_DECODED)
}
/// Open with a tighter decompression bound for object and cross-reference streams.
/// Inspection can use this without changing the editing reader's default limit. Such a
/// document is read-only: objects in a larger object stream read as missing, so the
/// writers refuse to save it ([`CosError::ReadOnlyLimit`]).
pub fn open_with_stream_limit(data: Arc<Vec<u8>>, password: Option<&str>, stream_limit: usize) -> Result<Self, CosError> {
Self::open_as(data, Credentials::Password(password), stream_limit)
}
/// Open `data`, an earlier revision of this file (the bytes a signature covers) or another
/// save of it, with this document's credentials: an encrypted one must have the same
/// `/Encrypt` dictionary and `/ID` as this document, whose file key is then reused, so no
/// password is needed or kept; unencrypted data opens as [`open`](Self::open) does. Data
/// encrypted differently fails with `NeedsPassword`.
pub fn open_with_same_credentials(&self, data: Arc<Vec<u8>>) -> Result<Self, CosError> {
let credentials = match &self.security {
Some(handler) => Credentials::Handler { handler, id0: self.first_id() },
None => Credentials::Password(None),
};
Self::open_as(data, credentials, self.stream_limit)
}
fn open_as(data: Arc<Vec<u8>>, credentials: Credentials<'_>, stream_limit: usize) -> Result<Self, CosError> {
// Viewers accept files whose header is missing or damaged as long as the body looks
// like PDF; so do we (a note goes to the repair log).
let header = find(&data, b"%PDF-", 0, 1024);
let headerless = header.is_none();
if headerless && find(&data, b" obj", 0, 4096).is_none() {
return Err(CosError::NotPdf);
}
let header_offset = header.unwrap_or(0);
let version = match header {
Some(h) => String::from_utf8_lossy(&data[(h + 5).min(data.len())..(h + 8).min(data.len())]).trim().to_string(),
None => "1.4".to_string(),
};
let mut doc = Document {
data,
endstreams: Arc::default(),
source_identity: Arc::new(()),
entries: Arc::new(XrefIndex::empty()),
trailer: Dict::new(),
revisions: Arc::new(Vec::new()),
xref_oddities: Arc::default(),
located: Arc::default(),
repair_log: Arc::new(Vec::new()),
unlinked_update: false,
cache: Arc::default(),
objstms: Arc::default(),
unreadable: Arc::default(),
jbig2_globals: Arc::default(),
stream_limit: stream_limit.min(crate::object::MAX_DECODED),
overlay: BTreeMap::new(),
next_num: 1,
header_offset,
version,
security: None,
encrypt_num: None,
encrypt_members: Arc::default(),
unfound: Arc::default(),
encryption_changed: false,
full_save: false,
new_id: false,
out_security: None,
out_encrypt_num: None,
};
let mut log = Vec::new();
if headerless {
log.push("the %PDF- header is missing; reading the file as PDF 1.4".into());
}
let parsed = doc.read_xref_chain(&mut log);
let mut authenticated = false;
// The newest section cut the page tree off from the chain (see `update_cuts_off_the_pages`).
let mut cut_off = false;
let ok = match parsed {
Ok((entries, trailer, revisions, oddities)) => {
doc.entries = Arc::new(entries);
doc.trailer = trailer;
doc.revisions = Arc::new(revisions);
doc.xref_oddities = Arc::new(oddities);
doc.fill_xref_gaps(&mut log);
// Authenticate before checking the catalog: it may sit in an encrypted object
// stream, which reads as garbage until the key is known.
if let Some(enc) = doc.trailer.get(b"Encrypt").cloned() {
doc.authenticate(&enc, &credentials)?;
authenticated = true;
}
let catalog = doc.root_is_catalog();
cut_off = catalog && doc.update_cuts_off_the_pages();
catalog && !cut_off
}
Err(e) => {
log.push(format!("cross-reference data unreadable ({e}); rebuilding from object headers"));
false
}
};
if cut_off {
log.push(UNLINKED_UPDATE.into());
doc.unlinked_update = true;
} else if !ok && !doc.entries.is_empty() {
log.push("cross-reference data does not lead to a valid catalog; rebuilding from object headers".into());
}
if !ok {
doc.reconstruct(&mut log)?;
}
// Header generations are plaintext, including in encrypted files. Inspect only the
// bounded headers, not object bodies, and retain the compact xref representation.
doc.note_xref_generation_repairs(&mut log);
doc.repair_log = Arc::new(log);
// Reconstruction may have found a different trailer: (re)authenticate against it.
if (!ok || !authenticated)
&& let Some(enc) = doc.trailer.get(b"Encrypt").cloned()
{
doc.authenticate(&enc, &credentials)?;
}
if Arc::make_mut(&mut doc.entries).remove_exhausted() {
Arc::make_mut(&mut doc.repair_log).push("skipped exhausted xref object number 4294967295".into());
}
let max = doc.entries.max_num();
let size = u32::try_from(doc.trailer.int(b"Size").unwrap_or(0).max(0)).unwrap_or(u32::MAX);
if size == u32::MAX {
Arc::make_mut(&mut doc.repair_log).push("exhausted trailer /Size; new objects use unused object numbers".into());
}
doc.next_num = max.saturating_add(1).max(size).max(1);
Ok(doc)
}
/// Whether the chain of cross-reference sections ends at the newest one (no `/Prev`, or one
/// that isn't an offset) although the file holds earlier sections, and the catalog's `/Pages`
/// is not among the objects that one section reaches. ISO 32000-2 §7.5.6 and §7.5.8.2 require
/// the `/Prev`; without it the earlier revisions, page tree and all, are cut off, so the file
/// is rebuilt from its object headers as a damaged one is (the newest definition of each
/// object wins), and the repair log says why. (A file whose `/Root` itself is missing is
/// rebuilt anyway.)
fn update_cuts_off_the_pages(&self) -> bool {
if self.revisions.len() != 1 {
return false;
}
let Some(root) = self.root() else { return false };
let Ok(catalog) = self.try_get_reference(root) else { return false };
let Object::Dict(catalog) = &*catalog else { return false };
// The page tree first: the scan for an earlier `startxref` reads the whole file.
!catalog.get(b"Pages").is_some_and(|pages| matches!(&*self.resolve(pages), Object::Dict(_))) && earlier_sections(&self.data)
}
fn authenticate(&mut self, enc: &Object, credentials: &Credentials<'_>) -> Result<(), CosError> {
// Authentication can change how the same input is decoded.
self.source_identity = Arc::new(());
let (num, dict) = match enc {
Object::Ref(r) => (Some(r.num), self.get(*r).as_dict().cloned()),
Object::Dict(d) => (None, Some(d.clone())),
_ => (None, None),
};
let Some(dict) = dict else {
// A dangling /Encrypt: viewers treat the file as unencrypted, and so do we.
let mut log = self.repair_log.as_ref().clone();
log.push("the trailer names an /Encrypt dictionary that does not exist; reading the file as unencrypted".into());
self.repair_log = Arc::new(log);
self.trailer.remove(b"Encrypt");
return Ok(());
};
let params = crate::security::encrypt_dict(self, &dict);
let id0 = self.first_id();
// Read here, while nothing is decrypted yet.
let members = self.encryption_members(enc);
let handler = match credentials {
Credentials::Password(password) => match pdfcraft_crypt::SecurityHandler::open(params.clone(), &id0, *password) {
Ok(handler) => Arc::new(handler),
// No password, or the empty one: what only the embedded files need isn't asked.
Err(pdfcraft_crypt::CryptError::WrongPassword) => match self.without_password(&dict, params) {
Some(handler) if password.is_none_or(str::is_empty) => Arc::new(handler),
_ if password.is_none() => return Err(CosError::NeedsPassword),
_ => return Err(CosError::WrongPassword),
},
Err(other) => return Err(CosError::Security(other.to_string())),
},
// The same encryption as the document whose key this is: that key opens it. A handler
// without the key opens only what it would open by itself.
Credentials::Handler { handler, id0: known } if *handler.dict() == params && *known == id0 => {
if !handler.has_key() && !crate::security::efopen_only(self, &dict) {
return Err(CosError::NeedsPassword);
}
Arc::clone(handler)
}
Credentials::Handler { .. } => return Err(CosError::NeedsPassword),
};
self.security = Some(handler);
self.encrypt_num = num;
self.encrypt_members = Arc::new(members);
// Objects read while locating the catalog were read before decryption was possible.
self.cache = Arc::default();
self.objstms = Arc::default();
self.unreadable = Arc::default();
Ok(())
}
/// The handler for a document opened without a password that only its embedded files need
/// (see [`crate::security::efopen_only`]). Not a repair: ISO 32000-2 §7.6.6 has such a
/// document open so; [`Document::embedded_files_locked`] tells it.
fn without_password(&self, dict: &Dict, params: pdfcraft_crypt::EncryptDict) -> Option<pdfcraft_crypt::SecurityHandler> {
crate::security::efopen_only(self, dict).then(|| pdfcraft_crypt::SecurityHandler::without_key(params)).flatten()
}
/// A document opened without the password its embedded files need takes either password
/// later: with the file key its embedded files decrypt, so what was read as stored is read
/// again. Refused while an edited stream holds data read as stored, which the key would
/// encrypt a second time.
fn unlock_embedded_files(&mut self, password: &str) -> Result<(), CosError> {
let Some(current) = self.security.as_deref() else { return Ok(()) };
if self.overlay_entries().any(|(_, _, o)| matches!(o.as_deref(), Some(Object::Stream(s)) if self.reads_as_stored(s))) {
return Err(CosError::Security("an embedded file was edited before the password was given: save or undo that edit first".into()));
}
let handler = pdfcraft_crypt::SecurityHandler::open(current.dict().clone(), &self.first_id(), Some(password)).map_err(|e| match e {
pdfcraft_crypt::CryptError::WrongPassword => CosError::WrongPassword,
other => CosError::Security(other.to_string()),
})?;
self.security = Some(Arc::new(handler));
self.source_identity = Arc::new(());
self.cache = Arc::default();
self.objstms = Arc::default();
Ok(())
}
/// Whether the document opened without the password its embedded files need (`/AuthEvent
/// /EFOpen`): their data is held, and saved, as stored, and isn't to be read as the files.
pub fn embedded_files_locked(&self) -> bool {
self.security.as_deref().is_some_and(|h| !h.has_key())
}
/// Whether `s`, a stream of this document, holds encrypted data that reads as stored because the
/// document opened without the password its embedded files need (see
/// [`Document::embedded_files_locked`]). Such data isn't the file it encrypts.
pub fn reads_as_stored(&self, s: &crate::Stream) -> bool {
self.security.as_deref().is_some_and(|h| crate::security::stream_reads_as_stored(self, h, s))
}
/// The first file identifier, part of the encryption key (empty when there is none).
fn first_id(&self) -> Vec<u8> {
let id = self.trailer.get(b"ID").map(|id| self.resolve(id));
let first = id.as_deref().and_then(Object::as_array).and_then(|a| a.first()).map(|s| self.resolve(s));
first.as_deref().and_then(Object::as_string).map(|s| s.bytes.clone()).unwrap_or_default()
}
/// The objects `/Encrypt`, given as `enc`, and `/ID` refer to, the ones they refer to, and so
/// on, up to [`MAX_ENCRYPT_MEMBERS`]. Their strings aren't encrypted (ISO 32000-2 §7.6.2): the
/// key is made from them, so they have to read as stored, and stay so. A number with no object
/// behind it, and a stream, which those dictionaries never hold, isn't one: an object added
/// under that number later, or a page's content a hostile `/Encrypt` names, is encrypted.
fn encryption_members(&self, enc: &Object) -> HashSet<u32> {
let mut pending = Vec::new();
crate::writer::collect_refs(enc, &mut |r| pending.push(r));
if let Some(id) = self.trailer.get(b"ID") {
crate::writer::collect_refs(id, &mut |r| pending.push(r));
}
let mut members = HashSet::new();
while let Some(r) = pending.pop() {
if members.len() >= MAX_ENCRYPT_MEMBERS || members.contains(&r.num) {
continue;
}
let held = self.get(r);
if matches!(&*held, Object::Null | Object::Stream(_)) {
continue;
}
members.insert(r.num);
crate::writer::collect_refs(&held, &mut |x| pending.push(x));
}
members
}
/// Whether object `num` is written as it is held, being one of [`Document::encryption_members`],
/// unedited, and the save writing the same `/Encrypt`. What was edited or added is encrypted
/// whatever its number.
pub(crate) fn stored_as_is(&self, num: u32) -> bool {
!self.encryption_changed && !self.is_edited(num) && self.encrypt_members.contains(&num)
}
/// Lift the restrictions of a document opened with its user password by entering the
/// owner (permissions) password (§7.6.4). Fails with `WrongPassword` for any other
/// password. Does nothing when the document is unencrypted or already opened as owner.
/// Both passwords unlock the same file key, so nothing read so far is decoded again.
pub fn unlock(&mut self, password: &str) -> Result<(), CosError> {
let Some(current) = self.security.as_deref() else { return Ok(()) };
if !current.has_key() {
return self.unlock_embedded_files(password);
}
if current.auth() == pdfcraft_crypt::Auth::Owner {
return Ok(());
}
let owner = current.authenticate_owner(&self.first_id(), password).map_err(|e| match e {
pdfcraft_crypt::CryptError::WrongPassword => CosError::WrongPassword,
other => CosError::Security(other.to_string()),
})?;
self.security = Some(Arc::new(owner));
Ok(())
}
/// The security handler, when the document is encrypted.
pub fn security(&self) -> Option<&pdfcraft_crypt::SecurityHandler> {
self.security.as_deref()
}
/// What the opening password allows (`None` for unencrypted documents: everything).
pub fn permissions(&self) -> Option<pdfcraft_crypt::Permissions> {
self.security().map(|s| s.permissions())
}
/// Make the next save a full rewrite: earlier revisions (which still hold what an edit
/// removed) must not stay in the file. Redaction requires this.
pub fn require_full_save(&mut self) {
self.full_save = true;
}
/// Like [`Document::require_full_save`], and the rewritten file also gets a new file identifier
/// (`/ID`), so that a redacted or sanitized copy never claims to be its source (ISO 32000-2
/// §14.4). An encrypted file keeps the first string of its identifier, which its key depends on.
pub fn require_full_save_with_new_id(&mut self) {
self.full_save = true;
self.new_id = true;
}
/// The next save must rewrite the whole file (see [`Document::require_full_save`]).
/// Repair-aware edits retaining two accepted generations cannot use one incremental slot.
pub fn full_save_required(&self) -> bool {
self.full_save || self.overlay.values().any(|slot| matches!(slot, Slot::RepairedSet(..)))
}
/// The next full save gives the file a new identifier (see [`Document::require_full_save_with_new_id`]).
pub fn new_id_required(&self) -> bool {
self.new_id
}
/// Security was set or removed since the document was opened (the next save applies it).
pub fn encryption_changed(&self) -> bool {
self.encryption_changed
}
/// The handler used to write: the new one after `set_encryption` / `remove_encryption`,
/// otherwise the one the document was opened with.
pub(crate) fn output_security(&self) -> (Option<&pdfcraft_crypt::SecurityHandler>, Option<u32>) {
if self.encryption_changed { (self.out_security.as_deref(), self.out_encrypt_num) } else { (self.security.as_deref(), self.encrypt_num) }
}
/// The security the next save writes: protection applied with `set_encryption`, none after
/// `remove_encryption`, otherwise the security the document was opened with.
pub fn output_handler(&self) -> Option<&pdfcraft_crypt::SecurityHandler> {
self.output_security().0
}
/// Protect the document with a password (§7.6.4). Takes effect on the next save, which is
/// always a full rewrite. Returns the handler (authenticated as owner).
pub fn set_encryption(&mut self, params: &pdfcraft_crypt::NewEncryption) -> Result<(), CosError> {
// The file identifier is part of the key; make sure it exists and keep it.
let id = self.trailer.get(b"ID").map(|id| self.resolve(id));
let id0 = match id.as_deref() {
Some(Object::Array(a)) if a.len() == 2 => self.first_id(),
_ => {
let id = pdfcraft_crypt::file_id(&params.seed);
let s = Object::String(crate::PdfString { bytes: id.clone(), hex: true });
self.trailer.set(b"ID".to_vec(), Object::Array(vec![s.clone(), s]));
id
}
};
let h = pdfcraft_crypt::create(params, &id0).map_err(|e| CosError::Security(e.to_string()))?;
let d = h.dict();
let s = |b: &[u8]| Object::String(crate::PdfString { bytes: b.to_vec(), hex: true });
let mut e = Dict::new();
e.set(b"Filter".to_vec(), Object::name("Standard"));
e.set(b"V".to_vec(), Object::Int(d.v));
e.set(b"R".to_vec(), Object::Int(d.r));
e.set(b"Length".to_vec(), Object::Int(d.length_bits));
e.set(b"O".to_vec(), s(&d.o));
e.set(b"U".to_vec(), s(&d.u));
e.set(b"P".to_vec(), Object::Int(i64::from(d.p)));
if d.v >= 4 {
let mut cf = Dict::new();
let mut std_cf = Dict::new();
std_cf.set(b"Type".to_vec(), Object::name("CryptFilter"));
let (cfm, len) = if d.v >= 5 { ("AESV3", 32) } else { ("AESV2", 16) };
std_cf.set(b"CFM".to_vec(), Object::name(cfm));
std_cf.set(b"Length".to_vec(), Object::Int(len));
std_cf.set(b"AuthEvent".to_vec(), Object::name("DocOpen"));
cf.set(b"StdCF".to_vec(), Object::Dict(std_cf));
e.set(b"CF".to_vec(), Object::Dict(cf));
e.set(b"StmF".to_vec(), Object::name("StdCF"));
e.set(b"StrF".to_vec(), Object::name("StdCF"));
if !d.encrypt_metadata {
e.set(b"EncryptMetadata".to_vec(), Object::Bool(false));
}
}
if d.v >= 5 {
e.set(b"OE".to_vec(), s(&d.oe));
e.set(b"UE".to_vec(), s(&d.ue));
e.set(b"Perms".to_vec(), s(&d.perms));
}
if let Some(Object::Ref(old)) = self.trailer.get(b"Encrypt").cloned() {
self.free(old);
}
let r = self.add(e);
self.trailer.set(b"Encrypt".to_vec(), Object::Ref(r));
// Objects are still read with the original handler; saves use the new one.
self.out_security = Some(Arc::new(h));
self.out_encrypt_num = Some(r.num);
self.encryption_changed = true;
Ok(())
}
/// Remove password protection (requires the document to be open — any authenticated
/// password; callers should require the owner password, §7.6.4). Applied by a full save.
pub fn remove_encryption(&mut self) {
if let Some(Object::Ref(r)) = self.trailer.get(b"Encrypt").cloned() {
self.free(r);
}
self.trailer.remove(b"Encrypt");
self.out_security = None;
self.out_encrypt_num = None;
self.encryption_changed = true;
}
/// Decrypt a freshly parsed indirect object if the document is encrypted.
pub(super) fn decrypted(&self, id: ObjRef, o: Object) -> Object {
match &self.security {
Some(h) if Some(id.num) != self.encrypt_num && !self.encrypt_members.contains(&id.num) => {
crate::security::transform(self, h, &o, id.num, id.generation, crate::security::Pass::Decrypt)
}
_ => o,
}
}
}
/// The repair-log line for a file rebuilt because its last update has no `/Prev`.
const UNLINKED_UPDATE: &str = "the file's last update has no /Prev link to its earlier revisions, so their objects (the page tree among them) are unreachable through the cross-reference chain; rebuilding from object headers";
/// Whether `data` has more than one `startxref`: an update was appended to it, or it is
/// linearized (whose first-page section links to the main one).
fn earlier_sections(data: &[u8]) -> bool {
find(data, b"startxref", 0, data.len()).and_then(|at| find(data, b"startxref", at.saturating_add(9), data.len())).is_some()
}
-957
View File
@@ -1,957 +0,0 @@
#[cfg(test)]
use super::state::SECTION_READS;
use super::state::Unreadable;
use super::xref::{
MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData,
};
use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, HashSet, Lexer, ObjRef, Object, is_whitespace, parse_indirect, parse_indirect_shared};
impl Document {
#[allow(clippy::type_complexity)]
pub(super) fn read_xref_chain(&self, log: &mut Vec<String>) -> Result<(XrefIndex, Dict, Vec<Revision>, Vec<(u64, u32)>), CosError> {
let mut compact_log = Vec::new();
if let Ok(Some((index, trailer, revisions))) = self.read_compact_xref_chain(&mut compact_log) {
let size = trailer.int(b"Size").unwrap_or(0).max(0);
if size > 0 && size <= i64::from(u32::MAX) && u64::from(index.max_num()) < size as u64 {
log.extend(compact_log);
// The compact reader declines any section with set-aside entries, so there are none here.
return Ok((index, trailer, revisions, Vec::new()));
}
}
let (entries, trailer, revisions, oddities, generation_repairs) = self.read_xref_chain_materialized(log)?;
Ok((XrefIndex::Materialized(Arc::new(entries), Arc::new(generation_repairs)), trailer, revisions, oddities))
}
fn read_compact_xref_chain(&self, log: &mut Vec<String>) -> Result<Option<(XrefIndex, Dict, Vec<Revision>)>, CosError> {
let data = &self.data;
let tail_start = data.len().saturating_sub(4096);
let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?;
let mut lx = Lexer::new(data, sx + 9);
let first = match lx.object() {
Ok(Object::Int(n)) if n >= 0 => n as u64,
_ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }),
};
let mut sections = Vec::new();
let mut retained_rows = 0usize;
let mut retained_bytes = 0usize;
let mut trailer: Option<Dict> = None;
let mut revisions = Vec::new();
let mut visited = HashSet::new();
let mut next = Some(first);
while let Some(off) = next.take() {
if !visited.insert(off) || visited.len() > 10_000 {
log.push(format!("cross-reference chain loops at offset {off}; stopped"));
break;
}
let ((primary, sect_trailer, is_stream), header_relative) = match self.read_compact_section(off as usize) {
Ok(Some(v)) => (v, false),
Ok(None) => return Ok(None),
Err(e) if self.header_offset > 0 => {
log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header"));
match self.read_compact_section(off as usize + self.header_offset)? {
Some(v) => (v, true),
None => return Ok(None),
}
}
// An unreadable older section loses every object only it locates: the
// materialized reader logs it and the file is rebuilt from object headers (#1656).
Err(e) => return Err(e),
};
revisions.push(Revision { xref_offset: off, is_stream, header_relative });
if primary.renumbered {
log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0"));
}
let Some(rows) = retained_rows.checked_add(primary.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) };
let Some(bytes) = retained_bytes.checked_add(primary.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES) else {
return Ok(None);
};
retained_rows = rows;
retained_bytes = bytes;
let hybrid = if let Some(x) = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0) {
match self.read_compact_section(x as usize) {
Ok(Some((section, _, _))) => {
let Some(rows) = retained_rows.checked_add(section.row_count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else {
return Ok(None);
};
let Some(bytes) = retained_bytes.checked_add(section.retained_bytes()).filter(|bytes| *bytes <= MAX_COMPACT_XREF_BYTES)
else {
return Ok(None);
};
retained_rows = rows;
retained_bytes = bytes;
Some(section)
}
Ok(None) => return Ok(None),
Err(e) => {
log.push(format!("hybrid /XRefStm at {x} unreadable: {e}"));
None
}
}
} else {
None
};
if let Some(hybrid) = &hybrid {
// A hybrid entry that disagrees with the table is set aside by the materialized reader.
let mut numbers = Vec::new();
hybrid.extend_numbers(&mut numbers);
for num in numbers {
let Some(entry) = hybrid.lookup(num, &self.data) else { continue };
if primary.lookup(num, &self.data).is_some_and(|old| !matches!(old, XrefEntry::Free { .. }) && old != entry) {
return Ok(None);
}
}
}
sections.push(XrefRevisionSection { primary, hybrid });
next = prev_offset(data, &sect_trailer, off, |num| sections.iter().find_map(|s| s.lookup(num, data)), log);
if trailer.is_none() {
trailer = Some(sect_trailer);
}
}
revisions.reverse();
let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?;
for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] {
trailer.remove(k);
}
let mut max_num = 0;
let mut has_entries = false;
for revision in &sections {
max_num = max_num.max(revision.primary.max_num);
has_entries |= revision.primary.has_entries;
if let Some(hybrid) = &revision.hybrid {
max_num = max_num.max(hybrid.max_num);
has_entries |= hybrid.has_entries;
}
}
Ok(Some((XrefIndex::Compact { revisions: sections, max_num, has_entries }, trailer, revisions)))
}
fn read_compact_section(&self, off: usize) -> Result<Option<(XrefSection, Dict, bool)>, CosError> {
let data = &self.data;
let mut lx = Lexer::new(data, off);
if lx.eat_keyword(b"xref") {
let mut ranges = Vec::new();
let mut layout_ok = true;
let mut max_num = 0;
let mut has_entries = false;
let mut previous_end = None;
let mut row_count = 0usize;
let mut renumbered = false;
loop {
lx.skip_ws();
if lx.eat_keyword(b"trailer") {
let t = lx.object()?;
let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?;
if !layout_ok {
return Ok(None);
}
return Ok(Some((XrefSection { data: XrefSectionData::Table, ranges, row_count, max_num, has_entries, renumbered }, d, false)));
}
let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else {
return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() });
};
if !valid_xref_range(start, count) || count > 10_000_000 {
return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() });
}
let shifted = subsection_start(data, self.header_offset, lx.pos, start, count);
renumbered |= shifted != start;
let start = shifted;
let Some(next_rows) = usize::try_from(count).ok().and_then(|count| row_count.checked_add(count)) else { return Ok(None) };
if next_rows > MAX_COMPACT_XREF_ROWS {
return Ok(None);
}
row_count = next_rows;
let mut range_offset = None;
let mut stride = None;
let mut previous_record = None;
for i in 0..count {
lx.skip_ws();
let record = lx.pos;
if range_offset.is_none() {
range_offset = Some(record);
}
if let Some(previous) = previous_record {
let Some(delta) = record.checked_sub(previous) else {
layout_ok = false;
continue;
};
if stride.is_some_and(|known| known != delta) {
layout_ok = false;
}
stride.get_or_insert(delta);
}
previous_record = Some(record);
let o = lx.token();
lx.skip_ws();
let _generation = lx.token();
lx.skip_ws();
let kind = lx.token();
let num = (start + i) as u32;
let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::<u64>().ok());
match (kind, offset) {
(b"n", Some(offset)) if offset != 0 => {
has_entries = true;
max_num = max_num.max(num);
}
(b"n", Some(_)) => {
// Set aside by the materialized reader, which records it.
if num != 0 {
return Ok(None);
}
}
(b"f", _) => {
has_entries = true;
max_num = max_num.max(num);
}
_ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }),
}
}
if count > 0 {
let range_start = start as u32;
let range_count = count as u32;
let end = u64::from(range_start) + u64::from(range_count);
if previous_end.is_some_and(|previous| u64::from(range_start) < previous) {
layout_ok = false;
}
previous_end = Some(end);
ranges.push(XrefRange {
start: range_start,
count: range_count,
offset: range_offset.unwrap_or(lx.pos),
stride: stride.unwrap_or(1),
});
}
}
}
let (_, obj) = parse_indirect(data, off, &|_| None)?;
let Object::Stream(stream) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) };
if stream.dict.name(b"Type") != Some(b"XRef") {
return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() });
}
let w: Vec<usize> = stream
.dict
.get(b"W")
.and_then(Object::as_array)
.map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect())
.unwrap_or_default();
if w.len() < 3 {
return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() });
}
let widths = [w[0], w[1], w[2]];
let row = widths.iter().sum::<usize>();
if row == 0 {
return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() });
}
let size = stream.dict.int(b"Size").unwrap_or(0).max(0);
let index: Vec<i64> = match stream.dict.get(b"Index").and_then(Object::as_array) {
Some(a) => a.iter().filter_map(Object::as_int).collect(),
None => vec![0, size],
};
let mut requested_rows = 0usize;
let mut previous_end = None;
for pair in index.chunks(2) {
let [start, count] = pair else { break };
if !valid_xref_range(*start, *count) {
return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() });
}
let Ok(count) = usize::try_from(*count) else { return Ok(None) };
let Some(next) = requested_rows.checked_add(count).filter(|rows| *rows <= MAX_COMPACT_XREF_ROWS) else { return Ok(None) };
if count > 0 {
let end = (*start as u64).saturating_add(count as u64);
if previous_end.is_some_and(|previous| (*start as u64) < previous) {
return Ok(None);
}
previous_end = Some(end);
}
requested_rows = next;
}
if requested_rows.checked_mul(row).is_none_or(|bytes| bytes > MAX_COMPACT_XREF_BYTES) {
return Ok(None);
}
let raw = stream.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit).min(MAX_COMPACT_XREF_BYTES))?;
if raw.capacity() > MAX_COMPACT_XREF_BYTES {
return Ok(None);
}
let mut ranges = Vec::new();
let mut row_start = 0usize;
let available_rows = raw.len() / row;
for pair in index.chunks(2) {
let [start, count] = pair else { break };
let range_start = *start as u32;
let Ok(requested) = usize::try_from(*count) else { return Ok(None) };
let range_count = requested.min(available_rows.saturating_sub(row_start));
if range_count > 0 {
let Ok(range_count_u32) = u32::try_from(range_count) else { return Ok(None) };
let Some(byte_offset) = row_start.checked_mul(row) else { return Ok(None) };
ranges.push(XrefRange { start: range_start, count: range_count_u32, offset: byte_offset, stride: row });
}
let Some(next) = row_start.checked_add(range_count) else { return Ok(None) };
row_start = next;
}
// Rows the materialized reader sets aside (reserved types, offset 0) are left to it.
for range in &ranges {
for i in 0..range.count as usize {
let at = range.offset + i * row;
let Some(r) = raw.get(at..at + row) else { break };
let field = |from: usize, width: usize, default: u64| -> u64 {
if width == 0 {
return default;
}
r[from..from + width].iter().fold(0u64, |acc, byte| acc << 8 | u64::from(*byte))
};
let kind = field(0, widths[0], 1);
let offset = field(widths[0], widths[1], 0);
if kind > 2 || (kind == 1 && offset == 0 && range.start + i as u32 != 0) {
return Ok(None);
}
}
}
let max_num = ranges.iter().filter_map(|range| range.start.checked_add(range.count.saturating_sub(1))).max().unwrap_or(0);
let row_count = row_start;
let section = XrefSection {
data: XrefSectionData::Stream { bytes: Arc::new(raw), widths },
ranges,
row_count,
max_num,
has_entries: false,
renumbered: false,
};
let has_entries = section.has_any_entry();
let section = XrefSection { has_entries, ..section };
Ok(Some((section, stream.dict.clone(), true)))
}
/// Read the cross-reference chain from `startxref` into one map of entries, newer sections
/// winning, with the trailer, the revisions oldest first and the set-aside entries. A chain
/// that lists more than [`MAX_XREF_OBJECTS`] rows in all is refused with
/// [`CosError::Syntax`], and the caller rebuilds the file from object headers (#5082).
#[allow(clippy::type_complexity)]
pub(super) fn read_xref_chain_materialized(
&self,
log: &mut Vec<String>,
) -> Result<(BTreeMap<u32, XrefEntry>, Dict, Vec<Revision>, Vec<(u64, u32)>, Vec<String>), CosError> {
self.read_xref_chain_materialized_with_limit(log, MAX_XREF_OBJECTS)
}
/// [`Self::read_xref_chain_materialized`] refusing a chain past `row_limit` rows in all
/// instead of [`MAX_XREF_OBJECTS`], so tests can reach the limit with a few rows.
#[allow(clippy::type_complexity)]
pub(super) fn read_xref_chain_materialized_with_limit(
&self,
log: &mut Vec<String>,
row_limit: usize,
) -> Result<(BTreeMap<u32, XrefEntry>, Dict, Vec<Revision>, Vec<(u64, u32)>, Vec<String>), CosError> {
let data = &self.data;
let tail_start = data.len().saturating_sub(4096);
let sx = rfind(data, b"startxref", tail_start).ok_or(CosError::Syntax { offset: data.len(), detail: "no startxref".into() })?;
let mut lx = Lexer::new(data, sx + 9);
let first = match lx.object() {
Ok(Object::Int(n)) if n >= 0 => n as u64,
_ => return Err(CosError::Syntax { offset: sx, detail: "bad startxref value".into() }),
};
let mut entries = BTreeMap::new();
let mut trailer: Option<Dict> = None;
let mut revisions = Vec::new();
let mut oddities = Vec::new();
let mut visited = HashSet::new();
// Hybrid streams already read, by offset: their entries, or `None` when unreadable. A newer
// section has merged every entry of one into `entries`, so an older section that names it
// again only has its own table compared with them, without decoding the stream again.
// At most MAX_XREF_OBJECTS entries are kept in all; a stream past that is read again.
let mut hybrids: HashMap<i64, Option<BTreeMap<u32, XrefEntry>>> = HashMap::new();
let (mut hybrid_rows, mut noted) = (0usize, HashSet::new());
let mut generation_repairs = Vec::new();
let mut recorded = HashSet::new();
let mut diagnose = true;
// Every row the sections (and their hybrid streams) list, counted by `read_section` as it
// reads them, before any are merged or deduplicated.
let mut merged_rows = 0usize;
let mut next = Some(first);
while let Some(off) = next.take() {
if !visited.insert(off) || visited.len() > 10_000 {
log.push(format!("cross-reference chain loops at offset {off}; stopped"));
break;
}
// This section's own entries; older sections only fill what newer ones left out.
let mut sect = BTreeMap::new();
// Object numbers whose entry in this section was set aside ([`Self::xref_oddities`]).
let mut odd = Vec::new();
let rows_before = merged_rows;
let ((sect_trailer, is_stream), header_relative) = match self.read_section(off as usize, &mut sect, &mut odd, &mut merged_rows, log) {
Ok(v) => (v, false),
Err(e) if self.header_offset > 0 => {
log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header"));
sect.clear();
odd.clear();
merged_rows = rows_before;
(self.read_section(off as usize + self.header_offset, &mut sect, &mut odd, &mut merged_rows, log)?, true)
}
Err(e) => {
// An unreadable older section loses every object only it locates (the page
// tree, often): rebuild from object headers, as readers do (#1656).
if !revisions.is_empty() {
log.push(format!("older cross-reference section at {off} is unreadable: {e}"));
}
return Err(e);
}
};
revisions.push(Revision { xref_offset: off, is_stream, header_relative });
for (&num, &entry) in &sect {
if !diagnose {
break;
}
diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs);
}
// Hybrid files (§7.5.8.4): the /XRefStm entries supplement the table and replace the
// free entries it gives objects only a cross-reference stream can locate.
let hybrid = sect_trailer.int(b"XRefStm").filter(|x| *x >= 0);
if let Some((x, read)) = hybrid.and_then(|x| hybrids.get(&x).map(|read| (x, read))) {
if noted.insert(x) {
log.push(format!("hybrid /XRefStm at {x} named again by the section at {off}; read once"));
}
if let Some(stm) = read {
for (num, old) in &sect {
// The table places the object somewhere other than the stream does.
if !matches!(old, XrefEntry::Free { .. }) && stm.get(num).is_some_and(|e| e != old) {
odd.push(*num);
}
}
}
} else if let Some(x) = hybrid {
let mut stm = BTreeMap::new();
let read = match self.read_section(x as usize, &mut stm, &mut odd, &mut merged_rows, log) {
Ok(_) => {
for (&num, &entry) in &stm {
if !diagnose {
break;
}
diagnose = self.inspect_xref_generation_repair(num, entry, &mut recorded, &mut generation_repairs);
}
for (&num, &e) in &stm {
match sect.get(&num) {
None | Some(XrefEntry::Free { .. }) => {
sect.insert(num, e);
}
// The table already places the object somewhere else.
Some(old) if *old != e => odd.push(num),
Some(_) => {}
}
}
Some(stm)
}
Err(e) => {
log.push(format!("hybrid /XRefStm at {x} unreadable: {e}"));
None
}
};
let rows = read.as_ref().map_or(0, BTreeMap::len);
if let Some(total) = hybrid_rows.checked_add(rows).filter(|total| *total <= MAX_XREF_OBJECTS) {
hybrid_rows = total;
hybrids.insert(x, read);
}
}
// Each section's rows are capped, so the chain is too: at most `row_limit` rows
// (MAX_XREF_OBJECTS outside tests) read in all, or the file is rebuilt from object
// headers, as when one section's stream decodes past that. This counts every row
// read, before deduplication, not the map's size: a chain that relists the same
// objects, or a stream whose `/Index` repeats a range (`[0 4 0 4]`), costs as much to
// merge (#5082). Rows set aside are among the rows read, and a row is set aside at
// most twice (in its section and against a hybrid stream), so they stay bounded too. A
// section's rows (its hybrid stream's included) never outnumber the rows the compact
// reader counts for it, and that reader stops at about a quarter of MAX_XREF_OBJECTS:
// no chain within its limit is refused.
if merged_rows > row_limit {
return Err(CosError::Syntax {
offset: usize::try_from(off).unwrap_or(usize::MAX),
detail: format!("cross-reference chain lists more than {row_limit} entries"),
});
}
oddities.extend(odd.into_iter().map(|num| (off, num)));
for (num, e) in sect {
entries.entry(num).or_insert(e);
}
next = prev_offset(data, &sect_trailer, off, |num| entries.get(&num).copied(), log);
if trailer.is_none() {
trailer = Some(sect_trailer);
}
}
revisions.reverse();
let mut trailer = trailer.ok_or(CosError::Syntax { offset: 0, detail: "no trailer".into() })?;
for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] {
trailer.remove(k);
}
Ok((entries, trailer, revisions, oddities, generation_repairs))
}
/// Read one section at `off`, adding entries not already known (newer sections win). `odd`
/// collects the object numbers whose entry here is set aside ([`Self::xref_oddities`]), and
/// `counted` grows by every row read, repeated and set-aside rows included (#5082).
fn read_section(
&self,
off: usize,
entries: &mut BTreeMap<u32, XrefEntry>,
odd: &mut Vec<u32>,
counted: &mut usize,
log: &mut Vec<String>,
) -> Result<(Dict, bool), CosError> {
#[cfg(test)]
SECTION_READS.with(|c| c.set(c.get() + 1));
let data = &self.data;
let mut lx = Lexer::new(data, off);
if lx.eat_keyword(b"xref") {
loop {
lx.skip_ws();
if lx.eat_keyword(b"trailer") {
let t = lx.object()?;
let d = t.as_dict().cloned().ok_or(CosError::Syntax { offset: lx.pos, detail: "trailer is not a dictionary".into() })?;
return Ok((d, false));
}
let (Ok(Object::Int(start)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else {
return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref subsection header".into() });
};
if !valid_xref_range(start, count) || count > 10_000_000 {
return Err(CosError::Syntax { offset: lx.pos, detail: "invalid xref object range".into() });
}
let shifted = subsection_start(data, self.header_offset, lx.pos, start, count);
if shifted != start {
log.push(format!("xref subsection at {off} is numbered from 1 but begins with object 0's entry; read from 0"));
}
let start = shifted;
for i in 0..count {
*counted = counted.saturating_add(1);
lx.skip_ws();
let o = lx.token();
lx.skip_ws();
let g = lx.token();
lx.skip_ws();
let kind = lx.token();
let num = (start + i) as u32;
let offset = std::str::from_utf8(o).ok().and_then(|s| s.parse::<u64>().ok());
let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::<u32>().ok()).unwrap_or(0).min(u16::MAX as u32) as u16;
let entry = match (kind, offset) {
(b"n", Some(offset)) => XrefEntry::InFile { offset, generation },
(b"f", _) => XrefEntry::Free { next_generation: generation },
_ => return Err(CosError::Syntax { offset: lx.pos, detail: "bad xref entry".into() }),
};
// Offset 0 "in use" entries are a known producer bug: ignore them. (Object 0
// is never a real object, so its entry cannot mislead anyone.)
if matches!(entry, XrefEntry::InFile { offset: 0, .. }) {
if num != 0 {
odd.push(num);
}
continue;
}
if *entries.entry(num).or_insert(entry) != entry {
odd.push(num);
}
}
}
}
// Cross-reference stream.
let (_, obj) = parse_indirect(data, off, &|_| None)?;
let Object::Stream(s) = obj else { return Err(CosError::Syntax { offset: off, detail: "expected xref table or stream".into() }) };
if s.dict.name(b"Type") != Some(b"XRef") {
return Err(CosError::Syntax { offset: off, detail: "stream at startxref is not /Type /XRef".into() });
}
let w: Vec<usize> = s
.dict
.get(b"W")
.and_then(Object::as_array)
.map(|a| a.iter().map(|x| x.as_int().unwrap_or(0).clamp(0, 8) as usize).collect())
.unwrap_or_default();
if w.len() < 3 {
return Err(CosError::Syntax { offset: off, detail: "xref stream /W invalid".into() });
}
let size = s.dict.int(b"Size").unwrap_or(0).max(0);
let index: Vec<i64> = match s.dict.get(b"Index").and_then(Object::as_array) {
Some(a) => a.iter().filter_map(Object::as_int).collect(),
None => vec![0, size],
};
let row = w[0] + w[1] + w[2];
if row == 0 {
return Err(CosError::Syntax { offset: off, detail: "xref stream row width 0".into() });
}
// One row per object, and a document has at most 8,388,607 indirect objects (the classic
// implementation limit): decoding more than that (about 200 MB at the widest rows) is a
// decompression bomb, not cross-reference data. The section is then reconstructed.
let raw = s.decoded_within(MAX_XREF_OBJECTS.saturating_mul(row).min(self.stream_limit))?;
let field = |r: &[u8], from: usize, len: usize, default: u64| -> u64 {
if len == 0 {
return default;
}
r[from..from + len].iter().fold(0u64, |acc, b| acc << 8 | *b as u64)
};
let mut rows = raw.chunks_exact(row);
for pair in index.chunks(2) {
let [start, count] = pair else { break };
if !valid_xref_range(*start, *count) {
return Err(CosError::Syntax { offset: off, detail: "invalid xref object range".into() });
}
for i in 0..*count {
let Some(r) = rows.next() else { break };
*counted = counted.saturating_add(1);
let t = field(r, 0, w[0], 1);
let a = field(r, w[0], w[1], 0);
let b = field(r, w[0] + w[1], w[2], 0);
let num = (*start + i) as u32;
let entry = match t {
0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 },
1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 },
2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 },
// Reserved types are treated as null references (§7.5.8.3).
_ => {
odd.push(num);
continue;
}
};
if matches!(entry, XrefEntry::InFile { offset: 0, .. }) {
if num != 0 {
odd.push(num);
}
continue;
}
if *entries.entry(num).or_insert(entry) != entry {
odd.push(num);
}
}
}
Ok((s.dict.clone(), true))
}
/// Cross-reference data that leaves out objects the file contains: a subsection numbered one
/// too low (object 0 in use, the last object without an entry), or an in-use entry at offset 0
/// that nothing else replaces. (A number no section lists is not evidence: it is a null
/// object, §7.3.10, and valid files have them below an oversized `/Size`.) Viewers rebuild
/// the table and show these objects; reading them as null would drop them from every copy.
/// When one of those shows, the file is scanned once for object headers, and wrong offsets
/// are looked up there (`load`). Only the numbers the damage leaves out take the last
/// header's location: one whose entry was set aside, or the last object of a subsection
/// numbered one too low. Any other unlisted number stays null, and existing
/// entries are never replaced: an object freed by an update stays freed.
pub(super) fn fill_xref_gaps(&mut self, log: &mut Vec<String>) {
let entries = &self.entries;
let shifted = matches!(entries.lookup(0, &self.data), Some(XrefEntry::InFile { .. }));
let set_aside = self.xref_oddities.iter().any(|(_, num)| entries.lookup(*num, &self.data).is_none());
if !(shifted || set_aside) {
return;
}
let mut located = BTreeMap::new();
for (num, offset, generation, _) in object_headers(&self.data) {
located.insert(num, (offset, generation));
}
let mut filled = entries.to_map(&self.data);
// A subsection `0 N` whose rows are those of objects 1 to N leaves out object N, whatever
// newer sections say about its neighbours.
let past_shift: HashSet<u32> = if shifted {
self.revisions
.iter()
.filter(|r| !r.is_stream)
.filter_map(|r| usize::try_from(r.xref_offset).ok().map(|o| if r.header_relative { o.saturating_add(self.header_offset) } else { o }))
.filter_map(|at| shifted_subsection_end(&self.data, at))
.collect()
} else {
HashSet::new()
};
let odd: HashSet<u32> = self.xref_oddities.iter().map(|(_, n)| *n).collect();
let mut found = 0usize;
for (&num, &(offset, generation)) in &located {
let left_out = past_shift.contains(&num) || odd.contains(&num);
if num != 0 && !filled.contains_key(&num) && left_out {
filled.insert(num, XrefEntry::InFile { offset, generation });
found += 1;
}
}
if found > 0 {
log.push(format!("the cross-reference data leaves out {found} objects the file contains; located them by their object headers"));
self.entries = Arc::new(XrefIndex::Materialized(Arc::new(filled), Arc::default()));
}
self.located = Arc::new(located);
}
/// Rebuild the cross-reference index by scanning for `N G obj` headers (§7.5.4 recovery).
pub(super) fn reconstruct(&mut self, log: &mut Vec<String>) -> Result<(), CosError> {
let data = self.data.clone();
let mut entries: BTreeMap<u32, XrefEntry> = BTreeMap::new();
let mut objstms = Vec::new();
// Offsets of objects that may be cross-reference streams: their dictionaries are the
// trailers of files that have no `trailer` keyword (§7.5.8).
let mut xref_streams = Vec::new();
let headers = object_headers(&data);
// Where `/XRef` occurs in the file, found in one pass.
let mut xref_marks = Vec::new();
let mut mark_from = 0;
while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) {
xref_marks.push(p);
mark_from = p + 1;
}
for &(num, offset, generation, body) in &headers {
entries.insert(num, XrefEntry::InFile { offset, generation });
objstms.push(num);
// `/XRef` within the 1024 bytes after the header (the first one that is there).
let first = xref_marks.partition_point(|&p| p < body);
if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len()))
&& let Ok(at) = usize::try_from(offset)
{
xref_streams.push(at);
}
}
if entries.is_empty() {
return Err(CosError::Syntax { offset: 0, detail: "no objects found".into() });
}
self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()), Arc::default()));
self.cache.lock().map_err(|_| CosError::Poisoned)?.clear();
*self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default();
// Objects inside object streams.
for num in objstms {
if let Ok(o) = self.try_get(num)
&& let Object::Stream(s) = &*o
&& s.dict.name(b"Type") == Some(b"ObjStm")
&& let Ok(stm) = self.objstm(num)
{
for (idx, member) in stm.index.iter().enumerate() {
if let Some((inner, _)) = member {
entries.entry(*inner).or_insert(XrefEntry::InStream { stream: num, index: idx as u32 });
}
}
}
}
let object_count = entries.len();
self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries), Arc::default()));
// Failures from the scan stand: the members added change no scanned entry, and a
// `/Length` found only now cannot make a stream unreadable.
self.cache.lock().map_err(|_| CosError::Poisoned)?.clear();
// Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a
// usable /Root (last in the file, whichever kind), else the catalog itself. One whose
// /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another
// dictionary, such as /Info, wherever it is; an FDF's /Root (no /Type, no /Pages) still
// counts when nothing better is there.
// A file whose only trailers are cross-reference streams (and whose `startxref` is gone
// or points nowhere) keeps its /Info, /ID and /Encrypt this way.
let rank = |at: usize, d: &Dict| {
let root = self.try_get_reference(d.reference(b"Root")?).ok()?;
let Object::Dict(c) = &*root else { return None };
Some((c.name(b"Type") == Some(b"Catalog") || c.contains(b"Pages"), at))
};
let mut trailer: Option<((bool, usize), Dict)> = None;
let mut from = 0;
while let Some(p) = find(&data, b"trailer", from, data.len()) {
from = p + 7;
let mut lx = Lexer::new(&data, from);
if let Ok(Object::Dict(d)) = lx.object()
&& let Some(r) = rank(p, &d)
&& trailer.as_ref().is_none_or(|(best, _)| r > *best)
{
trailer = Some((r, d));
}
}
for at in xref_streams {
if trailer.as_ref().is_some_and(|((catalog, p), _)| *catalog && *p > at) {
continue;
}
if let Ok((_, Object::Stream(s))) = parse_indirect_shared(&data, at, &|_| None, &self.endstreams)
&& s.dict.name(b"Type") == Some(b"XRef")
&& let Some(r) = rank(at, &s.dict)
&& trailer.as_ref().is_none_or(|(best, _)| r > *best)
{
trailer = Some((r, s.dict));
}
}
let trailer = match trailer {
Some((_, t)) => t,
None => {
let catalog = self
.entries
.object_numbers(&self.data)
.into_iter()
.find(|n| matches!(self.try_get(*n).as_deref(), Ok(Object::Dict(d)) if d.name(b"Type") == Some(b"Catalog")));
let Some(c) = catalog else { return Err(CosError::Syntax { offset: 0, detail: "no document catalog found".into() }) };
let mut t = Dict::new();
t.set(b"Root".to_vec(), Object::Ref(ObjRef::new(c, self.generation(c))));
t
}
};
let mut trailer = trailer;
// What a cross-reference stream's dictionary carries for itself goes, as the chain
// readers drop it: a save writes its own.
for k in [&b"Prev"[..], b"XRefStm", b"Type", b"W", b"Index", b"Filter", b"DecodeParms", b"Length"] {
trailer.remove(k);
}
self.trailer = trailer;
self.revisions = Arc::new(Vec::new());
log.push(format!("rebuilt cross-reference index from {object_count} object headers"));
Ok(())
}
}
/// For a classic table at `at` whose first subsection is `0 N` and opens with an in-use row
/// (the rows of objects 1 to N), the number N its rows really end at.
fn shifted_subsection_end(data: &[u8], at: usize) -> Option<u32> {
let mut lx = Lexer::new(data, at);
if !lx.eat_keyword(b"xref") {
return None;
}
let (Ok(Object::Int(0)), Ok(Object::Int(count))) = (lx.object(), lx.object()) else { return None };
lx.skip_ws();
let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::<u64>().ok());
lx.skip_ws();
lx.token();
lx.skip_ws();
match (offset, lx.token()) {
(Some(offset), b"n") if offset != 0 => u32::try_from(count).ok().filter(|n| *n > 0),
_ => None,
}
}
/// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position
/// after `obj`).
fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> {
let mut headers = Vec::new();
let mut i = 0;
while i < data.len() {
// Object headers start at a line start (or after whitespace).
let at_start = i == 0 || is_whitespace(data[i - 1]) || data[i - 1] == b'>';
if at_start && data[i].is_ascii_digit() {
let mut lx = Lexer::new(data, i);
let n = lx.token();
lx.skip_ws();
let g = lx.token();
if !n.is_empty() && !g.is_empty() && n.iter().all(u8::is_ascii_digit) && g.iter().all(u8::is_ascii_digit) && lx.eat_keyword(b"obj") {
let num = std::str::from_utf8(n).ok().and_then(|s| s.parse::<u32>().ok());
let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::<u16>().ok()).unwrap_or(0);
if let Some(num) = num {
headers.push((num, i as u64, generation, lx.pos));
}
i = lx.pos;
continue;
}
}
i += 1;
}
headers
}
/// The offset of the section before the one whose trailer this is (`/Prev`), if any. Some writers
/// give it as a reference, `N 0 R`. When the sections read so far (`lookup`) place that object
/// and it holds an offset (iText's `indirectPrev.pdf`), that is the offset; otherwise, as the
/// tiff2pdf files in JHOVE's PDF-HUL-157 cases need, the sections before are at N. Either way the
/// reference is noted. Any other value that is not an offset is noted and ends the chain.
fn prev_offset(data: &[u8], trailer: &Dict, off: u64, lookup: impl Fn(u32) -> Option<XrefEntry>, log: &mut Vec<String>) -> Option<u64> {
match trailer.get(b"Prev") {
Some(Object::Int(p)) if *p >= 0 => Some(*p as u64),
Some(Object::Ref(r)) => {
let held = match lookup(r.num) {
Some(XrefEntry::InFile { offset, generation }) if generation == r.generation => {
usize::try_from(offset).ok().and_then(|at| parse_indirect(data, at, &|_| None).ok()).and_then(|(at, value)| match value {
Object::Int(p) if at == *r && p >= 0 => Some(p as u64),
_ => None,
})
}
_ => None,
};
match held {
Some(p) => {
log.push(format!(
"the trailer at {off} gives /Prev as the reference {} {} R; reading the offset {p} it holds",
r.num, r.generation
));
Some(p)
}
None => {
log.push(format!(
"the trailer at {off} gives /Prev as the reference {} {} R; reading it as offset {}",
r.num, r.generation, r.num
));
Some(u64::from(r.num))
}
}
}
Some(_) => {
log.push(format!("the trailer at {off} has a /Prev that is not an offset; the sections before it are unreachable"));
None
}
None => None,
}
}
pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option<usize> {
let to = to.min(hay.len());
if from >= to || needle.is_empty() || to - from < needle.len() {
return None;
}
let hay = &hay[from..to];
// Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there.
const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz";
let (k, rare) =
needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0));
let mut i = k;
while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) {
let start = i + p - k;
if hay.get(start..start + needle.len()) == Some(needle) {
return Some(start + from);
}
i += p + 1;
}
None
}
/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which
/// the compiler turns into vector compares; only a block that holds the byte is searched.
fn position_of(hay: &[u8], byte: u8) -> Option<usize> {
const BLOCK: usize = 32;
let (blocks, rest) = hay.as_chunks::<BLOCK>();
for (i, block) in blocks.iter().enumerate() {
if block.iter().fold(false, |any, &b| any | (b == byte)) {
return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p);
}
}
rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p)
}
pub(super) fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option<usize> {
if hay.len() < needle.len() {
return None;
}
(from..=hay.len() - needle.len()).rev().find(|&i| &hay[i..i + needle.len()] == needle)
}
/// The object number a classic xref subsection really starts at. A common producer bug heads the
/// table `1 n` and then writes object 0's free entry (`0000000000 65535 f`) as its first row, so
/// every row would be read one number too high and object 1 as free. Such a subsection is read
/// from 0 when its second row, read that way, locates an object that is indeed numbered 1, at
/// that offset or, as `load` reads offsets, relative to a header shifted by `header_offset`.
/// `rows` is the position just after the subsection header.
fn subsection_start(data: &[u8], header_offset: usize, rows: usize, start: i64, count: i64) -> i64 {
if start != 1 || count < 2 {
return start;
}
let mut lx = Lexer::new(data, rows);
let mut row = || {
lx.skip_ws();
let offset = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::<u64>().ok());
lx.skip_ws();
let generation = std::str::from_utf8(lx.token()).ok().and_then(|s| s.parse::<u32>().ok());
lx.skip_ws();
(offset, generation, lx.token() == b"f")
};
let (Some(0), Some(65535), true) = row() else { return start };
let (Some(second), _, false) = row() else { return start };
let object_one = |at: usize| {
if at >= data.len() {
return false;
}
let mut obj = Lexer::new(data, at);
matches!((obj.object(), obj.object()), (Ok(Object::Int(1)), Ok(Object::Int(_)))) && obj.eat_keyword(b"obj")
};
let Ok(second) = usize::try_from(second) else { return start };
if object_one(second) || (header_offset != 0 && object_one(second.saturating_add(header_offset))) { 0 } else { start }
}
/// Object numbers must fit the representation used by references and the xref map.
/// Validate the whole subsection before adding entries, so neither arithmetic overflow nor
/// truncation can turn a damaged range into entries for unrelated objects.
fn valid_xref_range(start: i64, count: i64) -> bool {
u32::try_from(start).is_ok() && count >= 0 && start.checked_add(count).is_some_and(|end| end <= i64::from(u32::MAX) + 1)
}
/// The most indirect objects a document can have (the classic implementation limit, ISO
/// 32000-1 Annex C): it bounds how much data a cross-reference stream can hold.
const MAX_XREF_OBJECTS: usize = 8_388_607;
-504
View File
@@ -1,504 +0,0 @@
use super::xref::XrefEntry;
use super::{Arc, BTreeMap, CosError, Dict, Document, HashMap, ObjRef, Object, Weak};
thread_local! {
/// The objects being loaded on this thread, innermost last (see `Document::try_get`).
static LOADING: std::cell::RefCell<Vec<u32>> = const { std::cell::RefCell::new(Vec::new()) };
/// Indirect `/Length` values resolved during the current outermost load: a damaged file
/// retries a parse several ways, and each retry would resolve the same lengths again
/// (exponential in the chain of streams whose lengths point at further streams). Each is
/// kept with whether resolving it read unsaved edits.
pub(super) static LENGTHS: std::cell::RefCell<HashMap<ObjRef, Length>> = std::cell::RefCell::new(HashMap::new());
/// What the loads running on this thread have read ([`ReadFrame`]): how many frames are
/// open, the object numbers read inside them, and how often that list was full.
static READS: std::cell::RefCell<(u32, Vec<u32>, u64)> = const { std::cell::RefCell::new((0, Vec::new(), 0)) };
/// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`).
static OVERLAY_READS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
/// How deep `Document::scan_for` is nested on this thread, and how many indirect `/Length`
/// values the outermost scan and those inside it have resolved ([`ScanLengths`]).
static SCAN_LENGTHS: std::cell::Cell<(u32, u32)> = const { std::cell::Cell::new((0, 0)) };
}
/// The indirect `/Length` values a scan of the file for an object may resolve, shared with the
/// scans it leads to. Each header in the file that starts with the object's number is parsed, and
/// a `/Length` that names an object the cross-reference table cannot place scans the file for that
/// one: a damaged file with many such headers, each naming another missing object, would read the
/// whole file once per header. Past the budget a `/Length` is left unresolved, and the stream is
/// read as before resolution, by its `endstream`.
pub(super) struct ScanLengths;
impl ScanLengths {
/// A real object is defined once per revision, and its `/Length` once per definition.
pub(super) const MAX: u32 = 32;
/// Marks a scan as running; the outermost one starts the budget afresh.
pub(super) fn enter() -> Self {
SCAN_LENGTHS.with(|c| {
let (depth, used) = c.get();
c.set((depth.saturating_add(1), if depth == 0 { 0 } else { used }));
});
ScanLengths
}
/// Whether one more `/Length` may be resolved, counting it if so.
pub(super) fn take(&self) -> bool {
SCAN_LENGTHS.with(|c| {
let (depth, used) = c.get();
if used >= Self::MAX {
return false;
}
c.set((depth, used.saturating_add(1)));
true
})
}
}
impl Drop for ScanLengths {
fn drop(&mut self) {
SCAN_LENGTHS.with(|c| {
let (depth, used) = c.get();
c.set((depth.saturating_sub(1), used));
});
}
}
/// A resolved `/Length` in [`LENGTHS`]: its value, whether resolving it read unsaved edits, and
/// what it read.
pub(super) type Length = (Option<i64>, bool, Reads);
/// The other objects a cached load read: an object stream for a member, an indirect `/Length`,
/// and what those read in turn. Clones share the caches, so a document whose edits include one of
/// them reads the object again instead (`Document::try_get`); `None` is a load that read more
/// than [`Reads::MAX`], which such a document always reads again.
#[derive(Clone, Debug, Default)]
pub(super) struct Reads(Option<Arc<[u32]>>);
impl Reads {
/// Objects one entry keeps: a member's stream and a length or two, and what they read.
const MAX: usize = 64;
/// Object numbers the open frames keep between them.
const HELD: usize = 4096;
/// Whether `overlay` (a document's edits) holds any of these objects.
pub(super) fn touch(&self, overlay: &BTreeMap<u32, Slot>) -> bool {
!overlay.is_empty() && self.0.as_ref().is_none_or(|nums| nums.iter().any(|n| overlay.contains_key(n)))
}
fn bytes(&self) -> usize {
self.0.as_ref().map_or(0, |nums| nums.len().saturating_mul(std::mem::size_of::<u32>()))
}
}
/// Counts `num` as read by the loads open on this thread, with what loading it read (`reads`).
pub(super) fn note_read(num: u32, reads: Option<&Reads>) {
READS.with(|r| {
let (depth, held, full) = &mut *r.borrow_mut();
if *depth == 0 {
return;
}
let more = reads.map_or(Some(&[][..]), |r| r.0.as_deref());
match more {
Some(more) if held.len().saturating_add(more.len()) < Reads::HELD => {
held.push(num);
held.extend_from_slice(more);
}
_ => *full = full.wrapping_add(1),
}
});
}
/// Collects what one load reads, for [`Reads`]; frames nest, and each read counts for every
/// frame open around it.
pub(super) struct ReadFrame {
start: usize,
full: u64,
}
impl ReadFrame {
pub(super) fn enter() -> Self {
READS.with(|r| {
let (depth, held, full) = &mut *r.borrow_mut();
*depth = depth.saturating_add(1);
ReadFrame { start: held.len(), full: *full }
})
}
/// What was read since the frame opened.
pub(super) fn reads(&self) -> Reads {
READS.with(|r| {
let (_, held, full) = &*r.borrow();
if *full != self.full {
return Reads(None);
}
let mut nums = held.get(self.start..).unwrap_or_default().to_vec();
nums.sort_unstable();
nums.dedup();
Reads((nums.len() <= Reads::MAX).then(|| Arc::from(nums)))
})
}
}
impl Drop for ReadFrame {
fn drop(&mut self) {
READS.with(|r| {
let (depth, held, _) = &mut *r.borrow_mut();
*depth = depth.saturating_sub(1);
if *depth == 0 {
held.clear();
}
});
}
}
/// Unsaved edits read so far on this thread: a load during which this changes depended on one.
pub(super) fn overlay_reads() -> u64 {
OVERLAY_READS.with(std::cell::Cell::get)
}
pub(super) fn note_overlay_read() {
OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1)));
}
/// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream
/// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is
/// bounded as well.
pub(super) struct LoadGuard;
impl LoadGuard {
const MAX: usize = 32;
pub(super) fn enter(num: u32) -> Option<Self> {
LOADING.with(|l| {
let mut l = l.borrow_mut();
if l.len() >= Self::MAX || l.contains(&num) {
return None;
}
l.push(num);
Some(LoadGuard)
})
}
}
/// Whether an object is being loaded on this thread: [`LENGTHS`] holds only while one is, since
/// it is cleared when the outermost load ends.
pub(super) fn loading() -> bool {
LOADING.with(|l| !l.borrow().is_empty())
}
/// What loading a file object gave, kept in the document's cache either way.
pub(super) type Loaded = Result<Arc<Object>, CosError>;
/// The bytes a load result holds of its own ([`held_bytes`]); a failure holds none.
pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc<Vec<u8>>) -> usize {
loaded.as_ref().map_or(0, |o| held_bytes(o, data, 0))
}
impl Drop for LoadGuard {
fn drop(&mut self) {
let outermost = LOADING.with(|l| {
let mut l = l.borrow_mut();
l.pop();
l.is_empty()
});
if outermost {
LENGTHS.with(|m| m.borrow_mut().clear());
}
}
}
#[derive(Clone, Debug)]
pub(super) struct ObjStm {
/// One slot per member in stream order, since cross-reference entries name a member by its
/// position: (object number, byte offset within `data`), or `None` for a member whose number
/// or offset is unusable ([`objstm_member`]).
pub(super) index: Vec<Option<(u32, usize)>>,
pub(super) data: Arc<Vec<u8>>,
/// What parsing it read: the stream itself and its `/Length`, `/DecodeParms` and so on.
pub(super) reads: Reads,
}
/// Where member `num` of an object stream starts: its `offset` past `/First` (`first`), when the
/// number is an object number and the start lies inside the decoded `len` bytes. Checked on every
/// target, so a 32-bit build neither truncates nor wraps a large `/First` or offset (#1815).
pub(super) fn objstm_member(num: i64, offset: i64, first: Option<usize>, len: usize) -> Option<(u32, usize)> {
let num = u32::try_from(num).ok()?;
let start = first?.checked_add(usize::try_from(offset).ok()?)?;
(start < len).then_some((num, start))
}
/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`).
/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that
/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole
/// list holds at most `MAX` plus the objects edited in clones.
#[derive(Debug, Default)]
pub(super) struct Unreadable {
pub(super) objects: BTreeMap<u32, (u16, String)>,
pub(super) more: bool,
}
impl Unreadable {
pub(super) const MAX: usize = 100;
/// Characters of a reason kept: a parser's message can quote the bytes it stopped at.
const REASON: usize = 160;
/// Whether `num` is to be recorded: not listed yet and there is room among the entries the
/// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]),
/// so a clone's edits never use up its own allowance. A full list notes that there were more,
/// so a caller asks this before building the reason.
pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool {
if self.objects.contains_key(&num) {
return false;
}
if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX {
self.more = true;
return false;
}
true
}
/// Record `num` once; the first reason given is kept.
pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) {
if self.wants(num, hidden) {
self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect()));
}
}
/// One line per recorded object not `hidden`, then one saying there were more, if there were.
pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec<String> {
// Clones share the list and each hides its own edits, so the list may hold more than
// `MAX` entries in all; each document still shows at most `MAX`.
let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num));
let mut lines: Vec<String> = shown
.by_ref()
.take(Self::MAX)
.map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}"))
.collect();
if self.more || shown.next().is_some() {
lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX));
}
lines
}
}
/// Where an overlay slot came from.
#[derive(Clone, Debug, PartialEq)]
pub(super) enum Slot {
Set(u16, Arc<Object>),
/// Both generations accepted from a damaged live source slot survive a repair-aware edit.
/// A full rewrite is required: a new xref slot cannot encode two generations.
RepairedSet(u16, u16, Arc<Object>),
Freed(u16),
}
/// An immutable input/decode context shared by document clones. Holding this identity
/// retains neither the input bytes nor the parsed document. Weak ownership prevents an
/// expired identity from matching a later allocation at the same address.
#[derive(Clone, Debug)]
pub struct SourceIdentity(pub(super) Weak<()>);
impl PartialEq for SourceIdentity {
fn eq(&self, other: &Self) -> bool {
self.0.ptr_eq(&other.0)
}
}
impl Eq for SourceIdentity {}
/// What a document reads as: its source ([`SourceIdentity`]), its trailer and every object changed
/// since it was opened. A clone, or a snapshot that undo restores, has the same state, and every
/// change makes a different one. Changed objects are compared as allocations, which the state
/// keeps alive, so a later object can't take an address one of them had.
#[derive(Clone)]
pub struct ReadState {
pub(super) source: SourceIdentity,
pub(super) trailer: Dict,
pub(super) overlay: BTreeMap<u32, Slot>,
}
impl ReadState {
/// Whether `doc` reads as this state.
pub fn matches(&self, doc: &Document) -> bool {
self.source == doc.source_identity()
&& self.trailer == doc.trailer
&& self.overlay.len() == doc.overlay.len()
&& self.overlay.iter().zip(&doc.overlay).all(|((a, x), (b, y))| {
a == b
&& match (x, y) {
(Slot::Set(g, o), Slot::Set(h, p)) => g == h && Arc::ptr_eq(o, p),
(Slot::RepairedSet(g, a, o), Slot::RepairedSet(h, b, p)) => g == h && a == b && Arc::ptr_eq(o, p),
(Slot::Freed(g), Slot::Freed(h)) => g == h,
_ => false,
}
})
}
}
impl std::fmt::Debug for ReadState {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ReadState").field("changed_objects", &self.overlay.len()).finish_non_exhaustive()
}
}
/// A temporary reader for whole-document work. Its caches never populate the editor or
/// undo snapshots. Returned objects survive cache eviction; a stream read from the file is a
/// view into the input bytes (it keeps the whole input alive while held, not a copy).
pub(crate) struct ObjectReader {
pub(crate) document: Document,
pub(super) decoded_count: usize,
}
impl ObjectReader {
pub(super) const OBJECTS: usize = 128;
pub(super) const STREAM_BYTES: usize = 8 * 1024 * 1024;
pub(crate) fn get(&mut self, reference: ObjRef) -> Arc<Object> {
self.try_get(reference).unwrap_or_else(|_| Arc::new(Object::Null))
}
pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result<Arc<Object>, CosError> {
let object = self.document.try_get_reference(reference);
// A missed eviction here only costs a re-parse, so never fail the read on a poisoned
// lock. The editing-cache evictors propagate instead; they guard user edits.
if let Ok(mut streams) = self.document.objstms.lock() {
// Only insertion changes this private cache between reads. Most neighboring
// objects reuse a decoded stream: do not sum the whole store for every object.
if streams.len() != self.decoded_count {
let bytes = streams.values().fold(0usize, |bytes, stream| {
bytes
.saturating_add(stream.data.capacity())
.saturating_add(stream.index.capacity().saturating_mul(std::mem::size_of::<Option<(u32, usize)>>()))
});
if bytes > Self::STREAM_BYTES || streams.len() >= Self::OBJECTS {
// Retain one oversized active stream so its next object does not require
// decoding it again. That stream still obeys the document's decode cap.
let current = match self.document.xref_entry(reference.num) {
Some(XrefEntry::InStream { stream, .. }) => Some(stream),
_ => None,
};
streams.retain(|num, _| Some(*num) == current);
}
self.decoded_count = streams.len();
}
}
// Parsed objects stay cached (an indirect /Length or a dictionary read again costs a
// lookup, not a parse) until the cache passes its object count or holds more than
// STREAM_BYTES of data of its own. Streams read from the file are views into the
// input bytes, which the document keeps anyway, so they don't count; decrypted
// streams and strings do. Checked after failed loads too: resolving a /Length may
// have cached large objects.
if let Ok(mut cache) = self.document.cache.lock()
&& (cache.len() >= Self::OBJECTS || cache.owned() > Self::STREAM_BYTES)
{
cache.clear();
}
object
}
}
#[cfg(test)]
thread_local! {
/// Cross-reference sections read on this thread.
pub(super) static SECTION_READS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// What loading each object gave (a failure as well), by number, each with what it holds ([`held_bytes`]) measured once, when it goes
/// in, and their sum kept beside them.
///
/// Both caches are bounded by those bytes. Summing them on every read walked every cached object
/// whole each time: a file whose page dictionaries carry kilobytes of private data (InDesign's
/// `/PieceInfo`) took seconds to open, almost all of it re-measuring objects already measured
/// (#5873).
#[derive(Default)]
pub(crate) struct ObjectCache {
entries: HashMap<u32, (Loaded, usize, Reads)>,
owned: usize,
}
impl ObjectCache {
/// What loading `num` gave, and what that load read.
pub(super) fn get(&self, num: &u32) -> Option<(&Loaded, &Reads)> {
self.entries.get(num).map(|(object, _, reads)| (object, reads))
}
#[cfg(test)]
pub(crate) fn contains_key(&self, num: &u32) -> bool {
self.entries.contains_key(num)
}
/// Keep what loading `num` gave, which holds `owned` bytes ([`loaded_bytes`]), and what it read.
pub(super) fn insert(&mut self, num: u32, object: Loaded, owned: usize, reads: Reads) {
let owned = owned.saturating_add(reads.bytes());
if let Some((_, before, _)) = self.entries.insert(num, (object, owned, reads)) {
self.owned = self.owned.saturating_sub(before);
}
self.owned = self.owned.saturating_add(owned);
}
pub(crate) fn len(&self) -> usize {
self.entries.len()
}
#[cfg(test)]
pub(crate) fn is_empty(&self) -> bool {
self.entries.is_empty()
}
/// The bytes the cached objects hold between them.
pub(crate) fn owned(&self) -> usize {
self.owned
}
pub(crate) fn clear(&mut self) {
self.entries.clear();
self.owned = 0;
}
pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) {
self.entries.retain(|num, _| keep(num));
self.owned = self.entries.values().fold(0usize, |n, (_, owned, _)| n.saturating_add(*owned));
}
#[cfg(test)]
pub(crate) fn values(&self) -> impl Iterator<Item = &Arc<Object>> {
self.entries.values().filter_map(|(object, _, _)| object.as_ref().ok())
}
}
/// Nesting [`held_bytes`] walks: past the parser's own nesting limit, so every parsed object
/// is measured in full.
const OWNED_BYTES_DEPTH: usize = 256;
/// What keeping `object` parsed costs: string and stream data that is not a view into `data`
/// (the input file), and the parsed tree around it -- an [`Object`] per value and a key per
/// dictionary entry. An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured and
/// counts as too large to keep.
///
/// The tree counts. Measured by its strings and streams alone, a page dictionary carrying
/// InDesign's `/PieceInfo` -- thousands of small numeric arrays, parsed into tens of thousands of
/// values -- weighed nothing, so the caches kept a thousand of them: a 35 MB file held 1.5 GB
/// (#5873).
pub(crate) fn held_bytes(object: &Object, data: &Arc<Vec<u8>>, depth: usize) -> usize {
if depth > OWNED_BYTES_DEPTH {
return usize::MAX;
}
let node = std::mem::size_of::<Object>();
// A key, its slot beside the value, and room in the index a large dictionary keeps.
let entry = |key: &[u8]| key.len().saturating_add(std::mem::size_of::<(crate::Name, Object)>()).saturating_add(16);
let dict =
|d: &crate::Dict, start: usize| d.iter().fold(start, |n, (k, v)| n.saturating_add(entry(k)).saturating_add(held_bytes(v, data, depth + 1)));
match object {
Object::String(s) => node.saturating_add(s.bytes.len()),
Object::Name(n) => node.saturating_add(n.len()),
Object::Stream(s) => dict(&s.dict, node.saturating_add(if s.raw.shares(data) { 0 } else { s.raw.len() })),
Object::Array(a) => a.iter().fold(node, |n, v| n.saturating_add(held_bytes(v, data, depth + 1))),
Object::Dict(d) => dict(d, node),
_ => node,
}
}
/// How an encrypted document is opened: with a password, or with the file key of a document
/// that has the same encryption (an earlier revision of the same file).
pub(super) enum Credentials<'a> {
Password(Option<&'a str>),
Handler { handler: &'a Arc<pdfcraft_crypt::SecurityHandler>, id0: Vec<u8> },
}
File diff suppressed because it is too large Load Diff
-275
View File
@@ -1,275 +0,0 @@
use super::{Arc, BTreeMap, Lexer};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum XrefEntry {
Free { next_generation: u16 },
InFile { offset: u64, generation: u16 },
InStream { stream: u32, index: u32 },
}
#[derive(Clone, Copy, Debug)]
pub(super) struct XrefRange {
pub(super) start: u32,
pub(super) count: u32,
pub(super) offset: usize,
pub(super) stride: usize,
}
#[derive(Clone, Debug)]
pub(super) enum XrefSectionData {
Table,
Stream { bytes: Arc<Vec<u8>>, widths: [usize; 3] },
}
#[derive(Clone, Debug)]
pub(super) struct XrefSection {
pub(super) data: XrefSectionData,
pub(super) ranges: Vec<XrefRange>,
pub(super) row_count: usize,
pub(super) max_num: u32,
pub(super) has_entries: bool,
/// A table subsection was read one number lower than it says ([`subsection_start`]).
pub(super) renumbered: bool,
}
#[derive(Clone, Debug)]
pub(super) struct XrefRevisionSection {
pub(super) primary: XrefSection,
pub(super) hybrid: Option<XrefSection>,
}
#[derive(Clone, Debug)]
pub(super) enum XrefIndex {
Compact { revisions: Vec<XrefRevisionSection>, max_num: u32, has_entries: bool },
Materialized(Arc<BTreeMap<u32, XrefEntry>>, Arc<Vec<String>>),
}
pub(super) const MAX_COMPACT_XREF_ROWS: usize = 2_000_000;
pub(super) const MAX_COMPACT_XREF_BYTES: usize = 64 * 1024 * 1024;
impl XrefSection {
pub(super) fn has_any_entry(&self) -> bool {
match &self.data {
XrefSectionData::Table => self.has_entries,
XrefSectionData::Stream { bytes, widths } => {
let row_width = widths.iter().sum::<usize>();
if row_width == 0 {
return false;
}
for range in &self.ranges {
let available = bytes.len().saturating_sub(range.offset) / row_width;
for i in 0..available.min(range.count as usize) {
let Some(offset) = i.checked_mul(row_width).and_then(|delta| range.offset.checked_add(delta)) else { continue };
let Some(row) = bytes.get(offset..offset.saturating_add(row_width)) else { continue };
let read_field = |from: usize, width: usize, default: u64| -> Option<u64> {
if width == 0 {
return Some(default);
}
let end = from.checked_add(width)?;
Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte)))
};
let Some(kind) = read_field(0, widths[0], 1) else { continue };
if kind == 0 || kind == 2 {
return true;
}
if kind == 1 && read_field(widths[0], widths[1], 0).is_some_and(|offset| offset != 0) {
return true;
}
}
}
false
}
}
}
pub(super) fn retained_bytes(&self) -> usize {
match &self.data {
XrefSectionData::Table => 0,
XrefSectionData::Stream { bytes, .. } => bytes.capacity(),
}
}
pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option<XrefEntry> {
let after = self.ranges.partition_point(|range| range.start <= num);
let range = self.ranges.get(after.checked_sub(1)?)?;
let end = u64::from(range.start).checked_add(u64::from(range.count))?;
if u64::from(num) >= end {
return None;
}
let row = usize::try_from(num.checked_sub(range.start)?).ok()?;
let offset = range.offset.checked_add(row.checked_mul(range.stride)?)?;
match &self.data {
XrefSectionData::Table => {
let mut lexer = Lexer::new(file, offset);
lexer.skip_ws();
let object_offset = lexer.token();
lexer.skip_ws();
let generation = lexer.token();
lexer.skip_ws();
let kind = lexer.token();
match kind {
b"n" => {
let object_offset = std::str::from_utf8(object_offset).ok()?.parse::<u64>().ok()?;
if object_offset == 0 {
return None;
}
let generation = std::str::from_utf8(generation).ok()?.parse::<u32>().unwrap_or(0).min(u16::MAX as u32) as u16;
Some(XrefEntry::InFile { offset: object_offset, generation })
}
b"f" => {
let generation = std::str::from_utf8(generation).ok()?.parse::<u32>().unwrap_or(0).min(u16::MAX as u32) as u16;
Some(XrefEntry::Free { next_generation: generation })
}
_ => None,
}
}
XrefSectionData::Stream { bytes, widths } => {
let row_width = widths.iter().sum::<usize>();
let end = offset.checked_add(row_width)?;
let row = bytes.get(offset..end)?;
let field = |from: usize, width: usize, default: u64| -> Option<u64> {
if width == 0 {
return Some(default);
}
let end = from.checked_add(width)?;
Some(row.get(from..end)?.iter().fold(0u64, |acc, byte| acc.wrapping_shl(8) | u64::from(*byte)))
};
let kind = field(0, widths[0], 1)?;
let a = field(widths[0], widths[1], 0)?;
let b = field(widths[0] + widths[1], widths[2], 0)?;
let entry = match kind {
0 => XrefEntry::Free { next_generation: b.min(u16::MAX as u64) as u16 },
1 => XrefEntry::InFile { offset: a, generation: b.min(u16::MAX as u64) as u16 },
2 => XrefEntry::InStream { stream: a.min(u32::MAX as u64) as u32, index: b.min(u32::MAX as u64) as u32 },
_ => return None,
};
(!matches!(entry, XrefEntry::InFile { offset: 0, .. })).then_some(entry)
}
}
}
pub(super) fn numbers(&self) -> impl Iterator<Item = u32> + '_ {
self.ranges.iter().flat_map(|range| {
let available = match &self.data {
XrefSectionData::Table => range.count as usize,
XrefSectionData::Stream { bytes, widths } => {
let row_width = widths.iter().sum::<usize>();
bytes.len().saturating_sub(range.offset).checked_div(row_width).unwrap_or(0)
}
};
(0..available.min(range.count as usize)).filter_map(move |i| u32::try_from(i).ok().and_then(|i| range.start.checked_add(i)))
})
}
pub(super) fn extend_numbers(&self, numbers: &mut Vec<u32>) {
numbers.extend(self.numbers());
}
}
impl XrefRevisionSection {
pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option<XrefEntry> {
let primary = self.primary.lookup(num, file);
let hybrid = self.hybrid.as_ref().and_then(|section| section.lookup(num, file));
match primary {
None | Some(XrefEntry::Free { .. }) => hybrid.or(primary),
entry => entry,
}
}
fn extend_numbers(&self, numbers: &mut Vec<u32>) {
self.primary.extend_numbers(numbers);
if let Some(hybrid) = &self.hybrid {
hybrid.extend_numbers(numbers);
}
}
}
impl XrefIndex {
pub(super) fn empty() -> Self {
Self::Materialized(Arc::new(BTreeMap::new()), Arc::default())
}
pub(super) fn lookup(&self, num: u32, file: &[u8]) -> Option<XrefEntry> {
match self {
Self::Compact { revisions, .. } => revisions.iter().find_map(|revision| revision.lookup(num, file)),
Self::Materialized(entries, _) => entries.get(&num).copied(),
}
}
pub(super) fn is_empty(&self) -> bool {
match self {
Self::Compact { has_entries, .. } => !has_entries,
Self::Materialized(entries, _) => entries.is_empty(),
}
}
/// Drop an entry for object number 4294967295, which no incremental `/Size` can cover.
/// Only a materialized index can hold one: the compact reader requires `max_num < /Size`.
pub(super) fn remove_exhausted(&mut self) -> bool {
match self {
Self::Compact { .. } => false,
Self::Materialized(entries, _) => Arc::make_mut(entries).remove(&u32::MAX).is_some(),
}
}
/// Every entry, newest section first, as the materialized reader keeps them.
pub(super) fn to_map(&self, file: &[u8]) -> BTreeMap<u32, XrefEntry> {
match self {
Self::Materialized(entries, _) => entries.as_ref().clone(),
Self::Compact { revisions, .. } => {
let mut numbers = Vec::new();
for revision in revisions {
revision.extend_numbers(&mut numbers);
}
numbers.into_iter().filter_map(|num| self.lookup(num, file).map(|entry| (num, entry))).collect()
}
}
}
pub(super) fn max_num(&self) -> u32 {
match self {
Self::Compact { max_num, .. } => *max_num,
Self::Materialized(entries, _) => entries.keys().next_back().copied().unwrap_or(0),
}
}
pub(super) fn object_numbers(&self, file: &[u8]) -> Vec<u32> {
match self {
Self::Materialized(entries, _) => {
entries.iter().filter_map(|(&num, entry)| (!matches!(entry, XrefEntry::Free { .. })).then_some(num)).collect()
}
Self::Compact { revisions, .. } => {
if let [revision] = revisions.as_slice()
&& revision.hybrid.is_none()
{
let mut numbers = Vec::with_capacity(revision.primary.row_count);
revision.primary.extend_numbers(&mut numbers);
numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. })));
return numbers;
}
let mut numbers = Vec::new();
for revision in revisions {
revision.extend_numbers(&mut numbers);
}
numbers.sort_unstable();
numbers.dedup();
numbers.retain(|num| self.lookup(*num, file).is_some_and(|entry| !matches!(entry, XrefEntry::Free { .. })));
numbers
}
}
}
}
/// One cross-reference section of the file (one per save; the first is the original).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Revision {
/// Offset of the xref section (`startxref` value).
pub xref_offset: u64,
/// `true` if the section is a cross-reference stream.
pub is_stream: bool,
/// `true` if `xref_offset` counts from the `%PDF-` header rather than from the start of the
/// file. The two differ only in a file with bytes before its header, and an update appended
/// to such a file must count the same way as the section it points back to.
pub header_relative: bool,
}
+2 -57
View File
@@ -1,57 +1,2 @@
//! pdfcraft-cos — the PDF object layer (L1, standalone with `pdfcraft-filters`).
//!
//! Parses the COS object graph lazily and tolerantly, keeps edits in a copy-on-write overlay
//! (cheap snapshots for undo), and writes documents back either incrementally (original bytes
//! untouched) or as a full, garbage-collected rewrite. The PDF object graph *is* PdfCraft's
//! document model (plan/architecture.md §5, ADR-0010).
//!
//! Done: every cross-reference form (tables, streams, hybrid `/XRefStm`), object streams, repair
//! by scanning, encryption on load and save (R2–R6, crypt filters, `/EFF`), incremental writing,
//! and full saves that pack objects into compressed object streams with a cross-reference stream.
//! Not yet: streaming very large files within a memory budget (`ByteSource`,
//! `core.lazy-loading`), and linearized output (`core.linearization`, M11). See the README.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod budget;
mod bytes;
mod document;
mod object;
pub mod page_labels;
mod parser;
mod security;
mod writer;
pub use budget::{DecodeBudget, ImageBudget, MAX_PAGE_DECODED, MAX_PAGE_FORMS, MAX_REDACTED_IMAGE_PIXELS};
pub use bytes::Bytes;
pub use document::{DECODE_PARMS_KEYS, Document, ReadState, Revision, SourceIdentity, XrefEntry};
pub use object::{Dict, MAX_DECODED, MAX_JBIG2_GLOBALS, Name, ObjRef, Object, PdfString, Stream, TextChars};
pub use parser::{Lexer, is_whitespace, parse_indirect};
pub use pdfcraft_crypt::{
Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char,
};
pub use writer::{SaveOptions, civil_from_days, new_file_id, pdf_date, reachable_objects, serialize, write_full, write_incremental};
#[derive(Debug, thiserror::Error, Clone, PartialEq)]
pub enum CosError {
#[error("not a PDF file (no %PDF- header)")]
NotPdf,
#[error("syntax error at byte {offset}: {detail}")]
Syntax { offset: usize, detail: String },
#[error("object {0} is missing")]
MissingObject(u32),
#[error("object {0} is not a dictionary")]
NotADictionary(ObjRef),
#[error("stream data could not be decoded: {0}")]
Filter(String),
#[error("the document is protected by a password")]
NeedsPassword,
#[error("the password is incorrect")]
WrongPassword,
#[error("unsupported security: {0}")]
Security(String),
#[error("internal lock poisoned")]
Poisoned,
#[error("this document was opened read-only with a decode limit and cannot be saved")]
ReadOnlyLimit,
}
//! PDF object layer (`craft-pdf::cos`).
pub use craft_pdf::cos::*;
-834
View File
@@ -1,834 +0,0 @@
//! The PDF object model (ISO 32000-2 §7.3).
//!
//! Dictionaries keep their key order so rewritten objects stay recognisable in diffs and
//! byte-stable when re-serialized. Streams keep their *encoded* bytes; decoding is on demand.
use pdfcraft_filters::{Filter, Params};
use crate::{Bytes, CosError};
/// An indirect reference: object number and generation.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ObjRef {
pub num: u32,
pub generation: u16,
}
impl ObjRef {
pub const fn new(num: u32, generation: u16) -> Self {
Self { num, generation }
}
}
impl std::fmt::Display for ObjRef {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{} {} R", self.num, self.generation)
}
}
/// A string object. `hex` records how it was written, so it is re-emitted the same way.
#[derive(Clone, Debug, PartialEq)]
pub struct PdfString {
pub bytes: Vec<u8>,
pub hex: bool,
}
impl PdfString {
pub fn literal(bytes: impl Into<Vec<u8>>) -> Self {
Self { bytes: bytes.into(), hex: false }
}
/// A text string (ISO 32000-2 §7.9.2.2): PDFDocEncoding when possible, else UTF-16BE with BOM.
pub fn text(s: &str) -> Self {
if s.chars().all(|c| (' '..='~').contains(&c) || c == '\n' || c == '\r' || c == '\t') {
return Self::literal(s.as_bytes().to_vec());
}
let mut bytes = vec![0xFE, 0xFF];
for u in s.encode_utf16() {
bytes.extend_from_slice(&u.to_be_bytes());
}
Self { bytes, hex: false }
}
/// Decode a text string: UTF-16BE (BOM), UTF-8 (BOM, PDF 2.0) or PDFDocEncoding.
///
/// Lenient beyond the spec, as readers are: invalid sequences after a BOM decode lossily
/// instead of dropping the string, a BOM-less string whose bytes are valid UTF-8 with
/// non-ASCII bytes (written by some producers) is read as UTF-8 rather than as
/// PDFDocEncoding mojibake, and stray leading U+FEFF (doubled BOMs) are removed.
pub fn to_text(&self) -> String {
self.text_chars().collect()
}
/// The characters [`PdfString::to_text`] decodes, one at a time, without building a string.
pub fn text_chars(&self) -> TextChars<'_> {
let decode = match self.bytes.as_slice() {
[0xFE, 0xFF, rest @ ..] => {
let units: fn(&[u8; 2]) -> u16 = |c| u16::from_be_bytes(*c);
Decode::Utf16(char::decode_utf16(rest.as_chunks::<2>().0.iter().map(units)))
}
[0xEF, 0xBB, 0xBF, rest @ ..] => Decode::Utf8Lossy { chunks: rest.utf8_chunks(), valid: "".chars(), replace: false },
b => match std::str::from_utf8(b) {
// Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch.
Ok(utf8) if !b.is_ascii() => Decode::Utf8(utf8.chars()),
_ => Decode::PdfDoc(b.iter()),
},
};
TextChars { decode, leading: true }
}
/// Whether the string decodes to `s` ([`PdfString::to_text`]), compared without allocating.
pub fn text_eq(&self, s: &str) -> bool {
self.text_chars().eq(s.chars())
}
}
/// The characters of a text string: [`PdfString::text_chars`].
pub struct TextChars<'a> {
decode: Decode<'a>,
/// Still at the start, where stray U+FEFF (doubled BOMs) are dropped.
leading: bool,
}
/// The UTF-16BE code units after a BOM (an odd last byte is left out).
type Utf16Units<'a> = std::iter::Map<std::slice::Iter<'a, [u8; 2]>, fn(&[u8; 2]) -> u16>;
enum Decode<'a> {
Utf16(std::char::DecodeUtf16<Utf16Units<'a>>),
/// After a UTF-8 BOM: each invalid sequence is one U+FFFD, as `String::from_utf8_lossy` has it.
Utf8Lossy {
chunks: std::str::Utf8Chunks<'a>,
valid: std::str::Chars<'a>,
replace: bool,
},
Utf8(std::str::Chars<'a>),
PdfDoc(std::slice::Iter<'a, u8>),
}
impl Iterator for Decode<'_> {
type Item = char;
fn next(&mut self) -> Option<char> {
match self {
Decode::Utf16(units) => units.next().map(|u| u.unwrap_or(char::REPLACEMENT_CHARACTER)),
Decode::Utf8Lossy { chunks, valid, replace } => loop {
if let Some(c) = valid.next() {
return Some(c);
}
if *replace {
*replace = false;
return Some(char::REPLACEMENT_CHARACTER);
}
let chunk = chunks.next()?;
*valid = chunk.valid().chars();
*replace = !chunk.invalid().is_empty();
},
Decode::Utf8(chars) => chars.next(),
Decode::PdfDoc(bytes) => bytes.next().map(|&c| pdfcraft_crypt::pdfdoc_char(c)),
}
}
}
impl Iterator for TextChars<'_> {
type Item = char;
fn next(&mut self) -> Option<char> {
loop {
let c = self.decode.next()?;
if self.leading && c == '\u{FEFF}' {
continue;
}
self.leading = false;
return Some(c);
}
}
}
/// A name object (without the leading `/`, with `#xx` escapes already decoded).
pub type Name = Vec<u8>;
#[cfg(test)]
thread_local! {
/// Key comparisons and index lookups made by [`Dict`] on this thread.
static KEY_COMPARES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// Entries from which a [`Dict`] keeps a key index. Smaller dictionaries (nearly all of them)
/// search their entries; a parsed file controls the key count, and a linear search per
/// inserted key would make a dictionary of many distinct keys quadratic to build.
const INDEX_FROM: usize = 32;
/// An ordered dictionary.
#[derive(Clone, Default)]
pub struct Dict {
entries: Vec<(Name, Object)>,
/// Key to position in `entries`, kept once the dictionary reaches [`INDEX_FROM`] entries.
/// Boxed so that every `Dict`, and so every `Object`, grows by one pointer, not by a map.
#[allow(clippy::box_collection)]
index: Option<Box<std::collections::HashMap<Name, usize>>>,
}
impl PartialEq for Dict {
fn eq(&self, other: &Self) -> bool {
self.entries == other.entries
}
}
impl std::fmt::Debug for Dict {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Dict").field("entries", &self.entries).finish()
}
}
impl Dict {
pub fn new() -> Self {
Self::default()
}
fn position(&self, key: &[u8]) -> Option<usize> {
if let Some(index) = &self.index {
#[cfg(test)]
KEY_COMPARES.with(|c| c.set(c.get() + 1));
return index.get(key).copied();
}
self.entries.iter().position(|(k, _)| {
#[cfg(test)]
KEY_COMPARES.with(|c| c.set(c.get() + 1));
k == key
})
}
pub fn get(&self, key: &[u8]) -> Option<&Object> {
self.position(key).and_then(|i| self.entries.get(i)).map(|(_, v)| v)
}
pub fn get_mut(&mut self, key: &[u8]) -> Option<&mut Object> {
let i = self.position(key)?;
self.entries.get_mut(i).map(|(_, v)| v)
}
/// Insert or replace, keeping the position of an existing key.
pub fn set(&mut self, key: impl Into<Name>, value: impl Into<Object>) {
let key = key.into();
let value = value.into();
match self.position(&key).and_then(|i| self.entries.get_mut(i)) {
Some((_, v)) => *v = value,
None => {
let at = self.entries.len();
if let Some(index) = &mut self.index {
index.insert(key.clone(), at);
} else if at + 1 >= INDEX_FROM {
let index = self.entries.iter().enumerate().map(|(i, (k, _))| (k.clone(), i)).chain([(key.clone(), at)]).collect();
self.index = Some(Box::new(index));
}
self.entries.push((key, value));
}
}
}
pub fn remove(&mut self, key: &[u8]) -> Option<Object> {
let i = self.position(key)?;
if let Some(index) = &mut self.index {
index.remove(key);
index.values_mut().filter(|at| **at > i).for_each(|at| *at -= 1);
}
Some(self.entries.remove(i).1)
}
pub fn contains(&self, key: &[u8]) -> bool {
self.get(key).is_some()
}
pub fn iter(&self) -> impl Iterator<Item = (&Name, &Object)> {
self.entries.iter().map(|(k, v)| (k, v))
}
pub fn len(&self) -> usize {
self.entries.len()
}
pub fn is_empty(&self) -> bool {
self.entries.is_empty()
}
pub fn name(&self, key: &[u8]) -> Option<&[u8]> {
self.get(key).and_then(Object::as_name)
}
pub fn int(&self, key: &[u8]) -> Option<i64> {
self.get(key).and_then(Object::as_int)
}
pub fn reference(&self, key: &[u8]) -> Option<ObjRef> {
self.get(key).and_then(Object::as_ref)
}
}
impl FromIterator<(Name, Object)> for Dict {
fn from_iter<T: IntoIterator<Item = (Name, Object)>>(iter: T) -> Self {
let mut d = Dict::new();
for (k, v) in iter {
d.set(k, v);
}
d
}
}
/// A stream: dictionary plus its data *as stored* (still filtered).
#[derive(Clone, Debug, PartialEq)]
pub struct Stream {
pub dict: Dict,
pub raw: Bytes,
}
/// Upper bound for a single decoded stream (decompression-bomb defence).
pub const MAX_DECODED: usize = 1 << 30;
/// The most an image's `/JBIG2Globals` stream may decode to: the symbol and pattern dictionaries an
/// encoder shares between pages, themselves coded, so far less than any stream may decode to
/// ([`MAX_DECODED`]). They are decoded before the image's JBIG2 page is read, so the page's size
/// can't bound them: they have a limit of their own.
pub const MAX_JBIG2_GLOBALS: usize = 32 << 20;
impl Stream {
/// A new stream from already-encoded bytes (`/Length` is set at write time).
pub fn from_raw(dict: Dict, raw: Vec<u8>) -> Self {
Self { dict, raw: raw.into() }
}
/// A new stream holding `data` compressed with Flate.
pub fn flate(mut dict: Dict, data: &[u8]) -> Self {
dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec()));
dict.remove(b"DecodeParms");
Self { dict, raw: pdfcraft_filters::encode_flate(data).into() }
}
/// The filter chain declared in the dictionary.
pub fn filters(&self) -> Vec<(Filter, Params)> {
let names: Vec<&[u8]> = match self.dict.get(b"Filter") {
Some(Object::Name(n)) => vec![n],
Some(Object::Array(a)) => a.iter().filter_map(Object::as_name).collect(),
_ => Vec::new(),
};
let parms: Vec<Option<&Dict>> = match self.dict.get(b"DecodeParms") {
Some(Object::Dict(d)) => vec![Some(d)],
Some(Object::Array(a)) => a.iter().map(Object::as_dict).collect(),
_ => Vec::new(),
};
names
.into_iter()
.enumerate()
// `/Crypt` is applied by the security handler when the object is loaded (§7.4.10).
.filter(|(_, n)| *n != b"Crypt")
.map(|(i, n)| {
let p = parms.get(i).copied().flatten();
let get = |k: &[u8], d: i64| p.and_then(|p| p.int(k)).unwrap_or(d);
let params = Params {
predictor: get(b"Predictor", 1),
colors: get(b"Colors", 1),
bits_per_component: get(b"BitsPerComponent", 8),
columns: get(b"Columns", 1),
early_change: get(b"EarlyChange", 1),
};
(Filter::from_name(n), params)
})
.collect()
}
/// [`Stream::filters`], or an error when `/Filter` is present but is not a name or an
/// array of names (an indirect reference, say): the data is not unfiltered, so it must not
/// read as decoded.
pub(crate) fn readable_filters(&self) -> Result<Vec<(Filter, Params)>, pdfcraft_filters::FilterError> {
let readable = match self.dict.get(b"Filter") {
None | Some(Object::Null | Object::Name(_)) => true,
Some(Object::Array(a)) => a.iter().all(|f| f.as_name().is_some()),
Some(_) => false,
};
if !readable {
return Err(pdfcraft_filters::FilterError::Unsupported("/Filter is not a name or an array of names".into()));
}
Ok(self.filters())
}
/// Decoded data, tolerating truncated or corrupt encodings the way viewers do (the data
/// decoded before the damage is returned). Decoding past
/// [`crate::budget::MAX_PAGE_DECODED`] (256 MiB), the most the renderer reads from one
/// stream, is an error. Streams with image codecs fail with
/// `CosError::Filter` (keep them encoded). Use `decoded_strict` to detect damage.
pub fn decoded(&self) -> Result<Vec<u8>, CosError> {
self.decoded_within(crate::budget::MAX_PAGE_DECODED)
}
/// Like [`Stream::decoded`], for a stream whose decoded size has a tighter bound than
/// [`MAX_DECODED`]: decoding past `max` bytes is an error.
pub fn decoded_within(&self, max: usize) -> Result<Vec<u8>, CosError> {
let decode = |chain: &[(Filter, Params)], data: &[u8], max| {
pdfcraft_filters::decode_tolerant(chain, data, max).map(|(v, _)| (v, 0)).map_err(|e| (e, 0))
};
self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string()))
}
/// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept
/// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter
/// stages together, and the result gives whether the data stopped at damage (it is then what
/// decoded before it) and the bytes the stages decoded (the work done, on success and on
/// failure).
pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec<u8>, bool, usize), (pdfcraft_filters::FilterError, usize)> {
let mut damaged = false;
let decode = |chain: &[(Filter, Params)], data: &[u8], max| {
pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| {
damaged = partial;
(v, spent)
})
};
self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent))
}
/// [`Stream::decoded_within_counted`] with every filter stage also held to `each`, as
/// [`Stream::decoded_within`] holds it, while `max` bounds the stages together
/// ([`pdfcraft_filters::decode_tolerant_counted_each`]). Without filters the data is held to
/// both.
pub(crate) fn decoded_within_each_counted(
&self,
each: usize,
max: usize,
) -> Result<(Vec<u8>, bool, usize), (pdfcraft_filters::FilterError, usize)> {
let mut damaged = false;
let decode = |chain: &[(Filter, Params)], data: &[u8], max| {
pdfcraft_filters::decode_tolerant_counted_each(chain, data, each.min(MAX_DECODED), max).map(|(v, partial, spent)| {
damaged = partial;
(v, spent)
})
};
let max = if self.readable_filters().is_ok_and(|chain| chain.is_empty()) { max.min(each) } else { max };
self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent))
}
/// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a
/// document's images ([`pdfcraft_filters::decode_tolerant_within`]).
pub(crate) fn decoded_spending(&self, max: usize, work: &mut u64) -> Result<Vec<u8>, CosError> {
let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string());
let chain = self.readable_filters().map_err(filter)?;
pdfcraft_filters::decode_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), work).map_err(filter)
}
/// [`Stream::decoded_spending`] for the samples of an image `rows` high (its `/Height`):
/// [`pdfcraft_filters::decode_image_tolerant_within`].
pub(crate) fn decoded_image_rows_spending(&self, max: usize, rows: u32, work: &mut u64) -> Result<Vec<u8>, CosError> {
let filter = |e: pdfcraft_filters::FilterError| CosError::Filter(e.to_string());
let chain = self.readable_filters().map_err(filter)?;
pdfcraft_filters::decode_image_tolerant_within(&chain, &self.raw, max.min(MAX_DECODED), rows, work).map_err(filter)
}
/// The stream decoded through its filters by `decode`, or copied when it has none, within
/// `max` and [`MAX_DECODED`].
fn decode_chain(
&self,
max: usize,
decode: impl FnOnce(&[(Filter, Params)], &[u8], usize) -> Result<(Vec<u8>, usize), (pdfcraft_filters::FilterError, usize)>,
) -> Result<(Vec<u8>, usize), (pdfcraft_filters::FilterError, usize)> {
let max = max.min(MAX_DECODED);
let chain = self.readable_filters().map_err(|e| (e, 0))?;
if chain.is_empty() {
return self
.raw_within_typed(max)
.map(|v| {
let n = v.len();
(v, n)
})
.map_err(|e| (e, 0));
}
decode(&chain, &self.raw, max)
}
/// Decoded data, bounded by [`MAX_DECODED`]; any corruption is an error.
pub fn decoded_strict(&self) -> Result<Vec<u8>, CosError> {
self.decoded_strict_within(MAX_DECODED)
}
/// Like [`Stream::decoded_strict`], bounded by `max` (and [`MAX_DECODED`]): for data that
/// will be copied elsewhere, where a damaged stream must not pass for a shorter one.
pub fn decoded_strict_within(&self, max: usize) -> Result<Vec<u8>, CosError> {
let decode = |chain: &[(Filter, Params)], data: &[u8], max| pdfcraft_filters::decode(chain, data, max).map(|v| (v, 0)).map_err(|e| (e, 0));
self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string()))
}
/// [`Stream::decoded_strict`] bounded by `max` and counted like
/// [`Stream::decoded_within_counted`].
pub(crate) fn decoded_strict_within_counted(&self, max: usize) -> Result<(Vec<u8>, usize), (pdfcraft_filters::FilterError, usize)> {
self.decode_chain(max, pdfcraft_filters::decode_counted)
}
/// Data after the first filter alone, bounded by `max`: the rest of the chain stays encoded.
/// Any corruption in that first filter is an error.
pub fn decoded_first_filter_strict(&self, max: usize) -> Result<Vec<u8>, CosError> {
let mut chain = self.readable_filters().map_err(|e| CosError::Filter(e.to_string()))?;
chain.truncate(1);
if chain.is_empty() {
return self.raw_within(max);
}
pdfcraft_filters::decode(&chain, &self.raw, max.min(MAX_DECODED)).map_err(|e| CosError::Filter(e.to_string()))
}
fn raw_within(&self, max: usize) -> Result<Vec<u8>, CosError> {
self.raw_within_typed(max).map_err(|e| CosError::Filter(e.to_string()))
}
fn raw_within_typed(&self, max: usize) -> Result<Vec<u8>, pdfcraft_filters::FilterError> {
let max = max.min(MAX_DECODED);
// An empty decoder chain still produces an owned output buffer: check its limit
// before cloning, including streams whose /Crypt filter was applied on load.
if self.raw.len() > max {
return Err(pdfcraft_filters::FilterError::LimitExceeded(max));
}
Ok(self.raw.to_vec())
}
}
#[derive(Clone, Debug, PartialEq)]
pub enum Object {
Null,
Bool(bool),
Int(i64),
Real(f64),
String(PdfString),
Name(Name),
Array(Vec<Object>),
Dict(Dict),
Stream(Stream),
Ref(ObjRef),
}
impl Object {
pub fn name(n: &str) -> Self {
Object::Name(n.as_bytes().to_vec())
}
pub fn as_int(&self) -> Option<i64> {
match self {
Object::Int(i) => Some(*i),
Object::Real(r) if r.fract() == 0.0 && r.is_finite() => Some(*r as i64),
_ => None,
}
}
pub fn as_f64(&self) -> Option<f64> {
match self {
Object::Int(i) => Some(*i as f64),
Object::Real(r) => Some(*r),
_ => None,
}
}
pub fn as_name(&self) -> Option<&[u8]> {
match self {
Object::Name(n) => Some(n),
_ => None,
}
}
pub fn as_ref(&self) -> Option<ObjRef> {
match self {
Object::Ref(r) => Some(*r),
_ => None,
}
}
pub fn as_dict(&self) -> Option<&Dict> {
match self {
Object::Dict(d) => Some(d),
Object::Stream(s) => Some(&s.dict),
_ => None,
}
}
pub fn as_dict_mut(&mut self) -> Option<&mut Dict> {
match self {
Object::Dict(d) => Some(d),
Object::Stream(s) => Some(&mut s.dict),
_ => None,
}
}
pub fn as_array(&self) -> Option<&Vec<Object>> {
match self {
Object::Array(a) => Some(a),
_ => None,
}
}
pub fn as_string(&self) -> Option<&PdfString> {
match self {
Object::String(s) => Some(s),
_ => None,
}
}
}
impl From<i64> for Object {
fn from(v: i64) -> Self {
Object::Int(v)
}
}
impl From<i32> for Object {
fn from(v: i32) -> Self {
Object::Int(v as i64)
}
}
impl From<f64> for Object {
fn from(v: f64) -> Self {
Object::Real(v)
}
}
impl From<bool> for Object {
fn from(v: bool) -> Self {
Object::Bool(v)
}
}
impl From<ObjRef> for Object {
fn from(v: ObjRef) -> Self {
Object::Ref(v)
}
}
impl From<Dict> for Object {
fn from(v: Dict) -> Self {
Object::Dict(v)
}
}
impl From<Vec<Object>> for Object {
fn from(v: Vec<Object>) -> Self {
Object::Array(v)
}
}
impl From<PdfString> for Object {
fn from(v: PdfString) -> Self {
Object::String(v)
}
}
impl From<Stream> for Object {
fn from(v: Stream) -> Self {
Object::Stream(v)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_filter_that_is_not_a_direct_name_is_not_decoded_as_absent() {
let stream = |filter: Object| {
let mut d = Dict::new();
d.set(b"Filter".to_vec(), filter);
Stream::from_raw(d, b"48 69>".to_vec())
};
let hex = Object::Name(b"ASCIIHexDecode".to_vec());
assert_eq!(stream(hex.clone()).decoded_strict().ok(), Some(b"Hi".to_vec()));
assert_eq!(stream(Object::Array(vec![hex.clone()])).decoded().ok(), Some(b"Hi".to_vec()));
assert_eq!(stream(Object::Null).decoded_strict().ok(), Some(b"48 69>".to_vec()));
for filter in [
Object::Ref(ObjRef::new(5, 0)),
Object::Array(vec![Object::Ref(ObjRef::new(5, 0))]),
Object::Int(3),
Object::Array(vec![hex, Object::Int(1)]),
] {
let s = stream(filter.clone());
assert!(s.decoded_strict().is_err(), "{filter:?}");
assert!(s.decoded().is_err(), "{filter:?}");
assert!(s.decoded_first_filter_strict(1 << 20).is_err(), "{filter:?}");
}
}
#[test]
fn parsing_many_distinct_keys_is_not_quadratic() {
let n = 20_000;
let body: String = (0..n).map(|i| format!("/K{i} {i} ")).collect();
let text = format!("<< {body}/K7 true >>");
KEY_COMPARES.with(|c| c.set(0));
let Ok(Object::Dict(d)) = crate::parser::Lexer::new(text.as_bytes(), 0).object() else { panic!("not a dict") };
let compares = KEY_COMPARES.with(|c| c.get());
assert!(compares < 8 * n, "{compares} key comparisons for {n} keys");
// Order and replace-in-place semantics stay those of a plain ordered dictionary.
assert_eq!(d.len(), n);
assert_eq!(d.iter().next().map(|(k, _)| k.as_slice()), Some(&b"K0"[..]));
assert_eq!(d.get(b"K7"), Some(&Object::Bool(true)));
assert_eq!(d.iter().nth(7).map(|(_, v)| v), Some(&Object::Bool(true)));
let mut d = d;
assert_eq!(d.remove(b"K3"), Some(Object::Int(3)));
assert_eq!(d.get(b"K4"), Some(&Object::Int(4)));
assert_eq!(d.iter().nth(3).map(|(k, _)| k.as_slice()), Some(&b"K4"[..]));
d.set(b"K3".to_vec(), Object::Null);
assert_eq!(d.iter().last().map(|(k, _)| k.as_slice()), Some(&b"K3"[..]));
assert!(d.get(b"K20000").is_none());
}
#[test]
fn text_strings_round_trip() {
for s in ["Plain ASCII", "Café — naïve", "日本語", "emoji 📄"] {
assert_eq!(PdfString::text(s).to_text(), s);
}
assert!(!PdfString::text("abc").bytes.starts_with(&[0xFE, 0xFF]));
}
#[test]
fn text_string_encodings() {
// UTF-16BE with BOM.
let mut utf16 = vec![0xFE, 0xFF];
utf16.extend("第一章 概述".encode_utf16().flat_map(u16::to_be_bytes));
assert_eq!(PdfString::literal(utf16).to_text(), "第一章 概述");
// UTF-8 with BOM (PDF 2.0): the BOM is not part of the text.
let mut utf8 = vec![0xEF, 0xBB, 0xBF];
utf8.extend_from_slice("第一章".as_bytes());
assert_eq!(PdfString::literal(utf8).to_text(), "第一章");
// Invalid UTF-8 after the BOM decodes lossily instead of vanishing.
assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, b'A', 0xFF, b'B']).to_text(), "A\u{FFFD}B");
// BOM-less raw UTF-8 from producers that skip the BOM.
assert_eq!(PdfString::literal("目录".as_bytes()).to_text(), "目录");
// PDFDocEncoding Latin-1 (not valid UTF-8) stays PDFDocEncoding.
assert_eq!(PdfString::literal(vec![b'C', b'a', b'f', 0xE9]).to_text(), "Café");
// Doubled BOMs leave no U+FEFF behind.
assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A']).to_text(), "A");
assert_eq!(PdfString::literal(vec![0xEF, 0xBB, 0xBF, 0xEF, 0xBB, 0xBF, b'A']).to_text(), "A");
// Odd trailing byte and empty strings never panic.
assert_eq!(PdfString::literal(vec![0xFE, 0xFF, 0x00, b'A', 0x00]).to_text(), "A");
assert_eq!(PdfString::literal(vec![0xFE, 0xFF]).to_text(), "");
assert_eq!(PdfString::literal(Vec::new()).to_text(), "");
}
#[test]
fn text_eq_compares_what_to_text_decodes() {
let utf16 = |s: &str| [0xFE, 0xFF].into_iter().chain(s.encode_utf16().flat_map(u16::to_be_bytes)).collect::<Vec<u8>>();
// Each invalid UTF-8 sequence is one U+FFFD, as String::from_utf8_lossy has it.
let lossy = [0xEF, 0xBB, 0xBF, 0xC3, 0x28, b'x', 0xF0, 0x9F, 0x92, 0xFF];
assert_eq!(PdfString::literal(lossy.to_vec()).to_text(), String::from_utf8_lossy(&lossy[3..]));
let cases: Vec<Vec<u8>> = vec![
Vec::new(),
b"Plain".to_vec(),
vec![b'C', b'a', b'f', 0xE9, 0x18],
"目录".as_bytes().to_vec(),
utf16("第一章"),
vec![0xFE, 0xFF, 0xD8, 0x00, 0x00, b'A'],
vec![0xFE, 0xFF, 0xFE, 0xFF, 0x00, b'A', 0xFE, 0xFF],
lossy.to_vec(),
];
for bytes in cases {
let s = PdfString::literal(bytes);
let text = s.to_text();
assert!(s.text_eq(&text), "{text:?}");
assert!(!s.text_eq(&format!("{text}x")), "{text:?}");
assert!(text.is_empty() || !s.text_eq(&text[..text.len() - text.chars().last().map_or(0, char::len_utf8)]), "{text:?}");
}
// A U+FEFF after the first character is text and stays.
assert!(PdfString::literal(utf16("A\u{FEFF}")).text_eq("A\u{FEFF}"));
}
#[test]
fn pdfdoc_encoding_high_range() {
assert_eq!(PdfString::literal(vec![0x84, 0x92, 0xA0, b'A']).to_text(), "—™€A");
}
#[test]
fn dict_preserves_order_and_replaces_in_place() {
let mut d = Dict::new();
d.set(b"B".to_vec(), 1);
d.set(b"A".to_vec(), 2);
d.set(b"B".to_vec(), 3);
let keys: Vec<_> = d.iter().map(|(k, _)| k.clone()).collect();
assert_eq!(keys, vec![b"B".to_vec(), b"A".to_vec()]);
assert_eq!(d.int(b"B"), Some(3));
}
#[test]
fn raw_streams_refuse_decoding_past_a_caller_limit() {
let stream = Stream::from_raw(Dict::new(), b"abc".to_vec());
for max in [0, 2] {
assert!(matches!(stream.decoded_within(max), Err(CosError::Filter(_))));
}
}
#[test]
fn strict_decoding_is_bounded_by_the_callers_remaining_allowance() {
let stream = Stream::flate(Dict::new(), &vec![0u8; 1 << 20]);
assert!(matches!(stream.decoded_strict_within(1000), Err(CosError::Filter(_))));
assert_eq!(stream.decoded_strict_within(1 << 20).map(|v| v.len()).ok(), Some(1 << 20));
}
#[test]
fn empty_filter_arrays_refuse_decoding_past_a_caller_limit() {
let mut dict = Dict::new();
dict.set(b"Filter".to_vec(), Object::Array(Vec::new()));
let stream = Stream::from_raw(dict, b"abc".to_vec());
assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_))));
}
#[test]
fn crypt_only_streams_refuse_decoding_past_a_caller_limit() {
let mut dict = Dict::new();
dict.set(b"Filter".to_vec(), Object::name("Crypt"));
let mut parms = Dict::new();
parms.set(b"Name".to_vec(), Object::name("Identity"));
dict.set(b"DecodeParms".to_vec(), Object::Dict(parms));
let stream = Stream::from_raw(dict, b"abc".to_vec());
assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_))));
}
#[test]
fn identity_streams_preserve_bytes_within_a_caller_limit() {
let mut array = Dict::new();
array.set(b"Filter".to_vec(), Object::Array(Vec::new()));
let mut crypt = Dict::new();
crypt.set(b"Filter".to_vec(), Object::name("Crypt"));
for dict in [Dict::new(), array, crypt] {
let stream = Stream::from_raw(dict.clone(), b"abc".to_vec());
assert_eq!(stream.decoded_within(3).unwrap(), b"abc");
assert_eq!(stream.decoded_within(usize::MAX).unwrap(), b"abc");
assert_eq!(stream.decoded().unwrap(), b"abc");
assert_eq!(stream.decoded_strict().unwrap(), b"abc");
assert!(Stream::from_raw(dict, Vec::new()).decoded_within(0).unwrap().is_empty());
}
}
#[test]
fn filtered_streams_preserve_the_caller_limit() {
let stream = Stream::flate(Dict::new(), b"abc");
assert!(matches!(stream.decoded_within(2), Err(CosError::Filter(_))));
assert_eq!(stream.decoded_within(3).unwrap(), b"abc");
assert_eq!(stream.decoded_strict().unwrap(), b"abc");
}
#[test]
fn raw_flate_fallback_streams_preserve_the_caller_limit() {
// Raw deflate with a stored byte and a final empty block. 78 01 also
// passes the zlib header check, so decoding must retry the whole input.
let mut dict = Dict::new();
dict.set(b"Filter".to_vec(), Object::name("FlateDecode"));
let stream = Stream::from_raw(dict, vec![0x78, 0x01, 0x00, 0xfe, 0xff, b'A', 0x01, 0x00, 0x00, 0xff, 0xff]);
let result = stream.decoded_within(0);
assert!(matches!(&result, Err(CosError::Filter(detail)) if detail.contains("exceeds the limit of 0 bytes")), "{result:?}");
assert_eq!(stream.decoded_within(1).unwrap(), b"A");
assert_eq!(stream.decoded_strict().unwrap(), b"A");
}
#[test]
fn decoded_stops_at_renderer_cap_of_256_mib() {
let data = vec![0u8; (256 << 20) + 1];
let stream = Stream::flate(Dict::new(), &data);
assert!(matches!(stream.decoded(), Err(CosError::Filter(_))));
}
#[test]
fn first_filter_decodes_only_the_outer_wrapper() {
let data = "0 0 m 100 100 l S\n".repeat(64).into_bytes();
let inner = Stream::flate(Dict::new(), &data).raw.to_vec();
for (outer, name, corrupt) in [(Filter::Ascii85, "ASCII85Decode", &b"{~>"[..]), (Filter::AsciiHex, "ASCIIHexDecode", &b"G0>"[..])] {
let encoded = pdfcraft_filters::encode(&outer, &Params::default(), &inner).unwrap();
let mut dict = Dict::new();
dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name(name), Object::name("FlateDecode")]));
let stream = Stream::from_raw(dict.clone(), encoded);
assert_eq!(stream.decoded_first_filter_strict(usize::MAX).unwrap(), inner);
assert_eq!(stream.decoded_strict().unwrap(), data);
assert!(matches!(stream.decoded_first_filter_strict(inner.len() - 1), Err(CosError::Filter(_))));
assert!(matches!(Stream::from_raw(dict, corrupt.to_vec()).decoded_first_filter_strict(usize::MAX), Err(CosError::Filter(_))));
}
}
}
-96
View File
@@ -1,96 +0,0 @@
//! Bounded numbering for page labels (ISO 32000-2 §12.4.2).
//! Pure helpers shared by inspection and editing; no object graph or renderer dependency.
/// A complete custom label, including its prefix, may contain this many UTF-8 bytes.
pub const MAX_LABEL_BYTES: usize = 1024;
/// Raw prefixes are bounded before decoding; this admits a full UTF-16BE ASCII label.
pub const MAX_PREFIX_BYTES: usize = 2 * MAX_LABEL_BYTES + 2;
/// Sum of the UTF-8 bytes of custom labels for one document.
pub const MAX_LABEL_TOTAL_BYTES: usize = 4 << 20;
/// Sum of the UTF-8 bytes of the decoded prefixes of one `/PageLabels` tree. Each range decodes
/// its own prefix, and ranges may share one prefix object, so without this a small file could ask
/// for a copy per range: [`MAX_LABEL_TREE_WORK`] ranges naming one 1,024-byte prefix would hold
/// about 200 MiB. Real documents stay far below it.
pub const MAX_PREFIX_TOTAL_BYTES: usize = 4 << 20;
/// The most number-tree entries (nodes, kids and range pairs) read from one `/PageLabels`
/// tree. Scaled for real documents: one range per page on a ten-thousand-page PDF happens
/// (prefix labels; #307's file has 9,156 pages), so the cap is twenty times that while still
/// bounding hostile trees.
pub const MAX_LABEL_TREE_WORK: usize = 200_000;
/// Maximum nesting of a label number tree.
pub const MAX_LABEL_TREE_DEPTH: usize = 32;
/// Lowercase Roman numerals, or `None` if the output exceeds the remaining label budget.
/// The budget is capped at [`MAX_LABEL_BYTES`], including when `n` is hostile.
pub fn roman(mut n: u64, max_bytes: usize) -> Option<String> {
const PARTS: [(u64, &str); 13] = [
(1000, "m"),
(900, "cm"),
(500, "d"),
(400, "cd"),
(100, "c"),
(90, "xc"),
(50, "l"),
(40, "xl"),
(10, "x"),
(9, "ix"),
(5, "v"),
(4, "iv"),
(1, "i"),
];
let max_bytes = max_bytes.min(MAX_LABEL_BYTES);
let mut out = String::new();
for (value, part) in PARTS {
let count = usize::try_from(n / value).ok()?;
let bytes = count.checked_mul(part.len())?;
if bytes > max_bytes.saturating_sub(out.len()) {
return None;
}
// The quotient is checked against the tiny byte budget before repeating.
out.push_str(&part.repeat(count));
n %= value;
}
Some(out)
}
/// a..z, aa..zz, aaa.. (the repeated-letter numbering specified by PDF).
/// Returns `None` instead of truncating a label that exceeds the remaining byte budget.
pub fn alpha(n: u64, max_bytes: usize) -> Option<String> {
let n = n.max(1) - 1;
let count = usize::try_from(n / 26 + 1).ok()?;
if count > max_bytes.min(MAX_LABEL_BYTES) {
return None;
}
let letter = char::from(b'a' + (n % 26) as u8);
Some(std::iter::repeat_n(letter, count).collect())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn bounded_page_labels_follow_pdf_numbering() {
assert_eq!(roman(4, MAX_LABEL_BYTES).as_deref(), Some("iv"));
assert_eq!(roman(1994, MAX_LABEL_BYTES).as_deref(), Some("mcmxciv"));
for (n, expected) in [(0, "a"), (1, "a"), (26, "z"), (27, "aa"), (52, "zz"), (53, "aaa")] {
assert_eq!(alpha(n, MAX_LABEL_BYTES).as_deref(), Some(expected));
}
}
#[test]
fn bounded_page_labels_check_lengths_before_repeating() {
assert_eq!(roman(1_024_000, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES);
assert!(roman(1_024_001, MAX_LABEL_BYTES).is_none());
assert_eq!(alpha(26 * 1024, MAX_LABEL_BYTES).unwrap().len(), MAX_LABEL_BYTES);
assert!(alpha(26 * 1024 + 1, MAX_LABEL_BYTES).is_none());
assert_eq!(roman(4, 2).as_deref(), Some("iv"));
assert!(roman(4, 1).is_none());
assert_eq!(alpha(27, 2).as_deref(), Some("aa"));
assert!(alpha(27, 1).is_none());
for max in [0, MAX_LABEL_BYTES, usize::MAX] {
assert!(roman(u64::MAX, max).is_none());
assert!(alpha(u64::MAX, max).is_none());
}
}
}
-886
View File
@@ -1,886 +0,0 @@
//! Lexer and object parser (ISO 32000-2 §7.2–7.3), tolerant of common real-world damage.
//!
//! The parser works on a byte slice and a position; it never allocates for skipped data and
//! never panics on malformed input (every index is bounds-checked). Depth is limited to defend
//! against pathological nesting.
use std::sync::Arc;
use crate::object::{Dict, ObjRef, Object, PdfString, Stream};
use crate::{Bytes, CosError};
const MAX_DEPTH: usize = 128;
/// Objects one top-level [`Lexer::object`] call may build. Real arrays (page trees, parent
/// trees, widths) stay far below; a decompressed object stream of `0 0 0 ...` does not.
const MAX_NODES: usize = 1 << 22;
/// One of the six PDF whitespace characters (ISO 32000-2 §7.2.3, Table 1).
pub fn is_whitespace(b: u8) -> bool {
matches!(b, b'\0' | b'\t' | b'\n' | b'\x0C' | b'\r' | b' ')
}
pub(crate) fn is_delimiter(b: u8) -> bool {
matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%')
}
fn is_regular(b: u8) -> bool {
!is_whitespace(b) && !is_delimiter(b)
}
/// A cursor over PDF bytes.
pub struct Lexer<'a> {
pub data: &'a [u8],
pub pos: usize,
/// Objects built by the current top-level [`Lexer::object`] call.
nodes: usize,
}
impl<'a> Lexer<'a> {
pub fn new(data: &'a [u8], pos: usize) -> Self {
Self { data, pos: pos.min(data.len()), nodes: 0 }
}
fn peek(&self) -> Option<u8> {
self.data.get(self.pos).copied()
}
/// Skip whitespace and comments.
pub fn skip_ws(&mut self) {
while let Some(b) = self.peek() {
if is_whitespace(b) {
self.pos += 1;
} else if b == b'%' {
while let Some(c) = self.peek() {
if c == b'\n' || c == b'\r' {
break;
}
self.pos += 1;
}
} else {
break;
}
}
}
/// Read a run of regular characters (a keyword or number).
pub fn token(&mut self) -> &'a [u8] {
let start = self.pos.min(self.data.len());
while self.peek().is_some_and(is_regular) {
self.pos += 1;
}
&self.data[start..self.pos]
}
/// `true` (and consumes it) if the next token is exactly `kw`.
pub fn eat_keyword(&mut self, kw: &[u8]) -> bool {
self.skip_ws();
let end = self.pos + kw.len();
if self.data.get(self.pos..end) == Some(kw) && self.data.get(end).is_none_or(|b| !is_regular(*b)) {
self.pos = end;
true
} else {
false
}
}
fn err(&self, what: &str) -> CosError {
CosError::Syntax { offset: self.pos, detail: what.to_string() }
}
/// Parse one direct object. References `N G R` are recognised.
pub fn object(&mut self) -> Result<Object, CosError> {
self.nodes = 0;
self.object_depth(0)
}
fn over_budget(&self) -> bool {
self.nodes > MAX_NODES
}
fn object_depth(&mut self, depth: usize) -> Result<Object, CosError> {
if depth > MAX_DEPTH {
return Err(self.err("nesting too deep"));
}
self.nodes = self.nodes.saturating_add(1);
if self.over_budget() {
return Err(self.err("object too large"));
}
self.skip_ws();
let Some(b) = self.peek() else { return Err(self.err("unexpected end of data")) };
match b {
b'/' => {
self.pos += 1;
Ok(Object::Name(self.name_body()))
}
b'(' => {
self.pos += 1;
Ok(Object::String(PdfString { bytes: self.literal_string()?, hex: false }))
}
b'<' if self.data.get(self.pos + 1) == Some(&b'<') => {
self.pos += 2;
Ok(Object::Dict(self.dict_body(depth)?))
}
b'<' => {
self.pos += 1;
Ok(Object::String(PdfString { bytes: self.hex_string()?, hex: true }))
}
b'[' => {
self.pos += 1;
let mut items = Vec::new();
loop {
self.skip_ws();
match self.peek() {
None => return Err(self.err("unterminated array")),
Some(b']') => {
self.pos += 1;
break;
}
_ => {
let before = self.pos;
match self.object_depth(depth + 1) {
Ok(o) => items.push(o),
Err(e) if self.over_budget() => return Err(e),
// Tolerate junk inside arrays by skipping one byte.
Err(_) if self.pos == before => self.pos += 1,
Err(e) => return Err(e),
}
}
}
}
Ok(Object::Array(items))
}
b'+' | b'-' | b'.' | b'0'..=b'9' => self.number_or_ref(),
_ => {
let start = self.pos;
let tok = self.token();
match tok {
b"true" => Ok(Object::Bool(true)),
b"false" => Ok(Object::Bool(false)),
b"null" => Ok(Object::Null),
b"" => {
self.pos = start + 1;
Err(CosError::Syntax { offset: start, detail: format!("unexpected byte 0x{b:02x}") })
}
other => Err(CosError::Syntax { offset: start, detail: format!("unexpected keyword {:?}", String::from_utf8_lossy(other)) }),
}
}
}
}
fn name_body(&mut self) -> Vec<u8> {
let mut out = Vec::new();
while let Some(b) = self.peek() {
if !is_regular(b) {
break;
}
self.pos += 1;
if b == b'#'
&& let (Some(h), Some(l)) = (self.data.get(self.pos).and_then(|c| hexval(*c)), self.data.get(self.pos + 1).and_then(|c| hexval(*c)))
{
out.push(h << 4 | l);
self.pos += 2;
continue;
}
out.push(b);
}
out
}
fn literal_string(&mut self) -> Result<Vec<u8>, CosError> {
let mut out = Vec::new();
let mut depth = 1usize;
while let Some(b) = self.peek() {
self.pos += 1;
match b {
b'(' => {
depth += 1;
out.push(b);
}
b')' => {
depth -= 1;
if depth == 0 {
return Ok(out);
}
out.push(b);
}
b'\\' => {
let Some(e) = self.peek() else { break };
self.pos += 1;
match e {
b'n' => out.push(b'\n'),
b'r' => out.push(b'\r'),
b't' => out.push(b'\t'),
b'b' => out.push(8),
b'f' => out.push(12),
b'(' | b')' | b'\\' => out.push(e),
b'\r' => {
if self.peek() == Some(b'\n') {
self.pos += 1;
}
}
b'\n' => {}
b'0'..=b'7' => {
let mut v = (e - b'0') as u32;
for _ in 0..2 {
match self.peek() {
Some(d @ b'0'..=b'7') => {
v = v * 8 + (d - b'0') as u32;
self.pos += 1;
}
_ => break,
}
}
out.push((v & 0xFF) as u8);
}
other => out.push(other),
}
}
// End-of-line markers inside strings are normalised to \n (§7.3.4.2).
b'\r' => {
if self.peek() == Some(b'\n') {
self.pos += 1;
}
out.push(b'\n');
}
other => out.push(other),
}
}
Err(self.err("unterminated string"))
}
fn hex_string(&mut self) -> Result<Vec<u8>, CosError> {
let mut out = Vec::new();
let mut hi: Option<u8> = None;
while let Some(b) = self.peek() {
self.pos += 1;
if b == b'>' {
if let Some(h) = hi {
out.push(h << 4);
}
return Ok(out);
}
if is_whitespace(b) {
continue;
}
let Some(v) = hexval(b) else { return Err(self.err("invalid hex digit")) };
match hi.take() {
None => hi = Some(v),
Some(h) => out.push(h << 4 | v),
}
}
Err(self.err("unterminated hex string"))
}
fn dict_body(&mut self, depth: usize) -> Result<Dict, CosError> {
let mut d = Dict::new();
loop {
self.skip_ws();
match self.peek() {
None => return Err(self.err("unterminated dictionary")),
Some(b'>') if self.data.get(self.pos + 1) == Some(&b'>') => {
self.pos += 2;
return Ok(d);
}
Some(b'/') => {
self.pos += 1;
let key = self.name_body();
self.skip_ws();
// A key directly followed by `>>` has no value: treat as null. (A following
// `/` is a name *value*, so it cannot signal a missing value.)
if self.peek() == Some(b'>') && self.data.get(self.pos + 1) == Some(&b'>') {
d.set(key, Object::Null);
continue;
}
let value = self.object_depth(depth + 1)?;
d.set(key, value);
}
Some(_) => {
// Tolerate garbage between entries (seen in damaged files): skip a token.
let before = self.pos;
let _ = self.object_depth(depth + 1);
if self.over_budget() {
return Err(self.err("object too large"));
}
if self.pos == before {
self.pos += 1;
}
}
}
}
}
fn number_or_ref(&mut self) -> Result<Object, CosError> {
let start = self.pos;
let first = self.number()?;
// Look ahead for "G R" to form a reference.
if let Object::Int(n) = first
&& n >= 0
{
let save = self.pos;
self.skip_ws();
if self.peek().is_some_and(|b| b.is_ascii_digit()) {
let gstart = self.pos;
let g = self.token();
if g.iter().all(u8::is_ascii_digit) && self.eat_keyword(b"R") {
let generation = std::str::from_utf8(g).ok().and_then(|s| s.parse::<u32>().ok()).unwrap_or(0);
if n <= u32::MAX as i64 {
return Ok(Object::Ref(ObjRef::new(n as u32, generation.min(u16::MAX as u32) as u16)));
}
}
let _ = gstart;
}
self.pos = save;
}
let _ = start;
Ok(first)
}
fn number(&mut self) -> Result<Object, CosError> {
let start = self.pos;
let tok = self.token();
let s = std::str::from_utf8(tok).map_err(|_| self.err("bad number"))?;
if s.contains('.') {
// Tolerate forms like "--5", "5-" or "1.2.3" by parsing the longest valid prefix.
// Only a doubled sign needs a cleaned copy: a page of InDesign `/PieceInfo` is
// thousands of plain reals, and building a copy and a substring search for each
// was a large part of reading one (#5873).
let t = s.trim_start_matches('+');
let cleaned = if t.as_bytes().windows(2).any(|w| w == b"--") { std::borrow::Cow::Owned(t.replacen("--", "-", 1)) } else { t.into() };
let v = cleaned.get(..real_prefix_len(cleaned.as_bytes())).and_then(|p| parse_counted::<f64>(p).ok());
return Ok(Object::Real(v.unwrap_or(0.0)));
}
// Tolerate a doubled sign ("--280") like the real branch does.
let t = s.trim_start_matches('+');
let cleaned = t.strip_prefix('-').filter(|r| r.starts_with('-')).unwrap_or(t);
match cleaned.parse::<i64>() {
Ok(v) => Ok(Object::Int(v)),
Err(_) => {
// Out-of-range integers are treated as reals (§7.3.3 allows implementation limits).
cleaned.parse::<f64>().map(Object::Real).map_err(|_| CosError::Syntax { offset: start, detail: format!("bad number {s:?}") })
}
}
}
}
/// Length of the longest prefix of `s` that reads as a decimal number: a sign, digits with at
/// most one `.`, and an exponent with digits. Zero when that prefix has no digit. Found in one
/// pass so a long damaged token is converted once, not once per shorter prefix.
fn real_prefix_len(s: &[u8]) -> usize {
let digits = |from: usize| s.get(from..).map_or(0, |r| r.iter().take_while(|b| b.is_ascii_digit()).count());
let mut at = usize::from(matches!(s.first(), Some(b'+' | b'-')));
let int = digits(at);
at += int;
let mut frac = 0;
if s.get(at) == Some(&b'.') {
frac = digits(at + 1);
at += 1 + frac;
}
if int + frac == 0 {
return 0;
}
if matches!(s.get(at), Some(b'e' | b'E')) {
let sign = usize::from(matches!(s.get(at + 1), Some(b'+' | b'-')));
let exp = digits(at + 1 + sign);
if exp > 0 {
at += 1 + sign + exp;
}
}
at
}
#[cfg(test)]
thread_local! {
/// Bytes searched for `endstream` on this thread.
pub(crate) static ENDSTREAM_SCAN: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
/// Bytes handed to number conversion on this thread.
static NUMBER_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
fn parse_counted<T: std::str::FromStr>(s: &str) -> Result<T, T::Err> {
#[cfg(test)]
NUMBER_BYTES.with(|c| c.set(c.get() + s.len()));
s.parse()
}
fn hexval(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
/// Parse an indirect object at `offset`: `N G obj <object> [stream…endstream] endobj`.
/// `resolve_length` resolves an indirect `/Length`. Returns the object and its (num, gen).
pub fn parse_indirect(data: &[u8], offset: usize, resolve_length: &dyn Fn(ObjRef) -> Option<i64>) -> Result<(ObjRef, Object), CosError> {
parse_indirect_in(data, None, offset, resolve_length, None)
}
/// [`parse_indirect`] for a document's own bytes: stream data points into `data` instead of
/// being copied out of it.
/// `endstreams` is the document's [`EndstreamIndex`] for `data`.
pub(crate) fn parse_indirect_shared(
data: &Arc<Vec<u8>>,
offset: usize,
resolve_length: &dyn Fn(ObjRef) -> Option<i64>,
endstreams: &EndstreamIndex,
) -> Result<(ObjRef, Object), CosError> {
parse_indirect_in(data, Some(data), offset, resolve_length, Some(endstreams))
}
/// Where `endstream` occurs in a document's bytes, found once, the first time a stream's
/// /Length is wrong or missing. Searching the rest of the file for each such stream instead
/// makes many of them ending at one late `endstream` quadratic to read.
pub(crate) type EndstreamIndex = std::sync::OnceLock<Vec<usize>>;
fn endstream_offsets(data: &[u8]) -> Vec<usize> {
#[cfg(test)]
ENDSTREAM_SCAN.with(|c| c.set(c.get() + data.len()));
data.windows(9).enumerate().filter(|(i, w)| *w == b"endstream" && endstream_at(data, *i)).map(|(i, _)| i).collect()
}
fn parse_indirect_in(
data: &[u8],
shared: Option<&Arc<Vec<u8>>>,
offset: usize,
resolve_length: &dyn Fn(ObjRef) -> Option<i64>,
endstreams: Option<&EndstreamIndex>,
) -> Result<(ObjRef, Object), CosError> {
let mut lx = Lexer::new(data, offset);
lx.skip_ws();
let num_tok = lx.token();
lx.skip_ws();
let gen_tok = lx.token();
let parse_u = |t: &[u8]| std::str::from_utf8(t).ok().and_then(|s| s.parse::<u64>().ok());
let (Some(num), Some(generation)) = (parse_u(num_tok), parse_u(gen_tok)) else {
return Err(CosError::Syntax { offset, detail: "expected object header".into() });
};
if !lx.eat_keyword(b"obj") {
return Err(CosError::Syntax { offset: lx.pos, detail: "expected 'obj'".into() });
}
let id = ObjRef::new(num.min(u32::MAX as u64) as u32, generation.min(u16::MAX as u64) as u16);
let obj = match lx.object() {
Ok(o) => o,
// "N G obj endobj" (empty) is treated as null.
Err(_) if lx.eat_keyword(b"endobj") => return Ok((id, Object::Null)),
Err(e) => return Err(e),
};
lx.skip_ws();
if let Object::Dict(dict) = &obj
&& lx.eat_keyword(b"stream")
{
// The keyword is followed by CRLF or LF (a lone CR is tolerated). Microsoft Reporting
// Services writes a space first (`stream \r\n`); spaces and tabs count only when a line
// break follows, so they are never taken from the data.
let mut eol = lx.pos;
while matches!(data.get(eol), Some(b' ' | b'\t')) {
eol += 1;
}
if matches!(data.get(eol), Some(b'\r' | b'\n')) {
lx.pos = eol;
}
match (data.get(lx.pos), data.get(lx.pos + 1)) {
(Some(b'\r'), Some(b'\n')) => lx.pos += 2,
(Some(b'\n'), _) | (Some(b'\r'), _) => lx.pos += 1,
_ => {}
}
let start = lx.pos;
let declared = match dict.get(b"Length") {
Some(Object::Int(n)) => Some(*n),
Some(Object::Ref(r)) => resolve_length(*r),
_ => None,
};
// A /Length that fits in the file (not negative, and no wrap on a 32-bit target).
let fits = declared.and_then(|l| usize::try_from(l).ok()).filter(|l| start.checked_add(*l).is_some_and(|end| end <= data.len()));
// What follows the /Length bytes, white space aside.
let after = |len: usize| {
let mut after = Lexer::new(data, start.saturating_add(len));
after.skip_ws();
data.get(after.pos..).unwrap_or_default()
};
let len = match fits {
// The /Length ends at `endstream`, or where it would be: misspelt (`endstrea`), or
// left out before `endobj` or the end of the file. The stream is the bytes its /Length
// gives, "how many bytes of the PDF file are used for the stream's data" (ISO 32000-2
// §7.3.8.2), as other readers take it. A producer that wrote `endstrea` after every
// stream lost every font and every page's content, and searching on would run into a
// later object's `endstream`.
Some(l) if at_stream_end(after(l)) => l,
// Wrong or missing /Length: search for "endstream" (§7.3.8.1 recovery). With none
// left in the file, a /Length that fits is still the stream's extent.
_ => match find_endstream(data, start, endstreams) {
Some(end) => end - start,
None => fits.ok_or_else(|| CosError::Syntax { offset: start, detail: "stream without endstream".into() })?,
},
};
let range = start..start + len;
let raw = match shared {
Some(buf) => Bytes::view(buf, range),
None => data.get(range).map(|r| Bytes::from(r.to_vec())),
}
.ok_or_else(|| CosError::Syntax { offset: start, detail: "stream data past the end of the file".into() })?;
let mut dict = dict.clone();
dict.set(b"Length".to_vec(), Object::Int(raw.len() as i64));
return Ok((id, Object::Stream(Stream { dict, raw })));
}
Ok((id, obj))
}
/// Whether `rest`, what follows a stream's /Length bytes with white space skipped, is where the
/// stream ends. A whole `endstream` token is: a token runs on through regular characters (ISO
/// 32000-2 §7.2.3), so payload such as `endstreamer` is not one. A misspelt `endstream`
/// (`endstrea`), an `endobj` with the `endstream` left out, or both, end it only where the object
/// ends as well, before the end of the data, another object's `N G obj` header, or `xref`,
/// `trailer` or `startxref`: payload such as `endobj more` or `endobject`, where a wrong /Length
/// stops, is not, and the search for `endstream` runs instead. So does anything else but the end
/// of the data.
fn at_stream_end(rest: &[u8]) -> bool {
let mut lx = Lexer::new(rest, 0);
if lx.eat_keyword(b"endstream") {
return true;
}
let misspelt = lx.eat_keyword(b"endstrea");
let endobj = lx.eat_keyword(b"endobj");
lx.skip_ws();
if lx.pos >= rest.len() {
return true;
}
if !(misspelt || endobj) {
return false;
}
if [&b"xref"[..], b"trailer", b"startxref"].iter().any(|kw| Lexer::new(rest, lx.pos).eat_keyword(kw)) {
return true;
}
let number = |lx: &mut Lexer| {
lx.skip_ws();
let t = lx.token();
!t.is_empty() && t.iter().all(u8::is_ascii_digit)
};
number(&mut lx) && number(&mut lx) && lx.eat_keyword(b"obj")
}
/// Whether a whole `endstream` token starts at `at` in `data`: one followed by a regular
/// character (`endstreamer`) is payload (ISO 32000-2 §7.2.3).
fn endstream_at(data: &[u8], at: usize) -> bool {
data.get(at..).is_some_and(|d| d.starts_with(b"endstream")) && !data.get(at.saturating_add(9)).copied().is_some_and(is_regular)
}
/// Offset where stream data ends (before the EOL preceding "endstream").
fn find_endstream(data: &[u8], from: usize, endstreams: Option<&EndstreamIndex>) -> Option<usize> {
let at = match endstreams {
Some(index) => {
let index = index.get_or_init(|| endstream_offsets(data));
index.get(index.partition_point(|&p| p < from)).copied()?
}
None => {
let hay = data.get(from..)?;
let found = hay.windows(9).enumerate().position(|(i, w)| w == b"endstream" && endstream_at(hay, i));
#[cfg(test)]
ENDSTREAM_SCAN.with(|c| c.set(c.get() + found.map_or(hay.len(), |p| p + 9)));
found? + from
}
};
let mut end = at;
if end > from && data[end - 1] == b'\n' {
end -= 1;
}
if end > from && data[end - 1] == b'\r' {
end -= 1;
}
Some(end)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn numbers_with_multibyte_garbage_do_not_panic() {
// From `cargo xtask fuzz`: "2 0 obj\n.\u{2014}" (an em dash after a dot).
let doc = crate::Document::open(std::sync::Arc::new("2 0 obj\n.\u{2014}\n".as_bytes().to_vec()));
assert!(doc.is_err());
for t in [".\u{2014}", "1.\u{e9}5", "-.\u{1F600}", "+.5\u{2014}"] {
let _ = Lexer::new(t.as_bytes(), 0).object();
}
assert_eq!(Lexer::new("1.5\u{2014}".as_bytes(), 0).object().unwrap(), Object::Real(1.5));
}
#[test]
fn a_damaged_real_is_converted_once() {
let token = format!("1.{}{}", "5".repeat(2_000), "x".repeat(20_000));
NUMBER_BYTES.with(|c| c.set(0));
let Ok(Object::Real(v)) = Lexer::new(token.as_bytes(), 0).object() else { panic!("not a real") };
let converted = NUMBER_BYTES.with(|c| c.get());
assert!(converted <= token.len(), "{converted} bytes converted for a {}-byte token", token.len());
assert!((v - 1.5555).abs() < 1e-3);
// The longest numeric prefix wins, as before.
for (t, want) in
[("1.2.3", 1.2), ("--5.", -5.0), ("+.5x", 0.5), ("5.-", 5.0), ("1.5e3", 1500.0), ("1.5e+", 1.5), ("-.x", 0.0), (".", 0.0), ("-inf.", 0.0)]
{
assert_eq!(Lexer::new(t.as_bytes(), 0).object().ok(), Some(Object::Real(want)), "{t}");
}
}
fn obj(s: &str) -> Object {
Lexer::new(s.as_bytes(), 0).object().unwrap_or_else(|e| panic!("{s}: {e}"))
}
#[test]
fn doubled_sign_integer_reads_like_the_real_form() {
// Some writers emit a TJ kerning as "--280"; the real form "--280.0" is already read.
assert_eq!(obj("--280.0"), Object::Real(-280.0));
assert_eq!(obj("--280"), Object::Int(-280));
match obj("[(Hello) --280 (World)]") {
Object::Array(items) => {
assert_eq!(items.len(), 3);
assert_eq!(items[1], Object::Int(-280));
}
other => panic!("not an array: {other:?}"),
}
}
#[test]
fn scalars() {
assert_eq!(obj("true"), Object::Bool(true));
assert_eq!(obj("null"), Object::Null);
assert_eq!(obj(" -17 "), Object::Int(-17));
assert_eq!(obj("+42"), Object::Int(42));
assert_eq!(obj("-.5"), Object::Real(-0.5));
assert_eq!(obj("4."), Object::Real(4.0));
assert_eq!(obj("99999999999999999999"), Object::Real(1e20));
assert_eq!(obj("12 0 R"), Object::Ref(ObjRef::new(12, 0)));
assert_eq!(obj("12 0 obj").as_int(), Some(12), "12 followed by 0 obj is not a reference");
}
#[test]
fn names_decode_hex_escapes() {
assert_eq!(obj("/A#20B"), Object::Name(b"A B".to_vec()));
assert_eq!(obj("/Adobe#"), Object::Name(b"Adobe#".to_vec()));
assert_eq!(obj("/"), Object::Name(Vec::new()));
}
#[test]
fn literal_strings() {
let s = |x: &str| obj(x).as_string().expect("string").bytes.clone();
assert_eq!(s("(a (nested) b)"), b"a (nested) b");
assert_eq!(s(r"(\n\t\\\(\))"), b"\n\t\\()");
assert_eq!(s(r"(\101\1011)"), b"AA1");
assert_eq!(s("(line\\\ncontinued)"), b"linecontinued");
assert_eq!(s("(cr\r\nlf)"), b"cr\nlf");
}
#[test]
fn hex_strings() {
let s = obj("<48 65 6C6c6f7>");
assert_eq!(s.as_string().unwrap().bytes, b"Hellop");
assert!(s.as_string().unwrap().hex);
}
#[test]
fn arrays_and_dicts() {
let o = obj("<< /Type /Page /Kids [1 0 R 2 0 R] /Box [0 0 612.5 792] /Nested << /A true >> /Empty >>");
let d = o.as_dict().unwrap();
assert_eq!(d.name(b"Type"), Some(&b"Page"[..]));
assert_eq!(d.get(b"Kids").unwrap().as_array().unwrap().len(), 2);
assert_eq!(d.get(b"Box").unwrap().as_array().unwrap()[2], Object::Real(612.5));
assert_eq!(d.get(b"Empty"), Some(&Object::Null));
}
#[test]
fn comments_are_whitespace() {
assert_eq!(obj("% hello\n [1 % two\n 3]"), Object::Array(vec![Object::Int(1), Object::Int(3)]));
}
#[test]
fn indirect_stream_with_wrong_length_recovers() {
let data = b"7 0 obj\n<< /Length 999 >>\nstream\nHELLO\nendstream\nendobj\n";
let (id, o) = parse_indirect(data, 0, &|_| None).unwrap();
assert_eq!(id, ObjRef::new(7, 0));
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO");
assert_eq!(s.dict.int(b"Length"), Some(5));
}
/// With no `endstream` left in the file, as a producer that writes `endstrea` leaves every
/// stream, a stream is the bytes its /Length gives, "how many bytes of the PDF file are used
/// for the stream's data" (ISO 32000-2 §7.3.8.2): it was an error, and the object null, so
/// such a file lost every font and page content. A /Length past the file, or none, is still
/// no stream.
#[test]
fn indirect_stream_without_endstream_is_its_length() {
for end in [&b"\r\nendstrea\rendobj\r\n"[..], b"\nendobj\n", b""] {
let mut data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO".to_vec();
data.extend_from_slice(end);
let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO", "{end:?}");
}
assert!(parse_indirect(b"7 0 obj\n<< /Length 6 >>\nstream\nHELLO", 0, &|_| None).is_err(), "past the file");
assert!(parse_indirect(b"7 0 obj\n<< >>\nstream\nHELLO\nendstrea\nendobj\n", 0, &|_| None).is_err(), "no /Length");
}
/// A stream ending in `endstrea` is its /Length even when a later object does have an
/// `endstream`: the search for one would otherwise run on into that object and take its
/// header and data as this stream's.
#[test]
fn a_stream_without_endstream_is_not_read_into_a_later_one() {
let data = b"7 0 obj\n<< /Length 5 >>\nstream\r\nHELLO\r\nendstrea\rendobj\r\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n";
let second = data.windows(7).position(|w| w == b"8 0 obj").unwrap();
for (offset, want) in [(0, &b"HELLO"[..]), (second, b"WORLD")] {
let (_, o) = parse_indirect(data, offset, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected at {offset}") };
assert_eq!(&s.raw[..], want, "at {offset}");
}
// Through the loader's shared index of `endstream` offsets as well.
let index = EndstreamIndex::new();
let (_, o) = parse_indirect_in(data, None, 0, &|_| None, Some(&index)).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO");
}
/// The same for a stream whose `endstream` is left out before `endobj`: the later object's
/// `endstream` is not this stream's end.
#[test]
fn a_stream_ending_at_endobj_is_not_read_into_a_later_one() {
let data = b"7 0 obj\n<< /Length 5 >>\nstream\nHELLO\nendobj\n8 0 obj\n<< /Length 5 >>\nstream\nWORLD\nendstream\nendobj\n";
let (_, o) = parse_indirect(data, 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO");
}
/// A wrong /Length that stops before payload bytes beginning like an end marker (`endobject`,
/// `endstreak`, `endstreamer`) is not taken for the stream's end: a token runs on through
/// regular characters (ISO 32000-2 §7.2.3), so the search for `endstream` finds the whole
/// payload, and skips `endstreamer` too, by itself and through the loader's shared index.
#[test]
fn a_length_ending_inside_a_longer_word_is_not_an_end_marker() {
for word in ["endobject", "endstreak", "endstreamer"] {
let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {word} more\nendstream\nendobj\n");
let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}");
let index = EndstreamIndex::new();
let (_, o) = parse_indirect_in(pdf.as_bytes(), None, 0, &|_| None, Some(&index)).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(s.raw[..], *format!("HELLO {word} more").as_bytes(), "{word}, indexed");
}
// Whole markers still end it, followed by white space, a delimiter or the end of the data.
for tail in ["endobj\n", "endstrea%", "endobj"] {
let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{tail}");
let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO", "{tail:?}");
}
}
/// A whole `endobj` or `endstrea` token where a wrong /Length stops is payload, not the
/// stream's end, unless the object ends there too: what follows is the end of the data,
/// another object's header, or `xref`, `trailer` or `startxref`. Otherwise the search for
/// `endstream` finds the whole payload.
#[test]
fn an_end_marker_inside_the_payload_does_not_end_the_stream() {
for payload in ["endobj more", "endstrea more", "endstrea endobj more", "endobj 12 more"] {
let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO {payload}\nendstream\nendobj\n");
let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(s.raw[..], *format!("HELLO {payload}").as_bytes(), "{payload}");
}
// Where the object does end, the /Length is the stream, also with an `endstream` later.
for next in ["8 0 obj\n<< >>", "%c\n12 0 obj", "xref", "trailer", "startxref"] {
for marker in ["endobj", "endstrea\nendobj", "endstrea"] {
let pdf = format!("7 0 obj\n<< /Length 5 >>\nstream\nHELLO\n{marker}\n{next}\nstream\nWORLD\nendstream\nendobj\n");
let (_, o) = parse_indirect(pdf.as_bytes(), 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO", "{marker:?} then {next:?}");
}
}
}
/// Without an `endstream`, the /Length is trusted as written when it fits, also when it is
/// wrong, and an indirect one is resolved first; one that doesn't fit, a negative one and one
/// too large for any file are errors rather than wrapped into a short read.
#[test]
fn a_stream_without_endstream_takes_its_length_only_when_it_fits() {
let stream = |length: &str, tail: &str| format!("7 0 obj\n<< /Length {length} >>\nstream\nHELLO{tail}");
let raw = |pdf: &str, lookup: &dyn Fn(ObjRef) -> Option<i64>| match parse_indirect(pdf.as_bytes(), 0, lookup) {
Ok((_, Object::Stream(s))) => Ok(s.raw.to_vec()),
Ok((_, other)) => panic!("{pdf:?} is {other:?}"),
Err(e) => Err(e),
};
let none = |_: ObjRef| None;
// Wrong but fitting: the bytes it gives, not the bytes up to `endstrea`.
assert_eq!(raw(&stream("3", "\nendstrea\nendobj\n"), &none).unwrap(), b"HEL");
// Exactly to the end of the file, and nothing at all.
assert_eq!(raw(&stream("5", ""), &none).unwrap(), b"HELLO");
assert_eq!(raw(&stream("0", "\nendstrea\nendobj\n"), &none).unwrap(), b"");
// Indirect, resolved through the lookup.
let lookup = |r: ObjRef| (r == ObjRef::new(9, 0)).then_some(4);
assert_eq!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &lookup).unwrap(), b"HELL");
assert!(raw(&stream("9 0 R", "\nendstrea\nendobj\n"), &none).is_err(), "an unresolved /Length");
// One past the end, negative, the largest integer and one beyond it.
for length in ["6", "-1", "-9223372036854775808", "9223372036854775807", "18446744073709551621"] {
assert!(raw(&stream(length, ""), &none).is_err(), "/Length {length}");
}
}
#[test]
fn indirect_stream_with_a_space_before_the_line_break() {
// `stream \r\n` as Microsoft Reporting Services writes it: the data starts after the
// line break, so the /Length is still right and the data keeps no leading space.
for eol in [&b" \r\n"[..], b" \n", b"\t \r"] {
let mut data = b"7 0 obj\n<< /Length 5 >>\nstream".to_vec();
data.extend_from_slice(eol);
data.extend_from_slice(b"HELLO\r\nendstream\nendobj\n");
let (_, o) = parse_indirect(&data, 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b"HELLO", "{eol:?}");
}
// Spaces with no line break after them are data, not a keyword terminator.
let data = b"7 0 obj\n<< /Length 7 >>\nstream HELLO\nendstream\nendobj\n";
let (_, o) = parse_indirect(data, 0, &|_| None).unwrap();
let Object::Stream(s) = o else { panic!("stream expected") };
assert_eq!(&s.raw[..], b" HELLO");
}
#[test]
fn indirect_length_is_resolved() {
let data = b"7 0 obj\n<< /Length 8 0 R >>\nstream\r\nAB\r\nendstream\nendobj\n";
let (_, o) = parse_indirect(data, 0, &|r| (r.num == 8).then_some(2)).unwrap();
let Object::Stream(s) = o else { panic!() };
assert_eq!(&s.raw[..], b"AB");
}
#[test]
fn garbage_never_panics() {
for input in ["<<", "[", "(", "<", "<< /A", "[1 2", ")", "]", ">>", "<<>>>", "1 0", "\0\0", "/A#", "<zz>", "((("] {
let _ = Lexer::new(input.as_bytes(), 0).object();
}
let deep = "[".repeat(10_000);
assert!(Lexer::new(deep.as_bytes(), 0).object().is_err());
}
#[test]
fn one_object_has_a_node_budget() {
// A wide array (from a tiny compressed object stream) must not grow without bound.
let wide = |n: usize| format!("[{}]", "0 ".repeat(n));
let at_cap = Lexer::new(wide(MAX_NODES - 1).as_bytes(), 0).object();
assert!(matches!(at_cap, Ok(Object::Array(ref items)) if items.len() == MAX_NODES - 1));
assert!(Lexer::new(wide(MAX_NODES).as_bytes(), 0).object().is_err());
// The same inside a dictionary value and behind tolerated junk.
assert!(Lexer::new(format!("<< /A {} >>", wide(MAX_NODES)).as_bytes(), 0).object().is_err());
assert!(Lexer::new(format!("[[{}] ]", "0 ".repeat(MAX_NODES)).as_bytes(), 0).object().is_err());
// The budget is per top-level object: a lexer reused for many operands keeps working.
let two = wide(MAX_NODES - 1).repeat(2);
let mut lx = Lexer::new(two.as_bytes(), 0);
assert!(lx.object().is_ok() && lx.object().is_ok());
}
/// A plain real is read without a cleaned copy of its token; the forms that need one read
/// as they did (#5873).
#[test]
fn reals_read_the_same_with_and_without_a_doubled_sign() {
for (token, want) in [("1.5", 1.5), ("-375.993", -375.993), ("+2.25", 2.25), ("--5.5", -5.5), ("1.2.3", 1.2), ("-.5", -0.5), ("0.0", 0.0)] {
assert_eq!(Lexer::new(token.as_bytes(), 0).object().unwrap(), Object::Real(want), "{token}");
}
}
}
-376
View File
@@ -1,376 +0,0 @@
//! Bridging the COS object model and `pdfcraft-crypt`: reading `/Encrypt` dictionaries and
//! decrypting / encrypting the strings and streams of an object (§7.6.2).
//!
//! Not encrypted, per the spec: the `/Encrypt` dictionary itself, cross-reference streams, the
//! trailer `/ID`, objects inside object streams (the object stream is encrypted as a whole),
//! the `/Contents` of signature dictionaries, and metadata streams when `/EncryptMetadata` is
//! false. Streams naming a crypt filter (`/Crypt` in `/Filter`) use that filter; embedded files
//! use `/EFF`.
use std::cell::Cell;
use pdfcraft_crypt::{EncryptDict, Method, SecurityHandler, StreamKind};
use pdfcraft_filters::MAX_CHAIN;
use crate::document::Held;
use crate::{Dict, Document, Object, PdfString, Stream};
/// How many references one read or write of an object may follow, with all that it loads on the
/// way. How an object is treated can be given by reference, and what that loads can do the same: a
/// file could make one read a tree of them, each object read again once the cache has let it go.
/// An object takes a few of these, and two dozen if every part of it is given by reference; past
/// this they read as absent. The parameters of an object stream, read together, count as one.
const MAX_FOLLOWED: u32 = 32;
std::thread_local! {
/// How many [`FollowScope`]s this thread is inside.
static SCOPES: Cell<u32> = const { Cell::new(0) };
/// The references followed in the outermost of them.
static FOLLOWED: Cell<u32> = const { Cell::new(0) };
/// The references refused in the outermost of them, past [`MAX_FOLLOWED`].
static REFUSED: Cell<u32> = const { Cell::new(0) };
}
/// One read or write of an object, with everything it loads on the way: what [`MAX_FOLLOWED`] counts.
pub(crate) struct FollowScope;
impl FollowScope {
pub(crate) fn enter() -> Self {
if SCOPES.replace(SCOPES.get().saturating_add(1)) == 0 {
FOLLOWED.set(0);
REFUSED.set(0);
}
FollowScope
}
}
impl Drop for FollowScope {
fn drop(&mut self) {
SCOPES.set(SCOPES.get().saturating_sub(1));
}
}
/// What `f` gives, counted against [`MAX_FOLLOWED`], or `spent` past it. Called inside a
/// [`FollowScope`], which is what starts the count again.
pub(crate) fn following<T>(spent: T, f: impl FnOnce() -> T) -> T {
if FOLLOWED.get() >= MAX_FOLLOWED {
REFUSED.set(REFUSED.get().saturating_add(1));
return spent;
}
FOLLOWED.set(FOLLOWED.get() + 1);
f()
}
/// `o`, or the object it refers to, for what decides how a stream or dictionary is treated. Past
/// [`MAX_FOLLOWED`] a reference reads as absent.
fn followed<'o>(doc: &Document, o: &'o Object) -> Held<'o> {
match o {
Object::Ref(_) => following(Held::Direct(&Object::Null), || doc.held(o)),
o => Held::Direct(o),
}
}
fn bytes_of(doc: &Document, d: &Dict, key: &[u8]) -> Vec<u8> {
d.get(key).map(|v| doc.resolve(v)).and_then(|v| v.as_string().map(|s| s.bytes.clone())).unwrap_or_default()
}
/// Read the handler parameters from an `/Encrypt` dictionary.
pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict {
let int = |k: &[u8], default: i64| d.get(k).map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(default);
let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec));
let mut crypt_filters = Vec::new();
if let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) {
for (k, v) in cf.iter() {
let Some(f) = doc.resolve(v).as_dict().cloned() else { continue };
let method = match name(&f, b"CFM").as_deref() {
Some(b"V2") => Method::Rc4,
Some(b"AESV2") => Method::Aes128,
Some(b"AESV3") => Method::Aes256,
_ => Method::Identity,
};
crypt_filters.push((k.clone(), method));
}
}
let v = int(b"V", 0);
// V4: the key length comes from the standard crypt filter's /Length (bytes, or bits in
// some files); /Length on the dictionary is often a stale 40. A length that isn't
// positive means nothing and reads as if absent.
let cf_len = d
.get(b"CF")
.map(|c| doc.resolve(c))
.and_then(|c| c.as_dict().and_then(|c| c.get(b"StdCF").map(|f| doc.resolve(f))))
.and_then(|f| f.as_dict().and_then(|f| f.get(b"Length").map(|n| doc.resolve(n))))
.and_then(|n| n.as_int())
.filter(|n| *n > 0);
let length_bits = match (v, cf_len) {
(5.., _) => 256,
(4, Some(n)) if n <= 32 => n * 8,
(4, Some(n)) => n,
(4, None) => 128,
_ => int(b"Length", 40),
};
EncryptDict {
filter: name(d, b"Filter").unwrap_or_default(),
v,
r: int(b"R", 0),
length_bits,
o: bytes_of(doc, d, b"O"),
u: bytes_of(doc, d, b"U"),
oe: bytes_of(doc, d, b"OE"),
ue: bytes_of(doc, d, b"UE"),
perms: bytes_of(doc, d, b"Perms"),
p: int(b"P", -1) as i32,
encrypt_metadata: !matches!(d.get(b"EncryptMetadata").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(false))),
crypt_filters,
stm_f: name(d, b"StmF").unwrap_or_else(|| b"Identity".to_vec()),
str_f: name(d, b"StrF").unwrap_or_else(|| b"Identity".to_vec()),
ef_f: name(d, b"EFF").unwrap_or_default(),
}
}
/// Whether `d`, an `/Encrypt` dictionary, has crypt filters that encrypt (`/CFM` `V2`, `AESV2` or
/// `AESV3`), each authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO
/// 32000-2 §7.6.6, Table 25): what a document asks for its embedded files alone, which it opens
/// without (§7.6.6: "EFOpen Authorization shall be required when accessing embedded files"). The
/// files use them through `/EFF` or through a `/Crypt` filter of their own. Its strings and streams
/// have to be in the clear too, which [`SecurityHandler::without_key`] checks.
pub(crate) fn efopen_only(doc: &Document, d: &Dict) -> bool {
let name = |d: &Dict, k: &[u8]| d.get(k).and_then(|v| doc.resolve(v).as_name().map(<[u8]>::to_vec));
let Some(cf) = d.get(b"CF").map(|c| doc.resolve(c)).and_then(|c| c.as_dict().cloned()) else { return false };
let mut encrypting = Vec::new();
for (k, v) in cf.iter() {
let Some(f) = doc.resolve(v).as_dict().cloned() else { continue };
if !matches!(name(&f, b"CFM").as_deref(), Some(b"V2" | b"AESV2" | b"AESV3")) {
continue;
}
if name(&f, b"AuthEvent").as_deref() != Some(b"EFOpen") {
return false;
}
encrypting.push(k.clone());
}
!encrypting.is_empty()
}
/// Whether `s`, read from a document whose handler has no file key, holds data that handler would
/// decrypt, which therefore reads as stored: still encrypted.
pub(crate) fn stream_reads_as_stored(doc: &Document, h: &SecurityHandler, s: &Stream) -> bool {
if h.has_key() {
return false;
}
let _scope = FollowScope::enter();
let named = named_crypt_filter(doc, &s.dict);
let ty = s.dict.get(b"Type").map(|t| followed(doc, t));
let kind = match (&named, ty.as_deref().and_then(Object::as_name)) {
(Some(n), _) => StreamKind::Named(n),
(None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile,
_ => StreamKind::Normal,
};
h.decrypts(kind)
}
/// Whether `f`, an entry of a `/Filter` chain or its only filter, is `/Crypt`.
fn is_crypt(doc: &Document, f: &Object) -> bool {
followed(doc, f).as_name() == Some(b"Crypt")
}
/// The crypt filter a stream names in its `/Filter` chain (`/Crypt` + `/DecodeParms /Name`), each
/// given directly or by reference, in an array or not. A chain longer than any decoder takes names
/// none: nothing can read that stream anyway, and every stream of a file can share one long array.
/// A `/Name` that reads as absent means `/Identity`, unless a reference on the way to it was refused
/// (see [`MAX_FOLLOWED`]): that says nothing about the stream, so it names none, and the document's
/// own filter is used.
fn named_crypt_filter(doc: &Document, dict: &Dict) -> Option<Vec<u8>> {
let refused = REFUSED.get();
let filter = followed(doc, dict.get(b"Filter")?);
let i = match &*filter {
Object::Array(a) if a.len() <= MAX_CHAIN + 1 => a.iter().position(|f| is_crypt(doc, f))?,
f if is_crypt(doc, f) => 0,
_ => return None,
};
let parms = dict.get(b"DecodeParms").map(|p| followed(doc, p));
let entry = match parms.as_deref() {
Some(Object::Array(a)) => a.get(i).map(|p| followed(doc, p)),
Some(p) if i == 0 => Some(Held::Direct(p)),
_ => None,
};
let name = entry.as_deref().and_then(Object::as_dict).and_then(|p| p.get(b"Name")).map(|n| followed(doc, n));
match name.as_deref().and_then(Object::as_name) {
Some(name) => Some(name.to_vec()),
// /Identity here would read ciphertext as data, and write the stream in the clear.
None if REFUSED.get() > refused => None,
None => Some(b"Identity".to_vec()),
}
}
/// A signature dictionary (field signature, document timestamp or usage-rights signature),
/// whose `/Contents` is never encrypted (ISO 32000-2 §7.6.2, §12.8.1): the signer writes a
/// placeholder, hashes the file around it and patches the value in place, so encrypting it
/// would make signing impossible. Identified as other readers do: by `/Type`, or by a string
/// `/Contents` alongside an array `/ByteRange` (`/Type` is optional). A signature dictionary
/// has no `/Subtype`; a dictionary with one (an annotation, whose `/Contents` is the comment
/// text) or with another `/Type` is never exempt, whatever else it carries.
/// With `doc` its entries are read through references; a dictionary nested in another object
/// has none (see [`transform`]).
fn is_signature_dict(doc: Option<&Document>, d: &Dict) -> bool {
// All this decides is what happens to such a /Contents.
if !matches!(d.get(b"Contents"), Some(Object::String(_))) {
return false;
}
// A null entry, or one that refers to nothing, is an absent one (ISO 32000-2 §7.3.7, §7.3.10).
let entry = |key: &[u8]| {
let value = d.get(key)?;
let held = doc.map_or(Held::Direct(value), |doc| followed(doc, value));
(!matches!(*held, Object::Null)).then_some(held)
};
if entry(b"Subtype").is_some() {
return false;
}
match entry(b"Type") {
Some(t) => matches!(t.as_name(), Some(b"Sig" | b"DocTimeStamp")),
None => matches!(entry(b"ByteRange").as_deref(), Some(Object::Array(_))),
}
}
/// Longer than any filter's name. A name can be of any length, and a chain every stream shares can
/// hold a long one.
const MAX_FILTER_NAME: usize = 127;
/// The dictionary of `source` without the `/Crypt` entries of its filter chain and their decode
/// parameters. The two come out direct when given by reference, which every stream of a file can
/// share, so of what was shared only small values are copied: filter names, and the parameters
/// decoders read ([`Document::with_direct_parms`]). The chain is short, as [`named_crypt_filter`]
/// found a filter in it.
fn drop_crypt_filter(doc: &Document, source: &Stream) -> Dict {
let entry = |f: &Object| match f {
f if is_crypt(doc, f) => Object::name("Crypt"),
Object::Name(n) if n.len() <= MAX_FILTER_NAME => f.clone(),
Object::Ref(_) => f.clone(),
_ => Object::Null,
};
let filter = source.dict.get(b"Filter").map(|f| followed(doc, f));
let chain: Vec<Object> = match filter.as_deref() {
Some(Object::Array(a)) => a.iter().map(entry).collect(),
Some(f) => vec![entry(f)],
None => Vec::new(),
};
let crypt: Vec<bool> = chain.iter().map(|f| f.as_name() == Some(b"Crypt")).collect();
let mut chained = Stream::from_raw(source.dict.clone(), Vec::new());
chained.dict.set(b"Filter".to_vec(), Object::Array(chain.clone()));
// A lone parameter dictionary belongs to the first filter (as `named_crypt_filter` reads it).
let parms = match doc.with_direct_parms(&chained).dict.get(b"DecodeParms") {
Some(Object::Array(p)) => Some(Object::Array(p.iter().zip(&crypt).filter(|(_, c)| !**c).map(|(p, _)| p.clone()).collect())),
Some(Object::Dict(_)) if crypt.first() == Some(&true) => None,
_ => source.dict.get(b"DecodeParms").cloned(),
};
let mut dict = source.dict.clone();
match parms {
Some(p) => dict.set(b"DecodeParms".to_vec(), p),
None => {
dict.remove(b"DecodeParms");
}
}
let filters: Vec<Object> = chain.into_iter().zip(&crypt).filter(|(_, c)| !**c).map(|(f, _)| f).collect();
if filters.is_empty() {
dict.remove(b"Filter");
if matches!(dict.get(b"DecodeParms"), Some(Object::Array(p)) if p.is_empty()) {
dict.remove(b"DecodeParms");
}
} else {
dict.set(b"Filter".to_vec(), Object::Array(filters));
}
dict
}
/// What [`transform`] does to an object.
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Pass {
/// Decrypt, with the handler the document was opened with.
Decrypt,
/// Encrypt with the handler the document was opened with: streams keep their crypt filters.
Encrypt,
/// Encrypt with protection applied by `set_encryption`. A stream's crypt filter came from
/// the source's `/Encrypt`; unless the new one defines that name it is dropped, so the
/// stream follows the new policy instead of reading as `/Identity` and staying in clear.
EncryptReplacing,
}
/// Decrypt or encrypt every string and stream in an indirect object, except a signature
/// dictionary's `/Contents` (see [`is_signature_dict`]).
///
/// `o` is as `doc` holds it: what a stream's dictionary gives by reference is read from `doc`, so
/// its references have to be the document's, not a save's renumbered ones.
pub(crate) fn transform(doc: &Document, h: &SecurityHandler, o: &Object, num: u32, generation: u16, pass: Pass) -> Object {
let _scope = FollowScope::enter();
let decrypt = pass == Pass::Decrypt;
let strings = |s: &PdfString| {
let bytes = if decrypt { h.decrypt_string(num, generation, &s.bytes) } else { h.encrypt_string(num, generation, &s.bytes) };
PdfString { bytes, hex: s.hex || !decrypt }
};
// References are followed here only, for the object itself, never for what is nested in it.
// `walk` goes deep, and a load from down there adds that depth again for each object loaded,
// which overflowed the stack on a 10 KB file. A stream can't be nested, and a dictionary that
// is reads as it says directly.
match o {
Object::Stream(s) => stream(doc, s, &strings, h, num, generation, pass),
Object::Dict(d) if is_signature_dict(Some(doc), d) => signature_dict(d, &strings),
o => walk(o, &strings),
}
}
/// `d`, a signature dictionary, with the strings of everything but its `/Contents` transformed.
fn signature_dict(d: &Dict, strings: &dyn Fn(&PdfString) -> PdfString) -> Object {
Object::Dict(
d.iter()
.map(|(k, v)| match (k.as_slice(), v) {
(b"Contents", Object::String(_)) => (k.clone(), v.clone()),
_ => (k.clone(), walk(v, strings)),
})
.collect(),
)
}
fn walk(o: &Object, strings: &dyn Fn(&PdfString) -> PdfString) -> Object {
match o {
Object::String(s) => Object::String(strings(s)),
Object::Array(a) => Object::Array(a.iter().map(|x| walk(x, strings)).collect()),
Object::Dict(d) if is_signature_dict(None, d) => signature_dict(d, strings),
Object::Dict(d) => Object::Dict(d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect()),
other => other.clone(),
}
}
fn stream(
doc: &Document,
s: &Stream,
strings: &dyn Fn(&PdfString) -> PdfString,
h: &SecurityHandler,
num: u32,
generation: u16,
pass: Pass,
) -> Object {
let decrypt = pass == Pass::Decrypt;
let walked = |d: &Dict| -> Dict { d.iter().map(|(k, v)| (k.clone(), walk(v, strings))).collect() };
let ty = s.dict.get(b"Type").map(|t| followed(doc, t));
let ty = ty.as_deref().and_then(Object::as_name);
// Cross-reference streams are never encrypted (§7.5.8), but only reading may trust the
// label: a stream a producer calls /XRef stays plaintext if writing believes it, and the
// writer builds its own cross-reference streams outside this transform.
if (decrypt && ty == Some(b"XRef")) || (ty == Some(b"Metadata") && !h.encrypts_metadata()) {
return Object::Stream(Stream { dict: walked(&s.dict), raw: s.raw.clone() });
}
let mut named = named_crypt_filter(doc, &s.dict);
// Before the walk, which then encrypts what is in the dictionary this leaves.
let foreign = pass == Pass::EncryptReplacing && named.as_deref().is_some_and(|n| !h.dict().crypt_filters.iter().any(|(f, _)| f == n));
let dropped = foreign.then(|| drop_crypt_filter(doc, s));
if dropped.is_some() {
named = None;
}
let dict = walked(dropped.as_ref().unwrap_or(&s.dict));
let kind = match (&named, ty) {
(Some(n), _) => StreamKind::Named(n),
(None, Some(b"EmbeddedFile")) => StreamKind::EmbeddedFile,
_ => StreamKind::Normal,
};
let raw = if decrypt { h.decrypt_stream(num, generation, &s.raw, kind) } else { h.encrypt_stream(num, generation, &s.raw, kind) };
Object::Stream(Stream { dict, raw: raw.into() })
}
File diff suppressed because it is too large Load Diff
+1 -6
View File
@@ -11,12 +11,7 @@ repository.workspace = true
workspace = true
[dependencies]
aes = "0.9"
md-5 = "0.11"
sha2 = "0.11"
stringprep = "0.1"
thiserror.workspace = true
zeroize.workspace = true
craft-pdf = { workspace = true }
[dev-dependencies]
proptest.workspace = true
-89
View File
@@ -1,89 +0,0 @@
//! AES-CBC (128 or 256-bit key, chosen by key length) with optional PKCS#5 padding.
//!
//! Decryption is tolerant, as viewers are: a trailing partial block is ignored, and invalid
//! padding leaves the data unpadded rather than failing.
use aes::cipher::{Array, BlockCipherDecrypt, BlockCipherEncrypt, KeyInit};
use aes::{Aes128, Aes256};
enum Cipher {
A128(Box<Aes128>),
A256(Box<Aes256>),
}
impl Cipher {
fn new(key: &[u8]) -> Self {
if let Some(k) = key.first_chunk::<32>() {
Cipher::A256(Box::new(Aes256::new(&Array::from(*k))))
} else {
let mut k = [0u8; 16];
let n = key.len().min(16);
k[..n].copy_from_slice(&key[..n]);
Cipher::A128(Box::new(Aes128::new(&Array::from(k))))
}
}
fn encrypt(&self, block: &mut [u8; 16]) {
let mut b = Array::from(*block);
match self {
Cipher::A128(c) => c.encrypt_block(&mut b),
Cipher::A256(c) => c.encrypt_block(&mut b),
}
*block = b.into();
}
fn decrypt(&self, block: &mut [u8; 16]) {
let mut b = Array::from(*block);
match self {
Cipher::A128(c) => c.decrypt_block(&mut b),
Cipher::A256(c) => c.decrypt_block(&mut b),
}
*block = b.into();
}
}
/// Encrypt; with `pad`, PKCS#5 padding is added (otherwise `data` must be whole blocks, and a
/// trailing partial block is zero-padded).
pub fn aes_cbc_encrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec<u8> {
let c = Cipher::new(key);
let mut buf = data.to_vec();
if pad {
let n = 16 - buf.len() % 16;
buf.extend(std::iter::repeat_n(n as u8, n));
} else if !buf.len().is_multiple_of(16) {
buf.resize(buf.len().div_ceil(16) * 16, 0);
}
let mut prev = *iv;
for b in buf.as_chunks_mut::<16>().0 {
for (x, p) in b.iter_mut().zip(prev) {
*x ^= p;
}
c.encrypt(b);
prev = *b;
}
buf
}
/// Decrypt; with `pad`, PKCS#5 padding is removed when valid.
pub fn aes_cbc_decrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec<u8> {
let c = Cipher::new(key);
let blocks = data.as_chunks::<16>().0;
let mut out = Vec::with_capacity(blocks.len() * 16);
let mut prev = *iv;
for &cipher in blocks {
let mut b = cipher;
c.decrypt(&mut b);
for (x, p) in b.iter_mut().zip(prev) {
*x ^= p;
}
out.extend_from_slice(&b);
prev = cipher;
}
if pad && let Some(&n) = out.last() {
let n = n as usize;
if (1..=16).contains(&n) && n <= out.len() && out[out.len() - n..].iter().all(|b| *b as usize == n) {
out.truncate(out.len() - n);
}
}
out
}
+2 -797
View File
@@ -1,797 +1,2 @@
//! pdfcraft-crypt — the PDF standard security handler (ISO 32000-2 §7.6), standalone (L0).
//!
//! Supports every revision of the standard handler:
//! - R2 (40-bit RC4), R3 (RC4 up to 128-bit), R4 (crypt filters: RC4 or AES-128);
//! - R5 (the deprecated AES-256 extension) and R6 (AES-256, PDF 2.0).
//!
//! It authenticates with the user or owner password (owner first, as viewers do) and derives
//! per-object keys. It decrypts and encrypts strings and streams, and honours crypt filters,
//! including `Identity` and embedded-file-only encryption. It can also *create* encryption
//! dictionaries, which security settings (M8) and our own test fixtures use.
//!
//! The crate knows nothing about the COS object model. Callers pass in the values of the
//! `/Encrypt` dictionary and the first element of the trailer `/ID`.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod aes_cbc;
mod rc4;
use md5::{Digest as _, Md5};
use sha2::{Sha256, Sha384, Sha512};
use zeroize::Zeroizing;
pub use aes_cbc::{aes_cbc_decrypt, aes_cbc_encrypt};
pub use rc4::rc4;
#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)]
pub enum CryptError {
#[error("the password is incorrect")]
WrongPassword,
#[error("unsupported security handler: {0}")]
Unsupported(String),
#[error("malformed encryption dictionary: {0}")]
Malformed(String),
}
/// 32-byte padding string used by revisions 2–4 (§7.6.4.3.2, step a).
pub const PADDING: [u8; 32] = [
0x28, 0xBF, 0x4E, 0x5E, 0x4E, 0x75, 0x8A, 0x41, 0x64, 0x00, 0x4E, 0x56, 0xFF, 0xFA, 0x01, 0x08, 0x2E, 0x2E, 0x00, 0xB6, 0xD0, 0x68, 0x3E, 0x80,
0x2F, 0x0C, 0xA9, 0xFE, 0x64, 0x53, 0x69, 0x7A,
];
/// How a crypt filter encrypts (`/CFM`).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Method {
/// `None` / the `Identity` filter: data is not encrypted.
Identity,
/// `V2`: RC4.
Rc4,
/// `AESV2`: AES-128-CBC.
Aes128,
/// `AESV3`: AES-256-CBC.
Aes256,
}
/// The values of an `/Encrypt` dictionary that the handler needs.
#[derive(Clone, Debug, PartialEq, Default)]
pub struct EncryptDict {
/// `/Filter` (must be `Standard`).
pub filter: Vec<u8>,
pub v: i64,
pub r: i64,
/// Key length in bits (`/Length`; 40 when absent).
pub length_bits: i64,
pub o: Vec<u8>,
pub u: Vec<u8>,
pub oe: Vec<u8>,
pub ue: Vec<u8>,
pub perms: Vec<u8>,
pub p: i32,
pub encrypt_metadata: bool,
/// Crypt filters (`/CF`): name → (method, key length in bytes if given).
pub crypt_filters: Vec<(Vec<u8>, Method)>,
pub stm_f: Vec<u8>,
pub str_f: Vec<u8>,
pub ef_f: Vec<u8>,
}
/// Which password authenticated the document.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Auth {
Owner,
User,
}
/// Permission flags from `/P`, read the way the handler's revision reads them (ISO 32000-1:2008,
/// Table 22). Owners have every permission.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Permissions {
pub bits: i32,
pub owner: bool,
/// The `/R` value of the handler that read `bits`.
pub revision: i64,
}
impl Permissions {
fn bit(&self, n: u32) -> bool {
self.owner || (self.bits >> (n - 1)) & 1 == 1
}
/// Revision 3 and later read the full flag list of Table 22; revision 2 has its own.
fn is_revision_3_or_later(&self) -> bool {
self.revision >= 3
}
pub fn print(&self) -> bool {
self.bit(3)
}
/// Modify contents other than the operations below.
pub fn modify(&self) -> bool {
self.bit(4)
}
pub fn copy(&self) -> bool {
self.bit(5)
}
pub fn annotate(&self) -> bool {
self.bit(6)
}
pub fn fill_forms(&self) -> bool {
if self.is_revision_3_or_later() { self.bit(9) || self.bit(6) } else { self.bit(6) }
}
/// Revision 2 covers accessibility with the copy flag (bit 5); revision 3 and later with bit 10.
pub fn extract_for_accessibility(&self) -> bool {
if self.is_revision_3_or_later() { self.bit(10) } else { self.bit(5) }
}
/// Insert, rotate or delete pages; create bookmarks and thumbnails. Revision 3 and later grant
/// it by bit 11 alone. Table 22 defines bit 11 for those revisions only, so at revision 2 the
/// modify flag (bit 4) covers it.
pub fn assemble(&self) -> bool {
if self.is_revision_3_or_later() { self.bit(11) } else { self.bit(4) }
}
/// Take pages out into another document (Extract, Split, a source of Combine or Insert
/// Pages): [`Self::assemble`] and copying (bit 5), since the pages carry their text and
/// graphics into a new file without the document's protection (#5081). "Any except
/// extracting pages" sets bit 4 but not bit 11, so at revision 3 and later it grants neither.
pub fn extract_pages(&self) -> bool {
self.assemble() && self.copy()
}
/// Revision 3 and later need bit 3 as well as bit 12. Table 22 defines bit 12 for those
/// revisions only: at revision 2 the print flag (bit 3) is all there is.
pub fn print_high_quality(&self) -> bool {
if self.is_revision_3_or_later() { self.print() && self.bit(12) } else { self.print() }
}
}
/// What kind of stream is being decrypted (selects the crypt filter).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum StreamKind<'a> {
Normal,
/// An embedded file stream (uses `/EFF`).
EmbeddedFile,
/// The stream names its own crypt filter (`/Filter [/Crypt …] /DecodeParms << /Name … >>`).
Named(&'a [u8]),
}
/// An authenticated security handler for one document.
#[derive(Clone, Debug)]
pub struct SecurityHandler {
dict: EncryptDict,
key: Zeroizing<Vec<u8>>,
auth: Auth,
recovered_user: Option<Zeroizing<Vec<u8>>>,
methods: Methods,
}
/// The crypt filter methods of a handler, resolved once: a document controls both the number of
/// filters and the number of strings, so a search of `/CF` per string would multiply them.
#[derive(Clone, Debug)]
struct Methods {
string: Method,
stream: Method,
embedded_file: Method,
named: std::collections::HashMap<Vec<u8>, Method>,
}
impl Methods {
fn new(dict: &EncryptDict) -> Self {
let mut named = std::collections::HashMap::with_capacity(dict.crypt_filters.len());
for (name, method) in &dict.crypt_filters {
#[cfg(test)]
tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1));
// The first of two filters with one name wins, as a search would find it.
named.entry(name.clone()).or_insert(*method);
}
let mut methods = Methods { string: Method::Identity, stream: Method::Identity, embedded_file: Method::Identity, named };
methods.string = methods.get(&dict.str_f);
methods.stream = methods.get(&dict.stm_f);
methods.embedded_file = methods.get(if dict.ef_f.is_empty() { &dict.stm_f } else { &dict.ef_f });
methods
}
fn get(&self, name: &[u8]) -> Method {
if name == b"Identity" || name.is_empty() {
return Method::Identity;
}
#[cfg(test)]
tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1));
self.named.get(name).copied().unwrap_or(Method::Identity)
}
}
impl SecurityHandler {
fn new(dict: EncryptDict, key: Zeroizing<Vec<u8>>, auth: Auth, recovered_user: Option<Zeroizing<Vec<u8>>>) -> Self {
let methods = Methods::new(&dict);
Self { dict, key, auth, recovered_user, methods }
}
/// Authenticate with `password` (tried as owner password, then user password). `None`
/// tries the empty password, which opens documents that only have an owner password.
pub fn open(dict: EncryptDict, id0: &[u8], password: Option<&str>) -> Result<Self, CryptError> {
if !dict.filter.is_empty() && dict.filter != b"Standard" {
return Err(CryptError::Unsupported(format!("/Filter /{}", String::from_utf8_lossy(&dict.filter))));
}
let pw = password.unwrap_or("");
let mut recovered_user = None;
let (key, auth) = match dict.r {
2..=4 => {
let mut found = None;
for legacy in legacy_candidates(pw) {
if let Some((k, user)) = owner_key_r4(&dict, id0, &legacy) {
recovered_user = Some(user);
found = Some((k, Auth::Owner));
} else if let Some(k) = user_key_r4(&dict, id0, &legacy) {
found = Some((k, Auth::User));
}
if found.is_some() {
break;
}
}
found.ok_or(CryptError::WrongPassword)?
}
5 | 6 => {
let pw = sasl_password(pw);
if let Some(k) = key_r6(&dict, &pw, true)? {
(k, Auth::Owner)
} else if let Some(k) = key_r6(&dict, &pw, false)? {
(k, Auth::User)
} else {
return Err(CryptError::WrongPassword);
}
}
r => return Err(CryptError::Unsupported(format!("revision {r}"))),
};
Ok(Self::new(dict, key, auth, recovered_user))
}
/// A handler with no file key, for a document opened without a password whose strings and
/// streams aren't encrypted and whose encrypted crypt filters are all ones the caller found
/// authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO 32000-2 §7.6.6,
/// Table 25). Data such a filter encrypts is left as stored, read and written alike, so it
/// keeps its encrypted bytes; [`SecurityHandler::decrypts`] tells which. `None` when strings or
/// streams are encrypted, which needs the key, or the handler is older than crypt filters (V4).
pub fn without_key(dict: EncryptDict) -> Option<Self> {
if dict.v < 4 {
return None;
}
let methods = Methods::new(&dict);
if methods.string != Method::Identity || methods.stream != Method::Identity {
return None;
}
Some(Self { dict, key: Zeroizing::new(Vec::new()), auth: Auth::User, recovered_user: None, methods })
}
/// Whether the handler has the file key: false for [`SecurityHandler::without_key`].
pub fn has_key(&self) -> bool {
!self.key.is_empty()
}
/// Whether data of `kind` is encrypted, so that reading it needs the file key.
pub fn decrypts(&self, kind: StreamKind) -> bool {
self.method(Some(kind)) != Method::Identity
}
/// Whether data left as stored by a handler without the key still decrypts after its object
/// is renumbered: only AES-256 doesn't mix the object number into the key (Algorithm 1.A).
pub fn survives_renumbering(&self) -> bool {
[self.methods.string, self.methods.stream, self.methods.embedded_file]
.into_iter()
.chain(self.methods.named.values().copied())
.all(|m| matches!(m, Method::Identity | Method::Aes256))
}
/// Upgrade to owner access with the owner (permissions) password, as when a restricted
/// document is unlocked after opening. The user password, or any other, is `WrongPassword`.
/// Both passwords unlock the same file key, so what was decrypted so far stays valid; a
/// different key is refused too.
pub fn authenticate_owner(&self, id0: &[u8], password: &str) -> Result<Self, CryptError> {
let owner = Self::open(self.dict.clone(), id0, Some(password))?;
if owner.auth != Auth::Owner || owner.key != self.key {
return Err(CryptError::WrongPassword);
}
Ok(owner)
}
pub fn auth(&self) -> Auth {
self.auth
}
/// For R2–R4 documents opened with the owner password: the user password, which owner
/// authentication recovers (Algorithm 7). Lets components that only accept user passwords
/// open the document too.
pub fn recovered_user_password(&self) -> Option<Zeroizing<Vec<u8>>> {
self.recovered_user.clone()
}
/// For R5/R6, `/P` is stored in clear and again in `/Perms`, encrypted with the file key
/// (Algorithm 13). Where the two differ, a permission counts only when both grant it, so an
/// edited `/P` grants nothing extra. Without a readable `/Perms`, `/P` applies as before.
pub fn permissions(&self) -> Permissions {
let bits = self.perms_p().map_or(self.dict.p, |p| self.dict.p & p);
Permissions { bits, owner: self.auth == Auth::Owner, revision: self.dict.r }
}
/// `/P` as `/Perms` holds it: AES-256 ECB of one block (= CBC with a zero IV), marked `adb`.
fn perms_p(&self) -> Option<i32> {
if !matches!(self.dict.r, 5 | 6) || !self.has_key() {
return None;
}
let plain = Zeroizing::new(aes_cbc_decrypt(&self.key, &[0; 16], self.dict.perms.first_chunk::<16>()?, false));
let p = i32::from_le_bytes(*plain.first_chunk::<4>()?);
(plain.get(9..12) == Some(&b"adb"[..])).then_some(p)
}
pub fn dict(&self) -> &EncryptDict {
&self.dict
}
/// The file encryption key (for tests and diagnostics).
pub fn file_key(&self) -> &[u8] {
&self.key
}
/// Whether `/Metadata` streams are encrypted.
pub fn encrypts_metadata(&self) -> bool {
self.dict.encrypt_metadata
}
/// The method for strings or a kind of stream.
fn method(&self, stream: Option<StreamKind>) -> Method {
if self.dict.v < 4 {
return Method::Rc4;
}
match stream {
None => self.methods.string,
Some(StreamKind::Normal) => self.methods.stream,
Some(StreamKind::EmbeddedFile) => self.methods.embedded_file,
Some(StreamKind::Named(n)) => self.methods.get(n),
}
}
/// Algorithm 1 (§7.6.3.2): the key for one object. AES-256 uses the file key directly
/// (Algorithm 1.A, §7.6.3.3).
///
/// For an RC4 crypt filter under `/V 4`, a file key shorter than 16 bytes is first extended
/// to 16 bytes with zeros. The standard doesn't say so (Algorithm 1 takes the n-byte key as
/// it is), but that is how Acrobat writes and reads such files: a `/V 4` RC4 crypt filter
/// with a 40- or 48-bit key (pdf.js issue 19484). The renderer's `hayro-syntax` reads them
/// the same way, so a page it shows is one this crate can read and save. AES-128 keys are
/// left as they are: `AESV2` takes a 128-bit key (Table 25), and Acrobat's extension is only
/// seen with RC4. Every key PdfCraft writes under `/V 4` is 16 bytes already.
fn object_key(&self, method: Method, num: u32, generation: u16) -> Zeroizing<Vec<u8>> {
if method == Method::Aes256 {
return self.key.clone();
}
let mut key = self.key.clone();
if method == Method::Rc4 && self.dict.v == 4 && key.len() < 16 {
key.resize(16, 0);
}
let mut h = Md5::new();
h.update(&key);
h.update(&num.to_le_bytes()[..3]);
h.update(generation.to_le_bytes());
if method == Method::Aes128 {
h.update(b"sAlT");
}
let mut digest = Zeroizing::new(h.finalize().to_vec());
digest.truncate((key.len() + 5).min(16));
digest
}
fn apply(&self, method: Method, num: u32, generation: u16, data: &[u8], encrypt: bool) -> Vec<u8> {
// Without the key, encrypted data stays as stored (see `without_key`).
if !self.has_key() {
return data.to_vec();
}
match method {
Method::Identity => data.to_vec(),
Method::Rc4 => rc4(&self.object_key(method, num, generation), data),
Method::Aes128 | Method::Aes256 => {
let key = self.object_key(method, num, generation);
if encrypt {
// Deterministic IV derived from the key, object and content: saves are
// reproducible and no IV repeats for different plaintexts.
let mut h = Sha256::new();
h.update(&key);
h.update(num.to_le_bytes());
h.update(generation.to_le_bytes());
h.update(data);
let digest = h.finalize();
let mut iv = [0u8; 16];
for (d, s) in iv.iter_mut().zip(digest.iter()) {
*d = *s;
}
let mut out = iv.to_vec();
out.extend(aes_cbc_encrypt(&key, &iv, data, true));
out
} else {
let Some((iv, body)) = data.split_first_chunk::<16>() else {
return Vec::new(); // only (part of) an IV: empty plaintext
};
aes_cbc_decrypt(&key, iv, body, true)
}
}
}
}
pub fn decrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec<u8> {
self.apply(self.method(None), num, generation, data, false)
}
pub fn encrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec<u8> {
self.apply(self.method(None), num, generation, data, true)
}
pub fn decrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec<u8> {
self.apply(self.method(Some(kind)), num, generation, data, false)
}
pub fn encrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec<u8> {
self.apply(self.method(Some(kind)), num, generation, data, true)
}
/// Whether strings are encrypted at all (false with `/StrF /Identity`).
pub fn encrypts_strings(&self) -> bool {
self.method(None) != Method::Identity
}
}
// ── Passwords ──────────────────────────────────────────────────────────────────────────────────
/// PDFDocEncoding (ISO 32000-2 Annex D) where it differs from Latin-1: bytes 0x18–0x1F and
/// 0x80–0xA0 (0x9F is undefined). Here rather than in pdfcraft-cos, which decodes text strings
/// with it, because R2–R4 passwords need it too and cos depends on this crate.
const PDFDOC_LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜'];
const PDFDOC_HIGH: [char; 33] = [
'•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł', 'œ',
'š', 'ž', '\u{FFFD}', '€',
];
/// PDFDocEncoding → Unicode.
pub fn pdfdoc_char(c: u8) -> char {
match c {
0x18..=0x1F => PDFDOC_LOW[usize::from(c - 0x18)],
0x80..=0xA0 => PDFDOC_HIGH[usize::from(c - 0x80)],
_ => char::from(c),
}
}
/// Unicode → PDFDocEncoding, `None` for a character it lacks (ğ, ş, İ, Cyrillic, CJK, …).
pub fn pdfdoc_byte(c: char) -> Option<u8> {
let at = |table: &[char], base: u8| table.iter().position(|&t| t == c && t != '\u{FFFD}').and_then(|i| u8::try_from(i).ok()).map(|i| base + i);
at(&PDFDOC_LOW, 0x18).or_else(|| at(&PDFDOC_HIGH, 0x80)).or_else(|| match u32::from(c) {
0x18..=0x1F | 0x80..=0xA0 => None,
n => u8::try_from(n).ok(),
})
}
/// Revisions 2–4: the password in PDFDocEncoding (§7.6.4.3.2, Algorithm 2), at most 32 bytes.
/// Characters PDFDocEncoding lacks are dropped.
fn legacy_password(pw: &str) -> Zeroizing<Vec<u8>> {
Zeroizing::new(pw.chars().filter_map(pdfdoc_byte).take(32).collect())
}
/// The bytes an R2–R4 password may have been written as: PDFDocEncoding, as the specification
/// asks; and for a password with characters PDFDocEncoding lacks (ğ, ş, İ, …), which it gives
/// no bytes, also its UTF-8 bytes, which qpdf and other tools write for such passwords.
fn legacy_candidates(pw: &str) -> Vec<Zeroizing<Vec<u8>>> {
let mut out = vec![legacy_password(pw)];
if pw.chars().any(|c| pdfdoc_byte(c).is_none()) {
out.push(Zeroizing::new(pw.bytes().take(32).collect()));
}
out
}
/// Revisions 5–6: SASLprep (RFC 4013), UTF-8, at most 127 bytes.
fn sasl_password(pw: &str) -> Zeroizing<Vec<u8>> {
let prepped = Zeroizing::new(stringprep::saslprep(pw).map(|c| c.into_owned()).unwrap_or_else(|_| pw.to_string()));
let mut bytes = Zeroizing::new(prepped.as_bytes().to_vec());
bytes.truncate(127);
bytes
}
fn pad(pw: &[u8]) -> Zeroizing<[u8; 32]> {
let mut out = Zeroizing::new(PADDING);
let n = pw.len().min(32);
out[..n].copy_from_slice(&pw[..n]);
out[n..].copy_from_slice(&PADDING[..32 - n]);
out
}
fn key_len(d: &EncryptDict) -> usize {
if d.r == 2 { 5 } else { (d.length_bits.clamp(40, 128) / 8) as usize }
}
// ── Revisions 2–4 ──────────────────────────────────────────────────────────────────────────────
/// Algorithm 2: file key from a (user) password.
fn file_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Zeroizing<Vec<u8>> {
let n = key_len(d);
let mut h = Md5::new();
h.update(*pad(pw));
h.update(&d.o[..d.o.len().min(32)]);
h.update(d.p.to_le_bytes());
h.update(id0);
if d.r >= 4 && !d.encrypt_metadata {
h.update([0xFF; 4]);
}
let mut digest = Zeroizing::new(h.finalize().to_vec());
if d.r >= 3 {
for _ in 0..50 {
digest = Zeroizing::new(Md5::digest(&digest[..n]).to_vec());
}
}
digest.truncate(n);
digest
}
/// Algorithms 4/5: the `/U` value a key produces.
fn u_value_r4(d: &EncryptDict, id0: &[u8], key: &[u8]) -> Vec<u8> {
if d.r == 2 {
return rc4(key, &PADDING);
}
let mut h = Md5::new();
h.update(PADDING);
h.update(id0);
let mut x = rc4(key, &h.finalize());
for i in 1..=19u8 {
let k = Zeroizing::new(key.iter().map(|b| b ^ i).collect::<Vec<u8>>());
x = rc4(&k, &x);
}
x.extend_from_slice(&[0u8; 16]);
x
}
/// Algorithm 6: authenticate a user password; returns the file key.
///
/// Some producers ignore `/EncryptMetadata false` when deriving the key (they omit the
/// `FF FF FF FF` of Algorithm 2, step f). Viewers open those files, so if the strict derivation
/// fails we retry the other way.
fn user_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option<Zeroizing<Vec<u8>>> {
let check = |d: &EncryptDict| {
let key = file_key_r4(d, id0, pw);
let u = u_value_r4(d, id0, &key);
let n = if d.r == 2 { 32 } else { 16 };
(d.u.len() >= n && u[..n] == d.u[..n]).then_some(key)
};
check(d).or_else(|| (d.r >= 4 && !d.encrypt_metadata).then(|| check(&EncryptDict { encrypt_metadata: true, ..d.clone() })).flatten())
}
/// The RC4 key derived from an owner password (Algorithm 3, steps a–d).
fn owner_rc4_key(d: &EncryptDict, owner_pw: &[u8]) -> Zeroizing<Vec<u8>> {
let n = key_len(d);
let mut digest = Zeroizing::new(Md5::digest(*pad(owner_pw)).to_vec());
if d.r >= 3 {
for _ in 0..50 {
digest = Zeroizing::new(Md5::digest(&*digest).to_vec());
}
}
digest.truncate(n);
digest
}
type SecretBytes = Zeroizing<Vec<u8>>;
/// Algorithm 7: authenticate an owner password (recover the user password from `/O`).
/// Returns the file key and the recovered user password (padding removed).
fn owner_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option<(SecretBytes, SecretBytes)> {
let rk = owner_rc4_key(d, pw);
let mut user = Zeroizing::new(d.o[..d.o.len().min(32)].to_vec());
if d.r == 2 {
user = Zeroizing::new(rc4(&rk, &user));
} else {
for i in (0..=19u8).rev() {
let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::<Vec<u8>>());
user = Zeroizing::new(rc4(&k, &user));
}
}
let key = user_key_r4(d, id0, &user)?;
// The recovered value is the padded password: strip the padding suffix.
let len = (0..=32).find(|&i| user.len() >= 32 && user[i..32] == PADDING[..32 - i]).unwrap_or(user.len().min(32));
user.truncate(len);
Some((key, user))
}
// ── Revisions 5–6 ──────────────────────────────────────────────────────────────────────────────
/// The hash of a password for R5 (SHA-256) or R6 (Algorithm 2.B).
fn hash_r6(r: i64, pw: &[u8], salt: &[u8], udata: &[u8]) -> Zeroizing<Vec<u8>> {
let mut k = Zeroizing::new({
let mut h = Sha256::new();
h.update(pw);
h.update(salt);
h.update(udata);
h.finalize().to_vec()
});
if r == 5 {
return k;
}
let mut round = 0u32;
loop {
let mut k1 = Zeroizing::new(Vec::with_capacity(64 * (pw.len() + k.len() + udata.len())));
for _ in 0..64 {
k1.extend_from_slice(pw);
k1.extend_from_slice(&k);
k1.extend_from_slice(udata);
}
// `k` is always a SHA-2 digest (32, 48 or 64 bytes): key = its first 16 bytes, IV = the next 16.
let Some((key, rest)) = k.split_first_chunk::<16>() else { break };
let Some(iv) = rest.first_chunk::<16>() else { break };
let e = Zeroizing::new(aes_cbc_encrypt(key, iv, &k1, false));
let m = e.iter().take(16).map(|b| u32::from(*b)).sum::<u32>() % 3;
k = Zeroizing::new(match m {
0 => Sha256::digest(&*e).to_vec(),
1 => Sha384::digest(&*e).to_vec(),
_ => Sha512::digest(&*e).to_vec(),
});
round += 1;
if round >= 64 && e.last().is_none_or(|b| u32::from(*b) <= round - 32) {
break;
}
}
k.truncate(32);
k
}
/// Algorithm 2.A: authenticate as owner (`owner = true`) or user; returns the file key.
fn key_r6(d: &EncryptDict, pw: &[u8], owner: bool) -> Result<Option<Zeroizing<Vec<u8>>>, CryptError> {
if d.u.len() < 48 || d.o.len() < 48 {
return Err(CryptError::Malformed("/O and /U must be 48 bytes for R5/R6".into()));
}
let (hashed, encrypted, udata) = if owner { (&d.o[..48], &d.oe, &d.u[..48]) } else { (&d.u[..48], &d.ue, &[][..]) };
if *hash_r6(d.r, pw, &hashed[32..40], udata) != hashed[..32] {
return Ok(None);
}
if encrypted.len() < 32 {
return Err(CryptError::Malformed("/OE or /UE must be 32 bytes".into()));
}
let k = hash_r6(d.r, pw, &hashed[40..48], udata);
Ok(Some(Zeroizing::new(aes_cbc_decrypt(&k, &[0; 16], &encrypted[..32], false))))
}
// ── Creating encryption ────────────────────────────────────────────────────────────────────────
/// Which encryption to create.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Algorithm {
/// R2, 40-bit RC4 (legacy; for tests and compatibility only).
Rc4_40,
/// R3, 128-bit RC4.
Rc4_128,
/// R4, AES-128 crypt filters.
Aes128,
/// R6, AES-256 (PDF 2.0; the default for new documents).
Aes256,
}
/// A password as [`create`] prepares it: for R2–R4 its PDFDocEncoding (characters it lacks
/// dropped), at most 32 bytes; for R6 its SASLprep (RFC 4013: soft hyphens and the like removed,
/// other spaces made plain, then NFKC; the password as given if SASLprep refuses it), in UTF-8,
/// at most 127 bytes.
fn prepared_password(algorithm: Algorithm, password: &str) -> Zeroizing<Vec<u8>> {
match algorithm {
Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => legacy_password(password),
Algorithm::Aes256 => sasl_password(password),
}
}
/// The bytes new encryption with `algorithm` derives its keys from for `password`: for R2–R4 its
/// PDFDocEncoding cut or padded to exactly 32 bytes (Algorithm 2); for R6 its SASLprep (the
/// password as given if SASLprep refuses it), at most 127 bytes of UTF-8. Two passwords with the
/// same bytes are the same password to the file, and a password with the bytes of `""` opens it
/// without asking, so callers can refuse either before encrypting. Different bytes can still give
/// the same key (RC4-40 keeps 5 bytes of a hash). `create` still uses the user password for an
/// empty R2–R4 owner password.
pub fn effective_password(algorithm: Algorithm, password: &str) -> Vec<u8> {
let prepared = prepared_password(algorithm, password);
match algorithm {
Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => pad(&prepared).to_vec(),
Algorithm::Aes256 => prepared.to_vec(),
}
}
/// Parameters for new encryption.
#[derive(Clone, Debug)]
pub struct NewEncryption<'a> {
pub algorithm: Algorithm,
pub user_password: &'a str,
pub owner_password: &'a str,
pub permissions: i32,
pub encrypt_metadata: bool,
/// 32 bytes of entropy for keys and salts (the caller supplies randomness, so this crate
/// needs no OS access and tests are deterministic).
pub seed: [u8; 32],
}
/// Expand `seed` into `n` bytes (SHA-256 in counter mode).
fn expand(seed: &[u8; 32], label: &[u8], n: usize) -> Vec<u8> {
let mut out = Vec::new();
let mut counter = 0u32;
while out.len() < n {
let mut h = Sha256::new();
h.update(seed);
h.update(label);
h.update(counter.to_le_bytes());
out.extend_from_slice(&h.finalize());
counter = counter.saturating_add(1);
}
out.truncate(n);
out
}
/// Sixteen bytes of file identifier from `seed` (SHA-256 in counter mode, label `id`).
///
/// `/ID` is written in the clear. Deriving it the same way as the file key keeps that public
/// value from being an unkeyed hash of the seed.
pub fn file_id(seed: &[u8; 32]) -> Vec<u8> {
expand(seed, b"id", 16)
}
/// Build the `/Encrypt` values and an authenticated (owner) handler for a new encryption.
pub fn create(params: &NewEncryption, id0: &[u8]) -> Result<SecurityHandler, CryptError> {
// Bits 7–8 and 13–32 must be 1; bits 1–2 must be 0 (§7.6.4.2).
let p = (params.permissions | !0xFFF | 0xC0) & !0x3;
let mut d = EncryptDict { filter: b"Standard".to_vec(), p, encrypt_metadata: params.encrypt_metadata, ..Default::default() };
let user = prepared_password(params.algorithm, params.user_password);
let owner = prepared_password(params.algorithm, params.owner_password);
match params.algorithm {
Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => {
(d.v, d.r, d.length_bits) = match params.algorithm {
Algorithm::Rc4_40 => (1, 2, 40),
Algorithm::Rc4_128 => (2, 3, 128),
_ => (4, 4, 128),
};
if params.algorithm == Algorithm::Aes128 {
d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)];
d.stm_f = b"StdCF".to_vec();
d.str_f = b"StdCF".to_vec();
} else {
d.encrypt_metadata = true; // V < 4 always encrypts metadata
}
let owner = if params.owner_password.is_empty() { user.clone() } else { owner };
// Algorithm 3: /O
let rk = owner_rc4_key(&d, &owner);
let mut o = rc4(&rk, &*pad(&user));
if d.r >= 3 {
for i in 1..=19u8 {
let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::<Vec<u8>>());
o = rc4(&k, &o);
}
}
d.o = o;
let key = file_key_r4(&d, id0, &user);
d.u = u_value_r4(&d, id0, &key);
Ok(SecurityHandler::new(d, key, Auth::Owner, Some(user)))
}
Algorithm::Aes256 => {
(d.v, d.r, d.length_bits) = (5, 6, 256);
d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes256)];
d.stm_f = b"StdCF".to_vec();
d.str_f = b"StdCF".to_vec();
let key = Zeroizing::new(expand(&params.seed, b"file key", 32));
let salts = expand(&params.seed, b"salts", 32);
// Algorithm 8: /U, /UE
let mut u = hash_r6(6, &user, &salts[0..8], &[]).to_vec();
u.extend_from_slice(&salts[0..16]);
let uk = hash_r6(6, &user, &salts[8..16], &[]);
d.ue = aes_cbc_encrypt(&uk, &[0; 16], &key, false);
// Algorithm 9: /O, /OE
let mut o = hash_r6(6, &owner, &salts[16..24], &u).to_vec();
o.extend_from_slice(&salts[16..32]);
let ok = hash_r6(6, &owner, &salts[24..32], &u);
d.oe = aes_cbc_encrypt(&ok, &[0; 16], &key, false);
d.u = u;
d.o = o;
// Algorithm 10: /Perms
let mut perms = [0u8; 16];
perms[..4].copy_from_slice(&p.to_le_bytes());
perms[4..8].copy_from_slice(&[0xFF; 4]);
perms[8] = if d.encrypt_metadata { b'T' } else { b'F' };
perms[9..12].copy_from_slice(b"adb");
perms[12..16].copy_from_slice(&expand(&params.seed, b"perms", 4));
d.perms = aes_cbc_encrypt(&key, &[0; 16], &perms, false);
Ok(SecurityHandler::new(d, key, Auth::Owner, None))
}
}
}
#[cfg(test)]
mod tests;
//! PDF standard security handler (`craft-pdf::crypt`).
pub use craft_pdf::crypt::*;
-22
View File
@@ -1,22 +0,0 @@
//! RC4 (used by PDF security handler revisions 2–4). Encryption and decryption are the same.
pub fn rc4(key: &[u8], data: &[u8]) -> Vec<u8> {
if key.is_empty() {
return data.to_vec();
}
let mut s: [u8; 256] = std::array::from_fn(|i| i as u8);
let mut j: u8 = 0;
for i in 0..256 {
j = j.wrapping_add(s[i]).wrapping_add(key[i % key.len()]);
s.swap(i, j as usize);
}
let (mut i, mut j) = (0u8, 0u8);
data.iter()
.map(|b| {
i = i.wrapping_add(1);
j = j.wrapping_add(s[i as usize]);
s.swap(i as usize, j as usize);
b ^ s[s[i as usize].wrapping_add(s[j as usize]) as usize]
})
.collect()
}
-447
View File
@@ -1,447 +0,0 @@
use super::*;
fn hex(s: &str) -> Vec<u8> {
(0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect()
}
#[test]
fn file_id_is_a_one_way_16_byte_identifier() {
let seed = [7u8; 32];
let id = file_id(&seed);
assert_eq!(id.len(), 16);
assert_eq!(id, file_id(&seed));
assert_ne!(id, seed[..16]);
assert_ne!(id, file_id(&[8u8; 32]));
}
#[test]
fn rc4_reference_vectors() {
assert_eq!(rc4(b"Key", b"Plaintext"), hex("bbf316e8d940af0ad3"));
assert_eq!(rc4(b"Wiki", b"pedia"), hex("1021bf0420"));
assert_eq!(rc4(b"Secret", b"Attack at dawn"), hex("45a01f645fc35b383552544b9bf5"));
}
#[test]
fn aes_cbc_nist_sp800_38a_vectors() {
let iv: [u8; 16] = hex("000102030405060708090a0b0c0d0e0f").try_into().unwrap();
let plain = hex("6bc1bee22e409f96e93d7e117393172aae2d8a571e03ac9c9eb76fac45af8e51");
let k128 = hex("2b7e151628aed2a6abf7158809cf4f3c");
let c128 = aes_cbc_encrypt(&k128, &iv, &plain, false);
assert_eq!(c128, hex("7649abac8119b246cee98e9b12e9197d5086cb9b507219ee95db113a917678b2"));
assert_eq!(aes_cbc_decrypt(&k128, &iv, &c128, false), plain);
let k256 = hex("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4");
let c256 = aes_cbc_encrypt(&k256, &iv, &plain, false);
assert_eq!(c256, hex("f58c4c04d6e5f1ba779eabfb5f7bfbd69cfc4e967edb808d679f777bc6702c7d"));
assert_eq!(aes_cbc_decrypt(&k256, &iv, &c256, false), plain);
}
#[test]
fn aes_padding_is_added_and_tolerantly_removed() {
let key = [7u8; 16];
let iv = [1u8; 16];
for n in [0, 1, 15, 16, 17, 100] {
let data: Vec<u8> = (0..n as u8).collect();
let c = aes_cbc_encrypt(&key, &iv, &data, true);
assert_eq!(c.len() % 16, 0);
assert_eq!(aes_cbc_decrypt(&key, &iv, &c, true), data);
}
// Garbage (bad padding, partial block) decrypts without panicking.
let _ = aes_cbc_decrypt(&key, &iv, &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], true);
}
const ID: &[u8] = b"0123456789abcdef";
fn new(alg: Algorithm, user: &str, owner: &str, perms: i32) -> SecurityHandler {
create(
&NewEncryption { algorithm: alg, user_password: user, owner_password: owner, permissions: perms, encrypt_metadata: true, seed: [42; 32] },
ID,
)
.unwrap()
}
const ALL: [Algorithm; 4] = [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256];
#[test]
fn created_encryption_authenticates_owner_and_user() {
for alg in ALL {
let made = new(alg, "user pw", "owner pw", 0b0100); // print only
let d = made.dict().clone();
let owner = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap();
assert_eq!(owner.auth(), Auth::Owner, "{alg:?}");
assert_eq!(owner.file_key(), made.file_key(), "{alg:?}");
assert!(owner.permissions().modify(), "owners can do everything");
let user = SecurityHandler::open(d.clone(), ID, Some("user pw")).unwrap();
assert_eq!(user.auth(), Auth::User, "{alg:?}");
assert_eq!(user.file_key(), made.file_key(), "{alg:?}");
assert!(user.permissions().print() && !user.permissions().modify() && !user.permissions().copy(), "{alg:?}");
assert_eq!(SecurityHandler::open(d.clone(), ID, Some("nope")).unwrap_err(), CryptError::WrongPassword, "{alg:?}");
assert_eq!(SecurityHandler::open(d, ID, None).unwrap_err(), CryptError::WrongPassword, "{alg:?}");
}
}
#[test]
fn the_owner_password_upgrades_a_user_session() {
for alg in ALL {
let d = new(alg, "", "secret", 0b0100).dict().clone();
let user = SecurityHandler::open(d, ID, None).unwrap();
assert!(!user.permissions().modify(), "{alg:?}");
// The (empty) user password and wrong ones don't unlock anything.
assert_eq!(user.authenticate_owner(ID, "").unwrap_err(), CryptError::WrongPassword, "{alg:?}");
assert_eq!(user.authenticate_owner(ID, "nope").unwrap_err(), CryptError::WrongPassword, "{alg:?}");
let owner = user.authenticate_owner(ID, "secret").unwrap();
assert_eq!(owner.auth(), Auth::Owner, "{alg:?}");
assert_eq!(owner.file_key(), user.file_key(), "{alg:?}");
assert!(owner.permissions().modify(), "{alg:?}");
}
// A separate open password is not the permissions password.
for alg in ALL {
let d = new(alg, "open", "perms", 0b0100).dict().clone();
let user = SecurityHandler::open(d, ID, Some("open")).unwrap();
assert_eq!(user.authenticate_owner(ID, "open").unwrap_err(), CryptError::WrongPassword, "{alg:?}");
assert_eq!(user.authenticate_owner(ID, "perms").unwrap().auth(), Auth::Owner, "{alg:?}");
}
}
#[test]
fn empty_user_password_opens_without_asking() {
for alg in ALL {
let d = new(alg, "", "secret", -1).dict().clone();
let h = SecurityHandler::open(d, ID, None).unwrap();
assert_eq!(h.auth(), Auth::User, "{alg:?}");
}
}
#[test]
fn strings_and_streams_round_trip_and_are_scrambled() {
for alg in ALL {
let h = new(alg, "", "o", -1);
let text = b"Confidential quarterly numbers".to_vec();
let enc = h.encrypt_string(12, 0, &text);
assert_ne!(enc, text, "{alg:?}");
assert_eq!(h.decrypt_string(12, 0, &enc), text, "{alg:?}");
if alg != Algorithm::Aes256 {
// R2–R4 derive a key per object; R6 uses the file key everywhere (§7.6.3.3).
assert_ne!(h.decrypt_string(13, 0, &enc), text, "object keys differ per object ({alg:?})");
}
let data: Vec<u8> = (0..1000u32).map(|i| (i * 7) as u8).collect();
let enc = h.encrypt_stream(5, 0, &data, StreamKind::Normal);
assert_eq!(h.decrypt_stream(5, 0, &enc, StreamKind::Normal), data, "{alg:?}");
// Deterministic: the same object encrypts to the same bytes (reproducible saves).
assert_eq!(h.encrypt_stream(5, 0, &data, StreamKind::Normal), enc);
}
}
thread_local! {
/// Crypt filter names compared or looked up, on this thread.
pub(crate) static FILTER_COMPARES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[test]
fn crypt_filters_are_looked_up_once_per_handler() {
let plain = new(Algorithm::Aes128, "", "o", -1);
let mut d = plain.dict().clone();
let n = 5_000;
let mut filters: Vec<_> = (0..n).map(|i| (format!("F{i}").into_bytes(), Method::Identity)).collect();
filters.append(&mut d.crypt_filters);
d.crypt_filters = filters;
FILTER_COMPARES.with(|c| c.set(0));
let h = SecurityHandler::open(d, ID, None).unwrap();
let strings = 1_000;
let enc = plain.encrypt_string(7, 0, b"secret");
for _ in 0..strings {
assert_eq!(h.decrypt_string(7, 0, &enc), b"secret");
assert_eq!(h.decrypt_stream(7, 0, &plain.encrypt_stream(7, 0, b"s", StreamKind::Normal), StreamKind::Normal), b"s");
}
assert_eq!(h.decrypt_stream(7, 0, b"x", StreamKind::Named(b"F3")), b"x");
let compares = FILTER_COMPARES.with(|c| c.get());
assert!(compares <= 2 * n + 3 * strings, "{compares} filter name comparisons");
}
#[test]
fn identity_crypt_filters_leave_data_alone() {
let mut d = new(Algorithm::Aes256, "", "o", -1).dict().clone();
// Embedded-file-only encryption: strings and normal streams are not encrypted.
d.str_f = b"Identity".to_vec();
d.stm_f = b"Identity".to_vec();
d.ef_f = b"StdCF".to_vec();
let h = SecurityHandler::open(d, ID, None).unwrap();
assert!(!h.encrypts_strings());
assert_eq!(h.decrypt_stream(3, 0, b"plain", StreamKind::Normal), b"plain");
let enc = h.encrypt_stream(3, 0, b"attachment", StreamKind::EmbeddedFile);
assert_ne!(enc, b"attachment");
assert_eq!(h.decrypt_stream(3, 0, &enc, StreamKind::EmbeddedFile), b"attachment");
assert_eq!(h.decrypt_stream(3, 0, b"x", StreamKind::Named(b"Identity")), b"x");
}
#[test]
fn r6_passwords_are_saslprepped() {
// U+00AD (soft hyphen) is mapped to nothing by SASLprep; "ª" normalises to "a" (NFKC).
let d = new(Algorithm::Aes256, "SªSL\u{AD}prep", "o", -1).dict().clone();
assert!(SecurityHandler::open(d.clone(), ID, Some("SaSLprep")).is_ok());
assert!(SecurityHandler::open(d, ID, Some("SaSL-prep")).is_err());
}
#[test]
fn effective_passwords_are_the_bytes_the_keys_are_made_from() {
// AES-256: SASLprep drops soft hyphens, makes other spaces plain and folds compatibility
// characters (NFKC); at most 127 bytes of UTF-8.
let r6 = |pw: &str| effective_password(Algorithm::Aes256, pw);
assert_eq!(r6("\u{AD}"), r6(""), "a soft hyphen alone is no password");
assert_eq!(r6("\u{FF21}"), r6("A"), "fullwidth A");
assert_eq!(r6("\u{FB01}"), r6("fi"), "the fi ligature");
assert_eq!(r6("a\u{3000}b"), b"a b", "an ideographic space");
assert_ne!(r6("A"), r6("a"), "case matters");
assert_eq!(r6(&"é".repeat(100)).len(), 127);
// RC4 and AES-128: PDFDocEncoding, padded or cut to exactly 32 bytes.
let (a31, a32) = ("a".repeat(31), "a".repeat(32));
for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] {
assert_eq!(effective_password(alg, &format!("{a32}1")), a32.as_bytes(), "{alg:?}");
assert_eq!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("1{a32}2")), "{alg:?}");
assert_ne!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("2{a32}")), "{alg:?}");
assert_eq!(effective_password(alg, &a31), effective_password(alg, &format!("{a31}(")), "{alg:?}: the padding starts with '('");
assert_eq!(effective_password(alg, ""), PADDING, "{alg:?}");
assert_eq!(effective_password(alg, "ılık€")[..5], [0x9A, b'l', 0x9A, b'k', 0xA0], "{alg:?}");
}
}
#[test]
fn passwords_with_the_same_effective_bytes_are_one_password() {
// A file protected with "\u{FF21}" (fullwidth A) opens with "A".
let d = new(Algorithm::Aes256, "\u{FF21}", "owner", 0b0100).dict().clone();
assert_eq!(SecurityHandler::open(d.clone(), ID, Some("A")).unwrap().auth(), Auth::User);
assert_eq!(SecurityHandler::open(d, ID, Some("a")).unwrap_err(), CryptError::WrongPassword);
// A soft hyphen is no open password at all.
let d = new(Algorithm::Aes256, "\u{AD}", "owner", 0b0100).dict().clone();
assert_eq!(SecurityHandler::open(d, ID, None).unwrap().auth(), Auth::User);
// Open and permissions passwords with the same bytes: the open password gives owner rights.
let d = new(Algorithm::Aes256, "\u{FB01}", "fi", 0b0100).dict().clone();
assert_eq!(effective_password(Algorithm::Aes256, "\u{FB01}"), effective_password(Algorithm::Aes256, "fi"));
assert_eq!(SecurityHandler::open(d, ID, Some("\u{FB01}")).unwrap().auth(), Auth::Owner);
let a31 = "a".repeat(31);
let pairs = [(format!("{a31}a1"), format!("{a31}a2")), (a31.clone(), format!("{a31}("))];
for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] {
for (open, perms) in &pairs {
assert_eq!(effective_password(alg, open), effective_password(alg, perms), "{alg:?}");
let d = new(alg, open, perms, 0b0100).dict().clone();
assert_eq!(SecurityHandler::open(d, ID, Some(open)).unwrap().auth(), Auth::Owner, "{alg:?} {open:?}");
}
}
}
#[test]
fn unsupported_handlers_are_reported() {
let mut d = new(Algorithm::Aes128, "", "o", -1).dict().clone();
d.filter = b"Adobe.PubSec".to_vec();
assert!(matches!(SecurityHandler::open(d.clone(), ID, None), Err(CryptError::Unsupported(_))));
d.filter = b"Standard".to_vec();
d.r = 7;
assert!(matches!(SecurityHandler::open(d, ID, None), Err(CryptError::Unsupported(_))));
}
#[test]
fn permission_bits_follow_table_22() {
let p = Permissions { bits: -3904, owner: false, revision: 3 }; // 0xFFFFF0C0: only reserved bits set
assert!(!p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.assemble());
let p = Permissions { bits: -1, owner: false, revision: 3 };
assert!(p.print() && p.modify() && p.copy() && p.annotate() && p.assemble() && p.fill_forms());
let p = Permissions { bits: 0, owner: true, revision: 3 };
assert!(p.modify(), "owners ignore /P");
}
/// Page extraction (#5081) is assembly as the revision reads it, and copying (bit 5).
#[test]
fn page_extraction_needs_assembly_and_copying() {
let bit = |n: u32| 1i32 << (n - 1);
let user = |bits: i32, revision: i64| Permissions { bits, owner: false, revision };
// "Any except extracting pages" (bits 4, 6, 9; copying off) at revision 3 and later.
assert!(!user(bit(4) | bit(6) | bit(9), 4).extract_pages());
// Pages may change in place, but copying is withheld.
assert!(user(bit(11), 6).assemble() && !user(bit(11), 6).extract_pages());
assert!(user(bit(11) | bit(5), 6).extract_pages());
// Revision 2: bit 4 covers assembly, and its reserved bit 11 grants nothing.
assert!(user(bit(4), 2).assemble() && !user(bit(4) | bit(11), 2).extract_pages(), "copying withheld");
assert!(user(bit(4) | bit(5), 2).extract_pages());
assert!(!user(bit(5) | bit(11), 2).extract_pages(), "no assembly without bit 4 at revision 2");
assert!(Permissions { bits: 0, owner: true, revision: 2 }.extract_pages(), "owners ignore /P");
}
#[test]
fn perms_limits_what_an_edited_p_grants() {
let made = new(Algorithm::Aes256, "user pw", "owner pw", 0b0100); // print only
let open = |d: &EncryptDict, pw| SecurityHandler::open(d.clone(), ID, Some(pw)).unwrap().permissions();
let mut d = made.dict().clone();
assert_eq!(open(&d, "user pw").bits, d.p, "/P and /Perms agree");
d.p = -4; // /P edited to allow everything, /Perms as written
let p = open(&d, "user pw");
assert!(p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.fill_forms() && !p.assemble());
assert!(open(&d, "owner pw").modify(), "owners ignore /P");
d.p = -3904; // a stricter /P still applies
assert!(!open(&d, "user pw").print());
// No /Perms, or one that does not decrypt to an `adb` block: /P as before.
for perms in [Vec::new(), vec![0; 16], made.dict().perms[..15].to_vec()] {
let d = EncryptDict { p: -4, perms, ..made.dict().clone() };
assert!(open(&d, "user pw").modify());
}
// R2 to R4 have no /Perms: /P is part of their file key, so an edited /P does not open.
let d = EncryptDict { p: -4, ..new(Algorithm::Aes128, "user pw", "owner pw", 0b0100).dict().clone() };
assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap_err(), CryptError::WrongPassword);
}
proptest::proptest! {
#[test]
fn malformed_dictionaries_never_panic(o in proptest::collection::vec(proptest::num::u8::ANY, 0..60),
u in proptest::collection::vec(proptest::num::u8::ANY, 0..60),
r in 0i64..9, len in -10i64..300) {
let d = EncryptDict { filter: b"Standard".to_vec(), v: 4, r, length_bits: len, o, u, p: -1, encrypt_metadata: true, ..Default::default() };
let _ = SecurityHandler::open(d, ID, Some("x"));
}
}
/// A `/V 4` RC4 crypt filter with a 40- or 48-bit key, as Acrobat writes it (pdf.js issue 19484):
/// the file key is extended to 16 bytes with zeros before Algorithm 1, so each object's key is
/// all 16 bytes of MD5(key, zeros, object, generation). Everything else keeps Algorithm 1 as the
/// standard has it, n + 5 bytes (at most 16) of MD5(key, object, generation[, "sAlT"]): a `/V 2`
/// file with the same short key, a full 128-bit `/V 4` RC4 key, and AESV2, short (88 bits) or
/// full. Each object key is computed here by hand; data written with it must decrypt.
#[test]
fn a_v4_rc4_key_shorter_than_16_bytes_is_extended_with_zeros() {
let text = b"Page tree in an object stream".to_vec();
let cases =
[(4, 40, Method::Rc4), (4, 48, Method::Rc4), (2, 40, Method::Rc4), (4, 128, Method::Rc4), (4, 88, Method::Aes128), (4, 128, Method::Aes128)];
for (v, bits, method) in cases {
let mut d = EncryptDict {
filter: b"Standard".to_vec(),
v,
r: if v == 4 { 4 } else { 3 },
length_bits: bits,
o: vec![7; 32],
p: -3392,
encrypt_metadata: v != 4,
..Default::default()
};
if v == 4 {
d.crypt_filters = vec![(b"StdCF".to_vec(), method)];
d.stm_f = b"StdCF".to_vec();
d.str_f = b"StdCF".to_vec();
}
let key = file_key_r4(&d, ID, b"");
assert_eq!(key.len(), (bits / 8) as usize);
d.u = u_value_r4(&d, ID, &key);
let h = SecurityHandler::open(d, ID, None).unwrap();
let (num, generation) = (13u32, 0u16);
let extended = v == 4 && method == Method::Rc4 && key.len() < 16;
let mut md5 = Md5::new();
md5.update(&key);
if extended {
md5.update(vec![0u8; 16 - key.len()]);
}
md5.update(&num.to_le_bytes()[..3]);
md5.update(generation.to_le_bytes());
if method == Method::Aes128 {
md5.update(b"sAlT");
}
let digest = md5.finalize();
let n = if extended { 16 } else { key.len() };
let object_key = digest[..(n + 5).min(16)].to_vec();
let case = format!("V{v}, {bits}-bit key, {method:?}");
let written = if method == Method::Rc4 {
rc4(&object_key, &text)
} else {
let iv = [3u8; 16];
[iv.to_vec(), aes_cbc_encrypt(&object_key, &iv, &text, true)].concat()
};
assert_eq!(h.decrypt_string(num, generation, &written), text, "{case}");
assert_eq!(h.decrypt_stream(num, generation, &written, StreamKind::Normal), text, "{case}");
let saved = h.encrypt_stream(num, generation, &text, StreamKind::Normal);
if method == Method::Rc4 {
assert_eq!(saved, written, "saved as it was read: {case}");
}
assert_eq!(h.decrypt_stream(num, generation, &saved, StreamKind::Normal), text, "{case}");
}
}
#[test]
fn owner_authentication_recovers_the_user_password() {
for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] {
let d = new(alg, "user pw", "owner pw", -1).dict().clone();
let h = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap();
let _: &Zeroizing<Vec<u8>> = &h.key;
let _: &Option<Zeroizing<Vec<u8>>> = &h.recovered_user;
assert_eq!(h.recovered_user_password().as_deref().map(Vec::as_slice), Some(&b"user pw"[..]), "{alg:?}");
assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap().recovered_user_password(), None);
}
let d = new(Algorithm::Rc4_128, "", "owner", -1).dict().clone();
assert_eq!(SecurityHandler::open(d, ID, Some("owner")).unwrap().recovered_user_password().as_deref().map(Vec::as_slice), Some(&b""[..]));
}
/// /O, /U and the first /ID of files qpdf 12.4.2 wrote (`--static-id`), with the file keys
/// `qpdf --show-encryption-key` reports.
fn qpdf_file(r: i64, o: &str, u: &str) -> EncryptDict {
let mut d = EncryptDict {
filter: b"Standard".to_vec(),
v: if r == 4 { 4 } else { 2 },
r,
length_bits: 128,
o: hex(o),
u: hex(u),
p: -4,
encrypt_metadata: true,
..Default::default()
};
if r == 4 {
d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)];
d.stm_f = b"StdCF".to_vec();
d.str_f = b"StdCF".to_vec();
}
d
}
const QPDF_ID: &str = "56b57383aafa8314f3f98222995f5604";
#[test]
fn legacy_passwords_are_pdfdoc_encoded_with_a_utf8_fallback() {
let id = hex(QPDF_ID);
// R3 RC4: user "ılık€" in PDFDocEncoding (ı = 0x9A, € = 0xA0); owner "Öğretmen", which
// PDFDocEncoding can't hold, so qpdf wrote its UTF-8 bytes.
let r3 = qpdf_file(
3,
"d519aa8120ce8c54812b45e5391916fed167b1c84961d90c4ee4c182ad83534b",
"4985c03ddfa3eeb8a90c343512913ec40021446990b9e4114071a4d9104984c1",
);
let user = SecurityHandler::open(r3.clone(), &id, Some("ılık€")).unwrap();
assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("3564ffcea186122595ad55ee157da71f")));
let owner = SecurityHandler::open(r3.clone(), &id, Some("Öğretmen")).unwrap();
assert_eq!((owner.auth(), owner.file_key().to_vec()), (Auth::Owner, hex("3564ffcea186122595ad55ee157da71f")));
assert_eq!(owner.recovered_user_password(), Some(Zeroizing::new(vec![0x9A, b'l', 0x9A, b'k', 0xA0])));
// Dropping ı and € (the old Latin-1 conversion) or a near miss doesn't open it.
for wrong in ["lk", "ilik€", "ılık", "Ogretmen"] {
assert_eq!(SecurityHandler::open(r3.clone(), &id, Some(wrong)).unwrap_err(), CryptError::WrongPassword, "{wrong}");
}
// R4 AES-128: user "şifre" (UTF-8 bytes), owner "owner".
let r4 = qpdf_file(
4,
"bb4f8f4bd2236569e765906caf64e4429a4c20d6e996fdef963e9b5080f9e083",
"4a965fb91882772cd09ce8bbff2c33270021446990b9e4114071a4d9104984c1",
);
let user = SecurityHandler::open(r4.clone(), &id, Some("şifre")).unwrap();
assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("9f19d465cfd8e525d32aa0bdf10acd06")));
assert_eq!(SecurityHandler::open(r4.clone(), &id, Some("owner")).unwrap().auth(), Auth::Owner);
assert_eq!(SecurityHandler::open(r4, &id, Some("ifre")).unwrap_err(), CryptError::WrongPassword);
}
#[test]
fn pdfdoc_encoding_round_trips() {
for b in 0..=255u8 {
if b != 0x9F {
assert_eq!(pdfdoc_byte(pdfdoc_char(b)), Some(b), "{b:#04x}");
}
}
assert_eq!(pdfdoc_byte('ı'), Some(0x9A));
assert_eq!(pdfdoc_byte('€'), Some(0xA0));
assert_eq!(pdfdoc_byte('ç'), Some(0xE7));
for c in ['ğ', 'ş', 'İ', '\u{A0}', '\u{85}', 'Ж', '日'] {
assert_eq!(pdfdoc_byte(c), None, "{c}");
}
assert_eq!(legacy_candidates("Çok"), [Zeroizing::new(vec![0xC7, b'o', b'k'])]);
assert_eq!(legacy_candidates("şifre"), [Zeroizing::new(b"ifre".to_vec()), Zeroizing::new("şifre".as_bytes().to_vec())]);
}
+1
View File
@@ -11,6 +11,7 @@ repository.workspace = true
workspace = true
[dependencies]
craft-rtf.workspace = true
craft-zip.workspace = true
[dev-dependencies]
+2 -18
View File
@@ -4,24 +4,8 @@ use super::layout::{Block, Item, Page, count_lookup_steps, items};
use std::collections::HashMap;
fn rtf_text(s: &str) -> String {
let mut o = String::new();
for c in s.chars().filter(|c| allowed(*c)) {
match c {
'\\' | '{' | '}' => {
o.push('\\');
o.push(c);
}
c if (c as u32) < 128 => o.push(c),
c => {
// \uN with a ? fallback; UTF-16 units as signed 16-bit numbers.
let mut buf = [0u16; 2];
for u in c.encode_utf16(&mut buf) {
o.push_str(&format!("\\u{}?", *u as i16));
}
}
}
}
o
let cleaned: String = s.chars().filter(|c| allowed(*c)).collect();
craft_rtf::esc(&cleaned)
}
/// Rich Text Format: paragraphs with their sizes and bold/italic, real table rows, page breaks
+1 -6
View File
@@ -11,12 +11,7 @@ repository.workspace = true
workspace = true
[dependencies]
fax.workspace = true
# CCITT fax and JBIG2 decoding (`decode_ccitt`, `decode_jbig2`); already in the tree through hayro.
hayro-ccitt.workspace = true
hayro-jbig2.workspace = true
flate2.workspace = true
thiserror.workspace = true
craft-pdf = { workspace = true }
[dev-dependencies]
proptest.workspace = true
-156
View File
@@ -1,156 +0,0 @@
//! ASCII85Decode (ISO 32000-2 §7.4.3): groups of five characters `!`..`u` encode four
//! bytes base-85, `z` stands for four zero bytes, `~>` is EOD, white space is ignored and
//! a final partial group of n characters (2–5) yields n − 1 bytes. A leading `<~` and a
//! missing EOD are tolerated.
use crate::{Failure, Step, is_pdf_whitespace};
const NAME: &str = "ASCII85Decode";
fn push(out: &mut Vec<u8>, bytes: &[u8], max: usize) -> Result<(), Failure> {
if out.len() + bytes.len() > max {
return Err(Failure::limit(max));
}
out.extend_from_slice(bytes);
Ok(())
}
pub(crate) fn decode(data: &[u8], max: usize) -> Step {
let start = data.iter().position(|&b| !is_pdf_whitespace(b)).unwrap_or(data.len());
let body = data[start..].strip_prefix(b"<~").unwrap_or(&data[start..]);
let mut out = Vec::with_capacity((body.len() / 5 * 4 + 4).min(max));
let mut acc: u64 = 0;
let mut n = 0usize;
for (i, &b) in body.iter().enumerate() {
match b {
b'!'..=b'u' => {
acc = acc * 85 + u64::from(b - b'!');
n += 1;
if n == 5 {
let Ok(v) = u32::try_from(acc) else {
return Err(Failure::corrupt(NAME, format!("group ending at {i} exceeds 2^32"), out));
};
push(&mut out, &v.to_be_bytes(), max)?;
acc = 0;
n = 0;
}
}
b'z' if n == 0 => push(&mut out, &[0; 4], max)?,
b'z' => return Err(Failure::corrupt(NAME, format!("'z' inside a group at {i}"), out)),
// `~` starts the EOD marker; tolerate it without the following `>`.
b'~' => break,
b if is_pdf_whitespace(b) => {}
b => return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at {i}"), out)),
}
}
match n {
0 => {}
1 => return Err(Failure::corrupt(NAME, "final group has a single character", out)),
_ => {
// Pad with `u` (84) to a full group and keep n − 1 bytes.
for _ in n..5 {
acc = acc * 85 + 84;
}
let Ok(v) = u32::try_from(acc) else {
return Err(Failure::corrupt(NAME, "final group exceeds 2^32", out));
};
push(&mut out, &v.to_be_bytes()[..n - 1], max)?;
}
}
Ok(out)
}
/// Encodes with `z` for zero groups, a newline every 75 characters, and a `~>` EOD.
pub(crate) fn encode(data: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(data.len() * 5 / 4 + data.len() / 60 + 8);
let mut col = 0;
let mut put = |out: &mut Vec<u8>, s: &[u8]| {
for &c in s {
if col == 75 {
out.push(b'\n');
col = 0;
}
out.push(c);
col += 1;
}
};
for chunk in data.chunks(4) {
let mut word = [0u8; 4];
word[..chunk.len()].copy_from_slice(chunk);
let mut v = u32::from_be_bytes(word);
if v == 0 && chunk.len() == 4 {
put(&mut out, b"z");
continue;
}
let mut digits = [0u8; 5];
for d in digits.iter_mut().rev() {
*d = (v % 85) as u8 + b'!';
v /= 85;
}
put(&mut out, &digits[..chunk.len() + 1]);
}
out.extend_from_slice(b"~>");
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FilterError;
fn dec(d: &[u8]) -> Result<Vec<u8>, FilterError> {
decode(d, 1000).map_err(|f| f.error)
}
#[test]
fn vectors() {
// "Man " = 0x4D616E20 = 24·85⁴ + 73·85³ + 80·85² + 78·85 + 61 -> "9jqo^".
assert_eq!(dec(b"9jqo^~>").unwrap(), b"Man ");
assert_eq!(encode(b"Man "), b"9jqo^~>");
// "M" padded to 0x4D000000 -> "9`" (24, 63); decoding pads with 'u'.
assert_eq!(encode(b"M"), b"9`~>");
assert_eq!(dec(b"9`~>").unwrap(), b"M");
assert_eq!(dec(b"z~>").unwrap(), [0; 4]);
assert_eq!(encode(&[0; 4]), b"z~>");
// A partial zero group is not `z`.
assert_eq!(encode(&[0; 3]), b"!!!!~>");
assert_eq!(dec(b"!!!!~>").unwrap(), [0; 3]);
assert_eq!(dec(b"s8W-!~>").unwrap(), [0xff; 4]);
}
#[test]
fn syntax_tolerance() {
assert_eq!(dec(b"<~9jqo^~>").unwrap(), b"Man ");
assert_eq!(dec(b" \n<~9j\r\nq o\t^z~>").unwrap(), b"Man \0\0\0\0");
assert_eq!(dec(b"9jqo^").unwrap(), b"Man "); // missing EOD
assert_eq!(dec(b"9jqo^~").unwrap(), b"Man "); // bare ~
assert_eq!(dec(b"9jqo^~>garbage{").unwrap(), b"Man ");
assert_eq!(dec(b"~>").unwrap(), b"");
assert_eq!(dec(b"").unwrap(), b"");
}
#[test]
fn corrupt() {
assert!(matches!(dec(b"9jz~>"), Err(FilterError::Corrupt { .. })));
assert!(matches!(dec(b"9jqo^9~>"), Err(FilterError::Corrupt { .. })));
assert!(matches!(dec(b"uuuuu~>"), Err(FilterError::Corrupt { .. }))); // > 2^32 - 1
assert!(matches!(dec(b"9jqo^v~>"), Err(FilterError::Corrupt { .. })));
let f = decode(b"9jqo^{", 100).err().unwrap();
assert_eq!(f.partial, b"Man ");
}
#[test]
fn limit() {
assert!(matches!(decode(b"zz", 7).map_err(|f| f.error), Err(FilterError::LimitExceeded(7))));
assert!(matches!(decode(b"z9`", 4).map_err(|f| f.error), Err(FilterError::LimitExceeded(4))));
assert_eq!(decode(b"zz", 8).ok().unwrap().len(), 8);
}
#[test]
fn line_wrapping() {
let data: Vec<u8> = (0..=255u8).collect();
let e = encode(&data);
assert!(e.split(|&b| b == b'\n').all(|l| l.len() <= 77));
assert_eq!(dec(&e).unwrap_or_default().len(), 256);
}
}
-106
View File
@@ -1,106 +0,0 @@
//! ASCIIHexDecode (ISO 32000-2 §7.4.2): pairs of hex digits, white space ignored,
//! `>` is EOD, an odd final digit is padded with 0. A missing EOD is tolerated.
use crate::{Failure, Step, is_pdf_whitespace};
const NAME: &str = "ASCIIHexDecode";
fn nibble(b: u8) -> Option<u8> {
match b {
b'0'..=b'9' => Some(b - b'0'),
b'a'..=b'f' => Some(b - b'a' + 10),
b'A'..=b'F' => Some(b - b'A' + 10),
_ => None,
}
}
pub(crate) fn decode(data: &[u8], max: usize) -> Step {
let mut out = Vec::with_capacity((data.len() / 2).min(max));
let mut hi: Option<u8> = None;
for (i, &b) in data.iter().enumerate() {
if is_pdf_whitespace(b) {
continue;
}
if b == b'>' {
break;
}
let Some(n) = nibble(b) else {
return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at offset {i}"), out));
};
match hi.take() {
None => hi = Some(n),
Some(h) => {
if out.len() == max {
return Err(Failure::limit(max));
}
out.push(h << 4 | n);
}
}
}
if let Some(h) = hi {
if out.len() == max {
return Err(Failure::limit(max));
}
out.push(h << 4);
}
Ok(out)
}
/// Upper-case hex, a newline every 64 digits, terminated by `>`.
pub(crate) fn encode(data: &[u8]) -> Vec<u8> {
const HEX: &[u8; 16] = b"0123456789ABCDEF";
let mut out = Vec::with_capacity(data.len() * 2 + data.len() / 32 + 1);
for (i, &b) in data.iter().enumerate() {
if i > 0 && i % 32 == 0 {
out.push(b'\n');
}
out.push(HEX[usize::from(b >> 4)]);
out.push(HEX[usize::from(b & 15)]);
}
out.push(b'>');
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FilterError;
fn dec(d: &[u8]) -> Result<Vec<u8>, FilterError> {
decode(d, 1000).map_err(|f| f.error)
}
#[test]
fn vectors() {
assert_eq!(dec(b"48656c6C6F>").unwrap(), b"Hello");
assert_eq!(dec(b" 4 8\n6\r5\t6\x0c c\x006c 6f >").unwrap(), b"Hello");
assert_eq!(dec(b"901FA>").unwrap(), [0x90, 0x1f, 0xa0]);
assert_eq!(dec(b"901FA").unwrap(), [0x90, 0x1f, 0xa0]); // missing EOD
assert_eq!(dec(b"41>zz garbage after EOD").unwrap(), b"A");
assert_eq!(dec(b">").unwrap(), b"");
assert_eq!(dec(b"").unwrap(), b"");
}
#[test]
fn invalid_char() {
let f = decode(b"4142G43>", 1000).err().unwrap();
assert!(matches!(f.error, FilterError::Corrupt { .. }));
assert_eq!(f.partial, b"AB");
}
#[test]
fn limit() {
assert!(matches!(decode(b"414243", 2).map_err(|f| f.error), Err(FilterError::LimitExceeded(2))));
assert!(matches!(decode(b"41424", 2).map_err(|f| f.error), Err(FilterError::LimitExceeded(2))));
assert_eq!(decode(b"4142", 2).ok().unwrap(), b"AB");
}
#[test]
fn encoding() {
assert_eq!(encode(b"Hello"), b"48656C6C6F>");
assert_eq!(encode(b""), b">");
let long = encode(&[0xab; 40]);
assert_eq!(long[64], b'\n');
assert_eq!(dec(&long).unwrap(), [0xab; 40]);
}
}
-547
View File
@@ -1,547 +0,0 @@
//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`,
//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the
//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give.
use fax::encoder::Encoder;
use fax::{Color, VecWriter};
use crate::FilterError;
/// The widest row [`encode_g4`] encodes. The encoder counts the positions in a row in 16 bits
/// and adds two of them, so a row stays within half that range.
pub const MAX_G4_COLUMNS: u32 = 32_767;
/// `rows` rows of `columns` 1-bit samples, most significant bit first, each row starting on a new
/// byte (as a PDF image's samples are packed), encoded as CCITT Group 4 with an end-of-block mark.
///
/// A 0 bit is coded black and a 1 bit white, so the stream decodes to the same bits with
/// `/BlackIs1 false`, the default: an image keeps its colour space and decode array. Bytes after
/// the last row are ignored, as are the bits that pad a row to a whole byte.
///
/// Errors: no rows or columns, or rows wider than [`MAX_G4_COLUMNS`] ([`FilterError::Unsupported`]);
/// fewer bytes than the rows need ([`FilterError::Corrupt`]).
pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, FilterError> {
let width = match u16::try_from(columns) {
Ok(w) if w > 0 && columns <= MAX_G4_COLUMNS => w,
_ => return Err(FilterError::Unsupported(format!("CCITT Group 4 encoding of rows {columns} samples wide"))),
};
if rows == 0 {
return Err(FilterError::Unsupported("CCITT Group 4 encoding of an image without rows".into()));
}
let stride = usize::from(width).div_ceil(8);
let samples = usize::try_from(rows).ok().and_then(|r| r.checked_mul(stride)).and_then(|n| data.get(..n)).ok_or_else(|| FilterError::Corrupt {
filter: "CCITTFaxDecode",
detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()),
})?;
let mut encoder = Encoder::new(VecWriter::new());
for row in samples.chunks_exact(stride) {
let pels = (0..usize::from(width)).map(|x| {
let white = row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);
if white { Color::White } else { Color::Black }
});
// `VecWriter` writes to memory: its error type has no values.
if let Err(never) = encoder.encode_line(pels, width) {
match never {}
}
}
match encoder.finish() {
Ok(writer) => Ok(writer.finish()),
Err(never) => match never {},
}
}
/// The widest row [`decode_ccitt_within`] decodes: its row buffer stays within 128 KiB.
pub const MAX_CCITT_COLUMNS: u32 = 1 << 20;
/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the
/// table's defaults.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct CcittParams {
/// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed.
pub k: i64,
pub columns: i64,
/// `/Rows`: 0 when not given.
pub rows: i64,
pub end_of_line: bool,
pub encoded_byte_align: bool,
pub end_of_block: bool,
pub black_is_1: bool,
}
impl Default for CcittParams {
fn default() -> Self {
CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false }
}
}
/// What [`decode_ccitt_within`] decodes once it is paid for.
fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result<Vec<u8>, FilterError> {
let columns = u32::try_from(p.columns)
.ok()
.filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c))
.ok_or_else(|| FilterError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?;
let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows));
if rows == 0 {
return Ok(Vec::new());
}
let settings = hayro_ccitt::DecodeSettings {
columns,
rows,
end_of_block: p.end_of_block,
end_of_line: p.end_of_line,
// Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter
// always accepts them), with the fill starting each code on a byte or ending it on one (T.4's
// fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without
// codes, the rows are read on bytes. A reading that comes up short isn't tried the other
// way: data cut short would then read as other rows.
rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)),
encoding: match p.k {
..0 => hayro_ccitt::EncodingMode::Group4,
0 => hayro_ccitt::EncodingMode::Group3_1D,
k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) },
},
invert_black: false,
};
// u32 to usize: wasm32 and up have at least 32-bit usize.
let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1);
// The rows decoded before damaged data are kept.
let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings));
Ok(out.out)
}
/// The steps of reading a byte of fax code ([`decode_ccitt_within`]): noise takes 114 ns a byte of
/// rows and about two bytes of code to a byte of rows, so that and a step a byte of rows is about
/// what a step of JBIG2 decoding takes.
const CODE_WORK: u64 = 4;
/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most
/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`).
///
/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by
/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a
/// caller that needs a number of rows checks it got them.
///
/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes,
/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4
/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever
/// `end_of_line` says. Group 4 is read as its parameters say. Data with codes but none before the
/// first row is read on bytes, as PdfCraft's renderer draws it; with `k` 0 and the fill ending each
/// code on a byte, that may be other rows than were coded.
///
/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`FilterError::Unsupported`]).
///
/// Decoding spends from `work`, what decoding may still cost across a run of images, and there is
/// no way in without it: a
/// byte of rows a step, whether or not the data holds them, and [`CODE_WORK`] a byte of `data`.
/// A blank row takes a bit of code, so the rows are what a few bytes can ask for. It is all paid
/// before anything is decoded, and what `work` can't pay for spends all of it.
pub fn decode_ccitt_within(data: &[u8], p: &CcittParams, max_rows: u32, work: &mut u64) -> Result<Vec<u8>, FilterError> {
let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows));
let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8);
crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?;
decode_ccitt(data, p, max_rows)
}
/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill
/// included. T.4 sends one before the first row when it sends them at all, and no other code starts
/// with more than eight zeros.
fn starts_with_end_of_line(data: &[u8]) -> bool {
let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false };
i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11
}
/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at
/// most `rows` rows. Pixels past a row's end and rows past the last are dropped. The JBIG2 decoder
/// ([`crate::decode_jbig2_within`]) fills them too.
pub(crate) struct Rows {
pub(crate) out: Vec<u8>,
row: Vec<u8>,
x: usize,
columns: usize,
rows: usize,
done: usize,
black_is_1: bool,
}
impl Rows {
pub(crate) fn new(columns: usize, rows: usize, black_is_1: bool) -> Self {
Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 }
}
/// `count` pixels, white or black.
pub(crate) fn push(&mut self, white: bool, count: usize) {
let end = self.x.saturating_add(count);
if white != self.black_is_1 {
for x in self.x..end.min(self.columns) {
if let Some(b) = self.row.get_mut(x / 8) {
*b |= 0x80 >> (x % 8);
}
}
}
self.x = end;
}
/// The row filled so far kept, if fewer than `rows` are, and the next one begun.
pub(crate) fn end_row(&mut self) {
if self.done < self.rows {
self.out.extend_from_slice(&self.row);
self.done += 1;
}
self.row.fill(0);
self.x = 0;
}
}
impl hayro_ccitt::Decoder for Rows {
fn push_pixel(&mut self, white: bool) {
self.push(white, 1);
}
fn push_pixel_chunk(&mut self, white: bool, chunks: u32) {
self.push(white, (chunks as usize).saturating_mul(8));
}
fn next_line(&mut self) {
self.end_row();
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A run pays for the rows an image asks for and the code it reads, before any of it is
/// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number.
#[test]
fn a_run_of_images_pays_for_the_rows_it_asks_for() {
let (w, h) = (4096u32, 64u32);
let stride = (w as usize).div_ceil(8);
let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap();
let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() };
let whole = decode_ccitt(&data, &p, h).unwrap();
let cost = (whole.len() + 4 * data.len()) as u64;
let mut work = cost;
assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough");
assert_eq!(work, 0);
let mut work = cost + 7;
decode_ccitt_within(&data, &p, h, &mut work).unwrap();
assert_eq!(work, 7, "only what it took");
let mut work = cost - 1;
let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err();
assert!(matches!(refused, FilterError::Spent), "a step short: {refused}");
assert_eq!(work, 0, "spends all that was left");
// Nothing left: not read at all, so bad columns aren't reported as bad columns.
let bad = CcittParams { columns: 0, ..p };
assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| matches!(e, FilterError::Spent)));
// No code at all, for the same rows.
let mut work = u64::MAX;
decode_ccitt_within(&[], &p, h, &mut work).unwrap();
assert_eq!(u64::MAX - work, whole.len() as u64);
// `/Rows` below what is asked for, and no rows.
let mut work = u64::MAX;
decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap();
assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64);
let mut work = u64::MAX;
decode_ccitt_within(&data, &p, 0, &mut work).unwrap();
assert_eq!(u64::MAX - work, 4 * data.len() as u64);
}
/// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black.
pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
struct Rows(Vec<Vec<bool>>, Vec<bool>, usize);
impl hayro_ccitt::Decoder for Rows {
fn push_pixel(&mut self, white: bool) {
self.1.push(white);
}
fn push_pixel_chunk(&mut self, white: bool, chunks: u32) {
self.1.extend(std::iter::repeat_n(white, chunks as usize * 8));
}
fn next_line(&mut self) {
let mut row = std::mem::take(&mut self.1);
row.truncate(self.2);
self.0.push(row);
}
}
let settings = hayro_ccitt::DecodeSettings {
columns,
rows,
end_of_block: true,
end_of_line: false,
rows_are_byte_aligned: false,
encoding: hayro_ccitt::EncodingMode::Group4,
invert_black: false,
};
let mut out = Rows(Vec::new(), Vec::new(), columns as usize);
hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap();
out.0
}
/// The rows of packed samples `data`, as 1 for a 1 bit.
pub(crate) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
let stride = (columns as usize).div_ceil(8);
(0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect()
}
#[test]
fn a_page_of_text_like_marks_round_trips_and_shrinks() {
let (w, h) = (1700u32, 220u32);
let stride = (w as usize).div_ceil(8);
let mut data = vec![0xFF; stride * h as usize];
// Short black strokes in lines, like glyphs on a white page.
for y in (20..h as usize).step_by(40) {
for dy in 0..12 {
for x in (100..1600).step_by(9) {
if (x / 9 + dy) % 5 != 0 {
data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8));
}
}
}
}
let g4 = encode_g4(&data, w, h).unwrap();
assert_eq!(decode(&g4, w, h), unpack(&data, w, h));
assert!(g4.len() * 4 < data.len(), "{} bytes from {}", g4.len(), data.len());
}
#[test]
fn padding_bits_and_trailing_bytes_are_ignored() {
// 3 columns: the last 5 bits of each byte pad the row; set them to anything.
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD];
let g4 = encode_g4(&data, 3, 3).unwrap();
assert_eq!(decode(&g4, 3, 3), [[true, false, true], [false, true, false], [true, true, true]]);
}
#[test]
fn what_it_cannot_encode_is_an_error() {
assert!(matches!(encode_g4(&[0; 8], 0, 1), Err(FilterError::Unsupported(_))));
assert!(matches!(encode_g4(&[0; 8], 8, 0), Err(FilterError::Unsupported(_))));
assert!(matches!(encode_g4(&vec![0; 4097], MAX_G4_COLUMNS + 1, 1), Err(FilterError::Unsupported(_))));
assert!(matches!(encode_g4(&[0; 3], 16, 2), Err(FilterError::Corrupt { .. })), "two rows of 16 need 4 bytes");
assert!(matches!(encode_g4(&[], 1, u32::MAX), Err(FilterError::Corrupt { .. })));
let widest = vec![0x55; (MAX_G4_COLUMNS as usize).div_ceil(8) * 2];
let g4 = encode_g4(&widest, MAX_G4_COLUMNS, 2).unwrap();
assert_eq!(decode(&g4, MAX_G4_COLUMNS, 2), unpack(&widest, MAX_G4_COLUMNS, 2));
}
/// Rows of `columns` samples built from `runs`: alternating white and black runs one longer
/// than those values (so never empty, even when shrinking makes them all 0), repeated until the
/// rows are full; or, with `noise`, the bytes of `runs` as they are.
fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec<u8> {
let stride = (columns as usize).div_ceil(8);
let mut data = vec![0u8; stride * rows as usize];
if runs.is_empty() {
return data;
}
if noise {
data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r);
return data;
}
let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1);
for y in 0..rows as usize {
for x in 0..columns as usize {
if left == 0 {
white = !white;
left = 1 + usize::from(runs[next % runs.len()]);
next += 1;
}
if white {
data[y * stride + x / 8] |= 0x80 >> (x % 8);
}
left -= 1;
}
}
data
}
fn g4(columns: u32) -> CcittParams {
CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() }
}
#[test]
fn decoding_gives_the_rows_back() {
// 3 columns, padded with anything: the decoded padding is 0.
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010];
let coded = encode_g4(&data, 3, 3).unwrap();
assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]);
// /BlackIs1 gives black as 1.
let p = CcittParams { black_is_1: true, ..g4(3) };
assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]);
// At most /Rows rows, and at most max_rows.
assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2);
assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1);
assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows");
assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]);
// Damaged data gives the rows before the damage; none here.
assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3);
}
#[test]
fn what_it_cannot_decode_is_an_error() {
for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] {
let p = CcittParams { columns, ..CcittParams::default() };
assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(FilterError::Unsupported(_))), "{columns}");
}
assert_eq!(CcittParams::default().columns, 1728, "Table 11");
}
/// Group 3 data: rows of run lengths below 64, white first, coded one-dimensionally with T.4's
/// terminating codes (ITU-T T.4 Tables 2 and 3). Each row starts with an end-of-line code with
/// `eol`, then with `tag` a 1 bit (the next row one-dimensional, as `k` above 0 has it). Fill
/// bits (0s) go where `fill` says.
fn group_3(rows: &[Vec<u32>], eol: bool, tag: bool, fill: Fill) -> Vec<u8> {
// The codes for runs 0 to 63, in order.
const WHITE: &str = concat!(
"00110101 000111 0111 1000 1011 1100 1110 1111 10011 10100 00111 01000 001000 000011 110100 110101 101010 101011 ",
"0100111 0001100 0001000 0010111 0000011 0000100 0101000 0101011 0010011 0100100 0011000 00000010 00000011 00011010 ",
"00011011 00010010 00010011 00010100 00010101 00010110 00010111 00101000 00101001 00101010 00101011 00101100 00101101 ",
"00000100 00000101 00001010 00001011 01010010 01010011 01010100 01010101 00100100 00100101 01011000 01011001 01011010 ",
"01011011 01001010 01001011 00110010 00110011 00110100",
);
const BLACK: &str = concat!(
"0000110111 010 11 10 011 0011 0010 00011 000101 000100 0000100 0000101 0000111 00000100 00000111 000011000 0000010111 ",
"0000011000 0000001000 00001100111 00001101000 00001101100 00000110111 00000101000 00000010111 00000011000 ",
"000011001010 000011001011 000011001100 000011001101 000001101000 000001101001 000001101010 000001101011 000011010010 ",
"000011010011 000011010100 000011010101 000011010110 000011010111 000001101100 000001101101 000011011010 000011011011 ",
"000001010100 000001010101 000001010110 000001010111 000001100100 000001100101 000001010010 000001010011 000000100100 ",
"000000110111 000000111000 000000100111 000000101000 000001011000 000001011001 000000101011 000000101100 000001011010 ",
"000001100110 000001100111",
);
let code = |white: bool, run: u32| (if white { WHITE } else { BLACK }).split(' ').nth(run as usize).expect("a run below 64");
let mut bits = String::new();
for row in rows {
// The end-of-line code is 12 bits.
let ends_code = if fill == Fill::CodeEnd && eol { 12 } else { 0 };
while fill != Fill::None && !(bits.len() + ends_code).is_multiple_of(8) {
bits.push('0');
}
if eol {
bits.push_str("000000000001");
}
if tag {
bits.push('1');
}
row.iter().enumerate().for_each(|(i, run)| bits.push_str(code(i % 2 == 0, *run)));
}
bits.as_bytes().chunks(8).map(|c| c.iter().enumerate().fold(0u8, |b, (i, d)| b | (u8::from(*d == b'1') << (7 - i)))).collect()
}
/// Where [`group_3`] puts fill bits: none; so each row, its end-of-line code first, starts on a
/// byte; or so each end-of-line code ends on one (T.4's fill, as TIFF writes it).
#[derive(Clone, Copy, Debug, PartialEq)]
enum Fill {
None,
RowStart,
CodeEnd,
}
/// `rows` text-like rows of `columns` samples as runs, white first, from `seed`: long white runs
/// and short black ones; a row may start black.
fn text_like(seed: u64, columns: u32, rows: u32) -> Vec<Vec<u32>> {
let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1;
let mut random = |n: u64| {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
(seed % n) as u32
};
(0..rows)
.map(|_| {
let (mut runs, mut left) = (Vec::new(), columns);
while left > 0 {
let run = match runs.len() % 2 {
0 if runs.is_empty() => random(31),
0 => 1 + random(30),
_ => 1 + random(5),
};
runs.push(run.min(left));
left -= run.min(left);
}
runs
})
.collect()
}
/// Rows of runs, white first, packed as `decode_ccitt` gives them (white 1).
fn pack(runs: &[Vec<u32>], columns: u32) -> Vec<u8> {
let stride = (columns as usize).div_ceil(8);
let mut out = vec![0u8; stride * runs.len()];
for (y, row) in runs.iter().enumerate() {
let mut x = 0;
for (i, run) in row.iter().enumerate() {
if i % 2 == 0 {
(x..x + *run as usize).for_each(|x| out[y * stride + x / 8] |= 0x80 >> (x % 8));
}
x += *run as usize;
}
}
out
}
/// Group 3 rows on bytes in each layout, over many pictures, as how a misreading comes out
/// depends on the picture: no codes (as TIFF's Modified Huffman has it), and `end_of_line`
/// without them; codes, each starting on a byte or ending on one, `end_of_line` or not. Each
/// decodes to its picture, and a byte short to fewer rows, not to other rows read another way.
/// Group 4 rows on bytes keep their aligned read, `end_of_line` or not.
#[test]
fn group_3_rows_on_bytes_read_by_the_codes_in_the_data() {
for seed in 0..32u64 {
let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20);
let runs = text_like(seed, columns, rows);
let picture = pack(&runs, columns);
for k in [0, 1] {
for (codes, fill, end_of_line) in [
(false, Fill::RowStart, false),
(false, Fill::RowStart, true),
(true, Fill::RowStart, false),
(true, Fill::RowStart, true),
(true, Fill::CodeEnd, false),
(true, Fill::CodeEnd, true),
] {
let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align: true, ..CcittParams::default() };
let mut data = group_3(&runs, codes, k > 0, fill);
let case = format!("seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, end_of_line {end_of_line}");
assert_eq!(decode_ccitt(&data, &p, rows).unwrap(), picture, "{case}");
data.pop();
assert!(decode_ccitt(&data, &p, rows).unwrap().len() < picture.len(), "{case}, a byte short");
}
}
}
// White rows, each the one code 1 (vertical, no change) and its fill.
for end_of_line in [false, true] {
let p = CcittParams { k: -1, columns: 64, end_of_line, encoded_byte_align: true, ..CcittParams::default() };
assert_eq!(decode_ccitt(&[0x80; 16], &p, 16).unwrap(), [0xFF; 8 * 16], "Group 4, end_of_line {end_of_line}");
}
}
proptest::proptest! {
/// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder
/// and through `decode_ccitt`.
#[test]
fn bitmaps_round_trip(
columns in 1u32..400,
rows in 1u32..24,
runs in proptest::collection::vec(0u8..=255, 1..48),
noise in proptest::bool::ANY,
) {
let data = bitmap(columns, rows, &runs, noise);
let coded = encode_g4(&data, columns, rows).unwrap();
proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows));
let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap();
proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows));
}
/// Whatever it is given, it returns.
#[test]
fn any_input_returns(
data in proptest::collection::vec(0u8..=255, 0..600),
columns in 0u32..40_000,
rows in 0u32..80,
k in -2i64..3,
end_of_line in proptest::bool::ANY,
encoded_byte_align in proptest::bool::ANY,
) {
let _ = encode_g4(&data, columns, rows);
let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align, ..CcittParams::default() };
if let Ok(out) = decode_ccitt(&data, &p, rows) {
proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize);
}
}
}
}
-304
View File
@@ -1,304 +0,0 @@
//! FlateDecode (ISO 32000-2 §7.4.4): zlib (RFC 1950) wrapping deflate (RFC 1951).
//!
//! Real-world quirks handled:
//! - the Adler-32 trailer is not required (some producers omit or corrupt it), because the
//! deflate stream is inflated raw after the 2-byte zlib header;
//! - streams without a valid zlib header are inflated as raw deflate;
//! - data after the end of the deflate stream is ignored.
//!
//! A back-reference to before the start of the output points at nothing, and is damage, as zlib
//! has it: read strictly the stream is refused, and read tolerantly what came before it is kept.
//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) So is a dynamic
//! block whose code has no end-of-block symbol, and nothing of that block is kept.
use std::io::Write;
use flate2::{Compression, Decompress, FlushDecompress, Status};
use crate::{Failure, Step};
const NAME: &str = "FlateDecode";
/// Checks the RFC 1950 header: CM = 8, CINFO ≤ 7 and the FCHECK multiple-of-31 rule.
fn zlib_header(data: &[u8]) -> Option<(u8, u8)> {
let (&cmf, &flg) = (data.first()?, data.get(1)?);
let ok = cmf & 0x0f == 8 && cmf >> 4 <= 7 && (u16::from(cmf) << 8 | u16::from(flg)) % 31 == 0;
ok.then_some((cmf, flg))
}
/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked
/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has
/// left for it once it has paid for the second reading. `None`, that reading can't be paid for,
/// and is [`FilterError::Spent`](crate::FilterError::Spent).
pub(crate) fn decode(data: &[u8], max: usize, again: Option<usize>) -> (Step, bool) {
match zlib_header(data) {
Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false),
Some(_) => match inflate_raw(&data[2..], max) {
// A header that only looked valid: retry the whole input as raw deflate.
Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => {
let Some(again) = again else {
return (Err(Failure { error: crate::FilterError::Spent, partial: Vec::new(), discarded: 0 }), false);
};
let step = match inflate_raw(data, again) {
// Keep the original corruption diagnostic only if the retry is corrupt
// too. A limit error must remain fatal even for tolerant decoding. What the
// retry decoded isn't kept, but a caller charging for work has to hear of it.
Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }),
retry => retry,
};
(step, true)
}
r => (r, false),
},
None => (inflate_raw(data, max), false),
}
}
#[cfg(test)]
thread_local! {
/// Bytes inflating gave back on this thread, its partial output included.
pub(crate) static INFLATED: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
fn inflate_raw(input: &[u8], max: usize) -> Step {
let step = inflate(input, max);
#[cfg(test)]
INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len)));
step
}
fn inflate(input: &[u8], max: usize) -> Step {
let mut d = Decompress::new(false);
let mut out: Vec<u8> = Vec::new();
// One byte beyond the limit lets us tell "exactly max" from "more than max".
let cap = max.saturating_add(1);
loop {
if out.len() == out.capacity() {
if out.len() >= cap {
return Err(Failure::limit(max));
}
let grow = out.capacity().max(input.len().saturating_mul(2)).max(4096).min(cap - out.len());
out.reserve_exact(grow);
}
let consumed = d.total_in() as usize;
let produced = out.len();
let res = d.decompress_vec(&input[consumed.min(input.len())..], &mut out, FlushDecompress::None);
if out.len() > max {
return Err(Failure::limit(max));
}
match res {
Err(e) => return Err(Failure::corrupt(NAME, e.to_string(), out)),
Ok(Status::StreamEnd) => return Ok(out),
Ok(Status::Ok | Status::BufError) => {
let progressed = d.total_in() as usize != consumed || out.len() != produced;
let room_left = out.len() < out.capacity();
if room_left && (d.total_in() as usize >= input.len() || !progressed) {
return Err(Failure::corrupt(NAME, "unexpected end of deflate data", out));
}
}
}
}
}
/// zlib at level 6.
// The documented never-crash exception: writing into a `Vec` cannot fail.
#[allow(clippy::expect_used)]
pub(crate) fn encode(data: &[u8]) -> Vec<u8> {
let mut e = flate2::write::ZlibEncoder::new(Vec::with_capacity(data.len() / 2 + 64), Compression::new(6));
e.write_all(data).expect("in-memory write");
e.finish().expect("in-memory write")
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FilterError;
fn dec(d: &[u8]) -> Result<Vec<u8>, FilterError> {
decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error)
}
#[test]
fn known_zlib_vector() {
// zlib("a"): header 78 9C, fixed-Huffman block, Adler-32 0x00620062.
assert_eq!(dec(&[0x78, 0x9c, 0x4b, 0x04, 0x00, 0x00, 0x62, 0x00, 0x62]).unwrap(), b"a");
}
#[test]
fn hand_built_stored_block() {
// RFC 1951 §3.2.4 stored block: BFINAL=1 BTYPE=00, LEN=5, NLEN=!5, "hello",
// then Adler-32("hello") = 0x062C0215.
let v = [0x78, 0x01, 0x01, 0x05, 0x00, 0xfa, 0xff, b'h', b'e', b'l', b'l', b'o', 0x06, 0x2c, 0x02, 0x15];
assert_eq!(dec(&v).unwrap(), b"hello");
// Missing Adler-32 is tolerated.
assert_eq!(dec(&v[..12]).unwrap(), b"hello");
// Wrong Adler-32 is tolerated (as in other readers).
let mut bad = v;
bad[15] ^= 0xff;
assert_eq!(dec(&bad).unwrap(), b"hello");
// Trailing garbage after the stream is ignored.
let mut trailing = v.to_vec();
trailing.extend_from_slice(b"\r\nendstream");
assert_eq!(dec(&trailing).unwrap(), b"hello");
}
#[test]
fn raw_deflate_fallback() {
let data = b"raw deflate without any zlib header, repeated repeated repeated".repeat(4);
let mut e = flate2::write::DeflateEncoder::new(Vec::new(), Compression::new(6));
e.write_all(&data).unwrap();
let raw = e.finish().unwrap();
assert!(zlib_header(&raw).is_none());
assert_eq!(dec(&raw).unwrap(), data);
}
#[test]
fn round_trip_and_missing_checksum() {
let data: Vec<u8> = (0..50_000u32).map(|i| (i * i % 251) as u8).collect();
let z = encode(&data);
assert_eq!(&z[..2], &[0x78, 0x9c]);
assert_eq!(dec(&z).unwrap(), data);
assert_eq!(dec(&z[..z.len() - 4]).unwrap(), data);
assert_eq!(dec(&z[..z.len() - 2]).unwrap(), data);
}
#[test]
fn empty_input() {
assert_eq!(dec(&encode(b"")).unwrap(), b"");
assert!(matches!(dec(b""), Err(FilterError::Corrupt { .. })));
}
#[test]
fn corrupt_and_truncated() {
let data: Vec<u8> = (0..10_000u32).map(|i| (i % 13) as u8).collect();
let z = encode(&data);
match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 {
Err(f) => {
assert!(matches!(f.error, FilterError::Corrupt { .. }));
assert_eq!(&data[..f.partial.len()], &f.partial[..]);
}
Ok(_) => panic!("truncated stream decoded"),
}
// BTYPE = 11 is reserved (RFC 1951 §3.2.3).
assert!(matches!(dec(&[0x78, 0x9c, 0x07, 0, 0, 0]), Err(FilterError::Corrupt { .. })));
// Preset dictionary.
assert!(matches!(dec(&[0x78, 0x20, 0, 0, 0, 0]), Err(FilterError::Corrupt { .. })));
// One whose header is followed by data that inflates: refused all the same.
assert!(matches!(dec(&[0x78, 0xBB, 0x4b, 0x04, 0x00]), Err(FilterError::Corrupt { .. })));
}
#[test]
fn bomb_is_bounded() {
let z = encode(&vec![0u8; 16 << 20]);
assert!(z.len() < 32 * 1024);
assert!(matches!(dec_limit(&z, 1 << 20), Err(FilterError::LimitExceeded(n)) if n == 1 << 20));
assert_eq!(dec_limit(&z, 16 << 20).unwrap().len(), 16 << 20);
assert!(matches!(dec_limit(&z, (16 << 20) - 1), Err(FilterError::LimitExceeded(_))));
assert!(matches!(dec_limit(&z, 0), Err(FilterError::LimitExceeded(0))));
}
fn dec_limit(d: &[u8], max: usize) -> Result<Vec<u8>, FilterError> {
decode(d, max, Some(max)).0.map_err(|f| f.error)
}
/// Deflate's bits: header fields and extra bits least significant first, Huffman codes most
/// significant first (RFC 1951 §3.1.1).
#[derive(Default)]
struct Bits {
out: Vec<u8>,
acc: u32,
n: u32,
}
impl Bits {
fn put(&mut self, value: u32, len: u32) {
for i in 0..len {
self.acc |= ((value >> i) & 1) << self.n;
self.n += 1;
if self.n == 8 {
self.out.push(self.acc as u8);
(self.acc, self.n) = (0, 0);
}
}
}
fn code(&mut self, code: u32, len: u32) {
for i in (0..len).rev() {
self.put((code >> i) & 1, 1);
}
}
fn finish(mut self) -> Vec<u8> {
if self.n > 0 {
self.out.push(self.acc as u8);
}
self.out
}
}
/// A dynamic block whose literal and length code has no end-of-block symbol can't end (RFC 1951
/// §3.2.5): it is refused, and nothing of it is kept.
#[test]
fn a_dynamic_code_without_an_end_is_damage() {
let mut bits = Bits::default();
bits.put(1, 1); // the final block,
bits.put(2, 2); // dynamic Huffman codes (§3.2.7)
bits.put(0, 5); // 257 literal and length codes
bits.put(0, 5); // 1 distance code
bits.put(14, 4); // 18 code length codes, in their order up to the one for 1
for symbol in [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1] {
bits.put(u32::from(symbol == 18 || symbol == 1), 3);
}
// Code length codes: 1 is 0, 18 is 1. 'A' and 'B' get a bit each, 256 none.
let zeros = |bits: &mut Bits, n: u32| {
bits.code(1, 1);
bits.put(n - 11, 7);
};
zeros(&mut bits, 65);
bits.code(0, 1);
bits.code(0, 1);
zeros(&mut bits, 138);
zeros(&mut bits, 52);
bits.code(0, 1); // the distance code: a bit
for _ in 0..4 {
bits.code(0, 1); // 'A'
}
let data = bits.finish();
assert!(zlib_header(&data).is_none(), "read as raw deflate");
match decode(&data, 1 << 20, Some(1 << 20)).0 {
Err(f) => {
assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error);
assert_eq!(f.partial, b"");
}
Ok(out) => panic!("read as {out:?}"),
}
}
/// A back-reference to before the start of the output points at nothing: read strictly the
/// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide,
/// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which
/// backend inflates (the price of reading Flate, `FLATE_READ_WORK`, is set for zlib-rs).
#[test]
fn a_copy_from_before_the_start_is_damage() {
let literal = |bits: &mut Bits, b: u8| bits.code(0x30 + u32::from(b), 8);
let mut bits = Bits::default();
bits.put(1, 1); // the final block,
bits.put(1, 2); // fixed Huffman codes (§3.2.6)
literal(&mut bits, b'A');
literal(&mut bits, b'B');
bits.code(1, 7); // length 3
bits.code(4, 5); // distance 5 (code 4 and an extra bit of 0), three bytes before the start
bits.put(0, 1);
literal(&mut bits, b'C');
bits.code(0, 7); // end of block
let data = bits.finish();
assert!(zlib_header(&data).is_none(), "read as raw deflate");
match decode(&data, 1 << 20, Some(1 << 20)).0 {
Err(f) => {
assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error);
assert_eq!(f.partial, b"AB");
}
Ok(out) => panic!("read as {out:?}"),
}
}
}
-467
View File
@@ -1,467 +0,0 @@
//! JBIG2 encoding (ITU-T T.88 | ISO/IEC 14492), for `JBIG2Decode` (ISO 32000-2 §7.4.7), lossless:
//! a generic region, template 0 with its nominal adaptive pixels, typical prediction on, MQ
//! arithmetic coding ([`crate::mq`]); or the image's marks as symbols ([`symbols`]), which a group
//! of images may share through `/JBIG2Globals`. Written from the standard's 6.2.5 (coding a generic
//! region), 7.2 (segment headers), 7.4.1, 7.4.6 and 7.4.8 (the segments) and D.3 (the embedded
//! organisation PDF uses). Decoding is in [`decode`].
mod decode;
mod symbols;
pub use decode::{MAX_JBIG2_PIXELS, decode_jbig2_within};
use crate::FilterError;
use crate::mq::{Context, MqEncoder};
/// The widest row, and the most rows, [`encode_jbig2_generic`] encodes: what `hayro-jbig2`, the
/// decoder behind `JBIG2Decode` here, takes on a side. JBIG2 allows 32-bit sizes, but past this
/// nothing here could read the stream, and the optimizer, which doesn't read back what it stores,
/// replaced such an image with one that draws nothing.
pub const MAX_JBIG2_COLUMNS: u32 = 65_535;
/// Template 0's pixels (Figure 3), as (x, y) offsets from the pixel being coded, each a bit of the
/// context in this order: the four on the current row, the five fixed ones on the row above, the
/// three on the row above that, then the adaptive pixels A1–A4 at their nominal places (Table 5).
const TEMPLATE: [(i32, i32); 16] = [
(-1, 0),
(-2, 0),
(-3, 0),
(-4, 0),
(-2, -1),
(-1, -1),
(0, -1),
(1, -1),
(2, -1),
(-1, -2),
(0, -2),
(1, -2),
(3, -1),
(-3, -1),
(2, -2),
(-2, -2),
];
/// The context that codes SLTP when template 0 is used (Figure 8): those pixel values, in
/// [`TEMPLATE`]'s order. Figure 8 reads, by row from x = -4: `0 1 0 1` on the current row,
/// `0 1 1 0 0 1 0` (x = -3 to 3) above it, `1 0 0 1 1` (x = -2 to 2) above that.
const SLTP: [u8; 16] = [1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1];
/// Columns of padding either side of a row, so every template pixel of a row's pixels is in it.
const PAD: usize = 4;
/// `rows` rows of `columns` 1-bit PDF samples as lossless JBIG2, the smaller of two codings:
/// [`encode_jbig2_generic`]'s, or one that stores each distinct mark of the image once, as a
/// symbol, and places its copies (see [`symbols`]). Either way the stream is in the embedded
/// organisation and decodes to the same bits without `/JBIG2Globals`.
///
/// Errors: as [`encode_jbig2_generic`].
pub fn encode_jbig2(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, FilterError> {
let generic = encode_jbig2_generic(data, columns, rows)?;
let samples = rows_to_encode(data, columns, rows).ok();
Ok(match samples.and_then(|s| symbols::encode(s, columns, rows)) {
Some(coded) if coded.len() < generic.len() => coded,
_ => generic,
})
}
/// `rows` rows of `columns` 1-bit PDF samples (most significant bit first, each row starting on
/// a new byte), as a JBIG2 stream in the embedded organisation: a page information segment and an
/// immediate lossless generic region segment, which `JBIG2Decode` decodes to the same bits
/// without `/JBIG2Globals`.
///
/// A 0 sample is coded as a JBIG2 1 (black), since `JBIG2Decode` gives black as 0, as a grey image's
/// samples have it. Bytes after the last row are ignored, as are the bits that pad a row to a whole
/// byte.
///
/// Errors: no rows or columns, more of either than [`MAX_JBIG2_COLUMNS`], or more samples than
/// [`MAX_JBIG2_PIXELS`], what the decoder takes ([`FilterError::Unsupported`]); fewer bytes than the
/// rows need ([`FilterError::Corrupt`]).
pub fn encode_jbig2_generic(data: &[u8], columns: u32, rows: u32) -> Result<Vec<u8>, FilterError> {
let samples = rows_to_encode(data, columns, rows)?;
let width = columns as usize;
let stride = width.div_ceil(8);
let mut contexts: Vec<Context> = vec![0; 1 << 16];
let mut mq = MqEncoder::new();
let height = samples.len() / stride;
code_generic(&mut mq, &mut contexts, width, height, true, |y, row| {
let packed = samples.get(y * stride..).unwrap_or_default();
for (x, pixel) in row.iter_mut().enumerate() {
let sample = packed.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0);
*pixel = u8::from(!sample);
}
});
let region = mq.finish();
let mut out = Vec::with_capacity(region.len() + 64);
segment(&mut out, 0, 48, &[], false, &page_information(columns, rows));
segment(&mut out, 1, 39, &[], false, &generic_region(columns, rows, 0, 0, &region));
Ok(out)
}
/// The samples of `rows` rows of `columns` 1-bit samples in `data`, when the encoders here take
/// that size; the errors are [`encode_jbig2_generic`]'s.
fn rows_to_encode(data: &[u8], columns: u32, rows: u32) -> Result<&[u8], FilterError> {
let too_large = columns > MAX_JBIG2_COLUMNS || rows > MAX_JBIG2_COLUMNS || u64::from(columns) * u64::from(rows) > MAX_JBIG2_PIXELS;
if columns == 0 || rows == 0 || too_large {
return Err(FilterError::Unsupported(format!("JBIG2 encoding of {columns} × {rows} samples")));
}
let stride = (columns as usize).div_ceil(8);
usize::try_from(rows)
.ok()
.and_then(|r| r.checked_mul(stride))
.and_then(|n| data.get(..n))
.ok_or_else(|| FilterError::Corrupt { filter: "JBIG2Decode", detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()) })
}
/// A group of images coded by [`encode_jbig2_shared`]: the data of the `/JBIG2Globals` stream
/// they share, and each image's own stream, in the order given, or `None` for one that shares
/// nothing with the others.
#[derive(Debug)]
pub struct SharedJbig2 {
pub globals: Vec<u8>,
pub images: Vec<Option<Vec<u8>>>,
}
/// Images, each `rows` rows of `columns` 1-bit PDF samples, as lossless JBIG2 sharing one symbol
/// dictionary: the marks found in more than one of them are stored once, in a symbol dictionary
/// segment not associated with a page, which is the `/JBIG2Globals` stream's data, and each image's
/// stream, in the embedded organisation, places them from there, with its other marks coded as
/// [`encode_jbig2`]'s symbol coding codes them (see [`symbols`]). With the globals, each decodes to
/// its own bits. An image that holds none of those marks, or that symbol coding doesn't take, has
/// no stream here; it is stored alone. `None` when the images share fewer than two marks.
///
/// What each image's stream saves is left for the caller to weigh against what the globals take,
/// which the images that use them share.
///
/// Errors: as [`encode_jbig2_generic`], for the first image it would refuse.
pub fn encode_jbig2_shared(images: &[(&[u8], u32, u32)]) -> Result<Option<SharedJbig2>, FilterError> {
let images =
images.iter().map(|(data, columns, rows)| Ok((rows_to_encode(data, *columns, *rows)?, *columns, *rows))).collect::<Result<Vec<_>, _>>()?;
Ok(symbols::encode_shared(&images).map(|(globals, images)| SharedJbig2 { globals, images }))
}
/// A page information segment's data (7.4.8): the page's size, resolution unknown, eventually
/// lossless, default pixel 0 (white), OR combination, not striped.
fn page_information(columns: u32, rows: u32) -> Vec<u8> {
let mut page = Vec::with_capacity(19);
page.extend_from_slice(&columns.to_be_bytes());
page.extend_from_slice(&rows.to_be_bytes());
page.extend_from_slice(&[0; 8]);
page.push(0x01);
page.extend_from_slice(&[0, 0]);
page
}
/// A region segment information field (7.4.1): a `width` × `height` region at (`x`, `y`) on the
/// page, combined with it by OR.
fn region_information(out: &mut Vec<u8>, width: u32, height: u32, x: u32, y: u32) {
for v in [width, height, x, y] {
out.extend_from_slice(&v.to_be_bytes());
}
out.push(0);
}
/// An immediate lossless generic region segment's data (7.4.6): its region information, flags
/// (MMR 0, template 0, TPGDON 1), the AT pixels at their nominal places, then `coded`.
fn generic_region(width: u32, height: u32, x: u32, y: u32, coded: &[u8]) -> Vec<u8> {
let mut data = Vec::with_capacity(26 + coded.len());
region_information(&mut data, width, height, x, y);
data.push(0x08);
at_pixels(&mut data);
data.extend_from_slice(coded);
data
}
/// Template 0's four AT pixels at their nominal places, as AT flags (7.4.6.3, 7.4.2.1.2).
fn at_pixels(out: &mut Vec<u8>) {
for (x, y) in &TEMPLATE[12..] {
out.push(*x as i8 as u8);
out.push(*y as i8 as u8);
}
}
/// A segment (7.2) of page 1: its header (number, type, the segments it refers to, the page, data
/// length), then its data. `retained`: a later segment refers to this one. `refers_to` holds each
/// referred-to segment's number and whether a segment after this one refers to it too (its
/// retain bit, 7.2.4). Segment numbers here stay below 256, so each referred-to one takes a byte
/// (7.2.5); a segment refers to at most four, so their count and retain bits take one byte too.
fn segment(out: &mut Vec<u8>, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) {
segment_of(out, 1, number, kind, refers_to, retained, data);
}
/// [`segment`], of page `page`: 0 for a segment associated with no page (7.2.6), such as the
/// global ones `/JBIG2Globals` holds, which a page's segments may refer to.
fn segment_of(out: &mut Vec<u8>, page: u8, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) {
out.extend_from_slice(&number.to_be_bytes());
out.push(kind);
// At most four, so the count fits its three bits.
let refers_to = refers_to.get(..refers_to.len().min(4)).unwrap_or_default();
let kept = refers_to.iter().enumerate().fold(0u8, |bits, (k, (_, kept))| bits | u8::from(*kept) << (k + 1));
out.push((refers_to.len() as u8) << 5 | kept | u8::from(retained));
out.extend(refers_to.iter().map(|(n, _)| *n));
out.push(page);
// A segment's data never comes near 4 GiB here: an image is at most 2^28 samples.
out.extend_from_slice(&u32::try_from(data.len()).unwrap_or(u32::MAX).to_be_bytes());
out.extend_from_slice(data);
}
/// Codes a `width` × `height` bitmap as a generic region (6.2.5.7) with template 0 and the nominal
/// AT pixels, its decisions in `contexts` (2^16 of them), with typical prediction when `typical`
/// (TPGDON). `fill(y, row)` sets row `y`'s `width` pixels, 1 for black. Pixels outside the bitmap
/// count as 0 (white), as the decoder takes them.
fn code_generic(mq: &mut MqEncoder, contexts: &mut [Context], width: usize, height: usize, typical: bool, mut fill: impl FnMut(usize, &mut [u8])) {
let sltp = SLTP.iter().enumerate().fold(0usize, |cx, (k, v)| cx | usize::from(*v) << k);
// The two rows above the current one and the current one, as JBIG2 pixels (1 black), padded.
let blank = vec![0u8; width + 2 * PAD];
let (mut above2, mut above1) = (blank.clone(), blank.clone());
let mut row = blank;
let mut ltp = false;
for y in 0..height {
if let Some(pixels) = row.get_mut(PAD..PAD + width) {
fill(y, pixels);
}
if typical {
// Typical prediction (6.2.5.5): a row the same as the one above is coded as one decision.
let same = row == above1;
if let Some(cx) = contexts.get_mut(sltp) {
mq.encode(cx, same != ltp);
}
ltp = same;
}
if !ltp {
for x in 0..width {
let at = |dx: i32, dy: i32| {
let line = match dy {
0 => &row,
-1 => &above1,
_ => &above2,
};
let i = (PAD + x).checked_add_signed(dx as isize);
i.and_then(|i| line.get(i)).copied().unwrap_or(0)
};
let cx = TEMPLATE.iter().enumerate().fold(0usize, |cx, (k, (dx, dy))| cx | usize::from(at(*dx, *dy)) << k);
if let Some(state) = contexts.get_mut(cx) {
mq.encode(state, row.get(PAD + x) == Some(&1));
}
}
}
std::mem::swap(&mut above2, &mut above1);
std::mem::swap(&mut above1, &mut row);
}
}
#[cfg(test)]
mod tests {
use super::*;
/// The rows an independent decoder (`hayro-jbig2`) reads from `data`, as PDF samples: true
/// for white (a JBIG2 0).
pub(super) fn decode(data: &[u8]) -> Vec<Vec<bool>> {
decode_with(data, None)
}
/// [`decode`], with `/JBIG2Globals`.
pub(super) fn decode_with(data: &[u8], globals: Option<&[u8]>) -> Vec<Vec<bool>> {
struct Rows(Vec<Vec<bool>>, Vec<bool>, usize);
impl hayro_jbig2::Decoder for Rows {
fn push_pixel(&mut self, black: bool) {
self.1.push(!black);
}
fn push_pixel_chunk(&mut self, black: bool, chunks: u32) {
self.1.extend(std::iter::repeat_n(!black, chunks as usize * 8));
}
fn next_line(&mut self) {
let mut row = std::mem::take(&mut self.1);
row.truncate(self.2);
self.0.push(row);
}
}
let image = hayro_jbig2::Image::new_embedded(data, globals).unwrap();
let mut rows = Rows(Vec::new(), Vec::new(), image.width() as usize);
image.decode(&mut rows).unwrap();
rows.0
}
/// The rows of packed samples `data`, true for a 1 bit.
pub(super) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec<Vec<bool>> {
let stride = (columns as usize).div_ceil(8);
(0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect()
}
/// Rows of `columns` samples from `runs`: alternating white and black runs one longer than
/// those values (never empty, even when shrinking makes them all 0), until the rows are full;
/// or, with `noise`, the bytes of `runs` as they are.
fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec<u8> {
let stride = (columns as usize).div_ceil(8);
let mut data = vec![0u8; stride * rows as usize];
if runs.is_empty() {
return data;
}
if noise {
data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r);
return data;
}
let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1);
for y in 0..rows as usize {
for x in 0..columns as usize {
if left == 0 {
white = !white;
left = 1 + usize::from(runs[next % runs.len()]);
next += 1;
}
if white {
data[y * stride + x / 8] |= 0x80 >> (x % 8);
}
left -= 1;
}
}
data
}
#[test]
fn a_page_of_text_like_marks_round_trips_and_shrinks() {
let (w, h) = (1700u32, 220u32);
let stride = (w as usize).div_ceil(8);
let mut data = vec![0xFF; stride * h as usize];
for y in (20..h as usize).step_by(40) {
for dy in 0..12 {
for x in (100..1600).step_by(9) {
if (x / 9 + dy) % 5 != 0 {
data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8));
}
}
}
}
let jbig2 = encode_jbig2_generic(&data, w, h).unwrap();
assert_eq!(decode(&jbig2), unpack(&data, w, h));
let g4 = crate::encode_g4(&data, w, h).unwrap();
assert!(jbig2.len() < g4.len(), "JBIG2 {} bytes, Group 4 {}", jbig2.len(), g4.len());
}
#[test]
fn edges_padding_and_typical_rows_round_trip() {
// 3 columns: the last 5 bits of each byte pad the row; set them to anything.
let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD];
assert_eq!(decode(&encode_jbig2_generic(&data, 3, 3).unwrap()), [[true, false, true], [false, true, false], [true, true, true]]);
for (w, h, fill) in [(1, 1, 0x00), (1, 1, 0xFF), (13, 9, 0x00), (13, 9, 0xFF), (64, 40, 0x5A)] {
let data = vec![fill; (w as usize).div_ceil(8) * h as usize];
assert_eq!(decode(&encode_jbig2_generic(&data, w, h).unwrap()), unpack(&data, w, h), "{w} × {h} of {fill:#04x}");
}
}
/// Packed samples from JBIG2 pixels (1 black): a 0 sample for each 1.
pub(super) fn samples(pixels: &[Vec<u8>]) -> Vec<u8> {
let width = pixels.first().map_or(0, Vec::len);
let stride = width.div_ceil(8);
let mut data = vec![0xFF; stride * pixels.len()];
for (y, row) in pixels.iter().enumerate() {
for (x, p) in row.iter().enumerate() {
if *p == 1 {
data[y * stride + x / 8] &= !(0x80 >> (x % 8));
}
}
}
data
}
/// A row the same as the one above is typical (6.2.5.5): one decision, not a row of pixels.
#[test]
fn a_repeated_row_is_one_decision() {
let row: Vec<u8> = (0..800u32).map(|x| u8::from((x * 7919 + x / 3) % 5 < 2)).collect();
let pixels = vec![row; 1000];
let data = samples(&pixels);
crate::mq::DECISIONS.with(|n| n.set(0));
let jbig2 = encode_jbig2_generic(&data, 800, 1000).unwrap();
// The first row's 800 pixels, and one typical-row decision for each of the 1000 rows.
assert_eq!(crate::mq::DECISIONS.with(|n| n.get()), 800 + 1000);
assert_eq!(decode(&jbig2), unpack(&data, 800, 1000));
}
/// SLTP is coded in the context Figure 8 shows, which ordinary pixels can share: an image
/// tiled with that very neighbourhood decodes only if both coders use the same context.
#[test]
fn the_typical_row_flag_shares_figure_8s_context() {
// Figure 8, around a pixel at x = 4 of an 8-column tile: two rows up x = 2..6, one row up
// x = 1..7, the pixel's own row x = 0..3.
let tile = [[0, 0, 1, 0, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 1, 0], [0, 1, 0, 1, 1, 0, 0, 0]];
let pixels: Vec<Vec<u8>> = (0..90).map(|y| (0..400).map(|x| tile[y % 3][x % 8]).collect()).collect();
let data = samples(&pixels);
assert_eq!(decode(&encode_jbig2_generic(&data, 400, 90).unwrap()), unpack(&data, 400, 90));
}
#[test]
fn what_it_cannot_encode_is_an_error() {
assert!(matches!(encode_jbig2_generic(&[0; 8], 0, 1), Err(FilterError::Unsupported(_))));
assert!(matches!(encode_jbig2_generic(&[0; 8], 8, 0), Err(FilterError::Unsupported(_))));
assert!(matches!(encode_jbig2_generic(&[0; 3], 16, 2), Err(FilterError::Corrupt { .. })), "two rows of 16 need 4 bytes");
assert!(matches!(encode_jbig2_generic(&[], 1, 2), Err(FilterError::Corrupt { .. })));
}
/// The encoder stops where the decoder does, on each side and in total: past that it would
/// write a stream nothing here can read. 16,384 squared is exactly the bound on samples.
/// `encode_jbig2_shared` refuses what `encode_jbig2_generic` refuses: an image taller than
/// the decoder takes is `Unsupported`, one at the bound isn't refused. (Its one tall mark is
/// too large to be a symbol, so nothing is shared: `Ok(None)`.)
#[test]
fn shared_coding_takes_only_what_the_decoder_takes() {
let past = vec![0x7F; MAX_JBIG2_COLUMNS as usize + 1];
let tall = (&past[..], 1, MAX_JBIG2_COLUMNS + 1);
assert!(matches!(encode_jbig2_shared(&[tall, tall]), Err(FilterError::Unsupported(_))), "too tall");
let at = (&past[..MAX_JBIG2_COLUMNS as usize], 1, MAX_JBIG2_COLUMNS);
assert!(matches!(encode_jbig2_shared(&[at, at]), Ok(None)), "at the bound");
}
#[test]
fn it_encodes_only_what_the_decoder_takes() {
let unsupported = |columns, rows| matches!(encode_jbig2_generic(&[], columns, rows), Err(FilterError::Unsupported(_)));
assert!(unsupported(MAX_JBIG2_COLUMNS + 1, 1), "too wide");
assert!(unsupported(1, MAX_JBIG2_COLUMNS + 1), "too tall");
assert!(unsupported(16_385, 16_384), "too many samples, each side within the bound");
assert!(unsupported(u32::MAX, u32::MAX), "the product doesn't overflow");
// At the bounds what it writes reads back, through the decoder that has to read it.
let wide = vec![0x0F; (MAX_JBIG2_COLUMNS as usize).div_ceil(8) * 4];
assert_eq!(decode(&encode_jbig2_generic(&wide, MAX_JBIG2_COLUMNS, 4).unwrap()), unpack(&wide, MAX_JBIG2_COLUMNS, 4));
let tall = vec![0x7F; MAX_JBIG2_COLUMNS as usize];
assert_eq!(decode(&encode_jbig2_generic(&tall, 1, MAX_JBIG2_COLUMNS).unwrap()), unpack(&tall, 1, MAX_JBIG2_COLUMNS));
struct Rows(u32, u64);
impl hayro_jbig2::Decoder for Rows {
fn push_pixel(&mut self, black: bool) {
self.1 += u64::from(black);
}
fn push_pixel_chunk(&mut self, black: bool, chunks: u32) {
self.1 += u64::from(black) * u64::from(chunks) * 8;
}
fn next_line(&mut self) {
self.0 += 1;
}
}
// One black pixel, so a page that came back blank wouldn't pass.
let mut largest = vec![0xFF; 2_048 * 16_384];
largest[0] = 0x7F;
let largest = encode_jbig2_generic(&largest, 16_384, 16_384).unwrap();
let mut rows = Rows(0, 0);
hayro_jbig2::Image::new_embedded(&largest, None).unwrap().decode(&mut rows).unwrap();
assert_eq!(rows.0, 16_384);
assert_eq!(rows.1, 1);
}
proptest::proptest! {
/// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder.
#[test]
fn bitmaps_round_trip(
columns in 1u32..300,
rows in 1u32..24,
runs in proptest::collection::vec(0u8..=255, 1..48),
noise in proptest::bool::ANY,
) {
let data = bitmap(columns, rows, &runs, noise);
let jbig2 = encode_jbig2_generic(&data, columns, rows).unwrap();
proptest::prop_assert_eq!(decode(&jbig2), unpack(&data, columns, rows));
}
/// Whatever it is given, it returns.
#[test]
fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) {
let _ = encode_jbig2_generic(&data, columns, rows);
}
}
}
-607
View File
@@ -1,607 +0,0 @@
//! JBIG2 decoding for `JBIG2Decode` (ISO 32000-2 §7.4.7), through `hayro-jbig2`.
use crate::FilterError;
use crate::ccitt::Rows;
/// The largest JBIG2 page [`decode_jbig2_within`] decodes, in pixels. The decoder decodes a page whole,
/// and takes pages of up to this many (20,000 × 10,000 decodes). It is also the most
/// [`encode_jbig2_generic`](crate::encode_jbig2_generic) encodes, so what it stores reads back.
pub const MAX_JBIG2_PIXELS: u64 = 1 << 28;
/// A `JBIG2Decode` stream's `data`, with what its `/JBIG2Globals` stream decodes to, decoded as the
/// filter gives it: `rows` rows of `columns` 1-bit samples, most significant bit first, each
/// starting on a new byte, with black as 0 (JBIG2 codes black as 1).
///
/// The JBIG2 page has to be the image's size, `columns` × `rows`: PdfCraft's renderer draws the
/// whole page into the image, so the rows of a page of another height wouldn't be the image's. The
/// decoder decodes the page whole, so it is checked before it is decoded, and has to fit in `max`
/// bytes as such rows and have at most [`MAX_JBIG2_PIXELS`]. Regions inside it are bounded by the
/// decoder alone: a region may claim more pixels than its page, and is decoded before it is cut to
/// the page. With no rows asked for, nothing is decoded.
///
/// Errors: `data` or `globals` that don't decode, or a page of another size
/// ([`FilterError::Corrupt`]); a page past `max` ([`FilterError::LimitExceeded`]) or
/// [`MAX_JBIG2_PIXELS`] ([`FilterError::Unsupported`], or `Corrupt` from the decoder, which has the
/// same cap and can refuse the page first).
///
/// Decoding spends from `work`: what decoding may still cost across a run of images, in the
/// decoder's count of steps. There is no way in without it, so a run of images can't decode them
/// without sharing an allowance. Reading `data` and `globals` is charged first
/// ([`hayro_jbig2::charge_reading`]), so an image that turns out not to decode has paid for that,
/// then what decoding takes ([`hayro_jbig2::Image::decode_within`]). An image takes at most what
/// one decode may on top of its reading and its page, whatever `work` allows, and one that `work`
/// can't pay for spends all of it. With nothing left an image isn't read at all, so the images after
/// that one cost nothing but this check, however large their globals.
pub fn decode_jbig2_within(data: &[u8], globals: Option<&[u8]>, columns: u32, rows: u32, max: usize, work: &mut u64) -> Result<Vec<u8>, FilterError> {
if rows == 0 {
return Ok(Vec::new());
}
if *work == 0 {
return Err(FilterError::Spent);
}
hayro_jbig2::charge_reading(work, data.len().saturating_add(globals.map_or(0, <[u8]>::len))).map_err(|_| FilterError::Spent)?;
let corrupt = |detail: String| FilterError::Corrupt { filter: "JBIG2Decode", detail };
let image = hayro_jbig2::Image::new_embedded(data, globals).map_err(|e| corrupt(e.to_string()))?;
let (width, height) = (image.width(), image.height());
if (width, height) != (columns, rows) {
return Err(corrupt(format!("a page of {width} × {height} pixels for an image of {columns} × {rows}")));
}
if u64::from(width) * u64::from(height) > MAX_JBIG2_PIXELS {
return Err(FilterError::Unsupported(format!("JBIG2 pages of {width} × {height} pixels")));
}
// u32 to usize: wasm32 and up have at least 32-bit usize.
let (width, height) = (width as usize, height as usize);
if width.div_ceil(8).checked_mul(height).is_none_or(|bytes| bytes > max) {
return Err(FilterError::LimitExceeded(max));
}
let mut out = Rows::new(width, height, false);
// Stopped by what was left, it isn't a damaged image, whatever the decoder calls it.
image.decode_within(&mut out, work).map_err(|e| if *work == 0 { FilterError::Spent } else { corrupt(e.to_string()) })?;
Ok(out.out)
}
/// JBIG2 codes black as 1, which `JBIG2Decode` gives as 0.
impl hayro_jbig2::Decoder for Rows {
fn push_pixel(&mut self, black: bool) {
self.push(!black, 1);
}
fn push_pixel_chunk(&mut self, black: bool, chunks: u32) {
self.push(!black, (chunks as usize).saturating_mul(8));
}
fn next_line(&mut self) {
self.end_row();
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{encode_jbig2, encode_jbig2_generic};
/// [`decode_jbig2_within`] with no limit on the work.
fn decode_jbig2(data: &[u8], globals: Option<&[u8]>, columns: u32, rows: u32, max: usize) -> Result<Vec<u8>, FilterError> {
let mut unbounded = u64::MAX;
decode_jbig2_within(data, globals, columns, rows, max, &mut unbounded)
}
/// `rows` rows of `columns` samples of noise, the bits that pad each row to a byte 0.
fn noise(columns: u32, rows: u32) -> Vec<u8> {
let stride = (columns as usize).div_ceil(8);
let mut seed = 0x2545_F491_4F6C_DD1Du64;
let mut data: Vec<u8> = (0..stride * rows as usize)
.map(|_| {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed as u8
})
.collect();
let pad = (stride * 8 - columns as usize) as u32;
data.chunks_mut(stride).for_each(|row| *row.last_mut().unwrap() &= 0xFFu8.checked_shl(pad).unwrap_or(0));
data
}
/// The same stream with its page information (the first segment's data, after its 11-byte
/// header: width, then height) claiming a page of `width` × `height`.
fn page_of(data: &[u8], width: u32, height: u32) -> Vec<u8> {
let mut data = data.to_vec();
data[11..15].copy_from_slice(&width.to_be_bytes());
data[15..19].copy_from_slice(&height.to_be_bytes());
data
}
/// The JBIG2 file header's ID string (ISO 14492 D.4.1).
const FILE_HEADER_ID: [u8; 8] = [0x97, 0x4A, 0x42, 0x32, 0x0D, 0x0A, 0x1A, 0x0A];
/// `embedded`, segments as [`encode_jbig2_generic`] writes them (11-byte headers: no references,
/// a one-byte page association), behind a file header with `flags` and the page count `pages`,
/// in the organization `flags` declares (D.4.2, bit 0): 1 sequential, each segment's header
/// before its data (D.1); 0 random access, every header first, the last an end of file segment
/// numbered 2, which tells where they end, then every segment's data in the same order (D.2).
fn with_file_header(embedded: &[u8], flags: u8, pages: Option<u32>) -> Vec<u8> {
let mut data = FILE_HEADER_ID.to_vec();
data.push(flags);
data.extend(pages.map(u32::to_be_bytes).into_iter().flatten());
if flags & 0x01 != 0 {
data.extend_from_slice(embedded);
return data;
}
let (mut headers, mut bodies, mut rest) = (Vec::new(), Vec::new(), embedded);
while !rest.is_empty() {
let (header, after) = rest.split_at(11);
assert_eq!(header[5], 0, "no references");
let (body, after) = after.split_at(u32::from_be_bytes(header[7..11].try_into().unwrap()) as usize);
headers.extend_from_slice(header);
bodies.extend_from_slice(body);
rest = after;
}
headers.extend_from_slice(&[0, 0, 0, 2, 51, 0, 1, 0, 0, 0, 0]);
data.extend(headers);
data.extend(bodies);
data
}
/// A stream that starts with a JBIG2 file header (ISO 14492 D.4), which some producers write
/// into `JBIG2Decode` data although ISO 32000-1 7.4.7 has the embedded organization there,
/// decodes past it as Acrobat and PDFBox draw it (hayro-jbig2 patch 17), in either organization
/// the header declares, with the page count and without it. "Bit 0: File organisation type. If
/// this bit is 0, the file uses the random-access organisation. If this bit is 1, the file uses
/// the sequential organisation." (D.4.2) A header with its reserved flag bits set is an error,
/// not a panic.
#[test]
fn a_stream_with_a_jbig2_file_header_decodes_past_it() {
let (w, h) = (203u32, 41u32);
let bits = noise(w, h);
let max = bits.len();
let embedded = encode_jbig2_generic(&bits, w, h).unwrap();
let decode = |flags: u8, pages: Option<u32>| decode_jbig2(&with_file_header(&embedded, flags, pages), None, w, h, max);
// Sequential organization (bit 0 is 1), one page.
assert_eq!(decode(0x01, Some(1)).unwrap(), bits, "sequential, page count given");
// Sequential, number of pages unknown (bit 1 is 1): no count follows (D.4.3).
assert_eq!(decode(0x03, None).unwrap(), bits, "sequential, page count unknown");
// Random-access organization (bit 0 is 0): the segment headers, then their data.
assert_ne!(with_file_header(&embedded, 0x00, Some(1))[13..], embedded[..], "laid out otherwise");
assert_eq!(decode(0x00, Some(1)).unwrap(), bits, "random access, page count given");
assert_eq!(decode(0x02, None).unwrap(), bits, "random access, page count unknown");
assert!(decode(0x11, Some(1)).is_err(), "reserved bits");
}
/// A file header cut short, or the segments after it, is an error and not a panic: the ID string
/// alone or in part, a header whose flags promise a page count with fewer than its four bytes,
/// a header with no segments after it, and segments cut anywhere in either organization. No
/// prefix of a whole stream decodes to its samples.
#[test]
fn a_truncated_jbig2_file_header_is_an_error() {
let (w, h) = (24u32, 8u32);
let bits = noise(w, h);
let max = bits.len();
let embedded = encode_jbig2_generic(&bits, w, h).unwrap();
let decode = |data: &[u8]| decode_jbig2(data, None, w, h, max);
assert!(decode(&FILE_HEADER_ID[..5]).is_err(), "part of the ID string");
assert!(decode(&FILE_HEADER_ID).is_err(), "the ID string, no flags");
for count in 0..4 {
let mut data = FILE_HEADER_ID.to_vec();
data.push(0x01);
data.extend_from_slice(&1u32.to_be_bytes()[..count]);
assert!(decode(&data).is_err(), "{count} bytes of the page count");
}
for (flags, pages) in [(0x01, Some(1)), (0x03, None), (0x00, Some(1)), (0x02, None)] {
let whole = with_file_header(&embedded, flags, pages);
let header = 9 + if pages.is_some() { 4 } else { 0 };
assert!(decode(&whole[..header]).is_err(), "flags {flags:#04x}: the header alone");
assert!(decode(&whole[..whole.len() - 1]).is_err(), "flags {flags:#04x}: the last byte cut");
for end in 0..whole.len() {
assert_ne!(decode(&whole[..end]).ok(), Some(bits.clone()), "flags {flags:#04x}: {end} bytes");
}
assert_eq!(decode(&whole).unwrap(), bits, "flags {flags:#04x}: whole");
}
}
/// The segments after a file header count toward hayro-jbig2's cap on the segments of a stream
/// together with its `/JBIG2Globals`, in either organization (patches 8 and 17): globals of
/// extension segments, which decode to nothing, fill the cap to the image's segments, and one
/// more is refused.
#[test]
fn segments_after_a_jbig2_file_header_count_with_the_globals() {
/// hayro-jbig2's `MAX_SEGMENTS`, the most segments of a stream with its globals.
const MAX_SEGMENTS: u32 = 1 << 20;
let (w, h) = (24u32, 8u32);
let bits = noise(w, h);
let max = bits.len();
let embedded = encode_jbig2_generic(&bits, w, h).unwrap();
// Numbered past the image's segments (0 to 2), of no page, with no data.
let globals = |count: u32| -> Vec<u8> {
(16..16 + count)
.flat_map(|number| {
let mut segment = number.to_be_bytes().to_vec();
segment.extend_from_slice(&[62, 0, 0, 0, 0, 0, 0]);
segment
})
.collect()
};
// The image's page information and region, and in random access its end of file segment.
for (flags, organization, image) in [(0x01, "sequential", 2), (0x00, "random access", 3)] {
let data = with_file_header(&embedded, flags, Some(1));
let (at_cap, past_cap) = (globals(MAX_SEGMENTS - image), globals(MAX_SEGMENTS - image + 1));
assert_eq!(decode_jbig2(&data, Some(&at_cap), w, h, max).unwrap(), bits, "{organization}: at the cap");
let refused = decode_jbig2(&data, Some(&past_cap), w, h, max).unwrap_err();
assert!(refused.to_string().contains("too many segments"), "{organization}: {refused}");
}
}
/// Either coding decodes to its samples, black as 0, the bits that pad each row 0: a generic
/// region of noise, and symbols for letters repeated across the page. With no rows asked for,
/// nothing is decoded.
#[test]
fn both_codings_decode_to_their_samples() {
let (w, h) = (203u32, 41u32);
let bits = noise(w, h);
let generic = encode_jbig2_generic(&bits, w, h).unwrap();
assert_eq!(decode_jbig2(&generic, None, w, h, usize::MAX).unwrap(), bits);
assert_eq!(decode_jbig2(b"not JBIG2", None, w, 0, usize::MAX).unwrap(), b"", "no rows asked for");
// Four lines of three letters, black on white.
const LETTERS: [[&str; 7]; 3] = [
["#####", "#....", "####.", "#....", "#....", "#....", "#####"],
[".###.", "#...#", "#....", "#....", "#....", "#...#", ".###."],
["#...#", "#...#", "#####", "#...#", "#...#", "#...#", "#...#"],
];
let (w, h) = (400u32, 40u32);
let stride = (w as usize).div_ceil(8);
let mut marks = vec![0xFFu8; stride * h as usize];
for (line, top) in [2usize, 12, 22, 32].into_iter().enumerate() {
for (k, x) in (3..w as usize - 6).step_by(7).enumerate() {
for (dy, row) in LETTERS[(k + line) % 3].iter().enumerate() {
for (dx, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'#') {
marks[(top + dy) * stride + (x + dx) / 8] &= !(0x80 >> ((x + dx) % 8));
}
}
}
}
let symbols = encode_jbig2(&marks, w, h).unwrap();
assert!(symbols.len() < encode_jbig2_generic(&marks, w, h).unwrap().len(), "symbol coded");
assert_eq!(decode_jbig2(&symbols, None, w, h, usize::MAX).unwrap(), marks);
}
/// A region's location is unsigned (ISO/IEC 14492 5.4.4), so one past `i32::MAX` is off the
/// page. The decoder took it for a signed one: a region at 0xFFFFFFFC came back onto the page
/// four pixels left of it, and a refinement anywhere past `i32::MAX` was refused, and the page
/// with it. Each stream is a page of 16 × 12 covered by a generic region, then a region of 8 × 6
/// of one kind past `i32::MAX`, coded by the standard's procedures, and decodes to that page
/// alone; moved onto the page, the region draws. Vendored hayro-jbig2 patch 14.
#[test]
fn a_region_located_past_i32_max_is_off_the_page() {
const PAGE: &str = "0000000030000100000013000000100000000c00000000000000004200000000000126000100000035000000100000000c0000000000000000040003fffdff02fefefed906aa2cb6ee4ccb4bc975d02669ca7a9c14c20c638f5a5e1bffac";
let cases = [
// At 0xFFFFFFFC, 1.
("generic", "00000002260001000000220000000800000006fffffffc00000001000003fffdff02fefefe17c876ae6f77ffac"),
// At 1, 0xFFFFFFFE, with its symbol dictionary.
(
"text",
"000000020000010000001e000003fffdff02fefefe00000001000000014fdda14149cec477ed5fffac00000003062002010000001b000000080000000600000001fffffffe00001000000001a943ffac",
),
// At 0xFFFFFFFC, 1, with its pattern dictionary.
(
"halftone",
"000000021000010000000a000101000000011fffac00000003162002010000002e0000000800000006fffffffc0000000100000000000800000006000000000000000001000000d09301774485ffac",
),
// At 0xFFFFFFFC, 1, reading the page there (7.4.7.4): 0s.
("refinement of the page", "000000022a00010000001e0000000800000006fffffffc000000010400ffffffff0bd98f188428ffac"),
// Of an intermediate region at the same place (7.4.7.5), 1, 0xFFFFFFFE.
(
"refinement of a region",
"0000000224000100000023000000080000000600000001fffffffe000003fffdff02fefefe3d81322da16b7fffac000000032a2002010000001e000000080000000600000001fffffffe0000ffffffffe29946b82229ffac",
),
// Of an intermediate refinement of an intermediate region, all at 0xFFFFFFFC, 1.
(
"refinement of a refinement",
"00000002240001000000230000000800000006fffffffc00000001000003fffdff02fefefecdfd921ec8c47fffac00000003282002010000001e0000000800000006fffffffc000000010000ffffffff903cdb447a47ffac000000042a2003010000001e0000000800000006fffffffc000000010000ffffffffdcd6078cab86ffac",
),
];
let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::<Vec<u8>>();
let decode = |segments: &str| decode_jbig2(&[hex(PAGE), hex(segments)].concat(), None, 16, 12, usize::MAX).unwrap();
let page = decode("");
assert!(page.iter().any(|&byte| byte != 0xFF), "the page has ink");
for (kind, segments) in cases {
assert_eq!(decode(segments), page, "{kind}");
// Moved onto the page, to 4, 1 or 1, 2, the same regions draw: they decode, and are
// left out for where they are.
let (off, on) = [("fffffffc00000001", "0000000400000001"), ("00000001fffffffe", "0000000100000002")]
.into_iter()
.find(|(off, _)| segments.contains(off))
.unwrap();
assert_ne!(decode(&segments.replace(off, on)), page, "{kind} on the page");
}
// A refinement at 2, 0 of an intermediate region at 0x80000000, 0. A refinement of a
// region refines it where it is, "GRREFERENCEDX 0, GRREFERENCEDY 0" (Table 35), and the two
// should be at one place (7.4.7.5). Where they aren't, it decodes as it does where they are.
const ELSEWHERE: &str = "000000022400010000002300000008000000068000000000000000000003fffdff02fefefe1bca3149568cdfffac000000032a2002010000001e000000080000000600000002000000000000ffffffff784a909cc761ffac";
let refined = decode(ELSEWHERE);
assert_eq!(refined, decode(&ELSEWHERE.replace("8000000000000000", "0000000200000000")), "the region at 2, 0 too");
assert_ne!(refined, page, "drawn");
}
/// A refinement of the page refines "the current contents of the page buffer, restricted to the
/// area of the page buffer specified by this segment's region segment information field"
/// (ISO/IEC 14492 7.4.7.4), at an offset of 0 (Table 35), and reads 0 outside that area
/// (6.3.5.2). The decoder read the page around the area too, and refused a refinement wider or
/// taller than the page. A refinement of another region refines that region, not the page. And
/// a reference 32 or more left of a refinement (GRREFERENCEDX below −31, as a symbol refined
/// with an RDX below −31 has, Table 18) is read from where each row starts in it: each row's
/// first context read 0 for the reference column it starts at, and from 33 left for the one
/// before it too. Each stream is coded by the standard's procedures and decodes to its
/// picture, `#` for black. Vendored hayro-jbig2 patch 16.
#[test]
fn a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is() {
let cases: [(&str, &str, &[&str]); 5] = [
(
"a 48 × 6 page a generic region has inked, refined at 33: ink on both sides of the area, template 0",
"000000003000010000001300000030000000060000000000000000420000000000012600010000004100000030000000060000000000000000040003fffdff02fefefe331433558a5c5056cf412003ec9a38164df39bd4463b9331502c4fb1c267ab77b665aa5962ffac000000022a0001000000210000000c0000000600000021000000000400ffffffff9a6c35bbeee221f687ffac",
&[
".#..##.######..#.#..#####.##..#........##.##..##",
".#..####.####.#...###.##.##.#.#.....#.#.####.###",
"##.#......#..#....#..#.###.#..#.#####.###.######",
"......#....###.#.##..#.##........#.#.#...#####..",
"#.##..###.#.#.##.....#...#.#..#.##.#..#...#...#.",
".###...#..#..#..##..#.#..#####.#.##.......###.##",
],
),
(
"a 24 × 8 inked page refined at 5, 1, template 1",
"000000003000010000001300000018000000080000000000000000420000000000012600010000003500000018000000080000000000000000040003fffdff02fefefeb1e133cb8bcbb8510e81d219bb880dfb40a311dc5c298819efffac000000022a00010000001b0000000a0000000500000005000000010401e55c0078dcd30fffac",
&[
"#.##.##...##....##.##..#",
"#.##.##...#.####..##..#.",
"##..##.##...#...###..#.#",
".....#.#.##.#...###....#",
"..#.#...#.#...#.#...##.#",
"....######...#.####....#",
"..######..##.#.#..##.#..",
"...#..##.####.#..##...#.",
],
),
(
"a 16 × 5 inked page refined by a region 4 wider than it, template 0",
"000000003000010000001300000010000000050000000000000000420000000000012600010000002600000010000000050000000000000000040003fffdff02fefefe22e5e253a429cbf2c701ffac000000022a000100000023000000140000000500000000000000000400ffffffff3afa516532bbe3d481926fffac",
&[".#.#.#.#..#.#.#.", "##.####..###...#", "..####.#..####.#", ".###.#.....#..#.", "#.###..#########"],
),
(
"an intermediate region at 4, 1 on an inked 20 × 8 page, refined there (OR): the region, not the page, template 0",
"000000003000010000001300000014000000080000000000000000420000000000012600010000003000000014000000080000000000000000040003fffdff02fefefeb91e434e1726ca89ed06c2bffb0fe5fa7fccc80fffac00000002240001000000230000000a000000050000000400000001000003fffdff02fefefece7a9826ba81bfffac000000032a2002010000001e0000000a0000000500000004000000010000ffffffffd046ee6f6190ffac",
&[
"#.#.#....#.#...#....",
"##..#.#.#.##.....##.",
"##.#.#.#.#.###.##...",
"###.##.####..###.#.#",
"...#.###..###.#.#..#",
"#...#######.#...####",
"..##.##...#..##..#.#",
"#...##...#.##....#..",
],
),
(
"a symbol refined in a symbol dictionary from a 40 × 6 one, RDX −33: 33 left, template 1",
"0000000030000100000013000000070000000600000000000000004200000000000100000100000036000003fffdff02fefefe00000001000000014f913296209e2187f37bddb8589c8beb7d70109de7124ebe47d04914ef7e655c3c4cffac00000002002001010000001f100203fffdff02fefefe00000001000000014fe451a27bddf49f7ef7e7ffac00000003062002010000001b0000000700000006000000000000000004001000000001a943ffac",
&[".####.#", ".##..##", "......#", "...#..#", "#.#.###", "....##."],
),
];
let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::<Vec<u8>>();
for (form, stream, picture) in cases {
let (w, h) = (picture[0].len(), picture.len());
// As the filter gives it: black as 0, each row padded to a byte with 0s.
let mut rows = vec![0u8; w.div_ceil(8) * h];
for (y, row) in picture.iter().enumerate() {
for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') {
rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8);
}
}
assert_eq!(decode_jbig2(&hex(stream), None, w as u32, h as u32, usize::MAX).unwrap(), rows, "{form}");
}
}
/// What a page costs is the same however much a run has left. Given exactly that it decodes,
/// and less stops it: spending all that's left, and what comes after isn't read. A page that
/// fails still spends what it took.
#[test]
fn a_run_of_images_spends_its_allowance_down_to_the_step() {
let (w, h) = (203u32, 41u32);
let data = encode_jbig2_generic(&noise(w, h), w, h).unwrap();
let whole = decode_jbig2(&data, None, w, h, usize::MAX).unwrap();
let mut work = u64::MAX;
assert_eq!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap(), whole);
let cost = u64::MAX - work;
assert!(cost >= u64::from(w) * u64::from(h), "a pixel at a time: {cost}");
let mut work = cost;
assert_eq!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap(), whole, "exactly enough");
assert_eq!(work, 0);
let mut work = cost + 7;
decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).unwrap();
assert_eq!(work, 7, "only what it took");
let mut work = cost - 1;
assert!(matches!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work), Err(FilterError::Spent)), "a step short");
assert_eq!(work, 0, "spends all that was left");
// Nothing left: not read at all, so this garbage isn't reported as garbage.
assert!(matches!(decode_jbig2_within(b"garbage", None, w, h, usize::MAX, &mut work), Err(FilterError::Spent)));
// Too little even for its page.
let mut work = 3;
assert!(decode_jbig2_within(&data, None, w, h, usize::MAX, &mut work).is_err());
assert_eq!(work, 0);
// A region that can't be read after the one that can: the page fails having spent the
// first, and the segment it fails on, and that isn't being out of allowance.
let bad_region = [0, 0, 0, 9, 38, 0, 1, 0, 0, 0, 1, 0xFF];
let mut work = u64::MAX;
assert!(decode_jbig2_within(&[&data[..], &bad_region].concat(), None, w, h, usize::MAX, &mut work).is_err());
assert_eq!(u64::MAX - work, cost + 1 + 4 * bad_region.len() as u64, "the first region, a segment more, and reading it");
// Not a stream at all: it pays for being read, and globals count with it.
let mut work = u64::MAX;
assert!(matches!(decode_jbig2_within(b"garbage", Some(b"globals"), w, h, usize::MAX, &mut work), Err(FilterError::Corrupt { .. })));
assert_eq!(u64::MAX - work, 4 * 14);
}
/// A refinement whose reference is offset right of it (GRREFERENCEDX above 0, as a refined text
/// instance has from a wider bitmap, ISO/IEC 14492 Table 12, and a refined symbol from an RDX
/// above 0, Table 18) reads the reference from its first column. Left of the reference the
/// decoder emptied what it had read of it, and at its first column didn't read it again, so its
/// first 32 columns read as 0: such a refinement decoded to other pixels than were coded, or
/// failed. Each stream is a refinement coded by the standard's procedures on a page of its own
/// size, and decodes to its picture, `#` for black. Vendored hayro-jbig2 patch 15.
#[test]
fn a_refinement_reads_a_reference_right_of_it_from_its_first_column() {
let cases: [(&str, &str, &[&str]); 4] = [
(
"a text instance of an 8 × 6 symbol refined 6 wider: 3 right, template 0",
"00000000300001000000130000000e000000060000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fdd3dba48bdddde35ffac00000002062001010000002c0000000e000000060000000000000000040012ffffffff00000001a96b4a6b78b7fc094fadde1b3e881fffac",
&[".#.#.#....#...", "##.###....##.#", ".###...###..#.", "#.......#...#.", "#.####.#...#..", "#.##...###.#.."],
),
(
"refined 2 wider and 2 taller, RDX and RDY 1: 2 right, 2 down, template 1",
"00000000300001000000130000000a000000080000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fdd9c52d3ea827a43ffac0000000206200101000000280000000a00000008000000000000000004801200000001a96c00cdac6dd5f6f848ce872411cfffac",
&["####.##...", "####..####", "####..##..", "...##...#.", "#......#..", "..#.....#.", "##....####", ".#........"],
),
(
"a symbol refined in a symbol dictionary, RDX 5, RDY −1, template 0 with its AT pixels moved",
"00000000300001000000130000000c000000060000000000000000420000000000010000010000001e000003fffdff02fefefe00000001000000014fdda13efc4029efc95fffac000000020020010100000027000203fffdff02fefefefeff010100000001000000014fc98c352dd84a906f120ffb49793fffac00000003062002010000001b0000000c00000006000000000000000004001000000001a943ffac",
&["#.#.##.#.###", "..#.#.....##", "##..#...#...", "..##....##..", "##.#.##.####", ".#..#####..#"],
),
(
"a text instance of a 40 × 5 symbol refined 2 wider, RDX 1: 2 right, across a word, template 0",
"00000000300001000000130000002a0000000500000000000000004200000000000100000100000031000003fffdff02fefefe00000001000000015240f590d953ae3f4069dede672090d4d61b377906d3f41aafa4f77107ffac00000002062001010000003c0000002a000000050000000000000000040012ffffffff00000001a96c0b9386d4b140486bdc3673dbd031490e468c7f7813d19068ab04a79cbfffac",
&[
"##.#.#..##.##.#####.#....#.#.#..#..######.",
".#.#..###....#..##.#..#.####.#..#.###...##",
".#.#......#....#####....#...###...#####...",
"#...##.##.......#...###.###...##..##.###.#",
"..#..###..###..#...#.......##.#........##.",
],
),
];
let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::<Vec<u8>>();
for (form, stream, picture) in cases {
let (w, h) = (picture[0].len(), picture.len());
// As the filter gives it: black as 0, each row padded to a byte with 0s.
let mut rows = vec![0u8; w.div_ceil(8) * h];
for (y, row) in picture.iter().enumerate() {
for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') {
rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8);
}
}
assert_eq!(decode_jbig2(&hex(stream), None, w as u32, h as u32, usize::MAX).unwrap(), rows, "{form}");
}
}
/// The page has to be the image's size, and is bounded before it is decoded, being decoded
/// whole: within `max` as rows of one bit, and within [`MAX_JBIG2_PIXELS`]. Globals are given
/// to the decoder: ones it skips decode, ones that aren't JBIG2 are an error, as is damaged
/// data, which never panics.
#[test]
fn what_it_cannot_decode_is_an_error() {
let (w, h) = (203u32, 41u32);
let data = encode_jbig2_generic(&noise(w, h), w, h).unwrap();
for (columns, rows) in [(w - 1, h), (w + 1, h), (w, h - 1), (w, h + 1)] {
let e = decode_jbig2(&data, None, columns, rows, usize::MAX).unwrap_err();
assert!(matches!(e, FilterError::Corrupt { .. }), "{columns} × {rows}: {e}");
}
let page = 26 * h as usize;
assert!(matches!(decode_jbig2(&data, None, w, h, page - 1), Err(FilterError::LimitExceeded(m)) if m == page - 1), "a byte short");
assert_eq!(decode_jbig2(&data, None, w, h, page).unwrap().len(), page, "the page within max");
// 16,384 × 16,384 is 2^28 pixels, which decodes; a row more is past the cap, here or in the
// decoder, which has the same one and can refuse it first.
let large = |rows| decode_jbig2(&page_of(&data, 16_384, rows), None, 16_384, rows, usize::MAX);
assert!(matches!(large(16_385), Err(FilterError::Unsupported(_) | FilterError::Corrupt { .. })), "{:?}", large(16_385));
assert!(large(16_384).is_ok(), "{:?}", large(16_384).map(|page| page.len()));
// An extension segment (ISO/IEC 14492 7.4.14), which a decoder skips: its 11-byte header
// gives the length of the data after it.
let skipped = [0, 0, 0, 0, 0x3E, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1];
let generic = decode_jbig2(&data, None, w, h, usize::MAX).unwrap();
assert_eq!(decode_jbig2(&data, Some(&skipped), w, h, usize::MAX).unwrap(), generic);
assert!(matches!(decode_jbig2(&data, Some(b"not JBIG2"), w, h, usize::MAX), Err(FilterError::Corrupt { .. })));
for end in 0..data.len() {
let _ = decode_jbig2(&data[..end], None, w, h, usize::MAX);
}
}
/// Pixels past a bitmap's edge are 0 (ISO/IEC 14492 6.3.5.2). A page whose default pixel is 1,
/// and an intermediate text or halftone region whose default pixel is 1, kept 1s past their
/// width in their last word of each row, and a refinement, which reads its reference a word at
/// a time, read them as pixels: of rows not a multiple of 32 wide, it decoded to other pixels
/// than were coded. Each stream is a refinement coded by the standard's procedures from a
/// reference read as 0 past its edge, and decodes to the picture given, `#` for black.
/// Vendored hayro-jbig2 patch 13.
#[test]
fn a_refinement_reads_nothing_past_its_references_width() {
// A page of 7 × 6, then of 33 × 6, whose default pixel is 1, refined whole.
const PAGE_7: &str = "000000003000010000001300000007000000060000000000000000460000000000012a00010000001e000000070000000600000000000000000400ffffffffe42d7e889ebcffac";
const PAGE_33: &str = "000000003000010000001300000021000000060000000000000000460000000000012a000100000034000000210000000600000000000000000400ffffffff314a8d598037eb3f915ab3d526ccae915ad6a6a7b814111b124ae60fffac";
// An intermediate text region of 31 × 6 whose default pixel is 1, then one of 7 × 6 on a
// page of 40, each refined into the page.
const TEXT_31: &str = "00000000300001000000130000001f0000000600000000000000004200000000000100000100000030000003fffdff02fefefe00000001000000014f96dceb76c7f1ff585971969cd9f88e538261a6f713be68d77e8e1fffac00000002042001010000001b0000001f00000006000000000000000000029000000001a943ffac000000032a2002010000002f0000001f0000000600000000000000000400ffffffff9ac311688adbc8f570a2f96aef93d332e3b1f4b27b704bffac";
const TEXT_7: &str = "000000003000010000001300000028000000060000000000000000420000000000010000010000001d000003fffdff02fefefe00000001000000014fe1f79f4a95c5b77fffac00000002042001010000001b0000000700000006000000000000000000029000000001a943ffac000000032a2002010000001e000000070000000600000000000000000400ffffffffa71dae12bd7fffac";
// An intermediate halftone region of 33 × 6 whose default pixel is 1, refined into the page.
const HALFTONE_33: &str = "000000003000010000001300000021000000060000000000000000420000000000011000010000000a000101000000011fffac00000002142001010000004100000021000000060000000000000000009000000021000000060000000000000000010000004ea6618f551d48ff24c2c5c98124d449f2adea6fca76e4297fffac000000032a20020100000032000000210000000600000000000000000400ffffffffe87db809393a3251e565e7a749e6470ab37f7aef186ceaf5167bffac";
let cases: [(&str, &str, [&str; 6]); 5] = [
("page 7", PAGE_7, ["###.#.#", ".......", "#.####.", "######.", "#.#...#", ".#...##"]),
(
"page 33",
PAGE_33,
[
".#..##........###..####....#.##.#",
".#.###.###.##.##..#..#...#.....##",
".....##..#....#.##.######.###..#.",
"##...#..##.###..###.##..#.#.#.#.#",
".##.#........##.#.#.....###......",
"...#.#####.#..#.###......###..###",
],
),
(
"text 31",
TEXT_31,
[
"....##....###......#.##.#.#....",
".#.....##.......##..###.#...#..",
".#.#.#.####...#####.####...#.#.",
"......#...#...#.##...##...#..#.",
".#..##.###.#.###..####...#...#.",
"#...#...#..#.#....#.##.#.....#.",
],
),
(
"text 7 on a page of 40",
TEXT_7,
[
"......#.................................",
".....##.................................",
"##...##.................................",
"..#.....................................",
"##......................................",
"..#.....................................",
],
),
(
"halftone 33",
HALFTONE_33,
[
"##...#.###.#.###.....#.####..####",
"##.##.#...#.##.##.##.#...#.#.###.",
"###..#.#....##.###...#..##.##..##",
"##..#...###...##..#.####.###.#...",
"####.###.###.#..#..#.#.####...###",
"#.#..#........#....#.#.##.##..#.#",
],
),
];
let hex = |text: &str| (0..text.len() / 2).map(|i| u8::from_str_radix(&text[i * 2..i * 2 + 2], 16).unwrap()).collect::<Vec<u8>>();
for (form, stream, picture) in cases {
let w = picture[0].len();
// As the filter gives it: black as 0, each row padded to a byte with 0s.
let mut rows = vec![0u8; w.div_ceil(8) * picture.len()];
for (y, row) in picture.iter().enumerate() {
for (x, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'.') {
rows[y * w.div_ceil(8) + x / 8] |= 0x80 >> (x % 8);
}
}
assert_eq!(decode_jbig2(&hex(stream), None, w as u32, 6, usize::MAX).unwrap(), rows, "{form}");
}
}
}
File diff suppressed because it is too large Load Diff
+2 -958
View File
@@ -1,958 +1,2 @@
//! PDF stream filters (ISO 32000-2 §7.4).
//!
//! Layer L0, standalone: this crate depends on no other PdfCraft crate.
//!
//! Decoding covers the general-purpose filters (`FlateDecode`, `LZWDecode` with
//! TIFF/PNG predictors, `ASCIIHexDecode`, `ASCII85Decode`, `RunLengthDecode`).
//! Image codecs (`DCTDecode`, `JPXDecode`, `JBIG2Decode`, `CCITTFaxDecode`),
//! `Crypt` and unknown filters are reported as [`FilterError::Unsupported`] so
//! callers keep such data encoded. CCITT fax and JBIG2 images decode on their own
//! ([`decode_ccitt_within`], [`decode_jbig2_within`]), each spending from an allowance its caller
//! shares across a run of images, as `pdfcraft_cos::ImageBudget::decode` does after the filters
//! before them. Neither decodes without one:
//!
//! ```compile_fail
//! let _ = pdfcraft_filters::decode_jbig2(&[], None, 1, 1, 1 << 20);
//! ```
//!
//! ```compile_fail
//! let _ = pdfcraft_filters::decode_ccitt(&[], &pdfcraft_filters::CcittParams::default(), 1);
//! ```
//!
//! ```
//! let mut work = 1 << 20;
//! assert!(pdfcraft_filters::decode_jbig2_within(&[], None, 1, 1, 1 << 20, &mut work).is_err());
//! let p = pdfcraft_filters::CcittParams { columns: 8, ..Default::default() };
//! assert!(pdfcraft_filters::decode_ccitt_within(&[], &p, 1, &mut work).is_ok());
//! ```
//!
//! 1-bit samples can be encoded as CCITT Group 4 ([`encode_g4`]) or lossless JBIG2
//! ([`encode_jbig2`]), a group of images sharing their marks through `/JBIG2Globals`
//! ([`encode_jbig2_shared`]).
//!
//! Every decoder is bounded by a caller-supplied `max_output` (decompression-bomb
//! defence) and never panics on malformed input.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod ascii85;
mod asciihex;
mod ccitt;
mod flate;
mod jbig2;
mod lzw;
mod mq;
mod predictor;
mod runlength;
use std::borrow::Cow;
pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt_within, encode_g4};
pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, SharedJbig2, decode_jbig2_within, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared};
/// A PDF stream filter, identified by its `/Filter` name.
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
pub enum Filter {
Flate,
Lzw,
AsciiHex,
Ascii85,
RunLength,
Dct,
Jpx,
Jbig2,
CcittFax,
Crypt,
/// Any other name (kept verbatim, lossily converted to UTF-8).
Unknown(String),
}
impl Filter {
/// Parses a filter name. Accepts the full names (ISO 32000-2 Table 6) and the
/// inline-image abbreviations (Table 92): `Fl`, `LZW`, `AHx`, `A85`, `RL`, `DCT`, `CCF`.
/// A leading `/` is ignored.
pub fn from_name(name: &[u8]) -> Filter {
let n = name.strip_prefix(b"/").unwrap_or(name);
match n {
b"FlateDecode" | b"Fl" => Filter::Flate,
b"LZWDecode" | b"LZW" => Filter::Lzw,
b"ASCIIHexDecode" | b"AHx" => Filter::AsciiHex,
b"ASCII85Decode" | b"A85" => Filter::Ascii85,
b"RunLengthDecode" | b"RL" => Filter::RunLength,
b"DCTDecode" | b"DCT" => Filter::Dct,
b"JPXDecode" => Filter::Jpx,
b"JBIG2Decode" => Filter::Jbig2,
b"CCITTFaxDecode" | b"CCF" => Filter::CcittFax,
b"Crypt" => Filter::Crypt,
other => Filter::Unknown(String::from_utf8_lossy(other).into_owned()),
}
}
/// The full PDF name (without the leading `/`).
pub fn name(&self) -> &str {
match self {
Filter::Flate => "FlateDecode",
Filter::Lzw => "LZWDecode",
Filter::AsciiHex => "ASCIIHexDecode",
Filter::Ascii85 => "ASCII85Decode",
Filter::RunLength => "RunLengthDecode",
Filter::Dct => "DCTDecode",
Filter::Jpx => "JPXDecode",
Filter::Jbig2 => "JBIG2Decode",
Filter::CcittFax => "CCITTFaxDecode",
Filter::Crypt => "Crypt",
Filter::Unknown(s) => s,
}
}
/// True for the image codecs (DCT, JPX, JBIG2, CCITT fax).
pub fn is_image_codec(&self) -> bool {
matches!(self, Filter::Dct | Filter::Jpx | Filter::Jbig2 | Filter::CcittFax)
}
/// True for the general-purpose filters, which a chain can hold ([`decode`]).
pub fn is_decodable(&self) -> bool {
matches!(self, Filter::Flate | Filter::Lzw | Filter::AsciiHex | Filter::Ascii85 | Filter::RunLength)
}
}
/// `/DecodeParms` values relevant to the general-purpose filters (ISO 32000-2 Tables 8–9).
#[derive(Clone, Debug, PartialEq)]
pub struct Params {
/// 1 = none, 2 = TIFF predictor 2, 10–15 = PNG predictors.
pub predictor: i64,
pub colors: i64,
pub bits_per_component: i64,
pub columns: i64,
/// LZW only: 1 (default) switches code width one code early, 0 does not.
pub early_change: i64,
}
impl Default for Params {
fn default() -> Self {
Params { predictor: 1, colors: 1, bits_per_component: 8, columns: 1, early_change: 1 }
}
}
/// Errors from decoding or encoding.
#[derive(Debug, thiserror::Error)]
pub enum FilterError {
#[error("{filter}: corrupt data: {detail}")]
Corrupt { filter: &'static str, detail: String },
#[error("decoded output exceeds the limit of {0} bytes")]
LimitExceeded(usize),
#[error("unsupported filter: {0}")]
Unsupported(String),
/// A run of images has nothing left to spend ([`decode_tolerant_within`]).
#[error("more image decoding than one operation may do")]
Spent,
}
/// A decoder failure carrying whatever was decoded before the error.
pub(crate) struct Failure {
pub(crate) error: FilterError,
pub(crate) partial: Vec<u8>,
/// Bytes decoded on the way that `partial` doesn't hold: they took as long.
pub(crate) discarded: usize,
}
impl Failure {
pub(crate) fn corrupt(filter: &'static str, detail: impl Into<String>, partial: Vec<u8>) -> Self {
Failure { error: FilterError::Corrupt { filter, detail: detail.into() }, partial, discarded: 0 }
}
pub(crate) fn limit(max: usize) -> Self {
Failure { error: FilterError::LimitExceeded(max), partial: Vec::new(), discarded: 0 }
}
}
pub(crate) type Step = Result<Vec<u8>, Failure>;
/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends
/// all of it.
pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> {
match work.checked_sub(steps) {
Some(left) => {
*work = left;
Ok(())
}
None => {
*work = 0;
Err(FilterError::Spent)
}
}
}
/// Decodes `data` through `chain` (first element applied first).
///
/// Image codecs, `Crypt` and unknown filters anywhere in the chain give
/// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than
/// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer.
pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<Vec<u8>, FilterError> {
run(chain, data, max_output, None, true, false, None).map(|(v, _, _)| v).map_err(|(e, _)| e)
}
/// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages
/// decode together, each stage getting what the earlier ones left, and the result gives those
/// bytes, on success and on failure. A stage's output counts even when a later stage or its
/// predictor shrinks it.
pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, usize), (FilterError, usize)> {
run(chain, data, max_output, Some(max_output), true, false, None).map(|(v, _, spent)| (v, spent))
}
/// Like [`decode`], but on corrupt data returns what was decoded before the error and
/// reports `partial = true` (the behaviour viewers expect). The remaining filters of the
/// chain still run on the partial data. Limit and unsupported errors are still errors.
pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, bool), FilterError> {
run(chain, data, max_output, None, false, false, None).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e)
}
/// [`decode_tolerant`] bounded and counted as [`decode_counted`] is.
pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
run(chain, data, max_output, Some(max_output), false, false, None)
}
/// [`decode_tolerant_counted`] with the two bounds apart: `max_stage` bounds every stage, as
/// [`decode_tolerant`]'s bound does, and `max_output` the bytes all stages decode together. A
/// stage stopped by either gives [`FilterError::LimitExceeded`] and the bytes decoded before it.
pub fn decode_tolerant_counted_each(
chain: &[(Filter, Params)],
data: &[u8],
max_stage: usize,
max_output: usize,
) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
run(chain, data, max_stage, Some(max_output), false, false, None)
}
/// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images:
/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each
/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each
/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for
/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of
/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all
/// costs nothing, whatever is left, so what an image's filters are is known from the error.
///
/// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`.
pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result<Vec<u8>, FilterError> {
spending(chain, data, max_output, work, false, None)
}
/// [`decode_tolerant_within`] for the samples of an image `rows` high (its `/Height`), which the
/// last stage gives. When that stage is Flate with a PNG predictor and gives exactly `rows` rows
/// without the filter-type byte each predicted row starts with (ISO 32000-2 §7.4.4.4), its writer
/// declared the predictor without applying it: the output is taken as the samples, as other
/// readers take it, and reported as the damage it is, like any other corruption tolerated here.
/// Any other output has its predictor undone as [`decode_tolerant_within`] does.
pub fn decode_image_tolerant_within(
chain: &[(Filter, Params)],
data: &[u8],
max_output: usize,
rows: u32,
work: &mut u64,
) -> Result<Vec<u8>, FilterError> {
spending(chain, data, max_output, work, false, usize::try_from(rows).ok())
}
/// [`decode`], spending from `work` as [`decode_tolerant_within`] does: corrupt data is an error,
/// having paid for what was decoded before it.
pub fn decode_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result<Vec<u8>, FilterError> {
spending(chain, data, max_output, work, true, None)
}
fn spending(
chain: &[(Filter, Params)],
data: &[u8],
max_output: usize,
work: &mut u64,
strict: bool,
rows: Option<usize>,
) -> Result<Vec<u8>, FilterError> {
decodable(chain)?;
pay(work, data.len() as u64)?;
if chain.is_empty() {
return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) };
}
let left = usize::try_from(*work).unwrap_or(usize::MAX);
match run(chain, data, max_output, Some(left), strict, true, rows) {
Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out),
// The stage stopped at the lower of the two, which took as much to find out.
Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => {
pay(work, (decoded as u64).saturating_add(max_output as u64))?;
Err(FilterError::LimitExceeded(max_output))
}
// Past what is left, a stage stops, or a Flate stage doesn't start.
Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => {
*work = 0;
Err(FilterError::Spent)
}
Err((e, decoded)) => pay(work, decoded as u64).and(Err(e)),
}
}
/// Longest filter chain decoded. Real streams use one to three filters; every stage may produce
/// up to the output limit, so the chain length multiplies the work.
pub const MAX_CHAIN: usize = 8;
/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`],
/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes.
/// Inflating does work for every block it reads, whether or not the block makes anything: an empty
/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs
/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read
/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the
/// worst per byte read comes to about 7 ns a step, inside what a step of JBIG2 decoding takes.
/// (miniz_oxide 0.9.1, flate2's backend here before, took about 1.7 µs for every fixed-Huffman
/// block, empty or not: 1,383 ns a byte read on empty ones.) A stage's start isn't priced, about
/// 0.25 µs: every way to repeat one takes bytes of the file, so what starts cost is bounded by the
/// file's length. Data a real encoder makes took 0.2 to 2.3 ns a byte made, its reading included, so
/// an image pays twice its stored length more than it would unpriced.
pub const FLATE_READ_WORK: u64 = 2;
/// Whether a chain is one [`run`] takes, and if not, the error.
fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> {
if chain.len() > MAX_CHAIN {
return Err(FilterError::Unsupported(format!("a chain of {} filters", chain.len())));
}
match chain.iter().find(|(f, _)| !f.is_decodable()) {
Some((f, _)) => Err(FilterError::Unsupported(f.name().to_owned())),
None => Ok(()),
}
}
/// The decoded data, whether it is partial and the bytes all stages decoded, or the error and
/// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. With
/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it
/// reads, before it runs, so a stage `total` can't pay for doesn't run
/// ([`FilterError::Spent`]). `rows` is the height of the image whose samples the last stage
/// produces, if it does ([`decode_image_tolerant_within`]).
fn run(
chain: &[(Filter, Params)],
data: &[u8],
max: usize,
total: Option<usize>,
strict: bool,
priced: bool,
rows: Option<usize>,
) -> Result<(Vec<u8>, bool, usize), (FilterError, usize)> {
decodable(chain).map_err(|e| (e, 0))?;
let mut cur: Cow<[u8]> = Cow::Borrowed(data);
let mut partial = false;
let mut spent = 0usize;
let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX);
for (i, (filter, params)) in chain.iter().enumerate() {
let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 };
let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent));
if read > room {
return Err((FilterError::Spent, spent));
}
spent = spent.saturating_add(read);
#[cfg(test)]
STAGE_BYTES.with(|c| c.set(c.get() + cur.len()));
let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent)));
// A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and
// makes no more than is left once it has been.
let rows = rows.filter(|_| i + 1 == chain.len());
let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left));
let (step, work, again) = decode_one(filter, params, &cur, limit, again, rows);
if again {
spent = spent.saturating_add(read);
}
spent = spent.saturating_add(work);
cur = Cow::Owned(match step {
Ok(v) => v,
Err(Failure { error: FilterError::Corrupt { .. }, partial: out, .. }) if !strict => {
partial = true;
out
}
Err(Failure { error: FilterError::LimitExceeded(_), .. }) => return Err((FilterError::LimitExceeded(max), spent)),
Err(f) => return Err((f.error, spent)),
});
}
Ok((cur.into_owned(), partial, spent))
}
/// One stage's result, the bytes its decoder produced (before any predictor), and whether it
/// read its input twice, which it does only within `again` ([`flate::decode`]); `rows` as for
/// [`run`].
fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option<usize>, rows: Option<usize>) -> (Step, usize, bool) {
let once = |(step, n): (Step, usize)| (step, n, false);
match filter {
Filter::Flate => {
let (step, twice) = flate::decode(data, max, again);
let (step, n) = with_predictor(step, params, "FlateDecode", rows);
(step, n, twice)
}
// LZW rows are predicted whatever their length, as the renderer reads them.
Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode", None)),
Filter::AsciiHex => once(counted(asciihex::decode(data, max))),
Filter::Ascii85 => once(counted(ascii85::decode(data, max))),
Filter::RunLength => once(counted(runlength::decode(data, max))),
other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false),
}
}
/// A step with the bytes it produced: its output, or its partial output and what it threw away.
fn counted(step: Step) -> (Step, usize) {
let n = match &step {
Ok(v) => v.len(),
Err(f) => f.partial.len().saturating_add(f.discarded),
};
(step, n)
}
/// Applies the predictor to a decoder's result, including to partial output on failure. The
/// count is the decoder's output: a PNG predictor drops a tag byte per row, but the decoder
/// produced it. `rows` (an image's height, for its last stage) applies only to data that decoded
/// whole: data cut short by damage has its predictor undone as it always had.
fn with_predictor(step: Step, params: &Params, name: &'static str, rows: Option<usize>) -> (Step, usize) {
let (step, n) = counted(step);
let step = match step {
Ok(v) => predictor::decode(params, v, name, rows),
Err(Failure { error, partial, discarded }) => {
let partial = match predictor::decode(params, partial, name, None) {
Ok(v) => v,
Err(f) => f.partial,
};
Err(Failure { error, partial, discarded })
}
};
(step, n)
}
/// Encodes `data` with `filter`. Supported: Flate (zlib, level 6), LZW, ASCIIHex,
/// ASCII85, RunLength. For Flate and LZW, `params.predictor >= 2` applies the
/// predictor first (TIFF 2, PNG 10–15; 15 chooses the per-row optimum).
pub fn encode(filter: &Filter, params: &Params, data: &[u8]) -> Result<Vec<u8>, FilterError> {
match filter {
Filter::Flate => Ok(flate::encode(&predictor::encode(params, data)?)),
Filter::Lzw => Ok(lzw::encode(&predictor::encode(params, data)?, params.early_change != 0)),
Filter::AsciiHex => Ok(asciihex::encode(data)),
Filter::Ascii85 => Ok(ascii85::encode(data)),
Filter::RunLength => Ok(runlength::encode(data)),
other => Err(FilterError::Unsupported(format!("encoding with {}", other.name()))),
}
}
/// zlib (RFC 1950) at compression level 6.
pub fn encode_flate(data: &[u8]) -> Vec<u8> {
flate::encode(data)
}
/// PDF white-space characters (ISO 32000-2 Table 1).
pub(crate) fn is_pdf_whitespace(b: u8) -> bool {
matches!(b, 0 | 9 | 10 | 12 | 13 | 32)
}
#[cfg(test)]
thread_local! {
/// Bytes handed to decoding stages on this thread.
static STAGE_BYTES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
#[cfg(test)]
mod tests {
#[test]
fn a_long_filter_chain_is_refused_before_decoding() {
// `FF FF` run-length decodes to itself, so every stage of the chain repeats the work.
let data = vec![0xFF; 4096];
let chain = |n: usize| vec![(super::Filter::RunLength, super::Params::default()); n];
super::STAGE_BYTES.with(|c| c.set(0));
let long = super::decode(&chain(1000), &data, 1 << 20);
let read = super::STAGE_BYTES.with(|c| c.get());
assert!(read <= 16 * data.len(), "{read} bytes decoded through the chain");
assert!(matches!(long, Err(super::FilterError::Unsupported(_))), "{long:?}");
assert_eq!(super::decode(&chain(super::MAX_CHAIN), &data, 1 << 20).ok(), Some(data));
}
/// [`decode_tolerant_counted_each`] holds every stage to its own bound as well as the stages
/// to their total: a Flate stage that would make more than that bound stops there, though
/// the total has room, and the stage after it gets nothing; within both, the chain reads
/// whole, one stage's output charged on top of another's.
#[test]
fn each_stage_is_held_to_its_own_bound_within_the_total() {
let hex = "00".repeat(1 << 16).into_bytes();
let data = super::flate::encode(&hex);
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
// The total alone lets the Flate stage make all of the hex text, and the hex stage read it.
STAGE_BYTES.with(|c| c.set(0));
let whole = decode_tolerant_counted(&chain, &data, 3 << 16).map(|(v, _, _)| v);
assert_eq!(whole.ok(), Some(vec![0u8; 1 << 16]));
assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len() + hex.len());
// Each stage held to 64 KiB, the Flate stage stops there and the hex stage gets nothing.
STAGE_BYTES.with(|c| c.set(0));
let held = decode_tolerant_counted_each(&chain, &data, 1 << 16, 3 << 16);
assert!(matches!(held, Err((FilterError::LimitExceeded(m), 0)) if m == 1 << 16), "{held:?}");
assert_eq!(STAGE_BYTES.with(|c| c.get()), data.len());
// Within both, it reads whole.
let (out, _, spent) = decode_tolerant_counted_each(&chain, &data, 2 << 16, 3 << 16).unwrap();
assert_eq!((out.len(), spent), (1 << 16, hex.len() + (1 << 16)));
}
use super::*;
#[test]
fn names_and_abbreviations() {
for (full, abbr, f) in [
(&b"FlateDecode"[..], &b"Fl"[..], Filter::Flate),
(b"LZWDecode", b"LZW", Filter::Lzw),
(b"ASCIIHexDecode", b"AHx", Filter::AsciiHex),
(b"ASCII85Decode", b"A85", Filter::Ascii85),
(b"RunLengthDecode", b"RL", Filter::RunLength),
(b"DCTDecode", b"DCT", Filter::Dct),
(b"CCITTFaxDecode", b"CCF", Filter::CcittFax),
] {
assert_eq!(Filter::from_name(full), f);
assert_eq!(Filter::from_name(abbr), f);
assert_eq!(f.name().as_bytes(), full);
}
assert_eq!(Filter::from_name(b"/JPXDecode"), Filter::Jpx);
assert_eq!(Filter::from_name(b"JBIG2Decode"), Filter::Jbig2);
assert_eq!(Filter::from_name(b"Crypt"), Filter::Crypt);
assert_eq!(Filter::from_name(b"Foo"), Filter::Unknown("Foo".into()));
assert_eq!(Filter::Unknown("Foo".into()).name(), "Foo");
assert!(Filter::Dct.is_image_codec() && Filter::CcittFax.is_image_codec());
assert!(!Filter::Flate.is_image_codec() && !Filter::Crypt.is_image_codec());
}
#[test]
fn default_params() {
let p = Params::default();
assert_eq!((p.predictor, p.colors, p.bits_per_component, p.columns, p.early_change), (1, 1, 8, 1, 1));
}
#[test]
fn unsupported_filters() {
for f in [Filter::Dct, Filter::Jpx, Filter::Jbig2, Filter::CcittFax, Filter::Crypt, Filter::Unknown("X".into())] {
let chain = [(Filter::AsciiHex, Params::default()), (f.clone(), Params::default())];
assert!(matches!(decode(&chain, b"00>", 100), Err(FilterError::Unsupported(_))));
assert!(matches!(decode_tolerant(&chain, b"00>", 100), Err(FilterError::Unsupported(_))));
assert!(matches!(encode(&f, &Params::default(), b"x"), Err(FilterError::Unsupported(_))));
}
}
#[test]
fn empty_chain_is_identity() {
assert_eq!(decode(&[], b"abc", 1).unwrap(), b"abc");
}
#[test]
fn chain_ascii85_flate() {
let data = b"Hello, chained filters! Hello, chained filters!".repeat(10);
let z = encode_flate(&data);
let a = encode(&Filter::Ascii85, &Params::default(), &z).unwrap();
let chain = [(Filter::Ascii85, Params::default()), (Filter::Flate, Params::default())];
assert_eq!(decode(&chain, &a, 1 << 20).unwrap(), data);
}
#[test]
fn chain_three_filters_with_predictor() {
let p = Params { predictor: 12, colors: 3, columns: 5, ..Params::default() };
let data: Vec<u8> = (0..300u32).map(|i| (i * 7 % 251) as u8).collect();
let lzw = encode(&Filter::Lzw, &p, &data).unwrap();
let rl = encode(&Filter::RunLength, &Params::default(), &lzw).unwrap();
let hex = encode(&Filter::AsciiHex, &Params::default(), &rl).unwrap();
let chain = [(Filter::AsciiHex, Params::default()), (Filter::RunLength, Params::default()), (Filter::Lzw, p)];
assert_eq!(decode(&chain, &hex, 1 << 20).unwrap(), data);
}
#[test]
fn limit_applies_to_intermediate_buffers() {
let hex = encode(&Filter::AsciiHex, &Params::default(), &[7u8; 100]).unwrap();
let chain = [(Filter::AsciiHex, Params::default())];
assert!(matches!(decode(&chain, &hex, 99), Err(FilterError::LimitExceeded(99))));
assert_eq!(decode(&chain, &hex, 100).unwrap().len(), 100);
}
#[test]
fn raw_flate_fallback_preserves_output_limits() {
// RFC 1951 stored blocks: the first is nonfinal with LEN=257 and
// NLEN=!257, followed by a final empty block. Its ignored alignment
// bits make the first two bytes also look like a valid zlib header.
let data = vec![b'A'; 257];
let mut raw = vec![0x78, 0x01, 0x01, 0xfe, 0xfe];
raw.extend_from_slice(&data);
raw.extend_from_slice(&[0x01, 0x00, 0x00, 0xff, 0xff]);
let chain = [(Filter::Flate, Params::default())];
for max in [0, 1, 128, 256] {
assert!(matches!(decode(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max));
assert!(matches!(decode_tolerant(&chain, &raw, max), Err(FilterError::LimitExceeded(m)) if m == max));
}
for max in [257, 258] {
assert_eq!(decode(&chain, &raw, max).unwrap(), data);
assert_eq!(decode_tolerant(&chain, &raw, max).unwrap(), (data.clone(), false));
}
}
#[test]
fn tolerant_flate_returns_partial() {
let data: Vec<u8> = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect();
let mut z = encode_flate(&data);
let cut = z.len() / 2;
z.truncate(cut);
let chain = [(Filter::Flate, Params::default())];
assert!(matches!(decode(&chain, &z, 1 << 20), Err(FilterError::Corrupt { .. })));
let (out, partial) = decode_tolerant(&chain, &z, 1 << 20).unwrap();
assert!(partial);
assert!(!out.is_empty() && out.len() < data.len());
assert_eq!(&data[..out.len()], &out[..]);
// Complete data is not partial.
let (out, partial) = decode_tolerant(&chain, &encode_flate(&data), 1 << 20).unwrap();
assert!(!partial);
assert_eq!(out, data);
}
#[test]
fn tolerant_partial_flows_through_predictor() {
let p = Params { predictor: 12, columns: 4, ..Params::default() };
let data: Vec<u8> = (0..4000u32).map(|i| (i * 31 % 256) as u8).collect();
let mut z = encode(&Filter::Flate, &p, &data).unwrap();
z.truncate(z.len() - 20);
let (out, partial) = decode_tolerant(&[(Filter::Flate, p)], &z, 1 << 20).unwrap();
assert!(partial);
assert_eq!(&data[..out.len()], &out[..]);
}
/// A counted decode bounds what all stages decode together: Flate inflating 1 MiB of hex
/// that ASCIIHex halves is 1.5 MiB of work, past a 1.25 MiB allowance. A plain decode keeps
/// the limit per stage.
#[test]
fn a_counted_decode_bounds_all_stages_together() {
let hex: Vec<u8> = std::iter::repeat_n(b"ab".as_slice(), 1 << 19).flatten().copied().collect();
let data = encode_flate(&hex);
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
let max = (1 << 20) + (1 << 18);
assert_eq!(decode(&chain, &data, max).map(|d| d.len()).ok(), Some(1 << 19));
assert!(matches!(decode_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max));
assert_eq!(decode_counted(&chain, &data, 2 << 20).map(|(_, spent)| spent).ok(), Some((1 << 20) + (1 << 19)));
let tolerant = decode_tolerant_counted(&chain, &data, 2 << 20).map(|(out, partial, spent)| (out.len(), partial, spent));
assert_eq!(tolerant.ok(), Some((1 << 19, false, (1 << 20) + (1 << 19))));
assert!(matches!(decode_tolerant_counted(&chain, &data, max), Err((FilterError::LimitExceeded(m), _)) if m == max));
}
/// A PNG predictor drops a tag byte per row, but Flate produced it: the count includes it,
/// also when the predictor fails on a bad tag.
#[test]
fn a_counted_decode_counts_output_before_the_predictor() {
let params = Params { predictor: 12, ..Params::default() };
let rows = vec![7u8; 1 << 16];
let data = encode(&Filter::Flate, &params, &rows).unwrap();
let chain = [(Filter::Flate, params.clone())];
let (out, spent) = decode_counted(&chain, &data, usize::MAX).unwrap();
assert!(out == rows);
assert_eq!(spent, 2 << 16);
let bad: Vec<u8> = std::iter::repeat_n([0u8, 7], 1 << 15).flatten().chain([9, 7]).collect();
let (e, spent) = decode_counted(&chain, &encode_flate(&bad), usize::MAX).unwrap_err();
assert!(matches!(e, FilterError::Corrupt { .. }), "{e}");
assert_eq!(spent, bad.len());
}
fn refused(e: &FilterError) -> bool {
matches!(e, FilterError::Spent)
}
/// A chain pays for what it reads and what each stage decodes, down to the byte: Flate inflates
/// hex that ASCIIHex halves, so a stage counts when a later one shrinks it.
#[test]
fn a_chain_spends_down_to_the_byte() {
let hex = b"ab".repeat(1 << 10);
let data = encode_flate(&hex);
let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())];
let whole = decode(&chain, &data, 1 << 20).unwrap();
let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64;
let mut work = u64::MAX;
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole);
assert_eq!(u64::MAX - work, cost);
let mut work = cost;
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole, "exactly enough");
assert_eq!(work, 0);
let mut work = cost + 7;
decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap();
assert_eq!(work, 7, "only what it took");
let mut work = cost - 1;
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()), "a step short");
assert_eq!(work, 0, "spends all that was left");
// Each stage has `max_output` to itself, as in a plain decode: together they are past it.
let mut work = u64::MAX;
assert_eq!(decode_tolerant_within(&chain, &data, hex.len(), &mut work).unwrap(), whole);
}
/// No stage runs past what is left: the first stage of this bomb stops there, so the second
/// isn't given anything to read, and with too little to read it nothing runs at all.
#[test]
fn a_chain_stops_at_what_is_left() {
let inner = encode_flate(&vec![0; 1 << 20]);
let bomb = encode_flate(&inner);
let chain = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())];
let read = |work: u64| {
super::STAGE_BYTES.with(|c| c.set(0));
let mut work = work;
assert!(refused(&decode_tolerant_within(&chain, &bomb, usize::MAX, &mut work).unwrap_err()));
assert_eq!(work, 0);
super::STAGE_BYTES.with(|c| c.get())
};
let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64;
assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops");
assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read");
assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs");
assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read");
assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read");
}
/// Past `max_output` the chain has failed, and hasn't run out of anything: it pays what it took
/// to find out, and what is left stays. Whichever limit a stage stops at is the one reported.
#[test]
fn a_stage_past_its_limit_pays_for_finding_out() {
let data = encode_flate(&vec![0; 1 << 16]);
let chain = [(Filter::Flate, Params::default())];
let mut work = u64::MAX;
assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000))));
let read = |n: usize| FLATE_READ_WORK * n as u64;
assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000);
// The later stage of two, which pays for the first as well.
let bomb = encode_flate(&data);
let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())];
let mut work = u64::MAX;
assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_))));
assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len()));
// Both limits at once: a run that can't go on is what a caller needs to hear.
let mut work = data.len() as u64 + read(data.len()) + 1000;
assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err()));
assert_eq!(work, 0);
let mut work = data.len() as u64 + read(data.len()) + 1001;
assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000))));
assert_eq!(work, 1, "the image's limit, with a step to spare");
}
/// What a damaged stream decoded is paid for, however the decode ends.
#[test]
fn a_chain_that_fails_has_paid_for_what_it_did() {
let data: Vec<u8> = (0..20_000u32).map(|i| (i % 97) as u8 ^ (i / 300) as u8).collect();
let mut cut = encode_flate(&data);
cut.truncate(cut.len() / 2);
let chain = [(Filter::Flate, Params::default())];
let (partial, _) = decode_tolerant(&chain, &cut, 1 << 20).unwrap();
assert!(!partial.is_empty());
let mut work = u64::MAX;
assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial);
let read = |n: usize| FLATE_READ_WORK * n as u64;
assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len()));
// Read strictly it is an error, for the same work.
let mut work = u64::MAX;
assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. })));
assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len()));
let whole = encode_flate(&data);
let mut work = u64::MAX;
assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data);
assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len()));
// A predictor that comes to nothing, after all of this was inflated for it.
let p = Params { predictor: 99, ..Params::default() };
let data = encode_flate(&vec![0; 1 << 16]);
let mut work = u64::MAX;
assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty());
assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16));
}
/// A stream that reads as zlib only until it fails, and as raw deflate until that fails too,
/// gives back no data, and has still decoded everything the retry did.
#[test]
fn a_retry_that_fails_pays_for_what_it_decoded() {
// Stored as raw deflate: this many bytes, then a block of a reserved type. As zlib: the
// same reserved type at once, so the first reading fails with nothing.
let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9];
data.extend(vec![0u8; 0x0601]);
data.push(0x07);
let chain = [(Filter::Flate, Params::default())];
let (out, partial) = decode_tolerant(&chain, &data, 1 << 20).unwrap();
assert!(out.is_empty() && partial, "what is read stays as it was");
let (_, _, spent) = decode_tolerant_counted(&chain, &data, 1 << 20).unwrap();
assert_eq!(spent, 0x0601);
let (e, spent) = decode_counted(&chain, &data, 1 << 20).unwrap_err();
assert!(matches!(e, FilterError::Corrupt { .. }), "{e}");
assert_eq!(spent, 0x0601);
let mut work = u64::MAX;
assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty());
// Read twice: as zlib, then as raw deflate.
assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64);
// Through a predictor, which would otherwise lose the count.
let p = Params { predictor: 12, columns: 4, ..Params::default() };
let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap();
assert_eq!(spent, 0x0601);
}
/// Inflating does work for every block it reads, whether or not the block makes anything: empty
/// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them
/// is paid for at Flate's price before any is read.
#[test]
fn a_flate_stage_pays_for_what_it_reads() {
// Four empty blocks in five bytes, as raw deflate, then an empty final block.
let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12);
hollow.extend([0x03, 0x00]);
let chain = [(Filter::Flate, Params::default())];
assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b"");
let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64;
let mut work = u64::MAX;
assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b"");
assert_eq!(u64::MAX - work, cost);
let mut work = cost;
assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough");
assert_eq!(work, 0);
super::STAGE_BYTES.with(|c| c.set(0));
let mut work = cost - 1;
assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err()));
assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated");
// Counted for a byte allowance, as a page's content is, reading costs nothing.
assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0));
}
/// A zlib header that only looked valid is read again as raw deflate, and the second reading
/// is paid for as the first was. One that can't be paid for is refused, not read as damaged.
#[test]
fn a_second_reading_is_paid_for() {
// As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many
// zero bytes, then an empty final block.
let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9];
data.extend(vec![0u8; 0x0601]);
data.extend([0x03, 0x00]);
let chain = [(Filter::Flate, Params::default())];
assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]);
let read = FLATE_READ_WORK * data.len() as u64;
let mut work = (data.len() + 0x0601) as u64 + 2 * read;
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]);
assert_eq!(work, 0, "exactly enough");
// Enough to read it once and not twice: it isn't read again.
super::flate::INFLATED.with(|c| c.set(0));
let mut work = data.len() as u64 + 2 * read - 1;
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()));
assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0));
// Enough to read it twice, and a byte short of what it makes: the second reading makes no
// more than is left once it has been paid for.
super::flate::INFLATED.with(|c| c.set(0));
let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1;
assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err()));
assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short");
}
/// With nothing left nothing is read. What can't be decoded at all never costs anything, and
/// says so whatever is left. Without filters, what is read is the data.
#[test]
fn what_a_chain_is_shows_whatever_is_left() {
let flate = [(Filter::Flate, Params::default())];
let mut work = 0;
assert!(refused(&decode_tolerant_within(&flate, b"garbage", 1 << 20, &mut work).unwrap_err()));
let long = vec![(Filter::Flate, Params::default()); MAX_CHAIN + 1];
for chain in [vec![(Filter::Dct, Params::default())], vec![(Filter::Jbig2, Params::default())], long] {
for left in [0, 100] {
let mut work = left;
let e = decode_tolerant_within(&chain, b"data", 1 << 20, &mut work).unwrap_err();
assert!(matches!(e, FilterError::Unsupported(_)) && !refused(&e), "{e}");
assert_eq!(work, left);
}
}
let mut work = 100;
assert_eq!(decode_tolerant_within(&[], b"data", 4, &mut work).unwrap(), b"data");
assert_eq!(work, 96);
assert!(matches!(decode_tolerant_within(&[], b"data", 3, &mut work), Err(FilterError::LimitExceeded(3))));
assert_eq!(work, 92);
let mut work = 3;
assert!(refused(&decode_tolerant_within(&[], b"data", 4, &mut work).unwrap_err()));
assert_eq!(work, 0);
}
/// Four rows of three RGB pixels, every sample above 4 (none reads as a PNG filter type).
fn rgb_rows() -> (Params, Vec<u8>) {
let p = Params { predictor: 15, colors: 3, columns: 3, ..Params::default() };
(p, (0..36u8).map(|i| 200 - i).collect())
}
#[test]
fn an_image_whose_rows_carry_no_png_tags_is_read_as_its_samples_and_reported() {
let (p, samples) = rgb_rows();
let chain = [(Filter::Flate, p.clone())];
let data = encode_flate(&samples);
let mut work = u64::MAX;
assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut work).unwrap(), samples);
// The repair is reported as the damage it is: tolerant decoding says so, and strict refuses.
let (out, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap();
assert_eq!((out, partial), (samples.clone(), true));
let strict = run(&chain, &data, 1 << 20, None, true, false, Some(4)).map(|_| ()).map_err(|(e, _)| e.to_string());
assert!(strict.as_ref().is_err_and(|e| e.contains("carry no filter-type bytes")), "{strict:?}");
// The same data that isn't known to be an image's rows still goes through the predictor.
let mut work = u64::MAX;
assert_ne!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), samples);
// LZW is read as before, as the renderer reads it.
let lzw = [(Filter::Lzw, p)];
let data = encode(&Filter::Lzw, &Params::default(), &samples).unwrap();
let mut plain = u64::MAX;
assert_eq!(
decode_image_tolerant_within(&lzw, &data, 1 << 20, 4, &mut work).ok(),
decode_tolerant_within(&lzw, &data, 1 << 20, &mut plain).ok()
);
}
#[test]
fn an_image_whose_rows_carry_png_tags_has_its_predictor_undone() {
let (p, samples) = rgb_rows();
let chain = [(Filter::Flate, p.clone())];
let data = encode(&Filter::Flate, &p, &samples).unwrap();
let (mut a, mut b) = (u64::MAX, u64::MAX);
assert_eq!(decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).unwrap(), samples);
assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut b).unwrap(), samples);
assert_eq!(a, b, "the same work");
let (_, partial, _) = run(&chain, &data, 1 << 20, None, false, false, Some(4)).map_err(|(e, _)| e).unwrap();
assert!(!partial);
}
#[test]
fn image_data_of_neither_length_decodes_as_it_did() {
let (p, _) = rgb_rows();
let chain = [(Filter::Flate, p.clone())];
for len in [0, 1, 9, 27, 35, 37, 39, 40, 41, 72] {
let data = encode_flate(&(0..len).map(|i| (i * 7 % 5) as u8).collect::<Vec<_>>());
let (mut a, mut b) = (u64::MAX, u64::MAX);
let image = decode_image_tolerant_within(&chain, &data, 1 << 20, 4, &mut a).map_err(|e| e.to_string());
let plain = decode_tolerant_within(&chain, &data, 1 << 20, &mut b).map_err(|e| e.to_string());
assert_eq!(image, plain, "{len} bytes");
}
// Rows of the untagged length for another height, or no height.
let samples = encode_flate(&[9; 36]);
for rows in [0, 3, 5, u32::MAX] {
let (mut a, mut b) = (u64::MAX, u64::MAX);
assert_eq!(
decode_image_tolerant_within(&chain, &samples, 1 << 20, rows, &mut a).ok(),
decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok(),
"{rows} rows"
);
}
// Only the last stage produces the rows: an earlier one is predicted as before.
let hex: Vec<u8> = [9u8; 36].iter().flat_map(|b| format!("{b:02x}").into_bytes()).collect();
let earlier = [(Filter::Flate, p.clone()), (Filter::AsciiHex, Params::default())];
let data = encode_flate(&hex[..36]);
let (mut a, mut b) = (u64::MAX, u64::MAX);
assert_eq!(
decode_image_tolerant_within(&earlier, &data, 1 << 20, 4, &mut a).ok(),
decode_tolerant_within(&earlier, &data, 1 << 20, &mut b).ok()
);
// TIFF predictor 2 and no predictor are left alone.
for predictor in [1, 2] {
let chain = [(Filter::Flate, Params { predictor, ..p.clone() })];
let (mut a, mut b) = (u64::MAX, u64::MAX);
assert_eq!(
decode_image_tolerant_within(&chain, &samples, 1 << 20, 4, &mut a).ok(),
decode_tolerant_within(&chain, &samples, 1 << 20, &mut b).ok()
);
}
}
#[test]
fn image_rows_of_no_bytes_or_past_memory_do_not_panic() {
for (colors, bits_per_component, columns) in [(0, 8, 3), (3, 8, 0), (3, 8, -1), (255, 16, i64::MAX), (1, 1, i64::MAX), (3, 3, 3)] {
let chain = [(Filter::Flate, Params { predictor: 15, colors, bits_per_component, columns, early_change: 1 })];
for len in [0, 1, 36] {
let data = encode_flate(&vec![7; len]);
for rows in [0, 1, 4, u32::MAX] {
let mut work = u64::MAX;
let _ = decode_image_tolerant_within(&chain, &data, 1 << 20, rows, &mut work);
}
}
}
}
}
//! PDF stream filters (`craft-pdf::filters`).
pub use craft_pdf::filters::*;
-314
View File
@@ -1,314 +0,0 @@
//! LZWDecode (ISO 32000-2 §7.4.4.2): variable-width codes of 9–12 bits, MSB first,
//! 256 = clear-table, 257 = EOD, first free code 258. With `EarlyChange` 1 (the
//! default) the code width grows one code early.
//!
//! Tolerated: data ending without EOD, a missing initial clear-table code, a table that
//! fills up without a clear-table code (entries are then no longer added).
use std::collections::HashMap;
use crate::{Failure, Step};
const NAME: &str = "LZWDecode";
const CLEAR: u16 = 256;
const EOD: u16 = 257;
const FIRST: usize = 258;
const TABLE: usize = 4096;
/// The encoder emits clear-table when its next free code reaches this value.
const ENCODER_RESET_AT: usize = TABLE;
/// The code width a decoder uses when its next free code is `next`.
fn width(next: usize, early: bool) -> u32 {
match next + usize::from(early) {
0..512 => 9,
512..1024 => 10,
1024..2048 => 11,
_ => 12,
}
}
struct BitReader<'a> {
data: &'a [u8],
pos: usize,
acc: u32,
bits: u32,
}
impl BitReader<'_> {
fn read(&mut self, n: u32) -> Option<u16> {
while self.bits < n {
let &b = self.data.get(self.pos)?;
self.pos += 1;
self.acc = (self.acc << 8) | u32::from(b);
self.bits += 8;
}
self.bits -= n;
let v = (self.acc >> self.bits) & ((1 << n) - 1);
self.acc &= (1 << self.bits) - 1;
Some(v as u16)
}
}
pub(crate) fn decode(data: &[u8], early: bool, max: usize) -> Step {
// Table entry = (prefix code, last byte, first byte, length).
let mut prefix = vec![0u16; TABLE];
let mut last = vec![0u8; TABLE];
let mut first = vec![0u8; TABLE];
let mut len = vec![0u16; TABLE];
for i in 0..256 {
last[i] = i as u8;
first[i] = i as u8;
len[i] = 1;
}
let mut next = FIRST;
let mut w = 9;
let mut prev: Option<usize> = None;
let mut r = BitReader { data, pos: 0, acc: 0, bits: 0 };
let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max));
while let Some(code) = r.read(w) {
match code {
CLEAR => {
next = FIRST;
w = 9;
prev = None;
continue;
}
EOD => break,
_ => {}
}
let code = usize::from(code);
if let Some(p) = prev {
let head = if code < next {
first[code]
} else if code == next && next < TABLE {
first[p]
} else {
return Err(Failure::corrupt(NAME, format!("code {code} not in table (next {next})"), out));
};
if next < TABLE {
prefix[next] = p as u16;
last[next] = head;
first[next] = first[p];
len[next] = len[p] + 1;
next += 1;
}
} else if code >= 256 {
return Err(Failure::corrupt(NAME, format!("code {code} before any table entry"), out));
}
// Emit the string for `code`, walking the prefix chain backwards.
let n = usize::from(len[code]);
if out.len() + n > max {
return Err(Failure::limit(max));
}
let start = out.len();
out.resize(start + n, 0);
let mut c = code;
for slot in out[start..].iter_mut().rev() {
*slot = last[c];
c = usize::from(prefix[c]);
}
prev = Some(code);
w = width(next, early);
}
Ok(out)
}
struct BitWriter {
out: Vec<u8>,
acc: u32,
bits: u32,
}
impl BitWriter {
fn write(&mut self, code: u16, n: u32) {
self.acc = (self.acc << n) | u32::from(code);
self.bits += n;
while self.bits >= 8 {
self.bits -= 8;
self.out.push((self.acc >> self.bits) as u8);
}
self.acc &= (1 << self.bits) - 1;
}
fn finish(mut self) -> Vec<u8> {
if self.bits > 0 {
self.out.push((self.acc << (8 - self.bits)) as u8);
}
self.out
}
}
/// Greedy LZW with a leading clear-table code and a trailing EOD.
pub(crate) fn encode(data: &[u8], early: bool) -> Vec<u8> {
let mut wr = BitWriter { out: Vec::with_capacity(data.len() / 2 + 8), acc: 0, bits: 0 };
let mut dict: HashMap<(u16, u8), u16> = HashMap::new();
// `next` is the encoder's next free code; `dnext` mirrors the decoder's, which lags
// one entry behind (the decoder adds an entry only once it sees the following code).
let mut next = FIRST;
let mut dnext = FIRST;
let mut codes_since_clear = 0usize;
wr.write(CLEAR, 9);
let emit = |wr: &mut BitWriter, code: u16, dnext: &mut usize, since: &mut usize| {
wr.write(code, width(*dnext, early));
if *since > 0 && *dnext < TABLE {
*dnext += 1;
}
*since += 1;
};
let Some((&b0, rest)) = data.split_first() else {
wr.write(EOD, 9);
return wr.finish();
};
let mut cur = u16::from(b0);
for &b in rest {
if let Some(&c) = dict.get(&(cur, b)) {
cur = c;
continue;
}
emit(&mut wr, cur, &mut dnext, &mut codes_since_clear);
dict.insert((cur, b), next as u16);
next += 1;
if next == ENCODER_RESET_AT {
wr.write(CLEAR, width(dnext, early));
dict.clear();
next = FIRST;
dnext = FIRST;
codes_since_clear = 0;
}
cur = u16::from(b);
}
emit(&mut wr, cur, &mut dnext, &mut codes_since_clear);
wr.write(EOD, width(dnext, early));
wr.finish()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FilterError;
fn dec(d: &[u8], early: bool) -> Result<Vec<u8>, FilterError> {
decode(d, early, 1 << 26).map_err(|f| f.error)
}
/// ISO 32000-2 §7.4.4.2 example: codes 256 45 258 258 65 259 66 257.
const SPEC_IN: [u8; 10] = [45, 45, 45, 45, 45, 65, 45, 45, 45, 66];
const SPEC_OUT: [u8; 9] = [0x80, 0x0B, 0x60, 0x50, 0x22, 0x0C, 0x0C, 0x85, 0x01];
#[test]
fn spec_example() {
assert_eq!(dec(&SPEC_OUT, true).unwrap(), SPEC_IN);
assert_eq!(encode(&SPEC_IN, true), SPEC_OUT);
// Below 511 codes the early-change setting makes no difference.
assert_eq!(dec(&SPEC_OUT, false).unwrap(), SPEC_IN);
}
#[test]
fn missing_eod_and_clear() {
// Drop the EOD (last 9 bits span the final 2 bytes; keep bits for 66 only).
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
for c in [45u16, 258, 258, 65, 259, 66] {
w.write(c, 9);
}
assert_eq!(dec(&w.finish(), true).unwrap(), SPEC_IN);
}
#[test]
fn invalid_codes() {
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
for c in [256u16, 65, 300] {
w.write(c, 9);
}
let f = decode(&w.finish(), true, 1000).err().unwrap();
assert!(matches!(f.error, FilterError::Corrupt { .. }));
assert_eq!(f.partial, b"A");
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
w.write(258, 9);
assert!(matches!(dec(&w.finish(), true), Err(FilterError::Corrupt { .. })));
}
#[test]
fn widths() {
assert_eq!((width(510, true), width(511, true), width(511, false), width(512, false)), (9, 10, 9, 10));
assert_eq!((width(1022, true), width(1023, true), width(2047, true), width(4095, false)), (10, 11, 12, 12));
}
fn noisy(n: usize, seed: u32) -> Vec<u8> {
let mut s = seed;
(0..n)
.map(|_| {
s = s.wrapping_mul(1_103_515_245).wrapping_add(12_345);
((s >> 16) % 7) as u8 * 31
})
.collect()
}
#[test]
fn all_widths_and_clears_round_trip() {
for early in [false, true] {
for data in [noisy(200_000, 1), (0..=255u8).cycle().take(70_000).collect(), vec![9u8; 100_000]] {
let e = encode(&data, early);
assert_eq!(dec(&e, early).unwrap(), data);
}
}
}
#[test]
fn cross_check_with_weezl() {
use weezl::BitOrder::Msb;
for data in [noisy(150_000, 7), SPEC_IN.to_vec(), vec![]] {
// early change 1 is the TIFF size switch; 0 is the plain variant.
let tiff = weezl::encode::Encoder::with_tiff_size_switch(Msb, 8).encode(&data).unwrap();
assert_eq!(dec(&tiff, true).unwrap(), data);
let plain = weezl::encode::Encoder::new(Msb, 8).encode(&data).unwrap();
assert_eq!(dec(&plain, false).unwrap(), data);
let ours = encode(&data, true);
assert_eq!(weezl::decode::Decoder::with_tiff_size_switch(Msb, 8).decode(&ours).unwrap(), data);
let ours = encode(&data, false);
assert_eq!(weezl::decode::Decoder::new(Msb, 8).decode(&ours).unwrap(), data);
}
}
#[test]
fn full_table_without_clear() {
// Hand-roll a stream that fills the table and keeps going at 12 bits.
let data = noisy(40_000, 3);
let mut w = BitWriter { out: vec![], acc: 0, bits: 0 };
let mut dict: HashMap<Vec<u8>, u16> = (0..=255u8).map(|b| (vec![b], u16::from(b))).collect();
let (mut next, mut dnext, mut since) = (FIRST, FIRST, 0usize);
let mut cur: Vec<u8> = vec![];
let mut put = |w: &mut BitWriter, code: u16, dnext: &mut usize| {
w.write(code, width(*dnext, true));
if since > 0 && *dnext < TABLE {
*dnext += 1;
}
since += 1;
};
w.write(CLEAR, 9);
for &b in &data {
let mut ext = cur.clone();
ext.push(b);
if dict.contains_key(&ext) {
cur = ext;
continue;
}
put(&mut w, dict[&cur], &mut dnext);
if next < TABLE {
dict.insert(ext, next as u16);
next += 1;
}
cur = vec![b];
}
put(&mut w, dict[&cur], &mut dnext);
w.write(EOD, width(dnext, true));
assert_eq!(dec(&w.finish(), true).unwrap(), data);
}
#[test]
fn limit() {
let e = encode(&vec![0u8; 100_000], true);
assert!(matches!(decode(&e, true, 50_000).map_err(|f| f.error), Err(FilterError::LimitExceeded(50_000))));
assert_eq!(decode(&e, true, 100_000).ok().unwrap().len(), 100_000);
}
}
-209
View File
@@ -1,209 +0,0 @@
//! The MQ arithmetic encoder of JBIG2 (ITU-T T.88 | ISO/IEC 14492, Annex E), written from the
//! procedures in its Figures E.2–E.12 and the probability estimates of Table E.1.
/// Table E.1: for each index, the LPS probability estimate Qe, the next index after an MPS and
/// after an LPS renormalisation, and whether an LPS switches the MPS sense.
const STATES: [(u32, u8, u8, bool); 47] = [
(0x5601, 1, 1, true),
(0x3401, 2, 6, false),
(0x1801, 3, 9, false),
(0x0AC1, 4, 12, false),
(0x0521, 5, 29, false),
(0x0221, 38, 33, false),
(0x5601, 7, 6, true),
(0x5401, 8, 14, false),
(0x4801, 9, 14, false),
(0x3801, 10, 14, false),
(0x3001, 11, 17, false),
(0x2401, 12, 18, false),
(0x1C01, 13, 20, false),
(0x1601, 29, 21, false),
(0x5601, 15, 14, true),
(0x5401, 16, 14, false),
(0x5101, 17, 15, false),
(0x4801, 18, 16, false),
(0x3801, 19, 17, false),
(0x3401, 20, 18, false),
(0x3001, 21, 19, false),
(0x2801, 22, 19, false),
(0x2401, 23, 20, false),
(0x2201, 24, 21, false),
(0x1C01, 25, 22, false),
(0x1801, 26, 23, false),
(0x1601, 27, 24, false),
(0x1401, 28, 25, false),
(0x1201, 29, 26, false),
(0x1101, 30, 27, false),
(0x0AC1, 31, 28, false),
(0x09C1, 32, 29, false),
(0x08A1, 33, 30, false),
(0x0521, 34, 31, false),
(0x0441, 35, 32, false),
(0x02A1, 36, 33, false),
(0x0221, 37, 34, false),
(0x0141, 38, 35, false),
(0x0111, 39, 36, false),
(0x0085, 40, 37, false),
(0x0049, 41, 38, false),
(0x0025, 42, 39, false),
(0x0015, 43, 40, false),
(0x0009, 44, 41, false),
(0x0005, 45, 42, false),
(0x0001, 45, 43, false),
(0x5601, 46, 46, false),
];
#[cfg(test)]
thread_local! {
/// Decisions coded on this thread.
pub(crate) static DECISIONS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
/// A context's adaptive state: the index into [`STATES`] (low 7 bits) and the MPS sense (bit 7).
/// 0, the initial value, is index 0 with MPS 0 (E.2.8).
pub(crate) type Context = u8;
/// The encoder's registers (E.2.1) and the bytes written so far.
pub(crate) struct MqEncoder {
a: u32,
c: u32,
ct: u32,
/// The output, after the byte before the first one (B at INITENC), which is 0x00 and is
/// dropped by [`MqEncoder::finish`]: a carry can't reach it before the first byte is out.
out: Vec<u8>,
}
impl MqEncoder {
/// INITENC (Figure E.10). The byte before the first is 0x00, not 0xFF, so CT starts at 12.
pub(crate) fn new() -> Self {
MqEncoder { a: 0x8000, c: 0, ct: 12, out: vec![0] }
}
/// The byte B, the last one written (or the one before the first).
fn b(&self) -> u8 {
self.out.last().copied().unwrap_or(0)
}
/// ENCODE (Figures E.3–E.7): code decision `d` in context `cx`, updating its state.
pub(crate) fn encode(&mut self, cx: &mut Context, d: bool) {
#[cfg(test)]
DECISIONS.with(|n| n.set(n.get() + 1));
let (index, mps) = (usize::from(*cx & 0x7F), *cx >> 7 == 1);
let Some(&(qe, nmps, nlps, switch)) = STATES.get(index) else { return };
self.a -= qe;
if d == mps {
// CODEMPS.
if self.a & 0x8000 != 0 {
self.c += qe;
return;
}
if self.a < qe {
self.a = qe;
} else {
self.c += qe;
}
*cx = nmps | (*cx & 0x80);
} else {
// CODELPS.
if self.a < qe {
self.c += qe;
} else {
self.a = qe;
}
let mps = mps != switch;
*cx = nlps | (u8::from(mps) << 7);
}
self.renorm();
}
/// RENORME (Figure E.8).
fn renorm(&mut self) {
loop {
self.a <<= 1;
self.c <<= 1;
self.ct -= 1;
if self.ct == 0 {
self.byte_out();
}
if self.a & 0x8000 != 0 {
break;
}
}
}
/// BYTEOUT (Figure E.9), with bit stuffing after 0xFF and carry into the byte before.
fn byte_out(&mut self) {
if self.b() == 0xFF {
self.stuffed();
} else if self.c < 0x800_0000 {
self.plain();
} else {
let b = self.b().wrapping_add(1);
if let Some(last) = self.out.last_mut() {
*last = b;
}
if b == 0xFF {
self.c &= 0x7FF_FFFF;
self.stuffed();
} else {
self.plain();
}
}
}
fn plain(&mut self) {
self.out.push(((self.c >> 19) & 0xFF) as u8);
self.c &= 0x7_FFFF;
self.ct = 8;
}
fn stuffed(&mut self) {
self.out.push(((self.c >> 20) & 0xFF) as u8);
self.c &= 0xF_FFFF;
self.ct = 7;
}
/// FLUSH (Figures E.11 and E.12): the final bytes, then the 0xFF 0xAC marker.
pub(crate) fn finish(mut self) -> Vec<u8> {
let top = self.c + self.a;
self.c |= 0xFFFF;
if self.c >= top {
self.c -= 0x8000;
}
self.c <<= self.ct;
self.byte_out();
self.c <<= self.ct;
self.byte_out();
if self.b() != 0xFF {
self.out.push(0xFF);
}
self.out.push(0xAC);
self.out.split_off(1)
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Annex H.2's test sequence: 256 decisions in one context give these 30 bytes.
#[test]
fn the_annex_h_test_sequence_encodes_to_its_bytes() {
let decisions = [
0x00, 0x02, 0x00, 0x51, 0x00, 0x00, 0x00, 0xC0, 0x03, 0x52, 0x87, 0x2A, 0xAA, 0xAA, 0xAA, 0xAA, 0x82, 0xC0, 0x20, 0x00, 0xFC, 0xD7, 0x9E,
0xF6, 0xBF, 0x7F, 0xED, 0x90, 0x4F, 0x46, 0xA3, 0xBF,
];
let mut mq = MqEncoder::new();
let mut cx = 0;
for byte in decisions {
for bit in (0..8).rev() {
mq.encode(&mut cx, byte >> bit & 1 == 1);
}
}
let expected = [
0x84, 0xC7, 0x3B, 0xFC, 0xE1, 0xA1, 0x43, 0x04, 0x02, 0x20, 0x00, 0x00, 0x41, 0x0D, 0xBB, 0x86, 0xF4, 0x31, 0x7F, 0xFF, 0x88, 0xFF, 0x37,
0x47, 0x1A, 0xDB, 0x6A, 0xDF, 0xFF, 0xAC,
];
assert_eq!(mq.finish(), expected);
}
}
-356
View File
@@ -1,356 +0,0 @@
//! Predictor functions for FlateDecode and LZWDecode (ISO 32000-2 §7.4.4.4).
//!
//! - Predictor 2: TIFF 6.0 horizontal differencing (Section 14), per component, for
//! 1, 2, 4, 8 and 16 bits per component.
//! - Predictors 10–15: PNG filters (RFC 2083 §6): each row carries its own filter-type
//! byte (0 None, 1 Sub, 2 Up, 3 Average, 4 Paeth), whatever the predictor value is.
//!
//! A final partial row is decoded as far as its data goes.
//!
//! The samples of an image whose rows carry no filter-type byte, although a PNG predictor is
//! declared, are taken as they are and reported as damage ([`decode`]).
use crate::{Failure, FilterError, Params, Step};
struct Layout {
bpc: usize,
colors: usize,
/// Bytes per complete pixel, at least 1 (the PNG `bpp`).
bpp: usize,
/// Bytes per row (without the PNG tag byte).
row: usize,
/// Samples per row (colors × columns).
samples: usize,
}
fn layout(p: &Params) -> Result<Layout, String> {
let bpc = match p.bits_per_component {
b @ (1 | 2 | 4 | 8 | 16) => b as usize,
b => return Err(format!("invalid BitsPerComponent {b}")),
};
if !(1..=255).contains(&p.colors) {
return Err(format!("invalid Colors {}", p.colors));
}
if p.columns < 1 {
return Err(format!("invalid Columns {}", p.columns));
}
let colors = p.colors as usize;
let samples = usize::try_from(p.columns).ok().and_then(|c| c.checked_mul(colors)).ok_or_else(|| format!("Columns {} too large", p.columns))?;
let bits = samples.checked_mul(bpc).ok_or_else(|| format!("Columns {} too large", p.columns))?;
Ok(Layout { bpc, colors, bpp: (colors * bpc).div_ceil(8), row: bits.div_ceil(8), samples })
}
/// Undoes the predictor named by `p.predictor` (1 = identity). `rows` is the height of the image
/// `data` is the samples of, if it is: with a PNG predictor, data of exactly `rows` rows without
/// their filter-type bytes ([`untagged`]) is given back as it is, as a [`Failure`] whose partial
/// output is all of it, so that tolerant decoding takes the samples and reports the repair, and
/// strict decoding refuses them.
pub(crate) fn decode(p: &Params, data: Vec<u8>, name: &'static str, rows: Option<usize>) -> Step {
match p.predictor {
1 => Ok(data),
2 | 10..=15 => {
let l = layout(p).map_err(|e| Failure::corrupt(name, e, Vec::new()))?;
if p.predictor == 2 {
return Ok(tiff_decode(&l, data));
}
if let Some(rows) = rows.filter(|&rows| untagged(&l, data.len(), rows)) {
let detail =
format!("PNG predictor {} declared, but its {rows} rows carry no filter-type bytes: read as unpredicted samples", p.predictor);
return Err(Failure::corrupt(name, detail, data));
}
png_decode(&l, &data, name)
}
other => Err(Failure::corrupt(name, format!("invalid Predictor {other}"), Vec::new())),
}
}
/// Whether `len` bytes are exactly `rows` rows of `l` without the filter-type byte that starts each
/// PNG-predicted row (ISO 32000-2 §7.4.4.4), where predicted data would be `rows` × (row + 1): a
/// writer that declared the predictor and didn't apply it. Read with the predictor, every row's
/// first sample was taken for a filter type and the image came out a row short and shifted.
/// PDFium, MuPDF and Poppler read such data as the samples it is.
fn untagged(l: &Layout, len: usize, rows: usize) -> bool {
rows > 0 && l.row > 0 && rows.checked_mul(l.row) == Some(len)
}
/// Applies the predictor named by `p.predictor` (1 = identity; 15 = per-row optimum).
pub(crate) fn encode(p: &Params, data: &[u8]) -> Result<Vec<u8>, FilterError> {
match p.predictor {
1 => Ok(data.to_vec()),
2 | 10..=15 => {
let l = layout(p).map_err(FilterError::Unsupported)?;
Ok(if p.predictor == 2 { tiff_encode(&l, data) } else { png_encode(&l, data, p.predictor) })
}
other => Err(FilterError::Unsupported(format!("Predictor {other}"))),
}
}
// ---- TIFF predictor 2 ----
fn get(row: &[u8], i: usize, bpc: usize) -> u16 {
match bpc {
16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]),
8 => u16::from(row[i]),
_ => {
let bit = i * bpc;
let shift = 8 - bpc - bit % 8;
u16::from(row[bit / 8] >> shift) & ((1 << bpc) - 1)
}
}
}
fn set(row: &mut [u8], i: usize, bpc: usize, v: u16) {
match bpc {
16 => row[2 * i..2 * i + 2].copy_from_slice(&v.to_be_bytes()),
8 => row[i] = v as u8,
_ => {
let bit = i * bpc;
let shift = 8 - bpc - bit % 8;
let mask = (((1u16 << bpc) - 1) << shift) as u8;
row[bit / 8] = (row[bit / 8] & !mask) | (((v << shift) as u8) & mask);
}
}
}
/// Complete samples available in a (possibly partial) row.
fn samples_in(l: &Layout, row_len: usize) -> usize {
(row_len * 8 / l.bpc).min(l.samples)
}
fn tiff_decode(l: &Layout, mut data: Vec<u8>) -> Vec<u8> {
let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 };
for row in data.chunks_mut(l.row) {
for i in l.colors..samples_in(l, row.len()) {
let v = get(row, i, l.bpc).wrapping_add(get(row, i - l.colors, l.bpc)) & mask;
set(row, i, l.bpc, v);
}
}
data
}
fn tiff_encode(l: &Layout, data: &[u8]) -> Vec<u8> {
let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 };
let mut out = data.to_vec();
for row in out.chunks_mut(l.row) {
// Descending, so the left neighbour still holds its original value.
for i in (l.colors..samples_in(l, row.len())).rev() {
let v = get(row, i, l.bpc).wrapping_sub(get(row, i - l.colors, l.bpc)) & mask;
set(row, i, l.bpc, v);
}
}
out
}
// ---- PNG predictors ----
fn paeth(a: u8, b: u8, c: u8) -> u8 {
let p = i16::from(a) + i16::from(b) - i16::from(c);
let (pa, pb, pc) = ((p - i16::from(a)).abs(), (p - i16::from(b)).abs(), (p - i16::from(c)).abs());
if pa <= pb && pa <= pc {
a
} else if pb <= pc {
b
} else {
c
}
}
/// The PNG predictor value for byte `i` given the reconstructed current row `cur`
/// (bytes before `i`) and the prior row `up`.
fn predict(tag: u8, cur: &[u8], up: &[u8], i: usize, bpp: usize) -> u8 {
let a = if i >= bpp { cur[i - bpp] } else { 0 };
let b = up[i];
let c = if i >= bpp { up[i - bpp] } else { 0 };
match tag {
1 => a,
2 => b,
3 => ((u16::from(a) + u16::from(b)) / 2) as u8,
4 => paeth(a, b, c),
_ => 0,
}
}
fn png_decode(l: &Layout, data: &[u8], name: &'static str) -> Step {
let mut out = Vec::with_capacity(data.len());
// The prior row: never longer than the input, even for absurd Columns values.
let mut up = vec![0u8; l.row.min(data.len())];
for chunk in data.chunks(l.row.saturating_add(1)) {
let (tag, src) = (chunk[0], &chunk[1..]);
if tag > 4 {
return Err(Failure::corrupt(name, format!("invalid PNG filter type {tag}"), out));
}
let start = out.len();
for (i, &x) in src.iter().enumerate() {
let pred = predict(tag, &out[start..], &up, i, l.bpp);
out.push(x.wrapping_add(pred));
}
up[..src.len()].copy_from_slice(&out[start..]);
}
Ok(out)
}
fn png_filter(tag: u8, cur: &[u8], up: &[u8], bpp: usize, dst: &mut Vec<u8>) {
dst.clear();
dst.push(tag);
dst.extend(cur.iter().enumerate().map(|(i, &x)| x.wrapping_sub(predict(tag, cur, up, i, bpp))));
}
/// Sum of residuals as signed bytes: the usual heuristic for choosing a PNG filter.
fn cost(filtered: &[u8]) -> u64 {
filtered[1..].iter().map(|&b| u64::from((b as i8).unsigned_abs())).sum()
}
fn png_encode(l: &Layout, data: &[u8], predictor: i64) -> Vec<u8> {
let rows = data.len().div_ceil(l.row.max(1));
let mut out = Vec::with_capacity(data.len() + rows);
let zeros = vec![0u8; l.row.min(data.len())];
let (mut best, mut trial) = (Vec::new(), Vec::new());
let mut up: &[u8] = &zeros;
for cur in data.chunks(l.row) {
let up_row = &up[..cur.len()];
if predictor == 15 {
png_filter(0, cur, up_row, l.bpp, &mut best);
let mut best_cost = cost(&best);
for tag in 1..=4 {
png_filter(tag, cur, up_row, l.bpp, &mut trial);
let c = cost(&trial);
if c < best_cost {
best_cost = c;
std::mem::swap(&mut best, &mut trial);
}
}
} else {
png_filter((predictor - 10) as u8, cur, up_row, l.bpp, &mut best);
}
out.extend_from_slice(&best);
up = cur;
}
out
}
#[cfg(test)]
mod tests {
use super::*;
fn p(predictor: i64, colors: i64, bpc: i64, columns: i64) -> Params {
Params { predictor, colors, bits_per_component: bpc, columns, early_change: 1 }
}
fn dec(params: &Params, data: &[u8]) -> Result<Vec<u8>, FilterError> {
decode(params, data.to_vec(), "FlateDecode", None).map_err(|f| f.error)
}
#[test]
fn png_hand_vectors() {
// 3 columns, 1 color, 8 bpc. Rows: None, Sub, Up, Average, Paeth.
let enc = [
0, 10, 20, 30, // None -> 10 20 30
1, 5, 1, 1, // Sub -> 5 6 7
2, 1, 1, 1, // Up -> 6 7 8
3, 10, 10, 10, // Avg -> 10+3=13, 10+(13+7)/2=20, 10+(20+8)/2=24
4, 1, 1,
1, // Paeth: a=0,b=13,c=0 -> p=13 -> b: 14; a=14,b=20,c=13: p=21 -> pa=7,pb=1,pc=8 -> b: 21;
// a=21,b=24,c=20: p=25 -> pa=4,pb=1,pc=5 -> b: 25
];
let out = dec(&p(15, 1, 8, 3), &enc).unwrap();
assert_eq!(out, [10, 20, 30, 5, 6, 7, 6, 7, 8, 13, 20, 24, 14, 21, 25]);
}
#[test]
fn paeth_tie_breaks() {
// RFC 2083 order: a, then b, then c.
assert_eq!(paeth(1, 1, 1), 1);
assert_eq!(paeth(10, 20, 15), 15); // p=15: pa=5 pb=5 pc=0 -> c
assert_eq!(paeth(20, 10, 10), 20); // p=20: pa=0 -> a
assert_eq!(paeth(5, 9, 7), 7); // p=7: pa=2 pb=2 pc=0 -> c
}
#[test]
fn png_multibyte_pixels() {
// colors=3 bpc=16 -> bpp=6; 1 column; Sub on the first pixel uses a=0 throughout.
let enc = [1, 1, 2, 3, 4, 5, 6, 2, 1, 1, 1, 1, 1, 1];
let out = dec(&p(11, 3, 16, 1), &enc).unwrap();
assert_eq!(out, [1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7]);
// colors=3 bpc=8, 2 columns: Sub looks 3 bytes back.
let enc = [1, 1, 2, 3, 1, 1, 1];
assert_eq!(dec(&p(11, 3, 8, 2), &enc).unwrap(), [1, 2, 3, 2, 3, 4]);
}
#[test]
fn png_partial_final_row() {
let enc = [0, 1, 2, 3, 4, 2, 1, 1];
assert_eq!(dec(&p(12, 1, 8, 4), &enc).unwrap(), [1, 2, 3, 4, 2, 3]);
// A lone tag byte at the end yields nothing.
assert_eq!(dec(&p(12, 1, 8, 4), &[0, 1, 2, 3, 4, 2]).unwrap(), [1, 2, 3, 4]);
}
#[test]
fn png_invalid_tag() {
let r = decode(&p(10, 1, 8, 2), vec![0, 1, 2, 5, 1, 1], "FlateDecode", None);
let f = r.err().unwrap();
assert!(matches!(f.error, FilterError::Corrupt { .. }));
assert_eq!(f.partial, [1, 2]);
}
#[test]
fn tiff_8bit() {
// 2 colors, 3 columns: differences per component.
let enc = [10, 20, 1, 2, 3, 4, /* row 2 */ 5, 5, 250, 10, 1, 1];
assert_eq!(dec(&p(2, 2, 8, 3), &enc).unwrap(), [10, 20, 11, 22, 14, 26, 5, 5, 255, 15, 0, 16]);
}
#[test]
fn tiff_16bit() {
let enc = [0x01, 0x00, 0xff, 0xff, 0x00, 0x02];
assert_eq!(dec(&p(2, 1, 16, 3), &enc).unwrap(), [0x01, 0x00, 0x00, 0xff, 0x01, 0x01]);
}
#[test]
fn tiff_sub_byte() {
// 1 bpc, 1 color, 10 columns: bits 1 0 0 1 1 0 0 0 | 1 1 (pad 000000)
// Cumulative XOR: 1 1 1 0 1 1 1 1 | 0 1
let enc = [0b1001_1000, 0b1100_0000];
assert_eq!(dec(&p(2, 1, 1, 10), &enc).unwrap(), [0b1110_1111, 0b0100_0000]);
// 4 bpc, 1 color, 3 columns: samples 5, 15, 3 -> 5, 4, 7 (mod 16).
assert_eq!(dec(&p(2, 1, 4, 3), &[0x5f, 0x30]).unwrap(), [0x54, 0x70]);
// 2 bpc, 2 colors, 2 columns: samples 1 2 3 3 -> 1 2 0 1.
assert_eq!(dec(&p(2, 2, 2, 2), &[0b0110_1111]).unwrap(), [0b0110_0001]);
// Padding bits are left alone.
assert_eq!(dec(&p(2, 1, 4, 1), &[0x5f]).unwrap(), [0x5f]);
}
#[test]
fn tiff_partial_rows() {
// 16-bit with an odd trailing byte: the lone byte is left as is.
assert_eq!(dec(&p(2, 1, 16, 4), &[0, 1, 0, 1, 9]).unwrap(), [0, 1, 0, 2, 9]);
assert_eq!(dec(&p(2, 1, 8, 4), &[1, 1, 1, 1, 2, 2]).unwrap(), [1, 2, 3, 4, 2, 4]);
}
#[test]
fn invalid_params() {
for bad in [p(2, 1, 3, 1), p(12, 0, 8, 1), p(12, 1, 8, 0), p(3, 1, 8, 1), p(12, 1, 8, i64::MAX), p(2, 300, 8, 1)] {
assert!(matches!(dec(&bad, &[0, 1, 2]), Err(FilterError::Corrupt { .. })), "{bad:?}");
assert!(encode(&bad, &[1, 2]).is_err());
}
// Huge but representable Columns does not allocate a huge row.
assert_eq!(dec(&p(12, 1, 8, 1 << 40), &[2, 7, 8]).unwrap(), [7, 8]);
}
#[test]
fn encode_choices() {
let data = [1u8, 2, 3, 4, 5, 6];
let up = encode(&p(12, 1, 8, 3), &data).unwrap();
assert_eq!(up, [2, 1, 2, 3, 2, 3, 3, 3]);
let paeth = encode(&p(14, 1, 8, 3), &data).unwrap();
assert_eq!(paeth[0], 4);
assert_eq!(paeth[4], 4);
// Per-row optimum picks Sub for a ramp.
let ramp: Vec<u8> = (0..64).collect();
let opt = encode(&p(15, 1, 8, 64), &ramp).unwrap();
assert_eq!(opt[0], 1);
for pred in [10, 11, 12, 13, 14, 15] {
let e = encode(&p(pred, 1, 8, 3), &data).unwrap();
assert_eq!(dec(&p(pred, 1, 8, 3), &e).unwrap(), data);
}
}
}
-117
View File
@@ -1,117 +0,0 @@
//! RunLengthDecode (ISO 32000-2 §7.4.5): a length byte L of 0–127 copies the next
//! L + 1 bytes, 129–255 repeats the next byte 257 − L times, 128 is EOD.
//! A missing EOD is tolerated; a truncated run is corrupt.
use crate::{Failure, Step};
const NAME: &str = "RunLengthDecode";
pub(crate) fn decode(data: &[u8], max: usize) -> Step {
let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max));
let mut i = 0;
while let Some(&l) = data.get(i) {
i += 1;
match l {
128 => break,
0..=127 => {
let n = usize::from(l) + 1;
let avail = n.min(data.len() - i);
if out.len() + avail > max {
return Err(Failure::limit(max));
}
out.extend_from_slice(&data[i..i + avail]);
i += avail;
if avail < n {
return Err(Failure::corrupt(NAME, "truncated literal run", out));
}
}
_ => {
let Some(&b) = data.get(i) else {
return Err(Failure::corrupt(NAME, "truncated repeat run", out));
};
i += 1;
let n = 257 - usize::from(l);
if out.len() + n > max {
return Err(Failure::limit(max));
}
out.resize(out.len() + n, b);
}
}
}
Ok(out)
}
pub(crate) fn encode(data: &[u8]) -> Vec<u8> {
let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2);
let n = data.len();
let mut i = 0;
while i < n {
let mut run = 1;
while i + run < n && run < 128 && data[i + run] == data[i] {
run += 1;
}
if run >= 2 {
out.push((257 - run) as u8);
out.push(data[i]);
i += run;
continue;
}
// Literal: extend until a repeat begins or 128 bytes.
let start = i;
let mut j = i + 1;
while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) {
j += 1;
}
out.push((j - start - 1) as u8);
out.extend_from_slice(&data[start..j]);
i = j;
}
out.push(128);
out
}
#[cfg(test)]
mod tests {
use super::*;
use crate::FilterError;
fn dec(d: &[u8]) -> Result<Vec<u8>, FilterError> {
decode(d, 10_000).map_err(|f| f.error)
}
#[test]
fn vectors() {
assert_eq!(dec(&[2, b'a', b'b', b'c', 254, b'x', 128]).unwrap(), b"abcxxx");
assert_eq!(dec(&[129, 7, 128]).unwrap(), [7; 128]);
assert!(dec(&[127]).is_err());
assert_eq!(dec(&[0, b'q']).unwrap(), b"q"); // missing EOD
assert_eq!(dec(&[128, 0, b'q']).unwrap(), b""); // data after EOD ignored
assert_eq!(dec(&[]).unwrap(), b"");
}
#[test]
fn truncated() {
let f = decode(&[4, b'a', b'b'], 100).err().unwrap();
assert!(matches!(f.error, FilterError::Corrupt { .. }));
assert_eq!(f.partial, b"ab");
assert!(matches!(dec(&[0, b'a', 200]), Err(FilterError::Corrupt { .. })));
}
#[test]
fn limit() {
assert!(matches!(decode(&[129, 0, 129, 0], 200).map_err(|f| f.error), Err(FilterError::LimitExceeded(200))));
assert!(matches!(decode(&[3, 1, 2, 3, 4], 3).map_err(|f| f.error), Err(FilterError::LimitExceeded(3))));
assert_eq!(decode(&[129, 0, 129, 0], 256).ok().unwrap().len(), 256);
}
#[test]
fn encoding() {
assert_eq!(encode(b"abcxxx"), [2, b'a', b'b', b'c', 254, b'x', 128]);
assert_eq!(encode(b""), [128]);
assert_eq!(encode(&[5; 300]), [129, 5, 129, 5, 213, 5, 128]);
let lit: Vec<u8> = (0..200u8).collect();
let e = encode(&lit);
assert_eq!((e[0], e[129]), (127, 71));
assert_eq!(dec(&e).unwrap(), lit);
}
}
+1
View File
@@ -13,6 +13,7 @@ pdfcraft-cos = { workspace = true }
pdfcraft-annot = { workspace = true }
pdfcraft-forms = { workspace = true }
pdfcraft-model = { workspace = true }
craft-csv = { workspace = true }
roxmltree = "0.21"
base64 = "0.23"
thiserror = { workspace = true }
+14 -2
View File
@@ -123,7 +123,13 @@ pub fn export_data(doc: &Document, format: Format) -> String {
// A formula-like cell gets a leading `'`, as in `merge_csv`; import takes it off again.
let q = |s: &str| {
let s = if csv_guarded(s) { format!("'{s}") } else { s.to_string() };
if s.contains([',', '"', '\n', '\r', '\t']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s }
if s.contains('\t') {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
let mut out = String::new();
craft_csv::push_field(&mut out, &s, ',');
out
}
};
format!(
"{}\n{}\n",
@@ -202,7 +208,13 @@ pub fn merge_csv(files: &[Vec<(String, Vec<String>)>]) -> String {
let literal = |s: &str| if runs_as_formula(s) { format!("'{s}") } else { s.to_string() };
let q = |s: &str| {
let s = literal(s);
if s.contains([',', '"', '\n', '\r', '\t']) { format!("\"{}\"", s.replace('"', "\"\"")) } else { s }
if s.contains('\t') {
format!("\"{}\"", s.replace('"', "\"\""))
} else {
let mut out = String::new();
craft_csv::push_field(&mut out, &s, ',');
out
}
};
let mut out = columns.iter().map(|c| q(c)).collect::<Vec<_>>().join(",");
out.push('\n');