diff --git a/crates/redact/src/globals.rs b/crates/redact/src/globals.rs index 476b3d955b..735bbc3667 100644 --- a/crates/redact/src/globals.rs +++ b/crates/redact/src/globals.rs @@ -7,6 +7,7 @@ use std::collections::hash_map::DefaultHasher; use std::collections::{HashMap, HashSet}; use std::hash::{Hash, Hasher}; +use std::rc::Rc; use std::sync::Arc; use pdfcraft_cos::{DECODE_PARMS_KEYS, Dict, Document, MAX_JBIG2_GLOBALS, Object, Stream, reachable}; @@ -33,39 +34,75 @@ pub(crate) fn followed(doc: &Document, o: &Object) -> Arc { at } -/// Every object `s` names as `/JBIG2Globals`, and each reference on the way to one: in any entry of -/// its parameters, whichever filter that entry belongs to and however the filters are given. A +/// What a parameters value names as `/JBIG2Globals`, and the key it is remembered by: a value the +/// file refers to, by its object number and whether it is read as a list of entries. +pub(crate) type Listed = (Rc<[u32]>, Option<(u32, bool)>); + +/// Every object a stream names as `/JBIG2Globals`, and each reference on the way to one: in any entry +/// of its parameters, whichever filter that entry belongs to and however the filters are given. A /// reader may use less of this, but leaving a dictionary behind is the failure that matters here. -fn named(doc: &Document, s: &Stream) -> Vec { - let mut found = Vec::new(); - let mut read = |entry: &Object| { - let held = followed(doc, entry); - let Some(mut at) = held.as_dict().and_then(|d| d.get(b"JBIG2Globals")).and_then(Object::as_ref) else { return }; - for _ in 0..=MAX_HOPS { - found.push(at.num); - match &*doc.get(at) { - Object::Ref(next) => at = *next, - _ => return, - } +/// +/// What a value the file refers to names is read once a run, so streams that share one, by +/// reference or through an entry, cost it once however many there are. +#[derive(Default)] +pub(crate) struct Names { + memo: HashMap<(u32, bool), Rc<[u32]>>, + /// How many values were read rather than remembered. + pub(crate) read: usize, +} + +impl Names { + /// What `s` names, one list for each key that gives it parameters. + pub(crate) fn of(&mut self, doc: &Document, s: &Stream) -> Vec { + // `/FDecodeParms` too: those of data in another file, which a reader may draw instead. + [b"DecodeParms".as_slice(), b"DP", b"FDecodeParms"].into_iter().filter_map(|key| s.dict.get(key)).map(|v| self.value(doc, v, true)).collect() + } + + /// What `v` names: one entry, or (`list`) an array of them or one. + fn value(&mut self, doc: &Document, v: &Object, list: bool) -> Listed { + let key = v.as_ref().map(|r| (r.num, list)); + if let Some(hit) = key.and_then(|k| self.memo.get(&k)) { + return (hit.clone(), key); } - }; - // `/FDecodeParms` too: those of data in another file, which a reader may draw instead. - for key in [b"DecodeParms".as_slice(), b"DP", b"FDecodeParms"] { - let Some(parms) = s.dict.get(key).map(|p| followed(doc, p)) else { continue }; - match &*parms { - Object::Array(entries) => entries.iter().for_each(&mut read), - other => read(other), + self.read += 1; + let names: Rc<[u32]> = match (&*followed(doc, v), list) { + (Object::Array(entries), true) => { + let mut all: Vec = entries.iter().flat_map(|e| self.value(doc, e, false).0.to_vec()).collect(); + all.sort_unstable(); + all.dedup(); + all.into() + } + (entry, _) => entry_names(doc, entry).into(), + }; + if let Some(k) = key { + self.memo.insert(k, names.clone()); + } + (names, key) + } +} + +/// What one parameters entry names as `/JBIG2Globals`, and each reference on the way to it. +fn entry_names(doc: &Document, entry: &Object) -> Vec { + let mut found = Vec::new(); + let Some(mut at) = entry.as_dict().and_then(|d| d.get(b"JBIG2Globals")).and_then(Object::as_ref) else { return found }; + for _ in 0..=MAX_HOPS { + found.push(at.num); + match &*doc.get(at) { + Object::Ref(next) => at = *next, + _ => break, } } found } /// A `/Filter` through any references, as a decoder reads it: a name, or an array of names, each -/// entry through references too. Anything else is null, so nothing large is copied however often -/// the file refers to it. +/// entry through references too. Anything else is null, a chain longer than any decoder takes +/// (`MAX_CHAIN`, and a `/Crypt`) included, so nothing large is copied however often the file refers +/// to it, and a stream that names it is read as one with no filter. fn direct_filter(doc: &Document, v: &Object) -> Object { let name = |o: &Object| if matches!(o, Object::Name(_)) { o.clone() } else { Object::Null }; match &*followed(doc, v) { + Object::Array(a) if a.len() > pdfcraft_filters::MAX_CHAIN + 1 => Object::Null, Object::Array(a) => Object::Array(a.iter().map(|e| name(&followed(doc, e))).collect()), other => name(other), } @@ -80,8 +117,8 @@ fn direct_number(doc: &Document, v: &Object) -> Object { } /// `/DecodeParms` as a reader takes them, through any references, for `filters` filters: of each -/// entry, the numbers and flags the decoders read ([`DECODE_PARMS_KEYS`]), and `/JBIG2Globals` as -/// given. An entry that isn't a dictionary is null. As `Document::with_direct_parms`, nothing large +/// entry, the numbers and flags the decoders read ([`DECODE_PARMS_KEYS`]), and `/JBIG2Globals` when it +/// is a reference. An entry that isn't a dictionary is null. As `Document::with_direct_parms`, nothing large /// is copied however often the file refers to it. fn direct_parms(doc: &Document, v: &Object, filters: usize) -> Object { let entry = |e: &Object| match &*followed(doc, e) { @@ -94,7 +131,9 @@ fn direct_parms(doc: &Document, v: &Object, filters: usize) -> Object { made.set(key.to_vec(), (*value).clone()); } } - if let Some(globals) = d.get(b"JBIG2Globals") { + // As a reference only, as `Document::with_direct_parms` keeps it: a direct value is no + // stream, so no decoder reads it, and it may be large. + if let Some(globals @ Object::Ref(_)) = d.get(b"JBIG2Globals") { made.set(b"JBIG2Globals".to_vec(), globals.clone()); } Object::Dict(made) @@ -122,8 +161,8 @@ pub(crate) fn readable(doc: &Document, s: &Stream) -> Stream { if let Some(v) = s.dict.get(b"DecodeParms") { let filters = match readable.dict.get(b"Filter") { Some(Object::Array(a)) => a.len(), - Some(_) => 1, - None => 0, + Some(Object::Name(_)) => 1, + _ => 0, }; readable.dict.set(b"DecodeParms".to_vec(), direct_parms(doc, v, filters)); } @@ -196,12 +235,19 @@ fn without_dictionary(doc: &Document, s: &Stream) -> Option { /// changed by then: a caller works on a copy it can drop. pub(crate) fn unshare(doc: &mut Document, before: &HashSet) -> Result { let now = kept(doc); + let mut names = Names::default(); let mut dictionaries: HashSet = HashSet::new(); + // A list the file refers to is added once, however many streams share it. + let mut added: HashSet<(u32, bool)> = HashSet::new(); for &num in before.difference(&now) { if let Ok(object) = doc.try_get(num) && let Object::Stream(s) = &*object { - dictionaries.extend(named(doc, s)); + for (list, key) in names.of(doc, s) { + if key.is_none_or(|k| added.insert(k)) { + dictionaries.extend(list.iter()); + } + } } } if dictionaries.is_empty() { @@ -209,6 +255,11 @@ pub(crate) fn unshare(doc: &mut Document, before: &HashSet) -> Result = dictionaries.iter().filter_map(|n| content(doc, *n)).collect(); let mut read: HashMap> = HashMap::new(); + // Whether a name matches doesn't depend on order: it is a dictionary that left, or a copy of one + // (what is added below is only ever such a copy). So what a shared list matches is found once. + let left = dictionaries.clone(); + let mut matches: HashMap<(u32, bool), Rc<[u32]>> = HashMap::new(); + added.clear(); let mut users = Vec::new(); for (r, object) in doc.scan_objects() { let Object::Stream(s) = &*object else { continue }; @@ -220,9 +271,26 @@ pub(crate) fn unshare(doc: &mut Document, before: &HashSet) -> Result = named(doc, s).into_iter().filter(|n| dictionaries.contains(n) || copy(n, &mut read)).collect(); - if !matched.is_empty() { - dictionaries.extend(matched); + let mut uses = false; + for (list, key) in names.of(doc, s) { + let matched = match key.and_then(|k| matches.get(&k)) { + Some(hit) => hit.clone(), + None => { + let found: Rc<[u32]> = list.iter().copied().filter(|n| left.contains(n) || copy(n, &mut read)).collect(); + if let Some(k) = key { + matches.insert(k, found.clone()); + } + found + } + }; + if !matched.is_empty() { + uses = true; + if key.is_none_or(|k| added.insert(k)) { + dictionaries.extend(matched.iter()); + } + } + } + if uses { users.push(r); } } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index eb1d1d96dd..dd1d9b4b43 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -1798,11 +1798,25 @@ fn a_dictionary_that_names_itself_isnt_counted_as_stored() { fn what_is_made_direct_is_small_however_often_a_large_object_is_named() { let big = format!("[{}]", "1 ".repeat(100_000)); let keys: String = (0..50).map(|k| format!(" /K{k} 2 0 R")).collect(); - // Three filters, and parameters for 2,001: the entries past the filters are dropped. - let (filters, entries) = (" 2 0 R".repeat(2), " 2 0 R".repeat(2000)); + // Three filters, and parameters for 2,001: the entries past the filters are dropped. Two name + // object 6, whose `/JBIG2Globals` is the large array given directly, which no decoder reads. + let (filters, entries) = (" 2 0 R".repeat(2), format!(" 6 0 R 6 0 R{}", " 2 0 R".repeat(1998))); let dict = format!("/Width 2 0 R /Height 2 0 R /Filter [/FlateDecode{filters}] /DecodeParms [<< /Predictor 3 0 R /Columns 2 0 R{keys} >>{entries}]"); - let doc = pdf(vec![b"<< /Type /Catalog >>".to_vec(), big.into_bytes(), b"4 0 R".to_vec(), b"12".to_vec(), stream(&dict, b"")]); + let globals = format!("<< /JBIG2Globals {big} >>"); + // A chain of more filters than any decoder takes is no filter, and its parameters aren't read: + // ten here, against eight and a `/Crypt`. + let long = format!("/Filter [{}] /DecodeParms 2 0 R", "/FlateDecode ".repeat(10)); + let doc = pdf(vec![ + b"<< /Type /Catalog >>".to_vec(), + big.into_bytes(), + b"4 0 R".to_vec(), + b"12".to_vec(), + stream(&dict, b""), + globals.into_bytes(), + stream(&long, b""), + stream(&format!("/Filter [{}]", "/FlateDecode ".repeat(9)), b""), + ]); let Object::Stream(s) = &*doc.get(ObjRef::new(5, 0)) else { panic!("object 5 is the stream") }; let made = crate::globals::readable(&doc, s); let mut bytes = Vec::new(); @@ -1811,6 +1825,33 @@ fn what_is_made_direct_is_small_however_often_a_large_object_is_named() { let parms = made.dict.get(b"DecodeParms").and_then(Object::as_array).unwrap(); assert_eq!(parms.first().and_then(Object::as_dict).and_then(|d| d.get(b"Predictor")), Some(&Object::Int(12))); assert_eq!(parms.len(), 3, "an entry for each filter"); + assert!(parms[1].as_dict().is_some_and(|d| !d.contains(b"JBIG2Globals")), "{:?}", parms[1]); + let Object::Stream(s) = &*doc.get(ObjRef::new(7, 0)) else { panic!("object 7 is the stream") }; + let made = crate::globals::readable(&doc, s); + assert_eq!(made.dict.get(b"Filter"), Some(&Object::Null)); + assert_eq!(made.dict.get(b"DecodeParms").and_then(Object::as_array).map(Vec::len), Some(0)); + // Eight filters and a `/Crypt` are as many as a decoder takes: those are read. + let Object::Stream(s) = &*doc.get(ObjRef::new(8, 0)) else { panic!("object 8 is the stream") }; + let made = crate::globals::readable(&doc, s); + assert_eq!(made.dict.get(b"Filter").and_then(Object::as_array).map(Vec::len), Some(9)); +} + +/// What streams share by reference is read once a run, however many share it: a parameters array +/// of 5,000 entries named by 100 streams costs the array and its one entry, once. +#[test] +fn a_shared_parameters_array_is_read_once_however_many_streams_share_it() { + let entries = " 4 0 R".repeat(5000); + let mut objects = + vec![b"<< /Type /Catalog >>".to_vec(), format!("[{entries}]").into_bytes(), stream("", b"symbols"), b"<< /JBIG2Globals 3 0 R >>".to_vec()]; + objects.extend((0..100).map(|_| stream("/Filter /JBIG2Decode /DecodeParms 2 0 R", b""))); + let doc = pdf(objects); + let mut names = crate::globals::Names::default(); + for n in 5..105 { + let Object::Stream(s) = &*doc.get(ObjRef::new(n, 0)) else { panic!("object {n} is a stream") }; + let lists = names.of(&doc, s); + assert_eq!(lists.iter().map(|(list, _)| list.to_vec()).collect::>(), [vec![3]]); + } + assert_eq!(names.read, 2); } /// A copy of the dictionary stored with a predictor is found whatever way its parameters are given,