mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-10 13:35:21 +00:00
M8: a shared JBIG2 dictionary leaves with whatever used it, by any route
Redaction starts from the images a run replaces or removes and the forms and soft masks under them, so a dictionary used by something that left another way stays, with apply returning Ok: a stamp or widget appearance, a Type 3 glyph, an /Alternates entry, a /Mask, or an image with /Filter by reference or /DecodeParms through a reference to a reference. A byte-identical copy stays too, which is what pdfseparate then pdfunite leaves, and Remove Hidden Information never takes a dictionary out. Compare what a full save would keep before and after the run instead, the same walk write_full makes, now cos::reachable. What left names the dictionaries, in any entry of its parameters; what stays that names one, or a copy, is stored again without it; and the walk is taken once more, so a dictionary still kept refuses the run, whatever keeps it. An image with /Width, /Height or /Filter by reference is read, and is one bit in the copy, as it was when drawn. /Alternates stays with what is stored again, and a stream that isn't JBIG2 refuses rather than being stored. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY
This commit is contained in:
Grzegorz Martens
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
0cba87ae6f
commit
580abc51cf
@@ -326,17 +326,6 @@ impl Document {
|
||||
pdfcraft_filters::decode_ccitt(&data, &p, rows).map_err(filter)
|
||||
}
|
||||
|
||||
/// The `/JBIG2Globals` stream a JBIG2 image `s` names by reference in its last filter's
|
||||
/// `/DecodeParms`: `None` for a stream whose last filter isn't `JBIG2Decode`, or one without
|
||||
/// globals. Redaction finds with it the other images that share a redacted image's globals.
|
||||
pub fn jbig2_globals_ref(&self, s: &Stream) -> Option<ObjRef> {
|
||||
let chain = s.readable_filters().ok()?;
|
||||
if !matches!(chain.last(), Some((pdfcraft_filters::Filter::Jbig2, _))) {
|
||||
return None;
|
||||
}
|
||||
self.last_filter_parms(s).get(b"JBIG2Globals").and_then(Object::as_ref)
|
||||
}
|
||||
|
||||
/// What the `/JBIG2Globals` stream of JBIG2 image `s` (in its last filter's `/DecodeParms`)
|
||||
/// decodes to, within [`MAX_JBIG2_GLOBALS`]; `None` without one (a reference to no object is
|
||||
/// one to null, ISO 32000-2 §7.3.10, which is an absent entry, §7.3.7). The image is then read
|
||||
|
||||
@@ -1,5 +1,26 @@
|
||||
use super::*;
|
||||
|
||||
/// What [`crate::reachable`] gives is what a full save keeps: not an object nothing refers to, nor one
|
||||
/// a reference only dangles at, and a ring of references ends.
|
||||
#[test]
|
||||
fn what_is_reachable_is_what_a_full_save_keeps() {
|
||||
let mut doc = Document::new_empty();
|
||||
let root = doc.root().unwrap();
|
||||
let kept = doc.add(Object::Int(7));
|
||||
let dangling = ObjRef::new(4_000, 0);
|
||||
let list = doc.add(Object::Null);
|
||||
let ring = doc.add(Object::Array(vec![Object::Ref(list)]));
|
||||
doc.set(list, Object::Array(vec![Object::Ref(kept), Object::Ref(ring), Object::Ref(dangling)]));
|
||||
let orphan = doc.add(Object::Int(8));
|
||||
doc.update_dict(root, |d| d.set(b"Held".to_vec(), Object::Ref(list))).unwrap();
|
||||
let found = crate::reachable(&doc);
|
||||
assert!([root, kept, list, ring].iter().all(|r| found.contains(r)));
|
||||
assert!(!found.contains(&orphan) && !found.contains(&dangling));
|
||||
let saved = Document::open(Arc::new(crate::write_full(&doc, &crate::SaveOptions::default()).unwrap())).unwrap();
|
||||
let holds = |n: i64| saved.scan_objects().any(|(_, o)| matches!(*o, Object::Int(v) if v == n));
|
||||
assert!(holds(7) && !holds(8));
|
||||
}
|
||||
|
||||
/// /DecodeParms given by reference, whole, as array entries or as the numbers inside, are read
|
||||
/// once made direct: here a Flate predictor, which the stream alone doesn't see. Only the entries the filters use are
|
||||
/// resolved, and nothing is for a chain longer than a decoder takes.
|
||||
|
||||
@@ -30,7 +30,7 @@ pub use parser::{Lexer, is_whitespace, parse_indirect};
|
||||
pub use pdfcraft_crypt::{
|
||||
Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char,
|
||||
};
|
||||
pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental};
|
||||
pub use writer::{SaveOptions, pdf_date, reachable, serialize, write_full, write_incremental};
|
||||
|
||||
#[derive(Debug, thiserror::Error, Clone, PartialEq)]
|
||||
pub enum CosError {
|
||||
|
||||
+44
-30
@@ -276,36 +276,9 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result<Vec<u8>, CosErro
|
||||
let mut reader = doc.object_reader();
|
||||
stamp_mod_date(&mut reader.document, opts);
|
||||
let trailer_in = reader.document.trailer().clone();
|
||||
// Breadth-first walk from the trailer, assigning new numbers in visit order.
|
||||
let mut map: HashMap<ObjRef, u32> = HashMap::new();
|
||||
// Record the type while traversing so classifying packed objects needs no second parse.
|
||||
let mut order: Vec<(ObjRef, bool)> = Vec::new();
|
||||
let mut queue: VecDeque<ObjRef> = VecDeque::new();
|
||||
let visit_refs = |o: &Object, queue: &mut VecDeque<ObjRef>| collect_refs(o, &mut |r| queue.push_back(r));
|
||||
for key in [&b"Root"[..], b"Info", b"Encrypt"] {
|
||||
if let Some(r) = trailer_in.reference(key) {
|
||||
queue.push_back(r);
|
||||
}
|
||||
}
|
||||
// Unknown trailer entries are document data. Objects they alone reference must be written
|
||||
// too, or a full rewrite would keep the key and drop its value.
|
||||
for (key, value) in trailer_in.iter() {
|
||||
if !trailer_key_rewritten(key) {
|
||||
collect_refs(value, &mut |r| queue.push_back(r));
|
||||
}
|
||||
}
|
||||
while let Some(r) = queue.pop_front() {
|
||||
if map.contains_key(&r) {
|
||||
continue;
|
||||
}
|
||||
let o = reader.get(r);
|
||||
if matches!(*o, Object::Null) {
|
||||
continue; // dangling reference: written as null by renumbering below
|
||||
}
|
||||
map.insert(r, output_number(order.len())?);
|
||||
order.push((r, matches!(*o, Object::Stream(_))));
|
||||
visit_refs(&o, &mut queue);
|
||||
}
|
||||
// New numbers go in visit order.
|
||||
let order = walk(&mut reader, &trailer_in);
|
||||
let map: HashMap<ObjRef, u32> = order.iter().enumerate().map(|(i, (r, _))| Ok((*r, output_number(i)?))).collect::<Result<_, CosError>>()?;
|
||||
let version = reader.document.version().chars().take(3).collect::<String>();
|
||||
let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version };
|
||||
let mut out = Vec::new();
|
||||
@@ -486,6 +459,47 @@ pub(crate) fn unlinked_additions(doc: &Document) -> HashSet<u32> {
|
||||
pending.into_keys().collect()
|
||||
}
|
||||
|
||||
/// The objects a full save keeps: breadth first from `trailer`, each with whether it is a stream, so
|
||||
/// classifying packed objects needs no second parse. A dangling reference is left out, and written
|
||||
/// as null by renumbering.
|
||||
fn walk(reader: &mut ObjectReader, trailer: &Dict) -> Vec<(ObjRef, bool)> {
|
||||
let mut seen: HashSet<ObjRef> = HashSet::new();
|
||||
let mut order: Vec<(ObjRef, bool)> = Vec::new();
|
||||
let mut queue: VecDeque<ObjRef> = VecDeque::new();
|
||||
for key in [&b"Root"[..], b"Info", b"Encrypt"] {
|
||||
if let Some(r) = trailer.reference(key) {
|
||||
queue.push_back(r);
|
||||
}
|
||||
}
|
||||
// Unknown trailer entries are document data. Objects they alone reference must be written
|
||||
// too, or a full rewrite would keep the key and drop its value.
|
||||
for (key, value) in trailer.iter() {
|
||||
if !trailer_key_rewritten(key) {
|
||||
collect_refs(value, &mut |r| queue.push_back(r));
|
||||
}
|
||||
}
|
||||
while let Some(r) = queue.pop_front() {
|
||||
if seen.contains(&r) {
|
||||
continue;
|
||||
}
|
||||
let o = reader.get(r);
|
||||
if matches!(*o, Object::Null) {
|
||||
continue;
|
||||
}
|
||||
seen.insert(r);
|
||||
order.push((r, matches!(*o, Object::Stream(_))));
|
||||
collect_refs(&o, &mut |r| queue.push_back(r));
|
||||
}
|
||||
order
|
||||
}
|
||||
|
||||
/// The objects [`write_full`] would keep of `doc`: anything else leaves the file on a full save.
|
||||
pub fn reachable(doc: &Document) -> HashSet<ObjRef> {
|
||||
let mut reader = doc.object_reader();
|
||||
let trailer = reader.document.trailer().clone();
|
||||
walk(&mut reader, &trailer).into_iter().map(|(r, _)| r).collect()
|
||||
}
|
||||
|
||||
fn collect_refs(o: &Object, f: &mut impl FnMut(ObjRef)) {
|
||||
match o {
|
||||
Object::Ref(r) => f(*r),
|
||||
|
||||
+10
-4
@@ -24,10 +24,16 @@ clear_marks(&mut doc, None)?; // remove marks without app
|
||||
is Flate, or for a one-bit image of one component (a scan as a rule) CCITT Group 4, or JBIG2
|
||||
when it was JBIG2, whichever is smallest as stored, coded from the cleared samples; it refers to
|
||||
no globals. Globals hold the shapes of every image that shares them (a scan's pages, often),
|
||||
those of marks under the area included, so every other image that uses the globals of a removed
|
||||
or replaced image is decoded and stored again in place without them, as a copy is (Flate,
|
||||
Group 4 or JBIG2, whichever is smallest; `Report::images_restored`), and they leave the file on the full rewrite; one that can't be
|
||||
decoded for certain refuses the redaction (`RedactError::SharedJbig2Globals`). Codecs redaction doesn't decode (DCT, JPX) are
|
||||
those of marks under the area included, so whatever left the document, by any route, takes its
|
||||
globals with it on the full rewrite: anything else the save would keep that names them, or a
|
||||
byte-for-byte copy of them, is stored again without them, as a copy is (Flate, Group 4 or JBIG2,
|
||||
whichever is smallest; `Report::images_restored`). When one can't be (it isn't JBIG2, it's past
|
||||
the size redaction clears, or it can't be decoded), or something else refers to them, `apply`
|
||||
refuses and changes nothing (`RedactError::SharedJbig2Globals`). `remove_hidden` takes them out
|
||||
the same way. Another dictionary holding the same shapes in other bytes isn't found, nor is a copy
|
||||
nothing names, and an image that hasn't left keeps its globals with its pixels: one another page
|
||||
still draws, or one something that doesn't draw it still refers to. Each image is decoded once,
|
||||
with no total across the document yet. Codecs redaction doesn't decode (DCT, JPX) are
|
||||
removed whole (fail-closed), and so is a JBIG2 image whose page isn't its size (the renderer
|
||||
draws the whole page into the image, so the rows cleared wouldn't be the ones shown) or of more
|
||||
than one bit or component, and Group 3 on bytes (`/K` 0 or more, `/EncodedByteAlign`): PdfCraft's
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
//! A `/JBIG2Globals` stream is a dictionary of symbols that JBIG2 images can share, and scanners
|
||||
//! and `jbig2enc -s` share one across a scan's pages. It holds the shape of every distinct mark of
|
||||
//! those pages, including a mark found only under a redaction, so taking the image out isn't enough:
|
||||
//! while something else uses the dictionary, a full save keeps it. [`unshare`] stores what uses it
|
||||
//! again without it.
|
||||
|
||||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
|
||||
use pdfcraft_cos::{Document, MAX_JBIG2_GLOBALS, Object, Stream, reachable};
|
||||
|
||||
/// The references followed from `/JBIG2Globals` towards a stream, so that a ring of them ends.
|
||||
const MAX_HOPS: usize = 32;
|
||||
|
||||
/// The objects a full save would keep, by number: a reference reads the object of its number,
|
||||
/// whatever generation it gives.
|
||||
pub(crate) fn kept(doc: &Document) -> HashSet<u32> {
|
||||
reachable(doc).into_iter().map(|r| r.num).collect()
|
||||
}
|
||||
|
||||
/// `o`, or what references lead to from it. A ring of them ends as a reference, which is no dictionary.
|
||||
fn followed(doc: &Document, o: &Object) -> Arc<Object> {
|
||||
let mut at = doc.resolve(o);
|
||||
for _ in 0..MAX_HOPS {
|
||||
let next = match &*at {
|
||||
Object::Ref(next) => *next,
|
||||
_ => break,
|
||||
};
|
||||
at = doc.get(next);
|
||||
}
|
||||
at
|
||||
}
|
||||
|
||||
/// Every object `s` names as `/JBIG2Globals`, and each reference on the way to one: in any entry of
|
||||
/// its parameters, whichever filter that entry belongs to and however the filters are given. A
|
||||
/// reader may use less of this, but leaving a dictionary behind is the failure that matters here.
|
||||
fn named(doc: &Document, s: &Stream) -> Vec<u32> {
|
||||
let mut found = Vec::new();
|
||||
let mut read = |entry: &Object| {
|
||||
let held = followed(doc, entry);
|
||||
let Some(mut at) = held.as_dict().and_then(|d| d.get(b"JBIG2Globals")).and_then(Object::as_ref) else { return };
|
||||
for _ in 0..=MAX_HOPS {
|
||||
found.push(at.num);
|
||||
match &*doc.get(at) {
|
||||
Object::Ref(next) => at = *next,
|
||||
_ => return,
|
||||
}
|
||||
}
|
||||
};
|
||||
for key in [b"DecodeParms".as_slice(), b"DP"] {
|
||||
let Some(parms) = s.dict.get(key).map(|p| followed(doc, p)) else { continue };
|
||||
match &*parms {
|
||||
Object::Array(entries) => entries.iter().for_each(&mut read),
|
||||
other => read(other),
|
||||
}
|
||||
}
|
||||
found
|
||||
}
|
||||
|
||||
/// `v`, or what it refers to, inside an array too.
|
||||
fn direct(doc: &Document, v: &Object) -> Object {
|
||||
match &*doc.resolve(v) {
|
||||
Object::Array(a) => Object::Array(a.iter().map(|e| (*doc.resolve(e)).clone()).collect()),
|
||||
other => other.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `s` with what `decoded_image` and `decoded_within` read as given made direct: its size and filters.
|
||||
fn readable(doc: &Document, s: &Stream) -> Stream {
|
||||
let mut readable = s.clone();
|
||||
for key in [b"Width".as_slice(), b"Height", b"Filter"] {
|
||||
if let Some(v) = s.dict.get(key) {
|
||||
readable.dict.set(key.to_vec(), direct(doc, v));
|
||||
}
|
||||
}
|
||||
readable
|
||||
}
|
||||
|
||||
/// What dictionary `num` holds, to know a copy of it by: a file put together from pages split apart
|
||||
/// has one per page, each byte for byte the others.
|
||||
fn content(doc: &Document, num: u32) -> Option<u64> {
|
||||
let object = doc.try_get(num).ok()?;
|
||||
let Object::Stream(s) = &*object else { return None };
|
||||
let readable = readable(doc, s);
|
||||
let data = doc.with_direct_parms(&readable).decoded_within(MAX_JBIG2_GLOBALS).unwrap_or_else(|_| s.raw.to_vec());
|
||||
let mut hash = DefaultHasher::new();
|
||||
data.hash(&mut hash);
|
||||
Some(hash.finish())
|
||||
}
|
||||
|
||||
/// JBIG2 stream `s` stored alone, as a cleared copy is: what it shows unchanged and no dictionary
|
||||
/// referred to. `None` for a stream that isn't JBIG2, whose decoder took no dictionary and whose
|
||||
/// samples another filter may give otherwise; for one past the size of an image redaction clears,
|
||||
/// or known to have more than one component, which JBIG2 is drawn as one of whatever the dictionary
|
||||
/// says; and for one that can't be decoded.
|
||||
fn without_dictionary(doc: &Document, s: &Stream) -> Option<Stream> {
|
||||
let readable = readable(doc, s);
|
||||
let jbig2 = match readable.dict.get(b"Filter") {
|
||||
Some(Object::Array(a)) => a.iter().any(|f| f.as_name() == Some(b"JBIG2Decode".as_slice())),
|
||||
Some(f) => f.as_name() == Some(b"JBIG2Decode".as_slice()),
|
||||
None => false,
|
||||
};
|
||||
if !jbig2 || (s.dict.contains(b"ColorSpace") && crate::image::components(doc, s).is_some_and(|n| n > 1)) {
|
||||
return None;
|
||||
}
|
||||
let side = |key: &[u8]| readable.dict.int(key).and_then(|v| usize::try_from(v).ok());
|
||||
let (w, h) = (side(b"Width")?, side(b"Height")?);
|
||||
if (w as u64).saturating_mul(h as u64) > crate::image::MAX_PIXELS {
|
||||
return None;
|
||||
}
|
||||
// As for `clear`: the filters before the JBIG2 decoder may give up to three bytes for each one
|
||||
// of the raster and a byte a row.
|
||||
let allowance = w.div_ceil(8).checked_mul(h)?.checked_add(h)?.checked_mul(3)?.checked_add(4096)?;
|
||||
let data = doc.decoded_image_within(&readable, allowance).ok()?;
|
||||
let mut dict = readable.dict.clone();
|
||||
for key in [b"Length".as_slice(), b"DecodeParms", b"DP"] {
|
||||
dict.remove(key);
|
||||
}
|
||||
dict.set(b"BitsPerComponent".to_vec(), Object::Int(1));
|
||||
Some(crate::image::smallest(dict, &data, w, h, true, true))
|
||||
}
|
||||
|
||||
/// Takes the `/JBIG2Globals` dictionaries of what left the document since `before` ([`kept`]) out of
|
||||
/// the file: everything a full save would keep that names one, or a copy of one, is stored again
|
||||
/// without it, in place, so what refers to that stream stays as it is. Gives the number stored
|
||||
/// again. A copy is found byte for byte; another dictionary that happens to hold the same shapes isn't.
|
||||
///
|
||||
/// What left, not what a run says it removed: an image goes with a stamp, a widget, a Type 3 glyph,
|
||||
/// an `/Alternates` entry or a form that can't be read, none of which it was drawn through directly.
|
||||
/// An image another page still draws hasn't left: it keeps its pixels, so its dictionary keeps nothing
|
||||
/// more.
|
||||
///
|
||||
/// Errors: [`RedactError::SharedJbig2Globals`](crate::RedactError::SharedJbig2Globals) when a dictionary would
|
||||
/// still be kept, as what names it can't be stored without it or something else refers to it. Such
|
||||
/// an image may still be drawn, on a page no mark reached, so it isn't removed, and `doc` is part
|
||||
/// changed by then: a caller works on a copy it can drop.
|
||||
pub(crate) fn unshare(doc: &mut Document, before: &HashSet<u32>) -> Result<usize, crate::RedactError> {
|
||||
let now = kept(doc);
|
||||
let mut dictionaries: HashSet<u32> = HashSet::new();
|
||||
for &num in before.difference(&now) {
|
||||
if let Ok(object) = doc.try_get(num)
|
||||
&& let Object::Stream(s) = &*object
|
||||
{
|
||||
dictionaries.extend(named(doc, s));
|
||||
}
|
||||
}
|
||||
if dictionaries.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let contents: HashSet<u64> = dictionaries.iter().filter_map(|n| content(doc, *n)).collect();
|
||||
let mut read: HashMap<u32, Option<u64>> = HashMap::new();
|
||||
let mut users = Vec::new();
|
||||
for (r, object) in doc.scan_objects() {
|
||||
let Object::Stream(s) = &*object else { continue };
|
||||
if !now.contains(&r.num) {
|
||||
continue;
|
||||
}
|
||||
let names = named(doc, s);
|
||||
let copy =
|
||||
|n: &u32, read: &mut HashMap<u32, Option<u64>>| read.entry(*n).or_insert_with(|| content(doc, *n)).is_some_and(|c| contents.contains(&c));
|
||||
if names.iter().any(|n| dictionaries.contains(n) || copy(n, &mut read)) {
|
||||
dictionaries.extend(names);
|
||||
users.push(r);
|
||||
}
|
||||
}
|
||||
let mut stuck = 0;
|
||||
for &r in &users {
|
||||
let object = doc.get(r);
|
||||
let Object::Stream(s) = &*object else { continue };
|
||||
match without_dictionary(doc, s) {
|
||||
Some(stored) => doc.set(r, Object::Stream(stored)),
|
||||
None => stuck += 1,
|
||||
}
|
||||
}
|
||||
let after = kept(doc);
|
||||
if dictionaries.iter().any(|n| after.contains(n)) {
|
||||
return Err(crate::RedactError::SharedJbig2Globals(stuck.max(1)));
|
||||
}
|
||||
Ok(users.len())
|
||||
}
|
||||
@@ -11,9 +11,9 @@ use pdfcraft_content::{Matrix, overlaps};
|
||||
use pdfcraft_cos::{Dict, Document, Object, Stream};
|
||||
|
||||
/// Most pixels an image may have to be cleared in place (larger ones are removed).
|
||||
const MAX_PIXELS: u64 = 64 << 20;
|
||||
pub(crate) const MAX_PIXELS: u64 = 64 << 20;
|
||||
|
||||
fn components(doc: &Document, s: &Stream) -> Option<usize> {
|
||||
pub(crate) fn components(doc: &Document, s: &Stream) -> Option<usize> {
|
||||
let cs = doc.resolve(s.dict.get(b"ColorSpace")?);
|
||||
let cs = &*cs;
|
||||
let name = match cs {
|
||||
@@ -175,7 +175,7 @@ fn decode_and_clear(doc: &Document, s: &Stream, ctm: &Matrix, rects: &[[f64; 4]]
|
||||
/// is), whichever is smallest as stored, dictionary included (a tiny image can lose by Group 4's
|
||||
/// parameters). Both decode to `data`, cleared before it is coded, so nothing of the original's
|
||||
/// code is carried over; Group 4 or JBIG2 that can't code the rows (too wide) is left out.
|
||||
fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream {
|
||||
pub(crate) fn smallest(dict: Dict, data: &[u8], w: usize, h: usize, bilevel: bool, jbig2: bool) -> Stream {
|
||||
let flate = Stream::flate(dict.clone(), data);
|
||||
let Some((columns, rows)) = u32::try_from(w).ok().zip(u32::try_from(h).ok()).filter(|_| bilevel) else { return flate };
|
||||
let g4 = pdfcraft_filters::encode_g4(data, columns, rows).ok().map(|coded| {
|
||||
@@ -202,38 +202,3 @@ fn stored_len(s: &Stream) -> usize {
|
||||
pdfcraft_cos::serialize(&Object::Dict(s.dict.clone()), &mut dict);
|
||||
dict.len().saturating_add(s.raw.len())
|
||||
}
|
||||
|
||||
/// JBIG2 image `s` stored again without its `/JBIG2Globals`: decoded through them, as it is drawn,
|
||||
/// and written as the same samples, Flate, Group 4 or JBIG2 coded alone, whichever is smallest
|
||||
/// (`smallest`), its other entries kept. `None` when it can't
|
||||
/// be decoded for certain (too large, data that doesn't decode, a JBIG2 page of another size than
|
||||
/// the image, or not a one-bit image of one component). Redaction does this to every image that
|
||||
/// shares a redacted image's globals, since their symbol dictionary holds the shapes of marks drawn
|
||||
/// under the area too.
|
||||
pub(crate) fn without_globals(doc: &Document, s: &Stream) -> Option<Stream> {
|
||||
let (w, h) = (s.dict.int(b"Width")?, s.dict.int(b"Height")?);
|
||||
if w <= 0 || h <= 0 || (w as u64) * (h as u64) > MAX_PIXELS {
|
||||
return None;
|
||||
}
|
||||
let (w, h) = (w as usize, h as usize);
|
||||
let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true)));
|
||||
// JBIG2 codes one-bit images of one component (ISO 32000-2 §7.4.7).
|
||||
if !mask && (components(doc, s)?, s.dict.int(b"BitsPerComponent")?) != (1, 1) {
|
||||
return None;
|
||||
}
|
||||
let bytes = w.div_ceil(8).checked_mul(h)?;
|
||||
// As for `clear`: the filters before the JBIG2 decoder may give up to three bytes for each one
|
||||
// of the raster and a byte a row.
|
||||
let allowance = bytes.checked_add(h)?.checked_mul(3)?.checked_add(4096)?;
|
||||
let mut data = doc.decoded_image_within(s, allowance).ok()?;
|
||||
if data.len() < bytes {
|
||||
return None;
|
||||
}
|
||||
data.truncate(bytes);
|
||||
let mut dict = s.dict.clone();
|
||||
dict.remove(b"Length");
|
||||
// The globals are named in the parameters; the copy is coded alone, as `clear`'s are.
|
||||
dict.remove(b"DecodeParms");
|
||||
dict.remove(b"Alternates");
|
||||
Some(smallest(dict, &data, w, h, true, true))
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ use std::collections::{HashMap, HashSet};
|
||||
use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream};
|
||||
|
||||
pub mod codes;
|
||||
mod globals;
|
||||
mod image;
|
||||
mod interp;
|
||||
mod overlay;
|
||||
@@ -70,10 +71,12 @@ pub enum RedactError {
|
||||
/// could not be shown complete.
|
||||
#[error("the document's objects nest too deeply to remove hidden information completely; the operation was refused")]
|
||||
TooDeep,
|
||||
/// A JBIG2 image shares its `/JBIG2Globals` with a redacted one and can't be stored again
|
||||
/// without them, so the shapes they hold, those of marks under the area included, would stay.
|
||||
/// `/JBIG2Globals` used by something removed would stay: a JBIG2 image shares them and can't be
|
||||
/// stored again without them, or something else refers to them. Either way the shapes they
|
||||
/// hold, those of marks under the area included, would stay. Counts the images, or 1 for another
|
||||
/// reference.
|
||||
#[error(
|
||||
"{0} JBIG2 image(s) share their symbol data with a redacted image and can't be decoded to be stored without it, so that data, which can show what is under the marks, would stay in the file; nothing was redacted"
|
||||
"{0} item(s) share JBIG2 symbol data with something being removed and can't be stored without it, so that data, which can show what was removed, would stay in the file; nothing was changed"
|
||||
)]
|
||||
SharedJbig2Globals(usize),
|
||||
#[error(transparent)]
|
||||
@@ -139,7 +142,7 @@ pub struct Report {
|
||||
pub images_removed: usize,
|
||||
pub images_cleared: usize,
|
||||
/// JBIG2 images, anywhere in the document, stored again (as a cleared copy is) without the
|
||||
/// `/JBIG2Globals` they shared with a redacted image, so those globals leave the file.
|
||||
/// `/JBIG2Globals` they shared with something this removed, so those globals leave the file.
|
||||
pub images_restored: usize,
|
||||
pub paths_removed: usize,
|
||||
pub paths_clipped: usize,
|
||||
@@ -405,6 +408,7 @@ fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Redac
|
||||
pdfcraft_fonts::undrawable_message(&format!("the overlay text of a redaction mark on page {}", m.page + 1), c)
|
||||
)));
|
||||
}
|
||||
let kept_before = globals::kept(doc);
|
||||
let mut report = Report { marks: chosen.len(), ..Report::default() };
|
||||
// `marks` visits pages in order. Keep slices of that order for appearance generation,
|
||||
// content rewriting and verification instead of scanning all marks for every page.
|
||||
@@ -518,7 +522,7 @@ fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Redac
|
||||
// A removed image or form must not stay reachable through another resource dictionary (an
|
||||
// inherited or shared /Resources): a full save keeps whatever is still referenced.
|
||||
interp::unlink_doomed(doc, &doomed_xobjects, &retired_properties)?;
|
||||
report.images_restored = restore_shared_jbig2(doc, &doomed_xobjects)?;
|
||||
report.images_restored = globals::unshare(doc, &kept_before)?;
|
||||
|
||||
// 5. Verify: nothing readable may remain under a region.
|
||||
let mut residue = 0;
|
||||
@@ -543,69 +547,6 @@ fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result<Report, Redac
|
||||
Ok(report)
|
||||
}
|
||||
|
||||
/// Most XObjects followed down from the removed and replaced ones to find their JBIG2 images.
|
||||
const MAX_JBIG2_WALK: usize = 100_000;
|
||||
|
||||
/// The `/JBIG2Globals` of a redacted JBIG2 image hold the symbol dictionary of every image that
|
||||
/// shares them (`jbig2enc -s` and most scanners give one to all the pages of a scan): the bitmaps
|
||||
/// of marks drawn only under the area too. The image itself is replaced or removed, but the globals
|
||||
/// would stay for as long as another image uses them. So every image that uses the globals of a
|
||||
/// removed or replaced image (`doomed`), or of one drawn in a removed or replaced form, is decoded
|
||||
/// and stored again in place without them, as a cleared copy is, and a full save then leaves them
|
||||
/// out. An image that can't be stored again refuses the redaction (nothing changes, as `apply`
|
||||
/// works on a copy), but for a removed or replaced one, which is either unlinked or still drawn
|
||||
/// whole on another page. Returns how many images that weren't removed or replaced were stored again.
|
||||
fn restore_shared_jbig2(doc: &mut Document, doomed: &[ObjRef]) -> Result<usize, RedactError> {
|
||||
let mut globals = HashSet::new();
|
||||
let mut seen = HashSet::new();
|
||||
let mut stack: Vec<ObjRef> = doomed.to_vec();
|
||||
while let Some(r) = stack.pop() {
|
||||
if !seen.insert(r) || seen.len() > MAX_JBIG2_WALK {
|
||||
continue;
|
||||
}
|
||||
let object = doc.get(r);
|
||||
let Object::Stream(s) = &*object else { continue };
|
||||
if let Some(g) = doc.jbig2_globals_ref(s) {
|
||||
globals.insert(g);
|
||||
}
|
||||
// A form's images, and its forms' (an image's /SMask is a stream of its own, found below).
|
||||
let resources = s.dict.get(b"Resources").map(|r| doc.resolve(r));
|
||||
let xobjects = resources.as_deref().and_then(Object::as_dict).and_then(|d| d.get(b"XObject")).map(|x| doc.resolve(x));
|
||||
if let Some(x) = xobjects.as_deref().and_then(Object::as_dict) {
|
||||
stack.extend(x.iter().filter_map(|(_, v)| v.as_ref()));
|
||||
}
|
||||
if let Some(m) = s.dict.get(b"SMask").and_then(Object::as_ref) {
|
||||
stack.push(m);
|
||||
}
|
||||
}
|
||||
if globals.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let sharing: Vec<ObjRef> = doc
|
||||
.scan_objects()
|
||||
.filter(|(_, o)| matches!(&**o, Object::Stream(s) if doc.jbig2_globals_ref(s).is_some_and(|g| globals.contains(&g))))
|
||||
.map(|(r, _)| r)
|
||||
.collect();
|
||||
let doomed: HashSet<ObjRef> = doomed.iter().copied().collect();
|
||||
let (mut restored, mut stuck) = (0, 0);
|
||||
for r in sharing {
|
||||
let object = doc.get(r);
|
||||
let Object::Stream(s) = &*object else { continue };
|
||||
match image::without_globals(doc, s) {
|
||||
Some(new) => {
|
||||
doc.set(r, Object::Stream(new));
|
||||
restored += usize::from(!doomed.contains(&r));
|
||||
}
|
||||
None if doomed.contains(&r) => {}
|
||||
None => stuck += 1,
|
||||
}
|
||||
}
|
||||
if stuck > 0 {
|
||||
return Err(RedactError::SharedJbig2Globals(stuck));
|
||||
}
|
||||
Ok(restored)
|
||||
}
|
||||
|
||||
/// What applying the marks collects across pages.
|
||||
struct Applying {
|
||||
report: Report,
|
||||
|
||||
@@ -521,6 +521,7 @@ pub fn remove_hidden(doc: &mut Document, which: &[Hidden]) -> Result<Vec<(Hidden
|
||||
let count = |h: Hidden| counts.iter().find(|c| c.0 == h).map_or(0, |c| c.1);
|
||||
let (root, _) = catalog(doc).ok_or(RedactError::NothingToApply)?;
|
||||
let mut done = Vec::new();
|
||||
let kept_before = crate::globals::kept(doc);
|
||||
// Every annotation the pages list now: whatever is no longer listed at the end (flattened
|
||||
// widgets included) leaves the structure tree too.
|
||||
let listed_before = listed_annotations(doc);
|
||||
@@ -673,6 +674,7 @@ pub fn remove_hidden(doc: &mut Document, which: &[Hidden]) -> Result<Vec<(Hidden
|
||||
if done.is_empty() {
|
||||
return Err(RedactError::NothingToApply);
|
||||
}
|
||||
crate::globals::unshare(doc, &kept_before)?;
|
||||
doc.require_full_save();
|
||||
Ok(done)
|
||||
}
|
||||
|
||||
@@ -1242,6 +1242,470 @@ fn globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file() {
|
||||
assert!(matches!(apply(&mut doc, None), Err(RedactError::SharedJbig2Globals(1))));
|
||||
}
|
||||
|
||||
/// A scan's parts when it keeps its symbols in `/JBIG2Globals`: the dictionary, moved out as segment
|
||||
/// data belonging to no page, and the page information and regions that are left.
|
||||
struct SharedScan {
|
||||
bits: Vec<u8>,
|
||||
symbols: Vec<u8>,
|
||||
page: Vec<u8>,
|
||||
}
|
||||
|
||||
fn shared_scan() -> SharedScan {
|
||||
const LETTERS: [[&str; 7]; 3] = [
|
||||
["#####", "#....", "####.", "#....", "#....", "#....", "#####"],
|
||||
[".###.", "#...#", "#....", "#....", "#....", "#...#", ".###."],
|
||||
["#...#", "#...#", "#####", "#...#", "#...#", "#...#", "#...#"],
|
||||
];
|
||||
let (w, h) = (400usize, 40usize);
|
||||
let stride = w / 8;
|
||||
let mut bits = vec![0xFFu8; stride * h];
|
||||
for (line, top) in [2usize, 12, 22, 32].into_iter().enumerate() {
|
||||
for (k, x) in (3..w - 6).step_by(7).enumerate() {
|
||||
for (dy, row) in LETTERS[(k + line) % 3].iter().enumerate() {
|
||||
for (dx, _) in row.bytes().enumerate().filter(|(_, c)| *c == b'#') {
|
||||
bits[(top + dy) * stride + (x + dx) / 8] &= !(0x80 >> ((x + dx) % 8));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let coded = pdfcraft_filters::encode_jbig2(&bits, w as u32, h as u32).unwrap();
|
||||
// The end of the segment at `at`, and where its page association is (7.2.2 to 7.2.7).
|
||||
let end = |at: usize| {
|
||||
let page = at + 6 + usize::from(coded[at + 5] >> 5);
|
||||
let length = u32::from_be_bytes(coded[page + 1..page + 5].try_into().unwrap());
|
||||
(page + 5 + length as usize, page)
|
||||
};
|
||||
let (dictionary, _) = end(0);
|
||||
assert_eq!(coded[dictionary + 4], 0, "the second segment is the symbol dictionary");
|
||||
let (rest, page) = end(dictionary);
|
||||
let mut symbols = coded[dictionary..rest].to_vec();
|
||||
symbols[page - dictionary] = 0;
|
||||
SharedScan { page: [&coded[..dictionary], &coded[rest..]].concat(), symbols, bits }
|
||||
}
|
||||
|
||||
/// An image of [`SharedScan`]'s size that uses the dictionary, object 9, by `parms`.
|
||||
fn shared_image(parms: &str) -> String {
|
||||
format!("/Type /XObject /Subtype /Image /Width 400 /Height 40 /BitsPerComponent 1 /ColorSpace /DeviceGray /Filter /JBIG2Decode {parms}")
|
||||
}
|
||||
|
||||
/// Two pages of a scan sharing object 9: its symbols, and `LEAK_GLOBALS`, for the shape of a mark found
|
||||
/// only under the area. The first page draws image 7 unless told otherwise; the second draws image 8.
|
||||
/// `extra` are objects 10 on.
|
||||
struct SharedPages {
|
||||
/// What the first page draws, its `/Resources` entries and anything else in its dictionary.
|
||||
content: Vec<u8>,
|
||||
resources: String,
|
||||
page: String,
|
||||
/// Image 7's dictionary and data; image 8's.
|
||||
first: String,
|
||||
first_data: Vec<u8>,
|
||||
other: String,
|
||||
other_data: Vec<u8>,
|
||||
extra: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
const DRAWS: &[u8] = b"q 200 0 0 20 0 100 cm /Im Do Q";
|
||||
|
||||
impl SharedPages {
|
||||
fn new(scan: &SharedScan) -> Self {
|
||||
let image = shared_image(SHARED);
|
||||
Self {
|
||||
content: DRAWS.to_vec(),
|
||||
resources: "/XObject << /Im 7 0 R >>".into(),
|
||||
page: String::new(),
|
||||
first: image.clone(),
|
||||
first_data: scan.page.clone(),
|
||||
other: image,
|
||||
other_data: scan.page.clone(),
|
||||
extra: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn build(self, scan: &SharedScan) -> Document {
|
||||
let Self { content, resources, page, first, first_data, other, other_data, extra } = self;
|
||||
let mut objs: Vec<Vec<u8>> = vec![
|
||||
b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(),
|
||||
b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(),
|
||||
format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << {resources} >> {page} >>").into_bytes(),
|
||||
b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 6 0 R /Resources << /XObject << /Im 8 0 R >> >> >>".to_vec(),
|
||||
stream("", content),
|
||||
stream("", DRAWS),
|
||||
stream(&first, first_data),
|
||||
stream(&other, other_data),
|
||||
stream("", [jbig2_globals(b"LEAK_GLOBALS"), scan.symbols.clone()].concat()),
|
||||
];
|
||||
objs.extend(extra);
|
||||
pdf(objs)
|
||||
}
|
||||
}
|
||||
|
||||
/// The samples of the image the page draws as `/Im`.
|
||||
fn drawn_samples(doc: &Document, page: usize) -> Vec<u8> {
|
||||
let p = pdfcraft_model::pages(doc).swap_remove(page);
|
||||
let res = doc.resolve(p.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap();
|
||||
let xo = doc.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap();
|
||||
match &*doc.get(xo.get(b"Im").unwrap().as_ref().unwrap()) {
|
||||
Object::Stream(s) => doc.decoded_image(s).unwrap(),
|
||||
other => panic!("not an image: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
const SHARED: &str = "/DecodeParms << /JBIG2Globals 9 0 R >>";
|
||||
const CLEARS: [f64; 4] = [0.25, 113.75, 49.75, 120.0];
|
||||
const COVERS: [f64; 4] = [0.0, 100.0, 200.0, 120.0];
|
||||
|
||||
/// The ways image 7 can leave the document while taking the dictionary's only reason to stay with it.
|
||||
fn ways_out(scan: &SharedScan) -> Vec<(&'static str, [f64; 4], SharedPages)> {
|
||||
let base = || SharedPages::new(scan);
|
||||
let in_form = |filter: &str, data: &[u8]| {
|
||||
stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im 7 0 R >> >> {filter}"), data)
|
||||
};
|
||||
let plain = "/Type /XObject /Subtype /Image /Width 1 /Height 1 /BitsPerComponent 8 /ColorSpace /DeviceGray";
|
||||
let through_form = |filter: &str, data: &[u8]| SharedPages {
|
||||
content: b"/Fm Do".to_vec(),
|
||||
resources: "/XObject << /Fm 10 0 R >>".into(),
|
||||
extra: vec![in_form(filter, data)],
|
||||
..base()
|
||||
};
|
||||
let annotated = |subtype: &str| SharedPages {
|
||||
content: Vec::new(),
|
||||
resources: String::new(),
|
||||
page: "/Annots [10 0 R]".into(),
|
||||
extra: vec![
|
||||
format!("<< /Type /Annot /Subtype /{subtype} /T (Field) /FT /Tx /Rect [0 100 200 120] /AP << /N 11 0 R >> >>").into_bytes(),
|
||||
in_form("", DRAWS),
|
||||
],
|
||||
..base()
|
||||
};
|
||||
let glyph = SharedPages {
|
||||
content: b"BT /T3 200 Tf 0 100 Td (a) Tj ET".to_vec(),
|
||||
resources: "/Font << /T3 10 0 R >>".into(),
|
||||
extra: vec![
|
||||
b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 11 0 R >> \
|
||||
/Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] /Resources << /XObject << /Im 7 0 R >> >> >>"
|
||||
.to_vec(),
|
||||
stream("", b"1000 0 d0 q 1000 0 0 100 0 0 cm /Im Do Q"),
|
||||
],
|
||||
..base()
|
||||
};
|
||||
vec![
|
||||
("cleared", CLEARS, base()),
|
||||
("removed", COVERS, base()),
|
||||
("in a form", COVERS, through_form("", DRAWS)),
|
||||
("in a form that can't be read", COVERS, through_form("/Filter /FlateDecode", b"not Flate")),
|
||||
("a stamp's appearance", COVERS, annotated("Stamp")),
|
||||
("a widget's appearance", COVERS, annotated("Widget")),
|
||||
("a Type 3 glyph", COVERS, glyph),
|
||||
(
|
||||
"a mask",
|
||||
COVERS,
|
||||
SharedPages { first: format!("{plain} /SMask 10 0 R"), extra: vec![stream(&shared_image(SHARED), scan.page.clone())], ..base() },
|
||||
),
|
||||
(
|
||||
"an alternate",
|
||||
COVERS,
|
||||
SharedPages {
|
||||
first: format!("{plain} /Alternates [<< /Image 10 0 R >>]"),
|
||||
extra: vec![stream(&shared_image(SHARED), scan.page.clone())],
|
||||
..base()
|
||||
},
|
||||
),
|
||||
(
|
||||
"/Filter by reference",
|
||||
COVERS,
|
||||
SharedPages {
|
||||
first: shared_image("/DecodeParms [null << /JBIG2Globals 9 0 R >>]").replace("/Filter /JBIG2Decode", "/Filter 10 0 R"),
|
||||
extra: vec![b"[/ASCIIHexDecode /JBIG2Decode]".to_vec()],
|
||||
..base()
|
||||
},
|
||||
),
|
||||
(
|
||||
"a reference to it",
|
||||
COVERS,
|
||||
SharedPages { first: shared_image("/DecodeParms << /JBIG2Globals 10 0 R >>"), extra: vec![b"9 0 R".to_vec()], ..base() },
|
||||
),
|
||||
(
|
||||
"parameters through a reference to them",
|
||||
COVERS,
|
||||
SharedPages {
|
||||
first: shared_image("/DecodeParms 10 0 R"),
|
||||
extra: vec![b"11 0 R".to_vec(), b"<< /JBIG2Globals 9 0 R >>".to_vec()],
|
||||
..base()
|
||||
},
|
||||
),
|
||||
(
|
||||
"an entry through a reference to it",
|
||||
COVERS,
|
||||
SharedPages {
|
||||
first: shared_image("/DecodeParms [10 0 R]"),
|
||||
extra: vec![b"11 0 R".to_vec(), b"<< /JBIG2Globals 9 0 R >>".to_vec()],
|
||||
..base()
|
||||
},
|
||||
),
|
||||
(
|
||||
"an array of them through a reference to it",
|
||||
COVERS,
|
||||
SharedPages {
|
||||
first: shared_image("/DecodeParms 10 0 R"),
|
||||
extra: vec![b"11 0 R".to_vec(), b"[<< /JBIG2Globals 9 0 R >>]".to_vec()],
|
||||
..base()
|
||||
},
|
||||
),
|
||||
("/DP", COVERS, SharedPages { first: shared_image("/DP << /JBIG2Globals 9 0 R >>"), ..base() }),
|
||||
("another generation", COVERS, SharedPages { first: shared_image("/DecodeParms << /JBIG2Globals 9 1 R >>"), ..base() }),
|
||||
]
|
||||
}
|
||||
|
||||
/// Scanners and `jbig2enc -s` share one symbol dictionary across a scan's pages, and it holds the
|
||||
/// shape of every distinct mark, one found only under a redaction included: a signature, a stamp.
|
||||
/// The dictionary stayed in the file while any other image used it. Now the image left on the second
|
||||
/// page is stored again without it, whichever way the first one left the document.
|
||||
#[test]
|
||||
fn a_redacted_scans_shared_jbig2_dictionary_leaves_the_file() {
|
||||
let scan = shared_scan();
|
||||
for (way, rect, pages) in ways_out(&scan) {
|
||||
let mut doc = pages.build(&scan);
|
||||
assert_eq!(drawn_samples(&doc, 1), scan.bits, "{way}: the second page reads through the dictionary");
|
||||
mark(&mut doc, 0, &[rect], "");
|
||||
let report = apply(&mut doc, None).unwrap();
|
||||
assert_eq!(report.images_restored, 1, "{way}");
|
||||
assert_gone(&doc, b"LEAK_GLOBALS");
|
||||
assert_gone(&doc, b"JBIG2Globals");
|
||||
assert_eq!(drawn_samples(&reopen(&doc), 1), scan.bits, "{way}: its pixels are unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
/// However the image that stays names the dictionary, it comes out without it: its parameters by
|
||||
/// reference, behind another filter, or a copy of the dictionary, as a file put back together from
|
||||
/// pages split apart has, each page with its own. It is also read when its size or filter is given
|
||||
/// by reference, which `decoded_image` alone refuses.
|
||||
#[test]
|
||||
fn what_stays_is_stored_without_the_dictionary_however_it_names_it() {
|
||||
let scan = shared_scan();
|
||||
let hex: Vec<u8> = scan.page.iter().map(|b| format!("{b:02x}")).collect::<String>().bytes().chain(*b">").collect();
|
||||
let copy = stream("", [jbig2_globals(b"LEAK_GLOBALS"), scan.symbols.clone()].concat());
|
||||
let image = shared_image(SHARED);
|
||||
for (other, data, extra) in [
|
||||
(shared_image("/DecodeParms 10 0 R"), scan.page.clone(), vec![b"<< /JBIG2Globals 9 0 R >>".to_vec()]),
|
||||
(
|
||||
shared_image("/DecodeParms [null << /JBIG2Globals 9 0 R >>]").replace("/Filter /JBIG2Decode", "/Filter [/ASCIIHexDecode /JBIG2Decode]"),
|
||||
hex.clone(),
|
||||
vec![],
|
||||
),
|
||||
(shared_image("/DecodeParms << /JBIG2Globals 10 0 R >>"), scan.page.clone(), vec![copy]),
|
||||
(image.replace("/Width 400", "/Width 10 0 R"), scan.page.clone(), vec![b"400".to_vec()]),
|
||||
(image.replace("/Height 40", "/Height 10 0 R"), scan.page.clone(), vec![b"40".to_vec()]),
|
||||
(image.replace("/Filter /JBIG2Decode", "/Filter 10 0 R"), scan.page.clone(), vec![b"/JBIG2Decode".to_vec()]),
|
||||
(
|
||||
shared_image("/DecodeParms [null << /JBIG2Globals 9 0 R >>]").replace("/Filter /JBIG2Decode", "/Filter [/ASCIIHexDecode 10 0 R]"),
|
||||
hex.clone(),
|
||||
vec![b"/JBIG2Decode".to_vec()],
|
||||
),
|
||||
(image.replace("/ColorSpace /DeviceGray ", ""), scan.page.clone(), vec![]),
|
||||
(
|
||||
image.replace("/ColorSpace /DeviceGray", "/ColorSpace [/DeviceN [/Black] /DeviceGray 10 0 R]"),
|
||||
scan.page.clone(),
|
||||
vec![b"<< /FunctionType 2 /Domain [0 1] /N 1 >>".to_vec()],
|
||||
),
|
||||
// Parameters no filter reads, on a picture that needs no dictionary.
|
||||
(shared_image("/DP << /JBIG2Globals 9 0 R >>"), pdfcraft_filters::encode_jbig2_generic(&scan.bits, 400, 40).unwrap(), vec![]),
|
||||
] {
|
||||
let mut doc = SharedPages { other: other.clone(), other_data: data, extra, ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
let report = apply(&mut doc, None).unwrap();
|
||||
assert_eq!(report.images_restored, 1, "{other}");
|
||||
assert_gone(&doc, b"LEAK_GLOBALS");
|
||||
assert_eq!(drawn_samples(&reopen(&doc), 1), scan.bits, "{other}: its pixels are unchanged");
|
||||
}
|
||||
}
|
||||
|
||||
/// JBIG2 is drawn as one bit of one component whatever the dictionary says, so the Flate copy has to
|
||||
/// say so too: a stream with no `/BitsPerComponent`, or 8, shows the same picture stored again.
|
||||
#[test]
|
||||
fn a_stored_jbig2_image_is_one_bit_whatever_its_dictionary_said() {
|
||||
let scan = shared_scan();
|
||||
let hex: Vec<u8> = scan.page.iter().map(|b| format!("{b:02x}")).collect::<String>().bytes().chain(*b">").collect();
|
||||
let chained =
|
||||
shared_image("/DecodeParms [null << /JBIG2Globals 9 0 R >>]").replace("/Filter /JBIG2Decode", "/Filter [/ASCIIHexDecode /JBIG2Decode]");
|
||||
let lone = shared_image(SHARED).replace("/Filter /JBIG2Decode", "/Filter [/JBIG2Decode]");
|
||||
for (other, other_data, bits) in [
|
||||
(shared_image(SHARED), scan.page.clone(), ""),
|
||||
(shared_image(SHARED), scan.page.clone(), "/BitsPerComponent 8"),
|
||||
(chained, hex, ""),
|
||||
(lone, scan.page.clone(), ""),
|
||||
] {
|
||||
let other = other.replace("/BitsPerComponent 1", bits);
|
||||
let mut doc = SharedPages { other, other_data, ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
apply(&mut doc, None).unwrap();
|
||||
let saved = reopen(&doc);
|
||||
assert_eq!(drawn_samples(&saved, 1), scan.bits, "{bits}");
|
||||
let p = pdfcraft_model::pages(&saved).swap_remove(1);
|
||||
let res = saved.resolve(p.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap();
|
||||
let xo = saved.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap();
|
||||
let image = saved.get(xo.get(b"Im").unwrap().as_ref().unwrap());
|
||||
let Object::Stream(s) = &*image else { panic!("not an image") };
|
||||
assert_eq!(s.dict.int(b"BitsPerComponent"), Some(1), "{bits}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A mark that takes nothing using the dictionary leaves it, and what uses it, alone: here it covers
|
||||
/// text in a form that also draws the scan, away from the scan. A stream nothing reaches isn't decoded
|
||||
/// for nothing either: this one can't be, and it uses the dictionary.
|
||||
#[test]
|
||||
fn a_dictionary_stays_when_nothing_using_it_left() {
|
||||
let scan = shared_scan();
|
||||
let text = b"BT /F1 10 Tf 10 250 Td (SECRET) Tj ET q 200 0 0 20 0 100 cm /Im Do Q";
|
||||
let pages = SharedPages {
|
||||
content: b"/Fm Do".to_vec(),
|
||||
resources: "/XObject << /Fm 10 0 R >>".into(),
|
||||
extra: vec![
|
||||
stream(
|
||||
"/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> /XObject << /Im 7 0 R >> >>",
|
||||
text,
|
||||
),
|
||||
stream(&shared_image(SHARED), b"not JBIG2"),
|
||||
],
|
||||
..SharedPages::new(&scan)
|
||||
};
|
||||
let mut doc = pages.build(&scan);
|
||||
mark(&mut doc, 0, &[[0.0, 240.0, 200.0, 270.0]], "");
|
||||
let report = apply(&mut doc, None).unwrap();
|
||||
assert_eq!((report.images_removed, report.images_cleared, report.images_restored), (0, 0, 0));
|
||||
assert_eq!(drawn_samples(&reopen(&doc), 1), scan.bits);
|
||||
}
|
||||
|
||||
/// A dictionary of other bytes is another dictionary: the scan using it keeps it and isn't stored again.
|
||||
#[test]
|
||||
fn another_jbig2_dictionary_is_left_alone() {
|
||||
let scan = shared_scan();
|
||||
let own = stream("", [jbig2_globals(b"OWN_GLOBALS"), scan.symbols.clone()].concat());
|
||||
let other = shared_image("/DecodeParms << /JBIG2Globals 10 0 R >>");
|
||||
let mut doc = SharedPages { other, extra: vec![own], ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
assert_eq!(apply(&mut doc, None).unwrap().images_restored, 0);
|
||||
let saved = reopen(&doc);
|
||||
let holds = |secret: &[u8]| {
|
||||
saved.scan_objects().any(|(_, o)| matches!(&*o, Object::Stream(s) if s.decoded().is_ok_and(|d| d.windows(secret.len()).any(|w| w == secret))))
|
||||
};
|
||||
assert!(!holds(b"LEAK_GLOBALS") && holds(b"OWN_GLOBALS"));
|
||||
assert_eq!(drawn_samples(&saved, 1), scan.bits);
|
||||
}
|
||||
|
||||
/// An image that can't be stored without the dictionary may still be drawn: the renderer draws a
|
||||
/// JBIG2 page of another size than its image, which `decoded_image` refuses, and draws three components
|
||||
/// of one bit as grey. Removing it would lose a page no mark reached, and keeping the dictionary would
|
||||
/// keep what the mark covered, so the redaction refuses and the document stays as it was. So it does
|
||||
/// for anything else that refers to the dictionary, a reference that holds a reference to it among them.
|
||||
#[test]
|
||||
fn a_shared_jbig2_dictionary_that_cant_leave_refuses_the_redaction() {
|
||||
let scan = shared_scan();
|
||||
let image = shared_image(SHARED);
|
||||
let elsewhere = "/Type /XObject /Subtype /Image /Width 1 /Height 1 /BitsPerComponent 1 /ColorSpace /DeviceGray /Note 9 0 R";
|
||||
let not_jbig2 = image.replace("/Filter /JBIG2Decode", "/Filter /FlateDecode");
|
||||
for (other, data, extra) in [
|
||||
(image.replace("/Height 40", "/Height 41"), scan.page.clone(), vec![]),
|
||||
(image.replace("/DeviceGray", "/DeviceRGB"), scan.page.clone(), vec![]),
|
||||
(shared_image("/DecodeParms << /JBIG2Globals 10 0 R >>"), scan.page.clone(), vec![b"9 0 R".to_vec()]),
|
||||
(elsewhere.to_string(), vec![0], vec![]),
|
||||
(not_jbig2, pdfcraft_filters::encode_flate(&scan.bits), vec![]),
|
||||
// A copy with its filter by reference is known for a copy, and the decoder doesn't read it.
|
||||
(
|
||||
shared_image("/DecodeParms << /JBIG2Globals 10 0 R >>"),
|
||||
scan.page.clone(),
|
||||
vec![
|
||||
stream("/Filter 11 0 R", pdfcraft_filters::encode_flate(&[jbig2_globals(b"LEAK_GLOBALS"), scan.symbols.clone()].concat())),
|
||||
b"/FlateDecode".to_vec(),
|
||||
],
|
||||
),
|
||||
] {
|
||||
let mut doc = SharedPages { other: other.clone(), other_data: data, extra, ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
let before = doc.clone();
|
||||
assert!(matches!(apply(&mut doc, None), Err(RedactError::SharedJbig2Globals(_))), "{other}");
|
||||
assert_eq!(drawn_samples(&doc, 0), drawn_samples(&before, 0), "{other}: nothing was redacted");
|
||||
}
|
||||
}
|
||||
|
||||
/// What is stored again is bounded as what is cleared is: an image of [`image::MAX_PIXELS`] is, and one
|
||||
/// past it, which decodes, refuses the redaction.
|
||||
#[test]
|
||||
fn an_image_past_the_size_redaction_clears_refuses_it() {
|
||||
let scan = shared_scan();
|
||||
let h = 8192u32;
|
||||
for (w, stored) in [(8192u32, true), (8200, false)] {
|
||||
let blank = vec![0xFFu8; (w as usize).div_ceil(8) * h as usize];
|
||||
let other = shared_image(SHARED).replace("/Width 400", &format!("/Width {w}")).replace("/Height 40", &format!("/Height {h}"));
|
||||
let data = pdfcraft_filters::encode_jbig2_generic(&blank, w, h).unwrap();
|
||||
let mut doc = SharedPages { other, other_data: data, ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
let run = apply(&mut doc, None);
|
||||
assert_eq!(run.as_ref().is_ok_and(|r| r.images_restored == 1), stored, "{w}");
|
||||
assert_eq!(matches!(run, Err(RedactError::SharedJbig2Globals(1))), !stored, "{w}");
|
||||
}
|
||||
}
|
||||
|
||||
/// An alternate of an image stored again stays with it, stored without the dictionary in turn: a page
|
||||
/// no mark reached keeps what it had.
|
||||
#[test]
|
||||
fn an_alternate_of_what_stays_stays() {
|
||||
let scan = shared_scan();
|
||||
let other = format!("{} /Alternates [<< /Image 10 0 R >>]", shared_image(SHARED));
|
||||
let mut doc = SharedPages { other, extra: vec![stream(&shared_image(SHARED), scan.page.clone())], ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
assert_eq!(apply(&mut doc, None).unwrap().images_restored, 2);
|
||||
assert_gone(&doc, b"LEAK_GLOBALS");
|
||||
let saved = reopen(&doc);
|
||||
let kept = saved.scan_objects().filter(|(_, o)| matches!(&**o, Object::Stream(s) if s.dict.contains(b"Alternates"))).count();
|
||||
assert_eq!(kept, 1);
|
||||
}
|
||||
|
||||
/// The refusal counts the images that can't be stored, or says 1 for something else that refers to
|
||||
/// the dictionary.
|
||||
#[test]
|
||||
fn a_refused_redaction_counts_what_cant_be_stored() {
|
||||
let scan = shared_scan();
|
||||
let wrong = shared_image(SHARED).replace("/Height 40", "/Height 41");
|
||||
let other = format!("{wrong} /SMask 10 0 R");
|
||||
let mut doc = SharedPages { other, extra: vec![stream(&wrong, scan.page.clone())], ..SharedPages::new(&scan) }.build(&scan);
|
||||
mark(&mut doc, 0, &[CLEARS], "");
|
||||
assert!(matches!(apply(&mut doc, None), Err(RedactError::SharedJbig2Globals(2))));
|
||||
}
|
||||
|
||||
/// Remove Hidden Information leaves the same way: comments go, with what their appearances drew.
|
||||
#[test]
|
||||
fn removing_hidden_information_takes_a_shared_jbig2_dictionary_too() {
|
||||
let scan = shared_scan();
|
||||
let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im 7 0 R >> >>", DRAWS);
|
||||
let annotation = b"<< /Type /Annot /Subtype /Stamp /Rect [0 100 200 120] /AP << /N 11 0 R >> >>".to_vec();
|
||||
let mut doc = SharedPages {
|
||||
content: Vec::new(),
|
||||
resources: String::new(),
|
||||
page: "/Annots [10 0 R]".into(),
|
||||
extra: vec![annotation, form],
|
||||
..SharedPages::new(&scan)
|
||||
}
|
||||
.build(&scan);
|
||||
crate::sanitize::remove_hidden(&mut doc, &[crate::sanitize::Hidden::Comments]).unwrap();
|
||||
assert_gone(&doc, b"LEAK_GLOBALS");
|
||||
assert_eq!(drawn_samples(&reopen(&doc), 1), scan.bits);
|
||||
}
|
||||
|
||||
/// [`assert_no_secret`] for a file that may hold JBIG2 streams, which `decoded` doesn't read: a stream
|
||||
/// stored again can be coded alone.
|
||||
fn assert_gone(doc: &Document, secret: &[u8]) {
|
||||
for (r, object) in reopen(doc).scan_objects() {
|
||||
let mut bytes = Vec::new();
|
||||
pdfcraft_cos::serialize(&object, &mut bytes);
|
||||
assert!(!bytes.windows(secret.len()).any(|w| w == secret), "secret remains in object {r:?}");
|
||||
if let Object::Stream(s) = &*object
|
||||
&& let Ok(data) = s.decoded()
|
||||
{
|
||||
assert!(!data.windows(secret.len()).any(|w| w == secret), "secret remains in stream {r:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply Redactions gives the page its own pruned /Resources, but the dictionary it drew through
|
||||
/// before (inherited from the /Pages node, or shared with another page) still named the removed
|
||||
/// image and form, so a full-rewrite save kept them.
|
||||
|
||||
@@ -5829,8 +5829,8 @@ milestone = "M8"
|
||||
status = "shipped"
|
||||
commands = ["redact.apply"]
|
||||
tools = ["redact_apply"]
|
||||
evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"]
|
||||
notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; every other image using a redacted image's globals is stored again without them (Flate, Group 4 or JBIG2, whichever is smallest), so they leave the file (one that can't be decoded refuses the redaction)"
|
||||
evidence = ["crates/redact/src/tests.rs::images_are_removed_or_have_their_pixels_cleared", "crates/redact/src/tests.rs::a_redacted_scans_shared_jbig2_dictionary_leaves_the_file", "crates/redact/src/tests.rs::redaction_clears_pixels_intersected_without_covering_their_centres", "crates/redact/src/tests.rs::a_fax_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::group_3_on_bytes_is_removed_whole_and_other_fax_images_cleared", "crates/redact/src/tests.rs::an_image_whose_predictor_is_given_by_reference_has_the_covered_pixel_cleared", "crates/redact/src/tests.rs::a_jbig2_image_has_its_covered_pixels_cleared_instead_of_being_removed", "crates/redact/src/tests.rs::a_jbig2_image_that_cannot_be_cleared_for_certain_is_removed_whole", "crates/redact/src/tests.rs::a_redacted_jbig2_scan_keeps_neither_its_code_nor_its_globals", "crates/redact/src/tests.rs::a_cleared_black_and_white_scan_is_stored_as_group_4", "crates/redact/src/tests.rs::globals_a_redacted_jbig2_image_shares_with_another_page_leave_the_file"]
|
||||
notes = "intersected pixel footprints are cleared conservatively in a copy, CCITT fax and JBIG2 images (scans stored as Group 4 or JBIG2, with their /JBIG2Globals) included; DCT/JPX images, Group 3 on bytes (which the bootstrap renderer may draw as other rows than the decoder reads), and a JBIG2 image whose page isn't its size, removed whole. The copy is Flate, or for a one-bit image of one component Group 4, or JBIG2 when it was JBIG2, whichever is smallest as stored, and refers to no globals; whatever left the file takes its globals with it: what else names them, or an exact copy, is stored again without them (Flate, Group 4 or JBIG2, whichever is smallest), and apply refuses when one can't be. Not found: a dictionary holding the same shapes in other bytes. An image that hasn't left the file keeps its globals with its pixels: one another page draws, or one a pattern, soft mask or structure element refers to"
|
||||
|
||||
[[feature]]
|
||||
id = "protect.redact-vectors"
|
||||
|
||||
Reference in New Issue
Block a user