From 98907b3907bfd54efe57e60f7e19dede7f33f9fc Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sat, 24 Oct 2026 04:43:33 -0700 Subject: [PATCH] M11: PDF Optimizer can discard an embedded search index PDF Optimizer > Discard Objects gets "Discard embedded search index", off by default. It removes the entries of the catalog's /PieceInfo whose application name contains "index" in any letter case, such as IBM-ODIndexes, and removes /PieceInfo when nothing else is left in it. Other application data and the pages' own /PieceInfo stay. An indirect /PieceInfo is not edited: the catalog gets a direct copy without those entries, and other objects that refer to the original keep it. The full save leaves out the objects that only the removed entries refer to. doc_optimize takes discard_search_index and reports search_indexes. --- crates/automation/src/lib.rs | 2 + crates/automation/src/tools.rs | 3 +- crates/automation/tests/it/automation.rs | 30 ++++++++ crates/optimize/README.md | 7 ++ crates/optimize/src/lib.rs | 51 +++++++++++++ crates/optimize/tests/optimize.rs | 97 ++++++++++++++++++++++++ crates/ui-egui/src/optimize_ui.rs | 1 + parity/acrobat-features.toml | 10 +-- 8 files changed, 195 insertions(+), 6 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index d2114ac7bf..ada8aee4bb 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -1254,6 +1254,7 @@ impl Automation { ("discard_thumbnails", &mut settings.discard_thumbnails), ("discard_alternate_images", &mut settings.discard_alternate_images), ("discard_tags", &mut settings.discard_tags), + ("discard_search_index", &mut settings.discard_search_index), ("discard_print_settings", &mut settings.discard_print_settings), ("flate_unencoded", &mut settings.flate_unencoded), ("remove_invalid_links", &mut settings.remove_invalid_links), @@ -1292,6 +1293,7 @@ impl Automation { "image_bytes_after": o.image_bytes_after, "thumbnails": o.thumbnails, "alternate_images": o.alternate_images, + "search_indexes": o.search_indexes, "tags_removed": o.tags_removed, "streams_compressed": o.streams_compressed, "invalid_links": o.invalid_links, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 1e009c1170..1661dfcc40 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1375,7 +1375,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ @@ -1388,6 +1388,7 @@ pub fn tools() -> Vec { "discard_thumbnails": { "type": "boolean" }, "discard_alternate_images": { "type": "boolean" }, "discard_tags": { "type": "boolean" }, + "discard_search_index": { "type": "boolean" }, "discard_print_settings": { "type": "boolean" }, "flate_unencoded": { "type": "boolean" }, "remove_invalid_links": { "type": "boolean" }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 9fd5c4269a..327e25d6cd 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -5685,6 +5685,36 @@ fn signing_with_translated_labels_through_tools() { assert!(stamped.contains("(Genehmigt)") && !stamped.contains("Reason") && !stamped.contains("Date"), "{stamped}"); } +#[test] +fn optimizing_discards_an_embedded_search_index_through_tools() { + use pdfcraft_cos::{Dict, Object}; + let dir = workdir("optimize-search-index"); + let mut cos = pdfcraft_cos::Document::open(std::sync::Arc::new(fixture(1))).unwrap(); + let mut info = Dict::new(); + for name in ["IBM-ODIndexes", "Illustrator"] { + let mut data = Dict::new(); + data.set(b"Private".to_vec(), Object::Int(1)); + info.set(name.as_bytes().to_vec(), Object::Dict(data)); + } + let root = cos.root().unwrap(); + cos.update_dict(root, |c| c.set(b"PieceInfo".to_vec(), Object::Dict(info))).unwrap(); + std::fs::write(dir.join("indexed.pdf"), pdfcraft_cos::write_full(&cos, &pdfcraft_cos::SaveOptions::default()).unwrap()).unwrap(); + let piece_info = |name: &str| { + let d = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join(name)).unwrap())).unwrap(); + let cat = d.get(d.root().unwrap()).as_dict().cloned().unwrap(); + let info = d.dict(cat.get(b"PieceInfo").unwrap()).unwrap(); + info.iter().map(|(k, _)| String::from_utf8_lossy(k).into_owned()).collect::>() + }; + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "indexed.pdf" }))["doc"].as_u64().unwrap(); + let kept = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "kept.pdf" })); + assert_eq!(kept["search_indexes"], 0, "{kept}"); + assert_eq!(piece_info("kept.pdf"), ["IBM-ODIndexes", "Illustrator"]); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "small.pdf", "discard_search_index": true })); + assert_eq!(r["search_indexes"], 1, "{r}"); + assert_eq!(piece_info("small.pdf"), ["Illustrator"]); +} + #[test] fn optimizing_through_tools() { let dir = workdir("optimize"); diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 3d517eef3a..dde9d9f0f4 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -37,3 +37,10 @@ then this, then merges identical objects and writes a full, compressed save (`pdfcraft_engine::optimizer::OptimizeJob`, which the UI runs on a worker thread with a progress bar). JPEG decoding and encoding and resampling come from the `image` crate. + +Discard embedded search index (`discard_search_index`, off by default) removes the entries of +the catalog's `/PieceInfo` whose application name contains "index" in any letter case, such as +`IBM-ODIndexes`, and removes `/PieceInfo` when no other entry is left. An indirect `/PieceInfo` +is copied into the catalog without those entries, and the original object is not changed. The +full save leaves out the objects that only the removed entries refer to. Pages keep their own +`/PieceInfo`. diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index c8b9b67690..b31a2af8fa 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -15,6 +15,8 @@ //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted //! image); Flate-compress streams that have no filter; remove a redundant outer ASCII85 or //! ASCIIHex layer from streams that have one. +//! - **Discard embedded search index** (off by default): the search indexes stored in the +//! catalog's `/PieceInfo` (see [`Settings::discard_search_index`]). //! //! Metadata, attachments, comments, scripts, private data, hidden layers, bookmarks and form //! fields are discarded by `pdfcraft-redact`'s Remove Hidden Information, which the engine @@ -96,6 +98,11 @@ pub struct Settings { pub discard_thumbnails: bool, pub discard_alternate_images: bool, pub discard_tags: bool, + /// Remove the entries of the catalog's `/PieceInfo` (data an application keeps for the whole + /// document, ISO 32000-2 §14.5) whose application name contains "index" in any letter case, + /// and `/PieceInfo` itself when nothing else is left in it. Objects that only those entries + /// refer to are left out of the saved file. + pub discard_search_index: bool, pub discard_print_settings: bool, /// Flate-compress streams stored without a filter, and strip a redundant outer ASCII85 or /// ASCIIHex layer from streams that have one. @@ -116,6 +123,7 @@ impl Default for Settings { discard_thumbnails: true, discard_alternate_images: true, discard_tags: false, + discard_search_index: false, discard_print_settings: false, flate_unencoded: true, remove_invalid_links: true, @@ -176,6 +184,8 @@ pub struct Report { pub thumbnails: usize, pub alternate_images: usize, pub tags_removed: bool, + /// Search index entries removed from the catalog's `/PieceInfo`. + pub search_indexes: usize, pub print_settings: usize, pub streams_compressed: usize, /// Links removed and bookmarks whose destination was cleared (Clean Up). @@ -247,6 +257,11 @@ pub fn optimize_with_progress(doc: &mut Document, settings: &Settings, progress: } report.tags_removed = true; } + if settings.discard_search_index + && let Some(root) = root + { + report.search_indexes = discard_search_index(doc, root)?; + } if settings.discard_print_settings && let Some(root) = root { @@ -335,3 +350,39 @@ fn stored_len(s: &Stream) -> Option { serialize(&Object::Dict(dict), &mut out); out.len().checked_add(s.raw.len()) } + +/// Remove the search index entries of the catalog's `/PieceInfo` ([`Settings::discard_search_index`]) +/// and return how many there were. An indirect `/PieceInfo` is not edited: the catalog gets a +/// direct copy without those entries, so other objects that refer to it keep theirs. The objects +/// the removed entries held are dropped by the full save, which writes only what the trailer +/// reaches. +fn discard_search_index(doc: &mut Document, root: ObjRef) -> Result { + let Some(info) = doc.get(root).as_dict().and_then(|c| c.get(b"PieceInfo").cloned()) else { + return Ok(0); + }; + let (info, removed) = match &*doc.resolve(&info) { + Object::Dict(d) => { + let kept: pdfcraft_cos::Dict = d.iter().filter(|(k, _)| !is_search_index(k)).map(|(k, v)| (k.clone(), v.clone())).collect(); + let removed = d.len().saturating_sub(kept.len()); + (kept, removed) + } + _ => return Ok(0), + }; + if removed == 0 { + return Ok(0); + } + doc.update_dict(root, |c| { + if info.is_empty() { + c.remove(b"PieceInfo"); + } else { + c.set(b"PieceInfo".to_vec(), Object::Dict(info)); + } + })?; + Ok(removed) +} + +/// Whether a `/PieceInfo` application name names a search index: it contains "index" in any +/// letter case (such as `IBM-ODIndexes`). +fn is_search_index(name: &[u8]) -> bool { + name.windows(5).any(|w| w.eq_ignore_ascii_case(b"index")) +} diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 6d9a9bb44e..bd3f3a3880 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -1139,3 +1139,100 @@ fn named_dest_keys(doc: &Document) -> Vec> { let arr = doc.resolve(tree.get(b"Names").unwrap()).as_array().cloned().unwrap(); arr.chunks(2).filter_map(|c| doc.resolve(&c[0]).as_string().map(|s| s.bytes.clone())).collect() } + +/// A `/PieceInfo` data dictionary holding `private`. +fn app_data(private: Object) -> Object { + let mut d = Dict::new(); + d.set(b"LastModified".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"D:20260101000000Z".to_vec()))); + d.set(b"Private".to_vec(), private); + Object::Dict(d) +} + +fn marked(doc: &mut Document, marker: &str, extra: &[(&str, Object)]) -> ObjRef { + let mut d = Dict::new(); + d.set(b"Marker".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(marker.as_bytes().to_vec()))); + for (k, v) in extra { + d.set(k.as_bytes().to_vec(), v.clone()); + } + doc.add(Object::Dict(d)) +} + +fn saved(doc: &Document) -> Vec { + write_full(doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap() +} + +fn contains(bytes: &[u8], text: &str) -> bool { + bytes.windows(text.len()).any(|w| w == text.as_bytes()) +} + +#[test] +fn search_indexes_in_the_catalog_go_and_other_application_data_stays() { + let mut doc = Document::new_empty(); + let p = page(&mut doc, &[], "BT ET"); + // The first index holds two objects that refer to each other, and one the page also uses. + let shared = marked(&mut doc, "SHARED-DATA", &[]); + let a = marked(&mut doc, "INDEX-A", &[("Shared", Object::Ref(shared))]); + let b = marked(&mut doc, "INDEX-B", &[("Next", Object::Ref(a))]); + doc.update_dict(a, |d| d.set(b"Next".to_vec(), Object::Ref(b))).unwrap(); + let c = marked(&mut doc, "INDEX-C", &[]); + let keep = marked(&mut doc, "KEEP-APP", &[]); + let mut info = Dict::new(); + info.set(b"IBM-ODIndexes".to_vec(), app_data(Object::Ref(a))); + info.set(b"MySearchINDEX".to_vec(), app_data(Object::Ref(c))); + info.set(b"Illustrator".to_vec(), app_data(Object::Ref(keep))); + let root = doc.root().unwrap(); + doc.update_dict(root, |cat| cat.set(b"PieceInfo".to_vec(), Object::Dict(info))).unwrap(); + let mut page_info = Dict::new(); + page_info.set(b"PageIndexer".to_vec(), app_data(Object::Ref(shared))); + doc.update_dict(p, |d| d.set(b"PieceInfo".to_vec(), Object::Dict(page_info))).unwrap(); + + // Off by default: nothing changes. + let mut untouched = doc.clone(); + assert!(!Settings::default().discard_search_index); + assert_eq!(optimize(&mut untouched, &Settings::default()).unwrap().search_indexes, 0); + let bytes = saved(&untouched); + assert!(["INDEX-A", "INDEX-B", "INDEX-C", "KEEP-APP"].iter().all(|m| contains(&bytes, m))); + + let r = optimize(&mut doc, &Settings { discard_search_index: true, ..Settings::default() }).unwrap(); + assert_eq!(r.search_indexes, 2); + let cat = doc.get(root).as_dict().cloned().unwrap(); + let info = cat.get(b"PieceInfo").and_then(Object::as_dict).cloned().unwrap(); + assert_eq!(info.iter().map(|(k, _)| k.clone()).collect::>(), vec![b"Illustrator".to_vec()]); + // The saved file leaves out the objects only the indexes held, the cycle included; the + // object the page also uses, the other application's data and the page's own entry stay. + let bytes = saved(&doc); + assert!(!["INDEX-A", "INDEX-B", "INDEX-C"].iter().any(|m| contains(&bytes, m))); + assert!(["SHARED-DATA", "KEEP-APP", "PageIndexer"].iter().all(|m| contains(&bytes, m))); + let reopened = Document::open(Arc::new(bytes)).unwrap(); + let cat = reopened.get(reopened.root().unwrap()).as_dict().cloned().unwrap(); + let info = reopened.dict(cat.get(b"PieceInfo").unwrap()).unwrap(); + assert!(info.contains(b"Illustrator") && info.len() == 1); +} + +#[test] +fn an_indirect_piece_info_is_left_for_its_other_users() { + let mut doc = Document::new_empty(); + let p = page(&mut doc, &[], "BT ET"); + let idx = marked(&mut doc, "INDEX-DATA", &[]); + let mut info = Dict::new(); + info.set(b"Index".to_vec(), app_data(Object::Ref(idx))); + let info = doc.add(Object::Dict(info)); + let root = doc.root().unwrap(); + doc.update_dict(root, |cat| cat.set(b"PieceInfo".to_vec(), Object::Ref(info))).unwrap(); + doc.update_dict(p, |d| d.set(b"PieceInfo".to_vec(), Object::Ref(info))).unwrap(); + let settings = Settings { discard_search_index: true, ..Settings::default() }; + let r = optimize(&mut doc, &settings).unwrap(); + // The catalog's only entry was an index, so /PieceInfo goes; the page keeps the shared one. + assert_eq!(r.search_indexes, 1); + assert!(!doc.get(root).as_dict().unwrap().contains(b"PieceInfo")); + assert!(doc.get(info).as_dict().unwrap().contains(b"Index")); + assert!(contains(&saved(&doc), "INDEX-DATA")); + + // A /PieceInfo that is not a dictionary is left as it is. + let mut doc = Document::new_empty(); + page(&mut doc, &[], "BT ET"); + let root = doc.root().unwrap(); + doc.update_dict(root, |cat| cat.set(b"PieceInfo".to_vec(), Object::Int(7))).unwrap(); + assert_eq!(optimize(&mut doc, &settings).unwrap().search_indexes, 0); + assert_eq!(doc.get(root).as_dict().unwrap().int(b"PieceInfo"), Some(7)); +} diff --git a/crates/ui-egui/src/optimize_ui.rs b/crates/ui-egui/src/optimize_ui.rs index 34188e184a..9b485289b4 100644 --- a/crates/ui-egui/src/optimize_ui.rs +++ b/crates/ui-egui/src/optimize_ui.rs @@ -205,6 +205,7 @@ pub(crate) fn body(ui: &mut egui::Ui, d: &mut OptimizeDraft, t: &Tokens) -> (boo ui.add(widgets::checkbox(&mut s.discard_alternate_images, tl!("Discard alternate images"))); ui.add(widgets::checkbox(&mut s.discard_thumbnails, tl!("Discard embedded page thumbnails"))); ui.add(widgets::checkbox(&mut s.discard_tags, tl!("Discard document tags"))); + ui.add(widgets::checkbox(&mut s.discard_search_index, tl!("Discard embedded search index"))); ui.add(widgets::checkbox(&mut s.discard_print_settings, tl!("Discard embedded print settings"))); discard_box(ui, &mut d.discard, Hidden::Bookmarks, "Discard bookmarks"); discard_box(ui, &mut d.discard, Hidden::FormFields, "Flatten form fields"); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 4ec39848c5..bbcf68f66d 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7418,11 +7418,11 @@ title = "Discard embedded search index" area = "K" tier = "P2" milestone = "M11" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "shipped" +commands = ["optimize.advanced"] +tools = ["doc_optimize"] +evidence = ["crates/optimize/tests/optimize.rs::search_indexes_in_the_catalog_go_and_other_application_data_stays", "crates/optimize/tests/optimize.rs::an_indirect_piece_info_is_left_for_its_other_users", "crates/automation/tests/it/automation.rs::optimizing_discards_an_embedded_search_index_through_tools"] +notes = "PDF Optimizer ▸ Discard Objects, off by default. The public help page does not say where the index is stored; PdfCraft's rule: an entry of the catalog's /PieceInfo whose application name contains \"index\" in any letter case (such as IBM-ODIndexes) is a search index and is removed, with /PieceInfo itself when nothing else is left. An indirect /PieceInfo is copied into the catalog without those entries and the original object is left for its other users. The full save leaves out objects only the removed entries reach, cycles included; objects something else also uses stay. Page /PieceInfo is not changed. An index stored under an application name that does not contain \"index\" is kept." [[feature]] id = "optimize.discard-print-settings"