From a0b60a3b92c44121763f73997e2c21111c72bc7e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:29:38 +0200 Subject: [PATCH 01/79] Optimize: optional target_bytes retries with stronger image settings (#1889) Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 54 +++++++++++++++++++++++++-- crates/automation/src/tools.rs | 3 +- crates/automation/tests/automation.rs | 43 +++++++++++++++++++++ parity/acrobat-features.toml | 4 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 88856305b2..17c1f7293c 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -92,6 +92,8 @@ const DEFAULT_DPI: f64 = 96.0; const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; +/// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. const BOOKMARK_PAGE: usize = 100; @@ -1120,10 +1122,52 @@ impl Automation { }) .collect::>()?, }; - let (bytes, r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let target = match a.opt_int("target_bytes")? { + None => None, + Some(t) if t >= 1 => Some(t as usize), + Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), + }; + let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let mut attempts = 1; + let mut step_used = 0; + if let Some(t) = target { + // Stronger image settings, one rung per attempt, until the copy fits. Every attempt + // starts from the open document, so a failed one changes nothing; the number of + // attempts is fixed. The smallest copy is kept if none fits. + for (k, &(ppi, quality)) in TARGET_LADDER.iter().enumerate() { + if bytes.len() <= t { + break; + } + let mut s = settings.clone(); + for img in [&mut s.color, &mut s.gray] { + img.downsample = true; + img.target_ppi = img.target_ppi.min(ppi); + img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); + img.compression = match img.compression { + Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), + _ => Compression::Jpeg(quality), + }; + } + let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + attempts += 1; + if b2.len() < bytes.len() { + (bytes, r, step_used) = (b2, r2, k + 1); + } + } + } write_atomic(&path, &bytes)?; let o = &r.optimize; - Ok(json!({ + let mut extra = json!({}); + if let Some(t) = target { + let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + extra = json!({ + "target_bytes": t, + "target_reached": bytes.len() <= t, + "attempts": attempts, + "target_settings": { "step": step_used, "max_ppi": ppi, "max_jpeg_quality": quality }, + }); + } + let mut out = json!({ "path": path.to_string_lossy(), "bytes_before": before, "bytes_after": bytes.len(), @@ -1144,7 +1188,11 @@ impl Automation { "unused_xobjects": o.unused_xobjects, "merged_objects": r.merged, "discarded": r.discarded.iter().map(|(h, n)| json!({ "category": h.id(), "count": n })).collect::>(), - })) + }); + if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) { + o.extend(e.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + Ok(out) } /// A watermark or background picture: `file` (an image or a PDF) and `file_page` (1-based). diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..2365086a33 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ @@ -1313,6 +1313,7 @@ pub fn tools() -> Vec { "merge_fragments": { "type": "boolean" }, "color": { "type": "object" }, "gray": { "type": "object" }, + "target_bytes": { "type": "integer", "minimum": 1 }, "discard_thumbnails": { "type": "boolean" }, "discard_alternate_images": { "type": "boolean" }, "discard_tags": { "type": "boolean" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..8018c83321 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4752,6 +4752,49 @@ fn a_page_printed_in_strips_gets_small_through_tools() { assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "preset": "tiny" })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn optimizing_to_a_target_size() { + let dir = workdir("optimize-target"); + let (w, h) = (2400u32, 1600u32); + let px: Vec = (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + [(x * 255 / w) as u8 ^ ((x * y) & 15) as u8, (y * 255 / h) as u8, ((x + y) & 255) as u8] + }) + .collect(); + let mut jpeg = Vec::new(); + let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 95); + enc.set_pixel_density(image::codecs::jpeg::PixelDensity::dpi(600)); + enc.encode(&px, w, h, image::ExtendedColorType::Rgb8).unwrap(); + std::fs::write(dir.join("photo.jpg"), &jpeg).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["photo.jpg"] }))["doc"].as_u64().unwrap(); + // Without a target nothing changes: no target fields. + let base = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "base.pdf" })); + assert!(base.get("target_reached").is_none(), "{base}"); + let base_len = base["bytes_after"].as_u64().unwrap(); + // A target below the default result is met by a stronger step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "fit.pdf", "target_bytes": base_len - 1 })); + assert_eq!(r["target_reached"], true, "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(r["target_settings"]["step"].as_u64().unwrap() >= 1, "{r}"); + assert_eq!(std::fs::metadata(dir.join("fit.pdf")).unwrap().len(), r["bytes_after"].as_u64().unwrap()); + // A target already met by the first attempt tries nothing more. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "easy.pdf", "target_bytes": base_len })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(true), Some(1)), "{r}"); + // An impossible target: bounded attempts, smallest copy written, said plainly. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "tiny.pdf", "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(6)), "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(std::fs::metadata(dir.join("tiny.pdf")).is_ok()); + // The open document is untouched by the failed attempts, and bad targets are refused. + let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); + assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); + assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // Confinement still applies. + assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); +} + #[test] fn initial_view_through_tools() { let dir = workdir("initial-view"); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..aee6fda80f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7260,8 +7260,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced"] tools = ["doc_optimize"] -evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] -notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit" +evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/automation/tests/automation.rs::optimizing_to_a_target_size", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] +notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit; headless-only optional target_bytes retries up to 5 stronger image steps (an automation extension, not an Acrobat feature)" [[feature]] id = "optimize.downsample-images" From 68d7b4cf3b31516672f124e1a8bf65e8df150714 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:20:48 +0200 Subject: [PATCH 02/79] Redact: mark text by caller-supplied regex with optional checksum (#1886) Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 33 ++++ Cargo.toml | 2 + ROADMAP.md | 1 + crates/automation/src/redact.rs | 30 +++- crates/automation/src/tools.rs | 4 +- crates/automation/tests/automation.rs | 28 ++++ crates/engine/src/lib.rs | 1 + crates/redact/Cargo.toml | 1 + crates/redact/src/lib.rs | 1 + crates/redact/src/user_regex.rs | 211 ++++++++++++++++++++++++++ parity/acrobat-features.toml | 12 ++ 11 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 crates/redact/src/user_regex.rs diff --git a/Cargo.lock b/Cargo.lock index 91ed3c4904..268f3c745d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4284,6 +4293,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "regex", "thiserror 2.0.21", ] @@ -5136,6 +5146,29 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.11" diff --git a/Cargo.toml b/Cargo.toml index 597dd316fa..cc0ceaa4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ weezl = "0.2" # foundation thiserror = "2" +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. +regex = { version = "1", default-features = false, features = ["std", "unicode"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 66b9677569..d1e1c6febf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 1b6283c0ad..1367cb819c 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -2,8 +2,8 @@ //! (removing what they cover for good) or clear them. use pdfcraft_engine::{ - Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad, - redact_word_list, + Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator, + RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list, }; use serde_json::{Value, json}; @@ -49,16 +49,20 @@ impl Automation { a.get("find").is_some(), a.get("words").is_some(), a.get("pattern").is_some(), + a.get("regex").is_some(), a.opt_bool("whole_pages")?.unwrap_or(false), ]; if chosen.iter().filter(|c| **c).count() != 1 { - return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into())); + return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into())); } let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false); let whole_words = a.opt_bool("whole_words")?.unwrap_or(false); if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() { return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into())); } + if a.get("validate").is_some() && a.get("regex").is_none() { + return Err(ToolError::InvalidArgs("validate applies only to regex".into())); + } // Per searched word or phrase: how many marks it made. let mut matched_words: Vec<(String, usize)> = Vec::new(); if a.get("rect").is_some() { @@ -79,6 +83,19 @@ impl Automation { Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?), None => None, }; + let user_re = match a.opt_str("regex")? { + Some(src) => { + let validator = match a.opt_str("validate")? { + Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| { + let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect(); + ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", "))) + })?), + None => None, + }; + Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?) + } + None => None, + }; let needles: Vec = match (a.opt_str("find")?, a.get("words")) { (Some(f), _) => vec![f.to_owned()], (None, Some(_)) => { @@ -98,9 +115,10 @@ impl Automation { let texts = self.page_texts(id, &pages)?; let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { - let hits: Vec<(Option, _)> = match pattern { - Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), - None => needles + let hits: Vec<(Option, _)> = match (&user_re, pattern) { + (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), + (None, None) => needles .iter() .enumerate() .flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h))) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..84831b79a4 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -416,7 +416,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -428,6 +428,8 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all)"), "overlay": { "type": "string" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..b1cb1920cb 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -3295,6 +3295,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn redacting_with_a_caller_regex_through_tools() { + let dir = workdir("redact-regex"); + std::fs::write( + dir.join("ids.txt"), + "Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468", + ) + .unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap(); + let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" }))); + assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" }))); + assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_)))); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" })); + assert_eq!(r["marked"].as_u64(), Some(2)); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" })); + assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum"); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}"); + assert!(text.contains("6468"), "{text}"); +} + #[test] fn redacting_with_codes_through_tools() { let dir = workdir("redact-codes"); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 01f162ae7d..22743fdc65 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -152,6 +152,7 @@ pub use pdfcraft_redact::patterns::{ MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list, }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; +pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator}; pub use pdfcraft_sign as sign; pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 49593855e2..f6f99995ac 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +regex = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 61231bd982..fc44d941d1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,6 +27,7 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +pub mod user_regex; use interp::{Mode, Scope, process}; diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs new file mode 100644 index 0000000000..043b51d0b0 --- /dev/null +++ b/crates/redact/src/user_regex.rs @@ -0,0 +1,211 @@ +//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`). +//! +//! The expression is untrusted input. It runs on the `regex` crate, which matches in time +//! linear in the text (no backtracking, no look-around or back-references), and is compiled +//! under hard limits on source length, nesting and compiled size, so a hostile expression is +//! refused with an error instead of exhausting time or memory. An optional [`Validator`] +//! keeps a match only when its checksum holds. + +use std::ops::Range; + +use regex::RegexBuilder; +use thiserror::Error; + +/// Longest expression accepted, in bytes. +pub const MAX_REGEX_LEN: usize = 1024; +/// Most matches returned for one page; the rest are ignored. +pub const MAX_MATCHES_PER_TEXT: usize = 10_000; +/// Cap on the compiled program and on the lazy-DFA cache, in bytes. +const SIZE_LIMIT: usize = 1 << 20; +/// Cap on parenthesis nesting. +const NEST_LIMIT: u32 = 32; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum RegexError { + #[error("regex is empty")] + Empty, + #[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")] + TooLong(usize), + #[error("invalid or too complex regex: {0}")] + Invalid(String), + #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] + MatchesEmpty, +} + +/// Checksum a match must pass to be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validator { + /// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored. + Luhn, + /// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and + /// digits only, spaces and hyphens ignored. + Mod97, + /// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10. + Iban, +} + +/// Ids accepted for `validate`. +pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban]; + +impl Validator { + pub fn id(self) -> &'static str { + match self { + Validator::Luhn => "luhn", + Validator::Mod97 => "mod97", + Validator::Iban => "iban", + } + } + + pub fn from_id(id: &str) -> Option { + VALIDATORS.into_iter().find(|v| v.id() == id) + } + + /// Whether `s` passes the checksum. + pub fn check(self, s: &str) -> bool { + match self { + Validator::Luhn => luhn(s), + Validator::Mod97 => mod97(s.chars()), + Validator::Iban => { + let kept: Vec = s.chars().filter(|c| *c != ' ' && *c != '-').collect(); + let (head, tail) = kept.split_at(kept.len().min(4)); + mod97(tail.iter().chain(head).copied()) + } + } + } +} + +fn luhn(s: &str) -> bool { + let (mut sum, mut digits, mut double) = (0u32, 0usize, false); + for c in s.chars().rev() { + if c == ' ' || c == '-' { + continue; + } + let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false }; + let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d }; + sum = sum.saturating_add(d); + digits += 1; + double = !double; + } + digits >= 2 && sum % 10 == 0 +} + +fn mod97(s: impl Iterator) -> bool { + let (mut rem, mut n) = (0u32, 0usize); + for c in s { + if c == ' ' || c == '-' { + continue; + } + if !c.is_ascii_alphanumeric() { + return false; + } + // Digits stay one decimal digit, letters become two (A = 10 .. Z = 35). + let v = c.to_digit(36).unwrap_or(0); + rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 }; + n += 1; + } + n >= 2 && rem == 1 +} + +/// A compiled caller expression with an optional checksum. +#[derive(Debug, Clone)] +pub struct UserRegex { + re: regex::Regex, + validator: Option, +} + +impl UserRegex { + pub fn new(source: &str, validator: Option) -> Result { + if source.is_empty() { + return Err(RegexError::Empty); + } + if source.len() > MAX_REGEX_LEN { + return Err(RegexError::TooLong(source.len())); + } + let re = RegexBuilder::new(source) + .size_limit(SIZE_LIMIT) + .dfa_size_limit(SIZE_LIMIT) + .nest_limit(NEST_LIMIT) + .build() + .map_err(|e| RegexError::Invalid(e.to_string()))?; + if re.is_match("") { + return Err(RegexError::MatchesEmpty); + } + Ok(Self { re, validator }) + } + + /// Character ranges of `text` the expression matches (non-overlapping, left to right). + pub fn find(&self, text: &[char]) -> Vec> { + let s: String = text.iter().collect(); + // Char index of every byte offset that starts a char (and of the end). + let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; + for (ci, (bi, _)) in s.char_indices().enumerate() { + if let Some(slot) = byte_to_char.get_mut(bi) { + *slot = ci; + } + } + if let Some(slot) = byte_to_char.get_mut(s.len()) { + *slot = text.len(); + } + let mut out = Vec::new(); + for m in self.re.find_iter(&s) { + if out.len() >= MAX_MATCHES_PER_TEXT { + break; + } + if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { + continue; + } + if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end())) + && a < b + && b != usize::MAX + { + out.push(a..b); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chars(s: &str) -> Vec { + s.chars().collect() + } + + #[test] + fn finds_matches_as_char_ranges_with_multibyte_text() { + let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); + let t = chars("Ré 85 05 78 et 91 12 34 fin"); + let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + assert_eq!(found, ["85 05 78", "91 12 34"]); + } + + #[test] + fn validators_filter_matches() { + let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); + let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); + let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); + assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); + } + + #[test] + fn hostile_expressions_are_refused_or_stay_linear() { + assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty); + assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); + assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); + assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); + assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); + assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); + // Classic catastrophic pattern finishes at once on a long non-matching input. + let r = UserRegex::new(r"(a+)+$", None).unwrap(); + let mut t = chars(&"a".repeat(50_000)); + t.push('!'); + assert!(r.find(&t).is_empty()); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..159f580635 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5611,6 +5611,18 @@ tools = ["redact_mark"] evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"] notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded" +[[feature]] +id = "protect.redact-regex" +title = "Redact by caller-supplied regular expression" +area = "G" +tier = "P2" +milestone = "M8" +status = "shipped" +commands = [] +tools = ["redact_mark"] +evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/automation.rs::redacting_with_a_caller_regex_through_tools"] +notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)." + [[feature]] id = "protect.redact-pattern-locales" title = "Pattern sets per locale" From f6e335f3fe0c0ddc00f93b28a8c9f7499bcefa05 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:49:37 +0200 Subject: [PATCH 03/79] UI: allow a half-screen window, and fold the mode bar when narrow (#1913) The viewport minimum was 820x520 points, wider than half of a 1440 px screen. It is now MIN_WINDOW_SIZE = 480x520 points. At that width the five mode tabs were hidden under the search field, so the mode bar folds them into one menu named after the current mode, shrinks the search field and turns it into an icon button when it gets too narrow. Notice bars stack their buttons under the message in a narrow document area. Co-Authored-By: Claude Sonnet 5.5 --- apps/pdfcraft/src/main.rs | 2 +- crates/ui-egui/src/canvas.rs | 16 +++++++++++ crates/ui-egui/src/chrome.rs | 43 +++++++++++++++++++++++++----- crates/ui-egui/src/lib.rs | 6 +++++ crates/ui-egui/src/widgets.rs | 7 +++++ crates/ui-egui/tests/min_window.rs | 38 ++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 crates/ui-egui/tests/min_window.rs diff --git a/apps/pdfcraft/src/main.rs b/apps/pdfcraft/src/main.rs index 7a0bb20432..edc0652d0e 100644 --- a/apps/pdfcraft/src/main.rs +++ b/apps/pdfcraft/src/main.rs @@ -309,7 +309,7 @@ fn native_options(integrated: bool, app_frame: bool, renderer: eframe::Renderer) let mut viewport = egui::ViewportBuilder::default() .with_title("PdfCraft") .with_inner_size([1440.0, 920.0]) - .with_min_inner_size([820.0, 520.0]) + .with_min_inner_size(pdfcraft_ui_egui::MIN_WINDOW_SIZE) .with_drag_and_drop(true) // Wayland app id: matches packaging/linux/ai.storyteller.pdfcraft.desktop. .with_app_id(APP_ID); diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 989b731b7e..72a8316ab7 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4248,8 +4248,24 @@ fn notices( fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, text: String, buttons: impl FnOnce(&mut egui::Ui)) { const ROW: f32 = 28.0; const ICON: f32 = 16.0; + /// Content width below which the buttons stack under the message. + const STACK_BELOW: f32 = 300.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); + if width < STACK_BELOW { + // A narrow document area (a half-width window): the message takes the full width and + // the buttons sit on a row of their own below it, rather than squeezing it to nothing. + ui.with_layout(egui::Layout::left_to_right(egui::Align::Min), |ui| { + let (slot, _) = ui.allocate_exact_size(vec2(ICON, ICON), Sense::hover()); + icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); + ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); + }); + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.set_min_height(ROW); + buttons(ui); + }); + return; + } ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { ui.set_min_height(ROW); buttons(ui); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index e8bf3b48dd..88165102ad 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -487,6 +487,11 @@ fn tab( }) } +/// Width the print, save and info buttons and their gaps take in the mode bar. +const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 2.0 * 4.0 + 8.0; +/// Narrowest search field the mode bar shows; below it the field becomes an icon button. +const MODE_BAR_SEARCH_MIN: f32 = 140.0; + pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); egui::Panel::top("mode_bar") @@ -499,13 +504,32 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(6.0); ui.painter().vline(ui.cursor().left(), ui.max_rect().y_range().shrink(12.0), Stroke::new(1.0, t.divider)); ui.add_space(10.0); - for (mode, label) in - [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")] - { - if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { - app.switch_mode(mode, true); + let modes = + [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. + let avail = ui.available_width(); + let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); + let inline_tabs = avail >= tabs_w + MODE_BAR_RIGHT + MODE_BAR_SEARCH_MIN; + if inline_tabs { + for (mode, label) in modes { + if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { + app.switch_mode(mode, true); + } } + } else { + // Narrow window: the five modes fold into one menu named after the current one. + let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); + let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + egui::Popup::menu(&resp).show(|ui| { + ui.set_min_width(160.0); + for (mode, label) in modes { + if ui.add(egui::Button::selectable(app.mode == mode, tl!(label))).clicked() { + app.switch_mode(mode, true); + } + } + }); } + let left = ui.available_width(); ui.with_layout(Layout::right_to_left(Align::Center), |ui| { ui.spacing_mut().item_spacing.x = 4.0; let has_doc = app.active.is_some(); @@ -524,7 +548,14 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } }); ui.add_space(8.0); - if widgets::search_box(ui, tl!("Find tools and commands"), 260.0).clicked() { + let search_w = (left - MODE_BAR_RIGHT).clamp(0.0, 260.0); + let opened = if search_w >= MODE_BAR_SEARCH_MIN { + widgets::search_box(ui, tl!("Find tools and commands"), search_w).clicked() + } else { + // Too narrow for the field: a button opens the same command search. + icons::button(ui, "search", 32.0, false, tl!("Find tools and commands")).clicked() + }; + if opened { app.palette_open = true; } }); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d101d812e0..832ae30cb4 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -149,6 +149,12 @@ mod open_worker_tests; use pdfcraft_engine::{DocId, Session}; pub use canvas::{DocView, RasterMemory}; + +/// Smallest window the shell is laid out for, in egui points (logical pixels, so the OS scale +/// factor is already applied: 480 points are 600 px at 125 % and 720 px at 150 %). 480 is half of a +/// 1440-wide screen at 150 % scaling, the tightest common half-screen split on Windows; the mode +/// bar, tab strip and notice bars adapt below the old 820 minimum (#1913). +pub const MIN_WINDOW_SIZE: [f32; 2] = [480.0, 520.0]; pub use editing::{CloseRequest, SaveTarget}; pub use files::{ExtractDraft, FilePurpose, FileRequest, RotateDraft, SplitDraft, SplitMode, SplitPlan}; pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore}; diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index 4f5aaa79b6..cae7405ac5 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -108,6 +108,7 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { let t = Tokens::get(ui.ctx()); let font = if active { theme::semibold(13.5) } else { theme::medium(13.5) }; let w = ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), font.clone(), t.text).size().x); + // Keep in step with `mode_tab_width`. let (rect, resp) = ui.allocate_exact_size(vec2(w + 22.0, 48.0), Sense::click()); resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, ui.is_enabled(), active, label)); let press = theme::Press::track(ui, &resp); @@ -128,6 +129,12 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { theme::hand(resp) } +/// Width [`mode_tab`] takes for `label` (the semibold, active, form is the widest). +pub fn mode_tab_width(ui: &egui::Ui, label: &str) -> f32 { + let t = Tokens::get(ui.ctx()); + ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), theme::semibold(13.5), t.text).size().x) + 22.0 +} + /// Rounded pill button; `primary` fills with the accent. pub fn pill_button(ui: &mut egui::Ui, label: &str, primary: bool) -> Response { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs new file mode 100644 index 0000000000..b06e81d12d --- /dev/null +++ b/crates/ui-egui/tests/min_window.rs @@ -0,0 +1,38 @@ +//! The smallest window (#1913): at `MIN_WINDOW_SIZE` the mode bar folds the five modes into a menu +//! and keeps the command search reachable, instead of hiding them under the search field. + +use egui::accesskit::Role; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_ui_egui::{MIN_WINDOW_SIZE, PdfCraftApp}; + +fn harness(size: egui::Vec2) -> Harness<'static, PdfCraftApp> { + let mut h = Harness::builder().with_size(size).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app + }); + h.run_steps(8); + h +} + +#[test] +fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { + let mut h = harness(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + assert!(h.query_by_role_and_label(Role::Button, "Read").is_none(), "no room for five tabs"); + assert!(h.query_by_role_and_label(Role::Button, "Find tools and commands").is_some(), "command search is reachable"); + h.get_by_role_and_label(Role::Button, "All tools").click(); + h.run_steps(4); + h.get_by_role_and_label(Role::Button, "E-Sign").click(); + h.run_steps(4); + assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); + assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); +} + +#[test] +fn a_wide_window_keeps_the_five_mode_tabs() { + let h = harness(egui::vec2(1400.0, 900.0)); + for mode in ["All tools", "Read", "Edit", "Convert", "E-Sign"] { + assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); + } +} From 03a4c21cfe7f73b890ee4d93a204fe38ad2c9e11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:40:36 +0200 Subject: [PATCH 04/79] UI: stack notice buttons only below 200 pt so the 900-wide notice keeps its layout (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 72a8316ab7..ca1ff12337 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4249,7 +4249,7 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te const ROW: f32 = 28.0; const ICON: f32 = 16.0; /// Content width below which the buttons stack under the message. - const STACK_BELOW: f32 = 300.0; + const STACK_BELOW: f32 = 200.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); if width < STACK_BELOW { From c3cf27558a6d0f6c8c245b5a4bad813e6ae5de30 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:53:33 +0200 Subject: [PATCH 05/79] UI: wrap stacked notice buttons and scroll the rail in short windows (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 3 +- crates/ui-egui/src/chrome.rs | 241 +++++++++++++++-------------- crates/ui-egui/tests/min_window.rs | 82 ++++++++++ 3 files changed, 211 insertions(+), 115 deletions(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index ca1ff12337..349b4deded 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4260,7 +4260,8 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); }); - ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + // The buttons wrap onto further rows when even the full width is too narrow for them all. + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center).with_main_wrap(true), |ui| { ui.set_min_height(ROW); buttons(ui); }); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 88165102ad..d4c6f09503 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,6 +648,9 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } +/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 316.0; + pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); let Some((index, id)) = app.active_ids() else { return }; @@ -668,122 +671,132 @@ pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { .exact_size(48.0) .frame(egui::Frame::NONE.fill(t.chrome).inner_margin(egui::Margin::symmetric(6, 8)).stroke(Stroke::new(1.0, t.divider))) .show(ui, |ui| { - ui.spacing_mut().item_spacing.y = 4.0; - let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { - let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); - let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); - if has && !selected { - let c = r.rect.right_top() + vec2(-8.0, 8.0); - ui.painter().circle_filled(c, 3.0, t.accent); - } - if r.clicked() { - app.choose_right_panel(if selected { None } else { Some(panel) }); - } - }; - rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); - rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); - rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); - rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); - rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); - rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); - rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); - if has_check { - rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); - } - rail_button(ui, RightPanel::Tags, "tag", "Tags", false); + let panel_h = ui.available_height(); + // A window too short for both clusters scrolls the rail instead of overlapping them (#1913). + egui::ScrollArea::vertical().auto_shrink(false).scroll_bar_visibility(egui::scroll_area::ScrollBarVisibility::AlwaysHidden).show( + ui, + |ui| { + ui.spacing_mut().item_spacing.y = 4.0; + let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { + let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); + let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); + if has && !selected { + let c = r.rect.right_top() + vec2(-8.0, 8.0); + ui.painter().circle_filled(c, 3.0, t.accent); + } + if r.clicked() { + app.choose_right_panel(if selected { None } else { Some(panel) }); + } + }; + rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); + rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); + rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); + rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); + rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); + rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); + rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); + if has_check { + rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); + } + rail_button(ui, RightPanel::Tags, "tag", "Tags", false); - // Page navigation cluster at the bottom (as in Acrobat's rail). - let view = &mut app.views[index]; - let mut page_display = None; - ui.with_layout(Layout::bottom_up(Align::Center), |ui| { - ui.spacing_mut().item_spacing.y = 2.0; - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); - if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { - view.zoom_step(false); - } - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); - if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { - view.zoom_step(true); - } - // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. - let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); - let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); - page_display = egui::Popup::menu(&resp) - .show(|ui| { - ui.set_min_width(220.0); - rail_view_menu(ui, view) - }) - .and_then(|r| r.inner); - let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); - if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { - view.rotate_view(true); - } - // Not `columns-2` while fitting the page: that is the two-page view's icon. - let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; - if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { - view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; - view.goto = Some((view.current, 0.0)); - } - let zoom = format!("{:.0}%", view.zoom * 100.0); - ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { - ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); - }); - ui.add_space(6.0); - if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { - view.step_page(true); - } - if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { - view.step_page(false); - } - // History is distinct from stepping to the adjacent page, and belongs to this tab. - for (forward, icon, label, shortcut, enabled) in [ - (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), - (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), - ] { - let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); - if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { - view.view_history(forward); + // Page navigation cluster at the bottom (as in Acrobat's rail). + let view = &mut app.views[index]; + let mut page_display = None; + let cluster_w = ui.available_width(); + // Inside the scroll area `available_height` is unbounded: the panel's height was taken outside. + let cluster_h = (panel_h - ui.min_rect().height() - ui.spacing().item_spacing.y).max(RAIL_BOTTOM_H); + ui.allocate_ui_with_layout(vec2(cluster_w, cluster_h), Layout::bottom_up(Align::Center), |ui| { + ui.spacing_mut().item_spacing.y = 2.0; + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); + if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { + view.zoom_step(false); + } + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); + if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { + view.zoom_step(true); + } + // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. + let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); + let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); + page_display = egui::Popup::menu(&resp) + .show(|ui| { + ui.set_min_width(220.0); + rail_view_menu(ui, view) + }) + .and_then(|r| r.inner); + let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); + if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { + view.rotate_view(true); + } + // Not `columns-2` while fitting the page: that is the two-page view's icon. + let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; + if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { + view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; + view.goto = Some((view.current, 0.0)); + } + let zoom = format!("{:.0}%", view.zoom * 100.0); + ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { + ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); + }); + ui.add_space(6.0); + if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { + view.step_page(true); + } + if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { + view.step_page(false); + } + // History is distinct from stepping to the adjacent page, and belongs to this tab. + for (forward, icon, label, shortcut, enabled) in [ + (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), + (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), + ] { + let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); + if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { + view.view_history(forward); + } + } + ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); + let page_input_id = view.page_input_id(); + // Consume replacement selection once, including when the editor has lost focus. + let reselect = std::mem::take(&mut view.page_input_reselect); + // Navigation updates the idle display, never the text being edited. + if !ui.memory(|m| m.has_focus(page_input_id)) { + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } else if reselect { + // The replacement changed the label while egui retained this editor's cursor. + let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); + let end = egui::text::CCursor::new(view.page_input.chars().count()); + state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); + state.store(ui.ctx(), page_input_id); + } + let edit = egui::TextEdit::singleline(&mut view.page_input) + .id(page_input_id) + .desired_width(34.0) + .horizontal_align(Align::Center) + .font(theme::medium(12.0)) + .margin(vec2(2.0, 4.0)); + let r = egui::Frame::NONE + .fill(t.field) + .stroke(Stroke::new(1.0, t.border)) + .corner_radius(CornerRadius::same(5)) + .show(ui, |ui| ui.add(edit)) + .inner + .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); + if r.lost_focus() { + if ui.input(|i| i.key_pressed(egui::Key::Enter)) { + // A page label first (logical page numbers, as Acrobat), then a number. + let typed = view.page_input.clone(); + view.go_to_typed(&typed, &labels); + } + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } + }); + if let Some(id) = page_display { + app.execute(id); } - } - ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); - let page_input_id = view.page_input_id(); - // Consume replacement selection once, including when the editor has lost focus. - let reselect = std::mem::take(&mut view.page_input_reselect); - // Navigation updates the idle display, never the text being edited. - if !ui.memory(|m| m.has_focus(page_input_id)) { - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } else if reselect { - // The replacement changed the label while egui retained this editor's cursor. - let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); - let end = egui::text::CCursor::new(view.page_input.chars().count()); - state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); - state.store(ui.ctx(), page_input_id); - } - let edit = egui::TextEdit::singleline(&mut view.page_input) - .id(page_input_id) - .desired_width(34.0) - .horizontal_align(Align::Center) - .font(theme::medium(12.0)) - .margin(vec2(2.0, 4.0)); - let r = egui::Frame::NONE - .fill(t.field) - .stroke(Stroke::new(1.0, t.border)) - .corner_radius(CornerRadius::same(5)) - .show(ui, |ui| ui.add(edit)) - .inner - .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); - if r.lost_focus() { - if ui.input(|i| i.key_pressed(egui::Key::Enter)) { - // A page label first (logical page numbers, as Acrobat), then a number. - let typed = view.page_input.clone(); - view.go_to_typed(&typed, &labels); - } - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } - }); - if let Some(id) = page_display { - app.execute(id); - } + }, + ); }); } diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index b06e81d12d..889d952520 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -36,3 +36,85 @@ fn a_wide_window_keeps_the_five_mode_tabs() { assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); } } + +fn fixture(n: usize) -> Vec { + let mut objs: Vec = vec!["<< /Type /Catalog /Pages 2 0 R >>".into()]; + let kids: Vec = (0..n).map(|i| format!("{} 0 R", 4 + 2 * i)).collect(); + objs.push(format!("<< /Type /Pages /Kids [{}] /Count {n} /MediaBox [0 0 200 300] >>", kids.join(" "))); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into()); + for i in 0..n { + objs.push(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i)); + let body = format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1); + objs.push(format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len())); + } + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn protected(user: &str, owner: &str, permissions: i32) -> Vec { + protected_with(pdfcraft_cos::Algorithm::Aes256, user, owner, permissions) +} + +fn protected_with(algorithm: pdfcraft_cos::Algorithm, user: &str, owner: &str, permissions: i32) -> Vec { + let mut doc = pdfcraft_cos::Document::open(std::sync::Arc::new(fixture(2))).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm, + user_password: user, + owner_password: owner, + permissions, + encrypt_metadata: true, + seed: [4; 32], + }) + .unwrap(); + pdfcraft_cos::write_full(&doc, &Default::default()).unwrap() +} + +/// A narrow window: the stacked notice bar keeps both buttons inside the window and apart. +#[test] +fn the_stacked_notice_keeps_its_buttons_in_view() { + let mut h = Harness::builder().with_size(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("locked.pdf", None, protected("", "owner", 0b0100)).unwrap(); + app + }); + h.run_steps(8); + let window = egui::Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + let msg = h.get_by_label_contains("This document is secured").rect(); + let (unlock, settings) = (h.get_by_label("Unlock…").rect(), h.get_by_label("Security settings").rect()); + for (name, r) in [("unlock", unlock), ("settings", settings)] { + assert!(window.contains_rect(r), "{name} stays inside the window: {r:?}"); + assert!(r.left() >= msg.left() - 30.0, "{name} is not cut off at the bar's left edge: {r:?} vs {msg:?}"); + assert!(r.top() >= msg.bottom() - 0.5 || r.bottom() <= msg.top() + 0.5, "{name} does not cover the message"); + } + assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); +} + +/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +#[test] +fn a_short_window_does_not_overlap_the_rail_clusters() { + let h = { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("a.pdf", None, fixture(2)).unwrap(); + app + }); + h.run_steps(8); + h + }; + let tags = h.get_by_label("Tags").rect(); + let next = h.get_by_label("Previous page").rect(); + assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); +} From fb7f7252dbc470ef5736295da94134f2c3e5146f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 15 Oct 2026 23:33:25 +0200 Subject: [PATCH 06/79] UI: size the rail's bottom cluster for the history buttons, fix a pixel-fragile flick test (#1913) The rail's page-navigation cluster measures 360 pt since the two view-history buttons landed; RAIL_BOTTOM_H was 316, so in a short window the cluster overflowed its allocation and overlapped. The min-window test now uses the minimum height and checks the page box, history and zoom buttons. a_quick_flick_on_either_edge_rewraps_the_paragraph grabbed the widened edge 2 px outside the box, at the limit of the grab zone; the rail's scroll area moved the canvas width by 3 px and the press missed. It now grabs 1 px outside. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 4 ++-- crates/ui-egui/tests/editing.rs | 4 +++- crates/ui-egui/tests/min_window.rs | 12 ++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index d4c6f09503..51f269dc43 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,8 +648,8 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } -/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. -const RAIL_BOTTOM_H: f32 = 316.0; +/// Height the rail's page-navigation cluster (zoom, fit, page, history buttons, page box) needs (360 pt measured); below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 364.0; pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/editing.rs b/crates/ui-egui/tests/editing.rs index f8db525751..cef7214485 100644 --- a/crates/ui-egui/tests/editing.rs +++ b/crates/ui-egui/tests/editing.rs @@ -3487,7 +3487,9 @@ fn a_quick_flick_on_either_edge_rewraps_the_paragraph() { assert_eq!(text, ["Page", "1"], "narrowed from the right edge"); // Flick the right edge outward, past the box and its handle: back onto one line. let b = block(&h); - let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(2.0, 0.0); + // 1 px outside the box: the grab zone's outer limit sits near 2 px, where a pixel of page + // scale (the canvas width depends on the rail) decides whether the press lands on the handle. + let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(1.0, 0.0); flick(&mut h, edge, edge + egui::vec2(80.0 * k, 0.0)); let text: Vec = lines(&h).iter().map(|l| l.text.clone()).collect(); assert_eq!(text, ["Page 1"], "widened by a fast drag that left the handle at once"); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 889d952520..8d28c6aff6 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -101,11 +101,11 @@ fn the_stacked_notice_keeps_its_buttons_in_view() { assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); } -/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +/// #1913 (the minimum height): the rail's top buttons and its page-navigation cluster must not overlap. #[test] fn a_short_window_does_not_overlap_the_rail_clusters() { let h = { - let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { let mut app = PdfCraftApp::new(); app.set_option("language", "en").unwrap(); app.open_bytes("a.pdf", None, fixture(2)).unwrap(); @@ -117,4 +117,12 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { let tags = h.get_by_label("Tags").rect(); let next = h.get_by_label("Previous page").rect(); assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); + // The history buttons are the cluster's topmost buttons (above the page box). + let history = h.get_by_label_contains("Previous view").rect(); + assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); + // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. + let page_box = h.get_by_role(Role::TextInput).rect(); + assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + let zoom_out = h.get_by_label_contains("Zoom out").rect(); + assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From d6d4999a355325e701506df85318ddf50f1689dd Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 19 Oct 2026 02:39:14 +0200 Subject: [PATCH 07/79] Test: the rail now has two text fields (zoom and page box) After merging main's editable zoom field, get_by_role(TextInput) in the min-window test found two nodes. Check every text field of the rail sits below the rail buttons. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/min_window.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 8d28c6aff6..005c97dc5f 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -120,9 +120,11 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { // The history buttons are the cluster's topmost buttons (above the page box). let history = h.get_by_label_contains("Previous view").rect(); assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); - // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. - let page_box = h.get_by_role(Role::TextInput).rect(); - assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + // The cluster holds two text fields, the zoom field and the page box; the page box is the topmost thing in it, the lowest is the zoom-out button. + for field in h.get_all_by_role(Role::TextInput) { + let r = field.rect(); + assert!(r.top() >= tags.bottom(), "the rail's text fields sit below the rail buttons: {tags:?} vs {r:?}"); + } let zoom_out = h.get_by_label_contains("Zoom out").rect(); assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From 8239f9c4eda76ed126ddf2f063544cc5a7819289 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 20 Oct 2026 00:55:51 +0200 Subject: [PATCH 08/79] Optimize target_bytes: keep zip/retain image encoding on retries, only tighten ppi Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 11 +++++++---- crates/automation/src/tools.rs | 2 +- crates/automation/tests/automation.rs | 8 ++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 17c1f7293c..a279856fb5 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -93,6 +93,7 @@ const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; /// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +/// The quality applies only to images the caller left on JPEG; zip and retain keep their encoding. const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. @@ -1143,10 +1144,10 @@ impl Automation { img.downsample = true; img.target_ppi = img.target_ppi.min(ppi); img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); - img.compression = match img.compression { - Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), - _ => Compression::Jpeg(quality), - }; + // Only a caller who chose JPEG gets a lower quality; zip and retain stay as asked. + if let Compression::Jpeg(q) = img.compression { + img.compression = Compression::Jpeg(q.min(quality)); + } } let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; attempts += 1; @@ -1160,6 +1161,8 @@ impl Automation { let mut extra = json!({}); if let Some(t) = target { let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + let lossy = matches!(settings.color.compression, Compression::Jpeg(_)) || matches!(settings.gray.compression, Compression::Jpeg(_)); + let quality = quality.filter(|_| lossy); extra = json!({ "target_bytes": t, "target_reached": bytes.len() <= t, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 2365086a33..550fa119d8 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 8018c83321..b6e5305a69 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4791,6 +4791,14 @@ fn optimizing_to_a_target_size() { let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // A caller who asked for lossless images keeps them: retries only tighten the ppi, no JPEG quality. + let lossless = json!({ "compression": "zip" }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "zip.pdf", "color": lossless, "gray": lossless, "target_bytes": 1 })); + assert_eq!(r["attempts"].as_u64(), Some(6), "{r}"); + assert!(r["target_settings"]["max_ppi"].is_number() && r["target_settings"]["max_jpeg_quality"].is_null(), "{r}"); + // A JPEG caller gets the quality step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); + assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); // Confinement still applies. assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); } From 86af84171f004d1e06564a01152406bf4d997ff4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:11:43 +0200 Subject: [PATCH 09/79] doc_optimize: reject a target_bytes too large for usize Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 53e99e008d..ffde1dfab3 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -1308,7 +1308,7 @@ impl Automation { }; let target = match a.opt_int("target_bytes")? { None => None, - Some(t) if t >= 1 => Some(t as usize), + Some(t) if t >= 1 => Some(usize::try_from(t).map_err(|_| ToolError::InvalidArgs(format!("target_bytes {t} is too large")))?), Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), }; let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; From 0f64d9589ed5469490bd5d176ce47101f2c1d45a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:45:07 +0200 Subject: [PATCH 10/79] Run rustfmt on the mode bar after the upstream merge Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index b08c48518d..e5b239ecbb 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -507,8 +507,7 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(10.0); // Acrobat's mode bar has no Read tab: reading is what the viewer does with no tool // in use. Read mode (the panes hidden) stays a View command and a default workspace. - let modes = - [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + let modes = [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. let avail = ui.available_width(); let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); From 589c4af8e7138aa22eb51322353835d2ab73b772 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 25 Oct 2026 10:19:17 +0100 Subject: [PATCH 11/79] Redact regex: fail when a page exceeds the match cap UserRegex::find now returns an error instead of silently stopping at 10,000 matches, so redact_mark cannot report success while later matches stay readable. Empty-match detection probes several short texts so patterns like \b are refused with the clear message. Schema notes that the 1024 limit is in bytes. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/redact.rs | 19 ++++++++++++- crates/automation/src/tools.rs | 4 +-- crates/redact/src/user_regex.rs | 48 ++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 840020c570..808b6eabb7 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -126,7 +126,24 @@ impl Automation { let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { let hits: Vec<(Option, _)> = match (&user_re, pattern) { - (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (Some(re), _) => { + // A page with too many matches fails the call: marking only the first ones + // would leave the rest readable after redact_apply. + let too_many = std::cell::Cell::new(false); + let found = text.find_with(|chars| { + re.find(chars).unwrap_or_else(|_| { + too_many.set(true); + Vec::new() + }) + }); + if too_many.get() { + return Err(ToolError::InvalidArgs(format!( + "regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked", + p + 1 + ))); + } + found.into_iter().map(|h| (None, h)).collect() + } (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), (None, None) => needles .iter() diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4c651c00dd..0a41cacc0a 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -486,7 +486,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -498,7 +498,7 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, - "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." }, "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs index 043b51d0b0..9d1c1b0004 100644 --- a/crates/redact/src/user_regex.rs +++ b/crates/redact/src/user_regex.rs @@ -13,7 +13,8 @@ use thiserror::Error; /// Longest expression accepted, in bytes. pub const MAX_REGEX_LEN: usize = 1024; -/// Most matches returned for one page; the rest are ignored. +/// Most matches accepted for one page; one more makes [`UserRegex::find`] fail, so a page is never +/// marked only in part. pub const MAX_MATCHES_PER_TEXT: usize = 10_000; /// Cap on the compiled program and on the lazy-DFA cache, in bytes. const SIZE_LIMIT: usize = 1 << 20; @@ -30,6 +31,8 @@ pub enum RegexError { Invalid(String), #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] MatchesEmpty, + #[error("regex has more than {MAX_MATCHES_PER_TEXT} matches on one page; narrow it so that every match can be marked")] + TooManyMatches, } /// Checksum a match must pass to be kept. @@ -106,6 +109,12 @@ fn mod97(s: impl Iterator) -> bool { n >= 2 && rem == 1 } +/// Whether `re` can match the empty string, which `is_match("")` alone misses for assertions such +/// as `\b`. Tries a few short texts with letters, digits, space, newline and a non-ASCII letter. +fn can_match_empty(re: ®ex::Regex) -> bool { + ["", " ", "a", "1", "a a", "a\n", "\n", "é", "aa 11"].iter().any(|p| re.find_iter(p).any(|m| m.is_empty())) +} + /// A compiled caller expression with an optional checksum. #[derive(Debug, Clone)] pub struct UserRegex { @@ -127,14 +136,17 @@ impl UserRegex { .nest_limit(NEST_LIMIT) .build() .map_err(|e| RegexError::Invalid(e.to_string()))?; - if re.is_match("") { + if can_match_empty(&re) { return Err(RegexError::MatchesEmpty); } Ok(Self { re, validator }) } /// Character ranges of `text` the expression matches (non-overlapping, left to right). - pub fn find(&self, text: &[char]) -> Vec> { + /// + /// Fails with [`RegexError::TooManyMatches`] when more than [`MAX_MATCHES_PER_TEXT`] matches + /// are kept, instead of returning a partial list the caller would take for complete. + pub fn find(&self, text: &[char]) -> Result>, RegexError> { let s: String = text.iter().collect(); // Char index of every byte offset that starts a char (and of the end). let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; @@ -148,9 +160,6 @@ impl UserRegex { } let mut out = Vec::new(); for m in self.re.find_iter(&s) { - if out.len() >= MAX_MATCHES_PER_TEXT { - break; - } if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { continue; } @@ -158,10 +167,13 @@ impl UserRegex { && a < b && b != usize::MAX { + if out.len() >= MAX_MATCHES_PER_TEXT { + return Err(RegexError::TooManyMatches); + } out.push(a..b); } } - out + Ok(out) } } @@ -177,7 +189,7 @@ mod tests { fn finds_matches_as_char_ranges_with_multibyte_text() { let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); let t = chars("Ré 85 05 78 et 91 12 34 fin"); - let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + let found: Vec = r.find(&t).unwrap().into_iter().map(|g| t[g].iter().collect()).collect(); assert_eq!(found, ["85 05 78", "91 12 34"]); } @@ -185,10 +197,10 @@ mod tests { fn validators_filter_matches() { let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); } @@ -199,13 +211,25 @@ mod tests { assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); - assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + for empty in ["a*", r"\b", "(?m)^", r"\B", "(?m)$", r"\d*|x"] { + assert_eq!(UserRegex::new(empty, None).unwrap_err(), RegexError::MatchesEmpty, "{empty}"); + } + assert!(UserRegex::new(r"\bfoo\b", None).is_ok()); assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); // Classic catastrophic pattern finishes at once on a long non-matching input. let r = UserRegex::new(r"(a+)+$", None).unwrap(); let mut t = chars(&"a".repeat(50_000)); t.push('!'); - assert!(r.find(&t).is_empty()); + assert!(r.find(&t).unwrap().is_empty()); + } + + #[test] + fn more_than_the_cap_of_matches_is_an_error_not_a_partial_list() { + let r = UserRegex::new("x", None).unwrap(); + let at_cap = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT)); + assert_eq!(r.find(&at_cap).unwrap().len(), MAX_MATCHES_PER_TEXT); + let over = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT + 1)); + assert_eq!(r.find(&over).unwrap_err(), RegexError::TooManyMatches); } } From 90fb9b8c9e37adb0663bacca887248672f68d8e4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 19:18:16 +0100 Subject: [PATCH 12/79] Test the menu bar's File item instead of the removed Menu button Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/min_window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/min_window.rs b/crates/ui-egui/tests/it/min_window.rs index fc395e7cdf..f384eef0eb 100644 --- a/crates/ui-egui/tests/it/min_window.rs +++ b/crates/ui-egui/tests/it/min_window.rs @@ -26,7 +26,7 @@ fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { h.get_by_role_and_label(Role::Button, "E-Sign").click(); h.run_steps(4); assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); - assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); + assert!(h.query_by_role_and_label(Role::MenuItem, "File").is_some(), "main menu reachable"); } #[test] From 88649a8955fa674adcaa35af48340de74209e294 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 20:35:42 +0100 Subject: [PATCH 13/79] Name Read mode in the folded mode menu The narrow-window mode menu fell back to "All tools" while Read was active. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index f831d26d9c..5e1fef8860 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -505,8 +505,13 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } } else { // Narrow window: the modes fold into one menu named after the current one. - let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); - let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + // Read is not one of the four choices but can be active (View command): name it too. + let current = if app.mode == Mode::Read { + tl!("Read") + } else { + tl!(modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l)) + }; + let resp = widgets::ghost_button(ui, "chevron-down", current); egui::Popup::menu(&resp).show(|ui| { ui.set_min_width(160.0); for (mode, label) in modes { From fe03703a3b71a92c96a1d5318b3ad4dd9d02ab88 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:55:16 +0100 Subject: [PATCH 14/79] perf(cos): keep a running total of the scan reader's cache size ObjectReader::try_get measured every parsed object in its cache (up to 128) after each read to check the byte budget. Keep a running total instead and measure the whole cache only when something other than one freshly parsed object changed it. Eviction timing is unchanged, so output is identical. Fixes #4867 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..74b773e026 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -445,6 +445,10 @@ pub struct Document { pub(crate) struct ObjectReader { pub(crate) document: Document, decoded_count: usize, + /// Objects in the parsed-object cache and the bytes of their own, as last measured. Kept + /// as a running total so a read does not measure the whole cache again. + counted_objects: usize, + counted_bytes: usize, } impl ObjectReader { @@ -488,8 +492,25 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))) > Self::STREAM_BYTES { + // The common read parsed exactly this object and nothing else (a hit changes + // nothing): add its size to the running total. Anything else, such as a failed + // load, a /Length that cached further objects or a cache changed from outside, + // measures the whole cache again. + let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); + if object.is_ok() && cache.len() == self.counted_objects { + // Unchanged: a hit, or an object held by the overlay. + } else if let (true, Ok(read)) = (grew_by_this_object, &object) { + self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); + self.counted_objects = cache.len(); + } else { + self.counted_bytes = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))); + self.counted_objects = cache.len(); + } + if cache.len() >= Self::OBJECTS || self.counted_bytes > Self::STREAM_BYTES { cache.clear(); + self.counted_objects = 0; + self.counted_bytes = 0; } } object @@ -1176,7 +1197,7 @@ impl Document { let mut document = self.clone(); document.cache = Arc::default(); document.objstms = Arc::default(); - ObjectReader { document, decoded_count: 0 } + ObjectReader { document, decoded_count: 0, counted_objects: 0, counted_bytes: 0 } } fn load(&self, num: u32, depth: u32) -> Result { @@ -3004,11 +3025,27 @@ endobj // Data of its own (decrypted streams, strings) counts: past the budget the cache is // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned)); + reader.document.cache.lock().unwrap().insert(5, Arc::new(owned)); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } + #[test] + fn temporary_reader_running_total_matches_the_cache() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>".to_owned(), "<< /Type /Pages /Kids [] /Count 0 >>".to_owned()]; + bodies.extend((0..300).map(|i| format!("<< /Value {i} /Name ({}) >>", "s".repeat(i % 40)))); + let refs: Vec<_> = bodies.iter().map(String::as_str).collect(); + let doc = Document::open(Arc::new(build(&refs, "/Root 1 0 R"))).unwrap(); + let mut reader = doc.object_reader(); + for num in (1..=302).chain([5, 301, 99_999, 7]) { + let _ = reader.try_get(ObjRef::new(num, 0)); + let cache = reader.document.cache.lock().unwrap(); + let measured = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &reader.document.data, 0))); + assert_eq!((reader.counted_objects, reader.counted_bytes), (cache.len(), measured), "after object {num}"); + assert!(cache.len() < ObjectReader::OBJECTS); + } + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From be8345aa3f2d9930201e766a4c8a8f315a48f8d6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:57:29 +0100 Subject: [PATCH 15/79] perf(cos): rebuild a damaged cross-reference index without parsing every object Reconstruct parsed every object to see whether it is an object stream, and searched each object's next 1,024 bytes and the whole file byte by byte. Only objects whose dictionary holds ObjStm or a # (a name can hide letters as #xx) are now parsed for that check. Every /XRef position is found once, and find() scans for the needle's rarest byte in 32-byte blocks. The index, trailer and repair log are unchanged. Fixes #4868 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 72 ++++++++++++++++++++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/reconstruct.rs | 57 +++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 crates/cos/tests/it/reconstruct.rs diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..4ad24a11b1 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2052,10 +2052,29 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - for (num, offset, generation, body) in object_headers(&data) { + // Objects whose bytes (the last object of a number wins, as in `entries`) could hold an + // `/ObjStm` type; the others are not parsed for the object-stream check below. + let mut candidates: HashSet = HashSet::new(); + let headers = object_headers(&data); + // Where `/XRef` occurs in the file, found in one pass. + let mut xref_marks = Vec::new(); + let mut mark_from = 0; + while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { + xref_marks.push(p); + mark_from = p + 1; + } + for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if find(&data, b"/XRef", body, body.saturating_add(1024)).is_some() + let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); + if may_be_object_stream(&data, body, end) { + candidates.insert(num); + } else { + candidates.remove(&num); + } + // `/XRef` within the 1024 bytes after the header (the first one that is there). + let first = xref_marks.partition_point(|&p| p < body); + if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) && let Ok(at) = usize::try_from(offset) { xref_streams.push(at); @@ -2068,7 +2087,8 @@ impl Document { self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); // Objects inside object streams. for num in objstms { - if let Ok(o) = self.try_get(num) + if candidates.contains(&num) + && let Ok(o) = self.try_get(num) && let Object::Stream(s) = &*o && s.dict.name(b"Type") == Some(b"ObjStm") && let Ok(stm) = self.objstm(num) @@ -2165,6 +2185,24 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } +/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary +/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's +/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. +fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { + let Some(body) = data.get(from..to.min(data.len())) else { return false }; + let mut dict = body; + let mut at = 0; + while let Some(p) = find(body, b"stream", at, body.len()) { + at = p + 6; + let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); + if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { + dict = &body[..p]; + break; + } + } + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() +} + /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position /// after `obj`). fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { @@ -2241,7 +2279,33 @@ pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option< if from >= to || needle.is_empty() || to - from < needle.len() { return None; } - hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + let hay = &hay[from..to]; + // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. + const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; + let (k, rare) = + needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); + let mut i = k; + while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { + let start = i + p - k; + if hay.get(start..start + needle.len()) == Some(needle) { + return Some(start + from); + } + i += p + 1; + } + None +} + +/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which +/// the compiler turns into vector compares; only a block that holds the byte is searched. +fn position_of(hay: &[u8], byte: u8) -> Option { + const BLOCK: usize = 32; + let (blocks, rest) = hay.as_chunks::(); + for (i, block) in blocks.iter().enumerate() { + if block.iter().fold(false, |any, &b| any | (b == byte)) { + return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); + } + } + rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) } fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..07883e7b5a 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod reconstruct; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs new file mode 100644 index 0000000000..5f61867b1b --- /dev/null +++ b/crates/cos/tests/it/reconstruct.rs @@ -0,0 +1,57 @@ +//! Rebuilding the cross-reference index of a damaged file finds the objects inside object +//! streams however the stream's type is written. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; + +/// A file whose `startxref` points nowhere, with three object streams (type written plainly, as +/// `/Obj#53tm`, and after a string that contains the word `stream`), and one ordinary stream +/// whose data mentions `ObjStm` and `#`. +fn damaged() -> Vec { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut add = |n: u32, body: &[u8]| { + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let objstm = |dict_prefix: &str, members: &[(u32, &str)]| { + let mut header = String::new(); + let mut body = String::new(); + for (n, text) in members { + header.push_str(&format!("{n} {} ", body.len())); + body.push_str(text); + body.push('\n'); + } + let mut b = + format!("<< {dict_prefix} /N {} /First {} /Length {} >>\nstream\n", members.len(), header.len(), header.len() + body.len()).into_bytes(); + b.extend_from_slice(header.as_bytes()); + b.extend_from_slice(body.as_bytes()); + b.extend_from_slice(b"\nendstream"); + b + }; + add(1, b"<< /Type /Catalog /Pages 2 0 R >>"); + add(2, b"<< /Type /Pages /Kids [] /Count 0 >>"); + add(3, &objstm("/Type /ObjStm", &[(10, "<< /From 3 >>"), (11, "[3]")])); + add(4, &objstm("/Type /Obj#53tm", &[(12, "<< /From 4 >>")])); + add(5, &objstm("/Note (a stream) /Type /ObjStm", &[(13, "<< /From 5 >>")])); + let data = b"/Type /ObjStm # 14 0 << /From 6 >>"; + let mut plain = format!("<< /Length {} >>\nstream\n", data.len()).into_bytes(); + plain.extend_from_slice(data); + plain.extend_from_slice(b"\nendstream"); + add(6, &plain); + out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); + out +} + +#[test] +fn objects_in_object_streams_are_found_however_the_type_is_written() { + let doc = Document::open(Arc::new(damaged())).expect("opens"); + assert!(doc.reconstructed()); + for n in [10, 11, 12, 13] { + let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); + assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); + } + // The data of an ordinary stream is not an object stream, whatever it says. + assert!(!doc.object_numbers().contains(&14)); +} From a7baf8a2af97eeed166325a990bb116cdf3e1d9b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:59:12 +0100 Subject: [PATCH 16/79] perf(cos): write_full keeps the objects it parsed while numbering write_full parsed each object once to number it and again, after the 128-object reader cache had dropped it, to write it. Keep the parsed object from the traversal and write that, within a budget of 64 MiB of owned bytes and 2M tree nodes; past it the object is read again as before, which keeps the memory bound and the "read differently while saving" check. Fixes #4869 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 2 +- crates/cos/src/writer.rs | 89 +++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..9dca286d7e 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -530,7 +530,7 @@ const OWNED_BYTES_DEPTH: usize = 256; /// Bytes `object` holds of its own: string and stream data that is not a view into `data` /// (the input file). An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured /// and counts as too large to keep. -fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { +pub(crate) fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { if depth > OWNED_BYTES_DEPTH { return usize::MAX; } diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..1ba4574318 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -15,7 +15,7 @@ use std::io::Write; use std::sync::Arc; use crate::CosError; -use crate::document::{Document, ObjectReader, XrefEntry}; +use crate::document::{Document, ObjectReader, XrefEntry, owned_bytes}; use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; /// Serialize one object. Deterministic: same object → same bytes. @@ -280,6 +280,10 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. let mut order: Vec<(ObjRef, bool)> = Vec::new(); + // The parsed object of each entry in `order`, kept for the write pass while the budget + // allows it. Past the budget the write pass reads the object again. + let mut kept: Vec>> = Vec::new(); + let mut kept_budget = KeptObjects::default(); let mut queue: VecDeque = VecDeque::new(); let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); for key in [&b"Root"[..], b"Info", b"Encrypt"] { @@ -305,6 +309,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro map.insert(r, output_number(order.len())?); order.push((r, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); + kept.push(kept_budget.admit(&o, reader.document.bytes()).then_some(o)); } let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; @@ -329,14 +334,14 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut rows: BTreeMap = BTreeMap::new(); // Keep only references, not renamed copies of the entire object graph. Standalone // objects still precede object streams, preserving the established output order. - let mut packed: Vec<(u32, ObjRef)> = Vec::new(); + let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); for (i, (r, is_stream)) in order.iter().enumerate() { let num = output_number(i)?; let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(r) && root_outside != Some(*r); if in_stream { - packed.push((num, *r)); + packed.push((num, *r, kept[i].take())); } else { - let o = renumber(&*reread(&mut reader, *r, *is_stream)?, &map); + let o = renumber(&*written_object(&mut reader, *r, *is_stream, kept[i].take())?, &map); rows.insert(num, Row::InFile(out.len() as u64, 0)); write_indirect(num, 0, &prepared(&reader.document, r.num, num, 0, &o), &mut out); } @@ -345,9 +350,9 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro // individually: the whole stream is (§7.5.7, §7.6.2). let mut next = output_number(order.len())?; let mut batch = ObjectStreamBatch::default(); - for (num, reference) in packed { + for (num, reference, parsed) in packed { let mut encoded = Vec::new(); - serialize(&renumber(&*reread(&mut reader, reference, false)?, &map), &mut encoded); + serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; } @@ -400,6 +405,56 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro Ok(out) } +/// What `write_full` keeps of the objects it parsed while numbering them: owned bytes (see +/// [`owned_bytes`]; streams read from the file are views and cost only their dictionary) and +/// tree nodes. Beyond either limit objects are read again when written, which bounds the memory +/// of a save of a huge document. +#[derive(Default)] +struct KeptObjects { + bytes: usize, + nodes: usize, +} + +impl KeptObjects { + const BYTES: usize = 64 * 1024 * 1024; + const NODES: usize = 2_000_000; + + /// Whether `object` still fits the budget; if so it is counted against it. + fn admit(&mut self, object: &Object, data: &Arc>) -> bool { + let bytes = self.bytes.saturating_add(owned_bytes(object, data, 0)); + let nodes = self.nodes.saturating_add(node_count(object, 0)); + if bytes > Self::BYTES || nodes > Self::NODES { + return false; + } + (self.bytes, self.nodes) = (bytes, nodes); + true + } +} + +/// Values in `object`'s tree, counting the object itself. Past the parser's nesting limit +/// the count is `usize::MAX`, so nothing that deep is kept. +fn node_count(object: &Object, depth: usize) -> usize { + if depth > 256 { + return usize::MAX; + } + let children = match object { + Object::Stream(s) => s.dict.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + Object::Array(a) => a.iter().fold(0usize, |n, v| n.saturating_add(node_count(v, depth + 1))), + Object::Dict(d) => d.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + _ => 0, + }; + children.saturating_add(1) +} + +/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read +/// again. +fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { + match kept { + Some(o) => Ok(o), + None => reread(reader, r, is_stream), + } +} + /// Object `r` read again for writing. The traversal read it once and only kept its number, so /// a second read that fails, finds nothing or finds a different kind of object must fail the /// save rather than write `null` in its place. @@ -674,6 +729,28 @@ mod tests { Lexer::new(&out, 0).object().expect("parses") } + #[test] + fn kept_objects_stop_at_their_byte_and_node_budget() { + let data = Arc::new(Vec::new()); + let string = Object::String(PdfString::literal(vec![b'x'; 1000])); + let array = Object::Array(vec![Object::Int(1), Object::Null]); + let mut kept = KeptObjects::default(); + assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); + assert_eq!((kept.bytes, kept.nodes), (1000, 4)); + let mut full = KeptObjects { bytes: KeptObjects::BYTES - 999, nodes: 0 }; + assert!(!full.admit(&string, &data), "one byte over the byte budget"); + assert_eq!((full.bytes, full.nodes), (KeptObjects::BYTES - 999, 0), "a refused object is not counted"); + assert!(full.admit(&Object::Int(1), &data)); + let mut full = KeptObjects { bytes: 0, nodes: KeptObjects::NODES - 2 }; + assert!(!full.admit(&array, &data), "a node past the node budget"); + assert!(full.admit(&Object::Int(1), &data)); + let mut deep = Object::Null; + for _ in 0..300 { + deep = Object::Array(vec![deep]); + } + assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); + } + #[test] fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { let mut doc = Document::new_empty(); From 929d350f1dfacb6da308569f2a7de8d412a4049a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:00:43 +0100 Subject: [PATCH 17/79] perf(cos): write hex strings, integers and references without the formatter serialize wrote every byte of a hex string, every integer and every object number through write!. A table lookup and a small digit loop give the same bytes in a fraction of the instructions. The octal escape in literal strings and the #xx escape in names use the same helpers. Fixes #4870 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/writer.rs | 53 ++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/serialize_format.rs | 110 ++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 crates/cos/tests/it/serialize_format.rs diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..b8a9c53e9c 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -23,9 +23,7 @@ pub fn serialize(o: &Object, out: &mut Vec) { match o { Object::Null => out.extend_from_slice(b"null"), Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), - Object::Int(i) => { - let _ = write!(out, "{i}"); - } + Object::Int(i) => write_int(*i, out), Object::Real(r) => write_real(*r, out), Object::String(s) => write_string(s, out), Object::Name(n) => write_name(n, out), @@ -42,11 +40,46 @@ pub fn serialize(o: &Object, out: &mut Vec) { Object::Dict(d) => write_dict(d, out), Object::Stream(s) => write_stream(s, out), Object::Ref(r) => { - let _ = write!(out, "{} {} R", r.num, r.generation); + write_digits(u64::from(r.num), out); + out.push(b' '); + write_digits(u64::from(r.generation), out); + out.extend_from_slice(b" R"); } } } +/// Upper-case hexadecimal digits. +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. +fn write_digits(mut v: u64, out: &mut Vec) { + // `u64::MAX` has 20 digits, so `at` never goes below 0. + let mut buf = [0u8; 20]; + let mut at = buf.len(); + loop { + at -= 1; + buf[at] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { + break; + } + } + out.extend_from_slice(&buf[at..]); +} + +/// A signed integer in decimal, as `write!(out, "{i}")` writes it. +fn write_int(i: i64, out: &mut Vec) { + if i < 0 { + out.push(b'-'); + } + write_digits(i.unsigned_abs(), out); +} + +/// `b` as two upper-case hexadecimal digits. +fn hex_pair(b: u8) -> [u8; 2] { + [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] +} + fn write_real(r: f64, out: &mut Vec) { if !r.is_finite() { out.push(b'0'); @@ -55,7 +88,7 @@ fn write_real(r: f64, out: &mut Vec) { // A whole number is written as an integer only within the integer range Annex C gives // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { - let _ = write!(out, "{}", r as i32); + write_int(i64::from(r as i32), out); return; } // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the @@ -76,7 +109,8 @@ fn write_name(n: &[u8], out: &mut Vec) { if regular { out.push(b); } else { - let _ = write!(out, "#{b:02X}"); + out.push(b'#'); + out.extend_from_slice(&hex_pair(b)); } } } @@ -84,9 +118,10 @@ fn write_name(n: &[u8], out: &mut Vec) { fn write_string(s: &PdfString, out: &mut Vec) { let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); if s.hex || binary * 4 > s.bytes.len().max(1) { + out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); out.push(b'<'); - for b in &s.bytes { - let _ = write!(out, "{b:02X}"); + for &b in &s.bytes { + out.extend_from_slice(&hex_pair(b)); } out.push(b'>'); return; @@ -105,7 +140,7 @@ fn write_string(s: &PdfString, out: &mut Vec) { 0x0C => out.extend_from_slice(b"\\f"), b' '..=b'~' => out.push(b), _ => { - let _ = write!(out, "\\{b:03o}"); + out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); } } } diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..4b2fe32560 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod serialize_format; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs new file mode 100644 index 0000000000..19781a802f --- /dev/null +++ b/crates/cos/tests/it/serialize_format.rs @@ -0,0 +1,110 @@ +//! `serialize` writes integers, references, hexadecimal strings, octal escapes and `#xx` name +//! escapes with its own digit loops. They must give the bytes the standard formatter gives. + +use pdfcraft_cos::{ObjRef, Object, PdfString, serialize}; +use proptest::prelude::*; + +fn bytes_of(o: &Object) -> Vec { + let mut v = Vec::new(); + serialize(o, &mut v); + v +} + +#[test] +fn integer_extremes_and_digit_boundaries() { + let mut cases = vec![0i64, 1, -1, 9, 10, 99, 100, i64::MAX, i64::MIN, i64::MIN + 1, i64::from(i32::MAX), i64::from(i32::MIN)]; + let mut p = 1i64; + while let Some(next) = p.checked_mul(10) { + cases.extend([next - 1, next, next + 1, -next]); + p = next; + } + for i in cases { + assert_eq!(bytes_of(&Object::Int(i)), i.to_string().into_bytes(), "{i}"); + } +} + +#[test] +fn reference_extremes() { + for (n, g) in [(0, 0), (1, 0), (u32::MAX, u16::MAX), (10, 65535), (4_000_000_000, 7)] { + assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } +} + +#[test] +fn every_byte_value_in_hex_strings_octal_escapes_and_names() { + let all: Vec = (0..=255).collect(); + let hex: String = all.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(bytes_of(&Object::String(PdfString { bytes: all.clone(), hex: true })), format!("<{hex}>").into_bytes()); + // A literal string with a few control bytes escapes them in octal. + let lit = bytes_of(&Object::String(PdfString { bytes: b"abcdefghijkl\x01\x7f\xffz".to_vec(), hex: false })); + assert_eq!(lit, b"(abcdefghijkl\\001\\177\\377z)"); + let name = bytes_of(&Object::Name(all.clone())); + let mut want = b"/".to_vec(); + for &b in &all { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + want.push(b); + } else { + want.extend_from_slice(format!("#{b:02X}").as_bytes()); + } + } + assert_eq!(name, want); +} + +proptest! { + #[test] + fn integers_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Int(i)), format!("{i}").into_bytes()); + } + + #[test] + fn whole_reals_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); + } + + #[test] + fn references_match_the_formatter(n in any::(), g in any::()) { + prop_assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } + + #[test] + fn hex_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + let mut want = String::from("<"); + for b in &bytes { + want.push_str(&format!("{b:02X}")); + } + want.push('>'); + prop_assert_eq!(bytes_of(&Object::String(PdfString { bytes, hex: true })), want.into_bytes()); + } + + #[test] + fn binary_literal_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + // Mostly-binary input is written as hex; mostly-text input with escapes is a literal. + let got = bytes_of(&Object::String(PdfString { bytes: bytes.clone(), hex: false })); + let binary = bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + let mut want = Vec::new(); + if binary * 4 > bytes.len().max(1) { + want.push(b'<'); + for b in &bytes { + want.extend_from_slice(format!("{b:02X}").as_bytes()); + } + want.push(b'>'); + } else { + want.push(b'('); + for &b in &bytes { + match b { + b'(' | b')' | b'\\' => want.extend_from_slice(&[b'\\', b]), + b'\n' => want.extend_from_slice(b"\\n"), + b'\r' => want.extend_from_slice(b"\\r"), + b'\t' => want.extend_from_slice(b"\\t"), + 0x08 => want.extend_from_slice(b"\\b"), + 0x0C => want.extend_from_slice(b"\\f"), + b' '..=b'~' => want.push(b), + _ => want.extend_from_slice(format!("\\{b:03o}").as_bytes()), + } + } + want.push(b')'); + } + prop_assert_eq!(got, want); + } +} From d04fad3acd79906445a47e46496fb09fbfffdc11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:45:53 +0100 Subject: [PATCH 18/79] fix(cos): recount the scan reader's cache when a read replaces entries The running total trusted cache length and pointer equality to mean that a read only added the requested object. A read whose nested loads made the editor cache evict (it then keeps only the newcomer) leaves the length unchanged, so a large string stayed cached past the 8 MiB budget. Count the inserts a read makes instead; anything but a hit or exactly one insert measures the whole cache again. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..b7fe54bc19 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -21,6 +21,13 @@ thread_local! { static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); } +thread_local! { + /// Objects this thread has put into a document's parsed-object cache. A reader compares + /// it before and after a read: a read that inserted exactly one object cannot have + /// evicted or replaced anything else, whatever the cache length says. + static CACHE_INSERTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream /// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is /// bounded as well. @@ -460,7 +467,9 @@ impl ObjectReader { } pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { + let inserts_before = CACHE_INSERTS.with(|c| c.get()); let object = self.document.try_get(reference.num); + let inserted = CACHE_INSERTS.with(|c| c.get()).wrapping_sub(inserts_before); // A missed eviction here only costs a re-parse, so never fail the read on a poisoned // lock. The editing-cache evictors propagate instead; they guard user edits. if let Ok(mut streams) = self.document.objstms.lock() { @@ -492,13 +501,16 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - // The common read parsed exactly this object and nothing else (a hit changes - // nothing): add its size to the running total. Anything else, such as a failed - // load, a /Length that cached further objects or a cache changed from outside, - // measures the whole cache again. - let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + // A hit inserted nothing; a read that inserted exactly this object (no nested + // /Length or object stream load) added only it. Cache length and pointer equality + // alone cannot tell that from a read that evicted and replaced entries (a large + // nested object makes the editor cache keep only the newcomer), so count the + // inserts. Anything else, such as a failed load, a /Length that cached further + // objects or a cache changed from outside, measures the whole cache again. + let grew_by_this_object = inserted == 1 + && cache.len() == self.counted_objects.saturating_add(1) && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); - if object.is_ok() && cache.len() == self.counted_objects { + if object.is_ok() && inserted == 0 && cache.len() == self.counted_objects { // Unchanged: a hit, or an object held by the overlay. } else if let (true, Ok(read)) = (grew_by_this_object, &object) { self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); @@ -1075,6 +1087,7 @@ impl Document { // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); + CACHE_INSERTS.with(|c| c.set(c.get().wrapping_add(1))); self.evict_editor_cache_if_over_budget(num, &obj)?; Ok(obj) } @@ -3046,6 +3059,22 @@ endobj } } + #[test] + fn temporary_reader_recounts_when_a_read_replaces_cache_entries() { + // One cached object, then a read of a string over the editor cache's byte budget: the + // editor eviction keeps only the newcomer, so the cache length stays at one. The + // string is data of its own and must not stay cached past the reader's budget. + let big = format!("({})", "a".repeat(Document::EDITOR_CACHE_BYTES + 1)); + let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Small 1 >>", &big], "/Root 1 0 R"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let mut reader = doc.object_reader(); + let _ = reader.get(ObjRef::new(3, 0)); + assert_eq!((reader.document.cache.lock().unwrap().len(), reader.counted_objects), (1, 1)); + assert!(matches!(&*reader.get(ObjRef::new(4, 0)), Object::String(_))); + assert!(reader.document.cache.lock().unwrap().is_empty(), "a replaced entry must be counted: the string is over budget"); + assert_eq!((reader.counted_objects, reader.counted_bytes), (0, 0)); + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From 8bdd2d37504689fbff50eb198740d384a6a2c054 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:46:44 +0100 Subject: [PATCH 19/79] fix(cos): read the dictionary's syntax before the object-stream pre-check The pre-check cut the dictionary at the next object header or at a '>>' + 'stream' byte pattern, so text inside a string could hide a later /Type /ObjStm. It now reads the dictionary as the parser does (strings, hex strings, comments, nesting) and treats an object whose end is not certain as a candidate, so the reconstructed object set stays the same. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 173 ++++++++++++++++++++++++++--- crates/cos/tests/it/reconstruct.rs | 7 +- 2 files changed, 162 insertions(+), 18 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 4ad24a11b1..ffc4f6be72 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2063,11 +2063,10 @@ impl Document { xref_marks.push(p); mark_from = p + 1; } - for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { + for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); - if may_be_object_stream(&data, body, end) { + if may_be_object_stream(&data, body) { candidates.insert(num); } else { candidates.remove(&num); @@ -2185,22 +2184,107 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } -/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary -/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's -/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. -fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { - let Some(body) = data.get(from..to.min(data.len())) else { return false }; - let mut dict = body; - let mut at = 0; - while let Some(p) = find(body, b"stream", at, body.len()) { - at = p + 6; - let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); - if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { - dict = &body[..p]; - break; +/// Longest dictionary [`may_be_object_stream`] reads before it gives up and says "maybe". +const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; + +/// Where the dictionary that starts at `data[from..]` (after `N G obj`) ends, reading it as the +/// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the +/// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end +/// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must +/// then treat the object as a possible object stream. +fn dictionary_end(data: &[u8], from: usize) -> Option> { + let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT)); + let mut i = from; + while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { + if data[i] == b'%' { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } else { + i += 1; + } + } + if data.get(i..i + 2) != Some(b"<<") { + return if i < limit { Some(None) } else { None }; + } + i += 2; + let (mut dicts, mut arrays) = (1usize, 0usize); + while i < limit { + match data[i] { + b'(' => { + let mut depth = 1usize; + i += 1; + while i < limit && depth > 0 { + match data[i] { + b'\\' => i += 1, + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + i += 1; + } + if depth > 0 { + return None; + } + } + b'<' if data.get(i + 1) == Some(&b'<') => { + dicts += 1; + i += 2; + } + b'<' => { + i += 1; + while i < limit && data[i] != b'>' { + if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { + return None; + } + i += 1; + } + i += 1; + } + b'>' if data.get(i + 1) == Some(&b'>') => { + if arrays > 0 { + return None; + } + dicts -= 1; + i += 2; + if dicts == 0 { + return Some(Some(i)); + } + } + b'%' => { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } + b'[' => { + arrays += 1; + i += 1; + } + b']' => { + arrays = arrays.saturating_sub(1); + i += 1; + } + _ => i += 1, + } + } + None +} + +/// Whether the object whose body starts at `data[from..]` could be an object stream: its +/// dictionary holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in +/// `/Obj#53tm`). Only the dictionary is searched, not the data of a stream. Where the +/// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a +/// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a +/// candidate, so a doubtful object is parsed as it always was. +fn may_be_object_stream(data: &[u8], from: usize) -> bool { + match dictionary_end(data, from) { + Some(None) => false, + None => true, + Some(Some(end)) => { + let dict = &data[from..end]; + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } } - dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position @@ -3584,6 +3668,61 @@ endobj /// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used /// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after /// editing one Info field dropped every other. + /// Whether parsing `body` as an object gives a stream whose /Type is /ObjStm. + fn parses_as_object_stream(file: &[u8]) -> bool { + matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) + } + + #[test] + fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { + // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a + // string that comes before the /Type. + for dict in [ + "<< /Note (9 0 obj\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (>>stream\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (a (nested\n9 0 obj\nstream) \\) )) /Type /ObjStm >>", + "<< /Note <3e3e> /Other << /A (>>\nstream\n) >> /Type /ObjStm >>", + ] { + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); + assert!(may_be_object_stream(&file, 7), "{dict}"); + } + // Ordinary streams stay out; their data is never read. + let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; + assert!(!may_be_object_stream(plain, 7)); + assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7)); + // Unterminated: not certain, so a candidate. + assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7)); + // Random dictionaries built from tricky pieces: whatever parses as an object stream is + // a candidate. + let pieces: [&str; 22] = [ + "/Type /ObjStm", "/Type/ObjStm", "/Type /Obj#53tm", "/T#79pe /ObjStm", "(", ")", "\\", "(>>stream\n)", "(9 0 obj\n)", "<3e3e>", "<", ">", ">>", + "<<", "[", "]", "% c\n", "% >>\n", "/N 0", " ", "\n", "", + ]; + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut hits = 0; + for _ in 0..200_000 { + let mut dict = String::from("<<"); + for _ in 0..(next() % 9) { + dict.push_str(pieces[(next() % pieces.len() as u64) as usize]); + dict.push(' '); + } + dict.push_str(if next() % 8 == 0 { "" } else { ">>" }); + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + if parses_as_object_stream(&file) { + hits += 1; + assert!(may_be_object_stream(&file, 7), "rejected an object stream: {dict:?}"); + } + } + assert!(hits > 1000, "the generator must produce object streams ({hits})"); + } + #[test] fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { let table = build( diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs index 5f61867b1b..4611c0d17e 100644 --- a/crates/cos/tests/it/reconstruct.rs +++ b/crates/cos/tests/it/reconstruct.rs @@ -40,6 +40,11 @@ fn damaged() -> Vec { plain.extend_from_slice(data); plain.extend_from_slice(b"\nendstream"); add(6, &plain); + // Strings before the type that look like an object header, or like the end of the + // dictionary and the `stream` keyword, and a hex string and a comment. + add(7, &objstm("/Note (\n99 0 obj\n) /Type /ObjStm", &[(15, "<< /From 7 >>")])); + add(8, &objstm("/Note (>>stream\n) /Type /ObjStm", &[(16, "<< /From 8 >>")])); + add(9, &objstm("/Note <3e3e> /Other << /A (a (b) \\) >>\nstream\n) >> % >>\nstream\n\n/Type /ObjStm", &[(17, "<< /From 9 >>")])); out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); out } @@ -48,7 +53,7 @@ fn damaged() -> Vec { fn objects_in_object_streams_are_found_however_the_type_is_written() { let doc = Document::open(Arc::new(damaged())).expect("opens"); assert!(doc.reconstructed()); - for n in [10, 11, 12, 13] { + for n in [10, 11, 12, 13, 15, 16, 17] { let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); } From 1a7663cb48d1ba6740a50c3a3a23b89f27e7a5f9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 31 Oct 2026 13:41:56 +0100 Subject: [PATCH 20/79] fix(ui): select the bookmark title when the rename editor opens (#5529) The inline rename field asked for focus without setting a selection, so the caret landed at the end of the default text and typing appended to it. The field now selects the whole title on the frame it takes focus, so typing replaces the old title at once, for both new bookmarks and rename. Later frames leave the user's own caret and selection alone. Tests: three unit tests on rename_field (first focus, multibyte title, caret kept across frames) and two kittest tests driving the panel like a user (typing replaces the title; the caret survives a collapse). Co-Authored-By: Zoo --- crates/ui-egui/src/panels/bookmarks.rs | 87 ++++++++++++++++++++++++-- crates/ui-egui/tests/it/bookmarks.rs | 34 ++++++++++ parity/acrobat-features.toml | 4 +- 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/crates/ui-egui/src/panels/bookmarks.rs b/crates/ui-egui/src/panels/bookmarks.rs index ccc45d4eb0..2a92a4c227 100644 --- a/crates/ui-egui/src/panels/bookmarks.rs +++ b/crates/ui-egui/src/panels/bookmarks.rs @@ -88,6 +88,24 @@ pub(crate) fn ellipsized_prefix(text: &str, mut fits: impl FnMut(&str) -> bool) } } +/// The inline rename field (#5529). On the frame it takes focus it selects the whole title, so +/// typing replaces the old text at once; later frames leave the user's own caret and selection +/// alone. Same pattern as the zoom box in `chrome.rs`. +fn rename_field(ui: &mut egui::Ui, text: &mut String, id: egui::Id) -> egui::Response { + let first_frame = !ui.memory(|m| m.has_focus(id)); + let resp = ui.add(crate::widgets::line(text).desired_width(ui.available_width() - 8.0).id(id)); + if !resp.has_focus() && !resp.lost_focus() { + if first_frame { + let mut state = egui::TextEdit::load_state(ui.ctx(), id).unwrap_or_default(); + let all = egui::text::CCursorRange::two(egui::text::CCursor::new(0), egui::text::CCursor::new(text.chars().count())); + state.cursor.set_char_range(Some(all)); + state.store(ui.ctx(), id); + } + resp.request_focus(); + } + resp +} + /// How a bookmark's title is drawn: in its own style and colour (ISO 32000-2 §12.3.3, Tables 151 /// and 152), bold in the heavier face and italic slanted, its colour kept readable on the panel. /// Top-level bookmarks without a style keep the medium face. @@ -138,11 +156,7 @@ pub(super) fn outline_item( ui.horizontal(|ui| { ui.add_space(x_text); crate::widgets::begin_dialog(ui); - let edit = crate::widgets::line(text).desired_width(ui.available_width() - 8.0).id(id.with("rename")); - let resp = ui.add(edit); - if !resp.has_focus() && !resp.lost_focus() { - resp.request_focus(); - } + let resp = rename_field(ui, text, id.with("rename")); if ui.input(|i| i.key_pressed(egui::Key::Escape)) { cancel = true; } else if resp.lost_focus() { @@ -360,3 +374,66 @@ mod bookmark_filter { assert!(outline_matches(&outline, &fold("izmir")).is_empty()); } } + +#[cfg(test)] +mod rename_field { + use super::rename_field as field; + + fn cursor_range(ctx: &egui::Context, id: egui::Id) -> Option<(usize, usize)> { + egui::TextEdit::load_state(ctx, id) + .and_then(|s| s.cursor.char_range()) + .map(|r| (r.primary.index.0.min(r.secondary.index.0), r.primary.index.0.max(r.secondary.index.0))) + } + + /// #5529: the rename field opens with the whole title selected, so typing replaces it. + #[test] + fn first_focus_selects_the_whole_title() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-test"); + let mut text = String::from("Untitled"); + for _ in 0..2 { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + } + assert_eq!(cursor_range(&ctx, id), Some((0, 8))); + assert!(ctx.memory(|m| m.has_focus(id)), "the field takes focus"); + } + + /// A Unicode title is counted in characters, not bytes. + #[test] + fn a_multibyte_title_is_selected_in_characters() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-unicode"); + let mut text = String::from("図面（一階平面図）"); + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + assert_eq!(cursor_range(&ctx, id), Some((0, 9))); + } + + /// A later frame must not reset the selection the user made; a caret stays a caret. + #[test] + fn a_later_frame_keeps_the_users_selection() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-keep"); + let mut text = String::from("Chapter"); + for _ in 0..2 { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + } + // The user moves the caret to 3 with no selection. + let mut state = egui::TextEdit::load_state(&ctx, id).unwrap_or_default(); + state.cursor.set_char_range(Some(egui::text::CCursorRange::one(egui::text::CCursor::new(3)))); + state.store(&ctx, id); + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + assert_eq!(cursor_range(&ctx, id), Some((3, 3)), "the user's caret is kept"); + } +} diff --git a/crates/ui-egui/tests/it/bookmarks.rs b/crates/ui-egui/tests/it/bookmarks.rs index 1f07102a18..d5cd5efd2f 100644 --- a/crates/ui-egui/tests/it/bookmarks.rs +++ b/crates/ui-egui/tests/it/bookmarks.rs @@ -119,6 +119,40 @@ fn the_panel_keeps_its_width_while_the_pointer_moves() { assert_eq!(panel_width(&h), before, "the Bookmarks panel drifted wider"); } +/// #5529: the title is selected when the rename editor opens, so typing replaces it at once, +/// with no Ctrl+A. New bookmarks and double-click rename both get this. +#[test] +fn typing_replaces_the_title_when_the_editor_opens() { + let mut h = harness(); + h.key_press_modifiers(Modifiers::COMMAND, Key::B); + h.run_steps(3); + field(&h, "Untitled").type_text("Intro"); // the whole default is replaced + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h), [("Intro".into(), Some(0), 0)]); + + menu(&mut h, "Intro", "Rename"); + field(&h, "Intro").type_text("Introduction"); + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h)[0].0, "Introduction"); +} + +/// The selection is one-shot: after the user collapses it to a caret, later frames must not +/// reselect, so typing at that caret inserts. +#[test] +fn a_rename_editor_keeps_the_users_caret_across_frames() { + let mut h = harness(); + h.key_press_modifiers(Modifiers::COMMAND, Key::B); + h.run_steps(3); + h.key_press(Key::ArrowLeft); // collapses the opening selection to the caret at 0 + h.run_steps(2); + field(&h, "Untitled").type_text("I"); // inserts at the caret, does not replace + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h)[0].0, "IUntitled"); +} + #[test] fn escape_cancels_a_rename() { let mut h = harness(); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0b6f658dd6..d0377fc883 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -3291,8 +3291,8 @@ milestone = "M4" status = "shipped" commands = [] tools = ["bookmark_rename"] -evidence = ["crates/ui-egui/tests/it/bookmarks.rs::new_rename_reorder_indent_delete_and_undo", "crates/ui-egui/tests/it/bookmarks.rs::escape_cancels_a_rename", "crates/organize/src/tests.rs::bookmarks_add_rename_move_delete", "crates/organize/src/tests.rs::bookmark_edits_keep_unknown_keys_and_touch_few_objects"] -notes = "" +evidence = ["crates/ui-egui/tests/it/bookmarks.rs::new_rename_reorder_indent_delete_and_undo", "crates/ui-egui/tests/it/bookmarks.rs::escape_cancels_a_rename", "crates/ui-egui/tests/it/bookmarks.rs::typing_replaces_the_title_when_the_editor_opens", "crates/ui-egui/tests/it/bookmarks.rs::a_rename_editor_keeps_the_users_caret_across_frames", "crates/organize/src/tests.rs::bookmarks_add_rename_move_delete", "crates/organize/src/tests.rs::bookmark_edits_keep_unknown_keys_and_touch_few_objects"] +notes = "The inline editor opens with the whole title selected, so typing replaces it at once (#5529); a caret the user moves afterwards is kept by later frames." [[feature]] id = "organize.bookmark-delete" From 52080ac8983c616e1916beea4c5ecc4d76dd78eb Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 31 Oct 2026 13:56:51 +0100 Subject: [PATCH 21/79] M3/M4: add Crop pages to the document quick toolbar (#5330) Co-Authored-By: Codex --- ROADMAP.md | 3 ++ crates/ui-egui/src/canvas/bars.rs | 6 +++ crates/ui-egui/tests/it/pageboxes.rs | 64 ++++++++++++++++++++++++++-- docs/roadmap.md | 5 ++- parity/acrobat-features.toml | 4 +- 5 files changed, 76 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 56f6237e8d..9c098748d2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,6 @@ # PdfCraft roadmap +> **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** minor (M3/M4, #5330: Crop pages is directly available in the document quick toolbar; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: GBK rasters compared with explicit Unicode and independently selected installed faces; wrong-glyph and wrong-family negative controls fail; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: integrate main fa5f9f8c; retain CJK viewer and upstream raster regressions, restore explicit display_config imports; estimates unchanged) > **Last reviewed:** 2026-10-28 · **Last updated:** 2026-10-28 · **Change:** patch (M2, PR #4172: integrate main 5af10276; retain both roadmap histories and viewer-only CJK font policy; estimates unchanged) @@ -213,6 +214,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-31 (M3/M4, #5330):** The document quick toolbar offers Crop pages directly through the existing page.crop command, with the command’s assembly-permission check and an active-tool highlight. The existing crop, undo and headless page_set_box paths are unchanged. No toolbar customization or new shortcut. Estimates unchanged. + - **2026-10-31 (M2, PR #4172 review):** Added same-machine pixel oracles for GBK Song, Hei, Kai and FangSong, including all four subset-prefixed Founder aliases. Reference PDFs use explicit Unicode mappings and independently selected installed faces; font data and rasters remain in memory. All eight cases ran; wrong-glyph and wrong-family production mutations each failed the new assertion and were restored. Validation: 422 renderer unit tests, renderer all-target Clippy with -D warnings, formatting, layers/assets/parity and renderer WASM checks passed; existing optional-patch/WASM warnings and parity notes remain. Estimates unchanged. - **2026-10-31 (M2, PR #4172):** Integrated main fa5f9f8c, retaining both roadmap histories and raster regression groups. Restored display_config and thumbnail_config imports after the upstream explicit-import migration. Validation: 422 renderer unit tests, engine/UI all-target checks, scoped all-target Clippy with -D warnings, formatting, layers, assets, parity and renderer/engine WASM checks passed. Existing optional-patch/WASM warnings and parity notes remain; no full engine/UI runtime test rerun. Display-only font policy and estimates unchanged. diff --git a/crates/ui-egui/src/canvas/bars.rs b/crates/ui-egui/src/canvas/bars.rs index 2472366672..7c2ef781d8 100644 --- a/crates/ui-egui/src/canvas/bars.rs +++ b/crates/ui-egui/src/canvas/bars.rs @@ -296,6 +296,12 @@ pub(super) fn quick_bar(app: &mut PdfCraftApp, area: Rect, ui: &mut egui::Ui) { if icons::button(ui, "hand", 32.0, app.quick_tool == QuickTool::Hand, tl!("Hand (H)")).clicked() { app.quick_tool = QuickTool::Hand; } + let may_crop = pdfcraft_engine::commands::command("page.crop").is_some_and(|spec| app.command_enabled(spec)); + ui.add_enabled_ui(may_crop, |ui| { + if icons::button(ui, "crop", 32.0, app.quick_tool == QuickTool::Crop, tl!("Crop pages")).clicked() { + app.execute("page.crop"); + } + }); // Comment ▸, Highlight ▸, Draw ▸ (Acrobat's comment toolbar groups). Clicking a // group selects its last-used tool; clicking it again opens the flyout. for g in 0..comments::GROUPS.len() { diff --git a/crates/ui-egui/tests/it/pageboxes.rs b/crates/ui-egui/tests/it/pageboxes.rs index 2e2829e05b..c54b4f01ef 100644 --- a/crates/ui-egui/tests/it/pageboxes.rs +++ b/crates/ui-egui/tests/it/pageboxes.rs @@ -1,7 +1,7 @@ //! Crop tool, Set Page Boxes and Duplicate pages in the real shell (egui_kittest). use egui_kittest::Harness; -use egui_kittest::kittest::Queryable; +use egui_kittest::kittest::{NodeT, Queryable}; use pdfcraft_ui_egui::{Dialog, PdfCraftApp, QuickTool}; const FIXTURE: &[u8] = b"%PDF-1.7 @@ -37,6 +37,64 @@ fn the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select() { assert!(h.state_mut().execute("page.crop")); assert_eq!(h.state().quick_tool, QuickTool::Crop); h.run_steps(2); + crop_middle_half(&mut h); +} + +#[test] +fn quick_toolbar_crop_crops_the_page_and_undo_restores_it() { + let mut h = harness(); + h.get_by_label("Crop pages").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Crop); + assert!(!h.state().views[0].organize); + crop_middle_half(&mut h); + assert!(h.state_mut().execute("edit.undo")); + h.run_steps(3); + assert_eq!(size(&h, 0), (300.0, 400.0)); +} + +#[test] +fn quick_toolbar_crop_is_disabled_when_page_assembly_is_forbidden() { + let mut doc = pdfcraft_cos::Document::open(std::sync::Arc::new(FIXTURE.to_vec())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0b0100, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + let bytes = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(move |_| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("restricted.pdf", None, bytes).unwrap(); + app.set_option("left", "closed").unwrap(); + app + }); + h.run_steps(6); + assert!(h.get_by_label("Crop pages").accesskit_node().is_disabled()); + assert!(!h.state_mut().execute("page.crop")); + assert_eq!(h.state().quick_tool, QuickTool::Select); + assert_eq!(size(&h, 0), (300.0, 400.0)); +} + +#[test] +fn quick_toolbar_crop_is_hidden_on_home_and_in_read_mode() { + let mut h = harness(); + h.state_mut().execute("view.read_mode"); + h.run_steps(3); + assert!(h.query_by_label("Crop pages").is_none()); + h.state_mut().execute("view.read_mode"); + h.run_steps(3); + h.get_by_label("Crop pages"); + h.state_mut().active = None; + h.run_steps(3); + assert!(h.query_by_label("Crop pages").is_none()); +} + +fn crop_middle_half(h: &mut Harness<'static, PdfCraftApp>) { let r = h.state().views[0].page_screen_rect(0).unwrap(); // The middle half of the page in both directions. let (a, b) = (r.min + r.size() * 0.25, r.min + r.size() * 0.75); @@ -50,9 +108,9 @@ fn the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select() { } h.drop_at(b); h.run_steps(4); - let (w, hh) = size(&h, 0); + let (w, hh) = size(h, 0); assert!((w - 150.0).abs() < 2.0 && (hh - 200.0).abs() < 2.0, "{w}×{hh}"); - assert_eq!(size(&h, 1), (300.0, 400.0)); + assert_eq!(size(h, 1), (300.0, 400.0)); assert_eq!(h.state().quick_tool, QuickTool::Select); assert_eq!(h.state().session.get(h.state().views[0].id).unwrap().can_undo(), Some("Crop page")); } diff --git a/docs/roadmap.md b/docs/roadmap.md index 0f1fb6f4d5..a09e597a36 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,6 @@ # PdfCraft roadmap: milestones and what's next +> **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** minor (M3/M4, #5330: Crop pages is directly available in the document quick toolbar; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: GBK rasters compared with explicit Unicode and independently selected installed faces; wrong-glyph and wrong-family negative controls fail; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: integrate main fa5f9f8c; retain CJK viewer and upstream raster regressions, restore explicit display_config imports; estimates unchanged) > **Last reviewed:** 2026-10-28 · **Last updated:** 2026-10-28 · **Change:** patch (M2, PR #4172: integrate main 5af10276; retain both roadmap histories and viewer-only CJK font policy; estimates unchanged) @@ -194,7 +195,7 @@ total matches the parity estimate. | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. Page marks, added content and print output retain small page-unit transform coefficients when serialized. Forms without Resources use page Properties for ActualText and layer visibility; own dictionaries stop fallback. Oversized missing inline-image samples, including header and terminator overhead, no longer pin the content window (#3107 / PR #3438); normal large streamed images retain deterministic sample boundaries. Radial luminosity fades have synthetic coverage for the existing #2132 mask guard, including per-pixel references, antialiased clips, previews, scales and tiles; no additional production fix was needed. Dotted TOC leaders no longer inflate letter tracking or hide word spaces (#3086); phrase search and selection are covered. Short tracked ellipses retain their gap samples and remain searchable. Missing spacing glyphs in non-embedded Japanese CID substitutes render blank without changing PDF advances or Unicode (#3601). The desktop viewer can use installed CJK fonts, including Windows GBK Song/Hei/Kai/FangSong and subset-prefixed Founder names; Combine thumbnails and accessibility figure previews use the same display-only fallback, without changing export or print font policy (#234, continuation of #1567, review #4172). Print previews exclude installed faces and isolate their font cache; GBK regressions cover view/print switching for inline and pooled whole-page/tile rendering. Installed-font helpers and their regression contracts have function documentation. GBK regression rasters match explicit-Unicode references with independently selected installed faces; wrong-glyph and wrong-family negative controls fail. Integration with main fa5f9f8c retains viewer fallback across the split engine session modules, renderer tests and Combine thumbnail workers. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. The renderer keeps its one-point crop-size floor in physical points after UserUnit scaling, with regression coverage for crop origins, rotations and f32 reciprocal rounding. Existing unit policy and nearly-zero fallback are retained. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings | -| M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). #3808: the Pages inspector offers a 50–150% thumbnail slider and buttons, also available through `ui.set`. Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. #3149: page-list changes discard render demand based on old geometry; actual raster dimensions are checked against the GPU limit before upload. Synthetic crash and continued-rendering regressions cover the replacement path. Save-prompt shortcuts yield to dialogs above them; stacked link-consent UI regression covers Escape and Return. Comment hover follows individual marked lines and original annotation paint order (#3541/#3620), independently of spatial inspection sorting; synthetic rotation, differing-upper-edge overlap and unchanged-document regressions. Optional page gaps join page edges without cropping PDF margins; current-document toggle, persisted default and UI control options are covered by viewer tests. Gap changes in Organize Pages preserve document scroll position, and both comparison panes follow menu and control choices. Repeated window resizes by an edge and moves by the tab strip on GNOME/Wayland work without a click in between. | +| M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). #3808: the Pages inspector offers a 50–150% thumbnail slider and buttons, also available through `ui.set`. Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. #3149: page-list changes discard render demand based on old geometry; actual raster dimensions are checked against the GPU limit before upload. Synthetic crash and continued-rendering regressions cover the replacement path. Save-prompt shortcuts yield to dialogs above them; stacked link-consent UI regression covers Escape and Return. Comment hover follows individual marked lines and original annotation paint order (#3541/#3620), independently of spatial inspection sorting; synthetic rotation, differing-upper-edge overlap and unchanged-document regressions. Optional page gaps join page edges without cropping PDF margins; current-document toggle, persisted default and UI control options are covered by viewer tests. Gap changes in Organize Pages preserve document scroll position, and both comparison panes follow menu and control choices. Repeated window resizes by an edge and moves by the tab strip on GNOME/Wayland work without a click in between. Crop pages is directly available in the quick toolbar (#5330). | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. Wrapped revision buttons stay above Binding in Document Properties (#2739); a synthetic UI test checks layout and saved revision content. | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Later native-open replies stay queued while a modal or picker owns the foreground; regressions cover recipient ownership, completion/cancellation and one-shot wakes. | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. #1980: exact quit modifiers preserve Ctrl+Alt+Q email input; Ctrl+Q/Cmd+Q asks before discarding unsaved documents. | @@ -274,6 +275,8 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st ## Revision history +**2026-10-31 · minor:** M3/M4 #5330: direct quick-toolbar Crop pages action reuses command permissions and the existing crop tool. Estimates unchanged. + **2026-10-26 · patch:** M2/PR #4172: reconcile current main and preserve upstream Japanese serif/weight selection alongside the installed CJK viewer and preview fallback. Estimates unchanged. **2026-10-25 · minor:** M14 localization current state: Arabic catalogs stay logical, Arabic/Kurdish labels take an explicit RTL direction, and stock chrome mirrors; native/browser RTL input, IME and complex workflows remain unqualified. Native Simplified Chinese uses LXGW WenKai Lite; the web build has no Hans face. Estimates unchanged. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0b6f658dd6..bb3f8e7762 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -3195,8 +3195,8 @@ milestone = "M4" status = "shipped" commands = ["page.crop"] tools = ["page_set_box"] -evidence = ["crates/ui-egui/tests/it/pageboxes.rs::the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select"] -notes = "drag to crop; double-click opens Set Page Boxes" +evidence = ["crates/ui-egui/tests/it/pageboxes.rs::the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_crops_the_page_and_undo_restores_it", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_is_disabled_when_page_assembly_is_forbidden", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_is_hidden_on_home_and_in_read_mode"] +notes = "drag to crop; double-click opens Set Page Boxes; Crop pages is also directly available in the document quick toolbar (#5330), using the existing command availability and assembly permission check. The quick toolbar is hidden on Home and in Read/full-screen modes; arbitrary toolbar customization remains outside this feature." [[feature]] id = "organize.remove-white-margins" From 0e7e6a00205a3e44d1e16a83daaf9f258402c92a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 1 Nov 2026 13:13:29 +0100 Subject: [PATCH 22/79] Test: pick the camera panel's Crop pages button now that the quick toolbar has one Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/camera.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/camera.rs b/crates/ui-egui/tests/it/camera.rs index 99d44ec3b7..88d79d29c1 100644 --- a/crates/ui-egui/tests/it/camera.rs +++ b/crates/ui-egui/tests/it/camera.rs @@ -147,7 +147,8 @@ fn after_enhance_page_crop_and_organize_take_over() { h.get_by_label("Enhance page").click(); h.run_steps(4); assert_eq!(h.state().views[0].camera_done.map(|d| d.page), Some(0)); - h.get_by_label(button).click(); + // The quick toolbar also has a "Crop pages" button; the camera panel's pill is the first match. + h.get_all_by_label(button).next().unwrap().click(); h.run_steps(3); assert_eq!(h.state().quick_tool, then, "{button}"); assert_eq!(h.state().views[0].camera_done, None, "{button}"); From 35eaeca321597b44026f5f8a71bd0af9c9076e64 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 1 Nov 2026 13:35:14 +0100 Subject: [PATCH 23/79] ci: re-run checks Co-Authored-By: Claude Sonnet 5.5 From 10172d7a94d93f9c5030478c81bf6fdf6760e54b Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 20:36:50 +0100 Subject: [PATCH 24/79] M2.6: a Flate stage pays for what it reads The image allowance charged a Flate stage the bytes it made, and its input only as what the stage before it made. Inflating does work for every block it reads, whether or not the block makes anything: a stream of empty fixed-Huffman blocks, ten bits each, took 1,383 ns a byte to make nothing (miniz_oxide 0.9.1, release), one of a literal each 768 ns a byte, empty dynamic blocks 106, against the 3 ns a step budget.rs states. A 250-byte stream of four Flates over empty blocks took 47 s and was charged about 34 million steps, so a review of #5698 worked out about 7 hours before a 240 KB file of them was refused, through clearing, Optimize, export, compare and redaction's images and dictionaries alike. Where decoding is paid for (decode_tolerant_within, decode_within), a Flate stage now pays FLATE_READ_WORK, 128 steps, for each byte it reads, before it runs: a stage the allowance can't pay for doesn't start. Per byte read, the worst of the shapes measured comes to about 11 ns a step at that price, inside what a JBIG2 step takes. The raw retry of a zlib header that only looked valid reads the bytes again, so it pays again, makes no more than is left once it has, and is refused when it can't be paid for. Byte counts for a byte allowance (decode_counted, decode_tolerant_counted, as a page's content uses) are unchanged. With a tiny file's allowance (2^34 steps), the 250-byte stream is decoded three times in 141 s, and the fourth stops before its last stage reads the empty blocks (8.2 ns a step, release). Data an encoder makes took 0.4 to 3.2 ns a byte it made, reading included, and each byte of a file adds 64 times the price to an operation's allowance. Tests: a_flate_stage_pays_for_what_it_reads (empty blocks: 129 steps a byte, refused a step short with nothing inflated) and a_second_reading_is_paid_for; the exact costs in filters, cos and optimize tests add what Flate reads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 37 ++++--- crates/filters/README.md | 2 +- crates/filters/src/flate.rs | 51 +++++++--- crates/filters/src/lib.rs | 159 ++++++++++++++++++++++++------- crates/optimize/src/fragments.rs | 4 +- crates/optimize/src/images.rs | 4 +- 6 files changed, 191 insertions(+), 66 deletions(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 376b108be3..d3e4a71974 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -11,12 +11,14 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the /// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step -/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A +/// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks +/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to -/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to +/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -361,6 +363,11 @@ mod image_tests { Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) } + /// What a Flate stage pays to read `n` bytes, besides what it makes. + fn read(n: usize) -> u64 { + pdfcraft_filters::FLATE_READ_WORK * n as u64 + } + /// Each image spends what it took from one allowance, and those past it are refused and /// counted, whatever their filters. What can't be decoded here costs nothing, and fails /// without being one of these. @@ -407,7 +414,7 @@ mod image_tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); - let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64; + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); let mut budget = ImageBudget::with_work(2 * cost + 1); for left in [cost + 1, 1] { assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); @@ -436,17 +443,17 @@ mod image_tests { let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); // Past this image's limit: an error, with what finding out took paid and the rest left. let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &bomb, 1000).is_err()); - assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + 1000); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); // A small allowance is a limit too: it stops there, and the image is refused. - let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + 10); + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } @@ -486,7 +493,7 @@ mod image_tests { let doc = Document::open(std::sync::Arc::new(file)).unwrap(); let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); let s = stream(&doc, image); - let filters = (raw.len() + skipped.len()) as u64; + let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); assert_eq!(spent(&doc, &s), first, "the first read undoes them"); assert_eq!(spent(&doc, &s), first - filters, "kept"); @@ -501,7 +508,7 @@ mod image_tests { let image = jbig2(Some(Object::Dict(parms))); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); - assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); assert_eq!(budget.refused(), 0); } @@ -522,7 +529,7 @@ mod image_tests { dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); let pixels = u64::from(w) * u64::from(h); - let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + pixels; + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); assert_eq!(budget.work_left(), 0, "exactly that"); @@ -547,7 +554,7 @@ mod image_tests { }; let rows = vec![0xFFu8; 512 * 1024]; let flate = blank(Some("FlateDecode"), pdfcraft_filters::encode_flate(&rows)); - let cost = (flate.raw.len() + rows.len()) as u64 + 4096 * 1024; + let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); @@ -603,8 +610,8 @@ mod image_tests { budget.decode(&doc, s, 1 << 24).unwrap(); u64::MAX - budget.work_left() }; - assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64, "its bytes, and a pixel a bit of them"); - assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + 64 * 64, "its bytes, and its pixels"); + assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); + assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); for decode in [ImageBudget::decode, ImageBudget::decode_filters] { let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); @@ -621,7 +628,7 @@ mod image_tests { let samples = vec![7u8; 1000]; let raw = pdfcraft_filters::encode_flate(&samples); let chain = [(Filter::Flate, Params::default())]; - let cost = (raw.len() + samples.len()) as u64; + let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); let mut budget = ImageBudget::with_work(cost + 10); assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); assert_eq!(budget.work_left(), 10); @@ -658,7 +665,7 @@ mod image_tests { assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); let cost = u64::MAX - budget.work_left(); // The tag byte of each of its 100 rows counts, which the predictor takes off. - assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64); + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); let mut budget = ImageBudget::with_work(cost - 1); assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); diff --git a/crates/filters/README.md b/crates/filters/README.md index 90e1485177..38666e6369 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index 1c38f72a53..ffe46207b7 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -21,25 +21,48 @@ fn zlib_header(data: &[u8]) -> Option<(u8, u8)> { ok.then_some((cmf, flg)) } -pub(crate) fn decode(data: &[u8], max: usize) -> Step { +/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked +/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has +/// left for it once it has paid for the second reading. `None`, that reading can't be paid for, +/// and is [`FilterError::Spent`](crate::FilterError::Spent). +pub(crate) fn decode(data: &[u8], max: usize, again: Option) -> (Step, bool) { match zlib_header(data) { - Some((_, flg)) if flg & 0x20 != 0 => Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), + Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false), Some(_) => match inflate_raw(&data[2..], max) { // A header that only looked valid: retry the whole input as raw deflate. - Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => match inflate_raw(data, max) { - // Keep the original corruption diagnostic only if the retry is corrupt - // too. A limit error must remain fatal even for tolerant decoding. What the - // retry decoded isn't kept, but a caller charging for work has to hear of it. - Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), - retry => retry, - }, - r => r, + Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => { + let Some(again) = again else { + return (Err(Failure { error: crate::FilterError::Spent, partial: Vec::new(), discarded: 0 }), false); + }; + let step = match inflate_raw(data, again) { + // Keep the original corruption diagnostic only if the retry is corrupt + // too. A limit error must remain fatal even for tolerant decoding. What the + // retry decoded isn't kept, but a caller charging for work has to hear of it. + Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), + retry => retry, + }; + (step, true) + } + r => (r, false), }, - None => inflate_raw(data, max), + None => (inflate_raw(data, max), false), } } +#[cfg(test)] +thread_local! { + /// Bytes inflating gave back on this thread, its partial output included. + pub(crate) static INFLATED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + fn inflate_raw(input: &[u8], max: usize) -> Step { + let step = inflate(input, max); + #[cfg(test)] + INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len))); + step +} + +fn inflate(input: &[u8], max: usize) -> Step { let mut d = Decompress::new(false); let mut out: Vec = Vec::new(); // One byte beyond the limit lets us tell "exactly max" from "more than max". @@ -87,7 +110,7 @@ mod tests { use crate::FilterError; fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1 << 24).map_err(|f| f.error) + decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error) } #[test] @@ -144,7 +167,7 @@ mod tests { fn corrupt_and_truncated() { let data: Vec = (0..10_000u32).map(|i| (i % 13) as u8).collect(); let z = encode(&data); - match decode(&z[..z.len() / 2], 1 << 24) { + match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 { Err(f) => { assert!(matches!(f.error, FilterError::Corrupt { .. })); assert_eq!(&data[..f.partial.len()], &f.partial[..]); @@ -168,6 +191,6 @@ mod tests { } fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { - decode(d, max).map_err(|f| f.error) + decode(d, max, Some(max)).0.map_err(|f| f.error) } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 181f96ab66..cf10d0586b 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -188,7 +188,7 @@ pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> { /// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than /// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer. pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result, FilterError> { - run(chain, data, max_output, None, true).map(|(v, _, _)| v).map_err(|(e, _)| e) + run(chain, data, max_output, None, true, false).map(|(v, _, _)| v).map_err(|(e, _)| e) } /// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages @@ -196,27 +196,28 @@ pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Res /// bytes, on success and on failure. A stage's output counts even when a later stage or its /// predictor shrinks it. pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), true).map(|(v, _, spent)| (v, spent)) + run(chain, data, max_output, Some(max_output), true, false).map(|(v, _, spent)| (v, spent)) } /// Like [`decode`], but on corrupt data returns what was decoded before the error and /// reports `partial = true` (the behaviour viewers expect). The remaining filters of the /// chain still run on the partial data. Limit and unsupported errors are still errors. pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool), FilterError> { - run(chain, data, max_output, None, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) + run(chain, data, max_output, None, false, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) } /// [`decode_tolerant`] bounded and counted as [`decode_counted`] is. pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), false) + run(chain, data, max_output, Some(max_output), false, false) } /// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images: -/// a step for each byte of `data`, paid before anything is decoded, and one for each byte a stage -/// decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for what it did. -/// No stage decodes past what is left: a chain that `work` can't pay for spends all of it and stops -/// there, so with nothing left nothing is read. A chain that can't be decoded at all costs nothing, -/// whatever is left, so what an image's filters are is known from the error. +/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each +/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each +/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for +/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of +/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all +/// costs nothing, whatever is left, so what an image's filters are is known from the error. /// /// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`. pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { @@ -236,14 +237,15 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) }; } let left = usize::try_from(*work).unwrap_or(usize::MAX); - match run(chain, data, max_output, Some(left), strict) { + match run(chain, data, max_output, Some(left), strict, true) { Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out), // The stage stopped at the lower of the two, which took as much to find out. Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => { pay(work, (decoded as u64).saturating_add(max_output as u64))?; Err(FilterError::LimitExceeded(max_output)) } - Err((FilterError::LimitExceeded(_), _)) => { + // Past what is left, a stage stops, or a Flate stage doesn't start. + Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => { *work = 0; Err(FilterError::Spent) } @@ -255,6 +257,17 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m /// up to the output limit, so the chain length multiplies the work. pub const MAX_CHAIN: usize = 8; +/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], +/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block +/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits +/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide +/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At +/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 +/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading +/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this +/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). +pub const FLATE_READ_WORK: u64 = 1 << 7; + /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { if chain.len() > MAX_CHAIN { @@ -268,22 +281,39 @@ fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { /// The decoded data, whether it is partial and the bytes all stages decoded, or the error and /// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. +/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it +/// reads, before it runs, so a stage `total` can't pay for doesn't run +/// ([`FilterError::Spent`]). fn run( chain: &[(Filter, Params)], data: &[u8], max: usize, total: Option, strict: bool, + priced: bool, ) -> Result<(Vec, bool, usize), (FilterError, usize)> { decodable(chain).map_err(|e| (e, 0))?; let mut cur: Cow<[u8]> = Cow::Borrowed(data); let mut partial = false; let mut spent = 0usize; + let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX); for (filter, params) in chain { + let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 }; + let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent)); + if read > room { + return Err((FilterError::Spent, spent)); + } + spent = spent.saturating_add(read); #[cfg(test)] STAGE_BYTES.with(|c| c.set(c.get() + cur.len())); let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent))); - let (step, work) = decode_one(filter, params, &cur, limit); + // A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and + // makes no more than is left once it has been. + let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left)); + let (step, work, again) = decode_one(filter, params, &cur, limit, again); + if again { + spent = spent.saturating_add(read); + } spent = spent.saturating_add(work); cur = Cow::Owned(match step { Ok(v) => v, @@ -298,15 +328,21 @@ fn run( Ok((cur.into_owned(), partial, spent)) } -/// One stage's result and the bytes its decoder produced (before any predictor). -fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize) -> (Step, usize) { +/// One stage's result, the bytes its decoder produced (before any predictor), and whether it +/// read its input twice, which it does only within `again` ([`flate::decode`]). +fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option) -> (Step, usize, bool) { + let once = |(step, n): (Step, usize)| (step, n, false); match filter { - Filter::Flate => with_predictor(flate::decode(data, max), params, "FlateDecode"), - Filter::Lzw => with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode"), - Filter::AsciiHex => counted(asciihex::decode(data, max)), - Filter::Ascii85 => counted(ascii85::decode(data, max)), - Filter::RunLength => counted(runlength::decode(data, max)), - other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0), + Filter::Flate => { + let (step, twice) = flate::decode(data, max, again); + let (step, n) = with_predictor(step, params, "FlateDecode"); + (step, n, twice) + } + Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode")), + Filter::AsciiHex => once(counted(asciihex::decode(data, max))), + Filter::Ascii85 => once(counted(ascii85::decode(data, max))), + Filter::RunLength => once(counted(runlength::decode(data, max))), + other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false), } } @@ -552,7 +588,7 @@ mod tests { let data = encode_flate(&hex); let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; let whole = decode(&chain, &data, 1 << 20).unwrap(); - let cost = (data.len() + hex.len() + whole.len()) as u64; + let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64; let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole); assert_eq!(u64::MAX - work, cost); @@ -584,8 +620,11 @@ mod tests { assert_eq!(work, 0); super::STAGE_BYTES.with(|c| c.get()) }; - assert_eq!(read((bomb.len() + inner.len()) as u64 + 10), bomb.len() + inner.len(), "the second stage stops"); - assert_eq!(read(bomb.len() as u64 + 10), bomb.len(), "the first stops, and the second never runs"); + let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64; + assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops"); + assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read"); + assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs"); + assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read"); assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read"); } @@ -597,18 +636,19 @@ mod tests { let chain = [(Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); - assert_eq!(u64::MAX - work, data.len() as u64 + 1000); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000); // The later stage of two, which pays for the first as well. let bomb = encode_flate(&data); let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_)))); - assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64); + assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len())); // Both limits at once: a run that can't go on is what a caller needs to hear. - let mut work = data.len() as u64 + 1000; + let mut work = data.len() as u64 + read(data.len()) + 1000; assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err())); assert_eq!(work, 0); - let mut work = data.len() as u64 + 1001; + let mut work = data.len() as u64 + read(data.len()) + 1001; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); assert_eq!(work, 1, "the image's limit, with a step to spare"); } @@ -624,21 +664,22 @@ mod tests { assert!(!partial.is_empty()); let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); // Read strictly it is an error, for the same work. let mut work = u64::MAX; assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. }))); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); let whole = encode_flate(&data); let mut work = u64::MAX; assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data); - assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64); + assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len())); // A predictor that comes to nothing, after all of this was inflated for it. let p = Params { predictor: 99, ..Params::default() }; let data = encode_flate(&vec![0; 1 << 16]); let mut work = u64::MAX; assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, data.len() as u64 + (1 << 16)); + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16)); } /// A stream that reads as zlib only until it fails, and as raw deflate until that fails too, @@ -660,13 +701,67 @@ mod tests { assert_eq!(spent, 0x0601); let mut work = u64::MAX; assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64); + // Read twice: as zlib, then as raw deflate. + assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64); // Through a predictor, which would otherwise lose the count. let p = Params { predictor: 12, columns: 4, ..Params::default() }; let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap(); assert_eq!(spent, 0x0601); } + /// Inflating does work for every block it reads, whether or not the block makes anything: empty + /// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them + /// is paid for at Flate's price before any is read. + #[test] + fn a_flate_stage_pays_for_what_it_reads() { + // Four empty blocks in five bytes, as raw deflate, then an empty final block. + let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12); + hollow.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b""); + let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64; + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b""); + assert_eq!(u64::MAX - work, cost); + let mut work = cost; + assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough"); + assert_eq!(work, 0); + super::STAGE_BYTES.with(|c| c.set(0)); + let mut work = cost - 1; + assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated"); + // Counted for a byte allowance, as a page's content is, reading costs nothing. + assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0)); + } + + /// A zlib header that only looked valid is read again as raw deflate, and the second reading + /// is paid for as the first was. One that can't be paid for is refused, not read as damaged. + #[test] + fn a_second_reading_is_paid_for() { + // As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many + // zero bytes, then an empty final block. + let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; + data.extend(vec![0u8; 0x0601]); + data.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]); + let read = FLATE_READ_WORK * data.len() as u64; + let mut work = (data.len() + 0x0601) as u64 + 2 * read; + assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]); + assert_eq!(work, 0, "exactly enough"); + // Enough to read it once and not twice: it isn't read again. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = data.len() as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0)); + // Enough to read it twice, and a byte short of what it makes: the second reading makes no + // more than is left once it has been paid for. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short"); + } + /// With nothing left nothing is read. What can't be decoded at all never costs anything, and /// says so whatever is left. Without filters, what is read is the data. #[test] diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index f74b091984..b5662a2bb6 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -394,8 +394,8 @@ mod tests { let mut budget = ImageBudget::with_work(0); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 0); assert_eq!(budget.refused(), 1); - // Reading each strip, and building the picture of 8 rows of 8 they make. - let cost = raw + 8 * (8 + 8) + 8 * 8; + // Reading each strip, inflating it, and building the picture of 8 rows of 8 they make. + let cost = (1 + pdfcraft_filters::FLATE_READ_WORK) * raw + 8 * (8 + 8) + 8 * 8; let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 8); assert_eq!(u64::MAX - budget.work_left(), cost); diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 92e68582c2..3ad9409f7d 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -1515,7 +1515,7 @@ mod dimension_tests { dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); let image = Stream::flate(dict, &samples); - let read = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let read = (image.raw.len() + samples.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * image.raw.len() as u64 + 256 * 256; let original = doc.add(Object::Stream(image.clone())); let mut xobjects = Dict::new(); xobjects.set(b"I0".to_vec(), Object::Ref(original)); @@ -1607,7 +1607,7 @@ mod dimension_tests { }; let mut doc = Document::new_empty(); let mask = image("DeviceGray"); - let cost = (mask.raw.len() + noise.len()) as u64 + 256 * 256; + let cost = (mask.raw.len() + noise.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * mask.raw.len() as u64 + 256 * 256; let masked = doc.add(Object::Stream(mask)); let mut with_mask = image("DeviceGray"); with_mask.dict.set(b"SMask".to_vec(), Object::Ref(masked)); From f16c7db8f4aef7c965f7111d77a184b6605b78d8 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 20:41:12 +0100 Subject: [PATCH 25/79] M2.6: a preset dictionary is refused whatever follows its header Mutation testing of the commit before this one left the preset-dictionary check in flate.rs alive: the test's bytes after the header fail to inflate either way. A header with FDICT set followed by data that inflates as raw deflate is refused too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/filters/src/flate.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index ffe46207b7..ce57121f4a 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -178,6 +178,8 @@ mod tests { assert!(matches!(dec(&[0x78, 0x9c, 0x07, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); // Preset dictionary. assert!(matches!(dec(&[0x78, 0x20, 0, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); + // One whose header is followed by data that inflates: refused all the same. + assert!(matches!(dec(&[0x78, 0xBB, 0x4b, 0x04, 0x00]), Err(FilterError::Corrupt { .. }))); } #[test] From 6771ddffe87b902b8d61b1b19d5914020d436fbd Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:07:28 +0100 Subject: [PATCH 26/79] M2.6: inflate with zlib-rs everywhere, so reading Flate costs 2 steps a byte The Resources review of the commits before this one found that pricing what Flate reads at 128 steps a byte refused real redactions. Redaction decodes a picture once for each page that draws it, so 240 pages sharing a 1 MB scan-like Flate picture, a mark on each, went from Ok (240 cleared, 20.3 s) to ImagesPastAllowance, its allowance paying for 188 decodes: each paid about 27 times what it had. The cost being priced is miniz_oxide's: about 1.7 us for every fixed-Huffman block, empty or not. flate2's zlib-rs backend took about 11 to 13 ns a fixed block, and the Windows build already used it (craft-update turns on zip's deflate-flate2-zlib-rs, and with both backends on, flate2 inflates with zlib-rs, measured). Every platform now builds flate2 with zlib-rs, which it prefers to the miniz_oxide other dependencies still build. In a release build (zlib-rs 0.6.8), per byte read: empty fixed blocks 9 ns, fixed blocks of one literal 6, empty dynamic 21, empty stored 1.3, against 1,383, 768, 106 and 2 before. Each backend inflating what its own encoder made, 16 MiB of text took 2.6 ms against 6.3 to 13.7, and a scan-like picture 38 ms against 53. FLATE_READ_WORK is now 2: per byte read, the worst shape comes to about 7 ns a step, and a picture an encoder made pays twice its stored length more than unpriced (an_image_an_encoder_made_pays_little_for_its_reading: at most half again). The same review: export and compare's Office export read an indexed image's lookup table outside the allowance, once per image, even with nothing left (1.4 s each for 1 MiB of empty blocks under miniz_oxide). It is read through ImageBudget::decode_filters now, and refused past it (a_lookup_table_is_read_within_the_exports_allowance). filters, cos, optimize, redact, create, edit, export and engine pass with the new backend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 4 +++- crates/cos/src/budget.rs | 35 ++++++++++++++++++++++++++++++++--- crates/create/src/extract.rs | 23 +++++++++++++++-------- crates/create/src/tests.rs | 32 ++++++++++++++++++++++++++++++++ crates/filters/README.md | 2 +- crates/filters/src/lib.rs | 19 ++++++++++--------- 6 files changed, 93 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index edd9e682b7..19714f41a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,9 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } +# zlib-rs, which the Windows build already got through craft-update: miniz_oxide spends about +# 1.7 µs on every deflate block, empty ones included (`pdfcraft_filters::FLATE_READ_WORK`). +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. fax = "0.2.7" diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index d3e4a71974..ef68bfffdf 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -13,12 +13,12 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step /// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A /// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks -/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. +/// that make nothing takes up to 21 ns a byte: about 7 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to -/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to +/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -591,6 +591,35 @@ mod image_tests { assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } + /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays + /// little for it: a scan-like picture stored in under half its size costs no more than half + /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so + /// a letterhead on hundreds of pages has to fit as it did unpriced. + #[test] + fn an_image_an_encoder_made_pays_little_for_its_reading() { + let doc = Document::new_empty(); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let samples: Vec = (0..256 * 256usize) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + ((i % 256) as u8 / 16) ^ (seed as u8 & 3) + }) + .collect(); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + let image = Stream::flate(dict, &samples); + assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); + let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &image, 1 << 20).unwrap(); + let paid = u64::MAX - budget.work_left(); + assert_eq!(paid, unpriced + read(image.raw.len())); + assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); + } + /// An image pays for the pixels its data holds, not those its size declares: one claiming /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the /// images after it are decoded as they would be without it. diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index d5103071e6..8fc7fea984 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -184,11 +184,11 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace")), Ok(Space::Cmyk)); + let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace"), budget), Ok(Space::Cmyk)); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"))? }; + let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"), budget)? }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); @@ -436,7 +436,10 @@ impl Space { /// A lookup table holds at most 256 entries of 4 bytes; decoding stops well past that. const MAX_LOOKUP: usize = 64 << 10; -fn space(doc: &Document, cs: Option<&Object>) -> Result { +/// The colour space `cs` names; an indexed one's lookup table is read within `budget`, as the +/// image's samples are, since each image reads it again and its filters can do work that makes +/// nothing. +fn space(doc: &Document, cs: Option<&Object>, budget: &mut ImageBudget) -> Result { let Some(cs) = cs else { return Err("the image has no colour space".into()) }; let cs = doc.resolve(cs); let unsupported = |n: &[u8]| Err(format!("{} images can't be exported yet", String::from_utf8_lossy(n))); @@ -470,7 +473,7 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { if nested { return Err("nested indexed colour spaces aren't valid".into()); } - let base = space(doc, a.get(1))?; + let base = space(doc, a.get(1), budget)?; let hival = match a.get(2).map(|value| doc.resolve(value)).as_deref() { Some(Object::Int(value)) if (0..=255).contains(value) => *value as u8, _ => return Err("an indexed colour space with an invalid hival".into()), @@ -478,10 +481,14 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { let table = match a.get(3).map(|t| doc.resolve(t)).as_deref() { Some(Object::String(s)) => s.bytes.clone(), // Through its parameters given by reference too, as the image's samples. - Some(Object::Stream(t)) => doc - .with_direct_parms(t) - .decoded_within(MAX_LOOKUP) - .map_err(|_| "an indexed colour space with an oversized or damaged lookup table".to_string())?, + Some(Object::Stream(t)) => { + let refused = budget.refused(); + match budget.decode_filters(doc, t, MAX_LOOKUP) { + Ok(table) => table, + Err(_) if budget.refused() > refused => return Err(FilterError::Spent.to_string()), + Err(_) => return Err("an indexed colour space with an oversized or damaged lookup table".into()), + } + } _ => return Err("an indexed colour space without a lookup table".into()), }; Ok(Space::Indexed(Box::new(base), hival, table)) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index a7a5611ffc..644fed4bdb 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2312,6 +2312,38 @@ fn flate_images_past_the_exports_allowance_are_left_out_with_the_reason() { assert_eq!(budget.refused(), 1); } +/// An indexed image's lookup table, given as a stream, is read within the export's allowance as +/// the image's samples are: each image reads it again, and its filters can do work that makes +/// nothing (Flate over empty blocks). Past the allowance the image is left out with the reason. +#[test] +fn a_lookup_table_is_read_within_the_exports_allowance() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let exported = |as_stream: bool, work: u64| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(2)); + dict.set(b"Height".to_vec(), Object::Int(1)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + let image = doc.add(Object::Stream(Stream::flate(dict, &[0, 1]))); + let mut budget = ImageBudget::with_work(work); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + (file, work - budget.work_left(), budget.refused()) + }; + let (direct, image_alone, _) = exported(false, u64::MAX); + let (streamed, both, _) = exported(true, u64::MAX); + assert_eq!(streamed.unwrap(), direct.unwrap(), "the same picture"); + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(both, image_alone + read, "the table is paid for as it is read"); + let (refused, _, count) = exported(true, image_alone); + assert!(refused.unwrap_err().contains("more image decoding than")); + assert_eq!(count, 1); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. diff --git a/crates/filters/README.md b/crates/filters/README.md index 38666e6369..6c831ebe54 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. Inflating is flate2's zlib-rs backend on every platform; miniz_oxide, before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index cf10d0586b..5a679d5270 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -258,15 +258,16 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m pub const MAX_CHAIN: usize = 8; /// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], -/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block -/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits -/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide -/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At -/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 -/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading -/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this -/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). -pub const FLATE_READ_WORK: u64 = 1 << 7; +/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes. +/// Inflating does work for every block it reads, whether or not the block makes anything: an empty +/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs +/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read +/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the +/// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide +/// 0.8.9, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns +/// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its +/// reading included, so an image pays twice its stored length more than it would unpriced. +pub const FLATE_READ_WORK: u64 = 2; /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { From f8788dff118f2e45a5029904d173ec71ff254a96 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:33:59 +0100 Subject: [PATCH 27/79] M2.6: say what zlib-rs reads differently, and test which backend inflates The Consumers review of the commit before this one: zlib-rs refuses a back-reference to before the start of the output, which miniz_oxide filled with zeros and went on from. Read strictly such a stream is now refused, and read tolerantly what came before it is kept, as zlib has it. flate.rs's quirks and the filters README say so, and a_copy_from_before_the_start_is_damage checks it, which also fails under miniz_oxide, so it shows which backend runs: the read price is set for zlib-rs. The same review: other dependencies still build flate2's default miniz_oxide backend, and zlib-rs runs because flate2 prefers it when both are built, not because it is the only one. The Cargo.toml comment says so. The miniz_oxide measured was 0.9.1, flate2's (0.8.9 is png's); FLATE_READ_WORK's doc says so. The filters README named miniz_oxide as the backend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 6 ++-- crates/filters/README.md | 6 ++-- crates/filters/src/flate.rs | 67 +++++++++++++++++++++++++++++++++++++ crates/filters/src/lib.rs | 2 +- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19714f41a6..51a598d53b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,8 +64,10 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -# zlib-rs, which the Windows build already got through craft-update: miniz_oxide spends about -# 1.7 µs on every deflate block, empty ones included (`pdfcraft_filters::FLATE_READ_WORK`). +# zlib-rs: miniz_oxide spends about 1.7 µs on every deflate block, empty ones included +# (`pdfcraft_filters::FLATE_READ_WORK`). Other dependencies still build flate2's default +# miniz_oxide backend; flate2 prefers zlib-rs when both are built, as the Windows build already did +# through craft-update. `flate::tests::a_copy_from_before_the_start_is_damage` shows which runs. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. diff --git a/crates/filters/README.md b/crates/filters/README.md index 6c831ebe54..6fb820f8d8 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -3,7 +3,7 @@ PDF stream filters: the decoders and encoders behind `/Filter` and `/DecodeParms`. - **Layer:** L0, standalone. No dependency on any other PdfCraft crate. `cos` depends on it (architecture §3 rule 3). -- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `miniz_oxide` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder; CCITT decoding (T.4 and T.6) is the `hayro-ccitt` crate's, and JBIG2 decoding the `hayro-jbig2` crate's (both MIT OR Apache-2.0, already in the tree through hayro), bounded here. JBIG2 generic region encoding is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 checks the arithmetic coder byte for byte. +- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `zlib-rs` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder; CCITT decoding (T.4 and T.6) is the `hayro-ccitt` crate's, and JBIG2 decoding the `hayro-jbig2` crate's (both MIT OR Apache-2.0, already in the tree through hayro), bounded here. JBIG2 generic region encoding is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 checks the arithmetic coder byte for byte. - `#![forbid(unsafe_code)]`. Builds for `wasm32-unknown-unknown`. @@ -37,14 +37,14 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. Inflating is flate2's zlib-rs backend on every platform; miniz_oxide, before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. flate2 inflates with its zlib-rs backend, which it prefers to miniz_oxide when other dependencies build that too; miniz_oxide, its backend here before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour | Filter | Decode | Encode | |---|---|---| -| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated → Corrupt (partial in tolerant mode) | zlib level 6 | +| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, or a back-reference to before the start of the output → Corrupt (partial in tolerant mode) | zlib level 6 | | LZWDecode | 9–12-bit MSB-first codes, clear (256), EOD (257), `EarlyChange` 0/1; missing EOD/initial clear tolerated; a full table stops growing | greedy, leading clear, clear-table when the table fills, EOD | | Predictors (Flate/LZW) | 2 = TIFF (1/2/4/8/16 bpc), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad params or tags → Corrupt | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum (min sum of signed residuals) | | ASCIIHexDecode | white space ignored, odd digit padded with 0, `>` EOD (optional), other characters → Corrupt | upper case, newline every 64 digits, `>` | diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index ce57121f4a..c013b709da 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -5,6 +5,10 @@ //! deflate stream is inflated raw after the 2-byte zlib header; //! - streams without a valid zlib header are inflated as raw deflate; //! - data after the end of the deflate stream is ignored. +//! +//! A back-reference to before the start of the output points at nothing, and is damage, as zlib +//! has it: read strictly the stream is refused, and read tolerantly what came before it is kept. +//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) use std::io::Write; @@ -195,4 +199,67 @@ mod tests { fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { decode(d, max, Some(max)).0.map_err(|f| f.error) } + + /// Deflate's bits: header fields and extra bits least significant first, Huffman codes most + /// significant first (RFC 1951 §3.1.1). + #[derive(Default)] + struct Bits { + out: Vec, + acc: u32, + n: u32, + } + + impl Bits { + fn put(&mut self, value: u32, len: u32) { + for i in 0..len { + self.acc |= ((value >> i) & 1) << self.n; + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + (self.acc, self.n) = (0, 0); + } + } + } + + fn code(&mut self, code: u32, len: u32) { + for i in (0..len).rev() { + self.put((code >> i) & 1, 1); + } + } + + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push(self.acc as u8); + } + self.out + } + } + + /// A back-reference to before the start of the output points at nothing: read strictly the + /// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide, + /// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which + /// backend inflates (the price of reading Flate, `FLATE_READ_WORK`, is set for zlib-rs). + #[test] + fn a_copy_from_before_the_start_is_damage() { + let literal = |bits: &mut Bits, b: u8| bits.code(0x30 + u32::from(b), 8); + let mut bits = Bits::default(); + bits.put(1, 1); // the final block, + bits.put(1, 2); // fixed Huffman codes (§3.2.6) + literal(&mut bits, b'A'); + literal(&mut bits, b'B'); + bits.code(1, 7); // length 3 + bits.code(4, 5); // distance 5 (code 4 and an extra bit of 0), three bytes before the start + bits.put(0, 1); + literal(&mut bits, b'C'); + bits.code(0, 7); // end of block + let data = bits.finish(); + assert!(zlib_header(&data).is_none(), "read as raw deflate"); + match decode(&data, 1 << 20, Some(1 << 20)).0 { + Err(f) => { + assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); + assert_eq!(f.partial, b"AB"); + } + Ok(out) => panic!("read as {out:?}"), + } + } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 5a679d5270..f8b073f967 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -264,7 +264,7 @@ pub const MAX_CHAIN: usize = 8; /// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read /// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the /// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide -/// 0.8.9, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns +/// 0.9.1, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns /// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its /// reading included, so an image pays twice its stored length more than it would unpriced. pub const FLATE_READ_WORK: u64 = 2; From 05300ea5bc389d312c169c6f5327233be40989ed Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:58:17 +0100 Subject: [PATCH 28/79] M2.6: say what was measured, not why, and test the other block zlib-rs refuses The Claims review of the branch: the docs said miniz_oxide "builds a block's tables for every block", which a timing doesn't show. They now say what was timed: about 1.7 us for every fixed-Huffman block, empty or not. "The worst comes to about 7 ns a step" was per byte read; a stage's start, about 0.25 us under zlib-rs (1.6 us for eight over an empty stream, release), isn't priced, since every way to repeat one takes bytes of the file. The letterhead test's doc promised more than it checks. The same review found a second shape zlib-rs refuses where miniz_oxide read on: a dynamic block whose code has no end-of-block symbol. flate.rs's quirks and the README row say so, and a_dynamic_code_without_an_end_is_damage checks that nothing of it is kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 2 +- crates/cos/src/budget.rs | 3 ++- crates/filters/README.md | 2 +- crates/filters/src/flate.rs | 41 ++++++++++++++++++++++++++++++++++++- crates/filters/src/lib.rs | 10 +++++---- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 51a598d53b..b73a85f1de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -# zlib-rs: miniz_oxide spends about 1.7 µs on every deflate block, empty ones included +# zlib-rs: miniz_oxide took about 1.7 µs for every fixed-Huffman block, empty or not # (`pdfcraft_filters::FLATE_READ_WORK`). Other dependencies still build flate2's default # miniz_oxide backend; flate2 prefers zlib-rs when both are built, as the Windows build already did # through craft-update. `flate::tests::a_copy_from_before_the_start_is_damage` shows which runs. diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index ef68bfffdf..7f37a39fb0 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -594,7 +594,8 @@ mod image_tests { /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays /// little for it: a scan-like picture stored in under half its size costs no more than half /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so - /// a letterhead on hundreds of pages has to fit as it did unpriced. + /// a letterhead on hundreds of pages has to fit about as often as it did unpriced (here, at + /// most half again the cost). #[test] fn an_image_an_encoder_made_pays_little_for_its_reading() { let doc = Document::new_empty(); diff --git a/crates/filters/README.md b/crates/filters/README.md index 6fb820f8d8..2aab578686 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -44,7 +44,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row | Filter | Decode | Encode | |---|---|---| -| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, or a back-reference to before the start of the output → Corrupt (partial in tolerant mode) | zlib level 6 | +| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, a back-reference to before the start of the output, or a dynamic block without an end-of-block code → Corrupt (partial in tolerant mode) | zlib level 6 | | LZWDecode | 9–12-bit MSB-first codes, clear (256), EOD (257), `EarlyChange` 0/1; missing EOD/initial clear tolerated; a full table stops growing | greedy, leading clear, clear-table when the table fills, EOD | | Predictors (Flate/LZW) | 2 = TIFF (1/2/4/8/16 bpc), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad params or tags → Corrupt | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum (min sum of signed residuals) | | ASCIIHexDecode | white space ignored, odd digit padded with 0, `>` EOD (optional), other characters → Corrupt | upper case, newline every 64 digits, `>` | diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index c013b709da..ab7c501488 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -8,7 +8,8 @@ //! //! A back-reference to before the start of the output points at nothing, and is damage, as zlib //! has it: read strictly the stream is refused, and read tolerantly what came before it is kept. -//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) +//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) So is a dynamic +//! block whose code has no end-of-block symbol, and nothing of that block is kept. use std::io::Write; @@ -235,6 +236,44 @@ mod tests { } } + /// A dynamic block whose literal and length code has no end-of-block symbol can't end (RFC 1951 + /// §3.2.5): it is refused, and nothing of it is kept. + #[test] + fn a_dynamic_code_without_an_end_is_damage() { + let mut bits = Bits::default(); + bits.put(1, 1); // the final block, + bits.put(2, 2); // dynamic Huffman codes (§3.2.7) + bits.put(0, 5); // 257 literal and length codes + bits.put(0, 5); // 1 distance code + bits.put(14, 4); // 18 code length codes, in their order up to the one for 1 + for symbol in [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1] { + bits.put(u32::from(symbol == 18 || symbol == 1), 3); + } + // Code length codes: 1 is 0, 18 is 1. 'A' and 'B' get a bit each, 256 none. + let zeros = |bits: &mut Bits, n: u32| { + bits.code(1, 1); + bits.put(n - 11, 7); + }; + zeros(&mut bits, 65); + bits.code(0, 1); + bits.code(0, 1); + zeros(&mut bits, 138); + zeros(&mut bits, 52); + bits.code(0, 1); // the distance code: a bit + for _ in 0..4 { + bits.code(0, 1); // 'A' + } + let data = bits.finish(); + assert!(zlib_header(&data).is_none(), "read as raw deflate"); + match decode(&data, 1 << 20, Some(1 << 20)).0 { + Err(f) => { + assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); + assert_eq!(f.partial, b""); + } + Ok(out) => panic!("read as {out:?}"), + } + } + /// A back-reference to before the start of the output points at nothing: read strictly the /// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide, /// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index f8b073f967..4b1e0ce7cb 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -263,10 +263,12 @@ pub const MAX_CHAIN: usize = 8; /// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs /// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read /// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the -/// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide -/// 0.9.1, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns -/// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its -/// reading included, so an image pays twice its stored length more than it would unpriced. +/// worst per byte read comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. +/// (miniz_oxide 0.9.1, flate2's backend here before, took about 1.7 µs for every fixed-Huffman +/// block, empty or not: 1,383 ns a byte read on empty ones.) A stage's start isn't priced, about +/// 0.25 µs: every way to repeat one takes bytes of the file, so what starts cost is bounded by the +/// file's length. Data a real encoder makes took 0.2 to 2.3 ns a byte made, its reading included, so +/// an image pays twice its stored length more than it would unpriced. pub const FLATE_READ_WORK: u64 = 2; /// Whether a chain is one [`run`] takes, and if not, the error. From 0c9cd267f502f8de80f8ffc87648a5b7ac44a05f Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 3 Nov 2026 00:43:53 +0100 Subject: [PATCH 29/79] M2.6: a JPEG's colour space is read once for an export CodeRabbit on #5825: a JPEG that isn't inverted read its colour space to know whether it is CMYK, and then, with a soft mask, a stencil mask or a colour key keeping it from being written as it is, read it again for its samples. An indexed one (not valid, written as it is in the end) paid for its lookup table twice, and could be refused with an allowance that covers the image and one reading. The first reading is kept and used. a_jpegs_lookup_table_is_paid_for_once: an indexed JPEG with a soft mask pays one reading of its table, and is written as it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/create/src/extract.rs | 11 +++++++--- crates/create/src/tests.rs | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index 8fc7fea984..863ad2da0a 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -183,12 +183,17 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat .unwrap_or(false); // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as - // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace"), budget), Ok(Space::Cmyk)); + // a negative. The space is read once: an indexed one's lookup table is paid for each time. + let read = (jpeg && !inverted).then(|| space(doc, s.dict.get(b"ColorSpace"), budget)); + let as_is_negative = matches!(read, Some(Ok(Space::Cmyk))); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"), budget)? }; + let mut space = match read { + _ if mask => Space::Gray, + Some(read) => read?, + None => space(doc, s.dict.get(b"ColorSpace"), budget)?, + }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 644fed4bdb..f701463eae 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2344,6 +2344,46 @@ fn a_lookup_table_is_read_within_the_exports_allowance() { assert_eq!(count, 1); } +/// A JPEG's colour space is read once for an export, its lookup table paid for once: whether it is +/// CMYK (a negative as is) and what its samples are both come from that reading. An indexed JPEG +/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. +#[test] +fn a_jpegs_lookup_table_is_paid_for_once() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let mut jpeg = Vec::new(); + image::GrayImage::from_pixel(2, 1, image::Luma([1])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); + let spent = |as_stream: bool| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut mask = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + mask.set(k.as_bytes().to_vec(), v); + } + mask.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let mask = doc.add(Object::Stream(Stream::flate(mask, &[255, 128]))); + let mut dict = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + dict.set(k.as_bytes().to_vec(), v); + } + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); + dict.set(b"SMask".to_vec(), Object::Ref(mask)); + let image = doc.add(Object::Stream(Stream::from_raw(dict, jpeg.clone()))); + let mut budget = ImageBudget::with_work(u64::MAX); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + assert_eq!(file, Ok(("jpg", jpeg.clone())), "written as it is"); + u64::MAX - budget.work_left() + }; + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(spent(true), spent(false) + read, "one reading of the table"); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. From a2b2bc5e6da2ce10ccdb07bcf5abcb4103bf0e5a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 3 Nov 2026 06:37:59 +0100 Subject: [PATCH 30/79] M2.6: test that a JPEG refused at its lookup table is left out CodeRabbit's follow-up on #5825: the indexed JPEG test ran only with an unlimited allowance, so nothing checked that a refused lookup-table read leaves a JPEG out (`a_lookup_table_is_read_within_the_exports_allowance` checks a Flate image). The test now measures what the soft mask costs alone (no colour space, so nothing is read after it) and allows a step less than the mask and the table: the export is refused with the reason, and only the table is refused, nothing after it. A step short of the mask, the mask alone is refused, so the mask is read first and the case above is refused at the table. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/create/src/tests.rs | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index f701463eae..8c7db432fe 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2346,7 +2346,8 @@ fn a_lookup_table_is_read_within_the_exports_allowance() { /// A JPEG's colour space is read once for an export, its lookup table paid for once: whether it is /// CMYK (a negative as is) and what its samples are both come from that reading. An indexed JPEG -/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. +/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. With the +/// mask, a table the allowance can't pay for leaves the image out, with the reason. #[test] fn a_jpegs_lookup_table_is_paid_for_once() { use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; @@ -2354,9 +2355,12 @@ fn a_jpegs_lookup_table_is_paid_for_once() { let lookup = Stream::flate(Dict::new(), &table); let mut jpeg = Vec::new(); image::GrayImage::from_pixel(2, 1, image::Luma([1])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); - let spent = |as_stream: bool| { + // `given`: None leaves the colour space out, Some(true) gives the table as a stream. + let exported = |given: Option, work: u64| { let mut doc = Document::new_empty(); - let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let table = given.map(|as_stream| { + if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) } + }); let mut mask = Dict::new(); for (k, v) in [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] @@ -2371,17 +2375,33 @@ fn a_jpegs_lookup_table_is_paid_for_once() { { dict.set(k.as_bytes().to_vec(), v); } - dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + if let Some(table) = table { + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + } dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); dict.set(b"SMask".to_vec(), Object::Ref(mask)); let image = doc.add(Object::Stream(Stream::from_raw(dict, jpeg.clone()))); - let mut budget = ImageBudget::with_work(u64::MAX); + let mut budget = ImageBudget::with_work(work); let file = crate::extract::image_file_within(&doc, image, &mut budget); - assert_eq!(file, Ok(("jpg", jpeg.clone())), "written as it is"); - u64::MAX - budget.work_left() + (file, work - budget.work_left(), budget.refused()) }; + let (direct, direct_cost, _) = exported(Some(false), u64::MAX); + let (streamed, streamed_cost, _) = exported(Some(true), u64::MAX); + assert_eq!(direct, Ok(("jpg", jpeg.clone())), "written as it is"); + assert_eq!(streamed, direct); let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; - assert_eq!(spent(true), spent(false) + read, "one reading of the table"); + assert_eq!(streamed_cost, direct_cost + read, "one reading of the table"); + + // Without a colour space nothing is read after the soft mask, so this is what the mask costs. + let (none, mask, _) = exported(None, u64::MAX); + assert_eq!(none.unwrap_err(), "the image has no colour space"); + // Read first, the table would be refused a step short of the mask too, and the mask after it. + assert!(read >= mask, "the table costs at least what the mask does"); + assert_eq!(exported(Some(true), mask - 1).2, 1, "the mask is read first, and refused alone"); + let (refused, _, count) = exported(Some(true), mask + read - 1); + let refused = refused.unwrap_err(); + assert!(refused.contains("more image decoding than"), "{refused}"); + assert_eq!(count, 1, "refused at the table, and nothing after it asks to be paid for"); } /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that From e0f3afe2e1bb900cb98f2b38a38f589d867b20c9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 02:12:24 +0100 Subject: [PATCH 31/79] contributors: set names for vangel76 Co-Authored-By: Claude Sonnet 5.5 --- contributors/people.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contributors/people.toml b/contributors/people.toml index fecfb68c90..f059791248 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -183,3 +183,8 @@ sync_github_name = true [people.Combiler01] sync_github_name = true + +[people.vangel76] +real_name = "Valentin Klein" +display_name = "vangel76" +sync_github_name = false From c4dd5fe24540474c360215f4466dbb247e8bb16d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 00:07:26 +0100 Subject: [PATCH 32/79] Format Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 7579c47265..0fa058e0b3 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,8 +24,8 @@ pub(crate) use objects::Held; pub(crate) use reader::find; #[cfg(test)] use reader::rfind; -pub(crate) use state::{ObjectReader, held_bytes}; use state::{ObjStm, Slot}; +pub(crate) use state::{ObjectReader, held_bytes}; pub use state::{ReadState, SourceIdentity}; #[cfg(test)] use state::{SECTION_READS, objstm_member}; From a62633e902c5e52d0dfefae9356ef0ac9e3565b9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 01:16:17 +0100 Subject: [PATCH 33/79] fix(sign): the first signature drops /NeedAppearances so Acrobat recognises it (#5939) Co-Authored-By: Claude Sonnet 5.5 --- crates/sign/src/pdf/signing.rs | 17 ++++++++++++ crates/sign/tests/it/pdf.rs | 50 ++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/crates/sign/src/pdf/signing.rs b/crates/sign/src/pdf/signing.rs index 71dd10b302..b41d7fe91a 100644 --- a/crates/sign/src/pdf/signing.rs +++ b/crates/sign/src/pdf/signing.rs @@ -153,6 +153,7 @@ fn sign_inner( } check_signing_permissions(doc, opts)?; let mut doc = doc.clone(); + let earlier_signatures = !super::validate::signature_contents(&doc).is_empty(); let root = doc.root().ok_or_else(|| SignError::Pdf("the document has no catalog".into()))?; let alg = id.key.preferred_digest(); let reserve = @@ -293,6 +294,22 @@ fn sign_inner( doc.update_dict(r, |af| af.set(b"SigFlags".to_vec(), Object::Int(3)))?; } } + // Acrobat does not recognise a signature in a form that asks viewers to redraw its fields + // (`/NeedAppearances true`): redrawing would change what the signature covers. The first + // signature drops the flag; a later one leaves it, as removing it would count as a change + // to the earlier signatures. + if !earlier_signatures { + if let Some(Object::Ref(r)) = doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm").cloned()) { + doc.update_dict(r, |af| { + af.remove(b"NeedAppearances"); + })?; + } + doc.update_dict(root, |c| { + if let Some(Object::Dict(af)) = c.get_mut(b"AcroForm") { + af.remove(b"NeedAppearances"); + } + })?; + } // The appearance. let ap = appearance(&mut doc, rect, turn, &name, &id.certificate, opts)?; let ap_ref = doc.add(Object::Stream(ap)); diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 869a4bebd5..ce4b30de9e 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -122,6 +122,56 @@ fn signing_then_validating_with_and_without_trust() { } } +/// The form dictionary of `bytes`, with the catalog's reference resolved. +fn form_of(bytes: &[u8]) -> Dict { + let doc = open(bytes); + let form = doc.get(doc.root().unwrap()).as_dict().and_then(|c| c.get(b"AcroForm").map(|a| doc.resolve(a))).unwrap(); + form.as_dict().unwrap().clone() +} + +/// A form with `/NeedAppearances true` makes Acrobat ignore the signature (#5939): the first +/// signature drops the flag, whether the AcroForm is inline or indirect, and leaves the rest of +/// the form alone. +#[test] +fn signing_drops_need_appearances() { + let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); + let inline = fixture_with(b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] /NeedAppearances true >> >>", &[]); + let indirect = + fixture_with(b"<< /Type /Catalog /Pages 2 0 R /AcroForm 7 0 R >>", &[b"<< /Fields [6 0 R] /NeedAppearances true /DA (/Helv 0 Tf 0 g) >>"]); + for (what, bytes) in [("inline", inline), ("indirect", indirect)] { + assert_eq!(form_of(&bytes).get(b"NeedAppearances"), Some(&Object::Bool(true)), "{what}"); + let signed = pdfcraft_sign::sign(&open(&bytes), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + let form = form_of(&signed); + assert!(form.get(b"NeedAppearances").is_none(), "{what}"); + assert_eq!(form.get(b"SigFlags"), Some(&Object::Int(3)), "{what}"); + assert!(form.get(b"Fields").is_some(), "{what}"); + let doc = open(&signed); + assert!(signatures(&doc, &signed, &TrustStore::default()).iter().any(|s| s.field == "Approval" && s.signed), "{what}"); + } +} + +/// A second signer leaves the form's flags alone: changing them would count as a change to +/// what the first signature covered. +#[test] +fn a_later_signature_does_not_change_need_appearances() { + let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); + let first = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + let mut doc = open(&first); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + if let Some(Object::Dict(af)) = c.get_mut(b"AcroForm") { + af.set(b"NeedAppearances".to_vec(), Object::Bool(true)); + } + }) + .unwrap(); + if let Some(Object::Ref(r)) = doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm").cloned()) { + doc.update_dict(r, |af| af.set(b"NeedAppearances".to_vec(), Object::Bool(true))).unwrap(); + } + let flagged = write_incremental(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + let second = pdfcraft_sign::sign(&open(&flagged), &id, &opts()).unwrap(); + assert_eq!(form_of(&second).get(b"NeedAppearances"), Some(&Object::Bool(true))); +} + #[test] fn signing_an_existing_field_and_counter_signing() { let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); From a371b24f2a1c0484288fd9700d429216f3917f02 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 01:25:41 +0100 Subject: [PATCH 34/79] test(sign): tighten the NeedAppearances tests (#5939) Co-Authored-By: Claude Sonnet 5.5 --- crates/sign/tests/it/pdf.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index ce4b30de9e..137960e83b 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -130,8 +130,8 @@ fn form_of(bytes: &[u8]) -> Dict { } /// A form with `/NeedAppearances true` makes Acrobat ignore the signature (#5939): the first -/// signature drops the flag, whether the AcroForm is inline or indirect, and leaves the rest of -/// the form alone. +/// signature drops the flag, whether the AcroForm is inline or indirect, and keeps /Fields and +/// sets /SigFlags. #[test] fn signing_drops_need_appearances() { let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); @@ -170,6 +170,8 @@ fn a_later_signature_does_not_change_need_appearances() { let flagged = write_incremental(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); let second = pdfcraft_sign::sign(&open(&flagged), &id, &opts()).unwrap(); assert_eq!(form_of(&second).get(b"NeedAppearances"), Some(&Object::Bool(true))); + let doc = open(&second); + assert!(signatures(&doc, &second, &TrustStore::default()).iter().any(|s| s.field == "Approval" && s.signed)); } #[test] From 712345d7a1ed2fb794d9dab5b6fa59d8bc632d30 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:47:29 +0100 Subject: [PATCH 35/79] M2.6: an edit never rewrites a content stream from the part before its damage A Flate stream damaged partway decodes to what came before the damage, and `DecodeBudget::decode` gave that back as the stream's data: the decoder's partial flag was dropped in `Stream::decoded_within_counted`. Editing a text line, a block, an image or an added item in such a stream wrote it again from that part, so everything after the damage was lost on save, though other readers may still read it. - `DecodeBudget::decode_with_damage` decodes as `decode_within` does and says whether the stream is damaged partway. A missing checksum or end-of-data marker is no damage, as before. - Edit reads the page's streams with it (`ContentStream`), and `rewritten_contents` refuses, before adding anything, a change to a damaged stream. Changes in the page's other streams go on, and the damaged one stays as stored. - Selecting objects (`editable_objects`, moving added items) refuses a page with a damaged stream, as it refuses one that doesn't decode: what follows the damage could change every later item's graphics state. - Marking a tag as an artifact or a figure as decorative refuses when the tag's content is in a damaged stream (`ArtifactError::Damaged`, `AltError::Damaged`), and changes nothing, the structure tree included. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 65 ++++++++++++++++++------- crates/a11y/src/tests.rs | 42 ++++++++++++++++ crates/cos/src/budget.rs | 63 +++++++++++++++++++----- crates/cos/src/object.rs | 15 ++++-- crates/edit/src/background.rs | 4 +- crates/edit/src/images.rs | 45 +++++++---------- crates/edit/src/objects.rs | 23 +++++---- crates/edit/src/tests.rs | 60 +++++++++++++++++++++++ crates/edit/src/text.rs | 4 +- crates/edit/src/text/blocks.rs | 8 +-- crates/edit/src/text/lines.rs | 67 +++++++++++++++++--------- crates/edit/src/text/rewrite.rs | 20 ++++---- crates/edit/src/text_regions/styled.rs | 4 +- 13 files changed, 308 insertions(+), 112 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index fde1048e47..973dc29e33 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,6 +25,8 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, + #[error("the figure's content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), } @@ -52,6 +54,8 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, + #[error("the tag's marked content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), } @@ -372,7 +376,9 @@ fn content_entries(doc: &Document, page: &Dict) -> Vec { } } -type DecodedStream = (Object, Option<(Stream, Vec)>); +/// A `/Contents` entry and, for a stream, the stream, its decoded data and whether it is damaged +/// partway (the data is then what decoded before the damage). +type DecodedStream = (Object, Option<(Stream, Vec, bool)>); /// Decode the page's content streams within the page decode budget. `None` when a stream is /// rejected because the budget is exhausted. @@ -383,14 +389,14 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let data = match budget.decode(s) { - Ok(data) => data, + let (data, damaged) = match budget.decode_with_damage(s, usize::MAX) { + Ok(read) => read, // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. Err(_) if budget.exhausted() => return None, - Err(_) => Vec::new(), + Err(_) => (Vec::new(), false), }; - Some((s.clone(), data)) + Some((s.clone(), data, damaged)) } _ => None, }; @@ -403,6 +409,18 @@ fn marked_id(doc: &Document, props: &Dict, op: &pdfcraft_content::Op, ids: &Hash (op.op == b"BDC").then(|| bdc_mcid(doc, props, op)).flatten().filter(|n| ids.contains(n)) } +/// What [`artifact_marked_content`] did. +#[derive(Debug, PartialEq)] +enum Marked { + /// The streams holding the content were rewritten. + Done, + /// None of the content is on the page, or with `require_all` not all of it: nothing changed. + NotFound, + /// Some of it is in a stream damaged partway, which can't be rewritten without losing what + /// follows the damage: nothing changed. + Damaged, +} + /// Turn the tag's marked-content blocks into artifacts, rewriting only the streams that /// contain them. Other content streams, and unknown entries on a rewritten stream, stay. /// With `require_all`, every id in `ids` must appear on the page before any stream is rewritten. @@ -412,9 +430,9 @@ fn artifact_marked_content( streams: Vec, ids: &HashSet, require_all: bool, -) -> Result { +) -> Result { let props = page_properties(doc, &page.dict); - let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes)| bytes.as_slice())).collect(); + let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes, _)| bytes.as_slice())).collect(); let pieces = pdfcraft_content::Pieces::join(&decoded); let mut found = HashSet::new(); let mut edits = Vec::new(); @@ -425,12 +443,17 @@ fn artifact_marked_content( } } if if require_all { found != *ids } else { found.is_empty() } { - return Ok(false); + return Ok(Marked::NotFound); } - let mut spliced = pieces.splice(edits).into_iter(); + let spliced = pieces.splice(edits); + // The pieces are the streams that decoded, in order. + if streams.iter().filter_map(|(_, s)| s.as_ref()).zip(&spliced).any(|((_, old, damaged), new)| *damaged && old != new) { + return Ok(Marked::Damaged); + } + let mut spliced = spliced.into_iter(); let mut next = Vec::with_capacity(streams.len()); for (entry, stream) in streams { - let Some((stream, old)) = stream else { + let Some((stream, old, _)) = stream else { next.push(entry); continue; }; @@ -456,7 +479,7 @@ fn artifact_marked_content( _ => Object::Array(next), }; doc.update_dict(page.obj, |p| p.set(b"Contents".to_vec(), contents))?; - Ok(true) + Ok(Marked::Done) } /// Decorative figure: its marked content on the page becomes an artifact, and the element @@ -468,11 +491,15 @@ pub fn mark_decorative(doc: &mut Document, obj: ObjRef) -> Result<(), AltError> let pg = d.get(b"Pg").and_then(Object::as_ref).or_else(|| elements(doc).into_iter().find(|e| e.object == obj).and_then(|e| e.page)); if let Some(page) = pg.and_then(|p| pages.iter().find(|x| x.obj == p)).filter(|_| !ids.is_empty()) { let streams = decode_page_streams(doc, &page.dict).ok_or(AltError::TooLarge)?; - // The parent tree no longer points those ids at the figure. - if artifact_marked_content(doc, page, streams, &ids, false)? - && let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) - { - clear_parent_tree(doc, key, obj)?; + match artifact_marked_content(doc, page, streams, &ids, false)? { + // The parent tree no longer points those ids at the figure. + Marked::Done => { + if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + clear_parent_tree(doc, key, obj)?; + } + } + Marked::NotFound => {} + Marked::Damaged => return Err(AltError::Damaged), } } // Out of its parent's kids. @@ -513,8 +540,10 @@ pub fn mark_tag_artifact(doc: &mut Document, obj: ObjRef) -> Result<(), Artifact let parent = element.dict.get(b"P").and_then(Object::as_ref).ok_or(ArtifactError::NoParent)?; let replacement = replacement_kids(doc, parent, obj)?; let streams = decode_page_streams(doc, &page.dict).ok_or(ArtifactError::TooLarge)?; - if !artifact_marked_content(doc, page, streams, &ids, true)? { - return Err(ArtifactError::ContentNotFound); + match artifact_marked_content(doc, page, streams, &ids, true)? { + Marked::Done => {} + Marked::NotFound => return Err(ArtifactError::ContentNotFound), + Marked::Damaged => return Err(ArtifactError::Damaged), } if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { clear_parent_tree(doc, key, obj).map_err(|e| ArtifactError::ParentTree(e.to_string()))?; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 3024319ec3..b551c2cb42 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1087,6 +1087,48 @@ fn mark_tag_artifact_validates_parent_and_all_mcids_before_mutating_document() { assert_eq!(*absent.get(ObjRef::new(9, 0)), before_pt); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), pdfcraft_cos::Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A tag's content in a stream damaged partway stays tagged: rewriting the stream from what +/// decoded would lose what follows the damage. Nothing changes, the structure tree included. +/// Content in a whole stream beside a damaged one is marked, and the damaged one kept as stored. +#[test] +fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { + use pdfcraft_cos::{ObjRef, Object}; + let content = b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n"; + let damaged = damaged_flate(content, b"0 0 1 rg 0 0 10 10 re f"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged.clone()); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let (before_contents, before_pt, before_kids) = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + assert_eq!(figures(&doc).len(), 2, "read up to the damage"); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), (before_contents, before_pt, before_kids)); + + let mut doc = figures_doc(); + let other = doc.add(damaged_flate(b"0 g\n", b"0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; + assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); + assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); +} + fn ref_(n: u32) -> Option { Some(pdfcraft_cos::ObjRef::new(n, 0)) } diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 65b750e7b3..3211efb273 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -38,9 +38,9 @@ const JPEG_WORK: u64 = 2; /// and images can share one, so reading it again for each has to count. const JPEG_READ_WORK: u64 = 4; -/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data or -/// the error, each with that count. -type Decoder = fn(&Stream, usize) -> Result<(Vec, usize), (FilterError, usize)>; +/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data and +/// whether it stopped at damage, or the error, each with that count. +type Decoder = fn(&Stream, usize) -> Result<(Vec, bool, usize), (FilterError, usize)>; /// What one operation over a document's images may still spend decoding them: exporting them, /// optimizing them, and storing LZW streams with Flate as it does, clearing them for redaction. @@ -263,45 +263,56 @@ impl DecodeBudget { } /// `stream` decoded like [`Stream::decoded`], charged to the allowance. Fails, without - /// decoding, once the allowance is spent. + /// decoding, once the allowance is spent. A stream damaged partway gives what decoded before + /// the damage, so code that writes the stream back reads it with + /// [`DecodeBudget::decode_with_damage`] instead. pub fn decode(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_within_counted) + self.charge(stream, usize::MAX, Stream::decoded_within_counted).map(|(data, _)| data) } /// Like [`DecodeBudget::decode`], for a stream with a tighter bound of its own: decoding /// past `max` bytes is an error that charges `max`, the work it took to find out. pub fn decode_within(&mut self, stream: &Stream, max: usize) -> Result, CosError> { + self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) + } + + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), also saying + /// whether the stream is damaged: `true` when a filter stopped at corrupt data, so the bytes + /// are only what decoded before it. A stream written back from them loses everything after + /// the damage, which other readers may still read. + pub fn decode_with_damage(&mut self, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { self.charge(stream, max, Stream::decoded_within_counted) } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to /// the allowance. pub fn decode_strict(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_strict_within_counted) + let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); + self.charge(stream, usize::MAX, strict).map(|(data, _)| data) } /// Decodes `stream` within `max` and the allowance, charging what every filter stage /// decoded: a stage's output counts even when a later stage shrinks it, and a corrupt stream /// pays for the bytes decoded before the error, so referencing a stream again and again /// spends the allowance by the work each decode does. - fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result, CosError> { + fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result<(Vec, bool), CosError> { if self.bytes == 0 && !stream.raw.is_empty() { return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); } if max < self.bytes { let result = decode(stream, max); let spent = match &result { - Ok((_, spent)) => *spent, + Ok((_, _, spent)) => *spent, Err((FilterError::LimitExceeded(_), _)) => max, Err((_, spent)) => *spent, }; self.bytes = self.bytes.saturating_sub(spent); - return result.map(|(data, _)| data).map_err(|(e, _)| CosError::Filter(e.to_string())); + return result.map(|(data, damaged, _)| (data, damaged)).map_err(|(e, _)| CosError::Filter(e.to_string())); } match decode(stream, self.bytes) { - Ok((data, spent)) => { + Ok((data, damaged, spent)) => { self.bytes = self.bytes.saturating_sub(spent); - Ok(data) + Ok((data, damaged)) } Err((FilterError::LimitExceeded(_), _)) => { self.bytes = 0; @@ -801,6 +812,36 @@ mod tests { assert_eq!(budget.remaining(), (10 << 20) - partial); } + /// A stream damaged partway decodes to what came before the damage, and says so, so that code + /// writing it back can leave it as it is. What every reader takes is no damage: a missing + /// checksum, no filter at all. + #[test] + fn a_stream_damaged_partway_says_so() { + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); + let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); + let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); + assert!(partial > 0, "a cut flate stream decodes in part"); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &cut, usize::MAX), Some((partial, true))); + assert_eq!(budget.remaining(), (10 << 20) - partial); + // Within a bound of its own, under what is left. + assert_eq!(read(&mut budget, &cut, 4 << 20), Some((partial, true))); + assert_eq!(budget.decode(&cut).map(|d| d.len()).ok(), Some(partial)); + + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &whole, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &whole, 4 << 20), Some((1 << 20, false))); + let unchecked = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() - 4).unwrap_or_default().to_vec()); + assert_eq!(read(&mut budget, &unchecked, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &Stream::from_raw(Dict::new(), b"0 0 m".to_vec()), usize::MAX), Some((5, false))); + // A later stage's damage is the stream's. + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); + let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); + assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, /// gives nothing back, and each reference still pays for what it decoded. #[test] diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 5d437faa82..bcb188afb1 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -297,13 +297,18 @@ impl Stream { /// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept /// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter - /// stages together, and the result gives the bytes they decoded (the work done, on success - /// and on failure). - pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { + /// stages together, and the result gives whether the data stopped at damage (it is then what + /// decoded before it) and the bytes the stages decoded (the work done, on success and on + /// failure). + pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, bool, usize), (pdfcraft_filters::FilterError, usize)> { + let mut damaged = false; let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, _, spent)| (v, spent)) + pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| { + damaged = partial; + (v, spent) + }) }; - self.decode_chain(max, decode) + self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) } /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a diff --git a/crates/edit/src/background.rs b/crates/edit/src/background.rs index 376cb5a165..27073ddc09 100644 --- a/crates/edit/src/background.rs +++ b/crates/edit/src/background.rs @@ -1,7 +1,7 @@ //! A page's background colour: the colour the page is painted with before anything else, as //! Edit a PDF ▸ Background and many producers do it, for exports that keep a page colour (#2992). -use pdfcraft_content::{Matrix, Op, Pieces}; +use pdfcraft_content::{Matrix, Op}; use pdfcraft_cos::{Dict, Document}; use crate::{EditError, Rgb}; @@ -51,7 +51,7 @@ pub fn page_background(doc: &Document, page: usize) -> Result, EditE let resources = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let gstates = resources.get(b"ExtGState").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).unwrap_or_default(); let streams = crate::text::content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; let mut st = State { ctm: Matrix::IDENTITY, space: Space::Gray, fill: Some([0.0; 3]), alpha: 1.0, plain: true, clip: Some(Vec::new()) }; diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 72656fa1bb..201612e5b2 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -4,12 +4,12 @@ //! or delete. Only the stream that draws it is rewritten, as a new object. use kurbo::{BezPath, Point, Shape}; -use pdfcraft_content::{Matrix, Op, Pieces, parse, serialize_ops}; +use pdfcraft_content::{Matrix, Op, parse, serialize_ops}; use pdfcraft_cos::{Dict, Document, ObjRef, Object}; use std::sync::Arc; use crate::EditError; -use crate::text::content_streams; +use crate::text::{ContentStream, content_streams, joined}; /// The identity of a soft mask (`/SMask`) in the PDF graphics state. #[derive(Clone, Debug)] @@ -47,7 +47,7 @@ pub struct PageImage { pub width: u32, pub height: u32, /// The content stream its `Do` (its first token) is in, and the `Do`'s index among the - /// operators of the page's streams joined ([`Pieces`]). + /// operators of the page's streams joined ([`pdfcraft_content::Pieces`]). pub stream: usize, op: usize, /// CTM before `Do`, excluding a Form's own Matrix. @@ -103,11 +103,6 @@ fn too_many_images() -> EditError { EditError::Invalid(format!("the page draws more than {MAX_PAGE_IMAGES} images")) } -/// The page's streams joined into the one stream they are: an operator may span two of them. -fn joined(streams: &[(Object, Vec)]) -> Pieces { - Pieces::join(&streams.iter().map(|(_, data)| data.as_slice()).collect::>()) -} - fn page_of(doc: &Document, page: usize) -> Result { pdfcraft_model::pages(doc).into_iter().nth(page).ok_or(EditError::NoSuchPage(page)) } @@ -1191,13 +1186,7 @@ fn build_relocated_unit_ops( } /// Change the stacking order of `img`, drawn on page `p` by its content streams `all`. -fn arrange_image( - doc: &mut Document, - p: &pdfcraft_model::Page, - all: &[(Object, Vec)], - img: &PageImage, - order: ArrangeOrder, -) -> Result<(), EditError> { +fn arrange_image(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStream], img: &PageImage, order: ArrangeOrder) -> Result<(), EditError> { let pieces = joined(all); let ops = pieces.parse().ops; let target = img.op; @@ -1264,7 +1253,7 @@ fn arrange_image( ops.get(dest_unit.start).map(|op| pieces.piece_of(op.span.start)) }; let owned_destination = destination_piece - .filter(|piece| all.get(*piece).is_some_and(|(obj, _)| doc.resolve(obj).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); + .filter(|piece| all.get(*piece).is_some_and(|s| doc.resolve(&s.entry).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); if let Some(piece) = owned_destination { let whole_unit = ops.iter().enumerate().all(|(index, op)| { let (first, last) = pieces.pieces_of(op); @@ -1440,7 +1429,7 @@ fn arrange_image( if let Some(piece) = owned_destination { let changed = pieces.splice([(target_span, removal_replacement)]); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; let mut list = match contents { Object::Array(list) => list, one => vec![one], @@ -1467,7 +1456,7 @@ fn arrange_image( } let changed = pieces.splice(edits); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1606,7 +1595,7 @@ fn app_crop_ranges(ops: &[Op], start: usize, target: usize) -> Vec)], img: &PageImage) -> Result<(), EditError> { +fn remove_image_crop(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStream], img: &PageImage) -> Result<(), EditError> { let pieces = joined(all); let ops = pieces.parse().ops; let target = img.op; @@ -1619,7 +1608,7 @@ fn remove_image_crop(doc: &mut Document, p: &pdfcraft_model::Page, all: &[(Objec return Err(EditError::Invalid("the image has no PdfCraft crop to remove".into())); } let changed = crate::text::splice(&pieces, &ops, |index| (Vec::new(), !crops.iter().any(|r| r.contains(&index)))); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1666,7 +1655,7 @@ pub fn change_image(doc: &mut Document, page: usize, index: usize, change: &Imag } }; let changed = crate::text::splice(&pieces, &ops, |index| if index == img.op { (replacement.clone(), false) } else { (Vec::new(), true) }); - let contents = crate::text::rewritten_contents(doc, &all, changed); + let contents = crate::text::rewritten_contents(doc, &all, changed)?; doc.update_dict(p.obj, |d| { d.set(b"Contents".to_vec(), contents); if let Some(res) = resources { @@ -1687,7 +1676,7 @@ pub(crate) fn selectable_images( Ok(page_images(doc, page)? .into_iter() .enumerate() - .filter(|(_, image)| !all.get(image.stream).and_then(|(o, _)| o.as_ref()).is_some_and(|r| excluded.contains(&r))) + .filter(|(_, image)| !all.get(image.stream).and_then(|s| s.entry.as_ref()).is_some_and(|r| excluded.contains(&r))) .collect()) } @@ -1700,7 +1689,7 @@ pub(crate) fn painted_over(doc: &Document, page: usize, rect: [f64; 4], from: us fn transform_image( doc: &mut Document, page: &pdfcraft_model::Page, - streams: &[(Object, Vec)], + streams: &[ContentStream], image: &PageImage, transform: [f64; 6], ) -> Result<(), EditError> { @@ -1733,7 +1722,7 @@ fn transform_image( let changed = crate::text::splice(&pieces, &ops, |index| { if index == at { (vec![Op::new("cm", folded.0.map(Object::Real).to_vec())], false) } else { (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; return Ok(()); } @@ -1754,7 +1743,7 @@ fn transform_image( (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1766,7 +1755,7 @@ fn transform_image( fn crop_image_impl( doc: &mut Document, page: &pdfcraft_model::Page, - streams: &[(Object, Vec)], + streams: &[ContentStream], image: &PageImage, keep: [f64; 4], ) -> Result<(), EditError> { @@ -1844,7 +1833,7 @@ fn crop_image_impl( (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1977,7 +1966,7 @@ pub(crate) fn translate_images(doc: &mut Document, page: usize, indexes: &[usize out.push(Op::new("Q", vec![])); (out, false) }); - let contents = crate::text::rewritten_contents(doc, &all, data); + let contents = crate::text::rewritten_contents(doc, &all, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index cdd289457d..dbdbf495ae 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -4,8 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use pdfcraft_content::{Matrix, Op, Pieces}; -use pdfcraft_cos::{Dict, Document, Object}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object}; +use crate::text::ContentStream; use crate::{Content, EditError}; pub const MAX_MOVE_OBJECTS: usize = 1_000; @@ -152,7 +153,7 @@ fn added_geometry(doc: &Document, page: usize) -> Result<(Pieces, Vec>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; let owned: BTreeSet<_> = added.iter().map(|a| a.obj).collect(); let mut geometry = BTreeMap::new(); @@ -161,7 +162,7 @@ fn added_geometry(doc: &Document, page: usize) -> Result<(Pieces, Vec, EditError>>()?; let data = pieces.splice(edits); - let contents = crate::text::rewritten_contents(doc, &streams, data); + let contents = crate::text::rewritten_contents(doc, &streams, data)?; let refs = match &contents { Object::Array(a) => a.clone(), one => vec![one.clone()], @@ -288,9 +289,9 @@ fn translate_added(doc: &mut Document, page: usize, indexes: &[usize], offset: [ Ok(()) } -/// A missing or undecodable preceding stream can hide graphics state that affects every +/// A missing, undecodable or damaged preceding stream can hide graphics state that affects every /// later item. Refuse group editing rather than inventing an inventory from partial data. -fn checked_streams(doc: &Document, page: usize) -> Result)>, EditError> { +fn checked_streams(doc: &Document, page: usize) -> Result, EditError> { let p = pdfcraft_model::pages(doc).into_iter().nth(page).ok_or(EditError::NoSuchPage(page))?; let contents = match p.dict.get(b"Contents") { None => Vec::new(), @@ -305,6 +306,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> }, }; let mut remaining = MAX_CONTENT_BYTES; + let mut decode = DecodeBudget::new(usize::MAX, 0); let mut depth = 0usize; let mut operations = 0usize; let mut streams = Vec::with_capacity(contents.len()); @@ -313,9 +315,12 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> let Object::Stream(stream) = &*resolved else { return Err(EditError::Invalid("a page content stream is missing".into())); }; - let bytes = stream - .decoded_within(remaining.min(64 * 1024 * 1024)) + let (bytes, damaged) = decode + .decode_with_damage(stream, remaining.min(64 * 1024 * 1024)) .map_err(|_| EditError::Invalid("the page content cannot be decoded within its size limit".into()))?; + if damaged { + return Err(EditError::Invalid("a page content stream is damaged partway".into())); + } remaining = remaining.checked_sub(bytes.len()).ok_or_else(|| EditError::Invalid("the page content is too large for group selection".into()))?; let parsed = pdfcraft_content::parse(&bytes); @@ -333,7 +338,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> return Err(EditError::Invalid("the page graphics state is nested too deeply".into())); } } - streams.push((object, bytes)); + streams.push(ContentStream { entry: object, data: bytes, damaged: false }); } Ok(streams) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 22fbd6e8ae..2f7a310648 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5148,6 +5148,66 @@ fn the_graphics_state_carries_over_between_content_streams() { assert!(close(l.rect[0], 100.0) && close(l.size, 10.0), "{l:?}"); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A page whose first content stream is damaged after its text, and whose second is whole. +fn damaged_page() -> (Document, pdfcraft_cos::Stream) { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let damaged = damaged_flate(b"BT /F1 12 Tf 72 700 Td (Damaged) Tj ET\n", b"BT /F1 12 Tf 72 500 Td (After the damage) Tj ET"); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), damaged.clone()); + (doc, damaged) +} + +/// An edit that would rewrite a content stream damaged partway is refused: written from what +/// decoded, it would lose what follows the damage, which other readers may still read. An edit +/// in the page's other stream goes on, and leaves the damaged one as it is stored. +#[test] +fn an_edit_never_rewrites_a_content_stream_damaged_partway() { + let (mut doc, damaged) = damaged_page(); + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + assert_eq!(texts(&doc), ["Damaged", "Whole"], "read up to the damage"); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + let listed = contents(&doc); + + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + let refused = text::rewrite_block(&mut doc, 0, 0, Some("Edited"), &text::BlockStyle::default()).unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + + text::replace_line(&mut doc, 0, 1, "Edited").unwrap(); + let doc = reopen(&doc); + assert_eq!(texts(&doc), ["Damaged", "Edited"]); + let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; + assert_eq!(list.first(), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(kept.raw, damaged.raw, "stored as it was"); +} + +/// Objects are selected only from content read whole: what a damaged stream hides past the +/// damage could change the graphics state of every later item. +#[test] +fn objects_are_not_listed_from_a_content_stream_damaged_partway() { + let (doc, _) = damaged_page(); + let refused = editable_objects(&doc, 0).unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(editable_objects(&split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); +} + fn two_stream_image_page(first: &str, second: &str) -> Document { let objs: Vec = vec![ "<< /Type /Catalog /Pages 2 0 R >>".into(), diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 3d1626a576..01c0acdb70 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -55,7 +55,9 @@ use fallback::{add_type3_font, is_type3_fallback}; pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; use interp::{Origin, Paint, union}; use lines::page_dict; -pub(crate) use lines::{FormBudget, content_streams, content_streams_within, dict_in_place, form_call, resource_dict, rewritten_contents, splice}; +pub(crate) use lines::{ + ContentStream, FormBudget, content_streams, content_streams_within, dict_in_place, form_call, joined, resource_dict, rewritten_contents, splice, +}; pub use lines::{MAX_FORM_DEPTH, MAX_FORM_VISITS, glyph_text_lines, text_lines}; pub use rewrite::{LineEdit, replace_line, rewrite_block}; use rewrite::{coincident_ops, rewrite_block_from}; diff --git a/crates/edit/src/text/blocks.rs b/crates/edit/src/text/blocks.rs index 48fe956f4f..fd2ca0ed18 100644 --- a/crates/edit/src/text/blocks.rs +++ b/crates/edit/src/text/blocks.rs @@ -1,7 +1,7 @@ use super::interp::{Carry, interpret}; -use super::lines::{content_streams, page_dict, reading_lines, rewritten_contents, splice, text_lines}; +use super::lines::{content_streams, joined, page_dict, reading_lines, rewritten_contents, splice, text_lines}; use super::rewrite::{LineEdit, coincident_ops, rewrite_block}; -use super::{Document, EditError, HashMap, Object, Op, Pieces, TextLine, group_blocks, regions}; +use super::{Document, EditError, HashMap, Object, Op, TextLine, group_blocks, regions}; /// An editable text region: adjacent runs on visual rows, followed by aligned rows. /// A region may contain several source fonts, sizes, colours and weights. @@ -270,7 +270,7 @@ pub fn clear_block(doc: &mut Document, page: usize, block: usize) -> Result<(), let p = page_dict(doc, page)?; let res = p.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let shown = interpret(doc, &ops, &res, &mut HashMap::new(), &mut Carry::new(), None, None)?; let mut patches = HashMap::new(); @@ -297,7 +297,7 @@ pub fn clear_block(doc: &mut Document, page: usize, block: usize) -> Result<(), patches.insert(s.op, replacement); } let data = splice(&pieces, &ops, |i| patches.remove(&i).map_or((Vec::new(), true), |v| (v, false))); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index 85209a5472..a28806078b 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -61,16 +61,27 @@ impl TextLine { } } -/// The page's content streams as listed, each with its decoded data (empty when it doesn't -/// decode), within one page's [`DecodeBudget`]. An error when the content runs past it: reading -/// or rewriting a page from part of its content would lose the rest. -pub(crate) fn content_streams(doc: &Document, page: &Dict) -> Result)>, EditError> { +/// One entry of a page's `/Contents`, and what it decoded to. +pub(crate) struct ContentStream { + /// The entry as listed. + pub(crate) entry: Object, + /// Its decoded data, empty when it doesn't decode. + pub(crate) data: Vec, + /// The stream is damaged partway: `data` is what decoded before the damage, so the stream + /// can't be written again from it without losing the rest ([`rewritten_contents`]). + pub(crate) damaged: bool, +} + +/// The page's content streams as listed, each with its decoded data, within one page's +/// [`DecodeBudget`]. An error when the content runs past it: reading or rewriting a page from +/// part of its content would lose the rest. +pub(crate) fn content_streams(doc: &Document, page: &Dict) -> Result, EditError> { content_streams_within(doc, page, &mut DecodeBudget::page()) } /// [`content_streams`], decoded against `budget`, which a walk of the page's forms then goes on /// spending. -pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut DecodeBudget) -> Result)>, EditError> { +pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut DecodeBudget) -> Result, EditError> { let list: Vec = match page.get(b"Contents") { None => Vec::new(), Some(c) => match &*doc.resolve(c) { @@ -79,20 +90,25 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D }, }; let mut out = Vec::with_capacity(list.len()); - for o in list { - let data = match &*doc.resolve(&o) { - Object::Stream(s) => match budget.decode(s) { - Ok(data) => data, + for entry in list { + let (data, damaged) = match &*doc.resolve(&entry) { + Object::Stream(s) => match budget.decode_with_damage(s, usize::MAX) { + Ok(read) => read, Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), - Err(_) => Vec::new(), + Err(_) => (Vec::new(), false), }, - _ => Vec::new(), + _ => (Vec::new(), false), }; - out.push((o, data)); + out.push(ContentStream { entry, data, damaged }); } Ok(out) } +/// The page's streams joined into the one stream they are: an operator may span two of them. +pub(crate) fn joined(streams: &[ContentStream]) -> Pieces { + Pieces::join(&streams.iter().map(|s| s.data.as_slice()).collect::>()) +} + /// Rewrite a page's content (its streams joined, see [`Pieces`]): `edit(i)` gives the operators /// to insert before operator `i` of `ops` (parsed from the joined pieces) and whether to keep it. /// Everything else is copied byte for byte. A page's streams are one stream in pieces, split @@ -121,25 +137,32 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> } /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams -/// (keeping their dictionaries), the others stay as they were. -pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[(Object, Vec)], new: Vec>) -> Object { +/// (keeping their dictionaries), the others stay as they were. An error, before anything is +/// added, when a piece that changed is a damaged stream: written from what decoded, it would +/// lose everything after the damage. +pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { + if streams.iter().zip(&new).any(|(s, data)| s.damaged && s.data != *data) { + return Err(EditError::Invalid( + "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), + )); + } let mut contents: Vec = Vec::with_capacity(streams.len()); - for ((obj, old), data) in streams.iter().zip(new) { - if *old == data { - contents.push(obj.clone()); + for (s, data) in streams.iter().zip(new) { + if s.data == data { + contents.push(s.entry.clone()); continue; } - let mut dict = match &*doc.resolve(obj) { + let mut dict = match &*doc.resolve(&s.entry) { Object::Stream(s) => s.dict.clone(), _ => Dict::new(), }; dict.remove(b"Length"); contents.push(Object::Ref(doc.add(Object::Stream(Stream::flate(dict, &data))))); } - match <[Object; 1]>::try_from(contents) { + Ok(match <[Object; 1]>::try_from(contents) { Ok([one]) => one, Err(many) => Object::Array(many), - } + }) } pub(crate) fn page_dict(doc: &Document, page: usize) -> Result { @@ -201,7 +224,7 @@ fn text_lines_in(doc: &Document, p: &pdfcraft_model::Page, actual_text: bool) -> let mut carry = Carry::new(); // The streams are one stream in pieces (§7.8.2), split between any two tokens: parse them // joined, so an operator whose operands end one piece still has them. - let pieces = Pieces::join(&content_streams(doc, &p.dict)?.into_iter().map(|(_, d)| d).collect::>()); + let pieces = joined(&content_streams(doc, &p.dict)?); let ops = pieces.parse().ops; let properties = res.get(b"Properties").and_then(|o| doc.dict(o)).unwrap_or_default(); let shown = interpret(doc, &ops, &res, &mut cache, &mut carry, None, actual_text.then_some(&properties))?; @@ -608,7 +631,7 @@ pub(super) fn reading_lines(doc: &Document, page: usize) -> Result<(Vec>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let stream_of = |i: usize| ops.get(i).map_or(0, |op| pieces.pieces_of(op).0); walk(&mut w, &ops, &res, &mut carry, &stream_of)?; diff --git a/crates/edit/src/text/rewrite.rs b/crates/edit/src/text/rewrite.rs index d27d23e0b5..e7fde8545a 100644 --- a/crates/edit/src/text/rewrite.rs +++ b/crates/edit/src/text/rewrite.rs @@ -4,10 +4,10 @@ use super::fallback::{ source_program_completes, source_show, source_space, source_width, type3_encode, type3_font, }; use super::interp::{Carry, Origin, TextState, interpret}; -use super::lines::{content_streams, page_dict, rewritten_contents, splice, text_lines}; +use super::lines::{content_streams, joined, page_dict, rewritten_contents, splice, text_lines}; use super::{ - Dict, Document, EditError, HashMap, Matrix, Metrics, Object, Op, PdfString, Pieces, TextLine, TextSession, alignment_frame, group_blocks, - is_centred, parse, rewrite_styled_block, + Dict, Document, EditError, HashMap, Matrix, Metrics, Object, Op, PdfString, TextLine, TextSession, alignment_frame, group_blocks, is_centred, + parse, rewrite_styled_block, }; /// What replacing a line did. @@ -56,7 +56,7 @@ pub(crate) fn replace_line_with_limit(doc: &mut Document, page: usize, line: usi let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let mut fonts_res = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let first = target.ops.first().copied().filter(|i| *i < ops.len()).ok_or_else(|| EditError::Invalid("the page's content changed".into()))?; // The line's own font, when it can show every character. @@ -127,7 +127,7 @@ pub(crate) fn replace_line_with_limit(doc: &mut Document, page: usize, line: usi } // Rebuild: the line's first operator becomes the replacement, its others go. let new_data = splice(&pieces, &ops, |i| if i == first { (std::mem::take(&mut replacement), false) } else { (Vec::new(), !drops.contains(&i)) }); - let contents = rewritten_contents(doc, &streams, new_data); + let contents = rewritten_contents(doc, &streams, new_data)?; let page_ref = p.obj; if substituted.is_some() { res.set(b"Font".to_vec(), Object::Dict(fonts_res)); @@ -834,7 +834,7 @@ pub(crate) fn rewrite_block_from( } let upright = dispatch.upright; let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let text = paragraph_text(text.unwrap_or(&b.text)); // The characters to show, for choosing fonts: the breaks are moves, not glyphs. @@ -970,7 +970,7 @@ pub(crate) fn rewrite_block_from( let insert = if i == at { std::mem::take(&mut block_ops) } else { Vec::new() }; (insert, !dropped.contains(&i)) }); - let contents = rewritten_contents(doc, &streams, new_data); + let contents = rewritten_contents(doc, &streams, new_data)?; let new_font = !reuse; if new_font { res.set(b"Font".to_vec(), Object::Dict(fonts_res)); @@ -1040,7 +1040,7 @@ pub(crate) fn selectable_blocks( !b.lines .iter() .filter_map(|i| lines.get(*i)) - .any(|l| streams.get(l.stream).and_then(|(o, _)| o.as_ref()).is_some_and(|r| excluded.contains(&r))) + .any(|l| streams.get(l.stream).and_then(|s| s.entry.as_ref()).is_some_and(|r| excluded.contains(&r))) }) .collect()) } @@ -1134,7 +1134,7 @@ pub(crate) fn translate_blocks(doc: &mut Document, page: usize, indexes: &[usize } let res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let fonts = res.get(b"Font").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let shown = interpret(doc, &ops, &res, &mut HashMap::new(), &mut Carry::new(), None, None)?; let mut patches = HashMap::new(); @@ -1205,7 +1205,7 @@ pub(crate) fn translate_blocks(doc: &mut Document, page: usize, indexes: &[usize return Err(EditError::Invalid("a selected text operation cannot be moved".into())); } let data = splice(&pieces, &ops, |i| patches.remove(&i).map_or((Vec::new(), true), |v| (v, false))); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 7a9204a8c0..442913b61b 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -962,7 +962,7 @@ pub(crate) fn rewrite_styled_block( }; let (rows, row_ranges, row_lines, row_soft, row_gap) = wrap_styled_rows(&chars, width, measure)?; let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; // A row the edit doesn't reach keeps its source operators, so its kerning, TJ word gaps and // glyph widths stay exactly as they were instead of being rebuilt from the text state. A row @@ -1206,7 +1206,7 @@ pub(crate) fn rewrite_styled_block( // when the rewrite splits one, and a kept row's operators come later in the stream. let marker = if marked { None } else { paragraph_marker(block, lines, &ops) }; let data = splice_with_marker(&pieces, &ops, marker, at, block_ops, &drop); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; res.set(b"Font".to_vec(), Object::Dict(fonts)); doc.update_dict(p.obj, |d| { d.set(b"Contents".to_vec(), contents); From 480fdb114abcb039aea93828085574b27045e0a5 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:55:04 +0100 Subject: [PATCH 36/79] M2.6: Optimize leaves a page damaged partway as it is stored Optimize read a page's content up to any damage in its streams as the whole of it. Merging a page's image strips then wrote the page's content again as one stream, without what followed the damage, and the image pass measured each image by its uses before the damage only, so one drawn larger past it could be resampled below that use. - `DecodeBudget::decode_contents_with_damage` reads a page's `/Contents` as `decode_contents` does, each stream also saying whether it is damaged partway; `decode_contents` is it without the flag. - `content_bytes` gives `None` for a page with a damaged stream, as for one that doesn't decode: the content is not all known. Its strips aren't merged (`merge_page` already leaves such a page alone), and no image is resampled (`effective_resolutions` gives every image `INFINITY`). - A form damaged partway leaves the measuring incomplete in the same way. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 28 +++++++++++++- crates/optimize/README.md | 5 ++- crates/optimize/src/images.rs | 13 ++++--- crates/optimize/tests/optimize.rs | 63 +++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 3211efb273..41dff04ed4 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -348,12 +348,18 @@ impl DecodeBudget { /// A stream that fails to decode, or arrives after the allowance is spent, is an `Err` in its /// place, so the caller chooses between skipping it and giving up. pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { + self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() + } + + /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway + /// ([`DecodeBudget::decode_with_damage`]). + pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode(s)], + Object::Stream(s) => vec![self.decode_with_damage(s, usize::MAX)], Object::Array(list) => list .iter() .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode(s)), + Object::Stream(s) => Some(self.decode_with_damage(s, usize::MAX)), _ => None, }) .collect(), @@ -842,6 +848,24 @@ mod tests { assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); } + /// A page's content streams each say whether they are damaged, in their places. + #[test] + fn a_pages_streams_each_say_whether_they_are_damaged() { + let mut doc = Document::new_empty(); + let whole = doc.add(Object::Stream(Stream::flate(Dict::new(), b"0 0 m 1 1 l S"))); + let Object::Stream(s) = &*doc.get(whole) else { panic!("not a stream") }; + let cut = Stream::from_raw(s.dict.clone(), s.raw.get(..s.raw.len() - 6).unwrap_or_default().to_vec()); + let cut = doc.add(Object::Stream(cut)); + let contents = Object::Array(vec![Object::Ref(whole), Object::Null, Object::Ref(cut)]); + let mut budget = DecodeBudget::page(); + let read: Vec<_> = budget.decode_contents_with_damage(&doc, &contents).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(read, [Some(false), Some(true)]); + let one: Vec<_> = + budget.decode_contents_with_damage(&doc, &Object::Ref(cut)).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(one, [Some(true)]); + assert_eq!(budget.decode_contents(&doc, &contents).iter().filter(|r| r.is_ok()).count(), 2, "read up to the damage"); + } + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, /// gives nothing back, and each reference still pays for what it decoded. #[test] diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 9acb076908..fd7c60ae85 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -22,11 +22,14 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going printed this way went from 4.3 MB to 231 KB (Standard) and 161 KB (Mobile). Not merged yet: side-by-side tiles, rotated strips, masks. Content arrays use the shared space boundary policy, so trailing comments, split operands and graphics state keep their meaning after merging. + A page whose content doesn't decode, or is damaged partway, is left as it is: written again from + what decoded, it would lose the rest. - **Images:** the effective resolution of each image is measured where pages draw it (the CTM through form XObjects, up to 12 deep; the smallest over all uses). Page-local `/UserUnit` scaling is included, so large-format pages are measured in physical inches. If a use can't be - measured (forms nested deeper, or drawn too many times), no image is resampled. Colour and grayscale images above a + measured (forms nested deeper or drawn too many times, or content that doesn't decode or is + damaged partway, which may draw an image larger past the damage), no image is resampled. Colour and grayscale images above a threshold are resampled with a bicubic (Catmull-Rom) filter to a target resolution and stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A stencil `/Mask` has its own resolution, so the picture changes under it and it stays as it is. diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 159ba04e8c..21f6953a8d 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -32,9 +32,10 @@ fn held_page_resources(doc: &Document, page: ObjRef) -> Option<(Dict, ObjRef)> { } /// The page content `contents` names, decoded against `budget` and joined. `None` when a stream -/// doesn't decode or the budget runs out: the content is not all known. +/// doesn't decode, is damaged partway or the budget runs out: the content is not all known. pub(crate) fn content_bytes(doc: &Document, contents: &Object, budget: &mut DecodeBudget) -> Option> { - let pieces = budget.decode_contents(doc, contents).into_iter().collect::, _>>().ok()?; + let reads = budget.decode_contents_with_damage(doc, contents); + let pieces = reads.into_iter().map(|read| read.ok().filter(|(_, damaged)| !damaged).map(|(data, _)| data)).collect::>>()?; Some(Pieces::join(&pieces).data().to_vec()) } @@ -214,7 +215,7 @@ impl Walk<'_> { self.complete = false; return; } - let Ok(data) = self.budget.decode(s) else { + let Ok((data, false)) = self.budget.decode_with_damage(s, usize::MAX) else { self.complete = false; return; }; @@ -245,9 +246,9 @@ fn remaps_device_spaces(doc: &Document, resources: &Dict) -> bool { /// The effective resolution (ppi) of every image drawn by `pages`: the smallest over its uses. /// -/// If forms nest too deeply, are drawn too many times, or a page's content doesn't decode within -/// its budget, every image is given `INFINITY` (never resampled), since its largest use may be -/// among those missed. +/// If forms nest too deeply, are drawn too many times, or a page's or form's content doesn't +/// decode within its budget or is damaged partway, every image is given `INFINITY` (never +/// resampled), since its largest use may be among those missed. pub fn effective_resolutions(doc: &Document, pages: &[ObjRef]) -> HashMap { walk_pages(doc, pages).0 } diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 1387cae528..7250e9a0a0 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -3157,6 +3157,69 @@ fn images_that_do_not_tile_a_picture_are_left_alone() { unmerged(&mut off, &Settings { merge_fragments: false, ..Settings::default() }); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + Stream::from_raw(dict, raw) +} + +/// A page whose content is damaged partway isn't all known: its strips aren't merged, which would +/// write its content again without what follows the damage. +#[test] +fn strips_on_a_page_damaged_partway_are_left_as_they_are() { + let (mut doc, _) = printed_page(40, |_| false, |_, s| s); + let p = pdfcraft_annot::page_refs(&doc).unwrap()[0]; + let bad = damaged_flate(b"0 g\n", b"q 1000 0 0 1 0 0 cm /R0 Do Q"); + let damaged = doc.add(Object::Stream(bad.clone())); + let first = doc.get(p).as_dict().unwrap().get(b"Contents").cloned().unwrap(); + let contents = Object::Array(vec![first, Object::Ref(damaged)]); + doc.update_dict(p, |d| d.set(b"Contents".to_vec(), contents.clone())).unwrap(); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!((r.merged_images, r.fragments_merged), (0, 0), "{r:?}"); + assert_eq!(doc.get(p).as_dict().unwrap().get(b"Contents"), Some(&contents)); + assert_eq!(stream(&doc, damaged), bad); +} + +/// An image may be drawn larger past the damage in a page's or a form's content, so no image is +/// resampled for the uses before it. +#[test] +fn content_damaged_partway_leaves_every_image_at_its_resolution() { + for in_form in [false, true] { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 720, 720, 1, false, None); + let p = page(&mut doc, &[("Im", img)], "q 72 0 0 72 0 0 cm /Im Do Q"); + assert_eq!(effective_resolutions(&doc, &[p])[&img], 720.0); + let mut bad = damaged_flate(b"q 72 0 0 72 0 0 cm /Im Do Q\n", b"q 720 0 0 720 0 0 cm /Im Do Q"); + let added = if in_form { + bad.dict.set(b"Subtype".to_vec(), Object::name("Form")); + bad.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + bad.dict.set(b"Resources".to_vec(), Object::Dict(resources(xobjects(&[("Im", img)])))); + let f = doc.add(Object::Stream(bad)); + let res = Object::Dict(resources(xobjects(&[("Im", img), ("F", f)]))); + doc.update_dict(p, |d| d.set(b"Resources".to_vec(), res)).unwrap(); + content(&mut doc, "/F Do") + } else { + doc.add(Object::Stream(bad)) + }; + let first = doc.get(p).as_dict().unwrap().get(b"Contents").cloned().unwrap(); + doc.update_dict(p, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(added)]))).unwrap(); + // Not measured, or measured as never resampled: the image pass reads only what was measured. + let ppi = effective_resolutions(&doc, &[p]).get(&img).copied().unwrap_or(f64::INFINITY); + assert_eq!(ppi, f64::INFINITY, "in a form: {in_form}"); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 0, "{r:?}"); + } +} + /// `levels` nested forms with their own resources, the innermost drawing `img` over the unit square. fn nested_forms(doc: &mut Document, img: ObjRef, levels: usize) -> ObjRef { let mut inner = form(doc, Some(Object::Dict(resources(xobjects(&[("Im", img)])))), "/Im Do"); From 50d0fcc403ff57bba98bf0a3285de692a72bf356 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:55:04 +0100 Subject: [PATCH 37/79] M2.6: pruning keeps what a page or form damaged partway may draw Extracting, splitting and combining pages drops the `/XObject` entries no page draws. A page or form damaged partway was read up to the damage as the whole of it, so an image named only after the damage was dropped, though the stream still names it and other readers may still read that far. - `content_bytes` says whether a page's stream is damaged, and pruning keeps everything such a page could reach, as it does for content that doesn't decode. A form is read the same way. - A page made a form XObject (stamps, watermarks, backgrounds from a PDF) still draws what decoded before the damage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/import.rs | 5 +++-- crates/organize/src/prune.rs | 29 +++++++++++++++---------- crates/organize/src/tests.rs | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/crates/organize/src/import.rs b/crates/organize/src/import.rs index 9f86d6455a..b52c9175b3 100644 --- a/crates/organize/src/import.rs +++ b/crates/organize/src/import.rs @@ -499,9 +499,10 @@ pub fn page_as_form(dst: &mut Document, src: &Document, page: usize) -> Result<( let bbox = rect(b"CropBox").or_else(|| rect(b"MediaBox")).unwrap_or([0.0, 0.0, 612.0, 792.0]); let (w, h) = (bbox[2] - bbox[0], bbox[3] - bbox[1]); let rotate = d.get(b"Rotate").and_then(|r| src.resolve(r).as_int()).unwrap_or(0).rem_euclid(360); - // Content: the page's streams, decoded and joined. + // Content: the page's streams, decoded and joined. Of a stream damaged partway, the copy draws + // what decoded before the damage. let content = match own.get(b"Contents") { - Some(c) => crate::prune::content_bytes(src, c).map_err(|e| OrganizeError::Invalid(format!("page {} content: {e}", page + 1)))?, + Some(c) => crate::prune::content_bytes(src, c).map_err(|e| OrganizeError::Invalid(format!("page {} content: {e}", page + 1)))?.0, None => Vec::new(), }; let mut copier = Copier::new(src, HashMap::new(), HashMap::new()); diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 5a2a319fb4..8683b4d810 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -219,10 +219,10 @@ impl Pruner<'_> { } #[cfg(test)] crate::work(1); - match s.decoded() { - Ok(data) => self.content(&data, &inner, inner_owner, depth + 1), - // What the form draws is unknown: keep everything it could reach. - Err(_) => { + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(s, usize::MAX) { + Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), + // What the form draws is unknown, past damage too: keep everything it could reach. + Ok((_, true)) | Err(_) => { self.protect_type3_resources(&inner, inner_owner); self.distrust(&inner, inner_owner); } @@ -232,16 +232,20 @@ impl Pruner<'_> { } /// The page's content, concatenated (`/Contents` may be one stream or an array) with a space -/// between pieces as the renderer joins them, decoded against one page's [`DecodeBudget`]. An +/// between pieces as the renderer joins them, decoded against one page's [`DecodeBudget`], and +/// whether a stream is damaged partway (its part is then what decoded before the damage). An /// error when a stream can't be decoded or the budget runs out: the page may draw names that /// can't be seen. -pub(crate) fn content_bytes(doc: &Document, contents: &Object) -> Result, CosError> { +pub(crate) fn content_bytes(doc: &Document, contents: &Object) -> Result<(Vec, bool), CosError> { let mut out = Vec::new(); - for data in DecodeBudget::page().decode_contents(doc, contents) { - out.extend(data?); + let mut damaged = false; + for read in DecodeBudget::page().decode_contents_with_damage(doc, contents) { + let (data, partial) = read?; + damaged |= partial; + out.extend(data); out.push(b' '); } - Ok(out) + Ok((out, damaged)) } /// The resources of a page (as `walk` reports them) with their owner. @@ -324,9 +328,10 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result {} - Ok(data) => pruner.content(&data, &res_dict, owner, 0), - Err(_) => { + Ok((data, false)) if data.is_empty() => {} + Ok((data, false)) => pruner.content(&data, &res_dict, owner, 0), + // What the page draws past damage is unknown too. + Ok((_, true)) | Err(_) => { pruner.protect_type3_resources(&res_dict, owner); pruner.distrust(&res_dict, owner); } diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index d5ff2116ff..40bb4210d4 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2628,6 +2628,47 @@ fn extract_keeps_images_an_undecodable_page_might_draw() { assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "an image the unreadable page may draw must stay"); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A page or form damaged partway may draw images past the damage, which other readers may still +/// read: pruning keeps them, as it does for content that doesn't decode. +#[test] +fn extract_keeps_images_content_damaged_partway_might_draw() { + use pdfcraft_cos::ObjRef; + let mut page = doc_images_with(&raw("/ImA Do"), ""); + page.set(ObjRef::new(6, 0), damaged_flate(b"/ImA Do\n", b"/ImB Do")); + + let mut form = doc_images_with(&raw("/Fm Do"), ""); + let mut fm = damaged_flate(b"/ImA Do\n", b"/ImB Do"); + fm.dict.set(b"Subtype".to_vec(), Object::name("Form")); + fm.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let fm = form.add(fm); + let xobjects: Dict = + [(b"ImA".to_vec(), Object::Ref(ObjRef::new(8, 0))), (b"ImB".to_vec(), Object::Ref(ObjRef::new(9, 0))), (b"Fm".to_vec(), Object::Ref(fm))] + .into_iter() + .collect(); + form.update_dict(ObjRef::new(7, 0), |d| d.set(b"XObject".to_vec(), Object::Dict(xobjects))).unwrap(); + + for (src, what) in [(page, "page"), (form, "form")] { + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 0); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{what}: an image drawn past the damage must stay: {names:?}"); + } +} + #[test] fn extract_keeps_images_type3_glyphs_may_draw() { // A Type 3 font without its own /Resources draws through the page's: its glyphs could `Do`. From 3fec35e1ea1f910219b9c2f429643d400c73a3e4 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:14:47 +0100 Subject: [PATCH 38/79] M2.6: tests pin that content which doesn't decode is kept and edited around A stream that doesn't decode at all is not damage to write over: edit and accessibility keep it as stored and go on with the page's other streams, and only running past the page's decode allowance refuses. Nothing pinned either, nor that selecting objects reads a stream of a few MB. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/tests.rs | 17 +++++++++++++++++ crates/edit/src/tests.rs | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index b551c2cb42..a129f207cd 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1129,6 +1129,23 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); } +/// A content stream that doesn't decode at all is kept as stored, and a tag in the page's other +/// streams is still marked: only running past the decode budget refuses. +#[test] +fn a_tag_beside_a_stream_that_does_not_decode_is_marked() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"0 g".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let contents = doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let Some(Object::Array(list)) = contents else { panic!("contents {contents:?}") }; + assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); +} + fn ref_(n: u32) -> Option { Some(pdfcraft_cos::ObjRef::new(n, 0)) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 2f7a310648..dc80cb6a29 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5198,6 +5198,26 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { assert_eq!(kept.raw, damaged.raw, "stored as it was"); } +/// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, +/// and the page's other streams are read and edited around it. A stream of a few MB is read for +/// selecting objects. +#[test] +fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let unknown = pdfcraft_cos::Stream::from_raw(dict, b"BT /F1 12 Tf 72 700 Td (Unread) Tj ET".to_vec()); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), unknown.clone()); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(), ["Edited"]); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(kept.raw, unknown.raw, "stored as it was"); + + let large = format!("{}0 0 m 10 10 l S", " ".repeat(2 << 20)); + assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); +} + /// Objects are selected only from content read whole: what a damaged stream hides past the /// damage could change the graphics state of every later item. #[test] From 69e26167e4035d385e905f2563d6b6156266fbf8 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:44:05 +0100 Subject: [PATCH 39/79] M2.6: a page's content is read with its /DecodeParms given by reference `decode_with_damage` and `decode_contents_with_damage` read a stream's `/DecodeParms` only when direct, as `Stream::decoded` does, so content behind a predictor whose parameters, or a number in them, are an indirect object decoded with the predictor's tag bytes left in, and was taken as whole. Edit then listed and wrote back the wrong bytes, and pruning dropped an image such a form draws. ISO 32000-2 lets those values be indirect. - `decode_with_damage` takes the document and reads the parameters through it, and so does `decode_contents_with_damage`, and `decode_contents` with it: printing, accessibility's reading of a page and a page copied as a form read them too. Code that calls `decode` on a stream itself doesn't yet. - It reads through `Document::decoding_view`: `with_direct_parms`, but a copy of the stream holds only its `/Filter` and the parameters, its data shared, so a stream a page's `/Contents` lists again and again isn't copied whole for each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 2 +- crates/cos/src/budget.rs | 68 +++++++++++++++++++++++------- crates/cos/src/document/objects.rs | 33 +++++++++++---- crates/edit/src/objects.rs | 2 +- crates/edit/src/tests.rs | 32 ++++++++++++++ crates/edit/src/text/lines.rs | 2 +- crates/optimize/src/images.rs | 2 +- crates/organize/src/prune.rs | 2 +- crates/organize/src/tests.rs | 31 ++++++++++++++ 9 files changed, 146 insertions(+), 28 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index 973dc29e33..dd4b43a641 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -389,7 +389,7 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let (data, damaged) = match budget.decode_with_damage(s, usize::MAX) { + let (data, damaged) = match budget.decode_with_damage(doc, s, usize::MAX) { Ok(read) => read, // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 41dff04ed4..d5d67503ee 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -276,12 +276,13 @@ impl DecodeBudget { self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) } - /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), also saying - /// whether the stream is damaged: `true` when a filter stopped at corrupt data, so the bytes - /// are only what decoded before it. A stream written back from them loses everything after - /// the damage, which other readers may still read. - pub fn decode_with_damage(&mut self, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { - self.charge(stream, max, Stream::decoded_within_counted) + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), with the stream's + /// `/DecodeParms` given by reference read in `doc`, also saying whether the stream is damaged: + /// `true` when a filter stopped at corrupt data, so the bytes are only what decoded before it. + /// A stream written back from them loses everything after the damage, which other readers may + /// still read. + pub fn decode_with_damage(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { + self.charge(&doc.decoding_view(stream), max, Stream::decoded_within_counted) } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to @@ -343,23 +344,24 @@ impl DecodeBudget { self.seen.insert(r) } - /// Each stream `contents` names, decoded against the allowance, in order. `contents` is a - /// page's `/Contents`: one stream, or an array whose non-stream entries are left out. - /// A stream that fails to decode, or arrives after the allowance is spent, is an `Err` in its - /// place, so the caller chooses between skipping it and giving up. + /// Each stream `contents` names, decoded against the allowance, in order, its `/DecodeParms` + /// given by reference read in `doc`. `contents` is a page's `/Contents`: one stream, or an + /// array whose non-stream entries are left out. A stream that fails to decode, or arrives + /// after the allowance is spent, is an `Err` in its place, so the caller chooses between + /// skipping it and giving up. pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() } /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway - /// ([`DecodeBudget::decode_with_damage`]). + /// ([`DecodeBudget::decode_with_damage`], which reads `/DecodeParms` given by reference). pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode_with_damage(s, usize::MAX)], + Object::Stream(s) => vec![self.decode_with_damage(doc, s, usize::MAX)], Object::Array(list) => list .iter() .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode_with_damage(s, usize::MAX)), + Object::Stream(s) => Some(self.decode_with_damage(doc, s, usize::MAX)), _ => None, }) .collect(), @@ -823,7 +825,8 @@ mod tests { /// checksum, no filter at all. #[test] fn a_stream_damaged_partway_says_so() { - let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let doc = Document::new_empty(); + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(&doc, s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); @@ -845,7 +848,42 @@ mod tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); - assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + + /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a + /// predicted stream decodes as with them direct. Of the stream, decoding copies no more than + /// its filters and their parameters. + #[test] + fn parameters_given_by_reference_are_read() { + let content = b"q 10 0 0 10 0 0 cm /Im Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let raw = Stream::flate(Dict::new(), &rows).raw; + let mut doc = Document::new_empty(); + let direct: Dict = [(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect(); + let by_reference = Object::Ref(doc.add(Object::Dict(direct.clone()))); + let predictor = doc.add(Object::Int(12)); + let inner = Object::Dict([(b"Predictor".to_vec(), Object::Ref(predictor))].into_iter().collect()); + for parms in [Object::Dict(direct), by_reference, inner] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), parms.clone()); + dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); + let s = Stream::from_raw(dict, raw.to_vec()); + assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + let r = doc.add(Object::Stream(s.clone())); + let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); + assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_contents(&doc, &Object::Ref(r)), [Ok(content.to_vec())], "{parms:?}"); + let view = doc.decoding_view(&s); + assert!(matches!(view, std::borrow::Cow::Borrowed(_)) || view.dict.len() == 2, "{:?}", view.dict); + } } /// A page's content streams each say whether they are damaged, in their places. diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 458840de3a..7ce9474018 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -288,7 +288,29 @@ impl Document { /// ([`pdfcraft_filters::MAX_CHAIN`]). Only small values are copied, so a file can't have a /// large object copied once for each key and filter. pub fn with_direct_parms<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { - let Some(p) = s.dict.get(b"DecodeParms") else { return Cow::Borrowed(s) }; + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut s = s.clone(); + s.dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(s) + } + + /// [`Document::with_direct_parms`] for decoding alone: where that copies `s`, this makes a + /// stream of `s`'s data, shared, and a dictionary of its `/Filter` and the parameters made + /// direct, all its filters read. A stream named again and again, as a page's `/Contents` may + /// list one, isn't copied whole each time. + pub(crate) fn decoding_view<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut dict = Dict::new(); + if let Some(f) = s.dict.get(b"Filter") { + dict.set(b"Filter".to_vec(), f.clone()); + } + dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(Stream { dict, raw: s.raw.clone() }) + } + + /// The `/DecodeParms` [`Document::with_direct_parms`] gives `s`, or `None` when it keeps `s`. + fn direct_parms(&self, s: &Stream) -> Option { + let p = s.dict.get(b"DecodeParms")?; let crypt = |f: &Object| matches!(f.as_name(), Some(b"Crypt")); let filters: Vec = match s.dict.get(b"Filter") { Some(Object::Array(a)) => a.iter().map(crypt).collect(), @@ -296,7 +318,7 @@ impl Document { None => Vec::new(), }; if filters.iter().filter(|c| !**c).count() > pdfcraft_filters::MAX_CHAIN { - return Cow::Borrowed(s); + return None; } let mut changed = matches!(p, Object::Ref(_)); // Its numbers and flags kept, resolved; anything else is no filter's parameters (null). @@ -323,12 +345,7 @@ impl Document { } _ => one(p, &mut changed), }; - if !changed { - return Cow::Borrowed(s); - } - let mut s = s.clone(); - s.dict.set(b"DecodeParms".to_vec(), made); - Cow::Owned(s) + changed.then_some(made) } /// `o`, or the object it refers to, without copying either. diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index dbdbf495ae..c0a57c358a 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -316,7 +316,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result, Ed return Err(EditError::Invalid("a page content stream is missing".into())); }; let (bytes, damaged) = decode - .decode_with_damage(stream, remaining.min(64 * 1024 * 1024)) + .decode_with_damage(doc, stream, remaining.min(64 * 1024 * 1024)) .map_err(|_| EditError::Invalid("the page content cannot be decoded within its size limit".into()))?; if damaged { return Err(EditError::Invalid("a page content stream is damaged partway".into())); diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index dc80cb6a29..df9bce657d 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5218,6 +5218,38 @@ fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); } +/// A content stream whose `/DecodeParms`, or a parameter in them, is given by reference is read +/// through the reference: its text is listed and edited, not read with the predictor's tag bytes +/// left in it. +#[test] +fn content_with_parameters_given_by_reference_is_read_and_edited() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let content = b"BT /F1 12 Tf 72 700 Td (Predicted) Tj ET"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + for given in ["parameters", "predictor"] { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let predictor = Object::Int(12); + let parms = if given == "parameters" { + Object::Ref(doc.add(Object::Dict([(b"Predictor".to_vec(), predictor)].into_iter().collect::()))) + } else { + Object::Dict([(b"Predictor".to_vec(), Object::Ref(doc.add(predictor)))].into_iter().collect()) + }; + let mut s = Stream::flate(Dict::new(), &rows); + s.dict.set(b"DecodeParms".to_vec(), parms); + doc.set(ObjRef::new(4, 0), s); + assert_eq!(texts(&doc), ["Predicted", "Whole"], "{given} by reference"); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); + assert_eq!(texts(&reopen(&doc)), ["Edited", "Whole"], "{given} by reference"); + } +} + /// Objects are selected only from content read whole: what a damaged stream hides past the /// damage could change the graphics state of every later item. #[test] diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index a28806078b..bf66cc0f8b 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -92,7 +92,7 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D let mut out = Vec::with_capacity(list.len()); for entry in list { let (data, damaged) = match &*doc.resolve(&entry) { - Object::Stream(s) => match budget.decode_with_damage(s, usize::MAX) { + Object::Stream(s) => match budget.decode_with_damage(doc, s, usize::MAX) { Ok(read) => read, Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), Err(_) => (Vec::new(), false), diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 21f6953a8d..6ad6413696 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -215,7 +215,7 @@ impl Walk<'_> { self.complete = false; return; } - let Ok((data, false)) = self.budget.decode_with_damage(s, usize::MAX) else { + let Ok((data, false)) = self.budget.decode_with_damage(doc, s, usize::MAX) else { self.complete = false; return; }; diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 8683b4d810..591e804b1e 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -219,7 +219,7 @@ impl Pruner<'_> { } #[cfg(test)] crate::work(1); - match DecodeBudget::new(usize::MAX, 0).decode_with_damage(s, usize::MAX) { + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), // What the form draws is unknown, past damage too: keep everything it could reach. Ok((_, true)) | Err(_) => { diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 40bb4210d4..8d41f474cc 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2669,6 +2669,37 @@ fn extract_keeps_images_content_damaged_partway_might_draw() { } } +/// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is +/// kept, not lost to content read with the predictor's tag bytes left in it. +#[test] +fn extract_keeps_images_a_form_with_parameters_by_reference_draws() { + use pdfcraft_cos::ObjRef; + let content = b"q 10 0 0 10 0 0 cm /ImB Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let mut src = doc_images_with(&raw("/Fm Do"), ""); + let parms = src.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + let mut fm = pdfcraft_cos::Stream::flate(Dict::new(), &rows); + fm.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + fm.dict.set(b"Subtype".to_vec(), Object::name("Form")); + fm.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let fm = src.add(fm); + let xobjects: Dict = + [(b"ImA".to_vec(), Object::Ref(ObjRef::new(8, 0))), (b"ImB".to_vec(), Object::Ref(ObjRef::new(9, 0))), (b"Fm".to_vec(), Object::Ref(fm))] + .into_iter() + .collect(); + src.update_dict(ObjRef::new(7, 0), |d| d.set(b"XObject".to_vec(), Object::Dict(xobjects))).unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[1]).unwrap()); + let names = xobject_names(&part, 0); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{names:?}"); + assert!(!names.iter().any(|n| n.as_slice() == b"ImA"), "what the part doesn't draw still goes: {names:?}"); +} + #[test] fn extract_keeps_images_type3_glyphs_may_draw() { // A Type 3 font without its own /Resources draws through the page's: its glyphs could `Do`. From 3bf6c6f027f5bad86a4a16fccf955b87d449266e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:44:13 +0100 Subject: [PATCH 40/79] M2.6: marking content as an artifact refuses what may be past the damage A tag's content not found before the damage in a damaged stream may be past it. Marking a figure decorative took it for absent: it succeeded, took the figure out of the structure tree and left its content tagged. Marking a tag as an artifact said its content wasn't on the page. Both now refuse with `Damaged`, as when the content is before the damage, and change nothing. Clearing the parent tree read the page's `/StructParents` only when direct, so with it given by reference (ISO 32000-2 lets it be) the tree went on pointing the content at the retired tag. It is read through the reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 11 ++++++++--- crates/a11y/src/tests.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index dd4b43a641..bdc72d2e33 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -417,7 +417,7 @@ enum Marked { /// None of the content is on the page, or with `require_all` not all of it: nothing changed. NotFound, /// Some of it is in a stream damaged partway, which can't be rewritten without losing what - /// follows the damage: nothing changed. + /// follows the damage, or may be past the damage: nothing changed. Damaged, } @@ -442,6 +442,11 @@ fn artifact_marked_content( edits.push((op.span, b"/Artifact BMC".to_vec())); } } + // Content not found may be past the damage in a damaged stream: whether it is on the page + // isn't known. + if found != *ids && streams.iter().any(|(_, s)| s.as_ref().is_some_and(|(_, _, damaged)| *damaged)) { + return Ok(Marked::Damaged); + } if if require_all { found != *ids } else { found.is_empty() } { return Ok(Marked::NotFound); } @@ -494,7 +499,7 @@ pub fn mark_decorative(doc: &mut Document, obj: ObjRef) -> Result<(), AltError> match artifact_marked_content(doc, page, streams, &ids, false)? { // The parent tree no longer points those ids at the figure. Marked::Done => { - if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { clear_parent_tree(doc, key, obj)?; } } @@ -545,7 +550,7 @@ pub fn mark_tag_artifact(doc: &mut Document, obj: ObjRef) -> Result<(), Artifact Marked::NotFound => return Err(ArtifactError::ContentNotFound), Marked::Damaged => return Err(ArtifactError::Damaged), } - if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { clear_parent_tree(doc, key, obj).map_err(|e| ArtifactError::ParentTree(e.to_string()))?; } doc.update_dict(parent, |p| p.set(b"K".to_vec(), replacement))?; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index a129f207cd..78a6d96589 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1129,6 +1129,42 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); } +/// A tag's content not found before the damage in a damaged stream may be past it: it isn't known +/// to be off the page, so neither marking changes anything. +#[test] +fn a_tags_content_past_the_damage_is_not_taken_for_absent() { + use pdfcraft_cos::ObjRef; + let damaged = + damaged_flate(b"/Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n", b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let before = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), before); +} + +/// A page's `/StructParents` given by reference is read through it: the parent tree no longer +/// points the marked content at the tag. +#[test] +fn struct_parents_given_by_reference_are_read() { + use pdfcraft_cos::{ObjRef, Object}; + let by_reference = || { + let mut doc = figures_doc(); + let key = doc.add(Object::Int(0)); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"StructParents".to_vec(), Object::Ref(key))).unwrap(); + doc + }; + let nums = |doc: &Document| doc.get(ObjRef::new(9, 0)).as_dict().and_then(|d| d.get(b"Nums").cloned()); + let mut doc = by_reference(); + mark_decorative(&mut doc, ObjRef::new(7, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Null, Object::Ref(ObjRef::new(8, 0))])]))); + let mut doc = by_reference(); + mark_tag_artifact(&mut doc, ObjRef::new(8, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Ref(ObjRef::new(7, 0)), Object::Null])]))); +} + /// A content stream that doesn't decode at all is kept as stored, and a tag in the page's other /// streams is still marked: only running past the decode budget refuses. #[test] From 91968610c35dc20189f1ebb0c27d23eec8ba50bb Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 12:19:30 +0100 Subject: [PATCH 41/79] M2.6: redaction reads content with its /DecodeParms given by reference `DecodeBudget::decode_strict` read a stream's `/DecodeParms` only when direct, so content behind a predictor whose parameters are an indirect object decoded with the predictor's tag bytes left in. Redaction then found no glyph under a mark, left the text in the file and reported the mark applied. ISO 32000-2 lets those parameters be indirect. `decode_strict` takes the document and reads through `decoding_view`, as `decode_with_damage` does. Its other readers read the parameters too: Remove Hidden Information's hidden text and layers, the audit's covered-text check, and edit's backdrop scan. The accessibility_mark_tag_artifact description says that content which is, or may be, in a stream damaged partway is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/audit/src/cover.rs | 4 +-- crates/automation/src/tools.rs | 2 +- crates/cos/src/budget.rs | 14 +++++----- crates/edit/src/backdrop.rs | 8 +++--- crates/redact/src/interp.rs | 2 +- crates/redact/src/lib.rs | 2 +- crates/redact/src/tests.rs | 48 ++++++++++++++++++++++++++++++++++ 7 files changed, 65 insertions(+), 15 deletions(-) diff --git a/crates/audit/src/cover.rs b/crates/audit/src/cover.rs index bb5adf7ff8..9f771f7040 100644 --- a/crates/audit/src/cover.rs +++ b/crates/audit/src/cover.rs @@ -283,7 +283,7 @@ impl<'a> Walker<'a> { return Err(AuditError::TooManyForms); } // A form stream that cannot be decoded fails the audit: it might hide a cover. - let data = self.budget.decode_strict(stream)?; + let data = self.budget.decode_strict(self.doc, stream)?; // The XObject matrix maps form space into the user space active at `Do`. let mut sub_gs = gs.clone(); sub_gs.ctm = matrix.then(&gs.ctm); @@ -326,7 +326,7 @@ pub fn filled_rects(doc: &Document, page: &Dict) -> Result, Audi let mut streams = Vec::with_capacity(contents.len()); for c in &contents { if let Object::Stream(s) = &*doc.resolve(c) { - streams.push(budget.decode_strict(s)?); + streams.push(budget.decode_strict(doc, s)?); } } let pieces = Pieces::join(&streams); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index af5146377a..e69a57dc33 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, or content not found on its page are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index d5d67503ee..51b05e034d 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -286,10 +286,10 @@ impl DecodeBudget { } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to - /// the allowance. - pub fn decode_strict(&mut self, stream: &Stream) -> Result, CosError> { + /// the allowance, with its `/DecodeParms` given by reference read in `doc`. + pub fn decode_strict(&mut self, doc: &Document, stream: &Stream) -> Result, CosError> { let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); - self.charge(stream, usize::MAX, strict).map(|(data, _)| data) + self.charge(&doc.decoding_view(stream), usize::MAX, strict).map(|(data, _)| data) } /// Decodes `stream` within `max` and the allowance, charging what every filter stage @@ -788,7 +788,7 @@ mod tests { assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new((1 << 20) - 1, 1); let Object::Stream(stream) = &*doc.get(bomb) else { panic!("not a stream") }; - assert!(budget.decode_strict(stream).is_err()); + assert!(budget.decode_strict(&doc, stream).is_err()); assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new(10 << 20, 1); @@ -812,8 +812,9 @@ mod tests { let corrupt = Stream::from_raw(whole.dict.clone(), whole.raw.get(..cut).unwrap_or_default().to_vec()); let partial = corrupt.decoded().map(|d| d.len()).unwrap_or(0); assert!(partial > 0, "a cut flate stream decodes in part"); + let doc = Document::new_empty(); let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_strict(&corrupt).is_err()); + assert!(budget.decode_strict(&doc, &corrupt).is_err()); assert_eq!(budget.remaining(), (10 << 20) - partial); let mut budget = DecodeBudget::new(10 << 20, 1); assert!(budget.decode_within(&corrupt, 1 << 30).is_ok()); @@ -877,6 +878,7 @@ mod tests { dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); let s = Stream::from_raw(dict, raw.to_vec()); assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_strict(&doc, &s).ok(), Some(content.to_vec()), "{parms:?}"); let r = doc.add(Object::Stream(s.clone())); let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); @@ -935,7 +937,7 @@ mod tests { assert_eq!(budget.remaining(), (10 << 20) - charged); assert!(budget.decode_within(&chained, 2 << 20).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 2 * charged); - assert!(budget.decode_strict(&chained).is_ok()); + assert!(budget.decode_strict(&Document::new_empty(), &chained).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 3 * charged); } } diff --git a/crates/edit/src/backdrop.rs b/crates/edit/src/backdrop.rs index 43e16b40ec..f6788bb51c 100644 --- a/crates/edit/src/backdrop.rs +++ b/crates/edit/src/backdrop.rs @@ -208,7 +208,7 @@ pub(crate) fn restore(doc: &mut Document, page: &pdfcraft_model::Page) -> Result let same = |w: &ObjRef| match &*doc.get(*w) { Object::Stream(s) => { [TAG, &b"PCMark"[..], b"Subtype"].iter().all(|k| s.dict.get(k).is_none()) - && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok().as_deref() == Some(bytes.as_slice()) + && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok().as_deref() == Some(bytes.as_slice()) } _ => false, }; @@ -248,7 +248,7 @@ fn restored_copy(doc: &Document, page: &pdfcraft_model::Page, copy: ObjRef) -> O return None; } let hash = note.get(b"Before")?.as_string()?; - let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok()?; + let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok()?; let parsed = pdfcraft_content::parse(head(&bytes)); // The `n` standing in for the fill, after exactly the operators that came before it. let fill = parsed.ops.get(index)?; @@ -384,11 +384,11 @@ fn scan(doc: &Document, p: &pdfcraft_model::Page, swap: Option<(ObjRef, &[u8])>) if b.len() > SCAN_BYTES.saturating_sub(total) { return None; } - budget.decode_strict(&Stream::from_raw(Dict::new(), b.to_vec())).ok()? + budget.decode_strict(doc, &Stream::from_raw(Dict::new(), b.to_vec())).ok()? } // Tolerant decoding can return an empty or partial stream after corruption. // That cannot prove which paint preceded the candidate backdrop. - _ => budget.decode_strict(s).ok()?, + _ => budget.decode_strict(doc, s).ok()?, }; if bytes.len() > SCAN_BYTES.saturating_sub(total).saturating_sub(1) { return None; diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b0a7257cb3..4cb95c81d8 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -979,7 +979,7 @@ fn xobject( } // Strict, like page content: a Form read only in part counts as undecodable. let data = if scope.budget.enter_form() { - scope.budget.decode_strict(s) + scope.budget.decode_strict(doc, s) } else { Err(CosError::Filter(format!("the page draws more than {MAX_PAGE_FORMS} form XObjects"))) }; diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 6d323004ad..8785cb357a 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -290,7 +290,7 @@ pub(crate) fn page_streams_within( match &*doc.resolve(o) { // Strict: a tolerant decode keeps what precedes corruption and drops the rest, which // still draws in viewers that read further; anything not read can't be cleaned. - Object::Stream(s) => data.push(budget.decode_strict(s).map_err(|_| RedactError::Unreadable(index + 1))?), + Object::Stream(s) => data.push(budget.decode_strict(doc, s).map_err(|_| RedactError::Unreadable(index + 1))?), Object::Null => data.push(Vec::new()), _ => return Err(RedactError::Unreadable(index + 1)), } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 8978e0e7d6..c98a21c14f 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -81,6 +81,54 @@ fn glyphs_under_a_mark_go_and_the_rest_stays_put() { assert!(!content(&doc, 0).contains("1234")); } +/// Content whose `/DecodeParms` is given by reference is read through the reference, on the page +/// and in a form: the text under a mark goes. Read with the predictor's tag bytes left in, no glyph +/// was found under the mark, and the redaction left the text in the file and said it was done. +#[test] +fn content_with_parameters_given_by_reference_is_redacted() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + // PNG Up, one column: each byte a row of its own, tagged 2. + let predicted = |doc: &mut Document, mut dict: Dict| { + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET" { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + let mut s = Stream::flate(Dict::new(), &rows); + for (k, v) in dict.iter() { + s.dict.set(k.to_vec(), v.clone()); + } + s + }; + let marked = [[5.0, 90.0, 120.0, 115.0]]; + let mut page = one_page(b"", "", vec![]); + let s = predicted(&mut page, Dict::new()); + page.set(ObjRef::new(4, 0), s); + // The parameters take object 7 and the form 8. + let mut form = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![]); + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set(b"BBox".to_vec(), Object::Array([0, 0, 300, 300].into_iter().map(Object::Int).collect())); + dict.set( + b"Resources".to_vec(), + Object::Dict( + [(b"Font".to_vec(), Object::Dict([(b"F1".to_vec(), Object::Ref(ObjRef::new(5, 0)))].into_iter().collect()))].into_iter().collect(), + ), + ); + let s = predicted(&mut form, dict); + form.set(ObjRef::new(8, 0), s); + for (mut doc, what) in [(page, "page"), (form, "form")] { + assert_eq!(under(&mut doc, 0, &marked), 6, "{what}"); + mark(&mut doc, 0, &marked, ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.marks, r.glyphs), (1, 6), "{what}"); + assert_eq!(under(&mut reopen(&doc), 0, &marked), 0, "{what}"); + } +} + /// #2130: a font with no /Widths is drawn with its embedded program's advances. Measured with a /// guess from its name instead (Courier's 600 units for "Mono"), the end of the line seemed to /// lie past a mark that covers all of it on screen, and was kept. From 2d672d951754dc2b69fa5819a55274ce091bcc13 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 12:45:11 +0100 Subject: [PATCH 42/79] M2.6: redaction keeps an image the page still draws behind parameters by reference Once a redaction removes an image or form under a mark, it takes the name out of the page's resources unless the page's content still draws it elsewhere. It read the page's streams again for that with `decoded`, which reads `/DecodeParms` only when direct, so a stream behind a predictor whose parameters are an indirect object seemed not to draw it, and the page lost the image there. It now looks in what it read for the redaction, with the parameters by reference, and decodes again only the streams it has just written, which have none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/lib.rs | 7 +++- crates/redact/src/tests.rs | 68 +++++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 8785cb357a..db729715f1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -820,7 +820,12 @@ fn redact_page_content( // longer draws so the removed image or form bytes cannot survive in the saved file. let page_content: Vec> = new_list .iter() - .map(|o| match &*doc.resolve(o) { + .zip(&list) + .zip(&data) + .map(|((now, was), read)| match &*doc.resolve(now) { + // Left as it was: what was read above, with its parameters by reference. + _ if now == was => read.clone(), + // Written above by `Stream::flate`, which leaves no `/DecodeParms`. Object::Stream(s) => s.decoded().unwrap_or_default(), _ => Vec::new(), }) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index c98a21c14f..2de12ba374 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -81,31 +81,36 @@ fn glyphs_under_a_mark_go_and_the_rest_stays_put() { assert!(!content(&doc, 0).contains("1234")); } +/// `content` behind a PNG Up predictor whose `/DecodeParms` is an indirect object added to `doc`, +/// in a stream with `dict`'s entries too. +fn predicted(doc: &mut Document, dict: pdfcraft_cos::Dict, content: &[u8]) -> pdfcraft_cos::Stream { + use pdfcraft_cos::{Dict, Object, Stream}; + // One column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + let mut s = Stream::flate(Dict::new(), &rows); + for (k, v) in dict.iter() { + s.dict.set(k.to_vec(), v.clone()); + } + s.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + s +} + /// Content whose `/DecodeParms` is given by reference is read through the reference, on the page /// and in a form: the text under a mark goes. Read with the predictor's tag bytes left in, no glyph /// was found under the mark, and the redaction left the text in the file and said it was done. #[test] fn content_with_parameters_given_by_reference_is_redacted() { - use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; - // PNG Up, one column: each byte a row of its own, tagged 2. - let predicted = |doc: &mut Document, mut dict: Dict| { - let mut rows = Vec::new(); - let mut above = 0u8; - for &b in b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET" { - rows.extend([2, b.wrapping_sub(above)]); - above = b; - } - let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); - dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); - let mut s = Stream::flate(Dict::new(), &rows); - for (k, v) in dict.iter() { - s.dict.set(k.to_vec(), v.clone()); - } - s - }; + use pdfcraft_cos::{Dict, ObjRef, Object}; + let secret = b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET"; let marked = [[5.0, 90.0, 120.0, 115.0]]; let mut page = one_page(b"", "", vec![]); - let s = predicted(&mut page, Dict::new()); + let s = predicted(&mut page, Dict::new(), secret); page.set(ObjRef::new(4, 0), s); // The parameters take object 7 and the form 8. let mut form = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![]); @@ -118,7 +123,7 @@ fn content_with_parameters_given_by_reference_is_redacted() { [(b"Font".to_vec(), Object::Dict([(b"F1".to_vec(), Object::Ref(ObjRef::new(5, 0)))].into_iter().collect()))].into_iter().collect(), ), ); - let s = predicted(&mut form, dict); + let s = predicted(&mut form, dict, secret); form.set(ObjRef::new(8, 0), s); for (mut doc, what) in [(page, "page"), (form, "form")] { assert_eq!(under(&mut doc, 0, &marked), 6, "{what}"); @@ -129,6 +134,31 @@ fn content_with_parameters_given_by_reference_is_redacted() { } } +/// An image removed under a mark stays in the page's resources while another of its content +/// streams, with `/DecodeParms` given by reference, still draws it. Read without the parameters, +/// that stream seemed not to draw it, and the page lost the image there. +#[test] +fn an_image_drawn_again_in_content_with_parameters_by_reference_stays() { + use pdfcraft_cos::{Dict, ObjRef, Object}; + let img = stream("/Type /XObject /Subtype /Image /Width 4 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", [255, 255, 255, 255]); + let mut doc = one_page(b"q 10 0 0 10 10 10 cm /Im1 Do Q", "/XObject << /Im1 7 0 R >>", vec![img]); + let again = predicted(&mut doc, Dict::new(), b"q 10 0 0 10 200 200 cm /Im1 Do Q"); + let again = doc.add(Object::Stream(again)); + doc.update_dict(ObjRef::new(3, 0), |d| d.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(again)]))) + .unwrap(); + mark(&mut doc, 0, &[[5.0, 5.0, 25.0, 25.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.images_removed, 1, "{r:?}"); + let doc = reopen(&doc); + assert!(content(&doc, 0).contains("200 200 cm /Im1 Do"), "{}", content(&doc, 0)); + let p = pdfcraft_model::pages(&doc).swap_remove(0); + let res = doc.resolve(p.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap(); + let xo = doc.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap(); + // The save numbers objects afresh: the image is told by what it is. + let Some(Object::Stream(image)) = xo.reference(b"Im1").map(|r| (*doc.get(r)).clone()) else { panic!("no /Im1: {xo:?}") }; + assert_eq!((image.dict.name(b"Subtype"), image.decoded().ok()), (Some(&b"Image"[..]), Some(vec![255; 4]))); +} + /// #2130: a font with no /Widths is drawn with its embedded program's advances. Measured with a /// guess from its name instead (Courier's 600 units for "Mono"), the end of the line seemed to /// lie past a mark that covers all of it on screen, and was kept. From ebf9c7a3ce19eb71d58dc723181914f843dfb56d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 13:07:06 +0100 Subject: [PATCH 43/79] M2.6: a damaged-stream refusal says the content may be past the damage Since marking a figure decorative or a tag as an artifact refuses content not found beside a damaged stream, the refusal also covers content that only may be past the damage. Its message said the content is in a damaged stream; it now says it is, or may be. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index bdc72d2e33..607a1ec5c4 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,7 +25,9 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, - #[error("the figure's content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + #[error( + "the figure's content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + )] Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), @@ -54,7 +56,9 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, - #[error("the tag's marked content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + #[error( + "the tag's marked content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + )] Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), From 0a350f6809ded06bdc519f5fdbea86d1e78e5482 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:00:43 +0100 Subject: [PATCH 44/79] M2.6: nothing after a damaged content stream, or in one that doesn't decode, is rewritten A page's content streams are read as one stream, so what a stream damaged partway hides past the damage is unknown to the streams after it too. A string or comment the damage leaves open runs on into the next stream, and operators the damage hides can change the graphics state the later streams draw in. Marking a tag's content as an artifact took what read as a BDC in a later stream for the tag's own, and could rewrite bytes inside a string. Moving an image after the damage worked the move out without a cm the damage may hide. Now an edit refuses a change to any stream after one damaged partway (rewritten_contents), and an image isn't arranged into a new stream placed after one. Marking an artifact or a decorative figure refuses content that reads as the tag's in a stream after the damage. Edits and marking before the damage go on as before. A content stream that doesn't decode at all was read as empty and whole. A styled edit whose paragraph marker closes past the page's last operator appended the close to it, so the edit replaced that stream with one holding only the close. Edits now refuse a change to such a stream, and still go on around it. Marking an artifact or a decorative figure whose content isn't found takes it as possibly in such a stream, as it does for content past the damage, and refuses rather than take it for absent. Tests: an_edit_never_rewrites_a_content_stream_damaged_partway (an edit after the damage refused, one before it made), text_a_damaged_stream_runs_on_into_is_not_edited, images_after_a_content_stream_damaged_partway_are_not_moved, a_content_stream_that_does_not_decode_is_never_written_over, a_tags_content_after_a_damaged_stream_is_not_rewritten, a_tags_content_not_found_beside_a_stream_that_does_not_decode_is_not_taken_for_absent, and a_tags_content_in_a_stream_damaged_partway_is_not_rewritten now checks the damaged stream itself and marks content before the damage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 49 ++++++++++----- crates/a11y/src/tests.rs | 64 +++++++++++++++++--- crates/automation/src/tools.rs | 2 +- crates/edit/src/images.rs | 3 +- crates/edit/src/objects.rs | 2 +- crates/edit/src/tests.rs | 107 ++++++++++++++++++++++++++++++--- crates/edit/src/text.rs | 3 +- crates/edit/src/text/lines.rs | 74 ++++++++++++++++++----- 8 files changed, 255 insertions(+), 49 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index 607a1ec5c4..dfe083bec9 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -26,7 +26,7 @@ pub enum AltError { #[error("the page has more content than can be edited")] TooLarge, #[error( - "the figure's content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + "the figure's content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" )] Damaged, #[error("{0}")] @@ -57,7 +57,7 @@ pub enum ArtifactError { #[error("the page has more content than can be edited")] TooLarge, #[error( - "the tag's marked content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + "the tag's marked content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" )] Damaged, #[error(transparent)] @@ -380,9 +380,19 @@ fn content_entries(doc: &Document, page: &Dict) -> Vec { } } -/// A `/Contents` entry and, for a stream, the stream, its decoded data and whether it is damaged -/// partway (the data is then what decoded before the damage). -type DecodedStream = (Object, Option<(Stream, Vec, bool)>); +/// How much of a content stream decoded. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Read { + Whole, + /// Damaged partway: the data is what decoded before the damage. What follows it is unknown, + /// the syntax the later streams start in too. + Damaged, + /// Not at all (a filter this reader lacks, say): the data is empty. + Unread, +} + +/// A `/Contents` entry and, for a stream, the stream, its decoded data and how much of it decoded. +type DecodedStream = (Object, Option<(Stream, Vec, Read)>); /// Decode the page's content streams within the page decode budget. `None` when a stream is /// rejected because the budget is exhausted. @@ -393,14 +403,14 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let (data, damaged) = match budget.decode_with_damage(doc, s, usize::MAX) { - Ok(read) => read, + let (data, read) = match budget.decode_with_damage(doc, s, usize::MAX) { + Ok((data, damaged)) => (data, if damaged { Read::Damaged } else { Read::Whole }), // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. Err(_) if budget.exhausted() => return None, - Err(_) => (Vec::new(), false), + Err(_) => (Vec::new(), Read::Unread), }; - Some((s.clone(), data, damaged)) + Some((s.clone(), data, read)) } _ => None, }; @@ -420,8 +430,8 @@ enum Marked { Done, /// None of the content is on the page, or with `require_all` not all of it: nothing changed. NotFound, - /// Some of it is in a stream damaged partway, which can't be rewritten without losing what - /// follows the damage, or may be past the damage: nothing changed. + /// Some of it is, or may be, in a stream damaged partway or one that doesn't decode, neither of + /// which can be rewritten without losing what wasn't read, or after the damage: nothing changed. Damaged, } @@ -436,27 +446,34 @@ fn artifact_marked_content( require_all: bool, ) -> Result { let props = page_properties(doc, &page.dict); + // The pieces are the streams, in order. let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes, _)| bytes.as_slice())).collect(); + let read: Vec = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, _, read)| *read)).collect(); + let damage = read.iter().position(|r| *r == Read::Damaged); let pieces = pdfcraft_content::Pieces::join(&decoded); let mut found = HashSet::new(); let mut edits = Vec::new(); for op in pieces.parse().ops { if let Some(id) = marked_id(doc, &props, &op, ids) { + // What follows the damage is unknown: a string or comment the damage left open runs + // on into the next stream, so what reads as an operator after it may not be one. + if damage.is_some_and(|d| pieces.piece_of(op.span.start) > d) { + return Ok(Marked::Damaged); + } found.insert(id); edits.push((op.span, b"/Artifact BMC".to_vec())); } } - // Content not found may be past the damage in a damaged stream: whether it is on the page - // isn't known. - if found != *ids && streams.iter().any(|(_, s)| s.as_ref().is_some_and(|(_, _, damaged)| *damaged)) { + // Content not found may be past the damage, or in a stream that doesn't decode: whether it + // is on the page isn't known. + if found != *ids && read.iter().any(|r| *r != Read::Whole) { return Ok(Marked::Damaged); } if if require_all { found != *ids } else { found.is_empty() } { return Ok(Marked::NotFound); } let spliced = pieces.splice(edits); - // The pieces are the streams that decoded, in order. - if streams.iter().filter_map(|(_, s)| s.as_ref()).zip(&spliced).any(|((_, old, damaged), new)| *damaged && old != new) { + if read.iter().zip(&decoded).zip(&spliced).any(|((read, old), new)| *read != Read::Whole && *old != new.as_slice()) { return Ok(Marked::Damaged); } let mut spliced = spliced.into_iter(); diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 78a6d96589..7e719acabb 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1103,8 +1103,9 @@ fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { } /// A tag's content in a stream damaged partway stays tagged: rewriting the stream from what -/// decoded would lose what follows the damage. Nothing changes, the structure tree included. -/// Content in a whole stream beside a damaged one is marked, and the damaged one kept as stored. +/// decoded would lose what follows the damage. Nothing changes, the structure tree and the stream +/// included. Content in a whole stream before a damaged one is marked, and the damaged one kept +/// as stored. #[test] fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { use pdfcraft_cos::{ObjRef, Object}; @@ -1113,20 +1114,69 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { let mut doc = figures_doc(); doc.set(ObjRef::new(4, 0), damaged.clone()); let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); - let (before_contents, before_pt, before_kids) = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + let state = |doc: &Document| (contents(doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0)), doc.get(ObjRef::new(4, 0))); + let before = state(&doc); assert_eq!(figures(&doc).len(), 2, "read up to the damage"); assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); - assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), (before_contents, before_pt, before_kids)); + assert_eq!(state(&doc), before); let mut doc = figures_doc(); let other = doc.add(damaged_flate(b"0 g\n", b"0 0 1 rg 0 0 10 10 re f")); - doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(other)]))) .unwrap(); mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; - assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); - assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); + assert_ne!(list.first(), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); + assert_eq!(list.get(1), Some(&Object::Ref(other)), "kept as stored"); +} + +/// What follows the damage is unknown, the syntax the next stream starts in too, so a tag's +/// content in a stream after a damaged one stays tagged. Here a comment the damage leaves open +/// runs on into the next stream, up to its first end of line, and a string starting there reads +/// as operators, among them what looks like the figure's content: rewritten, it would change the +/// string. Nothing changes, the structure tree included. +#[test] +fn a_tags_content_after_a_damaged_stream_is_not_rewritten() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let in_a_string = b"(a note\n/Figure << /MCID 0 >> BDC 0 g EMC\n) pop /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC"; + for (damaged, after) in [(b"0 g\n".as_slice(), None), (b"0 g % a comment".as_slice(), Some(in_a_string.as_slice()))] { + let mut doc = figures_doc(); + if let Some(after) = after { + doc.set(ObjRef::new(4, 0), Stream::from_raw(Dict::new(), after.to_vec())); + } + let first = doc.add(damaged_flate(damaged, b"\n0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged), "{damaged:?}"); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged), "{damaged:?}"); + assert_eq!(state(&doc), before, "{damaged:?}"); + } +} + +/// A tag's content not found on the page may be in a stream that doesn't decode: it isn't known +/// to be off the page, so neither marking changes anything, the structure tree included. +#[test] +fn a_tags_content_not_found_beside_a_stream_that_does_not_decode_is_not_taken_for_absent() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + doc.update_dict(ObjRef::new(7, 0), |d| d.set(b"K".to_vec(), Object::Int(2))).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"/Figure << /MCID 2 >> BDC 0 g EMC".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(state(&doc), before); } /// A tag's content not found before the damage in a damaged stream may be past it: it isn't known diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index e69a57dc33..4f3b0c374e 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway or that doesn't decode, or after the damage, are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 201612e5b2..5e5c44fab7 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -1428,13 +1428,14 @@ fn arrange_image(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStr }; if let Some(piece) = owned_destination { + let insertion = piece.saturating_add(usize::from(insert_after)); + crate::text::drawn_before_damage(all, insertion)?; let changed = pieces.splice([(target_span, removal_replacement)]); let contents = crate::text::rewritten_contents(doc, all, changed)?; let mut list = match contents { Object::Array(list) => list, one => vec![one], }; - let insertion = piece.saturating_add(usize::from(insert_after)); if insertion > list.len() { return Err(EditError::Invalid("the destination content stream is missing".into())); } diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index c0a57c358a..6a64e3cfce 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -338,7 +338,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result, Ed return Err(EditError::Invalid("the page graphics state is nested too deeply".into())); } } - streams.push(ContentStream { entry: object, data: bytes, damaged: false }); + streams.push(ContentStream { entry: object, data: bytes, read: crate::text::Read::Whole }); } Ok(streams) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index df9bce657d..f3b4e3cc2f 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5172,8 +5172,10 @@ fn damaged_page() -> (Document, pdfcraft_cos::Stream) { } /// An edit that would rewrite a content stream damaged partway is refused: written from what -/// decoded, it would lose what follows the damage, which other readers may still read. An edit -/// in the page's other stream goes on, and leaves the damaged one as it is stored. +/// decoded, it would lose what follows the damage, which other readers may still read. So is an +/// edit in a later stream: what the damage hides can change how that stream reads, a string or +/// comment it leaves open, or the graphics state. An edit in a stream before the damage goes on, +/// and leaves the damaged one as it is stored. #[test] fn an_edit_never_rewrites_a_content_stream_damaged_partway() { let (mut doc, damaged) = damaged_page(); @@ -5183,21 +5185,64 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { let listed = contents(&doc); let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); - assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(refused.to_string().contains("is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); let refused = text::rewrite_block(&mut doc, 0, 0, Some("Edited"), &text::BlockStyle::default()).unwrap_err(); - assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(refused.to_string().contains("is damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + let refused = text::replace_line(&mut doc, 0, 1, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after a content stream that is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); - text::replace_line(&mut doc, 0, 1, "Edited").unwrap(); + let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), damaged.clone()); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); let doc = reopen(&doc); - assert_eq!(texts(&doc), ["Damaged", "Edited"]); + assert_eq!(texts(&doc), ["Edited", "Damaged"]); let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; - assert_eq!(list.first(), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))); - let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(list.get(1), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(5, 0)))); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(5, 0)) else { panic!("not a stream") }; assert_eq!(kept.raw, damaged.raw, "stored as it was"); } +/// A comment the damage leaves open runs on into the next stream, up to its first end of line, +/// so a string starting there reads as operators: a line of text inside it. That line is in a +/// stream after the damage, and isn't edited. +#[test] +fn text_a_damaged_stream_runs_on_into_is_not_edited() { + let mut doc = split_page("", "(note\nBT /F1 12 Tf 72 600 Td (Inside a string) Tj ET\n) pop"); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), damaged_flate(b"0 g % a comment", b"\n")); + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + assert_eq!(texts(&doc), ["Inside a string"], "the comment runs on into the next stream"); + let listed = pdfcraft_model::pages(&doc)[0].dict.get(b"Contents").cloned(); + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(pdfcraft_model::pages(&doc)[0].dict.get(b"Contents").cloned(), listed); +} + +/// A content stream that doesn't decode is never written over. A partial rewrite of the page's +/// last paragraph closes its marker past the last operator read, in the page's last stream, here +/// one that doesn't decode: the rewrite is refused, not that stream replaced by the close. An +/// empty stream there takes the close. +#[test] +fn a_content_stream_that_does_not_decode_is_never_written_over() { + let src = "BT /F1 12 Tf 14.4 TL 1 0 0 1 72 700 Tm (Alpha first row.) Tj T* (Alpha next row.) Tj ET"; + let mut doc = split_page(src, ""); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), pdfcraft_cos::Stream::from_raw(dict, b"0 0 1 rg 0 0 10 10 re f".to_vec())); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + let listed = contents(&doc); + let refused = text::replace_block(&mut doc, 0, 0, "One.\nAlpha next row.").unwrap_err(); + assert!(refused.to_string().contains("can't be decoded"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = split_page(src, ""); + text::replace_block(&mut doc, 0, 0, "One.\nAlpha next row.").unwrap(); + let texts: Vec<_> = text::text_blocks(&reopen(&doc), 0).unwrap().into_iter().map(|b| b.text).collect(); + assert_eq!(texts, ["One. Alpha next row."]); +} + /// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, /// and the page's other streams are read and edited around it. A stream of a few MB is read for /// selecting objects. @@ -5310,6 +5355,52 @@ fn arrange_and_crop_image_in_second_content_stream_succeeds() { assert_eq!(after_crop[1].visible_rect, Some(after_crop[1].rect)); } +/// What follows the damage in a content stream is unknown, the graphics state the later streams +/// draw in too: here it hides a `cm` that doubles them. An image after the damage isn't moved, as +/// the move is worked out without that `cm`, and one before it isn't arranged to after it, beside +/// an added item there. An image before the damage is moved. +#[test] +fn images_after_a_content_stream_damaged_partway_are_not_moved() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let image = "q 100 0 0 100 200 200 cm /Im1 Do Q"; + let hidden_scale = || damaged_flate(b"0 g\n", b"2 0 0 2 0 0 cm"); + let shift = images::ImageChange::Transform([1.0, 0.0, 0.0, 1.0, 10.0, 0.0]); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + + let mut doc = two_stream_image_page("", image); + doc.set(ObjRef::new(4, 0), hidden_scale()); + let listed = contents(&doc); + let refused = images::change_image(&mut doc, 0, 0, &shift).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = two_stream_image_page(image, ""); + let damaged = doc.add(hidden_scale()); + let mut added = Dict::new(); + added.set(b"PCAdded".to_vec(), Object::Dict(Dict::new())); + let added = doc.add(Object::Stream(Stream::from_raw(added, b"q 0 0 1 rg 0 0 10 10 re f Q".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(damaged), Object::Ref(added)])) + }) + .unwrap(); + let listed = contents(&doc); + let refused = images::change_image(&mut doc, 0, 0, &images::ImageChange::Arrange(images::ArrangeOrder::BringToFront)).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); + // Arranged within the content before the damage, it goes on. + doc.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(added), Object::Ref(damaged)])) + }) + .unwrap(); + images::change_image(&mut doc, 0, 0, &images::ImageChange::Arrange(images::ArrangeOrder::BringToFront)).unwrap(); + + let mut doc = two_stream_image_page(image, ""); + doc.set(ObjRef::new(5, 0), hidden_scale()); + images::change_image(&mut doc, 0, 0, &shift).unwrap(); + let moved = images::page_images(&reopen(&doc), 0).unwrap(); + assert!(close(moved[0].rect, [210.0, 200.0, 310.0, 300.0]), "{:?}", moved[0].rect); +} + /// One page whose font draws a few Hebrew letters (ToUnicode): A ב, B ד, C י, D ו, E ק, F ש, /// G ל, H ם. Content streams draw Hebrew in visual order, so "בדיוק" is `(EDCBA)`. fn hebrew_page(content: &str) -> Document { diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 01c0acdb70..40675b4892 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -56,7 +56,8 @@ pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; use interp::{Origin, Paint, union}; use lines::page_dict; pub(crate) use lines::{ - ContentStream, FormBudget, content_streams, content_streams_within, dict_in_place, form_call, joined, resource_dict, rewritten_contents, splice, + ContentStream, FormBudget, Read, content_streams, content_streams_within, dict_in_place, drawn_before_damage, form_call, joined, resource_dict, + rewritten_contents, splice, }; pub use lines::{MAX_FORM_DEPTH, MAX_FORM_VISITS, glyph_text_lines, text_lines}; pub use rewrite::{LineEdit, replace_line, rewrite_block}; diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index bf66cc0f8b..58b4ac6289 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -67,9 +67,19 @@ pub(crate) struct ContentStream { pub(crate) entry: Object, /// Its decoded data, empty when it doesn't decode. pub(crate) data: Vec, - /// The stream is damaged partway: `data` is what decoded before the damage, so the stream - /// can't be written again from it without losing the rest ([`rewritten_contents`]). - pub(crate) damaged: bool, + /// How much of it decoded. + pub(crate) read: Read, +} + +/// How much of a content stream decoded ([`rewritten_contents`] says what each allows). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Read { + Whole, + /// Damaged partway: the data is what decoded before the damage. What follows it is unknown, + /// the graphics state and syntax the later streams start in too. + Damaged, + /// Not at all (a filter this reader lacks, say): the data is empty. + Unread, } /// The page's content streams as listed, each with its decoded data, within one page's @@ -91,15 +101,16 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D }; let mut out = Vec::with_capacity(list.len()); for entry in list { - let (data, damaged) = match &*doc.resolve(&entry) { + let (data, read) = match &*doc.resolve(&entry) { Object::Stream(s) => match budget.decode_with_damage(doc, s, usize::MAX) { - Ok(read) => read, + Ok((data, damaged)) => (data, if damaged { Read::Damaged } else { Read::Whole }), Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), - Err(_) => (Vec::new(), false), + Err(_) => (Vec::new(), Read::Unread), }, - _ => (Vec::new(), false), + // Nothing to lose: a null or missing entry draws nothing. + _ => (Vec::new(), Read::Whole), }; - out.push(ContentStream { entry, data, damaged }); + out.push(ContentStream { entry, data, read }); } Ok(out) } @@ -136,15 +147,50 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> pieces.splice(edits) } +/// An error when content at `/Contents` index `at` (a stream there, or one inserted there) is drawn +/// after a stream damaged partway: what the damage hides can change how it reads. +pub(crate) fn drawn_before_damage(streams: &[ContentStream], at: usize) -> Result<(), EditError> { + if at > first_damaged(streams) { + return Err(EditError::Invalid( + "this change comes after a content stream that is damaged partway, and what the damage hides can change how the rest of the page reads" + .into(), + )); + } + Ok(()) +} + +/// The index of the first stream damaged partway, or the number of streams. +fn first_damaged(streams: &[ContentStream]) -> usize { + streams.iter().position(|s| s.read == Read::Damaged).unwrap_or(streams.len()) +} + /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams /// (keeping their dictionaries), the others stay as they were. An error, before anything is -/// added, when a piece that changed is a damaged stream: written from what decoded, it would -/// lose everything after the damage. +/// added, when a piece that changed is a stream that didn't decode whole, or comes after one +/// damaged partway. Written from what decoded, a damaged stream would lose everything after the +/// damage, and one that doesn't decode all of it. What follows the damage is unknown, so a later +/// stream may not read as it seems: a string or comment the damage left open runs on into it, and +/// operators the damage hides can change the graphics state it draws in. pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { - if streams.iter().zip(&new).any(|(s, data)| s.damaged && s.data != *data) { - return Err(EditError::Invalid( - "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), - )); + // The last stream that changed: when it is drawn before the damage, so are the others. + if let Some(i) = streams.iter().zip(&new).rposition(|(s, data)| s.data != *data) { + drawn_before_damage(streams, i)?; + } + for (s, data) in streams.iter().zip(&new) { + if s.data == *data { + continue; + } + match s.read { + Read::Whole => {} + Read::Damaged => { + return Err(EditError::Invalid( + "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), + )); + } + Read::Unread => { + return Err(EditError::Invalid("this change is in a content stream that can't be decoded, and rewriting it would lose it".into())); + } + } } let mut contents: Vec = Vec::with_capacity(streams.len()); for (s, data) in streams.iter().zip(new) { From 8bf741222016dc9fd997f96100897edc95c4049e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:01:24 +0100 Subject: [PATCH 45/79] M2.6: pruning reads every form that damaged content may draw Earlier in this change, a page or form damaged partway, like one that doesn't decode, kept its own /XObject dictionary whole and was read no further. So a form it draws was never read: when the form's resources are shared with another page, the names the form draws there went, and extracted or split pages lost those images. On main the part before the damage was read, so forms drawn there kept theirs, and forms drawn past it didn't. Now every form the resources of such content name is read as drawn, as any of them may be drawn before the damage or past it. Each set of resources has its forms read once. Test: extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources, with a page and a form each damaged before and past the form they draw, and each not decoding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 92 +++++++++++++++++++++--------------- crates/organize/src/tests.rs | 63 ++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 591e804b1e..bac8ad5f90 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -53,6 +53,9 @@ struct Pruner<'a> { /// Forms already analysed, with the owner of the resources they drew against: drawing one /// again records nothing new, and re-reading it on every `Do` is exponential in the nesting. forms_seen: HashSet<(ObjRef, Owner)>, + /// Owners of resources that content not read whole drew against, whose every form was read + /// as drawn ([`Pruner::unread`]): once is enough. + partly_read: HashSet, } impl Pruner<'_> { @@ -194,39 +197,57 @@ impl Pruner<'_> { } let Some(name) = op.name(0) else { continue }; self.used.entry(key).or_default().insert(name.to_vec()); - let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { continue }; - let Object::Stream(s) = &*self.doc.get(r) else { continue }; - if s.dict.name(b"Subtype") != Some(b"Form") { - continue; - } - let (inner, inner_owner) = match s.dict.get(b"Resources") { - Some(res) => match res.as_ref() { - Some(rr) => match self.doc.get(rr).as_dict().cloned() { - Some(d) => (d, Owner::Resources(rr)), - None => continue, - }, - None => match self.doc.resolve(res).as_dict().cloned() { - // The form carries its resources inline: filter them on the form itself. - Some(d) => (d, Owner::Resources(r)), - None => continue, - }, + self.form(&xobjects, name, resources, owner, depth); + } + } + + /// For content that can't be read whole (damaged partway, or not decoding at all): what it + /// draws is unknown, so its `/XObject` dictionary is kept whole, and every form there is read + /// as drawn, which also reads every form the part before any damage draws. A form whose + /// resources are shared keeps what it draws from them too. + fn unread(&mut self, resources: &Dict, owner: Owner, depth: u8) { + self.protect_type3_resources(resources, owner); + let Some((xobjects, key)) = self.xobjects(resources, owner) else { return }; + self.unsure.insert(key); + if depth > MAX_FORM_DEPTH || !self.partly_read.insert(owner) { + return; + } + for (name, _) in xobjects.iter() { + self.form(&xobjects, name, resources, owner, depth); + } + } + + /// Record what the XObject `name` of `xobjects` draws when it is a form, drawn from content + /// with `resources`. + fn form(&mut self, xobjects: &Dict, name: &[u8], resources: &Dict, owner: Owner, depth: u8) { + let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { return }; + let Object::Stream(s) = &*self.doc.get(r) else { return }; + if s.dict.name(b"Subtype") != Some(b"Form") { + return; + } + let (inner, inner_owner) = match s.dict.get(b"Resources") { + Some(res) => match res.as_ref() { + Some(rr) => match self.doc.get(rr).as_dict().cloned() { + Some(d) => (d, Owner::Resources(rr)), + None => return, }, - // A form without its own resources draws against the ones it was invoked with. - None => (resources.clone(), owner), - }; - if !self.forms_seen.insert((r, inner_owner)) { - continue; - } - #[cfg(test)] - crate::work(1); - match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { - Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), - // What the form draws is unknown, past damage too: keep everything it could reach. - Ok((_, true)) | Err(_) => { - self.protect_type3_resources(&inner, inner_owner); - self.distrust(&inner, inner_owner); - } - } + None => match self.doc.resolve(res).as_dict().cloned() { + // The form carries its resources inline: filter them on the form itself. + Some(d) => (d, Owner::Resources(r)), + None => return, + }, + }, + // A form without its own resources draws against the ones it was invoked with. + None => (resources.clone(), owner), + }; + if !self.forms_seen.insert((r, inner_owner)) { + return; + } + #[cfg(test)] + crate::work(1); + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { + Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), + Ok((_, true)) | Err(_) => self.unread(&inner, inner_owner, depth + 1), } } } @@ -321,6 +342,7 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result Result {} Ok((data, false)) => pruner.content(&data, &res_dict, owner, 0), - // What the page draws past damage is unknown too. - Ok((_, true)) | Err(_) => { - pruner.protect_type3_resources(&res_dict, owner); - pruner.distrust(&res_dict, owner); - } + Ok((_, true)) | Err(_) => pruner.unread(&res_dict, owner, 0), } } let Pruner { used, unsure, glyph_budget_exhausted, .. } = pruner; diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 8d41f474cc..d6273cc65b 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2669,6 +2669,69 @@ fn extract_keeps_images_content_damaged_partway_might_draw() { } } +/// Two pages: the first draws `/Fm` through resources of its own, `/Fm` draws `/Nested` through +/// resources of its own too, and `/Nested` draws `/ImA` through resources it shares with the second +/// page, which draws `/ImB` through them. The three streams are the first page's content, `/Fm` +/// and `/Nested`. +fn doc_shared_form_resources(page: pdfcraft_cos::Stream, fm: pdfcraft_cos::Stream, nested: pdfcraft_cos::Stream) -> Document { + use pdfcraft_cos::ObjRef; + let b: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 200 200] >>".into(), + "<< /Type /Page /Parent 2 0 R /Contents 5 0 R /Resources << /XObject << /Fm 7 0 R >> >> >>".into(), + "<< /Type /Page /Parent 2 0 R /Contents 6 0 R /Resources 8 0 R >>".into(), + raw(""), + raw("/ImB Do"), + raw(""), + "<< /XObject << /ImA 9 0 R /ImB 10 0 R >> >>".into(), + image(1, 1, 1), + image(1, 1, 2000), + raw(""), + ]; + let mut doc = open(build(&b, "/Root 1 0 R")); + doc.set(ObjRef::new(5, 0), page); + let nested_res: Dict = + [(b"XObject".to_vec(), Object::Dict([(b"Nested".to_vec(), Object::Ref(ObjRef::new(11, 0)))].into_iter().collect()))].into_iter().collect(); + for (num, mut form, resources) in [(7, fm, Object::Dict(nested_res)), (11, nested, Object::Ref(ObjRef::new(8, 0)))] { + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + form.dict.set(b"Resources".to_vec(), resources); + doc.set(ObjRef::new(num, 0), form); + } + doc +} + +/// Every form the resources of a page or form damaged partway name may be drawn, before the +/// damage or past it, so each is read: one whose resources another page shares keeps what it draws +/// from them. So is each one content that doesn't decode may draw. +#[test] +fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources() { + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let undecodable = |content: &[u8]| { + let mut s = whole(content); + s.dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + s + }; + let cases = [ + ("page, before the damage", damaged_flate(b"/Fm Do\n", b"0 g"), whole(b"/Nested Do")), + ("page, past the damage", damaged_flate(b"0 g\n", b"/Fm Do"), whole(b"/Nested Do")), + ("page that doesn't decode", undecodable(b"/Fm Do"), whole(b"/Nested Do")), + ("form, before the damage", whole(b"/Fm Do"), damaged_flate(b"/Nested Do\n", b"0 g")), + ("form, past the damage", whole(b"/Fm Do"), damaged_flate(b"0 g\n", b"/Nested Do")), + ("form that doesn't decode", whole(b"/Fm Do"), undecodable(b"/Nested Do")), + ]; + for (what, page, fm) in cases { + let part = full_roundtrip(&extract_pages(&doc_shared_form_resources(page, fm, whole(b"/ImA Do")), &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{what}: {names:?}"); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{what}: {names:?}"); + } + // What nothing draws still goes. + let src = doc_shared_form_resources(whole(b"0 g"), whole(b"/Nested Do"), whole(b"/ImA Do")); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()]); +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 749a3233422614f3a9c97cf7d71a40f5696935b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:01:24 +0100 Subject: [PATCH 46/79] M2.6: tests pin damage in an earlier filter stage, on another page, and in the saved file - cos: a filter chain whose first stage stops at damage and whose second reads all it was given is damaged. - optimize: a page damaged partway stops resampling for the whole document, so another page drawing the same image whole doesn't get it resampled. - redact: after redacting content with /DecodeParms by reference, no stream in the saved file holds the text, read with its parameters. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 11 +++++++++++ crates/optimize/tests/optimize.rs | 18 ++++++++++++++++++ crates/redact/src/tests.rs | 9 ++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 51b05e034d..23bcaf1af7 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -850,6 +850,17 @@ mod tests { dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + // So is an earlier stage's, though the stage after it reads all it was given: a stored + // block of whole hex, then a block of the reserved type (RFC 1951 §3.2.3). + let readable = b"4142>"; + let len = u16::try_from(readable.len()).unwrap_or_default(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + let flate_damaged = Stream::from_raw(hex.dict.clone(), raw); + assert_eq!(budget.decode_with_damage(&doc, &flate_damaged, usize::MAX).ok(), Some((b"AB".to_vec(), true))); } /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7250e9a0a0..2785ddac6c 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -3220,6 +3220,24 @@ fn content_damaged_partway_leaves_every_image_at_its_resolution() { } } +/// One page damaged partway stops resampling for the whole document: another page drawing the same +/// image at its full resolution, whole, doesn't get it resampled either. +#[test] +fn content_damaged_partway_on_one_page_leaves_the_image_another_page_draws() { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 720, 720, 1, false, None); + let damaged = page(&mut doc, &[("Im", img)], ""); + let bad = doc.add(Object::Stream(damaged_flate(b"0 g\n", b"q 720 0 0 720 0 0 cm /Im Do Q"))); + doc.update_dict(damaged, |d| d.set(b"Contents".to_vec(), Object::Ref(bad))).unwrap(); + let whole = page(&mut doc, &[("Im", img)], "q 72 0 0 72 0 0 cm /Im Do Q"); + assert_eq!(effective_resolutions(&doc, &[whole])[&img], 720.0); + let ppi = effective_resolutions(&doc, &[damaged, whole]).get(&img).copied().unwrap_or(f64::INFINITY); + assert_eq!(ppi, f64::INFINITY); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 0, "{r:?}"); + assert_eq!(doc.get(img).as_dict().and_then(|d| d.int(b"Width")), Some(720)); +} + /// `levels` nested forms with their own resources, the innermost drawing `img` over the unit square. fn nested_forms(doc: &mut Document, img: ObjRef, levels: usize) -> ObjRef { let mut inner = form(doc, Some(Object::Dict(resources(xobjects(&[("Im", img)])))), "/Im Do"); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 2de12ba374..f080df0e46 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -130,7 +130,14 @@ fn content_with_parameters_given_by_reference_is_redacted() { mark(&mut doc, 0, &marked, ""); let r = apply(&mut doc, None).unwrap(); assert_eq!((r.marks, r.glyphs), (1, 6), "{what}"); - assert_eq!(under(&mut reopen(&doc), 0, &marked), 0, "{what}"); + let mut saved = reopen(&doc); + assert_eq!(under(&mut saved, 0, &marked), 0, "{what}"); + // Nor is the text left in a stream the page no longer draws, read with its parameters. + for (r, o) in saved.scan_objects() { + let Object::Stream(s) = &*o else { continue }; + let data = pdfcraft_cos::DecodeBudget::new(usize::MAX, 0).decode_with_damage(&saved, s, usize::MAX).map(|(d, _)| d).unwrap_or_default(); + assert!(!data.windows(6).any(|w| w == b"SECRET"), "{what}: {r:?}"); + } } } From 75bf5ed264ba3995247e779308d1d67a689d1fc6 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:35:15 +0100 Subject: [PATCH 47/79] M2.6: a test pins that pruning reads the forms of damaged content at the depth limit Forms are read 8 deep. A damaged form that deep has every form it names read as drawn, so one drawing through resources another page shares keeps what it draws: a_damaged_form_at_the_depth_limit_has_its_forms_read. Mutation testing found nothing pinned that boundary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/tests.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index d6273cc65b..3d956727e3 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2732,6 +2732,34 @@ fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources( assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()]); } +/// Forms are read 8 deep (`prune.rs`'s `MAX_FORM_DEPTH`). A damaged form that deep still has +/// every form it names read as drawn, and one drawing through resources another page shares keeps +/// what it draws from them. +#[test] +fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let mut src = doc_shared_form_resources(whole(b"/C Do"), damaged_flate(b"0 g\n", b"/Nested Do"), whole(b"/ImA Do")); + // Seven forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 8 deep. + let mut next = ObjRef::new(7, 0); + for _ in 1..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 084fa0927edbb387bd7cee5365224e22142096b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:37:51 +0100 Subject: [PATCH 48/79] M2.6: say why pruning reads a form's /Subtype only when direct The sweep flags the read in Pruner::form, moved there by this change. A /Subtype given by reference reads as no form wherever the workspace tells forms from images; resolving it is a change across all of them, and the comment says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index bac8ad5f90..85ddb5f743 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -222,6 +222,9 @@ impl Pruner<'_> { fn form(&mut self, xobjects: &Dict, name: &[u8], resources: &Dict, owner: Owner, depth: u8) { let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { return }; let Object::Stream(s) = &*self.doc.get(r) else { return }; + // A `/Subtype` given by reference reads as no form here, as wherever the workspace tells + // forms from images (redaction, edit and Optimize among them): resolving it is a change + // across all of them. if s.dict.name(b"Subtype") != Some(b"Form") { return; } From d289038ca01432a97e164e29af181c55500bbb76 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:52:43 +0100 Subject: [PATCH 49/79] M2.6: autotagging reads page content with its /DecodeParms given by reference Autotagging a text-only page rewrites its content, marked, from what it read. It read each stream through its filters with /DecodeParms only when direct, so content behind a predictor whose parameters are given by reference was read with the predictor's tag bytes in it, and written back so: with eight-byte rows, a \0 every eight bytes split `24 Tf` into `2 4 Tf` and the text's string, and the tool returned ok. Found by the consumers review of the CodeRabbit round; inherited from main. It now reads through DecodeBudget::decode_strict, which reads the parameters in the document and is strict on damage as before, within the same 16 MiB. pdfcraft-a11y no longer uses pdfcraft-filters. Test: autotag_text_reads_content_with_parameters_given_by_reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.lock | 1 - crates/a11y/Cargo.toml | 1 - crates/a11y/src/autotag.rs | 19 +++++++++---------- crates/a11y/src/tests.rs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8617c45522..520413f003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4254,7 +4254,6 @@ version = "0.18.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", - "pdfcraft-filters", "pdfcraft-fonts", "pdfcraft-model", "thiserror 2.0.21", diff --git a/crates/a11y/Cargo.toml b/crates/a11y/Cargo.toml index 42a86bbad2..adef6053a1 100644 --- a/crates/a11y/Cargo.toml +++ b/crates/a11y/Cargo.toml @@ -14,6 +14,5 @@ workspace = true pdfcraft-cos = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-content = { workspace = true } -pdfcraft-filters = { workspace = true } pdfcraft-fonts = { workspace = true } thiserror.workspace = true diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 9b65f6972b..5f3a75ceb0 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -4,7 +4,7 @@ //! which can then be retagged with the existing tag-property tool after human review. use pdfcraft_content::{Op, parse, serialize_ops}; -use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; const MAX_PAGES: usize = 1_000; const MAX_PAGE_BYTES: usize = 16 * 1024 * 1024; @@ -59,16 +59,15 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R if remaining == 0 { return Err(unsupported(index, "decoded content exceeds 16 MiB")); } - let chain = stream.filters(); - let decoded = if chain.is_empty() { - if stream.raw.len() > remaining { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + // Strictly, and with `/DecodeParms` given by reference: the content is written back from it. + let mut budget = DecodeBudget::new(remaining, 0); + let decoded = budget.decode_strict(doc, stream).map_err(|e| { + if budget.remaining() == 0 { + unsupported(index, "decoded content exceeds 16 MiB") + } else { + unsupported(index, format!("content cannot be decoded strictly: {e}")) } - stream.raw.to_vec() - } else { - pdfcraft_filters::decode(&chain, &stream.raw, remaining) - .map_err(|e| unsupported(index, format!("content cannot be decoded strictly: {e}")))? - }; + })?; all.extend_from_slice(&decoded); all.push(b' '); } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 7e719acabb..304e4ad8c4 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -79,6 +79,39 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { assert_eq!(status(&check(&reopened, &Options::default()), Rule::TaggedContent), Status::Passed); } +/// Page content behind a predictor whose `/DecodeParms` are given by reference is read through +/// them: autotagging writes the text back as it was, marked, not with the predictor's tag bytes in +/// it (a `\0` every eight bytes, which split `24` into `2 4`). +#[test] +fn autotag_text_reads_content_with_parameters_given_by_reference() { + use pdfcraft_cos::{Dict, ObjRef}; + let mut content = b"BT /F1 24 Tf 72 700 Td (Hello World) Tj ET".to_vec(); + content.resize(content.len().div_ceil(8) * 8, b' '); + // PNG rows of eight bytes, each tagged 0 (no prediction). + let rows: Vec = content.chunks(8).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + let parms: Dict = [(b"Predictor".to_vec(), Object::Int(12)), (b"Columns".to_vec(), Object::Int(8))].into_iter().collect(); + let parms = doc.add(Object::Dict(parms)); + let mut predicted = pdfcraft_cos::Stream::flate(Dict::new(), &rows); + predicted.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + doc.set(ObjRef::new(4, 0), predicted); + assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1 }); + let page = &pdfcraft_model::pages(&doc)[0]; + let written = doc.resolve(page.dict.get(b"Contents").unwrap()); + let Object::Stream(written) = &*written else { panic!("page content") }; + let text = String::from_utf8_lossy(&written.decoded_strict().unwrap()).into_owned(); + assert!(text.contains("/F1 24 Tf") && text.contains("(Hello World) Tj"), "{text}"); +} + #[test] fn autotag_text_refuses_paint_and_does_not_modify_the_document() { let mut doc = pdf( From 477f0abd49ea318bede4314e67d4842fd41fa9b1 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:57:10 +0100 Subject: [PATCH 50/79] M2.6: a test pins why autotagging refuses a page it cannot read Content past 16 MiB says so, and content that does not decode strictly says that: autotag_text_says_why_it_cannot_read_a_page. Mutation testing found nothing pinned which message each gets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/tests.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 304e4ad8c4..6275c50437 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -112,6 +112,35 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { assert!(text.contains("/F1 24 Tf") && text.contains("(Hello World) Tj"), "{text}"); } +/// Autotagging refuses a page whose content it can't read whole, and says why: content past +/// 16 MiB, or content that doesn't decode strictly. +#[test] +fn autotag_text_says_why_it_cannot_read_a_page() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let reason = |content: Stream| { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + doc.set(ObjRef::new(4, 0), content); + match autotag_text(&mut doc) { + Err(AutoTagError::Unsupported { reason, .. }) => reason, + other => panic!("{other:?}"), + } + }; + assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoded content exceeds 16 MiB"); + let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); + corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let why = reason(corrupt); + assert!(why.starts_with("content cannot be decoded strictly"), "{why}"); +} + #[test] fn autotag_text_refuses_paint_and_does_not_modify_the_document() { let mut doc = pdf( From 79ea9154cfd63fff70391f63eeaf87b480bf4148 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 02:23:10 +0100 Subject: [PATCH 51/79] M2.6: autotagging says decoding takes more than 16 MiB, not that the content does Read through DecodeBudget::decode_strict since cbc3fc221, a stream counts what each of its filter stages decodes against the 16 MiB, as a page's decode allowance does, where main bounded each stage alone. A page of 12 MiB behind ASCIIHex and Flate, with 9.1 MiB between them, was tagged on main and is refused now; the refusal said the decoded content exceeds 16 MiB, which it does not. It now says decoding the content takes more than that. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/autotag.rs | 5 +++-- crates/a11y/src/tests.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 5f3a75ceb0..b9ffa46669 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -57,13 +57,14 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R let Object::Stream(stream) = &*value else { return Err(unsupported(index, "contents include a non-stream object")) }; let remaining = MAX_PAGE_BYTES.saturating_sub(all.len().saturating_add(1)); if remaining == 0 { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + return Err(unsupported(index, "decoding the content takes more than 16 MiB")); } // Strictly, and with `/DecodeParms` given by reference: the content is written back from it. let mut budget = DecodeBudget::new(remaining, 0); let decoded = budget.decode_strict(doc, stream).map_err(|e| { + // The allowance counts what every filter stage decodes, as a page's does. if budget.remaining() == 0 { - unsupported(index, "decoded content exceeds 16 MiB") + unsupported(index, "decoding the content takes more than 16 MiB") } else { unsupported(index, format!("content cannot be decoded strictly: {e}")) } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 6275c50437..0ad6ecd3e5 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -134,7 +134,7 @@ fn autotag_text_says_why_it_cannot_read_a_page() { other => panic!("{other:?}"), } }; - assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoded content exceeds 16 MiB"); + assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoding the content takes more than 16 MiB"); let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); let why = reason(corrupt); From c3ca0633737be8a96a3e5d83b5e6751ae0c64d70 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 02:35:33 +0100 Subject: [PATCH 52/79] M2.6: a test pins that pruning reads a long chain of forms only to the depth limit A chain of 100,000 forms, whole or each damaged partway, is read only down to the depth limit, so the walk's recursion stays bounded: a_long_chain_of_forms_is_read_only_to_the_depth_limit. Nothing pinned it: with the depth step dropped in either of the two calls that go one form deeper (reading a form whole, and reading one that is damaged), or the depth check in the walk of damaged content made to apply only with the read-once guard, the walk follows the chain to its end and the stack overflows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/tests.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 3d956727e3..dfd9eecf35 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2760,6 +2760,29 @@ fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); } +/// A chain of forms nested far past the depth limit, whole or each damaged partway, is read only +/// down to it: the walk's depth stays bounded however long the chain is, and pruning goes on. +#[test] +fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let xobjects = |names: Dict| Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect()); + for damaged in [false, true] { + let mut doc = doc_shared_form_resources(whole(b"/C Do"), whole(b""), whole(b"")); + let mut names = Dict::new(); + for _ in 0..100_000 { + let mut form = if damaged { damaged_flate(b"", b"/C Do") } else { whole(b"/C Do") }; + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + form.dict.set(b"Resources".to_vec(), xobjects(names)); + let next = doc.add(Object::Stream(form)); + names = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + } + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Resources".to_vec(), xobjects(names))).unwrap(); + assert!(crate::prune::prune_unused_xobjects(&mut doc).is_ok(), "damaged: {damaged}"); + } +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 104e22904ae34c7fbd421b8dcc830ea89c5c5eca Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 03:23:18 +0100 Subject: [PATCH 53/79] M2.6: shorter wording for content in or after damage Marking a figure decorative or a tag's content as an artifact refuses content that may be in a damaged or unreadable content stream, or after the damage. The two errors now say so in one clause, and that nothing was changed, which holds: both return before touching the document. The mark-as-artifact tool's description names the case in a few words. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 8 ++------ crates/automation/src/tools.rs | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index dfe083bec9..77dda106ed 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,9 +25,7 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, - #[error( - "the figure's content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" - )] + #[error("the figure's content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), @@ -56,9 +54,7 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, - #[error( - "the tag's marked content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" - )] + #[error("the tag's marked content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4f3b0c374e..e206f042bc 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway or that doesn't decode, or after the damage, are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that may be in or after a damaged content stream are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, From d1de0a4badd11203ef19ebf7e881c2c9a5c74484 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 03:52:04 +0100 Subject: [PATCH 54/79] M2.6: autotagging's reason for content it can't decode reads as on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since autotagging reads page content through DecodeBudget::decode_strict, the reason it gave for content that doesn't decode carried that error's own text, so it said twice that the data couldn't be decoded: "content cannot be decoded strictly: stream data could not be decoded: FlateDecode: …". It now gives the decoder's reason right after, as main does: "content cannot be decoded strictly: FlateDecode: …". Test: autotag_text_says_why_it_cannot_read_a_page now checks the decoder's reason follows the prefix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/autotag.rs | 8 +++++++- crates/a11y/src/tests.rs | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index b9ffa46669..5c0dc2bd19 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -66,7 +66,13 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R if budget.remaining() == 0 { unsupported(index, "decoding the content takes more than 16 MiB") } else { - unsupported(index, format!("content cannot be decoded strictly: {e}")) + // The decoder's own reason: `CosError::Filter`'s text would say again that the data + // couldn't be decoded. + let why = match e { + pdfcraft_cos::CosError::Filter(why) => why, + other => other.to_string(), + }; + unsupported(index, format!("content cannot be decoded strictly: {why}")) } })?; all.extend_from_slice(&decoded); diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 0ad6ecd3e5..960eb39d3d 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -113,7 +113,7 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { } /// Autotagging refuses a page whose content it can't read whole, and says why: content past -/// 16 MiB, or content that doesn't decode strictly. +/// 16 MiB, or content that doesn't decode strictly, with the decoder's reason right after. #[test] fn autotag_text_says_why_it_cannot_read_a_page() { use pdfcraft_cos::{Dict, ObjRef, Stream}; @@ -138,7 +138,7 @@ fn autotag_text_says_why_it_cannot_read_a_page() { let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); let why = reason(corrupt); - assert!(why.starts_with("content cannot be decoded strictly"), "{why}"); + assert!(why.starts_with("content cannot be decoded strictly: FlateDecode: "), "{why}"); } #[test] From 1d0c3034662aad0b697094afda59753237dba3af Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 13:15:45 +0100 Subject: [PATCH 55/79] M2.6: the redaction proof reads /DecodeParms given by reference The proof that came with #5557 decodes every stream it sweeps through `Ctx::decode`, which called `decode_strict` with the stream alone. This PR's `decode_strict` also takes the document, to read `/DecodeParms` given by reference, so the branch merged with main no longer built. `Ctx::decode` now takes the document, and each of its callers passes the one it reads, so the proof reads such a stream as redaction reads page content and forms. Before, it decoded the stream with the parameters' defaults: a PNG predictor's rows kept their tag bytes, and a copy of the removed text split by them went unreported (`a_survivor_behind_parameters_given_by_reference_is_found`). The test helpers' streams are direct, so they pass an empty document. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests_verify.rs | 18 ++++++++++++++++++ crates/redact/src/verify/extract.rs | 6 +++--- crates/redact/src/verify/geometry.rs | 8 ++++---- crates/redact/src/verify/mod.rs | 10 +++++----- crates/redact/src/verify/snapshot.rs | 2 +- crates/redact/src/verify/sweep.rs | 8 ++++---- crates/redact/src/verify/util.rs | 5 +++-- 7 files changed, 38 insertions(+), 19 deletions(-) diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index ee443f3209..dc3c3c89d9 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -332,6 +332,24 @@ fn a_survivor_in_an_xfa_packet_is_found() { assert!(!proof.passed(), "{}", proof.to_text()); } +#[test] +fn a_survivor_behind_parameters_given_by_reference_is_found() { + // Object 7 holds the text behind Flate and a PNG predictor whose `/DecodeParms` is object 8, + // a reference. Read without them, its rows keep their tag bytes, which split the text; the + // sweep reads them as redaction does, and reports the copy. + let text = format!("note: {SECRET} xx"); + let rows: Vec = text.as_bytes().chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let packed = stream("/Filter /FlateDecode /DecodeParms 8 0 R", Stream::flate(Dict::new(), &rows).raw.as_ref()); + let parms = b"<< /Predictor 12 /Columns 4 >>".to_vec(); + let doc = marked(Spec { extra: vec![packed, parms], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + #[test] fn a_survivor_in_an_unused_object_is_found() { // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index f6db96229b..fe92e4e78d 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -58,7 +58,7 @@ impl Decoder { let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); if let Some(Object::Stream(s)) = tu.as_deref() { - let data = ctx.decode(s)?; + let data = ctx.decode(doc, s)?; if data.len() > MAX_CMAP { return Err(()); } @@ -435,7 +435,7 @@ pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: for o in &list { match &*resolve_strict(doc, o)? { Object::Stream(s) => { - let d = ctx.decode(s)?; + let d = ctx.decode(doc, s)?; if skip_overlay && ctx.is_overlay(&d) { continue; } @@ -536,7 +536,7 @@ impl Extractor<'_> { self.stack.push(r); } let result = (|| { - let data = self.ctx.decode(s)?; + let data = self.ctx.decode(self.doc, s)?; let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) })(); diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index fb69f91351..7ea0de5699 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -98,7 +98,7 @@ impl Geo<'_> { // own placement: where it lands comes from each `/Rect`, so the first // placement tells nothing about the others. Repeated decodes spend the // same bounded budget as everything else. - let Ok(data) = self.ctx.decode(&st) else { + let Ok(data) = self.ctx.decode(self.doc, &st) else { self.unverifiable("an annotation appearance can't be read in full"); continue; }; @@ -278,7 +278,7 @@ impl Geo<'_> { { return; } - let Ok(data) = self.ctx.decode(s) else { + let Ok(data) = self.ctx.decode(self.doc, s) else { self.unverifiable("a form under the region can't be decoded in full"); return; }; @@ -336,7 +336,7 @@ impl Geo<'_> { if depth > MAX_DEPTH { return Err(()); } - let data = self.ctx.decode(s)?; + let data = self.ctx.decode(self.doc, s)?; let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); let res = own.as_ref().unwrap_or(parent); let xobjects = res_dict(self.doc, res, b"XObject"); @@ -464,7 +464,7 @@ pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { return Err(()); } - let data = ctx.decode(s)?; + let data = ctx.decode(doc, s)?; let bits = ncomp.checked_mul(bpc).ok_or(())?; let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); if data.len() < row.checked_mul(h).ok_or(())? { diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs index b374f93fdc..3a36c9e08e 100644 --- a/crates/redact/src/verify/mod.rs +++ b/crates/redact/src/verify/mod.rs @@ -252,7 +252,7 @@ pub struct ProofOptions { #[cfg(test)] pub(crate) mod tests_support { use pdfcraft_content::parse; - use pdfcraft_cos::{Dict, Object, Stream}; + use pdfcraft_cos::{Dict, Document, Object, Stream}; use super::extract::{carries_glyphs, parse_cmap}; use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; @@ -308,9 +308,9 @@ pub(crate) mod tests_support { pub fn decode_budget_runs_out() -> (bool, bool) { let ctx = Ctx::new(Default::default()); let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); - let before = ctx.decode(&s).is_ok(); + let before = ctx.decode(&Document::new_empty(), &s).is_ok(); let spent = Ctx::with_decode_left(50); - (before, spent.decode(&s).is_err()) + (before, spent.decode(&Document::new_empty(), &s).is_err()) } /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb @@ -322,9 +322,9 @@ pub(crate) mod tests_support { drop(big); let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); - let refused = ctx.decode(&bomb).is_err(); + let refused = ctx.decode(&Document::new_empty(), &bomb).is_err(); let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; - let gone = ctx.decode(&small).is_err(); + let gone = ctx.decode(&Document::new_empty(), &small).is_err(); (refused, charged, gone) } diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs index fe74571bb4..8421fc1136 100644 --- a/crates/redact/src/verify/snapshot.rs +++ b/crates/redact/src/verify/snapshot.rs @@ -200,7 +200,7 @@ impl Snapshot { for r in created { let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; let Object::Stream(s) = &*obj else { continue }; - let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + let data = ctx.decode(doc, s).map_err(|_| RedactError::OverlayUnreadable)?; digests.insert(sha256(&[data])); } self.overlay = digests; diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs index 73b44c005d..3c607ed8e7 100644 --- a/crates/redact/src/verify/sweep.rs +++ b/crates/redact/src/verify/sweep.rs @@ -495,7 +495,7 @@ fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, ac for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); } - match acc.ctx.decode(s) { + match acc.ctx.decode(doc, s) { Ok(d) => { let loc = format!("{loc} (decoded)"); for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { @@ -508,7 +508,7 @@ fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, ac Err(()) => acc.raw_only(Surface::DecodedStreams), } } else { - match acc.ctx.decode(s) { + match acc.ctx.decode(doc, s) { // The boxes and labels this run drew are not text to look for. Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), Ok(d) => { @@ -578,7 +578,7 @@ fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet p, Err(()) => { acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); @@ -742,7 +742,7 @@ pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut acc.problem(Surface::Xmp, format!("{loc} is not a stream")); return; }; - let Ok(data) = acc.ctx.decode(s) else { + let Ok(data) = acc.ctx.decode(doc, s) else { acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); return; }; diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs index 77c4309cdd..2051ae15a5 100644 --- a/crates/redact/src/verify/util.rs +++ b/crates/redact/src/verify/util.rs @@ -33,11 +33,12 @@ impl Ctx { /// attempt is charged to the cumulative allowance, failures included: a decode that fails /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded /// before the error — the work of finding out is not work a repeated reference gets for free. - pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + /// `/DecodeParms` given by reference are read in `doc`, as redaction itself reads them. + pub(super) fn decode(&self, doc: &Document, s: &Stream) -> Result, ()> { let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); let mut budget = DecodeBudget::new(cap, usize::MAX); - let result = budget.decode_strict(s); + let result = budget.decode_strict(doc, s); // The attempt's own accounting covers a success (what it decoded), a stream cut short // by corruption (what it decoded first) and one refused past the cap (the cap): charge // it through, whichever way the attempt went. From 0916bec6effba931a260da5215906699d6946233 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 13:15:45 +0100 Subject: [PATCH 56/79] M2.6: a font's streams read /DecodeParms given by reference The redaction proof now reads a font's /ToUnicode with its /DecodeParms given by reference, and `Metrics::from_dict`, which redaction takes the removed text from, read it only with direct ones. Behind a predictor so described, the two got different text for the same codes, so a copy the redaction rightly left counted as a survivor and the proof failed a sound redaction (`a_unicode_map_behind_parameters_given_by_reference_is_read_alike`). Every stream `from_dict` reads now goes through `Document::with_direct_parms`: the /ToUnicode; a composite font's /Encoding, which sets how its codes split (`cmap_streams_read_their_parameters_given_by_reference`); a subset's program and /CIDToGIDMap, which say which of its CIDs have a glyph (`a_subsets_program_and_cid_map_read_their_parameters_given_by_reference`); and the program a simple font without /Widths is measured with (`a_font_without_widths_measures_with_its_embedded_program`). Read without them, as on `main`, such a font was measured with widths guessed from its name, so redaction removed glyphs the mark doesn't cover and left some it does, and the proof, which also guesses from the name, passed. Redaction now measures such a font with its program. The proof still guesses, so it fails that redaction, as it already does on `main` when the program's parameters are direct. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf.rs | 13 +-- crates/fonts/src/pdf/tests.rs | 133 ++++++++++++++++++++++++++---- crates/redact/src/tests_verify.rs | 21 +++++ 3 files changed, 144 insertions(+), 23 deletions(-) diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 240b1c4d55..e9a909ea5c 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -156,13 +156,13 @@ fn cids_with_outlines(doc: &Document, font: &Dict) -> Vec { let Some(program) = dict(doc, descendant.get(b"FontDescriptor")).and_then(|d| d.get(b"FontFile2").cloned()) else { return Vec::new() }; let program = doc.resolve(&program); let Object::Stream(stream) = program.as_ref() else { return Vec::new() }; - let Ok(bytes) = stream.decoded_within(16 << 20) else { return Vec::new() }; + let Ok(bytes) = doc.with_direct_parms(stream).decoded_within(16 << 20) else { return Vec::new() }; let Some(loca) = skrifa::FontRef::from_index(&bytes, 0).ok().and_then(|face| face.loca(None).ok()) else { return Vec::new() }; let outlined = |gid: u32| loca.get_raw(gid as usize).zip(loca.get_raw(gid as usize + 1)).is_some_and(|(start, end)| end > start); match descendant.get(b"CIDToGIDMap").map(|o| doc.resolve(o)).as_deref() { None | Some(Object::Name(_)) => bit_set_below_64k((0..loca.len().min(0x1_0000) as u32).filter(|&gid| outlined(gid))), Some(Object::Stream(map)) => { - let Ok(map) = map.decoded_within(1 << 17) else { return Vec::new() }; + let Ok(map) = doc.with_direct_parms(map).decoded_within(1 << 17) else { return Vec::new() }; bit_set_below_64k( map.as_chunks::<2>() .0 @@ -189,7 +189,7 @@ fn embedded_widths(doc: &Document, font: &Dict, m: &mut Metrics) { let Some(object) = descriptor.get(key) else { continue }; let object = doc.resolve(object); let Object::Stream(stream) = object.as_ref() else { continue }; - let Ok(bytes) = stream.decoded_within(16 << 20) else { continue }; + let Ok(bytes) = doc.with_direct_parms(stream).decoded_within(16 << 20) else { continue }; let Ok(face) = skrifa::FontRef::from_index(&bytes, 0) else { continue }; let upem = f64::from(face.metrics(Size::unscaled(), LocationRef::default()).units_per_em); if upem == 0.0 { @@ -691,7 +691,7 @@ impl Metrics { m.codes = Codes::Two; if let Some(e) = font.get(b"Encoding").map(|e| doc.resolve(e)) && let Object::Stream(s) = &*e - && let Ok(data) = s.decoded_within(MAX_CMAP_BYTES) + && let Ok(data) = doc.with_direct_parms(s).decoded_within(MAX_CMAP_BYTES) { let (spaces, cids) = parse_cmap(&data); if !spaces.is_empty() { @@ -1293,7 +1293,10 @@ fn unicode_map(doc: &Document, font: &Dict, composite: bool) -> Unicode { } match font.get(b"ToUnicode").map(|t| doc.resolve(t)).as_deref() { Some(Object::Stream(s)) => { - if let Ok(data) = s.decoded_within(MAX_CMAP_BYTES) { + // With `/DecodeParms` given by reference, as every stream of the font is read here, + // and as the redaction proof reads the map: the two must get the same text, or a copy + // redaction rightly left counts as a survivor. + if let Ok(data) = doc.with_direct_parms(s).decoded_within(MAX_CMAP_BYTES) { parse_to_unicode(&data, &mut out); } } diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index c2acef191c..69c309c0e9 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -104,6 +104,18 @@ fn font(doc: &mut Document, entries: Vec<(&str, Object)>) -> Dict { d } +/// `data` behind Flate and a PNG predictor whose `/DecodeParms` is a reference: read without +/// it, every row of four bytes keeps its tag byte. +fn behind_parms_by_reference(doc: &mut Document, data: &[u8]) -> Object { + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(12)), (b"Columns".to_vec(), Object::Int(4))]))); + let mut data = data.to_vec(); + data.resize(data.len().next_multiple_of(4), 0); + let rows: Vec = data.chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let mut stream = Stream::flate(Dict::new(), &rows); + stream.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + Object::Ref(doc.add(Object::Stream(stream))) +} + #[test] fn embedded_editor_fonts_are_validated_without_installing_or_exporting() { use pdfcraft_cos::Stream; @@ -124,34 +136,46 @@ fn embedded_editor_fonts_are_validated_without_installing_or_exporting() { } /// #2130: with no `/Widths`, an embedded program's advances win over a guess from the name -/// ("Mono" would mean Courier, 600 units for every glyph). +/// ("Mono" would mean Courier, 600 units for every glyph), the program's `/DecodeParms` given +/// directly or by reference. #[test] fn a_font_without_widths_measures_with_its_embedded_program() { use pdfcraft_cos::Stream; use skrifa::MetadataProvider as _; - let mut doc = Document::new_empty(); let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); let face = skrifa::FontRef::new(bytes).unwrap(); let (size, location) = (skrifa::instance::Size::unscaled(), skrifa::instance::LocationRef::default()); let upem = f64::from(face.metrics(size, location).units_per_em); let advance = |c: char| f64::from(face.glyph_metrics(size, location).advance_width(face.charmap().map(c).unwrap()).unwrap()) / upem; - let program = doc.add(Object::Stream(Stream::flate(Dict::new(), bytes))); - let mut descriptor = Dict::new(); - descriptor.set(b"FontFile2".to_vec(), Object::Ref(program)); - let f = font( - &mut doc, - vec![ - ("Subtype", Object::name("TrueType")), - ("BaseFont", Object::name("HouseMono")), - ("Encoding", Object::name("WinAnsiEncoding")), - ("FontDescriptor", Object::Dict(descriptor)), - ], - ); - let m = Metrics::from_dict(&doc, &f); - for c in ['i', 'l', 'W', ' '] { - assert!((m.width(u32::from(c)) - advance(c)).abs() < 1e-9, "{c}: {} vs {}", m.width(u32::from(c)), advance(c)); + for by_reference in [false, true] { + let mut doc = Document::new_empty(); + let program = if by_reference { + behind_parms_by_reference(&mut doc, bytes) + } else { + Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), bytes)))) + }; + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("TrueType")), + ("BaseFont", Object::name("HouseMono")), + ("Encoding", Object::name("WinAnsiEncoding")), + ("FontDescriptor", Object::Dict(descriptor)), + ], + ); + let m = Metrics::from_dict(&doc, &f); + for c in ['i', 'l', 'W', ' '] { + assert!( + (m.width(u32::from(c)) - advance(c)).abs() < 1e-9, + "{c}, by reference {by_reference}: {} vs {}", + m.width(u32::from(c)), + advance(c) + ); + } + assert!(m.width(u32::from('i')) < 0.4, "not Courier's 0.6, by reference {by_reference}"); } - assert!(m.width(u32::from('i')) < 0.4, "not Courier's 0.6"); } #[test] @@ -342,6 +366,79 @@ fn cmap_streams_decode_within_a_size_cap() { } } +#[test] +fn cmap_streams_read_their_parameters_given_by_reference() { + // Both maps behind a predictor whose `/DecodeParms` is a reference: read without it, every + // row keeps its tag byte inside the map's tokens. + let to_unicode = b"1 begincodespacerange <0000> endcodespacerange 1 beginbfrange <0024> <0026> <0041> endbfrange"; + let encoding = b"1 begincodespacerange <0000> endcodespacerange 1 begincidrange <0000> 0 endcidrange"; + let mut doc = Document::new_empty(); + let (tu, enc) = (behind_parms_by_reference(&mut doc, to_unicode), behind_parms_by_reference(&mut doc, encoding)); + let mut desc = Dict::new(); + desc.set(b"Subtype".to_vec(), Object::name("CIDFontType2")); + let f = font( + &mut doc, + vec![("Subtype", Object::name("Type0")), ("Encoding", enc), ("DescendantFonts", Object::Array(vec![Object::Dict(desc)])), ("ToUnicode", tu)], + ); + let m = Metrics::from_dict(&doc, &f); + assert_eq!(m.decode(&[0, 0x24]), "A", "ToUnicode"); + assert!(matches!(m.codes, Codes::Ranges(_)), "Encoding"); +} + +/// A Type 0 subset over Inter with `/DW` alone, whose CIDs 1 to 7 draw "A B C D" (ToUnicode +/// gives CIDs 1 and 2), its program and its `/CIDToGIDMap` each behind a predictor whose +/// `/DecodeParms` is a reference, or behind Flate alone. +fn inter_subset(program_by_reference: bool, map_by_reference: bool) -> Metrics { + use skrifa::MetadataProvider as _; + let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let face = skrifa::FontRef::new(bytes).unwrap(); + let gid = |c: char| u16::try_from(face.charmap().map(c).unwrap().to_u32()).unwrap(); + let gids: Vec = std::iter::once(0).chain("A B C D".chars().map(gid)).flat_map(u16::to_be_bytes).collect(); + let stream = |doc: &mut Document, data: &[u8], by_reference: bool| { + if by_reference { behind_parms_by_reference(doc, data) } else { Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), data)))) } + }; + let mut doc = Document::new_empty(); + let program = stream(&mut doc, bytes, program_by_reference); + let map = stream(&mut doc, &gids, map_by_reference); + let to_unicode = b"1 begincodespacerange <0000> endcodespacerange 2 beginbfchar <0001> <0041> <0002> <0020> endbfchar"; + let to_unicode = stream(&mut doc, to_unicode, false); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let mut desc = Dict::new(); + desc.set(b"Subtype".to_vec(), Object::name("CIDFontType2")); + desc.set(b"DW".to_vec(), Object::Int(600)); + desc.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + desc.set(b"CIDToGIDMap".to_vec(), map); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("Type0")), + ("BaseFont", Object::name("ABCDEF+Inter")), + ("Encoding", Object::name("Identity-H")), + ("DescendantFonts", Object::Array(vec![Object::Dict(desc)])), + ("ToUnicode", to_unicode), + ], + ); + Metrics::from_dict(&doc, &f) +} + +/// A subset with `/DW` alone has a glyph for a CID when the program's glyph it maps to has an +/// outline, so the space, Inter's empty glyph, is refused. Its program, or its `/CIDToGIDMap`, +/// behind parameters given by reference says the same: read without them, no program says +/// which glyphs are empty, or the map points the CIDs at other glyphs. +#[test] +fn a_subsets_program_and_cid_map_read_their_parameters_given_by_reference() { + let direct = inter_subset(false, false); + assert!(!direct.outlined_cids.is_empty()); + assert_eq!(direct.encode("A").as_deref(), Some(&[0, 1][..])); + assert_eq!(direct.encode(" "), None, "the space's glyph is empty"); + for (program, map) in [(true, false), (false, true)] { + let m = inter_subset(program, map); + assert_eq!(m.outlined_cids, direct.outlined_cids, "program by reference: {program}, map: {map}"); + assert_eq!(m.encode(" "), None, "program by reference: {program}, map: {map}"); + } +} + #[test] fn composite_fonts_use_to_unicode() { let mut doc = Document::new_empty(); diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index dc3c3c89d9..28853cc4a4 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -350,6 +350,27 @@ fn a_survivor_behind_parameters_given_by_reference_is_found() { assert!(!proof.passed(), "{}", proof.to_text()); } +#[test] +fn a_unicode_map_behind_parameters_given_by_reference_is_read_alike() { + // The font's /ToUnicode (object 7) reads codes A..H as "SECRETVA", behind Flate and a PNG + // predictor whose `/DecodeParms` is object 8, a reference. The page draws the codes twice, + // once under the mark. Redaction takes the removed text from that map, and the proof the page + // text it allows for: read by one and not the other, the copy that stays counted as a + // survivor of a sound redaction. + let cmap = b"/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapName /Test def\n1 begincodespacerange\n<00> \nendcodespacerange\n8 beginbfchar\n<41> <0053>\n<42> <0045>\n<43> <0043>\n<44> <0052>\n<45> <0045>\n<46> <0054>\n<47> <0056>\n<48> <0041>\nendbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"; + let mut data = cmap.to_vec(); + data.resize(data.len().next_multiple_of(4), b' '); + let rows: Vec = data.chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let to_unicode = stream("/Filter /FlateDecode /DecodeParms 8 0 R", Stream::flate(Dict::new(), &rows).raw.as_ref()); + let parms = b"<< /Predictor 12 /Columns 4 >>".to_vec(); + let content = b"BT /F1 10 Tf 10 200 Td (ABCDEFGH) Tj ET BT /F1 10 Tf 10 100 Td (ABCDEFGH) Tj ET"; + let mut doc = marked_content(content, Spec { extra: vec![to_unicode, parms], ..Spec::default() }); + doc.update_dict(ObjRef::new(5, 0), |font| font.set(b"ToUnicode".to_vec(), Object::Ref(ObjRef::new(7, 0)))).unwrap(); + let (result, proof) = run(&mut doc); + assert!(result.is_ok(), "{result:?}"); + assert!(proof.passed(), "{}", proof.to_text()); +} + #[test] fn a_survivor_in_an_unused_object_is_found() { // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that From 8ac51f833b891e29a40e3a8563e53a08c9320fb5 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:21:28 +0100 Subject: [PATCH 57/79] cos: charge the object-stream pre-scan to one budget per rebuild Overlapping unterminated dictionaries could each be read up to 1 MiB. A shared budget (the file length) is now taken off on every path; a spent budget keeps the object a candidate, as before. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document/reader.rs | 163 +++++++++++++++++------------- crates/cos/src/document/tests.rs | 27 ++++- 2 files changed, 112 insertions(+), 78 deletions(-) diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 7e8a4e3b49..6a325ec933 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -623,6 +623,9 @@ impl Document { // `/ObjStm` type; the others are not parsed for the object-stream check below. let mut candidates: HashSet = HashSet::new(); let headers = object_headers(&data); + // What the dictionary pre-check may read in all, so overlapping unterminated dictionaries + // cost no more than one more pass over the file. + let mut scan_budget = data.len(); // Where `/XRef` occurs in the file, found in one pass. let mut xref_marks = Vec::new(); let mut mark_from = 0; @@ -633,7 +636,7 @@ impl Document { for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if may_be_object_stream(&data, body) { + if may_be_object_stream(&data, body, &mut scan_budget) { candidates.insert(num); } else { candidates.remove(&num); @@ -758,83 +761,86 @@ const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; /// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the /// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end /// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must -/// then treat the object as a possible object stream. -fn dictionary_end(data: &[u8], from: usize) -> Option> { - let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT)); +/// then treat the object as a possible object stream. `limit` ends the read; the second value is +/// how far the read got, for the caller's budget. +fn dictionary_end(data: &[u8], from: usize, limit: usize) -> (Option>, usize) { let mut i = from; - while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { - if data[i] == b'%' { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } else { - i += 1; - } - } - if data.get(i..i + 2) != Some(b"<<") { - return if i < limit { Some(None) } else { None }; - } - i += 2; - let (mut dicts, mut arrays) = (1usize, 0usize); - while i < limit { - match data[i] { - b'(' => { - let mut depth = 1usize; - i += 1; - while i < limit && depth > 0 { - match data[i] { - b'\\' => i += 1, - b'(' => depth += 1, - b')' => depth -= 1, - _ => {} - } - i += 1; - } - if depth > 0 { - return None; - } - } - b'<' if data.get(i + 1) == Some(&b'<') => { - dicts += 1; - i += 2; - } - b'<' => { - i += 1; - while i < limit && data[i] != b'>' { - if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { - return None; - } - i += 1; - } - i += 1; - } - b'>' if data.get(i + 1) == Some(&b'>') => { - if arrays > 0 { - return None; - } - dicts -= 1; - i += 2; - if dicts == 0 { - return Some(Some(i)); - } - } - b'%' => { + let result = 'scan: { + while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { + if data[i] == b'%' { while i < limit && !matches!(data[i], b'\n' | b'\r') { i += 1; } - } - b'[' => { - arrays += 1; + } else { i += 1; } - b']' => { - arrays = arrays.saturating_sub(1); - i += 1; - } - _ => i += 1, } - } - None + if data.get(i..i + 2) != Some(b"<<") { + break 'scan if i < limit { Some(None) } else { None }; + } + i += 2; + let (mut dicts, mut arrays) = (1usize, 0usize); + while i < limit { + match data[i] { + b'(' => { + let mut depth = 1usize; + i += 1; + while i < limit && depth > 0 { + match data[i] { + b'\\' => i += 1, + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + i += 1; + } + if depth > 0 { + break 'scan None; + } + } + b'<' if data.get(i + 1) == Some(&b'<') => { + dicts += 1; + i += 2; + } + b'<' => { + i += 1; + while i < limit && data[i] != b'>' { + if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { + break 'scan None; + } + i += 1; + } + i += 1; + } + b'>' if data.get(i + 1) == Some(&b'>') => { + if arrays > 0 { + break 'scan None; + } + dicts -= 1; + i += 2; + if dicts == 0 { + break 'scan Some(Some(i)); + } + } + b'%' => { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } + b'[' => { + arrays += 1; + i += 1; + } + b']' => { + arrays = arrays.saturating_sub(1); + i += 1; + } + _ => i += 1, + } + } + None + }; + (result, i.min(limit)) } /// Whether the object whose body starts at `data[from..]` could be an object stream: its @@ -843,8 +849,19 @@ fn dictionary_end(data: &[u8], from: usize) -> Option> { /// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a /// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a /// candidate, so a doubtful object is parsed as it always was. -pub(super) fn may_be_object_stream(data: &[u8], from: usize) -> bool { - match dictionary_end(data, from) { +/// +/// `budget` is the bytes all the calls of one rebuild may still read. A call reads at most +/// [`DICTIONARY_SCAN_LIMIT`] bytes or what is left, takes what it read off the budget on every +/// path, and says "maybe" once the budget cannot cover it. Dictionaries that do not overlap +/// add up to no more than the file, so only unterminated ones can use the budget up. +pub(super) fn may_be_object_stream(data: &[u8], from: usize, budget: &mut usize) -> bool { + let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT.min(*budget))); + if *budget == 0 { + return true; + } + let (end, read) = dictionary_end(data, from, limit); + *budget = budget.saturating_sub(read.saturating_sub(from).max(1)); + match end { Some(None) => false, None => true, Some(Some(end)) => { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index d2234fca41..ddd7965eda 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1193,6 +1193,11 @@ fn parses_as_object_stream(file: &[u8]) -> bool { matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) } +/// A scan budget that never runs out. +fn unlimited() -> usize { + usize::MAX +} + #[test] fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a @@ -1205,14 +1210,26 @@ fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { ] { let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); - assert!(may_be_object_stream(&file, 7), "{dict}"); + assert!(may_be_object_stream(&file, 7, &mut unlimited()), "{dict}"); } // Ordinary streams stay out; their data is never read. let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; - assert!(!may_be_object_stream(plain, 7)); - assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7)); + assert!(!may_be_object_stream(plain, 7, &mut unlimited())); + assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7, &mut unlimited())); // Unterminated: not certain, so a candidate. - assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7)); + assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7, &mut unlimited())); + // The scan is charged to the shared budget on every path, and a spent budget says "maybe" + // without reading. + let open = b"1 0 obj\n<< /Note (open "; + let mut budget = 1000; + assert!(may_be_object_stream(open, 7, &mut budget)); + assert_eq!(budget, 1000 - (open.len() - 7)); + let mut budget = 3; + assert!(may_be_object_stream(plain, 7, &mut budget), "a budget too small for the dictionary is a candidate"); + assert_eq!(budget, 0); + let mut budget = 0; + assert!(may_be_object_stream(plain, 7, &mut budget)); + assert_eq!(budget, 0); // Random dictionaries built from tricky pieces: whatever parses as an object stream is // a candidate. let pieces: [&str; 22] = [ @@ -1257,7 +1274,7 @@ fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); if parses_as_object_stream(&file) { hits += 1; - assert!(may_be_object_stream(&file, 7), "rejected an object stream: {dict:?}"); + assert!(may_be_object_stream(&file, 7, &mut unlimited()), "rejected an object stream: {dict:?}"); } } assert!(hits > 1000, "the generator must produce object streams ({hits})"); From a77cd9cb784fda453650f812ccdeb3787d5d7338 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:23:12 +0100 Subject: [PATCH 58/79] cos: reachable_objects keeps no parsed objects full_save_order retained up to 64 MiB of parsed objects for the write pass, which reachable_objects then dropped unused. It now asks for no retention. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/writer.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 6f8fbc92c1..a90ecea525 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -279,7 +279,10 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, /// object where the budget allowed keeping it. type SaveOrder = (HashMap, Vec<(ObjRef, bool)>, Vec>>); -fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { +/// Number the objects reachable from the trailer. `retain` keeps parsed objects for the write +/// pass, up to the budget. A caller that needs only the numbers passes `false`, so the traversal +/// holds no parsed objects beyond the current one. +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict, retain: bool) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -326,7 +329,7 @@ fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result Result Result, CosError> { let mut reader = doc.object_reader(); let trailer_in = reader.document.trailer().clone(); - let (_, order, _) = full_save_order(&mut reader, &trailer_in)?; + let (_, order, _) = full_save_order(&mut reader, &trailer_in, false)?; Ok(order.into_iter().map(|(r, _)| r.num).collect()) } @@ -351,7 +354,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut reader = doc.object_reader(); stamp_mod_date(&mut reader.document, opts); let trailer_in = reader.document.trailer().clone(); - let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in)?; + let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in, true)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); From 82c5fe2e42c1c393f9479f04fe2e11b9581c0c00 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:24:20 +0100 Subject: [PATCH 59/79] cos tests: keep the whole-real property inside the integer range i32::MIN is outside Annex C's integer range and stays a real token (-2147483648.). The generator now stops at -i32::MAX and a separate test pins the i32::MIN output. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/tests/it/serialize_format.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs index 19781a802f..8cfba2baf1 100644 --- a/crates/cos/tests/it/serialize_format.rs +++ b/crates/cos/tests/it/serialize_format.rs @@ -51,6 +51,12 @@ fn every_byte_value_in_hex_strings_octal_escapes_and_names() { assert_eq!(name, want); } +#[test] +fn whole_real_at_i32_min_stays_a_real_token() { + // -2^31 is outside the Annex C integer range (±2^31-1), so it keeps its trailing point. + assert_eq!(bytes_of(&Object::Real(f64::from(i32::MIN))), b"-2147483648."); +} + proptest! { #[test] fn integers_match_the_formatter(i in any::()) { @@ -58,7 +64,7 @@ proptest! { } #[test] - fn whole_reals_match_the_formatter(i in any::()) { + fn whole_reals_match_the_formatter(i in -i32::MAX..=i32::MAX) { prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); } From 442a6b56e3343a6a9d78a9a77a32ef8c5734c3b0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:38:17 +0100 Subject: [PATCH 60/79] automation: skip optimize-to-target rungs that repeat earlier settings A rung equal to settings already run is not optimized again, and attempts counts only runs. The target test uses a TempDir so it cleans up even when it fails. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/documents.rs | 6 ++++++ crates/automation/tests/it/automation.rs | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index a3a517d298..f6cb33d4c4 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -391,6 +391,8 @@ impl Automation { let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; let mut attempts = 1; let mut step_used = 0; + // Settings already optimized with; a rung that changes nothing is not run again. + let mut tried = vec![settings.clone()]; if let Some(t) = target { // Stronger image settings, one rung per attempt, until the copy fits. Every attempt // starts from the open document, so a failed one changes nothing; the number of @@ -409,7 +411,11 @@ impl Automation { img.compression = Compression::Jpeg(q.min(quality)); } } + if tried.contains(&s) { + continue; + } let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + tried.push(s); attempts += 1; if b2.len() < bytes.len() { (bytes, r, step_used) = (b2, r2, k + 1); diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index a54791f5f0..e8042fe9ce 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -7196,7 +7196,9 @@ fn a_page_printed_in_strips_gets_small_through_tools() { #[test] fn optimizing_to_a_target_size() { - let dir = workdir("optimize-target"); + // Dropped at the end of the test, also when an assertion fails. + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); let (w, h) = (2400u32, 1600u32); let px: Vec = (0..w * h) .flat_map(|i| { @@ -7241,6 +7243,10 @@ fn optimizing_to_a_target_size() { // A JPEG caller gets the quality step. let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); + // Settings already at the ladder's floor leave every rung equal to the first run: no retries. + let floor = json!({ "ppi": 9, "above_ppi": 9, "compression": "jpeg", "quality": 1 }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "floor.pdf", "color": floor, "gray": floor, "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(1)), "{r}"); // Confinement still applies. assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); } From cc4bd4914caa11f6db73d029edd7ff7dcafcf57d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:40:37 +0100 Subject: [PATCH 61/79] regex: enable perf-dfa so the dfa_size_limit on user patterns applies Without the feature the lazy-DFA cache does not exist and the limit set in UserRegex::new does nothing. Matching stays linear-time. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29d28051d8..a684ca9857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,8 +80,9 @@ hayro-jbig2 = "0.3" # foundation thiserror = "2" -# Linear-time matching (no backtracking) for caller-supplied redaction patterns. -regex = { version = "1", default-features = false, features = ["std", "unicode"] } +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. `perf-dfa` gives +# the lazy-DFA cache that `RegexBuilder::dfa_size_limit` bounds (the limit is inert without it). +regex = { version = "1", default-features = false, features = ["std", "unicode", "perf-dfa"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" From 95f8eaa63f3a2a9661f2cd168ff704367338b3a8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:42:02 +0100 Subject: [PATCH 62/79] ui: count the third item gap in the mode bar's right-hand width The print, save and info buttons each have 4 pt after them (three gaps), so the reserved width is 116 pt, not 112; the search field no longer overlaps the tabs. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 5e1fef8860..02b78d31fb 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -470,8 +470,9 @@ fn tab( theme::hand(resp).on_hover_text(if dirty { crate::i18n::fmt(tl!("{name} — unsaved changes"), &[("name", &shown)]) } else { shown }) } -/// Width the print, save and info buttons and their gaps take in the mode bar. -const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 2.0 * 4.0 + 8.0; +/// Width the print, save and info buttons and their gaps take in the mode bar: three buttons, the +/// 4 pt `item_spacing` after each of them (three gaps, not two), and the 8 pt spacer. +const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 3.0 * 4.0 + 8.0; /// Narrowest search field the mode bar shows; below it the field becomes an icon button. const MODE_BAR_SEARCH_MIN: f32 = 140.0; From 7b6baec065f29f4fe15158542575fbd94fd9f4a3 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:54:58 +0100 Subject: [PATCH 63/79] sign: an empty /Contents is not an earlier signature Only a non-empty signature value counts when deciding whether the first real signature may drop /NeedAppearances. Co-Authored-By: Claude Sonnet 5.5 --- crates/sign/src/pdf/signing.rs | 3 ++- crates/sign/tests/it/pdf.rs | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/sign/src/pdf/signing.rs b/crates/sign/src/pdf/signing.rs index b41d7fe91a..fb3569602b 100644 --- a/crates/sign/src/pdf/signing.rs +++ b/crates/sign/src/pdf/signing.rs @@ -153,7 +153,8 @@ fn sign_inner( } check_signing_permissions(doc, opts)?; let mut doc = doc.clone(); - let earlier_signatures = !super::validate::signature_contents(&doc).is_empty(); + // A signature field whose /V holds an empty /Contents is not a signature yet. + let earlier_signatures = super::validate::signature_contents(&doc).iter().any(|der| !der.is_empty()); let root = doc.root().ok_or_else(|| SignError::Pdf("the document has no catalog".into()))?; let alg = id.key.preferred_digest(); let reserve = diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 137960e83b..bcfe41631e 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -150,6 +150,22 @@ fn signing_drops_need_appearances() { } } +/// A field whose /V has an empty /Contents is no earlier signature: the first real one still +/// drops `/NeedAppearances`. +#[test] +fn an_empty_signature_value_does_not_count_as_an_earlier_signature() { + let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); + let bytes = fixture_with( + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R 7 0 R] /NeedAppearances true >> >>", + &[ + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Empty) /Rect [10 20 100 70] /P 3 0 R /F 4 /V 8 0 R >>", + b"<< /Type /Sig /Filter /Adobe.PPKLite /SubFilter /ETSI.CAdES.detached /Contents <> >>", + ], + ); + let signed = pdfcraft_sign::sign(&open(&bytes), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + assert!(form_of(&signed).get(b"NeedAppearances").is_none()); +} + /// A second signer leaves the form's flags alone: changing them would count as a change to /// what the first signature covered. #[test] From 761fd4ff5112c454eaefa8fcc9b10452b599b1d6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 18:35:28 +0100 Subject: [PATCH 64/79] Notice bar: stack the buttons only where the message still has room Below about 100 pt of content width the stacked message wrapped to a column thousands of points tall and pushed the hotbar off screen, which broke the narrow-viewport hotbar picker test merged from main. That width is under the smallest supported window, so the single row stays there. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas/bars.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas/bars.rs b/crates/ui-egui/src/canvas/bars.rs index e588013a74..3e12bd6ce3 100644 --- a/crates/ui-egui/src/canvas/bars.rs +++ b/crates/ui-egui/src/canvas/bars.rs @@ -253,9 +253,13 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te const ICON: f32 = 16.0; /// Content width below which the buttons stack under the message. const STACK_BELOW: f32 = 200.0; + /// Content width under which stacking is not tried: the message would wrap to a column of a + /// few characters, thousands of points tall, and push the hotbar off the screen. Such a width + /// is below the smallest supported window, so the single row stays as it was. + const STACK_ABOVE: f32 = 100.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); - if width < STACK_BELOW { + if (STACK_ABOVE..STACK_BELOW).contains(&width) { // A narrow document area (a half-width window): the message takes the full width and // the buttons sit on a row of their own below it, rather than squeezing it to nothing. ui.with_layout(egui::Layout::left_to_right(egui::Align::Min), |ui| { From 86cb375869428d0a69c0638321bfde366fb117c1 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 18:41:02 +0100 Subject: [PATCH 65/79] M2.6: edits stop at a content stream that doesn't decode, as at damage An edit was refused after a content stream damaged partway, since what the damage hides can change how the later streams read, the graphics state they draw in among it. A stream that doesn't decode at all hides all of its operators, and an edit after one went ahead. It now stops there too: text lines and blocks, and images (arrange, crop, transform, move), as selecting objects on `main` already does for a page with such a stream. An edit in a stream before it goes on, and the stream is kept as stored. A file may give a stream's /Filter by reference, which isn't read here, so the case needs no damage: `content_after_a_stream_that_does_not_decode_is_not_edited` has such a stream move the text and image after it with a `cm`. `a_content_stream_that_does_not_decode_is_kept_and_edited_around` becomes `..._kept_and_edited_before`, its edit now in the stream before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/tests.rs | 43 ++++++++++++++++++++++++++++++----- crates/edit/src/text/lines.rs | 20 ++++++++-------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index f3b4e3cc2f..5f9787a8e6 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5191,7 +5191,7 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { assert!(refused.to_string().contains("is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); let refused = text::replace_line(&mut doc, 0, 1, "Edited").unwrap_err(); - assert!(refused.to_string().contains("comes after a content stream that is damaged partway"), "{refused}"); + assert!(refused.to_string().contains("comes after a content stream that is damaged or can't be decoded"), "{refused}"); assert_eq!(contents(&doc), listed); let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); @@ -5244,25 +5244,56 @@ fn a_content_stream_that_does_not_decode_is_never_written_over() { } /// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, -/// and the page's other streams are read and edited around it. A stream of a few MB is read for +/// and the page's streams before it are read and edited. A stream of a few MB is read for /// selecting objects. #[test] -fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { - let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); +fn a_content_stream_that_does_not_decode_is_kept_and_edited_before() { + let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); let mut dict = pdfcraft_cos::Dict::new(); dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); let unknown = pdfcraft_cos::Stream::from_raw(dict, b"BT /F1 12 Tf 72 700 Td (Unread) Tj ET".to_vec()); - doc.set(pdfcraft_cos::ObjRef::new(4, 0), unknown.clone()); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), unknown.clone()); text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); let doc = reopen(&doc); assert_eq!(text::text_lines(&doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(), ["Edited"]); - let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(5, 0)) else { panic!("not a stream") }; assert_eq!(kept.raw, unknown.raw, "stored as it was"); let large = format!("{}0 0 m 10 10 l S", " ".repeat(2 << 20)); assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); } +/// A content stream that doesn't decode hides its operators as damage does. Here its `/Filter` is +/// given by reference, which a file may do and which isn't read here, and it holds a `cm` that +/// moves what the page draws after it. Text and images there aren't edited, as the edit would be +/// worked out without the `cm`. +#[test] +fn content_after_a_stream_that_does_not_decode_is_not_edited() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let hidden_move = |doc: &mut Document| { + let filter = doc.add(Object::name("FlateDecode")); + let mut s = Stream::flate(Dict::new(), b"1 0 0 1 0 -300 cm"); + s.dict.set(b"Filter".to_vec(), Object::Ref(filter)); + doc.set(ObjRef::new(4, 0), s); + }; + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (After) Tj ET"); + hidden_move(&mut doc); + let listed = contents(&doc); + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after a content stream that is damaged or can't be decoded"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = two_stream_image_page("", "q 100 0 0 100 200 200 cm /Im1 Do Q"); + hidden_move(&mut doc); + let listed = contents(&doc); + let shift = images::ImageChange::Transform([1.0, 0.0, 0.0, 1.0, 10.0, 0.0]); + let refused = images::change_image(&mut doc, 0, 0, &shift).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); +} + /// A content stream whose `/DecodeParms`, or a parameter in them, is given by reference is read /// through the reference: its text is listed and edited, not read with the predictor's tag bytes /// left in it. diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index 58b4ac6289..bc0812e00d 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -148,29 +148,31 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> } /// An error when content at `/Contents` index `at` (a stream there, or one inserted there) is drawn -/// after a stream damaged partway: what the damage hides can change how it reads. +/// after a stream not read whole, damaged partway or not decoding at all: what wasn't read can +/// change how it reads, the graphics state it draws in among it. pub(crate) fn drawn_before_damage(streams: &[ContentStream], at: usize) -> Result<(), EditError> { if at > first_damaged(streams) { return Err(EditError::Invalid( - "this change comes after a content stream that is damaged partway, and what the damage hides can change how the rest of the page reads" + "this change comes after a content stream that is damaged or can't be decoded, and what wasn't read can change how the rest of the page reads" .into(), )); } Ok(()) } -/// The index of the first stream damaged partway, or the number of streams. +/// The index of the first stream not read whole, or the number of streams. fn first_damaged(streams: &[ContentStream]) -> usize { - streams.iter().position(|s| s.read == Read::Damaged).unwrap_or(streams.len()) + streams.iter().position(|s| s.read != Read::Whole).unwrap_or(streams.len()) } /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams /// (keeping their dictionaries), the others stay as they were. An error, before anything is -/// added, when a piece that changed is a stream that didn't decode whole, or comes after one -/// damaged partway. Written from what decoded, a damaged stream would lose everything after the -/// damage, and one that doesn't decode all of it. What follows the damage is unknown, so a later -/// stream may not read as it seems: a string or comment the damage left open runs on into it, and -/// operators the damage hides can change the graphics state it draws in. +/// added, when a piece that changed is a stream that didn't decode whole, or comes after one. +/// Written from what decoded, a damaged stream would lose everything after the damage, and one +/// that doesn't decode all of it. What follows the damage is unknown, so a later stream may not +/// read as it seems: a string or comment the damage left open runs on into it, and operators the +/// damage hides can change the graphics state it draws in. A stream that doesn't decode hides its +/// operators all the same. pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { // The last stream that changed: when it is drawn before the damage, so are the others. if let Some(i) = streams.iter().zip(&new).rposition(|(s, data)| s.data != *data) { From ae677644e0b54860a16b2053a0044e19366df18d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 18:41:21 +0100 Subject: [PATCH 66/79] M2.6: Extract and Split prune nothing past the form depth limit Pruning reads forms 8 deep. Past that a form's /XObject dictionary was kept whole, but the forms it draws weren't read, nor what they draw through resources of their own. When those resources are shared with content that was read, they were cut to the names that content draws, and the images only the deep forms draw were dropped from the part. This was so on `main` for a form read whole, and in this change's reading of a damaged form too. A walk the depth limit stops now leaves the file unpruned, as an overlarge Type 3 resource graph already does: `nothing_is_pruned_past_the_depth_limit` (a form 9 deep, whole or damaged partway, drawing a form that draws through resources the second page shares). That also settles a form first reached past the limit and drawn again higher up, whose second reading `forms_seen` skips. Down to the limit pruning goes on, under a page whole or damaged partway and with the form 8 deep whole or damaged: `pruning_goes_on_down_to_the_depth_limit`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 22 ++++++---- crates/organize/src/tests.rs | 78 ++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 85ddb5f743..b771b07da9 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -40,12 +40,15 @@ enum Owner { struct Pruner<'a> { doc: &'a Document, used: HashMap>>, - /// Dictionaries whose drawn names can't be known for certain (undecodable content, nesting - /// past the depth limit, Type 3 glyphs drawing through them). They are left untouched: - /// pruning must never drop an XObject something still draws. + /// Dictionaries whose drawn names can't be known for certain (undecodable content, Type 3 + /// glyphs drawing through them). They are left untouched: pruning must never drop an XObject + /// something still draws. unsure: HashSet, /// An overlarge glyph-resource graph is not safe to prune at all. glyph_budget_exhausted: bool, + /// Nor is a file with forms nested past [`MAX_FORM_DEPTH`]: what the forms there draw isn't + /// read, and they may draw it through resources shared with content that was. + too_deep: bool, glyph_work_left: usize, glyph_seen: HashSet, /// An indirect font's explicit resources have one owner regardless of which page uses it. @@ -188,7 +191,7 @@ impl Pruner<'_> { return; }; if depth > MAX_FORM_DEPTH { - self.unsure.insert(key); + self.too_deep = true; return; } for op in pdfcraft_content::parse(data).ops { @@ -209,7 +212,11 @@ impl Pruner<'_> { self.protect_type3_resources(resources, owner); let Some((xobjects, key)) = self.xobjects(resources, owner) else { return }; self.unsure.insert(key); - if depth > MAX_FORM_DEPTH || !self.partly_read.insert(owner) { + if depth > MAX_FORM_DEPTH { + self.too_deep = true; + return; + } + if !self.partly_read.insert(owner) { return; } for (name, _) in xobjects.iter() { @@ -341,6 +348,7 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result Result pruner.unread(&res_dict, owner, 0), } } - let Pruner { used, unsure, glyph_budget_exhausted, .. } = pruner; - if glyph_budget_exhausted { + let Pruner { used, unsure, glyph_budget_exhausted, too_deep, .. } = pruner; + if glyph_budget_exhausted || too_deep { return Ok(0); } let mut removed = 0; diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index dfd9eecf35..7eb677a84c 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2733,8 +2733,8 @@ fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources( } /// Forms are read 8 deep (`prune.rs`'s `MAX_FORM_DEPTH`). A damaged form that deep still has -/// every form it names read as drawn, and one drawing through resources another page shares keeps -/// what it draws from them. +/// every form it names read as drawn: here `/Nested`, which draws through resources another page +/// shares. Read 9 deep, it leaves the file unpruned, so what it draws from them stays. #[test] fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { use pdfcraft_cos::ObjRef; @@ -2760,8 +2760,78 @@ fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); } +/// Past the depth limit a form isn't read, nor what the forms it draws draw: one of those may draw +/// through resources another page shares, and keep images there that page doesn't draw, so nothing +/// is pruned. Here `/Fm` is 9 deep, whole or damaged partway, and `/Nested`, which it draws, draws +/// `/ImA` through the resources the second page draws `/ImB` through. +#[test] +fn nothing_is_pruned_past_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + for (what, fm) in [("whole", whole(b"/Nested Do")), ("damaged", damaged_flate(b"0 g\n", b"/Nested Do"))] { + let mut src = doc_shared_form_resources(whole(b"/C Do"), fm, whole(b"/ImA Do")); + // Eight forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 9 deep. + let mut next = ObjRef::new(7, 0); + for _ in 0..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{what}: {names:?}"); + } +} + +/// Down to the depth limit pruning goes on: under a page whole or damaged partway, a form 8 deep, +/// whole or damaged, is read, and the image nothing draws, `/ImA`, still goes from the resources +/// the second page draws `/ImB` through. +#[test] +fn pruning_goes_on_down_to_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let cases = [ + ("whole", whole(b"/C Do"), whole(b"0 g")), + ("damaged form", whole(b"/C Do"), damaged_flate(b"0 g\n", b"0 g")), + ("damaged page", damaged_flate(b"/C Do\n", b"0 g"), whole(b"0 g")), + ]; + for (what, page, fm) in cases { + let mut src = doc_shared_form_resources(page, fm, whole(b"")); + // `/Fm` names an image, not `/Nested`: nothing is read 9 deep. + let image: Dict = [(b"Img".to_vec(), Object::Ref(ObjRef::new(9, 0)))].into_iter().collect(); + let resources = Object::Dict([(b"XObject".to_vec(), Object::Dict(image))].into_iter().collect()); + src.update_dict(ObjRef::new(7, 0), |f| f.set(b"Resources".to_vec(), resources)).unwrap(); + // Seven forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 8 deep. + let mut next = ObjRef::new(7, 0); + for _ in 1..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()], "{what}"); + } +} + /// A chain of forms nested far past the depth limit, whole or each damaged partway, is read only -/// down to it: the walk's depth stays bounded however long the chain is, and pruning goes on. +/// down to it: the walk's depth stays bounded however long the chain is, and the run ends without +/// an error, having pruned nothing. #[test] fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { use pdfcraft_cos::ObjRef; @@ -2779,7 +2849,7 @@ fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { names = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); } doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Resources".to_vec(), xobjects(names))).unwrap(); - assert!(crate::prune::prune_unused_xobjects(&mut doc).is_ok(), "damaged: {damaged}"); + assert!(matches!(crate::prune::prune_unused_xobjects(&mut doc), Ok(0)), "damaged: {damaged}"); } } From f19ecc4635a0a9e99d22dd032d8b24ec3438fdde Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 18:49:31 +0100 Subject: [PATCH 67/79] Merge upstream/main; drop the object-stream pre-check from reconstruct Main now reports unreadable objects from the parse of every object in Document::reconstruct, so the pre-check that skipped that parse is gone. The single-pass /XRef positions and the rarest-byte find stay. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document/reader.rs | 131 +---------------------------- crates/cos/src/document/tests.rs | 119 ++++++-------------------- crates/cos/tests/it/main.rs | 1 - crates/cos/tests/it/reconstruct.rs | 62 -------------- 4 files changed, 27 insertions(+), 286 deletions(-) delete mode 100644 crates/cos/tests/it/reconstruct.rs diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 4b2da4fc8b..18f5ce848a 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -635,13 +635,7 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - // Objects whose bytes (the last object of a number wins, as in `entries`) could hold an - // `/ObjStm` type; the others are not parsed for the object-stream check below. - let mut candidates: HashSet = HashSet::new(); let headers = object_headers(&data); - // What the dictionary pre-check may read in all, so overlapping unterminated dictionaries - // cost no more than one more pass over the file. - let mut scan_budget = data.len(); // Where `/XRef` occurs in the file, found in one pass. let mut xref_marks = Vec::new(); let mut mark_from = 0; @@ -652,11 +646,6 @@ impl Document { for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if may_be_object_stream(&data, body, &mut scan_budget) { - candidates.insert(num); - } else { - candidates.remove(&num); - } // `/XRef` within the 1024 bytes after the header (the first one that is there). let first = xref_marks.partition_point(|&p| p < body); if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) @@ -673,8 +662,7 @@ impl Document { *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Objects inside object streams. for num in objstms { - if candidates.contains(&num) - && let Ok(o) = self.try_get(num) + if let Ok(o) = self.try_get(num) && let Object::Stream(s) = &*o && s.dict.name(b"Type") == Some(b"ObjStm") && let Ok(stm) = self.objstm(num) @@ -773,123 +761,6 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } -/// Longest dictionary [`may_be_object_stream`] reads before it gives up and says "maybe". -const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; - -/// Where the dictionary that starts at `data[from..]` (after `N G obj`) ends, reading it as the -/// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the -/// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end -/// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must -/// then treat the object as a possible object stream. `limit` ends the read; the second value is -/// how far the read got, for the caller's budget. -fn dictionary_end(data: &[u8], from: usize, limit: usize) -> (Option>, usize) { - let mut i = from; - let result = 'scan: { - while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { - if data[i] == b'%' { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } else { - i += 1; - } - } - if data.get(i..i + 2) != Some(b"<<") { - break 'scan if i < limit { Some(None) } else { None }; - } - i += 2; - let (mut dicts, mut arrays) = (1usize, 0usize); - while i < limit { - match data[i] { - b'(' => { - let mut depth = 1usize; - i += 1; - while i < limit && depth > 0 { - match data[i] { - b'\\' => i += 1, - b'(' => depth += 1, - b')' => depth -= 1, - _ => {} - } - i += 1; - } - if depth > 0 { - break 'scan None; - } - } - b'<' if data.get(i + 1) == Some(&b'<') => { - dicts += 1; - i += 2; - } - b'<' => { - i += 1; - while i < limit && data[i] != b'>' { - if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { - break 'scan None; - } - i += 1; - } - i += 1; - } - b'>' if data.get(i + 1) == Some(&b'>') => { - if arrays > 0 { - break 'scan None; - } - dicts -= 1; - i += 2; - if dicts == 0 { - break 'scan Some(Some(i)); - } - } - b'%' => { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } - b'[' => { - arrays += 1; - i += 1; - } - b']' => { - arrays = arrays.saturating_sub(1); - i += 1; - } - _ => i += 1, - } - } - None - }; - (result, i.min(limit)) -} - -/// Whether the object whose body starts at `data[from..]` could be an object stream: its -/// dictionary holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in -/// `/Obj#53tm`). Only the dictionary is searched, not the data of a stream. Where the -/// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a -/// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a -/// candidate, so a doubtful object is parsed as it always was. -/// -/// `budget` is the bytes all the calls of one rebuild may still read. A call reads at most -/// [`DICTIONARY_SCAN_LIMIT`] bytes or what is left, takes what it read off the budget on every -/// path, and says "maybe" once the budget cannot cover it. Dictionaries that do not overlap -/// add up to no more than the file, so only unterminated ones can use the budget up. -pub(super) fn may_be_object_stream(data: &[u8], from: usize, budget: &mut usize) -> bool { - let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT.min(*budget))); - if *budget == 0 { - return true; - } - let (end, read) = dictionary_end(data, from, limit); - *budget = budget.saturating_sub(read.saturating_sub(from).max(1)); - match end { - Some(None) => false, - None => true, - Some(Some(end)) => { - let dict = &data[from..end]; - dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() - } - } -} - /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position /// after `obj`). fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 59221dc0ce..f69ffae042 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1,4 +1,3 @@ -use super::reader::may_be_object_stream; use super::*; std::thread_local! { @@ -1920,98 +1919,6 @@ fn a_prev_reference_to_an_object_holding_the_offset_reads_that_offset() { /// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used /// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after /// editing one Info field dropped every other. -/// Whether parsing `body` as an object gives a stream whose /Type is /ObjStm. -fn parses_as_object_stream(file: &[u8]) -> bool { - matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) -} - -/// A scan budget that never runs out. -fn unlimited() -> usize { - usize::MAX -} - -#[test] -fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { - // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a - // string that comes before the /Type. - for dict in [ - "<< /Note (9 0 obj\n) /Type /ObjStm /N 0 /First 0 >>", - "<< /Note (>>stream\n) /Type /ObjStm /N 0 /First 0 >>", - "<< /Note (a (nested\n9 0 obj\nstream) \\) )) /Type /ObjStm >>", - "<< /Note <3e3e> /Other << /A (>>\nstream\n) >> /Type /ObjStm >>", - ] { - let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); - assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); - assert!(may_be_object_stream(&file, 7, &mut unlimited()), "{dict}"); - } - // Ordinary streams stay out; their data is never read. - let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; - assert!(!may_be_object_stream(plain, 7, &mut unlimited())); - assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7, &mut unlimited())); - // Unterminated: not certain, so a candidate. - assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7, &mut unlimited())); - // The scan is charged to the shared budget on every path, and a spent budget says "maybe" - // without reading. - let open = b"1 0 obj\n<< /Note (open "; - let mut budget = 1000; - assert!(may_be_object_stream(open, 7, &mut budget)); - assert_eq!(budget, 1000 - (open.len() - 7)); - let mut budget = 3; - assert!(may_be_object_stream(plain, 7, &mut budget), "a budget too small for the dictionary is a candidate"); - assert_eq!(budget, 0); - let mut budget = 0; - assert!(may_be_object_stream(plain, 7, &mut budget)); - assert_eq!(budget, 0); - // Random dictionaries built from tricky pieces: whatever parses as an object stream is - // a candidate. - let pieces: [&str; 22] = [ - "/Type /ObjStm", - "/Type/ObjStm", - "/Type /Obj#53tm", - "/T#79pe /ObjStm", - "(", - ")", - "\\", - "(>>stream\n)", - "(9 0 obj\n)", - "<3e3e>", - "<", - ">", - ">>", - "<<", - "[", - "]", - "% c\n", - "% >>\n", - "/N 0", - " ", - "\n", - "", - ]; - let mut seed = 0x2545_F491_4F6C_DD1Du64; - let mut next = move || { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - seed - }; - let mut hits = 0; - for _ in 0..200_000 { - let mut dict = String::from("<<"); - for _ in 0..(next() % 9) { - dict.push_str(pieces[(next() % pieces.len() as u64) as usize]); - dict.push(' '); - } - dict.push_str(if next() % 8 == 0 { "" } else { ">>" }); - let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); - if parses_as_object_stream(&file) { - hits += 1; - assert!(may_be_object_stream(&file, 7, &mut unlimited()), "rejected an object stream: {dict:?}"); - } - } - assert!(hits > 1000, "the generator must produce object streams ({hits})"); -} - #[test] fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { let table = build( @@ -2282,3 +2189,29 @@ fn the_editor_cache_charges_a_dictionary_for_the_values_it_holds() { let sum = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); assert_eq!(cache.owned(), sum, "the kept total is the cache's own"); } + +#[test] +fn find_agrees_with_a_plain_window_search() { + let naive = |hay: &[u8], needle: &[u8], from: usize, to: usize| { + if from >= to || needle.is_empty() || to - from < needle.len() { + return None; + } + hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + }; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let alphabet = b"/XRef t\n<>0"; + for _ in 0..20_000 { + let hay: Vec = (0..next() % 200).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); + let needles: [&[u8]; 5] = [b"/XRef", b"trailer", b"R", b"ff", b"<<"]; + let needle = needles[(next() % 5) as usize]; + let from = (next() % (hay.len() as u64 + 1)) as usize; + let to = from + (next() % (hay.len() as u64 + 1 - from as u64)) as usize; + assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}"); + } +} diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index d0b73b16d4..1bd7ee1fd6 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -5,6 +5,5 @@ mod corpus; mod encryption; mod generations; mod proptests; -mod reconstruct; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs deleted file mode 100644 index 4611c0d17e..0000000000 --- a/crates/cos/tests/it/reconstruct.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Rebuilding the cross-reference index of a damaged file finds the objects inside object -//! streams however the stream's type is written. - -use std::sync::Arc; - -use pdfcraft_cos::{Document, Object}; - -/// A file whose `startxref` points nowhere, with three object streams (type written plainly, as -/// `/Obj#53tm`, and after a string that contains the word `stream`), and one ordinary stream -/// whose data mentions `ObjStm` and `#`. -fn damaged() -> Vec { - let mut out = b"%PDF-1.5\n".to_vec(); - let mut add = |n: u32, body: &[u8]| { - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - let objstm = |dict_prefix: &str, members: &[(u32, &str)]| { - let mut header = String::new(); - let mut body = String::new(); - for (n, text) in members { - header.push_str(&format!("{n} {} ", body.len())); - body.push_str(text); - body.push('\n'); - } - let mut b = - format!("<< {dict_prefix} /N {} /First {} /Length {} >>\nstream\n", members.len(), header.len(), header.len() + body.len()).into_bytes(); - b.extend_from_slice(header.as_bytes()); - b.extend_from_slice(body.as_bytes()); - b.extend_from_slice(b"\nendstream"); - b - }; - add(1, b"<< /Type /Catalog /Pages 2 0 R >>"); - add(2, b"<< /Type /Pages /Kids [] /Count 0 >>"); - add(3, &objstm("/Type /ObjStm", &[(10, "<< /From 3 >>"), (11, "[3]")])); - add(4, &objstm("/Type /Obj#53tm", &[(12, "<< /From 4 >>")])); - add(5, &objstm("/Note (a stream) /Type /ObjStm", &[(13, "<< /From 5 >>")])); - let data = b"/Type /ObjStm # 14 0 << /From 6 >>"; - let mut plain = format!("<< /Length {} >>\nstream\n", data.len()).into_bytes(); - plain.extend_from_slice(data); - plain.extend_from_slice(b"\nendstream"); - add(6, &plain); - // Strings before the type that look like an object header, or like the end of the - // dictionary and the `stream` keyword, and a hex string and a comment. - add(7, &objstm("/Note (\n99 0 obj\n) /Type /ObjStm", &[(15, "<< /From 7 >>")])); - add(8, &objstm("/Note (>>stream\n) /Type /ObjStm", &[(16, "<< /From 8 >>")])); - add(9, &objstm("/Note <3e3e> /Other << /A (a (b) \\) >>\nstream\n) >> % >>\nstream\n\n/Type /ObjStm", &[(17, "<< /From 9 >>")])); - out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); - out -} - -#[test] -fn objects_in_object_streams_are_found_however_the_type_is_written() { - let doc = Document::open(Arc::new(damaged())).expect("opens"); - assert!(doc.reconstructed()); - for n in [10, 11, 12, 13, 15, 16, 17] { - let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); - assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); - } - // The data of an ordinary stream is not an object stream, whatever it says. - assert!(!doc.object_numbers().contains(&14)); -} From 569f631f2b074784bf479020fce8fb862347fda9 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 20:18:07 +0100 Subject: [PATCH 68/79] M2.6: an image's move changes no state a stream not read whole draws in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving or resizing an image in a stream before one damaged partway, or one that doesn't decode, goes on, as the edit changes no stream past it. But the image's `q cm … Q` can close past that stream, and what the stream draws inside the scope is unknown. When the image looked alone there, the move changed the scope's `cm` in place, and anything the stream draws in it moved too: a square drawn there in the image's `cm` moved with the image, as on `main`. A scope still open after such a stream now counts as one that paints something else (`first_op_after_damage`, read by `image_scopes` and `enclosing_transform`). An operator counts by its keyword there: a page may split its content between an operator's operands and its keyword, so a `Q` whose operands come before the hidden stream still closes after it. The move gives the `Do` a `cm` of its own and leaves the scope's as it was; with a clip on the image, the clip stays, as one the stream may share. A crop goes around the `Do` alone rather than at the scope's start, where it would clip what the stream draws. A crop PdfCraft put in such a scope isn't removed, and the refusal says why rather than that there is no crop. Arrange is unchanged: it takes out the scope it finds, which changes the stream past it and is refused. `an_image_whose_scope_closes_past_a_stream_not_read_whole_moves_alone`: a stream that doesn't decode and one damaged, each also with the square's operands before what is hidden (ending the stream before, or in the damaged stream before its damage); the image with no clip, a clip, and a PdfCraft crop; moved, cropped, and its crop removed. A scope that closes in a damaged stream before its damage still takes the move in its `cm`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/images.rs | 57 ++++++++++++++++++-------- crates/edit/src/tests.rs | 77 +++++++++++++++++++++++++++++++++++ crates/edit/src/text.rs | 4 +- crates/edit/src/text/lines.rs | 9 ++++ 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 5e5c44fab7..fbb37ec1f0 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -527,12 +527,17 @@ fn exclusive_clip(ops: &[Op], op: usize) -> Option<(usize, Matrix)> { /// The `q cm … Q` whose only content after its `cm` is `start..=end` (a `Do`, or a balanced /// `q … Q` scope), with the index and value of that `cm`. Another transform of the same /// placement is folded into it, so repeated moves and resizes don't nest one more `q` each. -fn enclosing_transform(ops: &[Op], start: usize, end: usize) -> Option<(usize, Matrix)> { +/// None when its `Q` is at or past `hidden` ([`crate::text::first_op_after_damage`]): a stream +/// between that wasn't read may draw in that `cm` too. +fn enclosing_transform(ops: &[Op], start: usize, end: usize, hidden: usize) -> Option<(usize, Matrix)> { let at = start.checked_sub(1)?; let cm = ops.get(at)?; if !ops.get(at.checked_sub(1)?)?.is("q") || !ops.get(end.checked_add(1)?)?.is("Q") || !cm.is("cm") || cm.operands.len() != 6 { return None; } + if end.checked_add(1)? >= hidden { + return None; + } let m = Matrix(cm.nums::<6>()?); m.0.iter().all(|v| v.is_finite()).then_some((at, m)) } @@ -1052,7 +1057,9 @@ fn determine_arrange_spans( has_other_paint: bool, marked_unit: bool, ) -> Result<(std::ops::Range, usize, usize, usize), EditError> { - let scope = image_scopes(ops, target, "arranged") + // Nothing is hidden for Arrange: it takes out the scope it finds, so a scope that closes past a + // stream not read whole changes a stream past it, which `rewritten_contents` refuses. + let scope = image_scopes(ops, target, "arranged", ops.len()) .ok() .and_then(|s| s.isolated) .filter(|&(start, end, _)| start >= target_unit.start && end < target_unit.end); @@ -1491,8 +1498,10 @@ impl ImageScopes { } } -/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. -fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result { +/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. A scope +/// still open at op `hidden` ([`crate::text::first_op_after_damage`]) holds a stream that wasn't +/// read, which may paint there: it counts as painting something else. +fn image_scopes(ops: &[Op], target: usize, done: &str, hidden: usize) -> Result { struct Scope { start: usize, matrix: Matrix, @@ -1511,6 +1520,11 @@ fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result