From 71c5d2cc35f703ee3aadf859b0587a7a18f2fe23 Mon Sep 17 00:00:00 2001 From: jonathanwxh-cell Date: Sun, 25 Oct 2026 01:05:50 +0800 Subject: [PATCH] fix(a11y): mark vector graphics and images as artifacts in autotag_text (#4080) --- crates/a11y/src/autotag.rs | 264 ++++++++++++++++++++++++------------- crates/a11y/src/tests.rs | 107 ++++++++++++++- 2 files changed, 274 insertions(+), 97 deletions(-) diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 9b65f6972b..0889215bdb 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -1,9 +1,12 @@ -//! Create a valid, reviewable structure tree for PDFs whose painted content is text only. +//! Create a valid, reviewable structure tree for untagged PDFs. //! -//! The writer refuses mixed content and existing tags. Each text object becomes a paragraph, -//! which can then be retagged with the existing tag-property tool after human review. +//! Each text object becomes a paragraph, which can then be retagged with the existing +//! tag-property tool after human review. Painted paths, shadings and images outside text objects +//! are wrapped in `/Artifact BMC … EMC` so the resulting page passes the tagged-content check. -use pdfcraft_content::{Op, parse, serialize_ops}; +use std::collections::{HashMap, HashSet}; + +use pdfcraft_content::{Op, Pieces, serialize_ops}; use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; const MAX_PAGES: usize = 1_000; @@ -37,25 +40,27 @@ struct Plan { page: ObjRef, ops: Vec, spans: Vec<(usize, usize)>, + artifacts: HashMap, } fn unsupported(page: usize, reason: impl Into) -> AutoTagError { AutoTagError::Unsupported { page: page + 1, reason: reason.into() } } -fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> Result, AutoTagError> { - let Some(contents) = page.dict.get(b"Contents") else { return Ok(Vec::new()) }; +fn content_pieces(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> Result { + let Some(contents) = page.dict.get(b"Contents") else { return Ok(Pieces::join::<&[u8]>(&[])) }; let resolved = doc.resolve(contents); - let pieces: Vec = match &*resolved { + let items: Vec = match &*resolved { Object::Array(items) => items.clone(), Object::Stream(_) => vec![contents.clone()], _ => return Err(unsupported(index, "contents are not a stream or array of streams")), }; - let mut all = Vec::new(); - for item in pieces { + let mut pieces = Vec::with_capacity(items.len()); + let mut total = 0usize; + for item in items { let value = doc.resolve(&item); let Object::Stream(stream) = &*value else { return Err(unsupported(index, "contents include a non-stream object")) }; - let remaining = MAX_PAGE_BYTES.saturating_sub(all.len().saturating_add(1)); + let remaining = MAX_PAGE_BYTES.saturating_sub(total.saturating_add(1)); if remaining == 0 { return Err(unsupported(index, "decoded content exceeds 16 MiB")); } @@ -69,10 +74,18 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R pdfcraft_filters::decode(&chain, &stream.raw, remaining) .map_err(|e| unsupported(index, format!("content cannot be decoded strictly: {e}")))? }; - all.extend_from_slice(&decoded); - all.push(b' '); + total = total.saturating_add(decoded.len()).saturating_add(1); + pieces.push(decoded); } - Ok(all) + Ok(Pieces::join(&pieces)) +} + +fn page_xobjects(doc: &Document, page: &Dict) -> Dict { + page.get(b"Resources") + .map(|r| doc.resolve(r)) + .and_then(|r| r.as_dict().and_then(|r| r.get(b"XObject").map(|x| doc.resolve(x)))) + .and_then(|x| x.as_dict().cloned()) + .unwrap_or_default() } fn harmless_outside(op: &[u8]) -> bool { @@ -100,9 +113,35 @@ fn harmless_outside(op: &[u8]) -> bool { | b"sc" | b"SCN" | b"scn" + | b"m" + | b"l" + | b"c" + | b"v" + | b"y" + | b"h" + | b"re" + | b"W" + | b"W*" + | b"n" ) } +fn painted_outside(doc: &Document, xobjects: &Dict, op: &Op) -> bool { + match op.op.as_slice() { + b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s" | b"sh" | b"BI" => true, + b"Do" => op + .name(0) + .and_then(|n| xobjects.get(n)) + .map(|o| doc.resolve(o)) + .and_then(|o| match &*o { + Object::Stream(s) => s.dict.name(b"Subtype").map(<[u8]>::to_vec), + _ => None, + }) + .is_some_and(|st| st == b"Image"), + _ => false, + } +} + fn harmless_inside(op: &[u8]) -> bool { matches!( op, @@ -132,17 +171,31 @@ fn harmless_inside(op: &[u8]) -> bool { ) } -/// The `(BT, ET)` operator index pairs of each text object that shows text, or why the page cannot be tagged. -fn text_spans(ops: &[Op], index: usize) -> Result, AutoTagError> { - let mut start = None; - let mut has_text = false; - let mut spans = Vec::new(); - for (at, op) in ops.iter().enumerate() { +fn plan_page(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> Result { + if page.dict.contains(b"StructParents") { + return Err(unsupported(index, "page already has a structure parent key")); + } + if page.dict.get(b"Annots").is_some_and(|a| match &*doc.resolve(a) { + Object::Array(items) => !items.is_empty(), + _ => true, + }) { + return Err(unsupported(index, "annotations need separate structure tags")); + } + let pieces = content_pieces(doc, page, index)?; + let parsed = pieces.parse(); + if parsed.skipped > 0 || parsed.ambiguous_images > 0 || parsed.ops.len() > MAX_OPERATORS { + return Err(unsupported(index, "content has malformed or too many operators")); + } + let xobjects = page_xobjects(doc, &page.dict); + let (mut start, mut has_text, mut spans, mut artifacts) = (None, false, Vec::new(), HashMap::new()); + // Marked-content boundaries must enclose the whole path, not just its paint operator. + let mut path = None; + for (at, op) in parsed.ops.iter().enumerate() { + if path.is_some() && matches!(op.op.as_slice(), b"BT" | b"Do" | b"BI" | b"sh") { + return Err(unsupported(index, "painted content interrupts an unfinished path")); + } match op.op.as_slice() { - b"BT" if start.is_none() => { - start = Some(at); - has_text = false; - } + b"BT" if start.is_none() => (start, has_text) = (Some(at), false), b"ET" => { let Some(first) = start.take() else { return Err(unsupported(index, "unmatched ET operator")) }; if has_text { @@ -154,14 +207,31 @@ fn text_spans(ops: &[Op], index: usize) -> Result, AutoTagEr } b"Tj" | b"TJ" | b"'" | b"\"" if start.is_some() => has_text = true, name if start.is_some() && harmless_inside(name) => {} - name if start.is_none() && harmless_outside(name) => {} + name if start.is_none() && harmless_outside(name) => match name { + b"m" | b"re" => { + path.get_or_insert(at); + } + b"n" => path = None, + _ => {} + }, + _ if start.is_none() && painted_outside(doc, &xobjects, op) => { + let first = if matches!(op.op.as_slice(), b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s") { + path.take().unwrap_or(at) + } else { + at + }; + artifacts.insert(first, at); + } _ => return Err(unsupported(index, format!("unsupported or painted operator {}", String::from_utf8_lossy(&op.op)))), } } if start.is_some() { return Err(unsupported(index, "unterminated BT text object")); } - Ok(spans) + if path.is_some() { + return Err(unsupported(index, "unterminated path object")); + } + Ok(Plan { page: page.obj, ops: parsed.ops, spans, artifacts }) } fn plan(doc: &Document) -> Result, AutoTagError> { @@ -181,23 +251,9 @@ fn plan(doc: &Document) -> Result, AutoTagError> { let mut out = Vec::with_capacity(pages.len()); let mut total = 0usize; for (index, page) in pages.iter().enumerate() { - if page.dict.contains(b"StructParents") { - return Err(unsupported(index, "page already has a structure parent key")); - } - if page.dict.get(b"Annots").is_some_and(|a| match &*doc.resolve(a) { - Object::Array(items) => !items.is_empty(), - _ => true, - }) { - return Err(unsupported(index, "annotations need separate structure tags")); - } - let bytes = content_bytes(doc, page, index)?; - let parsed = parse(&bytes); - if parsed.skipped > 0 || parsed.ops.len() > MAX_OPERATORS { - return Err(unsupported(index, "content has malformed or too many operators")); - } - let spans = text_spans(&parsed.ops, index)?; - total = total.saturating_add(spans.len()); - out.push(Plan { page: page.obj, ops: parsed.ops, spans }); + let planned = plan_page(doc, page, index)?; + total = total.saturating_add(planned.spans.len()); + out.push(planned); } if total == 0 { return Err(AutoTagError::NoText); @@ -205,62 +261,36 @@ fn plan(doc: &Document) -> Result, AutoTagError> { Ok(out) } -/// Tag every text object in an untagged, text-only PDF as a paragraph. The full structure tree, -/// parent tree and marked-content IDs are written together. Mixed content is refused before any -/// mutation. On success the caller can use tag-property edits to assign reviewed heading roles. -pub fn autotag_text(doc: &mut Document) -> Result { - let plans = plan(doc)?; - let mut next = doc.clone(); - let catalog = next.root().ok_or(AutoTagError::NoCatalog)?; - let root = next.add(Object::Null); - let document = next.add(Object::Null); - let mut children = Vec::new(); - let mut numbers = Vec::new(); - let mut tagged_pages = 0usize; - let mut paragraphs = 0usize; - for page in plans { - if page.spans.is_empty() { - continue; +fn mark_ops(ops: Vec, starts: &HashMap, ends: &HashSet, artifacts: &HashMap) -> Result, AutoTagError> { + let artifact_ends: HashSet = artifacts.values().copied().collect(); + let extra = starts.len().saturating_add(artifacts.len()).saturating_mul(2); + let mut marked = Vec::with_capacity(ops.len().saturating_add(extra)); + for (at, op) in ops.into_iter().enumerate() { + let is_artifact = artifacts.contains_key(&at); + if let Some(mcid) = starts.get(&at) { + let mut properties = Dict::new(); + properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(*mcid).map_err(|_| AutoTagError::TooManyPages)?)); + marked.push(Op::new("BDC", vec![Object::name("P"), Object::Dict(properties)])); + } else if is_artifact { + marked.push(Op::new("BMC", vec![Object::name("Artifact")])); } - let key = i64::try_from(tagged_pages).map_err(|_| AutoTagError::TooManyPages)?; - let mut parents = Vec::with_capacity(page.spans.len()); - let mut starts = std::collections::HashMap::new(); - let mut ends = std::collections::HashSet::new(); - for (mcid, (first, last)) in page.spans.iter().enumerate() { - let mut element = Dict::new(); - element.set(b"Type".to_vec(), Object::name("StructElem")); - element.set(b"S".to_vec(), Object::name("P")); - element.set(b"P".to_vec(), Object::Ref(document)); - element.set(b"Pg".to_vec(), Object::Ref(page.page)); - element.set(b"K".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); - let reference = next.add(element); - parents.push(Object::Ref(reference)); - children.push(Object::Ref(reference)); - starts.insert(*first, mcid); - ends.insert(*last); - paragraphs = paragraphs.saturating_add(1); + marked.push(op); + if ends.contains(&at) || artifact_ends.contains(&at) { + marked.push(Op::new("EMC", Vec::new())); } - let mut marked = Vec::with_capacity(page.ops.len().saturating_add(page.spans.len().saturating_mul(2))); - for (at, op) in page.ops.into_iter().enumerate() { - if let Some(mcid) = starts.get(&at) { - let mut properties = Dict::new(); - properties.set(b"MCID".to_vec(), Object::Int(i64::try_from(*mcid).map_err(|_| AutoTagError::TooManyPages)?)); - marked.push(Op::new("BDC", vec![Object::name("P"), Object::Dict(properties)])); - } - marked.push(op); - if ends.contains(&at) { - marked.push(Op::new("EMC", Vec::new())); - } - } - let stream = next.add(Stream::flate(Dict::new(), &serialize_ops(&marked))); - next.update_dict(page.page, |d| { - d.set(b"Contents".to_vec(), Object::Ref(stream)); - d.set(b"StructParents".to_vec(), Object::Int(key)); - })?; - numbers.push(Object::Int(key)); - numbers.push(Object::Array(parents)); - tagged_pages += 1; } + Ok(marked) +} + +fn write_structure( + next: &mut Document, + catalog: ObjRef, + root: ObjRef, + document: ObjRef, + children: Vec, + numbers: Vec, + tagged_pages: usize, +) -> Result<(), AutoTagError> { let mut tree = Dict::new(); tree.set(b"Nums".to_vec(), Object::Array(numbers)); let parent_tree = next.add(tree); @@ -283,6 +313,50 @@ pub fn autotag_text(doc: &mut Document) -> Result { d.set(b"StructTreeRoot".to_vec(), Object::Ref(root)); d.set(b"MarkInfo".to_vec(), Object::Dict(mark_info)); })?; + Ok(()) +} + +/// Tag every text object in an untagged PDF as a paragraph and mark painted paths, shadings and +/// images outside text objects as artifacts. The full structure tree, parent tree and +/// marked-content IDs are written together. Unsupported operators are refused before any +/// mutation. On success the caller can use tag-property edits to assign reviewed heading roles. +pub fn autotag_text(doc: &mut Document) -> Result { + let plans = plan(doc)?; + let mut next = doc.clone(); + let catalog = next.root().ok_or(AutoTagError::NoCatalog)?; + let (root, document) = (next.add(Object::Null), next.add(Object::Null)); + let (mut children, mut numbers, mut tagged_pages, mut paragraphs) = (Vec::new(), Vec::new(), 0usize, 0usize); + for page in plans { + if page.spans.is_empty() && page.artifacts.is_empty() { + continue; + } + let (mut starts, mut ends) = (HashMap::new(), HashSet::new()); + if !page.spans.is_empty() { + let key = i64::try_from(tagged_pages).map_err(|_| AutoTagError::TooManyPages)?; + let mut parents = Vec::with_capacity(page.spans.len()); + for (mcid, (first, last)) in page.spans.iter().enumerate() { + let mut element = Dict::new(); + element.set(b"Type".to_vec(), Object::name("StructElem")); + element.set(b"S".to_vec(), Object::name("P")); + element.set(b"P".to_vec(), Object::Ref(document)); + element.set(b"Pg".to_vec(), Object::Ref(page.page)); + element.set(b"K".to_vec(), Object::Int(i64::try_from(mcid).map_err(|_| AutoTagError::TooManyPages)?)); + let reference = next.add(element); + parents.push(Object::Ref(reference)); + children.push(Object::Ref(reference)); + starts.insert(*first, mcid); + ends.insert(*last); + paragraphs = paragraphs.saturating_add(1); + } + next.update_dict(page.page, |d| d.set(b"StructParents".to_vec(), Object::Int(key)))?; + numbers.extend([Object::Int(key), Object::Array(parents)]); + tagged_pages += 1; + } + let marked = mark_ops(page.ops, &starts, &ends, &page.artifacts)?; + let stream = next.add(Stream::flate(Dict::new(), &serialize_ops(&marked))); + next.update_dict(page.page, |d| d.set(b"Contents".to_vec(), Object::Ref(stream)))?; + } + write_structure(&mut next, catalog, root, document, children, numbers, tagged_pages)?; *doc = next; Ok(AutoTagResult { pages: tagged_pages, paragraphs }) } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index a6dcdba199..5c73d4181b 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -80,17 +80,120 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { } #[test] -fn autotag_text_refuses_paint_and_does_not_modify_the_document() { +fn autotag_text_marks_vector_graphics_and_images_as_artifacts() { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R 7 0 R] /Count 2 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /XObject << /Im1 6 0 R >> >> >>".into(), + stream("0 0 200 200 m 200 0 l 200 200 l 0 200 l h f 10 10 50 50 re B BT /F1 12 Tf 20 100 Td (Hello) Tj ET q 20 0 0 20 80 80 cm /Im1 Do Q"), + FONT.into(), + "<< /Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length 1 >>\nstream\n\u{0}\nendstream".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 8 0 R >>".into(), + stream("0 0 100 100 re S"), + ], + "", + ); + let result = autotag_text(&mut doc).unwrap(); + assert_eq!(result, AutoTagResult { pages: 1, paragraphs: 1 }); + let report = check(&doc, &Options { rules: [Rule::TaggedPdf, Rule::TaggedContent].into(), pages: None }); + assert_eq!(status(&report, Rule::TaggedPdf), Status::Passed); + assert_eq!(status(&report, Rule::TaggedContent), Status::Passed); + let bytes = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap(); + let reopened = Document::open(Arc::new(bytes)).unwrap(); + let reopened_report = check(&reopened, &Options { rules: [Rule::TaggedPdf, Rule::TaggedContent].into(), pages: None }); + assert_eq!(status(&reopened_report, Rule::TaggedPdf), Status::Passed); + assert_eq!(status(&reopened_report, Rule::TaggedContent), Status::Passed); +} + +#[test] +fn autotag_artifacts_wrap_complete_paths_and_leave_unpainted_clips_outside() { + let content = "0 0 m 10 0 l 10 10 l h f 0 0 20 20 re W B 0 0 50 50 re W* n /Shade sh BT /F1 12 Tf (Text) Tj ET 3 3 5 5 re S"; let mut doc = pdf( &[ "<< /Type /Catalog /Pages 2 0 R >>".into(), "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), - stream("BT /F1 12 Tf 20 100 Td (Hello) Tj ET 0 0 50 50 re f"), + stream(content), FONT.into(), ], "", ); + autotag_text(&mut doc).unwrap(); + let bytes = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap(); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let page = pdfcraft_model::pages(&doc).remove(0); + let content = doc.resolve(page.dict.get(b"Contents").unwrap()); + let Object::Stream(content) = &*content else { panic!("page stream") }; + let ops = pdfcraft_content::parse(&content.decoded_strict().unwrap()).ops; + let names: Vec<&[u8]> = ops.iter().map(|op| op.op.as_slice()).collect(); + assert_eq!( + names, + vec![ + b"BMC".as_slice(), + b"m", + b"l", + b"l", + b"h", + b"f", + b"EMC", + b"BMC", + b"re", + b"W", + b"B", + b"EMC", + b"re", + b"W*", + b"n", + b"BMC", + b"sh", + b"EMC", + b"BDC", + b"BT", + b"Tf", + b"Tj", + b"ET", + b"EMC", + b"BMC", + b"re", + b"S", + b"EMC", + ] + ); +} + +#[test] +fn autotag_rejects_text_interleaved_with_an_unfinished_path_atomically() { + for tail in ["10 10 l S", "n"] { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(&format!("0 0 m BT /F1 12 Tf (Text) Tj ET {tail}")), + FONT.into(), + ], + "", + ); + let before = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap(); + assert!(matches!(autotag_text(&mut doc), Err(AutoTagError::Unsupported { .. }))); + assert_eq!(pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions::default()).unwrap(), before); + } +} + +#[test] +fn autotag_text_refuses_form_xobjects_and_does_not_modify_the_document() { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /XObject << /Fm1 6 0 R >> >> >>".into(), + stream("BT /F1 12 Tf 20 100 Td (Hello) Tj ET /Fm1 Do"), + FONT.into(), + "<< /Type /XObject /Subtype /Form /BBox [0 0 50 50] /Length 12 >>\nstream\n0 0 10 10 re f\nendstream".into(), + ], + "", + ); assert!(matches!(autotag_text(&mut doc), Err(AutoTagError::Unsupported { .. }))); assert!(!doc.get(doc.root().unwrap()).as_dict().unwrap().contains(b"StructTreeRoot")); assert_eq!(autotag_text(&mut pdf(&good(), "")), Err(AutoTagError::AlreadyTagged));