craft-pdf: a dictionary key written with raw spaces no longer fails the whole dictionary (cos) (#1905)

This commit is contained in:
Ken Mantle
2026-11-16 21:30:08 -05:00
committed by GitHub
parent 38e5614b9c
commit 79f862597d
7 changed files with 278 additions and 34 deletions
+4 -4
View File
@@ -25,7 +25,7 @@ pub(crate) use objects::Held;
pub(crate) use reader::find;
#[cfg(test)]
use reader::rfind;
use state::{ObjStm, Slot, Unreadable};
use state::{ObjStm, ObjectNotes, Slot};
pub(crate) use state::{ObjectReader, held_bytes};
pub use state::{ReadState, SourceIdentity};
#[cfg(test)]
@@ -56,9 +56,9 @@ pub struct Document {
/// decodes the stream again (`Document::load`).
cache: Arc<Mutex<state::ObjectCache>>,
objstms: Arc<Mutex<HashMap<u32, Arc<ObjStm>>>>,
/// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the
/// source, and is reset wherever they are.
unreadable: Arc<Mutex<Unreadable>>,
/// See [`Self::unreadable_objects`] and [`Self::repaired_objects`]. Shared by clones, like
/// the caches: it describes the source, and is reset wherever they are.
object_notes: Arc<Mutex<ObjectNotes>>,
/// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan
/// share them, and each would otherwise decode them again (see `Document::jbig2_globals`).
jbig2_globals: Arc<Mutex<HashMap<u32, Arc<Vec<u8>>>>>,
+48 -9
View File
@@ -2,8 +2,8 @@ use std::collections::HashSet;
use super::reader::find;
use super::state::{
LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadFrame, ReadState, ScanLengths, Slot, SourceIdentity, loaded_bytes, loading, note_overlay_read,
note_read, objstm_member, overlay_reads,
LENGTHS, LoadGuard, Noted, ObjStm, ObjectNotes, ObjectReader, ReadFrame, ReadState, ScanLengths, Slot, SourceIdentity, loaded_bytes, loading,
note_overlay_read, note_read, objstm_member, overlay_reads,
};
use super::xref::{Revision, XrefEntry, XrefIndex};
use super::{
@@ -106,18 +106,49 @@ impl Document {
/// line says there were more. Opening again, authenticating or rebuilding the index starts
/// the list afresh, as it does the object cache.
pub fn unreadable_objects(&self) -> Vec<String> {
let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() };
unreadable.lines(|num| self.overlay.contains_key(&num))
let Ok(notes) = self.object_notes.lock() else { return Vec::new() };
notes.unreadable.lines(|num| self.overlay.contains_key(&num), "could not be read")
}
/// Objects of the file that were read only by repairing their syntax, lowest number first, as
/// lines such as "object 9 0 was repaired: a dictionary key written with spaces was read as one
/// key". Unlike [`Self::unreadable_objects`] nothing was lost, but the object is not what the
/// file wrote. Bounded, listed and reset as [`Self::unreadable_objects`] is; each object is
/// listed once however often it is read.
pub fn repaired_objects(&self) -> Vec<String> {
let Ok(notes) = self.object_notes.lock() else { return Vec::new() };
notes.repaired.lines(|num| self.overlay.contains_key(&num), "was repaired")
}
/// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file
/// whose broken object is asked for again and again pays for it once.
fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) {
// A poisoned lock only means the object goes unlisted; the read itself still fails.
let Ok(mut unreadable) = self.unreadable.lock() else { return };
self.note_object(num, |notes| &mut notes.unreadable, reason);
}
fn note_spaced_key(&self, num: u32) {
self.note_object(num, |notes| &mut notes.repaired, || "a dictionary key written with spaces was read as one key".into());
}
/// [`Self::scan_for`] counted as a read of `num` of its own: a spaced key it joins is noted
/// under `num`, not under the object whose read called it.
fn scan_for_noting(&self, num: u32) -> Option<(ObjRef, Object)> {
let outer = crate::cos::parser::swap_spaced_keys(0);
let found = self.scan_for(num);
if crate::cos::parser::swap_spaced_keys(outer) > 0 && found.is_some() {
self.note_spaced_key(num);
}
found
}
/// Record `num` in the list `pick` chooses, once.
fn note_object(&self, num: u32, pick: impl FnOnce(&mut ObjectNotes) -> &mut Noted, reason: impl FnOnce() -> String) {
// A poisoned lock only means the object goes unlisted; the read itself is unaffected.
let Ok(mut notes) = self.object_notes.lock() else { return };
let list = pick(&mut notes);
let hidden = |n: u32| self.overlay.contains_key(&n);
if unreadable.wants(num, hidden) {
unreadable.note(num, self.generation(num), &reason(), hidden);
if list.wants(num, hidden) {
list.note(num, self.generation(num), &reason(), hidden);
}
}
@@ -582,10 +613,18 @@ impl Document {
LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?;
let _scope = crate::cos::security::FollowScope::enter();
let edits_read = overlay_reads();
// Counts every attempt `load` makes, so an object found only at a second offset is listed
// if an abandoned copy had a spaced key: a rare false report, never a missed one.
let outer = crate::cos::parser::swap_spaced_keys(0);
let frame = ReadFrame::enter();
let loaded = self.load(num, 0).map(Arc::new);
let reads = frame.reads();
drop(frame);
// The notes are shared by clones, so a load that read this document's edits (an edited
// object stream) is not noted, as it is not cached below.
if crate::cos::parser::swap_spaced_keys(outer) > 0 && loaded.is_ok() && overlay_reads() == edits_read {
self.note_spaced_key(num);
}
if let Err(e) = &loaded
&& !matches!(e, CosError::Poisoned)
{
@@ -759,7 +798,7 @@ impl Document {
_ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => {
return Err(CosError::MissingObject(num));
}
_ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?),
_ => Arc::new(self.scan_for_noting(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?),
};
let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) };
let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize;
+3 -3
View File
@@ -24,7 +24,7 @@ impl Document {
unlinked_update: false,
cache: Arc::default(),
objstms: Arc::default(),
unreadable: Arc::default(),
object_notes: Arc::default(),
jbig2_globals: Arc::default(),
stream_limit: crate::cos::object::MAX_DECODED,
overlay: BTreeMap::new(),
@@ -113,7 +113,7 @@ impl Document {
unlinked_update: false,
cache: Arc::default(),
objstms: Arc::default(),
unreadable: Arc::default(),
object_notes: Arc::default(),
jbig2_globals: Arc::default(),
stream_limit: stream_limit.min(crate::cos::object::MAX_DECODED),
overlay: BTreeMap::new(),
@@ -256,7 +256,7 @@ impl Document {
// Objects read while locating the catalog were read before decryption was possible.
self.cache = Arc::default();
self.objstms = Arc::default();
self.unreadable = Arc::default();
self.object_notes = Arc::default();
Ok(())
}
+2 -2
View File
@@ -1,6 +1,6 @@
use super::state::ObjectNotes;
#[cfg(test)]
use super::state::SECTION_READS;
use super::state::Unreadable;
use super::xref::{
MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData,
};
@@ -699,7 +699,7 @@ impl Document {
}
self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()), Arc::default()));
self.cache.lock().map_err(|_| CosError::Poisoned)?.clear();
*self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default();
*self.object_notes.lock().map_err(|_| CosError::Poisoned)? = ObjectNotes::default();
// Objects inside object streams.
for num in objstms {
if let Ok(o) = self.try_get(num)
+20 -14
View File
@@ -226,17 +226,25 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option<usize>, len: us
(start < len).then_some((num, start))
}
/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`).
/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that
/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole
/// list holds at most `MAX` plus the objects edited in clones.
/// What reading single objects found, shared by clones and reset with the object cache: the objects
/// that could not be read (`Document::unreadable_objects`) and those read with a repair
/// (`Document::repaired_objects`).
#[derive(Debug, Default)]
pub(super) struct Unreadable {
pub(super) struct ObjectNotes {
pub(super) unreadable: Noted,
pub(super) repaired: Noted,
}
/// Objects, each with a reason. Bounded: a document records past [`Noted::MAX`] of the objects it
/// shows only the fact that there were more. Entries a clone has edited are hidden from it and do
/// not count, so the whole list holds at most `MAX` plus the objects edited in clones.
#[derive(Debug, Default)]
pub(super) struct Noted {
pub(super) objects: BTreeMap<u32, (u16, String)>,
pub(super) more: bool,
}
impl Unreadable {
impl Noted {
pub(super) const MAX: usize = 100;
/// Characters of a reason kept: a parser's message can quote the bytes it stopped at.
const REASON: usize = 160;
@@ -263,18 +271,16 @@ impl Unreadable {
}
}
/// One line per recorded object not `hidden`, then one saying there were more, if there were.
pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec<String> {
/// One line per recorded object not `hidden`, saying what happened to it (`what`, as "could
/// not be read"), then one saying there were more, if there were.
pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool, what: &str) -> Vec<String> {
// Clones share the list and each hides its own edits, so the list may hold more than
// `MAX` entries in all; each document still shows at most `MAX`.
let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num));
let mut lines: Vec<String> = shown
.by_ref()
.take(Self::MAX)
.map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}"))
.collect();
let mut lines: Vec<String> =
shown.by_ref().take(Self::MAX).map(|(num, (generation, reason))| format!("object {num} {generation} {what}: {reason}")).collect();
if self.more || shown.next().is_some() {
lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX));
lines.push(format!("more objects {what}; only the first {} are listed", Self::MAX));
}
lines
}
+121 -1
View File
@@ -823,7 +823,7 @@ fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() {
/// edit of that number still answers for it.
#[test]
fn an_unreadable_object_is_searched_for_once() {
let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"];
let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd ) >>"];
let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap();
FILE_SCANS.with(|scans| scans.set(0));
let first = doc.try_get(3);
@@ -2711,3 +2711,123 @@ fn find_agrees_with_a_plain_window_search() {
assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}");
}
}
/// A key written with spaces is joined (as in some Info dictionaries), and the object it was read
/// from is listed as repaired, once however often it is read. A sound object (a stream with an
/// indirect `/Length`) adds nothing.
#[test]
fn a_spaced_key_is_listed_once_as_a_repair_of_its_object() {
let file = build(
&[
"<< /Type /Catalog /Pages 2 0 R >>",
"<< /Type /Pages /Kids [] /Count 0 >>",
"<< /Title (Report) /Universal PDF(Acme) >>",
"<< /Length 5 0 R >>\nstream\nabcd\nendstream",
"4",
],
"/Root 1 0 R /Info 3 0 R",
);
let doc = Document::open(Arc::new(file)).unwrap();
assert!(doc.repaired_objects().is_empty(), "nothing has read object 3 yet");
for _ in 0..3 {
let info = doc.get(ObjRef::new(3, 0));
assert!(info.as_dict().is_some_and(|d| d.get(b"Universal PDF").is_some()), "{info:?}");
}
assert!(doc.try_get(4).is_ok());
let lines = doc.repaired_objects();
assert_eq!(lines, ["object 3 0 was repaired: a dictionary key written with spaces was read as one key"], "{lines:?}");
assert_eq!(doc.clone().repaired_objects(), lines, "clones share the list");
assert!(doc.unreadable_objects().is_empty());
}
/// A scan of a fresh document lists the repaired object as a read does, and once.
#[test]
fn a_scan_lists_a_spaced_key_repair_once() {
let file = build(
&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Universal PDF(Acme) >>"],
"/Root 1 0 R /Info 3 0 R",
);
let doc = Document::open(Arc::new(file)).unwrap();
assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_ok()).count(), 3);
assert_eq!(doc.scan_objects_checked().count(), 3);
assert_eq!(doc.repaired_objects(), ["object 3 0 was repaired: a dictionary key written with spaces was read as one key"]);
}
/// A file holding object stream 4 with `members` (one object, numbered 3), the catalog and pages
/// as objects 1 and 2, and a cross-reference stream as object 5. `extra` is added to the stream's
/// dictionary; with `lost`, the table says the stream itself sits in stream 9, which is absent, so
/// only a scan of the file finds it.
fn objstm_file(members: &[u8], extra: &str, lost: bool) -> Vec<u8> {
let mut out = b"%PDF-1.5\n".to_vec();
let mut offs = [0usize; 6];
let mut obj = |out: &mut Vec<u8>, n: usize, body: &[u8]| {
offs[n] = out.len();
out.extend_from_slice(format!("{n} 0 obj\n").as_bytes());
out.extend_from_slice(body);
out.extend_from_slice(b"\nendobj\n");
};
obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>");
obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 >>");
let mut stm = format!("<< /Type /ObjStm /N 1 /First 4 /Length {} {extra} >>\nstream\n", members.len()).into_bytes();
stm.extend_from_slice(members);
stm.extend_from_slice(b"\nendstream");
obj(&mut out, 4, &stm);
let xref_at = out.len();
let mut rows = vec![0u8, 0, 0, 0, 0, 255];
for (n, off) in offs.iter().enumerate().take(6).skip(1) {
let row = match n {
3 => [2, 0, 0, 0, 4, 0],
4 if lost => [2, 0, 0, 0, 9, 0],
5 => [1, (xref_at >> 24) as u8, (xref_at >> 16) as u8, (xref_at >> 8) as u8, xref_at as u8, 0],
_ => [1, (off >> 24) as u8, (off >> 16) as u8, (off >> 8) as u8, *off as u8, 0],
};
rows.extend_from_slice(&row);
}
let mut xs = format!("<< /Type /XRef /Size 6 /W [1 4 1] /Root 1 0 R /Info 3 0 R /Length {} >>\nstream\n", rows.len()).into_bytes();
xs.extend_from_slice(&rows);
xs.extend_from_slice(b"\nendstream");
let at = out.len();
out.extend_from_slice(b"5 0 obj\n");
out.extend_from_slice(&xs);
out.extend_from_slice(b"\nendobj\n");
out.extend_from_slice(format!("startxref\n{at}\n%%EOF\n").as_bytes());
out
}
/// A member of an object stream with a spaced key is listed under its own number, not the
/// stream's.
#[test]
fn a_spaced_key_in_an_object_stream_member_is_listed_under_the_member() {
let doc = Document::open(Arc::new(objstm_file(b"3 0 << /Title (Report) /Universal PDF(Acme) >>", "", false))).unwrap();
let info = doc.get(ObjRef::new(3, 0));
assert!(info.as_dict().is_some_and(|d| d.get(b"Universal PDF").is_some()), "{info:?}");
assert_eq!(doc.repaired_objects(), ["object 3 0 was repaired: a dictionary key written with spaces was read as one key"]);
}
/// An object stream found only by a scan, whose dictionary has a spaced key, is listed under its
/// own number; the member read through it, which has none, is not listed.
#[test]
fn a_scanned_object_stream_with_a_spaced_key_is_listed_under_the_stream() {
let doc = Document::open(Arc::new(objstm_file(b"3 0 << /Title (Report) >>", "/Universal PDF(Acme)", true))).unwrap();
let info = doc.get(ObjRef::new(3, 0));
assert!(info.as_dict().is_some_and(|d| d.get(b"Title").is_some()), "{info:?}");
assert_eq!(doc.repaired_objects(), ["object 4 0 was repaired: a dictionary key written with spaces was read as one key"]);
}
/// A clone's edited object stream with a spaced key is not listed for the document it was cloned
/// from, whose own member has none.
#[test]
fn a_spaced_key_read_from_a_clone_edit_is_not_listed_for_the_source() {
let doc = Document::open(Arc::new(objstm_file(b"3 0 << /Title (Report) >>", "", false))).unwrap();
let mut clone = doc.clone();
let members = b"3 0 << /Title (Edited) /Universal PDF(Acme) >>".to_vec();
let mut d = Dict::new();
d.set(b"Type".to_vec(), Object::name("ObjStm"));
d.set(b"N".to_vec(), Object::Int(1));
d.set(b"First".to_vec(), Object::Int(4));
clone.set(ObjRef::new(4, 0), Object::Stream(Stream::from_raw(d, members)));
let edited = clone.get(ObjRef::new(3, 0));
assert!(edited.as_dict().is_some_and(|d| d.get(b"Universal PDF").is_some()), "{edited:?}");
assert!(doc.repaired_objects().is_empty(), "{:?}", doc.repaired_objects());
assert!(doc.get(ObjRef::new(3, 0)).as_dict().is_some_and(|d| d.get(b"Universal PDF").is_none()));
}
+80 -1
View File
@@ -294,8 +294,9 @@ impl<'a> Lexer<'a> {
}
Some(b'/') => {
self.pos += 1;
let key = self.name_body();
let mut key = self.name_body();
self.skip_ws();
self.spaced_key_words(&mut key);
// A key directly followed by `>>` has no value: treat as null. (A following
// `/` is a name *value*, so it cannot signal a missing value.)
if self.peek() == Some(b'>') && self.data.get(self.pos + 1) == Some(&b'>') {
@@ -325,6 +326,37 @@ impl<'a> Lexer<'a> {
}
}
/// The rest of a key written with raw spaces, as in `/Universal PDF(...)` (seen in Info
/// dictionaries). Bare words join the key only when a string follows them: before a name,
/// number or keyword they are more likely a value missing its slash, and the dictionary
/// fails as before. At most [`Self::KEY_WORDS`] words.
fn spaced_key_words(&mut self, key: &mut Vec<u8>) {
let start = self.pos;
let mut words = Vec::new();
while words.len() < Self::KEY_WORDS && self.peek().is_some_and(|b| is_regular(b) && !matches!(b, b'+' | b'-' | b'.' | b'0'..=b'9')) {
let at = self.pos;
if matches!(self.token(), b"true" | b"false" | b"null" | b"R" | b"obj" | b"endobj" | b"stream" | b"endstream") {
self.pos = at;
break;
}
self.pos = at;
words.push(self.name_body());
self.skip_ws();
}
if words.is_empty() || !matches!(self.peek(), Some(b'(' | b'<')) {
self.pos = start;
return;
}
for word in words {
key.push(b' ');
key.extend(word);
}
let _ = SPACED_KEYS.try_with(|c| c.set(c.get().saturating_add(1)));
}
/// The most bare words [`Self::spaced_key_words`] joins to one key.
const KEY_WORDS: usize = 8;
fn number_or_ref(&mut self) -> Result<Object, CosError> {
let start = self.pos;
let first = self.number()?;
@@ -404,6 +436,18 @@ fn real_prefix_len(s: &[u8]) -> usize {
at
}
thread_local! {
/// Keys [`Lexer::spaced_key_words`] joined on this thread (see [`swap_spaced_keys`]).
static SPACED_KEYS: std::cell::Cell<u64> = const { std::cell::Cell::new(0) };
}
/// Set the count of keys joined from words written with spaces, returning the count it replaces:
/// a reader swaps in 0 before parsing an object and the old count back after, so the count it gets
/// is that object's alone, objects parsed meanwhile included only through their own swaps.
pub(crate) fn swap_spaced_keys(count: u64) -> u64 {
SPACED_KEYS.try_with(|c| c.replace(count)).unwrap_or(0)
}
#[cfg(test)]
thread_local! {
/// Bytes searched for `endstream` on this thread.
@@ -611,6 +655,41 @@ fn find_endstream(data: &[u8], from: usize, endstreams: Option<&EndstreamIndex>)
mod tests {
use super::*;
/// A key written with raw spaces reads as one key, and the rest of the dictionary reads.
#[test]
fn a_key_with_raw_spaces_keeps_the_dictionary() {
let Ok(Object::Dict(d)) =
Lexer::new(b"<< /Title (Report) /Universal PDF(Acme) /A B C#41 <4B> /T x\n<< /K 1 >> /Author (Ken) /On true /Off null >>", 0).object()
else {
panic!("the dictionary did not parse")
};
let text = |k: &[u8]| d.get(k).and_then(|v| v.as_string().map(|s| s.to_text()));
assert_eq!(text(b"Title").as_deref(), Some("Report"));
assert_eq!(text(b"Universal PDF").as_deref(), Some("Acme"));
assert_eq!(text(b"A B CA").as_deref(), Some("K"));
assert!(matches!(d.get(b"T x"), Some(Object::Dict(_))), "a dictionary value");
assert_eq!(text(b"Author").as_deref(), Some("Ken"));
assert_eq!(d.get(b"On"), Some(&Object::Bool(true)), "true is a value");
assert_eq!(d.get(b"Off"), Some(&Object::Null), "null is a value");
assert_eq!(d.len(), 7);
}
/// A bare word before a name, a number or a keyword is a value missing its slash, not part of
/// the key: the dictionary fails as before, for the file's own repair, rather than read
/// with entries silently moved.
#[test]
fn a_bare_word_that_is_not_a_key_still_fails_the_dictionary() {
for bad in [
&b"<< /Type Catalog /Pages 2 0 R >>"[..],
b"<< /Length 10 /Filter FlateDecode >>",
b"<< /Contents endobj 4 0 obj (x) >>",
b"<< /S Span >> BDC",
b"<< /K a b c d e f g h i (x) >>",
] {
assert!(Lexer::new(bad, 0).object().is_err(), "{}", String::from_utf8_lossy(bad));
}
}
#[test]
fn numbers_with_multibyte_garbage_do_not_panic() {
// From `cargo xtask fuzz`: "2 0 obj\n.\u{2014}" (an em dash after a dot).