diff --git a/crates/pdf/src/patch.rs b/crates/pdf/src/patch.rs index 8b9d69b908..9235f22cea 100644 --- a/crates/pdf/src/patch.rs +++ b/crates/pdf/src/patch.rs @@ -107,6 +107,42 @@ impl Xref { /// A replacement of bytes `start..end` of the file. type Edit = (usize, usize, Vec); +/// Where byte offsets of the old file land in the new one, for edits sorted by start that do not +/// overlap, whose ends therefore never decrease. +struct Moves { + /// Each edit's end, in order. + ends: Vec, + /// The change in length made by the edits up to and including each one. + shifts: Vec, +} + +impl Moves { + /// `None` when a change in length doesn't fit an `isize`. + fn new(edits: &[Edit]) -> Option { + let mut total: isize = 0; + let mut shifts = Vec::with_capacity(edits.len()); + for (start, end, new) in edits { + let removed = isize::try_from(end.checked_sub(*start)?).ok()?; + total = total.checked_add(isize::try_from(new.len()).ok()?)?.checked_sub(removed)?; + shifts.push(total); + } + Some(Self { ends: edits.iter().map(|e| e.1).collect(), shifts }) + } + + /// Where `off` lands: moved by every edit that ends at or before it. + fn at(&self, off: usize) -> Option { + match self.ends.partition_point(|&end| end <= off) { + 0 => Some(off), + n => off.checked_add_signed(*self.shifts.get(n - 1)?), + } + } +} + +/// `off` as the ten digits of a cross-reference entry; `None` past what ten digits hold. +fn entry_offset(off: usize) -> Option { + (u64::try_from(off).ok()? < 10_000_000_000).then(|| format!("{off:010}")) +} + /// The edits of one PDF, applied together by [`Self::apply`]. pub(crate) struct Patch { edits: Vec, @@ -139,18 +175,17 @@ impl Patch { } /// `pdf` with the edits, new objects and trailer entries, and a cross-reference table that - /// matches. Refused when an edit overlaps another or reaches past the objects. + /// matches. Refused when an edit overlaps another or reaches past the objects, or when an + /// object would start past the ten digits a cross-reference entry gives its offset in. pub fn apply(mut self, pdf: &[u8], xref: &Xref) -> Result, PdfError> { let bad = |why: &str| PdfError::Write(format!("can't edit the written PDF: {why}")); self.edits.sort_by_key(|e| (e.0, e.1)); if self.edits.windows(2).any(|w| w[0].1 > w[1].0) || self.edits.iter().any(|e| e.0 > e.1 || e.1 > xref.at) { return Err(bad("edits overlap")); } - // Where a byte offset of the old file lands in the new one. let edits = &self.edits; - let moved = |off: usize| -> usize { - edits.iter().filter(|e| e.1 <= off).fold(off as isize, |o, e| o + e.2.len() as isize - (e.1 - e.0) as isize) as usize - }; + let moves = Moves::new(edits).ok_or_else(|| bad("edits too large"))?; + let too_far = || bad("an object lies past what the cross-reference table holds"); let added: usize = self.edits.iter().map(|e| e.2.len()).sum::() + self.objects.iter().map(|o| o.len() + 32).sum::(); let mut out = Vec::with_capacity(pdf.len() + added + 64); let mut at = 0; @@ -175,7 +210,7 @@ impl Patch { for e in old.chunks(20) { match number(e, 0) { Some((off, _)) if e.get(17) == Some(&b'n') => { - out.extend_from_slice(format!("{:010}", moved(off)).as_bytes()); + out.extend_from_slice(moves.at(off).and_then(entry_offset).ok_or_else(too_far)?.as_bytes()); out.extend_from_slice(e.get(10..).unwrap_or_default()); } _ => out.extend_from_slice(e), @@ -183,7 +218,8 @@ impl Patch { } let eol = old.get(18..20).unwrap_or(b"\r\n"); for off in offsets { - out.extend_from_slice(format!("{off:010} 00000 n").as_bytes()); + out.extend_from_slice(entry_offset(off).ok_or_else(too_far)?.as_bytes()); + out.extend_from_slice(b" 00000 n"); out.extend_from_slice(eol); } // The trailer, with its size and the new entries. @@ -201,3 +237,279 @@ impl Patch { Ok(out) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::{Duration, Instant}; + + /// A PDF written as the export writes one: objects `<< /N n >>` numbered from 1, those `free` + /// picks left out with a free entry, then one cross-reference table and the trailer. + fn written(count: usize, free: impl Fn(usize) -> bool) -> Vec { + let mut pdf = b"%PDF-1.7\n".to_vec(); + let mut entries = b"0000000000 65535 f\r\n".to_vec(); + for n in 1..=count { + if free(n) { + entries.extend_from_slice(b"0000000000 00001 f\r\n"); + continue; + } + entries.extend(format!("{:010} 00000 n\r\n", pdf.len()).bytes()); + pdf.extend(format!("{n} 0 obj\n<< /N {n} >>\nendobj\n").bytes()); + } + let at = pdf.len(); + pdf.extend(format!("xref\n0 {}\n", count + 1).bytes()); + pdf.extend(entries); + pdf.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{at}\n%%EOF", count + 1).bytes()); + pdf + } + + /// A small linear congruential generator, so every run makes the same edits. + struct Rng(u64); + + impl Rng { + /// A number in `0..n` (0 when `n` is 0). + fn below(&mut self, n: usize) -> usize { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + usize::try_from(self.0 >> 33).unwrap() % n.max(1) + } + + fn bytes(&mut self) -> Vec { + (0..self.below(9)).map(|i| b'a' + u8::try_from(i).unwrap()).collect() + } + } + + /// About `wanted` edits that don't overlap, in shuffled order, at random places among the + /// objects and at object starts: replacements that grow, shrink or remove bytes, edits that end + /// at an object's start, and inserts, several of them at one place. None when `wanted` is 0. + fn edits(pdf: &[u8], xref: &Xref, rng: &mut Rng, wanted: usize) -> Vec { + if wanted == 0 { + return vec![]; + } + let first = b"%PDF-1.7\n".len(); + let mut points: Vec = xref.objects(pdf).into_iter().map(|(_, off)| off).collect(); + points.extend((0..wanted).map(|_| first + rng.below(xref.at() - first + 1))); + points.push(xref.at()); + points.sort_unstable(); + points.dedup(); + let mut out = vec![]; + for w in points.windows(2) { + let (p, q) = (w[0], w[1]); + match rng.below(6) { + 0 => {} + 1 => out.push((p, q, rng.bytes())), + 2 => out.push((p, p, rng.bytes())), + 3 => { + out.push((p, p, rng.bytes())); + out.push((p, p, rng.bytes())); + out.push((p, q.min(p + rng.below(4)), rng.bytes())); + } + 4 => out.push((p + (q - p) / 2, q, rng.bytes())), + _ => out.push((q, q, rng.bytes())), + } + } + for i in (1..out.len()).rev() { + out.swap(i, rng.below(i + 1)); + } + out + } + + fn patch(xref: &Xref, edits: &[Edit], objects: bool) -> Patch { + let mut patch = Patch::new(xref); + for (start, end, with) in edits { + patch.replace(*start, *end, with.clone()); + } + if objects { + patch.add_object(b"<< /Added 1 >>".to_vec()); + patch.trailer_entry(b" /Info 2 0 R"); + } + patch + } + + /// Where `off` lands, by walking every edit: how [`Patch::apply`] found it before [`Moves`]. + fn scan(edits: &[Edit], off: usize) -> usize { + edits.iter().filter(|e| e.1 <= off).fold(off as isize, |o, e| o + e.2.len() as isize - (e.1 - e.0) as isize) as usize + } + + /// [`Patch::apply`] as it was written before [`Moves`], walking every edit for every object. + fn apply_by_scan(mut patch: Patch, pdf: &[u8], xref: &Xref) -> Result, PdfError> { + let bad = |why: &str| PdfError::Write(format!("can't edit the written PDF: {why}")); + patch.edits.sort_by_key(|e| (e.0, e.1)); + if patch.edits.windows(2).any(|w| w[0].1 > w[1].0) || patch.edits.iter().any(|e| e.0 > e.1 || e.1 > xref.at) { + return Err(bad("edits overlap")); + } + let edits = &patch.edits; + let moved = |off: usize| -> usize { scan(edits, off) }; + let added: usize = patch.edits.iter().map(|e| e.2.len()).sum::() + patch.objects.iter().map(|o| o.len() + 32).sum::(); + let mut out = Vec::with_capacity(pdf.len() + added + 64); + let mut at = 0; + for (start, end, new) in edits.iter() { + out.extend_from_slice(pdf.get(at..*start).ok_or_else(|| bad("bad edit"))?); + out.extend_from_slice(new); + at = *end; + } + out.extend_from_slice(pdf.get(at..xref.at).ok_or_else(|| bad("bad edit"))?); + let mut offsets = Vec::with_capacity(patch.objects.len()); + for (i, body) in patch.objects.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", patch.size + i).as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + } + let table = out.len(); + let size = patch.size + patch.objects.len(); + out.extend_from_slice(format!("xref\n0 {size}\n").as_bytes()); + let old = pdf.get(xref.entries..xref.entries + 20 * xref.count).ok_or_else(|| bad("bad cross-reference table"))?; + for e in old.chunks(20) { + match number(e, 0) { + Some((off, _)) if e.get(17) == Some(&b'n') => { + out.extend_from_slice(format!("{:010}", moved(off)).as_bytes()); + out.extend_from_slice(e.get(10..).unwrap_or_default()); + } + _ => out.extend_from_slice(e), + } + } + let eol = old.get(18..20).unwrap_or(b"\r\n"); + for off in offsets { + out.extend_from_slice(format!("{off:010} 00000 n").as_bytes()); + out.extend_from_slice(eol); + } + let (ts, te) = xref.trailer; + let dict = pdf.get(ts..te.saturating_sub(2)).ok_or_else(|| bad("bad trailer"))?; + let size_at = find(dict, b"/Size", 0).ok_or_else(|| bad("the trailer has no size"))? + 5; + let (_, size_end) = number(dict, size_at).ok_or_else(|| bad("the trailer has no size"))?; + let (Some(head), Some(tail)) = (dict.get(..size_at), dict.get(size_end..)) else { return Err(bad("bad trailer")) }; + out.extend_from_slice(b"trailer\n"); + out.extend_from_slice(head); + out.extend_from_slice(format!(" {size}").as_bytes()); + out.extend_from_slice(tail); + out.extend_from_slice(&patch.trailer); + out.extend_from_slice(format!(">>\nstartxref\n{table}\n%%EOF").as_bytes()); + Ok(out) + } + + #[test] + fn every_offset_moves_by_the_edits_that_end_at_or_before_it() { + let mut rng = Rng(7661); + for round in 0..100 { + let pdf = written(1 + rng.below(30), |n| n % 7 == 3); + let xref = Xref::read(&pdf).unwrap(); + let mut edits = edits(&pdf, &xref, &mut rng, [0, 1, 3, 20, 80][round % 5]); + edits.sort_by_key(|e| (e.0, e.1)); + let moves = Moves::new(&edits).unwrap(); + for off in 0..=pdf.len() { + assert_eq!(moves.at(off), Some(scan(&edits, off)), "round {round}, offset {off}, edits {edits:?}"); + } + } + } + + #[test] + fn apply_writes_the_bytes_a_walk_of_every_edit_wrote() { + let mut rng = Rng(1378); + for round in 0..300 { + let pdf = written(1 + rng.below(40), |n| n % 7 == 3); + let xref = Xref::read(&pdf).unwrap(); + let edits = edits(&pdf, &xref, &mut rng, [0, 1, 3, 20, 80][round % 5]); + let objects = round % 3 == 0; + let fast = patch(&xref, &edits, objects).apply(&pdf, &xref).unwrap(); + let walked = apply_by_scan(patch(&xref, &edits, objects), &pdf, &xref).unwrap(); + assert_eq!(String::from_utf8_lossy(&fast), String::from_utf8_lossy(&walked), "round {round}"); + } + } + + #[test] + fn every_entry_points_at_its_object_after_the_edits() { + let pdf = written(30, |n| n % 7 == 3); + let xref = Xref::read(&pdf).unwrap(); + let mut patch = Patch::new(&xref); + for (n, off) in xref.objects(&pdf) { + let (at, end) = xref.dict(&pdf, n).unwrap(); + match n % 4 { + 0 => patch.replace(at, at, b" /Grown [1 2 3]".to_vec()), + 1 => patch.replace(at, end, b">>\n".to_vec()), + // An insert at an object's start goes before the object. + 2 => patch.replace(off, off, b"% before\n".to_vec()), + _ => {} + } + } + let added = patch.add_object(b"<< /Added 1 >>".to_vec()); + let out = patch.apply(&pdf, &xref).unwrap(); + let after = Xref::read(&out).unwrap(); + assert_eq!((after.count(), added), (32, 31)); + for n in 1..32u32 { + let Some(off) = after.offset(&out, n) else { + assert!(n % 7 == 3, "object {n} is in use"); + let e = xref.entries + 20 * n as usize; + assert_eq!(out[after.entries + 20 * n as usize..][..20], pdf[e..e + 20], "free entry {n}"); + continue; + }; + assert!(out[off..].starts_with(format!("{n} 0 obj\n").as_bytes()), "object {n} at {off}"); + } + } + + #[test] + fn an_object_moved_past_ten_digits_is_refused() { + let mut pdf = written(2, |_| false); + let entry = Xref::read(&pdf).unwrap().entries + 40; + // Object 2's entry gives the last offset ten digits hold. + pdf[entry..entry + 10].copy_from_slice(b"9999999999"); + let xref = Xref::read(&pdf).unwrap(); + let out = Patch::new(&xref).apply(&pdf, &xref).unwrap(); + assert!(find(&out, b"9999999999 00000 n\r\n", 0).is_some()); + let mut patch = Patch::new(&xref); + patch.replace(9, 9, b"%\n".to_vec()); + let refused = patch.apply(&pdf, &xref); + assert!(matches!(&refused, Err(PdfError::Write(why)) if why.contains("cross-reference table holds")), "{refused:?}"); + } + + /// CPU time this thread has used, where the system reports it (`/proc` on Linux). + fn thread_cpu() -> Option { + let stat = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?; + stat.split_whitespace().next()?.parse().ok().map(Duration::from_nanos) + } + + /// How long `f` takes, in thread CPU time where the system reports it, else wall time. + fn time(f: impl FnOnce() -> T) -> (Duration, T) { + let (cpu, wall) = (thread_cpu(), Instant::now()); + let out = f(); + (thread_cpu().zip(cpu).map_or_else(|| wall.elapsed(), |(b, a)| b.saturating_sub(a)), out) + } + + fn median(mut runs: Vec) -> Duration { + runs.sort_unstable(); + runs[runs.len() / 2] + } + + /// `cargo test --release -p vectorcraft-pdf --lib patch::tests::bench -- --ignored --nocapture`. + /// The edits are those of forms in overprint: two inside each form's stream (its `/Length` and + /// data) and an insert into its dictionary, for `n` forms among `n + 20` objects. Each run + /// applies the patch `reps` times, so that small files take long enough for the clock. + #[test] + #[ignore = "timing; run explicitly"] + fn bench_apply_with_many_edits() { + println!("clock: {}", if thread_cpu().is_some() { "thread CPU time" } else { "wall time" }); + println!("| forms | objects | edits | walk every edit (median of 7) | binary search (median of 7) |"); + println!("| ---: | ---: | ---: | ---: | ---: |"); + for n in [100, 1_000, 5_000, 20_000] { + let pdf = written(n + 20, |_| false); + let xref = Xref::read(&pdf).unwrap(); + let mut edits = vec![]; + for (obj, _) in xref.objects(&pdf).into_iter().take(n) { + let (at, _) = xref.dict(&pdf, obj).unwrap(); + edits.push((at, at + 1, b" /Length 120 ".to_vec())); + edits.push((at + 1, at + 3, b"/Data".to_vec())); + edits.push((at + 3, at + 3, b" /Resources << /ExtGState << /GS0 9 0 R >> >>".to_vec())); + } + let reps = (200_000_000 / (3 * n * n)).clamp(1, 2_000); + let (mut walked, mut searched) = (vec![], vec![]); + for _ in 0..7 { + let (t, old) = time(|| (0..reps).fold(None, |_, _| apply_by_scan(patch(&xref, &edits, false), &pdf, &xref).ok())); + walked.push(t); + let (t, new) = time(|| (0..reps).fold(None, |_, _| patch(&xref, &edits, false).apply(&pdf, &xref).ok())); + searched.push(t); + assert!(old.is_some() && old == new, "{n} forms: the files differ"); + } + let ms = |d: Duration| d.as_secs_f64() * 1000.0 / reps as f64; + println!("| {n} | {} | {} | {:.3} ms | {:.3} ms |", n + 20, edits.len(), ms(median(walked)), ms(median(searched))); + } + } +}