mirror of
https://github.com/storytold/vectorcraft.git
synced 2026-11-18 12:27:37 +00:00
Merge pull request #8491 from dotCooCoo/perf/pdf-patch-offset-search
PDF export: Patch::apply finds where each object moved with a binary search
This commit is contained in:
+319
-7
@@ -107,6 +107,42 @@ impl Xref {
|
||||
/// A replacement of bytes `start..end` of the file.
|
||||
type Edit = (usize, usize, Vec<u8>);
|
||||
|
||||
/// Where byte offsets of the old file land in the new one, for edits sorted by start that do not
|
||||
/// overlap, whose ends therefore never decrease.
|
||||
struct Moves {
|
||||
/// Each edit's end, in order.
|
||||
ends: Vec<usize>,
|
||||
/// The change in length made by the edits up to and including each one.
|
||||
shifts: Vec<isize>,
|
||||
}
|
||||
|
||||
impl Moves {
|
||||
/// `None` when a change in length doesn't fit an `isize`.
|
||||
fn new(edits: &[Edit]) -> Option<Self> {
|
||||
let mut total: isize = 0;
|
||||
let mut shifts = Vec::with_capacity(edits.len());
|
||||
for (start, end, new) in edits {
|
||||
let removed = isize::try_from(end.checked_sub(*start)?).ok()?;
|
||||
total = total.checked_add(isize::try_from(new.len()).ok()?)?.checked_sub(removed)?;
|
||||
shifts.push(total);
|
||||
}
|
||||
Some(Self { ends: edits.iter().map(|e| e.1).collect(), shifts })
|
||||
}
|
||||
|
||||
/// Where `off` lands: moved by every edit that ends at or before it.
|
||||
fn at(&self, off: usize) -> Option<usize> {
|
||||
match self.ends.partition_point(|&end| end <= off) {
|
||||
0 => Some(off),
|
||||
n => off.checked_add_signed(*self.shifts.get(n - 1)?),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `off` as the ten digits of a cross-reference entry; `None` past what ten digits hold.
|
||||
fn entry_offset(off: usize) -> Option<String> {
|
||||
(u64::try_from(off).ok()? < 10_000_000_000).then(|| format!("{off:010}"))
|
||||
}
|
||||
|
||||
/// The edits of one PDF, applied together by [`Self::apply`].
|
||||
pub(crate) struct Patch {
|
||||
edits: Vec<Edit>,
|
||||
@@ -139,18 +175,17 @@ impl Patch {
|
||||
}
|
||||
|
||||
/// `pdf` with the edits, new objects and trailer entries, and a cross-reference table that
|
||||
/// matches. Refused when an edit overlaps another or reaches past the objects.
|
||||
/// matches. Refused when an edit overlaps another or reaches past the objects, or when an
|
||||
/// object would start past the ten digits a cross-reference entry gives its offset in.
|
||||
pub fn apply(mut self, pdf: &[u8], xref: &Xref) -> Result<Vec<u8>, PdfError> {
|
||||
let bad = |why: &str| PdfError::Write(format!("can't edit the written PDF: {why}"));
|
||||
self.edits.sort_by_key(|e| (e.0, e.1));
|
||||
if self.edits.windows(2).any(|w| w[0].1 > w[1].0) || self.edits.iter().any(|e| e.0 > e.1 || e.1 > xref.at) {
|
||||
return Err(bad("edits overlap"));
|
||||
}
|
||||
// Where a byte offset of the old file lands in the new one.
|
||||
let edits = &self.edits;
|
||||
let moved = |off: usize| -> usize {
|
||||
edits.iter().filter(|e| e.1 <= off).fold(off as isize, |o, e| o + e.2.len() as isize - (e.1 - e.0) as isize) as usize
|
||||
};
|
||||
let moves = Moves::new(edits).ok_or_else(|| bad("edits too large"))?;
|
||||
let too_far = || bad("an object lies past what the cross-reference table holds");
|
||||
let added: usize = self.edits.iter().map(|e| e.2.len()).sum::<usize>() + self.objects.iter().map(|o| o.len() + 32).sum::<usize>();
|
||||
let mut out = Vec::with_capacity(pdf.len() + added + 64);
|
||||
let mut at = 0;
|
||||
@@ -175,7 +210,7 @@ impl Patch {
|
||||
for e in old.chunks(20) {
|
||||
match number(e, 0) {
|
||||
Some((off, _)) if e.get(17) == Some(&b'n') => {
|
||||
out.extend_from_slice(format!("{:010}", moved(off)).as_bytes());
|
||||
out.extend_from_slice(moves.at(off).and_then(entry_offset).ok_or_else(too_far)?.as_bytes());
|
||||
out.extend_from_slice(e.get(10..).unwrap_or_default());
|
||||
}
|
||||
_ => out.extend_from_slice(e),
|
||||
@@ -183,7 +218,8 @@ impl Patch {
|
||||
}
|
||||
let eol = old.get(18..20).unwrap_or(b"\r\n");
|
||||
for off in offsets {
|
||||
out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
|
||||
out.extend_from_slice(entry_offset(off).ok_or_else(too_far)?.as_bytes());
|
||||
out.extend_from_slice(b" 00000 n");
|
||||
out.extend_from_slice(eol);
|
||||
}
|
||||
// The trailer, with its size and the new entries.
|
||||
@@ -201,3 +237,279 @@ impl Patch {
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// A PDF written as the export writes one: objects `<< /N n >>` numbered from 1, those `free`
|
||||
/// picks left out with a free entry, then one cross-reference table and the trailer.
|
||||
fn written(count: usize, free: impl Fn(usize) -> bool) -> Vec<u8> {
|
||||
let mut pdf = b"%PDF-1.7\n".to_vec();
|
||||
let mut entries = b"0000000000 65535 f\r\n".to_vec();
|
||||
for n in 1..=count {
|
||||
if free(n) {
|
||||
entries.extend_from_slice(b"0000000000 00001 f\r\n");
|
||||
continue;
|
||||
}
|
||||
entries.extend(format!("{:010} 00000 n\r\n", pdf.len()).bytes());
|
||||
pdf.extend(format!("{n} 0 obj\n<< /N {n} >>\nendobj\n").bytes());
|
||||
}
|
||||
let at = pdf.len();
|
||||
pdf.extend(format!("xref\n0 {}\n", count + 1).bytes());
|
||||
pdf.extend(entries);
|
||||
pdf.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{at}\n%%EOF", count + 1).bytes());
|
||||
pdf
|
||||
}
|
||||
|
||||
/// A small linear congruential generator, so every run makes the same edits.
|
||||
struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
/// A number in `0..n` (0 when `n` is 0).
|
||||
fn below(&mut self, n: usize) -> usize {
|
||||
self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
|
||||
usize::try_from(self.0 >> 33).unwrap() % n.max(1)
|
||||
}
|
||||
|
||||
fn bytes(&mut self) -> Vec<u8> {
|
||||
(0..self.below(9)).map(|i| b'a' + u8::try_from(i).unwrap()).collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// About `wanted` edits that don't overlap, in shuffled order, at random places among the
|
||||
/// objects and at object starts: replacements that grow, shrink or remove bytes, edits that end
|
||||
/// at an object's start, and inserts, several of them at one place. None when `wanted` is 0.
|
||||
fn edits(pdf: &[u8], xref: &Xref, rng: &mut Rng, wanted: usize) -> Vec<Edit> {
|
||||
if wanted == 0 {
|
||||
return vec![];
|
||||
}
|
||||
let first = b"%PDF-1.7\n".len();
|
||||
let mut points: Vec<usize> = xref.objects(pdf).into_iter().map(|(_, off)| off).collect();
|
||||
points.extend((0..wanted).map(|_| first + rng.below(xref.at() - first + 1)));
|
||||
points.push(xref.at());
|
||||
points.sort_unstable();
|
||||
points.dedup();
|
||||
let mut out = vec![];
|
||||
for w in points.windows(2) {
|
||||
let (p, q) = (w[0], w[1]);
|
||||
match rng.below(6) {
|
||||
0 => {}
|
||||
1 => out.push((p, q, rng.bytes())),
|
||||
2 => out.push((p, p, rng.bytes())),
|
||||
3 => {
|
||||
out.push((p, p, rng.bytes()));
|
||||
out.push((p, p, rng.bytes()));
|
||||
out.push((p, q.min(p + rng.below(4)), rng.bytes()));
|
||||
}
|
||||
4 => out.push((p + (q - p) / 2, q, rng.bytes())),
|
||||
_ => out.push((q, q, rng.bytes())),
|
||||
}
|
||||
}
|
||||
for i in (1..out.len()).rev() {
|
||||
out.swap(i, rng.below(i + 1));
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn patch(xref: &Xref, edits: &[Edit], objects: bool) -> Patch {
|
||||
let mut patch = Patch::new(xref);
|
||||
for (start, end, with) in edits {
|
||||
patch.replace(*start, *end, with.clone());
|
||||
}
|
||||
if objects {
|
||||
patch.add_object(b"<< /Added 1 >>".to_vec());
|
||||
patch.trailer_entry(b" /Info 2 0 R");
|
||||
}
|
||||
patch
|
||||
}
|
||||
|
||||
/// Where `off` lands, by walking every edit: how [`Patch::apply`] found it before [`Moves`].
|
||||
fn scan(edits: &[Edit], off: usize) -> usize {
|
||||
edits.iter().filter(|e| e.1 <= off).fold(off as isize, |o, e| o + e.2.len() as isize - (e.1 - e.0) as isize) as usize
|
||||
}
|
||||
|
||||
/// [`Patch::apply`] as it was written before [`Moves`], walking every edit for every object.
|
||||
fn apply_by_scan(mut patch: Patch, pdf: &[u8], xref: &Xref) -> Result<Vec<u8>, PdfError> {
|
||||
let bad = |why: &str| PdfError::Write(format!("can't edit the written PDF: {why}"));
|
||||
patch.edits.sort_by_key(|e| (e.0, e.1));
|
||||
if patch.edits.windows(2).any(|w| w[0].1 > w[1].0) || patch.edits.iter().any(|e| e.0 > e.1 || e.1 > xref.at) {
|
||||
return Err(bad("edits overlap"));
|
||||
}
|
||||
let edits = &patch.edits;
|
||||
let moved = |off: usize| -> usize { scan(edits, off) };
|
||||
let added: usize = patch.edits.iter().map(|e| e.2.len()).sum::<usize>() + patch.objects.iter().map(|o| o.len() + 32).sum::<usize>();
|
||||
let mut out = Vec::with_capacity(pdf.len() + added + 64);
|
||||
let mut at = 0;
|
||||
for (start, end, new) in edits.iter() {
|
||||
out.extend_from_slice(pdf.get(at..*start).ok_or_else(|| bad("bad edit"))?);
|
||||
out.extend_from_slice(new);
|
||||
at = *end;
|
||||
}
|
||||
out.extend_from_slice(pdf.get(at..xref.at).ok_or_else(|| bad("bad edit"))?);
|
||||
let mut offsets = Vec::with_capacity(patch.objects.len());
|
||||
for (i, body) in patch.objects.iter().enumerate() {
|
||||
offsets.push(out.len());
|
||||
out.extend_from_slice(format!("{} 0 obj\n", patch.size + i).as_bytes());
|
||||
out.extend_from_slice(body);
|
||||
out.extend_from_slice(b"\nendobj\n");
|
||||
}
|
||||
let table = out.len();
|
||||
let size = patch.size + patch.objects.len();
|
||||
out.extend_from_slice(format!("xref\n0 {size}\n").as_bytes());
|
||||
let old = pdf.get(xref.entries..xref.entries + 20 * xref.count).ok_or_else(|| bad("bad cross-reference table"))?;
|
||||
for e in old.chunks(20) {
|
||||
match number(e, 0) {
|
||||
Some((off, _)) if e.get(17) == Some(&b'n') => {
|
||||
out.extend_from_slice(format!("{:010}", moved(off)).as_bytes());
|
||||
out.extend_from_slice(e.get(10..).unwrap_or_default());
|
||||
}
|
||||
_ => out.extend_from_slice(e),
|
||||
}
|
||||
}
|
||||
let eol = old.get(18..20).unwrap_or(b"\r\n");
|
||||
for off in offsets {
|
||||
out.extend_from_slice(format!("{off:010} 00000 n").as_bytes());
|
||||
out.extend_from_slice(eol);
|
||||
}
|
||||
let (ts, te) = xref.trailer;
|
||||
let dict = pdf.get(ts..te.saturating_sub(2)).ok_or_else(|| bad("bad trailer"))?;
|
||||
let size_at = find(dict, b"/Size", 0).ok_or_else(|| bad("the trailer has no size"))? + 5;
|
||||
let (_, size_end) = number(dict, size_at).ok_or_else(|| bad("the trailer has no size"))?;
|
||||
let (Some(head), Some(tail)) = (dict.get(..size_at), dict.get(size_end..)) else { return Err(bad("bad trailer")) };
|
||||
out.extend_from_slice(b"trailer\n");
|
||||
out.extend_from_slice(head);
|
||||
out.extend_from_slice(format!(" {size}").as_bytes());
|
||||
out.extend_from_slice(tail);
|
||||
out.extend_from_slice(&patch.trailer);
|
||||
out.extend_from_slice(format!(">>\nstartxref\n{table}\n%%EOF").as_bytes());
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_offset_moves_by_the_edits_that_end_at_or_before_it() {
|
||||
let mut rng = Rng(7661);
|
||||
for round in 0..100 {
|
||||
let pdf = written(1 + rng.below(30), |n| n % 7 == 3);
|
||||
let xref = Xref::read(&pdf).unwrap();
|
||||
let mut edits = edits(&pdf, &xref, &mut rng, [0, 1, 3, 20, 80][round % 5]);
|
||||
edits.sort_by_key(|e| (e.0, e.1));
|
||||
let moves = Moves::new(&edits).unwrap();
|
||||
for off in 0..=pdf.len() {
|
||||
assert_eq!(moves.at(off), Some(scan(&edits, off)), "round {round}, offset {off}, edits {edits:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_writes_the_bytes_a_walk_of_every_edit_wrote() {
|
||||
let mut rng = Rng(1378);
|
||||
for round in 0..300 {
|
||||
let pdf = written(1 + rng.below(40), |n| n % 7 == 3);
|
||||
let xref = Xref::read(&pdf).unwrap();
|
||||
let edits = edits(&pdf, &xref, &mut rng, [0, 1, 3, 20, 80][round % 5]);
|
||||
let objects = round % 3 == 0;
|
||||
let fast = patch(&xref, &edits, objects).apply(&pdf, &xref).unwrap();
|
||||
let walked = apply_by_scan(patch(&xref, &edits, objects), &pdf, &xref).unwrap();
|
||||
assert_eq!(String::from_utf8_lossy(&fast), String::from_utf8_lossy(&walked), "round {round}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_entry_points_at_its_object_after_the_edits() {
|
||||
let pdf = written(30, |n| n % 7 == 3);
|
||||
let xref = Xref::read(&pdf).unwrap();
|
||||
let mut patch = Patch::new(&xref);
|
||||
for (n, off) in xref.objects(&pdf) {
|
||||
let (at, end) = xref.dict(&pdf, n).unwrap();
|
||||
match n % 4 {
|
||||
0 => patch.replace(at, at, b" /Grown [1 2 3]".to_vec()),
|
||||
1 => patch.replace(at, end, b">>\n".to_vec()),
|
||||
// An insert at an object's start goes before the object.
|
||||
2 => patch.replace(off, off, b"% before\n".to_vec()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
let added = patch.add_object(b"<< /Added 1 >>".to_vec());
|
||||
let out = patch.apply(&pdf, &xref).unwrap();
|
||||
let after = Xref::read(&out).unwrap();
|
||||
assert_eq!((after.count(), added), (32, 31));
|
||||
for n in 1..32u32 {
|
||||
let Some(off) = after.offset(&out, n) else {
|
||||
assert!(n % 7 == 3, "object {n} is in use");
|
||||
let e = xref.entries + 20 * n as usize;
|
||||
assert_eq!(out[after.entries + 20 * n as usize..][..20], pdf[e..e + 20], "free entry {n}");
|
||||
continue;
|
||||
};
|
||||
assert!(out[off..].starts_with(format!("{n} 0 obj\n").as_bytes()), "object {n} at {off}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_object_moved_past_ten_digits_is_refused() {
|
||||
let mut pdf = written(2, |_| false);
|
||||
let entry = Xref::read(&pdf).unwrap().entries + 40;
|
||||
// Object 2's entry gives the last offset ten digits hold.
|
||||
pdf[entry..entry + 10].copy_from_slice(b"9999999999");
|
||||
let xref = Xref::read(&pdf).unwrap();
|
||||
let out = Patch::new(&xref).apply(&pdf, &xref).unwrap();
|
||||
assert!(find(&out, b"9999999999 00000 n\r\n", 0).is_some());
|
||||
let mut patch = Patch::new(&xref);
|
||||
patch.replace(9, 9, b"%\n".to_vec());
|
||||
let refused = patch.apply(&pdf, &xref);
|
||||
assert!(matches!(&refused, Err(PdfError::Write(why)) if why.contains("cross-reference table holds")), "{refused:?}");
|
||||
}
|
||||
|
||||
/// CPU time this thread has used, where the system reports it (`/proc` on Linux).
|
||||
fn thread_cpu() -> Option<Duration> {
|
||||
let stat = std::fs::read_to_string("/proc/thread-self/schedstat").ok()?;
|
||||
stat.split_whitespace().next()?.parse().ok().map(Duration::from_nanos)
|
||||
}
|
||||
|
||||
/// How long `f` takes, in thread CPU time where the system reports it, else wall time.
|
||||
fn time<T>(f: impl FnOnce() -> T) -> (Duration, T) {
|
||||
let (cpu, wall) = (thread_cpu(), Instant::now());
|
||||
let out = f();
|
||||
(thread_cpu().zip(cpu).map_or_else(|| wall.elapsed(), |(b, a)| b.saturating_sub(a)), out)
|
||||
}
|
||||
|
||||
fn median(mut runs: Vec<Duration>) -> Duration {
|
||||
runs.sort_unstable();
|
||||
runs[runs.len() / 2]
|
||||
}
|
||||
|
||||
/// `cargo test --release -p vectorcraft-pdf --lib patch::tests::bench -- --ignored --nocapture`.
|
||||
/// The edits are those of forms in overprint: two inside each form's stream (its `/Length` and
|
||||
/// data) and an insert into its dictionary, for `n` forms among `n + 20` objects. Each run
|
||||
/// applies the patch `reps` times, so that small files take long enough for the clock.
|
||||
#[test]
|
||||
#[ignore = "timing; run explicitly"]
|
||||
fn bench_apply_with_many_edits() {
|
||||
println!("clock: {}", if thread_cpu().is_some() { "thread CPU time" } else { "wall time" });
|
||||
println!("| forms | objects | edits | walk every edit (median of 7) | binary search (median of 7) |");
|
||||
println!("| ---: | ---: | ---: | ---: | ---: |");
|
||||
for n in [100, 1_000, 5_000, 20_000] {
|
||||
let pdf = written(n + 20, |_| false);
|
||||
let xref = Xref::read(&pdf).unwrap();
|
||||
let mut edits = vec![];
|
||||
for (obj, _) in xref.objects(&pdf).into_iter().take(n) {
|
||||
let (at, _) = xref.dict(&pdf, obj).unwrap();
|
||||
edits.push((at, at + 1, b" /Length 120 ".to_vec()));
|
||||
edits.push((at + 1, at + 3, b"/Data".to_vec()));
|
||||
edits.push((at + 3, at + 3, b" /Resources << /ExtGState << /GS0 9 0 R >> >>".to_vec()));
|
||||
}
|
||||
let reps = (200_000_000 / (3 * n * n)).clamp(1, 2_000);
|
||||
let (mut walked, mut searched) = (vec![], vec![]);
|
||||
for _ in 0..7 {
|
||||
let (t, old) = time(|| (0..reps).fold(None, |_, _| apply_by_scan(patch(&xref, &edits, false), &pdf, &xref).ok()));
|
||||
walked.push(t);
|
||||
let (t, new) = time(|| (0..reps).fold(None, |_, _| patch(&xref, &edits, false).apply(&pdf, &xref).ok()));
|
||||
searched.push(t);
|
||||
assert!(old.is_some() && old == new, "{n} forms: the files differ");
|
||||
}
|
||||
let ms = |d: Duration| d.as_secs_f64() * 1000.0 / reps as f64;
|
||||
println!("| {n} | {} | {} | {:.3} ms | {:.3} ms |", n + 20, edits.len(), ms(median(walked)), ms(median(searched)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user