mirror of
https://github.com/storytold/pdfcraft.git
synced 2026-11-10 19:22:00 +00:00
export: the Office zip writer outgrows the classic records with ZIP64 (#2554)
This commit is contained in:
midasmatterhorn
+277
-39
@@ -1,4 +1,6 @@
|
||||
//! A minimal zip writer (deflate or store, no zip64) for Office files.
|
||||
//! A minimal zip writer (deflate or store) for Office files. Archives that outgrow the classic
|
||||
//! end-of-central-directory record's 16-bit entry count or 32-bit sizes are written with the
|
||||
//! ZIP64 records (APPNOTE 4.4), which every reader understands; small ones are unchanged.
|
||||
|
||||
use std::io::Write;
|
||||
|
||||
@@ -22,12 +24,102 @@ fn crc32(data: &[u8]) -> u32 {
|
||||
c ^ 0xFFFF_FFFF
|
||||
}
|
||||
|
||||
/// Builds a zip archive in memory.
|
||||
/// The largest entry count the classic end-of-central-directory record holds (APPNOTE 4.3.12).
|
||||
const MAX_CLASSIC_COUNT: u64 = u16::MAX as u64;
|
||||
/// The largest size or offset the classic records hold: past it the field is a sentinel and the
|
||||
/// ZIP64 record carries the real value (APPNOTE 4.4.1).
|
||||
const MAX_CLASSIC_U32: u64 = u32::MAX as u64;
|
||||
|
||||
/// The ZIP64 extended-information extra field (APPNOTE 4.5.3) holding `fields` — the values that
|
||||
/// no longer fit their classic fields — in specification order. Empty when everything fits.
|
||||
fn zip64_extra(fields: &[u64]) -> Vec<u8> {
|
||||
let mut e = Vec::with_capacity(4 + 8 * fields.len());
|
||||
if !fields.is_empty() {
|
||||
e.extend(0x0001u16.to_le_bytes());
|
||||
e.extend((8 * fields.len() as u16).to_le_bytes());
|
||||
for &f in fields {
|
||||
e.extend(f.to_le_bytes());
|
||||
}
|
||||
}
|
||||
e
|
||||
}
|
||||
|
||||
/// The classic 32-bit field value for `value`: the value itself, or the 0xFFFFFFFF sentinel when it
|
||||
/// no longer fits — including when it equals the sentinel, which a reader cannot tell from a real
|
||||
/// value — in which case the ZIP64 extra field carries the real value (APPNOTE 4.4.3).
|
||||
fn classic_u32(value: u64) -> u32 {
|
||||
if value >= MAX_CLASSIC_U32 { u32::MAX } else { value as u32 }
|
||||
}
|
||||
|
||||
/// Builds one member's local file header and central directory entry. The sizes and `offset` are
|
||||
/// parameters, not read from the body, so tests can inject values past the classic fields without
|
||||
/// allocating the data.
|
||||
fn entry_headers(name: &str, crc: u32, method: u16, compressed: u64, uncompressed: u64, offset: u64) -> (Vec<u8>, Vec<u8>) {
|
||||
// A value at or above the sentinel cannot be held by a classic field, so it is offloaded.
|
||||
let size_is_zip64 = uncompressed >= MAX_CLASSIC_U32 || compressed >= MAX_CLASSIC_U32;
|
||||
// A local header that carries ZIP64 sizes carries both, original then compressed, even when only
|
||||
// one overflowed (APPNOTE 4.5.3). A central entry carries one value for each field it set to its
|
||||
// sentinel, and only those, so a reader walking the fields by their classic ones stays in step.
|
||||
let local_extra = if size_is_zip64 { zip64_extra(&[uncompressed, compressed]) } else { Vec::new() };
|
||||
let mut central_fields = Vec::new();
|
||||
if uncompressed >= MAX_CLASSIC_U32 {
|
||||
central_fields.push(uncompressed);
|
||||
}
|
||||
if compressed >= MAX_CLASSIC_U32 {
|
||||
central_fields.push(compressed);
|
||||
}
|
||||
if offset >= MAX_CLASSIC_U32 {
|
||||
central_fields.push(offset);
|
||||
}
|
||||
let central_extra = zip64_extra(¢ral_fields);
|
||||
// An entry that carries ZIP64 fields needs version 4.5 to extract (APPNOTE 4.4.3); the version
|
||||
// in the local header and the central entry agree.
|
||||
let version_needed = if size_is_zip64 || offset >= MAX_CLASSIC_U32 { 45u16 } else { 20u16 };
|
||||
let header = |sig: u32, central: bool| {
|
||||
let mut h = Vec::new();
|
||||
h.extend(sig.to_le_bytes());
|
||||
if central {
|
||||
h.extend(20u16.to_le_bytes()); // made by
|
||||
}
|
||||
h.extend(version_needed.to_le_bytes()); // version needed
|
||||
h.extend(0x0800u16.to_le_bytes()); // UTF-8 names
|
||||
h.extend(method.to_le_bytes());
|
||||
h.extend([0, 0, 0x21, 0]); // 00:00, 1980-01-01 (a zero date is not a valid DOS date)
|
||||
h.extend(crc.to_le_bytes());
|
||||
// A local ZIP64 extra carries both sizes, so both corresponding classic fields are
|
||||
// sentinels even when only one size needs offloading (APPNOTE 4.5.3). Central entries
|
||||
// remain selective and use a sentinel only for fields represented in their extra.
|
||||
let (compressed_field, uncompressed_field) =
|
||||
if !central && size_is_zip64 { (u32::MAX, u32::MAX) } else { (classic_u32(compressed), classic_u32(uncompressed)) };
|
||||
h.extend(compressed_field.to_le_bytes());
|
||||
h.extend(uncompressed_field.to_le_bytes());
|
||||
h.extend((name.len() as u16).to_le_bytes());
|
||||
let extra = if central { ¢ral_extra } else { &local_extra };
|
||||
h.extend((extra.len() as u16).to_le_bytes()); // extra
|
||||
if central {
|
||||
h.extend(0u16.to_le_bytes()); // comment
|
||||
h.extend(0u16.to_le_bytes()); // disk
|
||||
h.extend(0u16.to_le_bytes()); // internal attributes
|
||||
h.extend(0u32.to_le_bytes()); // external attributes
|
||||
h.extend(classic_u32(offset).to_le_bytes());
|
||||
}
|
||||
h.extend(name.as_bytes());
|
||||
h.extend(extra);
|
||||
h
|
||||
};
|
||||
(header(0x0403_4b50, false), header(0x0201_4b50, true))
|
||||
}
|
||||
|
||||
/// Builds a zip archive in memory. An archive past the classic record's 16-bit entry count or
|
||||
/// 32-bit sizes and offsets is written with a ZIP64 end-of-central-directory record: a
|
||||
/// PowerPoint export writes two entries per page plus ten fixed parts, so a document with
|
||||
/// 32 763 pages already passes 65 535 entries, and without the ZIP64 record the count wraps
|
||||
/// and readers see a central directory that does not describe the archive.
|
||||
#[derive(Default)]
|
||||
pub struct Zip {
|
||||
out: Vec<u8>,
|
||||
central: Vec<u8>,
|
||||
count: u16,
|
||||
count: usize,
|
||||
}
|
||||
|
||||
impl Zip {
|
||||
@@ -41,49 +133,44 @@ impl Zip {
|
||||
} else {
|
||||
(0u16, data.to_vec())
|
||||
};
|
||||
let offset = self.out.len() as u32;
|
||||
let header = |sig: u32, central: bool| {
|
||||
let mut h = Vec::new();
|
||||
h.extend(sig.to_le_bytes());
|
||||
if central {
|
||||
h.extend(20u16.to_le_bytes()); // made by
|
||||
}
|
||||
h.extend(20u16.to_le_bytes()); // version needed
|
||||
h.extend(0x0800u16.to_le_bytes()); // UTF-8 names
|
||||
h.extend(method.to_le_bytes());
|
||||
h.extend([0, 0, 0x21, 0]); // 00:00, 1980-01-01 (a zero date is not a valid DOS date)
|
||||
h.extend(crc.to_le_bytes());
|
||||
h.extend((body.len() as u32).to_le_bytes());
|
||||
h.extend((data.len() as u32).to_le_bytes());
|
||||
h.extend((name.len() as u16).to_le_bytes());
|
||||
h.extend(0u16.to_le_bytes()); // extra
|
||||
if central {
|
||||
h.extend(0u16.to_le_bytes()); // comment
|
||||
h.extend(0u16.to_le_bytes()); // disk
|
||||
h.extend(0u16.to_le_bytes()); // internal attributes
|
||||
h.extend(0u32.to_le_bytes()); // external attributes
|
||||
h.extend(offset.to_le_bytes());
|
||||
}
|
||||
h.extend(name.as_bytes());
|
||||
h
|
||||
};
|
||||
self.out.extend(header(0x0403_4b50, false));
|
||||
let offset = self.out.len() as u64;
|
||||
let (local, central) = entry_headers(name, crc, method, body.len() as u64, data.len() as u64, offset);
|
||||
self.out.extend(local);
|
||||
self.out.extend(&body);
|
||||
self.central.extend(header(0x0201_4b50, true));
|
||||
self.central.extend(central);
|
||||
self.count += 1;
|
||||
}
|
||||
|
||||
pub fn finish(mut self) -> Vec<u8> {
|
||||
let start = self.out.len() as u32;
|
||||
let size = self.central.len() as u32;
|
||||
let start = self.out.len() as u64;
|
||||
let size = self.central.len() as u64;
|
||||
let count = self.count as u64;
|
||||
self.out.append(&mut self.central);
|
||||
let classic_count = if count > MAX_CLASSIC_COUNT { u16::MAX } else { count as u16 };
|
||||
if count > MAX_CLASSIC_COUNT || size >= MAX_CLASSIC_U32 || start >= MAX_CLASSIC_U32 {
|
||||
let eocd64_at = self.out.len() as u64;
|
||||
self.out.extend(0x0606_4b50u32.to_le_bytes()); // ZIP64 end of central directory record
|
||||
self.out.extend(44u64.to_le_bytes()); // size of the rest of this record
|
||||
self.out.extend(45u16.to_le_bytes()); // version made by (4.5, DOS)
|
||||
self.out.extend(45u16.to_le_bytes()); // version needed to extract
|
||||
self.out.extend(0u32.to_le_bytes()); // number of this disk
|
||||
self.out.extend(0u32.to_le_bytes()); // disk with the start of the central directory
|
||||
self.out.extend(count.to_le_bytes()); // entries on this disk
|
||||
self.out.extend(count.to_le_bytes()); // entries in total
|
||||
self.out.extend(size.to_le_bytes()); // size of the central directory
|
||||
self.out.extend(start.to_le_bytes()); // offset of the central directory
|
||||
self.out.extend(0x0706_4b50u32.to_le_bytes()); // ZIP64 end of central directory locator
|
||||
self.out.extend(0u32.to_le_bytes()); // disk with the ZIP64 record
|
||||
self.out.extend(eocd64_at.to_le_bytes()); // offset of the ZIP64 record
|
||||
self.out.extend(1u32.to_le_bytes()); // total number of disks
|
||||
}
|
||||
self.out.extend(0x0605_4b50u32.to_le_bytes());
|
||||
self.out.extend([0u8; 4]);
|
||||
self.out.extend(self.count.to_le_bytes());
|
||||
self.out.extend(self.count.to_le_bytes());
|
||||
self.out.extend(size.to_le_bytes());
|
||||
self.out.extend(start.to_le_bytes());
|
||||
self.out.extend(0u16.to_le_bytes());
|
||||
self.out.extend([0u8; 4]); // number of this disk, disk with the central directory
|
||||
self.out.extend(classic_count.to_le_bytes()); // entries on this disk
|
||||
self.out.extend(classic_count.to_le_bytes()); // entries in total
|
||||
self.out.extend(classic_u32(size).to_le_bytes());
|
||||
self.out.extend(classic_u32(start).to_le_bytes());
|
||||
self.out.extend(0u16.to_le_bytes()); // comment length
|
||||
self.out
|
||||
}
|
||||
}
|
||||
@@ -107,4 +194,155 @@ mod tests {
|
||||
assert_eq!(b.windows(4).filter(|w| *w == b"PK\x01\x02").count(), 2);
|
||||
assert_eq!(&b[b.len() - 22..b.len() - 18], b"PK\x05\x06");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn more_entries_than_the_classic_record_holds_writes_zip64() {
|
||||
// A PowerPoint export writes two entries per page plus ten fixed parts, so a
|
||||
// document with 32 763 pages already passes the classic record's 16-bit entry
|
||||
// count. The count must not wrap: it overflows in a debug build, and in a release
|
||||
// one it silently writes a central directory readers cannot match to the archive.
|
||||
let mut z = Zip::default();
|
||||
for i in 0..u16::MAX as usize + 1 {
|
||||
z.add(&format!("ppt/slides/slide{i}.xml"), b"x", true);
|
||||
z.add(&format!("ppt/slides/_rels/slide{i}.xml.rels"), b"y", true);
|
||||
}
|
||||
let b = z.finish();
|
||||
let entries = 2 * (u16::MAX as usize + 1);
|
||||
// The classic end-of-central-directory record holds sentinels…
|
||||
let eocd = &b[b.len() - 22..];
|
||||
assert_eq!(&eocd[..4], &0x0605_4b50u32.to_le_bytes());
|
||||
assert_eq!(&eocd[8..10], &u16::MAX.to_le_bytes(), "entries on this disk");
|
||||
assert_eq!(&eocd[10..12], &u16::MAX.to_le_bytes(), "entries in total");
|
||||
// …and the ZIP64 locator and record carry the truth.
|
||||
let locator = &b[b.len() - 42..b.len() - 22];
|
||||
assert_eq!(&locator[..4], &0x0706_4b50u32.to_le_bytes());
|
||||
let at = u64::from_le_bytes(locator[8..16].try_into().unwrap_or([0; 8])) as usize;
|
||||
assert_eq!(&b[at..at + 4], &0x0606_4b50u32.to_le_bytes());
|
||||
assert_eq!(u64::from_le_bytes(b[at + 24..at + 32].try_into().unwrap_or([0; 8])), entries as u64, "entries on this disk");
|
||||
assert_eq!(u64::from_le_bytes(b[at + 32..at + 40].try_into().unwrap_or([0; 8])), entries as u64, "entries in total");
|
||||
}
|
||||
|
||||
/// The extra field of a local file header (APPNOTE 4.3.7), which follows the file name.
|
||||
fn local_extra(h: &[u8]) -> &[u8] {
|
||||
let le = |i: usize, n: usize| h[i..i + n].iter().rev().fold(0usize, |v, b| v << 8 | *b as usize);
|
||||
let (name_len, extra_len) = (le(26, 2), le(28, 2));
|
||||
&h[30 + name_len..30 + name_len + extra_len]
|
||||
}
|
||||
|
||||
/// The extra field of a central directory entry (APPNOTE 4.3.12), which follows the file name.
|
||||
fn central_extra(h: &[u8]) -> &[u8] {
|
||||
let le = |i: usize, n: usize| h[i..i + n].iter().rev().fold(0usize, |v, b| v << 8 | *b as usize);
|
||||
let (name_len, extra_len) = (le(28, 2), le(30, 2));
|
||||
&h[46 + name_len..46 + name_len + extra_len]
|
||||
}
|
||||
|
||||
/// The classic compressed and original size fields of a local file header (`at` 18) or a
|
||||
/// central directory entry (`at` 20).
|
||||
fn classic_sizes(h: &[u8], at: usize) -> (u32, u32) {
|
||||
let field = |i: usize| u32::from_le_bytes(h[i..i + 4].try_into().unwrap_or([0; 4]));
|
||||
(field(at), field(at + 4))
|
||||
}
|
||||
|
||||
/// Asserts the local header carries both sizes whenever either needs ZIP64, with both classic
|
||||
/// size fields set to the sentinel (APPNOTE 4.5.3: a ZIP64 value appears only for a field set to
|
||||
/// its sentinel, and the local entry must carry both sizes). The central entry stays selective:
|
||||
/// only the fields that themselves overflow get a value, in specification order.
|
||||
fn assert_local_pair_and_central_selectivity(compressed: u64, uncompressed: u64) {
|
||||
let at_limit = |v: u64| v >= u32::MAX as u64;
|
||||
let (local, central) = entry_headers("a", 0, 0, compressed, uncompressed, 0);
|
||||
assert_eq!(classic_sizes(&local, 18), (u32::MAX, u32::MAX), "both local size fields are sentinels for ({compressed}, {uncompressed})");
|
||||
let extra = local_extra(&local);
|
||||
assert_eq!(
|
||||
u16::from_le_bytes(extra[2..4].try_into().unwrap_or([0; 2])),
|
||||
16,
|
||||
"both local sizes are present for ({compressed}, {uncompressed})"
|
||||
);
|
||||
assert_eq!(u64::from_le_bytes(extra[4..12].try_into().unwrap_or([0; 8])), uncompressed, "original size first");
|
||||
assert_eq!(u64::from_le_bytes(extra[12..20].try_into().unwrap_or([0; 8])), compressed, "compressed size second");
|
||||
let expected =
|
||||
(if at_limit(compressed) { u32::MAX } else { compressed as u32 }, if at_limit(uncompressed) { u32::MAX } else { uncompressed as u32 });
|
||||
assert_eq!(
|
||||
classic_sizes(¢ral, 20),
|
||||
expected,
|
||||
"the central entry offloads only the fields that overflow for ({compressed}, {uncompressed})"
|
||||
);
|
||||
let fields = usize::from(at_limit(uncompressed)) + usize::from(at_limit(compressed));
|
||||
let extra = central_extra(¢ral);
|
||||
assert_eq!(
|
||||
u16::from_le_bytes(extra[2..4].try_into().unwrap_or([0; 2])),
|
||||
8 * fields as u16,
|
||||
"one central value per sentinel field for ({compressed}, {uncompressed})"
|
||||
);
|
||||
if at_limit(uncompressed) {
|
||||
assert_eq!(u64::from_le_bytes(extra[4..12].try_into().unwrap_or([0; 8])), uncompressed, "original size first");
|
||||
}
|
||||
if at_limit(compressed) {
|
||||
let at = if at_limit(uncompressed) { 12 } else { 4 };
|
||||
assert_eq!(u64::from_le_bytes(extra[at..at + 8].try_into().unwrap_or([0; 8])), compressed, "compressed size in specification order");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_local_sizes_are_sentinels_when_only_one_needs_zip64() {
|
||||
// A local ZIP64 extra must carry both sizes, and its values correspond to classic fields set
|
||||
// to their sentinel — so a small classic size next to an overflowing one is wrong even
|
||||
// though the extra holds the truth. Both orders of which size overflows.
|
||||
assert_local_pair_and_central_selectivity(100, u32::MAX as u64 + 1);
|
||||
assert_local_pair_and_central_selectivity(u32::MAX as u64 + 1, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_local_sizes_are_sentinels_when_one_equals_the_sentinel() {
|
||||
// A size of exactly 0xFFFFFFFF is indistinguishable from the sentinel, so it needs ZIP64
|
||||
// just like a larger one; the other local size must then become a sentinel too. Both orders.
|
||||
assert_local_pair_and_central_selectivity(100, u32::MAX as u64);
|
||||
assert_local_pair_and_central_selectivity(u32::MAX as u64, 100);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn local_zip64_extra_carries_both_sizes() {
|
||||
// When only one size overflows the u32 limit, the local header still carries both sizes —
|
||||
// original then compressed — because a local ZIP64 extra that carries either must carry
|
||||
// both (APPNOTE 4.5.3). Either size may be the one past the limit.
|
||||
for (compressed, uncompressed) in [(100u64, u32::MAX as u64 + 1), (u32::MAX as u64 + 1, 100)] {
|
||||
let (local, _) = entry_headers("a", 0, 0, compressed, uncompressed, 0);
|
||||
let extra = local_extra(&local);
|
||||
assert_eq!(u16::from_le_bytes(extra[..2].try_into().unwrap_or([0; 2])), 0x0001, "ZIP64 extra id");
|
||||
assert_eq!(u16::from_le_bytes(extra[2..4].try_into().unwrap_or([0; 2])), 16, "both sizes are present");
|
||||
assert_eq!(u64::from_le_bytes(extra[4..12].try_into().unwrap_or([0; 8])), uncompressed, "original size first");
|
||||
assert_eq!(u64::from_le_bytes(extra[12..20].try_into().unwrap_or([0; 8])), compressed, "compressed size second");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zip64_entries_need_version_4_5() {
|
||||
// An entry that carries ZIP64 fields must advertise version 4.5 to extract, in both the
|
||||
// local file header and the central directory entry (APPNOTE 4.4.3). An entry that fits
|
||||
// every classic field stays at 2.0, so small archives are unchanged.
|
||||
let (local, central) = entry_headers("a", 0, 0, u32::MAX as u64 + 1, u32::MAX as u64 + 1, 0);
|
||||
assert_eq!(u16::from_le_bytes(local[4..6].try_into().unwrap_or([0; 2])), 45, "local version needed");
|
||||
assert_eq!(u16::from_le_bytes(central[6..8].try_into().unwrap_or([0; 2])), 45, "central version needed");
|
||||
let (local, central) = entry_headers("a", 0, 0, 4, 8, 0);
|
||||
assert_eq!(u16::from_le_bytes(local[4..6].try_into().unwrap_or([0; 2])), 20, "local version needed");
|
||||
assert_eq!(u16::from_le_bytes(central[6..8].try_into().unwrap_or([0; 2])), 20, "central version needed");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn size_equal_to_the_sentinel_is_offloaded() {
|
||||
// A real size of exactly u32::MAX is indistinguishable from the 0xFFFFFFFF sentinel, so it
|
||||
// too must be offloaded: every classic field set to the sentinel needs its 8-byte value in
|
||||
// the central extra, or a reader such as Python's zipfile reports "Compress size not
|
||||
// found". Both orders of which size is the MAX+1 one must work.
|
||||
for (compressed, uncompressed) in [(u32::MAX as u64, u32::MAX as u64 + 1), (u32::MAX as u64 + 1, u32::MAX as u64)] {
|
||||
let (local, central) = entry_headers("a", 0, 0, compressed, uncompressed, 0);
|
||||
assert_eq!(u32::from_le_bytes(local[18..22].try_into().unwrap_or([0; 4])), u32::MAX, "local compressed sentinel");
|
||||
assert_eq!(u32::from_le_bytes(local[22..26].try_into().unwrap_or([0; 4])), u32::MAX, "local original sentinel");
|
||||
assert_eq!(u32::from_le_bytes(central[20..24].try_into().unwrap_or([0; 4])), u32::MAX, "central compressed sentinel");
|
||||
assert_eq!(u32::from_le_bytes(central[24..28].try_into().unwrap_or([0; 4])), u32::MAX, "central original sentinel");
|
||||
let extra = central_extra(¢ral);
|
||||
assert_eq!(u16::from_le_bytes(extra[2..4].try_into().unwrap_or([0; 2])), 16, "a value for each sentinel field");
|
||||
assert_eq!(u64::from_le_bytes(extra[4..12].try_into().unwrap_or([0; 8])), uncompressed, "original size first");
|
||||
assert_eq!(u64::from_le_bytes(extra[12..20].try_into().unwrap_or([0; 8])), compressed, "compressed size second");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user