mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:24:24 +00:00
Merge pull request #1577 from storytold/codex/zip-seek-boundary
Share bounded seekable ZIP indexing
This commit is contained in:
@@ -70,3 +70,12 @@ calls it first.
|
||||
`preflight`, `read_entry` and `Budget`: the ZIP half of the craft-archive proposal (#126, #129).
|
||||
`check_entry_count`: craft-opc's `guard`. Duplicate detection: craft-penpot's rule, made exact
|
||||
by walking the directory. The writer: craft-opc's `ZipOut`.
|
||||
|
||||
## Seekable indexing
|
||||
|
||||
`MetadataReader::new(reader, metadata_bytes)` wraps a `Read + Seek` source without loading its
|
||||
payload into memory. During ZIP indexing it admits reads only from the last `metadata_bytes`
|
||||
bytes and charges repeated reads against twice that number. Call `Indexing::finish` on the
|
||||
returned handle after the ZIP index has been parsed to permit normal streaming payload reads.
|
||||
The caller must still enforce entry-count, declared-size and overlap limits: the wrapper only
|
||||
bounds metadata reads and does not validate the archive.
|
||||
|
||||
@@ -23,13 +23,17 @@ struct ReadmeExamples;
|
||||
|
||||
mod archive;
|
||||
mod guard;
|
||||
mod metadata_reader;
|
||||
pub mod preflight;
|
||||
mod seekable;
|
||||
mod write;
|
||||
|
||||
pub use archive::{Archive, Limits};
|
||||
pub use guard::check_entry_count;
|
||||
#[cfg(any(test, feature = "test-support"))]
|
||||
pub use guard::zip64_entry_count_sample;
|
||||
pub use metadata_reader::{Indexing, MetadataReader};
|
||||
pub use seekable::{InspectError, inspect_seekable};
|
||||
pub use write::{Compression, Writer};
|
||||
|
||||
/// The payload of a ZIP's first entry when it is a stored file named `mimetype` of at most 256
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
//! Bound the reads a ZIP indexer may perform before accessing entry payloads.
|
||||
|
||||
use std::io::{self, Read, Seek, SeekFrom};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
/// A seekable reader whose indexing phase is restricted to a tail window and read budget.
|
||||
///
|
||||
/// Keep the returned [`Indexing`] handle until the ZIP central directory has been parsed, then
|
||||
/// call [`Indexing::finish`] to allow normal entry reads. Repeated reads during indexing count
|
||||
/// towards the allowance, including reads after seeks. This bounds metadata scanning without
|
||||
/// buffering the whole file or restricting the size of its payload entries.
|
||||
pub struct MetadataReader<R> {
|
||||
inner: R,
|
||||
indexing: Arc<AtomicBool>,
|
||||
metadata_start: u64,
|
||||
remaining: u64,
|
||||
position: u64,
|
||||
}
|
||||
|
||||
/// Controls the indexing restriction for the associated reader.
|
||||
#[derive(Clone)]
|
||||
pub struct Indexing(Arc<AtomicBool>);
|
||||
|
||||
impl Indexing {
|
||||
/// Permit reads outside the metadata window once indexing is finished.
|
||||
pub fn finish(&self) {
|
||||
self.0.store(false, Ordering::Release);
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read + Seek> MetadataReader<R> {
|
||||
/// Limit indexing to the last `metadata_bytes` bytes and at most twice that many bytes read.
|
||||
pub fn new(mut inner: R, metadata_bytes: u64) -> io::Result<(Self, Indexing)> {
|
||||
let end = inner.seek(SeekFrom::End(0))?;
|
||||
let indexing = Arc::new(AtomicBool::new(true));
|
||||
let handle = Indexing(indexing.clone());
|
||||
Ok((
|
||||
Self { inner, indexing, metadata_start: end.saturating_sub(metadata_bytes), remaining: metadata_bytes.saturating_mul(2), position: end },
|
||||
handle,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for MetadataReader<R> {
|
||||
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
|
||||
if bytes.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
let indexing = self.indexing.load(Ordering::Acquire);
|
||||
if indexing && (self.position < self.metadata_start || self.remaining == 0) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "ZIP metadata exceeds the indexing limit"));
|
||||
}
|
||||
let length = if indexing { bytes.len().min(usize::try_from(self.remaining).unwrap_or(usize::MAX)) } else { bytes.len() };
|
||||
let read = self.inner.read(bytes.get_mut(..length).unwrap_or_default())?;
|
||||
self.position = self.position.saturating_add(read as u64);
|
||||
if indexing {
|
||||
self.remaining -= read as u64;
|
||||
}
|
||||
Ok(read)
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Seek> Seek for MetadataReader<R> {
|
||||
fn seek(&mut self, from: SeekFrom) -> io::Result<u64> {
|
||||
self.position = self.inner.seek(from)?;
|
||||
Ok(self.position)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
#[test]
|
||||
fn indexing_cannot_read_payloads_outside_the_metadata_window() {
|
||||
let (mut reader, indexing) = MetadataReader::new(Cursor::new(vec![7; 100]), 8).unwrap();
|
||||
reader.seek(SeekFrom::Start(91)).unwrap();
|
||||
assert!(reader.read(&mut [0; 1]).is_err());
|
||||
reader.seek(SeekFrom::Start(92)).unwrap();
|
||||
reader.read_exact(&mut [0; 8]).unwrap();
|
||||
indexing.finish();
|
||||
reader.seek(SeekFrom::Start(0)).unwrap();
|
||||
reader.read_exact(&mut [0; 100]).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn zip_indexing_is_tail_bounded_then_payload_reads_stream() {
|
||||
let mut writer = crate::Writer::new();
|
||||
writer.add("large.bin", &vec![42; 131_072], crate::Compression::Stored).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
let (reader, indexing) = MetadataReader::new(Cursor::new(bytes), 16_384).unwrap();
|
||||
let config = zip::read::Config { archive_offset: zip::read::ArchiveOffset::Known(0) };
|
||||
let mut archive = zip::ZipArchive::with_config(config, reader).unwrap();
|
||||
assert!(archive.by_name("large.bin").is_err(), "indexing must not read the payload header");
|
||||
indexing.finish();
|
||||
let mut entry = archive.by_name("large.bin").unwrap();
|
||||
let mut payload = Vec::new();
|
||||
entry.read_to_end(&mut payload).unwrap();
|
||||
assert_eq!(payload, vec![42; 131_072]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_metadata_scans_share_one_read_allowance() {
|
||||
let (mut reader, _indexing) = MetadataReader::new(Cursor::new(vec![7; 8]), 8).unwrap();
|
||||
for _ in 0..2 {
|
||||
reader.seek(SeekFrom::Start(0)).unwrap();
|
||||
reader.read_exact(&mut [0; 8]).unwrap();
|
||||
}
|
||||
reader.seek(SeekFrom::Start(0)).unwrap();
|
||||
assert!(reader.read(&mut [0; 1]).is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Checks for an already indexed, seekable ZIP archive.
|
||||
|
||||
use std::io::{Read, Seek};
|
||||
|
||||
/// Why an indexed seekable archive exceeded its caller's limits.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum InspectError {
|
||||
#[error("the archive has {actual} entries, more than {limit}")]
|
||||
TooManyEntries { actual: usize, limit: usize },
|
||||
#[error("declared entry sizes exceed {limit} bytes")]
|
||||
DeclaredBytesExceeded { limit: u64 },
|
||||
#[error("ZIP entries overlap")]
|
||||
OverlappingEntries,
|
||||
#[error("ZIP index: {0}")]
|
||||
Zip(#[from] zip::result::ZipError),
|
||||
}
|
||||
|
||||
/// Check every declared payload, including entries the caller may ignore, and reject overlapping
|
||||
/// entry headers and compressed ranges. This is a post-index check; it does not replace a pre-index count guard or
|
||||
/// bound bytes actually decompressed by a caller.
|
||||
pub fn inspect_seekable<R: Read + Seek>(archive: &mut zip::ZipArchive<R>, max_entries: usize, max_declared_bytes: u64) -> Result<(), InspectError> {
|
||||
if archive.len() > max_entries {
|
||||
return Err(InspectError::TooManyEntries { actual: archive.len(), limit: max_entries });
|
||||
}
|
||||
let mut remaining = max_declared_bytes;
|
||||
for index in 0..archive.len() {
|
||||
remaining =
|
||||
remaining.checked_sub(archive.by_index_raw(index)?.size()).ok_or(InspectError::DeclaredBytesExceeded { limit: max_declared_bytes })?;
|
||||
}
|
||||
if archive.has_overlapping_entries()? {
|
||||
return Err(InspectError::OverlappingEntries);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::unwrap_used, clippy::expect_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
#[test]
|
||||
fn counts_all_declared_entries_and_detects_aliases() {
|
||||
let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
writer.start_file("first", options).unwrap();
|
||||
writer.write_all(b"payload").unwrap();
|
||||
writer.shallow_copy_file("first", "alias").unwrap();
|
||||
let bytes = writer.finish().unwrap().into_inner();
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();
|
||||
assert!(matches!(inspect_seekable(&mut archive, 1, 14), Err(InspectError::TooManyEntries { .. })));
|
||||
assert!(matches!(inspect_seekable(&mut archive, 2, 13), Err(InspectError::DeclaredBytesExceeded { .. })));
|
||||
assert!(matches!(inspect_seekable(&mut archive, 2, 14), Err(InspectError::OverlappingEntries)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_aliases_still_overlap_at_the_header() {
|
||||
let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
writer.start_file("empty", zip::write::SimpleFileOptions::default()).unwrap();
|
||||
writer.shallow_copy_file("empty", "alias").unwrap();
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(writer.finish().unwrap().into_inner())).unwrap();
|
||||
assert!(matches!(inspect_seekable(&mut archive, 2, 0), Err(InspectError::OverlappingEntries)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn permits_exact_limits_and_empty_archive() {
|
||||
let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new()));
|
||||
writer.start_file("a", zip::write::SimpleFileOptions::default()).unwrap();
|
||||
writer.write_all(b"123").unwrap();
|
||||
let mut archive = zip::ZipArchive::new(Cursor::new(writer.finish().unwrap().into_inner())).unwrap();
|
||||
assert!(inspect_seekable(&mut archive, 1, 3).is_ok());
|
||||
let empty = zip::ZipWriter::new(Cursor::new(Vec::new())).finish().unwrap().into_inner();
|
||||
assert!(inspect_seekable(&mut zip::ZipArchive::new(Cursor::new(empty)).unwrap(), 0, 0).is_ok());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user