mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-17 21:52:32 +00:00
441 lines
20 KiB
Rust
441 lines
20 KiB
Rust
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
|
|
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
|
|
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
|
|
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
|
|
//! and the number of entries; a part is read no further than its declared size (a decompression
|
|
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
|
|
|
|
use std::collections::{BTreeMap, BTreeSet};
|
|
use std::io::{Read, Seek, SeekFrom};
|
|
use std::sync::Arc;
|
|
|
|
use serde_json::Value;
|
|
use sha2::{Digest, Sha256};
|
|
|
|
use crate::FormatError;
|
|
|
|
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
|
|
/// CPU time for space.
|
|
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
|
pub enum Compression {
|
|
#[default]
|
|
Fast,
|
|
Compact,
|
|
}
|
|
|
|
/// An app's budgets for its containers. The functions are read each time they are needed, so a
|
|
/// limit the app lets the user raise takes effect at once.
|
|
#[derive(Clone, Copy)]
|
|
pub struct Limits {
|
|
/// The most entries a container holds.
|
|
pub max_entries: usize,
|
|
/// The most bytes all parts may expand to, and all payloads may resolve to.
|
|
pub max_expanded: fn() -> usize,
|
|
/// The most bytes a part of this name may expand to.
|
|
pub part_limit: fn(&str) -> usize,
|
|
/// The Deflate level of a compressed part of this name and size.
|
|
pub level: fn(&str, usize) -> i64,
|
|
/// Whether a container of this many bytes may be read, or why not.
|
|
pub check_size: fn(u64) -> Result<(), String>,
|
|
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
|
|
/// a long part is written a piece at a time and stops before the next piece when this says
|
|
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
|
|
pub cancelled: Option<fn() -> bool>,
|
|
}
|
|
|
|
/// The most bytes of a part written between two cancellation checks.
|
|
const PIECE: usize = 1 << 20;
|
|
|
|
fn invalid(message: impl std::fmt::Display) -> FormatError {
|
|
FormatError::Invalid(message.to_string())
|
|
}
|
|
|
|
/// Whether the bytes start as a container does, whatever the file is called.
|
|
pub fn is_container(bytes: &[u8]) -> bool {
|
|
bytes.starts_with(b"PK\x03\x04")
|
|
}
|
|
|
|
/// The SHA-256 of the bytes.
|
|
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
|
|
Sha256::digest(bytes).into()
|
|
}
|
|
|
|
fn hex(digest: &[u8; 32]) -> String {
|
|
use std::fmt::Write as _;
|
|
let mut out = String::with_capacity(64);
|
|
for byte in digest {
|
|
// Writing to a `String` cannot fail.
|
|
let _ = write!(&mut out, "{byte:02x}");
|
|
}
|
|
out
|
|
}
|
|
|
|
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
|
|
/// the file it was written for.
|
|
pub fn digest(bytes: &[u8]) -> String {
|
|
hex(&sha256(bytes))
|
|
}
|
|
|
|
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
|
|
/// a container uses these to bind the parts without reading the file back.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct PartDigest {
|
|
pub name: String,
|
|
pub len: u64,
|
|
pub digest: [u8; 32],
|
|
}
|
|
|
|
/// A container being written.
|
|
pub struct Writer {
|
|
zip: craft_zip::Writer,
|
|
names: BTreeSet<String>,
|
|
expanded: usize,
|
|
resolved: usize,
|
|
limits: Limits,
|
|
/// Which written parts to record for a signature, and the records so far.
|
|
tracked: Option<TrackedParts>,
|
|
}
|
|
|
|
/// A tracking closure and the part digests collected so far.
|
|
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
|
|
|
|
impl Writer {
|
|
pub fn new(limits: Limits) -> Writer {
|
|
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
|
|
}
|
|
|
|
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
|
|
/// [`Writer::take_parts`]. Used by a caller that signs the container.
|
|
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
|
|
self.tracked = Some((include, Vec::new()));
|
|
}
|
|
|
|
/// The tracked parts written so far, oldest first, and stop tracking.
|
|
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
|
|
self.tracked.take().map(|(_, parts)| parts)
|
|
}
|
|
|
|
/// The container's bytes.
|
|
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
|
|
self.zip.finish().map_err(invalid)
|
|
}
|
|
|
|
/// Add a part, compressed or stored. A name already written is written once.
|
|
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
|
|
self.part_with_digest(name, bytes, compressed, None)
|
|
}
|
|
|
|
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
|
|
/// so a signed save does not hash the same bytes twice.
|
|
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
|
|
if bytes.len() > (self.limits.part_limit)(name) {
|
|
return Err(invalid("part too large"));
|
|
}
|
|
if self.names.contains(name) {
|
|
return Ok(());
|
|
}
|
|
// Check the counters this part would leave, and commit them only once it is written: a
|
|
// refused part is not remembered as written.
|
|
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
|
|
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
|
|
return Err(invalid("package too large"));
|
|
}
|
|
let cancelled = self.limits.cancelled;
|
|
if cancelled.is_some_and(|c| c()) {
|
|
return Err(FormatError::Cancelled);
|
|
}
|
|
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
|
|
let compression = if compressed {
|
|
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
|
|
} else {
|
|
craft_zip::Compression::Stored
|
|
};
|
|
match cancelled {
|
|
Some(cancelled) if bytes.len() > PIECE => {
|
|
let mut stopped = false;
|
|
let written = self.zip.add_with(name, compression, |out| {
|
|
for piece in bytes.chunks(PIECE) {
|
|
if cancelled() {
|
|
stopped = true;
|
|
return Err(std::io::Error::other("cancelled"));
|
|
}
|
|
out.write_all(piece)?;
|
|
}
|
|
Ok(())
|
|
});
|
|
if stopped {
|
|
return Err(FormatError::Cancelled);
|
|
}
|
|
written.map_err(invalid)?;
|
|
}
|
|
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
|
|
}
|
|
self.names.insert(name.to_owned());
|
|
self.expanded = expanded;
|
|
if let Some((include, parts)) = &mut self.tracked
|
|
&& include(name)
|
|
{
|
|
let digest = digest.unwrap_or_else(|| sha256(bytes));
|
|
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
|
|
/// file within `budget` and its expanded size within the reader's budget? The parts already
|
|
/// written count at their expanded size (an upper bound on what they take in the file).
|
|
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
|
|
self.remaining(entries, budget).is_some_and(|left| extra <= left)
|
|
}
|
|
|
|
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
|
|
pub fn space(&self, entries: usize, budget: usize) -> usize {
|
|
self.remaining(entries, budget).unwrap_or(0)
|
|
}
|
|
|
|
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
|
|
/// written already is past `budget` or the reader's budget.
|
|
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
|
|
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
|
|
let overhead = count.saturating_mul(256).saturating_add(65_536);
|
|
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
|
|
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
|
|
Some(in_file.min(expanded))
|
|
}
|
|
|
|
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
|
|
/// refers to it.
|
|
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
|
|
let resolved = self
|
|
.resolved
|
|
.checked_add(bytes.len())
|
|
.filter(|&n| n <= (self.limits.max_expanded)())
|
|
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
|
let digest = sha256(bytes);
|
|
let name = format!("blobs/{}.{extension}", hex(&digest));
|
|
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
|
|
self.resolved = resolved;
|
|
Ok(Value::String(name))
|
|
}
|
|
}
|
|
|
|
/// A container being read. Opening reads all bytes into memory and checks the directory
|
|
/// (the entry count, unique and plain names, every part's
|
|
/// declared size), not the parts.
|
|
pub struct Reader {
|
|
archive: craft_zip::Archive<'static>,
|
|
blobs: BTreeMap<String, Vec<u8>>,
|
|
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
|
|
pub hydrated: usize,
|
|
document: Option<Arc<[u8]>>,
|
|
limits: Limits,
|
|
/// Every byte part reads have expanded to, whatever the budget was at the time.
|
|
read_total: u64,
|
|
}
|
|
|
|
impl Reader {
|
|
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
|
|
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
|
|
(limits.check_size)(length).map_err(invalid)?;
|
|
|
|
// Read all bytes into memory for the archive, no more than the length just approved: a file
|
|
// that grew since is refused rather than read past its budget.
|
|
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
|
|
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
|
|
let mut bytes = Vec::new();
|
|
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
|
|
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
|
|
if bytes.len() as u64 != length {
|
|
return Err(invalid("the file changed while it was read"));
|
|
}
|
|
|
|
let mut archive = craft_zip::Archive::open_owned(
|
|
bytes,
|
|
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
|
|
)
|
|
.map_err(invalid)?;
|
|
// the directory lists distinct names, so a repeated name is caught here
|
|
if archive.has_duplicate_names() {
|
|
return Err(invalid("duplicate or invalid archive entry"));
|
|
}
|
|
|
|
// Validate entry names match format rules: no empty, no backslashes,
|
|
// no path components that are "." or "..", and no duplicates; and every part's declared
|
|
// size within its configured limit.
|
|
let mut name_set = BTreeSet::new();
|
|
for entry_idx in 0..archive.len() {
|
|
// an entry whose name can't be read would be left out of the package unnoticed
|
|
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
|
|
return Err(invalid("duplicate or invalid archive entry"));
|
|
};
|
|
{
|
|
if name.is_empty()
|
|
|| name.contains('\\')
|
|
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|
|
|| !name_set.insert(name.clone())
|
|
{
|
|
return Err(invalid("duplicate or invalid archive entry"));
|
|
}
|
|
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
|
|
return Err(invalid("part too large"));
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
|
|
}
|
|
|
|
/// The file's own part, `document.json`, read once and kept: provisional caches and the
|
|
/// authoritative reading are bound to the same bytes, even if another process overwrites the
|
|
/// open file in place. Blobs remain hash-checked.
|
|
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
|
|
if self.document.is_none() {
|
|
let bytes = self.limited_part("document.json", limit)?;
|
|
self.document = Some(bytes.into());
|
|
}
|
|
// a cached document still keeps the current part limit
|
|
let limit = limit.min((self.limits.part_limit)("document.json"));
|
|
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
|
|
}
|
|
|
|
/// Whether the container has a part of this name.
|
|
pub fn has_part(&self, name: &str) -> bool {
|
|
self.archive.contains(name)
|
|
}
|
|
|
|
/// The names of every part, in the container's order.
|
|
pub fn part_names(&self) -> Vec<String> {
|
|
self.archive.names().map(|n| n.into_owned()).collect()
|
|
}
|
|
|
|
/// The part of this name, at most `limit` bytes and never more than its configured limit.
|
|
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
|
|
let limit = limit.min((self.limits.part_limit)(name));
|
|
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
|
|
if expected > limit as u64 {
|
|
return Err(invalid("part too large"));
|
|
}
|
|
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
|
|
// bytes already read still counting against it
|
|
let budget = self.archive.budget_mut();
|
|
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
|
|
if budget.charge(self.read_total).is_err() {
|
|
return Err(invalid("package too large"));
|
|
}
|
|
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
|
|
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
|
|
if bytes.len() as u64 != expected {
|
|
return Err(invalid("part length mismatch"));
|
|
}
|
|
self.read_total = self.read_total.saturating_add(expected);
|
|
Ok(bytes)
|
|
}
|
|
|
|
/// The blob `value`'s `field` names, of this extension, within its part limit.
|
|
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
|
|
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
|
|
self.limited_blob(value, field, extension, limit)
|
|
}
|
|
|
|
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
|
|
/// reference counts against the resolved budget.
|
|
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
|
|
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
|
|
// a cached blob still keeps the part's current limit
|
|
let limit = limit.min((self.limits.part_limit)(name));
|
|
let hash = name
|
|
.strip_prefix("blobs/")
|
|
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
|
|
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
|
.ok_or_else(|| invalid("invalid blob reference"))?;
|
|
if !self.blobs.contains_key(name) {
|
|
let bytes = self.limited_part(name, limit)?;
|
|
if digest(&bytes) != hash {
|
|
return Err(invalid(format!("blob hash mismatch: {name}")));
|
|
}
|
|
self.blobs.insert(name.to_owned(), bytes);
|
|
}
|
|
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
|
|
if bytes.len() > limit {
|
|
return Err(invalid("blob exceeds its resource budget"));
|
|
}
|
|
// Count repeated references too: a tiny archive must not expand into unbounded owned
|
|
// strings or arrays when the same blob is referenced repeatedly.
|
|
self.hydrated = self
|
|
.hydrated
|
|
.checked_add(bytes.len())
|
|
.filter(|&n| n <= (self.limits.max_expanded)())
|
|
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
|
Ok(bytes)
|
|
}
|
|
}
|
|
|
|
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
|
|
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
|
|
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
|
|
#[derive(Clone, Copy, Debug)]
|
|
pub struct Thumbnail {
|
|
/// The key's format tag.
|
|
pub format: &'static str,
|
|
/// The build that wrote it: only the same build reads it back.
|
|
pub app: &'static str,
|
|
/// Its width and height in pixels.
|
|
pub size: [usize; 2],
|
|
/// The most bytes it may have.
|
|
pub max_bytes: usize,
|
|
}
|
|
|
|
impl Thumbnail {
|
|
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
|
|
/// consumer decodes with its own dimension and allocation limits).
|
|
pub fn valid(&self, bytes: &[u8]) -> bool {
|
|
bytes.len() <= self.max_bytes
|
|
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
|
|
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
|
|
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
|
|
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
|
|
}
|
|
|
|
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
|
|
/// room for it within `budget`; otherwise leave it out, never failing the save.
|
|
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
|
|
if !self.valid(png) {
|
|
return Ok(());
|
|
}
|
|
let key =
|
|
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
|
|
.map_err(invalid)?;
|
|
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
|
|
if !fits("cache/thumbnail.json", key.len())
|
|
|| !fits("cache/thumbnail.png", png.len())
|
|
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
|
|
{
|
|
return Ok(());
|
|
}
|
|
writer.part("cache/thumbnail.json", &key, false)?;
|
|
writer.part("cache/thumbnail.png", png, false)
|
|
}
|
|
|
|
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
|
|
/// read within `document_limit`).
|
|
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
|
|
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
|
|
let document = reader.document(document_limit)?;
|
|
if key.get("format").and_then(Value::as_str) != Some(self.format)
|
|
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|
|
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
|
|
{
|
|
return Err(invalid("thumbnail key"));
|
|
}
|
|
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
|
|
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
|
|
return Err(invalid("thumbnail payload"));
|
|
}
|
|
Ok(png)
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
#[path = "container_tests.rs"]
|
|
mod tests;
|