mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 20:25:02 +00:00
Share streaming gzip writer with stored segments
This commit is contained in:
Generated
+14
-7
@@ -1843,6 +1843,13 @@ dependencies = [
|
||||
"image",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-gzip"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-hdr"
|
||||
version = "0.1.0"
|
||||
@@ -2375,13 +2382,6 @@ dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-plist"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-platform"
|
||||
version = "0.1.0"
|
||||
@@ -2399,6 +2399,13 @@ dependencies = [
|
||||
"zbus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-plist"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-plotter"
|
||||
version = "0.1.0"
|
||||
|
||||
@@ -38,6 +38,7 @@ hayro-syntax = { version = "=0.7.2", default-features = false, features = ["std"
|
||||
bytemuck = "1"
|
||||
kurbo = "0.13"
|
||||
thiserror = "2"
|
||||
craft-gzip = { path = "crates/gzip" }
|
||||
craft-aaf = { path = "crates/aaf" }
|
||||
craft-atomic = { path = "crates/atomic" }
|
||||
craft-captions = { path = "crates/captions" }
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
[package]
|
||||
name = "craft-gzip"
|
||||
description = "Streaming gzip writer with stored and compressed segments"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
|
||||
@@ -0,0 +1,119 @@
|
||||
//! Streaming gzip output with optional stored segments.
|
||||
//!
|
||||
//! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors.
|
||||
//! Extracted from VectorCraft, licensed under MIT OR Apache-2.0.
|
||||
//!
|
||||
//! `flush` does not emit a deflate block; call [`Gz::finish`] to complete and flush the
|
||||
//! stream. After an I/O error, discard the writer: the output may be incomplete.
|
||||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||||
|
||||
use flate2::{Compress, Compression, Crc, FlushCompress, Status};
|
||||
use std::io::{self, Write};
|
||||
|
||||
/// Bytes of compressed output gathered before they go to the writer.
|
||||
const OUT_CHUNK: usize = 64 << 10;
|
||||
|
||||
/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the
|
||||
/// same header and the same data; the deflate blocks are the same too where flate2 deflates with
|
||||
/// miniz_oxide, not with zlib-rs, which writes them by how the data is fed), except that data which
|
||||
/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second
|
||||
/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level
|
||||
/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on,
|
||||
/// which makes one valid deflate stream that every reader unpacks.
|
||||
pub struct Gz<W: Write> {
|
||||
out: W,
|
||||
deflate: Compress,
|
||||
/// What `deflate` writes: stored blocks (packed data), else the default level.
|
||||
stored: bool,
|
||||
crc: Crc,
|
||||
buf: Vec<u8>,
|
||||
/// Bytes written (unpacked).
|
||||
total: u64,
|
||||
}
|
||||
|
||||
impl<W: Write> Gz<W> {
|
||||
/// Write a gzip header and construct a writer.
|
||||
pub fn new(mut out: W) -> io::Result<Self> {
|
||||
// flate2's header for the default level: no name, no time, unknown OS.
|
||||
out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?;
|
||||
Ok(Self {
|
||||
out,
|
||||
deflate: Compress::new(Compression::default(), false),
|
||||
stored: false,
|
||||
crc: Crc::new(),
|
||||
buf: Vec::with_capacity(OUT_CHUNK),
|
||||
total: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// Run the compressor over `input` with `flush` until it has taken it all (and, flushing,
|
||||
/// written everything out).
|
||||
fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> {
|
||||
loop {
|
||||
if self.buf.len() == self.buf.capacity() {
|
||||
self.out.write_all(&self.buf)?;
|
||||
self.buf.clear();
|
||||
}
|
||||
let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out());
|
||||
let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?;
|
||||
let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len());
|
||||
input = input.get(taken..).unwrap_or_default();
|
||||
let made = self.deflate.total_out() - before_out;
|
||||
let full = self.buf.len() == self.buf.capacity();
|
||||
match flush {
|
||||
FlushCompress::None if input.is_empty() => return Ok(()),
|
||||
FlushCompress::Finish if status == Status::StreamEnd => return Ok(()),
|
||||
// A flush is done once it left room in the output.
|
||||
FlushCompress::Sync if input.is_empty() && !full => return Ok(()),
|
||||
_ => {}
|
||||
}
|
||||
if taken == 0 && made == 0 && !full {
|
||||
return Err(io::Error::other("the compressor made no progress"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the number of uncompressed bytes supplied.
|
||||
pub fn written(&self) -> u64 {
|
||||
self.total
|
||||
}
|
||||
|
||||
/// What is written next is compressed already (stored as it is), or not (compressed at the
|
||||
/// default level).
|
||||
pub fn packed(&mut self, packed: bool) -> io::Result<()> {
|
||||
if packed == self.stored {
|
||||
return Ok(());
|
||||
}
|
||||
self.deflate(&[], FlushCompress::Sync)?;
|
||||
self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false);
|
||||
self.stored = packed;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Complete the gzip trailer and flush the underlying writer.
|
||||
pub fn finish(mut self) -> io::Result<()> {
|
||||
self.deflate(&[], FlushCompress::Finish)?;
|
||||
self.out.write_all(&self.buf)?;
|
||||
self.out.write_all(&self.crc.sum().to_le_bytes())?;
|
||||
self.out.write_all(&self.crc.amount().to_le_bytes())?;
|
||||
self.out.flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Write for Gz<W> {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
self.write_all(buf)?;
|
||||
Ok(buf.len())
|
||||
}
|
||||
|
||||
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
|
||||
self.total = self.total.checked_add(buf.len() as u64).ok_or_else(|| io::Error::other("uncompressed byte count overflow"))?;
|
||||
self.crc.update(buf);
|
||||
self.deflate(buf, FlushCompress::None)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
// Flushing would end a deflate block: the stream is flushed when it is finished.
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
use std::io::{self, Read, Write};
|
||||
|
||||
use craft_gzip::Gz;
|
||||
use flate2::{Compression, read::GzDecoder, write::GzEncoder};
|
||||
|
||||
fn decode(bytes: &[u8]) -> Vec<u8> {
|
||||
let mut result = Vec::new();
|
||||
GzDecoder::new(bytes).read_to_end(&mut result).unwrap();
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn matches_flate2_without_mode_changes() {
|
||||
let data = b"The quick brown fox jumps over the lazy dog. ".repeat(10000);
|
||||
let mut expected = GzEncoder::new(Vec::new(), Compression::default());
|
||||
let mut actual = Vec::new();
|
||||
{
|
||||
let mut gz = Gz::new(&mut actual).unwrap();
|
||||
for chunk in data.chunks(131) {
|
||||
expected.write_all(chunk).unwrap();
|
||||
gz.write_all(chunk).unwrap();
|
||||
}
|
||||
assert_eq!(gz.written(), data.len() as u64);
|
||||
gz.finish().unwrap();
|
||||
}
|
||||
assert_eq!(actual, expected.finish().unwrap());
|
||||
assert_eq!(decode(&actual), data);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_segments_round_trip_and_stored_bytes_remain_uncompressed() {
|
||||
let plain = b"repeat me ".repeat(20000);
|
||||
let packed: Vec<u8> = (0..200_000).map(|i| (i * 31) as u8).collect();
|
||||
let mut bytes = Vec::new();
|
||||
{
|
||||
let mut gz = Gz::new(&mut bytes).unwrap();
|
||||
gz.write_all(&plain).unwrap();
|
||||
gz.packed(true).unwrap();
|
||||
gz.packed(true).unwrap();
|
||||
gz.write_all(&packed).unwrap();
|
||||
gz.packed(false).unwrap();
|
||||
gz.write_all(&plain).unwrap();
|
||||
assert_eq!(gz.written(), (plain.len() * 2 + packed.len()) as u64);
|
||||
gz.finish().unwrap();
|
||||
}
|
||||
let expected = [plain.as_slice(), packed.as_slice(), plain.as_slice()].concat();
|
||||
assert_eq!(decode(&bytes), expected);
|
||||
assert!(bytes.windows(1024).any(|window| window == &packed[..1024]));
|
||||
assert!(bytes.len() < expected.len());
|
||||
}
|
||||
|
||||
struct FailAfter {
|
||||
remaining: usize,
|
||||
}
|
||||
impl Write for FailAfter {
|
||||
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
|
||||
if self.remaining == 0 {
|
||||
return Err(io::Error::other("injected failure"));
|
||||
}
|
||||
let n = self.remaining.min(data.len());
|
||||
self.remaining -= n;
|
||||
Ok(n)
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Err(io::Error::other("injected flush failure"))
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn propagates_header_body_and_flush_failures() {
|
||||
assert!(Gz::new(FailAfter { remaining: 0 }).is_err());
|
||||
let mut body = Gz::new(FailAfter { remaining: 10 }).unwrap();
|
||||
body.packed(true).unwrap();
|
||||
assert!(body.write_all(&vec![42; 200_000]).is_err());
|
||||
let empty = Gz::new(FailAfter { remaining: usize::MAX }).unwrap();
|
||||
assert_eq!(empty.finish().unwrap_err().to_string(), "injected flush failure");
|
||||
}
|
||||
Reference in New Issue
Block a user