Share streaming gzip writer with stored segments

This commit is contained in:
Dylan M. Taylor
2026-11-15 14:41:14 +00:00
parent e2e3b4213a
commit 424a2dc2be
5 changed files with 225 additions and 7 deletions
Generated
+14 -7
View File
@@ -1843,6 +1843,13 @@ dependencies = [
"image",
]
[[package]]
name = "craft-gzip"
version = "0.1.0"
dependencies = [
"flate2",
]
[[package]]
name = "craft-hdr"
version = "0.1.0"
@@ -2375,13 +2382,6 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-plist"
version = "0.1.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-platform"
version = "0.1.0"
@@ -2399,6 +2399,13 @@ dependencies = [
"zbus",
]
[[package]]
name = "craft-plist"
version = "0.1.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-plotter"
version = "0.1.0"
+1
View File
@@ -38,6 +38,7 @@ hayro-syntax = { version = "=0.7.2", default-features = false, features = ["std"
bytemuck = "1"
kurbo = "0.13"
thiserror = "2"
craft-gzip = { path = "crates/gzip" }
craft-aaf = { path = "crates/aaf" }
craft-atomic = { path = "crates/atomic" }
craft-captions = { path = "crates/captions" }
+14
View File
@@ -0,0 +1,14 @@
[package]
name = "craft-gzip"
description = "Streaming gzip writer with stored and compressed segments"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
+119
View File
@@ -0,0 +1,119 @@
//! Streaming gzip output with optional stored segments.
//!
//! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors.
//! Extracted from VectorCraft, licensed under MIT OR Apache-2.0.
//!
//! `flush` does not emit a deflate block; call [`Gz::finish`] to complete and flush the
//! stream. After an I/O error, discard the writer: the output may be incomplete.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
use flate2::{Compress, Compression, Crc, FlushCompress, Status};
use std::io::{self, Write};
/// Bytes of compressed output gathered before they go to the writer.
const OUT_CHUNK: usize = 64 << 10;
/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the
/// same header and the same data; the deflate blocks are the same too where flate2 deflates with
/// miniz_oxide, not with zlib-rs, which writes them by how the data is fed), except that data which
/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second
/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level
/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on,
/// which makes one valid deflate stream that every reader unpacks.
pub struct Gz<W: Write> {
out: W,
deflate: Compress,
/// What `deflate` writes: stored blocks (packed data), else the default level.
stored: bool,
crc: Crc,
buf: Vec<u8>,
/// Bytes written (unpacked).
total: u64,
}
impl<W: Write> Gz<W> {
/// Write a gzip header and construct a writer.
pub fn new(mut out: W) -> io::Result<Self> {
// flate2's header for the default level: no name, no time, unknown OS.
out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?;
Ok(Self {
out,
deflate: Compress::new(Compression::default(), false),
stored: false,
crc: Crc::new(),
buf: Vec::with_capacity(OUT_CHUNK),
total: 0,
})
}
/// Run the compressor over `input` with `flush` until it has taken it all (and, flushing,
/// written everything out).
fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> {
loop {
if self.buf.len() == self.buf.capacity() {
self.out.write_all(&self.buf)?;
self.buf.clear();
}
let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out());
let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?;
let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len());
input = input.get(taken..).unwrap_or_default();
let made = self.deflate.total_out() - before_out;
let full = self.buf.len() == self.buf.capacity();
match flush {
FlushCompress::None if input.is_empty() => return Ok(()),
FlushCompress::Finish if status == Status::StreamEnd => return Ok(()),
// A flush is done once it left room in the output.
FlushCompress::Sync if input.is_empty() && !full => return Ok(()),
_ => {}
}
if taken == 0 && made == 0 && !full {
return Err(io::Error::other("the compressor made no progress"));
}
}
}
/// Return the number of uncompressed bytes supplied.
pub fn written(&self) -> u64 {
self.total
}
/// What is written next is compressed already (stored as it is), or not (compressed at the
/// default level).
pub fn packed(&mut self, packed: bool) -> io::Result<()> {
if packed == self.stored {
return Ok(());
}
self.deflate(&[], FlushCompress::Sync)?;
self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false);
self.stored = packed;
Ok(())
}
/// Complete the gzip trailer and flush the underlying writer.
pub fn finish(mut self) -> io::Result<()> {
self.deflate(&[], FlushCompress::Finish)?;
self.out.write_all(&self.buf)?;
self.out.write_all(&self.crc.sum().to_le_bytes())?;
self.out.write_all(&self.crc.amount().to_le_bytes())?;
self.out.flush()
}
}
impl<W: Write> Write for Gz<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.write_all(buf)?;
Ok(buf.len())
}
fn write_all(&mut self, buf: &[u8]) -> io::Result<()> {
self.total = self.total.checked_add(buf.len() as u64).ok_or_else(|| io::Error::other("uncompressed byte count overflow"))?;
self.crc.update(buf);
self.deflate(buf, FlushCompress::None)
}
fn flush(&mut self) -> io::Result<()> {
// Flushing would end a deflate block: the stream is flushed when it is finished.
Ok(())
}
}
+77
View File
@@ -0,0 +1,77 @@
use std::io::{self, Read, Write};
use craft_gzip::Gz;
use flate2::{Compression, read::GzDecoder, write::GzEncoder};
fn decode(bytes: &[u8]) -> Vec<u8> {
let mut result = Vec::new();
GzDecoder::new(bytes).read_to_end(&mut result).unwrap();
result
}
#[test]
fn matches_flate2_without_mode_changes() {
let data = b"The quick brown fox jumps over the lazy dog. ".repeat(10000);
let mut expected = GzEncoder::new(Vec::new(), Compression::default());
let mut actual = Vec::new();
{
let mut gz = Gz::new(&mut actual).unwrap();
for chunk in data.chunks(131) {
expected.write_all(chunk).unwrap();
gz.write_all(chunk).unwrap();
}
assert_eq!(gz.written(), data.len() as u64);
gz.finish().unwrap();
}
assert_eq!(actual, expected.finish().unwrap());
assert_eq!(decode(&actual), data);
}
#[test]
fn mixed_segments_round_trip_and_stored_bytes_remain_uncompressed() {
let plain = b"repeat me ".repeat(20000);
let packed: Vec<u8> = (0..200_000).map(|i| (i * 31) as u8).collect();
let mut bytes = Vec::new();
{
let mut gz = Gz::new(&mut bytes).unwrap();
gz.write_all(&plain).unwrap();
gz.packed(true).unwrap();
gz.packed(true).unwrap();
gz.write_all(&packed).unwrap();
gz.packed(false).unwrap();
gz.write_all(&plain).unwrap();
assert_eq!(gz.written(), (plain.len() * 2 + packed.len()) as u64);
gz.finish().unwrap();
}
let expected = [plain.as_slice(), packed.as_slice(), plain.as_slice()].concat();
assert_eq!(decode(&bytes), expected);
assert!(bytes.windows(1024).any(|window| window == &packed[..1024]));
assert!(bytes.len() < expected.len());
}
struct FailAfter {
remaining: usize,
}
impl Write for FailAfter {
fn write(&mut self, data: &[u8]) -> io::Result<usize> {
if self.remaining == 0 {
return Err(io::Error::other("injected failure"));
}
let n = self.remaining.min(data.len());
self.remaining -= n;
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
Err(io::Error::other("injected flush failure"))
}
}
#[test]
fn propagates_header_body_and_flush_failures() {
assert!(Gz::new(FailAfter { remaining: 0 }).is_err());
let mut body = Gz::new(FailAfter { remaining: 10 }).unwrap();
body.packed(true).unwrap();
assert!(body.write_all(&vec![42; 200_000]).is_err());
let empty = Gz::new(FailAfter { remaining: usize::MAX }).unwrap();
assert_eq!(empty.finish().unwrap_err().to_string(), "injected flush failure");
}