Merge pull request #1461 from storytold/fix/png-bounded-row-stream

image-codecs: bound PNG row streaming without a full-frame allocation
This commit is contained in:
Dylan M. Taylor
2026-11-15 11:01:29 -05:00
committed by GitHub
2 changed files with 152 additions and 7 deletions
+58 -7
View File
@@ -141,12 +141,12 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
/// Reads the header and refuses what [`decode`] refuses before any pixel is decoded: sizes
/// past `limits`, and a file too small for the image it declares.
pub fn new(bytes: B, limits: &Limits) -> Result<Self, CodecError> {
Self::open(bytes, limits, true)
Self::open(bytes, limits, true, false)
}
/// [`PngDecodeRows::new`], optionally without reserving the whole image: a caller that hands
/// the rows on as they come ([`decode_rows`]) never holds them all.
fn open(bytes: B, limits: &Limits, reserve: bool) -> Result<Self, CodecError> {
fn open(bytes: B, limits: &Limits, reserve: bool, bounded_stream: bool) -> Result<Self, CodecError> {
let at = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0));
let bytes = SharedInput(std::sync::Arc::new(bytes));
let input = Tracked { cursor: Cursor::new(bytes.clone()), at: at.clone() };
@@ -187,7 +187,14 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
let (width, height) = info.frame_control.map_or_else(|| info.size(), |f| (f.width, f.height));
let line = reader.output_line_size(width).ok_or_else(|| err("image too large"))?;
// The output layout is known: charge its real bytes per pixel, not RGBA16's worst case.
limits.check(width, height, layout, sample)?;
if bounded_stream {
limits.check_size(width, height)?;
if line > limits.alloc_usize() {
return Err(CodecError::LimitExceeded("one PNG row exceeds max_alloc".into()));
}
} else {
limits.check(width, height, layout, sample)?;
}
let height = usize::try_from(height).map_err(|_| err("image too large"))?;
if width == 0 || height == 0 {
return Err(err("the image is empty"));
@@ -234,11 +241,35 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
/// decode the whole image after all (an interlaced file) calls
/// [`PngDecodeRows::reserve_output`] before [`PngDecodeRows::step`].
pub fn new_streaming(bytes: B, limits: &Limits) -> Result<Self, CodecError> {
Self::open(bytes, limits, false)
Self::open(bytes, limits, false, false)
}
/// Opens a bounded row stream without charging a full image that is never retained.
/// Dimension, pixel-count, plausibility, decoder and metadata limits remain unchanged;
/// `max_alloc` also bounds each row group. It is not a total process-memory limit.
/// Unlike [`Self::new_streaming`], a streamable image may exceed the full-image budget.
/// Non-streamable images still require that budget, as do [`Self::step`] and
/// [`Self::reserve_output`]. The caller must bound any output it retains in its sink.
pub fn new_bounded_streaming(bytes: B, limits: &Limits) -> Result<Self, CodecError> {
let decoder = Self::open(bytes, limits, false, true)?;
if !decoder.streamable() {
decoder.check_full_output()?;
}
Ok(decoder)
}
fn check_full_output(&self) -> Result<(), CodecError> {
let height = u32::try_from(self.height).map_err(|_| err("image too large"))?;
self.limits.check(self.width, height, self.layout, self.sample)?;
if self.size > self.limits.alloc_usize() {
return Err(CodecError::LimitExceeded("PNG output exceeds max_alloc".into()));
}
Ok(())
}
/// Reserves the buffer [`PngDecodeRows::step`] decodes the whole image into.
pub fn reserve_output(&mut self) -> Result<(), CodecError> {
self.check_full_output()?;
let size = self.size;
self.buf.try_reserve_exact(size).map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {size} bytes for the image")))
}
@@ -271,6 +302,7 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
/// Decodes at least one row (the whole image if it is interlaced), then goes on while `more()`
/// returns true. `Some` once the image is complete, after which the decoder is spent.
pub fn step(&mut self, more: &mut dyn FnMut() -> bool) -> Result<Option<Image>, CodecError> {
self.check_full_output()?;
if self.done {
return Err(err("the image was already decoded"));
}
@@ -338,7 +370,9 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
/// `more()` returns true; `Some` with the extras once the file is read to its end, with the
/// resolution a format that lacks one takes from its EXIF or XMP. A damaged file fails with
/// the error [`decode`] gives, after however many rows were delivered. An orientation (an
/// `eXIf` after the image data may set one) is for the caller to apply.
/// `eXIf` after the image data may set one) is for the caller to apply. A new group size
/// must exceed any partial group retained from an earlier yield; otherwise this returns
/// an error without consuming another row.
pub fn step_stream<E: From<CodecError>>(
&mut self,
group: u32,
@@ -348,12 +382,29 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
if self.done {
return Err(err("the image was already decoded").into());
}
let group = usize::try_from(group.max(1)).unwrap_or(usize::MAX);
// Tail EXIF may report an orientation after rows have already been delivered;
// that belongs in the returned extras, not a rejection of metadata continuation.
if self.rows == 0 && !self.tail && !self.streamable() {
return Err(err("this PNG requires whole-image decoding").into());
}
let group = usize::try_from(group.max(1)).unwrap_or(usize::MAX).min(self.height);
if self.rows - self.group_start >= group {
return Err(err("PNG row group is smaller than rows held from the previous step").into());
}
let group_bytes = group.checked_mul(self.line).ok_or_else(|| err("PNG row group is too large"))?;
if group_bytes > self.limits.alloc_usize() {
return Err(CodecError::LimitExceeded("PNG row group exceeds max_alloc".into()).into());
}
if !self.tail {
while self.rows < self.height {
let held = self.rows - self.group_start;
let at = held.checked_mul(self.line).ok_or_else(|| err("image too large"))?;
let end = at.checked_add(self.line).ok_or_else(|| err("image too large"))?;
// A caller may change its group size between yields. Bound the actual held
// bytes as well as the requested group before growing the row buffer.
if end > self.limits.alloc_usize() {
return Err(CodecError::LimitExceeded("PNG row group exceeds max_alloc".into()).into());
}
if self.buf.len() < end {
self.buf
.try_reserve_exact(end - self.buf.len())
@@ -368,7 +419,7 @@ impl<B: AsRef<[u8]>> PngDecodeRows<B> {
to_native_endian(row);
}
self.rows += 1;
if self.rows - self.group_start == group || self.rows == self.height {
if self.rows - self.group_start >= group || self.rows == self.height {
let data = self.buf.get(..end).ok_or_else(|| err("invalid PNG row group"))?;
sink(u32::try_from(self.group_start).map_err(|_| err("image too large"))?, data)?;
self.group_start = self.rows;
+94
View File
@@ -665,3 +665,97 @@ fn step_stream_gives_the_pixels_and_extras_of_decode() {
fn dec_line(w: u32, layout: ChannelLayout, sample: SampleType) -> usize {
w as usize * layout.channels() * sample.bytes()
}
#[test]
fn bounded_png_stream_does_not_charge_unretained_full_frame() {
let image = synth(256, 1024, ChannelLayout::Rgba, SampleType::U16, 77, 0.5);
let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap();
let limits = Limits { max_alloc: 512 * 1024, ..Limits::default() };
assert!(image.data().len() > limits.max_alloc as usize);
assert!(PngDecodeRows::new(bytes.as_slice(), &limits).is_err());
assert!(PngDecodeRows::new_streaming(bytes.as_slice(), &limits).is_err());
let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &limits).unwrap();
assert!(decoder.streamable());
// Opening a stream must not become a way to bypass whole-image allocation guards.
assert!(decoder.reserve_output().is_err());
assert!(decoder.step(&mut || true).is_err());
let mut next = 0;
let mut sink = |first: u32, rows: &[u8]| -> Result<(), CodecError> {
assert_eq!(first as usize, next);
assert!(rows.len() <= 3 * 256 * 8);
let start = next * 256 * 8;
assert_eq!(rows, &image.data()[start..start + rows.len()]);
next += rows.len() / (256 * 8);
Ok(())
};
while decoder.step_stream(3, &mut || false, &mut sink).unwrap().is_none() {}
assert_eq!(next, 1024);
}
#[test]
fn bounded_png_stream_keeps_group_dimension_and_fallback_limits() {
let image = synth(256, 1024, ChannelLayout::Rgba, SampleType::U16, 81, 0.5);
let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap();
let limits = Limits { max_alloc: 512 * 1024, ..Limits::default() };
let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &limits).unwrap();
assert!(decoder.step_stream::<CodecError>(1024, &mut || true, &mut |_, _| panic!("oversized group reached sink")).is_err());
for strict in [
Limits { max_width: 255, ..limits },
Limits { max_height: 1023, ..limits },
Limits { max_pixels: 256 * 1024 - 1, ..limits },
Limits { max_alloc: 1024, ..limits },
] {
assert!(PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &strict).is_err());
}
let adam7 = interlaced(&image);
assert!(PngDecodeRows::new_bounded_streaming(adam7.as_slice(), &limits).is_err());
}
#[test]
fn bounded_png_stream_rejects_shrinking_below_buffered_rows() {
let image = synth(16, 8, ChannelLayout::Rgba, SampleType::U16, 83, 0.5);
let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap();
let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &Limits::default()).unwrap();
for _ in 0..3 {
assert!(decoder.step_stream::<CodecError>(4, &mut || false, &mut |_, _| panic!("group is incomplete")).unwrap().is_none());
}
assert!(decoder.step_stream::<CodecError>(1, &mut || true, &mut |_, _| panic!("changed group reached sink")).is_err());
let mut output = Vec::new();
decoder
.step_stream::<CodecError>(4, &mut || true, &mut |_, rows| {
output.extend_from_slice(rows);
Ok(())
})
.unwrap();
assert_eq!(output, image.data());
}
#[test]
fn bounded_png_stream_preserves_late_exif_across_metadata_yields() {
let image = synth(16, 8, ChannelLayout::Rgba, SampleType::U16, 89, 0.5);
let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap();
let mut exif = sample_exif();
exif[18..22].copy_from_slice(&6u32.to_le_bytes());
let mut chunks = vec![png_chunk(b"eXIf", &exif)];
for i in 0..150 {
chunks.push(png_chunk(b"tEXt", format!("key{i}\0value").as_bytes()));
}
let bytes = with_chunks_before_iend(&bytes, &chunks);
let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &Limits::default()).unwrap();
assert!(decoder.streamable());
let mut output = Vec::new();
let extras = loop {
if let Some(extras) = decoder
.step_stream::<CodecError>(2, &mut || false, &mut |_, rows| {
output.extend_from_slice(rows);
Ok(())
})
.unwrap()
{
break extras;
}
};
assert_eq!(output, image.data());
assert_eq!(extras.orientation, 6);
assert_eq!(extras.meta.text.len(), 150);
}