From ab2bcccf845e35dc883b202d464f54c2c68bb54c Mon Sep 17 00:00:00 2001 From: RWoldberg <314159+rwoldberg@users.noreply.github.com> Date: Sat, 31 Oct 2026 17:22:59 -0600 Subject: [PATCH] crw: refuse an entropy stream cut short When the entropy stream ends before the frame does, the stream-end rule gives the remaining samples their predictor values. That was silent: a raw-data record cut short inside an intact CIFF heap decoded as a smeared image instead of an error. `decode` now counts those samples (`Image::stream_end_fill`) and returns `Error::Corrupt` when there are more than `Limits::max_stream_end_fill` (default 64, one block). `Limits` is non-exhaustive, so the field is additive. The frozen format is unchanged: a fill within the limit decodes exactly as before. Every real file measured fills none: 1,318 files of all seven verified bodies (1,308 EOS D60), whose last samples come from an end-of-block code. The CC0 sample test still decodes all ten files exactly. Co-Authored-By: Claude Opus 5.5 --- crates/crw/README.md | 11 ++++++++--- crates/crw/src/lib.rs | 29 +++++++++++++++++++++++------ crates/crw/src/tests.rs | 39 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 67 insertions(+), 12 deletions(-) diff --git a/crates/crw/README.md b/crates/crw/README.md index 4fa28b04..d05b6682 100644 --- a/crates/crw/README.md +++ b/crates/crw/README.md @@ -57,7 +57,7 @@ if craft_crw::probe(&bytes) { ### Output -`Image { header, samples, black, white }`: +`Image { header, samples, black, white, stream_end_fill }`: - `samples`: `width × height` `u16`s, row-major, the whole sensor including the masked borders, `bits` (10 or 12) deep, black not subtracted. Rows are in sensor order; apply @@ -79,6 +79,7 @@ if craft_crw::probe(&bytes) { offset. Convert with chrono (`DateTime::from_timestamp`), never by hand. - `header.make` / `header.model`: the MakeModel record, e.g. `Canon` / `Canon EOS D60`. CRW files carry no Exif block, so this is where the camera name comes from. +- `stream_end_fill`: how many samples were filled after the entropy stream ended (below). ### Errors and limits @@ -91,8 +92,12 @@ by depth (16) and by a per-file record budget (4,096), and `width × height` is buffers at its peak. `Limits` is `#[non_exhaustive]`: start from `Limits::default()` and set the fields you change. -A stream that ends early is not an error: the remaining samples keep their predictor values (the -frozen stream-end rule; real files end 1–3 samples short). +When the entropy stream ends before the frame does, the remaining samples keep their predictor +values (the frozen stream-end rule) and `Image::stream_end_fill` says how many. Up to +`Limits::max_stream_end_fill` (default 64, one block) that is accepted; more is `Corrupt`, a stream +cut short, which would otherwise decode as a smeared image. Every real file measured fills none: +1,318 files of all seven verified bodies (1,308 of them EOS D60), whose last samples come from an +end-of-block code. ### Performance diff --git a/crates/crw/src/lib.rs b/crates/crw/src/lib.rs index 345f40af..c7499268 100644 --- a/crates/crw/src/lib.rs +++ b/crates/crw/src/lib.rs @@ -63,11 +63,15 @@ pub struct Limits { /// `u16` buffers of this size at its peak. Default: 2²³ (8.4 million) on every target; real /// CRW frames hold at most 7.4 million samples. pub max_samples: usize, + /// Most samples [`decode`] may fill after the entropy stream ends (the stream-end rule: they + /// take their predictor values); more is [`Error::Corrupt`], a stream cut short. Default: 64, + /// one block; every real file measured (1,318 files, all seven verified bodies) fills none. + pub max_stream_end_fill: usize, } impl Default for Limits { fn default() -> Self { - Self { max_samples: 1 << 23 } + Self { max_samples: 1 << 23, max_stream_end_fill: 64 } } } @@ -179,6 +183,9 @@ pub struct Image { pub black: BlackLevel, /// White level: the saturation plateau, the sensor's clip level, or the full range. pub white: f32, + /// Samples filled after the entropy stream ended, at their predictor values (the stream-end + /// rule); at most [`Limits::max_stream_end_fill`]. Zero on every real file measured. + pub stream_end_fill: usize, } /// Result alias for this crate. @@ -368,13 +375,18 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result { let has_low = header_after_plane || (plane_fits && head.windows(2).any(|p| p[0] == 0xff && p[1] != 0x00)); let bits = if has_low { 12 } else { 10 }; let low_len = if has_low { q } else { 0 }; + let mut stream_end_fill = 0; let samples = if full { let entropy = payload.get(low_len.saturating_add(ent_off)..).ok_or_else(|| corrupt("payload shorter than its header"))?; let low = has_low.then(|| payload.get(..q)).flatten(); if has_low && low.is_none() { return Err(corrupt("payload shorter than its low-bits plane")); } - let hi = decode_entropy(entropy, w, h, &tables)?; + let (hi, tail) = decode_entropy(entropy, w, h, &tables)?; + if tail > limits.max_stream_end_fill { + return Err(Error::Corrupt(format!("entropy stream ends {tail} samples before the frame does"))); + } + stream_end_fill = tail; let mut samples = vec![0u16; n]; // `samples` is documented as `bits` deep, so a damaged stream whose chain value runs past // the range is clamped here: a consumer indexing a `1 << bits` LUT must not see more. @@ -429,7 +441,7 @@ fn parse(bytes: &[u8], limits: &Limits, full: bool) -> Result { exif_orientation, wb_multipliers: wb_from_color_balance(bytes, &rec, o), }; - Ok(Image { header, samples, black, white }) + Ok(Image { header, samples, black, white, stream_end_fill }) } /// As-shot white-balance multipliers from the ColorBalance record (0x10a9, D60/10D/300D): a @@ -645,8 +657,9 @@ fn build_lut(codes: &[(u16, u8, u8, u8)]) -> Vec { /// model (`clean-room/02-crw-format.md` §2): 64-sample blocks, first/main code tables, /// (run, size) leaves with JPEG-rule magnitudes, EOB, block-DC carry, and two parity DPCM chains /// initialised to 512 and reset at every row start. A stream that ends early leaves the remaining -/// samples at their chain values (the frozen stream-end rule). -fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result> { +/// samples at their chain values (the frozen stream-end rule); the second value returned is how +/// many, so [`decode`] can refuse a stream cut short ([`Limits::max_stream_end_fill`]). +fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables) -> Result<(Vec, usize)> { let d = unstuff(entropy); let nbits = d.len().saturating_mul(8); let peek16 = |pos: usize| -> u16 { @@ -661,12 +674,15 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables let mut out = Chains { dec: vec![0u16; n], w, qi: 0, ce: 512, co: 512 }; let mut carry: i32 = 0; let mut pos = 0usize; + // samples filled after the stream ended (the stream-end rule) + let mut tail = 0usize; while out.qi < n { let block_end = n.min((out.qi / 64 + 1) * 64); let mut tok = 0u32; while out.qi < block_end { // stream exhausted: every remaining sample keeps its chain value if pos >= nbits.saturating_sub(1) { + tail = n - out.qi; out.fill(n); break; } @@ -676,6 +692,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables } let (len, mut rr, s) = ((e >> 16) as usize, ((e >> 8) & 0xff) as usize, e & 0xff); if pos.saturating_add(len).saturating_add(s as usize) > nbits { + tail = n - out.qi; out.fill(n); break; } @@ -713,7 +730,7 @@ fn decode_entropy(entropy: &[u8], w: usize, h: usize, tables: &tables::SelTables } } } - Ok(out.dec) + Ok((out.dec, tail)) } /// Drop the stuffing byte: an 0x00 after an 0xff. diff --git a/crates/crw/src/tests.rs b/crates/crw/src/tests.rs index a858cef9..c67582bb 100644 --- a/crates/crw/src/tests.rs +++ b/crates/crw/src/tests.rs @@ -266,7 +266,7 @@ fn header_reads_geometry_without_samples() { fn limits_cap_the_sample_count() { let hi = test_hi(128); let f = roundtrip_file(16, 8, "Canon EOS D60", None, &hi, Order::Le); - let mut limits = Limits { max_samples: 127 }; + let mut limits = Limits { max_samples: 127, ..Limits::default() }; assert_eq!(decode(&f, &limits), Err(Error::LimitExceeded("image too large"))); assert!(header(&f, &limits).is_err()); limits.max_samples = 128; @@ -365,7 +365,7 @@ fn black_level_ignores_clamped_rows() { } /// The frozen stream-end rule: a truncated entropy stream leaves the remaining samples at -/// their chain values instead of failing the file. +/// their chain values instead of failing the file, when the limit allows that many. #[test] fn truncated_stream_keeps_chain_values() { let (w, h) = (16u16, 8u16); @@ -378,9 +378,42 @@ fn truncated_stream_keeps_chain_values() { Order::Le, ); let f = ciff(&[(RAW_DATA, payload(None, &entropy[..cut])), (0x300a, props)], Order::Le); - let img = full(&f).expect("truncated stream still decodes"); + let lenient = Limits { max_stream_end_fill: 128, ..Limits::default() }; + let img = decode(&f, &lenient).expect("truncated stream still decodes"); assert_eq!(img.samples.len(), 128); assert_eq!(&img.samples[..32], &hi[..32], "the leading samples decode exactly"); + assert!(img.stream_end_fill > 0, "the samples after the cut are filled"); +} + +/// A stream cut short decoded as a smeared image: the samples after its end take their chain +/// values. Past one block that is `Corrupt` by default; a complete stream fills none, and a fill +/// within the limit is accepted. +#[test] +fn a_stream_cut_short_is_corrupt() { + let (w, h) = (64u16, 32u16); + let hi = test_hi(2048); + let tables = tables::tables(0).expect("selector 0"); + let entropy = encode_hi(&hi, 64, &tables); + let file = |entropy: &[u8]| { + let props = heap( + &[ + (SENSOR_INFO, sensor_info(w, h, None, Order::Le)), + (DECODER_TABLE, decoder_table(0, Order::Le)), + (MAKE_MODEL, make_model("Canon EOS D60")), + ], + Order::Le, + ); + ciff(&[(RAW_DATA, payload(None, entropy)), (0x300a, props)], Order::Le) + }; + let whole = full(&file(&entropy)).expect("a complete stream decodes"); + assert_eq!(whole.stream_end_fill, 0); + let cut = file(&entropy[..entropy.len() / 2]); + let Err(Error::Corrupt(why)) = full(&cut) else { panic!("a stream cut in half must be refused") }; + assert!(why.contains("samples before the frame does"), "{why}"); + let filled = decode(&cut, &Limits { max_stream_end_fill: usize::MAX, ..Limits::default() }).expect("lenient").stream_end_fill; + assert!(filled > Limits::default().max_stream_end_fill, "{filled}"); + let at_limit = Limits { max_stream_end_fill: filled, ..Limits::default() }; + assert_eq!(decode(&cut, &at_limit).map(|i| i.stream_end_fill), Ok(filled), "a fill at the limit is accepted"); } /// Every record the decoder needs missing, bogus selectors, dimensions and offsets: errors,