Merge pull request #1659 from vangel76/perf/raw-dng-parallel-copy-1651

DNG files decode up to 2.4x faster on many cores (5x uncompressed)
This commit is contained in:
Dylan M. Taylor
2026-11-15 18:27:55 -05:00
committed by GitHub
3 changed files with 351 additions and 54 deletions
+154 -47
View File
@@ -2,7 +2,7 @@
#[cfg(feature = "dng")]
use crate::unpack::inflate;
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb};
use crate::unpack::{f16_to_f32, f24_to_f32, read_u16s, undo_diff_u16, undo_float_predictor, unpack_lsb, unpack_msb, unpack_words32_msb_with};
use crate::{Limits, RawData, RawError, Result, ljpeg};
use craft_tiff::image::{Chunk, ImageInfo, chunk_bytes};
use craft_tiff::{ByteOrder, tags::compression as comp};
@@ -18,8 +18,8 @@ pub enum Packing {
Lsb,
/// Each sample stored in 16 bits in file byte order, regardless of `bits`.
Word16,
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`]); rows start on a
/// word boundary.
/// MSB-first bit stream inside 32-bit words stored in file byte order ([`unpack_words32_msb`](crate::unpack::unpack_words32_msb));
/// rows start on a word boundary.
Words32Msb,
}
@@ -184,46 +184,72 @@ pub(crate) fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packin
.par_iter()
.map(|c| decode_chunk(data, info, c, order, packing, ccpp, bits, float, limits).map(|p| (*c, p)))
.collect();
for r in decoded {
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
let (c, px) = r?;
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h {
continue;
}
let copy_w = cw.min(w - x0);
let copy_h = ch.min(h - y0);
for y in 0..copy_h {
for x in 0..copy_w {
for s in 0..ccpp {
let src = (y * cw + x) * ccpp + s;
let dst_s = if planar { c.plane as usize } else { s };
if dst_s >= cpp {
continue;
}
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
match &px {
ChunkPx::U16(v) => {
if let (Some(o), Some(&s)) = (out_u16.get_mut(dst), v.get(src)) {
*o = s;
}
}
ChunkPx::F32(v) => {
if let (Some(o), Some(&s)) = (out_f32.get_mut(dst), v.get(src)) {
*o = s;
}
}
}
// a chunk that cannot be decoded is not a partial image: the file stays preview-only
let decoded = decoded.into_iter().collect::<Result<Vec<_>>>()?;
// The copy runs in parallel over bands of output rows: each band takes its rows of every chunk of the
// group that reaches it, in chunk order (so overlapping chunks resolve as a serial copy would).
let row_samples = w * cpp;
let threads = rayon::current_num_threads().max(1);
let band_rows = h.div_ceil(threads * 4).max(1);
if float {
out_f32.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
for (c, px) in &decoded {
if let ChunkPx::F32(v) = px {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
}
});
} else {
out_u16.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| {
for (c, px) in &decoded {
if let ChunkPx::U16(v) = px {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
});
}
decode_from = to;
}
Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) })
}
/// Copy the rows of chunk `c` (samples `src`, `c.width` per row, `ccpp` samples per pixel) that fall into the output
/// band `band`, which holds the image rows from `band_y0` on (`w` pixels of `cpp` samples each, `h` rows in the whole
/// image). Pixels past the image edge and samples `src` does not hold are skipped; a planar chunk (`planar`) writes
/// its one sample per pixel into the channel `c.plane`, a chunk of a plane the image does not have is skipped.
#[allow(clippy::too_many_arguments)]
fn place_chunk<T: Copy>(band: &mut [T], band_y0: usize, c: &Chunk, src: &[T], w: usize, h: usize, cpp: usize, planar: bool) {
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h || cpp == 0 {
return;
}
let plane = c.plane as usize;
if planar && plane >= cpp {
return;
}
let ccpp = if planar { 1 } else { cpp };
let copy_w = cw.min(w - x0);
let copy_h = ch.min(h - y0);
let band_rows = band.len() / (w * cpp);
let first = y0.max(band_y0);
let last = (y0 + copy_h).min(band_y0 + band_rows);
for y in first..last {
let s0 = (y - y0) * cw * ccpp;
let Some(avail) = src.get(s0..) else { break };
let d0 = ((y - band_y0) * w + x0) * cpp;
if planar {
let row = &mut band[d0..d0 + copy_w * cpp];
for (o, &v) in row[plane..].iter_mut().step_by(cpp).zip(&avail[..copy_w.min(avail.len())]) {
*o = v;
}
} else {
let n = (copy_w * cpp).min(avail.len());
band[d0..d0 + n].copy_from_slice(&avail[..n]);
}
}
}
/// Framing precedes allocation even in header-only probes: missing source rows may not
/// be reported as a decodable image and later silently filled with zero samples.
fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> {
@@ -512,13 +538,14 @@ fn unpack_chunk(
let row_bytes = row_n * bp;
let mut out = vec![0f32; row_n * ch];
let mut rowbuf = vec![0u8; row_bytes];
let mut scratch = Vec::new();
for y in 0..ch {
let s = src.get(y * row_bytes..).unwrap_or(&[]);
let len = s.len().min(row_bytes);
rowbuf[..len].copy_from_slice(&s[..len]);
rowbuf[len..].fill(0);
let big = if fp {
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor);
undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor, &mut scratch);
true
} else {
if factor > 0 {
@@ -526,16 +553,15 @@ fn unpack_chunk(
}
order == ByteOrder::Big
};
for (i, o) in out[y * row_n..(y + 1) * row_n].iter_mut().enumerate() {
let b = &rowbuf[i * bp..(i + 1) * bp];
*o = match (bp, big) {
(2, true) => f16_to_f32(u16::from_be_bytes([b[0], b[1]])),
(2, false) => f16_to_f32(u16::from_le_bytes([b[0], b[1]])),
(3, true) => f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]])),
(3, false) => f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0])),
(4, true) => f32::from_be_bytes([b[0], b[1], b[2], b[3]]),
_ => f32::from_le_bytes([b[0], b[1], b[2], b[3]]),
};
let dst = &mut out[y * row_n..(y + 1) * row_n];
let samples = rowbuf.chunks_exact(bp);
match (bp, big) {
(2, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_be_bytes([b[0], b[1]]))),
(2, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_le_bytes([b[0], b[1]]))),
(3, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]]))),
(3, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0]))),
(4, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_be_bytes([b[0], b[1], b[2], b[3]])),
_ => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_le_bytes([b[0], b[1], b[2], b[3]])),
}
}
return Ok(ChunkPx::F32(out));
@@ -544,6 +570,7 @@ fn unpack_chunk(
return Err(RawError::Unsupported("floating-point predictor on integer data".into()));
}
let mut out = vec![0u16; row_n * ch];
let mut words = Vec::new();
let (row_bytes, kind) = match (bits, packing) {
(8, _) => (row_n, 0),
(16, _) | (_, Packing::Word16) => (row_n * 2, 1),
@@ -559,7 +586,7 @@ fn unpack_chunk(
0 => row.iter_mut().zip(s).for_each(|(o, &b)| *o = b as u16),
1 => read_u16s(s, order, row),
2 => unpack_msb(s, bits, row),
4 => unpack_words32_msb(s, order, bits, row),
4 => unpack_words32_msb_with(s, order, bits, row, &mut words),
_ => unpack_lsb(s, bits, row),
}
if factor > 0 {
@@ -581,6 +608,86 @@ fn unpack_chunk(
Ok(ChunkPx::U16(out))
}
#[cfg(test)]
mod place_tests {
use super::*;
/// The serial per-sample copy `place_chunk` replaced: the oracle.
#[allow(clippy::too_many_arguments)]
fn reference(out: &mut [u16], c: &Chunk, v: &[u16], w: usize, h: usize, cpp: usize, planar: bool) {
let ccpp = if planar { 1 } else { cpp };
let (cw, ch) = (c.width as usize, c.height as usize);
let (x0, y0) = (c.x as usize, c.y as usize);
if x0 >= w || y0 >= h {
return;
}
for y in 0..ch.min(h - y0) {
for x in 0..cw.min(w - x0) {
for s in 0..ccpp {
let src = (y * cw + x) * ccpp + s;
let dst_s = if planar { c.plane as usize } else { s };
if dst_s >= cpp {
continue;
}
let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s;
if let (Some(o), Some(&s)) = (out.get_mut(dst), v.get(src)) {
*o = s;
}
}
}
}
}
fn chunk(x: u32, y: u32, width: u32, height: u32, plane: u16) -> Chunk {
Chunk { index: 0, x, y, width, height, plane, offset: 0, len: 0 }
}
/// Chunks that overlap (the later one wins), run past the image edge, hold fewer samples than their size, or name a
/// plane the image does not have, placed through bands of several heights, equal the serial copy.
#[test]
fn place_chunk_matches_the_serial_copy_for_odd_chunks() {
let (w, h) = (11, 9);
for cpp in [1usize, 3] {
for planar in [false, true] {
let ccpp = if planar { 1 } else { cpp };
let specs = [
(0, 0, 8, 6, 0, 1u16),
(4, 3, 8, 8, 0, 2), // overlaps the first and runs past both edges
(4, 3, 8, 8, 0, 3), // the same place again, later wins
(0, 5, 5, 4, 1, 4),
(2, 2, 6, 6, 2, 5),
(0, 0, 4, 4, 3, 6), // plane 3: not an image plane for cpp <= 3
(9, 0, 6, 12, 0, 7), // only two columns inside
(20, 0, 4, 4, 0, 8), // right of the image
(0, 0, 11, 9, 0, 9), // whole image, short source below
];
let chunks: Vec<(Chunk, Vec<u16>)> = specs
.iter()
.map(|&(x, y, cw, ch, plane, val)| {
let n = (cw * ch) as usize * ccpp;
// the last chunk holds only 40 samples
let n = if val == 9 { 40 } else { n };
(chunk(x, y, cw, ch, plane), (0..n).map(|i| val * 100 + i as u16).collect())
})
.collect();
let mut want = vec![0u16; w * h * cpp];
for (c, v) in &chunks {
reference(&mut want, c, v, w, h, cpp, planar);
}
for band_rows in [1usize, 2, 4, 9, 20] {
let mut got = vec![0u16; w * h * cpp];
for (bi, band) in got.chunks_mut(band_rows * w * cpp).enumerate() {
for (c, v) in &chunks {
place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar);
}
}
assert_eq!(got, want, "cpp {cpp} planar {planar} band rows {band_rows}");
}
}
}
}
}
#[cfg(test)]
#[cfg(feature = "dng")]
mod lossy_tests {
+31 -7
View File
@@ -46,12 +46,18 @@ pub fn unpack_lsb(src: &[u8], bits: u32, out: &mut [u16]) {
/// data, which is not the byte-wise MSB stream of [`unpack_msb`] unless `order` is big-endian. A short last word
/// is padded with zeros.
pub fn unpack_words32_msb(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16]) {
let words = src.chunks(4).flat_map(|c| {
unpack_words32_msb_with(src, order, bits, out, &mut Vec::new());
}
/// [`unpack_words32_msb`] with the byte-swapped copy of `src` kept in `scratch` (reused across rows).
pub fn unpack_words32_msb_with(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16], scratch: &mut Vec<u8>) {
scratch.clear();
scratch.extend(src.chunks(4).flat_map(|c| {
let mut w = [0u8; 4];
w[..c.len()].copy_from_slice(c);
order.u32(w).to_be_bytes()
});
unpack_msb(&words.collect::<Vec<u8>>(), bits, out);
}));
unpack_msb(scratch, bits, out);
}
pub fn read_u16s(src: &[u8], order: ByteOrder, out: &mut [u16]) {
@@ -126,10 +132,12 @@ pub fn undo_diff_u8(row: &mut [u8], stride: usize) {
/// Undo the floating-point predictor on one row: `row` holds `n` samples of `bytes_per` bytes, byte-differenced
/// with stride `stride` samples, then stored as byte planes (most significant first). Returns the samples as
/// big-endian byte groups in place.
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize) {
/// big-endian byte groups in place. `scratch` holds the byte planes (reused across rows).
pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize, scratch: &mut Vec<u8>) {
undo_diff_u8(row, stride);
let tmp = row.to_vec();
scratch.clear();
scratch.extend_from_slice(row);
let tmp = &scratch[..];
for i in 0..n {
for b in 0..bytes_per {
if let (Some(dst), Some(&s)) = (row.get_mut(i * bytes_per + b), tmp.get(b * n + i)) {
@@ -167,6 +175,22 @@ mod tests {
assert_eq!(o, [0x000, 0x012]); // the word 0x00001234, top bits first
}
#[test]
fn words32_scratch_is_reusable() {
let long: Vec<u8> = (0u8..24).map(|i| i.wrapping_mul(37).wrapping_add(5)).collect();
let short = &long[3..10];
let mut scratch = Vec::new();
for order in [ByteOrder::Little, ByteOrder::Big] {
for (src, n) in [(&long[..], 16usize), (short, 4)] {
let mut want = vec![0u16; n];
unpack_words32_msb(src, order, 12, &mut want);
let mut got = vec![0u16; n];
unpack_words32_msb_with(src, order, 12, &mut got, &mut scratch);
assert_eq!(got, want);
}
}
}
#[test]
fn packing() {
// 12-bit MSB: 0xABC, 0xDEF -> AB CD EF
@@ -210,7 +234,7 @@ mod tests {
for i in (1..row.len()).rev() {
row[i] = row[i].wrapping_sub(row[i - 1]);
}
undo_float_predictor(&mut row, 2, 4, 1);
undo_float_predictor(&mut row, 2, 4, 1, &mut Vec::new());
assert_eq!(f32::from_be_bytes([row[0], row[1], row[2], row[3]]), 1.0);
assert_eq!(f32::from_be_bytes([row[4], row[5], row[6], row[7]]), 2.0);
}
+166
View File
@@ -403,6 +403,172 @@ fn floating_point_dngs() {
assert_eq!(decode(&dng(raw, ByteOrder::Big)).unwrap().data, RawData::F32(vals));
}
/// The chunks (tiles of `tw` x `th`, or strips when `tw == w`) of an image of `cpp` interleaved samples per pixel, each
/// nominal-sized (strips: clipped to the image height) with zero padding past the image edge: one chunk per tile, and for `planar` one set of chunks per
/// plane (one sample per pixel), plane after plane.
fn chunk_samples<T: Copy + Default>(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec<Vec<T>> {
let planes = if planar { cpp } else { 1 };
let ccpp = if planar { 1 } else { cpp };
let mut out = Vec::new();
for plane in 0..planes {
for ty in 0..h.div_ceil(th) {
for tx in 0..w.div_ceil(tw) {
// a strip holds only the rows inside the image, a tile its nominal size
let rows = if tw == w { th.min(h - ty * th) } else { th };
let mut c = vec![T::default(); tw * rows * ccpp];
for y in 0..th.min(h - ty * th) {
for x in 0..tw.min(w - tx * tw) {
for s in 0..ccpp {
let src = ((ty * th + y) * w + tx * tw + x) * cpp + if planar { plane } else { s };
c[(y * tw + x) * ccpp + s] = px[src];
}
}
}
out.push(c);
}
}
}
out
}
fn layout_image(raw: &mut IfdBuilder, chunks: Vec<Vec<u8>>, w: usize, tile: (usize, usize)) {
raw.set_image(if tile.0 == w {
ImageData::Strips { rows_per_strip: tile.1 as u32, strips: chunks }
} else {
ImageData::Tiles { tile_width: tile.0 as u32, tile_height: tile.1 as u32, tiles: chunks }
});
}
/// Uncompressed integer DNGs in every chunk layout (chunky and planar, tiles and strips with edge chunks, several chunks
/// per image) at several bit depths and byte orders decode to the samples they were built from.
#[test]
fn uncompressed_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for cpp in [1usize, 3] {
for bits in [8u32, 12, 16] {
let px = pattern(w, h, cpp, bits);
// tiles overhanging both edges, strips with a shorter last strip
for tile in [(16, 16), (w, 5), (w, h), (16, 32)] {
for planar in [false, true] {
if planar && cpp == 1 {
continue;
}
for order in [ByteOrder::Little, ByteOrder::Big] {
let ccpp = if planar { 1 } else { cpp };
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
.iter()
.map(|c| match bits {
8 => c.iter().map(|&v| v as u8).collect(),
16 => u16_bytes(c, order),
_ => pack_msb(c, bits, tile.0 * ccpp),
})
.collect();
let mut raw = base_ifd(w, h, bits as u16, cpp, cpp == 1);
raw.set(t::COMPRESSION, Value::Short(vec![1]));
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, order)).unwrap();
assert_eq!(img.cpp, cpp);
assert_eq!(img.data, RawData::U16(px.clone()), "cpp {cpp} bits {bits} tile {tile:?} planar {planar} {order:?}");
}
}
}
}
}
}
/// Deflate with the integer and floating-point predictors, chunky and planar tiles and strips.
#[test]
fn deflate_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for planar in [false, true] {
let cpp = 3;
let ccpp = if planar { 1 } else { cpp };
let px = pattern(w, h, cpp, 16);
for tile in [(16, 16), (w, 5)] {
// integer predictor 2 on 16-bit samples
let chunks: Vec<Vec<u8>> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar)
.into_iter()
.map(|mut c| {
for row in c.chunks_mut(tile.0 * ccpp) {
for i in (ccpp..row.len()).rev() {
row[i] = row[i].wrapping_sub(row[i - ccpp]);
}
}
miniz_oxide::deflate::compress_to_vec_zlib(&u16_bytes(&c, ByteOrder::Little), 6)
})
.collect();
let mut raw = base_ifd(w, h, 16, cpp, false);
raw.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE]));
raw.set(t::PREDICTOR, Value::Short(vec![2]));
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
assert_eq!(img.data, RawData::U16(px.clone()), "integer predictor, tile {tile:?} planar {planar}");
}
}
}
/// Floating-point DNGs (uncompressed and with the floating-point predictor) in tiled and strip, chunky and planar
/// layouts with several chunks and edge chunks.
#[test]
fn float_chunk_layouts_place_every_sample() {
let (w, h) = (37, 23);
for cpp in [1usize, 3] {
let vals: Vec<f32> = (0..w * h * cpp).map(|i| ((i * 37) % 200) as f32 / 64.0).collect();
for tile in [(16, 16), (w, 5)] {
for planar in [false, true] {
if planar && cpp == 1 {
continue;
}
let ccpp = if planar { 1 } else { cpp };
let parts = chunk_samples(&vals, w, h, cpp, tile.0, tile.1, planar);
for bits in [16u16, 24, 32] {
let bp = bits as usize / 8;
for (pred, order) in [(1u16, ByteOrder::Little), (1, ByteOrder::Big), (3, ByteOrder::Little)] {
if pred == 1 && bits != 16 && bits != 32 {
continue; // uncompressed 24-bit floats are not written by this test
}
let chunks: Vec<Vec<u8>> = parts
.iter()
.map(|c| {
if pred == 3 {
let enc = fp_encode(c, bp, ccpp, tile.0 * ccpp);
miniz_oxide::deflate::compress_to_vec_zlib(&enc, 6)
} else if bits == 16 {
c.iter()
.flat_map(
|&v| if order == ByteOrder::Big { f32_to_f16(v).to_be_bytes() } else { f32_to_f16(v).to_le_bytes() },
)
.collect()
} else {
c.iter().flat_map(|&v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }).collect()
}
})
.collect();
let mut raw = base_ifd(w, h, bits, cpp, cpp == 1);
raw.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp]));
raw.set(t::COMPRESSION, Value::Short(vec![if pred == 3 { compression::DEFLATE } else { 1 }]));
if pred == 3 {
raw.set(t::PREDICTOR, Value::Short(vec![3]));
}
if planar {
raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2]));
}
layout_image(&mut raw, chunks, w, tile);
let img = decode(&dng(raw, order)).unwrap();
assert_eq!(img.data, RawData::F32(vals.clone()), "cpp {cpp} {bits}-bit pred {pred} tile {tile:?} planar {planar} {order:?}");
}
}
}
}
}
}
#[test]
fn non_finite_float_samples_decode_as_levels() {
// NaN samples in a float CFA DNG spread through any arithmetic downstream: the decoder hands on numbers only