raw: leave LinearizationTable unapplied when OpcodeList1 is present

DNG chapter 7 runs OpcodeList1 on the stored codes, then the
LinearizationTable maps them to sensor values. decode was applying the
table first, so a develop pipeline that later applied list 1 saw already
linearized samples.

When list 1 is present the table is now carried on RawImage and left
unapplied (`linearized` stays false). apply_linearization maps it after
the list. Files without list 1 are unchanged.

The dng-only build already compiles on main (d16ee792 gated
cr3::orientation). A comment at that gate records the E0433 failure
mode.

Part of #778.
This commit is contained in:
FredApps
2026-11-15 21:26:02 +01:00
parent 7d39dc4df2
commit a99c8f8e36
22 changed files with 158 additions and 21 deletions
+1 -1
View File
@@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins.
|---|---|
| Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area |
| Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) |
| Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) |
| Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour |
| Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves |
+1
View File
@@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
// makes a neutral surface magenta. WB edits remain relative to this as-shot RGB.
wb_multipliers: if linear_rgb { Some([1.0; 3]) } else { wb },
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
// only validated components become opcodes (see `arw_lens`, and `arw_lens_rm6` for the ILCE-7RM6)
opcodes: OpcodeLists { list3: warp.into_iter().chain(rm6).collect(), ..Default::default() },
+1
View File
@@ -311,6 +311,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -388,6 +388,7 @@ pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode, limits: &
// the camera's white balance is already applied, as in Sony's linear YCbCr
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+2
View File
@@ -269,6 +269,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: levels.wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: ifd0.map(Camera::from_ifd0).unwrap_or_default(),
@@ -824,6 +825,7 @@ mod tests {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
+1
View File
@@ -59,6 +59,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, caller: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: h.wb_multipliers,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
+30 -14
View File
@@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?;
}
// linearization table (integer data only)
let mut linearized = false;
// linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored
// codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`].
let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data);
// the largest value the table can give a sample: the white level when the file states none
let mut table_max = None;
if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) {
let table: Vec<u16> = table.iter().map(|&x| x.min(65535) as u16).collect();
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
table_max = table.iter().copied().max().filter(|&m| m > 0);
}
let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0));
// a reduced copy's own levels, where it states them
let levels = |tag| if pixels.contains(tag) { pixels } else { raw };
@@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
None
};
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let color = color_data(ifd0, raw, tiff.order);
let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2))
&& color.profile.gain_table_map.is_none();
@@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
color,
wb_multipliers: None,
linearized,
linearization_table,
pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()),
opcodes,
camera: Camera::from_ifd0(ifd0),
@@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f
Ok(img)
}
/// Parse opcode lists and, for integer data, the LinearizationTable.
///
/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7
/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the
/// table unapplied (`linearized` is false) and available on the image.
fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option<Vec<u16>>, crate::OpcodeLists) {
let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default();
let opcodes =
crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) };
let linearization_table = if matches!(data, RawData::U16(_)) {
pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::<Vec<u16>>())
} else {
None
};
let mut linearized = false;
if opcodes.list1.is_empty()
&& let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref())
{
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
linearized = true;
}
(linearized, linearization_table, opcodes)
}
/// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043).
const TIME_CODES: u16 = 51043;
/// CinemaDNG `FrameRate` (tag 51044).
+9
View File
@@ -159,6 +159,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result<Vec<u8>> {
} else {
ifd.set(t::WHITE_LEVEL, Value::Long(white));
}
// A table still to be applied (OpcodeList1 has to run on the stored codes first) is written
// with the samples. Once the samples have been mapped, writing the tag would apply it twice
// on the next read of a file without list 1.
if !raw.linearized
&& let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty())
{
ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec()));
}
let a = raw.active_area;
ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32]));
ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32]));
@@ -456,6 +464,7 @@ mod tests {
color: crate::ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+1
View File
@@ -115,6 +115,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+32 -5
View File
@@ -10,11 +10,13 @@
//!
//! # Where this crate stops
//!
//! The samples come out as the file stores them (after a DNG `LinearizationTable`): black is not subtracted,
//! nothing is scaled, cropped, white balanced or demosaiced, opcode lists and profile tables are parsed but not
//! applied, and no colour transform is computed from the matrices. Normalisation, demosaicing, highlight
//! handling, the DNG colour model's arithmetic, lens corrections, tone and everything after belong to the
//! develop pipeline, which reads the public fields of the types here.
//! The samples come out as the file stores them (after a DNG `LinearizationTable`, except when the file
//! also has an `OpcodeList1`: that list runs on the stored codes first, and applying it is the develop
//! pipeline's work, so the table is left unapplied and carried on [`RawImage::linearization_table`]).
//! Black is not subtracted, nothing is scaled, cropped, white balanced or demosaiced, opcode lists and
//! profile tables are parsed but not applied, and no colour transform is computed from the matrices.
//! Normalisation, demosaicing, highlight handling, the DNG colour model's arithmetic, lens corrections,
//! tone and everything after belong to the develop pipeline, which reads the public fields of the types here.
//!
//! # Formats
//!
@@ -645,6 +647,8 @@ pub fn orientation(bytes: &[u8]) -> Option<u16> {
}
// A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview
// fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start.
// The `cr3` module exists only with the `cr3` feature: this gate is required so
// `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise).
#[cfg(feature = "cr3")]
if probe(bytes) == Some(RawFormat::Cr3) {
return cr3::orientation(bytes);
@@ -1041,6 +1045,12 @@ pub struct RawImage {
pub wb_multipliers: Option<[f32; 3]>,
/// Whether a `LinearizationTable` was applied to `data`.
pub linearized: bool,
/// The file's `LinearizationTable`, when present. `data` has already been mapped through it when
/// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left
/// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call
/// [`Self::apply_linearization`] after that list.
#[serde(default)]
pub linearization_table: Option<Vec<u16>>,
/// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is
/// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by
/// this factor.
@@ -1119,6 +1129,23 @@ impl RawImage {
self.check(true)
}
/// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied.
/// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not
/// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7).
pub fn apply_linearization(&mut self) {
if self.linearized {
return;
}
let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else {
return;
};
if let RawData::U16(v) = &mut self.data {
let last = table.len() - 1;
v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]);
self.linearized = true;
}
}
fn check(&self, samples: bool) -> Result<()> {
if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) {
return Err(RawError::Corrupt("bad dimensions".into()));
+1
View File
@@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+1
View File
@@ -165,6 +165,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: tiff.as_ref().and_then(|t| t.ifds.first()).map(Camera::from_ifd0).unwrap_or_default(),
cinema: Default::default(),
+2
View File
@@ -329,6 +329,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect,
// Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift)
opcodes: OpcodeLists {
@@ -671,6 +672,7 @@ fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::ima
// the stored colour is already white balanced (see above)
wb_multipliers: Some([1.0; 3]),
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -357,6 +357,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -238,6 +238,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -298,6 +298,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData { kelvin_presets, baseline_exposure: baseline_exposure(&h), ..Default::default() },
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: crate::raf_lens::corrections(&camera_name(bytes), &ifd, active), ..Default::default() },
camera: ifd0
+1
View File
@@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists { list3: distortion(ifd0, active).into_iter().collect(), ..Default::default() },
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -186,6 +186,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result<RawIma
color: ColorData::default(),
wb_multipliers: wb,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(),
+1
View File
@@ -44,6 +44,7 @@ fn an_active_area_that_wraps_past_the_image_is_corrupt() {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
opcodes: OpcodeLists::default(),
camera: Camera::default(),
cinema: Default::default(),
+67 -1
View File
@@ -1,7 +1,7 @@
//! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92
//! encoder) and must decode bit-exactly.
use craft_raw::opcodes::{Area, Opcode};
use craft_raw::opcodes::{self, Area, Opcode};
use craft_raw::{
BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode,
embedded_preview, probe, write_dng,
@@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option<Cfa>, cpp: usize) -> RawImage {
color,
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera,
@@ -1177,12 +1178,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() {
};
let img = build(None);
assert!(img.linearized);
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect()));
assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top");
// a stated level still wins
assert_eq!(build(Some(40000)).white, vec![40000.0]);
}
/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to
/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline
/// can keep that order. Applying the table first would give a different picture.
#[test]
fn opcode_list1_runs_on_stored_samples_before_linearization() {
let (w, h) = (4, 2);
let stored = vec![2u16; w * h];
// table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree)
let mut table = vec![0u16; 32];
table[2] = 20;
table[7] = 30;
table[20] = 99;
// MapTable: 2 → 7, 20 → 8
let mut map: Vec<u16> = (0..32).collect();
map[2] = 7;
map[20] = 8;
let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 };
let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]);
let mut raw = base_ifd(w, h, 16, 1, true);
raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE]));
raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] });
raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone()));
raw.set(t::OPCODE_LIST_1, Value::Undefined(list1));
raw.set(t::WHITE_LEVEL, Value::Long(vec![100]));
let img = decode(&dng(raw, ByteOrder::Little)).unwrap();
assert!(!img.linearized, "list 1 is present, so the table must wait");
assert_eq!(img.data, RawData::U16(stored.clone()));
assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(img.opcodes.list1.len(), 1);
let apply_map = |samples: &mut [u16]| {
for s in samples {
let i = (*s as usize).min(map.len() - 1);
*s = map[i];
}
};
let mut correct = img.clone();
if let RawData::U16(v) = &mut correct.data {
apply_map(v);
}
correct.apply_linearization();
assert!(correct.linearized);
// 2 → 7 (MapTable) → 30 (linearization)
assert_eq!(correct.data, RawData::U16(vec![30; w * h]));
let mut wrong = img.clone();
wrong.apply_linearization();
if let RawData::U16(v) = &mut wrong.data {
apply_map(v);
}
// 2 → 20 (linearization) → 8 (MapTable)
assert_eq!(wrong.data, RawData::U16(vec![8; w * h]));
assert_ne!(correct.data, wrong.data);
// the unapplied table and list 1 survive a write / decode round trip
let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap();
assert!(!back.linearized);
assert_eq!(back.data, RawData::U16(stored));
assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice()));
assert_eq!(back.opcodes.list1.len(), 1);
}
/// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1).
#[test]
fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() {
+1
View File
@@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec<u8> {
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
linearization_table: None,
pixel_aspect: 1.0,
opcodes: OpcodeLists::default(),
camera: Default::default(),