//! The CC0 sample files (raw.pixls.us) pinned in `samples.json`: each decodes to exactly the
//! sensor samples the derivation verified against the black-box reference (`clean-room/
//! 03-verification.md`), recorded as the SHA-256 of the little-endian sample buffer, with the
//! levels, white balance and metadata the decoder read then.
//!
//! The files are not in the repository (5–11 MB each): `python scripts/fetch-crw-samples.py
//!
` downloads and verifies them, and `CRAFT_CRW_SAMPLES=` points the test at them.
//! Without the variable the test prints that it skipped and passes; `CRAFT_CRW_REQUIRE_SAMPLES=1`
//! (CI) turns the skip into a failure.
//!
//! `CRAFT_CRW_ORACLE=` additionally compares every sample against `.crw.tiff`
//! references next to the files (16-bit grey TIFFs of the whole sensor, as written by the
//! reference decoder's `unprocessed_raw -T`), the check the derivation used. Those references
//! are not distributable and live outside the repository.
use craft_crw::{Error, Limits, decode, probe};
use serde::Deserialize;
use sha2::{Digest, Sha256};
use std::path::{Path, PathBuf};
#[derive(Deserialize)]
struct Sample {
name: String,
body: String,
sha256: String,
expect: Expect,
}
#[derive(Deserialize)]
#[serde(untagged)]
enum Expect {
Unsupported(String),
Decoded(Decoded),
}
#[derive(Deserialize)]
struct Decoded {
width: usize,
height: usize,
bits: u32,
selector: u32,
cfa: String,
active: [usize; 4],
model: String,
exif_orientation: u8,
capture_unix: i64,
wb: Option<[f32; 3]>,
black: [f32; 4],
white: f32,
samples_sha256: String,
}
fn manifest() -> Vec {
serde_json::from_str(include_str!("samples.json")).expect("samples.json parses")
}
fn hex(digest: impl AsRef<[u8]>) -> String {
digest.as_ref().iter().map(|b| format!("{b:02x}")).collect()
}
fn samples_dir() -> Option {
match std::env::var_os("CRAFT_CRW_SAMPLES") {
Some(dir) => Some(PathBuf::from(dir)),
None if std::env::var_os("CRAFT_CRW_REQUIRE_SAMPLES").is_some_and(|v| v == "1") => {
panic!("CRAFT_CRW_REQUIRE_SAMPLES=1 but CRAFT_CRW_SAMPLES is unset")
}
None => {
eprintln!("CRAFT_CRW_SAMPLES not set; skipping the sample-file tests (scripts/fetch-crw-samples.py)");
None
}
}
}
#[test]
fn manifest_is_well_formed() {
let m = manifest();
assert_eq!(m.len(), 11);
for s in &m {
assert!(s.name.ends_with(".crw") && !s.name.contains('/') && !s.name.contains('\\'), "{}", s.name);
assert_eq!(s.sha256.len(), 64, "{}", s.name);
if let Expect::Decoded(d) = &s.expect {
assert_eq!(d.samples_sha256.len(), 64, "{}", s.name);
assert!(matches!(d.bits, 10 | 12), "{}", s.name);
} else {
assert!(s.body.contains("G5"), "only the PowerShot G5 is refused: {}", s.name);
}
}
}
#[test]
fn sample_files_decode_to_the_verified_samples() {
let Some(dir) = samples_dir() else { return };
let mut checked = 0;
for s in manifest() {
let path = dir.join(&s.name);
let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
assert_eq!(hex(Sha256::digest(&bytes)), s.sha256, "{}: not the pinned CC0 file", s.name);
assert!(probe(&bytes), "{}: not recognised", s.name);
let started = std::time::Instant::now();
let result = decode(&bytes, &Limits::default());
let took = started.elapsed();
match (&s.expect, result) {
(Expect::Unsupported(kind), Err(Error::Unsupported(why))) => {
assert_eq!(kind, "unsupported", "{}", s.name);
assert!(why.contains("G5"), "{}: {why}", s.name);
eprintln!("{}: refused as documented ({why})", s.name);
}
(Expect::Unsupported(_), other) => panic!("{}: expected Unsupported, got {other:?}", s.name.clone()),
(Expect::Decoded(d), Ok(img)) => {
let h = &img.header;
assert_eq!((h.width, h.height, h.bits, h.selector), (d.width, d.height, d.bits, d.selector), "{}", s.name);
assert_eq!(h.cfa.name(), d.cfa, "{}", s.name);
assert_eq!([h.active_area.x, h.active_area.y, h.active_area.width, h.active_area.height], d.active, "{}", s.name);
assert_eq!(h.model.as_deref(), Some(d.model.as_str()), "{}", s.name);
assert_eq!(h.exif_orientation, d.exif_orientation, "{}", s.name);
assert_eq!(h.capture_unix, Some(d.capture_unix), "{}", s.name);
match (h.wb_multipliers, d.wb) {
(None, None) => {}
(Some(got), Some(want)) => {
assert!(got.iter().zip(&want).all(|(a, b)| (a - b).abs() < 1e-4), "{}: wb {got:?} vs {want:?}", s.name)
}
(got, want) => panic!("{}: wb {got:?} vs {want:?}", s.name),
}
assert!(
img.black.values.iter().zip(&d.black).all(|(a, b)| (a - b).abs() < 0.01),
"{}: black {:?} vs {:?}",
s.name,
img.black.values,
d.black
);
assert_eq!(img.white, d.white, "{}", s.name);
assert_eq!(img.samples.len(), d.width * d.height, "{}", s.name);
let mut hasher = Sha256::new();
for v in &img.samples {
hasher.update(v.to_le_bytes());
}
assert_eq!(hex(hasher.finalize()), d.samples_sha256, "{}: the decoded samples differ from the verified decode", s.name);
eprintln!("{}: {}x{} {}-bit selector {} EXACT in {took:.0?}", s.name, h.width, h.height, h.bits, h.selector);
}
(Expect::Decoded(_), Err(e)) => panic!("{}: {e}", s.name),
}
checked += 1;
}
assert_eq!(checked, 11);
}
/// Minimal reader for the reference TIFFs: one 16-bit grey image, stripped, either byte order.
fn oracle_samples(bytes: &[u8]) -> (usize, usize, Vec) {
let be = match &bytes[..4] {
b"II*\0" => false,
b"MM\0*" => true,
other => panic!("not a TIFF: {other:?}"),
};
let u16_at = |at: usize| -> u16 {
let b: [u8; 2] = bytes[at..at + 2].try_into().unwrap();
if be { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) }
};
let u32_at = |at: usize| -> u32 {
let b: [u8; 4] = bytes[at..at + 4].try_into().unwrap();
if be { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) }
};
let ifd = u32_at(4) as usize;
let n = u16_at(ifd) as usize;
let (mut w, mut h, mut bits, mut spp) = (0usize, 0usize, 0u16, 1u16);
let (mut offsets, mut counts) = (Vec::new(), Vec::new());
for k in 0..n {
let e = ifd + 2 + k * 12;
let (tag, typ, cnt) = (u16_at(e), u16_at(e + 2), u32_at(e + 4) as usize);
let size = match typ {
3 => 2,
4 => 4,
_ => 1,
};
let values_at = if cnt * size <= 4 { e + 8 } else { u32_at(e + 8) as usize };
let value = |i: usize| -> usize { if typ == 3 { u16_at(values_at + i * 2) as usize } else { u32_at(values_at + i * 4) as usize } };
match tag {
256 => w = value(0),
257 => h = value(0),
258 => bits = value(0) as u16,
277 => spp = value(0) as u16,
273 => offsets = (0..cnt).map(value).collect(),
279 => counts = (0..cnt).map(value).collect(),
_ => {}
}
}
assert_eq!((bits, spp), (16, 1), "oracle TIFF is 16-bit grey");
let mut data = Vec::with_capacity(w * h);
for (&o, &c) in offsets.iter().zip(&counts) {
let strip = &bytes[o..o + c];
data.extend(strip.as_chunks::<2>().0.iter().map(|p| if be { u16::from_be_bytes(*p) } else { u16::from_le_bytes(*p) }));
}
data.truncate(w * h);
(w, h, data)
}
#[test]
fn oracle_tiffs_match_every_sample() {
let Some(dir) = std::env::var_os("CRAFT_CRW_ORACLE").map(PathBuf::from) else {
eprintln!("CRAFT_CRW_ORACLE not set; skipping the reference-TIFF comparison");
return;
};
let mut entries: Vec = std::fs::read_dir(&dir).expect("oracle directory").filter_map(|e| e.ok()).map(|e| e.path()).collect();
entries.sort();
let mut checked = 0;
for path in entries.iter().filter(|p| p.extension().and_then(|e| e.to_str()).is_some_and(|e| e.eq_ignore_ascii_case("crw"))) {
let name = path.file_name().and_then(|n| n.to_str()).unwrap_or_default();
let reference: &Path = &path.with_extension("crw.tiff");
if !reference.is_file() {
eprintln!("{name}: no reference TIFF, skipped");
continue;
}
let bytes = std::fs::read(path).expect("read crw");
let img = match decode(&bytes, &Limits::default()) {
Ok(img) => img,
Err(Error::Unsupported(why)) if why.contains("G5") => {
eprintln!("{name}: G5 refused as documented");
continue;
}
Err(e) => panic!("{name}: {e}"),
};
let (w, h, oracle) = oracle_samples(&std::fs::read(reference).expect("read oracle tiff"));
assert_eq!((img.header.width, img.header.height), (w, h), "{name}: dimensions");
assert_eq!(oracle.len(), w * h, "{name}: oracle sample count");
assert_eq!(img.samples.len(), w * h, "{name}: decoded sample count");
let mismatches = img.samples.iter().zip(&oracle).filter(|(a, b)| a != b).count();
assert_eq!(mismatches, 0, "{name}: {mismatches} mismatching samples");
eprintln!("{name}: EXACT ({w}x{h})");
checked += 1;
}
assert!(checked > 0, "no .crw files with references found in {}", dir.display());
}