diff --git a/Cargo.lock b/Cargo.lock index 2941a4097..95341b06d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -330,6 +330,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "astc-decode" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6058e6202e20b7fa445692ec4b77ca8eb02508091fcd479ea753d9f60e376b" + [[package]] name = "async-broadcast" version = "0.7.2" @@ -1445,6 +1451,14 @@ dependencies = [ "encoding_rs", ] +[[package]] +name = "craft-control-protocol" +version = "0.1.0" +dependencies = [ + "serde_json", + "subtle", +] + [[package]] name = "craft-crs" version = "0.1.0" @@ -1671,6 +1685,15 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-fcproj-format" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-fig" version = "0.1.0" @@ -1762,6 +1785,7 @@ name = "craft-font-catalog" version = "0.1.0" dependencies = [ "fontique", + "kurbo 0.13.1", "skrifa 0.47.0", "thiserror 2.0.21", "windows-registry", @@ -2759,6 +2783,8 @@ dependencies = [ name = "craft-texture" version = "0.1.0" dependencies = [ + "bytemuck", + "dds", "rayon", "thiserror 2.0.21", ] @@ -3304,6 +3330,19 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dds" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e917a58e17b27d2f15626c490754ccced7f46592dc5b5f39a3013b89cba5f068" +dependencies = [ + "astc-decode", + "bitflags 2.13.2", + "glam", + "resize", + "zerocopy", +] + [[package]] name = "der" version = "0.7.10" @@ -4552,6 +4591,12 @@ dependencies = [ "xml-rs", ] +[[package]] +name = "glam" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "151665d9be52f9bb40fc7966565d39666f2d1e69233571b71b87791c7e0528b3" + [[package]] name = "glifo" version = "0.1.1" @@ -8142,6 +8187,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "resize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71725ecd5e0197b54fe859055b108688472ab6a358f8fbe5cee4a556b1b5bfea" +dependencies = [ + "rgb", +] + [[package]] name = "resvg" version = "0.45.1" diff --git a/Cargo.toml b/Cargo.toml index 48a2edf1d..9558946f5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,6 +70,7 @@ craft-doc = { path = "crates/doc" } craft-docpath = { path = "crates/docpath" } craft-dpx = { path = "crates/dpx" } craft-dwg = { path = "crates/dwg" } +craft-fcproj-format = { path = "crates/fcproj-format" } craft-ecproj-format = { path = "crates/ecproj-format" } craft-figma-rest = { path = "crates/figma-rest" } craft-exr = { path = "crates/exr" } @@ -93,6 +94,7 @@ craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } +craft-control-protocol = { path = "crates/control-protocol" } craft-jsonrpc = { path = "crates/jsonrpc" } craft-mcp = { path = "crates/mcp" } craft-jxl = { path = "crates/jxl" } diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index 859ac20bc..fe28cec3f 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -29,6 +29,8 @@ encode = ["lossless-encode", "aac-encode", "alac-encode", "ac3-encode", "eac3-en lossless-encode = ["pcm-encode", "flac-encode"] # Encode WAV/BWF/RF64 and AIFF/AIFF-C files. pcm-encode = [] +# Decode FLAC files and frames natively (`flac::decode`, `flac::Decoder`). +flac-decode = [] # Encode FLAC files. flac-encode = ["craft-media/bitstream"] # The pure-Rust Vorbis encoder (`vorbis::encoder`) and Ogg Vorbis files through `encode` and @@ -86,6 +88,10 @@ required-features = ["decode", "lossless-encode", "peaks"] name = "codecs" required-features = ["decode", "encode"] +[[test]] +name = "flac_decode" +required-features = ["flac-decode", "flac-encode"] + [[test]] name = "qoa_oracle" required-features = ["qoa"] diff --git a/crates/audio/README.md b/crates/audio/README.md index a5af4b3f3..7469a9401 100644 --- a/crates/audio/README.md +++ b/crates/audio/README.md @@ -32,6 +32,7 @@ No default features. The shared types (`AudioBuffer`, `AudioInfo`, `ChannelLayou | `encode` | every encoder, and every format of `encode` and `FileEncoder` | `lossless-encode`, `aac-encode`, `alac-encode`, `ac3-encode`, `eac3-encode`, `mp3-encode`, `opus-encode`, `vorbis-encode` | | `lossless-encode` | WAV, AIFF and FLAC writing | `pcm-encode`, `flac-encode` | | `pcm-encode` | `encode`, `FileEncoder` for WAV and AIFF; `wav_float_header`; `pcm_wav_file`, `pcm_wav_header`, `pcm_aiff_file`, `pcm_aiff_header` (minimal files around integer PCM bytes, for essence embedded in AAF / OMF) | none | +| `flac-decode` | `flac::decode` (whole `.flac` files) and `flac::Decoder` (single frames, for callers that demux FLAC themselves); native, CRC-checked, 4 to 32 bits. `decode` still uses symphonia for FLAC files | none | | `flac-encode` | `encode`, `FileEncoder` for FLAC; `flac::Encoder`, the packet encoder for callers that mux FLAC themselves | craft-media `bitstream` | | `aac-decode`, `aac-encode` | `aac::Decoder`, `aac::Encoder`; with `aac-decode`, `aac::latm_config` / `latm_split` for LOAS / LATM frames (AAC in MPEG transport streams); with `aac-encode`, `encode`, `FileEncoder` for AAC in M4A | craft-media `bitstream`; `aac-encode` also `isobmff` | | `alac-decode` | `alac::Decoder` | none | diff --git a/crates/audio/src/flac_decode.rs b/crates/audio/src/flac_decode.rs new file mode 100644 index 000000000..bb56e65c5 --- /dev/null +++ b/crates/audio/src/flac_decode.rs @@ -0,0 +1,603 @@ +//! A native FLAC decoder (RFC 9639): [`decode`] for whole `.flac` files and [`Decoder`] for single +//! frames (the packets of FLAC in MP4, Matroska or Ogg). +//! +//! Every frame is checked against its CRC-8 and CRC-16. Subframe types constant, verbatim, fixed +//! and LPC; Rice partitions with 4- and 5-bit parameters and escaped partitions; wasted bits; +//! left/side, side/right and mid/side stereo; fixed and variable block sizes; 4 to 32 bits per +//! sample. The STREAMINFO MD5 is not verified. +//! +//! Samples come back as `i32`, one `Vec` per channel, in the FLAC channel order. + +/// What went wrong decoding FLAC. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum DecodeError { + /// The data ends early. + #[error("truncated FLAC data")] + Truncated, + /// The data is not valid FLAC. + #[error("invalid FLAC data: {0}")] + Invalid(&'static str), + /// A frame or header failed its CRC. + #[error("FLAC CRC mismatch in {0}")] + Crc(&'static str), + /// Valid FLAC that is not decoded. + #[error("unsupported FLAC feature: {0}")] + Unsupported(&'static str), + /// The stream is longer than the sample limit. + #[error("FLAC stream exceeds the sample limit")] + TooLarge, +} + +/// `Result` of this module. +pub type Result = std::result::Result; + +/// The STREAMINFO metadata block (RFC 9639 section 8.2). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StreamInfo { + /// Smallest block size in samples. + pub min_block_size: u16, + /// Largest block size in samples. + pub max_block_size: u16, + /// Smallest frame in bytes, 0 if unknown. + pub min_frame_size: u32, + /// Largest frame in bytes, 0 if unknown. + pub max_frame_size: u32, + /// Sample rate in Hz. + pub sample_rate: u32, + /// Channel count, 1 to 8. + pub channels: u8, + /// Bits per sample, 4 to 32. + pub bits_per_sample: u8, + /// Samples per channel in the stream, 0 if unknown. + pub total_samples: u64, + /// MD5 of the unencoded audio, all zero if not set. + pub md5: [u8; 16], +} + +impl StreamInfo { + /// Parse the 34 bytes of a STREAMINFO block body. + pub fn parse(b: &[u8]) -> Result { + let b = b.get(..34).ok_or(DecodeError::Truncated)?; + let packed = u64::from_be_bytes([b[10], b[11], b[12], b[13], b[14], b[15], b[16], b[17]]); + let info = StreamInfo { + min_block_size: u16::from_be_bytes([b[0], b[1]]), + max_block_size: u16::from_be_bytes([b[2], b[3]]), + min_frame_size: u32::from_be_bytes([0, b[4], b[5], b[6]]), + max_frame_size: u32::from_be_bytes([0, b[7], b[8], b[9]]), + sample_rate: (packed >> 44) as u32, + channels: ((packed >> 41) & 7) as u8 + 1, + bits_per_sample: ((packed >> 36) & 31) as u8 + 1, + total_samples: packed & 0xF_FFFF_FFFF, + md5: b[18..34].try_into().map_err(|_| DecodeError::Truncated)?, + }; + if info.bits_per_sample < 4 { + return Err(DecodeError::Invalid("bits per sample below 4")); + } + if info.min_block_size != 0 + && (info.min_block_size < 16 && info.min_block_size != info.max_block_size || info.max_block_size < info.min_block_size) + { + return Err(DecodeError::Invalid("inconsistent block sizes")); + } + Ok(info) + } +} + +/// One decoded frame. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + /// Sample rate in Hz. + pub sample_rate: u32, + /// Bits per sample. + pub bits_per_sample: u8, + /// `true` when `number` is the first sample's index (variable block size), `false` when it is + /// the frame's index (fixed block size). + pub variable_block_size: bool, + /// The coded number of the frame header. + pub number: u64, + /// Samples per channel in the frame. + pub block_size: usize, + /// One `Vec` per channel, after undoing any stereo decorrelation. + pub channels: Vec>, + /// Bytes the frame took, including the header and CRC footer. + pub size: usize, +} + +// ---- checksums ---- + +fn crc8(data: &[u8]) -> u8 { + let mut crc = 0u8; + for &b in data { + crc ^= b; + for _ in 0..8 { + crc = if crc & 0x80 != 0 { crc << 1 ^ 0x07 } else { crc << 1 }; + } + } + crc +} + +fn crc16(data: &[u8]) -> u16 { + let mut crc = 0u16; + for &b in data { + crc ^= u16::from(b) << 8; + for _ in 0..8 { + crc = if crc & 0x8000 != 0 { crc << 1 ^ 0x8005 } else { crc << 1 }; + } + } + crc +} + +// ---- bits ---- + +struct Bits<'a> { + data: &'a [u8], + /// Position in bits. + pos: usize, +} + +impl<'a> Bits<'a> { + fn new(data: &'a [u8]) -> Self { + Bits { data, pos: 0 } + } + + fn left(&self) -> usize { + self.data.len() * 8 - self.pos + } + + fn bit(&mut self) -> Result { + let byte = *self.data.get(self.pos / 8).ok_or(DecodeError::Truncated)?; + let v = (byte >> (7 - self.pos % 8)) & 1; + self.pos += 1; + Ok(u32::from(v)) + } + + /// `n` bits (0 to 32) as an unsigned number. + fn bits(&mut self, n: u32) -> Result { + if n as usize > self.left() { + return Err(DecodeError::Truncated); + } + let mut v = 0u64; + let mut n = n as usize; + while n > 0 { + let byte = u64::from(self.data[self.pos / 8]); + let avail = 8 - self.pos % 8; + let take = avail.min(n); + v = v << take | (byte >> (avail - take)) & ((1 << take) - 1); + self.pos += take; + n -= take; + } + Ok(v) + } + + /// `n` bits (0 to 33) as a two's complement number. + fn signed(&mut self, n: u32) -> Result { + if n == 0 { + return Ok(0); + } + let v = self.bits(n)? as i64; + Ok(if v >> (n - 1) & 1 == 1 { v - (1 << n) } else { v }) + } + + /// Zero bits up to the next one bit (which is consumed). + fn unary(&mut self) -> Result { + let mut n = 0u64; + loop { + // Skip whole zero bytes when aligned. + if self.pos.is_multiple_of(8) { + while self.data.get(self.pos / 8) == Some(&0) { + self.pos += 8; + n += 8; + } + } + if self.bit()? == 1 { + return Ok(n); + } + n += 1; + } + } + + fn align(&mut self) { + self.pos = self.pos.div_ceil(8) * 8; + } +} + +// ---- frame ---- + +const BLOCK_SIZES: [usize; 16] = [0, 192, 576, 1152, 2304, 4608, 0, 0, 256, 512, 1024, 2048, 4096, 8192, 16384, 32768]; + +/// A frame decoder. `info` supplies what a frame header leaves to the STREAMINFO. +#[derive(Debug, Clone, Copy, Default)] +pub struct Decoder { + info: Option, +} + +impl Decoder { + /// A decoder with the stream's STREAMINFO (needed for frames whose header defers the sample + /// rate or bit depth to it). + pub fn new(info: StreamInfo) -> Decoder { + Decoder { info: Some(info) } + } + + /// A decoder without STREAMINFO: frames that defer to it are an error. + pub fn without_streaminfo() -> Decoder { + Decoder { info: None } + } + + /// Decode the frame at the start of `data`. + pub fn decode_frame(&self, data: &[u8]) -> Result { + // StreamInfo has public fields: container callers can construct it without + // parse(), so validate before it can control shifts and sample reads. + if self.info.is_some_and(|i| !(4..=32).contains(&i.bits_per_sample) || !(1..=8).contains(&i.channels) || i.sample_rate == 0) { + return Err(DecodeError::Invalid("invalid decoder STREAMINFO")); + } + let mut r = Bits::new(data); + if r.bits(15)? != 0x7FFC { + return Err(DecodeError::Invalid("missing frame sync code")); + } + let variable = r.bit()? == 1; + let (bs_code, sr_code) = (r.bits(4)? as usize, r.bits(4)? as usize); + let (ch_code, bps_code) = (r.bits(4)? as usize, r.bits(3)? as usize); + if r.bit()? != 0 { + return Err(DecodeError::Invalid("reserved bit set")); + } + let number = coded_number(&mut r)?; + let block_size = match bs_code { + 0 => return Err(DecodeError::Invalid("reserved block size")), + 6 => r.bits(8)? as usize + 1, + 7 => r.bits(16)? as usize + 1, + c => BLOCK_SIZES[c], + }; + let sample_rate = match sr_code { + 0 => self.info.map(|i| i.sample_rate).ok_or(DecodeError::Invalid("sample rate only in STREAMINFO"))?, + 1 => 88_200, + 2 => 176_400, + 3 => 192_000, + 4 => 8_000, + 5 => 16_000, + 6 => 22_050, + 7 => 24_000, + 8 => 32_000, + 9 => 44_100, + 10 => 48_000, + 11 => 96_000, + 12 => r.bits(8)? as u32 * 1000, + 13 => r.bits(16)? as u32, + 14 => r.bits(16)? as u32 * 10, + _ => return Err(DecodeError::Invalid("forbidden sample rate code")), + }; + let header_len = r.pos.div_ceil(8); + let stored_crc = r.bits(8)? as u8; + if crc8(&data[..header_len]) != stored_crc { + return Err(DecodeError::Crc("a frame header")); + } + let bits = match bps_code { + 0 => self.info.map(|i| u32::from(i.bits_per_sample)).ok_or(DecodeError::Invalid("bit depth only in STREAMINFO"))?, + 1 => 8, + 2 => 12, + 4 => 16, + 5 => 20, + 6 => 24, + 7 => 32, + _ => return Err(DecodeError::Invalid("reserved bit depth")), + }; + let (channels, mode) = match ch_code { + 0..=7 => (ch_code + 1, Stereo::Independent), + 8 => (2, Stereo::LeftSide), + 9 => (2, Stereo::SideRight), + 10 => (2, Stereo::MidSide), + _ => return Err(DecodeError::Invalid("reserved channel assignment")), + }; + let mut out: Vec> = Vec::with_capacity(channels); + for c in 0..channels { + let extra = match (mode, c) { + (Stereo::LeftSide, 1) | (Stereo::SideRight, 0) | (Stereo::MidSide, 1) => 1, + _ => 0, + }; + out.push(subframe(&mut r, block_size, bits + extra)?); + } + r.align(); + let size = r.pos / 8; + let crc = r.bits(16)? as u16; + if crc16(&data[..size]) != crc { + return Err(DecodeError::Crc("a frame")); + } + let size = size + 2; + if mode != Stereo::Independent { + let (first, rest) = out.split_at_mut(1); + let (a, b) = (&mut first[0], &mut rest[0]); + for (x, y) in a.iter_mut().zip(b.iter_mut()) { + match mode { + Stereo::LeftSide => *y = x.wrapping_sub(*y), + Stereo::SideRight => *x = x.wrapping_add(*y), + _ => { + let m = x.wrapping_shl(1) | (*y & 1); + let side = *y; + *x = m.wrapping_add(side) >> 1; + *y = m.wrapping_sub(side) >> 1; + } + } + } + } + let mut chans = Vec::with_capacity(channels); + for c in out { + let v: Result> = c.into_iter().map(|s| checked_sample(s, bits)).collect(); + chans.push(v?); + } + Ok(Frame { sample_rate, bits_per_sample: bits as u8, variable_block_size: variable, number, block_size, channels: chans, size }) + } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Stereo { + Independent, + LeftSide, + SideRight, + MidSide, +} + +/// The UTF-8-like coded number of a frame header (up to 36 bits, 7 bytes). +fn coded_number(r: &mut Bits) -> Result { + let first = r.bits(8)? as u8; + let extra = match first { + 0x00..=0x7F => return Ok(u64::from(first)), + 0xC0..=0xDF => 1, + 0xE0..=0xEF => 2, + 0xF0..=0xF7 => 3, + 0xF8..=0xFB => 4, + 0xFC..=0xFD => 5, + 0xFE => 6, + _ => return Err(DecodeError::Invalid("bad coded number")), + }; + let mut v = if extra == 6 { 0 } else { u64::from(first & (0x3F >> extra)) }; + for _ in 0..extra { + let b = r.bits(8)?; + if b & 0xC0 != 0x80 { + return Err(DecodeError::Invalid("bad coded number")); + } + v = v << 6 | (b & 0x3F); + } + Ok(v) +} + +/// One subframe: `bits` is the frame bit depth (plus 1 for a side channel). +fn subframe(r: &mut Bits, n: usize, bits: u32) -> Result> { + if r.bit()? != 0 { + return Err(DecodeError::Invalid("subframe header padding bit set")); + } + let kind = r.bits(6)? as u32; + let wasted = if r.bit()? == 1 { r.unary()? as u32 + 1 } else { 0 }; + if wasted >= bits { + return Err(DecodeError::Invalid("wasted bits leave no sample bits")); + } + let bps = bits - wasted; + let mut s = vec![0i64; n]; + match kind { + 0 => { + let v = r.signed(bps)?; + s.fill(v); + } + 1 => { + for v in s.iter_mut() { + *v = r.signed(bps)?; + } + } + 8..=12 => { + let order = (kind - 8) as usize; + warmup(r, &mut s, order, bps)?; + residual(r, &mut s, order)?; + for i in order..n { + let at = |k: usize| s[i - k]; + // Wrapping: hostile residuals can overflow; such samples fail the range check later. + let p = match order { + 0 => 0, + 1 => at(1), + 2 => at(1).wrapping_mul(2).wrapping_sub(at(2)), + 3 => at(1).wrapping_mul(3).wrapping_sub(at(2).wrapping_mul(3)).wrapping_add(at(3)), + _ => at(1).wrapping_mul(4).wrapping_sub(at(2).wrapping_mul(6)).wrapping_add(at(3).wrapping_mul(4)).wrapping_sub(at(4)), + }; + s[i] = s[i].wrapping_add(p); + } + } + 32..=63 => { + let order = (kind - 31) as usize; + warmup(r, &mut s, order, bps)?; + let precision = r.bits(4)? as u32 + 1; + if precision == 16 { + return Err(DecodeError::Invalid("forbidden coefficient precision")); + } + let shift = r.signed(5)?; + if shift < 0 { + return Err(DecodeError::Invalid("negative prediction shift")); + } + let mut coefs = [0i64; 32]; + for c in coefs.iter_mut().take(order) { + *c = r.signed(precision)?; + } + residual(r, &mut s, order)?; + for i in order..n { + let mut sum = 0i64; + for (k, &c) in coefs.iter().take(order).enumerate() { + sum = sum.wrapping_add(c.wrapping_mul(s[i - 1 - k])); + } + s[i] = s[i].wrapping_add(sum >> shift); + } + } + _ => return Err(DecodeError::Invalid("reserved subframe type")), + } + if wasted > 0 { + for v in s.iter_mut() { + *v = v.wrapping_shl(wasted); + } + } + Ok(s) +} + +fn warmup(r: &mut Bits, s: &mut [i64], order: usize, bps: u32) -> Result<()> { + if order > s.len() { + return Err(DecodeError::Invalid("predictor order above the block size")); + } + for v in s.iter_mut().take(order) { + *v = r.signed(bps)?; + } + Ok(()) +} + +/// The coded residual (RFC 9639 section 9.2.7), into `s[order..]`. +fn residual(r: &mut Bits, s: &mut [i64], order: usize) -> Result<()> { + let method = r.bits(2)?; + let (param_bits, escape) = match method { + 0 => (4, 15), + 1 => (5, 31), + _ => return Err(DecodeError::Invalid("reserved residual coding method")), + }; + let part_order = r.bits(4)? as u32; + let n = s.len(); + let parts = 1usize << part_order; + if !n.is_multiple_of(parts) { + return Err(DecodeError::Invalid("partition order does not divide the block size")); + } + let per = n >> part_order; + if per < order { + return Err(DecodeError::Invalid("partition shorter than the predictor order")); + } + let mut i = order; + for p in 0..parts { + let count = if p == 0 { per - order } else { per }; + let k = r.bits(param_bits)? as u32; + if k == escape { + let width = r.bits(5)? as u32; + for v in s[i..i + count].iter_mut() { + *v = r.signed(width)?; + } + } else { + for v in s[i..i + count].iter_mut() { + let q = r.unary()?; + if q > (1 << 32) { + return Err(DecodeError::Invalid("Rice quotient too large")); + } + let low = r.bits(k)?; + let folded = q << k | low; + *v = if folded & 1 == 0 { (folded >> 1) as i64 } else { -((folded >> 1) as i64) - 1 }; + } + } + i += count; + } + Ok(()) +} + +// ---- files ---- + +/// A decoded file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Decoded { + /// The STREAMINFO block. + pub info: StreamInfo, + /// One `Vec` of samples per channel. + pub channels: Vec>, +} + +/// Most sample values (samples x channels) [`decode`] will produce. +pub const DEFAULT_MAX_VALUES: u64 = 1 << 28; + +/// Decode a whole `.flac` file (an ID3v2 tag in front is skipped). +pub fn decode(bytes: &[u8]) -> Result { + decode_with_limit(bytes, DEFAULT_MAX_VALUES) +} + +/// [`decode`] with a limit on the sample values (samples x channels) produced. +pub fn decode_with_limit(bytes: &[u8], max_values: u64) -> Result { + let mut pos = 0usize; + if bytes.starts_with(b"ID3") { + let size = bytes.get(6..10).ok_or(DecodeError::Truncated)?; + if size.iter().any(|&b| b & 0x80 != 0) { + return Err(DecodeError::Invalid("bad ID3v2 size")); + } + pos = 10 + size.iter().fold(0usize, |a, &b| a << 7 | usize::from(b)); + } + if bytes.get(pos..pos + 4) != Some(b"fLaC") { + return Err(DecodeError::Invalid("missing fLaC marker")); + } + pos += 4; + let mut info: Option = None; + loop { + let head = bytes.get(pos..pos + 4).ok_or(DecodeError::Truncated)?; + let last = head[0] & 0x80 != 0; + let kind = head[0] & 0x7F; + let len = usize::from(head[1]) << 16 | usize::from(head[2]) << 8 | usize::from(head[3]); + let body = bytes.get(pos + 4..pos + 4 + len).ok_or(DecodeError::Truncated)?; + match (kind, &info) { + (0, None) => info = Some(StreamInfo::parse(body)?), + (0, Some(_)) => return Err(DecodeError::Invalid("a second STREAMINFO block")), + (_, None) => return Err(DecodeError::Invalid("STREAMINFO is not the first block")), + (127, _) => return Err(DecodeError::Invalid("forbidden metadata block type")), + _ => {} + } + pos += 4 + len; + if last { + break; + } + } + let info = info.ok_or(DecodeError::Invalid("no STREAMINFO"))?; + let dec = Decoder::new(info); + let mut channels: Vec> = vec![Vec::new(); usize::from(info.channels)]; + let mut total = 0u64; + while pos + 2 <= bytes.len() && bytes[pos] == 0xFF && bytes[pos + 1] & 0xFE == 0xF8 { + let frame = dec.decode_frame(&bytes[pos..])?; + if frame.channels.len() != channels.len() { + return Err(DecodeError::Invalid("the channel count changes inside the stream")); + } + // One file has one sample rate and depth. Otherwise callers would play + // samples under incompatible STREAMINFO metadata. + if frame.sample_rate != info.sample_rate || frame.bits_per_sample != info.bits_per_sample { + return Err(DecodeError::Invalid("frame format differs from STREAMINFO")); + } + total += frame.block_size as u64; + if total.saturating_mul(channels.len() as u64) > max_values { + return Err(DecodeError::TooLarge); + } + for (dst, src) in channels.iter_mut().zip(frame.channels) { + dst.extend_from_slice(&src); + } + pos += frame.size; + } + // Unknown total_samples must not turn a corrupt/truncated next frame into + // a successfully decoded shorter recording. + if pos != bytes.len() { + return Err(DecodeError::Invalid("trailing data or missing frame sync")); + } + if info.total_samples != 0 && total != info.total_samples { + return Err(DecodeError::Truncated); + } + Ok(Decoded { info, channels }) +} + +// Validate the advertised precision, not just the storage type. Malformed +// predictors may produce an i32 that is still invalid for 4–24-bit audio. +fn checked_sample(sample: i64, bits: u32) -> Result { + if !(4..=32).contains(&bits) { + return Err(DecodeError::Invalid("invalid sample bit depth")); + } + let bound = 1i64 << (bits - 1); + if !(-bound..bound).contains(&sample) { + return Err(DecodeError::Invalid("sample outside the bit depth")); + } + i32::try_from(sample).map_err(|_| DecodeError::Invalid("sample outside the bit depth")) +} + +#[cfg(test)] +mod sample_bounds_tests { + use super::checked_sample; + + #[test] + fn samples_must_fit_declared_precision() { + for bits in 4..=32 { + let bound = 1i64 << (bits - 1); + assert_eq!(checked_sample(-bound, bits).map(i64::from), Ok(-bound)); + assert_eq!(checked_sample(bound - 1, bits).map(i64::from), Ok(bound - 1)); + assert!(checked_sample(-bound - 1, bits).is_err()); + assert!(checked_sample(bound, bits).is_err()); + } + assert!(checked_sample(0, 0).is_err()); + assert!(checked_sample(0, 255).is_err()); + } +} diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 4fe25b9e9..f653e9bba 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -29,8 +29,8 @@ //! - Codec modules for packet-level use, each with its own features: [`aac`] (`aac-decode`, //! `aac-encode`), [`alac`] (`alac-decode`, `alac-encode`), [`opus`] (`opus-decode`, and //! [`opus::encoder`] with `opus-encode`), [`ac3`] (`ac3-decode`), [`mpa`] (`mpa-decode`), [`mp3`] -//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder), [`qoa`] -//! (`qoa-decode`, `qoa-encode`: whole QOA files). +//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-decode`: the frame and file decoder; +//! `flac-encode`: the packet encoder), [`qoa`] (`qoa-decode`, `qoa-encode`: whole QOA files). //! //! The shared types below build with no features. @@ -105,10 +105,15 @@ pub use write::{OGG_BITRATES, OggVorbisOptions, encode_ogg, encode_ogg_vorbis, q /// assert!(file.len() < 48_000 * 2 * 2 / 2); // well under half of 16-bit PCM /// # Ok::<(), craft_audio::flac::Error>(()) /// ``` -#[cfg(feature = "flac-encode")] +#[cfg(any(feature = "flac-encode", feature = "flac-decode"))] pub mod flac { + #[cfg(feature = "flac-decode")] + pub use crate::flac_decode::{DEFAULT_MAX_VALUES, DecodeError, Decoded, Decoder, Frame, StreamInfo, decode, decode_with_limit}; + #[cfg(feature = "flac-encode")] pub use crate::write::flac::encoder::{Encoder, EncoderConfig, Error, STREAMINFO_OFFSET}; } +#[cfg(feature = "flac-decode")] +mod flac_decode; pub use buffer::AudioBuffer; /// The craft-media this crate is built with, for its containers: using it rather than a separate diff --git a/crates/audio/tests/flac_decode.rs b/crates/audio/tests/flac_decode.rs new file mode 100644 index 000000000..a1895789b --- /dev/null +++ b/crates/audio/tests/flac_decode.rs @@ -0,0 +1,211 @@ +//! The native FLAC decoder: round trips through this crate's encoder over bit depths, channel +//! counts, block sizes and effort levels; hostile input; and ffmpeg as the reference (oracle; +//! skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`) for files +//! ffmpeg's encoder writes at every compression level. + +use craft_audio::flac::{DecodeError, Decoder, Encoder, EncoderConfig, STREAMINFO_OFFSET, StreamInfo, decode}; +use std::process::Command; + +fn signal(channels: usize, frames: usize, bits: u32, seed: u32) -> Vec> { + let max = if bits == 32 { i32::MAX as f64 } else { ((1i64 << (bits - 1)) - 1) as f64 }; + let mut s = seed; + (0..channels) + .map(|c| { + (0..frames) + .map(|i| { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + let t = i as f64; + let v = match (i / 3000) % 4 { + 0 => (t * (0.01 + 0.007 * c as f64)).sin() * 0.6 + (t * 0.31).sin() * 0.1, + 1 => (f64::from(s) / f64::from(u32::MAX) - 0.5) * 1.8, + 2 => 0.0, + _ => { + if (i / 40) % 2 == 0 { + 0.99 + } else { + -0.99 + } + } + }; + (v * max).round() as i32 + }) + .collect() + }) + .collect() +} + +fn encode(channels: &[Vec], rate: u32, bits: u8, block: u16, level: u8) -> Vec { + let mut cfg = EncoderConfig::new(rate, channels.len() as u8, bits); + cfg.block_size = block; + cfg.level = level; + let mut enc = Encoder::new(cfg).expect("encoder"); + let mut file = enc.header(&[]); + let refs: Vec<&[i32]> = channels.iter().map(Vec::as_slice).collect(); + file.extend(enc.encode(&refs).expect("encode")); + file.extend(enc.finish()); + let at = STREAMINFO_OFFSET as usize; + file[at..at + 34].copy_from_slice(&enc.streaminfo()); + file +} + +#[test] +fn round_trips_through_the_encoder() { + for (channels, bits) in [(1usize, 8u8), (2, 8), (1, 12), (2, 16), (2, 20), (3, 24), (6, 24), (8, 16), (2, 32), (1, 32)] { + for (block, level) in [(4096u16, 5u8), (16, 0), (1000, 8), (65535, 3), (576, 1)] { + let frames = 9000 + usize::from(block % 97); + let src = signal(channels, frames, u32::from(bits), 0x1234_5679 + u32::from(bits)); + let file = encode(&src, 44_100, bits, block, level); + let got = decode(&file).unwrap_or_else(|e| panic!("{channels}ch {bits}-bit block {block} level {level}: {e}")); + assert_eq!((got.info.channels as usize, got.info.bits_per_sample, got.info.sample_rate), (channels, bits, 44_100)); + assert_eq!(got.channels, src, "{channels}ch {bits}-bit block {block} level {level}"); + } + } +} + +#[test] +fn frames_decode_one_at_a_time() { + let src = signal(2, 10_000, 16, 7); + let file = encode(&src, 48_000, 16, 4096, 5); + let info = StreamInfo::parse(&file[STREAMINFO_OFFSET as usize..]).expect("streaminfo"); + let dec = Decoder::new(info); + // The first frame follows the metadata blocks; find the first position a frame decodes at. + let at = (42..file.len()).find(|&p| dec.decode_frame(&file[p..]).is_ok()).expect("a frame"); + let first = dec.decode_frame(&file[at..]).expect("frame"); + assert_eq!((first.block_size, first.number, first.sample_rate, first.bits_per_sample), (4096, 0, 48_000, 16)); + assert_eq!(first.channels[0], src[0][..4096]); + // Without the STREAMINFO a frame that spells out its own rate and depth still decodes. + let _ = Decoder::without_streaminfo().decode_frame(&file[at..]); +} + +#[test] +fn hostile_input_is_an_error_not_a_panic() { + let src = signal(2, 6000, 16, 11); + let file = encode(&src, 44_100, 16, 1152, 5); + for n in 0..file.len().min(3000) { + let _ = decode(&file[..n]); + } + // A cut file is truncated, not silently shorter. + assert!(decode(&file[..file.len() - 5]).is_err()); + // Every bit flip in the first frames is caught by a CRC or another check, never a panic. + for i in 0..file.len().min(2500) { + for bit in 0..8 { + let mut c = file.clone(); + c[i] ^= 1 << bit; + let _ = decode(&c); + } + } + // A sample limit stops a stream that claims too much. + assert_eq!(craft_audio::flac::decode_with_limit(&file, 1000), Err(DecodeError::TooLarge)); + assert!(decode(b"nope").is_err()); + assert!(decode(b"fLaC").is_err()); +} + +// ---- ffmpeg ---- + +/// Little-endian 16- or 32-bit samples of raw PCM. +fn pcm_values(raw: &[u8], bytes: usize) -> Vec { + if bytes == 2 { + raw.as_chunks::<2>().0.iter().map(|b| i32::from(i16::from_le_bytes(*b))).collect() + } else { + raw.as_chunks::<4>().0.iter().map(|b| i32::from_le_bytes(*b)).collect() + } +} + +/// Planar channels as interleaved samples. +fn interleave(channels: &[Vec]) -> Vec { + let frames = channels.first().map_or(0, Vec::len); + (0..frames).flat_map(|i| channels.iter().map(move |c| c[i])).collect() +} + +fn ffmpeg_run(ff: &std::path::Path, args: &[&str]) -> Vec { + let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + out.stdout +} + +#[test] +fn ffmpeg_files_decode_to_ffmpegs_samples() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("flac_decode"); + let tones = "aevalsrc=exprs=0.4*sin(2*PI*441*t)+0.2*sin(2*PI*3100*t)|0.3*sin(2*PI*660*t)*sin(2*PI*3*t):s=44100:d=1.3"; + let noise = "anoisesrc=d=1.3:c=white:r=44100:a=0.5:s=3"; + for (src, name) in [(tones, "tones"), (noise, "noise")] { + for channels in [1usize, 2, 6] { + for (fmt, bytes, bits) in [("s16", 2usize, 16u32), ("s32", 4, 24)] { + for level in [0, 3, 5, 8, 12] { + let file = dir.join(format!("ffmpeg_{name}_{channels}ch_{bits}_l{level}.flac")); + let f = file.to_str().expect("path"); + let (ch, lv) = (channels.to_string(), level.to_string()); + ffmpeg_run(&ff, &["-f", "lavfi", "-i", src, "-ac", &ch, "-sample_fmt", fmt, "-compression_level", &lv, "-c:a", "flac", f]); + let theirs = ffmpeg_run(&ff, &["-i", f, "-f", if bits == 16 { "s16le" } else { "s32le" }, "-"]); + let got = + decode(&std::fs::read(&file).expect("read")).unwrap_or_else(|e| panic!("{name} {channels}ch {bits}-bit level {level}: {e}")); + assert_eq!(got.info.channels as usize, channels); + let frames = got.channels[0].len(); + assert_eq!(theirs.len(), frames * channels * bytes, "{name} {channels}ch {bits} l{level}: length"); + // ffmpeg returns 24-bit audio left-aligned in 32 bits. + let shift = if bits == 24 { 8 } else { 0 }; + let mine: Vec = interleave(&got.channels).into_iter().map(|v| v << shift).collect(); + let ok = mine == pcm_values(&theirs, bytes); + assert!(ok, "{name} {channels}ch {bits}-bit level {level}: samples differ from ffmpeg's decode"); + } + } + } + } +} + +#[test] +fn ffmpeg_decodes_our_files_to_our_samples() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("flac_decode"); + for (channels, bits) in [(1usize, 16u8), (2, 16), (2, 24), (6, 16)] { + let src = signal(channels, 20_000, u32::from(bits), 99); + let file = dir.join(format!("ours_{channels}ch_{bits}.flac")); + std::fs::write(&file, encode(&src, 44_100, bits, 4096, 5)).expect("write"); + let theirs = ffmpeg_run(&ff, &["-i", file.to_str().expect("path"), "-f", if bits == 16 { "s16le" } else { "s32le" }, "-"]); + let got = decode(&std::fs::read(&file).expect("read")).expect("decode"); + let bytes = if bits == 16 { 2 } else { 4 }; + let shift = if bytes == 4 { 8 } else { 0 }; + let want: Vec = pcm_values(&theirs, bytes).into_iter().map(|v| v >> shift).collect(); + assert!(interleave(&got.channels) == want, "{channels}ch {bits}-bit: samples differ from ffmpeg's decode"); + } +} + +#[test] +fn rejects_unknown_length_corruption_and_streaminfo_format_mismatch() { + let src = signal(1, 100, 16, 7); + let original = encode(&src, 44_100, 16, 4096, 5); + let at = STREAMINFO_OFFSET as usize; + // STREAMINFO is not CRC-covered. Clear only the 36-bit total sample count. + let mut unknown = original.clone(); + unknown[at + 13] &= 0xf0; + unknown[at + 14..at + 18].fill(0); + assert_eq!(decode(&unknown).unwrap().channels, src); + for tail in [vec![0xff], vec![0x00, 0x00]] { + let mut broken = unknown.clone(); + broken.extend(tail); + assert!(decode(&broken).is_err()); + } + // The encoder spells these in frame headers, independent of STREAMINFO. + let mut wrong_rate = original.clone(); + let mut packed = u64::from_be_bytes(wrong_rate[at + 10..at + 18].try_into().unwrap()); + packed = (packed & ((1u64 << 44) - 1)) | (48_000u64 << 44); + wrong_rate[at + 10..at + 18].copy_from_slice(&packed.to_be_bytes()); + assert!(decode(&wrong_rate).is_err()); + let mut wrong_depth = original; + wrong_depth[at + 12] = (wrong_depth[at + 12] & !1) | 1; + wrong_depth[at + 13] = (wrong_depth[at + 13] & 0x0f) | 0x70; + assert!(decode(&wrong_depth).is_err()); +} + +#[test] +fn caller_constructed_streaminfo_is_validated_before_decoding() { + let file = encode(&signal(1, 100, 16, 9), 44_100, 16, 4096, 5); + let info = StreamInfo::parse(&file[STREAMINFO_OFFSET as usize..]).unwrap(); + for bits in [0, 1, 3, 33, 64, 255] { + let invalid = StreamInfo { bits_per_sample: bits, ..info }; + assert!(matches!(Decoder::new(invalid).decode_frame(&[]), Err(DecodeError::Invalid(_)))); + } +} diff --git a/crates/control-protocol/Cargo.toml b/crates/control-protocol/Cargo.toml new file mode 100644 index 000000000..f00bbc1c6 --- /dev/null +++ b/crates/control-protocol/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-control-protocol" +description = "Authenticated bounded JSON-lines control protocol" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } +subtle = "2" diff --git a/crates/control-protocol/src/lib.rs b/crates/control-protocol/src/lib.rs new file mode 100644 index 000000000..8bdb08d66 --- /dev/null +++ b/crates/control-protocol/src/lib.rs @@ -0,0 +1,234 @@ +//! Authenticated JSON-lines request framing and dispatch for local control services. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +use serde_json::{Value, json}; +use std::io::{BufRead, BufReader, Read, Write}; +use subtle::ConstantTimeEq; + +/// Whether a session token has the control protocol's 64 hexadecimal character form. +pub fn valid_token(token: &str) -> bool { + token.len() == 64 && token.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Encode one client request, including its terminating newline. +pub fn request_line(id: u64, token: Option<&str>, method: &str, params: &Value) -> String { + format!("{}\n", json!({"id": id, "token": token, "method": method, "params": params})) +} + +/// Extract the result or application error from a parsed control reply. +pub fn reply_result(reply: &Value) -> Result { + if reply.get("ok").and_then(Value::as_bool) == Some(true) { + Ok(reply.get("result").cloned().unwrap_or(Value::Null)) + } else { + Err(reply.get("error").and_then(Value::as_str).unwrap_or("error").to_owned()) + } +} + +/// One parsed line. +enum Line { + Request { + id: Value, + method: String, + params: Value, + }, + Blank, + /// Not a request: reply with this error and close the connection. + Reject(String), +} + +fn parse_line(bytes: &[u8], token: &str) -> Line { + let Ok(text) = std::str::from_utf8(bytes) else { + return Line::Reject("not a control request (invalid UTF-8); closing the connection".into()); + }; + let text = text.trim(); + if text.is_empty() { + return Line::Blank; + } + let msg = match serde_json::from_str::(text) { + Ok(v) => v, + Err(e) => return Line::Reject(format!("bad JSON ({e}); expected one JSON request object per line; closing the connection")), + }; + let Some(obj) = msg.as_object() else { + return Line::Reject("not a control request (expected a JSON object); closing the connection".into()); + }; + let Some(method) = obj.get("method").and_then(Value::as_str) else { + return Line::Reject("not a control request (missing string `method`); closing the connection".into()); + }; + // Authenticate every request before constructing or enqueueing a ControlRequest. A valid + // earlier request must not authorize a later one with a missing or incorrect credential. + let authenticated = obj.get("token").and_then(Value::as_str).is_some_and(|supplied| bool::from(supplied.as_bytes().ct_eq(token.as_bytes()))); + if !authenticated { + return Line::Reject("control authentication failed; closing the connection".into()); + } + Line::Request { + id: obj.get("id").cloned().unwrap_or(Value::Null), + method: method.to_string(), + params: obj.get("params").cloned().unwrap_or(json!({})), + } +} + +#[derive(Debug, PartialEq, Eq)] +pub enum ReadFailure { + Timeout, + Rejected(String), +} + +/// Read one `\n`-terminated line of at most `max` bytes into `buf` (newline stripped). +/// `Ok(false)` at end of stream; `Err` when the line is too long or the read fails. +pub fn read_bounded_line(reader: &mut impl BufRead, buf: &mut Vec, max: usize) -> Result { + buf.clear(); + // `take` bounds the bytes buffered for one line: a peer can't make us allocate more than `max`. + let limit = u64::try_from(max).unwrap_or(u64::MAX).saturating_add(1); + let n = reader.take(limit).read_until(b'\n', buf).map_err(|e| match e.kind() { + std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock => ReadFailure::Timeout, + _ => ReadFailure::Rejected(e.to_string()), + })?; + if n == 0 { + return Ok(false); + } + if buf.last() == Some(&b'\n') { + buf.pop(); + if buf.last() == Some(&b'\r') { + buf.pop(); + } + return Ok(true); + } + if buf.len() > max { + return Err(ReadFailure::Rejected(format!("request line longer than {max} bytes; closing the connection"))); + } + // Last line without a newline, then end of stream. + Ok(true) +} + +/// Serve requests from `read`, answering on `out`, until the peer closes, sends something that is +/// not a request, or `handle` returns `None` (app gone). `handle` gets `(method, params)` and +/// returns the reply object (the request's `id` is added here). +pub fn serve_lines(read: R, mut out: W, token: &str, max_line: usize, mut handle: impl FnMut(String, Value) -> Option) { + let mut reader = BufReader::new(read); + let mut buf = Vec::new(); + loop { + let reject = match read_bounded_line(&mut reader, &mut buf, max_line) { + Ok(false) => return, + // Expiration is not a request: a pooled client must see EOF and reconnect, rather + // than read a stale error reply. A partial line must not be parsed on timeout. + Err(ReadFailure::Timeout) => return, + Err(ReadFailure::Rejected(e)) => e, + Ok(true) => match parse_line(&buf, token) { + Line::Blank => continue, + Line::Reject(e) => e, + Line::Request { id, method, params } => { + let Some(mut r) = handle(method, params) else { return }; + if let Some(o) = r.as_object_mut() { + o.insert("id".into(), id); + } + if writeln!(out, "{r}").and_then(|()| out.flush()).is_err() { + return; + } + continue; + } + }, + }; + let _ = writeln!(out, "{}", json!({"ok": false, "error": reject})).and_then(|()| out.flush()); + return; + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn rejects_bad_request_shapes_without_dispatching() { + for input in [&b"\xff\n"[..], &b"[]\n"[..], &b"{}\n"[..]] { + let mut out = Vec::new(); + serve_lines(input, &mut out, "secret", 1024, |_, _| panic!("must not dispatch")); + let reply: Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(reply["ok"], false); + assert!(reply["error"].as_str().unwrap().contains("closing the connection")); + } + } + #[test] + fn timeout_after_partial_line_never_dispatches() { + struct Partial(bool); + impl Read for Partial { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.0 { + self.0 = true; + buf[..2].copy_from_slice(b"{\""); + return Ok(2); + } + Err(std::io::ErrorKind::TimedOut.into()) + } + } + let mut out = Vec::new(); + serve_lines(Partial(false), &mut out, "secret", 1024, |_, _| panic!("must not dispatch")); + assert!(out.is_empty()); + } + #[test] + fn line_boundaries_and_closed_dispatch_are_preserved() { + let mut buf = Vec::new(); + assert_eq!(read_bounded_line(&mut &b"abc\r\n"[..], &mut buf, 4), Ok(true)); + assert_eq!(buf, b"abc"); + assert_eq!(read_bounded_line(&mut &b"abcd\n"[..], &mut buf, 4), Ok(true)); + assert_eq!(buf, b"abcd"); + let mut out = Vec::new(); + serve_lines(&b"{\"method\":\"x\",\"token\":\"s\"}\n"[..], &mut out, "s", 1024, |_, _| None); + assert!(out.is_empty()); + } + #[test] + fn output_failure_stops_following_requests() { + struct Closed; + impl Write for Closed { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::ErrorKind::BrokenPipe.into()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let input = b"{\"method\":\"x\",\"token\":\"s\"}\n{\"method\":\"y\",\"token\":\"s\"}\n"; + let mut calls = 0; + serve_lines(&input[..], Closed, "s", 1024, |_, _| { + calls += 1; + Some(json!({"ok": true})) + }); + assert_eq!(calls, 1); + } + #[test] + fn client_request_and_reply_keep_optional_fields() { + let line = request_line(7, None, "status", &json!({"a": 1})); + assert_eq!(serde_json::from_str::(&line).unwrap(), json!({"id": 7, "token": null, "method": "status", "params": {"a": 1}})); + assert_eq!(reply_result(&json!({"ok": true})).unwrap(), Value::Null); + assert_eq!(reply_result(&json!({"ok": false, "error": "busy"})).unwrap_err(), "busy"); + } + #[test] + fn validates_session_token_shape() { + assert!(valid_token(&"aB09".repeat(16))); + assert!(!valid_token(&"a".repeat(63))); + assert!(!valid_token(&"g".repeat(64))); + } + #[test] + fn authenticates_each_request_and_closes_after_failure() { + let input = + b"{\"id\":1,\"method\":\"first\",\"token\":\"secret\"}\n{\"id\":2,\"method\":\"second\"}\n{\"method\":\"third\",\"token\":\"secret\"}\n"; + let mut output = Vec::new(); + let mut calls = Vec::new(); + serve_lines(&input[..], &mut output, "secret", 4096, |method, _| { + calls.push(method); + Some(json!({"ok": true})) + }); + assert_eq!(calls, ["first"]); + let lines: Vec = std::str::from_utf8(&output).unwrap().lines().map(|s| serde_json::from_str(s).unwrap()).collect(); + assert_eq!(lines[0], json!({"ok": true, "id": 1})); + assert_eq!(lines[1]["error"], "control authentication failed; closing the connection"); + } + #[test] + fn bounds_requests_and_preserves_last_unterminated_line() { + let mut buf = Vec::new(); + assert_eq!(read_bounded_line(&mut &b"abc"[..], &mut buf, 3), Ok(true)); + assert_eq!(buf, b"abc"); + assert!(matches!(read_bounded_line(&mut &b"abcd"[..], &mut buf, 3), Err(ReadFailure::Rejected(_)))); + let mut out = Vec::new(); + serve_lines(&b"{broken}\n"[..], &mut out, "secret", 1024, |_, _| panic!("must not dispatch")); + let reply: Value = serde_json::from_slice(&out).unwrap(); + assert!(reply["error"].as_str().unwrap().starts_with("bad JSON (")); + } +} diff --git a/crates/docx/src/chart_spec.rs b/crates/docx/src/chart_spec.rs index e31363903..a090c5c96 100644 --- a/crates/docx/src/chart_spec.rs +++ b/crates/docx/src/chart_spec.rs @@ -6,7 +6,7 @@ //! [`spec_of`] turns a chart part back into the model only when writing that model again gives //! the same markup, so a chart with anything the model leaves out stays as it was read. -use craft_wordmodel::chart::{ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES}; +use craft_wordmodel::chart::{ChartAxis, ChartSeries, ChartSpec, ChartType, LegendPos, MAX_POINTS, MAX_SERIES}; use craft_wordmodel::graphic::GraphicItem; use craft_wordmodel::props::Rgb; @@ -102,11 +102,24 @@ fn bounded(spec: &ChartSpec) -> ChartSpec { .series .iter() .take(MAX_SERIES) - .map(|s| ChartSeries { name: s.name.clone(), values: s.values.iter().take(MAX_POINTS).copied().collect() }) + .map(|s| ChartSeries { + name: s.name.clone(), + values: s.values.iter().take(MAX_POINTS).copied().collect(), + fill: s.fill, + no_fill: s.no_fill, + line_w: s.line_w, + no_line: s.no_line, + markers: s.markers, + smooth: s.smooth, + labels: s.labels, + fmt: s.fmt.clone(), + }) .collect(), title: spec.title.clone(), legend: spec.legend, data_labels: spec.data_labels, + cat_axis: spec.cat_axis.clone(), + val_axis: spec.val_axis.clone(), } .sanitized() } @@ -185,17 +198,19 @@ fn plot(w: &mut W, spec: &ChartSpec) { match k { ChartType::Pie | ChartType::Doughnut => {} ChartType::Scatter => { - value_axis(w, CAT_AX, VAL_AX, "b", false, "midCat"); - value_axis(w, VAL_AX, CAT_AX, "l", true, "midCat"); + value_axis(w, CAT_AX, VAL_AX, "b", "midCat", &spec.cat_axis); + value_axis(w, VAL_AX, CAT_AX, "l", "midCat", &spec.val_axis); } _ => { - category_axis(w, if horizontal { "l" } else { "b" }); - value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, true, "between"); + category_axis(w, if horizontal { "l" } else { "b" }, &spec.cat_axis); + value_axis(w, VAL_AX, CAT_AX, if horizontal { "b" } else { "l" }, "between", &spec.val_axis); } } } -/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, cat/xVal, val/yVal, smooth). +/// One `c:ser` (ECMA-376 element order: idx, order, tx, spPr, marker, invertIfNegative, dLbls, +/// cat/xVal, val/yVal, smooth): the series' formatting (fill, line, markers, its own labels) sits +/// in `c:spPr`, `c:marker` and `c:dLbls`; the values carry their number format. fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) { let k = spec.kind; let i = i.to_string(); @@ -207,25 +222,26 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) { w.leaf("c:v", &[], &s.name); w.close("c:tx"); } - match k { - // Markers only: the series has no line. - ChartType::Scatter => { - w.open("c:spPr", &[]); - w.open("a:ln", &[("w", "19050")]); - w.empty("a:noFill", &[]); - w.close("a:ln"); - w.close("c:spPr"); - } - ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked => { - w.empty("c:invertIfNegative", &[("val", "0")]); - } - ChartType::Line => { - w.open("c:marker", &[]); - w.empty("c:symbol", &[("val", "none")]); - w.close("c:marker"); - } - _ => {} + series_props(w, s); + series_marker(w, k, s); + if matches!(k, ChartType::Column | ChartType::ColumnStacked | ChartType::Bar | ChartType::BarStacked) { + w.empty("c:invertIfNegative", &[("val", "0")]); } + if let Some(on) = s.labels { + w.open("c:dLbls", &[]); + for (name, show) in [ + ("c:showLegendKey", false), + ("c:showVal", on), + ("c:showCatName", false), + ("c:showSerName", false), + ("c:showPercent", false), + ("c:showBubbleSize", false), + ] { + w.empty(name, &[("val", if show { "1" } else { "0" })]); + } + w.close("c:dLbls"); + } + let code = s.fmt.as_deref().unwrap_or("General"); if k == ChartType::Scatter { let xs: Option>> = spec .categories @@ -234,27 +250,85 @@ fn series(w: &mut W, spec: &ChartSpec, i: usize, s: &ChartSeries) { .collect(); w.open("c:xVal", &[]); match xs { - Some(xs) => num_lit(w, &xs), + Some(xs) => num_lit(w, &xs, "General"), None => str_lit(w, &spec.categories), } w.close("c:xVal"); w.open("c:yVal", &[]); - num_lit(w, &s.values); + num_lit(w, &s.values, code); w.close("c:yVal"); } else { w.open("c:cat", &[]); str_lit(w, &spec.categories); w.close("c:cat"); w.open("c:val", &[]); - num_lit(w, &s.values); + num_lit(w, &s.values, code); w.close("c:val"); } if matches!(k, ChartType::Line | ChartType::Scatter) { - w.empty("c:smooth", &[("val", "0")]); + w.empty("c:smooth", &[("val", if s.smooth { "1" } else { "0" })]); } w.close("c:ser"); } +/// `c:spPr`: the series' fill (`a:solidFill` / `a:noFill`) and line (`a:ln`), written when the +/// model sets them. `a:ln` without `w` is the default width, `w` a fixed one, and `a:noFill` +/// inside it "no line" (a scatter series keeps its markers with no line). +fn series_props(w: &mut W, s: &ChartSeries) { + let fill_set = s.no_fill || s.fill.is_some(); + let ln_set = s.no_line || s.line_w.is_some(); + if !(fill_set || ln_set) { + return; + } + w.open("c:spPr", &[]); + if fill_set { + // "No fill" wins over a stored colour, so the pair reads back as it was written. + match &s.fill { + Some(c) if !s.no_fill => { + w.open("a:solidFill", &[]); + w.empty("a:srgbClr", &[("val", &c.hex())]); + w.close("a:solidFill"); + } + _ => w.empty("a:noFill", &[]), + } + } + if ln_set { + let w_attr = s.line_w.map(emu).unwrap_or_default(); + let attrs = [("w", w_attr.as_str())]; + w.open("a:ln", if w_attr.is_empty() { &[] } else { &attrs }); + if s.no_line { + w.empty("a:noFill", &[]); + } + w.close("a:ln"); + } + w.close("c:spPr"); +} + +/// `c:marker`: line and scatter series say which symbol each point gets; other types have none. +fn series_marker(w: &mut W, k: ChartType, s: &ChartSeries) { + match k { + ChartType::Line => { + w.open("c:marker", &[]); + w.empty("c:symbol", &[("val", if s.markers == Some(true) { "circle" } else { "none" })]); + w.close("c:marker"); + } + // Unset: no `c:marker`, so Word picks its automatic symbol for each series. + ChartType::Scatter => { + if let Some(on) = s.markers { + w.open("c:marker", &[]); + w.empty("c:symbol", &[("val", if on { "circle" } else { "none" })]); + w.close("c:marker"); + } + } + _ => {} + } +} + +/// A line width in points as the `w` attribute DrawingML wants (EMU). +fn emu(pt: f32) -> String { + ((pt * 12700.0).round().max(0.0) as i64).to_string() +} + fn str_lit(w: &mut W, items: &[String]) { w.open("c:strLit", &[]); w.empty("c:ptCount", &[("val", &items.len().to_string())]); @@ -266,9 +340,9 @@ fn str_lit(w: &mut W, items: &[String]) { w.close("c:strLit"); } -fn num_lit(w: &mut W, values: &[Option]) { +fn num_lit(w: &mut W, values: &[Option], code: &str) { w.open("c:numLit", &[]); - w.leaf("c:formatCode", &[], "General"); + w.leaf("c:formatCode", &[], code); w.empty("c:ptCount", &[("val", &values.len().to_string())]); for (i, v) in values.iter().enumerate() { if let Some(v) = v.filter(|v| v.is_finite()) { @@ -285,17 +359,21 @@ fn num(v: f64) -> String { if v == 0.0 { "0".to_string() } else { format!("{v}") } } -fn category_axis(w: &mut W, pos: &str) { +fn category_axis(w: &mut W, pos: &str, ax: &ChartAxis) { w.open("c:catAx", &[]); w.empty("c:axId", &[("val", CAT_AX)]); - w.open("c:scaling", &[]); - w.empty("c:orientation", &[("val", "minMax")]); - w.close("c:scaling"); + scaling(w, ax); w.empty("c:delete", &[("val", "0")]); w.empty("c:axPos", &[("val", pos)]); + if ax.gridlines { + w.empty("c:majorGridlines", &[]); + } + axis_title(w, ax); + num_fmt(w, ax); w.empty("c:majorTickMark", &[("val", "none")]); w.empty("c:minorTickMark", &[("val", "none")]); - w.empty("c:tickLblPos", &[("val", "nextTo")]); + w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]); + axis_line(w, ax); w.empty("c:crossAx", &[("val", VAL_AX)]); w.empty("c:crosses", &[("val", "autoZero")]); w.empty("c:auto", &[("val", "1")]); @@ -305,34 +383,81 @@ fn category_axis(w: &mut W, pos: &str) { w.close("c:catAx"); } -/// A value axis: with gridlines and no axis line (`grid`), else with its line and no gridlines. -fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, grid: bool, between: &str) { +/// A value axis (scatter x axes included): its scale, gridlines, title, labels and line, as the +/// model sets them. +fn value_axis(w: &mut W, id: &str, cross: &str, pos: &str, between: &str, ax: &ChartAxis) { w.open("c:valAx", &[]); w.empty("c:axId", &[("val", id)]); - w.open("c:scaling", &[]); - w.empty("c:orientation", &[("val", "minMax")]); - w.close("c:scaling"); + scaling(w, ax); w.empty("c:delete", &[("val", "0")]); w.empty("c:axPos", &[("val", pos)]); - if grid { + if ax.gridlines { w.empty("c:majorGridlines", &[]); } + axis_title(w, ax); + num_fmt(w, ax); w.empty("c:majorTickMark", &[("val", "none")]); w.empty("c:minorTickMark", &[("val", "none")]); - w.empty("c:tickLblPos", &[("val", "nextTo")]); - if grid { - w.open("c:spPr", &[]); - w.open("a:ln", &[]); - w.empty("a:noFill", &[]); - w.close("a:ln"); - w.close("c:spPr"); - } + w.empty("c:tickLblPos", &[("val", if ax.labels { "nextTo" } else { "none" })]); + axis_line(w, ax); w.empty("c:crossAx", &[("val", cross)]); w.empty("c:crosses", &[("val", "autoZero")]); w.empty("c:crossBetween", &[("val", between)]); w.close("c:valAx"); } +/// `c:scaling`: the orientation, then the fixed bounds when the model sets them. +fn scaling(w: &mut W, ax: &ChartAxis) { + w.open("c:scaling", &[]); + w.empty("c:orientation", &[("val", "minMax")]); + for (name, v) in [("c:max", ax.max), ("c:min", ax.min)] { + if let Some(v) = v.filter(|v| v.is_finite()) { + w.empty(name, &[("val", &num(v))]); + } + } + w.close("c:scaling"); +} + +/// The axis title, in the same rich-text shape the chart title uses. +fn axis_title(w: &mut W, ax: &ChartAxis) { + let Some(t) = &ax.title else { return }; + w.open("c:title", &[]); + w.open("c:tx", &[]); + w.open("c:rich", &[]); + w.empty("a:bodyPr", &[]); + w.empty("a:lstStyle", &[]); + w.open("a:p", &[]); + w.open("a:r", &[]); + w.leaf("a:t", &[], t); + w.close("a:r"); + w.close("a:p"); + w.close("c:rich"); + w.close("c:tx"); + w.empty("c:overlay", &[("val", "0")]); + w.close("c:title"); +} + +/// Tick-label number format: the model's code, unlinked from the source, else the default. +fn num_fmt(w: &mut W, ax: &ChartAxis) { + match &ax.fmt { + Some(code) => w.empty("c:numFmt", &[("formatCode", code), ("sourceLinked", "0")]), + None => w.empty("c:numFmt", &[("formatCode", "General"), ("sourceLinked", "1")]), + } +} + +/// `c:spPr`: an axis line only when the model keeps it (the default writes nothing, which reads +/// as a line); `a:noFill` when the axis line is off. +fn axis_line(w: &mut W, ax: &ChartAxis) { + if ax.line { + return; + } + w.open("c:spPr", &[]); + w.open("a:ln", &[]); + w.empty("a:noFill", &[]); + w.close("a:ln"); + w.close("c:spPr"); +} + /// The chart a chart part holds, when the part is exactly what [`chart_xml`] writes for it (so /// editing and saving it loses nothing). `None` for any other chart. pub(crate) fn spec_of(space: &El) -> Option { @@ -365,15 +490,40 @@ pub(crate) fn spec_of(space: &El) -> Option { let categories = sers.first().and_then(|s| s.child(if scatter { "c:xVal" } else { "c:cat" })).map(literal).unwrap_or_default(); let series = sers .iter() - .map(|s| ChartSeries { - name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(), - values: s - .child(if scatter { "c:yVal" } else { "c:val" }) - .map(literal) - .unwrap_or_default() - .iter() - .map(|v| v.parse::().ok().filter(|v| v.is_finite())) - .collect(), + .map(|s| { + let ln = s.child("c:spPr").and_then(|p| p.child("a:ln")); + let symbol = s.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|m| m.attr("val")); + ChartSeries { + name: s.child("c:tx").and_then(|t| t.child("c:v")).map(|v| v.text()).unwrap_or_default(), + values: s + .child(if scatter { "c:yVal" } else { "c:val" }) + .map(literal) + .unwrap_or_default() + .iter() + .map(|v| v.parse::().ok().filter(|v| v.is_finite())) + .collect(), + fill: s + .child("c:spPr") + .and_then(|p| p.child("a:solidFill")) + .and_then(|f| f.child("a:srgbClr")) + .and_then(|c| c.attr("val")) + .and_then(Rgb::parse), + no_fill: s.child("c:spPr").and_then(|p| p.child("a:noFill")).is_some(), + line_w: ln.and_then(|l| l.attr("w")).and_then(|w| w.parse::().ok()).filter(|w| w.is_finite()).map(|w| (w / 12700.0) as f32), + no_line: ln.and_then(|l| l.child("a:noFill")).is_some(), + // Line series show no markers by default: only a setting that differs from that + // belongs in the model. A scatter series keeps what the file says, so a missing + // `c:marker` (Word's automatic symbols) stays unset and is not written back. + markers: symbol.map(|v| v != "none").filter(|m| scatter || *m), + smooth: s.child("c:smooth").and_then(|v| v.attr("val")).is_some_and(|v| v == "1"), + labels: s.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")).map(|v| v == "1"), + fmt: s + .child(if scatter { "c:yVal" } else { "c:val" }) + .and_then(|v| v.els().find(|e| e.name == "c:numLit")) + .and_then(|l| l.child("c:formatCode")) + .map(|f| f.text()) + .filter(|f| *f != "General"), + } }) .collect(); let title = chart.child("c:title").and_then(|t| t.find("a:t")).map(|t| t.text()); @@ -384,12 +534,48 @@ pub(crate) fn spec_of(space: &El) -> Option { _ => LegendPos::Right, }); let data_labels = ty.child("c:dLbls").and_then(|d| d.child("c:showVal")).and_then(|v| v.attr("val")) == Some("1"); - let spec = ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized(); - // Editable only when nothing would be lost: writing the model again gives the same part. - let again = xml::parse(chart_xml(&spec).as_bytes()).ok()?; + let (cat_a, val_a) = crate::read::chart::chart_axes(plot, scatter); + let axis = |a: crate::read::chart::Axis| ChartAxis { + title: a.title, + min: a.min, + max: a.max, + fmt: a.fmt, + gridlines: a.grid, + labels: a.labels, + line: a.line, + }; + let spec = ChartSpec { kind, categories, series, title, legend, data_labels, cat_axis: axis(cat_a), val_axis: axis(val_a) }.sanitized(); + // Editable only when nothing would be lost: writing the model again gives the same part, as + // this writer or the earlier one wrote it. + let mut again = xml::parse(chart_xml(&spec).as_bytes()).ok()?; + if same(space, &again, 0) { + return Some(spec); + } + as_previous_writer(&mut again); same(space, &again, 0).then_some(spec) } +/// `part` (what [`chart_xml`] writes) as the earlier WordCraft writer wrote the same model: it +/// had no axis `c:numFmt`. Only that element goes, and only with the value its absence reads as +/// (General, linked to the source), so a part that matches still holds nothing the model leaves out. +fn as_previous_writer(part: &mut El) { + fn els_mut(e: &mut El) -> impl Iterator { + e.kids.iter_mut().filter_map(|n| match n { + Node::El(e) => Some(e), + Node::Text(_) => None, + }) + } + fn drop_els(e: &mut El, drop: impl Fn(&El) -> bool) { + e.kids.retain(|n| !matches!(n, Node::El(c) if drop(c))); + } + let Some(plot) = els_mut(part).find(|e| e.name == "c:chart").and_then(|c| els_mut(c).find(|e| e.name == "c:plotArea")) else { return }; + for e in els_mut(plot) { + if matches!(e.name.as_str(), "c:catAx" | "c:valAx") { + drop_els(e, |c| c.name == "c:numFmt" && c.attr("sourceLinked") == Some("1")); + } + } +} + /// The points of a `c:strLit` / `c:numLit` (or a cache) as texts by index, `ptCount` long. fn literal(e: &El) -> Vec { let Some(lit) = e.els().find(|c| matches!(c.name.as_str(), "c:strLit" | "c:numLit")) else { return Vec::new() }; @@ -461,10 +647,11 @@ mod tests { let spec = ChartSpec { kind: ChartType::Column, categories: (0..4500).map(|i| i.to_string()).collect(), - series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500] }).collect(), + series: (0..300).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 4500], ..Default::default() }).collect(), title: None, legend: None, data_labels: false, + ..Default::default() }; let x = chart_xml(&spec); let series = x.matches("").count(); @@ -491,6 +678,33 @@ mod tests { assert_eq!(spec_of(&parse(&plain)), Some(plain)); } + #[test] + fn no_fill_wins_over_a_stored_colour() { + let mut spec = ChartSpec::sample(ChartType::Column); + spec.series[0].fill = Some(Rgb(255, 0, 0)); + spec.series[0].no_fill = true; + let x = chart_xml(&spec); + assert!(!x.contains("FF0000"), "colour written next to no_fill"); + let back = spec_of(&parse(&spec)).expect("stays editable"); + assert!(back.series[0].no_fill); + assert_eq!(back.series[0].fill, None); + } + + #[test] + fn an_axis_format_is_limited_in_characters_not_bytes() { + use craft_wordmodel::chart::MAX_FMT; + let cjk = format!("0\"{}\"", "\u{5143}".repeat(90)); + let ascii = format!("0\"{}\"", "x".repeat(90)); + for fmt in [cjk, ascii] { + assert!(fmt.chars().count() <= MAX_FMT); + let mut spec = ChartSpec::sample(ChartType::Column); + spec.val_axis.fmt = Some(fmt.clone()); + let spec = spec.sanitized(); + assert_eq!(spec.val_axis.fmt.as_deref(), Some(fmt.as_str())); + assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{} chars", fmt.chars().count()); + } + } + #[test] fn a_chart_with_more_than_the_model_stays_read_only() { let spec = ChartSpec::sample(ChartType::Column); @@ -500,6 +714,61 @@ mod tests { assert_eq!(spec_of(&xml::parse(x.as_bytes()).unwrap()), None); } + /// Chart parts as the previous WordCraft chart writer saved them (its output for + /// `ChartSpec::sample`): no axis `c:numFmt`, and scatter series with no `c:marker`. + const PREVIOUS_COLUMN: &str = r#"Series 1Q1Q2Q3Q4General3.24.13.65Series 2Q1Q2Q3Q4General22.73.93.3Series 3Q1Q2Q3Q4General1.52.22.43.8"#; + const PREVIOUS_SCATTER: &str = r#"Series 1General1234General2.13.434.6Series 2General1234General1.21.92.83.1"#; + + #[test] + fn charts_saved_by_the_previous_writer_stay_editable() { + for (x, kind) in [(PREVIOUS_COLUMN, ChartType::Column), (PREVIOUS_SCATTER, ChartType::Scatter)] { + let spec = spec_of(&xml::parse(x.as_bytes()).unwrap()).unwrap_or_else(|| panic!("{kind:?}: the earlier part is read-only")); + assert_eq!((spec.kind, spec.series.len()), (kind, if kind == ChartType::Scatter { 2 } else { 3 })); + assert_eq!((spec.cat_axis.fmt.as_deref(), spec.val_axis.fmt.as_deref()), (None, None), "{kind:?}"); + assert!(spec.series.iter().all(|s| s.markers.is_none()), "{kind:?}: markers unset"); + // Saved again, it is in the current shape and still reads back as the same model. + assert_eq!(spec_of(&parse(&spec)).as_ref(), Some(&spec), "{kind:?}"); + } + // What neither writer writes still keeps such a part read-only: an axis format linked to + // the source that isn't General, a scatter marker other than the default, a marker size. + let edited = |x: &str, from: &str, to: &str| { + assert!(x.contains(from), "{from}"); + spec_of(&xml::parse(x.replacen(from, to, 1).as_bytes()).unwrap()) + }; + let ax = r#""#; + assert_eq!(edited(PREVIOUS_COLUMN, ax, &format!(r#"{ax}"#)), None); + let ln = r#""#; + assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}"#)), None); + assert_eq!(edited(PREVIOUS_SCATTER, ln, &format!(r#"{ln}"#)), None); + assert_eq!(edited(PREVIOUS_COLUMN, "", ""), None); + } + + /// A scatter series with no `c:marker` (Word's automatic symbols) stays without one after an + /// unrelated edit and a save; an explicit circle or "none" stays explicit. + #[test] + fn scatter_series_without_a_marker_stay_without_after_a_save() { + let marker_of = |x: &El| -> Vec> { + let sc = x.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap(); + sc.children("c:ser").map(|s| s.child("c:marker").map(|m| m.child("c:symbol").unwrap().attr("val").unwrap().to_string())).collect() + }; + let mut spec = spec_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()).unwrap(); + assert_eq!(marker_of(&xml::parse(PREVIOUS_SCATTER.as_bytes()).unwrap()), [None, None]); + spec.title = Some("Edited".into()); + spec.series[0].name = "Renamed".into(); + let saved = parse(&spec); + assert_eq!(marker_of(&saved), [None, None]); + let again = spec_of(&saved).unwrap(); + assert_eq!(again, spec); + assert!(again.series.iter().all(|s| s.markers.is_none())); + // Explicit choices are kept as written. + spec.series[0].markers = Some(true); + spec.series[1].markers = Some(false); + let saved = parse(&spec); + assert_eq!(marker_of(&saved), [Some("circle".to_string()), Some("none".to_string())]); + let again = spec_of(&saved).unwrap(); + assert_eq!((again.series[0].markers, again.series[1].markers), (Some(true), Some(false))); + } + #[test] fn elements_come_in_schema_order() { let mut spec = ChartSpec::sample(ChartType::ColumnStacked); @@ -517,6 +786,11 @@ mod tests { ["c:barDir", "c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:gapWidth", "c:overlap", "c:axId", "c:axId"] ); assert_eq!(names(bar.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:cat", "c:val"]); + // A series with its own labels keeps `c:invertIfNegative` ahead of `c:dLbls` (CT_BarSer). + spec.series[0].labels = Some(true); + let labelled = parse(&spec); + let ser = labelled.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:barChart").unwrap().child("c:ser").unwrap(); + assert_eq!(names(ser), ["c:idx", "c:order", "c:tx", "c:invertIfNegative", "c:dLbls", "c:cat", "c:val"]); let val_ax = plot.child("c:valAx").unwrap(); assert_eq!( names(val_ax), @@ -526,6 +800,7 @@ mod tests { "c:delete", "c:axPos", "c:majorGridlines", + "c:numFmt", "c:majorTickMark", "c:minorTickMark", "c:tickLblPos", @@ -535,13 +810,35 @@ mod tests { "c:crossBetween" ] ); + let cat_ax = plot.child("c:catAx").unwrap(); + assert_eq!( + names(cat_ax), + [ + "c:axId", + "c:scaling", + "c:delete", + "c:axPos", + "c:numFmt", + "c:majorTickMark", + "c:minorTickMark", + "c:tickLblPos", + "c:crossAx", + "c:crosses", + "c:auto", + "c:lblAlgn", + "c:lblOffset", + "c:noMultiLvlLbl" + ] + ); let line = parse(&ChartSpec { data_labels: true, ..ChartSpec::sample(ChartType::Line) }); let lc = line.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:lineChart").unwrap(); assert_eq!(names(lc), ["c:grouping", "c:varyColors", "c:ser", "c:ser", "c:ser", "c:dLbls", "c:marker", "c:axId", "c:axId"]); assert_eq!(names(lc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:marker", "c:cat", "c:val", "c:smooth"]); - let sc = parse(&ChartSpec::sample(ChartType::Scatter)); + let mut scatter = ChartSpec::sample(ChartType::Scatter); + scatter.series.iter_mut().for_each(|s| s.markers = Some(true)); + let sc = parse(&scatter); let sc = sc.child("c:chart").unwrap().child("c:plotArea").unwrap().child("c:scatterChart").unwrap(); - assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:xVal", "c:yVal", "c:smooth"]); + assert_eq!(names(sc.child("c:ser").unwrap()), ["c:idx", "c:order", "c:tx", "c:spPr", "c:marker", "c:xVal", "c:yVal", "c:smooth"]); let dn = parse(&ChartSpec::sample(ChartType::Doughnut)); let dn = dn.child("c:chart").unwrap().child("c:plotArea").unwrap(); assert_eq!(names(dn), ["c:layout", "c:doughnutChart"]); diff --git a/crates/docx/src/read/chart.rs b/crates/docx/src/read/chart.rs index a0087a3bb..5b02603d6 100644 --- a/crates/docx/src/read/chart.rs +++ b/crates/docx/src/read/chart.rs @@ -29,7 +29,9 @@ pub(super) use canvas::text_w; use canvas::{Rect, clip_text, legend, text}; #[cfg(test)] use format::{civil, excel_stamp, format_value}; -use model::{Axis, Grouping, Kind, Pos, Series}; +pub(crate) use model::Axis; +use model::{Grouping, Kind, Pos, Series}; +pub(crate) use parse::chart_axes; #[cfg(test)] use parse::{axis, category_text, labels, numbers}; use parse::{read_series, text_size, title_of}; diff --git a/crates/docx/src/read/chart/canvas.rs b/crates/docx/src/read/chart/canvas.rs index a8280e4ba..ab3fbf410 100644 --- a/crates/docx/src/read/chart/canvas.rs +++ b/crates/docx/src/read/chart/canvas.rs @@ -41,8 +41,13 @@ pub(super) fn widest(items: &[String], size: f32) -> f32 { items.iter().map(|s| text_w(s, size)).fold(0.0, f32::max) } +/// Text in `rect`; `angle` degrees clockwise turns it about the rect's centre (axis titles). pub(super) fn text(out: &mut Vec, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool) { - out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None }); + text_angle(out, rect, s, size, align, bold, 0.0); +} + +pub(super) fn text_angle(out: &mut Vec, rect: Rect, s: &str, size: f32, align: TextAlign, bold: bool, angle: f32) { + out.push(GraphicItem::Text { rect: rect.arr(), text: s.to_string(), size, color: TEXT, bold, align, font: None, angle }); } /// Text centred on `cx` with its top at `y`. diff --git a/crates/docx/src/read/chart/model.rs b/crates/docx/src/read/chart/model.rs index 71ee7040f..13da1d654 100644 --- a/crates/docx/src/read/chart/model.rs +++ b/crates/docx/src/read/chart/model.rs @@ -7,6 +7,33 @@ pub(super) struct DataLabel { pub(super) y: f32, pub(super) text: String, pub(super) inside: bool, + /// Where a label that is not `inside` sits relative to (`x`, `y`). + pub(super) pos: LabelPos, + /// Distance from (`x`, `y`) to the label for `pos` other than `Center`, points (half the marker). + pub(super) off: f32, +} + +/// Where a data label sits beside its point (`c:dLblPos`: `t`, `b`, `l`, `r`, `ctr`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum LabelPos { + Above, + Below, + Left, + Right, + Center, +} + +impl LabelPos { + pub(super) fn parse(v: &str) -> Option { + Some(match v.trim() { + "t" => LabelPos::Above, + "b" => LabelPos::Below, + "l" => LabelPos::Left, + "r" => LabelPos::Right, + "ctr" => LabelPos::Center, + _ => return None, + }) + } } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -52,16 +79,56 @@ pub(super) enum Pos { /// One value axis (or category axis) as the file states it. #[derive(Default)] -pub(super) struct Axis { - pub(super) deleted: bool, - pub(super) grid: bool, - pub(super) line: bool, +pub(crate) struct Axis { + pub(crate) deleted: bool, + pub(crate) grid: bool, + pub(crate) line: bool, /// `c:crossBetween val="midCat"`: categories sit on the tick marks, not between them. pub(super) mid: bool, - pub(super) min: Option, - pub(super) max: Option, + pub(crate) min: Option, + pub(crate) max: Option, /// Number format of the tick labels, when not linked to the source data. - pub(super) fmt: Option, + pub(crate) fmt: Option, + /// The axis title, when the file gives it one. + pub(crate) title: Option, + /// The title's `a:bodyPr/@rot` in degrees clockwise (`-5400000` is 90° counter-clockwise), + /// when the file gives one. + pub(crate) title_rot: Option, + /// Tick labels, as `c:tickLblPos` says (`none` hides them; a file that says nothing shows + /// them, as Word does). + pub(crate) labels: bool, +} + +/// The shape of a series' point markers (`ST_MarkerStyle`, ECMA-376 §21.2.3.28). +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub(super) enum Symbol { + #[default] + Circle, + Square, + Diamond, + Triangle, + X, + Plus, + Star, + Dash, + Dot, +} + +impl Symbol { + /// The symbol a `c:symbol` value names; anything unknown (`auto`, `picture`) is a circle. + pub(super) fn parse(name: &str) -> Symbol { + match name { + "square" => Symbol::Square, + "diamond" => Symbol::Diamond, + "triangle" => Symbol::Triangle, + "x" => Symbol::X, + "plus" => Symbol::Plus, + "star" => Symbol::Star, + "dash" => Symbol::Dash, + "dot" => Symbol::Dot, + _ => Symbol::Circle, + } + } } pub(super) struct Series { @@ -70,9 +137,21 @@ pub(super) struct Series { /// when missing or not a valid number. pub(super) idx: Option, pub(super) color: Option, - pub(super) line_w: f32, + /// Fixed line width in points, `None`: the chart's default. + pub(super) line_w: Option, /// Whether the series is drawn as a line (scatter series without one get markers). pub(super) line: bool, + /// The series' own fill: `Some` colour, `None` colour with `no_fill` set, else the chart's + /// accent for the series. + pub(super) no_fill: bool, + /// Markers on the series' points, when the file says. + pub(super) markers: Option, + /// The marker shape (`c:marker/c:symbol`; a circle when missing or unknown) and its size in + /// points (`c:marker/c:size`), when the file gives one. + pub(super) symbol: Symbol, + pub(super) marker_size: Option, + /// Draw the series' line as a smooth curve (`c:smooth`). + pub(super) smooth: bool, pub(super) values: Vec>, /// Scatter x values (index + 1 when the file has none). pub(super) xs: Vec>, @@ -83,6 +162,13 @@ pub(super) struct Series { pub(super) x_fmt: Option, /// Data labels on or off for this series (`c:dLbls` of its own), when it says. pub(super) labels: Option, + /// The labels' number format (`c:dLbls/c:numFmt`): `None` when not given (the chart type's + /// applies), `Some(None)` when linked to the source, else `Some(Some(code))`. + pub(super) label_fmt: Option>, + /// The data labels' `c:dLblPos`, when the file gives a known one. + pub(super) label_pos: Option, + /// The marker's own fill (`c:marker/c:spPr`); without it markers take the automatic colour. + pub(super) marker_color: Option, /// The values as the file has them, for data labels (`values` become shares in a 100% /// stacked chart); empty when the series shows no labels. pub(super) raw: Vec>, diff --git a/crates/docx/src/read/chart/parse.rs b/crates/docx/src/read/chart/parse.rs index 3c09fcfe9..522d50b3b 100644 --- a/crates/docx/src/read/chart/parse.rs +++ b/crates/docx/src/read/chart/parse.rs @@ -1,7 +1,7 @@ use super::format::{date_text, excel_stamp, format_value, is_date_code}; -use super::model::{Axis, Grouping, Kind, Pos, Series}; +use super::model::{Axis, Grouping, Kind, LabelPos, Pos, Series, Symbol}; use super::plot::label; -use super::{Chart, El, FONT, LINE_W, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot}; +use super::{Chart, El, FONT, MAX_CELLS, MAX_POINTS, MAX_RADAR_SERIES, MAX_SERIES, MAX_SPOKES, Rgb, fill_colour, int, measure, slot}; impl Chart { pub(super) fn parse(space: &El, theme: &[Rgb]) -> Option { @@ -39,10 +39,15 @@ impl Chart { cats.resize(n, String::new()); let radar_style = ty.child("c:radarStyle").and_then(|r| r.attr("val")); let type_labels = show_values(ty.child("c:dLbls")); + let type_fmt = label_format(ty.child("c:dLbls")); for s in &mut series { s.values.resize(n, None); s.xs.resize(n, None); s.labels = Some(s.labels.or(type_labels).unwrap_or(false)); + // Absent (`None`) inherits the chart type's; explicitly source-linked (`Some(None)`) does not. + if s.label_fmt.is_none() { + s.label_fmt.clone_from(&type_fmt); + } if s.labels == Some(true) { s.raw = s.values.clone(); } @@ -98,6 +103,13 @@ impl Chart { s.and_then(|s| s.color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j))) } + /// Colour of series `j`'s markers: the marker's own fill, else the automatic colour of the + /// series index (not the series' explicit line colour, as Word draws them). + pub(super) fn marker_colour(&self, j: usize) -> Rgb { + let s = self.series.get(j); + s.and_then(|s| s.marker_color).unwrap_or_else(|| self.accent(s.and_then(|s| s.idx).unwrap_or(j))) + } + /// Colour of point `i` of series `j`: its `c:dPt` override, else the varied accent (pie) or /// the series colour. pub(super) fn point_colour(&self, j: usize, i: usize) -> Rgb { @@ -149,7 +161,7 @@ fn to_percent(series: &mut [Series], n: usize) { } /// The category axis (scatter: the horizontal value axis) and the value axis of a plot area. -fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) { +pub(crate) fn chart_axes(plot: &El, scatter: bool) -> (Axis, Axis) { let vals_ax: Vec<&El> = plot.children("c:valAx").collect(); let horiz_pos = |a: &&El| matches!(a.child("c:axPos").and_then(|p| p.attr("val")), Some("b" | "t")); let (val_el, x_el) = if scatter { @@ -180,6 +192,17 @@ fn show_values(d: Option<&El>) -> Option { d.child("c:showVal").map(|v| v.attr("val").is_none_or(truthy)) } +/// The number format of data labels (`c:dLbls/c:numFmt`): `None` when the labels say nothing, +/// `Some(None)` when they are linked to the source (so a chart-level format does not apply), +/// `Some(Some(code))` for their own format code. +fn label_format(d: Option<&El>) -> Option> { + let f = d?.child("c:numFmt")?; + if f.attr("sourceLinked").is_some_and(truthy) { + return Some(None); + } + f.attr("formatCode").filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT).map(|f| Some(f.to_string())) +} + fn legend_pos(chart: &El) -> Option { chart.child("c:legend").map(|l| match l.child("c:legendPos").and_then(|p| p.attr("val")).unwrap_or("r") { "l" => Pos::Left, @@ -206,9 +229,24 @@ impl Series { let sp = ser.child("c:spPr"); let ln = sp.and_then(|p| p.child("a:ln")); let marker = ser.child("c:marker").and_then(|m| m.child("c:spPr")).and_then(|p| fill_colour(p, theme)); + let marker_color = marker; let color = sp.and_then(|p| fill_colour(p, theme)).or_else(|| ln.and_then(|l| fill_colour(l, theme))).or(marker); - let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map_or(LINE_W, |w| w.clamp(0.0, 20.0)); + let no_fill = sp.and_then(|p| p.child("a:noFill")).is_some(); + let line_w = ln.and_then(|l| l.attr("w")).and_then(|v| measure(v, 12_700.0)).map(|w| w.clamp(0.0, 20.0)); let line = ln.is_none_or(|l| l.child("a:noFill").is_none()); + // `c:marker/c:symbol`: which symbol each point gets (`none` for no markers). Scatter + // series show markers unless the file says not; line series only when it says so. + let symbol = ser.child("c:marker").and_then(|m| m.child("c:symbol")).and_then(|s| s.attr("val")).map(str::to_string); + let markers = symbol.as_deref().map(|s| s != "none").or_else(|| scatter.then_some(true)); + let symbol = symbol.as_deref().map_or(Symbol::Circle, Symbol::parse); + let marker_size = ser + .child("c:marker") + .and_then(|m| m.child("c:size")) + .and_then(|s| s.attr("val")) + .and_then(|v| v.trim().parse::().ok()) + .filter(|v| v.is_finite()) + .map(|v| v.clamp(2.0, 72.0)); + let smooth = ser.child("c:smooth").and_then(|s| s.attr("val")).is_some_and(truthy); let (values, fmt) = numbers(ser.child(if scatter { "c:yVal" } else { "c:val" }), cap); let (mut xs, x_fmt) = if scatter { numbers(ser.child("c:xVal"), cap) } else { (Vec::new(), None) }; if scatter && xs.is_empty() { @@ -223,7 +261,30 @@ impl Series { } } let labels = show_values(ser.child("c:dLbls")); - Series { name, idx, color, line_w, line, values, xs, points, fmt, x_fmt, labels, raw: Vec::new() } + let label_fmt = label_format(ser.child("c:dLbls")); + let label_pos = ser.child("c:dLbls").and_then(|d| d.child("c:dLblPos")).and_then(|p| p.attr("val")).and_then(LabelPos::parse); + Series { + name, + idx, + color, + line_w, + line, + no_fill, + markers, + symbol, + marker_size, + smooth, + values, + xs, + points, + fmt, + x_fmt, + labels, + label_fmt, + label_pos, + marker_color, + raw: Vec::new(), + } } } @@ -290,7 +351,7 @@ fn pts(cache: &El, cap: usize) -> Vec> { } pub(super) fn axis(a: Option<&El>) -> Axis { - let Some(a) = a else { return Axis { deleted: true, ..Default::default() } }; + let Some(a) = a else { return Axis { deleted: true, labels: true, line: true, ..Default::default() } }; let scaling = a.child("c:scaling"); let num = |n: &str| scaling.and_then(|s| s.child(n)).and_then(|e| e.attr("val")).and_then(|v| v.trim().parse::().ok()).filter(|v| v.is_finite()); @@ -305,8 +366,16 @@ pub(super) fn axis(a: Option<&El>) -> Axis { .child("c:numFmt") .filter(|f| f.attr("sourceLinked") != Some("1")) .and_then(|f| f.attr("formatCode")) - .filter(|f| f.len() <= MAX_FORMAT_CODE) + .filter(|f| f.chars().count() <= craft_wordmodel::chart::MAX_FMT) .map(str::to_string), + title: a.child("c:title").and_then(title_text), + title_rot: a + .child("c:title") + .and_then(|t| t.find("a:bodyPr")) + .and_then(|b| b.attr("rot")) + .and_then(|v| v.trim().parse::().ok()) + .map(|v| (v as f64 / 60_000.0).clamp(-360.0, 360.0) as f32), + labels: a.child("c:tickLblPos").and_then(|p| p.attr("val")) != Some("none"), } } diff --git a/crates/docx/src/read/chart/plot.rs b/crates/docx/src/read/chart/plot.rs index 652ee79a8..22d05b554 100644 --- a/crates/docx/src/read/chart/plot.rs +++ b/crates/docx/src/read/chart/plot.rs @@ -1,9 +1,12 @@ -use super::canvas::{Rect, fill, rect_segs, runs, seg, stroke, text, text_at, text_w, widest, wrap}; +use super::canvas::{Rect, clip_text, fill, rect_segs, runs, seg, stroke, text, text_angle, text_at, text_w, widest, wrap}; use super::format::format_value; -use super::model::{DataLabel, Grouping, Kind, Series}; +use super::model::{DataLabel, Grouping, Kind, LabelPos, Series, Symbol}; use super::parse::category_text; use super::polar::{draw_data_labels, label_of}; -use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure}; +use super::{AXIS, Chart, El, GAP, GRID, GraphicItem, LINE, LINE_W, MAX_LABEL, MAX_TICKS, PathSeg, Rgb, ShapeKind, TextAlign, fill_colour, measure}; + +/// Line width of the stroked marker symbols (x, plus, star), points. +const MARKER_STROKE: f32 = 1.5; pub(super) fn label(s: &str) -> String { s.chars().take(MAX_LABEL).collect() @@ -142,6 +145,8 @@ impl Axes { /// The room around the plot for axis labels, and the category labels as they'll be drawn. struct Margins { left: f32, + /// Part of `left` taken by the rotated category-axis title (bars): the labels start after it. + title: f32, top: f32, bottom: f32, /// Category labels in lines (none for scatter charts or a deleted category axis). @@ -151,22 +156,25 @@ struct Margins { } /// The margins the axis labels need around the plot in `area`. Category labels under a column -/// chart wrap to the band; labels that still don't fit show every k-th category. +/// chart wrap to the band; labels that still don't fit show every k-th category. Hidden tick +/// labels take no room; an axis title adds one line on its side. fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins { let fs = c.text_size; let lh = fs * LINE; let scatter = c.kind == Kind::Scatter; let (cat_on, val_on) = (!c.cat.deleted, !c.val.deleted); + let (cat_labels, val_labels) = (cat_on && c.cat.labels, val_on && c.val.labels); + let title = title_left(c, cat_on, val_on, area.w); let left = if c.horizontal { - if cat_on { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 } - } else if val_on { + if cat_labels { widest(&c.cats, fs).min(area.w * 0.4) + GAP } else { 0.0 } + } else if val_labels { ax.vticks.iter().map(|(_, s)| text_w(s, fs)).fold(0.0, f32::max) + GAP } else { 0.0 - }; + } + title; let band_w = (area.w - left).max(1.0) / c.n as f32; - let wrapped: Vec> = if cat_on && !scatter { - let width = if c.horizontal { left - GAP } else { band_w }; + let wrapped: Vec> = if cat_labels && !scatter { + let width = if c.horizontal { left - title - GAP } else { band_w }; c.cats.iter().map(|s| wrap(s, width, fs)).collect() } else { Vec::new() @@ -174,15 +182,48 @@ fn axis_margins(c: &Chart, ax: &Axes, area: Rect) -> Margins { let widest_line = wrapped.iter().flatten().map(|s| text_w(s, fs)).fold(0.0, f32::max); let skip = ((widest_line / band_w).ceil() as usize).max(1); let label_lines = wrapped.iter().map(Vec::len).max().unwrap_or(1) as f32; - let bottom = if cat_on && !c.horizontal { + let bottom = (if cat_labels && !c.horizontal { if scatter { lh + GAP } else { label_lines * lh + GAP } - } else if c.horizontal && val_on { + } else if c.horizontal && val_labels { lh + GAP } else { 0.0 - }; - let top = if !c.horizontal && val_on { lh / 2.0 } else { 0.0 }; - Margins { left, top, bottom, wrapped, skip } + }) + title_bottom(c, cat_on, val_on, lh); + let top = if !c.horizontal && val_labels { lh / 2.0 } else { 0.0 }; + Margins { left, title, top, bottom, wrapped, skip } +} + +/// Room for the vertical axis title on the plot's left: one line when the axis that sits there +/// (the value axis for columns, the category axis for bars) has a title. +/// A horizontal title (`rot="0"`) gets a strip as wide as its wrapped lines, at most a quarter of +/// `area_w`, so the plot keeps its room. +fn title_left(c: &Chart, cat_on: bool, val_on: bool, area_w: f32) -> f32 { + let side = if c.horizontal { cat_on && c.cat.title.is_some() } else { val_on && c.val.title.is_some() }; + if !side { + return 0.0; + } + match flat_title_lines(c, area_w) { + Some(lines) => widest(&lines, c.text_size).min((area_w * 0.25).max(0.0)) + GAP, + None => c.text_size * LINE, + } +} + +/// The wrapped lines of the left axis's title when the file keeps it horizontal (`rot="0"`). +fn flat_title_lines(c: &Chart, area_w: f32) -> Option> { + let left = if c.horizontal { &c.cat } else { &c.val }; + let r = left.title_rot?.rem_euclid(360.0); + if !(r < 1.0 || r > 359.0) { + return None; + } + let width = if area_w.is_finite() { (area_w * 0.18).max(1.0) } else { 1.0 }; + left.title.as_ref().map(|t| wrap(t, width, c.text_size)) +} + +/// Room for the horizontal axis title under the plot: one line when the axis that sits there +/// (the category axis for columns, the value axis for bars) has a title. +fn title_bottom(c: &Chart, cat_on: bool, val_on: bool, lh: f32) -> f32 { + let side = if c.horizontal { val_on && c.val.title.is_some() } else { cat_on && c.cat.title.is_some() }; + if side { lh } else { 0.0 } } /// A bar, column, line, area or scatter chart in `area`: gridlines, series, axes, then labels. @@ -200,6 +241,9 @@ pub(super) fn cartesian(out: &mut Vec, c: &Chart, area: Rect) { if c.val.grid { draw_grid(out, &g, &ax.vticks); } + if c.cat.grid { + draw_cat_grid(out, &g, &ax.xticks, c.kind); + } let mut labels = Vec::new(); match c.kind { Kind::Bar => bars(out, c, &g, &mut labels), @@ -210,12 +254,49 @@ pub(super) fn cartesian(out: &mut Vec, c: &Chart, area: Rect) { } draw_axes(out, c, &g); draw_data_labels(out, &labels, c.text_size, g.horiz && c.kind == Kind::Bar); - if !c.val.deleted { - draw_value_labels(out, c, &g, &ax.vticks, Rect { w: m.left, ..area }); + if !c.val.deleted && c.val.labels { + let room = title_left(c, !c.cat.deleted, true, area.w); + draw_value_labels(out, c, &g, &ax.vticks, Rect { x: area.x + room, w: m.left - room, ..area }); } - if !c.cat.deleted { + if !c.cat.deleted && c.cat.labels { draw_category_labels(out, c, &g, &ax.xticks, &m, area); } + draw_axis_titles(out, c, &g, area); +} + +/// The axis titles, one line: the title of the vertical axis (the value axis for columns, the +/// category axis for bars) is turned 90° on the plot's left; the horizontal axis's title sits +/// under the plot. +fn draw_axis_titles(out: &mut Vec, c: &Chart, g: &Frame, area: Rect) { + let lh = c.text_size * LINE; + // The axis drawn along the plot's left (the value axis for columns, the category axis for + // bars) gets its title rotated there; the other axis's title sits under the plot. + let (left, bottom) = if c.horizontal { (&c.cat, &c.val) } else { (&c.val, &c.cat) }; + // Word turns the vertical axis's title counter-clockwise (reading upwards) unless the file + // turns it the other way or keeps it horizontal (`rot="0"`, wrapped in its own strip); any + // other rotation would not fit the strip and is read as counter-clockwise. + let up = if left.title_rot.is_some_and(|r| (r.rem_euclid(360.0) - 90.0).abs() < 1.0) { 90.0 } else { -90.0 }; + if !left.deleted + && let Some(lines) = flat_title_lines(c, area.w) + { + let strip = (g.plot.x - area.x - GAP).max(1.0); + let top = g.plot.y + g.plot.h / 2.0 - lines.len() as f32 * lh / 2.0; + for (k, l) in lines.iter().enumerate() { + let l = clip_text(l, strip, c.text_size); + text_angle(out, Rect { x: area.x, y: top + k as f32 * lh, w: strip, h: lh }, &l, c.text_size, TextAlign::Center, false, 0.0); + } + } else if !left.deleted + && let Some(t) = &left.title + { + let t = clip_text(t, g.plot.h, c.text_size); + text_angle(out, Rect { x: area.x, y: g.plot.y, w: lh, h: g.plot.h }, &t, c.text_size, TextAlign::Center, false, up); + } + if !bottom.deleted + && let Some(t) = &bottom.title + { + let t = clip_text(t, g.plot.w, c.text_size); + text_angle(out, Rect { x: g.plot.x, y: area.bottom() - lh, w: g.plot.w, h: lh }, &t, c.text_size, TextAlign::Center, false, 0.0); + } } /// Major gridlines at the value ticks. @@ -228,6 +309,31 @@ fn draw_grid(out: &mut Vec, g: &Frame, ticks: &[(f64, String)]) { } } +/// Gridlines along the category axis: across the plot at each category's centre (where line and +/// area points sit, on the tick marks when the axis crosses "midCat"), at the x ticks of a +/// scatter chart. +fn draw_cat_grid(out: &mut Vec, g: &Frame, xticks: &[(f64, String)], kind: Kind) { + let plot = g.plot; + if kind == Kind::Scatter { + for (t, _) in xticks { + let p = g.scat_x(*t); + stroke(out, seg(p, plot.y, p, plot.bottom()), GRID, 0.75); + } + return; + } + for i in 0..g.n { + let segs = if g.horiz { + let p = g.band_start(i) + g.band() / 2.0; + seg(plot.x, p, plot.right(), p) + } else { + // Bars stay in their bands; only line and area points move onto the ticks. + let p = if matches!(kind, Kind::Line | Kind::Area) { g.cat_x(i) } else { g.band_start(i) + g.band() / 2.0 }; + seg(p, plot.y, p, plot.bottom()) + }; + stroke(out, segs, GRID, 0.75); + } +} + /// The category axis line, the plot area's outline and the value axis line, as the file shows them. fn draw_axes(out: &mut Vec, c: &Chart, g: &Frame) { let plot = g.plot; @@ -271,7 +377,7 @@ fn draw_category_labels(out: &mut Vec, c: &Chart, g: &Frame, xticks for (i, lines) in m.wrapped.iter().enumerate() { let top = g.band_start(i) + g.band() / 2.0 - lines.len() as f32 * lh / 2.0; for (k, s) in lines.iter().enumerate() { - text(out, Rect { x: area.x, y: top + k as f32 * lh, w: m.left - GAP, h: lh }, s, fs, TextAlign::Right, false); + text(out, Rect { x: area.x + m.title, y: top + k as f32 * lh, w: m.left - m.title - GAP, h: lh }, s, fs, TextAlign::Right, false); } } } else { @@ -361,7 +467,13 @@ fn bars(out: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec (if v >= 0.0 { p1 } else { p0 }, mid), (false, false) => (mid, if v >= 0.0 { p0 } else { p1 }), }; - labels.push(DataLabel { x, y, text, inside: stacked }); + labels.push(DataLabel { x, y, text, inside: stacked, pos: LabelPos::Above, off: 0.0 }); } } } @@ -401,13 +513,109 @@ fn lines(out: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec f32 { + if drawn { s.marker_size.filter(|v| v.is_finite()).unwrap_or(6.0).clamp(2.0, 72.0) / 2.0 } else { 0.0 } +} + +fn markers(out: &mut Vec, pts: impl Iterator, colour: Rgb, s: &Series) { + const MARKER: f32 = 6.0; + let size = s.marker_size.filter(|v| v.is_finite()).unwrap_or(MARKER).clamp(2.0, 72.0); + let h = size / 2.0; + let shape = |out: &mut Vec, rect: [f32; 4], kind: ShapeKind| { + out.push(GraphicItem::Shape { rect, kind, fill: Some(colour), stroke: None, stroke_width: 0.0 }); + }; + let mut strokes = Vec::new(); + for (x, y) in pts { + match s.symbol { + Symbol::Circle => shape(out, [x - h, y - h, size, size], ShapeKind::Ellipse), + Symbol::Square => shape(out, [x - h, y - h, size, size], ShapeKind::Rectangle), + Symbol::Dot => shape(out, [x - h / 2.0, y - h / 2.0, h, h], ShapeKind::Ellipse), + Symbol::Dash => shape(out, [x - h, y - size / 6.0, size, size / 3.0], ShapeKind::Rectangle), + Symbol::Diamond => fill( + out, + vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y), PathSeg::Line(x, y + h), PathSeg::Line(x - h, y), PathSeg::Close], + colour, + ), + Symbol::Triangle => { + fill(out, vec![PathSeg::Move(x, y - h), PathSeg::Line(x + h, y + h), PathSeg::Line(x - h, y + h), PathSeg::Close], colour) + } + Symbol::X | Symbol::Plus | Symbol::Star => { + if s.symbol != Symbol::Plus { + strokes.extend(seg(x - h, y - h, x + h, y + h)); + strokes.extend(seg(x - h, y + h, x + h, y - h)); + } + if s.symbol != Symbol::X { + strokes.extend(seg(x - h, y, x + h, y)); + strokes.extend(seg(x, y - h, x, y + h)); + } + } + } + } + stroke(out, strokes, colour, MARKER_STROKE); +} + +/// A polyline through `pts`, a smooth curve (Catmull-Rom pieces) when `smooth` and there are at +/// least three points in a row; gaps (`None`) end a run. +fn curve(pts: impl Iterator>, smooth: bool) -> Vec { + if !smooth { + return runs(pts); + } + let pts: Vec> = pts.collect(); + let mut segs = Vec::new(); + let mut i = 0; + while i < pts.len() { + let Some(first) = pts[i] else { + i += 1; + continue; + }; + // Collect one run of points. + let mut run = vec![first]; + while let Some(p) = pts.get(i + 1).copied().flatten() { + run.push(p); + i += 1; + } + i += 1; + if run.len() < 3 { + segs.extend(runs(run.into_iter().map(Some))); + continue; + } + segs.push(PathSeg::Move(run[0].0, run[0].1)); + for k in 0..run.len() - 1 { + // Catmull-Rom: tangent at p1 from p0 to p2 (ends mirror their own tangent). + let p0 = run[k.max(1) - 1]; + let p1 = run[k]; + let p2 = run[k + 1]; + let p3 = run.get(k + 2).copied().unwrap_or(p2); + let c1 = (p1.0 + (p2.0 - p0.0) / 6.0, p1.1 + (p2.1 - p0.1) / 6.0); + let c2 = (p2.0 - (p3.0 - p1.0) / 6.0, p2.1 - (p3.1 - p1.1) / 6.0); + segs.push(PathSeg::Cubic(c1.0, c1.1, c2.0, c2.1, p2.0, p2.1)); + } + } + segs +} + fn areas(out: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec) { let mut base = vec![0.0f64; g.n]; for (j, s) in c.series.iter().enumerate().filter(|(_, s)| has_values(s)) { @@ -422,10 +630,12 @@ fn areas(out: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec, c: &Chart, g: &Frame, labels: &mut Vec) { - const MARKER: f32 = 6.0; for (j, s) in c.series.iter().enumerate() { let pts: Vec> = s.xs.iter() @@ -448,16 +657,21 @@ fn draw_scatter(out: &mut Vec, c: &Chart, g: &Frame, labels: &mut V .collect(); for (i, p) in pts.iter().enumerate() { if let (Some((x, y)), Some(text)) = (p, label_of(s, i)) { - labels.push(DataLabel { x: *x, y: *y - MARKER / 2.0, text, inside: false }); + labels.push(DataLabel { + x: *x, + y: *y, + text, + inside: false, + pos: s.label_pos.unwrap_or(LabelPos::Right), + off: marker_half(s, s.markers != Some(false)) + 2.0, + }); } } if s.line { - stroke(out, runs(pts.iter().copied()), c.series_colour(j), s.line_w); - continue; + stroke(out, curve(pts.iter().copied(), s.smooth), c.series_colour(j), s.line_w.unwrap_or(LINE_W)); } - for (x, y) in pts.iter().flatten() { - let rect = [x - MARKER / 2.0, y - MARKER / 2.0, MARKER, MARKER]; - out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(c.series_colour(j)), stroke: None, stroke_width: 0.0 }); + if s.markers != Some(false) { + markers(out, pts.iter().flatten().copied(), c.marker_colour(j), s); } } } diff --git a/crates/docx/src/read/chart/polar.rs b/crates/docx/src/read/chart/polar.rs index 57ad10472..cf5e7452e 100644 --- a/crates/docx/src/read/chart/polar.rs +++ b/crates/docx/src/read/chart/polar.rs @@ -1,8 +1,8 @@ use super::canvas::{Rect, clip_text, fill, stroke, text, text_w, widest}; use super::format::format_value; -use super::model::{DataLabel, Kind, Series}; +use super::model::{DataLabel, Kind, LabelPos, Series}; use super::plot::{Axes, has_values}; -use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign}; +use super::{Chart, FRAC_PI_2, GAP, GRID, GraphicItem, LINE, LINE_W, PathSeg, Rgb, ShapeKind, TAU, TEXT, TextAlign}; pub(super) fn pie(out: &mut Vec, c: &Chart, area: Rect) { let Some(s) = c.series.first() else { return }; @@ -38,7 +38,7 @@ pub(super) fn pie(out: &mut Vec, c: &Chart, area: Rect) { // Halfway across the ring (pie: two thirds out from the centre). let at = if inner > 0.0 { (r + inner) / 2.0 } else { r * 0.65 }; let (x, y) = polar(cx, cy, at, (a + b) / 2.0); - labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true }); + labels.push(DataLabel { x: x as f32, y: y as f32, text, inside: true, pos: LabelPos::Above, off: 0.0 }); } a = b; } @@ -92,14 +92,14 @@ pub(super) fn radar(out: &mut Vec, c: &Chart, area: Rect) { if c.radar_fill { fill(out, segs.clone(), col); } - stroke(out, segs, col, s.line_w); + stroke(out, segs, col, s.line_w.unwrap_or(LINE_W)); for (i, p) in ring.iter().enumerate() { if c.radar_marks && s.values.get(i).copied().flatten().is_some() { let rect = [p.0 as f32 - MARKER / 2.0, p.1 as f32 - MARKER / 2.0, MARKER, MARKER]; out.push(GraphicItem::Shape { rect, kind: ShapeKind::Ellipse, fill: Some(col), stroke: None, stroke_width: 0.0 }); } if let Some(text) = label_of(s, i) { - labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false }); + labels.push(DataLabel { x: p.0 as f32, y: p.1 as f32, text, inside: false, pos: LabelPos::Above, off: 0.0 }); } } } @@ -134,13 +134,13 @@ pub(super) fn radar(out: &mut Vec, c: &Chart, area: Rect) { } /// The data label of point `i` of series `s`, when it shows one: the value as the file has it, -/// in the series' number format. +/// in the labels' own number format, else the series' one. pub(super) fn label_of(s: &Series, i: usize) -> Option { if s.labels != Some(true) { return None; } let v = s.raw.get(i).copied().flatten()?; - let fmt = s.fmt.as_deref().filter(|f| !f.eq_ignore_ascii_case("General")); + let fmt = s.label_fmt.as_ref().map_or(s.fmt.as_deref(), |f| f.as_deref().or(s.fmt.as_deref())).filter(|f| !f.eq_ignore_ascii_case("General")); Some(match fmt { Some(f) => format_value(v, f, 1.0), None => general_value(v), @@ -165,13 +165,22 @@ pub(super) fn draw_data_labels(out: &mut Vec, labels: &[DataLabel], let w = text_w(&l.text, fs) + 4.0; let (rect, align) = if l.inside { (Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center) + } else if l.off > 0.0 || l.pos != LabelPos::Above { + let o = l.off; + match l.pos { + LabelPos::Right => (Rect { x: l.x + o, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left), + LabelPos::Left => (Rect { x: l.x - o - w, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Right), + LabelPos::Below => (Rect { x: l.x - w / 2.0, y: l.y + o, w, h: lh }, TextAlign::Center), + LabelPos::Center => (Rect { x: l.x - w / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Center), + LabelPos::Above => (Rect { x: l.x - w / 2.0, y: l.y - o - lh, w, h: lh }, TextAlign::Center), + } } else if beside_right { (Rect { x: l.x + GAP / 2.0, y: l.y - lh / 2.0, w, h: lh }, TextAlign::Left) } else { (Rect { x: l.x - w / 2.0, y: l.y - lh, w, h: lh }, TextAlign::Center) }; let color = if l.inside { Rgb::WHITE } else { TEXT }; - out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None }); + out.push(GraphicItem::Text { rect: rect.arr(), text: l.text.clone(), size: fs, color, bold: false, align, font: None, angle: 0.0 }); } } diff --git a/crates/docx/src/read/chart/tests.rs b/crates/docx/src/read/chart/tests.rs index 6e8bcfd82..ae80c028e 100644 --- a/crates/docx/src/read/chart/tests.rs +++ b/crates/docx/src/read/chart/tests.rs @@ -386,9 +386,13 @@ fn axis_format_code_length_is_capped() { let x = format!(r#""#); axis(Some(&crate::xml::parse(x.as_bytes()).unwrap())).fmt }; - let ok = format!("{}0", "x".repeat(254)); + use craft_wordmodel::chart::MAX_FMT; + // The limit is the model's, counted in characters (a CJK character is one). + let ok = format!("{}0", "x".repeat(MAX_FMT - 1)); assert_eq!(fmt(&ok), Some(ok.clone())); - assert_eq!(fmt(&format!("{}0", "x".repeat(300))), None); + let cjk = format!("{}0", "\u{5143}".repeat(MAX_FMT - 1)); + assert_eq!(fmt(&cjk), Some(cjk.clone())); + assert_eq!(fmt(&format!("{}0", "x".repeat(MAX_FMT))), None); } #[test] @@ -590,3 +594,308 @@ fn series_without_a_usable_c_idx_take_their_place() { let big = [renumbered(&a, &format!(r#""#, u32::MAX))]; assert_eq!(column_colours(&big).0.len(), 1); } + +#[test] +fn bar_category_labels_start_after_the_axis_title() { + let long = "A category label that is much too long for its column"; + let title = r#"Region"#; + let plot = format!( + r#"{}{title}"#, + ser(0, "S", Some("FF0000"), &[long, "Short"], &["4", "8"]), + ); + let items = chart_items(&space(&plot), &theme(), 300.0, 200.0); + let rect_of = |f: &dyn Fn(&str, &TextAlign) -> bool| -> Vec<[f32; 4]> { + items + .iter() + .filter_map(|it| if let GraphicItem::Text { rect, text, align, .. } = it { f(text, align).then_some(*rect) } else { None }) + .collect() + }; + let title_rect = rect_of(&|t, _| t == "Region"); + assert_eq!(title_rect.len(), 1); + let strip_end = title_rect[0][0] + title_rect[0][2]; + let labels = rect_of(&|t, a| matches!(a, TextAlign::Right) && t != "Region" && (long.contains(t) || t == "Short")); + assert!(!labels.is_empty()); + assert!(labels.iter().all(|r| r[0] >= strip_end - 1e-3), "labels {labels:?} overlap the title strip ending at {strip_end}"); + assert!(labels.iter().all(|r| r[2] <= 0.4 * 300.0), "{labels:?}"); +} + +#[test] +fn category_gridlines_follow_the_points_when_the_axis_crosses_mid() { + let line = |cross: &str| { + let plot = format!( + r#"{}"#, + ser(0, "L", Some("FF0000"), &["Q1", "Q2", "Q3"], &["1", "5", "3"]), + ); + let items = chart_items(&space(&plot), &theme(), 300.0, 200.0); + let grid: Vec = items + .iter() + .filter_map(|it| match it { + GraphicItem::Path { segs, stroke: Some(c), .. } if *c == GRID => match segs.first() { + Some(PathSeg::Move(x, _)) => Some(*x), + _ => None, + }, + _ => None, + }) + .collect(); + let pts: Vec = items + .iter() + .filter_map(|it| match it { + GraphicItem::Path { segs, stroke: Some(c), .. } if *c == Rgb(255, 0, 0) => Some( + segs.iter().filter_map(|s| if let PathSeg::Move(x, _) | PathSeg::Line(x, _) = s { Some(*x) } else { None }).collect::>(), + ), + _ => None, + }) + .next() + .unwrap_or_default(); + (grid, pts) + }; + for cross in ["midCat", "between"] { + let (grid, pts) = line(cross); + assert_eq!(grid.len(), 3, "{cross}"); + assert_eq!(pts.len(), 3, "{cross}"); + assert!(grid.iter().zip(&pts).all(|(g, p)| (g - p).abs() < 1e-3), "{cross}: grid {grid:?} points {pts:?}"); + } +} + +/// A line chart with one series; `extra` goes into the series (`c:marker`, `c:dLbls`), `axes` +/// into the value axis. +fn line_with(extra: &str, val_axis: &str) -> Vec { + let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("", &format!("{extra}")); + let plot = format!( + r#"{s}{val_axis}"# + ); + chart_items(&space(&plot), &theme(), 300.0, 200.0) +} + +fn marker_xml(symbol: &str, size: u32) -> String { + format!(r#""#) +} + +#[test] +fn marker_symbols_are_drawn_as_their_shape() { + let shapes = |items: &[GraphicItem], kind: ShapeKind| { + items + .iter() + .filter(|it| matches!(it, GraphicItem::Shape { kind: k, rect, .. } if *k == kind && rect[2] > 3.0 && rect[2] < 20.0 && rect[3] > 1.0)) + .count() + }; + let paths = |items: &[GraphicItem], filled: bool| { + items + .iter() + .filter(|it| matches!(it, GraphicItem::Path { fill, segs, .. } if fill.is_some() == filled && segs.len() <= 12 && segs.len() >= 3)) + .count() + }; + // One marker per point, in the series' size. + let circle = line_with(&marker_xml("circle", 9), ""); + assert_eq!(shapes(&circle, ShapeKind::Ellipse), 3); + assert!(circle.iter().any(|it| matches!(it, GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, .. } if (rect[2] - 9.0).abs() < 1e-3))); + let square = line_with(&marker_xml("square", 7), ""); + assert_eq!(shapes(&square, ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3, "a square per point"); + assert_eq!(shapes(&square, ShapeKind::Ellipse), 0, "no circles"); + // Diamonds and triangles are filled polygons; x, plus and star are strokes. + let base = paths(&line_with("", ""), true); + assert_eq!(paths(&line_with(&marker_xml("diamond", 7), ""), true) - base, 3); + assert_eq!(paths(&line_with(&marker_xml("triangle", 7), ""), true) - base, 3); + for symbol in ["x", "plus", "star"] { + let items = line_with(&marker_xml(symbol, 7), ""); + assert!(items.iter().any(|it| matches!(it, GraphicItem::Path { fill: None, stroke: Some(_), segs, .. } if segs.len() >= 6)), "{symbol}"); + assert_eq!(shapes(&items, ShapeKind::Ellipse), 0, "{symbol} is not a circle"); + } + // Dash and dot are small shapes; `none` and unknown names never crash (unknown: a circle). + assert_eq!(shapes(&line_with(&marker_xml("dash", 7), ""), ShapeKind::Rectangle) - shapes(&circle, ShapeKind::Rectangle), 3); + assert_eq!(shapes(&line_with(&marker_xml("dot", 7), ""), ShapeKind::Ellipse), 3); + assert_eq!(shapes(&line_with(&marker_xml("none", 7), ""), ShapeKind::Ellipse), 0); + assert_eq!(shapes(&line_with(&marker_xml("wibble", 7), ""), ShapeKind::Ellipse), 3); + // Hostile sizes are clamped. + for size in ["0", "99999999999", "-4", "x"] { + let items = line_with(&format!(r#""#), ""); + assert!(items.iter().all(|it| match it { + GraphicItem::Shape { rect, .. } => rect.iter().all(|v| v.is_finite()) && rect[2] <= 300.0, + _ => true, + })); + } +} + +#[test] +fn data_labels_use_their_own_format_code() { + let dl = |num: &str| format!(r#"{num}"#); + let shown = |items: &[GraphicItem]| -> Vec { + items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect() + }; + let own = shown(&line_with(&dl(r#""#), "")); + assert!(own.contains(&"3.20".to_string()) && own.contains(&"5.00".to_string()), "{own:?}"); + // Linked to the source: the series' format (General here). + let linked = shown(&line_with(&dl(r#""#), "")); + assert!(linked.contains(&"3.2".to_string()) && linked.contains(&"5".to_string()), "{linked:?}"); + // A format code beyond the cap is ignored, not applied. + let long = "0".repeat(craft_wordmodel::chart::MAX_FMT + 1); + let capped = shown(&line_with(&dl(&format!(r#""#)), "")); + assert!(capped.contains(&"3.2".to_string()), "{capped:?}"); +} + +#[test] +fn value_axis_title_turns_counter_clockwise_like_word() { + let title = |body: &str| format!(r#"{body}Revenue"#); + let angle = |items: &[GraphicItem]| { + items.iter().find_map(|it| if let GraphicItem::Text { text, angle, .. } = it { (text == "Revenue").then_some(*angle) } else { None }) + }; + // The file's rotation (`-5400000` is 90° counter-clockwise), or Word's default when it says none. + assert_eq!(angle(&line_with("", &title(r#""#))), Some(-90.0)); + assert_eq!(angle(&line_with("", &title(""))), Some(-90.0)); + assert_eq!(angle(&line_with("", &title(r#""#))), Some(90.0)); + // A horizontal title (`rot="0"`) stays horizontal: the plot makes room for it (see below). + assert_eq!(angle(&line_with("", &title(r#""#))), Some(0.0)); + // Hostile values stay finite. + let odd = angle(&line_with("", &title(r#""#))); + assert!(odd.is_some_and(f32::is_finite)); +} + +fn first_ellipse(items: &[GraphicItem]) -> Option<([f32; 4], Rgb)> { + items.iter().find_map(|it| match it { + GraphicItem::Shape { kind: ShapeKind::Ellipse, rect, fill: Some(c), .. } if rect[2] > 3.0 && rect[2] < 20.0 => Some((*rect, *c)), + _ => None, + }) +} + +#[test] +fn markers_use_the_automatic_series_colour_not_the_line_colour() { + // The series has an explicit red line/fill but no marker fill: Word draws the markers in the + // palette colour of the series index (accent 1 here). + let items = line_with(&marker_xml("circle", 9), ""); + assert_eq!(first_ellipse(&items).map(|e| e.1), Some(theme()[4])); + // A marker's own fill wins. + let own = + r#""#; + assert_eq!(first_ellipse(&line_with(own, "")).map(|e| e.1), Some(Rgb(0, 0xAA, 0))); +} + +#[test] +fn line_data_labels_default_to_the_right_and_honour_dlblpos() { + let dl = |pos: &str| format!(r#"{}{pos}"#, marker_xml("circle", 9)); + let text_rect = |items: &[GraphicItem]| { + items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { (text == "3.2").then_some(*rect) } else { None }) + }; + let at = |pos: &str| { + let items = line_with(&dl(pos), ""); + let (m, _) = first_ellipse(&items).unwrap(); + let t = text_rect(&items).unwrap(); + // Centre of the marker and of the label box. + ((t[0] + t[2] / 2.0) - (m[0] + m[2] / 2.0), (t[1] + t[3] / 2.0) - (m[1] + m[3] / 2.0), t, m) + }; + let (dx, dy, t, m) = at(""); + assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "default is right of the marker: {dx} {dy}"); + let (dx, dy, t, m) = at(r#""#); + assert!(t[0] >= m[0] + m[2] && dy.abs() < 1.0, "{dx} {dy}"); + let (dx, dy, _, _) = at(r#""#); + assert!(dx.abs() < 1.0 && dy < -4.0, "{dx} {dy}"); + let (dx, dy, _, _) = at(r#""#); + assert!(dx.abs() < 1.0 && dy > 4.0, "{dx} {dy}"); + let (dx, dy, t, m) = at(r#""#); + assert!(t[0] + t[2] <= m[0] && dy.abs() < 1.0, "{dx} {dy}"); + let (dx, dy, _, _) = at(r#""#); + assert!(dx.abs() < 1.0 && dy.abs() < 1.0, "{dx} {dy}"); + // Unknown positions fall back to the default. + let (_, _, t, m) = at(r#""#); + assert!(t[0] >= m[0] + m[2]); +} + +#[test] +fn horizontal_value_axis_title_gets_its_own_strip() { + let title = |rot: &str, name: &str| { + format!(r#"{name}"#) + }; + let plot_left = |items: &[GraphicItem]| { + // The first gridline's start is the plot's left edge. + items.iter().find_map(|it| match it { + GraphicItem::Path { segs, stroke: Some(_), fill: None, .. } if segs.len() == 2 => match segs.first() { + Some(PathSeg::Move(x, _)) => Some(*x), + _ => None, + }, + _ => None, + }) + }; + let rect_of = |items: &[GraphicItem], name: &str| { + items.iter().find_map(|it| if let GraphicItem::Text { text, rect, .. } = it { text.starts_with(name).then_some(*rect) } else { None }) + }; + let turned = line_with("", &title(r#" rot="-5400000" vert="horz""#, "Revenue")); + let flat = line_with("", &title(r#" rot="0" vert="horz""#, "Revenue")); + let (a, b) = (plot_left(&turned).unwrap(), plot_left(&flat).unwrap()); + assert!(b > a + 10.0, "a horizontal title takes more room than a turned one: {a} {b}"); + // The title box lies left of the plot and inside the chart. + let r = rect_of(&flat, "Revenue").unwrap(); + assert!(r[0] >= 0.0 && r[0] + r[2] <= b + 0.5, "{r:?} {b}"); + // A hostile title never takes more than a quarter of the width and stays finite. + let long = "Wibble ".repeat(500); + let items = line_with("", &title(r#" rot="0""#, &long)); + let p = plot_left(&items).unwrap(); + assert!(p.is_finite() && p <= 300.0 * 0.3, "{p}"); +} + +/// A one-series chart whose series carries `sp` as its `c:spPr`. +fn chart_with_sp(kind: &str, sp: &str) -> Vec { + let s = ser(0, "S", None, &["Q1", "Q2"], &["4", "8"]).replace("", &format!("{sp}")); + let plot = match kind { + "area" => format!( + r#"{s}"# + ), + dir => format!( + r#"{s}"# + ), + }; + chart_items(&space(&plot), &theme(), 300.0, 200.0) +} + +/// The series' closed shapes: (filled, stroked with the given width). +fn series_shapes(items: &[GraphicItem], width: f32) -> (usize, usize) { + let closed = |segs: &[PathSeg]| segs.iter().any(|s| matches!(s, PathSeg::Close)); + let fills = items.iter().filter(|it| matches!(it, GraphicItem::Path { segs, fill: Some(_), .. } if closed(segs))).count(); + let strokes = items + .iter() + .filter(|it| matches!(it, GraphicItem::Path { segs, fill: None, stroke: Some(_), stroke_width, .. } if closed(segs) && (*stroke_width - width).abs() < 0.01)) + .count(); + (fills, strokes) +} + +#[test] +fn unfilled_bars_keep_their_explicit_outline() { + let outlined = r#""#; + let no_line = r#""#; + for dir in ["col", "bar"] { + // Two points, two outlines, no fill. + assert_eq!(series_shapes(&chart_with_sp(dir, outlined), 2.0), (0, 2), "{dir}"); + // Control: no fill and no line draws nothing. + assert_eq!(series_shapes(&chart_with_sp(dir, no_line), 2.0), (0, 0), "{dir}"); + } +} + +#[test] +fn no_fill_area_series_is_not_painted() { + let filled = r#""#; + let none = r#""#; + assert_eq!(series_shapes(&chart_with_sp("area", filled), 2.0).0, 1); + assert_eq!(series_shapes(&chart_with_sp("area", none), 2.0).0, 0); +} + +#[test] +fn source_linked_series_labels_do_not_inherit_the_chart_format() { + let dl = |fmt: &str| format!(r#""#); + let shown = |items: &[GraphicItem]| -> Vec { + items.iter().filter_map(|it| if let GraphicItem::Text { text, .. } = it { Some(text.clone()) } else { None }).collect() + }; + // Chart-level 0%; the series' own labels are `linked` (or have no labels element at all). + let run = |series_dl: &str| { + let s = ser(0, "S", Some("FF0000"), &["Q1", "Q2", "Q3"], &["3.2", "4.1", "5"]).replace("", &format!("{series_dl}")); + let chart_dl = dl("0%").replace("{}", "0"); + let plot = format!( + r#"{s}{chart_dl}"# + ); + shown(&chart_items(&space(&plot), &theme(), 300.0, 200.0)) + }; + for linked in ["1", "true"] { + let own = run(&dl("General").replace("{}", linked)); + assert!(own.contains(&"3.2".to_string()) && !own.contains(&"320%".to_string()), "sourceLinked={linked}: {own:?}"); + } + // Control: a series without a label format still takes the chart's. + let inherited = run(r#""#); + assert!(inherited.contains(&"320%".to_string()), "{inherited:?}"); +} diff --git a/crates/docx/src/read/diagram.rs b/crates/docx/src/read/diagram.rs index 7f03e8b51..7b7bbeebf 100644 --- a/crates/docx/src/read/diagram.rs +++ b/crates/docx/src/read/diagram.rs @@ -463,7 +463,16 @@ fn text(sp: &El, xf: Xf, b: [f64; 4], theme: &[Rgb], filled: bool, fit: f32, out let mut y = y0 + (th - total) / 2.0; for l in lines { let lh = l.size * LINE; - out.push(GraphicItem::Text { rect: [x0, y, tw, lh], text: l.text, size: l.size, color: l.color, bold: l.bold, align: l.align, font: l.font }); + out.push(GraphicItem::Text { + rect: [x0, y, tw, lh], + text: l.text, + size: l.size, + color: l.color, + bold: l.bold, + align: l.align, + font: l.font, + angle: 0.0, + }); y += lh; } } diff --git a/crates/docx/src/read/story/blocks.rs b/crates/docx/src/read/story/blocks.rs index 38c35a02b..878429454 100644 --- a/crates/docx/src/read/story/blocks.rs +++ b/crates/docx/src/read/story/blocks.rs @@ -249,10 +249,11 @@ impl Reader<'_> { .unwrap_or_default(); para.props.fmt_change = PropChange::boxed(self.revision(RevisionKind::Format, ch), old); } - if para.props.numbering.is_some() - && let Some(ch) = ppr.child("w:numPr").and_then(|n| n.child("w:numberingChange")) - { - let original = ch.attr("w:original").unwrap_or("").chars().take(256).collect(); + if let Some(ch) = ppr.child("w:numPr").and_then(|n| n.child("w:numberingChange")) { + // Multi-level numbering caches (`w:original`) routinely exceed 256 chars while staying + // bounded against hostile attributes. + const MAX_CHANGE_ORIGINAL: usize = 4096; + let original = ch.attr("w:original").unwrap_or("").chars().take(MAX_CHANGE_ORIGINAL).collect(); para.props.num_change = Some(Box::new(NumChange { rev: self.revision(RevisionKind::Format, ch), original })); } if let Some(s) = ppr.child("w:sectPr") { diff --git a/crates/docx/src/write/embed.rs b/crates/docx/src/write/embed.rs index 9a8051e09..7673a6f26 100644 --- a/crates/docx/src/write/embed.rs +++ b/crates/docx/src/write/embed.rs @@ -884,10 +884,11 @@ mod tests { let spec = ChartSpec { kind: ChartType::Column, categories: vec!["a".into(), "b".into()], - series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2] }).collect(), + series: (0..series).map(|i| ChartSeries { name: format!("S{i}"), values: vec![Some(1.0); 2], ..Default::default() }).collect(), title: None, legend: None, data_labels: false, + ..Default::default() }; let graphic = Graphic { kind: GraphicKind::Chart, w: 100.0, h: 100.0, chart: Some(Arc::new(spec)), ..Default::default() }; InlineObject::Graphic { w: 100.0, h: 100.0, alt: String::new(), float: Default::default(), graphic: Arc::new(graphic) } diff --git a/crates/docx/src/write/props.rs b/crates/docx/src/write/props.rs index 0a64fc20a..0b92ce191 100644 --- a/crates/docx/src/write/props.rs +++ b/crates/docx/src/write/props.rs @@ -261,10 +261,12 @@ pub fn ppr_inner(w: &mut W, p: &ParaProps, framed: bool, num_change: Option<&Cha toggle(w, "w:pageBreakBefore", p.page_break_before); write_frame(w, p, framed); toggle(w, "w:widowControl", p.widow_control); - if let Some(nr) = p.numbering { + if p.numbering.is_some() || num_change.is_some() { w.open("w:numPr", &[]); - w.val("w:ilvl", &n(nr.level.min(8) as i64)); - w.val("w:numId", &n(nr.num as i64)); + if let Some(nr) = p.numbering { + w.val("w:ilvl", &n(nr.level.min(8) as i64)); + w.val("w:numId", &n(nr.num as i64)); + } if let Some(a) = num_change { let refs: Vec<(&str, &str)> = a.iter().map(|(k, v)| (*k, v.as_str())).collect(); w.empty("w:numberingChange", &refs); diff --git a/crates/docx/tests/it/fixtures.rs b/crates/docx/tests/it/fixtures.rs index 868e8cbb0..348d6239e 100644 --- a/crates/docx/tests/it/fixtures.rs +++ b/crates/docx/tests/it/fixtures.rs @@ -2197,6 +2197,50 @@ fn formatting_revisions_round_trip() { assert_eq!(again.revisions, d.revisions); } +#[test] +fn numbering_change_without_active_list_roundtrips() { + let body = r#"Removed"#; + let d = read_body(body); + let p = paras(&d); + assert_eq!(p.len(), 1); + assert!(p[0].props.numbering.is_none()); + let nc = p[0].props.num_change.as_deref().expect("numberingChange read back"); + assert_eq!(nc.original, "%1:1:0:."); + + let bytes = craft_docx::write(&d).unwrap(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(&bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("word/document.xml").unwrap(), &mut xml).unwrap(); + assert!(xml.contains(r#""#), "{xml}"); + + let again = craft_docx::read(&bytes).unwrap(); + let ap = paras(&again); + assert!(ap[0].props.numbering.is_none()); + let anc = ap[0].props.num_change.as_deref().expect("numberingChange preserved after save"); + assert_eq!(anc.original, "%1:1:0:."); +} + +#[test] +fn numbering_change_original_preserves_long_values_within_bound() { + let long_orig = "%1:1:0:.".repeat(64); // 512 chars, well past the old 256-char cap + assert!(long_orig.len() > 256 && long_orig.len() < 4096); + let over_cap = "x".repeat(5000); + let body = format!( + r#"Long +Bounded"# + ); + let d = read_body(&body); + let p = paras(&d); + assert_eq!(p[0].props.num_change.as_deref().map(|n| n.original.as_str()), Some(long_orig.as_str())); + assert_eq!(p[1].props.num_change.as_deref().map(|n| n.original.chars().count()), Some(4096)); + + let bytes = craft_docx::write(&d).unwrap(); + let again = craft_docx::read(&bytes).unwrap(); + let ap = paras(&again); + assert_eq!(ap[0].props.num_change.as_deref().map(|n| n.original.as_str()), Some(long_orig.as_str())); +} + /// A package with `styles`, a theme (accent1 4472C4, accent2 ED7D31, major font "Major Face") and, /// when given, `settings`. fn themed_docx(body: &str, styles: &str, settings: Option<&str>) -> Document { diff --git a/crates/exr/src/lib.rs b/crates/exr/src/lib.rs index f1550f8de..78afd427e 100644 --- a/crates/exr/src/lib.rs +++ b/crates/exr/src/lib.rs @@ -18,6 +18,8 @@ //! - [`stream`]: one flat scan-line part read block by block, for pictures larger than memory. //! - [`cryptomatte_samples`]: a file's Cryptomatte layers and their per-pixel (ID, coverage) ranks. +/// Scanline OpenEXR writer using a caller-supplied linearization table. +pub mod rgba8; /// RFC 1950/1951 compressor for ZIP-compressed EXR chunks. pub mod zlib; diff --git a/crates/exr/src/rgba8.rs b/crates/exr/src/rgba8.rs new file mode 100644 index 000000000..6dc4bde4c --- /dev/null +++ b/crates/exr/src/rgba8.rs @@ -0,0 +1,235 @@ +//! Single-part scanline OpenEXR output from RGBA8 pixels. + +/// Scanline chunk compression. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compression { + /// Uncompressed. + None, + /// OpenEXR RLE. + Rle, + /// zlib per scanline. + Zips, + /// zlib per 16 scanlines. + Zip, +} +impl Compression { + fn code(self) -> u8 { + match self { + Self::None => 0, + Self::Rle => 1, + Self::Zips => 2, + Self::Zip => 3, + } + } + fn lines(self) -> usize { + if self == Self::Zip { 16 } else { 1 } + } +} +/// Why a frame cannot be encoded. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// Empty, overflowing or mismatched RGBA dimensions. + FrameSize, + /// Dimensions exceed the OpenEXR signed-size fields. + TooLarge, +} +fn checked_len(rgba: &[u8], w: u32, h: u32) -> Result { + let n = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(4)).ok_or(Error::FrameSize)?; + if w == 0 || h == 0 || rgba.len() != n { + return Err(Error::FrameSize); + } + Ok(n) +} + +/// IEEE 754 binary16 bits of `v`, rounded to nearest even (overflow → infinity, NaN → quiet NaN). +pub fn half(v: f32) -> u16 { + let b = v.to_bits(); + let sign = ((b >> 16) & 0x8000) as u16; + let exp = ((b >> 23) & 0xFF) as i32; + let man = b & 0x7F_FFFF; + if exp == 0xFF { + return sign | 0x7C00 | if man != 0 { 0x200 } else { 0 }; + } + // rounds `m >> shift` to nearest, ties to even (`shift` is 1..=24) + let round = |m: u32, shift: u32| { + let (q, rem, mid) = (m >> shift, m & ((1 << shift) - 1), 1 << (shift - 1)); + if rem > mid || (rem == mid && q & 1 == 1) { q + 1 } else { q } + }; + let e = exp - 127 + 15; + if e >= 0x1F { + sign | 0x7C00 + } else if e <= 0 { + if e < -10 { + return sign; + } + // subnormal half: the 24-bit significand scaled by 2^(e - 14); rounding up may carry + // into the smallest normal, which is the right encoding + sign | round(man | 0x80_0000, (14 - e) as u32) as u16 + } else { + // a carry out of the mantissa bumps the exponent (up to infinity), as it should + sign | round(((e as u32) << 23) | man, 13) as u16 + } +} + +/// An OpenEXR file: half-float scene-linear Rec.709 RGB, or RGBA with `alpha` (premultiplied, as +/// EXR defines alpha; the frame's straight alpha is multiplied in after linearisation). +pub fn write_rgba8(rgba: &[u8], w: u32, h: u32, alpha: bool, compression: Compression, linear: &[f32; 256]) -> Result, Error> { + let n = checked_len(rgba, w, h)?; + let too_large = || Error::TooLarge; + let (Ok(x_max), Ok(y_max)) = (i32::try_from(w - 1), i32::try_from(h - 1)) else { + return Err(too_large()); + }; + // channels in the alphabetical order the file layout requires + let names: &[u8] = if alpha { b"ABGR" } else { b"BGR" }; + let line = (w as usize).checked_mul(names.len() * 2).ok_or(Error::FrameSize)?; + // a chunk's byte count is a signed 32-bit field + let lines = compression.lines(); + if line.checked_mul(lines).and_then(|c| i32::try_from(c).ok()).is_none() { + return Err(too_large()); + } + + let mut hd = Vec::with_capacity(512); + hd.extend_from_slice(&[0x76, 0x2F, 0x31, 0x01]); // magic number 20000630 + hd.extend_from_slice(&[2, 0, 0, 0]); // version 2, single-part scan-line file, short names + let attr = |hd: &mut Vec, name: &str, ty: &str, value: &[u8]| { + for s in [name, ty] { + hd.extend_from_slice(s.as_bytes()); + hd.push(0); + } + hd.extend_from_slice(&(value.len() as i32).to_le_bytes()); + hd.extend_from_slice(value); + }; + let le = |v: &[f32]| v.iter().flat_map(|f| f.to_le_bytes()).collect::>(); + let mut channels = Vec::new(); + for &c in names { + channels.extend_from_slice(&[c, 0]); + channels.extend_from_slice(&1_i32.to_le_bytes()); // pixel type HALF + channels.extend_from_slice(&[0, 0, 0, 0]); // pLinear 0, reserved + channels.extend_from_slice(&1_i32.to_le_bytes()); // x sampling + channels.extend_from_slice(&1_i32.to_le_bytes()); // y sampling + } + channels.push(0); + let window: Vec = [0, 0, x_max, y_max].iter().flat_map(|v| v.to_le_bytes()).collect(); + attr(&mut hd, "channels", "chlist", &channels); + attr(&mut hd, "chromaticities", "chromaticities", &le(&[0.64, 0.33, 0.30, 0.60, 0.15, 0.06, 0.3127, 0.3290])); + attr(&mut hd, "compression", "compression", &[compression.code()]); + attr(&mut hd, "dataWindow", "box2i", &window); + attr(&mut hd, "displayWindow", "box2i", &window); + attr(&mut hd, "lineOrder", "lineOrder", &[0]); // INCREASING_Y + attr(&mut hd, "pixelAspectRatio", "float", &le(&[1.0])); + attr(&mut hd, "screenWindowCenter", "v2f", &le(&[0.0, 0.0])); + attr(&mut hd, "screenWindowWidth", "float", &le(&[1.0])); + hd.push(0); // end of header + + // offset table (one entry per chunk), then the chunks: first y, data size, data + let chunks = (h as usize).div_ceil(lines); + let table = chunks.checked_mul(8); + let body = (h as usize).checked_mul(line + 8); + let size = table.zip(body).and_then(|(t, b)| hd.len().checked_add(t)?.checked_add(b)).ok_or(Error::FrameSize)?; + // the uncompressed size: compressed chunks only shrink it + let mut out = Vec::with_capacity(size); + out.extend_from_slice(&hd); + let table_at = out.len(); + out.resize(table_at + chunks * 8, 0); + let lin = linear; + let opaque: [u16; 256] = std::array::from_fn(|i| half(lin[i])); + let row = w as usize * 4; + let wn = w as usize; + let mut px = vec![0_u16; wn * names.len()]; + let mut raw = Vec::with_capacity(line * lines); + // the deflate hash tables, shared by the frame's chunks instead of zeroed for each one + let mut tables = crate::zlib::Tables::default(); + for (k, block) in rgba[..n].chunks(row * lines).enumerate() { + raw.clear(); + for src in block.chunks_exact(row) { + // planar per line: every pixel of the first channel, then the next channel … + for (x, &[r, g, b, a]) in src.as_chunks::<4>().0.iter().enumerate() { + let rgb = [b, g, r]; + if alpha { + let k = f32::from(a) / 255.0; + px[x] = half(k); + for (c, &v) in rgb.iter().enumerate() { + px[(c + 1) * wn + x] = if a == 255 { opaque[v as usize] } else { half(lin[v as usize] * k) }; + } + } else { + for (c, &v) in rgb.iter().enumerate() { + px[c * wn + x] = opaque[v as usize]; + } + } + } + raw.extend(px.iter().flat_map(|v| v.to_le_bytes())); + } + let packed = match compression { + Compression::None => None, + Compression::Rle => Some(rle(&predict(&raw))), + Compression::Zips | Compression::Zip => Some(crate::zlib::zlib(&predict(&raw), &mut tables)), + }; + // a chunk that does not shrink is stored as it is (readers compare the sizes) + let data = packed.as_deref().filter(|p| p.len() < raw.len()).unwrap_or(&raw); + let offset = (out.len() as u64).to_le_bytes(); + out[table_at + k * 8..table_at + k * 8 + 8].copy_from_slice(&offset); + out.extend_from_slice(&((k * lines) as i32).to_le_bytes()); + out.extend_from_slice(&(data.len() as i32).to_le_bytes()); + out.extend_from_slice(data); + } + Ok(out) +} + +/// The OpenEXR RLE / ZIP predictor: bytes split into even and odd halves, then each byte replaced +/// by its difference from the previous one, plus 128. +fn predict(raw: &[u8]) -> Vec { + let half = raw.len().div_ceil(2); + let mut t = vec![0_u8; raw.len()]; + for (i, &b) in raw.iter().enumerate() { + t[if i % 2 == 0 { i / 2 } else { half + i / 2 }] = b; + } + let mut p = t.first().copied().unwrap_or(0); + for v in t.iter_mut().skip(1) { + let d = v.wrapping_sub(p).wrapping_add(128); + p = *v; + *v = d; + } + t +} + +/// OpenEXR run-length coding: a signed count byte, then either `count + 1` copies of one byte +/// (count ≥ 0, runs of 3 to 128) or `-count` literal bytes (1 to 127). +pub fn rle(src: &[u8]) -> Vec { + let mut out = Vec::with_capacity(src.len() + src.len() / 127 + 1); + let mut i = 0; + while i < src.len() { + let b = src[i]; + let run = src[i..].iter().take(128).take_while(|&&x| x == b).count(); + if run >= 3 { + out.extend_from_slice(&[(run - 1) as u8, b]); + i += run; + continue; + } + // literals up to the next run of three (or 127 bytes) + let s = i; + while i < src.len() && i - s < 127 { + if src.get(i..i + 3).is_some_and(|t| t[0] == t[1] && t[1] == t[2]) { + break; + } + i += 1; + } + out.push(((i - s) as u8).wrapping_neg()); + out.extend_from_slice(&src[s..i]); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn writes_all_scanline_compressions() { + let linear = std::array::from_fn(|i| i as f32 / 255.0); + for c in [Compression::None, Compression::Rle, Compression::Zips, Compression::Zip] { + let bytes = write_rgba8(&[128; 16 * 16 * 4], 16, 16, true, c, &linear).unwrap(); + assert_eq!(&bytes[..4], &[0x76, 0x2f, 0x31, 1]); + assert!(bytes.len() > 100); + } + assert_eq!(write_rgba8(&[], 1, 1, false, Compression::None, &linear), Err(Error::FrameSize)); + } +} diff --git a/crates/fcproj-format/Cargo.toml b/crates/fcproj-format/Cargo.toml new file mode 100644 index 000000000..aa210187d --- /dev/null +++ b/crates/fcproj-format/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "craft-fcproj-format" +description = "FilmCraft project envelope and schema migrations" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/fcproj-format/src/lib.rs b/crates/fcproj-format/src/lib.rs new file mode 100644 index 000000000..98e50cf59 --- /dev/null +++ b/crates/fcproj-format/src/lib.rs @@ -0,0 +1,297 @@ +//! FilmCraft `.fcproj` JSON envelope, version detection and historical migrations. +//! Project and editor-view types are supplied by the caller. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] +use serde::{Deserialize, Serialize}; +use serde_json::Value; +/// The `format` tag written in every envelope. +pub const FORMAT_ID: &str = "filmcraft.project"; + +/// A migration upgrades a whole document from schema `n` to `n + 1`. +pub type Migration = fn(Value) -> Result; + +/// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never +/// edit a shipped one. +pub const MIGRATIONS: &[Migration] = &[ + v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13, v13_to_v14, + v14_to_v15, v15_to_v16, v16_to_v17, +]; + +/// The schema version this build writes (and the newest it reads). +pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1; + +/// Oldest schema this build can still upgrade. +pub const OLDEST_SCHEMA: u32 = 1; + +#[derive(Debug, thiserror::Error)] +pub enum FormatError { + #[error("not a FilmCraft project: {0}")] + NotAProject(String), + #[error( + "this project was saved by a newer version of FilmCraft (project schema v{found}); this build reads up to v{supported}. Update FilmCraft to open it." + )] + TooNew { found: u32, supported: u32 }, + #[error("project schema v{found} is too old for this build (oldest supported: v{oldest})")] + TooOld { found: u32, oldest: u32 }, + #[error("upgrading project from schema v{from} to v{}: {msg}", from + 1)] + Migration { from: u32, msg: String }, + #[error("project file is damaged: {0}")] + Corrupt(String), +} + +/// Serialized current-schema envelope. +#[derive(Serialize)] +pub struct EnvelopeOut<'a, P, V> { + pub format: &'static str, + pub schema_version: u32, + pub generator: String, + pub project: &'a P, + #[serde(skip_serializing_if = "Option::is_none")] + pub view: Option<&'a V>, +} +/// Parsed envelope. View parsing remains optional to preserve forward compatibility. +#[derive(Deserialize)] +pub struct EnvelopeIn

{ + #[serde(default)] + pub generator: Option, + pub project: P, + #[serde(default)] + pub view: Option, +} +/// Serialize the current envelope, with optional editor view and caller-provided generator. +pub fn encode(project: &P, view: Option<&V>, generator: String, pretty: bool) -> Vec { + let envelope = EnvelopeOut { format: FORMAT_ID, schema_version: SCHEMA_VERSION, generator, project, view }; + let result = if pretty { serde_json::to_vec_pretty(&envelope) } else { serde_json::to_vec(&envelope) }; + result.unwrap_or_default() +} + +/// Decode an envelope, migrating older documents while retaining the current-version fast path. +pub fn decode(bytes: &[u8]) -> Result<(EnvelopeIn

, u32), FormatError> { + let found = detect_bytes(bytes)?; + if found == SCHEMA_VERSION { + let env = serde_json::from_slice(bytes).map_err(|e| FormatError::Corrupt(e.to_string()))?; + return Ok((env, found)); + } + let doc: Value = serde_json::from_slice(bytes).map_err(|e| FormatError::Corrupt(e.to_string()))?; + let doc = migrate_with(MIGRATIONS, doc, found)?; + let env = serde_json::from_value(doc).map_err(|e| FormatError::Corrupt(format!("after upgrading from schema v{found}: {e}")))?; + Ok((env, found)) +} + +/// Probe and validate the schema directly from bytes without building a JSON tree. +pub fn detect_bytes(bytes: &[u8]) -> Result { + let probe: Probe = serde_json::from_slice(bytes) + .map_err(|e| if e.is_syntax() || e.is_eof() { FormatError::Corrupt(e.to_string()) } else { FormatError::NotAProject(e.to_string()) })?; + let found = probe_version(&probe)?; + check_supported(found)?; + Ok(found) +} +/// Just enough of a document to know which schema it is. +#[derive(Deserialize)] +struct Probe { + #[serde(default)] + format: Option, + #[serde(default)] + schema_version: Option, + /// Bare (pre-envelope) project: its `version` field. + #[serde(default)] + version: Option, + #[serde(default)] + items: Option, + #[serde(default)] + root: Option, +} + +/// Detect the schema version of a parsed document. +pub fn detect_version(doc: &Value) -> Result { + let probe: Probe = serde_json::from_value(doc.clone()).map_err(|e| FormatError::NotAProject(e.to_string()))?; + probe_version(&probe) +} + +fn probe_version(p: &Probe) -> Result { + if let Some(v) = p.schema_version { + if let Some(f) = &p.format + && f != FORMAT_ID + { + return Err(FormatError::NotAProject(format!("format is `{f}`, expected `{FORMAT_ID}`"))); + } + return Ok(v); + } + // Before the envelope (v1) a file was the bare Project object with `version`. + // Its `version` was 1, or 2 on builds that briefly bumped it when caption tracks were added + // (same shape; the new field loads with its default). Anything higher is from the future. + if p.items.is_some() && p.root.is_some() { + return Ok(match p.version.unwrap_or(1) { + 0..=2 => 1, + v => v, + }); + } + Err(FormatError::NotAProject("no `schema_version` and not a legacy project".into())) +} + +pub fn check_supported(found: u32) -> Result<(), FormatError> { + if found > SCHEMA_VERSION { + return Err(FormatError::TooNew { found, supported: SCHEMA_VERSION }); + } + if found < OLDEST_SCHEMA { + return Err(FormatError::TooOld { found, oldest: OLDEST_SCHEMA }); + } + Ok(()) +} + +/// Run `table` from schema `from` up to `table.len() + 1`, stamping `schema_version` after each step. +pub fn migrate_with(table: &[Migration], mut doc: Value, from: u32) -> Result { + let target = table.len() as u32 + 1; + if from > target { + return Err(FormatError::TooNew { found: from, supported: target }); + } + if from == 0 { + return Err(FormatError::TooOld { found: 0, oldest: 1 }); + } + for v in from..target { + let step = table[(v - 1) as usize]; + doc = step(doc).map_err(|msg| FormatError::Migration { from: v, msg })?; + match doc.as_object_mut() { + Some(o) => { + o.insert("schema_version".into(), Value::from(v + 1)); + } + None => return Err(FormatError::Migration { from: v, msg: "migration produced a non-object document".into() }), + } + } + Ok(doc) +} + +/// v1 → v2: wrap the bare project in the envelope and drop the per-project `version` field. +fn v1_to_v2(doc: Value) -> Result { + let Value::Object(mut project) = doc else { return Err("expected a JSON object".into()) }; + project.remove("version"); + Ok(serde_json::json!({ + "format": FORMAT_ID, + "schema_version": 2, + "generator": "FilmCraft (schema v1)", + "project": Value::Object(project), + })) +} + +/// v2 → v3: graphic clips added; existing data needs no change. +fn v2_to_v3(doc: Value) -> Result { + Ok(doc) +} + +/// v3 → v4: Essential Sound fields added; existing data needs no change. +fn v3_to_v4(doc: Value) -> Result { + Ok(doc) +} + +/// v4 → v5: colour-management fields added; existing data needs no change. +fn v4_to_v5(doc: Value) -> Result { + Ok(doc) +} + +/// v5 → v6: media identity and ingest settings added; existing data needs no change. +fn v5_to_v6(doc: Value) -> Result { + Ok(doc) +} + +/// v6 → v7: effect masks (Bézier mask paths as keyframable parameters); existing data needs no change. +fn v6_to_v7(doc: Value) -> Result { + Ok(doc) +} + +/// v7 → v8: multi-camera source sequences / clips and merged clips added; existing data needs no +/// change. +fn v7_to_v8(doc: Value) -> Result { + Ok(doc) +} + +/// v8 → v9: per-media-item transcripts (`project.transcripts`) added; existing data needs no change. +fn v8_to_v9(doc: Value) -> Result { + Ok(doc) +} + +/// v9 → v10: clip time interpolation, Hold Filters, Field Options and audio source channels; the +/// Modify ▸ Audio Channels map; subclips' Restrict Trims flag. Existing data needs no change. +fn v9_to_v10(doc: Value) -> Result { + Ok(doc) +} + +/// v10 → v11: search bins (`project.search_bins`), Flash Cue markers, Project Settings safe areas, +/// capture format and scratch disks. Existing data needs no change. +fn v10_to_v11(doc: Value) -> Result { + Ok(doc) +} + +/// v11 → v12: graphics templates, rolls / crawls, responsive design and per-character text styles +/// on graphic clips, and source graphics. Existing data needs no change. +fn v11_to_v12(doc: Value) -> Result { + Ok(doc) +} + +/// v12 → v13: container audio streams and per-clip stream selection. `MediaInfo` reads the old +/// `audio` field as stream zero; `TrackItem::audio_stream` defaults to zero. +fn v12_to_v13(doc: Value) -> Result { + Ok(doc) +} + +/// v13 → v14: narrations (Text to Speech). Existing data needs no change. +fn v13_to_v14(doc: Value) -> Result { + Ok(doc) +} + +/// v14 → v15: project notes. Existing data needs no change. +fn v14_to_v15(doc: Value) -> Result { + Ok(doc) +} + +/// v15 → v16: existing colors remain presets; authors and the named custom palette default empty. +fn v15_to_v16(doc: Value) -> Result { + Ok(doc) +} + +/// v16 → v17: marker URLs default empty. Older builds must not silently drop saved URLs. +fn v16_to_v17(doc: Value) -> Result { + Ok(doc) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn legacy_and_current_envelopes_preserve_values() { + let bare = json!({"version": 2, "items": {}, "root": {}, "extra": [1, 2]}); + assert_eq!(detect_version(&bare).unwrap(), 1); + let migrated = migrate_with(MIGRATIONS, bare, 1).unwrap(); + assert_eq!(migrated["schema_version"], 17); + assert_eq!(migrated["project"]["extra"], json!([1, 2])); + assert_eq!(migrated["generator"], "FilmCraft (schema v1)"); + assert_eq!(detect_bytes(serde_json::to_string(&migrated).unwrap().as_bytes()).unwrap(), 17); + let parsed: EnvelopeIn = serde_json::from_value(migrated).unwrap(); + assert!(parsed.view.is_none()); + assert_eq!(parsed.project["extra"], json!([1, 2])); + } + #[test] + fn typed_encode_and_decode_current_and_legacy() { + let project = json!({"items": {}, "root": {}, "name": "Cut"}); + let view = json!({"active": 3}); + for pretty in [false, true] { + let bytes = encode(&project, Some(&view), "FilmCraft test".into(), pretty); + let (file, version): (EnvelopeIn, _) = decode(&bytes).unwrap(); + assert_eq!(version, SCHEMA_VERSION); + assert_eq!(file.project, project); + assert_eq!(file.view, Some(view.clone())); + } + let old = br#"{"version":1,"items":{},"root":{},"name":"Cut"}"#; + let (file, version): (EnvelopeIn, _) = decode(old).unwrap(); + assert_eq!(version, 1); + assert_eq!(file.project["name"], "Cut"); + } + #[test] + fn rejects_wrong_future_and_damaged_envelopes() { + assert!(matches!(detect_bytes(br#"{"format":"other","schema_version":1}"#), Err(FormatError::NotAProject(_)))); + assert!(matches!(detect_bytes(br#"{"schema_version":18}"#), Err(FormatError::TooNew { found: 18, supported: 17 }))); + assert!(matches!(detect_bytes(br#"{"schema_version":0}"#), Err(FormatError::TooOld { found: 0, oldest: 1 }))); + assert!(matches!(detect_bytes(b"{"), Err(FormatError::Corrupt(_)))); + assert!(matches!(migrate_with(MIGRATIONS, json!([]), 1), Err(FormatError::Migration { from: 1, .. }))); + } +} diff --git a/crates/flow/src/from_doc.rs b/crates/flow/src/from_doc.rs index 341ded449..810e67d55 100644 --- a/crates/flow/src/from_doc.rs +++ b/crates/flow/src/from_doc.rs @@ -493,6 +493,8 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec { kind, list, align: p.props.align, + // Direction can come from the paragraph's style (`w:bidi` in the style's pPr). + rtl: doc.styles.resolve_para(&p.props).bidi, page_break: p.props.page_break_before.unwrap_or(false), keep_next: p.props.keep_next.unwrap_or(false), inlines: Vec::new(), @@ -536,6 +538,7 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec { kind, list: None, align: p.props.align, + rtl: out.rtl, page_break: true, keep_next: p.props.keep_next.unwrap_or(false), inlines: Vec::new(), @@ -658,7 +661,10 @@ fn flow_paras_in(doc: &Document, p: &Paragraph, opts: FlowOpts) -> Vec { { (li.list, li.label) = (0, None); } - let old = Para { kind, list, align: c.old.align, page_break: c.old.page_break_before.unwrap_or(false), ..Default::default() }; + // Direction resolved like the current paragraph's: the old properties' own `w:bidi`, else + // the style's. + let rtl = doc.styles.resolve_para(&c.old).bidi; + let old = Para { kind, list, align: c.old.align, rtl, page_break: c.old.page_break_before.unwrap_or(false), ..Default::default() }; Box::new(FmtChange { mark: rev_mark(doc, c.rev), old }) }); } @@ -749,7 +755,7 @@ fn graphic_group( let inline = Inline::Shape { kind: *kind, w: rect[2] * sx, h: rect[3] * sy, float: None, blocks: Vec::new(), look, path: None }; Some(Member { x: rect[0] * sx, y: rect[1] * sy, inline }) } - GraphicItem::Text { rect, text, size, color, bold, align, font } if finite(rect) && !text.trim().is_empty() => { + GraphicItem::Text { rect, text, size, color, bold, align, font, .. } if finite(rect) && !text.trim().is_empty() => { let size = if size.is_finite() { size.clamp(1.0, 400.0) } else { 11.0 }; // One line, centred vertically in its place as the graphic draws it. let line = size * 1.2; diff --git a/crates/flow/src/to_doc.rs b/crates/flow/src/to_doc.rs index c10be85c2..ce29a4448 100644 --- a/crates/flow/src/to_doc.rs +++ b/crates/flow/src/to_doc.rs @@ -159,6 +159,9 @@ fn para_props(p: &Para, list_num: Option) -> ParaProps { out.numbering = Some(NumRef { num, level: li.level.min(8) }); } out.align = p.align; + if p.rtl { + out.bidi = Some(true); + } let sp = p.spacing; out.space_before = sp.before; out.space_after = sp.after; diff --git a/crates/flow/src/types.rs b/crates/flow/src/types.rs index e897778f9..f1c281c62 100644 --- a/crates/flow/src/types.rs +++ b/crates/flow/src/types.rs @@ -676,6 +676,8 @@ pub struct Para { pub kind: Kind, pub list: Option, pub align: Option, + /// The paragraph reads right to left (OOXML `w:bidi`, RTF `\rtlpar`). + pub rtl: bool, pub page_break: bool, /// Keep the paragraph on the same page as the next one (ODT `fo:keep-with-next`). pub keep_next: bool, diff --git a/crates/font-catalog/Cargo.toml b/crates/font-catalog/Cargo.toml index a95e1c28f..1f0830a63 100644 --- a/crates/font-catalog/Cargo.toml +++ b/crates/font-catalog/Cargo.toml @@ -15,6 +15,7 @@ default = ["system"] system = ["dep:fontique"] [dependencies] +kurbo = { workspace = true } skrifa = { version = "0.47", default-features = false, features = ["std"] } thiserror = { workspace = true } diff --git a/crates/font-catalog/src/lib.rs b/crates/font-catalog/src/lib.rs index 4970217b6..3f155d35d 100644 --- a/crates/font-catalog/src/lib.rs +++ b/crates/font-catalog/src/lib.rs @@ -24,8 +24,11 @@ mod catalog; mod embedding; mod env; mod fontconfig; +mod outline; mod registry; mod scan; +/// Bounded SFNT face metadata discovery over caller-provided random-access readers. +pub mod sfnt; mod suitcase; pub mod system; @@ -33,6 +36,7 @@ pub use catalog::{Catalog, FontData, FontFileError, Limits, Lister, LoadedFace, pub use embedding::{SubsetEmbedding, subset_embedding}; pub use env::{FontEnv, Os, font_dirs, font_store_dir, office_font_dirs}; pub use fontconfig::fontconfig_dirs; +pub use outline::{FontOutline, GlyphPen, OutlineError}; pub use scan::{FaceInfo, LocalizedName, Source, is_font_file, style_italic, style_weight, style_width}; #[cfg(test)] diff --git a/crates/font-catalog/src/outline.rs b/crates/font-catalog/src/outline.rs new file mode 100644 index 000000000..f48b61e4f --- /dev/null +++ b/crates/font-catalog/src/outline.rs @@ -0,0 +1,399 @@ +//! Unhinted glyph outlines and metrics for catalog faces. + +use kurbo::BezPath; +use skrifa::instance::{Location, Size}; +use skrifa::outline::{DrawSettings, OutlinePen}; +use skrifa::{FontRef, GlyphId, MetadataProvider, Tag}; + +use crate::{FontData, LoadedFace}; + +/// A parsed face and its variation location. Coordinates use font design units, with y up. +/// The bytes can come from a catalog or any other source, including a collection face. +pub struct FontOutline<'a> { + font: FontRef<'a>, + location: Location, +} + +/// Receives unhinted glyph commands in design units (y up). Implementations own their +/// storage and resource policy; drawing writes directly to the sink without an intermediate path. +pub trait GlyphPen { + /// Start a contour. + fn move_to(&mut self, x: f32, y: f32); + /// Add a straight segment. + fn line_to(&mut self, x: f32, y: f32); + /// Add a quadratic segment. + fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32); + /// Add a cubic segment. + fn curve_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32); + /// Close the current contour. + fn close(&mut self); +} + +struct Sink<'a>(&'a mut dyn GlyphPen); +impl OutlinePen for Sink<'_> { + fn move_to(&mut self, x: f32, y: f32) { + self.0.move_to(x, y); + } + fn line_to(&mut self, x: f32, y: f32) { + self.0.line_to(x, y); + } + fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32) { + self.0.quad_to(x1, y1, x, y); + } + fn curve_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) { + self.0.curve_to(x1, y1, x2, y2, x, y); + } + fn close(&mut self) { + self.0.close(); + } +} + +/// Why a font or glyph outline could not be read. +#[derive(Debug, thiserror::Error)] +pub enum OutlineError { + /// The face index or font data is invalid. + #[error("invalid font face")] + InvalidFace, + /// The glyph has no outline in this face. + #[error("glyph has no outline")] + MissingGlyph, + /// The glyph's outline is malformed. + #[error("cannot draw glyph outline: {0}")] + Draw(#[from] skrifa::outline::DrawError), + /// The glyph emitted more path elements than the safety limit. + #[error("glyph outline exceeds the element limit")] + TooComplex, + /// The outline starts without a move command. + #[error("glyph outline has no initial move")] + InvalidPath, +} + +impl<'a> FontOutline<'a> { + /// Parse the given face and select axis values in user units. Unknown axes are ignored and + /// values are clamped to their axis ranges by the font parser. + pub fn new(data: &'a FontData, variations: &[([u8; 4], f32)]) -> Result { + Self::from_bytes(&data.bytes, data.index, variations) + } + + /// Parse borrowed bytes and a collection face index without copying the font. + pub fn from_bytes(bytes: &'a [u8], index: u32, variations: &[([u8; 4], f32)]) -> Result { + let font = FontRef::from_index(bytes, index).map_err(|_| OutlineError::InvalidFace)?; + let location = font.axes().location(variations.iter().map(|(tag, value)| (Tag::new(tag), *value))); + Ok(Self { font, location }) + } + + /// Parse a catalog face, retaining the axis settings of a named instance. + pub fn from_loaded(face: &'a LoadedFace) -> Result { + Self::new(&face.data, &face.info.variations) + } + + /// Font-wide metrics in unscaled design units at this variation location. + pub fn metrics(&self) -> skrifa::metrics::Metrics { + self.font.metrics(Size::unscaled(), &self.location) + } + + /// Resolve a Unicode scalar to a glyph identifier. + pub fn glyph_for(&self, character: char) -> Option { + self.font.charmap().map(character).map(|id| id.to_u32()) + } + + /// Stream an unhinted glyph into the caller's sink. On an error, commands already sent + /// remain in the sink. The sink owns any allocation and enforces its own resource limit. + pub fn draw_to(&self, glyph: u32, pen: &mut dyn GlyphPen) -> Result<(), OutlineError> { + let outline = self.font.outline_glyphs().get(GlyphId::new(glyph)).ok_or(OutlineError::MissingGlyph)?; + outline.draw(DrawSettings::unhinted(Size::unscaled(), &self.location), &mut Sink(pen))?; + Ok(()) + } + + /// Draw a glyph in design units. Empty glyphs such as spaces can produce an empty path. + /// A malformed or excessively complex glyph returns an error instead of partial geometry. + pub fn outline(&self, glyph: u32) -> Result { + let (path, status) = self.outline_recovering(glyph)?; + status.map(|()| path) + } + + /// Draw a glyph while retaining the bounded prefix produced before a draw error. This is + /// useful for consumers preserving existing rendering of damaged fonts. The inner result + /// reports whether the returned path is complete; the outer error means no glyph was found. + /// Paths stop growing at 100,000 elements, even when the font is untrusted. + pub fn outline_recovering(&self, glyph: u32) -> Result<(BezPath, Result<(), OutlineError>), OutlineError> { + let outline = self.font.outline_glyphs().get(GlyphId::new(glyph)).ok_or(OutlineError::MissingGlyph)?; + let mut pen = PathPen::new(); + let result = outline.draw(DrawSettings::unhinted(Size::unscaled(), &self.location), &mut pen); + let status = if pen.invalid { + Err(OutlineError::InvalidPath) + } else if pen.truncated { + Err(OutlineError::TooComplex) + } else { + result.map(|_| ()).map_err(OutlineError::Draw) + }; + Ok((pen.path, status)) + } +} + +const MAX_OUTLINE_ELEMENTS: usize = 100_000; +struct PathPen { + path: BezPath, + truncated: bool, + invalid: bool, +} +impl PathPen { + fn new() -> Self { + Self { path: BezPath::new(), truncated: false, invalid: false } + } + fn has_start(&mut self) -> bool { + if self.path.elements().is_empty() { + self.invalid = true; + false + } else { + true + } + } + fn room(&mut self) -> bool { + if self.path.elements().len() >= MAX_OUTLINE_ELEMENTS { + self.truncated = true; + false + } else { + true + } + } +} +impl OutlinePen for PathPen { + fn move_to(&mut self, x: f32, y: f32) { + if self.room() { + self.path.move_to((f64::from(x), f64::from(y))); + } + } + fn line_to(&mut self, x: f32, y: f32) { + if self.has_start() && self.room() { + self.path.line_to((f64::from(x), f64::from(y))); + } + } + fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32) { + if self.has_start() && self.room() { + self.path.quad_to((f64::from(x1), f64::from(y1)), (f64::from(x), f64::from(y))); + } + } + fn curve_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) { + if self.has_start() && self.room() { + self.path.curve_to((f64::from(x1), f64::from(y1)), (f64::from(x2), f64::from(y2)), (f64::from(x), f64::from(y))); + } + } + fn close(&mut self) { + if self.has_start() && self.room() { + self.path.close_path(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kurbo::PathEl; + use std::sync::Arc; + + impl GlyphPen for PathPen { + fn move_to(&mut self, x: f32, y: f32) { + OutlinePen::move_to(self, x, y); + } + fn line_to(&mut self, x: f32, y: f32) { + OutlinePen::line_to(self, x, y); + } + fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32) { + OutlinePen::quad_to(self, x1, y1, x, y); + } + fn curve_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) { + OutlinePen::curve_to(self, x1, y1, x2, y2, x, y); + } + fn close(&mut self) { + OutlinePen::close(self); + } + } + + #[test] + fn rejects_invalid_face_and_collection_index() { + let data = FontData { bytes: Arc::from(&b"not a font"[..]), index: 0 }; + assert!(matches!(FontOutline::new(&data, &[]), Err(OutlineError::InvalidFace))); + } + + #[test] + fn pen_preserves_geometry_and_upward_y() { + let mut pen = PathPen::new(); + OutlinePen::move_to(&mut pen, 1.0, 2.0); + OutlinePen::line_to(&mut pen, 3.0, 4.0); + OutlinePen::quad_to(&mut pen, 5.0, 6.0, 7.0, 8.0); + OutlinePen::curve_to(&mut pen, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0); + OutlinePen::close(&mut pen); + assert_eq!( + pen.path.elements(), + &[ + PathEl::MoveTo((1.0, 2.0).into()), + PathEl::LineTo((3.0, 4.0).into()), + PathEl::QuadTo((5.0, 6.0).into(), (7.0, 8.0).into()), + PathEl::CurveTo((9.0, 10.0).into(), (11.0, 12.0).into(), (13.0, 14.0).into()), + PathEl::ClosePath, + ] + ); + } + + #[test] + fn streaming_adapter_forwards_each_command_and_rejects_an_invalid_prefix() { + let mut pen = PathPen::new(); + { + let mut sink = Sink(&mut pen); + OutlinePen::move_to(&mut sink, 1.0, 2.0); + OutlinePen::line_to(&mut sink, 3.0, 4.0); + OutlinePen::quad_to(&mut sink, 5.0, 6.0, 7.0, 8.0); + OutlinePen::curve_to(&mut sink, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0); + OutlinePen::close(&mut sink); + } + assert_eq!(pen.path.elements().len(), 5); + assert_eq!(pen.path.elements()[2], PathEl::QuadTo((5.0, 6.0).into(), (7.0, 8.0).into())); + let mut invalid = PathPen::new(); + OutlinePen::line_to(&mut invalid, 1.0, 2.0); + assert!(invalid.invalid); + assert!(invalid.path.elements().is_empty()); + } + + #[test] + fn pen_caps_path_growth() { + let mut pen = PathPen::new(); + OutlinePen::move_to(&mut pen, 0.0, 0.0); + for _ in 0..MAX_OUTLINE_ELEMENTS { + OutlinePen::line_to(&mut pen, 1.0, 2.0); + } + assert_eq!(pen.path.elements().len(), MAX_OUTLINE_ELEMENTS); + assert!(pen.truncated); + } + + #[test] + fn reads_craft_font_when_available() { + let Ok(dir) = std::env::var("CRAFT_FONTS_DIR") else { return }; + let bytes = std::fs::read(std::path::Path::new(&dir).join("fonts/noto-sans-hebrew/NotoSansHebrew-Regular.ttf")).unwrap(); + let data = FontData { bytes: bytes.into(), index: 0 }; + let face = FontOutline::new(&data, &[]).unwrap(); + assert!(face.metrics().units_per_em > 0); + let glyph = face.glyph_for('א').unwrap(); + assert!(!face.outline(glyph).unwrap().elements().is_empty()); + assert!(matches!(face.outline(u32::MAX), Err(OutlineError::MissingGlyph))); + let invalid_index = FontData { bytes: data.bytes.clone(), index: u32::MAX }; + assert!(matches!(FontOutline::new(&invalid_index, &[]), Err(OutlineError::InvalidFace))); + } + + #[test] + fn variation_changes_glyph_geometry() { + let Ok(dir) = std::env::var("CRAFT_FONTS_DIR") else { return }; + let bytes = std::fs::read(std::path::Path::new(&dir).join("fonts/noto-sans-arabic/NotoSansArabic.ttf")).unwrap(); + let data = FontData { bytes: bytes.into(), index: 0 }; + let light = FontOutline::new(&data, &[(*b"wght", 100.0)]).unwrap(); + let bold = FontOutline::new(&data, &[(*b"wght", 900.0)]).unwrap(); + let glyph = light.glyph_for('ب').unwrap(); + let light_path = light.outline(glyph).unwrap(); + let bold_path = bold.outline(glyph).unwrap(); + assert!(!light_path.elements().is_empty()); + assert_ne!(light_path, bold_path); + } + + // Construct a minimal CFF font so CI can exercise the public API without installed fonts. + fn cff_font(program: &[u8]) -> Vec { + let mut cff = vec![1, 0, 4, 4, 0, 1, 1, 1, 2, b'X', 0, 1, 1, 1, 7, 29, 0, 0, 0, 25, 17, 0, 0, 0, 0]; + cff.extend_from_slice(&[0, 1, 4]); + cff.extend_from_slice(&1u32.to_be_bytes()); + cff.extend_from_slice(&((program.len() as u32) + 1).to_be_bytes()); + cff.extend_from_slice(program); + let mut builder = write_fonts::FontBuilder::new(); + let mut head = vec![0; 54]; + head[18..20].copy_from_slice(&1000u16.to_be_bytes()); + let mut hhea = vec![0; 36]; + hhea[34..36].copy_from_slice(&1u16.to_be_bytes()); + builder.add_raw(write_fonts::types::Tag::new(b"head"), head); + builder.add_raw(write_fonts::types::Tag::new(b"hhea"), hhea); + builder.add_raw(write_fonts::types::Tag::new(b"hmtx"), vec![0, 100, 0, 0]); + builder.add_raw(write_fonts::types::Tag::new(b"maxp"), vec![0, 0, 0x50, 0, 0, 1]); + builder.add_raw(write_fonts::types::Tag::new(b"CFF "), cff); + builder.build() + } + + #[test] + fn malformed_cff_recovers_the_same_prefix_as_direct_drawing() { + // Move and line, then callsubr without an operand: a prefix is emitted before failure. + let bytes = cff_font(&[139, 139, 21, 239, 139, 5, 10, 14]); + let gid = 0; + let f = FontRef::new(&bytes).unwrap(); + let mut old = PathPen::new(); + let result = + f.outline_glyphs().get(GlyphId::new(gid)).unwrap().draw(DrawSettings::unhinted(Size::unscaled(), &Location::default()), &mut old); + assert!(result.is_err()); + assert!(!old.path.elements().is_empty()); + let shared = FontOutline::from_bytes(&bytes, 0, &[]).unwrap(); + assert_eq!(shared.metrics().units_per_em, 1000); + assert_eq!(shared.glyph_for('A'), None); + let (prefix, status) = shared.outline_recovering(gid).unwrap(); + assert!(matches!(status, Err(OutlineError::Draw(_)))); + assert_eq!(prefix, old.path); + let mut streamed = PathPen::new(); + assert!(matches!(shared.draw_to(gid, &mut streamed), Err(OutlineError::Draw(_)))); + assert_eq!(streamed.path, old.path); + assert!(matches!(shared.outline(gid), Err(OutlineError::Draw(_)))); + } + + #[test] + fn streaming_preserves_a_large_valid_cff_outline() { + // Minimal CFF with more than 100,000 line segments. + let mut program = vec![139, 139, 21]; + for _ in 0..100_001 { + program.extend_from_slice(&[140, 6]); + } + program.push(14); + let bytes = cff_font(&program); + let shared = FontOutline::from_bytes(&bytes, 0, &[]).unwrap(); + struct Count(usize); + impl GlyphPen for Count { + fn move_to(&mut self, _: f32, _: f32) { + self.0 += 1; + } + fn line_to(&mut self, _: f32, _: f32) { + self.0 += 1; + } + fn quad_to(&mut self, _: f32, _: f32, _: f32, _: f32) { + self.0 += 1; + } + fn curve_to(&mut self, _: f32, _: f32, _: f32, _: f32, _: f32, _: f32) { + self.0 += 1; + } + fn close(&mut self) { + self.0 += 1; + } + } + impl OutlinePen for Count { + fn move_to(&mut self, x: f32, y: f32) { + GlyphPen::move_to(self, x, y); + } + fn line_to(&mut self, x: f32, y: f32) { + GlyphPen::line_to(self, x, y); + } + fn quad_to(&mut self, x1: f32, y1: f32, x: f32, y: f32) { + GlyphPen::quad_to(self, x1, y1, x, y); + } + fn curve_to(&mut self, x1: f32, y1: f32, x2: f32, y2: f32, x: f32, y: f32) { + GlyphPen::curve_to(self, x1, y1, x2, y2, x, y); + } + fn close(&mut self) { + GlyphPen::close(self); + } + } + let mut old = Count(0); + FontRef::new(&bytes) + .unwrap() + .outline_glyphs() + .get(GlyphId::new(0)) + .unwrap() + .draw(DrawSettings::unhinted(Size::unscaled(), &Location::default()), &mut old) + .unwrap(); + let mut count = Count(0); + shared.draw_to(0, &mut count).unwrap(); + assert_eq!(count.0, old.0); + assert!(count.0 > MAX_OUTLINE_ELEMENTS); + assert!(matches!(shared.outline(0), Err(OutlineError::TooComplex))); + } +} diff --git a/crates/font-catalog/src/sfnt.rs b/crates/font-catalog/src/sfnt.rs new file mode 100644 index 000000000..83efa64dd --- /dev/null +++ b/crates/font-catalog/src/sfnt.rs @@ -0,0 +1,178 @@ +//! Minimal SFNT (TrueType / OpenType / collection) header reader for font discovery: reads only +//! the table directory, `name` and `OS/2` of each face, so scanning a system font folder does not +//! load whole files. + +use std::io::{Read, Seek, SeekFrom}; + +/// The tables skrifa reads glyph outlines from: TrueType, CFF, CFF2 and VARC. The renderer draws +/// no glyph of a face that has none of them, such as a face with bitmaps only or with its outlines +/// in Apple's `hvgl` table. +const OUTLINE_TABLES: [&[u8; 4]; 4] = [b"glyf", b"CFF ", b"CFF2", b"VARC"]; + +/// Names and style attributes of one face in a font file. +#[derive(Clone, Debug, PartialEq)] +pub struct FaceNames { + pub index: u32, + pub family: String, + pub style: String, + pub full_name: String, + pub weight: u16, + pub italic: bool, +} + +fn be16(b: &[u8], o: usize) -> Option { + Some(u16::from_be_bytes([*b.get(o)?, *b.get(o + 1)?])) +} +fn be32(b: &[u8], o: usize) -> Option { + Some(u32::from_be_bytes([*b.get(o)?, *b.get(o + 1)?, *b.get(o + 2)?, *b.get(o + 3)?])) +} + +fn read_at(r: &mut R, off: u64, len: usize) -> Option> { + if len > 4 << 20 { + return None; + } + r.seek(SeekFrom::Start(off)).ok()?; + let mut v = vec![0u8; len]; + r.read_exact(&mut v).ok()?; + Some(v) +} + +/// Read the names of every face in a font file (a single font or a collection). Faces with none +/// of the `OUTLINE_TABLES` are left out. +pub fn read_faces(r: &mut R) -> Vec { + let Some(head) = read_at(r, 0, 12) else { return Vec::new() }; + let offsets: Vec = if &head[0..4] == b"ttcf" { + let n = be32(&head, 8).unwrap_or(0).min(256) as usize; + let Some(t) = read_at(r, 12, n * 4) else { return Vec::new() }; + (0..n).filter_map(|i| be32(&t, i * 4)).collect() + } else { + vec![0] + }; + offsets.iter().enumerate().filter_map(|(i, &o)| read_face(r, o as u64, i as u32)).collect() +} + +/// Names from in-memory font data. +pub fn read_faces_bytes(data: &[u8]) -> Vec { + read_faces(&mut std::io::Cursor::new(data)) +} + +fn read_face(r: &mut R, off: u64, index: u32) -> Option { + let h = read_at(r, off, 12)?; + let tag = &h[0..4]; + if tag != [0, 1, 0, 0] && tag != b"OTTO" && tag != b"true" { + return None; + } + let n = be16(&h, 4)? as usize; + let dir = read_at(r, off + 12, n * 16)?; + if !dir.as_chunks::<16>().0.iter().any(|rec| OUTLINE_TABLES.iter().any(|t| rec.starts_with(*t))) { + return None; + } + let mut name = None; + let mut os2 = None; + for i in 0..n { + let rec = &dir[i * 16..i * 16 + 16]; + let (o, l) = (be32(rec, 8)? as u64, be32(rec, 12)? as usize); + match &rec[0..4] { + b"name" => name = Some((o, l)), + b"OS/2" => os2 = Some((o, l)), + _ => {} + } + } + let (no, nl) = name?; + let nb = read_at(r, no, nl)?; + let get = |id: u16| name_string(&nb, id); + let family = get(16).or_else(|| get(1))?; + let style = get(17).or_else(|| get(2)).unwrap_or_else(|| "Regular".into()); + let full_name = get(4).unwrap_or_else(|| format!("{family} {style}")); + let (mut weight, mut italic) = (400u16, false); + if let Some((oo, ol)) = os2 + && let Some(ob) = read_at(r, oo, ol.min(78)) + { + weight = be16(&ob, 4).unwrap_or(400); + italic = be16(&ob, 62).is_some_and(|fs| fs & 1 != 0); + } + let sl = style.to_ascii_lowercase(); + italic |= sl.contains("italic") || sl.contains("oblique"); + Some(FaceNames { index, family, style, full_name, weight, italic }) +} + +/// A name-table string by id, preferring Windows Unicode English, then any Unicode, then Mac Roman. +fn name_string(b: &[u8], id: u16) -> Option { + let count = be16(b, 2)? as usize; + let storage = be16(b, 4)? as usize; + let mut best: Option<(u8, String)> = None; + for i in 0..count { + let r = 6 + i * 12; + let (plat, enc, lang, nid, len, off) = + (be16(b, r)?, be16(b, r + 2)?, be16(b, r + 4)?, be16(b, r + 6)?, be16(b, r + 8)? as usize, be16(b, r + 10)? as usize); + if nid != id { + continue; + } + let rank = match plat { + 3 if enc == 1 || enc == 10 => { + if best.as_ref().is_some_and(|(current, _)| *current <= if lang == 0x409 { 0 } else { 1 }) { + continue; + } + if lang == 0x409 { 0 } else { 1 } + } + 0 => { + if best.as_ref().is_some_and(|(current, _)| *current <= 2) { + continue; + } + 2 + } + 1 if enc == 0 => { + if best.as_ref().is_some_and(|(current, _)| *current <= 3) { + continue; + } + 3 + } + _ => continue, + }; + // Only decode a record that can improve the selected name. A hostile name table may + // point thousands of records at the same huge string. + let raw = b.get(storage + off..storage + off + len)?; + let s = match plat { + 3 => { + let u: Vec = raw.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u) + } + 0 => { + let u: Vec = raw.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes([c[0], c[1]])).collect(); + String::from_utf16_lossy(&u) + } + 1 => raw.iter().map(|&c| if c < 128 { c as char } else { '?' }).collect(), + _ => return None, + }; + best = Some((rank, s)); + } + best.map(|(_, s)| s).filter(|s| !s.trim().is_empty()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn later_non_improving_name_records_are_not_decoded() { + // The first record is the preferred Windows-English value. The second names an invalid + // range: it must be ignored instead of making the whole name lookup fail. + let mut table = vec![0; 30]; + table[2..4].copy_from_slice(&2u16.to_be_bytes()); + table[4..6].copy_from_slice(&30u16.to_be_bytes()); + let first = 6usize; + table[first..first + 2].copy_from_slice(&3u16.to_be_bytes()); + table[first + 2..first + 4].copy_from_slice(&1u16.to_be_bytes()); + table[first + 4..first + 6].copy_from_slice(&0x409u16.to_be_bytes()); + table[first + 6..first + 8].copy_from_slice(&1u16.to_be_bytes()); + table[first + 8..first + 10].copy_from_slice(&4u16.to_be_bytes()); + let second = first + 12; + table[second..second + 2].copy_from_slice(&3u16.to_be_bytes()); + table[second + 2..second + 4].copy_from_slice(&1u16.to_be_bytes()); + table[second + 4..second + 6].copy_from_slice(&0x409u16.to_be_bytes()); + table[second + 6..second + 8].copy_from_slice(&1u16.to_be_bytes()); + table[second + 8..second + 10].copy_from_slice(&u16::MAX.to_be_bytes()); + table.extend_from_slice(&[0, b'A', 0, b'B']); + assert_eq!(name_string(&table, 1).as_deref(), Some("AB")); + } +} diff --git a/crates/font-catalog/src/tests.rs b/crates/font-catalog/src/tests.rs index 5137a4295..7967cc9c8 100644 --- a/crates/font-catalog/src/tests.rs +++ b/crates/font-catalog/src/tests.rs @@ -568,3 +568,20 @@ fn a_read_is_capped_where_the_file_has_grown() { assert!(matches!(read_capped(&path.with_extension("missing"), 9), Err(FontFileError::Io(_)))); let _ = std::fs::remove_file(&path); } + +#[test] +fn bounded_sfnt_reader_preserves_face_names_and_collection_indices() { + let mut bold = spec("Example", "Bold Italic", 700); + bold.italic = true; + let data = font(&bold); + let faces = crate::sfnt::read_faces_bytes(&data); + assert_eq!(faces.len(), 1); + assert_eq!((&*faces[0].family, &*faces[0].style, faces[0].weight, faces[0].italic), ("Example", "Bold Italic", 700, true)); + let mut hidden = spec("Hidden", "Regular", 400); + hidden.outlines = false; + let combined = collection(&[font(&hidden), data]); + let faces = crate::sfnt::read_faces_bytes(&combined); + assert_eq!(faces.len(), 1); + assert_eq!((faces[0].index, faces[0].family.as_str()), (1, "Example")); + assert!(crate::sfnt::read_faces_bytes(b"invalid").is_empty()); +} diff --git a/crates/media/src/mpegts/tests.rs b/crates/media/src/mpegts/tests.rs index 0215680ad..b9cb1af3b 100644 --- a/crates/media/src/mpegts/tests.rs +++ b/crates/media/src/mpegts/tests.rs @@ -791,6 +791,23 @@ fn scrambled_packet_is_skipped_with_a_warning() { assert!(total(&f) < total(&base)); } +#[test] +fn repeated_pat_and_pmt_packets_are_dropped_before_the_section_parser() { + let clean = audio_ts(2); + let first = |pid: u16| { + (0..clean.len() / 188).map(|i| i * 188).find(|&o| (((clean[o + 1] & 0x1F) as u16) << 8 | clean[o + 2] as u16) == pid).expect("packet exists") + }; + let mut dup = clean.clone(); + for pid in [0x100, 0] { + let o = first(pid); + let copy = dup[o..o + 188].to_vec(); + dup.splice(o + 188..o + 188, copy); + } + let f = open(&dup).unwrap(); + assert!(f.warnings.iter().any(|w| w.contains("2 duplicate packet(s) dropped")), "{:?}", f.warnings); + assert!(!f.warnings.iter().any(|w| w.contains("continuity") || w.contains("CRC")), "{:?}", f.warnings); +} + #[test] fn duplicate_packet_is_dropped_but_a_different_payload_with_the_same_counter_is_kept() { let clean = audio_ts(6); diff --git a/crates/media/src/mpegts/ts.rs b/crates/media/src/mpegts/ts.rs index a4c01a986..1d28bb736 100644 --- a/crates/media/src/mpegts/ts.rs +++ b/crates/media/src/mpegts/ts.rs @@ -497,6 +497,7 @@ pub(crate) fn scan(src: &S, packet_size: usize, sync_off let mut warnings: Vec = Vec::new(); let mut pat = Psi::default(); let mut pmt = PmtState::default(); + let mut psi_dup: HashMap = HashMap::new(); let mut es: HashMap = HashMap::new(); let mut order: Vec = Vec::new(); // one timestamp unwrapper for the program: its streams share one clock, so a stream whose @@ -560,6 +561,13 @@ pub(crate) fn scan(src: &S, packet_size: usize, sync_off } continue; } + if pk.pid == 0 || Some(pk.pid) == pmt.pmt_pid { + // a repeated packet would append its payload to the section in progress + if psi_dup.entry(pk.pid).or_default().is_dup(&pk) { + duplicates += 1; + continue; + } + } if pk.pid == 0 { feed_pat(&pk, &mut pat, &mut pmt, &mut crc_errors); continue; diff --git a/crates/odf/src/text/chart.rs b/crates/odf/src/text/chart.rs index 9d073ecd2..2a79352a1 100644 --- a/crates/odf/src/text/chart.rs +++ b/crates/odf/src/text/chart.rs @@ -229,7 +229,7 @@ impl Reader { }; let name = s.label.as_deref().and_then(|a| cells_of(a).first().copied()).map(|at| label(self.cell(at))).unwrap_or_default(); let name = if name.is_empty() { format!("Series {}", i + 1) } else { name }; - ChartSeries { name, values } + ChartSeries { name, values, ..Default::default() } }) .collect(); // A scatter chart's x values: its first series' domain. @@ -248,7 +248,10 @@ impl Reader { _ => LegendPos::Right, }); let title = self.title.as_deref().map(str::trim).filter(|t| !t.is_empty()).map(str::to_string); - Some(ChartSpec { kind, categories, series, title, legend, data_labels }.sanitized()) + // ODF axis formatting is not read; the value axis keeps the look every chart had before + // axes were formattable (gridlines, no axis line), a round chart has none. + let val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() }; + Some(ChartSpec { kind, categories, series, title, legend, data_labels, val_axis, ..Default::default() }.sanitized()) } } diff --git a/crates/odf/src/text/tests_more/charts.rs b/crates/odf/src/text/tests_more/charts.rs index 9ac0b6b96..5aa892cf4 100644 --- a/crates/odf/src/text/tests_more/charts.rs +++ b/crates/odf/src/text/tests_more/charts.rs @@ -57,8 +57,8 @@ fn spec(chart: &str) -> Option { fn north_south() -> Vec { vec![ - ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)] }, - ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }, + ChartSeries { name: "North".into(), values: vec![Some(10.0), Some(14.0), Some(8.0), Some(16.0)], ..Default::default() }, + ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() }, ] } @@ -78,6 +78,8 @@ fn a_column_chart_reads_its_data_title_and_legend() { title: Some("Sales by quarter".into()), legend: Some(LegendPos::Right), data_labels: false, + val_axis: craft_wordmodel::chart::value_axis(), + ..Default::default() } ); // The frame's size, in points. @@ -135,7 +137,7 @@ fn line_area_pie_doughnut_and_scatter_classes() { let s = spec(&chart_xml("chart:scatter", "", "", "", scatter)).unwrap(); assert_eq!(s.kind, ChartType::Scatter); assert_eq!(s.categories, vec!["10", "14", "8", "16"]); - assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)] }]); + assert_eq!(s.series, vec![ChartSeries { name: "South".into(), values: vec![Some(12.0), None, Some(11.0), Some(13.0)], ..Default::default() }]); } #[test] @@ -249,7 +251,15 @@ fn every_chart_type_is_saved_as_a_chart_object_and_reads_back() { let bytes = crate::text::export(&doc_with(spec.clone(), false)).unwrap(); let back = reread(&bytes); assert_eq!(back.len(), 1, "{kind:?}"); - assert_eq!(*back[0].spec, spec.clone().sanitized(), "{kind:?}"); + // ODF keeps no series or axis formatting here: only the data, the title and the legend + // come back; the value axis has the gridlines every chart had before axes were formattable. + let mut expected = spec.clone().sanitized(); + for s in &mut expected.series { + *s = ChartSeries { name: std::mem::take(&mut s.name), values: std::mem::take(&mut s.values), ..Default::default() }; + } + expected.cat_axis = Default::default(); + expected.val_axis = if kind.is_round() { Default::default() } else { craft_wordmodel::chart::value_axis() }; + assert_eq!(*back[0].spec, expected, "{kind:?}"); assert_eq!((back[0].w, back[0].h, back[0].alt.as_str()), (300.0, 180.0, "Sales chart"), "{kind:?}"); // The package lists the chart object. let z = crate::Package::open(&bytes, &crate::Limits::DEFAULT).unwrap(); diff --git a/crates/odf/src/text/tests_more/groups.rs b/crates/odf/src/text/tests_more/groups.rs index bf1b056a6..4fada68a9 100644 --- a/crates/odf/src/text/tests_more/groups.rs +++ b/crates/odf/src/text/tests_more/groups.rs @@ -130,6 +130,7 @@ fn a_diagram_is_saved_as_a_group_of_its_drawing() { bold: false, align: TextAlign::Center, font: None, + angle: 0.0, }, GraphicItem::Shape { rect: [90.0, 9.0, 18.0, 18.0], kind: ShapeKind::Arrow, fill: Some(blue), stroke: None, stroke_width: 0.0 }, ], diff --git a/crates/psd/src/duotone.rs b/crates/psd/src/duotone.rs new file mode 100644 index 000000000..6c9ae49e1 --- /dev/null +++ b/crates/psd/src/duotone.rs @@ -0,0 +1,169 @@ +//! Raw records in Photoshop's version 1 Duotone Color Mode Data. +//! +//! Layout extracted from PhotoCraft `duotone_map.rs`, originally by Gert Pellin +//! (772cf39c09930229e11b920196f486239bbbfab8), with colour-book integration by David Mann. +//! Colour conversion, curve sampling and fallback/display policy are caller-owned. +//! Parsing accepts the 412-byte record prefix as PhotoCraft did; dot gain and overprints +//! are optional trailing data, not prerequisites for interpreting the inks. + +/// Full block size emitted by Photoshop and PhotoCraft. +pub const BLOCK_LEN: usize = 524; +/// Alternate Duotone Colors image resource. +pub const ALTERNATE_DUOTONE_COLORS: u16 = 1066; +/// Raw colour space and four component words, without colour interpretation. +pub type ColorRecord = [u16; 5]; +/// Sentinel for an absent ink or computed overprint colour. +pub const NO_COLOR: ColorRecord = [0xffff, 0, 0, 0, 0]; + +/// Valid active ink counts for a serialized block. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u16)] +pub enum InkCount { + /// One active ink. + One = 1, + /// Two active inks. + Two = 2, + /// Three active inks. + Three = 3, + /// Four active inks. + Four = 4, +} + +/// One fixed-size ink slot, including uninterpreted reserved curve word. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct InkRecord { + /// Raw ink colour space and components. + pub color: ColorRecord, + /// Pascal name bytes, including its length byte; text encoding belongs to the caller. + pub name: [u8; 64], + /// Thirteen signed curve samples and the reserved final word. + pub curve: [i16; 14], +} + +fn word(data: &[u8], at: usize) -> Option { + let bytes = data.get(at..at.checked_add(2)?)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) +} + +/// Decode a raw colour record at a byte offset, without guessing unknown colour spaces. +pub fn color_at(data: &[u8], at: usize) -> Option { + let bytes = data.get(at..at.checked_add(10)?)?; + Some(std::array::from_fn(|i| u16::from_be_bytes([bytes[i * 2], bytes[i * 2 + 1]]))) +} + +/// Borrowed, validated view over at most four active ink records. No input-sized allocation. +#[derive(Clone, Copy, Debug)] +pub struct DuotoneView<'a> { + data: &'a [u8], + count: usize, +} + +impl<'a> DuotoneView<'a> { + /// Accept version 1, one to four inks and the full 412-byte record prefix. + pub fn parse(data: &'a [u8]) -> Option { + if data.len() < 412 || word(data, 0)? != 1 { + return None; + } + let count = usize::from(word(data, 2)?); + (1..=4).contains(&count).then_some(Self { data, count }) + } + + /// Number of active ink records (one through four). + pub fn ink_count(self) -> usize { + self.count + } + + /// Copy one small fixed-size record. Inactive/out-of-range slots are not returned. + pub fn ink(self, index: usize) -> Option { + if index >= self.count { + return None; + } + let name_at = 44 + 64 * index; + let name = self.data.get(name_at..name_at + 64)?.try_into().ok()?; + let curve_at = 300 + 28 * index; + let curve_bytes = self.data.get(curve_at..curve_at + 28)?; + Some(InkRecord { + color: color_at(self.data, 4 + 10 * index)?, + name, + curve: std::array::from_fn(|i| i16::from_be_bytes([curve_bytes[i * 2], curve_bytes[i * 2 + 1]])), + }) + } +} + +/// Read one raw alternate colour when the resource version/count permits it. +pub fn alternate_color(data: &[u8], index: usize) -> Option { + if word(data, 0)? != 1 || index >= usize::from(word(data, 2)?) { + return None; + } + color_at(data, 4usize.checked_add(index.checked_mul(10)?)?) +} + +/// Serialize four raw slots and explicit trailing policy into a fixed-size block. +/// The typed count cannot be invalid. Reserved bytes and inactive slots are preserved as supplied. +pub fn write_block(count: InkCount, inks: &[InkRecord; 4], dot_gain: u16, overprints: &[ColorRecord; 11]) -> [u8; BLOCK_LEN] { + let mut out = [0u8; BLOCK_LEN]; + out[..2].copy_from_slice(&1u16.to_be_bytes()); + out[2..4].copy_from_slice(&(count as u16).to_be_bytes()); + for (i, ink) in inks.iter().enumerate() { + for (j, value) in ink.color.iter().enumerate() { + let at = 4 + 10 * i + 2 * j; + out[at..at + 2].copy_from_slice(&value.to_be_bytes()); + } + out[44 + 64 * i..44 + 64 * (i + 1)].copy_from_slice(&ink.name); + for (j, value) in ink.curve.iter().enumerate() { + let at = 300 + 28 * i + 2 * j; + out[at..at + 2].copy_from_slice(&value.to_be_bytes()); + } + } + out[412..414].copy_from_slice(&dot_gain.to_be_bytes()); + for (i, color) in overprints.iter().enumerate() { + for (j, value) in color.iter().enumerate() { + let at = 414 + 10 * i + 2 * j; + out[at..at + 2].copy_from_slice(&value.to_be_bytes()); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fixed_layout_roundtrip_preserves_raw_fields() { + let mut ink = InkRecord { color: [7, 1, 2, 3, 65535], name: [0; 64], curve: [-1; 14] }; + ink.name[..4].copy_from_slice(&[3, b'I', b'n', b'k']); + ink.curve[0] = 0; + ink.curve[12] = 1000; + let slots = std::array::from_fn(|_| ink.clone()); + let bytes = write_block(InkCount::Four, &slots, 20, &[NO_COLOR; 11]); + assert_eq!(&bytes[..4], &[0, 1, 0, 4]); + assert_eq!(&bytes[44..48], &[3, b'I', b'n', b'k']); + assert_eq!(&bytes[412..416], &[0, 20, 255, 255]); + let view = DuotoneView::parse(&bytes); + assert_eq!(view.map(DuotoneView::ink_count), Some(4)); + for i in 0..4 { + assert_eq!(view.and_then(|v| v.ink(i)), Some(ink.clone())); + } + assert!(view.and_then(|v| v.ink(usize::MAX)).is_none()); + for n in 0..412 { + assert!(DuotoneView::parse(&bytes[..n]).is_none()); + } + assert!(DuotoneView::parse(&bytes[..412]).is_some()); + } + + #[test] + fn rejects_hostile_offsets_versions_counts_and_alternate_truncation() { + let mut bytes = [0u8; BLOCK_LEN]; + bytes[..4].copy_from_slice(&[0, 1, 255, 255]); + assert!(DuotoneView::parse(&bytes).is_none()); + bytes[..4].copy_from_slice(&[0, 2, 0, 1]); + assert!(DuotoneView::parse(&bytes).is_none()); + assert!(color_at(&bytes, usize::MAX).is_none()); + let alt = [0, 1, 0, 1, 0, 7, 0, 1, 0, 2, 0, 3, 0, 4]; + assert_eq!(alternate_color(&alt, 0), Some([7, 1, 2, 3, 4])); + assert!(alternate_color(&alt[..13], 0).is_none()); + assert!(alternate_color(&alt, 1).is_none()); + assert!(alternate_color(&alt, usize::MAX).is_none()); + } +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index d4db98065..efcc474b2 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -66,6 +66,7 @@ pub mod compression; pub mod csh; pub mod descriptor; pub mod displace_map; +pub mod duotone; pub mod engine_data; pub mod error; pub mod file; diff --git a/crates/raw/README.md b/crates/raw/README.md index e15e355c1..f0faf0670 100644 --- a/crates/raw/README.md +++ b/crates/raw/README.md @@ -33,7 +33,7 @@ The format layer ends where arithmetic on the picture begins. |---|---| | Recognising containers (`probe`), walking TIFF/IFD structure on `craft-tiff` | Black subtraction, white scaling, cropping to the active area | | Decompressing the samples: unpacking, lossless JPEG, lossy JPEG, Deflate, JPEG XL | Demosaicing, binned previews, hot-pixel and highlight handling | -| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | +| A DNG `LinearizationTable` (a storage code, applied so the samples are sensor values; left unapplied when `OpcodeList1` is present, so the develop pipeline can run that list on the stored codes first) | Raising samples stuck below the stated white level (`lift_clipped_samples`) | | Black and white levels, CFA layout, active area, default crop, orientation, pixel aspect | The DNG colour model's arithmetic (illuminant interpolation, camera to working space) | | Colour tags as data: matrices, calibration illuminants, as-shot neutral / white | White balance, Kelvin scales, spectral or fitted camera colour | | Opcode lists parsed and written (`opcodes`), profile tables and gain table maps as data | Applying opcodes (gain maps, warps, bad pixels), profile look tables, tone curves | diff --git a/crates/raw/docs/cr3.md b/crates/raw/docs/cr3.md index 40e375b54..9c17c809c 100644 --- a/crates/raw/docs/cr3.md +++ b/crates/raw/docs/cr3.md @@ -187,7 +187,7 @@ the six verified fixtures are unchanged. Partial-subband and rounded-plane coding, vertical C-RAW tiling and adaptive low-frequency gains remain unsupported (see the variant table above; `cr3_variants.rs` pins each rejection). The decoder bounds image dimensions, sample count, tile count, marker headers and temporary allocations. The full sensor buffer is -allocated only after the first plane decodes, and plane buffers grow row by row, +allocated only after every plane has decoded, and plane buffers grow row by row, so a small crafted file fails before reserving memory for the image it claims. ## Further bodies: one CC0 file each diff --git a/crates/raw/src/arw.rs b/crates/raw/src/arw.rs index 1b25aacda..389c464a5 100644 --- a/crates/raw/src/arw.rs +++ b/crates/raw/src/arw.rs @@ -852,6 +852,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result Result> let count = image_size(config)?; let tiles = layout(sample, config)?; validate_planes(&tiles, config)?; - // Allocated once the first plane decodes: a tiny crafted file can claim a huge image. - let mut output = Vec::new(); + // A tile's QP map error is reported by its first plane, so errors keep tile, then plane order. + let qp: Vec>> = tiles.iter().map(|tile| tile_qp(sample, tile)).collect(); let midpoint = 1i32 << (config.bit_depth - 1); let maximum = (1i32 << config.bit_depth) - 1; - for tile in tiles { - let qp = tile_qp(sample, &tile)?; - // Each plane is its own entropy stream: decode a tile's planes in parallel. - let planes: Vec>> = tile - .planes - .par_iter() - .enumerate() - .map(|(p, plane)| { - if config.levels == 0 { - let band = plane.bands.first().ok_or_else(|| corrupt("missing lossless band"))?; - lossless_plane(band_bytes(sample, band)?, tile.width / 2, tile.height / 2) - } else { - let edges = TileEdges { left: tile.x > 0, right: tile.x + tile.width < width, top: false, bottom: false }; - wavelet_plane(sample, plane, tile.width / 2, tile.height / 2, edges, qp.as_ref()) - } - .map_err(|error| RawError::Corrupt(format!("Canon CRX tile ({}, {}), plane {p}: {error}", tile.x, tile.y))) - }) - .collect(); - for (p, decoded) in planes.into_iter().enumerate() { - let decoded = decoded?; - if output.is_empty() { - output.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX sample allocation"))?; - output.resize(count, 0); - } - for (i, &value) in decoded.iter().enumerate() { - let sensor_value = sensor_sample(value, midpoint, maximum, config.levels != 0)?; - let (x, y) = (tile.x + 2 * (i % (tile.width / 2)) + p % 2, tile.y + 2 * (i / (tile.width / 2)) + p / 2); - let slot = y - .checked_mul(width) - .and_then(|n| n.checked_add(x)) - .and_then(|n| output.get_mut(n)) - .ok_or_else(|| corrupt("tile writes outside image"))?; - *slot = sensor_value; - } - } - } + // Every plane of every tile is its own entropy stream: decode them all in parallel, each straight + // to sensor samples, so a task holds only its own plane as i32. The first error in tile and plane + // order is reported, as when the tiles were decoded one after another. + let jobs: Vec<(&Tile, &Result>, usize, &Plane)> = + tiles.iter().zip(&qp).flat_map(|(tile, qp)| tile.planes.iter().enumerate().map(move |(p, plane)| (tile, qp, p, plane))).collect(); + let planes: Vec>> = jobs + .par_iter() + .map(|&(tile, qp, p, plane)| { + let qp = qp.as_ref().map_err(Clone::clone)?.as_ref(); + let decoded = decode_plane(sample, config, width, tile, plane, qp) + .map_err(|error| RawError::Corrupt(format!("Canon CRX tile ({}, {}), plane {p}: {error}", tile.x, tile.y)))?; + sensor_plane(&decoded, midpoint, maximum, config.levels != 0) + }) + .collect(); + let planes = planes.into_iter().collect::>>()?; + // Allocated once every plane has decoded: a tiny crafted file can claim a huge image. + let mut output = Vec::new(); + output.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX sample allocation"))?; + output.resize(count, 0); + interleave(&mut output, width, &tiles, &planes)?; Ok(output) } -/// A decoded plane value as a sensor sample of the CMP1 bit depth. +fn decode_plane(sample: &[u8], config: &Cr3Compression, width: usize, tile: &Tile, plane: &Plane, qp: Option<&QpMap>) -> Result> { + if config.levels == 0 { + let band = plane.bands.first().ok_or_else(|| corrupt("missing lossless band"))?; + lossless_plane(band_bytes(sample, band)?, tile.width / 2, tile.height / 2) + } else { + let edges = TileEdges { left: tile.x > 0, right: tile.x + tile.width < width, top: false, bottom: false }; + wavelet_plane(sample, plane, tile.width / 2, tile.height / 2, edges, qp) + } +} + +/// Decoded plane values as sensor samples of the CMP1 bit depth. /// /// Lossless planes reproduce the sensor exactly, so a value outside the bit depth is corrupt data. /// Wavelet (C-RAW) planes are quantized: next to clipped highlights the reconstruction overshoots /// the sensor's range by up to a few quantization steps (measured on 17 CC0 files: at most 36 codes /// above 16383 at 14 bits, never below 0), while lossless files of the same bodies clip at exactly -/// 16383. Those samples are clamped to the range the sensor can record. -fn sensor_sample(value: i32, midpoint: i32, maximum: i32, quantized: bool) -> Result { - let sample = value.checked_add(midpoint).ok_or_else(|| corrupt("decoded sample outside its bit depth"))?; - let sample = if quantized { sample.clamp(0, maximum) } else { sample }; - u16::try_from(sample).ok().filter(|_| sample <= maximum).ok_or_else(|| corrupt("decoded sample outside its bit depth")) +/// 16383. Those samples are clamped to the range the sensor can record. The range is checked once, +/// on the plane's extremes; `midpoint` is positive, so only the largest sum can overflow. +fn sensor_plane(values: &[i32], midpoint: i32, maximum: i32, quantized: bool) -> Result> { + let outside = || corrupt("decoded sample outside its bit depth"); + let (Some(&low), Some(&high)) = (values.iter().min(), values.iter().max()) else { return Ok(Vec::new()) }; + let high = high.checked_add(midpoint).ok_or_else(outside)?; + if !quantized && (low.saturating_add(midpoint) < 0 || high > maximum) { + return Err(outside()); + } + let mut samples = Vec::new(); + samples.try_reserve_exact(values.len()).map_err(|_| RawError::Limit("CRX sample allocation"))?; + // Every sum fits (the largest did), and the clamp leaves in-range lossless samples unchanged. + samples.extend(values.iter().map(|&value| (value + midpoint).clamp(0, maximum) as u16)); + Ok(samples) +} + +/// Writes each tile's four planes (`planes`: four per tile, in tile order) into the Bayer mosaic. +/// Plane `p` holds the samples at row parity `p / 2` and column parity `p % 2`. Runs in parallel +/// over pairs of output rows. +fn interleave(output: &mut [u16], width: usize, tiles: &[Tile], planes: &[Vec]) -> Result<()> { + let outside = || corrupt("tile writes outside image"); + if width == 0 || planes.len() != tiles.len() * 4 { + return Err(outside()); + } + output.par_chunks_mut(2 * width).enumerate().try_for_each(|(pair, rows)| -> Result<()> { + let y = 2 * pair; + for (tile, planes) in tiles.iter().zip(planes.as_chunks::<4>().0).filter(|(tile, _)| (tile.y..tile.y + tile.height).contains(&y)) { + let half = tile.width / 2; + let start = (y - tile.y) / 2 * half; + for (row, [even, odd]) in rows.chunks_exact_mut(width).zip(planes.as_chunks::<2>().0) { + let target = row.get_mut(tile.x..tile.x + tile.width).ok_or_else(outside)?; + let even = even.get(start..start + half).ok_or_else(outside)?; + let odd = odd.get(start..start + half).ok_or_else(outside)?; + for ([a, b], (&left, &right)) in target.as_chunks_mut::<2>().0.iter_mut().zip(even.iter().zip(odd)) { + *a = left; + *b = right; + } + } + } + Ok(()) + }) } /// Header-only probing must reject the same unsupported coding variants as full decoding. @@ -413,24 +443,58 @@ fn wavelet_plane(sample: &[u8], plane: &Plane, width: usize, height: usize, edge for high in high_dimensions.iter().rev() { shapes.extend_from_slice(high); } - let mut bands = Vec::with_capacity(10); - for (i, (&(w, h), band)) in shapes.iter().zip(&plane.bands).enumerate() { - let bytes = band_bytes(sample, band)?; - let mut data = if i == 0 { lossless_plane(bytes, w, h)? } else { coded_high_frequency_band(bytes, w, coded_band_width(i, w, edges), h)? }; - for (position, value) in data.iter_mut().enumerate() { - let step = match band.quant { - Quantization::Uniform(value) => quant_step(value, MAX_UNIFORM_QUANT)?, - Quantization::Adaptive { base, gain } => { - adaptive_step(qp.ok_or_else(|| corrupt("missing QP map"))?, i, position % w, position / w, base, gain)? - } - }; - *value = value.checked_mul(step).ok_or_else(|| corrupt("dequantized coefficient overflow"))?; - } - bands.push(WaveletBand { width: w, height: h, data }); - } + // Each subband is its own entropy stream: decode and dequantize them in parallel. The first error + // in band order is reported, as when they were decoded one after another. + let bands: Vec> = shapes + .par_iter() + .zip(plane.bands.par_iter()) + .enumerate() + .map(|(i, (&(w, h), band))| { + let bytes = band_bytes(sample, band)?; + let mut data = if i == 0 { lossless_plane(bytes, w, h)? } else { coded_high_frequency_band(bytes, w, coded_band_width(i, w, edges), h)? }; + dequantize(&mut data, i, w, &band.quant, qp)?; + Ok(WaveletBand { width: w, height: h, data }) + }) + .collect(); + let bands = bands.into_iter().collect::>>()?; crx_wavelet::reconstruct(&bands, width, height, 3, edges) } +/// Multiplies the coefficients of subband `band` (`width` columns) by their quantization step. +/// +/// The step is constant per subband, except for adaptive quantization with a gain, where it is +/// constant per QP cell: 2 coefficients wide in bands 4-6 and 4 wide in bands 7-9 (see +/// [`adaptive_step`]), so it is looked up once per cell rather than per coefficient. +fn dequantize(data: &mut [i32], band: usize, width: usize, quant: &Quantization, qp: Option<&QpMap>) -> Result<()> { + let (map, base, gain) = match *quant { + Quantization::Uniform(value) => return scale(data, quant_step(value, MAX_UNIFORM_QUANT)?), + Quantization::Adaptive { base, gain } => (qp.ok_or_else(|| corrupt("missing QP map"))?, base, gain), + }; + if gain == 0 { + return scale(data, adaptive_step(map, band, 0, 0, base, gain)?); + } + if width == 0 { + return Err(corrupt("empty subband row")); + } + let cell = if (4..=6).contains(&band) { 2 } else { 4 }; + for (y, row) in data.chunks_exact_mut(width).enumerate() { + for (column, values) in row.chunks_mut(cell).enumerate() { + scale(values, adaptive_step(map, band, column * cell, y, base, gain)?)?; + } + } + Ok(()) +} + +fn scale(values: &mut [i32], step: i32) -> Result<()> { + let mut overflow = false; + for value in values.iter_mut() { + let (product, wrapped) = value.overflowing_mul(step); + *value = product; + overflow |= wrapped; + } + if overflow { Err(corrupt("dequantized coefficient overflow")) } else { Ok(()) } +} + fn adaptive_step(map: &QpMap, band: usize, x: usize, y: usize, base: u32, gain: u32) -> Result { let step = if gain == 0 { u64::from(base) @@ -502,17 +566,60 @@ fn be32(src: &[u8], at: usize) -> Result { Ok(u32::from_be_bytes(bytes)) } +/// An MSB-first bit reader over one entropy stream. +/// +/// Up to 64 stream bits are held in `cache`, left-aligned: the top `count` bits are the next unread +/// ones. A refill loads 8 bytes at once and keeps whole bytes only, so the bits below `count` are +/// either zero or the stream's own next bits at their final positions, which a later refill ORs in +/// again unchanged. Reads never trust those bits: they are checked against `count`. struct Bits<'a> { src: &'a [u8], - at: usize, + /// The next byte to load into `cache`. + next: usize, + cache: u64, + count: u32, } -impl Bits<'_> { - /// The next 32 bits, MSB first, when at least that many remain before the end of the stream. - fn peek32(&self) -> Option { - let bytes = self.src.get(self.at / 8..self.at / 8 + 5)?; - let word = u64::from_be_bytes([0, 0, 0, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4]]); - Some((word >> (8 - self.at % 8)) as u32) +impl<'a> Bits<'a> { + fn new(src: &'a [u8]) -> Self { + Self { src, next: 0, cache: 0, count: 0 } + } + + /// The number of bits consumed so far. + fn position(&self) -> usize { + self.next * 8 - self.count as usize + } + + /// Loads whole bytes until more than 56 bits are cached or the stream ends. + fn refill(&mut self) { + if self.count > 56 { + return; + } + if let Some(bytes) = self.src.get(self.next..self.next + 8).and_then(|s| <[u8; 8]>::try_from(s).ok()) { + self.cache |= u64::from_be_bytes(bytes) >> self.count; + let loaded = (63 - self.count) / 8; + self.next += loaded as usize; + self.count += loaded * 8; + return; + } + // The last 7 bytes: one at a time. + while self.count <= 56 { + let Some(&byte) = self.src.get(self.next) else { break }; + self.cache |= u64::from(byte) << (56 - self.count); + self.next += 1; + self.count += 8; + } + } + + /// Drops `count` bits (at most `self.count`, below 64). + fn consume(&mut self, count: u32) { + self.cache <<= count; + self.count -= count; + } + + /// The top `count` cached bits (0 to 32) as a number; `>> 1 >> (63 - count)` is 0 for `count` 0. + fn top(&self, count: u32) -> u32 { + (self.cache >> 1 >> (63 - count)) as u32 } fn take(&mut self, count: u32) -> Result { @@ -522,27 +629,34 @@ impl Bits<'_> { if count == 0 { return Ok(0); } - if let Some(window) = self.peek32() { - self.at += count as usize; - return Ok(window >> (32 - count)); - } - // Near the end of the stream: bit by bit, so truncation is reported exactly. - let mut value = 0; - for _ in 0..count { - let byte = self.src.get(self.at / 8).ok_or_else(|| corrupt("truncated entropy stream"))?; - value = (value << 1) | u32::from((byte >> (7 - self.at % 8)) & 1); - self.at += 1; + if self.count < count { + self.refill(); + if self.count < count { + return Err(corrupt("truncated entropy stream")); + } } + let value = self.top(count); + self.consume(count); Ok(value) } fn rice(&mut self, k: u32) -> Result { - // Common case: the unary prefix and its terminating 1 lie within the next 32 bits. - if let Some(window) = self.peek32().filter(|&w| w != 0) { - let zeros = window.leading_zeros(); - self.at += zeros as usize + 1; - return if k <= 20 { Ok((zeros << k) | self.take(k)?) } else { Err(corrupt("Rice parameter exceeds sample precision")) }; + // Common case: the unary prefix (shorter than the 41-zero escape), its terminating 1 and the k + // low bits are all cached. + if k <= 20 { + let mut zeros = self.cache.leading_zeros(); + if zeros + 1 + k > self.count { + self.refill(); + zeros = self.cache.leading_zeros(); + } + if zeros < 41 && zeros + 1 + k <= self.count { + self.consume(zeros + 1); + let low = self.top(k); + self.consume(k); + return Ok((zeros << k) | low); + } } + // Escapes, k above 20 and the end of the stream: bit by bit, so truncation is reported exactly. let mut zeros = 0; while self.take(1)? == 0 { zeros += 1; @@ -580,11 +694,12 @@ impl Bits<'_> { fn finish(&self) -> Result<()> { let bits = self.src.len().checked_mul(8).ok_or(RawError::Limit("CRX entropy length"))?; - if bits.saturating_sub(self.at) > 7 { + let at = self.position(); + if bits.saturating_sub(at) > 7 { return Err(corrupt("unconsumed entropy data")); } if let Some(&last) = self.src.last() { - let remaining = (bits - self.at) as u32; + let remaining = (bits - at) as u32; if u32::from(last) & ((1 << remaining) - 1) != 0 { return Err(corrupt("nonzero entropy alignment bits")); } @@ -653,7 +768,7 @@ fn lossless_plane(src: &[u8], width: usize, height: usize) -> Result> { current.try_reserve_exact(width).map_err(|_| RawError::Limit("CRX row allocation"))?; current.resize(width, 0i32); let (mut k, mut run_index) = (0u32, 0usize); - let mut bits = Bits { src, at: 0 }; + let mut bits = Bits::new(src); for y in 0..height { let mut x = 0; let mut left = previous.first().copied().ok_or_else(|| corrupt("empty plane row"))?; @@ -716,7 +831,7 @@ fn high_frequency_band(src: &[u8], width: usize, height: usize) -> Result Result { Ok(values) }; let (mut previous, mut current) = (row()?, row()?); - let mut bits = Bits { src, at: 0 }; + let mut bits = Bits::new(src); let mut k = 0; for y in 0..height { let mut left = *previous.first().ok_or_else(|| corrupt("empty QP row"))?; @@ -879,11 +994,11 @@ mod tests { #[test] fn partial_final_run_segment_keeps_its_index() { - let mut bits = Bits { src: &[0xc0], at: 0 }; + let mut bits = Bits::new(&[0xc0]); let mut index = 11; assert_eq!(run(&mut bits, &mut index, 6).unwrap(), 6); assert_eq!(index, 12); - assert_eq!(bits.at, 2); + assert_eq!(bits.position(), 2); } #[test] @@ -1043,14 +1158,22 @@ mod tests { #[test] fn only_quantized_planes_clamp_out_of_range_samples() { - assert_eq!(sensor_sample(8200, 8192, 16383, true).unwrap(), 16383); - assert_eq!(sensor_sample(-8300, 8192, 16383, true).unwrap(), 0); - assert_eq!(sensor_sample(8191, 8192, 16383, false).unwrap(), 16383); - assert_eq!(sensor_sample(-8192, 8192, 16383, false).unwrap(), 0); + assert_eq!(sensor_plane(&[8200], 8192, 16383, true).unwrap(), [16383]); + assert_eq!(sensor_plane(&[-8300], 8192, 16383, true).unwrap(), [0]); + assert_eq!(sensor_plane(&[-8300, 0, 8200], 8192, 16383, true).unwrap(), [0, 8192, 16383]); + assert_eq!(sensor_plane(&[8191], 8192, 16383, false).unwrap(), [16383]); + assert_eq!(sensor_plane(&[-8192], 8192, 16383, false).unwrap(), [0]); + assert_eq!(sensor_plane(&[-8192, 5, 8191], 8192, 16383, false).unwrap(), [0, 8197, 16383]); // Lossless planes reproduce the sensor exactly: outside the bit depth is corrupt data. - assert!(matches!(sensor_sample(8192, 8192, 16383, false), Err(RawError::Corrupt(_)))); - assert!(matches!(sensor_sample(-8193, 8192, 16383, false), Err(RawError::Corrupt(_)))); - assert!(matches!(sensor_sample(i32::MAX, 8192, 16383, true), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[8192], 8192, 16383, false), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[-8193], 8192, 16383, false), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[0, 8192, 0], 8192, 16383, false), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[0, -8193, 0], 8192, 16383, false), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[i32::MAX], 8192, 16383, true), Err(RawError::Corrupt(_)))); + assert!(matches!(sensor_plane(&[0, i32::MAX], 8192, 16383, true), Err(RawError::Corrupt(_)))); + // The lowest i32 cannot overflow below: it is clamped in a quantized plane and corrupt in a lossless one. + assert_eq!(sensor_plane(&[i32::MIN], 8192, 16383, true).unwrap(), [0]); + assert!(matches!(sensor_plane(&[i32::MIN], 8192, 16383, false), Err(RawError::Corrupt(_)))); } #[test] @@ -1081,14 +1204,14 @@ mod tests { #[test] fn qp_escape_and_average_precede_the_quantization_table() { // 23 zeros, the terminating 1, then the symbol in 8 bits. - let mut bits = Bits { src: &[0, 0, 1, 0x18], at: 0 }; + let mut bits = Bits::new(&[0, 0, 1, 0x18]); assert_eq!(bits.qp_rice(0).unwrap(), 24); - assert_eq!(bits.at, 32); + assert_eq!(bits.position(), 32); // 22 zeros are an ordinary Rice prefix. - let mut bits = Bits { src: &[0, 0, 2, 0], at: 0 }; + let mut bits = Bits::new(&[0, 0, 2, 0]); assert_eq!(bits.qp_rice(0).unwrap(), 22); - assert_eq!(bits.at, 23); - assert!(Bits { src: &[0, 0, 0, 0x80, 0], at: 0 }.qp_rice(0).is_err()); + assert_eq!(bits.position(), 23); + assert!(Bits::new(&[0, 0, 0, 0x80, 0]).qp_rice(0).is_err()); // The map's first symbol is a 32-bit literal (300 = QP 150 from prediction 0). let map = qp_map(&[0, 0, 1, 0x2c, 0x94], 1, 4).unwrap(); assert_eq!(map.data, [150; 4]); @@ -1187,11 +1310,17 @@ mod tests { for end in 0..sample.len() { assert!(decode(&sample[..end], &config).is_err()); } + // A corrupt QP map (nonzero alignment bits) is reported as such, not as a plane error. + let at = sample.windows(5).position(|bytes| bytes == [0, 0, 1, 0x2c, 0x94]).unwrap() + 4; + sample[at] ^= 1; + let error = decode(&sample, &config).unwrap_err().to_string(); + assert!(error.contains("alignment") && !error.contains("plane"), "{error}"); } #[test] fn optional_cc0_samples_match_external_oracle() { let Some(directory) = std::env::var_os("LIGHTCRAFT_CR3_CORPUS") else { return }; + let mut checked = 0; for (name, expected) in [ ("cr3-canon-r100-raw.cr3", 0xcbe5299ab9c52630u64), ("cr3-canon-m50-raw.cr3", 0x62261f0ba81cfcd2u64), @@ -1203,9 +1332,18 @@ mod tests { ("cr3-canon-rp-craw.cr3", 0xf48df06ca1d67382u64), ("cr3-canon-rp-raw-crop.cr3", 0x46808a9de3b3dcc7u64), ("cr3-canon-rp-craw-crop.cr3", 0x39a60c4efed4e1f4u64), + // EOS R5 Mark II (version 0x200 C-RAW and its lossless pair): pinned to this decoder's + // own output, see docs/cr3.md. + ("cr3-canon-r5m2-raw.cr3", 0xff716fb30248496fu64), + ("cr3-canon-r5m2-craw.cr3", 0xaf32b45357b534e5u64), + ("cr3-canon-r5m2-raw-crop.cr3", 0x31cf630afe868958u64), + ("cr3-canon-r5m2-craw-crop.cr3", 0x4721605b20339332u64), ] { let path = std::path::Path::new(&directory).join(name); - let bytes = std::fs::read(path).unwrap(); + let Ok(bytes) = std::fs::read(&path) else { + eprintln!("skipped {}: not in LIGHTCRAFT_CR3_CORPUS", path.display()); + continue; + }; let container = craft_metadata::cr3::parse_cr3(&bytes).unwrap(); let mut candidates = container .tracks @@ -1224,6 +1362,285 @@ mod tests { .flat_map(|value| value.to_le_bytes()) .fold(0xcbf29ce484222325u64, |hash, byte| (hash ^ u64::from(byte)).wrapping_mul(0x100000001b3)); assert_eq!(hash, expected, "{name}"); + checked += 1; + } + assert!(checked > 0, "LIGHTCRAFT_CR3_CORPUS holds none of the pinned files"); + } + + /// The bit reader before the 64-bit cache: a checked 5-byte window per read, bit by bit near the end. + struct ReferenceBits<'a> { + src: &'a [u8], + at: usize, + } + + impl ReferenceBits<'_> { + fn peek32(&self) -> Option { + let bytes = self.src.get(self.at / 8..self.at / 8 + 5)?; + let word = u64::from_be_bytes([0, 0, 0, bytes[0], bytes[1], bytes[2], bytes[3], bytes[4]]); + Some((word >> (8 - self.at % 8)) as u32) + } + + fn take(&mut self, count: u32) -> Result { + if count > 24 { + return Err(corrupt("invalid Rice parameter")); + } + if count == 0 { + return Ok(0); + } + if let Some(window) = self.peek32() { + self.at += count as usize; + return Ok(window >> (32 - count)); + } + let mut value = 0; + for _ in 0..count { + let byte = self.src.get(self.at / 8).ok_or_else(|| corrupt("truncated entropy stream"))?; + value = (value << 1) | u32::from((byte >> (7 - self.at % 8)) & 1); + self.at += 1; + } + Ok(value) + } + + fn rice(&mut self, k: u32) -> Result { + if let Some(window) = self.peek32().filter(|&w| w != 0) { + let zeros = window.leading_zeros(); + self.at += zeros as usize + 1; + return if k <= 20 { Ok((zeros << k) | self.take(k)?) } else { Err(corrupt("Rice parameter exceeds sample precision")) }; + } + let mut zeros = 0; + while self.take(1)? == 0 { + zeros += 1; + if zeros > 41 { + return Err(corrupt("Rice escape prefix exceeds 41 bits")); + } + } + if zeros == 41 { + self.take(21) + } else if k <= 20 { + Ok((zeros << k) | self.take(k)?) + } else { + Err(corrupt("Rice parameter exceeds sample precision")) + } + } + + fn qp_rice(&mut self, k: u32) -> Result { + let mut zeros = 0; + while self.take(1)? == 0 { + zeros += 1; + if zeros > QP_ESCAPE_ZEROS { + return Err(corrupt("QP Rice escape prefix exceeds 23 bits")); + } + } + if zeros == QP_ESCAPE_ZEROS { + self.take(8) + } else if k <= 20 { + Ok((zeros << k) | self.take(k)?) + } else { + Err(corrupt("QP Rice parameter exceeds precision")) + } + } + + fn finish(&self) -> bool { + let bits = self.src.len() * 8; + bits.saturating_sub(self.at) <= 7 && self.src.last().is_none_or(|&last| u32::from(last) & ((1u32 << (bits - self.at)) - 1) == 0) } } + + /// A xorshift generator: deterministic pseudo-random test input without a dependency. + fn xorshift(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + } + + #[test] + fn cached_bit_reader_matches_the_bitwise_reference() { + let mut seed = 0x2545_f491_4f6c_dd1d; + for round in 0..400 { + let length = (xorshift(&mut seed) % 48) as usize; + // Mostly zero and small bytes, so long unary prefixes and both escapes occur. + let stream: Vec = (0..length) + .map(|_| match xorshift(&mut seed) % 4 { + 0 | 1 => 0, + 2 => (xorshift(&mut seed) & 0x0f) as u8, + _ => xorshift(&mut seed) as u8, + }) + .collect(); + let ops: Vec<(u64, u32)> = (0..48).map(|_| (xorshift(&mut seed) % 3, (xorshift(&mut seed) % 26) as u32)).collect(); + // Every truncation of the stream, so reads meet the end at every offset. + for end in 0..=stream.len() { + let src = &stream[..end]; + let (mut cached, mut reference) = (Bits::new(src), ReferenceBits { src, at: 0 }); + let mut complete = true; + for &(op, n) in &ops { + let (a, b) = match op { + 0 => (cached.take(n), reference.take(n)), + 1 => (cached.rice(n), reference.rice(n)), + _ => (cached.qp_rice(n % 22), reference.qp_rice(n % 22)), + }; + assert_eq!(a.as_ref().ok(), b.as_ref().ok(), "round {round}, end {end}, op {op} {n}, stream {stream:02x?}"); + assert_eq!( + a.as_ref().err().map(ToString::to_string), + b.as_ref().err().map(ToString::to_string), + "round {round}, end {end}, op {op} {n}" + ); + if a.is_err() { + complete = false; + break; + } + assert_eq!(cached.position(), reference.at, "round {round}, end {end}, op {op} {n}"); + } + if complete { + assert_eq!(cached.finish().is_ok(), reference.finish(), "round {round}, end {end}"); + } + } + } + } + + #[test] + fn rice_escape_is_an_error_when_cut_at_any_byte() { + // 41 zeros, the terminating 1 and the 21-bit literal 16400 (+8200): the whole stream is one escape. + let src = [0, 0, 0, 0, 0, 0x40, 0x80, 0x20]; + assert_eq!(lossless_plane(&src, 1, 1).unwrap(), [8200]); + for end in 0..src.len() { + assert!(lossless_plane(&src[..end], 1, 1).is_err(), "prefix {end}"); + } + // The same escape read from the cache, 9 bytes into a longer stream. + let mut long = vec![0xff; 9]; + long.extend(src); + let mut bits = Bits::new(&long); + for _ in 0..72 { + assert_eq!(bits.rice(0).unwrap(), 0); + } + assert_eq!(bits.rice(0).unwrap(), 16400); + bits.finish().unwrap(); + } + + #[test] + fn dequantization_per_cell_matches_a_step_per_coefficient() { + let mut seed = 0x9e37_79b9_7f4a_7c15; + // Odd map heights as on the EOS R7, and band sizes that fit the map and that overrun it by one. + for (map_width, map_height) in [(1, 1), (2, 3), (3, 4), (5, 7)] { + let map = QpMap { + width: map_width, + height: map_height, + data: (0..map_width * map_height).map(|_| 131 + (xorshift(&mut seed) % 45) as i32).collect(), + }; + for band in 0..10 { + for (width, height) in [ + (1, 1), + (3, 2), + (2 * map_width - 1, map_height.div_ceil(2)), + (2 * map_width + 1, map_height), + (4 * map_width, map_height), + (4 * map_width + 1, map_height + 1), + ] { + for quant in [ + Quantization::Uniform(4 + 4 * band as u8), + Quantization::Adaptive { base: 0, gain: 0 }, + Quantization::Adaptive { base: 3, gain: 0 }, + Quantization::Adaptive { base: 1, gain: 8 }, + Quantization::Adaptive { base: 0, gain: 16 }, + ] { + let data: Vec = (0..width * height).map(|_| (xorshift(&mut seed) % 2001) as i32 - 1000).collect(); + let mut cells = data.clone(); + let cells_result = dequantize(&mut cells, band, width, &quant, Some(&map)); + let mut reference = data; + let reference_result = reference.iter_mut().enumerate().try_for_each(|(position, value)| -> Result<()> { + let step = match quant { + Quantization::Uniform(uniform) => quant_step(uniform, MAX_UNIFORM_QUANT)?, + Quantization::Adaptive { base, gain } => adaptive_step(&map, band, position % width, position / width, base, gain)?, + }; + *value = value.checked_mul(step).ok_or_else(|| corrupt("dequantized coefficient overflow"))?; + Ok(()) + }); + let case = format!("map {map_width}x{map_height}, band {band}, {width}x{height}, {quant:?}"); + assert_eq!(cells_result.is_ok(), reference_result.is_ok(), "{case}"); + assert_eq!( + cells_result.as_ref().err().map(ToString::to_string), + reference_result.as_ref().err().map(ToString::to_string), + "{case}" + ); + if cells_result.is_ok() { + assert_eq!(cells, reference, "{case}"); + } + } + } + } + } + assert!(dequantize(&mut [1 << 30], 1, 1, &Quantization::Uniform(10), None).is_err()); + assert!(dequantize(&mut [1], 4, 1, &Quantization::Adaptive { base: 0, gain: 0 }, None).is_err()); + } + + #[test] + fn interleave_places_every_plane_of_every_tile_like_a_per_sample_scatter() { + // A 2 x 2 tile grid whose right column is narrower, as at the right edge of a real image. + let (width, height) = (12, 8); + let tile = |x, y, width, height| Tile { x, y, width, height, planes: Vec::new(), qp: None }; + let tiles = [tile(0, 0, 8, 4), tile(8, 0, 4, 4), tile(0, 4, 8, 4), tile(8, 4, 4, 4)]; + let planes: Vec> = tiles + .iter() + .enumerate() + .flat_map(|(t, tile)| { + (0..4).map(move |p| (0..tile.width * tile.height / 4).map(|i| ((t * 4 + p) * 100 + i) as u16).collect::>()) + }) + .collect(); + let mut expected = vec![0u16; width * height]; + for (t, tile) in tiles.iter().enumerate() { + for p in 0..4 { + for (i, &value) in planes[t * 4 + p].iter().enumerate() { + let (x, y) = (tile.x + 2 * (i % (tile.width / 2)) + p % 2, tile.y + 2 * (i / (tile.width / 2)) + p / 2); + expected[y * width + x] = value; + } + } + } + let mut output = vec![0u16; width * height]; + interleave(&mut output, width, &tiles, &planes).unwrap(); + assert_eq!(output, expected); + // A plane shorter than its tile is an error, not a partial image. + let mut short = planes.clone(); + short[5].pop(); + assert!(interleave(&mut output, width, &tiles, &short).is_err()); + assert!(interleave(&mut output, width, &tiles, &planes[..12]).is_err()); + } + + #[test] + fn lossless_tile_grid_decodes_each_plane_to_its_place() { + // Four 4 x 4 tiles in a 2 x 2 grid (horizontal and vertical tile boundaries). Each 2 x 2 plane + // is flat at value 1, 2 or 3, chosen per tile and plane. Bitstreams per value: row 0 has a + // zero-run flag 0, the Rice-coded first sample and a 0 symbol; row 1 is one run to the row end. + let stream = |value: usize| -> &'static [u8] { + match value { + 1 => &[0x1e], // 0, 001 (k 0), 1 (k 0), 1, 1 + 2 => &[0x06, 0xc0], // 0, 00001 (k 0), 10 (k 1), 1, 1 + _ => &[0x01, 0x98], // 0, 0000001 (k 0), 100 (k 2), 1, 1 + } + }; + let value = |t: usize, p: usize| 1 + (t + p) % 3; + let mut config = coding(); + config.width = 8; + config.height = 8; + config.header_size = 432; + let mut sample = Vec::new(); + let mut data = Vec::new(); + for t in 0..4 { + let sizes: Vec = (0..4).map(|p| stream(value(t, p)).len() as u32).collect(); + marker(&mut sample, 0xff01, sizes.iter().sum(), (t as u32) << 16); + for (p, &size) in sizes.iter().enumerate() { + marker(&mut sample, 0xff02, size, ((p as u32) << 28) | 0x0800_0000); + marker(&mut sample, 0xff03, size, 0x0020_0000); + data.extend_from_slice(stream(value(t, p))); + } + } + assert_eq!(sample.len(), 432); + sample.extend(data); + validate(&sample, &config).unwrap(); + let expected: Vec = + (0..64).map(|i| (i % 8, i / 8)).map(|(x, y)| 8192 + value((y / 4) * 2 + x / 4, (y % 2) * 2 + x % 2) as u16).collect(); + assert_eq!(decode(&sample, &config).unwrap(), expected); + // A corrupt plane in the last tile still fails the whole image. + let mut broken = sample.clone(); + *broken.last_mut().unwrap() ^= 1; + assert!(decode(&broken, &config).is_err()); + } } diff --git a/crates/raw/src/crx_wavelet.rs b/crates/raw/src/crx_wavelet.rs index ac5e51d1d..190840be7 100644 --- a/crates/raw/src/crx_wavelet.rs +++ b/crates/raw/src/crx_wavelet.rs @@ -6,6 +6,7 @@ //! neighbouring low/high coefficients; only outer image edges are symmetric. use crate::{RawError, Result}; +use rayon::prelude::*; #[derive(Clone, Debug)] pub(crate) struct Band { @@ -18,10 +19,6 @@ fn corrupt() -> RawError { RawError::Corrupt("invalid CRX wavelet subband geometry".into()) } -fn checked_sample(v: i64) -> Result { - i32::try_from(v).map_err(|_| RawError::Corrupt("CRX wavelet coefficient overflow".into())) -} - #[derive(Clone, Copy, Debug, Default)] pub(crate) struct TileEdges { pub left: bool, @@ -34,40 +31,112 @@ fn sizes(length: usize, before: bool, after: bool) -> (usize, usize) { (length.div_ceil(2) + usize::from(after && length.is_multiple_of(2)), length / 2 + usize::from(before) + usize::from(after)) } +/// Whether a lifted value still fits the i32 coefficient range. +fn fits(value: i64) -> bool { + i32::try_from(value).is_ok() +} + +fn overflow() -> RawError { + RawError::Corrupt("CRX wavelet coefficient overflow".into()) +} + fn inverse_line(low: &[i32], high: &[i32], out: &mut [i32], before: bool, after: bool) -> Result<()> { let (low_count, high_count) = sizes(out.len(), before, after); if out.is_empty() || low.len() != low_count || high.len() != high_count { return Err(corrupt()); } - if high.is_empty() { + let Some(&last_high) = high.last() else { let value = low.first().copied().ok_or_else(corrupt)?; *out.first_mut().ok_or_else(corrupt)? = value; return Ok(()); - } + }; // The high-pass storage starts at -1 when there is a preceding tile. // Only absent external neighbours are reflected at the image boundary. - let high_at = |index: usize| -> Result { high.get(index.min(high.len().saturating_sub(1))).copied().ok_or_else(corrupt) }; - let even = |i: usize| -> Result { - let left = high_at((i + usize::from(before)).saturating_sub(1))?; - let right = high_at(i + usize::from(before))?; - let update = (i64::from(left) + i64::from(right) + 2).div_euclid(4); - checked_sample(i64::from(*low.get(i).ok_or_else(corrupt)?) - update) + let shift = usize::from(before); + let high_at = |index: usize| i64::from(high.get(index).copied().unwrap_or(last_high)); + let even = |i: usize, value: i32| i64::from(value) - ((high_at((i + shift).saturating_sub(1)) + high_at(i + shift) + 2) >> 2); + let mut in_range = true; + for (i, (slot, &value)) in out.iter_mut().step_by(2).zip(low).enumerate() { + let lifted = even(i, value); + in_range &= fits(lifted); + *slot = lifted as i32; + } + // With a following tile, the last odd sample's right neighbour is the even sample past the end. + let length = out.len(); + let past_end = if after && length.is_multiple_of(2) { low.get(length / 2).map(|&value| even(length / 2, value)) } else { None }; + in_range &= past_end.is_none_or(fits); + for i in 0..length / 2 { + let left = out.get(2 * i).copied().map_or(0, i64::from); + let right = out.get(2 * i + 2).copied().map(i64::from).or(past_end).unwrap_or(left); + let lifted = high_at(i + shift) + ((left + right) >> 1); + in_range &= fits(lifted); + if let Some(slot) = out.get_mut(2 * i + 1) { + *slot = lifted as i32; + } + } + if in_range { Ok(()) } else { Err(overflow()) } +} + +/// `out = low - floor((left + right + 2) / 4)` per column: one even row of the vertical lifting. +fn even_row(out: &mut [i32], low: &[i32], left: &[i32], right: &[i32]) -> bool { + let mut in_range = true; + for (((slot, &l), &a), &b) in out.iter_mut().zip(low).zip(left).zip(right) { + let lifted = i64::from(l) - ((i64::from(a) + i64::from(b) + 2) >> 2); + in_range &= fits(lifted); + *slot = lifted as i32; + } + in_range +} + +/// `out = high + floor((above + below) / 2)` per column: one odd row of the vertical lifting. +fn odd_row(out: &mut [i32], high: &[i32], above: &[i32], below: &[i32]) -> bool { + let mut in_range = true; + for (((slot, &h), &a), &b) in out.iter_mut().zip(high).zip(above).zip(below) { + let lifted = i64::from(h) + ((i64::from(a) + i64::from(b)) >> 1); + in_range &= fits(lifted); + *slot = lifted as i32; + } + in_range +} + +/// [`inverse_line`] applied to every column of `out`, on whole rows instead of strided columns. +/// +/// `rows` holds the low-pass rows, then the high-pass rows, each `width` wide. Runs in parallel over +/// pairs of output rows: each pair lifts its even row, recomputes the next even row (the following +/// pair's, or the one a following tile supplies past the end) and lifts its odd row from both. +fn inverse_columns(rows: &[i32], out: &mut [i32], width: usize, before: bool, after: bool) -> Result<()> { + let height = out.len().checked_div(width).filter(|&h| h > 0 && h * width == out.len()).ok_or_else(corrupt)?; + let (low_count, high_count) = sizes(height, before, after); + if rows.len() != (low_count + high_count) * width { + return Err(corrupt()); + } + let (low, high) = (rows.get(..low_count * width).ok_or_else(corrupt)?, rows.get(low_count * width..).ok_or_else(corrupt)?); + let low_row = |i: usize| low.get(i * width..(i + 1) * width).ok_or_else(corrupt); + let Some(last_high) = high_count.checked_sub(1) else { + let first = out.get_mut(..width).ok_or_else(corrupt)?; + first.copy_from_slice(low_row(0)?); + return Ok(()); }; - for i in 0..out.len().div_ceil(2) { - *out.get_mut(i * 2).ok_or_else(corrupt)? = even(i)?; - } - for i in 0..out.len() / 2 { - let left = out.get(i * 2).copied().ok_or_else(corrupt)?; - let right = match out.get(i * 2 + 2).copied() { - Some(value) => value, - None if after => even(i + 1)?, - None => left, - }; - let prediction = (i64::from(left) + i64::from(right)).div_euclid(2); - let value = high_at(i + usize::from(before))?; - *out.get_mut(i * 2 + 1).ok_or_else(corrupt)? = checked_sample(i64::from(value) + prediction)?; - } - Ok(()) + let high_row = |i: usize| { + let i = i.min(last_high); + high.get(i * width..(i + 1) * width).ok_or_else(corrupt) + }; + let shift = usize::from(before); + out.par_chunks_mut(2 * width).enumerate().try_for_each_init(Vec::new, |next, (i, pair)| -> Result<()> { + let (even, odd) = pair.split_at_mut(width); + let mut in_range = even_row(even, low_row(i)?, high_row((i + shift).saturating_sub(1))?, high_row(i + shift)?); + if !odd.is_empty() { + let below: &[i32] = if 2 * i + 2 < height || after { + next.resize(width, 0); + in_range &= even_row(next, low_row(i + 1)?, high_row(i + shift)?, high_row(i + shift + 1)?); + next.as_slice() + } else { + &*even + }; + in_range &= odd_row(odd, high_row(i + shift)?, even, below); + } + if in_range { Ok(()) } else { Err(overflow()) } + }) } fn valid_band(band: &Band, width: usize, height: usize) -> bool { @@ -97,32 +166,18 @@ pub(crate) fn inverse_level(ll: &Band, hl: &Band, lh: &Band, hh: &Band, width: u let mut temporary = zeroed(intermediate_count)?; // CRX analysis is vertical then horizontal. Integer rounding means that // synthesis must reverse that order, including for quantized subbands. - for y in 0..lhgt + hhgt { + temporary.par_chunks_mut(width).enumerate().try_for_each(|(y, row)| { let (lo, hi, by) = if y < lhgt { (ll, hl, y) } else { (lh, hh, y - lhgt) }; inverse_line( lo.data.get(by * lw..(by + 1) * lw).ok_or_else(corrupt)?, hi.data.get(by * hw..(by + 1) * hw).ok_or_else(corrupt)?, - temporary.get_mut(y * width..(y + 1) * width).ok_or_else(corrupt)?, + row, edges.left, edges.right, - )?; - } + ) + })?; let mut data = zeroed(count)?; - let mut low = zeroed(lhgt)?; - let mut high = zeroed(hhgt)?; - let mut column = zeroed(height)?; - for x in 0..width { - for (y, value) in low.iter_mut().enumerate() { - *value = temporary.get(y * width + x).copied().ok_or_else(corrupt)?; - } - for (y, value) in high.iter_mut().enumerate() { - *value = temporary.get((y + lhgt) * width + x).copied().ok_or_else(corrupt)?; - } - inverse_line(&low, &high, &mut column, edges.top, edges.bottom)?; - for (y, &value) in column.iter().enumerate() { - *data.get_mut(y * width + x).ok_or_else(corrupt)? = value; - } - } + inverse_columns(&temporary, &mut data, width, edges.top, edges.bottom)?; Ok(Band { width, height, data }) } @@ -335,4 +390,139 @@ mod tests { assert!(reconstruct(&bands, 20, 9, 3, TileEdges::default()).is_err()); } } + + /// The lifting before the row-wise rewrite: a checked closure per sample, as a reference. + fn reference_line(low: &[i32], high: &[i32], out: &mut [i32], before: bool, after: bool) -> Result<()> { + let (low_count, high_count) = sizes(out.len(), before, after); + if out.is_empty() || low.len() != low_count || high.len() != high_count { + return Err(corrupt()); + } + if high.is_empty() { + out[0] = low[0]; + return Ok(()); + } + let checked = |v: i64| i32::try_from(v).map_err(|_| overflow()); + let high_at = |index: usize| high[index.min(high.len() - 1)]; + let even = |i: usize| -> Result { + let left = high_at((i + usize::from(before)).saturating_sub(1)); + let right = high_at(i + usize::from(before)); + checked(i64::from(low[i]) - (i64::from(left) + i64::from(right) + 2).div_euclid(4)) + }; + for i in 0..out.len().div_ceil(2) { + out[i * 2] = even(i)?; + } + for i in 0..out.len() / 2 { + let left = out[i * 2]; + let right = match out.get(i * 2 + 2).copied() { + Some(value) => value, + None if after => even(i + 1)?, + None => left, + }; + out[i * 2 + 1] = checked(i64::from(high_at(i + usize::from(before))) + (i64::from(left) + i64::from(right)).div_euclid(2))?; + } + Ok(()) + } + + /// One level before the row-wise rewrite: horizontal lines, then strided columns. + fn reference_level(bands: &[Band; 4], width: usize, height: usize, edges: TileEdges) -> Result> { + let [ll, hl, lh, hh] = bands; + let (lhgt, hhgt) = sizes(height, edges.top, edges.bottom); + let mut temporary = vec![0; width * (lhgt + hhgt)]; + for y in 0..lhgt + hhgt { + let (lo, hi, by) = if y < lhgt { (ll, hl, y) } else { (lh, hh, y - lhgt) }; + reference_line( + &lo.data[by * lo.width..(by + 1) * lo.width], + &hi.data[by * hi.width..(by + 1) * hi.width], + &mut temporary[y * width..(y + 1) * width], + edges.left, + edges.right, + )?; + } + let mut data = vec![0; width * height]; + let mut column = vec![0; height]; + for x in 0..width { + let low: Vec = (0..lhgt).map(|y| temporary[y * width + x]).collect(); + let high: Vec = (lhgt..lhgt + hhgt).map(|y| temporary[y * width + x]).collect(); + reference_line(&low, &high, &mut column, edges.top, edges.bottom)?; + for (y, &value) in column.iter().enumerate() { + data[y * width + x] = value; + } + } + Ok(data) + } + + fn xorshift(seed: &mut u64) -> u64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + *seed + } + + /// Small coefficients, or (with `extreme`) values near the i32 limits so lifting can overflow. + fn coefficients(seed: &mut u64, count: usize, extreme: bool) -> Vec { + (0..count) + .map(|_| { + let r = xorshift(seed); + if extreme && r.is_multiple_of(5) { + if r & 32 == 0 { i32::MAX - (r >> 40) as i32 % 4 } else { i32::MIN + (r >> 40) as i32 % 4 } + } else { + (r % 2001) as i32 - 1000 + } + }) + .collect() + } + + const EDGE_CASES: [(bool, bool); 4] = [(false, false), (true, false), (false, true), (true, true)]; + + #[test] + fn line_lifting_matches_the_reference_for_every_length_and_edge() { + let mut seed = 0x0123_4567_89ab_cdef; + for length in 1..=40 { + for (before, after) in EDGE_CASES { + let (low_count, high_count) = sizes(length, before, after); + for extreme in [false, true] { + let low = coefficients(&mut seed, low_count, extreme); + let high = coefficients(&mut seed, high_count, extreme); + let (mut fast, mut reference) = (vec![0; length], vec![0; length]); + let fast_result = inverse_line(&low, &high, &mut fast, before, after); + let reference_result = reference_line(&low, &high, &mut reference, before, after); + let case = format!("length {length}, before {before}, after {after}, extreme {extreme}"); + assert_eq!(fast_result.is_ok(), reference_result.is_ok(), "{case}"); + assert_eq!( + fast_result.as_ref().err().map(ToString::to_string), + reference_result.as_ref().err().map(ToString::to_string), + "{case}" + ); + if fast_result.is_ok() { + assert_eq!(fast, reference, "{case}"); + } + } + } + } + } + + #[test] + fn row_wise_level_matches_column_lifting_for_every_size_and_edge() { + let mut seed = 0xfeed_beef_dead_cafe; + for width in 1..=17 { + for height in 1..=17 { + for (left, right) in EDGE_CASES { + for (top, bottom) in EDGE_CASES { + let edges = TileEdges { left, right, top, bottom }; + let (lw, hw) = sizes(width, left, right); + let (lhgt, hhgt) = sizes(height, top, bottom); + // Overflow cases on the smaller sizes only; they fail early and are cheap. + let extreme = width * height < 40 && xorshift(&mut seed).is_multiple_of(2); + let mut band = |w, h| Band { width: w, height: h, data: coefficients(&mut seed, w * h, extreme) }; + let bands = [band(lw, lhgt), band(hw, lhgt), band(lw, hhgt), band(hw, hhgt)]; + let fast = inverse_level(&bands[0], &bands[1], &bands[2], &bands[3], width, height, edges); + let reference = reference_level(&bands, width, height, edges); + let case = format!("{width}x{height}, {edges:?}, extreme {extreme}"); + assert_eq!(fast.as_ref().ok().map(|band| &band.data), reference.as_ref().ok(), "{case}"); + assert_eq!(fast.as_ref().err().map(ToString::to_string), reference.as_ref().err().map(ToString::to_string), "{case}"); + } + } + } + } + } } diff --git a/crates/raw/src/dng.rs b/crates/raw/src/dng.rs index aaa08e49a..fb3968239 100644 --- a/crates/raw/src/dng.rs +++ b/crates/raw/src/dng.rs @@ -299,17 +299,11 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f deinterleave(&mut data, w, h, cpp, factor(t::ROW_INTERLEAVE_FACTOR), factor(t::COLUMN_INTERLEAVE_FACTOR))?; } - // linearization table (integer data only) - let mut linearized = false; + // linearization table (integer data only) and opcode lists. OpcodeList1 runs on the stored + // codes first (DNG 1.7 chapter 7); see [`linearization_and_opcodes`]. + let (linearized, linearization_table, opcodes) = linearization_and_opcodes(pixels, raw, &mut data); // the largest value the table can give a sample: the white level when the file states none - let mut table_max = None; - if let (RawData::U16(v), Some(table)) = (&mut data, pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty())) { - let table: Vec = table.iter().map(|&x| x.min(65535) as u16).collect(); - let last = table.len() - 1; - v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); - linearized = true; - table_max = table.iter().copied().max().filter(|&m| m > 0); - } + let table_max = linearization_table.as_deref().and_then(|t| t.iter().copied().max().filter(|&m| m > 0)); // a reduced copy's own levels, where it states them let levels = |tag| if pixels.contains(tag) { pixels } else { raw }; @@ -442,10 +436,6 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f None }; - let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); - let opcodes = - crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; - let color = color_data(ifd0, raw, tiff.order); let gain_table_omitted = [raw, ifd0].iter().any(|ifd| ifd.contains(t::PROFILE_GAIN_TABLE_MAP) || ifd.contains(t::PROFILE_GAIN_TABLE_MAP_2)) && color.profile.gain_table_map.is_none(); @@ -465,6 +455,7 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f color, wb_multipliers: None, linearized, + linearization_table, pixel_aspect: default_scale_aspect(raw.f64s(t::DEFAULT_SCALE).as_deref()), opcodes, camera: Camera::from_ifd0(ifd0), @@ -477,6 +468,31 @@ fn decode_ifd(bytes: &[u8], tiff: &Tiff, pixels: &Ifd, main: &Ifd, mode: Mode, f Ok(img) } +/// Parse opcode lists and, for integer data, the LinearizationTable. +/// +/// OpcodeList1 runs on the stored codes, then the table maps them to sensor values (DNG 1.7 +/// chapter 7). Applying opcodes is the develop pipeline's work, so a non-empty list 1 leaves the +/// table unapplied (`linearized` is false) and available on the image. +fn linearization_and_opcodes(pixels: &Ifd, raw: &Ifd, data: &mut RawData) -> (bool, Option>, crate::OpcodeLists) { + let opcode_list = |tag| raw.bytes(tag).map(opcodes::parse_list).unwrap_or_default(); + let opcodes = + crate::OpcodeLists { list1: opcode_list(t::OPCODE_LIST_1), list2: opcode_list(t::OPCODE_LIST_2), list3: opcode_list(t::OPCODE_LIST_3) }; + let linearization_table = if matches!(data, RawData::U16(_)) { + pixels.u64s(t::LINEARIZATION_TABLE).filter(|t| !t.is_empty()).map(|table| table.iter().map(|&x| x.min(65535) as u16).collect::>()) + } else { + None + }; + let mut linearized = false; + if opcodes.list1.is_empty() + && let (RawData::U16(v), Some(table)) = (data, linearization_table.as_deref()) + { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + linearized = true; + } + (linearized, linearization_table, opcodes) +} + /// CinemaDNG `TimeCodes` (CinemaDNG Image Data Format Specification 1.0, tag 51043). const TIME_CODES: u16 = 51043; /// CinemaDNG `FrameRate` (tag 51044). diff --git a/crates/raw/src/dngwrite.rs b/crates/raw/src/dngwrite.rs index ace0c07f6..3a715085a 100644 --- a/crates/raw/src/dngwrite.rs +++ b/crates/raw/src/dngwrite.rs @@ -165,6 +165,14 @@ pub fn write_dng(raw: &RawImage, opts: &DngWriteOptions) -> Result> { } else { ifd.set(t::WHITE_LEVEL, Value::Long(white)); } + // A table still to be applied (OpcodeList1 has to run on the stored codes first) is written + // with the samples. Once the samples have been mapped, writing the tag would apply it twice + // on the next read of a file without list 1. + if !raw.linearized + && let Some(table) = raw.linearization_table.as_deref().filter(|t| !t.is_empty()) + { + ifd.set(t::LINEARIZATION_TABLE, Value::Short(table.to_vec())); + } let a = raw.active_area; ifd.set(t::ACTIVE_AREA, Value::Long(vec![a.y as u32, a.x as u32, (a.y + a.height) as u32, (a.x + a.width) as u32])); ifd.set(t::DEFAULT_CROP_ORIGIN, Value::Long(vec![raw.crop.x as u32, raw.crop.y as u32])); @@ -514,6 +522,7 @@ mod tests { color: crate::ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), diff --git a/crates/raw/src/erf.rs b/crates/raw/src/erf.rs index fcdfa43cd..b7dd9d8be 100644 --- a/crates/raw/src/erf.rs +++ b/crates/raw/src/erf.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Option { } // A damaged/unsupported pixel IFD must not hide usable orientation metadata from preview // fallback. CR3 keeps IFD0 in its CMT1 box; TIFF-based raws keep it at the file start. + // The `cr3` module exists only with the `cr3` feature: this gate is required so + // `cargo check -p craft-raw --no-default-features --features dng` builds (E0433 otherwise). #[cfg(feature = "cr3")] if probe(bytes) == Some(RawFormat::Cr3) { return cr3::orientation(bytes); @@ -1041,6 +1045,12 @@ pub struct RawImage { pub wb_multipliers: Option<[f32; 3]>, /// Whether a `LinearizationTable` was applied to `data`. pub linearized: bool, + /// The file's `LinearizationTable`, when present. `data` has already been mapped through it when + /// [`Self::linearized`] is true. When `linearized` is false and this is `Some`, the table was left + /// unapplied so `OpcodeList1` can run on the stored samples first (DNG chapter 7); call + /// [`Self::apply_linearization`] after that list. + #[serde(default)] + pub linearization_table: Option>, /// Height of a sensor pixel over its width (1.0 for square pixels). The D1X and D1H write 2.0: their frame is /// 4028 x 1324 and the picture is 3:2 only once the rows are doubled. A developer stretches the rows by /// this factor. @@ -1119,6 +1129,23 @@ impl RawImage { self.check(true) } + /// Map integer `data` through [`Self::linearization_table`] when the table has not yet been applied. + /// No-op when [`Self::linearized`] is already true, the table is missing, or the samples are not + /// integers. Call this after `OpcodeList1`, which DNG applies to the stored codes (chapter 7). + pub fn apply_linearization(&mut self) { + if self.linearized { + return; + } + let Some(table) = self.linearization_table.as_deref().filter(|t| !t.is_empty()) else { + return; + }; + if let RawData::U16(v) = &mut self.data { + let last = table.len() - 1; + v.iter_mut().for_each(|s| *s = table[(*s as usize).min(last)]); + self.linearized = true; + } + } + fn check(&self, samples: bool) -> Result<()> { if self.width == 0 || self.height == 0 || !(1..=4).contains(&self.cpp) { return Err(RawError::Corrupt("bad dimensions".into())); diff --git a/crates/raw/src/mef.rs b/crates/raw/src/mef.rs index 36ba18e0f..1180fe063 100644 --- a/crates/raw/src/mef.rs +++ b/crates/raw/src/mef.rs @@ -123,6 +123,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result Result Result Result Result= sum`. `count << k0` has the bit length of `sum` (for `k0` the bit-length + /// difference), so it is at most one step short. + #[inline] fn k(self) -> u32 { - let mut k = 0; - while self.count << k < self.sum { - k += 1; - } - k + let k = self.count.leading_zeros().saturating_sub(self.sum.leading_zeros()); + k + u32::from(self.count << k < self.sum) } + #[inline] fn update(&mut self, delta: i32) { self.sum += delta.unsigned_abs(); if self.count == 64 { @@ -202,8 +203,7 @@ impl State { } #[inline] -fn quant(v: i32, near: u32) -> i32 { - let t = [18 + 3 * near, 67 + 5 * near, 276 + 7 * near]; +fn quant(v: i32, near: u32, t: &[u32; 3]) -> i32 { let a = v.unsigned_abs(); let bucket = if a <= near { 0 @@ -232,124 +232,169 @@ fn even_prediction(nw: i32, n: i32, ne: i32, nn: i32) -> i32 { (pair + 2 * n) >> 2 } -#[inline] -fn sample(rows: &[Vec], y: usize, x: usize) -> Result { - rows.get(y).and_then(|r| r.get(x)).copied().ok_or_else(|| corrupt("invalid predictor position")) +/// Residual coding parameters of one precision (`2 * precision + 1` is the quantizer step). +#[derive(Clone, Copy)] +struct Coding { + range: u32, + bits: u32, + limit: u32, + step: i32, } -fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result> { - let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 }; - let mut previous = vec![vec![vec![0; size + 2]; 2]; 3]; - let max = (1u32 << bits) - 1; - let range = |near: u32| (max + 2 * near) / (2 * near + 1) + 1; - let seed = |near: u32| State { sum: ((range(near) + 32) / 64).max(2), count: 1 }; - let mut states = [[[seed(0); 41]; 2]; 3]; - // Smooth regions retain finer precision and independent state across block quantizer changes. - let mut fine = [[[[seed(0); 5]; 2]; 3], [[[seed(1); 5]; 2]; 3], [[[seed(2); 5]; 2]; 3]]; - let mut last_near = None; - let mut reader = Bits::new(stripe.bytes); - let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?; - // a capacity hint only: the stripe's declared height is the file's word, so the reservation is capped at a sample - // per bit of the stripe's real bytes (all stripes reserve in parallel; a one-byte stripe must not take 2 GiB) - let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8))); - for group in 0..height / 6 { - let near = match stripe.quantizers { - Some(q) => u32::from(*q.get(group).ok_or_else(|| corrupt("truncated quantizer table"))?), - None => 0, +impl Coding { + fn new(max: u32, sample_bits: u32, precision: u32) -> Self { + let range = (max + 2 * precision) / (2 * precision + 1) + 1; + let bits = 32 - (range - 1).leading_zeros(); + Self { range, bits, limit: 4 * sample_bits - bits - 1, step: (2 * precision + 1) as i32 } + } +} + +/// Everything that depends only on a six-row group's quantizer, computed when the quantizer changes. +struct Group { + near: u32, + thresholds: [u32; 3], + main: Coding, + /// Precisions 0, 1 and 2 for the low-gradient sites. + fine: [Coding; 3], +} + +impl Group { + fn new(near: u32, max: u32, sample_bits: u32) -> Self { + Self { + near, + thresholds: [18 + 3 * near, 67 + 5 * near, 276 + 7 * near], + main: Coding::new(max, sample_bits, near), + fine: [0, 1, 2].map(|p| Coding::new(max, sample_bits, p)), + } + } +} + +fn seed(max: u32, near: u32) -> State { + let range = (max + 2 * near) / (2 * near + 1) + 1; + State { sum: ((range + 32) / 64).max(2), count: 1 } +} + +/// Colour vectors of one stripe, kept for the rows of a six-row group plus the two rows above it. Red and blue get +/// three rows per group, green six; they are stored one after the other in one buffer of rows `stride` wide. +const ROWS: [usize; 3] = [5, 8, 5]; +const BASE: [usize; 3] = [0, 5, 13]; +const TOTAL_ROWS: usize = 18; + +/// The three lines a sample is predicted from: the two above and the one being decoded. Each line has one edge +/// cell on both sides, which the new line takes from the line above. +struct Lines<'a> { + above2: &'a [i32], + above: &'a [i32], + current: &'a mut [i32], +} + +impl<'a> Lines<'a> { + fn new(plane: &'a mut [i32], y: usize, stride: usize) -> Self { + let (done, rest) = plane.split_at_mut(y * stride); + let done: &[i32] = done; + let current = &mut rest[..stride]; + let above = &done[(y - 1) * stride..]; + let above2 = &done[(y - 2) * stride..(y - 1) * stride]; + current[0] = above[1]; + current[stride - 1] = above[stride - 2]; + Self { above2, above, current } + } +} + +struct Coder<'a> { + reader: Bits<'a>, + states: [[[State; 41]; 2]; 3], + /// Smooth regions retain finer precision and independent state across block quantizer changes. + fine: [[[[State; 5]; 2]; 3]; 3], + max: u32, + xtrans: bool, +} + +impl Coder<'_> { + /// Decodes one sample's residual with the context its gradients select and applies it to the prediction. + #[inline] + fn residual(&mut self, g: &Group, pair: usize, odd: usize, (d1, d2): (i32, i32), predict: i32) -> Result { + let near = g.near; + let contrast = d1.unsigned_abs() + d2.unsigned_abs(); + let precision = match contrast { + 0..=5 => 0, + 6 => near.min(1), + 7 => near.min(2), + _ => near, }; - if last_near != Some(near) { - states = [[[seed(near); 41]; 2]; 3]; - last_near = Some(near); + let (context, state, coding) = if precision < near { + let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() }; + let context = 3 * sign(d1) + sign(d2); + (context, &mut self.fine[precision as usize][pair % 3][odd][context.unsigned_abs() as usize], g.fine[precision as usize]) + } else { + let context = 9 * quant(d1, near, &g.thresholds) + quant(d2, near, &g.thresholds); + (context, &mut self.states[pair % 3][odd][context.unsigned_abs() as usize], g.main) + }; + let mut delta = self.reader.residual(state.k(), coding.bits, coding.range, coding.limit)?; + state.update(delta); + if context < 0 { + delta = -delta; } - let mut rows = previous; + let mut value = predict + delta * coding.step; + if value < -(precision as i32) { + value += coding.range as i32 * coding.step; + } else if value > (self.max + precision) as i32 { + value -= coding.range as i32 * coding.step; + } + Ok(value.clamp(0, self.max as i32)) + } + + /// One position of the two colour lines of a pair: the even column (`odd == 0`) or the one seven behind it. + #[inline] + fn position(&mut self, g: &Group, pair: usize, odd: usize, pos: usize, lines: &mut [Lines<'_>; 2]) -> Result<()> { + let x = pos + 1; + for (which, l) in lines.iter_mut().enumerate() { + let (nw, north, ne) = (l.above[x - 1], l.above[x], l.above[x + 1]); + let real = !self.xtrans || (MASKS[pair][which] >> (pos % 4)) & 1 != 0; + l.current[x] = if odd == 0 { + let nn = l.above2[x]; + let predict = even_prediction(nw, north, ne, nn); + if real { self.residual(g, pair, 0, (north - nn, nw - north), predict)? } else { predict } + } else { + let (west, east) = (l.current[x - 1], l.current[x + 1]); + let predict = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 }; + if real { self.residual(g, pair, 1, (north - nw, nw - west), predict)? } else { predict } + }; + } + Ok(()) + } + + /// Decodes the six colour-vector pairs of one group into `buf`. + fn group(&mut self, g: &Group, buf: &mut [i32], size: usize) -> Result<()> { + let stride = size + 2; + let (red, rest) = buf.split_at_mut(ROWS[0] * stride); + let (green, blue) = rest.split_at_mut(ROWS[1] * stride); for pair in 0..6 { - let colours = if pair % 2 == 0 { [0, 1] } else { [1, 2] }; - for &colour in &colours { - let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?; - let mut row = vec![0; size + 2]; - let prev = lines.len() - 1; - let left = sample(lines, prev, 1)?; - let right = sample(lines, prev, size)?; - if let Some(v) = row.first_mut() { - *v = left; - } - if let Some(v) = row.last_mut() { - *v = right; - } - lines.push(row); - } + let mut lines = if pair % 2 == 0 { + [Lines::new(&mut *red, 2 + pair / 2, stride), Lines::new(&mut *green, 2 + pair, stride)] + } else { + [Lines::new(&mut *green, 2 + pair, stride), Lines::new(&mut *blue, 2 + pair / 2, stride)] + }; for even in (0..size + 8).step_by(2) { - for (odd, position) in [(0, Some(even)), (1, even.checked_sub(7))] { - let Some(pos) = position.filter(|&x| x < size) else { continue }; - let x = pos + 1; - for (which, &colour) in colours.iter().enumerate() { - let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?; - let y = lines.len() - 1; - let (nw, north, ne, nn) = - (sample(lines, y - 1, x - 1)?, sample(lines, y - 1, x)?, sample(lines, y - 1, x + 1)?, sample(lines, y - 2, x)?); - let west = sample(lines, y, x - 1)?; - let east = sample(lines, y, x + 1)?; - let (predict, d1, d2) = if odd == 0 { - (even_prediction(nw, north, ne, nn), north - nn, nw - north) - } else { - let p = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 }; - (p, north - nw, nw - west) - }; - let real = !xtrans || MASKS.get(pair).and_then(|m| m.get(which)).is_some_and(|m| (m >> (pos % 4)) & 1 != 0); - let value = if real { - let contrast = d1.unsigned_abs() + d2.unsigned_abs(); - let precision = match contrast { - 0..=5 => 0, - 6 => near.min(1), - 7 => near.min(2), - _ => near, - }; - let (context, state) = if precision < near { - let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() }; - let context = 3 * sign(d1) + sign(d2); - let state = fine - .get_mut(precision as usize) - .and_then(|f| f.get_mut(pair % 3)) - .and_then(|s| s.get_mut(odd)) - .and_then(|s| s.get_mut(context.unsigned_abs() as usize)) - .ok_or_else(|| corrupt("invalid fine coding context"))?; - (context, state) - } else { - let context = 9 * quant(d1, near) + quant(d2, near); - let state = states - .get_mut(pair % 3) - .and_then(|s| s.get_mut(odd)) - .and_then(|s| s.get_mut(context.unsigned_abs() as usize)) - .ok_or_else(|| corrupt("invalid coding context"))?; - (context, state) - }; - let coded_range = range(precision); - let coded_bits = 32 - (coded_range - 1).leading_zeros(); - let mut delta = reader.residual(state.k(), coded_bits, coded_range, 4 * bits - coded_bits - 1)?; - state.update(delta); - if context < 0 { - delta = -delta; - } - let step = (2 * precision + 1) as i32; - let mut value = predict + delta * step; - if value < -(precision as i32) { - value += coded_range as i32 * step; - } else if value > (max + precision) as i32 { - value -= coded_range as i32 * step; - } - value.clamp(0, max as i32) - } else { - predict - }; - *lines.get_mut(y).and_then(|r| r.get_mut(x)).ok_or_else(|| corrupt("invalid predictor position"))? = value; - } + if even < size { + self.position(g, pair, 0, even, &mut lines)?; + } + if even >= 7 && even - 7 < size { + self.position(g, pair, 1, even - 7, &mut lines)?; } } } - for y in 0..6 { - for x in 0..stripe.width { + Ok(()) + } +} + +/// For each of the six rows of a group, where each output sample sits in the colour-vector buffer. +fn placement(xtrans: bool, width: usize, stride: usize) -> [Vec; 6] { + std::array::from_fn(|y| { + (0..width) + .map(|x| { let colour = if xtrans { - XTRANS.get(y).and_then(|r| r.get(x % 6)).copied().ok_or_else(|| corrupt("invalid CFA position"))? + XTRANS[y][x % 6] } else { match (y % 2, x % 2) { (0, 0) => 0, @@ -359,10 +404,49 @@ fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> }; let row = 2 + if colour == 1 { y } else { y / 2 }; let col = 1 + if xtrans { 2 * (x / 3) + (x % 3).min(1) } else { x / 2 }; - output.push(sample(rows.get(colour).ok_or_else(|| corrupt("invalid colour vector"))?, row, col)? as u16); - } + (BASE[colour] + row) * stride + col + }) + .collect() + }) +} + +fn decode_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result> { + let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 }; + let stride = size + 2; + let max = (1u32 << bits) - 1; + let mut coder = Coder { + reader: Bits::new(stripe.bytes), + states: [[[seed(max, 0); 41]; 2]; 3], + fine: [[[[seed(max, 0); 5]; 2]; 3], [[[seed(max, 1); 5]; 2]; 3], [[[seed(max, 2); 5]; 2]; 3]], + max, + xtrans, + }; + let mut last_near = None; + let mut group = Group::new(0, max, bits); + let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?; + // a capacity hint only: the stripe's declared height is the file's word, so the reservation is capped at a sample + // per bit of the stripe's real bytes (all stripes reserve in parallel; a one-byte stripe must not take 2 GiB) + let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8))); + let place = placement(xtrans, stripe.width, stride); + let mut buf = vec![0i32; TOTAL_ROWS * stride]; + for index in 0..height / 6 { + let near = match stripe.quantizers { + Some(q) => u32::from(*q.get(index).ok_or_else(|| corrupt("truncated quantizer table"))?), + None => 0, + }; + if last_near != Some(near) { + coder.states = [[[seed(max, near); 41]; 2]; 3]; + group = Group::new(near, max, bits); + last_near = Some(near); + } + coder.group(&group, &mut buf, size)?; + for row in &place { + output.extend(row.iter().map(|&at| buf[at] as u16)); + } + // the last two rows of each colour are the next group's rows above + for (base, rows) in BASE.iter().zip(ROWS) { + buf.copy_within((base + rows - 2) * stride..(base + rows) * stride, base * stride); } - previous = rows.into_iter().map(|mut r| r.split_off(r.len() - 2)).collect(); } Ok(output) } @@ -590,3 +674,260 @@ mod tests { } } } + +/// The straightforward decoder this module started from (shift loop for `k`, an `Option`/`Result` neighbour read +/// per access, a `Vec` per colour line, one pixel placed at a time), kept as the oracle for the optimised one. +#[cfg(test)] +mod differential { + use super::*; + use sha2::{Digest, Sha256}; + + fn k_by_shifting(state: State) -> u32 { + let mut k = 0; + while state.count << k < state.sum { + k += 1; + } + k + } + + fn quant_by_table(v: i32, near: u32) -> i32 { + let t = [18 + 3 * near, 67 + 5 * near, 276 + 7 * near]; + quant(v, near, &t) + } + + fn sample(rows: &[Vec], y: usize, x: usize) -> Result { + rows.get(y).and_then(|r| r.get(x)).copied().ok_or_else(|| corrupt("invalid predictor position")) + } + + fn reference_stripe(stripe: &Stripe<'_>, height: usize, bits: u32, xtrans: bool) -> Result> { + let size = if xtrans { BLOCK * 2 / 3 } else { BLOCK / 2 }; + let mut previous = vec![vec![vec![0; size + 2]; 2]; 3]; + let max = (1u32 << bits) - 1; + let range = |near: u32| (max + 2 * near) / (2 * near + 1) + 1; + let seed = |near: u32| State { sum: ((range(near) + 32) / 64).max(2), count: 1 }; + let mut states = [[[seed(0); 41]; 2]; 3]; + let mut fine = [[[[seed(0); 5]; 2]; 3], [[[seed(1); 5]; 2]; 3], [[[seed(2); 5]; 2]; 3]]; + let mut last_near = None; + let mut reader = Bits::new(stripe.bytes); + let n = height.checked_mul(stripe.width).filter(|&n| n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?; + let mut output = Vec::with_capacity(n.min(stripe.bytes.len().saturating_mul(8))); + for group in 0..height / 6 { + let near = match stripe.quantizers { + Some(q) => u32::from(*q.get(group).ok_or_else(|| corrupt("truncated quantizer table"))?), + None => 0, + }; + if last_near != Some(near) { + states = [[[seed(near); 41]; 2]; 3]; + last_near = Some(near); + } + let mut rows = previous; + for pair in 0..6 { + let colours = if pair % 2 == 0 { [0, 1] } else { [1, 2] }; + for &colour in &colours { + let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?; + let mut row = vec![0; size + 2]; + let prev = lines.len() - 1; + let left = sample(lines, prev, 1)?; + let right = sample(lines, prev, size)?; + if let Some(v) = row.first_mut() { + *v = left; + } + if let Some(v) = row.last_mut() { + *v = right; + } + lines.push(row); + } + for even in (0..size + 8).step_by(2) { + for (odd, position) in [(0, Some(even)), (1, even.checked_sub(7))] { + let Some(pos) = position.filter(|&x| x < size) else { continue }; + let x = pos + 1; + for (which, &colour) in colours.iter().enumerate() { + let lines = rows.get_mut(colour).ok_or_else(|| corrupt("invalid colour vector"))?; + let y = lines.len() - 1; + let (nw, north, ne, nn) = + (sample(lines, y - 1, x - 1)?, sample(lines, y - 1, x)?, sample(lines, y - 1, x + 1)?, sample(lines, y - 2, x)?); + let west = sample(lines, y, x - 1)?; + let east = sample(lines, y, x + 1)?; + let (predict, d1, d2) = if odd == 0 { + (even_prediction(nw, north, ne, nn), north - nn, nw - north) + } else { + let p = if (nw.min(ne)..=nw.max(ne)).contains(&north) { (west + east) >> 1 } else { (west + east + 2 * north) >> 2 }; + (p, north - nw, nw - west) + }; + let real = !xtrans || MASKS.get(pair).and_then(|m| m.get(which)).is_some_and(|m| (m >> (pos % 4)) & 1 != 0); + let value = if real { + let contrast = d1.unsigned_abs() + d2.unsigned_abs(); + let precision = match contrast { + 0..=5 => 0, + 6 => near.min(1), + 7 => near.min(2), + _ => near, + }; + let (context, state) = if precision < near { + let sign = |v: i32| if v.unsigned_abs() <= precision { 0 } else { v.signum() }; + let context = 3 * sign(d1) + sign(d2); + let state = fine + .get_mut(precision as usize) + .and_then(|f| f.get_mut(pair % 3)) + .and_then(|s| s.get_mut(odd)) + .and_then(|s| s.get_mut(context.unsigned_abs() as usize)) + .ok_or_else(|| corrupt("invalid fine coding context"))?; + (context, state) + } else { + let context = 9 * quant_by_table(d1, near) + quant_by_table(d2, near); + let state = states + .get_mut(pair % 3) + .and_then(|s| s.get_mut(odd)) + .and_then(|s| s.get_mut(context.unsigned_abs() as usize)) + .ok_or_else(|| corrupt("invalid coding context"))?; + (context, state) + }; + let coded_range = range(precision); + let coded_bits = 32 - (coded_range - 1).leading_zeros(); + let mut delta = reader.residual(k_by_shifting(*state), coded_bits, coded_range, 4 * bits - coded_bits - 1)?; + state.update(delta); + if context < 0 { + delta = -delta; + } + let step = (2 * precision + 1) as i32; + let mut value = predict + delta * step; + if value < -(precision as i32) { + value += coded_range as i32 * step; + } else if value > (max + precision) as i32 { + value -= coded_range as i32 * step; + } + value.clamp(0, max as i32) + } else { + predict + }; + *lines.get_mut(y).and_then(|r| r.get_mut(x)).ok_or_else(|| corrupt("invalid predictor position"))? = value; + } + } + } + } + for y in 0..6 { + for x in 0..stripe.width { + let colour = if xtrans { + XTRANS.get(y).and_then(|r| r.get(x % 6)).copied().ok_or_else(|| corrupt("invalid CFA position"))? + } else { + match (y % 2, x % 2) { + (0, 0) => 0, + (1, 1) => 2, + _ => 1, + } + }; + let row = 2 + if colour == 1 { y } else { y / 2 }; + let col = 1 + if xtrans { 2 * (x / 3) + (x % 3).min(1) } else { x / 2 }; + output.push(sample(rows.get(colour).ok_or_else(|| corrupt("invalid colour vector"))?, row, col)? as u16); + } + } + previous = rows.into_iter().map(|mut r| r.split_off(r.len() - 2)).collect(); + } + Ok(output) + } + + /// Deterministic bytes: uniform (0), mostly zero (1: long unary prefixes and escapes) or one-heavy (2: short + /// prefixes and small residuals, which stay inside the range of every quantizer so the stream decodes to the end). + fn bytes(seed: u64, len: usize, kind: u8) -> Vec { + let mut x = seed | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + let b = (x >> 24) as u8; + match kind { + 1 if x & 0b11 != 0 => 0, + 2 => b | (x >> 40) as u8, + _ => b, + } + }) + .collect() + } + + #[test] + fn k_without_the_shift_loop_matches_it() { + let mut sums: Vec = (0..=4200).collect(); + for i in 0..=30 { + sums.extend([(1u32 << i) - 1, 1 << i, (1 << i) + 1]); + } + for count in 1..=64 { + for &sum in &sums { + let state = State { sum, count }; + assert_eq!(state.k(), k_by_shifting(state), "count {count}, sum {sum}"); + } + } + } + + /// Every variant the stripe decoder handles (lossless and lossy, Bayer and X-Trans, 12/14/16 bits, full and + /// partial stripes, quantizer changes, fine contexts) gives the oracle's samples, or its error, on + /// uniform pseudo-random entropy streams that reach all contexts, one-heavy ones that every quantizer table decodes + /// to the end, and sparse ones that reach the escapes. Each quantizer table must have streams that finish. + #[test] + fn stripe_decoder_matches_the_straightforward_one() { + let quantizers: [Option<&[u8]>; 4] = + [None, Some([0, 1, 2, 3].as_slice()), Some([2, 2, 7, 0, 40, 255].as_slice()), Some([1, 1, 0, 5, 5, 9, 2].as_slice())]; + // uniform and one-heavy streams that decode to the end, per quantizer table + let mut finished = [0usize; 4]; + for bits in [12, 14, 16] { + for xtrans in [false, true] { + for (table, q) in quantizers.into_iter().enumerate() { + for (seed, width, kind) in [(1, BLOCK, 0), (2, 301, 0), (3, 12, 0), (4, BLOCK, 1), (5, BLOCK, 2), (6, 301, 2)] { + let data = bytes(seed * 31 + u64::from(bits), 32 << 10, kind); + let groups = q.map_or(3, <[u8]>::len); + let stripe = Stripe { bytes: &data, quantizers: q, width }; + let want = reference_stripe(&stripe, groups * 6, bits, xtrans).map_err(|e| e.to_string()); + let got = decode_stripe(&stripe, groups * 6, bits, xtrans).map_err(|e| e.to_string()); + assert_eq!(got, want, "bits {bits}, xtrans {xtrans}, quantizers {q:?}, width {width}, stream kind {kind}"); + finished[table] += usize::from(want.is_ok() && kind != 1); + } + } + } + } + eprintln!("dense streams decoded to the end per quantizer table (of 30): {finished:?}"); + assert!(finished.iter().all(|&n| n > 0), "some quantizer table never decodes to the end: {finished:?}"); + } + + /// Optional: the CC0 files of docs/raf-corpus.sha256 in the directory named by `LIGHTCRAFT_RAF_CORPUS` (file + /// names as listed there, e.g. `raf-fuji-xt5-6122.raf`). Each present file's SHA-256 is checked, then the + /// decoded sensor samples are compared with the hash recorded from the decoder before the optimisation. + #[test] + fn optional_cc0_samples_decode_as_recorded() { + let Some(directory) = std::env::var_os("LIGHTCRAFT_RAF_CORPUS") else { return }; + let list = include_str!("../docs/raf-corpus.sha256"); + for line in list.lines().filter(|l| !l.starts_with('#') && !l.trim().is_empty()) { + let mut columns = line.split_whitespace(); + let (Some(digest), Some(path)) = (columns.next(), columns.next()) else { panic!("bad corpus row {line:?}") }; + let name = path.rsplit('/').next().unwrap(); + let Ok(bytes) = std::fs::read(std::path::Path::new(&directory).join(name)) else { continue }; + let sha = Sha256::digest(&bytes).iter().map(|b| format!("{b:02x}")).collect::(); + assert_eq!(sha, digest, "{name}: not the published file"); + let Some(&(_, expected)) = RECORDED.iter().find(|(n, _)| *n == name) else { panic!("{name}: no recorded hash") }; + let image = crate::decode(&bytes).unwrap(); + let crate::RawData::U16(data) = &image.data else { panic!("{name}: not 16-bit samples") }; + let hash = data.iter().flat_map(|v| v.to_le_bytes()).fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(b)).wrapping_mul(0x100000001b3)); + assert_eq!(hash, expected, "{name}"); + } + } + + /// FNV-1a over the little-endian decoded samples, recorded from `upstream/main` before the change. + const RECORDED: &[(&str, u64)] = &[ + ("raf-fuji-gfx100-3773.raf", 0x1195d3ab70da7cb4), + ("raf-fuji-gfx100-3775.raf", 0xb7c7c1a7717ef93c), + ("raf-fuji-gfx100rf-8091.raf", 0x49110f1b80eefe2e), + ("raf-fuji-gfx100s-4495.raf", 0x84ab46122f72796c), + ("raf-fuji-gfx100s-4503.raf", 0x63debf0afde73e2e), + ("raf-fuji-gfx50s-1435.raf", 0xb3e570a6c88455f5), + ("raf-fuji-xe5-8509.raf", 0x6a7d5cde2832d4f3), + ("raf-fuji-xh2-6001.raf", 0x466a5707ea484f9c), + ("raf-fuji-xh2-6002.raf", 0xbbf043676476925a), + ("raf-fuji-xm5-7748.raf", 0xedd17960a117a553), + ("raf-fuji-xt2-865.raf", 0xe020816ec114436f), + ("raf-fuji-xt20-compressed.raf", 0x85014025edc2cebd), + ("raf-fuji-xt4-3914.raf", 0x7c98e527dbe01f23), + ("raf-fuji-xt4-3918.raf", 0xfb4ba0df34c1c904), + ("raf-fuji-xt5-6122.raf", 0x8f76abd57de79a94), + ("raf-fuji-xt5-6123.raf", 0x0b2784853c31278d), + ("raf-fuji-xt50-7807.raf", 0xa8e2be9cac387ebc), + ]; +} diff --git a/crates/raw/src/rw2.rs b/crates/raw/src/rw2.rs index 1dc1a885e..c0101e85e 100644 --- a/crates/raw/src/rw2.rs +++ b/crates/raw/src/rw2.rs @@ -672,6 +672,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result Result= w || y0 >= h { - continue; - } - let copy_w = cw.min(w - x0); - let copy_h = ch.min(h - y0); - for y in 0..copy_h { - for x in 0..copy_w { - for s in 0..ccpp { - let src = (y * cw + x) * ccpp + s; - let dst_s = if planar { c.plane as usize } else { s }; - if dst_s >= cpp { - continue; - } - let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s; - match &px { - ChunkPx::U16(v) => { - if let (Some(o), Some(&s)) = (out_u16.get_mut(dst), v.get(src)) { - *o = s; - } - } - ChunkPx::F32(v) => { - if let (Some(o), Some(&s)) = (out_f32.get_mut(dst), v.get(src)) { - *o = s; - } - } - } + // a chunk that cannot be decoded is not a partial image: the file stays preview-only + let decoded = decoded.into_iter().collect::>>()?; + // The copy runs in parallel over bands of output rows: each band takes its rows of every chunk of the + // group that reaches it, in chunk order (so overlapping chunks resolve as a serial copy would). + let row_samples = w * cpp; + let threads = rayon::current_num_threads().max(1); + let band_rows = h.div_ceil(threads * 4).max(1); + if float { + out_f32.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| { + for (c, px) in &decoded { + if let ChunkPx::F32(v) = px { + place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar); } } - } + }); + } else { + out_u16.par_chunks_mut(band_rows * row_samples).enumerate().for_each(|(bi, band)| { + for (c, px) in &decoded { + if let ChunkPx::U16(v) = px { + place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar); + } + } + }); } decode_from = to; } Ok(if float { RawData::F32(out_f32) } else { RawData::U16(out_u16) }) } +/// Copy the rows of chunk `c` (samples `src`, `c.width` per row, `ccpp` samples per pixel) that fall into the output +/// band `band`, which holds the image rows from `band_y0` on (`w` pixels of `cpp` samples each, `h` rows in the whole +/// image). Pixels past the image edge and samples `src` does not hold are skipped; a planar chunk (`planar`) writes +/// its one sample per pixel into the channel `c.plane`, a chunk of a plane the image does not have is skipped. +#[allow(clippy::too_many_arguments)] +fn place_chunk(band: &mut [T], band_y0: usize, c: &Chunk, src: &[T], w: usize, h: usize, cpp: usize, planar: bool) { + let (cw, ch) = (c.width as usize, c.height as usize); + let (x0, y0) = (c.x as usize, c.y as usize); + if x0 >= w || y0 >= h || cpp == 0 { + return; + } + let plane = c.plane as usize; + if planar && plane >= cpp { + return; + } + let ccpp = if planar { 1 } else { cpp }; + let copy_w = cw.min(w - x0); + let copy_h = ch.min(h - y0); + let band_rows = band.len() / (w * cpp); + let first = y0.max(band_y0); + let last = (y0 + copy_h).min(band_y0 + band_rows); + for y in first..last { + let s0 = (y - y0) * cw * ccpp; + let Some(avail) = src.get(s0..) else { break }; + let d0 = ((y - band_y0) * w + x0) * cpp; + if planar { + let row = &mut band[d0..d0 + copy_w * cpp]; + for (o, &v) in row[plane..].iter_mut().step_by(cpp).zip(&avail[..copy_w.min(avail.len())]) { + *o = v; + } + } else { + let n = (copy_w * cpp).min(avail.len()); + band[d0..d0 + n].copy_from_slice(&avail[..n]); + } + } +} + /// Framing precedes allocation even in header-only probes: missing source rows may not /// be reported as a decodable image and later silently filled with zero samples. fn check_uncompressed_chunks(data: &[u8], info: &ImageInfo, chunks: &[Chunk], packing: Packing) -> Result<()> { @@ -512,13 +538,14 @@ fn unpack_chunk( let row_bytes = row_n * bp; let mut out = vec![0f32; row_n * ch]; let mut rowbuf = vec![0u8; row_bytes]; + let mut scratch = Vec::new(); for y in 0..ch { let s = src.get(y * row_bytes..).unwrap_or(&[]); let len = s.len().min(row_bytes); rowbuf[..len].copy_from_slice(&s[..len]); rowbuf[len..].fill(0); let big = if fp { - undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor); + undo_float_predictor(&mut rowbuf, row_n, bp, cpp * factor, &mut scratch); true } else { if factor > 0 { @@ -526,16 +553,15 @@ fn unpack_chunk( } order == ByteOrder::Big }; - for (i, o) in out[y * row_n..(y + 1) * row_n].iter_mut().enumerate() { - let b = &rowbuf[i * bp..(i + 1) * bp]; - *o = match (bp, big) { - (2, true) => f16_to_f32(u16::from_be_bytes([b[0], b[1]])), - (2, false) => f16_to_f32(u16::from_le_bytes([b[0], b[1]])), - (3, true) => f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]])), - (3, false) => f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0])), - (4, true) => f32::from_be_bytes([b[0], b[1], b[2], b[3]]), - _ => f32::from_le_bytes([b[0], b[1], b[2], b[3]]), - }; + let dst = &mut out[y * row_n..(y + 1) * row_n]; + let samples = rowbuf.chunks_exact(bp); + match (bp, big) { + (2, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_be_bytes([b[0], b[1]]))), + (2, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f16_to_f32(u16::from_le_bytes([b[0], b[1]]))), + (3, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_be_bytes([0, b[0], b[1], b[2]]))), + (3, false) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f24_to_f32(u32::from_le_bytes([b[0], b[1], b[2], 0]))), + (4, true) => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_be_bytes([b[0], b[1], b[2], b[3]])), + _ => dst.iter_mut().zip(samples).for_each(|(o, b)| *o = f32::from_le_bytes([b[0], b[1], b[2], b[3]])), } } return Ok(ChunkPx::F32(out)); @@ -544,6 +570,7 @@ fn unpack_chunk( return Err(RawError::Unsupported("floating-point predictor on integer data".into())); } let mut out = vec![0u16; row_n * ch]; + let mut words = Vec::new(); let (row_bytes, kind) = match (bits, packing) { (8, _) => (row_n, 0), (16, _) | (_, Packing::Word16) => (row_n * 2, 1), @@ -559,7 +586,7 @@ fn unpack_chunk( 0 => row.iter_mut().zip(s).for_each(|(o, &b)| *o = b as u16), 1 => read_u16s(s, order, row), 2 => unpack_msb(s, bits, row), - 4 => unpack_words32_msb(s, order, bits, row), + 4 => unpack_words32_msb_with(s, order, bits, row, &mut words), _ => unpack_lsb(s, bits, row), } if factor > 0 { @@ -581,6 +608,86 @@ fn unpack_chunk( Ok(ChunkPx::U16(out)) } +#[cfg(test)] +mod place_tests { + use super::*; + + /// The serial per-sample copy `place_chunk` replaced: the oracle. + #[allow(clippy::too_many_arguments)] + fn reference(out: &mut [u16], c: &Chunk, v: &[u16], w: usize, h: usize, cpp: usize, planar: bool) { + let ccpp = if planar { 1 } else { cpp }; + let (cw, ch) = (c.width as usize, c.height as usize); + let (x0, y0) = (c.x as usize, c.y as usize); + if x0 >= w || y0 >= h { + return; + } + for y in 0..ch.min(h - y0) { + for x in 0..cw.min(w - x0) { + for s in 0..ccpp { + let src = (y * cw + x) * ccpp + s; + let dst_s = if planar { c.plane as usize } else { s }; + if dst_s >= cpp { + continue; + } + let dst = ((y0 + y) * w + x0 + x) * cpp + dst_s; + if let (Some(o), Some(&s)) = (out.get_mut(dst), v.get(src)) { + *o = s; + } + } + } + } + } + + fn chunk(x: u32, y: u32, width: u32, height: u32, plane: u16) -> Chunk { + Chunk { index: 0, x, y, width, height, plane, offset: 0, len: 0 } + } + + /// Chunks that overlap (the later one wins), run past the image edge, hold fewer samples than their size, or name a + /// plane the image does not have, placed through bands of several heights, equal the serial copy. + #[test] + fn place_chunk_matches_the_serial_copy_for_odd_chunks() { + let (w, h) = (11, 9); + for cpp in [1usize, 3] { + for planar in [false, true] { + let ccpp = if planar { 1 } else { cpp }; + let specs = [ + (0, 0, 8, 6, 0, 1u16), + (4, 3, 8, 8, 0, 2), // overlaps the first and runs past both edges + (4, 3, 8, 8, 0, 3), // the same place again, later wins + (0, 5, 5, 4, 1, 4), + (2, 2, 6, 6, 2, 5), + (0, 0, 4, 4, 3, 6), // plane 3: not an image plane for cpp <= 3 + (9, 0, 6, 12, 0, 7), // only two columns inside + (20, 0, 4, 4, 0, 8), // right of the image + (0, 0, 11, 9, 0, 9), // whole image, short source below + ]; + let chunks: Vec<(Chunk, Vec)> = specs + .iter() + .map(|&(x, y, cw, ch, plane, val)| { + let n = (cw * ch) as usize * ccpp; + // the last chunk holds only 40 samples + let n = if val == 9 { 40 } else { n }; + (chunk(x, y, cw, ch, plane), (0..n).map(|i| val * 100 + i as u16).collect()) + }) + .collect(); + let mut want = vec![0u16; w * h * cpp]; + for (c, v) in &chunks { + reference(&mut want, c, v, w, h, cpp, planar); + } + for band_rows in [1usize, 2, 4, 9, 20] { + let mut got = vec![0u16; w * h * cpp]; + for (bi, band) in got.chunks_mut(band_rows * w * cpp).enumerate() { + for (c, v) in &chunks { + place_chunk(band, bi * band_rows, c, v, w, h, cpp, planar); + } + } + assert_eq!(got, want, "cpp {cpp} planar {planar} band rows {band_rows}"); + } + } + } + } +} + #[cfg(test)] #[cfg(feature = "dng")] mod lossy_tests { diff --git a/crates/raw/src/unpack.rs b/crates/raw/src/unpack.rs index fd749c386..27d77b55e 100644 --- a/crates/raw/src/unpack.rs +++ b/crates/raw/src/unpack.rs @@ -46,12 +46,18 @@ pub fn unpack_lsb(src: &[u8], bits: u32, out: &mut [u16]) { /// data, which is not the byte-wise MSB stream of [`unpack_msb`] unless `order` is big-endian. A short last word /// is padded with zeros. pub fn unpack_words32_msb(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16]) { - let words = src.chunks(4).flat_map(|c| { + unpack_words32_msb_with(src, order, bits, out, &mut Vec::new()); +} + +/// [`unpack_words32_msb`] with the byte-swapped copy of `src` kept in `scratch` (reused across rows). +pub fn unpack_words32_msb_with(src: &[u8], order: ByteOrder, bits: u32, out: &mut [u16], scratch: &mut Vec) { + scratch.clear(); + scratch.extend(src.chunks(4).flat_map(|c| { let mut w = [0u8; 4]; w[..c.len()].copy_from_slice(c); order.u32(w).to_be_bytes() - }); - unpack_msb(&words.collect::>(), bits, out); + })); + unpack_msb(scratch, bits, out); } pub fn read_u16s(src: &[u8], order: ByteOrder, out: &mut [u16]) { @@ -126,10 +132,12 @@ pub fn undo_diff_u8(row: &mut [u8], stride: usize) { /// Undo the floating-point predictor on one row: `row` holds `n` samples of `bytes_per` bytes, byte-differenced /// with stride `stride` samples, then stored as byte planes (most significant first). Returns the samples as -/// big-endian byte groups in place. -pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize) { +/// big-endian byte groups in place. `scratch` holds the byte planes (reused across rows). +pub fn undo_float_predictor(row: &mut [u8], n: usize, bytes_per: usize, stride: usize, scratch: &mut Vec) { undo_diff_u8(row, stride); - let tmp = row.to_vec(); + scratch.clear(); + scratch.extend_from_slice(row); + let tmp = &scratch[..]; for i in 0..n { for b in 0..bytes_per { if let (Some(dst), Some(&s)) = (row.get_mut(i * bytes_per + b), tmp.get(b * n + i)) { @@ -167,6 +175,22 @@ mod tests { assert_eq!(o, [0x000, 0x012]); // the word 0x00001234, top bits first } + #[test] + fn words32_scratch_is_reusable() { + let long: Vec = (0u8..24).map(|i| i.wrapping_mul(37).wrapping_add(5)).collect(); + let short = &long[3..10]; + let mut scratch = Vec::new(); + for order in [ByteOrder::Little, ByteOrder::Big] { + for (src, n) in [(&long[..], 16usize), (short, 4)] { + let mut want = vec![0u16; n]; + unpack_words32_msb(src, order, 12, &mut want); + let mut got = vec![0u16; n]; + unpack_words32_msb_with(src, order, 12, &mut got, &mut scratch); + assert_eq!(got, want); + } + } + } + #[test] fn packing() { // 12-bit MSB: 0xABC, 0xDEF -> AB CD EF @@ -210,7 +234,7 @@ mod tests { for i in (1..row.len()).rev() { row[i] = row[i].wrapping_sub(row[i - 1]); } - undo_float_predictor(&mut row, 2, 4, 1); + undo_float_predictor(&mut row, 2, 4, 1, &mut Vec::new()); assert_eq!(f32::from_be_bytes([row[0], row[1], row[2], row[3]]), 1.0); assert_eq!(f32::from_be_bytes([row[4], row[5], row[6], row[7]]), 2.0); } diff --git a/crates/raw/tests/it/dng.rs b/crates/raw/tests/it/dng.rs index 216937862..cc0747830 100644 --- a/crates/raw/tests/it/dng.rs +++ b/crates/raw/tests/it/dng.rs @@ -1,7 +1,7 @@ //! Synthetic DNG round trips: every storage variant we decode is written with our own TIFF writer (and LJ92 //! encoder) and must decode bit-exactly. -use craft_raw::opcodes::{Area, Opcode}; +use craft_raw::opcodes::{self, Area, Opcode}; use craft_raw::{ BlackLevel, Cfa, ColorData, DngCompression, DngWriteOptions, OpcodeLists, Orientation, RawData, RawFormat, RawImage, Rect, decode, embedded_preview, probe, write_dng, @@ -58,6 +58,7 @@ fn synthetic(w: usize, h: usize, cfa: Option, cpp: usize) -> RawImage { color, wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera, @@ -403,6 +404,172 @@ fn floating_point_dngs() { assert_eq!(decode(&dng(raw, ByteOrder::Big)).unwrap().data, RawData::F32(vals)); } +/// The chunks (tiles of `tw` x `th`, or strips when `tw == w`) of an image of `cpp` interleaved samples per pixel, each +/// nominal-sized (strips: clipped to the image height) with zero padding past the image edge: one chunk per tile, and for `planar` one set of chunks per +/// plane (one sample per pixel), plane after plane. +fn chunk_samples(px: &[T], w: usize, h: usize, cpp: usize, tw: usize, th: usize, planar: bool) -> Vec> { + let planes = if planar { cpp } else { 1 }; + let ccpp = if planar { 1 } else { cpp }; + let mut out = Vec::new(); + for plane in 0..planes { + for ty in 0..h.div_ceil(th) { + for tx in 0..w.div_ceil(tw) { + // a strip holds only the rows inside the image, a tile its nominal size + let rows = if tw == w { th.min(h - ty * th) } else { th }; + let mut c = vec![T::default(); tw * rows * ccpp]; + for y in 0..th.min(h - ty * th) { + for x in 0..tw.min(w - tx * tw) { + for s in 0..ccpp { + let src = ((ty * th + y) * w + tx * tw + x) * cpp + if planar { plane } else { s }; + c[(y * tw + x) * ccpp + s] = px[src]; + } + } + } + out.push(c); + } + } + } + out +} + +fn layout_image(raw: &mut IfdBuilder, chunks: Vec>, w: usize, tile: (usize, usize)) { + raw.set_image(if tile.0 == w { + ImageData::Strips { rows_per_strip: tile.1 as u32, strips: chunks } + } else { + ImageData::Tiles { tile_width: tile.0 as u32, tile_height: tile.1 as u32, tiles: chunks } + }); +} + +/// Uncompressed integer DNGs in every chunk layout (chunky and planar, tiles and strips with edge chunks, several chunks +/// per image) at several bit depths and byte orders decode to the samples they were built from. +#[test] +fn uncompressed_chunk_layouts_place_every_sample() { + let (w, h) = (37, 23); + for cpp in [1usize, 3] { + for bits in [8u32, 12, 16] { + let px = pattern(w, h, cpp, bits); + // tiles overhanging both edges, strips with a shorter last strip + for tile in [(16, 16), (w, 5), (w, h), (16, 32)] { + for planar in [false, true] { + if planar && cpp == 1 { + continue; + } + for order in [ByteOrder::Little, ByteOrder::Big] { + let ccpp = if planar { 1 } else { cpp }; + let chunks: Vec> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar) + .iter() + .map(|c| match bits { + 8 => c.iter().map(|&v| v as u8).collect(), + 16 => u16_bytes(c, order), + _ => pack_msb(c, bits, tile.0 * ccpp), + }) + .collect(); + let mut raw = base_ifd(w, h, bits as u16, cpp, cpp == 1); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + if planar { + raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2])); + } + layout_image(&mut raw, chunks, w, tile); + let img = decode(&dng(raw, order)).unwrap(); + assert_eq!(img.cpp, cpp); + assert_eq!(img.data, RawData::U16(px.clone()), "cpp {cpp} bits {bits} tile {tile:?} planar {planar} {order:?}"); + } + } + } + } + } +} + +/// Deflate with the integer and floating-point predictors, chunky and planar tiles and strips. +#[test] +fn deflate_chunk_layouts_place_every_sample() { + let (w, h) = (37, 23); + for planar in [false, true] { + let cpp = 3; + let ccpp = if planar { 1 } else { cpp }; + let px = pattern(w, h, cpp, 16); + for tile in [(16, 16), (w, 5)] { + // integer predictor 2 on 16-bit samples + let chunks: Vec> = chunk_samples(&px, w, h, cpp, tile.0, tile.1, planar) + .into_iter() + .map(|mut c| { + for row in c.chunks_mut(tile.0 * ccpp) { + for i in (ccpp..row.len()).rev() { + row[i] = row[i].wrapping_sub(row[i - ccpp]); + } + } + miniz_oxide::deflate::compress_to_vec_zlib(&u16_bytes(&c, ByteOrder::Little), 6) + }) + .collect(); + let mut raw = base_ifd(w, h, 16, cpp, false); + raw.set(t::COMPRESSION, Value::Short(vec![compression::ADOBE_DEFLATE])); + raw.set(t::PREDICTOR, Value::Short(vec![2])); + if planar { + raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2])); + } + layout_image(&mut raw, chunks, w, tile); + let img = decode(&dng(raw, ByteOrder::Little)).unwrap(); + assert_eq!(img.data, RawData::U16(px.clone()), "integer predictor, tile {tile:?} planar {planar}"); + } + } +} + +/// Floating-point DNGs (uncompressed and with the floating-point predictor) in tiled and strip, chunky and planar +/// layouts with several chunks and edge chunks. +#[test] +fn float_chunk_layouts_place_every_sample() { + let (w, h) = (37, 23); + for cpp in [1usize, 3] { + let vals: Vec = (0..w * h * cpp).map(|i| ((i * 37) % 200) as f32 / 64.0).collect(); + for tile in [(16, 16), (w, 5)] { + for planar in [false, true] { + if planar && cpp == 1 { + continue; + } + let ccpp = if planar { 1 } else { cpp }; + let parts = chunk_samples(&vals, w, h, cpp, tile.0, tile.1, planar); + for bits in [16u16, 24, 32] { + let bp = bits as usize / 8; + for (pred, order) in [(1u16, ByteOrder::Little), (1, ByteOrder::Big), (3, ByteOrder::Little)] { + if pred == 1 && bits != 16 && bits != 32 { + continue; // uncompressed 24-bit floats are not written by this test + } + let chunks: Vec> = parts + .iter() + .map(|c| { + if pred == 3 { + let enc = fp_encode(c, bp, ccpp, tile.0 * ccpp); + miniz_oxide::deflate::compress_to_vec_zlib(&enc, 6) + } else if bits == 16 { + c.iter() + .flat_map( + |&v| if order == ByteOrder::Big { f32_to_f16(v).to_be_bytes() } else { f32_to_f16(v).to_le_bytes() }, + ) + .collect() + } else { + c.iter().flat_map(|&v| if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }).collect() + } + }) + .collect(); + let mut raw = base_ifd(w, h, bits, cpp, cpp == 1); + raw.set(t::SAMPLE_FORMAT, Value::Short(vec![3; cpp])); + raw.set(t::COMPRESSION, Value::Short(vec![if pred == 3 { compression::DEFLATE } else { 1 }])); + if pred == 3 { + raw.set(t::PREDICTOR, Value::Short(vec![3])); + } + if planar { + raw.set(t::PLANAR_CONFIGURATION, Value::Short(vec![2])); + } + layout_image(&mut raw, chunks, w, tile); + let img = decode(&dng(raw, order)).unwrap(); + assert_eq!(img.data, RawData::F32(vals.clone()), "cpp {cpp} {bits}-bit pred {pred} tile {tile:?} planar {planar} {order:?}"); + } + } + } + } + } +} + #[test] fn non_finite_float_samples_decode_as_levels() { // NaN samples in a float CFA DNG spread through any arithmetic downstream: the decoder hands on numbers only @@ -1177,12 +1344,77 @@ fn white_level_defaults_to_the_linearization_tables_maximum() { }; let img = build(None); assert!(img.linearized); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); assert_eq!(img.data, RawData::U16(codes.iter().map(|&c| table[c as usize]).collect())); assert_eq!(img.white, vec![51000.0], "255 × 200, the table's top"); // a stated level still wins assert_eq!(build(Some(40000)).white, vec![40000.0]); } +/// DNG chapter 7: OpcodeList1 runs on the stored codes, then the LinearizationTable maps them to +/// sensor values. When list 1 is present, decode leaves the table unapplied so the develop pipeline +/// can keep that order. Applying the table first would give a different picture. +#[test] +fn opcode_list1_runs_on_stored_samples_before_linearization() { + let (w, h) = (4, 2); + let stored = vec![2u16; w * h]; + // table: 2 → 20, 7 → 30, 20 → 99 (so the two orders disagree) + let mut table = vec![0u16; 32]; + table[2] = 20; + table[7] = 30; + table[20] = 99; + // MapTable: 2 → 7, 20 → 8 + let mut map: Vec = (0..32).collect(); + map[2] = 7; + map[20] = 8; + let area = Area { top: 0, left: 0, bottom: h as u32, right: w as u32, plane: 0, planes: 1, row_pitch: 1, col_pitch: 1 }; + let list1 = opcodes::write_list(&[Opcode::MapTable { area, table: map.clone() }]); + + let mut raw = base_ifd(w, h, 16, 1, true); + raw.set(t::COMPRESSION, Value::Short(vec![compression::NONE])); + raw.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![u16_bytes(&stored, ByteOrder::Little)] }); + raw.set(t::LINEARIZATION_TABLE, Value::Short(table.clone())); + raw.set(t::OPCODE_LIST_1, Value::Undefined(list1)); + raw.set(t::WHITE_LEVEL, Value::Long(vec![100])); + let img = decode(&dng(raw, ByteOrder::Little)).unwrap(); + assert!(!img.linearized, "list 1 is present, so the table must wait"); + assert_eq!(img.data, RawData::U16(stored.clone())); + assert_eq!(img.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(img.opcodes.list1.len(), 1); + + let apply_map = |samples: &mut [u16]| { + for s in samples { + let i = (*s as usize).min(map.len() - 1); + *s = map[i]; + } + }; + + let mut correct = img.clone(); + if let RawData::U16(v) = &mut correct.data { + apply_map(v); + } + correct.apply_linearization(); + assert!(correct.linearized); + // 2 → 7 (MapTable) → 30 (linearization) + assert_eq!(correct.data, RawData::U16(vec![30; w * h])); + + let mut wrong = img.clone(); + wrong.apply_linearization(); + if let RawData::U16(v) = &mut wrong.data { + apply_map(v); + } + // 2 → 20 (linearization) → 8 (MapTable) + assert_eq!(wrong.data, RawData::U16(vec![8; w * h])); + assert_ne!(correct.data, wrong.data); + + // the unapplied table and list 1 survive a write / decode round trip + let back = decode(&write_dng(&img, &DngWriteOptions { compression: DngCompression::Uncompressed, ..Default::default() }).unwrap()).unwrap(); + assert!(!back.linearized); + assert_eq!(back.data, RawData::U16(stored)); + assert_eq!(back.linearization_table.as_deref(), Some(table.as_slice())); + assert_eq!(back.opcodes.list1.len(), 1); +} + /// PhotoCraft: `AsShotWhiteXY` is a chromaticity, so both coordinates lie inside (0, 1). #[test] fn an_as_shot_white_outside_the_chromaticity_range_is_ignored() { diff --git a/crates/raw/tests/it/robust.rs b/crates/raw/tests/it/robust.rs index 5ad1d1b32..28640a803 100644 --- a/crates/raw/tests/it/robust.rs +++ b/crates/raw/tests/it/robust.rs @@ -22,6 +22,7 @@ fn sample(comp: DngCompression) -> Vec { color: ColorData::default(), wb_multipliers: None, linearized: false, + linearization_table: None, pixel_aspect: 1.0, opcodes: OpcodeLists::default(), camera: Default::default(), diff --git a/crates/rtf/src/doc/export.rs b/crates/rtf/src/doc/export.rs index daa3afbda..0cd34498c 100644 --- a/crates/rtf/src/doc/export.rs +++ b/crates/rtf/src/doc/export.rs @@ -659,6 +659,10 @@ impl Writer { Some(Align::Distribute) => out.push_str("\\qd"), _ => out.push_str("\\ql"), } + // The paragraph's text direction, as Word writes it (`\rtlpar`; `\ltrpar` is the default). + if p.rtl { + out.push_str("\\rtlpar"); + } if p.page_break { out.push_str("\\pagebb"); } diff --git a/crates/rtf/src/doc/state.rs b/crates/rtf/src/doc/state.rs index 486b94a32..c27758d05 100644 --- a/crates/rtf/src/doc/state.rs +++ b/crates/rtf/src/doc/state.rs @@ -148,6 +148,8 @@ pub(super) struct St { pub(super) hidden: bool, // Paragraph properties. pub(super) align: Option, + /// `\rtlpar`: the paragraph reads right to left. + pub(super) rtldir: Option, pub(super) style: i32, pub(super) intbl: bool, pub(super) ls: Option, @@ -185,6 +187,7 @@ impl Default for St { uc: 1, hidden: false, align: None, + rtldir: None, style: 0, intbl: false, ls: None, diff --git a/crates/rtf/src/doc/styles.rs b/crates/rtf/src/doc/styles.rs index 014a105b4..6047bf3c4 100644 --- a/crates/rtf/src/doc/styles.rs +++ b/crates/rtf/src/doc/styles.rs @@ -30,6 +30,7 @@ impl Reader { out.italic = out.italic.or(d.italic); out.size = out.size.or(d.size); out.font = out.font.or(d.font); + out.rtl = out.rtl.or(d.rtl); cur = d.based_on; } out @@ -87,6 +88,8 @@ impl Reader { pub(super) fn para_props(&self, st: &St) -> Para { let mut p = Para::new(self.para_kind(st)); p.align = st.align; + // Direct `\rtlpar`/`\ltrpar` wins over the style's direction. + p.rtl = st.rtldir.or_else(|| self.file_style(st.style).rtl).unwrap_or(false); p.page_break = st.pagebb; p.spacing = spacing(st); p.borders = st.borders.any_visible().then_some(st.borders); @@ -135,6 +138,8 @@ pub(super) struct StyleDef { pub(super) size: Option, /// A `\fonttbl` index, named when it's used (the table may follow). pub(super) font: Option, + /// `\rtlpar` (`Some(true)`) or `\ltrpar`: the paragraph direction the style sets. + pub(super) rtl: Option, } /// Clear from `f` the formatting a paragraph of kind `kind` gets from its style, whose @@ -203,6 +208,10 @@ pub(super) fn is_format_word(w: &str) -> bool { | "s" | "outlinelevel" | "pagebb" + | "rtlpar" + | "ltrpar" + | "rtldir" + | "ltrdir" | "intbl" | "ls" | "ilvl" diff --git a/crates/rtf/src/doc/tests.rs b/crates/rtf/src/doc/tests.rs index 003b32872..2a4103eb1 100644 --- a/crates/rtf/src/doc/tests.rs +++ b/crates/rtf/src/doc/tests.rs @@ -38,6 +38,86 @@ fn basic_reader() { assert!(matches!(&l.inlines[0], Inline::Text(_, f) if f.link.as_deref() == Some("http://a.b"))); } +#[test] +fn rtl_paragraphs_read_and_write_their_direction() { + // Word writes an RTL paragraph as `\rtlpar` (`\ltrpar` for LTR); reading it keeps the + // direction as w:bidi on the document paragraph, and export writes it back. + let rtf = br#"{\rtf1\ansi\pard\rtlpar RTL\par\pard\ltrpar LTR\par\pard plain\par}"#; + let flow = parse(rtf).unwrap(); + let paras: Vec<&Para> = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect(); + assert!(paras[0].rtl, "rtlpar sets the direction"); + assert!(!paras[1].rtl && !paras[2].rtl, "ltrpar and no word: not RTL"); + // \rtldir and \ltrdir, which some writers used for paragraphs, are read too. + let old = parse(br#"{\rtf1\ansi\pard\rtldir RTL\par\pard\ltrdir LTR\par}"#).unwrap(); + let old: Vec<&Para> = old.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect(); + assert!(old[0].rtl && !old[1].rtl, "rtldir and ltrdir are read"); + // \pard resets the direction. + let reset = parse(br#"{\rtf1\ansi\pard\rtlpar A\par\pard B\par}"#).unwrap(); + let reset: Vec<&Para> = reset.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p) } else { None }).collect(); + assert!(reset[0].rtl && !reset[1].rtl, "pard resets the direction"); + // Import keeps it as bidi on the document paragraph. + let bidi = |d: &Document| { + d.body.iter().filter_map(|b| if let Block::Para(p) = &**b { Some(p.props.bidi.unwrap_or(false)) } else { None }).collect::>() + }; + let doc = import(rtf).unwrap(); + assert_eq!(bidi(&doc), [true, false, false]); + // Export writes \rtlpar once, and not the \rtldir. + let out = export(&doc); + assert_eq!(out.matches("\\rtlpar").count(), 1, "{out}"); + assert!(!out.contains("rtldir"), "{out}"); + assert_eq!(bidi(&import(out.as_bytes()).unwrap()), [true, false, false]); +} + +#[test] +fn rtl_paragraph_styles_carry_their_direction() { + // Word often writes the direction in the style: `{\s1\rtlpar RTL;}` then `\pard\s1`. + let rtf = br#"{\rtf1\ansi{\stylesheet{\s0 Normal;}{\s1\rtlpar RTL;}{\s2\sbasedon1 Child;}{\s3\sbasedon1\ltrpar Back;}} +\pard\s1 A\par\pard\s2 B\par\pard\s3 C\par\pard\s1\ltrpar D\par\pard\s0\rtlpar E\par\pard F\par}"#; + let flow = parse(rtf).unwrap(); + let rtl: Vec = flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some(p.rtl) } else { None }).collect(); + // A: style. B: inherited from the base style. C: the style overrides its base. D: direct + // \ltrpar beats the style. E: direct \rtlpar on Normal. F: \pard drops direct formatting. + assert_eq!(rtl, [true, true, false, false, true, false]); +} + +#[test] +fn oldpprops_keep_the_old_paragraph_direction() { + // A tracked change of direction: now LTR, before RTL (`\rtlpar` in the old properties). + let rtf = br"{\rtf1\ansi{\*\revtbl {Unknown;}{Alice;}} +\pard\ltrpar\prauth1\prdate-2014675576{\*\oldpprops \pard \rtlpar}A\par}"; + let flow = parse(rtf).unwrap(); + let Some(FBlock::Para(p)) = flow.blocks.first() else { panic!("no paragraph") }; + assert!(!p.rtl); + assert!(p.fmt_change.as_deref().expect("paragraph change").old.rtl, "old direction is RTL"); + let doc = import(rtf).unwrap(); + let c = doc_paras(&doc)[0].props.fmt_change.as_deref().expect("change"); + assert_eq!(c.old.bidi, Some(true)); +} + +#[test] +fn tracked_rtl_to_ltr_change_from_a_document_keeps_the_old_direction() { + use craft_wordmodel::props::{CharProps, ParaProps, PropChange}; + use craft_wordmodel::{Revision, RevisionKind}; + // Document origin (not RTF): now LTR, before RTL. + let mut p = craft_wordmodel::Paragraph::with_text("A", CharProps::default()); + p.props.bidi = Some(false); + p.props.fmt_change = PropChange::boxed(0, ParaProps { bidi: Some(true), ..Default::default() }); + let doc = doc_with(vec![p], vec![Revision::new(RevisionKind::Format, "Bob", "2025-01-02T08:05:00Z")]); + let flow = model::from_doc_tracked(&doc); + let Some(FBlock::Para(fp)) = flow.blocks.first() else { panic!("no paragraph") }; + assert!(!fp.rtl); + assert!(fp.fmt_change.as_deref().expect("change").old.rtl, "snapshot keeps RTL"); + let rtf = export(&doc); + assert!(rtf.contains("{\\*\\oldpprops"), "{rtf}"); + let back = import(rtf.as_bytes()).unwrap(); + let p = doc_paras(&back)[0]; + assert_ne!(p.props.bidi, Some(true), "current direction is LTR"); + let change = p.props.fmt_change.as_deref().expect("change read back"); + assert_eq!(change.old.bidi, Some(true)); + // Reject restores RTL. + assert_eq!(change.old.formatting().bidi, Some(true)); +} + #[test] fn stylesheet_names_take_unicode_escapes() { // \u116?=t \u105?=i \u116?=t \u108?=l \u101?=e — how Word writes a non-Latin style name. @@ -1863,3 +1943,46 @@ fn compound_borders_round_trip() { let back = import(export(&d).as_bytes()).unwrap(); assert_eq!(tops(&back), want); } + +fn para_aligns(flow: &Flow) -> Vec<(bool, Option)> { + flow.blocks.iter().filter_map(|b| if let FBlock::Para(p) = b { Some((p.rtl, p.align)) } else { None }).collect() +} + +#[test] +fn alignment_words_after_rtlpar_mean_start_and_end() { + // Word reads `\ql`/`\qr` that follow `\rtlpar` as start/end (the model's alignment is + // logical), so `\rtlpar\qr` sits at the right edge and `\rtlpar\ql` at the left edge. + // Words written before `\rtlpar` keep their plain meaning (checked in Word by saving to DOCX: + // `\qr\rtlpar` gives jc=right, i.e. end, in the bidi paragraph; `\ql\rtlpar` gives no jc). + let rtf = br#"{\rtf1\ansi\deff0{\fonttbl{\f0 Arial;}}\pard LTR one.\par \pard\rtlpar\qr A.\par \pard\rtlpar B.\par \pard\ltrpar Back.\par \pard\rtlpar\ql C.\par \pard\rtlpar\qc D.\par \pard\ql\rtlpar E.\par \pard\qr\rtlpar F.\par \pard\rtlpar\ltrpar\qr G.\par}"#; + let got = para_aligns(&parse(rtf).unwrap()); + assert_eq!( + got, + [ + (false, None), + (true, Some(Align::Left)), + (true, None), + (false, None), + (true, Some(Align::Right)), + (true, Some(Align::Center)), + (true, Some(Align::Left)), + (true, Some(Align::Right)), + (false, Some(Align::Right)), + ] + ); +} + +#[test] +fn alignment_survives_an_export_import_round_trip() { + let rtf = br#"{\rtf1\ansi\pard\ql L.\par \pard\qr R.\par \pard\qc C.\par \pard\rtlpar\qr start.\par \pard\rtlpar\ql end.\par \pard\rtlpar\qc mid.\par \pard\rtlpar plain.\par}"#; + let first = para_aligns(&parse(rtf).unwrap()); + let doc = import(rtf).unwrap(); + let out = export(&doc); + let second = para_aligns(&parse(out.as_bytes()).unwrap()); + let norm = |v: Vec<(bool, Option)>| v.into_iter().map(|(r, a)| (r, a.filter(|a| *a != Align::Left))).collect::>(); + assert_eq!(norm(first), norm(second.clone()), "{out}"); + // `\rtlpar\qr` is start (no explicit alignment in the model) and `\rtlpar\ql` is end. + assert_eq!(norm(second)[3..5], [(true, None), (true, Some(Align::Right))], "{out}"); + // The export writes the alignment before the direction, which Word reads as logical start/end. + assert!(out.contains("\\qr\\rtlpar"), "{out}"); +} diff --git a/crates/rtf/src/doc/word.rs b/crates/rtf/src/doc/word.rs index 28360c616..2eb86692f 100644 --- a/crates/rtf/src/doc/word.rs +++ b/crates/rtf/src/doc/word.rs @@ -3,6 +3,18 @@ use super::styles::{StyleDef, is_char_format_word, is_format_word}; use super::{Align, Fmt, Reader, VAlign, model}; impl Reader { + /// The logical alignment for a side word (`Left` for `\ql`, `Right` for `\qr`). The model's + /// alignment is logical (start/end), and Word reads a side word that comes after `\rtlpar` as + /// start/end: the sides swap in a paragraph already marked right to left. Before `\rtlpar` + /// the side keeps its plain meaning, which is why the writer puts the alignment first. + fn side_align(&self, side: Align) -> Align { + match (self.st.rtldir, side) { + (Some(true), Align::Left) => Align::Right, + (Some(true), Align::Right) => Align::Left, + _ => side, + } + } + pub(super) fn word(&mut self, w: &str, p: Option, starred: bool, first_in_group: bool) { // Every control word ends a multibyte run: the code page it is read in may change here // (`\f`, `\fcharset`, `\cpg`, `\ansicpg`, `\pard`), and `\par` must see the text before it. @@ -166,11 +178,15 @@ impl Reader { "cf" => self.st.fmt.color = self.colors.get(pv.max(0) as usize).copied().flatten(), "highlight" | "chcbpat" | "cb" => self.st.fmt.background = self.colors.get(pv.max(0) as usize).copied().flatten(), "v" => self.st.hidden = on, - "ql" => self.st.align = Some(Align::Left), + "ql" => self.st.align = Some(self.side_align(Align::Left)), "qc" => self.st.align = Some(Align::Center), - "qr" => self.st.align = Some(Align::Right), + "qr" => self.st.align = Some(self.side_align(Align::Right)), "qj" => self.st.align = Some(Align::Justify), "qd" => self.st.align = Some(Align::Distribute), + // `\rtldir`/`\ltrdir` are not paragraph words (they set a table's or section's + // direction in RTF); some writers used them for paragraphs, so they are read too. + "rtlpar" | "rtldir" => self.st.rtldir = Some(true), + "ltrpar" | "ltrdir" => self.st.rtldir = Some(false), "s" => self.st.style = pv, "outlinelevel" => self.st.outline = Some(pv.clamp(0, 9) as u8), "pagebb" => self.st.pagebb = on, @@ -417,6 +433,8 @@ impl Reader { "i" => def.italic = Some(on), "fs" => def.size = (value > 0).then(|| (value as f32 / 2.0).clamp(1.0, 1638.0)), "f" => def.font = Some(value), + "rtlpar" | "rtldir" => def.rtl = Some(true), + "ltrpar" | "ltrdir" => def.rtl = Some(false), _ => {} } true diff --git a/crates/texture/Cargo.toml b/crates/texture/Cargo.toml index 78868dd87..173abb0b0 100644 --- a/crates/texture/Cargo.toml +++ b/crates/texture/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true [features] default = [] +dds = ["dep:dds", "dep:bytemuck"] parallel = ["dep:rayon"] [lints] @@ -17,3 +18,11 @@ workspace = true [dependencies] rayon = { workspace = true, optional = true } thiserror = { workspace = true } + +# Same decoder used by MaterialCraft; no parallel backend enabled. +dds = { version = "0.2.0", default-features = false, optional = true } +bytemuck = { workspace = true, optional = true } + +[dev-dependencies] +dds = { version = "0.2.0", default-features = false } +bytemuck = { workspace = true } diff --git a/crates/texture/src/dds.rs b/crates/texture/src/dds.rs new file mode 100644 index 000000000..6971ad461 --- /dev/null +++ b/crates/texture/src/dds.rs @@ -0,0 +1,116 @@ +//! First-surface DDS decoding to unmodified RGBA floats. +//! +//! Adapted from MaterialCraft `dds_image.rs` (originally chasse20, cc052aeaa8c61fbe6975674b6d5e28bb3c597b9d). +//! Uses the same `dds` backend and first-surface semantics, including the first cube face or +//! volume slice. This is not a whole-resource decoder. Transfer functions and image policy +//! belong to callers. [`Limits`] bounds the output float plane, not backend scratch buffers. + +use crate::{ImageError, Kind, Limits}; + +/// A malformed DDS/backend failure or rejected output allocation. +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + /// Output size or allocation failed shared image limits. + #[error(transparent)] + Image(#[from] ImageError), + /// The decoder rejected input or output geometry. + #[error("DDS decode failed: {0}")] + Decode(String), +} + +/// Unmodified backend output. Alpha and out-of-range float values are not clamped. +#[derive(Debug)] +pub struct Surface { + /// Width of the first surface in pixels. + pub width: u32, + /// Height of the first surface in pixels. + pub height: u32, + /// Row-major RGBA float samples without transfer conversion. + pub rgba: Vec, + /// Exactly the backend's `Precision::F32` classification, as used by MaterialCraft. + pub source_precision_is_f32: bool, +} + +/// Decode the first surface after validating dimensions, pixels and output-byte limits. +/// The output reservation is fallible; no second full-sized float plane is created. +pub fn decode_first(bytes: &[u8], limits: &Limits) -> Result { + let error = |e: ::dds::DecodingError| DecodeError::Decode(e.to_string()); + let mut decoder = ::dds::Decoder::new(std::io::Cursor::new(bytes)).map_err(error)?; + let size = decoder.main_size(); + let len = limits.check(size.width, size.height, Kind::Color)?; + let mut rgba = Vec::new(); + rgba.try_reserve_exact(len).map_err(|_| ImageError::OutOfMemory(size.width, size.height))?; + rgba.resize(len, 0.0_f32); + let view = ::dds::ImageViewMut::new(bytemuck::cast_slice_mut(&mut rgba), size, ::dds::ColorFormat::RGBA_F32) + .ok_or_else(|| DecodeError::Decode("invalid DDS output dimensions".into()))?; + decoder.read_surface(view).map_err(error)?; + Ok(Surface { width: size.width, height: size.height, rgba, source_precision_is_f32: decoder.format().precision() == ::dds::Precision::F32 }) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn bytes(format: ::dds::Format, payload: &[u8]) -> Vec { + let mut bytes = Vec::new(); + ::dds::header::Header::new_image(1, 1, format).write(&mut bytes).unwrap(); + bytes.extend_from_slice(payload); + bytes + } + + #[test] + fn rgba8_keeps_transfer_and_alpha_unmodified() { + let data = bytes(::dds::Format::R8G8B8A8_UNORM, &[128, 64, 255, 32]); + let image = decode_first(&data, &Limits::default()).unwrap(); + assert_eq!((image.width, image.height), (1, 1)); + assert!(!image.source_precision_is_f32); + for (actual, expected) in image.rgba.iter().zip([128.0 / 255.0, 64.0 / 255.0, 1.0, 32.0 / 255.0]) { + assert!((actual - expected).abs() < 1e-6); + } + } + + #[test] + fn float_hdr_and_negative_values_survive() { + let values = [-0.5_f32, 2.0, 0.25, 0.75]; + let payload: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + let image = decode_first(&bytes(::dds::Format::R32G32B32A32_FLOAT, &payload), &Limits::default()).unwrap(); + assert!(image.source_precision_is_f32); + assert_eq!(image.rgba, values); + } + + #[test] + fn half_float_and_compressed_pixels_preserve_backend_values() { + let half: Vec = [0xb800u16, 0x4000, 0x3400, 0x3a00].iter().flat_map(|v| v.to_le_bytes()).collect(); + let image = decode_first(&bytes(::dds::Format::R16G16B16A16_FLOAT, &half), &Limits::default()).unwrap(); + assert_eq!(image.rgba, [-0.5, 2.0, 0.25, 0.75]); + assert_eq!(image.source_precision_is_f32, ::dds::Format::R16G16B16A16_FLOAT.precision() == ::dds::Precision::F32); + let mut compressed = Vec::new(); + ::dds::header::Header::new_image(4, 4, ::dds::Format::BC1_UNORM).write(&mut compressed).unwrap(); + compressed.extend_from_slice(&[0, 0xf8, 0, 0, 0, 0, 0, 0]); + let image = decode_first(&compressed, &Limits::default()).unwrap(); + assert!(!image.source_precision_is_f32); + assert!(image.rgba.as_chunks::<4>().0.iter().all(|p| *p == [1.0, 0.0, 0.0, 1.0])); + } + + #[test] + fn cube_keeps_first_surface_only_like_materialcraft() { + let mut data = Vec::new(); + ::dds::header::Header::new_cube_map(1, 1, ::dds::Format::R8G8B8A8_UNORM).write(&mut data).unwrap(); + data.extend_from_slice(&[255, 0, 0, 255]); + data.extend_from_slice(&[0; 20]); + let image = decode_first(&data, &Limits::default()).unwrap(); + assert_eq!(image.rgba, [1.0, 0.0, 0.0, 1.0]); + } + + #[test] + fn checks_output_budget_and_truncation() { + let data = bytes(::dds::Format::R8G8B8A8_UNORM, &[1, 2, 3, 4]); + let mut limits = Limits { max_bytes: 15, ..Limits::default() }; + assert!(matches!(decode_first(&data, &limits), Err(DecodeError::Image(ImageError::TooLarge(1, 1))))); + limits.max_bytes = 16; + assert!(decode_first(&data, &limits).is_ok()); + assert!(decode_first(&data[..data.len() - 1], &limits).is_err()); + assert!(decode_first(&[], &limits).is_err()); + } +} diff --git a/crates/texture/src/lib.rs b/crates/texture/src/lib.rs index 2aaef3e8b..e33b68d88 100644 --- a/crates/texture/src/lib.rs +++ b/crates/texture/src/lib.rs @@ -29,6 +29,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(any(feature = "dds", test))] +pub mod dds; + use std::sync::Arc; #[cfg(feature = "parallel")] diff --git a/crates/tiff/src/classic.rs b/crates/tiff/src/classic.rs new file mode 100644 index 000000000..c1ab6f64a --- /dev/null +++ b/crates/tiff/src/classic.rs @@ -0,0 +1,470 @@ +//! The TIFF container (TIFF 6.0 §2, as DNG uses it): byte order, image file directories (IFDs), +//! their entries and values, and the SubIFD tree. +//! +//! Every offset and count comes from the file and is checked before use: an entry whose value lies +//! outside the file is an error when it is read, the walk visits at most [`MAX_IFDS`] directories, +//! never the same one twice (cyclic chains end), and goes at most [`MAX_DEPTH`] SubIFD levels deep. + +use std::collections::HashSet; + +/// Why a classic TIFF directory could not be read. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The input has no supported classic TIFF header. + #[error("not a classic TIFF")] + NotTiff, + /// The directory or a referenced value is invalid. + #[error("{0}")] + Malformed(&'static str), +} +/// Result of the bounded classic TIFF reader. +pub type Result = std::result::Result; + +/// Most IFDs one file may have (main chain and SubIFDs together). +pub const MAX_IFDS: usize = 64; +/// Deepest SubIFD nesting followed. +pub const MAX_DEPTH: usize = 4; +/// Invalid SubIFD edges count too: otherwise a file can make failed offset probes quadratic. +const MAX_IFD_ATTEMPTS: usize = MAX_IFDS * 16; +/// SubIFD references (zero, repeated and invalid ones included) accepted over the whole walk, +/// counted before a list is read: many directories can name the same long SubIFD table. +const MAX_SUB_IFD_REFS: usize = MAX_IFD_ATTEMPTS; + +/// TIFF field types (TIFF 6.0 §2, plus type 13 IFD from TIFF Technical Note 1). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Type { + Byte, + Ascii, + Short, + Long, + Rational, + SByte, + Undefined, + SShort, + SLong, + SRational, + Float, + Double, + Ifd, +} + +impl Type { + fn from_code(c: u16) -> Option { + Some(match c { + 1 => Type::Byte, + 2 => Type::Ascii, + 3 => Type::Short, + 4 => Type::Long, + 5 => Type::Rational, + 6 => Type::SByte, + 7 => Type::Undefined, + 8 => Type::SShort, + 9 => Type::SLong, + 10 => Type::SRational, + 11 => Type::Float, + 12 => Type::Double, + 13 => Type::Ifd, + _ => return None, + }) + } + + /// Bytes per value. + pub fn size(self) -> usize { + match self { + Type::Byte | Type::Ascii | Type::SByte | Type::Undefined => 1, + Type::Short | Type::SShort => 2, + Type::Long | Type::SLong | Type::Float | Type::Ifd => 4, + Type::Rational | Type::SRational | Type::Double => 8, + } + } +} + +/// One directory entry. +#[derive(Clone, Copy, Debug)] +pub struct Entry { + pub tag: u16, + pub ty: Type, + pub count: u32, + /// Where the value bytes start: inside the entry when they fit in four bytes, else the offset + /// the entry holds (not yet checked against the file length). + value_at: u64, +} + +/// One image file directory. +#[derive(Clone, Debug, Default)] +pub struct Ifd { + /// File offset of the directory. + pub offset: u64, + pub entries: Vec, + /// Nesting: 0 for the main chain, 1 for its SubIFDs, … + pub depth: usize, +} + +impl Ifd { + pub fn get(&self, tag: u16) -> Option<&Entry> { + self.entries.iter().find(|e| e.tag == tag) + } +} + +/// A TIFF file in memory. +#[derive(Clone, Copy)] +pub struct Tiff<'a> { + pub data: &'a [u8], + pub big_endian: bool, +} + +impl<'a> Tiff<'a> { + /// Read the header. Classic TIFF only (BigTIFF, version 43, is not used by DNG). + pub fn new(data: &'a [u8]) -> Result<(Tiff<'a>, u64)> { + let big_endian = match data.get(0..4) { + Some(b"II*\0") => false, + Some(b"MM\0*") => true, + _ => return Err(Error::NotTiff), + }; + let t = Tiff { data, big_endian }; + let first = u64::from(t.u32_at(4)?); + Ok((t, first)) + } + + fn bytes_at(&self, off: u64, len: usize) -> Result<&'a [u8]> { + let start = usize::try_from(off).map_err(|_| Error::Malformed("offset outside the file"))?; + let end = start.checked_add(len).ok_or(Error::Malformed("offset outside the file"))?; + self.data.get(start..end).ok_or(Error::Malformed("offset outside the file")) + } + + pub fn u16_of(&self, b: [u8; 2]) -> u16 { + if self.big_endian { u16::from_be_bytes(b) } else { u16::from_le_bytes(b) } + } + + pub fn u32_of(&self, b: [u8; 4]) -> u32 { + if self.big_endian { u32::from_be_bytes(b) } else { u32::from_le_bytes(b) } + } + + fn u16_at(&self, off: u64) -> Result { + let b = self.bytes_at(off, 2)?; + Ok(self.u16_of([b[0], b[1]])) + } + + fn u32_at(&self, off: u64) -> Result { + let b = self.bytes_at(off, 4)?; + Ok(self.u32_of([b[0], b[1], b[2], b[3]])) + } + + /// Read the IFD at `off`: its entries and the offset of the next IFD (0 at the end of a chain). + pub fn read_ifd(&self, off: u64, depth: usize) -> Result<(Ifd, u64)> { + let n = self.u16_at(off)?; + let mut entries = Vec::with_capacity(usize::from(n)); + for i in 0..u64::from(n) { + let at = off + 2 + i * 12; + let e = self.bytes_at(at, 12)?; + let tag = self.u16_of([e[0], e[1]]); + let code = self.u16_of([e[2], e[3]]); + let count = self.u32_of([e[4], e[5], e[6], e[7]]); + // unknown field types are skipped (TIFF 6.0: readers ignore them) + let Some(ty) = Type::from_code(code) else { continue }; + let len = u64::from(count).saturating_mul(ty.size() as u64); + let value_at = if len <= 4 { at + 8 } else { u64::from(self.u32_of([e[8], e[9], e[10], e[11]])) }; + entries.push(Entry { tag, ty, count, value_at }); + } + let next = u64::from(self.u32_at(off + 2 + u64::from(n) * 12)?); + Ok((Ifd { offset: off, entries, depth }, next)) + } + + /// Every IFD of the file: the main chain from `first` and, depth first, the SubIFDs (tag 330) of + /// each. Bounded by [`MAX_IFDS`] and [`MAX_DEPTH`]; a directory reached twice ends that branch. + pub fn all_ifds(&self, first: u64) -> Result> { + let mut out: Vec = Vec::new(); + let mut seen = HashSet::new(); + let mut attempts = 0usize; + let mut sub_refs = 0usize; + let mut stack: Vec<(u64, usize)> = vec![(first, 0)]; + while let Some((off, depth)) = stack.pop() { + let mut next = off; + while next != 0 { + if !seen.insert(next) { + break; + } + attempts += 1; + if attempts > MAX_IFD_ATTEMPTS { + return Err(Error::Malformed("too many IFD offsets")); + } + if out.len() >= MAX_IFDS { + return Err(Error::Malformed("too many IFDs")); + } + let (ifd, after) = match self.read_ifd(next, depth) { + Ok(x) => x, + // a broken directory ends the chain; the first one must be readable + Err(e) if out.is_empty() => return Err(e), + Err(_) => break, + }; + if depth < MAX_DEPTH + && let Some(sub) = ifd.get(tags::SUB_IFDS) + { + sub_refs = sub_refs.saturating_add(usize::try_from(sub.count).unwrap_or(usize::MAX)); + if sub_refs > MAX_SUB_IFD_REFS { + return Err(Error::Malformed("too many SubIFD references")); + } + if let Ok(offs) = self.uints(sub) { + for o in offs.into_iter().rev().filter(|o| *o != 0 && !seen.contains(o)) { + stack.push((o, depth + 1)); + } + } + } + out.push(ifd); + next = after; + } + } + Ok(out) + } + + /// The value bytes of an entry. + pub fn value_bytes(&self, e: &Entry) -> Result<&'a [u8]> { + let len = (e.count as usize).checked_mul(e.ty.size()).ok_or(Error::Malformed("entry too large"))?; + self.bytes_at(e.value_at, len) + } + + /// The `i`-th value of a numeric entry as f64 (rationals divided; a zero denominator is an + /// error). + fn number(&self, e: &Entry, b: &[u8], i: usize) -> Result { + let s = e.ty.size(); + let v = b.get(i * s..i * s + s).ok_or(Error::Malformed("entry too short"))?; + let u16v = |o: usize| self.u16_of([v[o], v[o + 1]]); + let u32v = |o: usize| self.u32_of([v[o], v[o + 1], v[o + 2], v[o + 3]]); + Ok(match e.ty { + Type::Byte | Type::Undefined => f64::from(v[0]), + Type::SByte => f64::from(v[0] as i8), + Type::Short => f64::from(u16v(0)), + Type::SShort => f64::from(u16v(0) as i16), + Type::Long | Type::Ifd => f64::from(u32v(0)), + Type::SLong => f64::from(u32v(0) as i32), + Type::Rational => { + let d = u32v(4); + if d == 0 { + return Err(Error::Malformed("rational with a zero denominator")); + } + f64::from(u32v(0)) / f64::from(d) + } + Type::SRational => { + let d = u32v(4) as i32; + if d == 0 { + return Err(Error::Malformed("rational with a zero denominator")); + } + f64::from(u32v(0) as i32) / f64::from(d) + } + Type::Float => f64::from(f32::from_bits(u32v(0))), + Type::Double => { + let mut a = [0u8; 8]; + a.copy_from_slice(&v[..8]); + if self.big_endian { f64::from_be_bytes(a) } else { f64::from_le_bytes(a) } + } + Type::Ascii => return Err(Error::Malformed("text where a number belongs")), + }) + } + + /// All values of a numeric entry as f64. Non-finite values are an error. + pub fn floats(&self, e: &Entry) -> Result> { + let b = self.value_bytes(e)?; + let mut out = Vec::with_capacity(e.count as usize); + for i in 0..e.count as usize { + let v = self.number(e, b, i)?; + if !v.is_finite() { + return Err(Error::Malformed("non-finite number")); + } + out.push(v); + } + Ok(out) + } + + /// All values of an unsigned integer entry (BYTE, SHORT, LONG, IFD). + pub fn uints(&self, e: &Entry) -> Result> { + if !matches!(e.ty, Type::Byte | Type::Undefined | Type::Short | Type::Long | Type::Ifd) { + return Err(Error::Malformed("expected an unsigned integer field")); + } + let b = self.value_bytes(e)?; + let s = e.ty.size(); + Ok(b.chunks_exact(s) + .map(|v| match s { + 1 => u64::from(v[0]), + 2 => u64::from(self.u16_of([v[0], v[1]])), + _ => u64::from(self.u32_of([v[0], v[1], v[2], v[3]])), + }) + .collect()) + } + + /// The first value of an unsigned integer entry. + pub fn uint(&self, e: &Entry) -> Result { + self.uints(e)?.first().copied().ok_or(Error::Malformed("empty field")) + } + + /// An ASCII entry up to its first NUL, lossily decoded. + pub fn ascii(&self, e: &Entry) -> Result { + let b = self.value_bytes(e)?; + let end = b.iter().position(|&c| c == 0).unwrap_or(b.len()); + Ok(String::from_utf8_lossy(&b[..end]).trim().to_string()) + } +} + +/// Tag numbers (TIFF 6.0, TIFF/EP, DNG 1.7, CinemaDNG). +pub mod tags { + pub const NEW_SUBFILE_TYPE: u16 = 254; + pub const IMAGE_WIDTH: u16 = 256; + pub const IMAGE_LENGTH: u16 = 257; + pub const BITS_PER_SAMPLE: u16 = 258; + pub const COMPRESSION: u16 = 259; + pub const PHOTOMETRIC: u16 = 262; + pub const MAKE: u16 = 271; + pub const MODEL: u16 = 272; + pub const STRIP_OFFSETS: u16 = 273; + pub const SAMPLES_PER_PIXEL: u16 = 277; + pub const ROWS_PER_STRIP: u16 = 278; + pub const STRIP_BYTE_COUNTS: u16 = 279; + pub const PLANAR_CONFIGURATION: u16 = 284; + pub const PREDICTOR: u16 = 317; + pub const TILE_WIDTH: u16 = 322; + pub const TILE_LENGTH: u16 = 323; + pub const TILE_OFFSETS: u16 = 324; + pub const TILE_BYTE_COUNTS: u16 = 325; + pub const SUB_IFDS: u16 = 330; + pub const SAMPLE_FORMAT: u16 = 339; + pub const CFA_REPEAT_PATTERN_DIM: u16 = 33421; + pub const CFA_PATTERN: u16 = 33422; + pub const DNG_VERSION: u16 = 50706; + pub const UNIQUE_CAMERA_MODEL: u16 = 50708; + pub const CFA_PLANE_COLOR: u16 = 50710; + pub const CFA_LAYOUT: u16 = 50711; + pub const LINEARIZATION_TABLE: u16 = 50712; + pub const BLACK_LEVEL_REPEAT_DIM: u16 = 50713; + pub const BLACK_LEVEL: u16 = 50714; + pub const BLACK_LEVEL_DELTA_H: u16 = 50715; + pub const BLACK_LEVEL_DELTA_V: u16 = 50716; + pub const WHITE_LEVEL: u16 = 50717; + pub const DEFAULT_CROP_ORIGIN: u16 = 50719; + pub const DEFAULT_CROP_SIZE: u16 = 50720; + pub const COLOR_MATRIX_1: u16 = 50721; + pub const COLOR_MATRIX_2: u16 = 50722; + pub const CAMERA_CALIBRATION_1: u16 = 50723; + pub const CAMERA_CALIBRATION_2: u16 = 50724; + pub const ANALOG_BALANCE: u16 = 50727; + pub const AS_SHOT_NEUTRAL: u16 = 50728; + pub const AS_SHOT_WHITE_XY: u16 = 50729; + pub const BASELINE_EXPOSURE: u16 = 50730; + pub const CALIBRATION_ILLUMINANT_1: u16 = 50778; + pub const CALIBRATION_ILLUMINANT_2: u16 = 50779; + pub const ACTIVE_AREA: u16 = 50829; + pub const OPCODE_LIST_1: u16 = 51008; + pub const OPCODE_LIST_2: u16 = 51009; + pub const OPCODE_LIST_3: u16 = 51022; + pub const FORWARD_MATRIX_1: u16 = 50964; + pub const FORWARD_MATRIX_2: u16 = 50965; + /// CinemaDNG: SMPTE ST 12-1 time codes, 8 bytes each. + pub const TIME_CODES: u16 = 51043; + /// CinemaDNG: frames per second (SRATIONAL). + pub const FRAME_RATE: u16 = 51044; + /// CinemaDNG: the camera's reel name. + pub const REEL_NAME: u16 = 51081; +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A little-endian TIFF: a chain of `ifds` directories, each with one SubIFDs entry of + /// `count` LONGs, all naming the same table (of zeros, or of `target`). + fn chain(ifds: usize, count: u32, target: u32) -> Vec { + let mut b = b"II*\0".to_vec(); + b.extend_from_slice(&8u32.to_le_bytes()); + let table = 8 + ifds * 18; + for i in 0..ifds { + let next = if i + 1 < ifds { 8 + (i + 1) * 18 } else { 0 }; + b.extend_from_slice(&1u16.to_le_bytes()); + b.extend_from_slice(&tags::SUB_IFDS.to_le_bytes()); + b.extend_from_slice(&4u16.to_le_bytes()); + b.extend_from_slice(&count.to_le_bytes()); + b.extend_from_slice(&(table as u32).to_le_bytes()); + b.extend_from_slice(&(next as u32).to_le_bytes()); + } + for _ in 0..count.max(2) { + b.extend_from_slice(&target.to_le_bytes()); + } + b + } + + #[test] + fn shared_sub_ifd_tables_are_budgeted_before_they_are_queued() { + // 60 directories each naming one 100 000-entry table: 6 million queued edges before + let b = chain(60, 100_000, 0); + let (t, first) = Tiff::new(&b).unwrap(); + assert!(matches!(t.all_ifds(first), Err(Error::Malformed("too many SubIFD references")))); + } + + #[test] + fn matches_general_reader_for_a_simple_ifd_chain() { + let bytes = chain(3, 0, 0); + let (reader, first) = Tiff::new(&bytes).unwrap(); + let classic = reader.all_ifds(first).unwrap(); + let general = crate::Tiff::parse(&bytes).unwrap(); + assert_eq!(classic.len(), general.ifds.len()); + assert!(classic.iter().all(|ifd| ifd.get(tags::SUB_IFDS).is_some())); + } + + #[test] + fn ordinary_sub_ifds_are_still_walked() { + // one directory whose SubIFD is the same directory: visited once + let b = chain(1, 1, 8); + let (t, first) = Tiff::new(&b).unwrap(); + assert_eq!(t.all_ifds(first).unwrap().len(), 1); + // a short chain with a few zero SubIFD offsets + let b = chain(3, 4, 0); + let (t, first) = Tiff::new(&b).unwrap(); + assert_eq!(t.all_ifds(first).unwrap().len(), 3); + } + #[test] + fn numeric_field_types_and_malformed_values() { + fn field(code: u16, payload: &[u8]) -> Vec { + let mut b = b"II*\0".to_vec(); + b.extend_from_slice(&8u32.to_le_bytes()); + b.extend_from_slice(&1u16.to_le_bytes()); + b.extend_from_slice(&256u16.to_le_bytes()); + b.extend_from_slice(&code.to_le_bytes()); + b.extend_from_slice(&1u32.to_le_bytes()); + if payload.len() <= 4 { + let mut inline = [0; 4]; + inline[..payload.len()].copy_from_slice(payload); + b.extend_from_slice(&inline); + } else { + b.extend_from_slice(&26u32.to_le_bytes()); + } + b.extend_from_slice(&0u32.to_le_bytes()); + if payload.len() > 4 { + b.extend_from_slice(payload); + } + b + } + let cases: &[(u16, &[u8], f64)] = &[ + (1, &[7], 7.0), + (6, &[255], -1.0), + (3, &[2, 0], 2.0), + (8, &[255, 255], -1.0), + (4, &[3, 0, 0, 0], 3.0), + (9, &[255, 255, 255, 255], -1.0), + (5, &[3, 0, 0, 0, 2, 0, 0, 0], 1.5), + (10, &[253, 255, 255, 255, 2, 0, 0, 0], -1.5), + (11, &1.5_f32.to_le_bytes(), 1.5), + (12, &1.5_f64.to_le_bytes(), 1.5), + ]; + for &(code, data, expected) in cases { + let bytes = field(code, data); + let (t, at) = Tiff::new(&bytes).unwrap(); + let (ifd, _) = t.read_ifd(at, 0).unwrap(); + assert_eq!(t.floats(&ifd.entries[0]).unwrap(), [expected], "type {code}"); + } + let bytes = field(5, &[1, 0, 0, 0, 0, 0, 0, 0]); + let (t, at) = Tiff::new(&bytes).unwrap(); + let (ifd, _) = t.read_ifd(at, 0).unwrap(); + assert!(t.floats(&ifd.entries[0]).is_err()); + let bytes = field(11, &f32::NAN.to_le_bytes()); + let (t, at) = Tiff::new(&bytes).unwrap(); + let (ifd, _) = t.read_ifd(at, 0).unwrap(); + assert!(t.floats(&ifd.entries[0]).is_err()); + } +} diff --git a/crates/tiff/src/lib.rs b/crates/tiff/src/lib.rs index bd095f4f0..0e9d595c7 100644 --- a/crates/tiff/src/lib.rs +++ b/crates/tiff/src/lib.rs @@ -22,6 +22,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +/// Bounded classic TIFF directory reader with lazy value access. +pub mod classic; + pub mod image; pub mod makernote; pub mod packbits; diff --git a/crates/usd/README.md b/crates/usd/README.md index 9e9635185..fcf585760 100644 --- a/crates/usd/README.md +++ b/crates/usd/README.md @@ -2,6 +2,6 @@ Bounded wire primitives for USD Crate, independent of application scene and material models. -Provides bounded integer-delta array decoding and Crate-framed LZ4 decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and export remain outside this extraction. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. +Provides bounded integer-delta array decoding and Crate-framed LZ4 compression/decompression: a single unprefixed block or up to 127 length-prefixed blocks, under a caller-provided output-byte cap with fallible output allocation. Trailing bytes after counted chunks retain the original reader's ignored-tail policy. This is not a complete USD reader/writer; scene interpretation, integer packing and full scene export remain outside this extraction. Compression uses a caller cap on worst-case reserved output and fallible allocation. Integer decoding uses explicit 32/64-bit widths, signed wrapping arithmetic, a caller output-byte cap and fallible allocation. Adapted from TextureCraft `crates/formats/src/mesh_import/usd/codec.rs`, introduced by bflatastic in `bdbb52c9b66b221a55b370105b42f485a76e6fc7`. The LZ4 backend is MIT-licensed `lz4_flex`. diff --git a/crates/usd/src/compress.rs b/crates/usd/src/compress.rs new file mode 100644 index 000000000..b92fa3be8 --- /dev/null +++ b/crates/usd/src/compress.rs @@ -0,0 +1,76 @@ +//! Bounded single-block USD Crate LZ4 writing. + +/// Compression admission, allocation or backend failure. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// Input or worst-case output exceeds the supported/caller budget. + #[error("USD Crate LZ4 compression exceeds budget")] + Limit, + /// The bounded output reservation failed. + #[error("cannot allocate USD Crate LZ4 output")] + Allocation, + /// The backend could not encode into the preflighted output buffer. + #[error("USD Crate LZ4 compression: {0}")] + Encode(#[from] lz4_flex::block::CompressError), +} + +fn capacity(input: usize, max_output_bytes: usize) -> Result { + // Keep inputs below the LZ4 single-block limit and preflight the backend's + // 20 + input*110/100 sizing arithmetic before calling it, including the count byte. + if input > 0x7e00_0000 { + return Err(Error::Limit); + } + input + .checked_mul(110) + .map(|n| n / 100) + .and_then(|n| n.checked_add(21)) + .filter(|n| *n <= max_output_bytes && *n <= isize::MAX as usize) + .ok_or(Error::Limit) +} + +/// Write one Crate-framed LZ4 block, beginning with a zero chunk-count byte. +/// +/// `max_output_bytes` bounds the reserved output including framing. Admission uses +/// the backend's worst-case size, so even highly compressible input must fit that bound. +/// Allocation is fallible; backend scratch memory is not included in the output cap. +/// Uses TextureCraft's original LZ4 backend/configuration and produces the same block bytes. +pub fn compress(input: &[u8], max_output_bytes: usize) -> Result, Error> { + let capacity = capacity(input.len(), max_output_bytes)?; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.resize(capacity, 0); + let target = output.get_mut(1..).ok_or(Error::Limit)?; + let written = lz4_flex::block::compress_into(input, target)?; + output.truncate(written.checked_add(1).ok_or(Error::Limit)?); + Ok(output) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + #[test] + fn original_wire_bytes_and_roundtrip() { + for size in [0, 1, 15, 16, 255, 4096, 65536] { + for repeated in [false, true] { + let input: Vec = (0..size).map(|i| if repeated { 17 } else { (i % 251) as u8 }).collect(); + let mut old = vec![0]; + old.extend(lz4_flex::block::compress(&input)); + let encoded = compress(&input, 1 << 20).unwrap(); + assert_eq!(encoded, old); + assert_eq!(crate::decompress(&encoded, size).unwrap(), input); + } + } + } + + #[test] + fn checks_worst_case_before_allocation() { + let input = [0; 100]; + assert!(matches!(compress(&input, 130), Err(Error::Limit))); + assert!(compress(&input, 131).is_ok()); + assert!(capacity(usize::MAX, usize::MAX).is_err()); + assert!(capacity(0x7e00_0001, usize::MAX).is_err()); + assert_eq!(capacity(0, 21).unwrap(), 21); + } +} diff --git a/crates/usd/src/integers.rs b/crates/usd/src/integers.rs index c50dd0e0f..0d8e67df4 100644 --- a/crates/usd/src/integers.rs +++ b/crates/usd/src/integers.rs @@ -80,6 +80,72 @@ pub fn decode(data: &[u8], count: usize, width: Width, max_output_bytes: usize) Ok(output) } +/// Encode integers with their original signed wrapping and largest-delta tie-break policy. +/// +/// `max_bytes` bounds the combined worst-case output reservation and sorted delta scratch. +/// Both reservations are fallible. Sorting uses no extra heap storage; input storage and +/// allocator bookkeeping are outside this budget. Empty input emits no bytes. +pub fn encode(values: &[i64], width: Width, max_bytes: usize) -> Result, Error> { + if values.is_empty() { + return Ok(Vec::new()); + } + let size = width.bytes(); + let codes_len = values.len().div_ceil(4); + let capacity = values.len().checked_mul(size).and_then(|n| n.checked_add(size)).and_then(|n| n.checked_add(codes_len)).ok_or(Error::Limit)?; + let scratch = values.len().checked_mul(8).ok_or(Error::Limit)?; + if capacity > isize::MAX as usize || scratch > isize::MAX as usize || capacity.checked_add(scratch).is_none_or(|n| n > max_bytes) { + return Err(Error::Limit); + } + let wrap = |v: i64| match width { + Width::I32 => i64::from(v as i32), + Width::I64 => v, + }; + let mut deltas = Vec::new(); + deltas.try_reserve_exact(values.len()).map_err(|_| Error::Allocation)?; + let mut previous = 0i64; + for &value in values { + deltas.push(wrap(value.wrapping_sub(previous))); + previous = value; + } + deltas.sort_unstable(); + let mut best = (0usize, 0i64); + let mut at = 0; + while at < deltas.len() { + let value = deltas[at]; + let start = at; + while at < deltas.len() && deltas[at] == value { + at += 1; + } + best = best.max((at - start, value)); + } + let common = best.1; + let mut output = Vec::new(); + output.try_reserve_exact(capacity).map_err(|_| Error::Allocation)?; + output.extend_from_slice(&common.to_le_bytes()[..size]); + output.resize(size + codes_len, 0); + previous = 0; + for (i, &value) in values.iter().enumerate() { + let delta = wrap(value.wrapping_sub(previous)); + previous = value; + let small = |bits: u32| delta >= -(1i64 << (bits - 1)) && delta < (1i64 << (bits - 1)); + let code: u8 = if delta == common { + 0 + } else if small(size as u32 * 2) { + 1 + } else if small(size as u32 * 4) { + 2 + } else { + 3 + }; + if code != 0 { + let bytes = (size >> (3 - usize::from(code))).max(1); + output.extend_from_slice(&delta.to_le_bytes()[..bytes]); + } + output[size + i / 4] |= code << (2 * (i % 4)); + } + Ok(output) +} + #[cfg(test)] #[allow(clippy::unwrap_used)] mod tests { @@ -125,3 +191,81 @@ mod tests { } } } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod encode_tests { + use super::*; + fn oracle(values: &[i64], width: usize) -> Vec { + if values.is_empty() { + return Vec::new(); + } + let wrap = |v: i64| if width == 4 { i64::from(v as i32) } else { v }; + let mut prev = 0i64; + let deltas: Vec = values + .iter() + .map(|&v| { + let d = wrap(v.wrapping_sub(prev)); + prev = v; + d + }) + .collect(); + let mut counts = std::collections::HashMap::new(); + for d in &deltas { + *counts.entry(*d).or_insert(0usize) += 1; + } + let common = counts.into_iter().max_by_key(|&(v, c)| (c, v)).map_or(0, |(v, _)| v); + let mut out = Vec::with_capacity(width + values.len().div_ceil(4) + values.len() * width); + out.extend_from_slice(&common.to_le_bytes()[..width]); + let codes_at = out.len(); + out.resize(codes_at + values.len().div_ceil(4), 0); + for (i, &d) in deltas.iter().enumerate() { + let small = |bits: u32| d >= -(1i64 << (bits - 1)) && d < (1i64 << (bits - 1)); + let code: u8 = if d == common { + 0 + } else if small(width as u32 * 2) { + 1 + } else if small(width as u32 * 4) { + 2 + } else { + 3 + }; + if code > 0 { + let w = (width >> (3 - code as usize)).max(1); + out.extend_from_slice(&d.to_le_bytes()[..w]); + } + if let Some(b) = out.get_mut(codes_at + i / 4) { + *b |= code << (2 * (i % 4)); + } + } + out + } + + #[test] + fn matches_original_bytes_and_wrapping() { + for width in [Width::I32, Width::I64] { + for n in [0, 1, 2, 3, 4, 5, 127, 1024] { + let mut seed = 17u64; + let values: Vec = (0..n) + .map(|_| { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1); + seed as i64 + }) + .collect(); + let bytes = encode(&values, width, 32768).unwrap(); + assert_eq!(bytes, oracle(&values, width.bytes())); + let expected: Vec = values.iter().map(|&v| if width == Width::I32 { i64::from(v as i32) } else { v }).collect(); + assert_eq!(decode(&bytes, n, width, 32768).unwrap(), expected); + } + for values in [vec![1, 3, 4, 6], vec![i64::MAX, i64::MIN, 0], vec![7; 32]] { + assert_eq!(encode(&values, width, 4096).unwrap(), oracle(&values, width.bytes())); + } + } + } + #[test] + fn combined_reservation_budget_is_enforced() { + assert_eq!(encode(&[], Width::I64, 0).unwrap(), Vec::::new()); + assert!(matches!(encode(&[1], Width::I32, 16), Err(Error::Limit))); + assert!(encode(&[1], Width::I32, 17).is_ok()); + } +} diff --git a/crates/usd/src/lib.rs b/crates/usd/src/lib.rs index edaeb89a2..33c7f3484 100644 --- a/crates/usd/src/lib.rs +++ b/crates/usd/src/lib.rs @@ -5,6 +5,9 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +/// Bounded single-block Crate LZ4 compression. +pub mod compress; + /// Bounded Crate integer delta arrays. pub mod integers; diff --git a/crates/video/src/av1/decode/README.md b/crates/video/src/av1/decode/README.md index 1518df92c..e1b8e4732 100644 --- a/crates/video/src/av1/decode/README.md +++ b/crates/video/src/av1/decode/README.md @@ -73,11 +73,17 @@ with libdav1d: **bit-exact** on all 6 fixtures. `Decoder::new()` uses all cores (`Decoder::with_threads(1)` decodes on the calling thread; on wasm32, or without the default `threads` feature, everything runs on the calling thread). Headers are parsed in order; each frame is -then decoded by a frame worker once the reference frames it reads are finished, so independent -frames (hierarchical GOP layers) decode concurrently; tiles of a frame decode in parallel into -private buffers that are merged afterwards; the in-loop filters (deblocking, CDEF, loop -restoration) each run in parallel bands of 64 rows (`postfilter.rs` explains why the bands are -independent). Results never depend on scheduling (the oracle tests compare 1 thread, many threads +then decoded by a frame worker. With worker threads it first reads the symbols of all its tiles, +which needs only the parse results of its references (CDFs, segment ids, saved motion vectors), +publishes its own, and then reconstructs from the recorded blocks. Frames with several tiles +read the symbols of their tiles in parallel (one after another with intra block copy). A frame reconstructs superblock row by superblock +row (that row of every tile), each row waiting only for the reference rows its motion vectors +reach (row wavefront), and the in-loop filters (deblocking, CDEF, loop restoration) follow a few +64-row bands behind and publish each final band for the frames predicting from it +(`decoder.rs` and `postfilter.rs` give the dependency rules). Super-resolution frames wait for +their whole references and filter the whole frame. There is one frame worker per thread (at most +16), and the frames in flight are bounded by their size. Independent frames (hierarchical GOP +layers) decode concurrently. Results never depend on scheduling (the oracle tests compare 1 thread, many threads and libdav1d). With frame threads, pictures may be returned by a later `decode_pts` call or by `flush`; each carries the `pts` of the temporal unit that showed it. @@ -130,6 +136,44 @@ motion vectors nor CDFs reach another frame — skip deblocking, CDEF and loop r (super-resolution still runs, it sets the output size) and are flagged `Picture::draft`. In SVT-AV1's hierarchical GOPs that is the top layer, half of the frames. +Frame threading on a Ryzen 7800X3D (8 cores / 16 threads, release `examples/av1_av1dec`, median of 7 +alternating runs against Filmcraft `6909e461`, 2026-11-08), wall time for the whole file: + +| Fixture | Threads | Before | After | Speed-up | +|---|---|---|---|---| +| `perf_1080p_gop` (60 frames) | 1 | 69.2 fps | 69.1 fps | 1.00x | +| `perf_1080p_gop` | 16 | 159.6 fps | 227.3 fps | 1.42x | +| `perf_2160p_gop` (30 frames) | 1 | 17.9 fps | 17.8 fps | 1.00x | +| `perf_2160p_gop` | 16 | 40.2 fps | 57.7 fps | 1.43x | + +Peak RSS at 16 threads goes from 278 to 365 MB (1080p) and from 1031 to 1217 MB (2160p, +18%): +every frame in flight holds its parse record and its final samples are kept in 64-row bands. + +With one frame worker per thread, the row wavefront for frames with several tiles and parse +waits limited to what parsing reads (same machine, median of 5 alternating runs, 2026-11-08): + +| Fixture | Threads | Before (`6909e461`) | After | Speed-up | Peak RSS before / after | +|---|---|---|---|---|---| +| `perf_1080p_gop` | 4 | 95.7 fps | 185.2 fps | 1.94x | 148 / 236 MB | +| `perf_1080p_gop` | 16 | 155.0 fps | 259.7 fps | 1.68x | 274 / 560 MB | +| `perf_2160p_gop` | 4 | 23.6 fps | 47.1 fps | 2.00x | 512 / 798 MB | +| `perf_2160p_gop` | 16 | 39.9 fps | 65.8 fps | 1.65x | 994 / 1353 MB | +| `perf_1080p_gop_tiles` (4x2 tiles) | 16 | 186.3 fps | 237.2 fps | 1.27x | 328 / 617 MB | +| SVT-AV1 2160p, 4x2 tiles (30 frames) | 16 | 57.1 fps | 70.8 fps | 1.24x | 1197 / 1502 MB | + +Known limitations of the frame threading (the output is the same in every case, only the +overlap is smaller): + +- A frame whose parse record would pass 32 bytes per luma sample of a tile (pathological streams: + tiny blocks, large palettes, every coefficient coded), or that cannot reserve its record from + the decoder-wide budget of 1 GiB shared by the frames in flight, is decoded again in one pass. +- A superblock row using local warped motion, global motion beyond translation or a scaled + reference waits for the whole reference instead of the rows its motion vectors reach. +- Super-resolution frames do not use the row wavefront: they wait for their whole references + and filter the whole frame. +- The tile columns of a superblock row are reconstructed one after another on the frame's worker + (their symbols are still read in parallel). + ## Limits Frames beyond the AV1 level 6.x limits (Annex A: at most 16384 wide, 8704 high and 35 651 584 luma samples diff --git a/crates/video/src/av1/decode/cdef.rs b/crates/video/src/av1/decode/cdef.rs index edd119292..05eb0a2a7 100644 --- a/crates/video/src/av1/decode/cdef.rs +++ b/crates/video/src/av1/decode/cdef.rs @@ -10,19 +10,33 @@ use crate::av1::decode::state::FrameState; /// equals CurrFrame). Bands of 64 luma rows (whole 64x64 filter blocks) are filtered in /// parallel: every block reads only CurrFrame and writes only its own samples. pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option { + if !active(fs) { + return None; + } + let mut out = fs.cur.clone(); + let ranges = bands(pool, fs.fh.mi_rows as usize * 4); + filter_bands(fs, pool, out.row_bands(&ranges), &ranges); + Some(out) +} + +/// Whether CDEF filters any block of the frame (it reads only parse results). +pub(crate) fn active(fs: &FrameState) -> bool { let fh = &fs.fh; if fh.coded_lossless || fh.allow_intrabc || !fs.seq.enable_cdef { - return None; + return false; } let (mi_rows, mi_cols) = (fh.mi_rows as usize, fh.mi_cols as usize); let fb_rows = mi_rows.div_ceil(16); let fb_cols = mi_cols.div_ceil(16); - if !(0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some())) { - return None; - } - let mut out = fs.cur.clone(); - let ranges = bands(pool, mi_rows * 4); - let mut planes = out.row_bands(&ranges); + (0..fb_rows).any(|fr| (0..fb_cols).any(|fc| fb_skip_mask(fs, fr * 16, fc * 16).is_some())) +} + +/// Filter the bands `planes` covering luma rows `ranges` (whole 64x64 filter block rows), which +/// hold CurrFrame; reads CurrFrame up to 2 rows beyond each band. +pub(crate) fn filter_bands(fs: &FrameState, pool: &Pool, mut planes: Vec<[RowBand; 3]>, ranges: &[(usize, usize)]) { + let (mi_rows, mi_cols) = (fs.fh.mi_rows as usize, fs.fh.mi_cols as usize); + let fb_rows = mi_rows.div_ceil(16); + let fb_cols = mi_cols.div_ceil(16); pool.map_mut(&mut planes, |b, band| { let (start, end) = ranges[b]; let mut buf = [0i16; BUF_LEN]; @@ -34,8 +48,6 @@ pub(crate) fn apply(fs: &FrameState, pool: &Pool) -> Option { } } }); - drop(planes); - Some(out) } /// The filtered 8x8 blocks of the 64x64 filter block at (r, c): bit `(i << 3) | j` is set for diff --git a/crates/video/src/av1/decode/decoder.rs b/crates/video/src/av1/decode/decoder.rs index 65c3de6ef..4a922cce1 100644 --- a/crates/video/src/av1/decode/decoder.rs +++ b/crates/video/src/av1/decode/decoder.rs @@ -2,31 +2,37 @@ //! (7.20), show-existing-frame and output (7.18). //! //! Threading: headers are parsed in order on the calling thread; each frame then becomes a job -//! that waits for the reference frames it uses and decodes on a frame worker thread, so frames -//! that do not depend on each other decode concurrently. Within a frame, tiles and the -//! post-filter row bands run on a shared worker pool. Every result depends only on finished -//! reference frames, so the output is identical whatever the scheduling. Shown pictures are -//! returned in order; with frame threads they can come out of a later `decode` call (or -//! `flush`), each carrying the `pts` of the temporal unit that showed it. +//! on a frame worker thread. A job first reads the symbols of its tiles (parse), which needs +//! only the parse results of its references (CDFs, segment ids, saved motion vectors), and +//! publishes its own; it then reconstructs. A frame is reconstructed superblock row by +//! superblock row (that row of every tile), each row waiting only for the reference rows its +//! motion vectors reach, with the in-loop filters following a few bands behind and every final +//! band published for the frames that predict from it (row wavefront); super-resolution frames +//! wait for their whole references. So a frame can parse while its references are still being reconstructed, and reconstruct +//! while their bottom is still being filtered. Within a frame, tiles and the post-filter row +//! bands run on a shared worker pool; waits happen only on frame worker threads. Every result +//! depends only on final reference data, so the output is identical whatever the scheduling. +//! Shown pictures are returned in order; with frame threads they can come out of a later +//! `decode` call (or `flush`), each carrying the `pts` of the temporal unit that showed it. use std::collections::VecDeque; use std::sync::{Arc, Condvar, Mutex}; use crate::av1::decode::bits::{BitReader, leb128}; use crate::av1::decode::cdf::Cdfs; -use crate::av1::decode::frame::FrameBuf; +use crate::av1::decode::frame::{BandFrame, FrameBuf, PlaneRef}; use crate::av1::decode::header::{ ColorConfig, FilmGrainParams, FrameHeader, HeaderState, LoopFilterDeltas, RefInfo, SegmentationFeatures, SequenceHeader, default_gm_params, }; use crate::av1::decode::par::Pool; use crate::av1::decode::spec_tables::{ - ALTREF_FRAME, ALTREF2_FRAME, BWDREF_FRAME, DEFAULT_INTRA_FRAME_Y_MODE_CDF, DIV_MULT, GOLDEN_FRAME, INTRA_FRAME, INTRA_ONLY_FRAME, KEY_FRAME, - LAST_FRAME, LAST2_FRAME, MAX_FRAME_DISTANCE, MAX_OFFSET_HEIGHT, MAX_OFFSET_WIDTH, MFMV_STACK_SIZE, NUM_REF_FRAMES, PRIMARY_REF_NONE, - REFMVS_LIMIT, REFS_PER_FRAME, + ALTREF_FRAME, ALTREF2_FRAME, BWDREF_FRAME, DEFAULT_INTRA_FRAME_Y_MODE_CDF, DIV_MULT, GLOBAL_GLOBALMV, GLOBALMV, GOLDEN_FRAME, INTRA_FRAME, + INTRA_ONLY_FRAME, KEY_FRAME, LAST_FRAME, LAST2_FRAME, LOCALWARP, MAX_FRAME_DISTANCE, MAX_OFFSET_HEIGHT, MAX_OFFSET_WIDTH, MFMV_STACK_SIZE, + NUM_REF_FRAMES, PRIMARY_REF_NONE, REFMVS_LIMIT, REFS_PER_FRAME, TRANSLATION, }; use crate::av1::decode::state::{FrameShared, FrameState, TileState}; use crate::av1::decode::stats::{DecodeStats, Stage, Timer}; -use crate::av1::decode::tile::TileDecoder; +use crate::av1::decode::tile::{Pass, TileDecoder, TileRecord}; use crate::av1::decode::{Error, Picture, Result}; /// Largest accepted frame: the AV1 level 6.x limits (Annex A: MaxHSize 16384, MaxVSize 8704, MaxPicSize @@ -49,22 +55,156 @@ const OBU_FRAME_T: u32 = 6; const OBU_REDUNDANT_FRAME_HEADER_T: u32 = 7; const OBU_TILE_LIST_T: u32 = 8; -/// A stored reference frame (FrameStore plus the saved per-frame state). +/// The samples of a stored reference frame (FrameStore), after every in-loop filter: a whole +/// buffer (decoding on one thread) or the bands published while decoding (with worker threads). pub(crate) struct RefFrame { pub buf: FrameBuf, - pub cdfs: Box, + pub bands: Option>, + pub num_planes: usize, pub bit_depth: u8, pub subsampling_x: u8, pub subsampling_y: u8, + pub color: (u8, u8, u8, bool), +} + +impl RefFrame { + pub fn plane(&self, p: usize) -> PlaneRef<'_> { + match &self.bands { + Some(b) => PlaneRef::Bands(&b.planes[p]), + None => PlaneRef::Whole(&self.buf.planes[p]), + } + } +} + +/// The saved per-frame state that later frames read while parsing (CDFs, segment ids and the +/// motion vectors for motion field estimation). It depends only on the frame's symbols, so it +/// is published as soon as the tiles are decoded, before the in-loop filters run. +pub(crate) struct Parsed { + pub cdfs: Box, pub segment_ids: Vec, pub mi_cols: usize, pub mi_rows: usize, - pub color: (u8, u8, u8, bool), - /// SavedRefFrames / SavedMvs (MfRefFrames / MfMvs of 7.19), mi units. + /// SavedRefFrames / SavedMvs (MfRefFrames / MfMvs of 7.19) at the odd row and column of + /// every 8x8 block, the only positions the projection reads. pub saved_ref_frames: Vec, pub saved_mvs: Vec<[i32; 2]>, } +/// How many luma rows of a frame are final (every in-loop filter applied), or that the frame +/// failed. Rows only ever grow; a failure wakes every waiter with an error. +pub(crate) struct Progress { + state: Mutex, + cv: Condvar, +} + +#[derive(Clone, Copy, Default)] +struct ProgressState { + rows: usize, + failed: bool, +} + +impl Progress { + fn new() -> Progress { + Progress { state: Mutex::new(ProgressState::default()), cv: Condvar::new() } + } + + /// Rows [0, rows) are final. A smaller value than already published is ignored. + pub fn publish(&self, rows: usize) { + #[cfg(test)] + tests::jitter(); + let mut g = self.state.lock().unwrap_or_else(|e| e.into_inner()); + if rows > g.rows { + g.rows = rows; + self.cv.notify_all(); + } + } + + /// The frame will publish no more rows. + pub fn fail(&self) { + let mut g = self.state.lock().unwrap_or_else(|e| e.into_inner()); + g.failed = true; + self.cv.notify_all(); + } + + /// Rows published so far. + #[cfg(test)] + fn rows(&self) -> usize { + self.state.lock().map(|g| g.rows).unwrap_or(0) + } + + /// Blocks until rows [0, rows) are final; an error when the frame failed first. + pub fn wait(&self, rows: usize) -> Result<()> { + #[cfg(test)] + tests::jitter(); + let mut g = self.state.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if g.rows >= rows { + return Ok(()); + } + if g.failed { + return Err(Error::Invalid("reference frame failed to decode")); + } + g = self.cv.wait(g).unwrap_or_else(|e| e.into_inner()); + } + } +} + +/// Everything a decoding frame publishes for the frames that reference it and for the output: +/// its parse results, its final rows (bands, then the row progress) and the finished frame. The +/// parse results and the frame are set exactly once and the progress ends complete or failed, +/// also when decoding fails, so no waiter blocks forever. +pub(crate) struct FrameSlot { + parsed: Slot>, + pub progress: Progress, + /// The final samples, published band by band before the progress covers them (with + /// worker threads). + pub bands: Arc, + frame: Slot>, +} + +impl FrameSlot { + fn new(bands: BandFrame) -> Arc { + Arc::new(FrameSlot { parsed: Slot::new_plain(), progress: Progress::new(), bands: Arc::new(bands), frame: Slot::new_plain() }) + } + + /// The finished frame. + fn wait(&self) -> Result> { + self.frame.wait() + } + + fn is_set(&self) -> bool { + self.frame.is_set() + } + + /// The frame's parse results (available before its samples are final). + fn wait_parsed(&self) -> Result> { + self.parsed.wait() + } + + /// Publish the parse results (ignored when already set). + fn set_parsed(&self, v: Result>) { + self.parsed.set_once(v); + } + + /// Publish the finished frame: every row is final. On an error, the parse results (when + /// not yet published) and the progress fail too. + fn finish(&self, v: Result>, rows: usize) { + match &v { + Ok(_) => {} + Err(e) => { + self.parsed.set_once(Err(e.clone())); + } + } + let ok = v.is_ok(); + self.frame.set(v); + if ok { + self.progress.publish(rows); + } else { + self.progress.fail(); + } + } +} + /// A value produced by a decoding job, waited for by later jobs and by the output. pub(crate) struct Slot { value: Mutex>>, @@ -73,7 +213,11 @@ pub(crate) struct Slot { impl Slot { fn new() -> Arc> { - Arc::new(Slot { value: Mutex::new(None), cv: Condvar::new() }) + Arc::new(Slot::new_plain()) + } + + fn new_plain() -> Slot { + Slot { value: Mutex::new(None), cv: Condvar::new() } } fn set(&self, v: Result) { @@ -82,6 +226,15 @@ impl Slot { self.cv.notify_all(); } + /// [`Slot::set`] unless a value is already there. + fn set_once(&self, v: Result) { + let mut g = self.value.lock().unwrap_or_else(|e| e.into_inner()); + if g.is_none() { + *g = Some(v); + self.cv.notify_all(); + } + } + fn is_set(&self) -> bool { self.value.lock().map(|g| g.is_some()).unwrap_or(true) } @@ -108,8 +261,6 @@ impl Slot { } } -type FrameSlot = Slot>; - /// Everything needed to decode one frame, captured when its header is parsed. struct FrameJob { seq: SequenceHeader, @@ -130,6 +281,15 @@ struct FrameJob { draft: bool, #[cfg(test)] force_panic: bool, + /// Panic in the row wavefront at the first superblock row from this one after which rows + /// are published (tests). + #[cfg(test)] + panic_at_row: Option, + /// Overrides [`RECORD_BYTES_PER_SAMPLE`] (tests). + #[cfg(test)] + record_limit: Option, + /// The decoder's parse record budget. + record_budget: Arc, } /// A shown picture in output order. @@ -254,6 +414,8 @@ pub struct Decoder { draft: bool, stats: Arc>, pool: Pool, + /// Parse record bytes all frames in flight may hold together. + record_budget: Arc, #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] workers: Option, /// Frames submitted to the workers and not known to be finished (oldest first). @@ -268,6 +430,14 @@ pub struct Decoder { /// Make every frame job panic (tests of the worker panic handling). #[cfg(test)] force_panic: bool, + /// (frame number, superblock row): that frame panics in the row wavefront after that row, + /// once it has published rows that later frames may already be using (tests). + #[cfg(test)] + panic_at_row: Option<(usize, usize)>, + #[cfg(test)] + record_limit: Option, + #[cfg(test)] + frame_number: usize, } fn default_threads() -> usize { @@ -281,10 +451,25 @@ fn default_threads() -> usize { } } -/// Frames decoded concurrently with `threads` threads (each in-flight 1080p frame holds a few -/// tens of MB of state). +/// Frame worker threads with `threads` threads. A frame worker spends most of its time waiting +/// for the parse results and rows of its references, so there is one per thread (at least two, +/// at most [`MAX_FRAME_THREADS`]); the memory is bounded by `in_flight_limit` instead. fn frame_threads(threads: usize) -> usize { - if threads <= 1 { 1 } else { threads.div_ceil(2).clamp(2, 8) } + if threads <= 1 { 1 } else { threads.clamp(2, MAX_FRAME_THREADS) } +} + +const MAX_FRAME_THREADS: usize = 16; + +/// Luma samples of the frames in flight (each in-flight frame holds its state, parse record and +/// samples, roughly 10 bytes per luma sample): ten 2160p frames. +#[cfg(all(feature = "threads", not(target_arch = "wasm32")))] +const IN_FLIGHT_SAMPLES: usize = 10 * 3840 * 2160; + +/// Frames submitted to the frame workers that may be unfinished: two more than the workers, +/// fewer for large frames (at least four). +#[cfg(all(feature = "threads", not(target_arch = "wasm32")))] +fn in_flight_limit(frame_threads: usize, luma_samples: usize) -> usize { + (frame_threads + 2).min((IN_FLIGHT_SAMPLES / luma_samples.max(1)).max(4)) } impl Default for Decoder { @@ -391,6 +576,7 @@ impl Decoder { draft: false, stats, pool, + record_budget: Arc::new(crate::av1::decode::tile::RecordBudget::new(RECORD_BUDGET_BYTES)), #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] workers, in_flight: VecDeque::new(), @@ -400,6 +586,12 @@ impl Decoder { pts: 0, #[cfg(test)] force_panic: false, + #[cfg(test)] + panic_at_row: None, + #[cfg(test)] + record_limit: None, + #[cfg(test)] + frame_number: 0, } } @@ -667,6 +859,13 @@ impl Decoder { let picture = fh.show_frame.then(Slot::new); let draft = self.draft && fh.show_frame && fh.refresh_frame_flags == 0; let tile_count = fh.tile_info.cols.checked_mul(fh.tile_info.rows).ok_or(Error::Invalid("tile count overflows"))?; + let slot = FrameSlot::new(BandFrame::new( + fh.upscaled_width as usize, + fh.frame_height as usize, + seq.color.num_planes, + seq.color.subsampling_x as usize, + seq.color.subsampling_y as usize, + )); self.pending = Some(Box::new(FrameJob { seq, fh, @@ -674,14 +873,23 @@ impl Decoder { ref_info: self.refs.clone(), tiles: Vec::new(), tile_covered: vec![false; tile_count], - slot: Slot::new(), + slot, picture, pts: self.pts, apply_film_grain: self.apply_film_grain, draft, #[cfg(test)] force_panic: self.force_panic, + #[cfg(test)] + panic_at_row: self.panic_at_row.filter(|&(f, _)| f == self.frame_number).map(|(_, r)| r), + #[cfg(test)] + record_limit: self.record_limit, + record_budget: self.record_budget.clone(), })); + #[cfg(test)] + { + self.frame_number += 1; + } Ok(None) } @@ -779,6 +987,8 @@ impl Decoder { .clone() .map(|p| Output::Decoded(p, picture_bytes(job.fh.upscaled_width as usize, job.fh.frame_height as usize, &job.seq.color))); #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] + let fh_samples = (fh.upscaled_width as usize).max(fh.frame_width as usize) * fh.frame_height as usize; + #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] if let Some(w) = &self.workers && let Some(tx) = &w.tx { @@ -787,7 +997,8 @@ impl Decoder { Ok(()) => { self.in_flight.push_back(slot); // Bound the frames in flight (memory): wait for the oldest. - while self.in_flight.len() > self.max_in_flight { + let limit = in_flight_limit(w.handles.len(), fh_samples); + while self.in_flight.len() > limit { let Some(s) = self.in_flight.pop_front() else { break }; let _ = s.wait(); } @@ -808,18 +1019,19 @@ impl Decoder { fn run_job(job: FrameJob, pool: &Pool, stats: &Mutex) { let mut st = DecodeStats::default(); let res = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| decode_frame(&job, pool, &mut st))); + let rows = job.fh.frame_height as usize; match res.unwrap_or(Err(Error::Invalid("decoder panicked"))) { Ok((rf, pic)) => { if let Some(ps) = &job.picture { ps.set(pic.ok_or(Error::Invalid("shown frame without picture"))); } - job.slot.set(Ok(rf)); + job.slot.finish(Ok(rf), rows); } Err(e) => { if let Some(ps) = &job.picture { ps.set(Err(e.clone())); } - job.slot.set(Err(e)); + job.slot.finish(Err(e), rows); } } if let Ok(mut s) = stats.lock() { @@ -844,71 +1056,191 @@ fn decode_frame(job: &FrameJob, pool: &Pool, stats: &mut DecodeStats) -> Result< if fh.primary_ref_frame != PRIMARY_REF_NONE { used[fh.ref_frame_idx[fh.primary_ref_frame]] = true; } - let mut refs: [Option>; NUM_REF_FRAMES] = Default::default(); - for i in 0..NUM_REF_FRAMES { - if used[i] { - let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; - refs[i] = Some(slot.wait()?); + // Prediction reads the reference planes with this frame's chroma format. + if !fh.frame_is_intra { + for i in 0..NUM_REF_FRAMES { + let Some(slot) = job.refs[i].as_ref().filter(|_| used[i]) else { continue }; + let b = &slot.bands; + let c = &seq.color; + if b.num_planes != c.num_planes || b.subsampling_x != c.subsampling_x as usize || b.subsampling_y != c.subsampling_y as usize { + return Err(Error::Invalid("reference frame has another chroma format")); + } } } - let t = Timer::start(); - let mut fs = FrameState::new(seq, fh); - fs.refs = refs; - fs.ref_info = job.ref_info.clone(); + // Parse dependencies first, then the samples. Parsing reads the CDFs and segment ids of + // the primary reference frame, and the saved motion vectors of the inter references only + // with use_ref_frame_mvs; the parse results of the other references are not waited for + // (their failures still fail this frame through the waits for their samples below). + let mut parse_deps = [false; NUM_REF_FRAMES]; if fh.primary_ref_frame != PRIMARY_REF_NONE { - // load_previous_segment_ids( ) - let (mi_cols, mi_rows) = (fs.mi.cols, fs.mi.rows); - let sh = &mut fs.sh; - if let Some(rf) = &sh.refs[fh.ref_frame_idx[fh.primary_ref_frame]] - && fh.seg.enabled - && rf.mi_cols == mi_cols - && rf.mi_rows == mi_rows - { - sh.prev_segment_ids.copy_from_slice(&rf.segment_ids); - } + parse_deps[fh.ref_frame_idx[fh.primary_ref_frame]] = true; } if fh.use_ref_frame_mvs { - motion_field_estimation(&mut fs); + for i in 0..REFS_PER_FRAME { + parse_deps[fh.ref_frame_idx[i]] = true; + } } - let mut cdfs = if fh.primary_ref_frame == PRIMARY_REF_NONE { - Cdfs::new(fh.quant.base_q_idx) - } else { - let rf = fs.refs[fh.ref_frame_idx[fh.primary_ref_frame]].as_ref().ok_or(Error::Invalid("primary reference frame missing"))?; - let mut c = rf.cdfs.clone(); - c.clear_counts(); - c + let mut parsed: [Option>; NUM_REF_FRAMES] = Default::default(); + for i in 0..NUM_REF_FRAMES { + if parse_deps[i] { + let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; + parsed[i] = Some(slot.wait_parsed()?); + } + } + // The samples of the references predicted from (inter frames only). + let wait_samples = || -> Result<[Option>; NUM_REF_FRAMES]> { + let mut refs: [Option>; NUM_REF_FRAMES] = Default::default(); + if !fh.frame_is_intra { + for i in 0..NUM_REF_FRAMES { + if used[i] { + let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; + slot.progress.wait(job.ref_info[i].frame_height as usize)?; + refs[i] = Some(slot.wait()?); + } + } + } + Ok(refs) }; + // With worker threads, the symbols of every tile are read first, so the parse results + // (which the next frames need before they can read their own symbols) are published + // before reconstruction, and parsing does not wait for the reference samples; on one + // thread each block is reconstructed as it is read. + let split = pool.is_parallel(); + // The frame state before tile decoding (with the reference samples when `with_refs`). + let setup = |with_refs: bool| -> Result<(FrameState, Box)> { + let mut fs = FrameState::new(seq, fh); + if with_refs { + fs.refs = wait_samples()?; + } + fs.parsed = parsed.clone(); + fs.ref_info = job.ref_info.clone(); + if fh.primary_ref_frame != PRIMARY_REF_NONE { + // load_previous_segment_ids( ) + let (mi_cols, mi_rows) = (fs.mi.cols, fs.mi.rows); + let sh = &mut fs.sh; + if let Some(rf) = &sh.parsed[fh.ref_frame_idx[fh.primary_ref_frame]] + && fh.seg.enabled + && rf.mi_cols == mi_cols + && rf.mi_rows == mi_rows + { + sh.prev_segment_ids.copy_from_slice(&rf.segment_ids); + } + } + if fh.use_ref_frame_mvs { + motion_field_estimation(&mut fs); + } + let cdfs = if fh.primary_ref_frame == PRIMARY_REF_NONE { + Cdfs::new(fh.quant.base_q_idx) + } else { + let rf = fs.parsed[fh.ref_frame_idx[fh.primary_ref_frame]].as_ref().ok_or(Error::Invalid("primary reference frame missing"))?; + let mut c = rf.cdfs.clone(); + c.clear_counts(); + c + }; + Ok((fs, cdfs)) + }; + let t = Timer::start(); + let (mut fs, mut cdfs) = setup(!split)?; stats.add(Stage::Setup, t.secs()); let t = Timer::start(); - let saved = decode_tiles(pool, &mut fs, &cdfs, &job.tiles)?; + // Time spent waiting for reference samples is not decoding time. + let mut waited = 0.0; + #[cfg(not(test))] + let record_limit = RECORD_BYTES_PER_SAMPLE; + #[cfg(test)] + let record_limit = job.record_limit.unwrap_or(RECORD_BYTES_PER_SAMPLE); + let mut tiles = parse_tiles(pool, &mut fs, &cdfs, &job.tiles, if split { Pass::Parse } else { Pass::Full }, record_limit, &job.record_budget)?; + // A frame whose record would take too much memory (pathological streams) is decoded again + // in one pass, as on one thread. + let one_pass = split && tiles.records.iter().any(|(_, r)| r.over_limit()); + if one_pass { + // The fallback starts again from the tile data: release the records first. + drop(tiles); + let wait = Timer::start(); + (fs, cdfs) = setup(true)?; + waited = wait.secs(); + tiles = parse_tiles(pool, &mut fs, &cdfs, &job.tiles, Pass::Full, record_limit, &job.record_budget)?; + } stats.tiles += job.tiles.len() as u64; - stats.add(Stage::Tiles, t.secs()); if !fh.disable_frame_end_update_cdf - && let Some(s) = saved + && let Some(s) = tiles.saved.take() { cdfs = s; } - crate::av1::decode::postfilter::apply(&mut fs, stats, pool, job.draft); + if split && !one_pass { + if tiles.whole { + if let Some(state) = tiles.states.pop() { + fs.put_tile_state(state); + job.slot.set_parsed(Ok(Arc::new(parse_results(&fs, cdfs)))); + tiles.states.push(fs.take_tile_state()); + } + } else { + for state in &tiles.states { + fs.merge_tile_modes(state); + } + job.slot.set_parsed(Ok(Arc::new(parse_results(&fs, cdfs)))); + } + if !fh.use_superres { + if !tiles.whole { + // The tiles were parsed into private buffers, merged into the frame above: + // reconstruct them in the frame's own buffers. + tiles.states.clear(); + tiles.states.push(fs.take_tile_state()); + tiles.whole = true; + } + // Reconstruct superblock row by superblock row (every tile of the row), each + // waiting only for the reference rows it reads, with the in-loop filters following + // behind. + waited = reconstruct_wavefront(job, pool, &mut fs, &mut tiles, stats)?; + } else { + let wait = Timer::start(); + fs.refs = wait_samples()?; + waited = wait.secs(); + reconstruct_tiles(pool, &fs.sh, &mut tiles)?; + tiles.finish(&mut fs); + let f = Timer::start(); + if fh.use_superres { + crate::av1::decode::postfilter::apply(&mut fs, stats, pool, job.draft); + publish_bands(&job.slot, &fs.cur, 0, usize::MAX, fh.frame_height as usize); + } else { + let mut pipe = crate::av1::decode::postfilter::Pipeline::new(&fs, job.draft); + let done = pipe.advance(&mut fs, stats, pool, usize::MAX); + publish_final(&job.slot, done, fh.frame_height as usize); + } + waited += f.secs(); + } + } else { + tiles.finish(&mut fs); + job.slot.set_parsed(Ok(Arc::new(parse_results(&fs, cdfs)))); + } + stats.add(Stage::Tiles, t.secs() - waited); + if !split || one_pass { + crate::av1::decode::postfilter::apply(&mut fs, stats, pool, job.draft); + } + if one_pass { + publish_bands(&job.slot, &fs.cur, 0, usize::MAX, fh.frame_height as usize); + } + if split { + // A frame using a reference only for its parse results (an intra frame's CDFs) still + // fails with it, as it does on one thread. + for i in 0..NUM_REF_FRAMES { + if used[i] { + job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?.wait()?; + } + } + } stats.draft_frames += job.draft as u64; let t = Timer::start(); stats.frames += 1; - if fh.seg.enabled && !fh.seg.update_map { - let prev = std::mem::take(&mut fs.prev_segment_ids); - fs.mi.segment_id.copy_from_slice(&prev); - } - let (mf_refs, mf_mvs) = motion_vector_storage(&fs); let rf = Arc::new(RefFrame { - buf: std::mem::take(&mut fs.cur), - cdfs, + // With worker threads the samples are in the published bands. + buf: if split { FrameBuf::default() } else { std::mem::take(&mut fs.cur) }, + bands: split.then(|| job.slot.bands.clone()), + num_planes: seq.color.num_planes, bit_depth: seq.color.bit_depth, subsampling_x: seq.color.subsampling_x, subsampling_y: seq.color.subsampling_y, - segment_ids: std::mem::take(&mut fs.mi.segment_id), - mi_cols: fs.mi.cols, - mi_rows: fs.mi.rows, color: (seq.color.color_primaries, seq.color.transfer_characteristics, seq.color.matrix_coefficients, seq.color.color_range), - saved_ref_frames: mf_refs, - saved_mvs: mf_mvs, }); // job.picture is None for a shown frame refused for its size (see tile_group) let pic = job.picture.is_some().then(|| { @@ -920,6 +1252,232 @@ fn decode_frame(job: &FrameJob, pool: &Pool, stats: &mut DecodeStats) -> Result< Ok((rf, pic)) } +/// Publish bands [from, to) copied from the final samples `src`, and the row progress they make +/// final. +fn publish_bands(slot: &FrameSlot, src: &FrameBuf, from: usize, to: usize, height: usize) { + let to = to.min(slot.bands.bands()); + for b in from..to { + slot.bands.publish(b, src); + } + slot.progress.publish((to << crate::av1::decode::frame::BAND_LOG2).min(height)); +} + +/// Publish the bands that became final (in order, following those published before) and the +/// row progress they make final. +fn publish_final(slot: &FrameSlot, bands: crate::av1::decode::postfilter::FinalBands, height: usize) { + let mut end = None; + for (b, planes) in bands { + slot.bands.publish_owned(b, planes); + end = Some(b + 1); + } + if let Some(end) = end { + slot.progress.publish((end << crate::av1::decode::frame::BAND_LOG2).min(height)); + } +} + +/// Extra luma rows waited for below what a motion vector reaches: the 8-tap filters read 4 +/// rows below a block, chroma positions round, and the vertical sub-sample phase adds one. +const ROW_WAIT_MARGIN: usize = 16; + +/// The rows of each reference slot that reconstructing the blocks `blocks` (a superblock row +/// and the row above it, whose motion overlapped block motion compensation can reuse) reads, +/// for blocks ending at luma row `bottom`: their reach plus a margin, or everything +/// (`usize::MAX`) for warped motion and scaled references, whose reach is not bounded by the +/// motion vector. +fn rows_needed( + fh: &FrameHeader, + ref_info: &[RefInfo; NUM_REF_FRAMES], + blocks: impl Iterator, + bottom: usize, +) -> [usize; NUM_REF_FRAMES] { + let mut need = [0usize; NUM_REF_FRAMES]; + let mut used = [false; NUM_REF_FRAMES]; + let mut warp = false; + let global = (LAST_FRAME..=ALTREF_FRAME).any(|r| fh.gm_type[r] as usize > TRANSLATION); + for b in blocks { + if !b.inter { + continue; + } + if b.motion_mode as usize == LOCALWARP || (global && (b.y_mode == GLOBALMV || b.y_mode == GLOBAL_GLOBALMV)) { + warp = true; + } + let bottom = bottom.max((b.mi_row + b.bh4) * 4); + for list in 0..2 { + let rf = b.ref_frame[list]; + if rf <= INTRA_FRAME as i8 { + continue; + } + let Some(&i) = fh.ref_frame_idx.get(rf as usize - LAST_FRAME) else { continue }; + used[i] = true; + let ri = &ref_info[i]; + if ri.upscaled_width != fh.frame_width || ri.frame_height != fh.frame_height { + need[i] = usize::MAX; + continue; + } + let down = (b.mv[list][0].max(0) as usize).div_ceil(8); + need[i] = need[i].max(bottom.saturating_add(down).saturating_add(ROW_WAIT_MARGIN)); + } + } + if warp { + for i in 0..NUM_REF_FRAMES { + if used[i] { + need[i] = usize::MAX; + } + } + } + need +} + +/// Reconstructed luma rows between two runs of the in-loop filter pipeline in +/// [`reconstruct_wavefront`] (finer steps publish rows sooner, coarser ones filter more bands in +/// parallel). +const FILTER_STEP_ROWS: usize = 128; + +/// Reconstruct a frame superblock row by superblock row (row wavefront): before each row, wait +/// only for the reference rows it reads; then reconstruct that row in every tile it crosses +/// (tiles share no prediction context, so the order of the tiles within a row does not matter) +/// and run the in-loop filters on what that allows, publishing the bands that became final. +/// `tiles` holds the whole frame's buffers. Returns the seconds spent waiting for references +/// and filtering (not tile decoding). +fn reconstruct_wavefront(job: &FrameJob, pool: &Pool, fs: &mut FrameState, tiles: &mut FrameTiles, stats: &mut DecodeStats) -> Result { + let fh = &job.fh; + let mut other = 0.0; + let ti = fh.tile_info.clone(); + let Some(state) = tiles.states.pop() else { + return Err(Error::Invalid("no tile to reconstruct")); + }; + // The record of every tile, by tile number. + let mut recs: Vec> = (0..ti.rows * ti.cols).map(|_| None).collect(); + for (num, rec) in std::mem::take(&mut tiles.records) { + *recs.get_mut(num).ok_or(Error::Invalid("tile number out of range"))? = Some(rec); + } + let mut used = [false; NUM_REF_FRAMES]; + if !fh.frame_is_intra { + for i in 0..REFS_PER_FRAME { + used[fh.ref_frame_idx[i]] = true; + } + } + for i in 0..NUM_REF_FRAMES { + if used[i] { + let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; + fs.ref_rows[i] = Some(slot.bands.clone()); + } + } + fs.put_tile_state(state); + let mut pipe = crate::av1::decode::postfilter::Pipeline::new(fs, job.draft); + let sb4 = if job.seq.use_128x128_superblock { 32 } else { 16 }; + let mi_rows = fh.mi_rows as usize; + let sb_rows = mi_rows.div_ceil(sb4); + let mut bufs = (crate::av1::decode::tile::Buffers::new(), Cdfs::new(fh.quant.base_q_idx)); + let mut filtered = 0; + let mut tile_row = 0; + for row in 0..sb_rows { + // Tile rows start on superblock rows. + while tile_row + 1 < ti.rows && row * sb4 >= ti.mi_row_starts[tile_row + 1] as usize { + tile_row += 1; + } + let local = (row * sb4).saturating_sub(ti.mi_row_starts[tile_row] as usize) / sb4; + let w = Timer::start(); + if !fh.frame_is_intra { + let mut need = [0usize; NUM_REF_FRAMES]; + for col in 0..ti.cols { + let Some(rec) = &recs[tile_row * ti.cols + col] else { continue }; + let width = (ti.mi_col_starts[col + 1] as usize).min(fh.mi_cols as usize).saturating_sub(ti.mi_col_starts[col] as usize); + let n = rows_needed(fh, &job.ref_info, rec.row_blocks(width.div_ceil(sb4), local.saturating_sub(1), local + 1), (row + 1) * sb4 * 4); + for i in 0..NUM_REF_FRAMES { + need[i] = need[i].max(n[i]); + } + } + for i in 0..NUM_REF_FRAMES { + if used[i] && need[i] > 0 { + let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; + slot.progress.wait(need[i].min(job.ref_info[i].frame_height as usize))?; + } + } + } + other += w.secs(); + let mut t = fs.take_tile_state(); + let mut res = Ok(()); + for col in 0..ti.cols { + let Some(rec) = recs[tile_row * ti.cols + col].as_mut() else { continue }; + let (b, cdf) = bufs; + let mut td = TileDecoder::with_buffers(&fs.sh, &mut t, &[], cdf, tile_row, col, b); + td.pass = Pass::Reconstruct; + td.record = std::mem::take(rec); + res = td.reconstruct_sb_rows(1); + *rec = std::mem::take(&mut td.record); + bufs = td.into_buffers(); + if res.is_err() { + break; + } + } + fs.put_tile_state(t); + res?; + let f = Timer::start(); + let recon_rows = ((row + 1) * sb4 * 4).min(mi_rows * 4); + // Filter in steps of a few bands, so the bands of a step run in parallel. + if recon_rows >= filtered + FILTER_STEP_ROWS || recon_rows == mi_rows * 4 { + filtered = recon_rows; + let done = pipe.advance(fs, stats, pool, recon_rows); + publish_final(&job.slot, done, fh.frame_height as usize); + } + other += f.secs(); + #[cfg(test)] + assert!( + !(job.panic_at_row.is_some_and(|r| row >= r) && job.slot.progress.rows() > 0), + "forced panic in the row wavefront after publishing rows" + ); + } + // A reference that failed after publishing the rows used here still fails this frame, as + // it does when frames are decoded one after another. + for i in 0..NUM_REF_FRAMES { + if used[i] { + let slot = job.refs[i].as_ref().ok_or(Error::Invalid("missing reference frame"))?; + slot.wait()?; + } + } + Ok(other) +} + +/// Most bytes a tile's parse record may take per luma sample of the tile. Typical streams need +/// a fraction of this; past it (pathological streams: tiny blocks, large palettes, every +/// coefficient coded) the frame is decoded in one pass instead, so memory stays bounded. +const RECORD_BYTES_PER_SAMPLE: usize = 32; + +/// Most parse record bytes all frames of a decoder hold at once. Past it, the frames that +/// cannot reserve more are decoded in one pass, so the frames in flight together stay bounded +/// too (typical streams need a few bytes per luma sample: under 100 MB for a 2160p frame). +const RECORD_BUDGET_BYTES: usize = 1 << 30; + +/// The tiles of a frame between parsing and reconstruction. +struct FrameTiles { + /// One state per tile (decoded in parallel into private buffers), or one for the whole + /// frame (tiles one after another). + states: Vec, + /// Per tile: (tile number, record of [`Pass::Parse`]). + records: Vec<(usize, TileRecord)>, + /// The final CDFs of tile context_update_tile_id. + saved: Option>, + /// `states` holds the frame's own buffers (taken with [`FrameState::take_tile_state`]). + whole: bool, +} + +impl FrameTiles { + /// Move the tile buffers (mode info, filter parameters and, after reconstruction, the + /// samples) into the frame. + fn finish(self, fs: &mut FrameState) { + if self.whole { + if let Some(t) = self.states.into_iter().next() { + fs.put_tile_state(t); + } + } else { + for t in &self.states { + fs.merge_tile_state(t); + } + } + } +} + /// Bytes all parallel tile states may hold of whole-frame CDEF and restoration metadata; past it /// the tiles decode one after another on the frame's own metadata. const TILE_METADATA_BUDGET: usize = 256 << 20; @@ -928,16 +1486,30 @@ fn tile_metadata_fits(tiles: usize, per_tile: usize) -> bool { tiles.checked_mul(per_tile).is_some_and(|n| n <= TILE_METADATA_BUDGET) } -/// Decode the tiles of a frame (tile number, data), in parallel when several; returns the final -/// CDFs of tile context_update_tile_id. -fn decode_tiles(pool: &Pool, fs: &mut FrameState, cdfs: &Cdfs, tiles: &[(usize, Vec)]) -> Result>> { +/// Run `pass` (Full or Parse) over the tiles of a frame. +/// `record_limit`: most record bytes per luma sample of a tile (see [`RECORD_BYTES_PER_SAMPLE`]). +/// `budget`: the decoder-wide budget the records are reserved from. +fn parse_tiles( + pool: &Pool, + fs: &mut FrameState, + cdfs: &Cdfs, + tiles: &[(usize, Vec)], + pass: Pass, + record_limit: usize, + budget: &Arc, +) -> Result { let ti = fs.fh.tile_info.clone(); let ctx_tile = ti.context_update_tile_id as usize; - let run = |sh: &FrameShared, t: &mut TileState, num: usize, data: &[u8]| -> Result>> { + let run = |sh: &FrameShared, t: &mut TileState, num: usize, data: &[u8]| -> Result<(Option>, TileRecord)> { t.intra_frame_y_mode_cdf = DEFAULT_INTRA_FRAME_Y_MODE_CDF; let mut td = TileDecoder::new(sh, t, data, Box::new(cdfs.clone()), num / ti.cols, num % ti.cols); + td.pass = pass; + let (rows, cols) = (td.mi_row_end.saturating_sub(td.mi_row_start), td.mi_col_end.saturating_sub(td.mi_col_start)); + td.record.limit = rows.saturating_mul(cols).saturating_mul(16 * record_limit); + td.record.budget = Some(budget.clone()); td.decode_tile()?; - Ok((num == ctx_tile).then_some(td.cdf)) + let rec = std::mem::take(&mut td.record); + Ok(((num == ctx_tile).then_some(td.cdf), rec)) }; // Intra block copy reads the current frame through the tile's own buffer: keep whole-frame // buffers (tiles one after another) for it. @@ -953,42 +1525,75 @@ fn decode_tiles(pool: &Pool, fs: &mut FrameState, cdfs: &Cdfs, tiles: &[(usize, .collect(); let sh = &fs.sh; let results = pool.map_mut(&mut states, |i, t| run(sh, t, tiles[i].0, &tiles[i].1)); - for t in &states { - fs.merge_tile_state(t); - } let mut saved = None; - for r in results { - if let Some(c) = r? { - saved = Some(c); + let mut records = Vec::with_capacity(tiles.len()); + for (r, (num, _)) in results.into_iter().zip(tiles) { + let (c, rec) = r?; + if c.is_some() { + saved = c; } + records.push((*num, rec)); } - return Ok(saved); + return Ok(FrameTiles { states, records, saved, whole: false }); } let mut t = fs.take_tile_state(); let mut saved = None; + let mut records = Vec::with_capacity(tiles.len()); let mut res = Ok(()); for (num, data) in tiles { match run(&fs.sh, &mut t, *num, data) { - Ok(Some(c)) => saved = Some(c), - Ok(None) => {} + Ok((c, rec)) => { + if c.is_some() { + saved = c; + } + records.push((*num, rec)); + } Err(e) => { res = Err(e); break; } } } - fs.put_tile_state(t); - res.map(|_| saved) + if let Err(e) = res { + fs.put_tile_state(t); + return Err(e); + } + Ok(FrameTiles { states: vec![t], records, saved, whole: true }) +} + +/// Reconstruct parsed tiles (in parallel when each has its own state). +fn reconstruct_tiles(pool: &Pool, fs: &FrameShared, tiles: &mut FrameTiles) -> Result<()> { + let ti = fs.fh.tile_info.clone(); + let run = |t: &mut TileState, num: usize, rec: TileRecord| -> Result<()> { + let mut td = TileDecoder::new(fs, t, &[], Cdfs::new(fs.fh.quant.base_q_idx), num / ti.cols, num % ti.cols); + td.pass = Pass::Reconstruct; + td.record = rec; + td.decode_tile() + }; + let records = std::mem::take(&mut tiles.records); + if !tiles.whole { + let mut items: Vec<(&mut TileState, Option<(usize, TileRecord)>)> = tiles.states.iter_mut().zip(records.into_iter().map(Some)).collect(); + let results = pool.map_mut(&mut items, |_, (t, rec)| match rec.take() { + Some((num, rec)) => run(t, num, rec), + None => Ok(()), + }); + return results.into_iter().collect(); + } + let Some(t) = tiles.states.first_mut() else { return Ok(()) }; + for (num, rec) in records { + run(t, num, rec)?; + } + Ok(()) } fn output_picture(seq: &SequenceHeader, rf: &RefFrame, w: usize, h: usize, grain: &FilmGrainParams, apply_film_grain: bool, pts: i64) -> Picture { let ssx = rf.subsampling_x as usize; let ssy = rf.subsampling_y as usize; - let num_planes = rf.buf.num_planes; + let num_planes = rf.num_planes; let mut planes: [Vec; 3] = Default::default(); for p in 0..num_planes { let (pw, ph) = if p == 0 { (w, h) } else { ((w + ssx) >> ssx, (h + ssy) >> ssy) }; - let src = &rf.buf.planes[p]; + let src = rf.plane(p); let mut v = Vec::with_capacity(pw * ph); for y in 0..ph { v.extend_from_slice(&src.row(y)[..pw]); @@ -1016,25 +1621,40 @@ fn output_picture(seq: &SequenceHeader, rf: &RefFrame, w: usize, h: usize, grain pic } -/// Motion field motion vector storage process (7.19): (MfRefFrames, MfMvs). +/// What later frames read from this one while parsing: the final CDFs, the segment ids (the +/// previous frame's when the map is not updated) and the stored motion vectors (7.19). The +/// in-loop filters read the decoded segment ids, so the frame's own map is left alone. +fn parse_results(fs: &FrameState, cdfs: Box) -> Parsed { + let fh = &fs.fh; + let segment_ids = if fh.seg.enabled && !fh.seg.update_map { fs.prev_segment_ids.clone() } else { fs.mi.segment_id.clone() }; + let (saved_ref_frames, saved_mvs) = motion_vector_storage(fs); + Parsed { cdfs, segment_ids, mi_cols: fs.mi.cols, mi_rows: fs.mi.rows, saved_ref_frames, saved_mvs } +} + +/// Motion field motion vector storage process (7.19): (MfRefFrames, MfMvs), kept only where +/// the projection process (7.9.2) reads them, the odd row and column of every 8x8 block +/// (index `y8 * (MiCols >> 1) + x8`). fn motion_vector_storage(fs: &FrameState) -> (Vec, Vec<[i32; 2]>) { let mi = &fs.mi; - let n = mi.rows * mi.cols; - let mut refs = vec![-1i8; n]; - let mut mvs = vec![[0i32; 2]; n]; let fh = &fs.fh; - for i in 0..n { - for list in 0..2 { - let r = mi.ref_frame[i][list]; - if r > INTRA_FRAME as i8 { - let ref_idx = fh.ref_frame_idx[r as usize - LAST_FRAME]; - let dist = crate::av1::decode::header::relative_dist(&fs.seq, fs.ref_info[ref_idx].order_hint, fh.order_hint); - if dist < 0 { - let m = mi.mv[i][list]; - let (row, col) = (m.row as i32, m.col as i32); - if row.abs() <= REFMVS_LIMIT as i32 && col.abs() <= REFMVS_LIMIT as i32 { - refs[i] = r; - mvs[i] = [row, col]; + let (mi_cols, w8, h8) = (fh.mi_cols as usize, fh.mi_cols as usize >> 1, fh.mi_rows as usize >> 1); + let mut refs = vec![-1i8; w8 * h8]; + let mut mvs = vec![[0i32; 2]; w8 * h8]; + for y8 in 0..h8 { + for x8 in 0..w8 { + let (i, o) = ((2 * y8 + 1) * mi_cols + 2 * x8 + 1, y8 * w8 + x8); + for list in 0..2 { + let r = mi.ref_frame[i][list]; + if r > INTRA_FRAME as i8 { + let ref_idx = fh.ref_frame_idx[r as usize - LAST_FRAME]; + let dist = crate::av1::decode::header::relative_dist(&fs.seq, fs.ref_info[ref_idx].order_hint, fh.order_hint); + if dist < 0 { + let m = mi.mv[i][list]; + let (row, col) = (m.row as i32, m.col as i32); + if row.abs() <= REFMVS_LIMIT as i32 && col.abs() <= REFMVS_LIMIT as i32 { + refs[o] = r; + mvs[o] = [row, col]; + } } } } @@ -1092,7 +1712,7 @@ fn project_pos(v8: i32, delta: i32, dst_sign: i32, max8: i32, max_off8: i32) -> /// Projection process (7.9.2). fn project(fs: &mut FrameState, src: usize, dst_sign: i32) -> bool { - let FrameShared { seq, fh, refs, ref_info, motion_field, .. } = &mut fs.sh; + let FrameShared { seq, fh, parsed, ref_info, motion_field, .. } = &mut fs.sh; let src_idx = fh.ref_frame_idx[src - LAST_FRAME]; let w8 = fh.mi_cols as i32 >> 1; let h8 = fh.mi_rows as i32 >> 1; @@ -1100,20 +1720,18 @@ fn project(fs: &mut FrameState, src: usize, dst_sign: i32) -> bool { if ri.mi_rows != fh.mi_rows || ri.mi_cols != fh.mi_cols || ri.frame_type == INTRA_ONLY_FRAME as u8 || ri.frame_type == KEY_FRAME as u8 { return false; } - let Some(rf) = &refs[src_idx] else { return false }; + let Some(rf) = &parsed[src_idx] else { return false }; let rd = |a: u32, b: u32| crate::av1::decode::header::relative_dist(seq, a, b); - let mi_cols = fh.mi_cols as usize; for y8 in 0..h8 { for x8 in 0..w8 { - let row = (2 * y8 + 1) as usize; - let col = (2 * x8 + 1) as usize; - let src_ref = rf.saved_ref_frames[row * mi_cols + col]; + let at = (y8 * w8 + x8) as usize; + let src_ref = rf.saved_ref_frames[at]; if src_ref > INTRA_FRAME as i8 { let ref_to_cur = rd(fh.order_hints[src], fh.order_hint); let ref_offset = rd(fh.order_hints[src], ri.saved_order_hints[src_ref as usize]); let pos_valid = ref_to_cur.abs() <= MAX_FRAME_DISTANCE as i32 && ref_offset.abs() <= MAX_FRAME_DISTANCE as i32 && ref_offset > 0; if pos_valid { - let mv = rf.saved_mvs[row * mi_cols + col]; + let mv = rf.saved_mvs[at]; let proj = get_mv_projection(mv, ref_to_cur * dst_sign, ref_offset); let py = project_pos(y8, proj[0], dst_sign, h8, MAX_OFFSET_HEIGHT as i32); let px = project_pos(x8, proj[1], dst_sign, w8, MAX_OFFSET_WIDTH as i32); @@ -1145,13 +1763,18 @@ mod tests { ref_info: Default::default(), tiles: Vec::new(), tile_covered: vec![false; 3], - slot: Slot::new(), + slot: FrameSlot::new(BandFrame::new(0, 0, 1, 0, 0)), picture: None, pts: 0, apply_film_grain: false, draft: false, #[cfg(test)] force_panic: false, + #[cfg(test)] + panic_at_row: None, + #[cfg(test)] + record_limit: None, + record_budget: decoder.record_budget.clone(), })); decoder } @@ -1298,6 +1921,317 @@ mod tests { assert!(frame_size_allowed(1, 1)); } + /// Random short sleeps in row progress publishing and waiting, to shake out scheduling + /// orders a quiet machine never produces (enabled by [`JitterGuard`]). + static JITTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + + pub(super) fn jitter() { + use std::cell::Cell; + use std::sync::atomic::Ordering; + if JITTER.load(Ordering::Relaxed) == 0 { + return; + } + static SEED: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0x2545_f491_4f6c_dd1d); + thread_local!(static RNG: Cell = Cell::new(SEED.fetch_add(0x9e37_79b9_7f4a_7c15, Ordering::Relaxed) | 1)); + let r = RNG.with(|c| { + let mut x = c.get(); + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + c.set(x); + x + }); + match r % 4 { + 0 => std::thread::yield_now(), + 1 => std::thread::sleep(std::time::Duration::from_micros(r % 300)), + _ => {} + } + } + + struct JitterGuard; + impl JitterGuard { + fn on() -> JitterGuard { + JITTER.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + JitterGuard + } + } + impl Drop for JitterGuard { + fn drop(&mut self) { + JITTER.fetch_sub(1, std::sync::atomic::Ordering::SeqCst); + } + } + + /// Inter test streams made with ffmpeg / SVT-AV1 (None, reported, without them): several + /// superblock rows and hierarchical references; odd size; super-resolution; 2x2 tiles. + const STREAMS: [(&str, &str, &str); 4] = [ + ("wavefront_unit_352x288", "testsrc2=size=352x288:rate=25", "-preset 8 -crf 35"), + ("wavefront_unit_odd", "testsrc2=size=203x117:rate=25,noise=alls=15:allf=t", "-preset 8 -crf 40"), + ("wavefront_unit_superres", "testsrc2=size=416x240:rate=25", "-preset 8 -crf 35 -svtav1-params superres-mode=1:superres-denom=12"), + ("wavefront_unit_tiles", "mandelbrot=size=650x370:rate=25", "-preset 8 -crf 32 -svtav1-params tile-columns=1:tile-rows=1"), + ]; + + fn inter_stream((name, src, args): (&str, &str, &str)) -> Option>> { + let ff = craft_media_testkit::ffmpeg_or_skip("av1 wavefront")?; + if !craft_media_testkit::ffmpeg_has_encoder(&ff, "libsvtav1") { + craft_media_testkit::skip_test("av1 wavefront", "this ffmpeg has no libsvtav1 encoder"); + return None; + } + let out = craft_media_testkit::fixtures_dir("av1").join(format!("{name}.ivf")); + let path = craft_media_testkit::fixtures::generate(&out, |tmp| { + std::process::Command::new(&ff) + .args([ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + src, + "-frames:v", + "16", + "-pix_fmt", + "yuv420p", + "-c:v", + "libsvtav1", + ]) + .args(args.split_whitespace()) + .args(["-f", "ivf"]) + .arg(tmp) + .stdin(std::process::Stdio::null()) + .status() + .is_ok_and(|s| s.success()) + }); + let path = path.unwrap_or_else(|| panic!("ffmpeg has libsvtav1 but could not encode {name}")); + let d = std::fs::read(&path).unwrap_or_else(|e| panic!("read {}: {e}", path.display())); + let mut pos = usize::from(u16::from_le_bytes([d[6], d[7]])); + let mut out = Vec::new(); + while let Some(h) = d.get(pos..pos + 12) { + let n = u32::from_le_bytes([h[0], h[1], h[2], h[3]]) as usize; + pos += 12; + out.push(d[pos..pos + n].to_vec()); + pos += n; + } + assert_eq!(out.len(), 16, "{name}: temporal units"); + Some(out) + } + + /// Decode `tus` with `dec`: (pictures, errors). + fn decode_tus(mut dec: Decoder, tus: &[Vec]) -> (Vec, usize) { + let (mut pics, mut errors) = (Vec::new(), 0); + for (i, tu) in tus.iter().enumerate() { + match dec.decode_pts(tu, i as i64) { + Ok(p) => pics.extend(p), + Err(_) => errors += 1, + } + } + match dec.flush_result() { + Ok(p) => pics.extend(p), + Err(_) => errors += 1, + } + (pics, errors) + } + + #[test] + fn jittered_wavefront_gives_the_single_thread_pictures() { + for stream in STREAMS { + let Some(tus) = inter_stream(stream) else { return }; + let name = stream.0; + let (want, errors) = decode_tus(Decoder::with_threads(1), &tus); + assert_eq!(errors, 0, "{name}"); + let _jitter = JitterGuard::on(); + for threads in [2, 3, 5, 8] { + for round in 0..2 { + let tus = tus.clone(); + let (got, errors) = within_a_minute(move || decode_tus(Decoder::with_threads(threads), &tus)); + assert_eq!(errors, 0, "{name}: threads {threads} round {round}"); + assert!(got == want, "{name}: threads {threads} round {round}: pictures differ"); + } + } + } + } + + #[test] + fn frames_over_the_record_limit_decode_in_one_pass() { + for stream in [STREAMS[0], STREAMS[3]] { + let Some(tus) = inter_stream(stream) else { return }; + let name = stream.0; + let (want, _) = decode_tus(Decoder::with_threads(1), &tus); + for (threads, limit) in [(3, 0), (8, 0), (8, 1)] { + let tus = tus.clone(); + let (got, errors) = within_a_minute(move || { + let mut dec = Decoder::with_threads(threads); + dec.record_limit = Some(limit); + decode_tus(dec, &tus) + }); + assert_eq!(errors, 0, "{name}: threads {threads}, limit {limit}"); + assert!(got == want, "{name}: threads {threads}, limit {limit}: pictures differ"); + } + } + } + + #[test] + fn the_parse_records_of_all_frames_stay_within_the_decoder_budget() { + for stream in [STREAMS[0], STREAMS[3]] { + let Some(tus) = inter_stream(stream) else { return }; + let name = stream.0; + let (want, _) = decode_tus(Decoder::with_threads(1), &tus); + // Room for about one frame's records: the other frames in flight decode in one pass. + for (threads, budget) in [(3, 0), (8, 3 << 20), (16, 1 << 20)] { + let tus = tus.clone(); + let b = Arc::new(crate::av1::decode::tile::RecordBudget::new(budget)); + let mut dec = Decoder::with_threads(threads); + dec.record_budget = b.clone(); + let (got, errors) = within_a_minute(move || decode_tus(dec, &tus)); + assert_eq!(errors, 0, "{name}: threads {threads}, budget {budget}"); + assert!(got == want, "{name}: threads {threads}, budget {budget}: pictures differ"); + assert!(b.peak.load(std::sync::atomic::Ordering::Acquire) <= budget, "{name}: threads {threads}: over the budget"); + assert_eq!(b.used(), 0, "{name}: threads {threads}: records not released"); + } + } + } + + #[cfg(all(feature = "threads", not(target_arch = "wasm32")))] + #[test] + fn a_reference_panicking_after_publishing_rows_fails_without_a_hang() { + let Some(tus) = inter_stream(STREAMS[0]) else { return }; + let _jitter = JitterGuard::on(); + // Frame 1 is the first inter frame; later frames predict from it. + for threads in [2, 3, 8] { + for frame in [0, 1, 2] { + let tus = tus.clone(); + let (pics, errors) = within_a_minute(move || { + let mut dec = Decoder::with_threads(threads); + dec.panic_at_row = Some((frame, 1)); + decode_tus(dec, &tus) + }); + assert!(errors > 0, "threads {threads}, frame {frame}: the panic was not reported ({} pictures)", pics.len()); + } + } + } + + #[test] + fn dropping_a_busy_decoder_does_not_hang() { + let Some(tus) = inter_stream(STREAMS[0]) else { return }; + for threads in [2, 3, 8, 16] { + for cut in [1, 3, tus.len() / 2] { + let tus = tus.clone(); + within_a_minute(move || { + let mut dec = Decoder::with_threads(threads); + for (i, tu) in tus.iter().take(cut).enumerate() { + let _ = dec.decode_pts(tu, i as i64); + } + drop(dec); + }); + } + } + } + + #[test] + fn a_flush_mid_stream_keeps_the_pictures() { + let Some(tus) = inter_stream(STREAMS[0]) else { return }; + let (want, _) = decode_tus(Decoder::with_threads(1), &tus); + for threads in [2, 8] { + for cut in [2, 5, tus.len() / 2] { + let tus = tus.clone(); + let got = within_a_minute(move || { + let mut dec = Decoder::with_threads(threads); + let mut pics = Vec::new(); + for (i, tu) in tus.iter().enumerate() { + if i == cut { + pics.extend(dec.flush_result().expect("flush")); + } + pics.extend(dec.decode_pts(tu, i as i64).expect("decode")); + } + pics.extend(dec.flush_result().expect("flush")); + pics + }); + assert!(got == want, "threads {threads}, flush before unit {cut}"); + } + } + } + + #[test] + fn progress_wakes_every_waiter_with_the_rows_or_the_failure() { + let p = Arc::new(Progress::new()); + let waiters: Vec<_> = (1..=8) + .map(|k| { + let p = p.clone(); + std::thread::spawn(move || p.wait(k * 10)) + }) + .collect(); + p.publish(40); + p.publish(20); // never goes back + assert_eq!(p.wait(40), Ok(())); + p.fail(); + let res: Vec<_> = waiters.into_iter().map(|w| w.join().expect("waiter")).collect(); + assert!(res[..4].iter().all(|r| r.is_ok()), "{res:?}"); + assert!(res[4..].iter().all(|r| r.is_err()), "{res:?}"); + assert_eq!(p.wait(40), Ok(()), "rows published before a failure stay readable"); + } + + #[test] + fn a_failed_frame_keeps_its_published_parse_results() { + let slot = FrameSlot::new(BandFrame::new(16, 16, 1, 0, 0)); + let parsed = Parsed { cdfs: Cdfs::new(0), segment_ids: vec![], mi_cols: 4, mi_rows: 4, saved_ref_frames: vec![], saved_mvs: vec![] }; + slot.set_parsed(Ok(Arc::new(parsed))); + slot.finish(Err(Error::Invalid("x")), 16); + assert!(slot.wait_parsed().is_ok()); + assert!(slot.wait().is_err()); + assert!(slot.progress.wait(1).is_err()); + // A frame failing before its parse results: every waiter gets the error. + let slot = FrameSlot::new(BandFrame::new(16, 16, 1, 0, 0)); + slot.finish(Err(Error::Invalid("y")), 16); + assert!(slot.wait_parsed().is_err()); + } + + fn motion(mi_row: usize, bh4: usize, ref_frame: [i8; 2], mv: [[i32; 2]; 2]) -> crate::av1::decode::tile::BlockMotion { + crate::av1::decode::tile::BlockMotion { + mi_row, + bh4, + inter: true, + ref_frame, + mv, + motion_mode: 0, + y_mode: crate::av1::decode::spec_tables::NEARESTMV, + } + } + + #[test] + fn rows_needed_follow_the_motion_vectors() { + let mut fh = FrameHeader { frame_width: 64, upscaled_width: 64, frame_height: 256, ..Default::default() }; + fh.ref_frame_idx = [0, 1, 2, 3, 4, 5, 6]; + let mut ri: [RefInfo; NUM_REF_FRAMES] = Default::default(); + for r in ri.iter_mut() { + (r.upscaled_width, r.frame_height) = (64, 256); + } + let none = [[0; 2]; 2]; + // 41/8 rows down rounds up to 6; upward motion needs only the block's own rows. + let need = rows_needed(&fh, &ri, [motion(0, 16, [1, -1], [[41, 0], [0, 0]]), motion(0, 16, [2, 3], [[-80, 0], [7, 0]])].into_iter(), 64); + assert_eq!(need[0], 64 + 6 + ROW_WAIT_MARGIN); + assert_eq!(need[1], 64 + ROW_WAIT_MARGIN); + assert_eq!(need[2], 64 + 1 + ROW_WAIT_MARGIN); + assert_eq!(need[3], 0); + // Intra blocks and intra block copy read no reference. + let intra = crate::av1::decode::tile::BlockMotion { inter: false, ..motion(0, 16, [1, -1], none) }; + assert_eq!(rows_needed(&fh, &ri, [intra].into_iter(), 64), [0; NUM_REF_FRAMES]); + // Blocks reaching below the superblock row count from their own bottom. + assert_eq!(rows_needed(&fh, &ri, [motion(12, 8, [1, -1], none)].into_iter(), 64)[0], 80 + ROW_WAIT_MARGIN); + // Local warp: the whole of every reference the rows use. + let warp = + crate::av1::decode::tile::BlockMotion { motion_mode: crate::av1::decode::spec_tables::LOCALWARP as u8, ..motion(0, 16, [1, -1], none) }; + let need = rows_needed(&fh, &ri, [warp, motion(0, 16, [2, -1], none)].into_iter(), 64); + assert_eq!((need[0], need[1], need[2]), (usize::MAX, usize::MAX, 0)); + // Global motion beyond translation for GLOBALMV blocks. + fh.gm_type[3] = (TRANSLATION + 1) as u8; + let global = crate::av1::decode::tile::BlockMotion { y_mode: GLOBALMV, ..motion(0, 16, [1, -1], none) }; + assert_eq!(rows_needed(&fh, &ri, [global].into_iter(), 64)[0], usize::MAX); + fh.gm_type[3] = 0; + // A scaled reference. + ri[0].frame_height = 128; + assert_eq!(rows_needed(&fh, &ri, [motion(0, 16, [1, -1], none)].into_iter(), 64)[0], usize::MAX); + } + #[test] fn frames_beyond_the_limits_are_refused() { assert!(!frame_size_allowed(16384, 2177)); // one row over MaxPicSize diff --git a/crates/video/src/av1/decode/frame.rs b/crates/video/src/av1/decode/frame.rs index d0432e2a8..e28ca89e9 100644 --- a/crates/video/src/av1/decode/frame.rs +++ b/crates/video/src/av1/decode/frame.rs @@ -197,6 +197,50 @@ impl FrameBuf { } } + /// Copy luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows) of `src`, a + /// frame of the same geometry. + pub fn copy_rows_from(&mut self, src: &FrameBuf, y0: usize, y1: usize) { + for p in 0..self.num_planes { + let sy = if p == 0 { 0 } else { self.subsampling_y }; + let (d, s) = (&mut self.planes[p], &src.planes[p]); + if d.stride != s.stride { + continue; + } + let a = (y0 >> sy).min(d.rows).min(s.rows) * d.stride; + let b = (y1 >> sy).min(d.rows).min(s.rows) * d.stride; + if let (Some(dst), Some(src)) = (d.data.get_mut(a..b), s.data.get(a..b)) { + dst.copy_from_slice(src); + } + } + } + + /// Copies of luma rows [y0, y1) (chroma rows scaled down; clipped to the stored rows), one + /// plane region each. + pub fn rows_copy(&self, y0: usize, y1: usize) -> [Plane; 3] { + std::array::from_fn(|p| { + if p >= self.num_planes { + return Plane::default(); + } + let sy = if p == 0 { 0 } else { self.subsampling_y }; + let s = &self.planes[p]; + let (a, b) = ((y0 >> sy).min(s.rows), (y1 >> sy).min(s.rows)); + let mut out = Plane::region(0, a, s.stride, b - a); + if let Some(src) = s.data.get(a * s.stride..b * s.stride) { + out.data.copy_from_slice(src); + } + out + }) + } + + /// A frame of this geometry (samples not copied: zero). + pub fn same_geometry(&self) -> FrameBuf { + let planes = std::array::from_fn(|p| { + let pl = &self.planes[p]; + Plane { data: zeroed(pl.data.len()), stride: pl.stride, rows: pl.rows, ox: pl.ox, oy: pl.oy } + }); + FrameBuf { planes, ..*self } + } + /// [`Plane::row_bands`] of all three planes for ranges of luma rows (chroma rows scaled /// down): element `i` holds the bands of range `i`. pub fn row_bands(&mut self, luma: &[(usize, usize)]) -> Vec<[RowBand<'_>; 3]> { @@ -208,6 +252,128 @@ impl FrameBuf { } } +/// Luma rows per published band of a [`BandFrame`] (log2). +pub const BAND_LOG2: u32 = 6; + +/// One plane of a frame's final samples, published band by band while the rest of the frame +/// is still being decoded (see [`BandFrame`]). A band is written once and then only read. +pub struct BandPlane { + bands: Vec>, + stride: usize, + rows: usize, + log2: u32, + /// Returned for rows of a band that is not published: zeros instead of a panic. Readers wait + /// for the row progress first and references of another chroma format are refused, so + /// this only happens through a scheduling bug. + missing: Box<[u16]>, +} + +impl BandPlane { + fn new(stride: usize, rows: usize, log2: u32) -> BandPlane { + let n = rows.div_ceil(1 << log2); + BandPlane { bands: (0..n).map(|_| std::sync::OnceLock::new()).collect(), stride, rows, log2, missing: vec![0; stride].into_boxed_slice() } + } + + /// Row `y` (`stride` samples) of a published band. + #[inline(always)] + pub fn row(&self, y: usize) -> &[u16] { + let o = (y & ((1 << self.log2) - 1)) * self.stride; + match self.bands.get(y >> self.log2).and_then(|b| b.get()).and_then(|b| b.data.get(o..o + self.stride)) { + Some(r) => r, + None => { + debug_assert!(false, "row {y} read before its band was published"); + &self.missing + } + } + } + + /// Publish band `band` copied from the rows of the whole plane `src` (same geometry); a + /// band is published once. + fn publish(&self, band: usize, src: &Plane) { + let y0 = band << self.log2; + let y1 = ((band + 1) << self.log2).min(self.rows); + let mut p = Plane::region(0, y0, self.stride, y1.saturating_sub(y0)); + let w = self.stride.min(src.stride); + for (y, out) in (y0..y1).zip(p.data.chunks_exact_mut(self.stride.max(1))) { + if let Some(r) = src.data.get(y * src.stride..y * src.stride + w) { + out[..w].copy_from_slice(r); + } + } + self.publish_owned(band, p); + } + + /// Publish band `band` as `p`, whose rows start at the band's first row (stride `stride`; + /// rows past the band are ignored). + fn publish_owned(&self, band: usize, p: Plane) { + if let Some(slot) = self.bands.get(band) + && p.stride == self.stride + { + let _ = slot.set(p); + } + } +} + +/// A frame's final samples (every in-loop filter applied) in bands of 64 luma rows, so later +/// frames can predict from the top of a reference while its bottom is still being decoded. +/// Each band is published once, before the frame's row progress says it is final. +pub struct BandFrame { + pub planes: [BandPlane; 3], + pub num_planes: usize, + pub subsampling_x: usize, + pub subsampling_y: usize, +} + +impl BandFrame { + /// Bands for the visible rows of a `width` x `height` frame stored like [`FrameBuf::new`]. + pub fn new(width: usize, height: usize, num_planes: usize, subsampling_x: usize, subsampling_y: usize) -> BandFrame { + let aw = width.div_ceil(128) * 128 + 160; + let planes = std::array::from_fn(|p| { + if p >= num_planes { + return BandPlane::new(0, 0, BAND_LOG2); + } + let (sx, sy) = if p == 0 { (0, 0) } else { (subsampling_x, subsampling_y) }; + BandPlane::new(aw >> sx, (height + sy) >> sy, BAND_LOG2 - sy as u32) + }); + BandFrame { planes, num_planes, subsampling_x, subsampling_y } + } + + /// Number of bands (of 64 luma rows). + pub fn bands(&self) -> usize { + self.planes[0].bands.len() + } + + /// Publish luma band `band` (and the chroma rows it covers) copied from `src`. + pub fn publish(&self, band: usize, src: &FrameBuf) { + for p in 0..self.num_planes { + self.planes[p].publish(band, &src.planes[p]); + } + } + + /// Publish luma band `band` from per-plane buffers starting at the band's first row. + pub fn publish_owned(&self, band: usize, planes: [Plane; 3]) { + for (p, pl) in planes.into_iter().enumerate().take(self.num_planes) { + self.planes[p].publish_owned(band, pl); + } + } +} + +/// A plane to predict from: a whole stored plane or the published bands of a reference. +#[derive(Clone, Copy)] +pub enum PlaneRef<'a> { + Whole(&'a Plane), + Bands(&'a BandPlane), +} + +impl<'a> PlaneRef<'a> { + #[inline(always)] + pub fn row(&self, y: usize) -> &'a [u16] { + match *self { + PlaneRef::Whole(p) => p.row(y), + PlaneRef::Bands(b) => b.row(y), + } + } +} + /// A motion vector (row, col) in 1/8 sample units. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Mv { diff --git a/crates/video/src/av1/decode/inter.rs b/crates/video/src/av1/decode/inter.rs index 8b40717a3..8d80e96b0 100644 --- a/crates/video/src/av1/decode/inter.rs +++ b/crates/video/src/av1/decode/inter.rs @@ -3,7 +3,7 @@ use std::sync::OnceLock; -use crate::av1::decode::frame::Plane; +use crate::av1::decode::frame::{Plane, PlaneRef}; use crate::av1::decode::mvpred::round2_signed64; use crate::av1::decode::spec_tables::{ BLOCK_8X8, BLOCK_SIZES, COMPOUND_AVERAGE, COMPOUND_DIFFWTD, COMPOUND_DISTANCE, COMPOUND_INTRA, COMPOUND_WEDGE, DIV_LUT, DIV_LUT_BITS, @@ -227,12 +227,9 @@ impl TileDecoder<'_, '_> { (start_x as i32, start_y as i32, step_x as i32, step_y as i32) } - fn ref_plane(&self, ref_idx: i32, plane: usize) -> &Plane { + fn ref_plane(&self, ref_idx: i32, plane: usize) -> PlaneRef<'_> { // A missing reference (rejected earlier for valid streams) predicts from the current frame. - match usize::try_from(ref_idx).ok().and_then(|i| self.fs.refs.get(i)).and_then(|r| r.as_ref()) { - Some(r) => &r.buf.planes[plane], - None => &self.t.cur.planes[plane], - } + self.fs.ref_samples(ref_idx, plane).unwrap_or(PlaneRef::Whole(&self.t.cur.planes[plane])) } /// Block inter prediction process (7.11.3.4) into `pred` (stride PRED_STRIDE); `tmp` holds @@ -512,12 +509,12 @@ impl TileDecoder<'_, '_> { return false; } let fs = self.fs; - let Some(r) = usize::try_from(ref_idx).ok().and_then(|i| fs.refs.get(i)).and_then(|r| r.as_ref()) else { + let Some(refp) = fs.ref_samples(ref_idx, plane) else { return false; }; let dims = self.ref_plane_dims(ref_idx, plane); let max = ((1u32 << fs.bit_depth) - 1) as u16; - copy_fullpel(&r.buf.planes[plane], dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max); + copy_fullpel(refp, dims, McBlock { x: sx, y: sy, w, h }, &mut self.t.cur.planes[plane], (x, y), max); true } @@ -763,7 +760,7 @@ impl TileDecoder<'_, '_> { /// Unscaled single-reference prediction at a whole-sample position: both filter phases are the /// identity and Round2(128 * Round2(128 * v, InterRound0), InterRound1) == v for every bit /// depth, so the prediction is the (clamped) reference sample. -fn copy_fullpel(refp: &Plane, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) { +fn copy_fullpel(refp: PlaneRef, (last_x, last_y): (i32, i32), src_blk: McBlock, dst: &mut Plane, (dx, dy): (usize, usize), max: u16) { let McBlock { x, y, w, h } = src_blk; let x0 = x >> 10; let y0 = y >> 10; @@ -831,7 +828,7 @@ fn vpass(rows: &[i32], vf: &[i16; 8], out: &mut [i32], half: i32 /// Block inter prediction without reference scaling: every column / row uses the same filter /// phase, so the filters run over whole rows (bit-identical to the per-sample process). fn mc_unscaled( - refp: &Plane, + refp: PlaneRef, (last_x, last_y): (i32, i32), blk: McBlock, (hf, vf): (&[i16; 8], &[i16; 8]), @@ -982,9 +979,17 @@ mod tests { [(0, 0), (-5, -3), (7, 9), (last_x - w as i32 / 2, last_y - h as i32 / 2), (last_x + 3, last_y + 2), (-(w as i32) - 4, 3)] { let (x, y) = ((px << 10) | 32, (py << 10) | 32); - mc_unscaled(&refp, (last_x, last_y), McBlock { x, y, w, h }, (&IDENTITY_TAPS, &IDENTITY_TAPS), rnd, &mut pred, &mut tmp); + mc_unscaled( + PlaneRef::Whole(&refp), + (last_x, last_y), + McBlock { x, y, w, h }, + (&IDENTITY_TAPS, &IDENTITY_TAPS), + rnd, + &mut pred, + &mut tmp, + ); let mut dst = Plane::new(w + 3, h); - copy_fullpel(&refp, (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max); + copy_fullpel(PlaneRef::Whole(&refp), (last_x, last_y), McBlock { x, y, w, h }, &mut dst, (3, 0), max); for i in 0..h { for j in 0..w { let want = spec_phase0(&refp, (last_x, last_y), (px, py), (j, i), (w * 3 + h) % SUBPEL_FILTERS.len(), rnd) diff --git a/crates/video/src/av1/decode/postfilter.rs b/crates/video/src/av1/decode/postfilter.rs index 6304fdfb6..aa9c2e3ea 100644 --- a/crates/video/src/av1/decode/postfilter.rs +++ b/crates/video/src/av1/decode/postfilter.rs @@ -1,7 +1,7 @@ //! In-loop post filters run by decode_frame_wrapup: loop filter (7.14), CDEF (7.15), //! super-resolution upscaling (7.16) and loop restoration (7.17). -use crate::av1::decode::frame::RowBand; +use crate::av1::decode::frame::{FrameBuf, Plane, RowBand}; use crate::av1::decode::spec_tables::{ GLOBAL_GLOBALMV, GLOBALMV, INTRA_FRAME, MAX_LOOP_FILTER, NEARESTMV, NUM_4X4_BLOCKS_HIGH, NUM_4X4_BLOCKS_WIDE, SEG_LVL_ALT_LF_Y_V, TX_HEIGHT, TX_WIDTH, @@ -48,6 +48,121 @@ pub(crate) fn apply(fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate:: stats.add(Stage::Restoration, t.lap()); } +/// The in-loop filters run band by band behind reconstruction (row wavefront): each stage of a +/// band starts once the stage before has finished the band below it, which bounds what the +/// stage reads beyond the band (the loop filter changes up to 7 rows above a band and intra +/// prediction of the next superblock row reads the unfiltered row above it; CDEF reads 2 rows +/// and loop restoration 3 rows beyond a band). Bands are 64 luma rows. The result is the same +/// as running each filter over the whole frame. Super-resolution frames use [`apply`]. +/// +/// The last active stage writes each band into its own buffers, which are handed out as the +/// band's final samples without a copy. +pub(crate) struct Pipeline { + bands: Vec<(usize, usize)>, + /// Luma rows of the coded frame (mi_rows * 4). + rows: usize, + lf_on: bool, + cdef_on: bool, + lr_on: bool, + /// Bands finished by each stage. + lf: usize, + cdef: usize, + lr: usize, + pending: [Vec; 3], + /// CdefFrame when loop restoration reads it. + cdef_out: Option, +} + +/// Final bands from [`Pipeline::advance`]: (band, its samples starting at the band's first row). +pub(crate) type FinalBands = Vec<(usize, [Plane; 3])>; + +impl Pipeline { + pub fn new(fs: &FrameState, draft: bool) -> Pipeline { + let rows = fs.fh.mi_rows as usize * 4; + let n = rows.div_ceil(BAND_ROWS).max(1); + let bands = (0..n).map(|b| (b * BAND_ROWS, if b + 1 == n { usize::MAX } else { (b + 1) * BAND_ROWS })).collect(); + let lvl = fs.fh.lf.level; + let cdef_on = !draft && crate::av1::decode::cdef::active(fs); + let lr_on = fs.fh.lr.uses_lr && !draft; + let cdef_out = (cdef_on && lr_on).then(|| fs.cur.same_geometry()); + Pipeline { + bands, + rows, + lf_on: (lvl[0] != 0 || lvl[1] != 0) && !draft, + cdef_on, + lr_on, + lf: 0, + cdef: 0, + lr: 0, + pending: Default::default(), + cdef_out, + } + } + + /// Filter what luma rows [0, `recon_rows`) of reconstructed samples allow (everything once + /// they cover the frame); returns the bands whose samples became final. + pub fn advance(&mut self, fs: &mut FrameState, stats: &mut DecodeStats, pool: &crate::av1::decode::par::Pool, recon_rows: usize) -> FinalBands { + let n = self.bands.len(); + let all = recon_rows >= self.rows; + let mut out = FinalBands::new(); + let mut t = Timer::start(); + // The loop filter of a band waits for the band below it to be reconstructed. + let lf_to = if all { n } else { (recon_rows / BAND_ROWS).saturating_sub(1) }; + if lf_to > self.lf { + if self.lf_on { + loop_filter_bands(fs, pool, &self.bands, self.lf..lf_to, &mut self.pending); + } + self.lf = lf_to; + } + stats.add(Stage::LoopFilter, t.lap()); + let cdef_to = if all { n } else { self.lf.saturating_sub(1) }; + if cdef_to > self.cdef { + let r = &self.bands[self.cdef..cdef_to]; + if self.cdef_on { + if let Some(cdef) = &mut self.cdef_out { + cdef.copy_rows_from(&fs.cur, r[0].0, r[r.len() - 1].1); + crate::av1::decode::cdef::filter_bands(fs, pool, cdef.row_bands(r), r); + } else { + let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b)).collect(); + crate::av1::decode::cdef::filter_bands(fs, pool, own_bands(&mut own), r); + out.extend((self.cdef..cdef_to).zip(own)); + } + } else if !self.lr_on { + // The loop filter is the last stage: the band is final once the band below it + // is filtered. + out.extend((self.cdef..cdef_to).zip(r.iter().map(|&(a, b)| fs.cur.rows_copy(a, b)))); + } + self.cdef = cdef_to; + } + stats.add(Stage::Cdef, t.lap()); + let lr_to = if all { n } else { self.cdef.saturating_sub(1) }; + if self.lr_on && lr_to > self.lr { + let r = &self.bands[self.lr..lr_to]; + let cdef = self.cdef_out.as_ref().unwrap_or(&fs.cur); + let mut own: Vec<[Plane; 3]> = r.iter().map(|&(a, b)| cdef.rows_copy(a, b)).collect(); + crate::av1::decode::restoration::restore_bands(fs, &fs.cur, cdef, own_bands(&mut own), pool, r); + out.extend((self.lr..lr_to).zip(own)); + self.lr = lr_to; + } + stats.add(Stage::Restoration, t.lap()); + out + } +} + +/// Row bands over per-band buffers made by [`FrameBuf::rows_copy`]. +fn own_bands(own: &mut [[Plane; 3]]) -> Vec<[RowBand<'_>; 3]> { + fn band(p: &mut Plane) -> RowBand<'_> { + let (stride, y0) = (p.stride.max(1), p.oy); + RowBand { data: p.data.as_mut_slice(), stride, y0 } + } + own.iter_mut() + .map(|planes| { + let [a, b, c] = planes; + [band(a), band(b), band(c)] + }) + .collect() +} + // --------------------------------------------------------------------------------------------- // Loop filter (7.14) @@ -69,23 +184,40 @@ const LF_MARGIN: usize = 8; /// written back; the band above never touches them. fn loop_filter(fs: &mut FrameState, pool: &crate::av1::decode::par::Pool) { let luma = bands(pool, fs.fh.mi_rows as usize * 4); + loop_filter_bands(fs, pool, &luma, 0..luma.len(), &mut Default::default()); +} + +/// The loop filter on bands `range` of `all` (the frame's bands). The bands above the range +/// are filtered already: `pending` holds the unfiltered rows above the first band of the range +/// (empty for band 0), saved before the band above was filtered, and is set to those of the +/// band after the range. +fn loop_filter_bands( + fs: &mut FrameState, + pool: &crate::av1::decode::par::Pool, + all: &[(usize, usize)], + range: std::ops::Range, + pending: &mut [Vec; 3], +) { + let Some(luma) = all.get(range.clone()) else { return }; let planes: Vec = (0..fs.num_planes).filter(|&p| p == 0 || fs.fh.lf.level[1 + p] != 0).collect(); let mut cur = std::mem::take(&mut fs.cur); let f: &FrameState = fs; - // Unfiltered copies of the rows above each band but the first. - let margins: Vec<[Vec; 3]> = luma - .iter() - .map(|&(start, _)| { - std::array::from_fn(|p| { - let sub_y = if p == 0 { 0 } else { f.ssy }; - let pl = &cur.planes[p]; - let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y); - let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride); - if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() } - }) + // Unfiltered copies of the rows above a band (none above the first). + let margin = |cur: &crate::av1::decode::frame::FrameBuf, start: usize| -> [Vec; 3] { + std::array::from_fn(|p| { + let sub_y = if p == 0 { 0 } else { f.ssy }; + let pl = &cur.planes[p]; + let (y0, y1) = ((start.saturating_sub(LF_MARGIN)) >> sub_y, start >> sub_y); + let rows = pl.data.get(y0 * pl.stride..y1 * pl.stride); + if start == 0 || !planes.contains(&p) { Vec::new() } else { rows.map(<[u16]>::to_vec).unwrap_or_default() } }) - .collect(); - let mut bands: Vec<([RowBand; 3], [Vec; 3])> = cur.row_bands(&luma).into_iter().zip(margins).collect(); + }; + let margins: Vec<[Vec; 3]> = + luma.iter().enumerate().map(|(i, &(start, _))| if i == 0 { std::mem::take(pending) } else { margin(&cur, start) }).collect(); + if let Some(&(next, _)) = all.get(range.end) { + *pending = margin(&cur, next); + } + let mut bands: Vec<([RowBand; 3], [Vec; 3])> = cur.row_bands(luma).into_iter().zip(margins).collect(); let changed = pool.map_mut(&mut bands, |b, (band, margin)| { let mut changed: [Vec<(usize, u16)>; 3] = Default::default(); for &p in &planes { diff --git a/crates/video/src/av1/decode/restoration.rs b/crates/video/src/av1/decode/restoration.rs index efc5a751f..d252f7b06 100644 --- a/crates/video/src/av1/decode/restoration.rs +++ b/crates/video/src/av1/decode/restoration.rs @@ -70,10 +70,16 @@ struct LrPlane<'a> { /// Bands of 64 luma rows are restored in parallel: every unit reads only the two input frames /// and writes only its own samples. pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, pool: &Pool) -> FrameBuf { - let fh = &fs.fh; let mut lr = cdef.clone(); - let ranges = bands(pool, fh.frame_height as usize); - let mut planes = lr.row_bands(&ranges); + let ranges = bands(pool, fs.fh.frame_height as usize); + restore_bands(fs, cur, cdef, lr.row_bands(&ranges), pool, &ranges); + lr +} + +/// Restore the bands `planes` covering luma rows `ranges`, which hold UpscaledCdefFrame; reads +/// both inputs up to 3 rows beyond each band. +pub(crate) fn restore_bands(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, mut planes: Vec<[RowBand; 3]>, pool: &Pool, ranges: &[(usize, usize)]) { + let fh = &fs.fh; pool.map_mut(&mut planes, |b, band| { let (start, end) = ranges[b]; let mut scratch = Scratch::default(); @@ -100,8 +106,6 @@ pub(crate) fn loop_restoration(fs: &FrameState, cur: &FrameBuf, cdef: &FrameBuf, p.restore(out, start >> sub_y, end >> sub_y, &mut scratch); } }); - drop(planes); - lr } /// Row buffers reused across stripes. diff --git a/crates/video/src/av1/decode/state.rs b/crates/video/src/av1/decode/state.rs index 4c4806f16..f498a7f05 100644 --- a/crates/video/src/av1/decode/state.rs +++ b/crates/video/src/av1/decode/state.rs @@ -6,8 +6,8 @@ //! with several, each tile decodes into private buffers covering its area (tiles never read //! outside themselves), which are then copied into the frame. -use crate::av1::decode::decoder::RefFrame; -use crate::av1::decode::frame::{FrameBuf, MiInfo}; +use crate::av1::decode::decoder::{Parsed, RefFrame}; +use crate::av1::decode::frame::{BandFrame, FrameBuf, MiInfo, PlaneRef}; use crate::av1::decode::header::{FrameHeader, RefInfo, SequenceHeader}; use crate::av1::decode::spec_tables::{DEFAULT_INTRA_FRAME_Y_MODE_CDF, NUM_REF_FRAMES, RESTORE_NONE, SUBSAMPLED_SIZE}; use std::ops::{Deref, DerefMut}; @@ -26,6 +26,11 @@ pub(crate) struct FrameShared { pub cdef_cols: usize, /// Reference slots (FrameStore and saved state) used by this frame. pub refs: [Option>; NUM_REF_FRAMES], + /// Parse results (CDFs, segment ids, saved motion vectors) of the same slots. + pub parsed: [Option>; NUM_REF_FRAMES], + /// The published bands of references that are still being decoded (row wavefront): read + /// only below the row progress that was waited for. + pub ref_rows: [Option>; NUM_REF_FRAMES], pub ref_info: [RefInfo; NUM_REF_FRAMES], /// MotionFieldMvs[ ref ][ y8 * w8 + x8 ] (7.9); empty when unused. pub motion_field: [Vec<[i32; 2]>; 8], @@ -34,6 +39,16 @@ pub(crate) struct FrameShared { } impl FrameShared { + /// The samples of reference slot `ref_idx` to predict from (None for a missing reference). + #[inline] + pub fn ref_samples(&self, ref_idx: i32, plane: usize) -> Option> { + let i = usize::try_from(ref_idx).ok()?; + if let Some(b) = self.ref_rows.get(i)?.as_ref() { + return b.planes.get(plane).map(PlaneRef::Bands); + } + self.refs.get(i)?.as_ref().map(|r| r.plane(plane)) + } + pub fn plane_residual_size(&self, subsize: usize, plane: usize) -> usize { let sx = if plane > 0 { self.ssx } else { 0 }; let sy = if plane > 0 { self.ssy } else { 0 }; @@ -210,6 +225,8 @@ impl FrameState { lr_unit_cols, cdef_cols, refs: Default::default(), + parsed: Default::default(), + ref_rows: Default::default(), ref_info: Default::default(), motion_field: Default::default(), prev_segment_ids: vec![0; mi_cols * mi_rows], @@ -283,6 +300,14 @@ impl FrameState { pub fn merge_tile_state(&mut self, t: &TileState) { for p in 0..self.num_planes { t.cur.planes[p].copy_into(&mut self.cur.planes[p]); + } + self.merge_tile_modes(t); + } + + /// Copy a tile's private buffers except the samples (after parsing, before reconstruction) + /// into the frame. + pub fn merge_tile_modes(&mut self, t: &TileState) { + for p in 0..self.num_planes { t.lf_tx_size[p].copy_into(&mut self.lf_tx_size[p]); for &u in &t.lr_written[p] { self.lr.lr_type[p][u] = t.lr.lr_type[p][u]; diff --git a/crates/video/src/av1/decode/tile.rs b/crates/video/src/av1/decode/tile.rs index 16d59ed3a..2572f9d21 100644 --- a/crates/video/src/av1/decode/tile.rs +++ b/crates/video/src/av1/decode/tile.rs @@ -94,6 +94,191 @@ pub(crate) struct Block { pub local_warp_params: [i32; 6], } +/// What a tile decoder does with the blocks it visits. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Pass { + /// Read the symbols and reconstruct each block right away (single-threaded decoding). + Full, + /// Read the symbols only, recording what reconstruction needs in a [`TileRecord`]. + Parse, + /// Reconstruct from a [`TileRecord`], reading no symbols. + Reconstruct, +} + +/// The per-block values reconstruction needs beyond the mode info (which is final after +/// parsing): the block variables as parsed, its quantizer index and the warp samples / palette +/// color maps when used. +#[derive(Clone)] +pub(crate) struct BlockRecord { + b: Block, + current_q_index: i32, + warp_samples: Option>, + color_maps: Option, Vec)>>, +} + +/// A parsed tile: its blocks in decoding order, how many of them each superblock holds and the +/// coefficients of every coded transform block as a stream of +/// `eob, tx_type, count, (position, value) * count` (only non-zero values). +#[derive(Default)] +pub(crate) struct TileRecord { + blocks: Vec, + sb_blocks: Vec, + coeffs: Vec, + /// Read positions while reconstructing. + next_block: usize, + next_coeff: usize, + next_sb: usize, + next_sb_row: usize, + /// Bytes recorded and the most allowed. + pub bytes: usize, + pub limit: usize, + /// The decoder-wide budget the recorded bytes are reserved from, and how much is reserved. + pub budget: Option>, + reserved: usize, +} + +impl Drop for TileRecord { + fn drop(&mut self) { + if let Some(b) = &self.budget { + b.release(self.reserved); + } + } +} + +/// The parse record bytes all frames of a decoder may hold at once (frames in flight decode in +/// one pass once it is used up). +pub(crate) struct RecordBudget { + used: std::sync::atomic::AtomicUsize, + max: usize, + /// Most bytes ever reserved at once (tests). + #[cfg(test)] + pub peak: std::sync::atomic::AtomicUsize, +} + +/// Bytes reserved from a [`RecordBudget`] at a time. +const RESERVE_STEP: usize = 1 << 20; + +impl RecordBudget { + pub fn new(max: usize) -> RecordBudget { + RecordBudget { + used: Default::default(), + max, + #[cfg(test)] + peak: Default::default(), + } + } + + /// Reserve `n` bytes; false (nothing reserved) if that would go over the budget. + fn reserve(&self, n: usize) -> bool { + use std::sync::atomic::Ordering; + let ok = self.used.try_update(Ordering::AcqRel, Ordering::Acquire, |u| u.checked_add(n).filter(|&v| v <= self.max)).is_ok(); + #[cfg(test)] + if ok { + self.peak.fetch_max(self.used.load(Ordering::Acquire), Ordering::AcqRel); + } + ok + } + + fn release(&self, n: usize) { + self.used.fetch_sub(n, std::sync::atomic::Ordering::AcqRel); + } + + /// Bytes reserved now (tests). + #[cfg(test)] + pub fn used(&self) -> usize { + self.used.load(std::sync::atomic::Ordering::Acquire) + } +} + +/// The motion of a recorded block (what a reference row wait depends on). +pub(crate) struct BlockMotion { + pub mi_row: usize, + pub bh4: usize, + /// Inter prediction from reference frames (not intra block copy). + pub inter: bool, + pub ref_frame: [i8; 2], + /// Mv[ list ] (row, col) in 1/8 samples. + pub mv: [[i32; 2]; 2], + pub motion_mode: u8, + pub y_mode: usize, +} + +impl TileRecord { + /// Whether recording stopped at [`TileRecord::limit`] (the record is unusable). + pub fn over_limit(&self) -> bool { + self.bytes > self.limit + } + + /// Account for `n` more recorded bytes; false once over the limit (nothing more is kept). + fn grow(&mut self, n: usize) -> bool { + self.bytes = self.bytes.saturating_add(n); + if self.bytes > self.limit { + return false; + } + if self.bytes > self.reserved + && let Some(b) = &self.budget + { + let step = (self.bytes - self.reserved).max(RESERVE_STEP); + if !b.reserve(step) { + // The decoder-wide budget is used up: this frame is decoded in one pass. + self.limit = 0; + return false; + } + self.reserved += step; + } + true + } + /// The recorded blocks of superblock rows [first, end), for a tile `sbs_per_row` superblocks wide. + pub fn row_blocks(&self, sbs_per_row: usize, first: usize, end: usize) -> impl Iterator + '_ { + let start: usize = self.sb_blocks.iter().take(first * sbs_per_row).map(|&n| n as usize).sum(); + let count: usize = self.sb_blocks.iter().skip(first * sbs_per_row).take((end - first) * sbs_per_row).map(|&n| n as usize).sum(); + self.blocks.iter().skip(start).take(count).map(|r| BlockMotion { + mi_row: r.b.mi_row, + bh4: r.b.bh4, + inter: r.b.is_inter && !r.b.use_intrabc, + ref_frame: r.b.ref_frame, + mv: r.b.mv_i, + motion_mode: r.b.motion_mode, + y_mode: r.b.y_mode, + }) + } + + fn next_coeff(&mut self) -> Result { + let v = *self.coeffs.get(self.next_coeff).ok_or(crate::av1::decode::Error::Invalid("coefficient record exhausted"))?; + self.next_coeff += 1; + Ok(v) + } +} + +/// The work buffers of a [`TileDecoder`] (`quant` all zero between transform blocks). +pub(crate) struct Buffers { + quant: Vec, + dequant: Vec, + residual: Vec, + color_map_y: Vec, + color_map_uv: Vec, + pred_buf: [Vec; 2], + mask: Vec, + mc_tmp: Vec, + levels: Vec, +} + +impl Buffers { + pub fn new() -> Buffers { + Buffers { + quant: vec![0; 1024], + dequant: vec![0; 64 * 64], + residual: vec![0; 64 * 64], + color_map_y: vec![0; 64 * 64], + color_map_uv: vec![0; 64 * 64], + pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]], + mask: vec![0; 128 * 128], + mc_tmp: vec![0; (2 * 128 + 8) * 128], + levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)], + } + } +} + /// Decoder for one tile. pub(crate) struct TileDecoder<'a, 'f> { pub fs: &'f FrameShared, @@ -128,6 +313,9 @@ pub(crate) struct TileDecoder<'a, 'f> { pub mc_tmp: Vec, /// Coefficient levels of the current transform block, padded (see TX_PAD). levels: Vec, + pub pass: Pass, + /// Written by [`Pass::Parse`], read by [`Pass::Reconstruct`]. + pub record: TileRecord, } const SB_MAX: usize = 32; // superblock size in 4x4 units (128 / 4) @@ -141,6 +329,21 @@ fn block_height(bs: usize) -> usize { impl<'a, 'f> TileDecoder<'a, 'f> { pub fn new(fs: &'f FrameShared, t: &'f mut TileState, data: &'a [u8], cdf: Box, tile_row: usize, tile_col: usize) -> Self { + Self::with_buffers(fs, t, data, cdf, tile_row, tile_col, Buffers::new()) + } + + /// [`TileDecoder::new`] reusing the work buffers of an earlier decoder of the same frame + /// (see [`TileDecoder::into_buffers`]). + pub fn with_buffers( + fs: &'f FrameShared, + t: &'f mut TileState, + data: &'a [u8], + cdf: Box, + tile_row: usize, + tile_col: usize, + bufs: Buffers, + ) -> Self { + let Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels } = bufs; let ti = &fs.fh.tile_info; let (rs, re) = (ti.mi_row_starts[tile_row] as usize, ti.mi_row_starts[tile_row + 1] as usize); let (cs, ce) = (ti.mi_col_starts[tile_col] as usize, ti.mi_col_starts[tile_col + 1] as usize); @@ -164,19 +367,27 @@ impl<'a, 'f> TileDecoder<'a, 'f> { b: Block::default(), mvs: Default::default(), nb: Default::default(), - quant: vec![0; 1024], - dequant: vec![0; 64 * 64], - residual: vec![0; 64 * 64], - color_map_y: vec![0; 64 * 64], - color_map_uv: vec![0; 64 * 64], + quant, + dequant, + residual, + color_map_y, + color_map_uv, plane_tx_type: 0, - pred_buf: [vec![0; 128 * 128], vec![0; 128 * 128]], - mask: vec![0; 128 * 128], - mc_tmp: vec![0; (2 * 128 + 8) * 128], - levels: vec![0; (32 + TX_PAD) * (32 + TX_PAD)], + pred_buf, + mask, + mc_tmp, + levels, + pass: Pass::Full, + record: TileRecord::default(), } } + /// The work buffers and CDFs, for [`TileDecoder::with_buffers`]. + pub fn into_buffers(self) -> (Buffers, Box) { + let TileDecoder { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels, cdf, .. } = self; + (Buffers { quant, dequant, residual, color_map_y, color_map_uv, pred_buf, mask, mc_tmp, levels }, cdf) + } + fn seq(&self) -> &SequenceHeader { &self.fs.seq } @@ -189,8 +400,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> { c >= self.mi_col_start as isize && c < self.mi_col_end as isize && r >= self.mi_row_start as isize && r < self.mi_row_end as isize } - /// decode_tile( ) + /// decode_tile( ), or its reconstruction from [`TileDecoder::record`] in [`Pass::Reconstruct`]. pub fn decode_tile(&mut self) -> Result<()> { + if self.pass == Pass::Reconstruct { + return self.reconstruct_tile(); + } // clear_above_context( ) let t = &mut *self.t; for p in 0..3 { @@ -223,7 +437,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> { self.clear_cdef(r, c); self.clear_block_decoded_flags(r, c, sb4); self.read_lr(r, c, sb_size); + let before = self.record.blocks.len(); self.decode_partition(r, c, sb_size)?; + if self.pass == Pass::Parse { + self.record.sb_blocks.push((self.record.blocks.len() - before) as u32); + } c += sb4; } r += sb4; @@ -231,6 +449,76 @@ impl<'a, 'f> TileDecoder<'a, 'f> { Ok(()) } + /// The reconstruction half of decode_tile( ): the superblocks in the same order, each block + /// predicted and reconstructed from its record. + fn reconstruct_tile(&mut self) -> Result<()> { + self.reconstruct_sb_rows(usize::MAX) + } + + /// Reconstruct the next `rows` superblock rows of the tile (the record keeps the position). + pub fn reconstruct_sb_rows(&mut self, rows: usize) -> Result<()> { + let sb4 = if self.seq().use_128x128_superblock { 32 } else { 16 }; + let mut r = self.mi_row_start + self.record.next_sb_row * sb4; + let mut done = 0; + while r < self.mi_row_end && done < rows { + let mut c = self.mi_col_start; + while c < self.mi_col_end { + self.clear_block_decoded_flags(r, c, sb4); + let n = *self.record.sb_blocks.get(self.record.next_sb).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?; + self.record.next_sb += 1; + for _ in 0..n { + self.reconstruct_block()?; + } + c += sb4; + } + r += sb4; + done += 1; + self.record.next_sb_row += 1; + } + Ok(()) + } + + fn reconstruct_block(&mut self) -> Result<()> { + let rec = self.record.blocks.get(self.record.next_block).ok_or(crate::av1::decode::Error::Invalid("block record exhausted"))?; + self.record.next_block += 1; + self.b = rec.b.clone(); + self.current_q_index = rec.current_q_index; + if let Some(w) = &rec.warp_samples { + self.mvs.num_samples = w.0; + self.mvs.cand_list = w.1; + } + if let Some(maps) = &rec.color_maps { + if let Some(d) = self.color_map_y.get_mut(..maps.0.len()) { + d.copy_from_slice(&maps.0); + } + if let Some(d) = self.color_map_uv.get_mut(..maps.1.len()) { + d.copy_from_slice(&maps.1); + } + } + self.compute_prediction()?; + self.residual() + } + + /// Records the block for [`Pass::Reconstruct`] (its variables as parsed). + fn record_block(&mut self) { + let b = &self.b; + // The rows of the color maps (stride 64) the block's palette prediction reads. + let map_rows = |n: usize, h: usize| if n > 0 { (h + 2).min(64) * 64 } else { 0 }; + let rows_y = map_rows(b.palette_size_y, block_height(b.mi_size)); + let rows_uv = map_rows(b.palette_size_uv, block_height(b.mi_size) >> self.fs.ssy); + let size = std::mem::size_of::() + + rows_y + + rows_uv + + if b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP { 160 } else { 0 }; + if !self.record.grow(size) { + return; + } + let warp_samples = + (b.motion_mode as usize == crate::av1::decode::spec_tables::LOCALWARP).then(|| Box::new((self.mvs.num_samples, self.mvs.cand_list))); + let color_maps = (rows_y + rows_uv > 0).then(|| Box::new((self.color_map_y[..rows_y].to_vec(), self.color_map_uv[..rows_uv].to_vec()))); + self.record.blocks.push(BlockRecord { b: b.clone(), current_q_index: self.current_q_index, warp_samples, color_maps }); + } + fn clear_cdef(&mut self, r: usize, c: usize) { let t = &mut *self.t; t.set_cdef_idx(r, c, -1); @@ -461,7 +749,11 @@ impl<'a, 'f> TileDecoder<'a, 'f> { } } } - self.compute_prediction()?; + if self.pass == Pass::Parse { + self.record_block(); + } else { + self.compute_prediction()?; + } self.residual()?; let b = &self.b; let delta_lf = self.delta_lf; diff --git a/crates/video/src/av1/decode/tile/coeffs.rs b/crates/video/src/av1/decode/tile/coeffs.rs index 6a704f8ac..df358e68d 100644 --- a/crates/video/src/av1/decode/tile/coeffs.rs +++ b/crates/video/src/av1/decode/tile/coeffs.rs @@ -459,6 +459,58 @@ impl<'a, 'f> TileDecoder<'a, 'f> { } } + /// Moves the coefficients `coeffs` left in `quant` into the record (leaving `quant` zero, as + /// reconstruction does). + pub(super) fn record_coeffs(&mut self, eob: usize, tx_sz: usize) { + let scan = self.get_scan(tx_sz); + let scan = scan.get(..eob).unwrap_or(scan); + if !self.record.grow(4 * (3 + 2 * scan.len())) { + // Over the limit: the record is dropped (the frame is decoded again in one pass); + // `quant` must still be left zero. + for &pos in scan { + if let Some(q) = self.quant.get_mut(pos as usize) { + *q = 0; + } + } + return; + } + let rec = &mut self.record.coeffs; + rec.push(eob as i32); + if eob == 0 { + return; + } + rec.push(self.plane_tx_type as i32); + let count_at = rec.len(); + rec.push(0); + let mut count = 0; + for &pos in scan { + let v = std::mem::take(&mut self.quant[pos as usize]); + if v != 0 { + rec.push(pos as i32); + rec.push(v); + count += 1; + } + } + rec[count_at] = count; + } + + /// The next transform block's coefficients from the record into `quant` (and its type); + /// returns its eob. + pub(super) fn replay_coeffs(&mut self) -> crate::av1::decode::Result { + let eob = self.record.next_coeff()?; + if eob <= 0 { + return Ok(0); + } + self.plane_tx_type = usize::try_from(self.record.next_coeff()?).map_err(|_| crate::av1::decode::Error::Invalid("coefficient record"))?; + let count = self.record.next_coeff()?; + for _ in 0..count { + let pos = self.record.next_coeff()? as usize; + let v = self.record.next_coeff()?; + *self.quant.get_mut(pos).ok_or(crate::av1::decode::Error::Invalid("coefficient record"))? = v; + } + Ok(eob as usize) + } + /// Reconstruct process (7.12.3). pub(super) fn reconstruct(&mut self, plane: usize, x: usize, y: usize, tx_sz: usize, dc_only: bool) { // dqDenom (7.12.3) is 1, 2 or 4 and the dividend is non-negative. diff --git a/crates/video/src/av1/decode/tile/predict.rs b/crates/video/src/av1/decode/tile/predict.rs index b1a53f399..244be62c6 100644 --- a/crates/video/src/av1/decode/tile/predict.rs +++ b/crates/video/src/av1/decode/tile/predict.rs @@ -213,7 +213,8 @@ impl<'a, 'f> TileDecoder<'a, 'f> { if start_x >= max_x || start_y >= max_y { return Ok(()); } - if !self.b.is_inter { + let parse_only = self.pass == super::Pass::Parse; + if !self.b.is_inter && !parse_only { let palette = if plane == 0 { self.b.palette_size_y } else { self.b.palette_size_uv }; if palette > 0 { self.predict_palette(plane, start_x, start_y, x, y, tx_sz); @@ -266,9 +267,23 @@ impl<'a, 'f> TileDecoder<'a, 'f> { } } if !self.b.skip { - let eob = self.coeffs(plane, start_x, start_y, tx_sz); - if eob > 0 { - self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1); + match self.pass { + super::Pass::Full => { + let eob = self.coeffs(plane, start_x, start_y, tx_sz); + if eob > 0 { + self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1); + } + } + super::Pass::Parse => { + let eob = self.coeffs(plane, start_x, start_y, tx_sz); + self.record_coeffs(eob, tx_sz); + } + super::Pass::Reconstruct => { + let eob = self.replay_coeffs()?; + if eob > 0 { + self.reconstruct(plane, start_x, start_y, tx_sz, eob == 1); + } + } } } for i in 0..step_y { diff --git a/crates/video/src/hevc/decode/mvpred.rs b/crates/video/src/hevc/decode/mvpred.rs index 962bbaa8c..9abbe33aa 100644 --- a/crates/video/src/hevc/decode/mvpred.rs +++ b/crates/video/src/hevc/decode/mvpred.rs @@ -152,38 +152,35 @@ impl<'a, S: crate::hevc::decode::picture::Sample> SliceDecoder<'a, S> { col.ref_idx[1] = 0; col_avail = true; } - let mut cands: Vec = list[..n].to_vec(); if col_avail { - cands.push(col); + list[n] = col; + n += 1; } - if merge_idx < cands.len() { - return self.finish_merge(cands[merge_idx], w0, h0); + if merge_idx < n { + return self.finish_merge(list[merge_idx], w0, h0); } // combined bi-predictive candidates (8.5.3.2.4) - let num_orig = cands.len(); + let num_orig = n; if self.sh.is_b() && num_orig > 1 && num_orig < max { let mut comb = 0; - while comb < num_orig * (num_orig - 1) && cands.len() < max { + while comb < num_orig * (num_orig - 1) && n < max { let (i0, i1) = COMB[comb]; - let (c0, c1) = (cands[i0], cands[i1]); + let (c0, c1) = (list[i0], list[i1]); if c0.pred(0) && c1.pred(1) && (self.ref_pic(0, c0.ref_idx[0]).poc != self.ref_pic(1, c1.ref_idx[1]).poc || c0.mv[0] != c1.mv[1]) { - cands.push(MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] }); + list[n] = MvField { mv: [c0.mv[0], c1.mv[1]], ref_idx: [c0.ref_idx[0], c1.ref_idx[1]] }; + n += 1; } comb += 1; } } - if merge_idx < cands.len() { - return self.finish_merge(cands[merge_idx], w0, h0); + if merge_idx < n { + return self.finish_merge(list[merge_idx], w0, h0); } - // zero candidates (8.5.3.2.5) + // zero candidates (8.5.3.2.5): candidate merge_idx is the (merge_idx - n)-th one let num_ref = if self.sh.is_b() { self.sh.num_ref_idx[0].min(self.sh.num_ref_idx[1]) } else { self.sh.num_ref_idx[0] } as usize; - let mut zero_idx = 0; - while cands.len() <= merge_idx { - let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 }; - cands.push(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] }); - zero_idx += 1; - } - self.finish_merge(cands[merge_idx], w0, h0) + let zero_idx = merge_idx - n; + let r = if zero_idx < num_ref { zero_idx as i8 } else { 0 }; + self.finish_merge(MvField { mv: [[0; 2]; 2], ref_idx: [r, if self.sh.is_b() { r } else { -1 }] }, w0, h0) } fn finish_merge(&self, mut f: MvField, w: i32, h: i32) -> MvField { diff --git a/crates/video/src/hevc/decode/slicedec.rs b/crates/video/src/hevc/decode/slicedec.rs index 102b74719..8ebaeafb1 100644 --- a/crates/video/src/hevc/decode/slicedec.rs +++ b/crates/video/src/hevc/decode/slicedec.rs @@ -16,7 +16,7 @@ use crate::hevc::decode::spec_tables::{ INTRA_CHROMA, LAST_X_PREFIX, LAST_Y_PREFIX, MERGE_FLAG, MERGE_IDX, MVP_FLAG, PART_MODE, PRED_MODE, PREV_INTRA_LUMA, REF_IDX, RQT_ROOT_CBF, SAO_MERGE, SAO_TYPE, SIG_COEFF, SPLIT_CU, SPLIT_TRANSFORM, TRANSFORM_SKIP, }; -use crate::hevc::decode::tables::{CTX_IDX_MAP, INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, qpc, scan_order}; +use crate::hevc::decode::tables::{INTRA_CHROMA_MODE_422, LEVEL_SCALE, SCAN_4, SIG_CTX, qpc, scan_order, scan_pos}; use crate::hevc::decode::transform; use std::sync::Arc; @@ -340,6 +340,8 @@ pub struct SliceDecoder<'a, S> { pub cu_part: PartMode, // scratch pub coeffs: Vec, + /// Scaled non-zero coefficients of the current transform block: (index in `coeffs`, value). + pub nz: Vec<(u16, i32)>, pub pred0: Vec, pub pred1: Vec, pub mc: crate::hevc::decode::inter::McScratch, @@ -461,6 +463,7 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { intra_split: false, cu_part: PartMode::P2Nx2N, coeffs: vec![0; 32 * 32], + nz: vec![(0, 0); 32 * 32], pred0: vec![0; 64 * 64], pred1: vec![0; 64 * 64], mc: Default::default(), @@ -632,3 +635,30 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::hevc::decode::transform; + + /// `reconstruct` clears only the coefficient bounding box, so the inverse transform must not + /// read the stale values left outside it by the previous block. + #[test] + fn inverse_transform_reads_only_the_coefficient_box() { + for &(n, dst) in &[(4usize, true), (4, false), (8, false), (16, false), (32, false)] { + for (max_x, max_y) in [(0, 1), (1, 0), (2, 3), (n / 2, 1), (n - 1, 0), (n - 1, n - 1)] { + let (mut clean, mut stale) = (vec![0i32; n * n], vec![0i32; n * n]); + for (i, (c, s)) in clean.iter_mut().zip(stale.iter_mut()).enumerate() { + let (x, y) = (i % n, i / n); + *s = 12345 - i as i32; + if x <= max_x && y <= max_y { + *c = (i as i32 * 37) % 201 - 100; + *s = *c; + } + } + transform::inverse_transform(&mut clean, n, dst, max_x, max_y, 12); + transform::inverse_transform(&mut stale, n, dst, max_x, max_y, 12); + assert_eq!(clean, stale, "n {n} dst {dst} box ({max_x}, {max_y})"); + } + } + } +} diff --git a/crates/video/src/hevc/decode/slicedec/transform_tree.rs b/crates/video/src/hevc/decode/slicedec/transform_tree.rs index 8512c341a..1ffaebbff 100644 --- a/crates/video/src/hevc/decode/slicedec/transform_tree.rs +++ b/crates/video/src/hevc/decode/slicedec/transform_tree.rs @@ -1,7 +1,7 @@ use super::{ - CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CTX_IDX_MAP, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX, - LAST_Y_PREFIX, LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra, - scan_order, transform, + CBF_CHROMA, CBF_LUMA, CODED_SUB_BLOCK, CU_QP_DELTA_ABS, ChromaFormat, E_TU_H, E_TU_V, F_CBF, F_INTRA, GT1, GT2, LAST_X_PREFIX, LAST_Y_PREFIX, + LEVEL_SCALE, PartMode, Result, SCAN_4, SIG_COEFF, SIG_CTX, SPLIT_TRANSFORM, Sample, SliceDecoder, TRANSFORM_SKIP, ensure, for_runs, intra, + scan_order, scan_pos, transform, }; /// Significant scan positions of one sub-block, in decoding (descending) order. @@ -44,54 +44,26 @@ struct GreaterFlags { g1ctx: u32, } -/// sigCtx increment of sig_coeff_flag (9.3.4.2.5) for the position `(xp, yp)` inside the sub-block. -fn sig_ctx_inc(p: &SigMapParams, (xp, yp): (usize, usize)) -> usize { - let (xc, yc) = (p.xs * 4 + xp, p.ys * 4 + yp); +/// sigCtx increments (9.3.4.2.5) of the 16 scan positions of one sub-block. +fn sig_ctx_table(p: &SigMapParams) -> [u8; 16] { if p.log2 == 2 { - return CTX_IDX_MAP[(yc << 2) + xc] as usize; + return SIG_CTX[p.scan_idx][4]; } - if xc + yc == 0 { - return 0; - } - let mut s = match p.prev_csbf { - 0 => { - if xp + yp == 0 { - 2 - } else if xp + yp < 3 { - 1 - } else { - 0 - } - } - 1 => { - if yp == 0 { - 2 - } else if yp == 1 { - 1 - } else { - 0 - } - } - 2 => { - if xp == 0 { - 2 - } else if xp == 1 { - 1 - } else { - 0 - } - } - _ => 2, - }; - if p.c == 0 { - if p.xs + p.ys > 0 { - s += 3; - } - s += if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 }; + let off = if p.c == 0 { + (if p.xs + p.ys > 0 { 3 } else { 0 }) + if p.log2 == 3 { if p.scan_idx == 0 { 9 } else { 15 } } else { 21 } + } else if p.log2 == 3 { + 9 } else { - s += if p.log2 == 3 { 9 } else { 12 }; + 12 + }; + let mut t = SIG_CTX[p.scan_idx][p.prev_csbf as usize]; + for v in t.iter_mut() { + *v += off; } - s + if p.xs + p.ys == 0 { + t[0] = 0; + } + t } /// A node of the transform tree: the block at (`x0`, `y0`) of size 2^`log2`, its parent's origin @@ -333,14 +305,13 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { /// sig_coeff_flag decoding of one coded sub-block, appending the significant scan positions /// (descending) to `list`. fn read_sig_map(&mut self, p: &SigMapParams, list: &mut SigList) { - let pos_scan = &SCAN_4[p.scan_idx]; let c_off_sig = if p.c == 0 { 0 } else { 27 }; + let ctx_tab = sig_ctx_table(p); let mut infer_dc = p.infer_dc; let mut np = p.start; while np >= 0 { - let (xp, yp) = (pos_scan[np as usize].0 as usize, pos_scan[np as usize].1 as usize); if np > 0 || !infer_dc { - let sig_ctx = sig_ctx_inc(p, (xp, yp)); + let sig_ctx = ctx_tab[np as usize] as usize; if self.c.decode_decision(SIG_COEFF + c_off_sig + sig_ctx) == 1 { list.push(np as u8); infer_dc = false; @@ -438,9 +409,23 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { let sb_scan = scan_order(log2sb, scan_idx); let pos_scan = &SCAN_4[scan_idx]; let sb_w = 1usize << log2sb; - let last_sb = sb_scan.iter().position(|&(x, y)| x as u32 == lx >> 2 && y as u32 == ly >> 2).unwrap_or(0); - let last_pos = pos_scan.iter().position(|&(x, y)| x as u32 == lx & 3 && y as u32 == ly & 3).unwrap_or(0); - self.coeffs[..n * n].fill(0); + let last_sb = scan_pos(log2sb, scan_idx, lx >> 2, ly >> 2); + let last_pos = scan_pos(2, scan_idx, lx & 3, ly & 3); + // scaling (8.6.3), applied as each coefficient is decoded + let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c }; + let bd_shift = bd + log2 - 5; + let round = 1i64 << (bd_shift - 1); + let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6); + let matrix = match &self.pic.scaling { + Some(s) if !(bypass || (ts && n > 4)) => { + let size_id = (log2 - 2) as usize; + let matrix_id = if self.cu_intra { c } else { 3 + c }; + Some((s.clone(), size_id * 6 + matrix_id)) + } + _ => None, + }; + let m: Option<&[u8]> = matrix.as_ref().map(|(s, i)| &s[*i][..]); + let mut nnz = 0; let mut csbf = [0u8; 64]; let mut prev_g1: Option = None; let sdh = self.pps.sign_data_hiding && !bypass; @@ -485,12 +470,18 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { for k in 0..list.n { let np = list.pos[k] as usize; let (xc, yc) = (xs * 4 + pos_scan[np].0 as usize, ys * 4 + pos_scan[np].1 as usize); - self.coeffs[yc * n + xc] = values[k]; + let mut v = values[k]; + if !bypass { + let f = m.map_or(16, |m| m[yc * n + xc] as i64); + v = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32; + } + self.nz[nnz] = ((yc * n + xc) as u16, v); + nnz += 1; max_x = max_x.max(xc); max_y = max_y.max(yc); } } - self.reconstruct((x0, y0), log2, c, qp, ts, (max_x, max_y)) + self.reconstruct((x0, y0), log2, c, ts, nnz, (max_x, max_y)) } fn coeff_abs_level_remaining(&mut self, rice: u32) -> Result { @@ -508,55 +499,70 @@ impl<'a, S: Sample> SliceDecoder<'a, S> { } } - /// Scaling, transform and addition to the prediction. - fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, qp: i32, ts: bool, (max_x, max_y): (usize, usize)) -> Result<()> { + /// Transform of the scaled coefficients `nz[..nnz]` and addition to the prediction. + fn reconstruct(&mut self, (x0, y0): (i32, i32), log2: u32, c: usize, ts: bool, nnz: usize, (max_x, max_y): (usize, usize)) -> Result<()> { let n = 1usize << log2; let bd = if c == 0 { self.pic.bd_y } else { self.pic.bd_c }; - let coeffs = &mut self.coeffs[..n * n]; - let mut dc_only = false; - if !self.cu_transquant_bypass { - // scaling (8.6.3) - let bd_shift = bd + log2 - 5; - let round = 1i64 << (bd_shift - 1); - let scale = (LEVEL_SCALE[(qp % 6) as usize] as i64) << (qp / 6); - let m: Option<&[u8]> = match &self.pic.scaling { - Some(s) if !(ts && n > 4) => { - let size_id = (log2 - 2) as usize; - let matrix_id = if self.cu_intra { c } else { 3 + c }; - Some(&s[size_id * 6 + matrix_id]) - } - _ => None, - }; - for y in 0..=max_y { - for x in 0..=max_x { - let v = coeffs[y * n + x]; - if v != 0 { - let f = m.map_or(16, |m| m[y * n + x] as i64); - coeffs[y * n + x] = ((v as i64 * f * scale + round) >> bd_shift).clamp(-32768, 32767) as i32; - } - } + let bypass = self.cu_transquant_bypass; + let dst = self.cu_intra && c == 0 && n == 4; + let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1); + let off = y0 as usize * stride + x0 as usize; + let plane = &mut self.pic.planes[c]; + let bd_shift = 20 - bd; + if !bypass && !ts && max_x == 0 && max_y == 0 && !dst { + let v = transform::inverse_dc(self.nz[0].1, bd_shift); + match n { + 4 => add_dc::(plane, off, stride, v, max), + 8 => add_dc::(plane, off, stride, v, max), + 16 => add_dc::(plane, off, stride, v, max), + _ => add_dc::(plane, off, stride, v, max), } - let bd_shift = 20 - bd; + return Ok(()); + } + // the inverse transform reads only the bounding box of the coefficients + let coeffs = &mut self.coeffs[..n * n]; + if bypass || ts { + coeffs.fill(0); + } else { + for row in coeffs.chunks_exact_mut(n).take(max_y + 1) { + row[..=max_x].fill(0); + } + } + for &(p, v) in &self.nz[..nnz] { + coeffs[p as usize] = v; + } + if !bypass { if ts { transform::transform_skip(coeffs, n, bd_shift); - } else if max_x == 0 && max_y == 0 && !(self.cu_intra && c == 0 && n == 4) { - let v = transform::inverse_dc(coeffs[0], bd_shift); - coeffs.fill(v); - dc_only = true; } else { - let dst = self.cu_intra && c == 0 && n == 4; transform::inverse_transform(coeffs, n, dst, max_x, max_y, bd_shift); } } - let _ = dc_only; - let (stride, max) = (if c == 0 { self.pic.width } else { self.pic.cwidth }, (1i32 << bd) - 1); - let plane = &mut self.pic.planes[c]; - for y in 0..n { - let row = &mut plane[(y0 as usize + y) * stride + x0 as usize..][..n]; - for (x, s) in row.iter_mut().enumerate() { - *s = S::from_i32((s.to_i32() + coeffs[y * n + x]).clamp(0, max)); - } + match n { + 4 => add_residual::(plane, off, stride, coeffs, max), + 8 => add_residual::(plane, off, stride, coeffs, max), + 16 => add_residual::(plane, off, stride, coeffs, max), + _ => add_residual::(plane, off, stride, coeffs, max), } Ok(()) } } + +#[inline(always)] +fn add_residual(plane: &mut [S], off: usize, stride: usize, res: &[i32], max: i32) { + for (y, r) in res.as_chunks::().0.iter().take(N).enumerate() { + let row = &mut plane[off + y * stride..][..N]; + for (s, &d) in row.iter_mut().zip(r) { + *s = S::from_i32((s.to_i32() + d).max(0).min(max)); + } + } +} + +#[inline(always)] +fn add_dc(plane: &mut [S], off: usize, stride: usize, v: i32, max: i32) { + for y in 0..N { + for s in &mut plane[off + y * stride..][..N] { + *s = S::from_i32((s.to_i32() + v).max(0).min(max)); + } + } +} diff --git a/crates/video/src/hevc/decode/tables.rs b/crates/video/src/hevc/decode/tables.rs index 7e20bd99e..1c9c39c44 100644 --- a/crates/video/src/hevc/decode/tables.rs +++ b/crates/video/src/hevc/decode/tables.rs @@ -51,6 +51,72 @@ pub fn scan_order(log2: u32, scan_idx: usize) -> &'static [(u8, u8)] { } } +/// Inverse of a scan of an `N x N` block: scan position at `[y * N + x]`. +const fn inverse(scan: [(u8, u8); L], n: usize) -> [u8; L] { + let mut out = [0u8; L]; + let mut i = 0; + while i < L { + out[scan[i].1 as usize * n + scan[i].0 as usize] = i as u8; + i += 1; + } + out +} + +static INV_SCAN_1: [[u8; 1]; 3] = [[0], [0], [0]]; +static INV_SCAN_2: [[u8; 4]; 3] = [inverse(diag::<4>(2), 2), inverse(hv::<4>(2, false), 2), inverse(hv::<4>(2, true), 2)]; +static INV_SCAN_4: [[u8; 16]; 3] = [inverse(diag::<16>(4), 4), inverse(hv::<16>(4, false), 4), inverse(hv::<16>(4, true), 4)]; +static INV_SCAN_8: [[u8; 64]; 3] = [inverse(diag::<64>(8), 8), inverse(hv::<64>(8, false), 8), inverse(hv::<64>(8, true), 8)]; + +/// Position of (x, y) in ScanOrder[log2BlockSize][scanIdx] (log2BlockSize 0..=3). +#[inline] +pub fn scan_pos(log2: u32, scan_idx: usize, x: u32, y: u32) -> usize { + let i = ((y << log2) + x) as usize; + (match log2 { + 0 => INV_SCAN_1[scan_idx][i], + 1 => INV_SCAN_2[scan_idx][i], + 2 => INV_SCAN_4[scan_idx][i], + _ => INV_SCAN_8[scan_idx][i], + }) as usize +} + +/// sig_coeff_flag ctxInc (9.3.4.2.5) by scan position n inside a 4x4 sub-block, `[scanIdx][k][n]`: +/// k = prevCsbf (0..=3) gives sigCtx of blocks larger than 4x4 before the position and size +/// offsets; k = 4 gives ctxIdxMap of 4x4 blocks. +pub static SIG_CTX: [[[u8; 16]; 5]; 3] = [sig_ctx(diag::<16>(4)), sig_ctx(hv::<16>(4, false)), sig_ctx(hv::<16>(4, true))]; + +const fn sig_ctx(scan: [(u8, u8); 16]) -> [[u8; 16]; 5] { + let mut out = [[0u8; 16]; 5]; + let mut n = 0; + while n < 16 { + let (x, y) = (scan[n].0, scan[n].1); + out[0][n] = if x + y == 0 { + 2 + } else if x + y < 3 { + 1 + } else { + 0 + }; + out[1][n] = if y == 0 { + 2 + } else if y == 1 { + 1 + } else { + 0 + }; + out[2][n] = if x == 0 { + 2 + } else if x == 1 { + 1 + } else { + 0 + }; + out[3][n] = 2; + out[4][n] = CTX_IDX_MAP[(y as usize) * 4 + x as usize]; + n += 1; + } + out +} + /// Up-right diagonal scan for log2 size 2 or 3. pub fn scan_diag(log2: u32) -> &'static [(u8, u8)] { scan_order(log2, 0) @@ -81,7 +147,7 @@ pub fn qpc(qpi: i32, chroma_array_type_1: bool) -> i32 { } /// Table 9-50 ctxIdxMap for 4x4 sig_coeff_flag contexts. -pub static CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8]; +pub const CTX_IDX_MAP: [u8; 16] = [0, 1, 4, 5, 2, 3, 4, 5, 6, 6, 8, 8, 7, 7, 8, 8]; /// levelScale (8-309). pub static LEVEL_SCALE: [i32; 6] = [40, 45, 51, 57, 64, 72]; @@ -137,6 +203,17 @@ mod tests { assert_eq!(SCAN_4[1][1], (1, 0)); } + #[test] + fn scan_pos_inverts_scan_order() { + for log2 in 0..4 { + for scan_idx in 0..3 { + for (i, &(x, y)) in scan_order(log2, scan_idx).iter().enumerate() { + assert_eq!(scan_pos(log2, scan_idx, x as u32, y as u32), i); + } + } + } + } + #[test] fn dct_matrix_properties() { // rows are nearly orthogonal with norm ~ 64^2 * 32 diff --git a/crates/video/tests/av1_decode/fuzz_hostile.rs b/crates/video/tests/av1_decode/fuzz_hostile.rs index 05dbbe126..31cb66073 100644 --- a/crates/video/tests/av1_decode/fuzz_hostile.rs +++ b/crates/video/tests/av1_decode/fuzz_hostile.rs @@ -1,7 +1,6 @@ //! Hostile-input robustness: truncation, header-region sweeps and random mutation of small //! ffmpeg-generated AV1 streams (frame by frame) must never panic or run away. Skips without ffmpeg. -use std::panic::{AssertUnwindSafe, catch_unwind}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -20,8 +19,15 @@ impl Rng { } } -/// (name, encoder args, pix_fmt) -const SEEDS: &[(&str, &str, &str)] = &[("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p"), ("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le")]; +// Seeds are at least 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines. +/// (name, encoder args, pix_fmt, size, decoder thread counts) +const SEEDS: &[(&str, &str, &str, &str, &[usize])] = &[ + ("av1_fz_default", "-c:v libsvtav1 -g 4", "yuv420p", "64x64", &[1, 3]), + ("av1_fz_10bit", "-c:v libsvtav1 -g 6", "yuv420p10le", "64x64", &[1, 3]), + // Several superblock rows, so corrupt references meet frames already predicting from their + // top rows (row wavefront with frame threads). + ("av1_fz_rows", "-c:v libsvtav1 -g 6", "yuv420p", "64x136", &[1, 8]), +]; fn ivf_frames(data: &[u8]) -> Vec> { let mut out = Vec::new(); @@ -37,14 +43,13 @@ fn ivf_frames(data: &[u8]) -> Vec> { out } -// Seeds are 64x64: SVT-AV1 before 3.0 (the Ubuntu runners have 2.3) refuses a source under 64 lines. -fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str) -> Option>> { +fn make(ff: &std::path::Path, name: &str, args: &str, pix: &str, size: &str) -> Option>> { let out = craft_media_testkit::fixtures_dir("av1").join(format!("{name}.ivf")); let p = craft_media_testkit::fixtures::generate(&out, |tmp| { let a: Vec<&str> = args.split_whitespace().collect(); matches!( Command::new(ff) - .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", "testsrc2=size=64x64:rate=25", "-frames:v", "6", "-pix_fmt", pix]) + .args(["-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", "-i", &format!("testsrc2=size={size}:rate=25"), "-frames:v", "6", "-pix_fmt", pix]) .args(a) .args(["-f", "ivf"]) .arg(tmp) @@ -69,20 +74,29 @@ fn count_panics() { }); } -fn run_all(frames: &[Vec], label: &str) { +fn run_all(frames: &[Vec], threads: &[usize], label: &str) { craft_media_testkit::set_progress(label); count_panics(); let before = PANICS.load(Ordering::SeqCst); - let res = catch_unwind(AssertUnwindSafe(|| { - for threads in [1, 3] { - let mut dec = Decoder::with_threads(threads); - for (i, f) in frames.iter().enumerate() { - let _ = dec.decode_pts(f, i as i64); - } - let _ = dec.flush_result(); - } - })); - assert!(res.is_ok() && PANICS.load(Ordering::SeqCst) == before, "panic: {label}"); + // Every thread count decodes the stream at the same time, each on a thread of its own. + let ok = std::thread::scope(|s| { + let runs: Vec<_> = threads + .iter() + .map(|&threads| { + s.spawn(move || { + let mut dec = Decoder::with_threads(threads); + for (i, f) in frames.iter().enumerate() { + let _ = dec.decode_pts(f, i as i64); + } + let _ = dec.flush_result(); + }) + }) + .collect(); + // Join every run (no short cut), then check them all. + let joined: Vec = runs.into_iter().map(|r| r.join().is_ok()).collect(); + joined.into_iter().all(|ok| ok) + }); + assert!(ok && PANICS.load(Ordering::SeqCst) == before, "panic: {label}"); } /// Random-mutation rounds per seed (raise with FILMCRAFT_FUZZ_ITERS for a long hunt). @@ -95,17 +109,23 @@ const DEADLINE_SECS: u64 = 600; #[test] fn hostile_streams_never_panic() { - craft_media_testkit::with_deadline(DEADLINE_SECS, sweep); + craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[..2])); } -fn sweep() { +/// The seed with several superblock rows, in a test of its own so it runs beside the others. +#[test] +fn hostile_streams_with_superblock_rows_never_panic() { + craft_media_testkit::with_deadline(DEADLINE_SECS, || sweep(&SEEDS[2..])); +} + +fn sweep(seeds: &[(&str, &str, &str, &str, &[usize])]) { let Some(ff) = craft_media_testkit::ffmpeg_or_skip("AV1 fuzz") else { return }; - for &(name, args, pix) in SEEDS { + for &(name, args, pix, size, threads) in seeds { if !craft_media_testkit::ffmpeg_has_encoder(&ff, "libsvtav1") { craft_media_testkit::skip_test("AV1 fuzz", &format!("this ffmpeg has no libsvtav1 encoder (seed {name})")); continue; } - let Some(seed) = make(&ff, name, args, pix) else { + let Some(seed) = make(&ff, name, args, pix, size) else { panic!("ffmpeg has libsvtav1 but could not encode the {name} seed"); }; if seed.is_empty() { @@ -118,7 +138,7 @@ fn sweep() { for n in (0..len).step_by((len / 200).max(1)) { let mut d = seed.clone(); d[fi].truncate(n); - run_all(&d, &format!("{name} frame {fi} truncate {n}")); + run_all(&d, threads, &format!("{name} frame {fi} truncate {n}")); } } // header region of every frame: every byte to a few values, every bit @@ -127,12 +147,12 @@ fn sweep() { for v in [0u8, 0xff, 0x80, 0x01, 0x7f] { let mut d = seed.clone(); d[fi][p] = v; - run_all(&d, &format!("{name} frame {fi} byte {p}={v:#x}")); + run_all(&d, threads, &format!("{name} frame {fi} byte {p}={v:#x}")); } for b in 0..8 { let mut d = seed.clone(); d[fi][p] ^= 1 << b; - run_all(&d, &format!("{name} frame {fi} flip {p}.{b}")); + run_all(&d, threads, &format!("{name} frame {fi} flip {p}.{b}")); } } } @@ -151,7 +171,7 @@ fn sweep() { d[fi][p] = rng.next() as u8; } } - run_all(&d, &format!("{name} random {i}")); + run_all(&d, threads, &format!("{name} random {i}")); } } } diff --git a/crates/video/tests/av1_decode/main.rs b/crates/video/tests/av1_decode/main.rs index 50c3939f6..76a5adfcf 100644 --- a/crates/video/tests/av1_decode/main.rs +++ b/crates/video/tests/av1_decode/main.rs @@ -9,3 +9,4 @@ mod hostile_dims; mod oracle_inter; mod oracle_intra; mod perf; +mod wavefront; diff --git a/crates/video/tests/av1_decode/wavefront.rs b/crates/video/tests/av1_decode/wavefront.rs new file mode 100644 index 000000000..725b02544 --- /dev/null +++ b/crates/video/tests/av1_decode/wavefront.rs @@ -0,0 +1,93 @@ +//! Stress tests of the frame threading (parse / reconstruct split and the row wavefront across +//! frames): every thread count gives the single-thread pictures, run after run, and dropping +//! or flushing a busy decoder never hangs. A hang fails the test through the deadline. + +use crate::common; +use common::*; +use craft_video::av1::decode::{Decoder, Picture}; + +/// A GOP with many superblock rows (the wavefront has rows to overlap), hierarchical +/// references and motion reaching across rows. +fn spec() -> Spec { + Spec::new("wavefront_640x480", "testsrc2=s=640x480:r=25,rotate=a=t*0.5:c=black", 24, "yuv420p", &["-preset", "8", "-crf", "38"]) +} + +/// Seconds a test may take before it counts as a hang (override: FILMCRAFT_HOSTILE_TIMEOUT_SECS). +const DEADLINE_SECS: u64 = 300; + +fn decode(tus: &[Vec], threads: usize) -> Vec { + let mut dec = Decoder::with_threads(threads); + let mut pics = Vec::new(); + for (i, tu) in tus.iter().enumerate() { + pics.extend(dec.decode_pts(tu, i as i64).unwrap_or_else(|e| panic!("threads {threads}, unit {i}: {e}"))); + } + pics.extend(dec.flush_result().unwrap_or_else(|e| panic!("threads {threads}, flush: {e}"))); + pics +} + +#[test] +fn every_thread_count_gives_the_single_thread_pictures_repeatedly() { + let Some(ff) = ffmpeg() else { return }; + let tus = ivf_frames(&make(&ff, &spec())); + craft_media_testkit::with_deadline(DEADLINE_SECS, move || { + let want = decode(&tus, 1); + assert_eq!(want.len(), 24); + for threads in [2, 3, 4, 5, 8, 16] { + for round in 0..4 { + assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ"); + } + } + }); +} + +/// 128x128 superblocks in two tile columns (libaom), with motion field estimation: the +/// wavefront over the superblock rows of every tile column. +fn spec_sb128_tiles() -> Spec { + let enc = ["-cpu-used", "8", "-crf", "40", "-tile-columns", "1", "-lag-in-frames", "16", "-aom-params", "sb-size=128:enable-ref-frame-mvs=1"]; + Spec::new("wavefront_sb128_tiles", "testsrc2=s=640x400:r=25,rotate=a=t*0.5:c=black", 16, "yuv420p", &enc).encoder("libaom-av1") +} + +#[test] +fn superblocks_of_128_in_tile_columns_give_the_single_thread_pictures() { + let Some(ff) = ffmpeg_libaom() else { return }; + let tus = ivf_frames(&make(&ff, &spec_sb128_tiles())); + craft_media_testkit::with_deadline(DEADLINE_SECS, move || { + let want = decode(&tus, 1); + assert_eq!(want.len(), 16); + for threads in [2, 3, 4, 8, 16] { + for round in 0..2 { + assert!(decode(&tus, threads) == want, "threads {threads}, round {round}: pictures differ"); + } + } + }); +} + +#[test] +fn dropping_or_flushing_a_busy_decoder_never_hangs() { + let Some(ff) = ffmpeg() else { return }; + let tus = ivf_frames(&make(&ff, &spec())); + craft_media_testkit::with_deadline(DEADLINE_SECS, move || { + let want = decode(&tus, 1); + for threads in [2, 3, 8, 16] { + for cut in [1, 2, 7, 12, 23] { + // Dropped with frames in flight (no flush). + let mut dec = Decoder::with_threads(threads); + for (i, tu) in tus.iter().take(cut).enumerate() { + dec.decode_pts(tu, i as i64).expect("decode"); + } + drop(dec); + // Flushed mid-stream (as before a seek), then continued: the same pictures. + let mut dec = Decoder::with_threads(threads); + let mut pics = Vec::new(); + for (i, tu) in tus.iter().enumerate() { + if i == cut { + pics.extend(dec.flush_result().expect("flush")); + } + pics.extend(dec.decode_pts(tu, i as i64).expect("decode")); + } + pics.extend(dec.flush_result().expect("flush")); + assert!(pics == want, "threads {threads}, flush before unit {cut}: pictures differ"); + } + } + }); +} diff --git a/crates/wordmodel/src/chart.rs b/crates/wordmodel/src/chart.rs index b0cb6ad60..6c7296647 100644 --- a/crates/wordmodel/src/chart.rs +++ b/crates/wordmodel/src/chart.rs @@ -11,6 +11,8 @@ use serde::{Deserialize, Serialize}; +use crate::props::Rgb; + /// Most series in a chart. pub const MAX_SERIES: usize = 255; /// Most categories (points per series) in a chart. @@ -123,17 +125,111 @@ impl LegendPos { } } -/// One series: its name and a value per category (`None` = no value, a gap). +/// One series: its name, a value per category (`None` = no value, a gap) and how it is drawn +/// (Chart Design ▸ Format Series): fill, line, markers, smooth lines, its own data labels and +/// number format. Unset fields follow the chart's defaults. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ChartSeries { pub name: String, #[serde(default)] pub values: Vec>, + /// Fill colour; `None`: the chart's accent colour for the series. + #[serde(default)] + pub fill: Option, + /// "None" fill: the series draws no fill. + #[serde(default)] + pub no_fill: bool, + /// Line width in points; `None`: the chart's default. + #[serde(default)] + pub line_w: Option, + /// "None" line: the series draws no line. + #[serde(default)] + pub no_line: bool, + /// Markers on points; `None`: unset (scatter charts draw them; a saved scatter series then has no `c:marker`, so Word picks its own symbols). + #[serde(default)] + pub markers: Option, + /// Draw the line as a smooth curve. + #[serde(default)] + pub smooth: bool, + /// Data labels for this series; `None`: follow the chart's setting. + #[serde(default)] + pub labels: Option, + /// Number format of this series's labels and values; `None`: linked to the source. + #[serde(default)] + pub fmt: Option, +} + +/// One chart axis as Word formats it (Format Axis): title, scale bounds, tick-label number +/// format, gridlines, tick labels and the axis line. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ChartAxis { + /// `None`: no axis title; text cut to [`MAX_TEXT`] characters. + #[serde(default)] + pub title: Option, + /// Fixed lower bound; `None`: automatic. + #[serde(default)] + pub min: Option, + /// Fixed upper bound; `None`: automatic. + #[serde(default)] + pub max: Option, + /// Tick-label number format (`#,##0`, `0.00`, `0%`…); `None`: linked to the source data. + #[serde(default)] + pub fmt: Option, + /// Major gridlines drawn along this axis. + #[serde(default)] + pub gridlines: bool, + /// Tick labels; `false` hides the axis the way Word's Axis Labels ▸ None does. + #[serde(default = "the_true")] + pub labels: bool, + /// The axis line. + #[serde(default = "the_true")] + pub line: bool, +} + +fn the_true() -> bool { + true +} + +impl Default for ChartAxis { + fn default() -> Self { + ChartAxis { title: None, min: None, max: None, fmt: None, gridlines: false, labels: true, line: true } + } +} + +/// The value axis as a new chart has it: gridlines, labels, no axis line. +pub fn value_axis() -> ChartAxis { + ChartAxis { gridlines: true, line: false, ..ChartAxis::default() } +} + +impl ChartAxis { + /// Make the axis safe to store: texts cleaned, bounds finite, `min` under `max` (an + /// impossible pair drops both), the format code cut to [`MAX_FMT`] characters. + pub fn sanitize(&mut self) { + self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty()); + self.min = self.min.filter(|v| v.is_finite()); + self.max = self.max.filter(|v| v.is_finite()); + if matches!((self.min, self.max), (Some(lo), Some(hi)) if lo >= hi) { + self.min = None; + self.max = None; + } + self.fmt = self.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty()); + } +} + +/// Longest axis number format kept, characters. +pub const MAX_FMT: usize = 128; + +/// A number format code: control characters as spaces, ends trimmed, at most [`MAX_FMT`] +/// characters. +pub fn clean_fmt(s: &str) -> String { + let s: String = s.chars().take(MAX_FMT * 4).map(|c| if c.is_control() { ' ' } else { c }).collect(); + s.trim().chars().take(MAX_FMT).collect::().trim_end().to_string() } /// A chart the app can edit. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ChartSpec { #[serde(rename = "type")] @@ -151,6 +247,29 @@ pub struct ChartSpec { /// Show each point's value next to it. #[serde(default)] pub data_labels: bool, + /// The category axis (scatter charts: the x axis). + #[serde(default)] + pub cat_axis: ChartAxis, + /// The value axis (scatter charts: the y axis); gridlines on, no axis line, as a new chart has them. + #[serde(default = "value_axis")] + pub val_axis: ChartAxis, +} + +impl Default for ChartSpec { + /// An empty chart whose value axis is the one a new chart (and a missing JSON field) has: + /// gridlines on, no axis line. + fn default() -> Self { + ChartSpec { + kind: ChartType::default(), + categories: Vec::new(), + series: Vec::new(), + title: None, + legend: None, + data_labels: false, + cat_axis: ChartAxis::default(), + val_axis: value_axis(), + } + } } impl ChartSpec { @@ -168,10 +287,23 @@ impl ChartSpec { ChartSpec { kind, categories: categories.into_iter().map(str::to_string).collect(), - series: series.into_iter().map(|(n, v)| ChartSeries { name: n.to_string(), values: v.into_iter().map(Some).collect() }).collect(), + series: series + .into_iter() + .map(|(n, v)| ChartSeries { + name: n.to_string(), + values: v.into_iter().map(Some).collect(), + // Scatter series are drawn as markers without a line. + no_line: kind == ChartType::Scatter, + ..Default::default() + }) + .collect(), title: None, legend: Some(LegendPos::Bottom), data_labels: false, + cat_axis: ChartAxis::default(), + // Pie and doughnut charts have no axes: their axes stay plain, so the chart reads back + // as the same model. + val_axis: if kind.is_round() { ChartAxis::default() } else { value_axis() }, } } @@ -203,8 +335,12 @@ impl ChartSpec { *v = v.filter(|x| x.is_finite()); } s.values.resize(n, None); + s.fmt = s.fmt.as_deref().map(clean_fmt).filter(|f| !f.is_empty()); + s.line_w = s.line_w.filter(|w| w.is_finite()).map(|w| w.clamp(0.0, 20.0)); } self.title = self.title.as_deref().map(clean_text).filter(|t| !t.is_empty()); + self.cat_axis.sanitize(); + self.val_axis.sanitize(); } /// The chart after `sanitize`. @@ -229,7 +365,11 @@ mod tests { let mut c = ChartSpec { categories: vec!["a".into(); MAX_POINTS + 10], series: (0..MAX_SERIES + 5) - .map(|i| ChartSeries { name: format!("{}\n{}", "x".repeat(400), i), values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)] }) + .map(|i| ChartSeries { + name: format!("{}\n{}", "x".repeat(400), i), + values: vec![Some(f64::NAN), Some(f64::INFINITY), Some(1.0)], + ..Default::default() + }) .collect(), title: Some(" ".into()), ..Default::default() @@ -250,7 +390,7 @@ mod tests { fn short_categories_are_padded_to_the_longest_series() { let c = ChartSpec { categories: vec!["a".into()], - series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)] }], + series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0), Some(2.0)], ..Default::default() }], ..Default::default() } .sanitized(); @@ -258,4 +398,53 @@ mod tests { assert_eq!(ChartType::from_id("BARSTACKED"), Some(ChartType::BarStacked)); assert_eq!(LegendPos::from_id("top"), Some(LegendPos::Top)); } + + #[test] + fn default_and_missing_json_axis_fields_agree() { + let from_json: ChartSpec = serde_json::from_str(r#"{"type":"column"}"#).unwrap(); + let d = ChartSpec::default(); + assert_eq!((d.cat_axis.clone(), d.val_axis.clone()), (from_json.cat_axis, from_json.val_axis)); + assert!(d.val_axis.gridlines && !d.val_axis.line); + } + + #[test] + fn axis_bounds_and_formats_are_cleaned() { + let mut a = ChartAxis { + title: Some(" Sales\n".into()), + min: Some(f64::NAN), + max: Some(50.0), + fmt: Some("x".repeat(MAX_FMT + 40)), + gridlines: true, + labels: false, + line: true, + }; + a.sanitize(); + assert_eq!(a.title.as_deref(), Some("Sales")); + assert_eq!(a.min, None); + assert_eq!(a.max, Some(50.0)); + assert_eq!(a.fmt.as_deref().map(|f| f.chars().count()), Some(MAX_FMT)); + + // min not under max drops the pair; a format of only spaces is no format. + let mut b = ChartAxis { min: Some(50.0), max: Some(50.0), fmt: Some(" ".into()), ..Default::default() }; + b.sanitize(); + assert_eq!((b.min, b.max), (None, None)); + assert_eq!(b.fmt, None); + + let mut spec = ChartSpec { + series: vec![ChartSeries { name: "s".into(), values: vec![Some(1.0)], ..Default::default() }], + categories: vec!["a".into()], + ..Default::default() + }; + spec.series[0].line_w = Some(f32::NAN); + spec.series[0].fmt = Some("0.00".into()); + spec.sanitize(); + assert_eq!(spec.series[0].line_w, None); + assert_eq!(spec.series[0].fmt.as_deref(), Some("0.00")); + let mut spec = ChartSpec { + series: vec![ChartSeries { name: "s".into(), values: vec![], line_w: Some(100.0), ..Default::default() }], + ..Default::default() + }; + spec.sanitize(); + assert_eq!(spec.series[0].line_w, Some(20.0), "a wild width clamps to 20 pt"); + } } diff --git a/crates/wordmodel/src/graphic.rs b/crates/wordmodel/src/graphic.rs index c77c0e5e7..cba7e766a 100644 --- a/crates/wordmodel/src/graphic.rs +++ b/crates/wordmodel/src/graphic.rs @@ -115,7 +115,9 @@ pub enum GraphicItem { #[serde(default)] stroke_width: f32, }, - /// One line of text in `rect`, aligned horizontally by `align`, centred vertically. + /// One line of text in `rect`, aligned horizontally by `align`, centred vertically; `angle` + /// degrees clockwise turns the text about `rect`'s centre as it is drawn (chart axis titles + /// turn the vertical axis's title -90°, reading upwards, as Word does). Text { rect: [f32; 4], text: String, @@ -127,6 +129,8 @@ pub enum GraphicItem { align: TextAlign, #[serde(default)] font: Option, + #[serde(default)] + angle: f32, }, /// A picture (`Document::media` key) stretched into `rect`. Image { rect: [f32; 4], media: String }, diff --git a/crates/xlsx/README.md b/crates/xlsx/README.md index b8669288e..9d8e1b785 100644 --- a/crates/xlsx/README.md +++ b/crates/xlsx/README.md @@ -83,17 +83,22 @@ for new pictures (`vm = i + 1`) and the `XLDAPR` dynamic-array metadata (`cm = 1 The parts this crate writes use the transitional namespaces; a strict file's kept parts stay as they were. A binary workbook is not written. -A binary workbook gives the same `Workbook`, strings, rows and merged cells. Its styles are not -read (each cell keeps its format index). Its cell formulas are decoded from their tokens into the -text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from the four -ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks typed in -the formula, numbers as Excel spells them), with -`FormulaKind::Shared` or `FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell -metadata index. Every cell of a shared formula gets its own text, decoded at the cell, since the -file keeps the tokens once. A formula whose decoding needs something the reader does not read -keeps only its saved result, with a warning saying why: user defined names, table references, -other workbooks, Python code stored outside the cell, data tables. The XLSB function table -(`xlsb/ftab.rs`) and the expression printer are this crate's own, apart from `craft-xls`'s. +A binary workbook gives the same `Workbook`, strings, rows and merged cells, and the same `Styles`: +number formats, fonts, fills, borders, cell and cell style formats, cell styles, differential +formats and the indexed colours, as Excel writes them in `styles.xml` (default fill colours and +empty settings left out, a cell style format's `apply*="0"`, a number format with a newer locale tag +in its Excel 2007 spelling, as `formatCode` has it). Custom table styles and recent colours are not +read; a damaged styles part is a warning and no styles, while one past a cap (65,536 entries in a +table, 1,024 gradient stops) or past `binary_text` is too large. Its cell formulas are decoded from +their tokens into the text an `.xlsx` file holds (`_xlfn.` and `_xlpm.` names as written, apart from +the four ISO/IEC 29500 functions `.xlsx` names bare, no implicit `@`, the spaces and line breaks +typed in the formula, numbers as Excel spells them), with `FormulaKind::Shared` or +`FormulaKind::Array` and their ranges, and `BrtCellMeta` as the cell metadata index. Every cell of a +shared formula gets its own text, decoded at the cell, since the file keeps the tokens once. A +formula whose decoding needs something the reader does not read keeps only its saved result, with a +warning saying why: user defined names, table references, other workbooks, Python code stored +outside the cell, data tables. The XLSB function table (`xlsb/ftab.rs`) and the expression printer +are this crate's own, apart from `craft-xls`'s. ## Hostile input diff --git a/crates/xlsx/src/document.rs b/crates/xlsx/src/document.rs index 26e6a84c6..1be23029e 100644 --- a/crates/xlsx/src/document.rs +++ b/crates/xlsx/src/document.rs @@ -209,8 +209,8 @@ pub struct Document { } impl Document { - /// Reads every part. A binary workbook is refused: its formulas and styles are not read, so - /// it cannot be written back as SpreadsheetML without loss. + /// Reads every part. A binary workbook is refused: only some of its parts are read, so it + /// cannot be written back as SpreadsheetML without loss. pub fn read(bytes: &[u8], limits: &Limits) -> Result { let mut p = Package::open(bytes, limits)?; if p.is_binary() { diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs index d43a15a8c..62076b3fc 100644 --- a/crates/xlsx/src/package.rs +++ b/crates/xlsx/src/package.rs @@ -220,16 +220,28 @@ impl<'a> Package<'a> { Ok(items) } - /// The styles part, `None` when the workbook has none (or is binary: its styles are not read). + /// The styles part, `None` when the workbook has none. A damaged one is a warning and `None`; + /// only a limit is an error. pub fn styles(&mut self) -> Result, Error> { - if self.binary.is_some() { - return Ok(None); - } let rels = self.workbook_rels()?; let Some(rel) = rels.first("styles").filter(|r| !r.external) else { self.warn("no styles part"); return Ok(None); }; + if let Some(types) = &self.binary { + let read = xlsb::check_type(types, &rel.target, xlsb::STYLES).and_then(|()| { + let bytes = self.opc.read(&rel.target)?.ok_or_else(|| xlsb::invalid(format!("missing part {}", rel.target)))?; + xlsb::styles(&bytes, &self.limits, &self.budget, &mut self.warnings) + }); + return match read { + Ok(styles) => Ok(Some(styles)), + Err(Error::TooLarge(m)) => Err(Error::TooLarge(m)), + Err(e) => { + self.warn(format!("skipped {}: {e}", rel.target)); + Ok(None) + } + }; + } Ok(self.optional_xml(&rel.target.clone())?.map(Styles::read)) } diff --git a/crates/xlsx/src/xlsb.rs b/crates/xlsx/src/xlsb.rs index b3f94b5fa..c8fdba69f 100644 --- a/crates/xlsx/src/xlsb.rs +++ b/crates/xlsx/src/xlsb.rs @@ -1,11 +1,13 @@ //! Binary workbooks ([MS-XLSB]): the workbook part, shared strings and worksheet cells, from //! their records. Cell formulas are decoded from their tokens ([`formula`]) into the text an //! `.xlsx` file holds; one that can't be keeps only its saved result, with a warning saying why. -//! Styles are not read. A record that does not parse fails the read. +//! The styles part is read into the same [`crate::styles::Styles`] ([`styles`]). A record that +//! does not parse fails the read. mod expr; mod formula; mod ftab; +mod styles; use std::collections::HashMap; use std::sync::Arc; @@ -19,11 +21,13 @@ use crate::workbook::{SheetEntry, SheetKind, Workbook}; use crate::{Error, Limits, Warnings}; pub(crate) use formula::Links; use formula::{Book, Mode, Name, Xti}; +pub(crate) use styles::styles; pub(crate) const MAIN: &str = "application/vnd.ms-excel.main"; const BINARY_MAIN: &str = "application/vnd.ms-excel.sheet.binary.macroEnabled.main"; pub(crate) const WORKSHEET: &str = "application/vnd.ms-excel.worksheet"; pub(crate) const STRINGS: &str = "application/vnd.ms-excel.sharedStrings"; +pub(crate) const STYLES: &str = "application/vnd.ms-excel.styles"; /// Most sheets in the workbook. const MAX_SHEETS: usize = 1024; /// Supporting links (this workbook, add-ins, other workbooks) read at most; past it, formulas @@ -174,12 +178,16 @@ struct Records<'a, 'l> { bytes: Bytes<'a>, limits: &'l Limits, budget: &'l Budget, - blocks: Vec, + /// Each open block's end record, and whether its records are read. + blocks: Vec<(u16, bool)>, + /// Read the alternate content meant for Excel 2007 (version 12), as an `.xlsx` reader sees + /// the main attribute where Excel keeps a newer spelling in an extension. + excel_2007: bool, } impl<'a, 'l> Records<'a, 'l> { fn new(bytes: &'a [u8], limits: &'l Limits, budget: &'l Budget) -> Self { - Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![] } + Self { bytes: Bytes::new(bytes), limits, budget, blocks: vec![], excel_2007: false } } fn next(&mut self, warnings: &mut Warnings) -> Result, Error> { @@ -208,6 +216,7 @@ impl<'a, 'l> Records<'a, 'l> { // BrtFRTBegin, BrtACBegin. 35 | 37 => { let mut b = Bytes::new(payload); + let mut read = false; if id == 35 { b.take(4)?; } else { @@ -215,22 +224,32 @@ impl<'a, 'l> Records<'a, 'l> { if count == 0 { return Err(invalid("empty alternate-content version list")); } - b.take(usize::from(count) * 4)?; + // ACProductVersion: fileVersion, then fileProduct (15 bits) and + // fileExtension (the version and later). + for _ in 0..count { + let (version, product) = (b.u16()?, b.u16()?); + let later = product & 0x8000 != 0; + read |= self.excel_2007 && product & 0x7FFF == 0 && (version == 12 || later && version <= 12); + } } b.finish()?; if self.blocks.len() >= 64 { return Err(invalid("extension blocks nested too deeply")); } - self.blocks.push(id + 1); - warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); + // Inside a skipped block, everything is skipped. + let read = read && self.blocks.iter().all(|(_, r)| *r); + self.blocks.push((id + 1, read)); + if !read { + warnings.warn("unsupported XLSB future-record and alternate-content blocks were skipped"); + } } // BrtFRTEnd, BrtACEnd. 36 | 38 => { - if self.blocks.pop() != Some(id) || !payload.is_empty() { + if self.blocks.pop().map(|(end, _)| end) != Some(id) || !payload.is_empty() { return Err(invalid("mismatched extension block end")); } } - _ if !self.blocks.is_empty() => {} + _ if self.blocks.iter().any(|(_, read)| !read) => {} _ => return Ok(Some((id, payload))), } } diff --git a/crates/xlsx/src/xlsb/styles.rs b/crates/xlsx/src/xlsb/styles.rs new file mode 100644 index 000000000..88c6bee51 --- /dev/null +++ b/crates/xlsx/src/xlsb/styles.rs @@ -0,0 +1,484 @@ +//! The styles part of a binary workbook (`styles.bin`, [MS-XLSB] 2.1.7.50), read into the same +//! [`Styles`] the XML part gives: what Excel writes in `styles.xml` for the same workbook. Table +//! styles, recent colours and future records are not read. + +use craft_sheet::{BorderStyle, HAlign, Pattern, Script, Underline, VAlign}; +use craft_xml::El; + +use super::{Budget, Bytes, Records, invalid}; +use crate::styles::{ + Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, NumFmt, PatternFill, Protection, Styles, + Xf, +}; +use crate::{Error, Limits, Warnings}; + +/// Characters of a font name or number format kept, as the XML side keeps of a font name. +const MAX_NAME_CHARS: usize = 255; +/// Longest string read from the part (Excel's own limits are 31 and 255 characters). +const MAX_STRING_UNITS: u32 = 32_767; +/// Most stops in one gradient fill (Excel writes a handful). +const MAX_STOPS: u32 = 1024; +/// Most entries in one table of the part (Excel allows about 64,000 cell formats). Each entry is +/// an allocation, and the record allowance alone would let a small part hold millions. +const MAX_ENTRIES: usize = 65_536; + +/// `BrtColor` and `XFPropColor`: the same eight bytes. `None` for "not set" (`XFPropColor` type 4). +fn color(b: &mut Bytes<'_>) -> Result, Error> { + let kind = b.u8()? >> 1; + let index = b.u8()?; + let tint = i16::from_le_bytes(<[u8; 2]>::try_from(b.take(2)?).map_err(|_| invalid("truncated colour"))?); + let [r, g, bl, a] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?; + let source = match kind { + 0 => ColorSource::Auto, + 1 => ColorSource::Indexed(u32::from(index)), + 2 => ColorSource::Rgb(u32::from_be_bytes([a, r, g, bl])), + 3 => ColorSource::Theme(u32::from(index)), + _ => return Ok(None), + }; + Ok(Some(Color { source, tint: (f64::from(tint) / 32767.0).clamp(-1.0, 1.0) })) +} + +fn pattern(fls: u32) -> Result { + const PATTERNS: [Pattern; 19] = [ + Pattern::None, + Pattern::Solid, + Pattern::Gray50, + Pattern::Gray75, + Pattern::Gray25, + Pattern::DarkHorizontal, + Pattern::DarkVertical, + Pattern::DarkDown, + Pattern::DarkUp, + Pattern::DarkGrid, + Pattern::DarkTrellis, + Pattern::LightHorizontal, + Pattern::LightVertical, + Pattern::LightDown, + Pattern::LightUp, + Pattern::LightGrid, + Pattern::LightTrellis, + Pattern::Gray125, + Pattern::Gray0625, + ]; + PATTERNS.get(fls as usize).copied().ok_or_else(|| invalid("unknown fill pattern")) +} + +fn border_style(dg: u16) -> Result { + const STYLES: [BorderStyle; 14] = [ + BorderStyle::None, + BorderStyle::Thin, + BorderStyle::Medium, + BorderStyle::Dashed, + BorderStyle::Dotted, + BorderStyle::Thick, + BorderStyle::Double, + BorderStyle::Hair, + BorderStyle::MediumDashed, + BorderStyle::DashDot, + BorderStyle::MediumDashDot, + BorderStyle::DashDotDot, + BorderStyle::MediumDashDotDot, + BorderStyle::SlantDashDot, + ]; + STYLES.get(usize::from(dg)).copied().ok_or_else(|| invalid("unknown border style")) +} + +fn halign(alc: u32) -> Option { + [HAlign::General, HAlign::Left, HAlign::Center, HAlign::Right, HAlign::Fill, HAlign::Justify, HAlign::CenterAcross, HAlign::Distributed] + .get(alc as usize) + .copied() +} + +fn valign(alcv: u32) -> Option { + [VAlign::Top, VAlign::Center, VAlign::Bottom, VAlign::Justify, VAlign::Distributed].get(alcv as usize).copied() +} + +fn underline(uls: u32) -> Underline { + match uls { + 0 => Underline::None, + 2 => Underline::Double, + 0x21 => Underline::SingleAccounting, + 0x22 => Underline::DoubleAccounting, + _ => Underline::Single, + } +} + +fn script(sss: u32) -> Script { + match sss { + 1 => Script::Superscript, + 2 => Script::Subscript, + _ => Script::Baseline, + } +} + +fn scheme(s: u32) -> Option { + match s { + 0 => Some(FontScheme::None), + 1 => Some(FontScheme::Major), + 2 => Some(FontScheme::Minor), + _ => None, + } +} + +fn name(text: String) -> String { + text.chars().take(MAX_NAME_CHARS).collect() +} + +/// `BrtFont`: only what Excel's `.xlsx` states for the same font (no ``, no scheme +/// `none`, no family or character set 0). +fn font(b: &mut Bytes<'_>) -> Result { + let height = b.u16()?; + let flags = b.u16()?; + let weight = b.u16()?; + let sss = b.u16()?; + let uls = b.u8()?; + let family = b.u8()?; + let charset = b.u8()?; + b.u8()?; + let color = color(b)?; + let font_scheme = b.u8()?; + let font_name = name(b.text(MAX_STRING_UNITS)?); + let on = |bit: u16| (flags & bit != 0).then_some(true); + Ok(Font { + name: Some(font_name), + size: Some(f64::from(height) / 20.0), + bold: (weight >= 0x2BC).then_some(true), + italic: on(0x02), + strike: on(0x08), + outline: on(0x10), + shadow: on(0x20), + condense: on(0x40), + extend: on(0x80), + underline: (uls != 0).then(|| underline(u32::from(uls))), + vert_align: (sss != 0).then(|| script(u32::from(sss))), + family: (family != 0).then_some(u32::from(family)), + charset: (charset != 0).then_some(u32::from(charset)), + scheme: scheme(u32::from(font_scheme)).filter(|s| *s != FontScheme::None), + // An automatic colour is a font with no `` in `.xlsx`. + color: color.filter(|c| c.source != ColorSource::Auto), + }) +} + +/// A gradient's angle and inner rectangle, as `BrtFill` and `XFPropGradient` hold them. +fn gradient(b: &mut Bytes<'_>, path: bool) -> Result { + Ok(GradientFill { path, degree: b.f64()?, left: b.f64()?, right: b.f64()?, top: b.f64()?, bottom: b.f64()?, stops: vec![] }) +} + +/// `BrtFill`. Excel's `.xlsx` leaves out the default colours: system foreground (indexed 64) as +/// the pattern colour and system background (indexed 65) as the background, unless the pattern +/// colour has a tint (as the built-in "20% - Accent1" styles do), when it states the background. +fn fill(b: &mut Bytes<'_>) -> Result { + let fls = b.u32()?; + let fg = color(b)?; + let bg = color(b)?; + let kind = b.u32()?; + let mut g = gradient(b, kind == 1)?; + let count = b.u32()?; + if fls == 0x28 { + if count > MAX_STOPS { + return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops"))); + } + for _ in 0..count { + let c = color(b)?.unwrap_or_default(); + g.stops.push((b.f64()?, c)); + } + return Ok(Fill::Gradient(g)); + } + let system = |c: Color, i: u32| c.source == ColorSource::Indexed(i) && c.tint == 0.0; + let tinted = fg.is_some_and(|c| c.tint != 0.0); + Ok(Fill::Pattern(PatternFill { pattern: Some(pattern(fls)?), fg: fg.filter(|c| !system(*c, 64)), bg: bg.filter(|c| tinted || !system(*c, 65)) })) +} + +/// `BrtBorder`: the five edges a cell border has, each stated (`` when it has no line). +fn border(b: &mut Bytes<'_>) -> Result { + let flags = b.u8()?; + let mut edge = || -> Result { + let style = border_style(u16::from(b.u8()?))?; + b.u8()?; + let c = color(b)?; + Ok(BorderEdge { style, color: c.filter(|_| style != BorderStyle::None) }) + }; + let (top, bottom, left, right, diagonal) = (edge()?, edge()?, edge()?, edge()?, edge()?); + Ok(Border { + left: Some(left), + right: Some(right), + top: Some(top), + bottom: Some(bottom), + diagonal: Some(diagonal), + vertical: None, + horizontal: None, + diagonal_up: flags & 2 != 0, + diagonal_down: flags & 1 != 0, + outline: true, + }) +} + +/// `BrtXF`. A cell style format (`style`) marks what it leaves out (`apply*="0"`); a cell format +/// marks what it sets apart from its style (`apply*="1"`). +fn xf(b: &mut Bytes<'_>, style: bool) -> Result { + const APPLY: [(u8, &str); 6] = [ + (0x01, "applyNumberFormat"), + (0x02, "applyFont"), + (0x10, "applyFill"), + (0x08, "applyBorder"), + (0x04, "applyAlignment"), + (0x20, "applyProtection"), + ]; + let parent = b.u16()?; + let num_fmt_id = u32::from(b.u16()?); + let font_id = u32::from(b.u16()?); + let fill_id = u32::from(b.u16()?); + let border_id = u32::from(b.u16()?); + let rotation = u32::from(b.u8()?); + let indent = u32::from(b.u8()?); + let flags = u32::from(b.u16()?); + let apply = b.u8()?; + b.u8()?; + let alignment = Alignment { + horizontal: halign(flags & 7).filter(|h| *h != HAlign::General), + vertical: valign((flags >> 3) & 7).filter(|v| *v != VAlign::Bottom), + text_rotation: (rotation != 0).then_some(rotation), + wrap_text: flags & 0x40 != 0, + justify_last_line: flags & 0x80 != 0, + shrink_to_fit: flags & 0x100 != 0, + indent: (indent != 0).then_some(indent), + reading_order: Some((flags >> 10) & 3).filter(|r| *r != 0), + }; + let protection = Protection { locked: (flags & 0x1000 == 0).then_some(false), hidden: (flags & 0x2000 != 0).then_some(true) }; + let mut el = El::new("xf"); + for (bit, attr) in APPLY { + if apply & bit != 0 { + el = el.with_attr(attr, if style { "0" } else { "1" }); + } + } + Ok(Xf { + num_fmt_id, + font_id, + fill_id, + border_id, + xf_id: (!style).then_some(u32::from(parent)), + alignment: (alignment != Alignment::default()).then_some(alignment), + protection: (protection != Protection::default()).then_some(protection), + quote_prefix: flags & 0x8000 != 0, + el, + }) +} + +/// `BrtStyle`. +fn cell_style(b: &mut Bytes<'_>) -> Result { + let xf_id = b.u32()?; + let flags = b.u16()?; + let builtin = b.u8()?; + let level = b.u8()?; + let style_name = b.string(MAX_STRING_UNITS, true)?.map(name).unwrap_or_default(); + let is_builtin = flags & 1 != 0; + Ok(CellStyle { + name: style_name, + xf_id, + builtin_id: is_builtin.then_some(u32::from(builtin)), + level: (is_builtin && matches!(builtin, 1 | 2)).then_some(u32::from(level)), + hidden: flags & 2 != 0, + custom_builtin: flags & 4 != 0, + }) +} + +/// An unsigned integer of the size a property has (1, 2 or 4 bytes). +fn uint(data: &[u8]) -> Result { + match *data { + [a] => Ok(u32::from(a)), + [a, b] => Ok(u32::from(u16::from_le_bytes([a, b]))), + [a, b, c, d] => Ok(u32::from_le_bytes([a, b, c, d])), + _ => Err(invalid("unexpected formatting property size")), + } +} + +/// A string with a 2-byte length (`LPWideString`, and the number format of `XFProp` 0x26). +fn short_string(b: &mut Bytes<'_>) -> Result { + let count = usize::from(b.u16()?); + let mut text = String::new(); + for ch in char::decode_utf16(b.take(count * 2)?.as_chunks::<2>().0.iter().copied().map(u16::from_le_bytes)) { + text.push(ch.map_err(|_| invalid("invalid UTF-16 string"))?); + } + Ok(name(text)) +} + +fn dxf_font(d: &mut Dxf) -> &mut Font { + d.font.get_or_insert_with(Font::default) +} + +/// `BrtDXF`: its `XFProps`, each stating one property. +fn dxf(b: &mut Bytes<'_>) -> Result { + b.u16()?; + b.u16()?; + let count = b.u16()?; + let mut d = Dxf::default(); + let mut gradient_fill: Option = None; + let mut pattern_fill: Option = None; + for _ in 0..count { + let kind = b.u16()?; + let size = usize::from(b.u16()?).checked_sub(4).ok_or_else(|| invalid("formatting property too short"))?; + let data = b.take(size)?; + let mut p = Bytes::new(data); + match kind { + 0x00 => pattern_fill.get_or_insert_with(PatternFill::default).pattern = Some(pattern(uint(data)?)?), + 0x01 => pattern_fill.get_or_insert_with(PatternFill::default).fg = color(&mut p)?, + 0x02 => pattern_fill.get_or_insert_with(PatternFill::default).bg = color(&mut p)?, + 0x03 => { + let path = p.u32()? == 1; + gradient_fill = Some(gradient(&mut p, path)?); + } + 0x04 => { + let g = gradient_fill.as_mut().ok_or_else(|| invalid("gradient stop without a gradient"))?; + if g.stops.len() >= MAX_STOPS as usize { + return Err(Error::TooLarge(format!("more than {MAX_STOPS} gradient stops"))); + } + p.u16()?; + let position = p.f64()?; + g.stops.push((position, color(&mut p)?.unwrap_or_default())); + } + 0x05 => dxf_font(&mut d).color = color(&mut p)?, + 0x06..=0x0C => { + let c = color(&mut p)?; + let edge = Some(BorderEdge { style: border_style(p.u16()?)?, color: c }); + let border = d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }); + match kind { + 0x06 => border.top = edge, + 0x07 => border.bottom = edge, + 0x08 => border.left = edge, + 0x09 => border.right = edge, + 0x0A => border.diagonal = edge, + 0x0B => border.vertical = edge, + _ => border.horizontal = edge, + } + } + 0x0D => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_up = uint(data)? != 0, + 0x0E => d.border.get_or_insert_with(|| Border { outline: true, ..Border::default() }).diagonal_down = uint(data)? != 0, + 0x0F..=0x16 => { + let v = uint(data)?; + let a = d.alignment.get_or_insert_with(Alignment::default); + match kind { + 0x0F => a.horizontal = halign(v), + 0x10 => a.vertical = valign(v), + 0x11 => a.text_rotation = Some(v), + 0x12 => a.indent = Some(v), + 0x13 => a.reading_order = Some(v), + 0x14 => a.wrap_text = v != 0, + 0x15 => a.justify_last_line = v != 0, + _ => a.shrink_to_fit = v != 0, + } + } + 0x18 => dxf_font(&mut d).name = Some(short_string(&mut p)?), + 0x19 => dxf_font(&mut d).bold = Some(uint(data)? >= 0x2BC), + 0x1A => dxf_font(&mut d).underline = Some(underline(uint(data)?)), + 0x1B => dxf_font(&mut d).vert_align = Some(script(uint(data)?)), + 0x1C => dxf_font(&mut d).italic = Some(uint(data)? != 0), + 0x1D => dxf_font(&mut d).strike = Some(uint(data)? != 0), + 0x1E => dxf_font(&mut d).outline = Some(uint(data)? != 0), + 0x1F => dxf_font(&mut d).shadow = Some(uint(data)? != 0), + 0x20 => dxf_font(&mut d).condense = Some(uint(data)? != 0), + 0x21 => dxf_font(&mut d).extend = Some(uint(data)? != 0), + 0x22 => dxf_font(&mut d).charset = Some(uint(data)?), + 0x23 => dxf_font(&mut d).family = Some(uint(data)?), + 0x24 => dxf_font(&mut d).size = Some(f64::from(uint(data)?) / 20.0), + 0x25 => dxf_font(&mut d).scheme = scheme(uint(data)?), + 0x26 => d.num_fmt.get_or_insert_with(NumFmt::default).code = Some(short_string(&mut p)?), + 0x29 => d.num_fmt.get_or_insert_with(NumFmt::default).id = Some(uint(data)?), + 0x2B => d.protection.get_or_insert_with(Protection::default).locked = Some(uint(data)? != 0), + 0x2C => d.protection.get_or_insert_with(Protection::default).hidden = Some(uint(data)? != 0), + // Merged cells (0x17) and relative indents (0x2A) have no place in ``. + _ => {} + } + } + d.fill = gradient_fill.map(Fill::Gradient).or(pattern_fill.map(Fill::Pattern)); + Ok(d) +} + +/// The styles part: `BrtBeginStyleSheet` to `BrtEndStyleSheet`. +pub(crate) fn styles(bytes: &[u8], limits: &Limits, budget: &Budget, warnings: &mut Warnings) -> Result { + let mut records = Records::new(bytes, limits, budget); + // Excel keeps a number format with a newer locale tag in two alternate-content blocks: the + // newer spelling for itself, and for Excel 2007 the one `.xlsx` has as `formatCode`. + records.excel_2007 = true; + records.first(278, warnings)?; + let mut s = Styles::default(); + // Inside BrtBeginCellStyleXFs (true) or BrtBeginCellXFs (false). + let mut xfs: Option = None; + let mut palette = false; + while let Some((id, payload)) = records.next(warnings)? { + let mut b = Bytes::new(payload); + let entries = match id { + 44 => s.num_fmts.len(), + 43 => s.fonts.len(), + 45 => s.fills.len(), + 46 => s.borders.len(), + 47 => s.cell_style_xfs.len().max(s.cell_xfs.len()), + 48 => s.cell_styles.len(), + 507 => s.dxfs.len(), + 475 => s.indexed_colors.len(), + _ => 0, + }; + if entries >= MAX_ENTRIES { + return Err(Error::TooLarge(format!("more than {MAX_ENTRIES} entries in a styles table"))); + } + match id { + 279 => { + records.end(payload, warnings)?; + return Ok(s); + } + 44 => { + let id = u32::from(b.u16()?); + let code = name(b.text(MAX_STRING_UNITS)?); + budget.text(code.len(), limits)?; + s.num_fmts.push((id, code)); + } + 43 => { + let f = font(&mut b)?; + budget.text(f.name.as_ref().map_or(0, String::len), limits)?; + s.fonts.push(f); + } + 45 => s.fills.push(fill(&mut b)?), + 46 => s.borders.push(border(&mut b)?), + 626 => xfs = Some(true), + 617 => xfs = Some(false), + 627 | 618 => xfs = None, + 47 => match xfs { + Some(true) => s.cell_style_xfs.push(xf(&mut b, true)?), + Some(false) => s.cell_xfs.push(xf(&mut b, false)?), + None => return Err(invalid("cell format outside its list")), + }, + 48 => { + let c = cell_style(&mut b)?; + budget.text(c.name.len(), limits)?; + s.cell_styles.push(c); + } + 507 => { + let d = dxf(&mut b)?; + // What it keeps of its strings, after a later property has replaced an earlier. + let font = d.font.as_ref().and_then(|f| f.name.as_ref()).map_or(0, String::len); + let format = d.num_fmt.as_ref().and_then(|n| n.code.as_ref()).map_or(0, String::len); + budget.text(font + format, limits)?; + s.dxfs.push(d); + } + // BrtBeginIndexedColors, BrtEndIndexedColors. + 565 => palette = true, + 566 => palette = false, + 475 if palette => { + let [r, g, bl, _] = <[u8; 4]>::try_from(b.take(4)?).map_err(|_| invalid("truncated colour"))?; + s.indexed_colors.push(u32::from_be_bytes([0, r, g, bl])); + } + // BrtBeginTableStyles: the count of custom table styles, which are not read. + 508 => { + if b.u32()? > 0 { + warnings.warn("custom table styles in a binary workbook were not read"); + } + continue; + } + _ => continue, + } + // The records read above are read whole; the rest are skipped. + if matches!(id, 44 | 43 | 45 | 46 | 47 | 48 | 507 | 475) { + b.finish()?; + } + } + Err(invalid("missing styles end record")) +} diff --git a/crates/xlsx/tests/xlsb.rs b/crates/xlsx/tests/xlsb.rs index 748d1b86d..18a833388 100644 --- a/crates/xlsx/tests/xlsb.rs +++ b/crates/xlsx/tests/xlsb.rs @@ -5,7 +5,10 @@ mod common; use common::{read, read_with}; use craft_xlsx::testing::zip; -use craft_xlsx::{CellError, CellRef, Error, FormulaKind, Limits, Range, SheetKind, Value, Visibility, is_xlsb}; +use craft_xlsx::{ + CellError, CellRef, Color, ColorSource, Error, Fill, FormulaKind, Limits, Package, Pattern, PatternFill, Protection, Range, SheetKind, Styles, + Value, Visibility, is_xlsb, +}; const ROOT: &str = r#""#; const RELS: &str = r#""#; @@ -931,6 +934,280 @@ fn a_dynamic_array_kept_as_its_value_has_no_cell_metadata() { assert!(warned(&wb, "missing from the file"), "{:?}", wb.warnings); } +// Styles ([MS-XLSB] 2.1.7.50): a styles part assembled from records, and the `styles.xml` Excel +// writes for the same formats when it saves the workbook as `.xlsx`. + +const STYLES_RELS: &str = r#""#; +const STYLES_TYPES: &str = r#""#; + +fn styled_file(styles: &[u8], types: &str) -> Vec { + let sheet = worksheet(&[row(0), cell(5, 0, &1.0f64.to_le_bytes())].concat(), &[]); + zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", ROOT.as_bytes()), + ("records/book.bin", &book(false)), + ("records/_rels/book.bin.rels", STYLES_RELS.as_bytes()), + ("data/one.bin", &sheet), + ("data/two.bin", &worksheet(&[], &[])), + ("look/styles.bin", styles), + ]) +} + +/// The styles part of a package, and the warnings reading it gave. +fn styles_of(bytes: &[u8]) -> (Option, Vec) { + let mut p = Package::open(bytes, &Limits::default()).unwrap(); + let styles = p.styles().unwrap(); + (styles, p.warnings().to_vec()) +} + +/// `BrtColor`: auto 0, indexed 1, RGB 2, theme 3. +fn color(kind: u8, index: u8, tint: i16, rgba: [u8; 4]) -> Vec { + [&[kind << 1 | 1, index][..], &tint.to_le_bytes(), &rgba].concat() +} +fn indexed(i: u8) -> Vec { + color(1, i, 0, [0, 0, 0, 255]) +} +fn theme(i: u8, tint: i16) -> Vec { + color(3, i, tint, [0, 0, 0, 255]) +} +fn rgb(r: u8, g: u8, b: u8) -> Vec { + color(2, 255, 0, [r, g, b, 255]) +} +#[allow(clippy::too_many_arguments)] +fn font(twips: u16, flags: u16, weight: u16, script: u16, underline: u8, family: u8, charset: u8, c: &[u8], scheme: u8, name: &str) -> Vec { + let head = + [&twips.to_le_bytes()[..], &flags.to_le_bytes(), &weight.to_le_bytes(), &script.to_le_bytes(), &[underline, family, charset, 0]].concat(); + record(43, &[&head[..], c, &[scheme], &wide(name)].concat()) +} +fn pattern_fill(fls: u32, fg: &[u8], bg: &[u8]) -> Vec { + record(45, &[&fls.to_le_bytes()[..], fg, bg, &[0; 48]].concat()) +} +fn edge(dg: u8, c: &[u8]) -> Vec { + [&[dg, 0][..], c].concat() +} +#[allow(clippy::too_many_arguments)] +fn brt_xf(parent: u16, fmt: u16, font: u16, fill: u16, border: u16, rotation: u8, indent: u8, flags: u16, apply: u8) -> Vec { + let ids = [parent, fmt, font, fill, border].map(u16::to_le_bytes).concat(); + record(47, &[&ids[..], &[rotation, indent], &flags.to_le_bytes(), &[apply, 0]].concat()) +} +fn style(xf: u32, flags: u16, builtin: u8, level: u8, name: &str) -> Vec { + record(48, &[&xf.to_le_bytes()[..], &flags.to_le_bytes(), &[builtin, level], &wide(name)].concat()) +} +fn prop(kind: u16, data: &[u8]) -> Vec { + [&kind.to_le_bytes()[..], &(data.len() as u16 + 4).to_le_bytes(), data].concat() +} +fn short(text: &str) -> Vec { + let units: Vec = text.encode_utf16().collect(); + [(units.len() as u16).to_le_bytes().to_vec(), units.iter().flat_map(|u| u.to_le_bytes()).collect()].concat() +} +fn brt_dxf(props: &[Vec]) -> Vec { + record(507, &[&[0, 0x80, 0, 0][..], &(props.len() as u16).to_le_bytes(), &props.concat()].concat()) +} +fn begin(id: u16, count: u32) -> Vec { + record(id, &count.to_le_bytes()) +} + +/// Every kind of record the styles part has, as Excel writes them. +fn styles_bin() -> Vec { + let normal = font(220, 0, 400, 0, 0, 2, 0, &theme(1, 0), 2, "Aptos Narrow"); + // Bold, italic, struck out, superscript, double accounting underline; family and + // character set 0 and scheme "none" are not written in `.xlsx`. + let fancy = font(290, 0x0A, 700, 1, 0x22, 0, 2, &rgb(0, 128, 255), 0, "Wingdings"); + let heading = font(220, 0, 400, 0, 0, 2, 0, &theme(4, 13106), 1, "Aptos Display"); + // An automatic colour: no `` in `.xlsx`. + let auto = font(220, 0x02, 400, 0, 0, 2, 0, &color(0, 64, 0, [0, 0, 0, 255]), 2, "Aptos Narrow"); + let gradient = [ + &0x28u32.to_le_bytes()[..], + &indexed(64), + &indexed(65), + &1u32.to_le_bytes(), + &[0.0f64, 0.5, 0.5, 0.5, 0.5].map(f64::to_le_bytes).concat(), + &2u32.to_le_bytes(), + &theme(0, 0), + &0.0f64.to_le_bytes(), + &theme(4, 0), + &1.0f64.to_le_bytes(), + ] + .concat(); + let none = edge(0, &color(0, 0, 0, [0, 0, 0, 0])); + let parts: Vec> = vec![ + record(278, &[]), + begin(615, 2), + record(44, &[&164u16.to_le_bytes()[..], &wide(r"yyyy\-mm\-dd")].concat()), + // A format in two alternate-content blocks: the newer spelling for version 13 and later + // (skipped), and the one for versions 12 and 13, which `.xlsx` has as `formatCode`. + record(37, &[1, 0, 13, 0, 0, 0x80]), + record(44, &[&165u16.to_le_bytes()[..], &wide("[$-ja-JP-x-gannen]ge")].concat()), + record(38, &[]), + record(37, &[2, 0, 12, 0, 0, 0, 13, 0, 0, 0]), + record(44, &[&165u16.to_le_bytes()[..], &wide("[$]ge")].concat()), + record(38, &[]), + record(616, &[]), + begin(611, 4), + normal, + fancy, + heading, + auto, + record(612, &[]), + begin(603, 6), + pattern_fill(0, &indexed(64), &indexed(65)), + pattern_fill(17, &indexed(64), &indexed(65)), + pattern_fill(1, &rgb(255, 255, 0), &indexed(64)), + pattern_fill(9, &theme(5, -8191), &indexed(65)), + record(45, &gradient), + // A tinted pattern colour: `.xlsx` states the system background, as for darkGrid above. + pattern_fill(1, &theme(4, 26213), &indexed(65)), + record(604, &[]), + begin(613, 2), + record(46, &[&[0][..], &none, &none, &none, &none, &none].concat()), + record( + 46, + &[&[3][..], &edge(2, &rgb(200, 0, 0)), &none, &edge(7, &indexed(64)), &edge(13, &color(0, 0, 0, [0, 0, 0, 0])), &edge(1, &theme(4, 0))] + .concat(), + ), + record(614, &[]), + begin(626, 3), + brt_xf(0xFFFF, 0, 0, 0, 0, 0, 0, 0x1010, 0), + brt_xf(0xFFFF, 9, 0, 0, 0, 0, 0, 0x1010, 0x3E), + brt_xf(0xFFFF, 0, 2, 0, 0, 0, 0, 0x1010, 0), + record(627, &[]), + begin(617, 4), + brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0), + brt_xf(0, 164, 0, 0, 0, 0, 0, 0x1010, 0x01), + // Left, top, wrapped, shrunk, right to left, unlocked, hidden, quote prefix. + brt_xf(0, 0, 0, 0, 0, 180, 2, 0x01 | 0x40 | 0x100 | 0x800 | 0x2000 | 0x8000, 0x24), + brt_xf(1, 0, 1, 4, 1, 0, 0, 0x1010, 0x1A), + record(618, &[]), + begin(619, 3), + style(0, 1, 0, 0, "Normal"), + // Excel wraps each style in alternate content, which is skipped. + record(37, &[1, 0, 2, 0x0e, 0, 0x80]), + record(3072, &[0; 16]), + record(38, &[]), + style(1, 1 | 4, 1, 1, "RowLevel_2"), + style(2, 2, 0, 0, "Probe Style"), + record(620, &[]), + begin(505, 2), + brt_dxf(&[ + prop(0x05, &rgb(0x9C, 0, 6)), + prop(0x19, &700u16.to_le_bytes()), + prop(0x1C, &[0]), + prop(0x1A, &0x21u16.to_le_bytes()), + prop(0x24, &240u32.to_le_bytes()), + prop(0x18, &short("Arial")), + prop(0x29, &170u16.to_le_bytes()), + prop(0x26, &short("0.0")), + prop(0x02, &rgb(0xFF, 0xC7, 0xCE)), + prop(0x0F, &[2]), + prop(0x14, &[1]), + prop(0x2B, &[0]), + prop(0x07, &[&color(0, 0, 0, [0, 0, 0, 0])[..], &1u16.to_le_bytes()].concat()), + ]), + brt_dxf(&[ + prop(0x03, &[&0u32.to_le_bytes()[..], &[90.0f64, 0.0, 0.0, 0.0, 0.0].map(f64::to_le_bytes).concat()].concat()), + prop(0x04, &[&[0, 0][..], &0.0f64.to_le_bytes(), &rgb(255, 255, 255)].concat()), + prop(0x04, &[&[0, 0][..], &1.0f64.to_le_bytes(), &theme(4, 0)].concat()), + ]), + record(506, &[]), + record(508, &[&0u32.to_le_bytes()[..], &wide("TableStyleMedium2"), &wide("PivotStyleLight16")].concat()), + record(509, &[]), + record(473, &[]), + record(565, &[]), + record(475, &[0x11, 0x22, 0x33, 0]), + record(475, &[0xFF, 0xFF, 0xFF, 0]), + record(566, &[]), + record(474, &[]), + record(279, &[]), + ]; + parts.concat() +} + +const STYLES_XML: &str = r#""#; + +/// The same styles from a SpreadsheetML package. +fn xml_styles(xml: &str) -> Styles { + let types = r#""#; + let root = r#""#; + let rels = r#""#; + let bytes = zip(&[ + ("[Content_Types].xml", types.as_bytes()), + ("_rels/.rels", root.as_bytes()), + ("xl/workbook.xml", br#""#), + ("xl/_rels/workbook.xml.rels", rels.as_bytes()), + ("xl/styles.xml", xml.as_bytes()), + ]); + styles_of(&bytes).0.unwrap() +} + +/// What both readers lift out: an ``'s element keeps only its `apply*` flags, and the XML +/// part's own element is dropped. +fn lifted(mut s: Styles) -> Styles { + for xf in s.cell_style_xfs.iter_mut().chain(&mut s.cell_xfs) { + let attrs = xf.el.attrs.iter().filter(|(k, _)| k.starts_with("apply")).cloned().collect(); + xf.el = craft_xml::El::new("xf"); + xf.el.attrs = attrs; + } + s.root = craft_xml::El::default(); + s +} + +#[test] +fn styles_are_read_as_their_xlsx_spelling() { + let (binary, warnings) = styles_of(&styled_file(&styles_bin(), STYLES_TYPES)); + let binary = binary.expect("styles are read"); + assert_eq!(lifted(binary.clone()), lifted(xml_styles(STYLES_XML))); + // A few of the conversions, spelled out. + assert_eq!(binary.fonts[1].size, Some(14.5)); + assert_eq!(binary.fonts[1].family, None); + let solid = Fill::Pattern(PatternFill { + pattern: Some(Pattern::Solid), + fg: Some(Color { source: ColorSource::Rgb(0xFFFF_FF00), tint: 0.0 }), + bg: Some(Color { source: ColorSource::Indexed(64), tint: 0.0 }), + }); + assert_eq!(binary.fills[2], solid); + assert_eq!(binary.cell_xfs[2].protection, Some(Protection { locked: Some(false), hidden: Some(true) })); + assert_eq!(binary.palette()[0], 0x11_2233); + // Written back as SpreadsheetML, a cell style format says what it leaves out, as Excel's does. + let written = String::from_utf8(binary.to_xml()).unwrap(); + assert!(written.contains(r#"applyFont="0" applyFill="0" applyBorder="0" applyAlignment="0" applyProtection="0""#), "{written}"); + assert!(warnings.iter().any(|w| w.contains("alternate-content")), "{warnings:?}"); + assert!(!warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}"); +} + +#[test] +fn damaged_styles_are_a_warning_and_the_workbook_opens() { + let good = styles_bin(); + let end = record(279, &[]); + let body = &good[..good.len() - end.len()]; + let with = |extra: &[u8]| [body, extra, &end].concat(); + let cases: Vec<(&str, Vec, &str)> = vec![ + ("truncated", good[..good.len() - 1].to_vec(), STYLES_TYPES), + ("no end record", body.to_vec(), STYLES_TYPES), + ("unknown fill pattern", with(&pattern_fill(99, &indexed(64), &indexed(65))), STYLES_TYPES), + ("unknown border style", with(&record(46, &[&[0][..], &edge(14, &indexed(64)), &[0; 40]].concat())), STYLES_TYPES), + ("cell format outside its list", with(&brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0)), STYLES_TYPES), + ("short property", with(&brt_dxf(&[vec![0x19, 0, 2, 0]])), STYLES_TYPES), + ("wrong content type", good.clone(), TYPES), + ]; + for (why, part, types) in cases { + let bytes = styled_file(&part, types); + let (styles, warnings) = styles_of(&bytes); + assert!(styles.is_none(), "{why}"); + assert!(warnings.iter().any(|w| w.contains("look/styles.bin")), "{why}: {warnings:?}"); + assert_eq!(read(&bytes).unwrap().sheets[0].value(CellRef::new(0, 0)), Value::Number(1.0), "{why}"); + } + // A table past 65,536 entries is past a fixed cap: refused before it is held, as too large. + let xfs = brt_xf(0, 0, 0, 0, 0, 0, 0, 0x1010, 0).repeat(65_537); + let bytes = styled_file(&with(&[&begin(617, 65_537)[..], &xfs, &record(618, &[])].concat()), STYLES_TYPES); + let mut p = Package::open(&bytes, &Limits::default()).unwrap(); + assert!(matches!(p.styles(), Err(Error::TooLarge(m)) if m.contains("65536 entries"))); + // Custom table styles are not read, and say so. + let table_styles = [body, &record(508, &[&1u32.to_le_bytes()[..], &wide("Probe Table"), &wide("")].concat()), &end].concat(); + let (styles, warnings) = styles_of(&styled_file(&table_styles, STYLES_TYPES)); + assert!(styles.is_some()); + assert!(warnings.iter().any(|w| w.contains("table styles")), "{warnings:?}"); +} + /// `PtgAttrSpace` ([MS-XLSB] 2.5.98.31): its type (0 spaces before the next token, 1 line breaks /// before it, 2 spaces before a `(`, 4 spaces before a `)`) and count. fn space(kind: u8, n: u8) -> Vec { @@ -1063,6 +1340,16 @@ fn a_spent_formula_allowance_skips_decoding() { assert!(!warned(&wb, "unknown formula token"), "{:?}", wb.warnings); } +#[test] +fn differential_format_strings_are_charged_to_the_text_allowance() { + // A DXF's font name and number format are text the styles part keeps, charged like a font's. + let dxf = |name: &str| [record(278, &[]), begin(505, 1), brt_dxf(&[prop(0x18, &short(name))]), record(506, &[]), record(279, &[])].concat(); + let limits = Limits { binary_text: 100, ..Limits::default() }; + let open = |part: &[u8]| Package::open(&styled_file(part, STYLES_TYPES), &limits).unwrap().styles(); + assert!(open(&dxf("Arial")).is_ok_and(|s| s.is_some_and(|s| s.dxfs.len() == 1))); + assert!(matches!(open(&dxf(&"x".repeat(200))), Err(Error::TooLarge(_)))); +} + #[test] fn module_sheets_and_supporting_links_are_bounded() { // 1,025 very hidden module sheets (no relationship) before the worksheets: each keeps a place