From 7aedc1de73d5fe3e2a771696ddd5401abf665d63 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 06:42:41 -0500 Subject: [PATCH 01/98] Share small raster format implementations --- crates/image-codecs/src/codecs/pcx.rs | 35 +- crates/image-codecs/src/codecs/xpm.rs | 2 +- crates/image-codecs/src/lib.rs | 14 + crates/image-codecs/src/options.rs | 6 +- crates/raster-codecs/Cargo.toml | 3 - crates/raster-codecs/README.md | 16 +- crates/raster-codecs/src/bridge.rs | 79 ++ crates/raster-codecs/src/dds.rs | 612 +-------------- crates/raster-codecs/src/dds_bc.rs | 369 --------- crates/raster-codecs/src/dicom.rs | 530 +------------ crates/raster-codecs/src/fits.rs | 859 +-------------------- crates/raster-codecs/src/iff.rs | 543 +------------ crates/raster-codecs/src/lib.rs | 53 +- crates/raster-codecs/src/pcx.rs | 149 +--- crates/raster-codecs/src/pnm.rs | 360 +-------- crates/raster-codecs/src/xpm.rs | 244 +----- crates/raster-codecs/src/xpm/x11_colors.rs | 666 ---------------- 17 files changed, 225 insertions(+), 4315 deletions(-) create mode 100644 crates/raster-codecs/src/bridge.rs delete mode 100644 crates/raster-codecs/src/dds_bc.rs delete mode 100644 crates/raster-codecs/src/xpm/x11_colors.rs diff --git a/crates/image-codecs/src/codecs/pcx.rs b/crates/image-codecs/src/codecs/pcx.rs index 48dbf1169..bf1b2eee6 100644 --- a/crates/image-codecs/src/codecs/pcx.rs +++ b/crates/image-codecs/src/codecs/pcx.rs @@ -1,5 +1,5 @@ //! ZSoft PCX (Paintbrush) reader: strictly bounded scanline RLE, 16/256-colour -//! palettes and 8-bit three-plane RGB. Read-only: modern saves should use PNG. +//! palettes, 8-bit RGB and four-plane RGBA. Read-only: modern saves should use PNG. use crate::SampleType; use crate::error::CodecError; @@ -19,7 +19,7 @@ fn le16(b: &[u8], i: usize) -> u16 { /// PCX lacks a strong magic number. Require plausible complete header fields before /// identifying it, rather than classifying every file starting with 0x0a as PCX. pub(crate) fn looks_like_pcx(b: &[u8]) -> bool { - if b.len() < HEADER || b[0] != 0x0a || !matches!(b[1], 0 | 2 | 3 | 5) || b[2] != 1 { + if b.len() < HEADER || b[0] != 0x0a || !matches!(b[1], 0 | 2..=5) || b[2] != 1 { return false; } let bits = b[3]; @@ -31,7 +31,12 @@ pub(crate) fn looks_like_pcx(b: &[u8]) -> bool { } let width = u32::from(x1 - x0) + 1; let min_stride = (width * u32::from(bits)).div_ceil(8); - stride >= min_stride && stride > 0 && (bits != 8 || matches!(planes, 1 | 3)) && (bits == 8 || u32::from(bits) * u32::from(planes) <= 4) + stride >= min_stride && stride > 0 && (bits != 8 || matches!(planes, 1 | 3 | 4)) && (bits == 8 || u32::from(bits) * u32::from(planes) <= 4) +} + +/// The dimensions declared by a plausible PCX header, without decoding its pixels. +pub(crate) fn dimensions(b: &[u8]) -> Option<(u32, u32)> { + looks_like_pcx(b).then(|| (u32::from(le16(b, 8) - le16(b, 4)) + 1, u32::from(le16(b, 10) - le16(b, 6)) + 1)) } /// Read one compressed scanline, including all planes. RLE packets never cross a scanline. @@ -61,11 +66,12 @@ pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result { if !looks_like_pcx(b) { return Err(bad("invalid PCX header or unsupported pixel layout")); } - let (w, h) = (u32::from(le16(b, 8) - le16(b, 4)) + 1, u32::from(le16(b, 10) - le16(b, 6)) + 1); - limits.check(w, h, ChannelLayout::Rgb, SampleType::U8)?; + let (w, h) = dimensions(b).ok_or_else(|| bad("invalid PCX header or unsupported pixel layout"))?; + let layout = if b[3] == 8 && b[65] == 4 { ChannelLayout::Rgba } else { ChannelLayout::Rgb }; + limits.check(w, h, layout, SampleType::U8)?; let (bits, planes, stride) = (usize::from(b[3]), usize::from(b[65]), usize::from(le16(b, 66))); let row_len = stride.checked_mul(planes).ok_or_else(|| bad("row size overflow"))?; - let total = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(3)).ok_or_else(|| bad("pixel size overflow"))?; + let total = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(layout.channels())).ok_or_else(|| bad("pixel size overflow"))?; // The last 769 bytes are the optional VGA palette for 8-bit indexed images. let vga = bits == 8 && planes == 1 && b.len() >= HEADER + 769 && b[b.len() - 769] == 0x0c; if bits == 8 && planes == 1 && !vga && le16(b, 68) != 2 { @@ -102,8 +108,10 @@ pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result { for y in 0..h as usize { read_row(&b[..image_end], &mut p, &mut row)?; for x in 0..width { - let rgb = if bits == 8 && planes == 3 { - [row[x], row[stride + x], row[2 * stride + x]] + let rgba = if bits == 8 && planes == 4 { + [row[x], row[stride + x], row[2 * stride + x], row[3 * stride + x]] + } else if bits == 8 && planes == 3 { + [row[x], row[stride + x], row[2 * stride + x], 255] } else { let index = if bits == 8 { usize::from(row[x]) @@ -118,19 +126,20 @@ pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result { }; if vga { let o = image_end + 1 + index * 3; - [b[o], b[o + 1], b[o + 2]] + [b[o], b[o + 1], b[o + 2], 255] } else if bits == 8 { let g = index as u8; - [g, g, g] // palette-info 2: gray scale + [g, g, g, 255] // palette-info 2: gray scale } else { let o = 16 + index * 3; - [b[o], b[o + 1], b[o + 2]] + [b[o], b[o + 1], b[o + 2], 255] } }; - pixels[(y * width + x) * 3..(y * width + x + 1) * 3].copy_from_slice(&rgb); + let start = (y * width + x) * layout.channels(); + pixels[start..start + layout.channels()].copy_from_slice(&rgba[..layout.channels()]); } } - let mut image = Image::from_u8(w, h, ChannelLayout::Rgb, pixels)?; + let mut image = Image::from_u8(w, h, layout, pixels)?; let (xdpi, ydpi) = (le16(b, 12), le16(b, 14)); if xdpi > 0 && ydpi > 0 { image.meta.dpi = Some((f32::from(xdpi), f32::from(ydpi))); diff --git a/crates/image-codecs/src/codecs/xpm.rs b/crates/image-codecs/src/codecs/xpm.rs index c20f3c58e..2dbd867c9 100644 --- a/crates/image-codecs/src/codecs/xpm.rs +++ b/crates/image-codecs/src/codecs/xpm.rs @@ -96,7 +96,7 @@ fn channel(hex: &str) -> Option { Some(((v * 255 + max / 2) / max) as u8) } -fn named(name: &str) -> Option<[u8; 3]> { +pub(crate) fn named(name: &str) -> Option<[u8; 3]> { let s = name.to_ascii_lowercase().replace([' ', '-', '_'], ""); // XPM imports X11 names, whose RGB values differ from CSS for names // such as gray, maroon and purple. Keep these exact rather than guessing. diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index e02c438ec..8085ca44e 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -211,6 +211,20 @@ pub fn probe_size(bytes: &[u8]) -> Result<(u32, u32), CodecError> { Ok(if (5..=8).contains(&o) { (h, w) } else { (w, h) }) } +/// PCX's dimensions are available from its fixed-size header without decoding the RLE rows. +pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + codecs::pcx::dimensions(bytes) +} + +/// The X11 color-name lookup used by XPM files, expressed as normalized sRGB. +pub fn xpm_named_color_rgb(name: &str) -> Option<[f32; 3]> { + if name.is_empty() || name.len() > 64 || !name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()) { + return None; + } + let rgb = codecs::xpm::named(name)?; + Some(rgb.map(|channel| f32::from(channel) / 255.0)) +} + /// The colour table of a GIF's first frame (its Local Color Table, else the Global one) and the /// entry its Graphic Control Extension makes transparent, read without decoding pixels. pub fn gif_palette(bytes: &[u8]) -> Option<(Vec<[u8; 3]>, Option)> { diff --git a/crates/image-codecs/src/options.rs b/crates/image-codecs/src/options.rs index 9674b4edf..5dd0ac985 100644 --- a/crates/image-codecs/src/options.rs +++ b/crates/image-codecs/src/options.rs @@ -32,6 +32,10 @@ impl Default for Limits { } impl Limits { + /// Construct explicit bounds for a caller-owned format adapter. + pub const fn new(max_width: u32, max_height: u32, max_pixels: u64, max_alloc: u64) -> Self { + Limits { max_width, max_height, max_pixels, max_alloc } + } /// For a decoder that writes a band of rows at a time into the document's tiles instead of /// one buffer (TIFF, #8169): the largest document size, 300,000 px a side, on 64-bit /// targets, with `max_alloc` bounding each group of bands. 32-bit targets keep the @@ -85,7 +89,7 @@ impl Limits { /// never beats `max_ratio`:1. Called before the output buffer is allocated, so a /// header-only hostile file cannot make the decoder reserve gigabytes. A fixed 1 MiB of /// slack keeps tiny images and headers with large metadata unaffected. - pub(crate) fn check_plausible(declared_bytes: u64, file_len: usize, max_ratio: u64) -> Result<(), CodecError> { + pub fn check_plausible(declared_bytes: u64, file_len: usize, max_ratio: u64) -> Result<(), CodecError> { let capacity = (file_len as u64).saturating_mul(max_ratio).saturating_add(1 << 20); if declared_bytes > capacity { return Err(CodecError::LimitExceeded(format!("a {file_len} byte file cannot hold {declared_bytes} bytes of image data"))); diff --git a/crates/raster-codecs/Cargo.toml b/crates/raster-codecs/Cargo.toml index 002012b80..a29cda403 100644 --- a/crates/raster-codecs/Cargo.toml +++ b/crates/raster-codecs/Cargo.toml @@ -16,6 +16,3 @@ craft-image-codecs = { workspace = true } thiserror = { workspace = true } image = { version = "0.25", default-features = false } -# The DDS encoder compresses rows of blocks in parallel on native; wasm stays single-threaded. -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] -rayon = { workspace = true } diff --git a/crates/raster-codecs/README.md b/crates/raster-codecs/README.md index 6735769ae..907e73763 100644 --- a/crates/raster-codecs/README.md +++ b/crates/raster-codecs/README.md @@ -1,12 +1,9 @@ # craft-raster-codecs -Small raster image formats for the Craft apps, from PhotoCraft's `photocraft-codecs`: one module -per format, one implementation of each. Decoders take untrusted bytes and `Limits`, check the -declared size before allocating, and never panic. No `unsafe`; it builds for -`wasm32-unknown-unknown`. The `common` RGBA image facade delegates PNG/JPEG/BMP/GIF/TIFF/WebP -decoding and PNG/JPEG encoding to [`craft-image-codecs`](../image-codecs/README.md), the shared -depth- and metadata-preserving codec implementation. Other dependencies are `thiserror`, and -`rayon` on native targets (the DDS encoder compresses rows of blocks in parallel). +Compatibility facade for the small raster-format modules in older Craft app versions. The +format readers and writers are implemented once in [`craft-image-codecs`](../image-codecs/README.md); +these modules forward calls and adapt the older `Image` and `Limits` types. The `common` RGBA +image facade also delegates to that crate. ```toml craft-raster-codecs = { git = "https://github.com/storytold/craft-libs", rev = "" } @@ -37,9 +34,8 @@ assert_eq!(image.data(), &[0, 255]); let bytes = pnm::encode(&image)?; ``` -Each module has a `looks_like_*` (or `is_dicom`) sniffer for the format's signature. `dds` and -`fits` also have `has_additional_images`, which says without decoding whether a file holds -more than the one image `decode` returns (cube faces, array slices, further HDUs). +The module sniffers and the `dds`/`fits` additional-image queries forward to shared detection and +metadata inspection in `craft-image-codecs`. `xpm::x11_color_rgb` also looks up exact normalized X11 names as RGB channels in `0..=1`. It accepts only 1–64 lowercase ASCII letters or digits, without normalization, XPM-only aliases, diff --git a/crates/raster-codecs/src/bridge.rs b/crates/raster-codecs/src/bridge.rs new file mode 100644 index 000000000..a1bd061db --- /dev/null +++ b/crates/raster-codecs/src/bridge.rs @@ -0,0 +1,79 @@ +//! Compatibility between this crate's small-format facade and the canonical image codec model. + +use crate::{ChannelLayout, CodecError, DecodeWarning, Image, Limits, SampleType}; +use craft_image_codecs::{DecodeOptions, Format, Metadata}; + +pub(crate) fn format(format: Format, bytes: &[u8], limits: &Limits) -> Result { + let shared_limits = craft_image_codecs::Limits::new(limits.max_width, limits.max_height, limits.max_pixels, limits.max_alloc); + let options = DecodeOptions { limits: shared_limits, ..DecodeOptions::default() }; + let image = craft_image_codecs::decode_as_with(format, bytes, &options).map_err(from_shared_error)?; + from_shared_image(image) +} + +pub(crate) fn shared_image(image: &Image) -> Result { + let layout = match image.layout() { + ChannelLayout::Gray => craft_image_codecs::ChannelLayout::Gray, + ChannelLayout::GrayA => craft_image_codecs::ChannelLayout::GrayA, + ChannelLayout::Rgb => craft_image_codecs::ChannelLayout::Rgb, + ChannelLayout::Rgba => craft_image_codecs::ChannelLayout::Rgba, + ChannelLayout::Cmyk => craft_image_codecs::ChannelLayout::Cmyk, + ChannelLayout::CmykA => craft_image_codecs::ChannelLayout::CmykA, + }; + let sample = match image.sample_type() { + SampleType::U8 => craft_image_codecs::SampleType::U8, + SampleType::U16 => craft_image_codecs::SampleType::U16, + SampleType::F32 => craft_image_codecs::SampleType::F32, + }; + let meta = Metadata { dpi: image.dpi, text: image.text.clone(), ..Metadata::default() }; + craft_image_codecs::Image::from_raw(image.width(), image.height(), layout, sample, image.data().to_vec()) + .map(|image| image.with_meta(meta)) + .map_err(from_shared_error) +} + +fn from_shared_image(image: craft_image_codecs::Image) -> Result { + let layout = match image.layout() { + craft_image_codecs::ChannelLayout::Gray => ChannelLayout::Gray, + craft_image_codecs::ChannelLayout::GrayA => ChannelLayout::GrayA, + craft_image_codecs::ChannelLayout::Rgb => ChannelLayout::Rgb, + craft_image_codecs::ChannelLayout::Rgba => ChannelLayout::Rgba, + craft_image_codecs::ChannelLayout::Cmyk => ChannelLayout::Cmyk, + craft_image_codecs::ChannelLayout::CmykA => ChannelLayout::CmykA, + _ => return Err(CodecError::InvalidImage(format!("unsupported small-codec layout {:?}", image.layout()))), + }; + let sample = match image.sample_type() { + craft_image_codecs::SampleType::U8 => SampleType::U8, + craft_image_codecs::SampleType::U16 => SampleType::U16, + craft_image_codecs::SampleType::F32 => SampleType::F32, + _ => return Err(CodecError::InvalidImage(format!("unsupported small-codec sample type {:?}", image.sample_type()))), + }; + let mut out = Image::from_raw(image.width(), image.height(), layout, sample, image.data().to_vec())?; + out.dpi = image.meta.dpi; + out.text = image.meta.text; + out.warnings = image + .warnings + .into_iter() + .filter_map(|warning| match warning { + craft_image_codecs::DecodeWarning::MorePages { total } | craft_image_codecs::DecodeWarning::MoreParts { total } => { + Some(DecodeWarning::MorePages { total }) + } + _ => None, + }) + .collect(); + Ok(out) +} + +pub(crate) fn from_shared_error(error: craft_image_codecs::CodecError) -> CodecError { + match error { + craft_image_codecs::CodecError::UnknownFormat => CodecError::unsupported("image", "unrecognized image format"), + craft_image_codecs::CodecError::Unsupported { format, reason } => CodecError::unsupported(format.name(), reason), + craft_image_codecs::CodecError::Malformed { format, message } => CodecError::malformed(format.name(), message), + craft_image_codecs::CodecError::LimitExceeded(message) => CodecError::LimitExceeded(message), + craft_image_codecs::CodecError::Encode { format, message } => CodecError::Encode { format: format.name(), message }, + craft_image_codecs::CodecError::InvalidImage(message) => CodecError::InvalidImage(message), + } +} + +pub(crate) fn encode(image: &Image, format: Format, options: &craft_image_codecs::EncodeOptions) -> Result, CodecError> { + let image = shared_image(image)?; + craft_image_codecs::encode(&image, format, options).map_err(from_shared_error) +} diff --git a/crates/raster-codecs/src/dds.rs b/crates/raster-codecs/src/dds.rs index 1249cb5b7..789028fb4 100644 --- a/crates/raster-codecs/src/dds.rs +++ b/crates/raster-codecs/src/dds.rs @@ -1,616 +1,64 @@ -//! DirectDraw Surface (DDS) textures, after Microsoft's public DDS documentation ("DDS", -//! "DDS_HEADER", "DDS_PIXELFORMAT", "DDS_HEADER_DXT10"). -//! -//! Reading: the top mip level of a 2D texture, either uncompressed (8 to 32 bits per pixel -//! described by channel masks, RGB or luminance) or block-compressed BC1/BC2/BC3 -//! (DXT1/DXT3/DXT5), with or without the DX10 header. Cubemaps, texture arrays and volume -//! textures are refused rather than flattened to one face or slice. -//! -//! Writing: a 2D texture, uncompressed (BGRA or BGRX, luminance with or without alpha) or -//! BC1/BC2/BC3, optionally with its full mip chain. The legacy header describes all of these, -//! so no DX10 header is written; formats only DXGI can name (BC4–BC7, sRGB) aren't written. +//! DDS compatibility facade; implementation and BC block codecs live in craft-image-codecs. +use crate::{CodecError, Image, Limits, SampleType, bridge}; +use craft_image_codecs::Format; -use std::borrow::Cow; - -use crate::dds_bc::{self, Block, Mask}; -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -/// `DDS ` plus the 124-byte DDS_HEADER. -const HEADER: usize = 128; -/// DDS_HEADER_DXT10, present when the FourCC is `DX10`. -const DX10_HEADER: usize = 20; - -const DDSD_DEPTH: u32 = 0x80_0000; -const DDPF_ALPHAPIXELS: u32 = 0x1; -const DDPF_FOURCC: u32 = 0x4; -const DDPF_RGB: u32 = 0x40; -const DDPF_LUMINANCE: u32 = 0x2_0000; -const DDSCAPS2_CUBEMAP: u32 = 0x200; -const DDSCAPS2_CUBEMAP_FACES: u32 = 0xfc00; -const DDSCAPS2_VOLUME: u32 = 0x20_0000; -const DIMENSION_TEXTURE2D: u32 = 3; -const MISC_TEXTURECUBE: u32 = 0x4; - -fn bad(reason: impl std::fmt::Display) -> CodecError { - CodecError::malformed("DDS", reason) +pub fn looks_like_dds(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Dds) } -fn unsupported(reason: impl Into) -> CodecError { - CodecError::unsupported("DDS", reason) -} -fn le32(b: &[u8], at: usize) -> Result { - let bytes = b.get(at..at + 4).and_then(|s| <[u8; 4]>::try_from(s).ok()).ok_or_else(|| bad("truncated header"))?; - Ok(u32::from_le_bytes(bytes)) -} - -/// The `DDS ` magic. -pub fn looks_like_dds(b: &[u8]) -> bool { - b.starts_with(b"DDS ") -} - -/// Header-declared extra subresources cannot survive a single-image rewrite, even without presence flags. pub fn has_additional_images(bytes: &[u8]) -> Result { - let depth = le32(bytes, 24)?; - let mips = le32(bytes, 28)?; - let faces = (le32(bytes, 112)? & DDSCAPS2_CUBEMAP_FACES).count_ones(); - let dx10 = le32(bytes, 80)? & DDPF_FOURCC != 0 && bytes.get(84..88) == Some(b"DX10"); - let (cube, array) = if dx10 { (le32(bytes, 136)? & MISC_TEXTURECUBE != 0, le32(bytes, 140)?) } else { (false, 0) }; - Ok(depth > 1 || mips > 1 || faces > 1 || cube || array > 1) + craft_image_codecs::has_unretained_source_data(bytes).map_err(bridge::from_shared_error) +} +pub fn decode(bytes: &[u8], limits: &Limits) -> Result { + bridge::format(Format::Dds, bytes, limits) } -/// A FourCC as text when it is printable, else as the number legacy writers store there. -fn fourcc_name(fourcc: u32) -> String { - let bytes = fourcc.to_le_bytes(); - if bytes.iter().all(|c| c.is_ascii_graphic() || *c == b' ') { bytes.iter().map(|&c| char::from(c)).collect() } else { fourcc.to_string() } -} - -/// One channel of an uncompressed pixel: where it sits and its largest value. -#[derive(Clone, Copy)] -struct Field { - shift: u32, - max: u32, -} - -impl Field { - /// `None` for an empty mask; an error for one with gaps or wider than the pixel. - fn new(mask: u32, bits: u32) -> Result, CodecError> { - if mask == 0 { - return Ok(None); - } - let shift = mask.trailing_zeros(); - let max = mask >> shift; - if max & max.wrapping_add(1) != 0 || (bits < 32 && mask >> bits != 0) { - return Err(bad(format!("channel mask {mask:#x} is not a contiguous field of a {bits}-bit pixel"))); - } - Ok(Some(Field { shift, max })) - } - - fn get(self, pixel: u32) -> u8 { - let v = u64::from((pixel >> self.shift) & self.max); - let max = u64::from(self.max); - ((v * 255 + max / 2) / max) as u8 - } -} - -enum Encoding { - /// Uncompressed: bytes per pixel, then red (or luminance), green, blue and alpha. - Masks { - bytes: usize, - gray: bool, - fields: [Option; 4], - }, - Bc1, - Bc2, - Bc3, -} - -impl Encoding { - fn masks(bits: u32, masks: [u32; 4], gray: bool) -> Result { - if !matches!(bits, 8 | 16 | 24 | 32) { - return Err(bad(format!("{bits} bits per pixel is not 8, 16, 24 or 32"))); - } - let mut fields = [None; 4]; - for (field, mask) in fields.iter_mut().zip(masks) { - *field = Field::new(mask, bits)?; - } - let colour = if gray { fields[0].is_some() } else { fields[..3].iter().any(Option::is_some) }; - if !colour { - return Err(bad("the pixel format has no colour channel masks")); - } - Ok(Encoding::Masks { bytes: bits as usize / 8, gray, fields }) - } - - fn layout(&self) -> ChannelLayout { - match self { - Encoding::Masks { gray: true, fields, .. } if fields[3].is_some() => ChannelLayout::GrayA, - Encoding::Masks { gray: true, .. } => ChannelLayout::Gray, - Encoding::Masks { fields, .. } if fields[3].is_none() => ChannelLayout::Rgb, - _ => ChannelLayout::Rgba, - } - } - - /// Stored row stride and byte size of the top mip level (block rows for BC). - fn level_layout(&self, width: u32, height: u32, pitch: Option) -> Result<(usize, usize), CodecError> { - let overflow = || bad("pixel data size overflow"); - let (w, h) = (usize::try_from(width).map_err(|_| overflow())?, usize::try_from(height).map_err(|_| overflow())?); - let (row_bytes, rows) = match self { - Encoding::Masks { bytes, .. } => { - let packed = w.checked_mul(*bytes).ok_or_else(overflow)?; - let row_bytes = match pitch { - Some(pitch) => usize::try_from(pitch).map_err(|_| overflow())?, - None => packed, - }; - if row_bytes < packed { - return Err(bad(format!("row pitch {row_bytes} is smaller than the {packed}-byte pixel row"))); - } - (row_bytes, h) - } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block = if matches!(self, Encoding::Bc1) { 8 } else { 16 }; - (w.div_ceil(4).checked_mul(block).ok_or_else(overflow)?, h.div_ceil(4)) - } - }; - let size = row_bytes.checked_mul(rows).ok_or_else(overflow)?; - Ok((row_bytes, size)) - } -} - -/// The legacy pixel format: a FourCC, or channel masks. -fn legacy_encoding(b: &[u8]) -> Result { - let (flags, fourcc, bits) = (le32(b, 80)?, le32(b, 84)?, le32(b, 88)?); - let alpha = if flags & DDPF_ALPHAPIXELS != 0 { le32(b, 104)? } else { 0 }; - if flags & DDPF_FOURCC != 0 { - return match &fourcc.to_le_bytes() { - b"DXT1" => Ok(Encoding::Bc1), - b"DXT3" => Ok(Encoding::Bc2), - b"DXT5" => Ok(Encoding::Bc3), - b"DXT2" | b"DXT4" => Err(unsupported("premultiplied-alpha DXT2/DXT4 textures are not supported")), - _ => Err(unsupported(format!("compression `{}` is not supported (DXT1, DXT3 and DXT5 are)", fourcc_name(fourcc)))), - }; - } - if flags & DDPF_RGB != 0 { - return Encoding::masks(bits, [le32(b, 92)?, le32(b, 96)?, le32(b, 100)?, alpha], false); - } - if flags & DDPF_LUMINANCE != 0 { - return Encoding::masks(bits, [le32(b, 92)?, 0, 0, alpha], true); - } - Err(unsupported(format!("pixel format flags {flags:#x} are not RGB, luminance or a supported compression"))) -} - -/// The DX10 header's DXGI format, refusing anything but a single 2D texture. -fn dx10_encoding(b: &[u8]) -> Result { - let (dxgi, dimension, misc, array) = (le32(b, HEADER)?, le32(b, HEADER + 4)?, le32(b, HEADER + 8)?, le32(b, HEADER + 12)?); - if misc & MISC_TEXTURECUBE != 0 { - return Err(unsupported("cubemaps are not supported")); - } - if dimension != DIMENSION_TEXTURE2D { - return Err(unsupported(format!("only 2D textures are supported (resource dimension {dimension})"))); - } - if array > 1 { - return Err(unsupported(format!("texture arrays are not supported ({array} textures)"))); - } - const RGBA: [u32; 4] = [0xff, 0xff00, 0xff_0000, 0xff00_0000]; - const BGRA: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0xff00_0000]; - const BGRX: [u32; 4] = [0xff_0000, 0xff00, 0xff, 0]; - match dxgi { - 27..=29 => Encoding::masks(32, RGBA, false), - 87 | 90 | 91 => Encoding::masks(32, BGRA, false), - 88 | 92 | 93 => Encoding::masks(32, BGRX, false), - 70..=72 => Ok(Encoding::Bc1), - 73..=75 => Ok(Encoding::Bc2), - 76..=78 => Ok(Encoding::Bc3), - _ => Err(unsupported(format!("DXGI format {dxgi} is not supported (BC1, BC2, BC3 and 8-bit RGBA or BGRA are)"))), - } -} - -pub fn decode(b: &[u8], limits: &Limits) -> Result { - if !b.starts_with(b"DDS ") { - return Err(bad("missing the `DDS ` signature")); - } - if le32(b, 4)? != 124 { - return Err(bad("header size is not 124")); - } - let (flags, height, width, depth, caps2) = (le32(b, 8)?, le32(b, 12)?, le32(b, 16)?, le32(b, 24)?, le32(b, 112)?); - if caps2 & DDSCAPS2_CUBEMAP != 0 { - return Err(unsupported("cubemaps are not supported")); - } - if caps2 & DDSCAPS2_VOLUME != 0 || (flags & DDSD_DEPTH != 0 && depth > 1) { - return Err(unsupported("volume textures are not supported")); - } - let dx10 = le32(b, 80)? & DDPF_FOURCC != 0 && &le32(b, 84)?.to_le_bytes() == b"DX10"; - let (encoding, start) = if dx10 { (dx10_encoding(b)?, HEADER + DX10_HEADER) } else { (legacy_encoding(b)?, HEADER) }; - let layout = encoding.layout(); - limits.check(width, height, layout, SampleType::U8)?; - // The same header field is a total level size for BC, but a row stride when DDSD_PITCH is set. - let pitch = if flags & DDSD_PITCH != 0 { Some(le32(b, 20)?) } else { None }; - let (row_pitch, size) = encoding.level_layout(width, height, pitch)?; - let data = b.get(start..).and_then(|d| d.get(..size)).ok_or_else(|| bad(format!("truncated pixel data: {size} bytes needed")))?; - let (w, h) = (width as usize, height as usize); - let channels = layout.channels(); - let row_size = w.checked_mul(channels).ok_or_else(|| bad("pixel size overflow"))?; - let total = row_size.checked_mul(h).ok_or_else(|| bad("pixel size overflow"))?; - let mut pixels = vec![0u8; total]; - match encoding { - Encoding::Masks { bytes, gray, fields } => { - for (src_row, dst_row) in data.chunks_exact(row_pitch).zip(pixels.chunks_exact_mut(row_size)) { - // Stop at the output width, leaving any stored row padding unread. - for (src, dst) in src_row.chunks_exact(bytes).zip(dst_row.chunks_exact_mut(channels)) { - let pixel = src.iter().rev().fold(0u32, |v, &byte| v << 8 | u32::from(byte)); - let channel = |i: usize| fields[i].map_or(0, |f| f.get(pixel)); - let values = if gray { [channel(0), channel(3), 0, 0] } else { [channel(0), channel(1), channel(2), channel(3)] }; - dst.copy_from_slice(&values[..channels]); - } - } - } - Encoding::Bc1 | Encoding::Bc2 | Encoding::Bc3 => { - let block_size = if matches!(encoding, Encoding::Bc1) { 8 } else { 16 }; - let blocks_x = w.div_ceil(4); - for (i, block) in data.chunks_exact(block_size).enumerate() { - let texels = match encoding { - Encoding::Bc1 => bc1(block, true), - Encoding::Bc2 => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc2_alpha(block)), - _ => with_alpha(bc1(block.get(8..).unwrap_or_default(), false), bc3_alpha(block)), - }; - let (bx, by) = ((i % blocks_x) * 4, (i / blocks_x) * 4); - for (t, texel) in texels.iter().enumerate() { - let (x, y) = (bx + t % 4, by + t / 4); - if x < w && y < h { - let at = (y * w + x) * 4; - if let Some(dst) = pixels.get_mut(at..at + 4) { - dst.copy_from_slice(texel); - } - } - } - } - } - } - Image::from_u8(width, height, layout, pixels) -} - -/// A BC1 colour block. BC2 and BC3 colour blocks always have four colours; only BC1's -/// `c0 <= c1` order selects three colours and transparent black. -fn bc1(block: &[u8], one_bit_alpha: bool) -> [[u8; 4]; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let (c0, c1) = (u16::from_le_bytes([byte(0), byte(1)]), u16::from_le_bytes([byte(2), byte(3)])); - let palette = dds_bc::colour_palette(c0, c1, c0 > c1 || !one_bit_alpha); - let indices = u32::from_le_bytes([byte(4), byte(5), byte(6), byte(7)]); - std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) -} - -/// BC2: four explicit bits of alpha per texel. -fn bc2_alpha(block: &[u8]) -> [u8; 16] { - let bits = u64::from_le_bytes(std::array::from_fn(|i| block.get(i).copied().unwrap_or(0))); - std::array::from_fn(|t| ((bits >> (4 * t)) & 0xf) as u8 * 17) -} - -/// BC3: two alpha endpoints and a 3-bit index per texel into eight values, or into six plus -/// 0 and 255 when `a0 <= a1`. -fn bc3_alpha(block: &[u8]) -> [u8; 16] { - let byte = |i: usize| block.get(i).copied().unwrap_or(0); - let palette = dds_bc::alpha_palette(byte(0), byte(1)); - let bits = u64::from_le_bytes([byte(2), byte(3), byte(4), byte(5), byte(6), byte(7), 0, 0]); - std::array::from_fn(|t| palette[((bits >> (3 * t)) & 7) as usize]) -} - -fn with_alpha(mut texels: [[u8; 4]; 16], alpha: [u8; 16]) -> [[u8; 4]; 16] { - for (texel, a) in texels.iter_mut().zip(alpha) { - texel[3] = a; - } - texels -} - -const DDSD_CAPS: u32 = 0x1; -const DDSD_HEIGHT: u32 = 0x2; -const DDSD_WIDTH: u32 = 0x4; -const DDSD_PITCH: u32 = 0x8; -const DDSD_PIXELFORMAT: u32 = 0x1000; -const DDSD_MIPMAPCOUNT: u32 = 0x2_0000; -const DDSD_LINEARSIZE: u32 = 0x8_0000; -const DDSCAPS_COMPLEX: u32 = 0x8; -const DDSCAPS_TEXTURE: u32 = 0x1000; -const DDSCAPS_MIPMAP: u32 = 0x40_0000; - -fn encode_err(reason: impl std::fmt::Display) -> CodecError { - CodecError::encode("DDS", reason) -} - -/// How a written texture stores its pixels. -#[derive(Clone, Copy, PartialEq, Eq)] -enum Store { - /// 8-bit luminance. - Luminance, - /// 8-bit luminance, then 8-bit alpha. - LuminanceAlpha, - /// Blue, green, red and an unused byte (written as 255, so a reader that takes it for - /// alpha still sees an opaque texture). - Bgrx, - Bgra, - Bc1, - Bc2, - Bc3, -} - -impl Store { - /// The storage for an image of `layout`. - fn new(compression: DdsCompression, layout: ChannelLayout) -> Result { - Ok(match (compression, layout) { - (DdsCompression::None, ChannelLayout::Gray) => Store::Luminance, - (DdsCompression::None, ChannelLayout::GrayA) => Store::LuminanceAlpha, - (DdsCompression::None, ChannelLayout::Rgb) => Store::Bgrx, - (DdsCompression::None, ChannelLayout::Rgba) => Store::Bgra, - (DdsCompression::Bc1, ChannelLayout::Rgba) => Store::Bc1, - (DdsCompression::Bc2, ChannelLayout::Rgba) => Store::Bc2, - (DdsCompression::Bc3, ChannelLayout::Rgba) => Store::Bc3, - (_, layout) => return Err(encode_err(format!("{layout:?} pixels can't be written as `{}`", compression.name()))), - }) - } - - /// Bytes per pixel, or per 4×4 block for a block compression. - fn unit(self) -> usize { - match self { - Store::Luminance => 1, - Store::LuminanceAlpha => 2, - Store::Bgrx | Store::Bgra => 4, - Store::Bc1 => 8, - Store::Bc2 | Store::Bc3 => 16, - } - } - - fn is_block(self) -> bool { - matches!(self, Store::Bc1 | Store::Bc2 | Store::Bc3) - } - - /// Bytes of one row of pixels (or of 4×4 blocks) of a level `width` wide. - fn row_bytes(self, width: usize) -> Option { - if self.is_block() { width.div_ceil(4).checked_mul(self.unit()) } else { width.checked_mul(self.unit()) } - } - - fn level_bytes(self, width: usize, height: usize) -> Option { - let rows = if self.is_block() { height.div_ceil(4) } else { height }; - self.row_bytes(width)?.checked_mul(rows) - } - - /// DDS_PIXELFORMAT: flags, FourCC, bits per pixel and the red, green, blue and alpha masks. - fn pixel_format(self) -> (u32, [u8; 4], u32, [u32; 4]) { - match self { - Store::Luminance => (DDPF_LUMINANCE, [0; 4], 8, [0xff, 0, 0, 0]), - Store::LuminanceAlpha => (DDPF_LUMINANCE | DDPF_ALPHAPIXELS, [0; 4], 16, [0xff, 0, 0, 0xff00]), - Store::Bgrx => (DDPF_RGB, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0]), - Store::Bgra => (DDPF_RGB | DDPF_ALPHAPIXELS, [0; 4], 32, [0xff_0000, 0xff00, 0xff, 0xff00_0000]), - Store::Bc1 => (DDPF_FOURCC, *b"DXT1", 0, [0; 4]), - Store::Bc2 => (DDPF_FOURCC, *b"DXT3", 0, [0; 4]), - Store::Bc3 => (DDPF_FOURCC, *b"DXT5", 0, [0; 4]), - } - } -} - -/// Levels in a full mip chain: the larger side halves down to 1. -fn mip_levels(width: u32, height: u32) -> u32 { - 32 - width.max(height).max(1).leading_zeros() -} - -fn half((w, h): (usize, usize)) -> (usize, usize) { - ((w / 2).max(1), (h / 2).max(1)) -} - -/// How a DDS texture stores its pixels. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum DdsCompression { - /// Uncompressed, 8 bits per channel: BGRA (BGRX without alpha), or luminance with or - /// without alpha for grayscale. Lossless. #[default] None, - /// BC1 (DXT1): 4 bits per pixel. Alpha is on or off: texels under half opacity become - /// transparent black. Bc1, - /// BC2 (DXT3): 8 bits per pixel, with 4 bits of explicit alpha per texel. Bc2, - /// BC3 (DXT5): 8 bits per pixel, with smooth interpolated alpha. Bc3, } - impl DdsCompression { - pub const ALL: [DdsCompression; 4] = [DdsCompression::None, DdsCompression::Bc1, DdsCompression::Bc2, DdsCompression::Bc3]; - - /// The name scripts and the CLI use: `none`, `bc1`, `bc2` or `bc3`. + pub const ALL: [Self; 4] = [Self::None, Self::Bc1, Self::Bc2, Self::Bc3]; pub fn name(self) -> &'static str { match self { - DdsCompression::None => "none", - DdsCompression::Bc1 => "bc1", - DdsCompression::Bc2 => "bc2", - DdsCompression::Bc3 => "bc3", + Self::None => "none", + Self::Bc1 => "bc1", + Self::Bc2 => "bc2", + Self::Bc3 => "bc3", } } - - /// [`Self::name`] back to a compression, also taking the DXT names (`dxt1`, `dxt3`, - /// `dxt5`), in any case. Anything else is `None`. pub fn from_name(name: &str) -> Option { match name.to_ascii_lowercase().as_str() { - "none" => Some(DdsCompression::None), - "bc1" | "dxt1" => Some(DdsCompression::Bc1), - "bc2" | "dxt3" => Some(DdsCompression::Bc2), - "bc3" | "dxt5" => Some(DdsCompression::Bc3), + "none" => Some(Self::None), + "bc1" | "dxt1" => Some(Self::Bc1), + "bc2" | "dxt3" => Some(Self::Bc2), + "bc3" | "dxt5" => Some(Self::Bc3), _ => None, } } - - /// A block compression (lossy), not uncompressed pixels. pub fn is_block(self) -> bool { - self != DdsCompression::None + self != Self::None } } - -/// What a DDS file is written with. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub struct DdsOptions { pub compression: DdsCompression, - /// Write the full mip chain down to 1×1, each level a box filter of the one above. pub mipmaps: bool, } - -/// Writes `img` (8 bits per channel) as a DDS texture. Uncompressed stores gray, gray with -/// alpha, RGB and RGBA; the block compressions take RGBA only. -pub fn encode(img: &Image, opts: DdsOptions) -> Result, CodecError> { - if img.sample_type() != SampleType::U8 { - return Err(encode_err("DDS is written at 8 bits per channel")); +pub fn encode(image: &Image, options: DdsOptions) -> Result, CodecError> { + if image.sample_type() != SampleType::U8 { + return Err(CodecError::encode("DDS", "DDS is written at 8 bits per channel")); } - let pixels = img.data(); - let store = Store::new(opts.compression, img.layout())?; - let (width, height) = (img.width(), img.height()); - let levels = if opts.mipmaps { mip_levels(width, height) } else { 1 }; - let top = (width as usize, height as usize); - // Every level's size, checked before anything is allocated. - let mut sizes = Vec::new(); - let mut size = top; - for _ in 0..levels { - sizes.push(store.level_bytes(size.0, size.1).ok_or_else(|| encode_err("texture size overflow"))?); - size = half(size); - } - let total = sizes.iter().try_fold(HEADER, |total, &s| total.checked_add(s)).ok_or_else(|| encode_err("texture size overflow"))?; - let header = header(width, height, store, levels, sizes.first().copied().unwrap_or(0))?; - let mut out = crate::alloc_zeroed::(total)?; - out.get_mut(..HEADER).ok_or_else(|| encode_err("output buffer too small"))?.copy_from_slice(&header); - - let channels = img.layout().channels(); - let mut level = Cow::Borrowed(pixels); - let (mut size, mut at) = (top, HEADER); - for (i, &bytes) in sizes.iter().enumerate() { - let dst = out.get_mut(at..at + bytes).ok_or_else(|| encode_err("output buffer too small"))?; - write_level(store, &level, size, dst)?; - at += bytes; - if i + 1 < sizes.len() { - let next = half(size); - level = Cow::Owned(downsample(&level, size, next, channels)?); - size = next; - } - } - Ok(out) -} - -/// The `DDS ` signature and DDS_HEADER. `top` is the top level's size in bytes. -fn header(width: u32, height: u32, store: Store, levels: u32, top: usize) -> Result<[u8; HEADER], CodecError> { - let mut b = [0u8; HEADER]; - let mut put = |at: usize, v: u32| { - if let Some(d) = b.get_mut(at..at + 4) { - d.copy_from_slice(&v.to_le_bytes()); - } + let compression = match options.compression { + DdsCompression::None => craft_image_codecs::DdsCompression::None, + DdsCompression::Bc1 => craft_image_codecs::DdsCompression::Bc1, + DdsCompression::Bc2 => craft_image_codecs::DdsCompression::Bc2, + DdsCompression::Bc3 => craft_image_codecs::DdsCompression::Bc3, }; - // Uncompressed: the pitch of one row. Compressed: the whole top level (its "linear size"). - let (size_flag, pitch_or_size) = if store.is_block() { (DDSD_LINEARSIZE, Some(top)) } else { (DDSD_PITCH, store.row_bytes(width as usize)) }; - let pitch_or_size = pitch_or_size - .and_then(|v| u32::try_from(v).ok()) - .ok_or_else(|| encode_err(format!("a {width}x{height} level is too large for a DDS header")))?; - let mipmapped = levels > 1; - put(4, 124); - put(8, DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT | size_flag | if mipmapped { DDSD_MIPMAPCOUNT } else { 0 }); - put(12, height); - put(16, width); - put(20, pitch_or_size); - put(28, levels); - let (flags, fourcc, bits, masks) = store.pixel_format(); - put(76, 32); - put(80, flags); - put(84, u32::from_le_bytes(fourcc)); - put(88, bits); - for (i, mask) in masks.into_iter().enumerate() { - put(92 + 4 * i, mask); - } - put(108, DDSCAPS_TEXTURE | if mipmapped { DDSCAPS_COMPLEX | DDSCAPS_MIPMAP } else { 0 }); - b[..4].copy_from_slice(b"DDS "); - Ok(b) -} - -/// Runs `f` on each `len`-byte row of `out` with its index, in parallel on native targets. -fn rows(out: &mut [u8], len: usize, f: impl Fn(usize, &mut [u8]) + Send + Sync) { - #[cfg(not(target_arch = "wasm32"))] - { - use rayon::prelude::*; - out.par_chunks_mut(len.max(1)).enumerate().for_each(|(i, row)| f(i, row)); - } - #[cfg(target_arch = "wasm32")] - for (i, row) in out.chunks_mut(len.max(1)).enumerate() { - f(i, row); - } -} - -/// Writes one level of `width` × `height` pixels (in the image's layout) into `dst`. -fn write_level(store: Store, pixels: &[u8], (width, height): (usize, usize), dst: &mut [u8]) -> Result<(), CodecError> { - let row_bytes = store.row_bytes(width).ok_or_else(|| encode_err("texture size overflow"))?; - match store { - Store::Luminance | Store::LuminanceAlpha => { - // The image's gray (and alpha) samples, in the same order. - let src = pixels.get(..dst.len()).ok_or_else(|| encode_err("the level is smaller than its size"))?; - dst.copy_from_slice(src); - } - Store::Bgrx | Store::Bgra => { - let channels = if store == Store::Bgra { 4 } else { 3 }; - rows(dst, row_bytes, |y, row| { - let src = pixels.get(y * width * channels..).unwrap_or_default(); - for (d, s) in row.as_chunks_mut::<4>().0.iter_mut().zip(src.chunks_exact(channels)) { - let byte = |i: usize| s.get(i).copied().unwrap_or(255); - *d = [byte(2), byte(1), byte(0), byte(3)]; - } - }); - } - Store::Bc1 | Store::Bc2 | Store::Bc3 => { - let (texels, _) = pixels.as_chunks::<4>(); - rows(dst, row_bytes, |by, row| { - for (bx, d) in row.chunks_exact_mut(store.unit()).enumerate() { - let (block, inside) = gather(texels, (width, height), bx, by); - let encoded: &[u8] = match store { - Store::Bc1 => &dds_bc::encode_bc1(&block, inside), - Store::Bc2 => &dds_bc::encode_bc2(&block, inside), - _ => &dds_bc::encode_bc3(&block, inside), - }; - for (d, s) in d.iter_mut().zip(encoded) { - *d = *s; - } - } - }); - } - } - Ok(()) -} - -/// Block (`bx`, `by`)'s texels, and which of them lie inside the image. -fn gather(texels: &[[u8; 4]], (width, height): (usize, usize), bx: usize, by: usize) -> (Block, Mask) { - let mut block = [[0u8; 4]; 16]; - let mut inside: Mask = 0; - for (t, texel) in block.iter_mut().enumerate() { - let (x, y) = (bx * 4 + t % 4, by * 4 + t / 4); - if x < width - && y < height - && let Some(p) = texels.get(y * width + x) - { - *texel = *p; - inside |= 1 << t; - } - } - (block, inside) -} - -/// The next mip level: each pixel the average of the 2×2 pixels it covers, or of 2×3, 3×2 or -/// 3×3 along the last column and row of an odd-sized level, so no pixel is dropped. -fn downsample(src: &[u8], (w, h): (usize, usize), (nw, nh): (usize, usize), channels: usize) -> Result, CodecError> { - let len = nw.checked_mul(nh).and_then(|n| n.checked_mul(channels)).ok_or_else(|| encode_err("mip level size overflow"))?; - let mut dst = crate::alloc_zeroed::(len)?; - let span = |i: usize, n: usize, size: usize| 2 * i..=if i + 1 == n { size.saturating_sub(1) } else { 2 * i + 1 }; - rows(&mut dst, nw * channels, |y, row| { - let ys = span(y, nh, h); - for (x, px) in row.chunks_exact_mut(channels.max(1)).enumerate() { - let xs = span(x, nw, w); - for (c, v) in px.iter_mut().enumerate() { - let (mut sum, mut n) = (0u32, 0u32); - for sy in ys.clone() { - for sx in xs.clone() { - if let Some(&s) = src.get((sy * w + sx) * channels + c) { - sum += u32::from(s); - n += 1; - } - } - } - *v = ((sum + n / 2) / n.max(1)) as u8; - } - } - }); - Ok(dst) + let opts = + craft_image_codecs::EncodeOptions { dds: craft_image_codecs::DdsOptions { compression, mipmaps: options.mipmaps }, ..Default::default() }; + bridge::encode(image, Format::Dds, &opts) } diff --git a/crates/raster-codecs/src/dds_bc.rs b/crates/raster-codecs/src/dds_bc.rs deleted file mode 100644 index df60e462c..000000000 --- a/crates/raster-codecs/src/dds_bc.rs +++ /dev/null @@ -1,369 +0,0 @@ -//! BC1, BC2 and BC3 (DXT1, DXT3, DXT5) blocks, after Microsoft's public block compression -//! documentation ("Block Compression (Direct3D 10)" and "Texture Block Compression in Direct3D -//! 11"): the palettes the DDS reader decodes with, and the block encoder the DDS writer uses. -//! -//! The encoder fits a block's colours with two RGB565 endpoints along their principal axis, -//! refines them by least squares, and scores every candidate against the exact palette the -//! reader rebuilds, so the indices it picks are the best ones for what a decoder shows. - -/// The 16 texels of a 4×4 block, row by row, as RGBA. -pub(crate) type Block = [[u8; 4]; 16]; - -/// Bit `t` set for each texel `t` of a block that counts: edge blocks of a texture whose size is -/// not a multiple of four have texels outside the image. -pub(crate) type Mask = u16; - -pub(crate) fn rgb565(c: u16) -> [u8; 3] { - let scale = |v: u16, max: u16| ((u32::from(v) * 255 + u32::from(max) / 2) / u32::from(max)) as u8; - [scale(c >> 11, 31), scale((c >> 5) & 63, 63), scale(c & 31, 31)] -} - -/// The colours a colour block's indices select. Four-colour mode (BC1 with `c0 > c1`, and every -/// BC2 and BC3 block) adds two colours at thirds; three-colour mode adds the midpoint and -/// transparent black. -pub(crate) fn colour_palette(c0: u16, c1: u16, four: bool) -> [[u8; 4]; 4] { - let (a, b) = (rgb565(c0), rgb565(c1)); - let mix = |wa: u32, wb: u32| { - let d = wa + wb; - let at = |i: usize| ((wa * u32::from(a[i]) + wb * u32::from(b[i]) + d / 2) / d) as u8; - [at(0), at(1), at(2), 255] - }; - if four { - [[a[0], a[1], a[2], 255], [b[0], b[1], b[2], 255], mix(2, 1), mix(1, 2)] - } else { - [[a[0], a[1], a[2], 255], [b[0], b[1], b[2], 255], mix(1, 1), [0; 4]] - } -} - -/// A BC3 alpha block's eight values: six interpolated at sevenths when `a0 > a1`, else four -/// interpolated at fifths followed by 0 and 255. -pub(crate) fn alpha_palette(a0: u8, a1: u8) -> [u8; 8] { - let (a0, a1) = (u32::from(a0), u32::from(a1)); - let mut palette = [a0, a1, 0, 0, 0, 0, 0, 255]; - if a0 > a1 { - for (k, v) in palette.iter_mut().enumerate().skip(2) { - let k = k as u32; - *v = ((8 - k) * a0 + (k - 1) * a1 + 3) / 7; - } - } else { - for (k, v) in palette.iter_mut().enumerate().take(6).skip(2) { - let k = k as u32; - *v = ((6 - k) * a0 + (k - 1) * a1 + 2) / 5; - } - } - palette.map(|v| v as u8) -} - -/// A BC1 block. Texels under half opacity become BC1's transparent black, which needs the -/// three-colour mode; an opaque block takes whichever mode fits its colours better. -pub(crate) fn encode_bc1(block: &Block, inside: Mask) -> [u8; 8] { - let transparent = select(block, inside, |t| t[3] < 128); - colour_block(block, inside & !transparent, transparent, true) -} - -/// A BC2 block: four bits of alpha per texel, then a four-colour block. -pub(crate) fn encode_bc2(block: &Block, inside: Mask) -> [u8; 16] { - let bits = block.iter().enumerate().fold(0u64, |bits, (t, texel)| bits | ((u64::from(texel[3]) * 15 + 127) / 255) << (4 * t)); - join(bits.to_le_bytes(), colour_block(block, inside, 0, false)) -} - -/// A BC3 block: an interpolated alpha block, then a four-colour block. -pub(crate) fn encode_bc3(block: &Block, inside: Mask) -> [u8; 16] { - join(alpha_block(block, inside), colour_block(block, inside, 0, false)) -} - -fn join(a: [u8; 8], b: [u8; 8]) -> [u8; 16] { - std::array::from_fn(|i| if i < 8 { a.get(i) } else { b.get(i - 8) }.copied().unwrap_or(0)) -} - -/// The texels of `inside` for which `f` holds. -fn select(block: &Block, inside: Mask, f: impl Fn(&[u8; 4]) -> bool) -> Mask { - block.iter().enumerate().filter(|(t, texel)| inside >> t & 1 != 0 && f(texel)).fold(0, |m, (t, _)| m | 1 << t) -} - -fn rgb_of(texel: &[u8; 4]) -> [f32; 3] { - [f32::from(texel[0]), f32::from(texel[1]), f32::from(texel[2])] -} - -/// RGB565 of a colour, rounding each channel to the nearest level. -fn to565(c: [f32; 3]) -> u16 { - let q = |v: f32, max: f32| (v.clamp(0.0, 255.0) * max / 255.0).round() as u16; - q(c[0], 31.0) << 11 | q(c[1], 63.0) << 5 | q(c[2], 31.0) -} - -fn distance(a: &[u8; 4], b: &[u8; 4]) -> u32 { - (0..3).map(|i| u32::from(a[i].abs_diff(b[i])).pow(2)).sum() -} - -/// A colour block (endpoints, then 2-bit indices) for the `opaque` texels' colours, with the -/// `transparent` ones on BC1's transparent index. `bc1` allows the three-colour mode; BC2 and -/// BC3 always decode four colours. -fn colour_block(block: &Block, opaque: Mask, transparent: Mask, bc1: bool) -> [u8; 8] { - let colours: Vec<[f32; 3]> = block.iter().enumerate().filter(|(t, _)| opaque >> t & 1 != 0).map(|(_, texel)| rgb_of(texel)).collect(); - if colours.is_empty() { - // Nothing visible: every texel on the transparent index (or index 0 of black). - let indices = if transparent != 0 { u32::MAX } else { 0 }; - return pack(0, 0, indices); - } - let (lo, hi) = principal_endpoints(&colours); - let mut candidates = vec![(to565(lo), to565(hi))]; - let modes: &[bool] = if transparent != 0 { - &[false] - } else if bc1 { - &[true, false] - } else { - &[true] - }; - let mut best: Option<(u32, [u8; 8])> = None; - // Each refinement starts from the best block so far: its indices give the least-squares - // endpoints, which become one more candidate. - for round in 0..3 { - for &(e0, e1) in &candidates { - for &four in modes { - let (bytes, error) = fit(block, opaque, transparent, e0, e1, four); - if best.is_none_or(|(e, _)| error < e) { - best = Some((error, bytes)); - } - } - } - let Some((error, bytes)) = best else { break }; - if error == 0 || round == 2 { - break; - } - candidates = least_squares(block, opaque, &bytes).into_iter().collect(); - if candidates.is_empty() { - break; - } - } - best.map_or([0; 8], |(_, bytes)| bytes) -} - -/// Orders endpoints `e0`, `e1` for the mode (`c0 > c1` for four colours, `c0 <= c1` for three) -/// and picks each texel's nearest palette colour. Returns the block and its squared error. -fn fit(block: &Block, opaque: Mask, transparent: Mask, e0: u16, e1: u16, four: bool) -> ([u8; 8], u32) { - let (c0, c1) = if four == (e0 >= e1) { (e0, e1) } else { (e1, e0) }; - // Equal endpoints can't be in four-colour order; the palette is then one colour either way. - let palette = colour_palette(c0, c1, four && c0 > c1); - let choices = if four && c0 > c1 { 4 } else { 3 }; - let mut indices = 0u32; - let mut error = 0u32; - for (t, texel) in block.iter().enumerate() { - let index = if transparent >> t & 1 != 0 { - 3 - } else if opaque >> t & 1 != 0 { - let (i, d) = palette.iter().take(choices).enumerate().map(|(i, p)| (i, distance(texel, p))).min_by_key(|&(_, d)| d).unwrap_or((0, 0)); - error += d; - i as u32 - } else { - 0 - }; - indices |= index << (2 * t); - } - (pack(c0, c1, indices), error) -} - -fn pack(c0: u16, c1: u16, indices: u32) -> [u8; 8] { - let (c0, c1, i) = (c0.to_le_bytes(), c1.to_le_bytes(), indices.to_le_bytes()); - [c0[0], c0[1], c1[0], c1[1], i[0], i[1], i[2], i[3]] -} - -/// The ends of the colours' spread along their principal axis (the covariance's dominant -/// eigenvector, by power iteration), clamped to the RGB cube. -fn principal_endpoints(colours: &[[f32; 3]]) -> ([f32; 3], [f32; 3]) { - let n = colours.len().max(1) as f32; - let mean = [0, 1, 2].map(|i| colours.iter().map(|c| c[i]).sum::() / n); - let mut cov = [[0f32; 3]; 3]; - for c in colours { - let d = [c[0] - mean[0], c[1] - mean[1], c[2] - mean[2]]; - for (i, row) in cov.iter_mut().enumerate() { - for (j, v) in row.iter_mut().enumerate() { - *v += d[i] * d[j]; - } - } - } - let times = |a: [f32; 3]| [0, 1, 2].map(|i| cov[i][0] * a[0] + cov[i][1] * a[1] + cov[i][2] * a[2]); - let iterate = |start: usize| { - let mut axis = [0f32; 3]; - axis[start] = 1.0; - for _ in 0..8 { - let next = times(axis); - let len = next.iter().map(|v| v * v).sum::().sqrt(); - if !len.is_normal() { - break; - } - axis = next.map(|v| v / len); - } - axis - }; - // From each channel's axis, keeping the one the colours spread along most: a channel that - // varies on its own is already an eigenvector, so a single start there would never leave it - // even when the others vary more together. - let spread = |a: [f32; 3]| times(a).iter().zip(a).map(|(x, y)| x * y).sum::(); - let axis = [0, 1, 2].map(iterate).into_iter().max_by(|&a, &b| spread(a).total_cmp(&spread(b))).unwrap_or([1.0, 0.0, 0.0]); - let along = |c: &[f32; 3]| (c[0] - mean[0]) * axis[0] + (c[1] - mean[1]) * axis[1] + (c[2] - mean[2]) * axis[2]; - let (lo, hi) = colours.iter().map(along).fold((f32::MAX, f32::MIN), |(lo, hi), t| (lo.min(t), hi.max(t))); - let at = |t: f32| [0, 1, 2].map(|i| (mean[i] + t * axis[i]).clamp(0.0, 255.0)); - if lo > hi { (mean, mean) } else { (at(lo), at(hi)) } -} - -/// The endpoints that minimise the squared error for the indices `bytes` already chose: each -/// texel's index fixes its weights between the endpoints, giving 2×2 normal equations per -/// channel. `None` when the indices don't constrain both endpoints. -fn least_squares(block: &Block, opaque: Mask, bytes: &[u8; 8]) -> Option<(u16, u16)> { - let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]])); - let indices = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); - let four = c0 > c1; - // How much of c0 each index holds. - let weight = |i: u32| match (four, i) { - (_, 0) => Some(1.0f32), - (_, 1) => Some(0.0), - (true, 2) => Some(2.0 / 3.0), - (true, _) => Some(1.0 / 3.0), - (false, 2) => Some(0.5), - (false, _) => None, - }; - let (mut aa, mut ab, mut bb) = (0f32, 0f32, 0f32); - let (mut ax, mut bx) = ([0f32; 3], [0f32; 3]); - for (t, texel) in block.iter().enumerate() { - if opaque >> t & 1 == 0 { - continue; - } - let Some(a) = weight(indices >> (2 * t) & 3) else { continue }; - let b = 1.0 - a; - let c = rgb_of(texel); - aa += a * a; - ab += a * b; - bb += b * b; - for ((x0, x1), v) in ax.iter_mut().zip(bx.iter_mut()).zip(c) { - *x0 += a * v; - *x1 += b * v; - } - } - let det = aa * bb - ab * ab; - if det.abs() < 1e-3 { - return None; - } - let e0 = [0, 1, 2].map(|i| (ax[i] * bb - bx[i] * ab) / det); - let e1 = [0, 1, 2].map(|i| (bx[i] * aa - ax[i] * ab) / det); - Some((to565(e0), to565(e1))) -} - -/// A BC3 alpha block: whichever of the two modes (eight values between the extremes, or six -/// between the extremes other than 0 and 255, plus exact 0 and 255) fits better. -fn alpha_block(block: &Block, inside: Mask) -> [u8; 8] { - let alphas = || block.iter().enumerate().filter(move |(t, _)| inside >> t & 1 != 0).map(|(_, texel)| texel[3]); - let range = - |values: &mut dyn Iterator| values.fold(None, |r: Option<(u8, u8)>, a| Some(r.map_or((a, a), |(lo, hi)| (lo.min(a), hi.max(a))))); - let (lo, hi) = range(&mut alphas()).unwrap_or((0, 0)); - // Eight values need a0 > a1; equal ends fall to the six-value mode, which starts with a0 too. - let eight = alpha_fit(block, inside, hi, lo); - let (mid_lo, mid_hi) = range(&mut alphas().filter(|a| !matches!(a, 0 | 255))).unwrap_or((0, 0)); - let six = alpha_fit(block, inside, mid_lo, mid_hi); - if six.1 < eight.1 { six.0 } else { eight.0 } -} - -fn alpha_fit(block: &Block, inside: Mask, a0: u8, a1: u8) -> ([u8; 8], u32) { - let palette = alpha_palette(a0, a1); - let mut bits = 0u64; - let mut error = 0u32; - for (t, texel) in block.iter().enumerate() { - if inside >> t & 1 == 0 { - continue; - } - let (i, d) = palette.iter().enumerate().map(|(i, &p)| (i, u32::from(p.abs_diff(texel[3])).pow(2))).min_by_key(|&(_, d)| d).unwrap_or((0, 0)); - error += d; - bits |= (i as u64) << (3 * t); - } - let b = bits.to_le_bytes(); - ([a0, a1, b[0], b[1], b[2], b[3], b[4], b[5]], error) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn decode_colour(bytes: &[u8; 8], bc1: bool) -> Block { - let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]])); - let palette = colour_palette(c0, c1, c0 > c1 || !bc1); - let indices = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]); - std::array::from_fn(|t| palette[(indices >> (2 * t)) as usize & 3]) - } - - #[test] - fn solid_blocks_land_on_the_nearest_565_colour() { - for colour in [[0, 0, 0, 255], [255, 255, 255, 255], [12, 200, 77, 255]] { - let block = [colour; 16]; - let out = decode_colour(&encode_bc1(&block, u16::MAX), true); - for texel in out { - assert!(texel.iter().zip(colour).all(|(a, b)| a.abs_diff(b) <= 4), "{texel:?} for {colour:?}"); - } - } - } - - #[test] - fn two_colour_blocks_are_exact_when_the_colours_are_565() { - let (a, b) = ([255, 0, 0, 255], [0, 0, 255, 255]); - let block: Block = std::array::from_fn(|t| if t % 3 == 0 { a } else { b }); - for bc1 in [true, false] { - let bytes = if bc1 { encode_bc1(&block, u16::MAX) } else { colour_block(&block, u16::MAX, 0, false) }; - assert_eq!(decode_colour(&bytes, bc1), block); - } - } - - #[test] - fn bc1_punches_through_texels_under_half_alpha() { - let block: Block = std::array::from_fn(|t| if t < 8 { [200, 100, 50, 255] } else { [9, 9, 9, 127] }); - let bytes = encode_bc1(&block, u16::MAX); - let (c0, c1) = (u16::from_le_bytes([bytes[0], bytes[1]]), u16::from_le_bytes([bytes[2], bytes[3]])); - assert!(c0 <= c1, "three-colour mode"); - let out = decode_colour(&bytes, true); - assert!(out[..8].iter().all(|t| t[3] == 255) && out[8..].iter().all(|t| *t == [0; 4]), "{out:?}"); - } - - #[test] - fn four_colour_blocks_keep_c0_above_c1() { - // A ramp needs all four colours; BC2 and BC3 blocks must not rely on the order, but - // older decoders read c0 <= c1 as three colours, so the encoder never writes that. - let block: Block = std::array::from_fn(|t| [(t * 16) as u8, 0, 0, 255]); - let bytes = colour_block(&block, u16::MAX, 0, false); - assert!(u16::from_le_bytes([bytes[0], bytes[1]]) > u16::from_le_bytes([bytes[2], bytes[3]])); - } - - #[test] - fn alpha_blocks_pick_the_mode_that_keeps_0_and_255() { - // 0, 255 and a cluster in between: the six-value mode stores 0 and 255 exactly. - let block: Block = std::array::from_fn(|t| [0, 0, 0, [0, 255, 100, 110][t % 4]]); - let bytes = alpha_block(&block, u16::MAX); - assert!(bytes[0] <= bytes[1], "six-value mode"); - let palette = alpha_palette(bytes[0], bytes[1]); - let bits = u64::from_le_bytes([bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7], 0, 0]); - for (t, texel) in block.iter().enumerate() { - let got = palette[(bits >> (3 * t)) as usize & 7]; - assert!(got.abs_diff(texel[3]) <= 2, "texel {t}: {got} for {}", texel[3]); - } - } - - #[test] - fn the_principal_axis_is_found_when_one_channel_varies_on_its_own() { - // Red alternates on its own (variance 64); green and blue alternate together (variance 49 - // each, 98 along their shared axis), uncorrelated with red. Starting on red alone would - // keep red's axis and flatten green and blue; the block must do better than its bounding box. - let block: Block = std::array::from_fn(|t| { - let gb = if (t / 2) % 2 == 0 { 121 } else { 135 }; - [if t % 2 == 0 { 120 } else { 136 }, gb, gb, 255] - }); - let error = |bytes: &[u8; 8]| decode_colour(bytes, false).iter().zip(&block).map(|(a, b)| distance(a, b)).sum::(); - let (_, bounding_box) = fit(&block, u16::MAX, 0, to565([120.0, 121.0, 121.0]), to565([136.0, 135.0, 135.0]), true); - let ours = error(&colour_block(&block, u16::MAX, 0, false)); - assert!(ours < bounding_box, "{ours} vs the bounding box's {bounding_box}"); - } - - #[test] - fn texels_outside_the_image_are_ignored() { - // Only the top-left texel is inside: its colour alone decides the endpoints. - let mut block: Block = [[0, 255, 0, 255]; 16]; - block[0] = [255, 0, 255, 255]; - let out = decode_colour(&encode_bc1(&block, 1), true); - assert_eq!(out[0], [255, 0, 255, 255]); - } -} diff --git a/crates/raster-codecs/src/dicom.rs b/crates/raster-codecs/src/dicom.rs index 31f99839f..ee6c73208 100644 --- a/crates/raster-codecs/src/dicom.rs +++ b/crates/raster-codecs/src/dicom.rs @@ -1,526 +1,10 @@ -//! DICOM (medical imaging) reader, uncompressed transfer syntaxes only. -//! -//! Supports what a viewer needs from a single-frame image: MONOCHROME1/2 (gray) and RGB or -//! YBR_FULL, 8- and 16-bit samples, interleaved or planar, both byte orders and explicit or -//! implicit VR. Undefined-length sequences outside the pixel data are skipped (clinical files -//! contain them); compressed transfer syntaxes, encapsulated pixel data and other photometric -//! interpretations are refused with a clear error. Read-only in this crate. +//! DICOM compatibility facade; implementation and detection live in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use std::collections::HashMap; - -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -const F: &str = "DICOM"; -const PREAMBLE: usize = 132; // 128 zero bytes + "DICM" -const UNDEFINED: u32 = 0xFFFF_FFFF; - -fn bad(msg: impl std::fmt::Display) -> CodecError { - CodecError::malformed(F, msg) +pub fn is_dicom(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Dicom) } - -/// A DICOM file begins with a 128-byte preamble and the `DICM` magic. -pub fn is_dicom(b: &[u8]) -> bool { - b.get(128..132) == Some(b"DICM") -} - -#[derive(Clone, Copy)] -struct Endian { - big: bool, -} - -/// The transfer syntax: byte order and whether value representations are explicit. -#[derive(Clone, Copy)] -struct Syntax { - endian: Endian, - explicit: bool, -} - -/// The value representations whose length field is 2 reserved bytes + `u32` in explicit VR. -fn long_vr(vr: &[u8]) -> bool { - matches!(vr, b"OB" | b"OW" | b"OF" | b"SQ" | b"UT" | b"UN" | b"OD" | b"OL" | b"OV" | b"UC" | b"UR") -} - -/// A cursor over the byte stream. -struct Cur<'a> { - b: &'a [u8], - pos: usize, - big: bool, -} - -impl<'a> Cur<'a> { - fn take(&mut self, n: usize) -> Option<&'a [u8]> { - let s = self.b.get(self.pos..self.pos.checked_add(n)?)?; - self.pos += n; - Some(s) - } - fn u16(&mut self) -> Option { - let c = self.take(2)?; - Some(if self.big { u16::from_be_bytes([c[0], c[1]]) } else { u16::from_le_bytes([c[0], c[1]]) }) - } - fn u32(&mut self) -> Option { - let c = self.take(4)?; - Some(if self.big { u32::from_be_bytes([c[0], c[1], c[2], c[3]]) } else { u32::from_le_bytes([c[0], c[1], c[2], c[3]]) }) - } - /// Skip `len` bytes, refusing to run past the buffer. - fn skip(&mut self, len: usize) -> Result<(), CodecError> { - self.pos = self.pos.checked_add(len).filter(|p| *p <= self.b.len()).ok_or_else(|| bad("a value runs past the end of the file"))?; - Ok(()) - } -} - -/// Read one element's tag and value length, leaving the cursor at the value. Item and delimiter -/// tags (group `FFFE`) carry a `u32` length with no VR even under explicit VR. -fn read_header(c: &mut Cur, syn: &Syntax) -> Result<(u16, u16, [u8; 2], u32), CodecError> { - let group = c.u16().ok_or_else(|| bad("truncated element tag"))?; - let elem = c.u16().ok_or_else(|| bad("truncated element tag"))?; - let (vr, len) = if group == 0xFFFE { - ([0, 0], c.u32().ok_or_else(|| bad("truncated item length"))?) - } else if syn.explicit { - let vr = c.take(2).ok_or_else(|| bad("truncated VR"))?; - let v = [vr[0], vr[1]]; - let len = if long_vr(&v) { - c.take(2).ok_or_else(|| bad("truncated VR"))?; // reserved - c.u32().ok_or_else(|| bad("truncated length"))? - } else { - u32::from(c.u16().ok_or_else(|| bad("truncated length"))?) - }; - (v, len) - } else { - ([0, 0], c.u32().ok_or_else(|| bad("truncated length"))?) - }; - Ok((group, elem, vr, len)) -} - -/// Skip an undefined-length sequence, ending just after its `FFFE,E0DD` delimiter. Nesting is -/// tracked through undefined-length items and nested sequences; everything else is skipped by -/// its length. Returns the position after the delimiter. -fn skip_undefined(c: &mut Cur, syn: &Syntax) -> Result<(), CodecError> { - let mut depth = 1u32; - while depth > 0 { - let (group, elem, _, len) = read_header(c, syn)?; - // Item (E00D) and sequence (E0DD) delimiters close one undefined length; they carry no - // content of their own. - if group == 0xFFFE && matches!(elem, 0xE00D | 0xE0DD) { - depth -= 1; - continue; - } - if len == UNDEFINED { - depth += 1; - } else { - c.skip(len as usize)?; - } - } - Ok(()) -} - -/// Walk the elements from `start`, calling `f` with each `(group, element, value)`. An -/// undefined-length sequence outside the pixel data is skipped; encapsulated (undefined-length) -/// pixel data is refused. Returns the number of bytes walked. -fn elements(b: &[u8], start: usize, syn: &Syntax, mut f: impl FnMut(u16, u16, &Elem) -> Result<(), CodecError>) -> Result { - let mut c = Cur { b, pos: start, big: syn.endian.big }; - loop { - if c.pos >= b.len() { - return Ok(c.pos - start); - } - let (group, elem, vr, len) = read_header(&mut c, syn)?; - if len == UNDEFINED { - if (group, elem) == (0x7FE0, 0x0010) { - return Err(CodecError::unsupported(F, "encapsulated (compressed) pixel data")); - } - skip_undefined(&mut c, syn)?; - continue; - } - let len = len as usize; - let start = c.pos; - if b.len() < start.saturating_add(len) { - return Err(bad("an element value runs past the end of the file")); - } - f(group, elem, &Elem { vr, start, len })?; - c.pos = start + len; - } -} - -/// A parsed element: its VR and the byte range of its value. -struct Elem { - vr: [u8; 2], - start: usize, - len: usize, -} - -fn u16_val(b: &[u8], e: &Elem, big: bool) -> Option { - let c = b.get(e.start..e.start.checked_add(2)?)?; - Some(if big { u16::from_be_bytes([c[0], c[1]]) } else { u16::from_le_bytes([c[0], c[1]]) }) -} - -/// The transfer syntax (byte order and explicit/implicit VR) and the data-set start. The file -/// meta group is always explicit VR little-endian; the data set follows its `(0002,0010)` -/// TransferSyntaxUID. -fn transfer_syntax(b: &[u8]) -> Result<(Syntax, usize), CodecError> { - let meta = Syntax { endian: Endian { big: false }, explicit: true }; - let mut ts: Option = None; - let mut data_start = b.len(); - let mut c = Cur { b, pos: PREAMBLE, big: false }; - while c.pos + 8 <= b.len() { - let save = c.pos; - let (group, elem, _, len) = read_header(&mut c, &meta)?; - if group != 0x0002 { - // A non-meta group begins the data set. - data_start = save; - break; - } - if len == UNDEFINED { - return Err(CodecError::unsupported(F, "undefined-length meta element")); - } - let start = c.pos; - if b.len() < start.saturating_add(len as usize) { - return Err(bad("a meta element runs past the end of the file")); - } - if elem == 0x0010 { - ts = Some(String::from_utf8_lossy(&b[start..start + len as usize]).trim_end_matches(['\0', ' ']).to_string()); - } - c.pos = start + len as usize; - } - let syn = match ts.as_deref() { - Some("1.2.840.10008.1.2") => Syntax { endian: Endian { big: false }, explicit: false }, - Some("1.2.840.10008.1.2.1") => Syntax { endian: Endian { big: false }, explicit: true }, - Some("1.2.840.10008.1.2.2") => Syntax { endian: Endian { big: true }, explicit: true }, - Some(other) => return Err(CodecError::unsupported(F, format!("transfer syntax {other} (only uncompressed is read)"))), - // No meta: the historical default is implicit VR little-endian. - None => Syntax { endian: Endian { big: false }, explicit: false }, - }; - Ok((syn, data_start)) -} - -pub fn decode(b: &[u8], limits: &Limits) -> Result { - if !is_dicom(b) { - return Err(bad("not a DICOM file (no DICM magic)")); - } - let (syn, start) = transfer_syntax(b)?; - let mut tags: HashMap<(u16, u16), Elem> = HashMap::new(); - elements(b, start, &syn, |g, e, el| { - // Later duplicates must not silently override the first (DICOM forbids duplicates). - tags.entry((g, e)).or_insert_with(|| Elem { vr: el.vr, start: el.start, len: el.len }); - Ok(()) - })?; - let big = syn.endian.big; - let get_u16 = |tag: (u16, u16)| tags.get(&tag).and_then(|e| u16_val(b, e, big)); - let rows = u32::from(get_u16((0x0028, 0x0010)).ok_or_else(|| bad("missing Rows (0028,0010)"))?); - let cols = u32::from(get_u16((0x0028, 0x0011)).ok_or_else(|| bad("missing Columns (0028,0011)"))?); - let samples = u32::from(get_u16((0x0028, 0x0002)).unwrap_or(1)); - let bits = get_u16((0x0028, 0x0100)).ok_or_else(|| bad("missing BitsAllocated (0028,0100)"))?; - let bits_stored = get_u16((0x0028, 0x0101)).unwrap_or(bits); - let high_bit = get_u16((0x0028, 0x0102)).unwrap_or(bits_stored.saturating_sub(1)); - if u32::from(high_bit) >= u32::from(bits) { - return Err(bad("HighBit is outside the allocated bits")); - } - let signed = get_u16((0x0028, 0x0103)).unwrap_or(0) == 1; - let planar = get_u16((0x0028, 0x0006)).unwrap_or(0); - let photometric = tags - .get(&(0x0028, 0x0004)) - .and_then(|e| b.get(e.start..e.start.checked_add(e.len)?)) - .map(|v| String::from_utf8_lossy(v).trim_end_matches(['\0', ' ']).to_string()) - .ok_or_else(|| bad("missing PhotometricInterpretation (0028,0004)"))?; - let pixel = tags.get(&(0x7FE0, 0x0010)).ok_or_else(|| CodecError::unsupported(F, "no pixel data (7FE0,0010)"))?; - if !matches!(bits, 8 | 16) || !matches!(bits_stored, 1..=16) { - return Err(CodecError::unsupported(F, format!("{bits}-bit allocated / {bits_stored}-bit stored samples"))); - } - let mono = matches!(photometric.as_str(), "MONOCHROME1" | "MONOCHROME2"); - let rgb = photometric == "RGB"; - let ybr = photometric == "YBR_FULL"; - if !(mono || rgb || ybr) { - return Err(CodecError::unsupported(F, format!("photometric interpretation {photometric}"))); - } - // Mono stores one sample; RGB and YBR_FULL store three. Any other count would index past the - // pixel data, so it is refused rather than misread. - let want_samples = if mono { 1 } else { 3 }; - if samples != want_samples { - return Err(CodecError::unsupported(F, format!("{samples} samples per pixel with {photometric}"))); - } - if ybr && bits != 8 { - return Err(CodecError::unsupported(F, format!("{bits}-bit {photometric} (8-bit only)"))); - } - let layout = if mono { ChannelLayout::Gray } else { ChannelLayout::Rgb }; - let out_sample = if bits == 8 { SampleType::U8 } else { SampleType::U16 }; - limits.check(cols, rows, layout, out_sample)?; - - let pixels = (cols as usize).checked_mul(rows as usize).ok_or_else(|| CodecError::LimitExceeded("image too large".into()))?; - let n = pixels.checked_mul(samples as usize).ok_or_else(|| CodecError::LimitExceeded("image too large".into()))?; - let bytes_per = if bits == 16 { 2usize } else { 1 }; - let need = n.checked_mul(bytes_per).ok_or_else(|| CodecError::LimitExceeded("pixel data too large".into()))?; - let data = b - .get(pixel.start..pixel.start.checked_add(need).ok_or_else(|| CodecError::LimitExceeded("pixel data too large".into()))?) - .ok_or_else(|| bad("pixel data is truncated"))?; - let out_channels = layout.channels(); - let out_len = - pixels.checked_mul(out_channels).and_then(|k| k.checked_mul(bytes_per)).ok_or_else(|| CodecError::LimitExceeded("image too large".into()))?; - let mut out = Vec::new(); - out.try_reserve_exact(out_len).map_err(|_| CodecError::LimitExceeded("cannot allocate the image".into()))?; - // Raw sample of channel `ch` at pixel `p`, honouring the planar configuration. - let sample_at = |p: usize, ch: usize| -> Option { - let idx = if planar == 1 { ch.checked_mul(pixels)?.checked_add(p)? } else { p.checked_mul(samples as usize)?.checked_add(ch)? }; - let off = idx.checked_mul(bytes_per)?; - if bits == 16 { - let c = data.get(off..off.checked_add(2)?)?; - Some(if big { u16::from_be_bytes([c[0], c[1]]) } else { u16::from_le_bytes([c[0], c[1]]) }.into()) - } else { - Some(u32::from(*data.get(off)?)) - } - }; - let stored_max = f64::from((1u32 << bits_stored) - 1); - // Extract the `bits_stored` samples (the least significant one at `high_bit`) and scale to - // [0, 1]; signed values are shifted up by half a range first. `high_bit` is 1-based here. - let shift = (u32::from(high_bit) + 1).saturating_sub(u32::from(bits_stored)); - let mask = (1u32 << bits_stored) - 1; - let half = 1u32 << (bits_stored - 1); - let norm = move |raw: u32| -> f64 { - let v = (raw >> shift) & mask; - // Signed: flip the sign bit to map [-2^(b-1), 2^(b-1)-1] onto [0, 2^b-1]. - let v = if signed { v ^ half } else { v }; - (f64::from(v) / stored_max).clamp(0.0, 1.0) - }; - let q = |x: f64| -> u16 { (x * 65535.0).round() as u16 }; - let push = |out: &mut Vec, x: f64| { - if bits == 8 { - out.push((x * 255.0).round() as u8); - } else { - out.extend_from_slice(&q(x).to_ne_bytes()); - } - }; - for p in 0..pixels { - let s = |ch: usize| sample_at(p, ch).ok_or_else(|| bad("pixel data is shorter than the image")); - if mono { - let x = norm(s(0)?); - push(&mut out, if photometric == "MONOCHROME1" { 1.0 - x } else { x }); - } else if rgb { - for ch in 0..3 { - push(&mut out, norm(s(ch)?)); - } - } else { - // YBR_FULL (8-bit): BT.601 full-range to RGB. - let y = f64::from(s(0)?); - let cb = f64::from(s(1)?) - 128.0; - let cr = f64::from(s(2)?) - 128.0; - let r = (y + 1.402 * cr).clamp(0.0, 255.0); - let g = (y - 0.344_136 * cb - 0.714_136 * cr).clamp(0.0, 255.0); - let bl = (y + 1.772 * cb).clamp(0.0, 255.0); - for v in [r, g, bl] { - push(&mut out, v / 255.0); - } - } - } - Image::from_raw(cols, rows, layout, out_sample, out) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Build the file meta group (explicit VR little-endian), including the given extra - /// `(element, ascii)` pairs before the TransferSyntaxUID. - fn meta(b: &mut Vec, ts: &str, extra: &[(u16, &str)]) { - let ui = |e: u16, v: &str, b: &mut Vec| { - let mut s = v.as_bytes().to_vec(); - if s.len() % 2 == 1 { - s.push(0); - } - b.extend_from_slice(&0x0002u16.to_le_bytes()); - b.extend_from_slice(&e.to_le_bytes()); - b.extend_from_slice(b"UI"); - b.extend_from_slice(&(s.len() as u16).to_le_bytes()); - b.extend_from_slice(&s); - }; - for (e, v) in extra { - ui(*e, v, b); - } - ui(0x0010, ts, b); - } - - fn us(b: &mut Vec, g: u16, e: u16, v: u16, explicit: bool) { - b.extend_from_slice(&g.to_le_bytes()); - b.extend_from_slice(&e.to_le_bytes()); - if explicit { - b.extend_from_slice(b"US"); - b.extend_from_slice(&2u16.to_le_bytes()); - } else { - b.extend_from_slice(&2u32.to_le_bytes()); - } - b.extend_from_slice(&v.to_le_bytes()); - } - - fn cs(b: &mut Vec, g: u16, e: u16, v: &str, explicit: bool) { - let mut s = v.as_bytes().to_vec(); - if s.len() % 2 == 1 { - s.push(0); - } - b.extend_from_slice(&g.to_le_bytes()); - b.extend_from_slice(&e.to_le_bytes()); - if explicit { - b.extend_from_slice(b"CS"); - b.extend_from_slice(&(s.len() as u16).to_le_bytes()); - } else { - b.extend_from_slice(&(s.len() as u32).to_le_bytes()); - } - b.extend_from_slice(&s); - } - - /// A DICOM with the given transfer syntax UID and data-set encoding. - #[allow(clippy::too_many_arguments)] - fn synth( - ts: &str, - explicit: bool, - big: bool, - planar: u16, - rows: u16, - cols: u16, - samples: u16, - bits: u16, - photometric: &str, - pixels: &[u8], - ) -> Vec { - let mut b = vec![0u8; 128]; - b.extend_from_slice(b"DICM"); - // One extra meta element before the syntax, so a reader that only looks at the last - // group-0002 element takes the wrong one. - meta(&mut b, ts, &[(0x0002, "1.2.840.10008.5.1.4.1.1.7")]); - let _ = big; - us(&mut b, 0x0028, 0x0010, rows, explicit); - us(&mut b, 0x0028, 0x0011, cols, explicit); - us(&mut b, 0x0028, 0x0002, samples, explicit); - us(&mut b, 0x0028, 0x0006, planar, explicit); - cs(&mut b, 0x0028, 0x0004, photometric, explicit); - us(&mut b, 0x0028, 0x0100, bits, explicit); - us(&mut b, 0x0028, 0x0101, bits, explicit); - us(&mut b, 0x0028, 0x0102, bits - 1, explicit); - us(&mut b, 0x0028, 0x0103, 0, explicit); - b.extend_from_slice(&0x7FE0u16.to_le_bytes()); - b.extend_from_slice(&0x0010u16.to_le_bytes()); - if explicit { - b.extend_from_slice(b"OB"); - b.extend_from_slice(&0u16.to_le_bytes()); - } - b.extend_from_slice(&(pixels.len() as u32).to_le_bytes()); - b.extend_from_slice(pixels); - b - } - - #[test] - fn decodes_8bit_monochrome() { - let b = synth("1.2.840.10008.1.2.1", true, false, 0, 2, 2, 1, 8, "MONOCHROME2", &[0, 51, 102, 255]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!((img.width(), img.height(), img.layout(), img.sample_type()), (2, 2, ChannelLayout::Gray, SampleType::U8)); - assert_eq!(img.data(), &[0, 51, 102, 255]); - } - - #[test] - fn decodes_16bit_monochrome_and_scales() { - let px: Vec = [0u16, 32768, 65535].iter().flat_map(|v| v.to_le_bytes()).collect(); - let b = synth("1.2.840.10008.1.2.1", true, false, 0, 3, 1, 1, 16, "MONOCHROME2", &px); - let img = decode(&b, &Limits::default()).unwrap(); - let v = img.to_u16_samples().unwrap(); - assert_eq!(v[0], 0); - assert!((i32::from(v[1]) - 32768).abs() <= 1); - assert_eq!(v[2], 65535); - } - - #[test] - fn monochrome1_is_inverted() { - let b = synth("1.2.840.10008.1.2.1", true, false, 0, 2, 1, 1, 8, "MONOCHROME1", &[0, 255]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[255, 0]); - } - - #[test] - fn decodes_rgb_interleaved_and_planar() { - // 2 pixels: (10,20,30) and (40,50,60). - let interleaved = synth("1.2.840.10008.1.2.1", true, false, 0, 1, 2, 3, 8, "RGB", &[10, 20, 30, 40, 50, 60]); - let img = decode(&interleaved, &Limits::default()).unwrap(); - assert_eq!((img.layout(), img.data()), (ChannelLayout::Rgb, &[10u8, 20, 30, 40, 50, 60][..])); - // Planar 0: R plane, then G, then B. - let planar = synth("1.2.840.10008.1.2.1", true, false, 1, 1, 2, 3, 8, "RGB", &[10, 40, 20, 50, 30, 60]); - let img = decode(&planar, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[10, 20, 30, 40, 50, 60]); - } - - #[test] - fn decodes_ybr_full_to_rgb() { - // White (Y=128, Cb=Cr=128) stays neutral grey. - let b = synth("1.2.840.10008.1.2.1", true, false, 0, 1, 1, 3, 8, "YBR_FULL", &[128, 128, 128]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.layout(), ChannelLayout::Rgb); - let d = img.data(); - assert!(d[0].abs_diff(d[1]) <= 1 && d[1].abs_diff(d[2]) <= 1, "{d:?}"); - } - - #[test] - fn decodes_implicit_vr() { - let b = synth("1.2.840.10008.1.2", false, false, 0, 2, 2, 1, 8, "MONOCHROME2", &[0, 51, 102, 255]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.layout(), ChannelLayout::Gray); - assert_eq!(img.data(), &[0, 51, 102, 255]); - } - - #[test] - fn skips_an_undefined_length_sequence_outside_the_pixel_data() { - // Insert an undefined-length SQ (0008,2218) with one item, then a defined-length one, - // before the pixel data: the reader must skip both and still decode the image. - let mut b = vec![0u8; 128]; - b.extend_from_slice(b"DICM"); - meta(&mut b, "1.2.840.10008.1.2.1", &[]); - for (g, e, v) in [ - (0x0028, 0x0010, 1u16), - (0x0028, 0x0011, 1), - (0x0028, 0x0002, 1), - (0x0028, 0x0100, 8), - (0x0028, 0x0101, 8), - (0x0028, 0x0102, 7), - (0x0028, 0x0103, 0), - ] { - us(&mut b, g, e, v, true); - } - cs(&mut b, 0x0028, 0x0004, "MONOCHROME2", true); - // Undefined-length sequence (0008,2211), one undefined-length item containing a nested - // defined-length sequence, then the delimiters. - b.extend_from_slice(&[0x08, 0x00, 0x11, 0x22, b'S', b'Q', 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF]); - b.extend_from_slice(&[0xFE, 0xFF, 0x00, 0xE0, 0xFF, 0xFF, 0xFF, 0xFF]); // item, undefined - b.extend_from_slice(&[0x08, 0x00, 0x55, 0x22, b'S', b'Q', 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 1, 2, 3, 4]); // nested, defined - b.extend_from_slice(&[0xFE, 0xFF, 0x0D, 0xE0, 0x00, 0x00, 0x00, 0x00]); // item delimitation - b.extend_from_slice(&[0xFE, 0xFF, 0xDD, 0xE0, 0x00, 0x00, 0x00, 0x00]); // sequence delimitation - // Pixel data (defined length). - b.extend_from_slice(&0x7FE0u16.to_le_bytes()); - b.extend_from_slice(&0x0010u16.to_le_bytes()); - b.extend_from_slice(b"OB"); - b.extend_from_slice(&0u16.to_le_bytes()); - b.extend_from_slice(&1u32.to_le_bytes()); - b.push(200); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[200]); - } - - #[test] - fn truncated_and_bad_is_error_not_panic() { - let b = synth("1.2.840.10008.1.2.1", true, false, 0, 2, 2, 1, 8, "MONOCHROME2", &[0, 1, 2, 3]); - for cut in [0, 4, 128, 132, 140, b.len() - 1] { - assert!(decode(&b[..cut.min(b.len())], &Limits::default()).is_err(), "cut {cut}"); - } - assert!(decode(b"not dicom", &Limits::default()).is_err()); - } - - #[test] - fn high_bit_outside_the_allocated_bits_is_an_error_not_a_panic() { - let mut b = synth("1.2.840.10008.1.2.1", true, false, 0, 1, 1, 1, 8, "MONOCHROME2", &[7]); - let tag = [0x28, 0x00, 0x02, 0x01, b'U', b'S', 0x02, 0x00]; - let at = b.windows(tag.len()).position(|w| w == tag).unwrap() + tag.len(); - b[at..at + 2].copy_from_slice(&200u16.to_le_bytes()); - assert!(decode(&b, &Limits::default()).is_err()); - } - - #[test] - fn rejects_compressed_transfer_syntax() { - let mut b = synth("1.2.840.10008.1.2.1", true, false, 0, 1, 1, 1, 8, "MONOCHROME2", &[0]); - let uncompressed = b"1.2.840.10008.1.2.1\0"; - let compressed = b"1.2.840.10008.1.2.4\0"; // same length, a compressed family - let at = b.windows(uncompressed.len()).position(|w| w == uncompressed).expect("transfer syntax in the fixture"); - b[at..at + compressed.len()].copy_from_slice(compressed); - assert!(matches!(decode(&b, &Limits::default()), Err(CodecError::Unsupported { .. }))); - } +pub fn decode(bytes: &[u8], limits: &Limits) -> Result { + bridge::format(Format::Dicom, bytes, limits) } diff --git a/crates/raster-codecs/src/fits.rs b/crates/raster-codecs/src/fits.rs index 0d05c0fa1..4a662ab6e 100644 --- a/crates/raster-codecs/src/fits.rs +++ b/crates/raster-codecs/src/fits.rs @@ -1,857 +1,16 @@ -//! FITS (Flexible Image Transport System, FITS Standard 4.0), implemented here: the image in the -//! primary HDU, or the first `IMAGE` extension when the primary HDU holds no data. -//! -//! Reading: -//! -//! * `BITPIX` 8, 16, 32, 64, -32 and -64, with `BSCALE`/`BZERO` and integer `BLANK`. -//! * `NAXIS3 = 3` opens as RGB (the planes are R, G, B); any other plane count opens the first -//! plane as gray with [`crate::DecodeWarning::MorePages`]. -//! * Rows are stored bottom-to-top, as the standard's convention has it (Siril, DS9, Astropy), -//! unless `ROWORDER = 'TOP-DOWN'`. -//! * Physical values that fit (unsigned 8-bit, or integers in 0..=65535 such as a camera's -//! `BITPIX = 16`, `BZERO = 32768`) are kept exactly as U8/U16. Everything else becomes F32: -//! values up to 2 are taken as already normalised (Siril, PixInsight), larger ones are scaled by -//! 65535 when they fit in it (DeepSkyStacker, most stackers) and by the maximum otherwise. NaN -//! and blank pixels read as 0. -//! * The other header cards are kept as text: `KEY` → value (strings unquoted), `COMMENT` and -//! `HISTORY` by their text, `HIERARCH` keywords by their long name. Long strings continued -//! on `CONTINUE` cards are joined. -//! -//! Writing: gray or RGB, U8 (`BITPIX = 8`), U16 (`BITPIX = 16`, `BZERO = 32768`) or F32 -//! (`BITPIX = -32`, 0..1), bottom-to-top with `ROWORDER = 'BOTTOM-UP'`, plus the text cards -//! (strings longer than one card on `CONTINUE` cards). -//! Tile-compressed files (fpack `ZIMAGE` tables) are recognised and refused. +//! FITS compatibility facade; implementation lives in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use std::collections::HashMap; - -use crate::{ChannelLayout, CodecError, DecodeWarning, Image, Limits, SampleType}; - -const F: &str = "FITS"; -const BLOCK: usize = 2880; -const CARD: usize = 80; -/// Most HDUs walked looking for an image; FITS files from cameras and stackers hold a handful. -const MAX_HDUS: usize = 1000; - -/// Keywords that describe the data layout. They are written from the image, never copied from -/// (or kept as) text metadata: after decoding they no longer describe the pixels. -const STRUCTURAL: &[&str] = &[ - "SIMPLE", "BITPIX", "NAXIS", "EXTEND", "BZERO", "BSCALE", "BLANK", "END", "XTENSION", "PCOUNT", "GCOUNT", "ROWORDER", "CHECKSUM", "DATASUM", - "DATAMIN", "DATAMAX", "ZIMAGE", -]; - -fn err(msg: impl std::fmt::Display) -> CodecError { - CodecError::malformed(F, msg) +pub fn looks_like_fits(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Fits) } - -#[derive(Debug, Clone, PartialEq)] -enum Value { - /// A quoted string, unescaped, trailing blanks removed. - Str(String), - /// Anything else (number, logical, complex) as written. - Raw(String), -} - -#[derive(Debug, Clone)] -struct Card { - key: String, - /// `None` for commentary cards (`COMMENT`, `HISTORY`, blank keyword). - value: Option, - /// The text of a commentary card. - text: String, - /// A `HIERARCH` keyword (`key` holds its long name). - hierarch: bool, -} - -fn parse_value(field: &str) -> Value { - let t = field.trim_start(); - if let Some(rest) = t.strip_prefix('\'') { - let mut s = String::new(); - let mut chars = rest.chars().peekable(); - while let Some(c) = chars.next() { - if c == '\'' { - if chars.peek() == Some(&'\'') { - chars.next(); - s.push('\''); - } else { - break; - } - } else { - s.push(c); - } - } - Value::Str(s.trim_end().to_string()) - } else { - Value::Raw(t.split('/').next().unwrap_or("").trim().to_string()) - } -} - -fn parse_card(raw: &[u8]) -> Card { - // Header bytes are ASCII 32–126 by the standard; anything else is replaced, never trusted. - let s: String = raw.iter().map(|&b| if (32..=126).contains(&b) { b as char } else { '?' }).collect(); - let key_field = s.get(..8).unwrap_or(&s).trim_end().to_string(); - if key_field == "HIERARCH" { - let rest = s.get(9..).unwrap_or(""); - if let Some((k, v)) = rest.split_once('=') { - return Card { key: k.trim().to_string(), value: Some(parse_value(v)), text: String::new(), hierarch: true }; - } - } - if s.get(8..10) == Some("= ") { - let v = parse_value(s.get(10..).unwrap_or("")); - return Card { key: key_field, value: Some(v), text: String::new(), hierarch: false }; - } - Card { key: key_field, value: None, text: s.get(8..).unwrap_or("").trim_end().to_string(), hierarch: false } -} - -struct Header { - cards: Vec, - /// First-occurrence index of each non-HIERARCH keyword, so `get` is O(1). The card - /// count and the number of looked-up keywords (NAXIS1..NAXISn) are both file-controlled, - /// so a linear scan per lookup lets a crafted header multiply commentary cards by axis - /// count into quadratic work. - index: HashMap, - /// Offset of the data unit. - data_start: usize, -} - -impl Header { - fn get(&self, key: &str) -> Option<&Value> { - let c = &self.cards[*self.index.get(key)?]; - c.value.as_ref() - } - - fn int(&self, key: &str) -> Option { - match self.get(key)? { - Value::Raw(r) => r.parse::().ok().or_else(|| self.float(key).filter(|f| f.fract() == 0.0 && f.abs() < 9.0e15).map(|f| f as i64)), - Value::Str(_) => None, - } - } - - fn float(&self, key: &str) -> Option { - match self.get(key)? { - // Fortran-style `D` exponents are allowed in FITS. - Value::Raw(r) => r.replace(['D', 'd'], "E").parse::().ok().filter(|f| f.is_finite()), - Value::Str(_) => None, - } - } - - fn logical(&self, key: &str) -> bool { - matches!(self.get(key), Some(Value::Raw(r)) if r == "T") - } - - fn string(&self, key: &str) -> Option<&str> { - match self.get(key)? { - Value::Str(s) => Some(s.trim()), - Value::Raw(_) => None, - } - } -} - -fn parse_header(b: &[u8], start: usize) -> Result { - let mut cards = Vec::new(); - let mut pos = start; - loop { - let raw = b.get(pos..pos.checked_add(CARD).ok_or_else(|| err("header too long"))?).ok_or_else(|| err("header ends before END"))?; - pos += CARD; - if raw.starts_with(b"END") && raw.get(3..).is_some_and(|r| r.iter().all(|&c| c == b' ')) { - break; - } - let card = parse_card(raw); - // Long strings (FITS 4.0 section 4.2.1.2): a string ending in '&' goes on in the next - // `CONTINUE` record. An orphaned `CONTINUE` stays a card of its own (commentary). - if let Some(Card { value: Some(Value::Str(s)), .. }) = cards.last_mut() - && s.ends_with('&') - && card.key == "CONTINUE" - && card.text.starts_with(" ") - && let Value::Str(more) = parse_value(&card.text) - { - s.pop(); - s.push_str(&more); - continue; - } - cards.push(card); - } - let data_start = pos.div_ceil(BLOCK).checked_mul(BLOCK).ok_or_else(|| err("header too long"))?; - // Index each non-HIERARCH keyword to its first occurrence, matching the previous - // `find(|c| !c.hierarch && c.key == key)` lookup while making `get` O(1). - let mut index = HashMap::with_capacity(cards.len()); - for (i, c) in cards.iter().enumerate() { - if !c.hierarch { - index.entry(c.key.clone()).or_insert(i); - } - } - Ok(Header { cards, index, data_start }) -} - -/// The axis lengths (`NAXIS1`, `NAXIS2`, …). -fn axes(h: &Header) -> Result, CodecError> { - let n = h.int("NAXIS").ok_or_else(|| err("missing NAXIS"))?; - if !(0..=999).contains(&n) { - return Err(err(format!("invalid NAXIS {n}"))); - } - (1..=n) - .map(|i| h.int(&format!("NAXIS{i}")).and_then(|v| u64::try_from(v).ok()).ok_or_else(|| err(format!("missing or negative NAXIS{i}")))) - .collect() -} - -/// Bytes in the data unit (before padding to a block). -fn data_len(h: &Header, bitpix: i64) -> Result { - let ax = axes(h)?; - if ax.is_empty() { - return Ok(0); - } - let pcount = u64::try_from(h.int("PCOUNT").unwrap_or(0)).map_err(|_| err("negative PCOUNT"))?; - let gcount = u64::try_from(h.int("GCOUNT").unwrap_or(1)).map_err(|_| err("negative GCOUNT"))?; - let elems = ax.iter().try_fold(1u64, |a, &n| a.checked_mul(n)).ok_or_else(|| err("data size overflows"))?; - elems - .checked_add(pcount) - .and_then(|v| v.checked_mul(gcount)) - .and_then(|v| v.checked_mul(bitpix.unsigned_abs() / 8)) - .ok_or_else(|| err("data size overflows")) -} - -/// A primary header's first card, `SIMPLE =`. -pub fn looks_like_fits(b: &[u8]) -> bool { - b.starts_with(b"SIMPLE =") -} - -/// Inspect every HDU without decoding pixels: one gray plane or three RGB planes are retained, -/// while other cubes, image extensions, tables and other data-bearing HDUs cannot survive a flat save. -/// Incomplete or over-limit walks fail closed rather than treating an unchecked suffix as safe. pub fn has_additional_images(bytes: &[u8]) -> Result { - let mut start = 0; - let mut image_seen = false; - for index in 0..MAX_HDUS { - let h = parse_header(bytes, start)?; - if index == 0 && h.get("SIMPLE").is_none() { - return Err(err("missing SIMPLE")); - } - let ax = axes(&h)?; - let size = data_len(&h, h.int("BITPIX").unwrap_or(8))?; - let image_hdu = index == 0 || h.string("XTENSION") == Some("IMAGE"); - if h.logical("ZIMAGE") { - return Ok(true); - } - if !image_hdu && size > 0 { - return Ok(true); - } - // An image HDU with a zero axis has no data (FITS 4.0, NAXISn) and holds no image, like - // the empty primary: `find_image` skips it too. - if image_hdu && size > 0 && ax.len() >= 2 && ax[..2].iter().all(|n| *n > 0) { - let planes = ax.iter().skip(2).try_fold(1u64, |n, axis| n.checked_mul(*axis)).ok_or_else(|| err("plane count overflows"))?; - let retained = planes == 1 || (ax.get(2) == Some(&3) && planes == 3); - if image_seen || !retained || h.int("GCOUNT").unwrap_or(1) > 1 || h.int("PCOUNT").unwrap_or(0) > 0 { - return Ok(true); - } - image_seen = true; - } else if size > 0 { - return Ok(true); - } - let padded = size.div_ceil(BLOCK as u64).checked_mul(BLOCK as u64).ok_or_else(|| err("data size overflows"))?; - let next = usize::try_from(padded).ok().and_then(|size| h.data_start.checked_add(size)).ok_or_else(|| err("data size overflows"))?; - if next > bytes.len() { - return Err(err("truncated HDU data")); - } - if next == bytes.len() { - return Ok(false); - } - start = next; - } - Err(err("too many HDUs to inspect every source image")) + craft_image_codecs::has_unretained_source_data(bytes).map_err(bridge::from_shared_error) } - -/// Whether an HDU with these axes stores pixels: at least two axes and none of them zero (FITS 4.0, -/// NAXISn: a zero axis means no data follow the header). -fn holds_pixels(ax: &[u64]) -> bool { - ax.len() >= 2 && ax.iter().all(|n| *n > 0) -} - -/// Finds the image HDU: the primary one when it holds pixels, else the first `IMAGE` extension -/// that does. Returns its header and the primary header (for its descriptive cards). -fn find_image(b: &[u8]) -> Result<(Header, Option
), CodecError> { - let primary = parse_header(b, 0)?; - if primary.get("SIMPLE").is_none() { - return Err(err("missing SIMPLE")); - } - if holds_pixels(&axes(&primary)?) { - return Ok((primary, None)); - } - let mut compressed = false; - let mut cur_start = primary.data_start; - let mut cur_len = data_len(&primary, primary.int("BITPIX").unwrap_or(8))?; - for _ in 0..MAX_HDUS { - let next = usize::try_from(cur_len.div_ceil(BLOCK as u64).saturating_mul(BLOCK as u64)) - .ok() - .and_then(|l| cur_start.checked_add(l)) - .ok_or_else(|| err("data size overflows"))?; - if next >= b.len() { - break; - } - let h = parse_header(b, next)?; - let kind = h.string("XTENSION").unwrap_or("").to_string(); - if kind == "IMAGE" && holds_pixels(&axes(&h)?) { - return Ok((h, Some(primary))); - } - if kind == "BINTABLE" && h.logical("ZIMAGE") { - compressed = true; - } - cur_len = data_len(&h, h.int("BITPIX").unwrap_or(8))?; - cur_start = h.data_start; - } - if compressed { - return Err(CodecError::unsupported(F, "tile-compressed FITS (fpack) is not supported; decompress it with funpack first")); - } - Err(err("the file holds no image")) -} - pub fn decode(bytes: &[u8], limits: &Limits) -> Result { - let (h, primary) = find_image(bytes)?; - let bitpix = h.int("BITPIX").ok_or_else(|| err("missing BITPIX"))?; - if !matches!(bitpix, 8 | 16 | 32 | 64 | -32 | -64) { - return Err(err(format!("invalid BITPIX {bitpix}"))); - } - let ax = axes(&h)?; - let (Some(&w), Some(&hgt)) = (ax.first(), ax.get(1)) else { return Err(err("the image has fewer than two axes")) }; - let w = u32::try_from(w).map_err(|_| CodecError::LimitExceeded(format!("width {w}")))?; - let hgt = u32::try_from(hgt).map_err(|_| CodecError::LimitExceeded(format!("height {hgt}")))?; - let planes = ax.iter().skip(2).try_fold(1u64, |a, &n| a.checked_mul(n)).ok_or_else(|| err("too many planes"))?; - let (layout, nplanes) = if ax.get(2) == Some(&3) && planes == 3 { (ChannelLayout::Rgb, 3usize) } else { (ChannelLayout::Gray, 1) }; - - let bscale = h.float("BSCALE").unwrap_or(1.0); - let bzero = h.float("BZERO").unwrap_or(0.0); - let blank = if bitpix > 0 { h.int("BLANK") } else { None }; - let integral = bscale == 1.0 && bzero.fract() == 0.0; - let candidate = match bitpix { - 8 if integral && bzero == 0.0 => SampleType::U8, - 8 | 16 if integral => SampleType::U16, - _ => SampleType::F32, - }; - limits.check(w, hgt, layout, candidate)?; - - let bps = (bitpix.unsigned_abs() / 8) as usize; - let plane_len = (w as usize).checked_mul(hgt as usize).ok_or_else(|| CodecError::LimitExceeded(format!("{w}x{hgt} does not fit in memory")))?; - let n = plane_len.checked_mul(nplanes).ok_or_else(|| CodecError::LimitExceeded("image does not fit in memory".into()))?; - let need = n.checked_mul(bps).and_then(|l| l.checked_add(h.data_start)).ok_or_else(|| err("data size overflows"))?; - let data = bytes.get(h.data_start..need).ok_or_else(|| err("truncated data"))?; - - // Physical value of sample `i` (planar order); `None` for a blank or NaN pixel. - let physical = |i: usize| -> Option { - let s = data.get(i * bps..(i + 1) * bps)?; - let raw = match bitpix { - 8 => s.first().map(|&v| v as i64)?, - 16 => i16::from_be_bytes([s[0], s[1]]) as i64, - 32 => i32::from_be_bytes([s[0], s[1], s[2], s[3]]) as i64, - 64 => i64::from_be_bytes([s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]]), - -32 => return Some(f32::from_be_bytes([s[0], s[1], s[2], s[3]]) as f64 * bscale + bzero).filter(|v| v.is_finite()), - _ => return Some(f64::from_be_bytes([s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]]) * bscale + bzero).filter(|v| v.is_finite()), - }; - if blank == Some(raw) { - return None; - } - Some(raw as f64 * bscale + bzero) - }; - - let top_down = h.string("ROWORDER").is_some_and(|r| r.eq_ignore_ascii_case("TOP-DOWN")); - let (wu, hu) = (w as usize, hgt as usize); - // Planar, bottom-up (by default) → interleaved, top-down. - let dest = |i: usize| -> usize { - let (p, rem) = (i / plane_len, i % plane_len); - let (y, x) = (rem / wu, rem % wu); - let y = if top_down { y } else { hu - 1 - y }; - (y * wu + x) * nplanes + p - }; - - let mut img = match candidate { - SampleType::U8 => { - let mut out = vec![0u8; n]; - for i in 0..n { - out[dest(i)] = physical(i).unwrap_or(0.0) as u8; - } - Image::from_u8(w, hgt, layout, out)? - } - _ => { - let mut out16 = if candidate == SampleType::U16 { Some(vec![0u16; n]) } else { None }; - if let Some(out) = out16.as_mut() { - for i in 0..n { - let v = physical(i).unwrap_or(0.0); - if !(0.0..=65535.0).contains(&v) { - // Signed data with negative values: fall back to float. - out16 = None; - break; - } - out[dest(i)] = v as u16; - } - } - match out16 { - Some(out) => Image::from_u16(w, hgt, layout, &out)?, - None => { - let max = (0..n).filter_map(physical).fold(0.0f64, f64::max); - let white = if max <= 2.0 { - 1.0 - } else if max <= 65535.0 { - 65535.0 - } else { - max - }; - let mut out = vec![0f32; n]; - for i in 0..n { - out[dest(i)] = (physical(i).unwrap_or(0.0) / white) as f32; - } - Image::from_f32(w, hgt, layout, &out)? - } - } - } - }; - - if planes > nplanes as u64 { - img.warnings.push(DecodeWarning::MorePages { total: u32::try_from(planes).ok() }); - } - let mut text = Vec::new(); - for c in primary.iter().flat_map(|p| p.cards.iter()).chain(h.cards.iter()) { - if !c.hierarch && (STRUCTURAL.contains(&c.key.as_str()) || is_naxis(&c.key)) { - continue; - } - let entry = match &c.value { - Some(Value::Str(s) | Value::Raw(s)) => (c.key.clone(), s.clone()), - None if c.key.is_empty() && c.text.trim().is_empty() => continue, - None => (if c.key.is_empty() { "COMMENT".to_string() } else { c.key.clone() }, c.text.trim().to_string()), - }; - text.push(entry); - } - img.text = text; - Ok(img) + bridge::format(Format::Fits, bytes, limits) } - -fn is_naxis(key: &str) -> bool { - key.strip_prefix("NAXIS").is_some_and(|d| d.chars().all(|c| c.is_ascii_digit())) -} - -/// A standard keyword: 1–8 characters from `A-Z 0-9 _ -`. -fn standard_key(key: &str) -> bool { - (1..=8).contains(&key.len()) && key.bytes().all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_' || b == b'-') -} - -fn ascii(s: &str) -> String { - s.chars().map(|c| if (' '..='~').contains(&c) { c } else { '?' }).collect() -} - -/// A FITS literal (number or logical) that can be written unquoted. -fn literal(v: &str) -> bool { - v == "T" - || v == "F" - || (v.replace(['D', 'd'], "E").parse::().is_ok_and(|f| f.is_finite()) && v.bytes().all(|b| b"0123456789+-.EeDd".contains(&b))) -} - -fn quoted(v: &str, room: usize) -> String { - let mut s = String::from("'"); - for c in ascii(v).chars() { - let add = if c == '\'' { 2 } else { 1 }; - // Keep room for the closing quote. - if s.len() + add + 1 > room { - break; - } - s.push(c); - if c == '\'' { - s.push('\''); - } - } - // Strings are at least eight characters inside the quotes. - while s.len() < 9 { - s.push(' '); - } - s.push('\''); - s -} - -struct Writer { - out: Vec, -} - -impl Writer { - fn card(&mut self, text: &str) { - let mut c: Vec = ascii(text).into_bytes(); - c.truncate(CARD); - c.resize(CARD, b' '); - self.out.extend_from_slice(&c); - } - - fn value(&mut self, key: &str, v: &str) { - if literal(v) { - self.card(&format!("{key:<8}= {v:>20}")); - return; - } - // A string too long for one card goes on in `CONTINUE` cards, every piece but the last - // ending in '&' (FITS 4.0 section 4.2.1.2). A doubled quote is never split, and trailing - // blanks (not significant) are dropped so none are kept before an '&'. - let escaped = ascii(v).trim_end().replace('\'', "''"); - let mut rest = escaped.as_str(); - let mut head = format!("{key:<8}= "); - while rest.len() > CARD - 12 { - let cut = CARD - 13 - rest.get(..CARD - 13).unwrap_or("").bytes().rev().take_while(|&b| b == b'\'').count() % 2; - let (piece, tail) = rest.split_at_checked(cut).unwrap_or((rest, "")); - self.card(&format!("{head}'{piece}&'")); - head = "CONTINUE ".to_string(); - rest = tail; - } - self.card(&format!("{head}'{rest:<8}'")); - } - - fn commentary(&mut self, key: &str, text: &str) { - let chars: Vec = ascii(text).chars().collect(); - if chars.is_empty() { - self.card(key); - } - for chunk in chars.chunks(CARD - 8) { - self.card(&format!("{key:<8}{}", chunk.iter().collect::())); - } - } - - fn text(&mut self, key: &str, v: &str) { - let upper = key.to_ascii_uppercase(); - if matches!(upper.as_str(), "COMMENT" | "HISTORY") { - self.commentary(&upper, v); - } else if standard_key(key) { - // `CONTINUE` only ever carries a string's continuation, written by `value`. - if !STRUCTURAL.contains(&key) && !is_naxis(key) && key != "CONTINUE" { - self.value(key, v); - } - } else { - // Long or mixed-case names (`Description`, `Software`): the HIERARCH convention, - // when the card fits, else a comment. - let name = ascii(key); - let head = format!("HIERARCH {name} = "); - let fits_key = !name.contains('=') && !name.trim().is_empty(); - let body = if literal(v) { v.to_string() } else { quoted(v, CARD.saturating_sub(head.len())) }; - if fits_key && head.len() + body.len() <= CARD && (literal(v) || body.ends_with('\'')) && quoted_whole(v, &body) { - self.card(&format!("{head}{body}")); - } else { - self.commentary("COMMENT", &format!("{key} = {v}")); - } - } - } -} - -/// `body` holds all of `v` (it was not cut short to fit the card). -fn quoted_whole(v: &str, body: &str) -> bool { - literal(v) || body.trim_matches('\'').trim_end().replace("''", "'") == ascii(v).trim_end() -} - -/// Writes `img` (gray or RGB; 8-bit, 16-bit or float) as a single-HDU FITS file, with -/// `img.text` as header cards. -pub fn encode(img: &Image) -> Result, CodecError> { - let (w, h) = (img.width(), img.height()); - let layout = img.layout(); - let nc = match layout { - ChannelLayout::Gray => 1, - ChannelLayout::Rgb => 3, - l => return Err(CodecError::encode(F, format!("FITS cannot store {l:?}"))), - }; - let bitpix = match img.sample_type() { - SampleType::U8 => 8, - SampleType::U16 => 16, - SampleType::F32 => -32, - }; - let mut wr = Writer { out: Vec::new() }; - wr.value("SIMPLE", "T"); - wr.value("BITPIX", &bitpix.to_string()); - wr.value("NAXIS", if nc == 3 { "3" } else { "2" }); - wr.value("NAXIS1", &w.to_string()); - wr.value("NAXIS2", &h.to_string()); - if nc == 3 { - wr.value("NAXIS3", "3"); - } - if bitpix == 16 { - wr.value("BZERO", "32768"); - wr.value("BSCALE", "1"); - } - wr.value("ROWORDER", "BOTTOM-UP"); - for (k, v) in &img.text { - wr.text(k, v); - } - wr.card("END"); - let mut out = wr.out; - out.resize(out.len().div_ceil(BLOCK) * BLOCK, b' '); - - // Planar, bottom row first. - let (wu, hu) = (w as usize, h as usize); - let idx = |p: usize, y: usize, x: usize| ((hu - 1 - y) * wu + x) * nc + p; - match bitpix { - 8 => { - let d = img.data(); - for p in 0..nc { - for y in 0..hu { - for x in 0..wu { - out.push(d[idx(p, y, x)]); - } - } - } - } - 16 => { - let d = img.to_u16_samples().unwrap_or_default(); - for p in 0..nc { - for y in 0..hu { - for x in 0..wu { - out.extend_from_slice(&((d[idx(p, y, x)] as i32 - 32768) as i16).to_be_bytes()); - } - } - } - } - _ => { - let d = img.to_f32_samples().unwrap_or_default(); - for p in 0..nc { - for y in 0..hu { - for x in 0..wu { - // 0..1 as documented: a value above 2 would make readers (ours too) - // take the data as ADU and scale it by 65535. NaN is written as 0. - let v = d[idx(p, y, x)]; - let v = if v.is_nan() { 0.0 } else { v.clamp(0.0, 1.0) }; - out.extend_from_slice(&v.to_be_bytes()); - } - } - } - } - } - out.resize(out.len().div_ceil(BLOCK) * BLOCK, 0); - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn header(cards: &[&str]) -> Vec { - let mut w = Writer { out: Vec::new() }; - for c in cards { - w.card(c); - } - w.card("END"); - let mut out = w.out; - out.resize(out.len().div_ceil(BLOCK) * BLOCK, b' '); - out - } - - #[test] - fn continue_strings_decode() { - let mut b = header(&[ - "SIMPLE = T", - "BITPIX = 8", - "NAXIS = 2", - "NAXIS1 = 1", - "NAXIS2 = 1", - "OBJECT = 'aaa&'", - "CONTINUE 'bbb&' / a comment", - "CONTINUE '&'", - "CONTINUE ''", - "OBSERVER= 'R&'", - "COMMENT hi", - "CONTINUE 'orphan'", - ]); - b.push(0); - let img = decode(&b, &Limits::default()).unwrap(); - let text = [("OBJECT", "aaabbb"), ("OBSERVER", "R&"), ("COMMENT", "hi"), ("CONTINUE", "'orphan'")]; - assert_eq!(img.text, text.map(|(k, v)| (k.to_string(), v.to_string()))); - } - - #[test] - fn continue_strings_round_trip() { - for value in ["a".repeat(100), format!("{}' {}", "a".repeat(65), "b".repeat(100)), "'".repeat(100), "a".repeat(68)] { - let mut img = Image::from_u8(1, 1, ChannelLayout::Gray, vec![0]).unwrap(); - img.text = vec![("OBJECT".into(), value.clone()), ("CONTINUE".into(), "orphan".into())]; - let b = encode(&img).unwrap(); - assert!(!b.chunks(CARD).any(|c| c.starts_with(b"CONTINUE="))); - let decoded = decode(&b, &Limits::default()).unwrap(); - assert_eq!(decoded.text, vec![("OBJECT".into(), value)]); - } - } - - #[test] - fn camera_u16_with_bzero_reads_exactly_and_bottom_up() { - let mut b = header(&[ - "SIMPLE = T", - "BITPIX = 16", - "NAXIS = 2", - "NAXIS1 = 2", - "NAXIS2 = 2", - "BZERO = 32768", - "OBJECT = 'M 31 '", - "EXPTIME = 300.0 / seconds", - ]); - for v in [0u16, 1000, 40000, 65535] { - b.extend_from_slice(&((v as i32 - 32768) as i16).to_be_bytes()); - } - b.resize(b.len().div_ceil(BLOCK) * BLOCK, 0); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.sample_type(), SampleType::U16); - // The first stored row is the bottom one. - assert_eq!(img.to_u16_samples().unwrap(), vec![40000, 65535, 0, 1000]); - assert!(img.text.contains(&("OBJECT".into(), "M 31".into()))); - assert!(img.text.contains(&("EXPTIME".into(), "300.0".into()))); - assert!(!img.text.iter().any(|(k, _)| k == "BZERO" || k == "NAXIS1")); - } - - #[test] - fn roworder_top_down_is_honoured() { - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 2", "NAXIS1 = 1", "NAXIS2 = 2", "ROWORDER= 'TOP-DOWN'"]); - b.extend_from_slice(&[10, 20]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[10, 20]); - } - - #[test] - fn stacked_float_in_adu_is_scaled_by_65535() { - let mut b = header(&["SIMPLE = T", "BITPIX = -32", "NAXIS = 2", "NAXIS1 = 2", "NAXIS2 = 1"]); - for v in [65535.0f32, f32::NAN] { - b.extend_from_slice(&v.to_be_bytes()); - } - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.to_f32_samples().unwrap(), vec![1.0, 0.0]); - } - - #[test] - fn hdr_float_saves_clamped_and_reopens_as_saved() { - // One value above 2 used to make the reader scale everything by 65535 (all black). - let img = Image::from_f32(2, 1, ChannelLayout::Gray, &[0.5, 3.0]).unwrap(); - let bytes = encode(&img).unwrap(); - let back = decode(&bytes, &Limits::default()).unwrap().to_f32_samples().unwrap(); - assert!((back[0] - 0.5).abs() < 1e-6, "{back:?}"); - assert!((back[1] - 1.0).abs() < 1e-6, "{back:?}"); - } - - #[test] - fn signed_negative_data_falls_back_to_float() { - let mut b = header(&["SIMPLE = T", "BITPIX = 16", "NAXIS = 2", "NAXIS1 = 2", "NAXIS2 = 1"]); - for v in [-100i16, 1000] { - b.extend_from_slice(&v.to_be_bytes()); - } - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.sample_type(), SampleType::F32); - } - - #[test] - fn image_extension_after_empty_primary() { - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0", "EXTEND = T", "TELESCOP= 'Seestar S50'"]); - b.extend(header(&["XTENSION= 'IMAGE '", "BITPIX = 8", "NAXIS = 2", "NAXIS1 = 1", "NAXIS2 = 1", "PCOUNT = 0", "GCOUNT = 1"])); - b.push(42); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[42]); - assert!(img.text.contains(&("TELESCOP".into(), "Seestar S50".into()))); - } - - /// A zero axis (`NAXIS3 = 0`) means the HDU has no data: it is neither the opened image nor - /// an additional one that a flat save would discard. - #[test] - fn zero_axis_image_hdus_are_empty() { - let empty = |primary: bool| { - let first = if primary { "SIMPLE = T" } else { "XTENSION= 'IMAGE '" }; - header(&[first, "BITPIX = 8", "NAXIS = 3", "NAXIS1 = 1", "NAXIS2 = 1", "NAXIS3 = 0", "PCOUNT = 0", "GCOUNT = 1"]) - }; - let image = || { - let mut b = header(&["XTENSION= 'IMAGE '", "BITPIX = 8", "NAXIS = 2", "NAXIS1 = 1", "NAXIS2 = 1", "PCOUNT = 0", "GCOUNT = 1"]); - b.push(42); - b.resize(b.len().div_ceil(BLOCK) * BLOCK, 0); - b - }; - let mut after_empty_primary = empty(true); - after_empty_primary.extend(image()); - let mut after_empty_extension = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0", "EXTEND = T"]); - after_empty_extension.extend(empty(false)); - after_empty_extension.extend(image()); - let mut before_empty_extension = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0", "EXTEND = T"]); - before_empty_extension.extend(image()); - before_empty_extension.extend(empty(false)); - for b in [&after_empty_primary, &after_empty_extension, &before_empty_extension] { - assert!(!has_additional_images(b).unwrap()); - assert_eq!(decode(b, &Limits::default()).unwrap().data(), &[42]); - } - // Only empty HDUs: nothing to open, nothing to lose. - assert!(!has_additional_images(&empty(true)).unwrap()); - assert!(decode(&empty(true), &Limits::default()).is_err()); - // A second non-empty image is still additional data. - let mut two = after_empty_primary.clone(); - two.extend(image()); - assert!(has_additional_images(&two).unwrap()); - } - - #[test] - fn fpack_is_refused_as_unsupported() { - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0"]); - b.extend(header(&["XTENSION= 'BINTABLE'", "BITPIX = 8", "NAXIS = 2", "NAXIS1 = 0", "NAXIS2 = 0", "ZIMAGE = T"])); - b.extend(header(&["XTENSION= 'BINTABLE'", "BITPIX = 8", "NAXIS = 1", "NAXIS1 = 0"])); - assert!(matches!(decode(&b, &Limits::default()), Err(CodecError::Unsupported { .. }))); - } - - /// Hostile headers: absurd or negative axes, bad BITPIX, overflowing sizes, missing END, - /// a long chain of empty HDUs and declared data past the end all error, never panic or - /// allocate the declared size. - #[test] - fn hostile_headers_error_without_panicking() { - let base = ["SIMPLE = T", "BITPIX = 16"]; - let cases: &[&[&str]] = &[ - &["NAXIS = 2", "NAXIS1 = 4294967296", "NAXIS2 = 1"], - &["NAXIS = 2", "NAXIS1 = -5", "NAXIS2 = 1"], - &["NAXIS = 2", "NAXIS1 = 2000000", "NAXIS2 = 2000000"], - &["NAXIS = 3", "NAXIS1 = 65536", "NAXIS2 = 65536", "NAXIS3 = 9223372036854775807"], - &["NAXIS = 1000"], - &["NAXIS = -1"], - &["NAXIS = 2", "NAXIS1 = 0", "NAXIS2 = 0"], - &["NAXIS = 2", "NAXIS1 = 1.5D300", "NAXIS2 = 1"], - &["NAXIS = 2"], - &["NAXIS = 2", "NAXIS1 = 100", "NAXIS2 = 100"], - ]; - for extra in cases { - let cards: Vec<&str> = base.iter().chain(extra.iter()).copied().collect(); - let b = header(&cards); - assert!(decode(&b, &Limits::default()).is_err(), "{extra:?}"); - } - for bitpix in ["0", "7", "-16", "128", "'X'"] { - let card = format!("BITPIX = {bitpix}"); - let b = header(&["SIMPLE = T", &card, "NAXIS = 2", "NAXIS1 = 1", "NAXIS2 = 1"]); - assert!(decode(&b, &Limits::default()).is_err(), "BITPIX {bitpix}"); - } - // No END card: the header runs off the end of the file. - let mut w = Writer { out: Vec::new() }; - for c in ["SIMPLE = T", "BITPIX = 8", "NAXIS = 2"] { - w.card(c); - } - assert!(decode(&w.out, &Limits::default()).is_err()); - // An empty primary followed by many empty extensions: bounded walk, then "no image". - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0"]); - for _ in 0..(MAX_HDUS + 5) { - b.extend(header(&["XTENSION= 'IMAGE '", "BITPIX = 8", "NAXIS = 0"])); - } - assert!(decode(&b, &Limits::default()).is_err()); - // An extension whose declared data size overflows. - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 0"]); - b.extend(header(&[ - "XTENSION= 'BINTABLE'", - "BITPIX = 64", - "NAXIS = 2", - "NAXIS1 = 9223372036854775807", - "NAXIS2 = 4", - "PCOUNT = 0", - "GCOUNT = 1", - ])); - assert!(decode(&b, &Limits::default()).is_err()); - } - - #[test] - fn extra_planes_warn() { - let mut b = header(&["SIMPLE = T", "BITPIX = 8", "NAXIS = 3", "NAXIS1 = 1", "NAXIS2 = 1", "NAXIS3 = 4"]); - b.extend_from_slice(&[1, 2, 3, 4]); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.layout(), ChannelLayout::Gray); - assert_eq!(img.warnings, vec![DecodeWarning::MorePages { total: Some(4) }]); - } - - #[test] - fn strings_with_quotes_and_hierarch_round_trip() { - let mut w = Writer { out: Vec::new() }; - w.text("OBSERVER", "O'Brien"); - w.text("Software", "craft 1.0"); - w.text("HISTORY", "stacked"); - let cards: Vec = w.out.chunks(CARD).map(parse_card).collect(); - assert_eq!(cards[0].value, Some(Value::Str("O'Brien".into()))); - assert!(cards[1].hierarch && cards[1].key == "Software"); - assert_eq!(cards[1].value, Some(Value::Str("craft 1.0".into()))); - assert_eq!((cards[2].key.as_str(), cards[2].text.as_str()), ("HISTORY", "stacked")); - } +pub fn encode(image: &Image) -> Result, CodecError> { + bridge::encode(image, Format::Fits, &Default::default()) } diff --git a/crates/raster-codecs/src/iff.rs b/crates/raster-codecs/src/iff.rs index 087ae5046..282617e21 100644 --- a/crates/raster-codecs/src/iff.rs +++ b/crates/raster-codecs/src/iff.rs @@ -1,538 +1,13 @@ -//! Amiga IFF/ILBM reader and writer (EA "Interchange File Format", "InterLeaved BitMap" and the -//! chunky "PBM " variant). Written from the published IFF/ILBM layout: a `FORM` chunk of four-byte -//! chunks (`BMHD` bitmap header, `CMAP` palette, `CAMG` Amiga viewport mode, `BODY` pixel data). -//! -//! Read: 1–8 bitplanes with a palette (or a gray ramp when there is none) with masking 0/1/2, and -//! 24-bitplane "deep" ILBMs (8 planes each of red, green and blue, least significant bit first, -//! no palette) with masking 0/1, both stored (0) and ByteRun1/PackBits (1) compressed. Anything -//! else is a clear `Unsupported`/`Malformed` error, never a panic. -//! -//! Write: a 24-bitplane ILBM, ByteRun1 compressed one plane row at a time, with a mask plane -//! (masking 1) when the image has alpha. The mask plane is one bit per pixel: alpha of 128 or -//! more is opaque. +//! IFF/ILBM compatibility facade; implementation lives in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -const F: &str = "IFF"; - -fn bad(msg: impl std::fmt::Display) -> CodecError { - CodecError::malformed(F, msg) +pub fn looks_like_iff(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Iff) } - -fn unsupported(msg: impl Into) -> CodecError { - CodecError::unsupported(F, msg) +pub fn decode(bytes: &[u8], limits: &Limits) -> Result { + bridge::format(Format::Iff, bytes, limits) } - -fn be32(b: &[u8], at: usize) -> Option { - let s = b.get(at..at.checked_add(4)?)?.try_into().ok()?; - Some(u32::from_be_bytes(s)) -} - -fn be16(b: &[u8], at: usize) -> Option { - let s = b.get(at..at.checked_add(2)?)?.try_into().ok()?; - Some(u16::from_be_bytes(s)) -} - -/// `FORM` + a bitmap type (`ILBM` interleaved, `PBM ` chunky) at offset 8. -pub fn looks_like_iff(b: &[u8]) -> bool { - b.get(0..4) == Some(b"FORM") && matches!(b.get(8..12), Some(b"ILBM" | b"PBM ")) -} - -/// Bitplanes of a "deep" ILBM: red, green and blue, 8 each, least significant bit first. -const DEEP_PLANES: u8 = 24; - -/// The bitmap header fields decoding needs. -struct Header { - width: u32, - height: u32, - planes: u8, - masking: u8, - compression: u8, - transparent: u16, - /// `PBM ` stores one palette index per byte, padded to even row lengths. - chunky: bool, -} - -/// One four-byte chunk's id and byte range. -struct Chunk<'a> { - id: [u8; 4], - data: &'a [u8], -} - -/// Walk the `FORM`'s chunks from offset 12 to `end` (the form length), stopping at the first -/// malformed length so a hostile file can't run past the buffer. -fn chunks(b: &[u8]) -> impl Iterator> { - let end = be32(b, 4).map_or(b.len(), |n| (n as usize).saturating_add(8).min(b.len())); - let mut pos = 12usize; - std::iter::from_fn(move || { - if pos + 8 > end { - return None; - } - let id = b.get(pos..pos + 4)?.try_into().ok()?; - let len = be32(b, pos + 4)? as usize; - let start = pos + 8; - let data = b.get(start..start.checked_add(len)?)?; - // Chunks are padded to an even length. - pos = start.checked_add(len)?.checked_add(len & 1)?; - Some(Chunk { id, data }) - }) -} - -fn parse_header(b: &[u8], chunky: bool) -> Result { - let bmhd = chunks(b).find(|c| &c.id == b"BMHD").ok_or_else(|| bad("no BMHD chunk"))?; - let d = bmhd.data; - if d.len() < 20 { - return Err(bad("BMHD chunk is too short")); - } - let width = u32::from(be16(d, 0).ok_or_else(|| bad("BMHD width"))?); - let height = u32::from(be16(d, 2).ok_or_else(|| bad("BMHD height"))?); - let planes = *d.get(8).ok_or_else(|| bad("BMHD planes"))?; - let masking = *d.get(9).ok_or_else(|| bad("BMHD masking"))?; - let compression = *d.get(10).ok_or_else(|| bad("BMHD compression"))?; - if width == 0 || height == 0 { - return Err(bad("zero-sized bitmap")); - } - if !(1..=8).contains(&planes) && planes != DEEP_PLANES { - return Err(unsupported(format!("{planes} bitplanes (1–8 with a palette, or 24)"))); - } - if masking > 2 { - return Err(unsupported(format!("masking {masking}"))); - } - if planes == DEEP_PLANES && masking == 2 { - return Err(unsupported("a transparent color in a 24-bitplane ILBM")); - } - if compression > 1 { - return Err(unsupported(format!("compression {compression} (only stored and ByteRun1)"))); - } - if chunky && (planes != 8 || masking == 1) { - return Err(unsupported("PBM requires 8-bit indices without a mask plane")); - } - let transparent = be16(d, 12).ok_or_else(|| bad("BMHD transparent colour"))?; - Ok(Header { width, height, planes, masking, compression, transparent, chunky }) -} - -/// ByteRun1 / PackBits decode of one plane row into `out`; returns the bytes consumed. A `0x80` -/// control byte is a no-op, as the layout specifies. -fn unpack_byte_run1(src: &[u8], out: &mut [u8]) -> Result { - let (mut i, mut o) = (0usize, 0usize); - while o < out.len() { - let n = *src.get(i).ok_or_else(|| bad("truncated ByteRun1 data"))? as i8; - i += 1; - if n >= 0 { - let count = n as usize + 1; - let lit = src.get(i..i.checked_add(count).ok_or_else(|| bad("ByteRun1 overflow"))?).ok_or_else(|| bad("truncated ByteRun1 run"))?; - if count > out.len() - o { - return Err(bad("ByteRun1 run crosses the plane row")); - } - out[o..o + count].copy_from_slice(lit); - i += count; - o += count; - } else if n != -128 { - let count = (1 - n as i32) as usize; - let value = *src.get(i).ok_or_else(|| bad("truncated ByteRun1 repeat"))?; - i += 1; - if count > out.len() - o { - return Err(bad("ByteRun1 repeat crosses the plane row")); - } - out[o..o + count].fill(value); - o += count; - } - } - Ok(i) -} - -pub fn decode(b: &[u8], limits: &Limits) -> Result { - if !looks_like_iff(b) { - return Err(bad("not an IFF/ILBM file (no FORM ... ILBM)")); - } - let h = parse_header(b, b.get(8..12) == Some(b"PBM "))?; - if let Some(camg) = chunks(b).find(|c| &c.id == b"CAMG") { - let mode = be32(camg.data, 0).ok_or_else(|| bad("short CAMG chunk"))?; - if mode & (0x80 | 0x800) != 0 { - return Err(unsupported("EHB and HAM viewport modes")); - } - } - let layout = if h.masking == 0 { ChannelLayout::Rgb } else { ChannelLayout::Rgba }; - limits.check(h.width, h.height, layout, SampleType::U8)?; - - // Palette: three bytes per colour; missing entries read as black. - let cmap = chunks(b).find(|c| &c.id == b"CMAP").map(|c| c.data).filter(|p| !p.is_empty()); - let colors = 1usize << h.planes; - let palette = |idx: usize| -> [u8; 3] { - if let Some(p) = cmap { - p.get(idx * 3..idx * 3 + 3).map(|c| [c[0], c[1], c[2]]).unwrap_or([0; 3]) - } else { - let v = ((idx as u32 * 255) / (colors as u32 - 1).max(1)) as u8; - [v, v, v] - } - }; - - let body = chunks(b).find(|c| &c.id == b"BODY").map(|c| c.data).ok_or_else(|| bad("no BODY chunk"))?; - let (w, hh) = (h.width as usize, h.height as usize); - // Each plane row is stored as a whole number of 16-bit words. - let plane_row = if h.chunky { w.div_ceil(2) * 2 } else { w.div_ceil(16) * 2 }; - // A masking type 1 bitmap carries one extra (mask) plane per row. - let rows_planes: usize = if h.chunky { 1 } else { usize::from(h.planes) + usize::from(h.masking == 1) }; - - // Decode every stored plane row up front, then recombine. - let total_rows = hh.checked_mul(rows_planes).ok_or_else(|| CodecError::LimitExceeded("bitmap too large".into()))?; - let total_bytes = total_rows.checked_mul(plane_row).ok_or_else(|| CodecError::LimitExceeded("bitmap too large".into()))?; - if total_bytes as u64 > limits.max_alloc { - return Err(CodecError::LimitExceeded("bitmap storage exceeds max_alloc".into())); - } - Limits::check_plausible(total_bytes as u64, body.len(), if h.compression == 0 { 1 } else { 128 })?; - let mut stored = crate::alloc_zeroed::(total_bytes)?; - let mut cursor = 0usize; - for row in stored.chunks_mut(plane_row.max(1)).take(total_rows) { - match h.compression { - 0 => { - let src = - body.get(cursor..cursor.checked_add(plane_row).ok_or_else(|| bad("BODY overflow"))?).ok_or_else(|| bad("truncated BODY data"))?; - let n = row.len().min(src.len()); - row[..n].copy_from_slice(&src[..n]); - cursor += plane_row; - } - _ => { - let tail = body.get(cursor..).ok_or_else(|| bad("truncated BODY data"))?; - cursor += unpack_byte_run1(tail, row)?; - } - } - } - - let mut out = Vec::new(); - out.try_reserve_exact(w.saturating_mul(hh).saturating_mul(layout.channels())) - .map_err(|_| CodecError::LimitExceeded("cannot allocate the image".into()))?; - let planes = usize::from(h.planes); - for y in 0..hh { - for x in 0..w { - let index = if h.chunky { - usize::from(*stored.get(y * plane_row + x).ok_or_else(|| bad("PBM pixel outside BODY"))?) - } else { - (0..planes).filter(|&p| bit_at(&stored, (y * rows_planes + p) * plane_row, x)).fold(0usize, |acc, p| acc | (1 << p)) - }; - let c = if h.planes == DEEP_PLANES { [index as u8, (index >> 8) as u8, (index >> 16) as u8] } else { palette(index) }; - out.extend_from_slice(&c); - if h.masking != 0 { - let opaque = - if h.masking == 1 { bit_at(&stored, (y * rows_planes + planes) * plane_row, x) } else { index != usize::from(h.transparent) }; - out.push(if opaque { 255 } else { 0 }); - } - } - } - Image::from_u8(h.width, h.height, layout, out) -} - -/// The bit for pixel `x` of a stored plane row at `base` (bits run MSB-first). -fn bit_at(stored: &[u8], base: usize, x: usize) -> bool { - stored.get(base + x / 8).is_some_and(|byte| byte & (0x80 >> (x % 8)) != 0) -} - -/// ByteRun1 / PackBits encode of one plane row, appended to `out`: three or more equal bytes -/// become a repeat (at most 128), everything else literal runs (at most 128 bytes). -fn pack_byte_run1(row: &[u8], out: &mut Vec) { - let mut i = 0usize; - while let Some(&value) = row.get(i) { - let repeat = row[i..].iter().take(128).take_while(|&&b| b == value).count(); - if repeat >= 3 { - out.push((1 - repeat as i32) as i8 as u8); - out.push(value); - i += repeat; - continue; - } - let start = i; - while i < row.len() && i - start < 128 { - if row.get(i..i + 3).is_some_and(|r| r[0] == r[1] && r[1] == r[2]) { - break; - } - i += 1; - } - out.push((i - start - 1) as u8); - out.extend_from_slice(&row[start..i]); - } -} - -fn push_chunk(out: &mut Vec, id: &[u8; 4], data: &[u8]) -> Result<(), CodecError> { - let len = u32::try_from(data.len()).map_err(|_| CodecError::encode(F, "chunk too large for an IFF file (4 GiB)"))?; - out.extend_from_slice(id); - out.extend_from_slice(&len.to_be_bytes()); - out.extend_from_slice(data); - // Chunks are padded to an even length. - if data.len() % 2 == 1 { - out.push(0); - } - Ok(()) -} - -/// Writes a 24-bitplane `FORM ILBM`: `BMHD` then a ByteRun1-compressed `BODY`, each row stored as -/// the red, green and blue planes (least significant bit first) and, for RGBA, a mask plane. -pub fn encode(img: &Image) -> Result, CodecError> { - let enc = |msg: String| CodecError::encode(F, msg); - let masked = match img.layout() { - ChannelLayout::Rgb => false, - ChannelLayout::Rgba => true, - l => return Err(enc(format!("IFF/ILBM cannot store {l:?}"))), - }; - if img.sample_type() != SampleType::U8 { - return Err(enc(format!("IFF/ILBM cannot store {:?} samples", img.sample_type()))); - } - let (w, h) = (img.width(), img.height()); - let (Ok(w16), Ok(h16)) = (u16::try_from(w), u16::try_from(h)) else { - return Err(enc("IFF/ILBM stores at most 65535 × 65535 pixels".into())); - }; - let (w, ch) = (w as usize, img.layout().channels()); - let data = img.data(); - let row_len = w.checked_mul(ch).ok_or_else(|| enc("image too large".into()))?; - if data.len() != row_len.checked_mul(h as usize).ok_or_else(|| enc("image too large".into()))? { - return Err(enc("the image has the wrong number of samples".into())); - } - let planes = usize::from(DEEP_PLANES) + usize::from(masked); - let mut plane_row = vec![0u8; w.div_ceil(16) * 2]; - let mut body = Vec::new(); - for px in data.chunks_exact(row_len.max(1)) { - for plane in 0..planes { - plane_row.fill(0); - for (x, p) in px.chunks_exact(ch).enumerate() { - let on = if plane == usize::from(DEEP_PLANES) { p[3] >= 128 } else { (p[plane / 8] >> (plane % 8)) & 1 == 1 }; - if on { - plane_row[x / 8] |= 0x80 >> (x % 8); - } - } - pack_byte_run1(&plane_row, &mut body); - } - } - - let mut bmhd = Vec::with_capacity(20); - bmhd.extend_from_slice(&w16.to_be_bytes()); - bmhd.extend_from_slice(&h16.to_be_bytes()); - bmhd.extend_from_slice(&[0, 0, 0, 0]); // x/y origin - bmhd.push(DEEP_PLANES); - bmhd.push(u8::from(masked)); // masking: 1 = a mask plane follows the color planes - bmhd.push(1); // ByteRun1 - bmhd.push(0); // pad - bmhd.extend_from_slice(&[0, 0]); // transparent color (unused) - bmhd.extend_from_slice(&[1, 1]); // square pixels - bmhd.extend_from_slice(&w16.to_be_bytes()); // page size - bmhd.extend_from_slice(&h16.to_be_bytes()); - - let mut out = b"FORM".to_vec(); - out.extend_from_slice(&[0; 4]); // patched below - out.extend_from_slice(b"ILBM"); - push_chunk(&mut out, b"BMHD", &bmhd)?; - push_chunk(&mut out, b"BODY", &body)?; - let form_len = u32::try_from(out.len() - 8).map_err(|_| enc("image too large for an IFF file (4 GiB)".into()))?; - out[4..8].copy_from_slice(&form_len.to_be_bytes()); - Ok(out) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// A `FORM ILBM` with a `BMHD`, `CMAP` and `BODY`. `planes` plane rows per row, - /// `compression` 0 (stored) or 1 (ByteRun1), MSB-first. - fn ilbm(w: u16, h: u16, planes: u8, palette: &[[u8; 3]], compression: u8, body: &[u8]) -> Vec { - let mut b = b"FORM".to_vec(); - b.extend_from_slice(&0u32.to_be_bytes()); // patched below - b.extend_from_slice(b"ILBM"); - // BMHD (20 bytes) - b.extend_from_slice(b"BMHD"); - b.extend_from_slice(&20u32.to_be_bytes()); - b.extend_from_slice(&w.to_be_bytes()); - b.extend_from_slice(&h.to_be_bytes()); - b.extend_from_slice(&[0, 0, 0, 0]); // x/y origin - b.push(planes); - b.push(0); // masking - b.push(compression); - b.push(0); // pad - b.extend_from_slice(&[0, 0]); // transparent colour - b.extend_from_slice(&[1, 1]); // aspect - b.extend_from_slice(&[0, 0, 0, 0]); // page size - // CMAP - b.extend_from_slice(b"CMAP"); - b.extend_from_slice(&((palette.len() * 3) as u32).to_be_bytes()); - for c in palette { - b.extend_from_slice(c); - } - if palette.len() % 2 == 1 { - b.push(0); - } - // BODY - b.extend_from_slice(b"BODY"); - b.extend_from_slice(&(body.len() as u32).to_be_bytes()); - b.extend_from_slice(body); - if body.len() % 2 == 1 { - b.push(0); - } - let size = (b.len() - 8) as u32; - b[4..8].copy_from_slice(&size.to_be_bytes()); - b - } - - #[test] - fn reads_a_stored_two_plane_bitmap() { - // 2×1, 2 planes: pixel 0 index 0 (plane bits 0,0), pixel 1 index 3 (bits 1,1). - // Plane 0 = 0b0100_0000, plane 1 = 0b0100_0000, each padded to 2 bytes. - let body = [0b0100_0000, 0, 0b0100_0000, 0]; - let pal = [[10, 20, 30], [40, 50, 60], [70, 80, 90], [100, 110, 120]]; - let b = ilbm(2, 1, 2, &pal, 0, &body); - assert!(looks_like_iff(&b)); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!((img.width(), img.height(), img.layout()), (2, 1, ChannelLayout::Rgb)); - assert_eq!(img.data(), &[10, 20, 30, 100, 110, 120]); - } - - #[test] - fn reads_byte_run1_compressed_rows() { - // Same image, ByteRun1: literal run of two bytes (0x01) then the two plane bytes each row. - let body = [0x01, 0b0100_0000, 0, 0x01, 0b0100_0000, 0]; - let pal = [[1, 2, 3], [4, 5, 6], [7, 8, 9], [250, 251, 252]]; - let b = ilbm(2, 1, 2, &pal, 1, &body); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[1, 2, 3, 250, 251, 252]); - } - - #[test] - fn no_palette_falls_back_to_a_grey_ramp() { - // 2×1, 1 plane, no CMAP: indices 0 and 1 → black and white. - let body = [0b0100_0000, 0]; - let b = ilbm(2, 1, 1, &[], 0, &body); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[0, 0, 0, 255, 255, 255]); - } - - #[test] - fn pbm_reads_chunky_indices_with_odd_width_padding() { - let pal = [[10, 20, 30], [40, 50, 60], [70, 80, 90]]; - for (compression, body) in [(0, vec![2, 0, 1, 255, 1, 2, 0, 255]), (1, vec![3, 2, 0, 1, 255, 3, 1, 2, 0, 255])] { - let mut b = ilbm(3, 2, 8, &pal, compression, &body); - b[8..12].copy_from_slice(b"PBM "); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.data(), &[70, 80, 90, 10, 20, 30, 40, 50, 60, 40, 50, 60, 70, 80, 90, 10, 20, 30]); - } - } - - #[test] - fn transparency_uses_the_mask_plane_or_transparent_index() { - for masking in [1, 2] { - let mut b = ilbm(2, 1, 1, &[[10, 20, 30], [40, 50, 60]], 0, if masking == 1 { &[0x40, 0, 0x40, 0] } else { &[0x40, 0] }); - b[29] = masking; - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!(img.layout(), ChannelLayout::Rgba); - assert_eq!(img.data(), &[10, 20, 30, 0, 40, 50, 60, 255]); - } - let mut b = ilbm(2, 1, 8, &[[10, 20, 30], [40, 50, 60]], 0, &[0, 1]); - b[8..12].copy_from_slice(b"PBM "); - b[29] = 2; - b[32..34].copy_from_slice(&1u16.to_be_bytes()); - assert_eq!(decode(&b, &Limits::default()).unwrap().data(), &[10, 20, 30, 255, 40, 50, 60, 0]); - } - - #[test] - fn rejects_ham_and_ehb_instead_of_returning_wrong_colours() { - for mode in [0x80u32, 0x800] { - let mut b = ilbm(1, 1, 1, &[], 0, &[0, 0]); - b.extend_from_slice(b"CAMG"); - b.extend_from_slice(&4u32.to_be_bytes()); - b.extend_from_slice(&mode.to_be_bytes()); - let len = (b.len() - 8) as u32; - b[4..8].copy_from_slice(&len.to_be_bytes()); - assert!(matches!(decode(&b, &Limits::default()), Err(CodecError::Unsupported { .. }))); - } - } - - #[test] - fn missing_palette_entries_are_black() { - let b = ilbm(1, 1, 2, &[[10, 20, 30]], 0, &[0x80, 0, 0x80, 0]); - assert_eq!(decode(&b, &Limits::default()).unwrap().data(), &[0, 0, 0]); - } - - #[test] - fn reads_a_stored_24_plane_bitmap() { - // 2×1: pixel 0 = (1, 0, 128), pixel 1 = (0, 255, 0). Each plane row is 2 bytes; red bit 0 - // has pixel 0 set, green bits 0–7 have pixel 1 set, blue bit 7 has pixel 0 set. - let mut body = Vec::new(); - for plane in 0..24 { - let byte = match plane { - 0 | 23 => 0x80, - 8..=15 => 0x40, - _ => 0, - }; - body.extend_from_slice(&[byte, 0]); - } - let b = ilbm(2, 1, 24, &[], 0, &body); - let img = decode(&b, &Limits::default()).unwrap(); - assert_eq!((img.width(), img.height(), img.layout()), (2, 1, ChannelLayout::Rgb)); - assert_eq!(img.data(), &[1, 0, 128, 0, 255, 0]); - } - - #[test] - fn writes_24_plane_byte_run1_that_reads_back_exactly() { - // 37 pixels wide: three plane-row words, the last one partly padding. - let (w, h) = (37u32, 3u32); - let data: Vec = (0..w * h * 3).map(|i| (i * 7 % 256) as u8).collect(); - let img = Image::from_u8(w, h, ChannelLayout::Rgb, data.clone()).unwrap(); - let bytes = encode(&img).unwrap(); - assert!(looks_like_iff(&bytes)); - assert_eq!(be32(&bytes, 4).unwrap() as usize, bytes.len() - 8, "FORM length"); - let h = parse_header(&bytes, false).unwrap(); - assert_eq!((h.width, h.height, h.planes, h.masking, h.compression), (37, 3, 24, 0, 1)); - assert!(chunks(&bytes).all(|c| &c.id != b"CMAP"), "a deep ILBM has no palette"); - let back = decode(&bytes, &Limits::default()).unwrap(); - assert_eq!(back.layout(), ChannelLayout::Rgb); - assert_eq!(back.data(), &data[..]); - } - - #[test] - fn alpha_is_written_as_a_mask_plane() { - let data = [10, 20, 30, 0, 40, 50, 60, 127, 70, 80, 90, 128, 1, 2, 3, 255]; - let img = Image::from_u8(4, 1, ChannelLayout::Rgba, data.to_vec()).unwrap(); - let bytes = encode(&img).unwrap(); - assert_eq!(parse_header(&bytes, false).unwrap().masking, 1); - let back = decode(&bytes, &Limits::default()).unwrap(); - assert_eq!(back.layout(), ChannelLayout::Rgba); - assert_eq!(back.data(), &[10, 20, 30, 0, 40, 50, 60, 0, 70, 80, 90, 255, 1, 2, 3, 255]); - } - - #[test] - fn byte_run1_round_trips_runs_literals_and_long_rows() { - let mut rows: Vec> = vec![vec![], vec![5], vec![5, 5], vec![5, 5, 5], vec![1, 2, 2, 2, 3], vec![9; 300]]; - rows.push((0..300).map(|i| (i % 251) as u8).collect()); - rows.push([vec![1, 2], vec![7; 129], vec![3]].concat()); - for row in rows { - let mut packed = Vec::new(); - pack_byte_run1(&row, &mut packed); - let mut back = vec![0; row.len()]; - assert_eq!(unpack_byte_run1(&packed, &mut back).unwrap(), packed.len(), "{row:?}"); - assert_eq!(back, row); - } - // A long repeat packs into two-byte runs. - let mut packed = Vec::new(); - pack_byte_run1(&[0; 256], &mut packed); - assert_eq!(packed, [0x81, 0, 0x81, 0]); - } - - #[test] - fn rejects_what_ilbm_cannot_store() { - let gray = Image::from_u8(1, 1, ChannelLayout::Gray, vec![0]).unwrap(); - assert!(encode(&gray).is_err()); - let wide = Image::from_u8(65536, 1, ChannelLayout::Rgb, vec![0; 65536 * 3]).unwrap(); - assert!(encode(&wide).is_err()); - // A transparent color index has no meaning without a palette. - let mut b = ilbm(1, 1, 24, &[], 0, &[0; 48]); - b[29] = 2; - assert!(matches!(decode(&b, &Limits::default()), Err(CodecError::Unsupported { .. }))); - } - - #[test] - fn hostile_input_errors_not_panics() { - // Truncations of a valid file never panic. - let body = [0b0100_0000, 0, 0b0100_0000, 0]; - let full = ilbm(2, 1, 2, &[[1, 2, 3], [4, 5, 6], [7, 8, 9], [10, 11, 12]], 0, &body); - for cut in [0, 4, 8, 12, 40, full.len() - 1] { - assert!(decode(&full[..cut.min(full.len())], &Limits::default()).is_err(), "cut {cut}"); - } - // Too many planes is unsupported. - let bad = ilbm(1, 1, 9, &[], 0, &[0, 0]); - assert!(matches!(decode(&bad, &Limits::default()), Err(CodecError::Unsupported { .. }))); - } +pub fn encode(image: &Image) -> Result, CodecError> { + bridge::encode(image, Format::Iff, &Default::default()) } diff --git a/crates/raster-codecs/src/lib.rs b/crates/raster-codecs/src/lib.rs index 5b335a30c..7e705da99 100644 --- a/crates/raster-codecs/src/lib.rs +++ b/crates/raster-codecs/src/lib.rs @@ -18,10 +18,10 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod bridge; /// Common bounded decoding and PNG/JPEG encoding for common image formats. pub mod common; pub mod dds; -mod dds_bc; pub mod dicom; pub mod fits; pub mod iff; @@ -215,47 +215,32 @@ impl Limits { /// Validates declared dimensions for an output buffer of the given layout and sample type. pub fn check(&self, width: u32, height: u32, layout: ChannelLayout, sample: SampleType) -> Result<(), CodecError> { - if width == 0 || height == 0 { - return Err(CodecError::InvalidImage(format!("zero-sized image {width}x{height}"))); - } - if width > self.max_width || height > self.max_height { - return Err(CodecError::LimitExceeded(format!("dimensions {width}x{height} exceed max {}x{}", self.max_width, self.max_height))); - } - let pixels = width as u64 * height as u64; - if pixels > self.max_pixels { - return Err(CodecError::LimitExceeded(format!("{pixels} pixels exceed max {}", self.max_pixels))); - } - let bytes = pixels.saturating_mul((layout.channels() * sample.bytes()) as u64); - if bytes > self.max_alloc { - return Err(CodecError::LimitExceeded(format!("{bytes} bytes exceed max_alloc {}", self.max_alloc))); - } - if usize::try_from(bytes).is_err() { - return Err(CodecError::LimitExceeded(format!("{bytes} bytes do not fit in memory"))); - } - Ok(()) + let layout = match layout { + ChannelLayout::Gray => craft_image_codecs::ChannelLayout::Gray, + ChannelLayout::GrayA => craft_image_codecs::ChannelLayout::GrayA, + ChannelLayout::Rgb => craft_image_codecs::ChannelLayout::Rgb, + ChannelLayout::Rgba => craft_image_codecs::ChannelLayout::Rgba, + ChannelLayout::Cmyk => craft_image_codecs::ChannelLayout::Cmyk, + ChannelLayout::CmykA => craft_image_codecs::ChannelLayout::CmykA, + }; + let sample = match sample { + SampleType::U8 => craft_image_codecs::SampleType::U8, + SampleType::U16 => craft_image_codecs::SampleType::U16, + SampleType::F32 => craft_image_codecs::SampleType::F32, + }; + craft_image_codecs::Limits::new(self.max_width, self.max_height, self.max_pixels, self.max_alloc) + .check(width, height, layout, sample) + .map_err(crate::bridge::from_shared_error) } /// Decompression-bomb guard for formats that compress: refuses a file too small to hold /// `declared_bytes` of stored image data when the compression never beats `max_ratio`:1. /// Called before the output buffer is allocated. 1 MiB of slack leaves tiny images alone. - pub(crate) fn check_plausible(declared_bytes: u64, file_len: usize, max_ratio: u64) -> Result<(), CodecError> { - let capacity = (file_len as u64).saturating_mul(max_ratio).saturating_add(1 << 20); - if declared_bytes > capacity { - return Err(CodecError::LimitExceeded(format!("a {file_len} byte file cannot hold {declared_bytes} bytes of image data"))); - } - Ok(()) + pub fn check_plausible(declared_bytes: u64, file_len: usize, max_ratio: u64) -> Result<(), CodecError> { + craft_image_codecs::Limits::check_plausible(declared_bytes, file_len, max_ratio).map_err(crate::bridge::from_shared_error) } } -/// A zero-filled buffer of `len` elements, allocated fallibly: an allocation failure is -/// [`CodecError::LimitExceeded`] instead of an abort. -pub(crate) fn alloc_zeroed(len: usize) -> Result, CodecError> { - let mut v = Vec::new(); - v.try_reserve_exact(len).map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {len} elements for the image")))?; - v.resize(len, T::default()); - Ok(v) -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/raster-codecs/src/pcx.rs b/crates/raster-codecs/src/pcx.rs index 00117bcf1..61c476511 100644 --- a/crates/raster-codecs/src/pcx.rs +++ b/crates/raster-codecs/src/pcx.rs @@ -1,144 +1,13 @@ -//! ZSoft PCX (Paintbrush) reader: strictly bounded scanline RLE, 16/256-colour -//! palettes, 8-bit three-plane RGB and four-plane RGBA. Read-only: modern saves should use PNG. +//! PCX compatibility facade; implementation and header probing live in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -const HEADER: usize = 128; - -fn bad(reason: impl std::fmt::Display) -> CodecError { - CodecError::malformed("PCX", reason) +pub fn looks_like_pcx(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Pcx) } -fn le16(b: &[u8], i: usize) -> u16 { - u16::from_le_bytes([b[i], b[i + 1]]) +pub fn dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + craft_image_codecs::pcx_dimensions(bytes) } - -/// PCX lacks a strong magic number. Require plausible complete header fields before -/// identifying it, rather than classifying every file starting with 0x0a as PCX. -pub fn looks_like_pcx(b: &[u8]) -> bool { - if b.len() < HEADER || b[0] != 0x0a || !matches!(b[1], 0 | 2..=5) || b[2] != 1 { - return false; - } - let bits = b[3]; - let planes = b[65]; - let stride = u32::from(le16(b, 66)); - let (x0, y0, x1, y1) = (le16(b, 4), le16(b, 6), le16(b, 8), le16(b, 10)); - if x1 < x0 || y1 < y0 || !matches!(bits, 1 | 2 | 4 | 8) || !(1..=4).contains(&planes) { - return false; - } - let width = u32::from(x1 - x0) + 1; - let min_stride = (width * u32::from(bits)).div_ceil(8); - stride >= min_stride && stride > 0 && (bits != 8 || matches!(planes, 1 | 3 | 4)) && (bits == 8 || u32::from(bits) * u32::from(planes) <= 4) -} - -/// Read one compressed scanline, including all planes. RLE packets never cross a scanline. -fn read_row(b: &[u8], pos: &mut usize, out: &mut [u8]) -> Result<(), CodecError> { - let mut n = 0; - while n < out.len() { - let code = *b.get(*pos).ok_or_else(|| bad("truncated pixel RLE"))?; - *pos += 1; - let (count, value) = if code >= 0xc0 { - let count = usize::from(code & 0x3f); - let value = *b.get(*pos).ok_or_else(|| bad("truncated RLE packet"))?; - *pos += 1; - (count, value) - } else { - (1, code) - }; - if count == 0 || count > out.len() - n { - return Err(bad("RLE packet crosses a scanline or has zero length")); - } - out[n..n + count].fill(value); - n += count; - } - Ok(()) -} - -/// The pixel size a PCX header declares, without decoding; `None` when [`looks_like_pcx`] is false. -pub fn dimensions(b: &[u8]) -> Option<(u32, u32)> { - looks_like_pcx(b).then(|| (u32::from(le16(b, 8) - le16(b, 4)) + 1, u32::from(le16(b, 10) - le16(b, 6)) + 1)) -} - -/// Decodes to 8-bit RGB, or RGBA for four 8-bit planes (red, green, blue, alpha). -pub fn decode(b: &[u8], limits: &Limits) -> Result { - let (w, h) = dimensions(b).ok_or_else(|| bad("invalid PCX header or unsupported pixel layout"))?; - let (bits, planes, stride) = (usize::from(b[3]), usize::from(b[65]), usize::from(le16(b, 66))); - let layout = if bits == 8 && planes == 4 { ChannelLayout::Rgba } else { ChannelLayout::Rgb }; - let ch = layout.channels(); - limits.check(w, h, layout, SampleType::U8)?; - let row_len = stride.checked_mul(planes).ok_or_else(|| bad("row size overflow"))?; - let total = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(ch)).ok_or_else(|| bad("pixel size overflow"))?; - // The last 769 bytes are the optional VGA palette for 8-bit indexed images. - let vga = bits == 8 && planes == 1 && b.len() >= HEADER + 769 && b[b.len() - 769] == 0x0c; - if bits == 8 && planes == 1 && !vga && le16(b, 68) != 2 { - return Err(bad("8-bit indexed PCX has no 256-colour palette")); - } - let image_end = if vga { b.len() - 769 } else { b.len() }; - // A 63-byte RLE packet takes 2 bytes; a trailing remainder of 1 byte below 0xC0 - // can be encoded as a 1-byte literal, while any larger remainder takes at least 2 bytes. - // Because RLE packets may span planes within a scanline (`row_len`), compute the - // minimum encoded bytes across the full scanline (`row_len`) and multiply by `h`. - // Reject inputs that cannot possibly contain all scanlines before allocating - // the decoded image buffer. - let min_packets = (row_len / 63) - .checked_mul(2) - .and_then(|n| { - n.checked_add(match row_len % 63 { - 0 => 0, - 1 => 1, - _ => 2, - }) - }) - .ok_or_else(|| bad("compressed row size overflow"))?; - let min_encoded = min_packets.checked_mul(h as usize).ok_or_else(|| bad("compressed image size overflow"))?; - let encoded_len = image_end.saturating_sub(HEADER); - if encoded_len < min_encoded { - return Err(bad("PCX pixel data is too short for the declared image")); - } - let mut row = vec![0u8; row_len]; - let mut pixels = Vec::new(); - pixels.try_reserve_exact(total).map_err(|_| bad("PCX pixel buffer is too large"))?; - pixels.resize(total, 0); - let mut p = HEADER; - let width = w as usize; - for y in 0..h as usize { - read_row(&b[..image_end], &mut p, &mut row)?; - for x in 0..width { - let at = (y * width + x) * ch; - let rgb = if bits == 8 && planes >= 3 { - if planes == 4 { - pixels[at + 3] = row[3 * stride + x]; - } - [row[x], row[stride + x], row[2 * stride + x]] - } else { - let index = if bits == 8 { - usize::from(row[x]) - } else { - let bit = x * bits; - let shift = 8 - bits - (bit % 8); - let mask = (1usize << bits) - 1; - (0..planes).fold(0usize, |v, plane| { - let sample = (usize::from(row[plane * stride + bit / 8]) >> shift) & mask; - v | sample << (plane * bits) - }) - }; - if vga { - let o = image_end + 1 + index * 3; - [b[o], b[o + 1], b[o + 2]] - } else if bits == 8 { - let g = index as u8; - [g, g, g] // palette-info 2: gray scale - } else { - let o = 16 + index * 3; - [b[o], b[o + 1], b[o + 2]] - } - }; - pixels[at..at + 3].copy_from_slice(&rgb); - } - } - let mut image = Image::from_u8(w, h, layout, pixels)?; - let (xdpi, ydpi) = (le16(b, 12), le16(b, 14)); - if xdpi > 0 && ydpi > 0 { - image.dpi = Some((f32::from(xdpi), f32::from(ydpi))); - } - Ok(image) +pub fn decode(bytes: &[u8], limits: &Limits) -> Result { + bridge::format(Format::Pcx, bytes, limits) } diff --git a/crates/raster-codecs/src/pnm.rs b/crates/raster-codecs/src/pnm.rs index 6ef140902..834e6508a 100644 --- a/crates/raster-codecs/src/pnm.rs +++ b/crates/raster-codecs/src/pnm.rs @@ -1,357 +1,13 @@ -//! Netpbm family, implemented here: P1–P6 (PBM/PGM/PPM, ASCII and binary), -//! P7 (PAM, including the common `CMYK`/`CMYK_ALPHA` tuple types) and PFM -//! (`PF` colour / `Pf` gray float). -//! -//! Writing picks the subtype from the image: float → PFM (gray/RGB), gray -//! → P5, RGB → P6, anything with alpha or CMYK → P7. +//! Netpbm compatibility facade; implementation and bounds live in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -const F: &str = "PNM"; - -fn err(msg: impl std::fmt::Display) -> CodecError { - CodecError::malformed(F, msg) +pub fn looks_like_pnm(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Pnm) } - -struct Reader<'a> { - b: &'a [u8], - pos: usize, -} - -impl<'a> Reader<'a> { - fn skip_ws_and_comments(&mut self) { - while self.pos < self.b.len() { - let c = self.b[self.pos]; - if c == b'#' { - while self.pos < self.b.len() && self.b[self.pos] != b'\n' && self.b[self.pos] != b'\r' { - self.pos += 1; - } - } else if c.is_ascii_whitespace() { - self.pos += 1; - } else { - break; - } - } - } - - fn token(&mut self) -> Result<&'a [u8], CodecError> { - self.skip_ws_and_comments(); - let start = self.pos; - while self.pos < self.b.len() && !self.b[self.pos].is_ascii_whitespace() && self.b[self.pos] != b'#' { - self.pos += 1; - } - if start == self.pos { - return Err(err("unexpected end of header")); - } - Ok(&self.b[start..self.pos]) - } - - fn uint(&mut self) -> Result { - let t = self.token()?; - std::str::from_utf8(t).ok().and_then(|s| s.parse::().ok()).ok_or_else(|| err("invalid integer in header")) - } - - /// Consume the single whitespace byte that separates header and raster. - fn single_ws(&mut self) -> Result<(), CodecError> { - match self.b.get(self.pos) { - Some(c) if c.is_ascii_whitespace() => { - self.pos += 1; - Ok(()) - } - _ => Err(err("missing whitespace after header")), - } - } - - /// PAM ends its header with an LF-terminated line, unlike P4–P6's single separator. - fn pam_endhdr(&mut self) -> Result<(), CodecError> { - while let Some(&c) = self.b.get(self.pos) { - self.pos += 1; - if c == b'\n' { - // Stop at the first LF: whitespace and '#' after it are raster bytes. - return Ok(()); - } - if !c.is_ascii_whitespace() { - return Err(err("unexpected data after PAM ENDHDR")); - } - } - Err(err("unterminated PAM ENDHDR line")) - } - - fn rest(&self) -> &'a [u8] { - &self.b[self.pos.min(self.b.len())..] - } -} - -fn sample_for_maxval(maxval: u32) -> Result { - match maxval { - 1..=255 => Ok(SampleType::U8), - 256..=65535 => Ok(SampleType::U16), - _ => Err(err(format!("invalid maxval {maxval}"))), - } -} - -/// The decoded raster, built straight in the output representation. It is the only buffer sized -/// by the header, so `Limits::check` on the output budgets all of it: no wider scratch buffer -/// (such as one `u32` per sample) sits beside it. -struct Samples { - maxval: u32, - sample: SampleType, - bytes: Vec, -} - -impl Samples { - /// Reserve room for `cap` samples (callers bound `cap` by the file length where the header - /// alone could overstate it). - fn with_capacity(maxval: u32, cap: usize) -> Result { - let sample = sample_for_maxval(maxval)?; - let len = cap.checked_mul(sample.bytes()).ok_or_else(|| CodecError::LimitExceeded(format!("{cap} samples do not fit in memory")))?; - let mut bytes = Vec::new(); - bytes.try_reserve_exact(len).map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {len} bytes for the image")))?; - Ok(Samples { maxval, sample, bytes }) - } - - /// Append one raw value in `[0, maxval]`, rescaled to the full range of the output sample. - fn push(&mut self, v: u32) -> Result<(), CodecError> { - let m = self.maxval; - if v > m { - return Err(err("sample exceeds maxval")); - } - match (self.sample, m) { - (SampleType::U8, 255) => self.bytes.push(v as u8), - (SampleType::U8, m) => self.bytes.push(((v * 255 + m / 2) / m) as u8), - (_, 65535) => self.bytes.extend_from_slice(&(v as u16).to_ne_bytes()), - (_, m) => self.bytes.extend_from_slice(&(((v as u64 * 65535 + m as u64 / 2) / m as u64) as u16).to_ne_bytes()), - } - Ok(()) - } - - fn into_image(self, w: u32, h: u32, layout: ChannelLayout) -> Result { - Image::from_raw(w, h, layout, self.sample, self.bytes) - } -} - -/// `w × h × channels` samples. `Limits::none()` lets any header through, so the product is -/// checked here rather than trusted to the limits. -fn sample_count(w: u32, h: u32, channels: usize) -> Result { - (w as usize) - .checked_mul(h as usize) - .and_then(|p| p.checked_mul(channels)) - .ok_or_else(|| CodecError::LimitExceeded(format!("{w}x{h}x{channels} samples do not fit in memory"))) -} - -fn read_binary(data: &[u8], n: usize, maxval: u32) -> Result { - let mut out = Samples::with_capacity(maxval, n)?; - if maxval < 256 { - let Some(raw) = data.get(..n) else { - return Err(err("truncated raster")); - }; - for &v in raw { - out.push(v as u32)?; - } - } else { - let Some(raw) = n.checked_mul(2).and_then(|len| data.get(..len)) else { - return Err(err("truncated raster")); - }; - for c in raw.as_chunks::<2>().0 { - out.push(u16::from_be_bytes(*c) as u32)?; - } - } - Ok(out) -} - -/// `P1`..`P7`, `PF` or `Pf` followed by white space. -pub fn looks_like_pnm(b: &[u8]) -> bool { - matches!(b, [b'P', b'1'..=b'7' | b'F' | b'f', s, ..] if s.is_ascii_whitespace()) -} - pub fn decode(bytes: &[u8], limits: &Limits) -> Result { - if bytes.len() < 2 || bytes[0] != b'P' { - return Err(err("not a Netpbm file")); - } - let kind = bytes[1]; - let mut r = Reader { b: bytes, pos: 2 }; - match kind { - b'1' | b'4' => { - let (w, h) = (r.uint()?, r.uint()?); - limits.check(w, h, ChannelLayout::Gray, SampleType::U8)?; - let n = sample_count(w, h, 1)?; - let mut px; - if kind == b'4' { - r.single_ws()?; - let row = (w as usize).div_ceil(8); - let data = r.rest(); - if row.checked_mul(h as usize).is_none_or(|len| data.len() < len) { - return Err(err("truncated raster")); - } - px = Vec::with_capacity(n); - for y in 0..h as usize { - for x in 0..w as usize { - let bit = (data[y * row + x / 8] >> (7 - x % 8)) & 1; - px.push(if bit == 1 { 0 } else { 255 }); - } - } - } else { - // Each ASCII pixel takes at least a byte, so the file bounds the reservation. - px = Vec::with_capacity(n.min(r.rest().len())); - while px.len() < n { - r.skip_ws_and_comments(); - match r.b.get(r.pos) { - Some(b'0') => px.push(255), - Some(b'1') => px.push(0), - Some(_) => return Err(err("invalid PBM digit")), - None => return Err(err("truncated raster")), - } - r.pos += 1; - } - } - Image::from_u8(w, h, ChannelLayout::Gray, px) - } - b'2' | b'3' | b'5' | b'6' => { - let (w, h, maxval) = (r.uint()?, r.uint()?, r.uint()?); - let layout = if matches!(kind, b'2' | b'5') { ChannelLayout::Gray } else { ChannelLayout::Rgb }; - limits.check(w, h, layout, sample_for_maxval(maxval)?)?; - let n = sample_count(w, h, layout.channels())?; - let samples = if matches!(kind, b'5' | b'6') { - r.single_ws()?; - read_binary(r.rest(), n, maxval)? - } else { - // Each ASCII sample is a digit and a separator (the last may lack one), so the - // file, not the header, bounds the reservation. - let mut v = Samples::with_capacity(maxval, n.min(r.rest().len() / 2 + 1))?; - for _ in 0..n { - v.push(r.uint()?)?; - } - v - }; - samples.into_image(w, h, layout) - } - b'7' => decode_pam(&mut r, limits), - b'F' | b'f' => { - let layout = if kind == b'F' { ChannelLayout::Rgb } else { ChannelLayout::Gray }; - let (w, h) = (r.uint()?, r.uint()?); - let scale: f32 = std::str::from_utf8(r.token()?) - .ok() - .and_then(|s| s.parse().ok()) - .filter(|s: &f32| s.is_finite() && *s != 0.0) - .ok_or_else(|| err("invalid PFM scale"))?; - r.single_ws()?; - limits.check(w, h, layout, SampleType::F32)?; - let nc = layout.channels(); - let row = w as usize * nc; - let data = r.rest(); - if sample_count(w, h, nc)?.checked_mul(4).is_none_or(|len| data.len() < len) { - return Err(err("truncated raster")); - } - let le = scale < 0.0; - // Write native-endian bytes straight into the output buffer: a separate `Vec` - // would double the budgeted allocation. - let row_bytes = row * 4; - let mut out = vec![0u8; row_bytes * h as usize]; - for (y, src) in data.chunks_exact(row_bytes).take(h as usize).enumerate() { - // PFM rows are stored bottom-to-top. - let dst_start = (h as usize - 1 - y) * row_bytes; - let Some(dst) = out.get_mut(dst_start..dst_start + row_bytes) else { - return Err(err("truncated raster")); - }; - for (d, s) in dst.as_chunks_mut::<4>().0.iter_mut().zip(src.as_chunks::<4>().0) { - let v = if le { f32::from_le_bytes(*s) } else { f32::from_be_bytes(*s) }; - *d = v.to_ne_bytes(); - } - } - Image::from_raw(w, h, layout, SampleType::F32, out) - } - _ => Err(err("unknown Netpbm subtype")), - } + bridge::format(Format::Pnm, bytes, limits) } - -fn decode_pam(r: &mut Reader<'_>, limits: &Limits) -> Result { - let (mut w, mut h, mut depth, mut maxval) = (None, None, None, None); - let mut tupltype = String::new(); - loop { - let tok = r.token()?; - match tok { - b"WIDTH" => w = Some(r.uint()?), - b"HEIGHT" => h = Some(r.uint()?), - b"DEPTH" => depth = Some(r.uint()?), - b"MAXVAL" => maxval = Some(r.uint()?), - b"TUPLTYPE" => { - let t = r.token()?; - if !tupltype.is_empty() { - tupltype.push(' '); - } - tupltype.push_str(std::str::from_utf8(t).map_err(|_| err("bad TUPLTYPE"))?); - } - b"ENDHDR" => break, - _ => return Err(err("unknown PAM header field")), - } - } - r.pam_endhdr()?; - let (w, h, depth, maxval) = match (w, h, depth, maxval) { - (Some(a), Some(b), Some(c), Some(d)) => (a, b, c, d), - _ => return Err(err("incomplete PAM header")), - }; - let layout = match (tupltype.as_str(), depth) { - ("BLACKANDWHITE" | "GRAYSCALE", 1) | ("", 1) => ChannelLayout::Gray, - ("BLACKANDWHITE_ALPHA" | "GRAYSCALE_ALPHA", 2) | ("", 2) => ChannelLayout::GrayA, - ("RGB", 3) | ("", 3) => ChannelLayout::Rgb, - ("RGB_ALPHA", 4) | ("", 4) => ChannelLayout::Rgba, - ("CMYK", 4) => ChannelLayout::Cmyk, - ("CMYK_ALPHA", 5) => ChannelLayout::CmykA, - (t, d) => { - return Err(CodecError::unsupported(F, format!("PAM tuple type {t:?} with depth {d}"))); - } - }; - limits.check(w, h, layout, sample_for_maxval(maxval)?)?; - let n = sample_count(w, h, layout.channels())?; - read_binary(r.rest(), n, maxval)?.into_image(w, h, layout) -} - -/// Writes `img` as PGM/PPM (gray, RGB), PAM (alpha, CMYK) or PFM (float gray, RGB). -pub fn encode(img: &Image) -> Result, CodecError> { - let (w, h) = (img.width(), img.height()); - let layout = img.layout(); - let mut out = Vec::new(); - match img.sample_type() { - SampleType::F32 => { - let tag = match layout { - ChannelLayout::Gray => "Pf", - ChannelLayout::Rgb => "PF", - l => return Err(CodecError::encode(F, format!("PFM cannot store {l:?}"))), - }; - out.extend_from_slice(format!("{tag}\n{w} {h}\n-1.0\n").as_bytes()); - let s = img.to_f32_samples().unwrap_or_default(); - let row = w as usize * layout.channels(); - for y in (0..h as usize).rev() { - for v in &s[y * row..(y + 1) * row] { - out.extend_from_slice(&v.to_le_bytes()); - } - } - } - sample @ (SampleType::U8 | SampleType::U16) => { - let maxval = if sample == SampleType::U8 { 255 } else { 65535 }; - match layout { - ChannelLayout::Gray => out.extend_from_slice(format!("P5\n{w} {h}\n{maxval}\n").as_bytes()), - ChannelLayout::Rgb => out.extend_from_slice(format!("P6\n{w} {h}\n{maxval}\n").as_bytes()), - l => { - let t = match l { - ChannelLayout::GrayA => "GRAYSCALE_ALPHA", - ChannelLayout::Rgba => "RGB_ALPHA", - ChannelLayout::Cmyk => "CMYK", - _ => "CMYK_ALPHA", - }; - out.extend_from_slice( - format!("P7\nWIDTH {w}\nHEIGHT {h}\nDEPTH {}\nMAXVAL {maxval}\nTUPLTYPE {t}\nENDHDR\n", l.channels()).as_bytes(), - ); - } - } - if sample == SampleType::U8 { - out.extend_from_slice(img.data()); - } else { - for v in img.to_u16_samples().unwrap_or_default() { - out.extend_from_slice(&v.to_be_bytes()); - } - } - } - } - Ok(out) +pub fn encode(image: &Image) -> Result, CodecError> { + bridge::encode(image, Format::Pnm, &Default::default()) } diff --git a/crates/raster-codecs/src/xpm.rs b/crates/raster-codecs/src/xpm.rs index 05d93cd75..2d49a2d63 100644 --- a/crates/raster-codecs/src/xpm.rs +++ b/crates/raster-codecs/src/xpm.rs @@ -1,243 +1,13 @@ -//! X PixMap reader for XPM2 plain-text icons and XPM3 C-source icon arrays. -//! Supports transparency (None), RGB hex at 4/8/16 bits per channel and -//! common X11 colour names. Does not execute C code. +//! XPM compatibility facade; implementation and color-name data live in craft-image-codecs. +use crate::{CodecError, Image, Limits, bridge}; +use craft_image_codecs::Format; -use std::collections::HashMap; - -mod x11_colors; - -use crate::{ChannelLayout, CodecError, Image, Limits, SampleType}; - -const MAX_SOURCE: usize = 256 << 20; -const MAX_COLOURS: usize = 65_536; -const MAX_CPP: usize = 8; - -fn bad(msg: impl std::fmt::Display) -> CodecError { - CodecError::malformed("XPM", msg) -} - -/// Extract C string literals without treating quoted text or // in a string as a comment. -/// XPM uses one literal for the header, each palette entry and each row. -fn c_strings(bytes: &[u8]) -> Result>, CodecError> { - let mut lines = Vec::new(); - let mut i = 0usize; - while i < bytes.len() { - if bytes[i..].starts_with(b"/*") { - let tail = &bytes[i + 2..]; - let Some(end) = tail.windows(2).position(|p| p == b"*/") else { return Err(bad("unterminated C comment")) }; - i += end + 4; - } else if bytes[i..].starts_with(b"//") { - i += bytes[i..].iter().position(|&c| c == b'\n').unwrap_or(bytes.len() - i); - } else if bytes[i] == b'"' { - i += 1; - let mut s = Vec::new(); - let mut closed = false; - while i < bytes.len() { - let c = bytes[i]; - i += 1; - match c { - b'"' => { - closed = true; - break; - } - b'\\' => { - let next = *bytes.get(i).ok_or_else(|| bad("truncated C escape"))?; - i += 1; - s.push(match next { - b'n' => b'\n', - b'r' => b'\r', - b't' => b'\t', - b'0' => 0, - _ => next, // quoted backslash or quote - }); - } - _ => s.push(c), - } - } - if !closed { - return Err(bad("unterminated XPM string")); - } - lines.push(s); - } else { - i += 1; - } - } - Ok(lines) -} - -/// Extract XPM2's plain-text lines without trimming pixel rows: leading and -/// trailing spaces can be palette keys, and physical CRLF is not image data. -fn xpm2_lines(bytes: &[u8]) -> Result>, CodecError> { - let mut input = bytes.split(|&b| b == b'\n'); - let header = input.next().ok_or_else(|| bad("missing XPM2 marker"))?; - if header.strip_suffix(b"\r").unwrap_or(header) != b"! XPM2" { - return Err(bad("not an XPM2 image")); - } - let mut lines = Vec::new(); - for raw in input { - let line = raw.strip_suffix(b"\r").unwrap_or(raw); - // The header may be preceded by comments and blank lines. After the - // header, a leading '!' is allowed as a palette key or pixel value. - if lines.is_empty() && (line.is_empty() || line.starts_with(b"!")) { - continue; - } - lines.push(line.to_vec()); - } - Ok(lines) -} - -fn channel(hex: &str) -> Option { - let v = u32::from_str_radix(hex, 16).ok()?; - let max = (1u32 << (hex.len() * 4)) - 1; - Some(((v * 255 + max / 2) / max) as u8) -} - -fn named(name: &str) -> Option<[u8; 3]> { - let s = name.to_ascii_lowercase().replace([' ', '-', '_'], ""); - // XPM imports X11 names, whose RGB values differ from CSS for names - // such as gray, maroon and purple. Keep these exact rather than guessing. - if let Ok(index) = x11_colors::X11_COLORS.binary_search_by_key(&s.as_str(), |(name, _)| *name) { - return Some(x11_colors::X11_COLORS[index].1); - } - // Retain useful web color aliases not present in the X11 database. - Some(match s.as_str() { - "silver" => [192, 192, 192], - "olive" => [128, 128, 0], - "teal" => [0, 128, 128], - "fuchsia" => [255, 0, 255], - "navyblue" => [0, 0, 128], - _ => return None, - }) -} - -/// Look up an exact normalized X11 color name as RGB channels in `0..=1`. -/// -/// Names must contain 1–64 bytes of lowercase ASCII letters or digits. Unknown names, -/// unnormalized names and XPM-only aliases return `None`; this does not parse hex colors -/// or transparency. Values are the stored X11 bytes divided by 255, without CSS overrides -/// or color-space conversion. XPM decoding retains its own normalization and alias policy. pub fn x11_color_rgb(name: &str) -> Option<[f32; 3]> { - // Bound work on untrusted names before searching the shared table, without allocating. - if name.is_empty() || name.len() > 64 || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit()) { - return None; - } - let index = x11_colors::X11_COLORS.binary_search_by_key(&name, |(entry, _)| *entry).ok()?; - let (_, rgb) = x11_colors::X11_COLORS.get(index)?; - Some(rgb.map(|channel| f32::from(channel) / 255.0)) + craft_image_codecs::xpm_named_color_rgb(name) } - -fn color(name: &str) -> Option<[u8; 4]> { - if name.eq_ignore_ascii_case("none") { - return Some([0, 0, 0, 0]); - } - let rgb = if let Some(hex) = name.strip_prefix('#') { - // UTF-8 byte lengths can match a supported hex width while slicing would panic. - // `from_str_radix` would also take a leading `+`: accept hex digits only. - if !hex.bytes().all(|b| b.is_ascii_hexdigit()) { - return None; - } - let digits = match hex.len() { - 3 => 1, - 6 => 2, - 12 => 4, - _ => return None, - }; - let rgb = (0..3).map(|i| channel(&hex[i * digits..(i + 1) * digits])).collect::>>()?; - [rgb[0], rgb[1], rgb[2]] - } else { - named(name)? - }; - Some([rgb[0], rgb[1], rgb[2], 255]) +pub fn looks_like_xpm(bytes: &[u8]) -> bool { + craft_image_codecs::detect(bytes) == Some(Format::Xpm) } - -/// The XPM3 comment or the XPM2 first line. -pub fn looks_like_xpm(b: &[u8]) -> bool { - b.starts_with(b"/* XPM */") || b.starts_with(b"! XPM2\n") || b.starts_with(b"! XPM2\r\n") -} - pub fn decode(bytes: &[u8], limits: &Limits) -> Result { - if bytes.len() > MAX_SOURCE { - return Err(CodecError::LimitExceeded("XPM source exceeds 256 MiB".into())); - } - let lines = if bytes.starts_with(b"/* XPM */") { - c_strings(bytes)? - } else if bytes.starts_with(b"! XPM2\n") || bytes.starts_with(b"! XPM2\r\n") { - xpm2_lines(bytes)? - } else { - return Err(bad("not an XPM2/XPM3 image")); - }; - decode_lines(&lines, limits) -} - -fn decode_lines(lines: &[Vec], limits: &Limits) -> Result { - let header = lines.first().ok_or_else(|| bad("missing image header"))?; - let parts = std::str::from_utf8(header).map_err(|_| bad("non-UTF-8 XPM header"))?; - let mut parts = parts.split_ascii_whitespace(); - let number = |key: &str, s: Option<&str>| -> Result { - s.ok_or_else(|| bad(format!("missing {key}")))?.parse::().map_err(|_| bad(format!("invalid {key}"))) - }; - let width = number("width", parts.next())?; - let height = number("height", parts.next())?; - let colours = number("palette size", parts.next())?; - let cpp = number("chars per pixel", parts.next())?; - if width == 0 || height == 0 || colours == 0 || colours > MAX_COLOURS || !(1..=MAX_CPP).contains(&cpp) { - return Err(bad("invalid XPM dimensions, palette size or chars per pixel")); - } - let (w, h) = (u32::try_from(width).map_err(|_| bad("width overflow"))?, u32::try_from(height).map_err(|_| bad("height overflow"))?); - limits.check(w, h, ChannelLayout::Rgba, SampleType::U8)?; - let required = 1usize.checked_add(colours).and_then(|n| n.checked_add(height)).ok_or_else(|| bad("row count overflow"))?; - if lines.len() < required { - return Err(bad("truncated palette or image rows")); - } - let mut palette = HashMap::, [u8; 4]>::with_capacity(colours); - for line in &lines[1..1 + colours] { - if line.len() < cpp { - return Err(bad("truncated colour key")); - } - let fields = std::str::from_utf8(&line[cpp..]).map_err(|_| bad("invalid colour name"))?; - // XPM priority is colour, gray, four-level gray, then monochrome, - // regardless of the order of keys in the palette entry. - let terms: Vec<&str> = fields.split_ascii_whitespace().collect(); - let mut picked: Option<(u8, String)> = None; - let mut i = 0; - while i < terms.len() { - let priority = match terms[i] { - "c" => 4, - "g" => 3, - "g4" => 2, - "m" => 1, - _ => 0, // Symbolic names ('s') are not colour values. - }; - let start = i + 1; - i = start; - while i < terms.len() && !matches!(terms[i], "c" | "g" | "g4" | "m" | "s") { - i += 1; - } - if priority > 0 && i > start && picked.as_ref().is_none_or(|(current, _)| priority > *current) { - picked = Some((priority, terms[start..i].join(" "))); - } - } - let value = picked.as_ref().and_then(|(_, name)| color(name)).ok_or_else(|| bad("unsupported or missing XPM colour"))?; - if palette.insert(line[..cpp].to_vec(), value).is_some() { - return Err(bad("duplicate XPM palette key")); - } - } - let row_len = width.checked_mul(cpp).ok_or_else(|| bad("row byte count overflow"))?; - // Validate all source rows before reserving the decoded RGBA image. A malformed - // short row must not be able to make a large header-sized allocation first. - if lines[1 + colours..required].iter().any(|line| line.len() != row_len) { - return Err(bad("XPM row has the wrong width")); - } - let total = width.checked_mul(height).and_then(|n| n.checked_mul(4)).ok_or_else(|| bad("pixel size overflow"))?; - let mut pixels = Vec::new(); - pixels.try_reserve_exact(total).map_err(|_| bad("XPM pixel buffer is too large"))?; - pixels.resize(total, 0); - for (y, line) in lines[1 + colours..required].iter().enumerate() { - for x in 0..width { - let key = &line[x * cpp..(x + 1) * cpp]; - let rgba = palette.get(key).ok_or_else(|| bad("unknown palette key"))?; - pixels[(y * width + x) * 4..(y * width + x + 1) * 4].copy_from_slice(rgba); - } - } - Image::from_u8(w, h, ChannelLayout::Rgba, pixels) + bridge::format(Format::Xpm, bytes, limits) } diff --git a/crates/raster-codecs/src/xpm/x11_colors.rs b/crates/raster-codecs/src/xpm/x11_colors.rs deleted file mode 100644 index 31986999a..000000000 --- a/crates/raster-codecs/src/xpm/x11_colors.rs +++ /dev/null @@ -1,666 +0,0 @@ -//! X11 color-name database (X.Org rgb.txt) normalized to lowercase without -//! spaces, underscores, or hyphens. RGB values are *X11* values, not CSS values. -//! Source: -//! License: MIT (X.Org Foundation), see the upstream app/rgb COPYING file. -//! Aliases differing only in case or spaces have been collapsed. -//! Keep sorted: decoder uses binary search without allocation per candidate. - -pub(super) const X11_COLORS: &[(&str, [u8; 3])] = &[ - ("aliceblue", [240, 248, 255]), - ("antiquewhite", [250, 235, 215]), - ("antiquewhite1", [255, 239, 219]), - ("antiquewhite2", [238, 223, 204]), - ("antiquewhite3", [205, 192, 176]), - ("antiquewhite4", [139, 131, 120]), - ("aquamarine", [127, 255, 212]), - ("aquamarine1", [127, 255, 212]), - ("aquamarine2", [118, 238, 198]), - ("aquamarine3", [102, 205, 170]), - ("aquamarine4", [69, 139, 116]), - ("azure", [240, 255, 255]), - ("azure1", [240, 255, 255]), - ("azure2", [224, 238, 238]), - ("azure3", [193, 205, 205]), - ("azure4", [131, 139, 139]), - ("beige", [245, 245, 220]), - ("bisque", [255, 228, 196]), - ("bisque1", [255, 228, 196]), - ("bisque2", [238, 213, 183]), - ("bisque3", [205, 183, 158]), - ("bisque4", [139, 125, 107]), - ("black", [0, 0, 0]), - ("blanchedalmond", [255, 235, 205]), - ("blue", [0, 0, 255]), - ("blue1", [0, 0, 255]), - ("blue2", [0, 0, 238]), - ("blue3", [0, 0, 205]), - ("blue4", [0, 0, 139]), - ("blueviolet", [138, 43, 226]), - ("brown", [165, 42, 42]), - ("brown1", [255, 64, 64]), - ("brown2", [238, 59, 59]), - ("brown3", [205, 51, 51]), - ("brown4", [139, 35, 35]), - ("burlywood", [222, 184, 135]), - ("burlywood1", [255, 211, 155]), - ("burlywood2", [238, 197, 145]), - ("burlywood3", [205, 170, 125]), - ("burlywood4", [139, 115, 85]), - ("cadetblue", [95, 158, 160]), - ("cadetblue1", [152, 245, 255]), - ("cadetblue2", [142, 229, 238]), - ("cadetblue3", [122, 197, 205]), - ("cadetblue4", [83, 134, 139]), - ("chartreuse", [127, 255, 0]), - ("chartreuse1", [127, 255, 0]), - ("chartreuse2", [118, 238, 0]), - ("chartreuse3", [102, 205, 0]), - ("chartreuse4", [69, 139, 0]), - ("chocolate", [210, 105, 30]), - ("chocolate1", [255, 127, 36]), - ("chocolate2", [238, 118, 33]), - ("chocolate3", [205, 102, 29]), - ("chocolate4", [139, 69, 19]), - ("coral", [255, 127, 80]), - ("coral1", [255, 114, 86]), - ("coral2", [238, 106, 80]), - ("coral3", [205, 91, 69]), - ("coral4", [139, 62, 47]), - ("cornflowerblue", [100, 149, 237]), - ("cornsilk", [255, 248, 220]), - ("cornsilk1", [255, 248, 220]), - ("cornsilk2", [238, 232, 205]), - ("cornsilk3", [205, 200, 177]), - ("cornsilk4", [139, 136, 120]), - ("cyan", [0, 255, 255]), - ("cyan1", [0, 255, 255]), - ("cyan2", [0, 238, 238]), - ("cyan3", [0, 205, 205]), - ("cyan4", [0, 139, 139]), - ("darkblue", [0, 0, 139]), - ("darkcyan", [0, 139, 139]), - ("darkgoldenrod", [184, 134, 11]), - ("darkgoldenrod1", [255, 185, 15]), - ("darkgoldenrod2", [238, 173, 14]), - ("darkgoldenrod3", [205, 149, 12]), - ("darkgoldenrod4", [139, 101, 8]), - ("darkgray", [169, 169, 169]), - ("darkgreen", [0, 100, 0]), - ("darkgrey", [169, 169, 169]), - ("darkkhaki", [189, 183, 107]), - ("darkmagenta", [139, 0, 139]), - ("darkolivegreen", [85, 107, 47]), - ("darkolivegreen1", [202, 255, 112]), - ("darkolivegreen2", [188, 238, 104]), - ("darkolivegreen3", [162, 205, 90]), - ("darkolivegreen4", [110, 139, 61]), - ("darkorange", [255, 140, 0]), - ("darkorange1", [255, 127, 0]), - ("darkorange2", [238, 118, 0]), - ("darkorange3", [205, 102, 0]), - ("darkorange4", [139, 69, 0]), - ("darkorchid", [153, 50, 204]), - ("darkorchid1", [191, 62, 255]), - ("darkorchid2", [178, 58, 238]), - ("darkorchid3", [154, 50, 205]), - ("darkorchid4", [104, 34, 139]), - ("darkred", [139, 0, 0]), - ("darksalmon", [233, 150, 122]), - ("darkseagreen", [143, 188, 143]), - ("darkseagreen1", [193, 255, 193]), - ("darkseagreen2", [180, 238, 180]), - ("darkseagreen3", [155, 205, 155]), - ("darkseagreen4", [105, 139, 105]), - ("darkslateblue", [72, 61, 139]), - ("darkslategray", [47, 79, 79]), - ("darkslategray1", [151, 255, 255]), - ("darkslategray2", [141, 238, 238]), - ("darkslategray3", [121, 205, 205]), - ("darkslategray4", [82, 139, 139]), - ("darkslategrey", [47, 79, 79]), - ("darkturquoise", [0, 206, 209]), - ("darkviolet", [148, 0, 211]), - ("deeppink", [255, 20, 147]), - ("deeppink1", [255, 20, 147]), - ("deeppink2", [238, 18, 137]), - ("deeppink3", [205, 16, 118]), - ("deeppink4", [139, 10, 80]), - ("deepskyblue", [0, 191, 255]), - ("deepskyblue1", [0, 191, 255]), - ("deepskyblue2", [0, 178, 238]), - ("deepskyblue3", [0, 154, 205]), - ("deepskyblue4", [0, 104, 139]), - ("dimgray", [105, 105, 105]), - ("dimgrey", [105, 105, 105]), - ("dodgerblue", [30, 144, 255]), - ("dodgerblue1", [30, 144, 255]), - ("dodgerblue2", [28, 134, 238]), - ("dodgerblue3", [24, 116, 205]), - ("dodgerblue4", [16, 78, 139]), - ("firebrick", [178, 34, 34]), - ("firebrick1", [255, 48, 48]), - ("firebrick2", [238, 44, 44]), - ("firebrick3", [205, 38, 38]), - ("firebrick4", [139, 26, 26]), - ("floralwhite", [255, 250, 240]), - ("forestgreen", [34, 139, 34]), - ("gainsboro", [220, 220, 220]), - ("ghostwhite", [248, 248, 255]), - ("gold", [255, 215, 0]), - ("gold1", [255, 215, 0]), - ("gold2", [238, 201, 0]), - ("gold3", [205, 173, 0]), - ("gold4", [139, 117, 0]), - ("goldenrod", [218, 165, 32]), - ("goldenrod1", [255, 193, 37]), - ("goldenrod2", [238, 180, 34]), - ("goldenrod3", [205, 155, 29]), - ("goldenrod4", [139, 105, 20]), - ("gray", [190, 190, 190]), - ("gray0", [0, 0, 0]), - ("gray1", [3, 3, 3]), - ("gray10", [26, 26, 26]), - ("gray100", [255, 255, 255]), - ("gray11", [28, 28, 28]), - ("gray12", [31, 31, 31]), - ("gray13", [33, 33, 33]), - ("gray14", [36, 36, 36]), - ("gray15", [38, 38, 38]), - ("gray16", [41, 41, 41]), - ("gray17", [43, 43, 43]), - ("gray18", [46, 46, 46]), - ("gray19", [48, 48, 48]), - ("gray2", [5, 5, 5]), - ("gray20", [51, 51, 51]), - ("gray21", [54, 54, 54]), - ("gray22", [56, 56, 56]), - ("gray23", [59, 59, 59]), - ("gray24", [61, 61, 61]), - ("gray25", [64, 64, 64]), - ("gray26", [66, 66, 66]), - ("gray27", [69, 69, 69]), - ("gray28", [71, 71, 71]), - ("gray29", [74, 74, 74]), - ("gray3", [8, 8, 8]), - ("gray30", [77, 77, 77]), - ("gray31", [79, 79, 79]), - ("gray32", [82, 82, 82]), - ("gray33", [84, 84, 84]), - ("gray34", [87, 87, 87]), - ("gray35", [89, 89, 89]), - ("gray36", [92, 92, 92]), - ("gray37", [94, 94, 94]), - ("gray38", [97, 97, 97]), - ("gray39", [99, 99, 99]), - ("gray4", [10, 10, 10]), - ("gray40", [102, 102, 102]), - ("gray41", [105, 105, 105]), - ("gray42", [107, 107, 107]), - ("gray43", [110, 110, 110]), - ("gray44", [112, 112, 112]), - ("gray45", [115, 115, 115]), - ("gray46", [117, 117, 117]), - ("gray47", [120, 120, 120]), - ("gray48", [122, 122, 122]), - ("gray49", [125, 125, 125]), - ("gray5", [13, 13, 13]), - ("gray50", [127, 127, 127]), - ("gray51", [130, 130, 130]), - ("gray52", [133, 133, 133]), - ("gray53", [135, 135, 135]), - ("gray54", [138, 138, 138]), - ("gray55", [140, 140, 140]), - ("gray56", [143, 143, 143]), - ("gray57", [145, 145, 145]), - ("gray58", [148, 148, 148]), - ("gray59", [150, 150, 150]), - ("gray6", [15, 15, 15]), - ("gray60", [153, 153, 153]), - ("gray61", [156, 156, 156]), - ("gray62", [158, 158, 158]), - ("gray63", [161, 161, 161]), - ("gray64", [163, 163, 163]), - ("gray65", [166, 166, 166]), - ("gray66", [168, 168, 168]), - ("gray67", [171, 171, 171]), - ("gray68", [173, 173, 173]), - ("gray69", [176, 176, 176]), - ("gray7", [18, 18, 18]), - ("gray70", [179, 179, 179]), - ("gray71", [181, 181, 181]), - ("gray72", [184, 184, 184]), - ("gray73", [186, 186, 186]), - ("gray74", [189, 189, 189]), - ("gray75", [191, 191, 191]), - ("gray76", [194, 194, 194]), - ("gray77", [196, 196, 196]), - ("gray78", [199, 199, 199]), - ("gray79", [201, 201, 201]), - ("gray8", [20, 20, 20]), - ("gray80", [204, 204, 204]), - ("gray81", [207, 207, 207]), - ("gray82", [209, 209, 209]), - ("gray83", [212, 212, 212]), - ("gray84", [214, 214, 214]), - ("gray85", [217, 217, 217]), - ("gray86", [219, 219, 219]), - ("gray87", [222, 222, 222]), - ("gray88", [224, 224, 224]), - ("gray89", [227, 227, 227]), - ("gray9", [23, 23, 23]), - ("gray90", [229, 229, 229]), - ("gray91", [232, 232, 232]), - ("gray92", [235, 235, 235]), - ("gray93", [237, 237, 237]), - ("gray94", [240, 240, 240]), - ("gray95", [242, 242, 242]), - ("gray96", [245, 245, 245]), - ("gray97", [247, 247, 247]), - ("gray98", [250, 250, 250]), - ("gray99", [252, 252, 252]), - ("green", [0, 255, 0]), - ("green1", [0, 255, 0]), - ("green2", [0, 238, 0]), - ("green3", [0, 205, 0]), - ("green4", [0, 139, 0]), - ("greenyellow", [173, 255, 47]), - ("grey", [190, 190, 190]), - ("grey0", [0, 0, 0]), - ("grey1", [3, 3, 3]), - ("grey10", [26, 26, 26]), - ("grey100", [255, 255, 255]), - ("grey11", [28, 28, 28]), - ("grey12", [31, 31, 31]), - ("grey13", [33, 33, 33]), - ("grey14", [36, 36, 36]), - ("grey15", [38, 38, 38]), - ("grey16", [41, 41, 41]), - ("grey17", [43, 43, 43]), - ("grey18", [46, 46, 46]), - ("grey19", [48, 48, 48]), - ("grey2", [5, 5, 5]), - ("grey20", [51, 51, 51]), - ("grey21", [54, 54, 54]), - ("grey22", [56, 56, 56]), - ("grey23", [59, 59, 59]), - ("grey24", [61, 61, 61]), - ("grey25", [64, 64, 64]), - ("grey26", [66, 66, 66]), - ("grey27", [69, 69, 69]), - ("grey28", [71, 71, 71]), - ("grey29", [74, 74, 74]), - ("grey3", [8, 8, 8]), - ("grey30", [77, 77, 77]), - ("grey31", [79, 79, 79]), - ("grey32", [82, 82, 82]), - ("grey33", [84, 84, 84]), - ("grey34", [87, 87, 87]), - ("grey35", [89, 89, 89]), - ("grey36", [92, 92, 92]), - ("grey37", [94, 94, 94]), - ("grey38", [97, 97, 97]), - ("grey39", [99, 99, 99]), - ("grey4", [10, 10, 10]), - ("grey40", [102, 102, 102]), - ("grey41", [105, 105, 105]), - ("grey42", [107, 107, 107]), - ("grey43", [110, 110, 110]), - ("grey44", [112, 112, 112]), - ("grey45", [115, 115, 115]), - ("grey46", [117, 117, 117]), - ("grey47", [120, 120, 120]), - ("grey48", [122, 122, 122]), - ("grey49", [125, 125, 125]), - ("grey5", [13, 13, 13]), - ("grey50", [127, 127, 127]), - ("grey51", [130, 130, 130]), - ("grey52", [133, 133, 133]), - ("grey53", [135, 135, 135]), - ("grey54", [138, 138, 138]), - ("grey55", [140, 140, 140]), - ("grey56", [143, 143, 143]), - ("grey57", [145, 145, 145]), - ("grey58", [148, 148, 148]), - ("grey59", [150, 150, 150]), - ("grey6", [15, 15, 15]), - ("grey60", [153, 153, 153]), - ("grey61", [156, 156, 156]), - ("grey62", [158, 158, 158]), - ("grey63", [161, 161, 161]), - ("grey64", [163, 163, 163]), - ("grey65", [166, 166, 166]), - ("grey66", [168, 168, 168]), - ("grey67", [171, 171, 171]), - ("grey68", [173, 173, 173]), - ("grey69", [176, 176, 176]), - ("grey7", [18, 18, 18]), - ("grey70", [179, 179, 179]), - ("grey71", [181, 181, 181]), - ("grey72", [184, 184, 184]), - ("grey73", [186, 186, 186]), - ("grey74", [189, 189, 189]), - ("grey75", [191, 191, 191]), - ("grey76", [194, 194, 194]), - ("grey77", [196, 196, 196]), - ("grey78", [199, 199, 199]), - ("grey79", [201, 201, 201]), - ("grey8", [20, 20, 20]), - ("grey80", [204, 204, 204]), - ("grey81", [207, 207, 207]), - ("grey82", [209, 209, 209]), - ("grey83", [212, 212, 212]), - ("grey84", [214, 214, 214]), - ("grey85", [217, 217, 217]), - ("grey86", [219, 219, 219]), - ("grey87", [222, 222, 222]), - ("grey88", [224, 224, 224]), - ("grey89", [227, 227, 227]), - ("grey9", [23, 23, 23]), - ("grey90", [229, 229, 229]), - ("grey91", [232, 232, 232]), - ("grey92", [235, 235, 235]), - ("grey93", [237, 237, 237]), - ("grey94", [240, 240, 240]), - ("grey95", [242, 242, 242]), - ("grey96", [245, 245, 245]), - ("grey97", [247, 247, 247]), - ("grey98", [250, 250, 250]), - ("grey99", [252, 252, 252]), - ("honeydew", [240, 255, 240]), - ("honeydew1", [240, 255, 240]), - ("honeydew2", [224, 238, 224]), - ("honeydew3", [193, 205, 193]), - ("honeydew4", [131, 139, 131]), - ("hotpink", [255, 105, 180]), - ("hotpink1", [255, 110, 180]), - ("hotpink2", [238, 106, 167]), - ("hotpink3", [205, 96, 144]), - ("hotpink4", [139, 58, 98]), - ("indianred", [205, 92, 92]), - ("indianred1", [255, 106, 106]), - ("indianred2", [238, 99, 99]), - ("indianred3", [205, 85, 85]), - ("indianred4", [139, 58, 58]), - ("ivory", [255, 255, 240]), - ("ivory1", [255, 255, 240]), - ("ivory2", [238, 238, 224]), - ("ivory3", [205, 205, 193]), - ("ivory4", [139, 139, 131]), - ("khaki", [240, 230, 140]), - ("khaki1", [255, 246, 143]), - ("khaki2", [238, 230, 133]), - ("khaki3", [205, 198, 115]), - ("khaki4", [139, 134, 78]), - ("lavender", [230, 230, 250]), - ("lavenderblush", [255, 240, 245]), - ("lavenderblush1", [255, 240, 245]), - ("lavenderblush2", [238, 224, 229]), - ("lavenderblush3", [205, 193, 197]), - ("lavenderblush4", [139, 131, 134]), - ("lawngreen", [124, 252, 0]), - ("lemonchiffon", [255, 250, 205]), - ("lemonchiffon1", [255, 250, 205]), - ("lemonchiffon2", [238, 233, 191]), - ("lemonchiffon3", [205, 201, 165]), - ("lemonchiffon4", [139, 137, 112]), - ("lightblue", [173, 216, 230]), - ("lightblue1", [191, 239, 255]), - ("lightblue2", [178, 223, 238]), - ("lightblue3", [154, 192, 205]), - ("lightblue4", [104, 131, 139]), - ("lightcoral", [240, 128, 128]), - ("lightcyan", [224, 255, 255]), - ("lightcyan1", [224, 255, 255]), - ("lightcyan2", [209, 238, 238]), - ("lightcyan3", [180, 205, 205]), - ("lightcyan4", [122, 139, 139]), - ("lightgoldenrod", [238, 221, 130]), - ("lightgoldenrod1", [255, 236, 139]), - ("lightgoldenrod2", [238, 220, 130]), - ("lightgoldenrod3", [205, 190, 112]), - ("lightgoldenrod4", [139, 129, 76]), - ("lightgoldenrodyellow", [250, 250, 210]), - ("lightgray", [211, 211, 211]), - ("lightgreen", [144, 238, 144]), - ("lightgrey", [211, 211, 211]), - ("lightpink", [255, 182, 193]), - ("lightpink1", [255, 174, 185]), - ("lightpink2", [238, 162, 173]), - ("lightpink3", [205, 140, 149]), - ("lightpink4", [139, 95, 101]), - ("lightsalmon", [255, 160, 122]), - ("lightsalmon1", [255, 160, 122]), - ("lightsalmon2", [238, 149, 114]), - ("lightsalmon3", [205, 129, 98]), - ("lightsalmon4", [139, 87, 66]), - ("lightseagreen", [32, 178, 170]), - ("lightskyblue", [135, 206, 250]), - ("lightskyblue1", [176, 226, 255]), - ("lightskyblue2", [164, 211, 238]), - ("lightskyblue3", [141, 182, 205]), - ("lightskyblue4", [96, 123, 139]), - ("lightslateblue", [132, 112, 255]), - ("lightslategray", [119, 136, 153]), - ("lightslategrey", [119, 136, 153]), - ("lightsteelblue", [176, 196, 222]), - ("lightsteelblue1", [202, 225, 255]), - ("lightsteelblue2", [188, 210, 238]), - ("lightsteelblue3", [162, 181, 205]), - ("lightsteelblue4", [110, 123, 139]), - ("lightyellow", [255, 255, 224]), - ("lightyellow1", [255, 255, 224]), - ("lightyellow2", [238, 238, 209]), - ("lightyellow3", [205, 205, 180]), - ("lightyellow4", [139, 139, 122]), - ("limegreen", [50, 205, 50]), - ("linen", [250, 240, 230]), - ("magenta", [255, 0, 255]), - ("magenta1", [255, 0, 255]), - ("magenta2", [238, 0, 238]), - ("magenta3", [205, 0, 205]), - ("magenta4", [139, 0, 139]), - ("maroon", [176, 48, 96]), - ("maroon1", [255, 52, 179]), - ("maroon2", [238, 48, 167]), - ("maroon3", [205, 41, 144]), - ("maroon4", [139, 28, 98]), - ("mediumaquamarine", [102, 205, 170]), - ("mediumblue", [0, 0, 205]), - ("mediumorchid", [186, 85, 211]), - ("mediumorchid1", [224, 102, 255]), - ("mediumorchid2", [209, 95, 238]), - ("mediumorchid3", [180, 82, 205]), - ("mediumorchid4", [122, 55, 139]), - ("mediumpurple", [147, 112, 219]), - ("mediumpurple1", [171, 130, 255]), - ("mediumpurple2", [159, 121, 238]), - ("mediumpurple3", [137, 104, 205]), - ("mediumpurple4", [93, 71, 139]), - ("mediumseagreen", [60, 179, 113]), - ("mediumslateblue", [123, 104, 238]), - ("mediumspringgreen", [0, 250, 154]), - ("mediumturquoise", [72, 209, 204]), - ("mediumvioletred", [199, 21, 133]), - ("midnightblue", [25, 25, 112]), - ("mintcream", [245, 255, 250]), - ("mistyrose", [255, 228, 225]), - ("mistyrose1", [255, 228, 225]), - ("mistyrose2", [238, 213, 210]), - ("mistyrose3", [205, 183, 181]), - ("mistyrose4", [139, 125, 123]), - ("moccasin", [255, 228, 181]), - ("navajowhite", [255, 222, 173]), - ("navajowhite1", [255, 222, 173]), - ("navajowhite2", [238, 207, 161]), - ("navajowhite3", [205, 179, 139]), - ("navajowhite4", [139, 121, 94]), - ("navy", [0, 0, 128]), - ("navyblue", [0, 0, 128]), - ("oldlace", [253, 245, 230]), - ("olivedrab", [107, 142, 35]), - ("olivedrab1", [192, 255, 62]), - ("olivedrab2", [179, 238, 58]), - ("olivedrab3", [154, 205, 50]), - ("olivedrab4", [105, 139, 34]), - ("orange", [255, 165, 0]), - ("orange1", [255, 165, 0]), - ("orange2", [238, 154, 0]), - ("orange3", [205, 133, 0]), - ("orange4", [139, 90, 0]), - ("orangered", [255, 69, 0]), - ("orangered1", [255, 69, 0]), - ("orangered2", [238, 64, 0]), - ("orangered3", [205, 55, 0]), - ("orangered4", [139, 37, 0]), - ("orchid", [218, 112, 214]), - ("orchid1", [255, 131, 250]), - ("orchid2", [238, 122, 233]), - ("orchid3", [205, 105, 201]), - ("orchid4", [139, 71, 137]), - ("palegoldenrod", [238, 232, 170]), - ("palegreen", [152, 251, 152]), - ("palegreen1", [154, 255, 154]), - ("palegreen2", [144, 238, 144]), - ("palegreen3", [124, 205, 124]), - ("palegreen4", [84, 139, 84]), - ("paleturquoise", [175, 238, 238]), - ("paleturquoise1", [187, 255, 255]), - ("paleturquoise2", [174, 238, 238]), - ("paleturquoise3", [150, 205, 205]), - ("paleturquoise4", [102, 139, 139]), - ("palevioletred", [219, 112, 147]), - ("palevioletred1", [255, 130, 171]), - ("palevioletred2", [238, 121, 159]), - ("palevioletred3", [205, 104, 137]), - ("palevioletred4", [139, 71, 93]), - ("papayawhip", [255, 239, 213]), - ("peachpuff", [255, 218, 185]), - ("peachpuff1", [255, 218, 185]), - ("peachpuff2", [238, 203, 173]), - ("peachpuff3", [205, 175, 149]), - ("peachpuff4", [139, 119, 101]), - ("peru", [205, 133, 63]), - ("pink", [255, 192, 203]), - ("pink1", [255, 181, 197]), - ("pink2", [238, 169, 184]), - ("pink3", [205, 145, 158]), - ("pink4", [139, 99, 108]), - ("plum", [221, 160, 221]), - ("plum1", [255, 187, 255]), - ("plum2", [238, 174, 238]), - ("plum3", [205, 150, 205]), - ("plum4", [139, 102, 139]), - ("powderblue", [176, 224, 230]), - ("purple", [160, 32, 240]), - ("purple1", [155, 48, 255]), - ("purple2", [145, 44, 238]), - ("purple3", [125, 38, 205]), - ("purple4", [85, 26, 139]), - ("red", [255, 0, 0]), - ("red1", [255, 0, 0]), - ("red2", [238, 0, 0]), - ("red3", [205, 0, 0]), - ("red4", [139, 0, 0]), - ("rosybrown", [188, 143, 143]), - ("rosybrown1", [255, 193, 193]), - ("rosybrown2", [238, 180, 180]), - ("rosybrown3", [205, 155, 155]), - ("rosybrown4", [139, 105, 105]), - ("royalblue", [65, 105, 225]), - ("royalblue1", [72, 118, 255]), - ("royalblue2", [67, 110, 238]), - ("royalblue3", [58, 95, 205]), - ("royalblue4", [39, 64, 139]), - ("saddlebrown", [139, 69, 19]), - ("salmon", [250, 128, 114]), - ("salmon1", [255, 140, 105]), - ("salmon2", [238, 130, 98]), - ("salmon3", [205, 112, 84]), - ("salmon4", [139, 76, 57]), - ("sandybrown", [244, 164, 96]), - ("seagreen", [46, 139, 87]), - ("seagreen1", [84, 255, 159]), - ("seagreen2", [78, 238, 148]), - ("seagreen3", [67, 205, 128]), - ("seagreen4", [46, 139, 87]), - ("seashell", [255, 245, 238]), - ("seashell1", [255, 245, 238]), - ("seashell2", [238, 229, 222]), - ("seashell3", [205, 197, 191]), - ("seashell4", [139, 134, 130]), - ("sienna", [160, 82, 45]), - ("sienna1", [255, 130, 71]), - ("sienna2", [238, 121, 66]), - ("sienna3", [205, 104, 57]), - ("sienna4", [139, 71, 38]), - ("skyblue", [135, 206, 235]), - ("skyblue1", [135, 206, 255]), - ("skyblue2", [126, 192, 238]), - ("skyblue3", [108, 166, 205]), - ("skyblue4", [74, 112, 139]), - ("slateblue", [106, 90, 205]), - ("slateblue1", [131, 111, 255]), - ("slateblue2", [122, 103, 238]), - ("slateblue3", [105, 89, 205]), - ("slateblue4", [71, 60, 139]), - ("slategray", [112, 128, 144]), - ("slategray1", [198, 226, 255]), - ("slategray2", [185, 211, 238]), - ("slategray3", [159, 182, 205]), - ("slategray4", [108, 123, 139]), - ("slategrey", [112, 128, 144]), - ("snow", [255, 250, 250]), - ("snow1", [255, 250, 250]), - ("snow2", [238, 233, 233]), - ("snow3", [205, 201, 201]), - ("snow4", [139, 137, 137]), - ("springgreen", [0, 255, 127]), - ("springgreen1", [0, 255, 127]), - ("springgreen2", [0, 238, 118]), - ("springgreen3", [0, 205, 102]), - ("springgreen4", [0, 139, 69]), - ("steelblue", [70, 130, 180]), - ("steelblue1", [99, 184, 255]), - ("steelblue2", [92, 172, 238]), - ("steelblue3", [79, 148, 205]), - ("steelblue4", [54, 100, 139]), - ("tan", [210, 180, 140]), - ("tan1", [255, 165, 79]), - ("tan2", [238, 154, 73]), - ("tan3", [205, 133, 63]), - ("tan4", [139, 90, 43]), - ("thistle", [216, 191, 216]), - ("thistle1", [255, 225, 255]), - ("thistle2", [238, 210, 238]), - ("thistle3", [205, 181, 205]), - ("thistle4", [139, 123, 139]), - ("tomato", [255, 99, 71]), - ("tomato1", [255, 99, 71]), - ("tomato2", [238, 92, 66]), - ("tomato3", [205, 79, 57]), - ("tomato4", [139, 54, 38]), - ("turquoise", [64, 224, 208]), - ("turquoise1", [0, 245, 255]), - ("turquoise2", [0, 229, 238]), - ("turquoise3", [0, 197, 205]), - ("turquoise4", [0, 134, 139]), - ("violet", [238, 130, 238]), - ("violetred", [208, 32, 144]), - ("violetred1", [255, 62, 150]), - ("violetred2", [238, 58, 140]), - ("violetred3", [205, 50, 120]), - ("violetred4", [139, 34, 82]), - ("wheat", [245, 222, 179]), - ("wheat1", [255, 231, 186]), - ("wheat2", [238, 216, 174]), - ("wheat3", [205, 186, 150]), - ("wheat4", [139, 126, 102]), - ("white", [255, 255, 255]), - ("whitesmoke", [245, 245, 245]), - ("yellow", [255, 255, 0]), - ("yellow1", [255, 255, 0]), - ("yellow2", [238, 238, 0]), - ("yellow3", [205, 205, 0]), - ("yellow4", [139, 139, 0]), - ("yellowgreen", [154, 205, 50]), -]; From 4e4e9338d9e6610775eb47e00b0139330da8780b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 06:45:26 -0500 Subject: [PATCH 02/98] Update lockfile after raster facade cleanup --- Cargo.lock | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a8abed4c..0bc90ee3d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2284,7 +2284,6 @@ dependencies = [ "base64 0.23.1", "craft-image-codecs", "image", - "rayon", "thiserror 2.0.21", ] @@ -4941,17 +4940,17 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" -[[package]] -name = "indd" -version = "0.1.0" -source = "git+https://github.com/storytold/indd-utils?rev=7f3060e5bce78d21ea70e0d5eacb96edce15dfbe#7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" - [[package]] name = "imgref" version = "1.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e44b0a4eaa4c82f441d50a963f2d5f05a787240aeee097597033e72accfd22f" +[[package]] +name = "indd" +version = "0.1.0" +source = "git+https://github.com/storytold/indd-utils?rev=7f3060e5bce78d21ea70e0d5eacb96edce15dfbe#7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" + [[package]] name = "indexmap" version = "2.14.2" From 1d52e89405ff9237c53f34584ff97fef8decc611 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:11:11 -0500 Subject: [PATCH 03/98] Share raster metadata parsing with craft-metadata --- Cargo.lock | 1 + crates/image-codecs/Cargo.toml | 1 + crates/image-codecs/src/codecs/tiff/bands.rs | 35 +- crates/image-codecs/src/exif_color.rs | 68 +-- crates/image-codecs/src/orientation.rs | 231 +------- crates/image-codecs/src/resolution.rs | 578 +------------------ 6 files changed, 38 insertions(+), 876 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0bc90ee3d..9702836da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1758,6 +1758,7 @@ version = "0.1.0" dependencies = [ "craft-heif", "craft-jpeg", + "craft-metadata", "exr", "flate2", "half", diff --git a/crates/image-codecs/Cargo.toml b/crates/image-codecs/Cargo.toml index 30f7f7046..b893c347a 100644 --- a/crates/image-codecs/Cargo.toml +++ b/crates/image-codecs/Cargo.toml @@ -30,6 +30,7 @@ zune-jpeg = "0.5" zune-core = "0.5" jpeg-encoder = "0.6" craft-jpeg = { workspace = true } +craft-metadata = { workspace = true } zune-jpegxl = { version = "=0.5.2", default-features = false, features = ["std"] } craft-heif = { workspace = true, optional = true } diff --git a/crates/image-codecs/src/codecs/tiff/bands.rs b/crates/image-codecs/src/codecs/tiff/bands.rs index 093f90efd..117c95ded 100644 --- a/crates/image-codecs/src/codecs/tiff/bands.rs +++ b/crates/image-codecs/src/codecs/tiff/bands.rs @@ -230,16 +230,7 @@ fn decode_band( } continue; } - if pref.len > lay.scratch_cap { - return Err(CodecError::LimitExceeded(format!("a {} byte strip or tile", pref.len))); - } - if scratch.len() < pref.len { - scratch - .try_reserve_exact(pref.len - scratch.len()) - .map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {} bytes for a strip or tile", pref.len)))?; - scratch.resize(pref.len, 0); - } - let buf = scratch.get_mut(..pref.len).ok_or_else(|| err("scratch buffer"))?; + let buf = chunk_scratch(lay, scratch, pref.len)?; match dec.read_chunk_bytes(chunk, buf) { Ok(()) => tally.decoded += 1, Err(e) if is_eof(&e) => { @@ -284,6 +275,19 @@ fn deposit_extras(lay: &Layout, src: &[u8], dst: &mut [u8], n: usize, x0: usize) } } +fn chunk_scratch<'a>(lay: &Layout, scratch: &'a mut Vec, len: usize) -> Result<&'a mut [u8], CodecError> { + if len > lay.scratch_cap { + return Err(CodecError::LimitExceeded(format!("a {} byte strip or tile", len))); + } + if scratch.len() < len { + scratch + .try_reserve_exact(len - scratch.len()) + .map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {} bytes for a strip or tile", len)))?; + scratch.resize(len, 0); + } + scratch.get_mut(..len).ok_or_else(|| err("scratch buffer")) +} + /// Planar files store each extra sample as its own plane. Read those planes into `extra`. fn read_planar_extras( lay: &Layout, @@ -310,16 +314,7 @@ fn read_planar_extras( .ok_or_else(|| err("chunk index out of range"))?; let pref = dec.image_chunk_buffer_layout(chunk).map_err(map_err)?; let stride = pref.row_stride.map(|s| s.get()).ok_or_else(|| err("empty chunk row"))?; - if pref.len > lay.scratch_cap { - return Err(CodecError::LimitExceeded(format!("a {} byte strip or tile", pref.len))); - } - if scratch.len() < pref.len { - scratch - .try_reserve_exact(pref.len - scratch.len()) - .map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {} bytes for a strip or tile", pref.len)))?; - scratch.resize(pref.len, 0); - } - let buf = scratch.get_mut(..pref.len).ok_or_else(|| err("scratch buffer"))?; + let buf = chunk_scratch(lay, scratch, pref.len)?; match dec.read_chunk_bytes(chunk, buf) { Ok(()) => tally.decoded += 1, Err(e) if is_eof(&e) => { diff --git a/crates/image-codecs/src/exif_color.rs b/crates/image-codecs/src/exif_color.rs index 4259bb15c..8a35907ff 100644 --- a/crates/image-codecs/src/exif_color.rs +++ b/crates/image-codecs/src/exif_color.rs @@ -1,70 +1,16 @@ -//! The EXIF ColorSpace tag (0xA001, in the Exif sub-IFD that IFD0's tag 0x8769 points to): -//! 1 = sRGB, 0xFFFF = uncalibrated (EXIF 2.3, §4.6.5). -//! -//! Cameras write it instead of an ICC profile. Photoshop opens a file with no embedded profile but -//! ColorSpace = sRGB as sRGB, unless Preferences › File Handling › Ignore EXIF Profile Tag is on. -//! Anything malformed, truncated or missing reads as no tag. +//! Shared EXIF ColorSpace tag parsing. -use crate::orientation::{Order, ifd0_entries, tiff_body}; - -/// IFD0's pointer to the Exif sub-IFD. -const TAG_EXIF_IFD: u16 = 0x8769; -/// The ColorSpace tag in the Exif sub-IFD. -const TAG_COLOR_SPACE: u16 = 0xA001; -const TYPE_SHORT: u16 = 3; -const TYPE_LONG: u16 = 4; -/// TIFF type 13: an IFD offset (some writers use it for sub-IFD pointers). -const TYPE_IFD: u16 = 13; -/// Bytes per classic TIFF IFD entry; EXIF is never BigTIFF. -const ENTRY: usize = 12; -/// No IFD can start inside the 8-byte TIFF header. -const HEADER: usize = 8; - -/// The ColorSpace value meaning sRGB. -pub const EXIF_COLOR_SPACE_SRGB: u16 = 1; - -/// The EXIF ColorSpace value of a TIFF-structured block (an EXIF payload with or without the -/// `Exif\0\0` prefix), or `None` when it has none or it is malformed. -pub fn exif_color_space(exif: &[u8]) -> Option { - let b = tiff_body(exif); - let (order, first, count, entry_bytes, big_tiff) = ifd0_entries(b)?; - if big_tiff { - return None; - } - let ptr = find(order, b, first, count, entry_bytes, TAG_EXIF_IFD)?; - if !matches!(order.u16(b, ptr.checked_add(2)?)?, TYPE_LONG | TYPE_IFD) || order.u32(b, ptr.checked_add(4)?)? != 1 { - return None; - } - let ifd = usize::try_from(order.u32(b, ptr.checked_add(8)?)?).ok()?; - if ifd < HEADER { - return None; - } - let n = usize::from(order.u16(b, ifd)?); - let first = ifd.checked_add(2)?; - // The whole table must be present. - if first.checked_add(n.checked_mul(ENTRY)?)? > b.len() { - return None; - } - let e = find(order, b, first, n, ENTRY, TAG_COLOR_SPACE)?; - if order.u32(b, e.checked_add(4)?)? != 1 { - return None; - } - match order.u16(b, e.checked_add(2)?)? { - TYPE_SHORT => order.u16(b, e.checked_add(8)?), - TYPE_LONG => u16::try_from(order.u32(b, e.checked_add(8)?)?).ok(), - _ => None, - } -} - -/// Offset of the first entry tagged `tag` in the IFD whose `count` entries start at `first`. -fn find(order: Order, b: &[u8], first: usize, count: usize, entry_bytes: usize, tag: u16) -> Option { - (0..count).filter_map(|i| first.checked_add(i.checked_mul(entry_bytes)?)).find(|&e| order.u16(b, e) == Some(tag)) -} +pub use craft_metadata::edit::{EXIF_COLOR_SPACE_SRGB, exif_color_space}; #[cfg(test)] mod tests { use super::*; + const TAG_EXIF_IFD: u16 = 0x8769; + const TAG_COLOR_SPACE: u16 = 0xA001; + const TYPE_SHORT: u16 = 3; + const TYPE_LONG: u16 = 4; + /// A little- or big-endian EXIF block: IFD0 holding only the Exif pointer, then an Exif IFD /// holding `entries` (tag, type, count, value). fn exif(big_endian: bool, entries: &[(u16, u16, u32, u32)]) -> Vec { diff --git a/crates/image-codecs/src/orientation.rs b/crates/image-codecs/src/orientation.rs index f77e6691f..2db0de75f 100644 --- a/crates/image-codecs/src/orientation.rs +++ b/crates/image-codecs/src/orientation.rs @@ -1,232 +1,15 @@ -//! EXIF / TIFF orientation (tag 274): reading it, applying it to the pixels, -//! and rewriting it to 1 ("top-left", upright) once the pixels are upright. -//! -//! Decoders apply the orientation by default (like Photoshop and every photo -//! viewer), so a document is always edited upright. Encoders then write -//! Orientation = 1, so a viewer never rotates the already-upright pixels a -//! second time. -//! -//! Orientation values (TIFF 6.0, EXIF 2.3): 1 = upright, 2 = mirrored, 3 = -//! rotated 180°, 4 = flipped vertically, 5 = transposed, 6 = needs a 90° -//! clockwise turn, 7 = transverse, 8 = needs a 90° counter-clockwise turn. -//! Anything malformed, truncated or out of range reads as 1: an IFD0 that -//! starts inside the 8-byte header or is cut off before its next-IFD pointer, -//! or an Orientation entry that is not exactly one SHORT or LONG (the spec says -//! SHORT, but some cameras and tools write LONG). When IFD0 holds -//! several Orientation entries the first one wins (like libexif). - -use std::borrow::Cow; +//! EXIF orientation transforms for decoded images. Byte-level EXIF/XMP inspection and edits are +//! shared with `craft-metadata`. use crate::error::CodecError; use crate::image::Image; +use std::borrow::Cow; -/// The TIFF/EXIF Orientation tag. -const TAG_ORIENTATION: u16 = 274; -/// Field types accepted for the Orientation tag: SHORT (the spec) and LONG -/// (written by some cameras and tools). -const TYPE_SHORT: u16 = 3; -const TYPE_LONG: u16 = 4; -/// Bytes per IFD entry. -const ENTRY: usize = 12; -/// The TIFF header is 8 bytes, so no IFD can start before it. -const HEADER: usize = 8; -/// BigTIFF uses a 16-byte header, 20-byte entries, and 8-byte offsets. -const BIG_ENTRY: usize = 20; -const BIG_HEADER: usize = 16; +pub use craft_metadata::edit::{exif_orientation, upright_exif, upright_xmp}; -/// Byte order of a TIFF structure. -#[derive(Clone, Copy)] -pub(crate) enum Order { - Little, - Big, -} - -impl Order { - pub(crate) fn u16(self, b: &[u8], at: usize) -> Option { - let s: [u8; 2] = b.get(at..at.checked_add(2)?)?.try_into().ok()?; - Some(match self { - Order::Little => u16::from_le_bytes(s), - Order::Big => u16::from_be_bytes(s), - }) - } - - pub(crate) fn u32(self, b: &[u8], at: usize) -> Option { - let s: [u8; 4] = b.get(at..at.checked_add(4)?)?.try_into().ok()?; - Some(match self { - Order::Little => u32::from_le_bytes(s), - Order::Big => u32::from_be_bytes(s), - }) - } - - pub(crate) fn u64(self, b: &[u8], at: usize) -> Option { - let s: [u8; 8] = b.get(at..at.checked_add(8)?)?.try_into().ok()?; - Some(match self { - Order::Little => u64::from_le_bytes(s), - Order::Big => u64::from_be_bytes(s), - }) - } -} - -/// The TIFF payload of an EXIF block (a JPEG-style `Exif\0\0` prefix is skipped). -pub(crate) fn tiff_body(b: &[u8]) -> &[u8] { - b.strip_prefix(b"Exif\0\0").unwrap_or(b) -} - -/// IFD0 of a TIFF structure: byte order, entry layout, and entry offsets. -/// -/// The full table and next-IFD pointer must be present, and IFD0 must start -/// after the corresponding TIFF or BigTIFF header. -pub(crate) fn ifd0_entries(b: &[u8]) -> Option<(Order, usize, usize, usize, bool)> { - let (order, big_tiff) = match b.get(0..4)? { - [b'I', b'I', 42, 0] => (Order::Little, false), - [b'M', b'M', 0, 42] => (Order::Big, false), - [b'I', b'I', 43, 0] => (Order::Little, true), - [b'M', b'M', 0, 43] => (Order::Big, true), - _ => return None, - }; - let (ifd, count_bytes, entry_bytes, next_ifd_bytes, header) = if big_tiff { - if order.u16(b, 4)? != 8 || order.u16(b, 6)? != 0 { - return None; - } - (usize::try_from(order.u64(b, 8)?).ok()?, 8usize, BIG_ENTRY, 8usize, BIG_HEADER) - } else { - (usize::try_from(order.u32(b, 4)?).ok()?, 2usize, ENTRY, 4usize, HEADER) - }; - if ifd < header { - return None; - } - let count = if big_tiff { usize::try_from(order.u64(b, ifd)?).ok()? } else { usize::from(order.u16(b, ifd)?) }; - let first = ifd.checked_add(count_bytes)?; - let next_ifd = first.checked_add(count.checked_mul(entry_bytes)?)?; - if next_ifd.checked_add(next_ifd_bytes)? > b.len() { - return None; - } - // In bounds: every entry ends at or before `next_ifd`. - Some((order, first, count, entry_bytes, big_tiff)) -} - -/// A well-formed Orientation entry's value: where it sits and its field type. -#[derive(Clone, Copy)] -struct Value { - at: usize, - ty: u16, -} - -impl Value { - /// The stored value (a SHORT or LONG, left-justified in the 4-byte field). - fn read(self, order: Order, b: &[u8]) -> Option { - match self.ty { - TYPE_SHORT => order.u16(b, self.at).map(u32::from), - _ => order.u32(b, self.at), - } - } - - /// 1 in this entry's own type, width and byte order. - fn one(self, order: Order) -> Vec { - match (self.ty, order) { - (TYPE_SHORT, Order::Little) => 1u16.to_le_bytes().to_vec(), - (TYPE_SHORT, Order::Big) => 1u16.to_be_bytes().to_vec(), - (_, Order::Little) => 1u32.to_le_bytes().to_vec(), - (_, Order::Big) => 1u32.to_be_bytes().to_vec(), - } - } -} - -/// The value of the entry at `e` if it is a well-formed Orientation entry: -/// exactly one SHORT or LONG. -fn orientation_value(order: Order, b: &[u8], e: usize, big_tiff: bool) -> Option { - let count = if big_tiff { order.u64(b, e.checked_add(4)?)? } else { u64::from(order.u32(b, e.checked_add(4)?)?) }; - if order.u16(b, e)? != TAG_ORIENTATION || count != 1 { - return None; - } - let ty = order.u16(b, e.checked_add(2)?)?; - let value_offset = if big_tiff { 12 } else { 8 }; - matches!(ty, TYPE_SHORT | TYPE_LONG).then_some(Value { at: e.checked_add(value_offset)?, ty }) -} - -/// Locates the Orientation value in IFD0. -/// -/// The **first** Orientation entry wins (like libexif and Photoshop); if that -/// one is malformed (count ≠ 1, or neither SHORT nor LONG) the tag is ignored, -/// even when a later duplicate is well formed. -fn find_entry(b: &[u8]) -> Option<(Order, Value)> { - let (order, first, count, entry_bytes, big_tiff) = ifd0_entries(b)?; - let e = (0..count).filter_map(|i| first.checked_add(i.checked_mul(entry_bytes)?)).find(|&e| order.u16(b, e) == Some(TAG_ORIENTATION))?; - Some((order, orientation_value(order, b, e, big_tiff)?)) -} - -/// The orientation (1–8) recorded in a TIFF-structured block: an EXIF payload -/// (with or without the `Exif\0\0` prefix) or a whole TIFF file. Missing, -/// malformed or out-of-range values give 1. -pub fn exif_orientation(exif: &[u8]) -> u16 { - let b = tiff_body(exif); - find_entry(b).and_then(|(order, v)| v.read(order, b)).and_then(|v| u16::try_from(v).ok()).filter(|v| (1..=8).contains(v)).unwrap_or(1) -} - -/// `exif` with its Orientation rewritten to 1, borrowed when there is nothing -/// to change (no well-formed tag, already 1, or unparseable). -/// -/// Only the value bytes of each well-formed Orientation entry change, written -/// as 1 in the entry's existing type and width (SHORT or LONG); every other -/// byte (other tags, their offset-based values, sub-IFDs) is kept as is. -/// Well-formed duplicates are all set to 1, so a reader that lets a later -/// duplicate win cannot rotate the upright pixels either. -pub fn upright_exif(exif: &[u8]) -> Cow<'_, [u8]> { - let body = tiff_body(exif); - let prefix = exif.len() - body.len(); - let Some((order, first, count, entry_bytes, big_tiff)) = ifd0_entries(body) else { - return Cow::Borrowed(exif); - }; - let values: Vec = (0..count) - .filter_map(|i| first.checked_add(i.checked_mul(entry_bytes)?)) - .filter_map(|e| orientation_value(order, body, e, big_tiff)) - .collect(); - if values.iter().all(|v| v.read(order, body) == Some(1)) { - return Cow::Borrowed(exif); - } - let mut out = exif.to_vec(); - for v in values { - let one = v.one(order); - let slot = prefix.checked_add(v.at).and_then(|at| out.get_mut(at..at.checked_add(one.len())?)); - if let Some(slot) = slot { - slot.copy_from_slice(&one); - } - } - Cow::Owned(out) -} - -/// `xmp` with any `tiff:Orientation` (attribute or element form) set to 1, -/// borrowed when there is nothing to change. -pub fn upright_xmp(xmp: &str) -> Cow<'_, str> { - const KEY: &str = "tiff:Orientation"; - if !xmp.contains(KEY) { - return Cow::Borrowed(xmp); - } - let mut out = String::with_capacity(xmp.len()); - let mut rest = xmp; - let mut changed = false; - while let Some(i) = rest.find(KEY) { - let (head, tail) = rest.split_at(i + KEY.len()); - out.push_str(head); - rest = tail; - // `tiff:Orientation="6"`, `tiff:Orientation='6'` or `6<`. - let open = rest.char_indices().take_while(|&(_, c)| c.is_ascii_whitespace() || matches!(c, '=' | '"' | '\'' | '>')).last(); - let Some((at, c)) = open else { continue }; - if !matches!(c, '"' | '\'' | '>') { - continue; - } - let start = at + 1; - let digits = rest.get(start..).map_or(0, |s| s.bytes().take_while(u8::is_ascii_digit).count()); - if digits == 0 || rest.get(start..start + digits) == Some("1") { - continue; - } - out.push_str(rest.get(..start).unwrap_or_default()); - out.push('1'); - rest = rest.get(start + digits..).unwrap_or_default(); - changed = true; - } - out.push_str(rest); - if changed { Cow::Owned(out) } else { Cow::Borrowed(xmp) } +/// TIFF payload of an EXIF block (a JPEG-style `Exif\0\0` prefix is skipped). +pub(crate) fn tiff_body(bytes: &[u8]) -> &[u8] { + bytes.strip_prefix(b"Exif\0\0").unwrap_or(bytes) } /// Source pixel index for output pixel `(0, y2)` and the step per output column, for diff --git a/crates/image-codecs/src/resolution.rs b/crates/image-codecs/src/resolution.rs index a787cacbc..54e6759a0 100644 --- a/crates/image-codecs/src/resolution.rs +++ b/crates/image-codecs/src/resolution.rs @@ -1,575 +1,11 @@ -//! Image resolution (pixels per inch) recorded in metadata blocks: EXIF / TIFF IFD0 -//! (XResolution 282, YResolution 283, ResolutionUnit 296), XMP (`tiff:XResolution`, -//! `tiff:YResolution`, `tiff:ResolutionUnit`) and Photoshop's image resource -//! ResolutionInfo (`8BIM` resource 0x03ED); reading them, and rewriting the EXIF and XMP -//! copies on export so they agree with the resolution the file is written at. -//! -//! Implemented from TIFF 6.0 (section 8, "Baseline fields": XResolution / YResolution are -//! RATIONAL, ResolutionUnit is SHORT, 1 = no absolute unit, 2 = inch, 3 = centimetre, -//! default 2), EXIF 2.3 (same tags in the 0th IFD, same default) and the Adobe Photoshop File -//! Formats Specification (ResolutionInfo: Fixed 16.16 hRes in pixels per inch, then the -//! display units; the vertical resolution likewise). -//! -//! **Precedence** when a file carries several, as Photoshop resolves them: Photoshop's own -//! ResolutionInfo, then the XMP `tiff:` properties, then EXIF, and only then the format's own -//! density field when it is the generic JFIF one (see the JPEG decoder). JFIF density is last -//! because cameras record the real resolution in EXIF while many tools and libraries write a -//! default JFIF density of 72 (or a bare aspect ratio) without knowing it; Photoshop shows a -//! camera JPEG at its EXIF resolution whatever its JFIF header says. Formats whose own field is -//! written deliberately (PNG `pHYs`, TIFF tags) keep it first; the metadata blocks only fill in -//! when it is missing. -//! -//! Everything read here is untrusted: offsets and counts are bounds-checked, both byte orders -//! and BigTIFF-style EXIF are handled, and anything malformed reads as "no resolution" rather -//! than as a wrong one. +//! Shared EXIF/XMP/Photoshop resolution parsing and edits. -use std::borrow::Cow; +pub use craft_metadata::edit::{ + exif_resolution, exif_with_pixel_size, exif_with_resolution, exif_with_resolution_adding_missing, export_exif, + export_exif_adding_missing_resolution, export_xmp, photoshop_resolution, resolution, xmp_resolution, xmp_with_resolution, +}; -use crate::orientation::{Order, ifd0_entries, tiff_body, upright_exif, upright_xmp}; - -const TAG_X_RESOLUTION: u16 = 282; -const TAG_Y_RESOLUTION: u16 = 283; -const TAG_RESOLUTION_UNIT: u16 = 296; -const TYPE_SHORT: u16 = 3; -const TYPE_RATIONAL: u16 = 5; -const CM_PER_INCH: f64 = 2.54; -/// Photoshop's ResolutionInfo image resource. -const RESOLUTION_INFO: u16 = 0x03ED; -/// Resolutions above this (pixels per inch) are treated as damage, not data. -const MAX_PPI: f64 = 1_000_000.0; -/// Two resolutions this close (pixels per inch) are the same. -const SAME_PPI: f64 = 1e-3; - -/// A usable resolution pair, or `None`. -fn ppi_pair(x: f64, y: f64) -> Option<(f32, f32)> { - let ok = |v: f64| v.is_finite() && v > 0.0 && v <= MAX_PPI; - (ok(x) && ok(y)).then_some((x as f32, y as f32)) -} - -fn same(a: (f32, f32), b: (f32, f32)) -> bool { - (f64::from(a.0) - f64::from(b.0)).abs() < SAME_PPI && (f64::from(a.1) - f64::from(b.1)).abs() < SAME_PPI -} - -/// `ppi` as a TIFF RATIONAL: a whole number over 1 when it is one, else over 1000. -fn rational(ppi: f32) -> (u32, u32) { - let p = f64::from(ppi); - if (p - p.round()).abs() < SAME_PPI { (p.round() as u32, 1) } else { ((p * 1000.0).round() as u32, 1000) } -} - -/// One of the three resolution entries of IFD0. -#[derive(Clone, Copy)] -struct Field { - /// Offset of the 12- (BigTIFF: 20-) byte entry. - entry: usize, - tag: u16, - ty: u16, - count: u64, - /// Where the value bytes are, when the type is one we read and they lie inside the block. - value: Option, -} - -impl Field { - fn short(&self, order: Order, b: &[u8]) -> Option { - (self.ty == TYPE_SHORT && self.count == 1).then_some(())?; - order.u16(b, self.value?) - } - - fn rational(&self, order: Order, b: &[u8]) -> Option { - (self.ty == TYPE_RATIONAL && self.count == 1).then_some(())?; - let at = self.value?; - let (n, d) = (order.u32(b, at)?, order.u32(b, at.checked_add(4)?)?); - (d != 0).then(|| f64::from(n) / f64::from(d)) - } -} - -/// IFD0's resolution entries. -struct Ifd0 { - order: Order, - big: bool, - first: usize, - count: usize, - entry_bytes: usize, - fields: Vec, -} - -impl Ifd0 { - fn parse(b: &[u8]) -> Option { - let (order, first, count, entry_bytes, big) = ifd0_entries(b)?; - let mut fields = Vec::new(); - for i in 0..count { - let e = first.checked_add(i.checked_mul(entry_bytes)?)?; - let tag = order.u16(b, e)?; - if !matches!(tag, TAG_X_RESOLUTION | TAG_Y_RESOLUTION | TAG_RESOLUTION_UNIT) { - continue; - } - let ty = order.u16(b, e.checked_add(2)?)?; - let count = if big { order.u64(b, e.checked_add(4)?)? } else { u64::from(order.u32(b, e.checked_add(4)?)?) }; - let value = value_at(order, b, e, big, ty, count); - fields.push(Field { entry: e, tag, ty, count, value }); - } - Some(Ifd0 { order, big, first, count, entry_bytes, fields }) - } - - fn first_of(&self, tag: u16) -> Option<&Field> { - self.fields.iter().find(|f| f.tag == tag) - } -} - -/// Where an entry's SHORT or RATIONAL value bytes are: inline in the entry when they fit, -/// else at the offset it holds. `None` for other types or when they leave the block. -fn value_at(order: Order, b: &[u8], e: usize, big: bool, ty: u16, count: u64) -> Option { - let size = match ty { - TYPE_SHORT => 2u64, - TYPE_RATIONAL => 8, - _ => return None, - } - .checked_mul(count)?; - let (field, inline) = if big { (e.checked_add(12)?, 8) } else { (e.checked_add(8)?, 4) }; - let at = if size <= inline { - field - } else if big { - usize::try_from(order.u64(b, field)?).ok()? - } else { - usize::try_from(order.u32(b, field)?).ok()? - }; - (at.checked_add(usize::try_from(size).ok()?)? <= b.len()).then_some(at) -} - -/// The resolution (pixels per inch, horizontal and vertical) recorded in IFD0 of an EXIF block -/// (with or without the JPEG `Exif\0\0` prefix) or a TIFF structure. -/// -/// Both XResolution and YResolution must be one RATIONAL each with a non-zero denominator. -/// ResolutionUnit 2 (inch, also when the tag is absent) is used as is, 3 (centimetre) is -/// converted; 1 (no absolute unit), any other value or a malformed unit entry gives `None`. -/// The first of duplicated entries wins. -pub fn exif_resolution(exif: &[u8]) -> Option<(f32, f32)> { - let b = tiff_body(exif); - let ifd = Ifd0::parse(b)?; - let unit = match ifd.first_of(TAG_RESOLUTION_UNIT) { - None => 2, - Some(f) => f.short(ifd.order, b)?, - }; - let scale = match unit { - 2 => 1.0, - 3 => CM_PER_INCH, - _ => return None, - }; - let x = ifd.first_of(TAG_X_RESOLUTION)?.rational(ifd.order, b)?; - let y = ifd.first_of(TAG_Y_RESOLUTION)?.rational(ifd.order, b)?; - ppi_pair(x * scale, y * scale) -} - -/// `exif` with its IFD0 resolution set to `ppi` (pixels per inch), borrowed when it has no -/// resolution entries, already records that resolution, or cannot be parsed. -/// -/// Existing well-formed XResolution / YResolution (one RATIONAL) entries are rewritten in place, -/// and every well-formed ResolutionUnit (one SHORT) becomes 2 (inch). Malformed entries are -/// removed from IFD0 instead, so no stale value survives; all other bytes are kept. -pub fn exif_with_resolution(exif: &[u8], ppi: (f32, f32)) -> Cow<'_, [u8]> { - exif_with_resolution_inner(exif, ppi, false) -} - -/// Like [`exif_with_resolution`], but adds X/Y resolution tags when a valid IFD0 has none. -/// Existing TIFF data stays in place, so its offsets remain valid. Used by CMYK JPEG, which -/// cannot carry a JFIF density marker. -pub(crate) fn exif_with_resolution_adding_missing(exif: &[u8], ppi: (f32, f32)) -> Cow<'_, [u8]> { - exif_with_resolution_inner(exif, ppi, true) -} - -/// Rewrite valid IFD0 resolution tags and optionally append tags that are missing or malformed. -fn exif_with_resolution_inner(exif: &[u8], ppi: (f32, f32), add_missing: bool) -> Cow<'_, [u8]> { - if ppi_pair(f64::from(ppi.0), f64::from(ppi.1)).is_none() { - return Cow::Borrowed(exif); - } - if exif.is_empty() { - if !add_missing { - return Cow::Borrowed(exif); - } - // An image with DPI but no source EXIF still needs a valid EXIF block when the output - // format cannot carry resolution in its own header (CMYK JPEG has no valid JFIF marker). - return Cow::Owned(exif_with_resolution_inner(b"II*\0\x08\0\0\0\0\0\0\0\0\0\0", ppi, true).into_owned()); - } - let body = tiff_body(exif); - let prefix = exif.len() - body.len(); - let Some(ifd) = Ifd0::parse(body) else { return Cow::Borrowed(exif) }; - if exif_resolution(body).is_some_and(|r| same(r, ppi)) { - return Cow::Borrowed(exif); - } - let missing_xy = ifd.first_of(TAG_X_RESOLUTION).and_then(|f| f.rational(ifd.order, body)).is_none() - || ifd.first_of(TAG_Y_RESOLUTION).and_then(|f| f.rational(ifd.order, body)).is_none(); - if add_missing && missing_xy { - let Some(body) = append_resolution_ifd(body, &ifd, ppi) else { return Cow::Borrowed(exif) }; - let mut full = exif.get(..prefix).unwrap_or_default().to_vec(); - full.extend_from_slice(&body); - return Cow::Owned(full); - } - if ifd.fields.is_empty() { - return Cow::Borrowed(exif); - } - let order = ifd.order; - let mut out = body.to_vec(); - let mut removed = Vec::new(); - for f in &ifd.fields { - let bytes = match f.tag { - TAG_RESOLUTION_UNIT if f.short(order, body).is_some() => put_u16(order, 2).to_vec(), - TAG_X_RESOLUTION | TAG_Y_RESOLUTION if f.rational(order, body).is_some() => { - let (n, d) = rational(if f.tag == TAG_X_RESOLUTION { ppi.0 } else { ppi.1 }); - [put_u32(order, n), put_u32(order, d)].concat() - } - _ => { - removed.push(f.entry); - continue; - } - }; - let slot = f.value.and_then(|at| out.get_mut(at..at.checked_add(bytes.len())?)); - match slot { - Some(slot) => slot.copy_from_slice(&bytes), - None => removed.push(f.entry), - } - } - if !removed.is_empty() { - remove_entries(&mut out, &ifd, &removed); - } - let mut full = exif.get(..prefix).unwrap_or_default().to_vec(); - full.extend_from_slice(&out); - if full == exif { Cow::Borrowed(exif) } else { Cow::Owned(full) } -} - -/// Append a replacement IFD0 with fresh resolution tags. Every other entry keeps its original -/// bytes and offsets, since the TIFF block's existing data remains where it was. -fn append_resolution_ifd(body: &[u8], ifd: &Ifd0, ppi: (f32, f32)) -> Option> { - let (count_bytes, next_bytes, alignment) = if ifd.big { (8usize, 8usize, 8usize) } else { (2usize, 4usize, 2usize) }; - let mut entries = Vec::with_capacity(ifd.count.saturating_add(3)); - for index in 0..ifd.count { - let entry = ifd.first.checked_add(index.checked_mul(ifd.entry_bytes)?)?; - let tag = ifd.order.u16(body, entry)?; - if matches!(tag, TAG_X_RESOLUTION | TAG_Y_RESOLUTION | TAG_RESOLUTION_UNIT) { - continue; - } - entries.push((tag, body.get(entry..entry.checked_add(ifd.entry_bytes)?)?.to_vec())); - } - let count = entries.len().checked_add(3)?; - if !ifd.big && count > usize::from(u16::MAX) { - return None; - } - - let new_ifd = align_up(body.len(), alignment)?; - let table_end = new_ifd.checked_add(count_bytes)?.checked_add(count.checked_mul(ifd.entry_bytes)?)?; - let next_at = ifd.first.checked_add(ifd.count.checked_mul(ifd.entry_bytes)?)?; - let next_ifd = body.get(next_at..next_at.checked_add(next_bytes)?)?; - let rational_at = align_up(table_end.checked_add(next_bytes)?, if ifd.big { 8 } else { 4 })?; - let y_rational_at = rational_at.checked_add(8)?; - if !ifd.big && (new_ifd > u32::MAX as usize || y_rational_at.checked_add(8)? > u32::MAX as usize) { - return None; - } - - entries.push((TAG_X_RESOLUTION, resolution_entry(ifd.order, ifd.big, TAG_X_RESOLUTION, u64::try_from(rational_at).ok()?)?)); - entries.push((TAG_Y_RESOLUTION, resolution_entry(ifd.order, ifd.big, TAG_Y_RESOLUTION, u64::try_from(y_rational_at).ok()?)?)); - entries.push((TAG_RESOLUTION_UNIT, resolution_entry(ifd.order, ifd.big, TAG_RESOLUTION_UNIT, 2)?)); - entries.sort_by_key(|(tag, _)| *tag); - - let mut out = body.to_vec(); - out.resize(new_ifd, 0); - if ifd.big { - out.extend_from_slice(&put_u64(ifd.order, count as u64)); - } else { - out.extend_from_slice(&put_u16(ifd.order, count as u16)); - } - for (_, entry) in entries { - out.extend_from_slice(&entry); - } - out.extend_from_slice(next_ifd); - out.resize(rational_at, 0); - let (xn, xd) = rational(ppi.0); - let (yn, yd) = rational(ppi.1); - out.extend_from_slice(&put_u32(ifd.order, xn)); - out.extend_from_slice(&put_u32(ifd.order, xd)); - out.extend_from_slice(&put_u32(ifd.order, yn)); - out.extend_from_slice(&put_u32(ifd.order, yd)); - - let offset_at: usize = if ifd.big { 8 } else { 4 }; - if ifd.big { - out.get_mut(offset_at..offset_at.checked_add(8)?)?.copy_from_slice(&put_u64(ifd.order, new_ifd as u64)); - } else { - out.get_mut(offset_at..offset_at.checked_add(4)?)?.copy_from_slice(&put_u32(ifd.order, u32::try_from(new_ifd).ok()?)); - } - Some(out) -} - -/// One IFD entry. Rational fields store an offset; the SHORT unit is inline. -fn resolution_entry(order: Order, big: bool, tag: u16, value: u64) -> Option> { - let entry_bytes = if big { 20 } else { 12 }; - let count_at = 4; - let value_at = if big { 12 } else { 8 }; - let mut entry = vec![0; entry_bytes]; - entry.get_mut(..2)?.copy_from_slice(&put_u16(order, tag)); - entry.get_mut(2..4)?.copy_from_slice(&put_u16(order, if tag == TAG_RESOLUTION_UNIT { TYPE_SHORT } else { TYPE_RATIONAL })); - if big { - entry.get_mut(count_at..count_at + 8)?.copy_from_slice(&put_u64(order, 1)); - if tag == TAG_RESOLUTION_UNIT { - entry.get_mut(value_at..value_at + 2)?.copy_from_slice(&put_u16(order, value as u16)); - } else { - entry.get_mut(value_at..value_at + 8)?.copy_from_slice(&put_u64(order, value)); - } - } else { - entry.get_mut(count_at..count_at + 4)?.copy_from_slice(&put_u32(order, 1)); - if tag == TAG_RESOLUTION_UNIT { - entry.get_mut(value_at..value_at + 2)?.copy_from_slice(&put_u16(order, value as u16)); - } else { - entry.get_mut(value_at..value_at + 4)?.copy_from_slice(&put_u32(order, u32::try_from(value).ok()?)); - } - } - Some(entry) -} - -/// Round `value` up to the next `alignment` boundary, returning `None` on overflow. -fn align_up(value: usize, alignment: usize) -> Option { - value.checked_add((alignment - value % alignment) % alignment) -} - -fn put_u16(order: Order, v: u16) -> [u8; 2] { - match order { - Order::Little => v.to_le_bytes(), - Order::Big => v.to_be_bytes(), - } -} - -fn put_u32(order: Order, v: u32) -> [u8; 4] { - match order { - Order::Little => v.to_le_bytes(), - Order::Big => v.to_be_bytes(), - } -} - -fn put_u64(order: Order, v: u64) -> [u8; 8] { - match order { - Order::Little => v.to_le_bytes(), - Order::Big => v.to_be_bytes(), - } -} - -/// Removes the IFD0 entries at offsets `removed` (see [`exif_with_resolution`]). -fn remove_entries(b: &mut [u8], ifd: &Ifd0, removed: &[usize]) { - let eb = ifd.entry_bytes; - let next_bytes = if ifd.big { 8 } else { 4 }; - let count_bytes = if ifd.big { 8 } else { 2 }; - let Some(table_end) = ifd.count.checked_mul(eb).and_then(|n| ifd.first.checked_add(n)) else { return }; - let Some(old) = b.get(ifd.first..table_end.saturating_add(next_bytes)).map(<[u8]>::to_vec) else { return }; - let mut table = Vec::with_capacity(old.len()); - let mut kept = 0usize; - for (i, entry) in old.chunks_exact(eb).take(ifd.count).enumerate() { - if !removed.contains(&(ifd.first + i * eb)) { - table.extend_from_slice(entry); - kept += 1; - } - } - table.extend_from_slice(old.get(ifd.count * eb..).unwrap_or_default()); - table.resize(old.len(), 0); - let count: Vec = if ifd.big { put_u64(ifd.order, kept as u64).to_vec() } else { put_u16(ifd.order, kept as u16).to_vec() }; - let Some(count_at) = ifd.first.checked_sub(count_bytes) else { return }; - if let Some(slot) = b.get_mut(count_at..ifd.first) { - slot.copy_from_slice(&count); - } - if let Some(slot) = b.get_mut(ifd.first..ifd.first + table.len()) { - slot.copy_from_slice(&table); - } -} - -/// The resolution in a Photoshop image resource block (a sequence of `8BIM` resources, as in a -/// JPEG's APP13 "Photoshop 3.0" segment or TIFF tag 34377): its ResolutionInfo (0x03ED), whose -/// Fixed 16.16 hRes / vRes are pixels per inch whatever display unit follows them. -pub fn photoshop_resolution(resources: &[u8]) -> Option<(f32, f32)> { - let be16 = |at: usize| resources.get(at..at.checked_add(2)?).map(|s| u16::from_be_bytes([s[0], s[1]])); - let be32 = |at: usize| resources.get(at..at.checked_add(4)?).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]])); - let mut i = 0usize; - while i < resources.len() { - let signature = resources.get(i..i.checked_add(4)?)?; - let id = be16(i.checked_add(4)?)?; - // Pascal name: length byte plus text, padded to an even size. - let name_len = usize::from(*resources.get(i.checked_add(6)?)?); - let name = (name_len + 1).next_multiple_of(2); - let size_at = i.checked_add(6)?.checked_add(name)?; - let size = usize::try_from(be32(size_at)?).ok()?; - let data = size_at.checked_add(4)?; - if signature == b"8BIM" && id == RESOLUTION_INFO { - let info = resources.get(data..data.checked_add(size)?)?; - if info.len() < 16 { - return None; - } - let fixed = |at: usize| be32(data + at).map(|v| f64::from(v) / 65536.0); - return ppi_pair(fixed(0)?, fixed(8)?); - } - i = data.checked_add(size)?.checked_add(size % 2)?; - } - None -} - -/// The text ranges of every value of XMP property `key` (`key="v"`, `key='v'` or -/// `v`), in order. -fn xmp_values(xmp: &str, key: &str) -> Vec<(usize, usize)> { - let mut out = Vec::new(); - let mut from = 0; - while let Some(k) = xmp.get(from..).and_then(|s| s.find(key)).map(|k| k + from) { - from = k + key.len(); - // An attribute or an opening tag of exactly this name (not ``, not `xkey`). - if !xmp.get(..k).and_then(|h| h.chars().next_back()).is_some_and(|c| c == '<' || c.is_ascii_whitespace()) { - continue; - } - let rest = xmp.get(from..).unwrap_or_default(); - let after_key = rest.trim_start(); - let (value, close) = if let Some(r) = after_key.strip_prefix('=') { - let r = r.trim_start(); - match r.chars().next() { - Some(q @ ('"' | '\'')) => (r.get(1..).unwrap_or_default(), q), - _ => continue, - } - } else if let Some(r) = rest.strip_prefix('>') { - (r, '<') - } else { - continue; - }; - let start = xmp.len() - value.len(); - // Advance past this value before searching for the next key. Without this, `from` - // only moved past the key text, so property-like text repeated inside an unrelated - // attribute re-ran this full-suffix `find(close)` for every occurrence, which is - // quadratic in the packet size. A missing close means the value is unterminated, so - // no later value can parse either; stop scanning in that case. - let Some(len) = value.find(close) else { break }; - out.push((start, start + len)); - from = start + len + close.len_utf8(); - } - out -} - -/// An XMP rational (`300/1`) or plain number. -fn xmp_number(s: &str) -> Option { - let s = s.trim(); - match s.split_once('/') { - Some((n, d)) => { - let (n, d) = (n.trim().parse::().ok()?, d.trim().parse::().ok()?); - (d != 0.0).then(|| n / d) - } - None => s.parse().ok(), - } -} - -const XMP_X: &str = "tiff:XResolution"; -const XMP_Y: &str = "tiff:YResolution"; -const XMP_UNIT: &str = "tiff:ResolutionUnit"; - -/// The resolution (pixels per inch) in an XMP packet's `tiff:XResolution` / `tiff:YResolution` -/// (with `tiff:ResolutionUnit`, 2 when absent, 3 converted from centimetres), or `None`. -pub fn xmp_resolution(xmp: &str) -> Option<(f32, f32)> { - let first = |key: &str| xmp_values(xmp, key).first().and_then(|&(a, b)| xmp.get(a..b)); - let unit = match first(XMP_UNIT) { - None => 2.0, - Some(u) => xmp_number(u)?, - }; - let scale = if unit == 2.0 { - 1.0 - } else if unit == 3.0 { - CM_PER_INCH - } else { - return None; - }; - ppi_pair(xmp_number(first(XMP_X)?)? * scale, xmp_number(first(XMP_Y)?)? * scale) -} - -/// `xmp` with its `tiff:` resolution properties set to `ppi` (and the unit to 2, inch), -/// borrowed when it has none or already says so. -pub fn xmp_with_resolution(xmp: &str, ppi: (f32, f32)) -> Cow<'_, str> { - if ppi_pair(f64::from(ppi.0), f64::from(ppi.1)).is_none() || xmp_resolution(xmp).is_some_and(|r| same(r, ppi)) { - return Cow::Borrowed(xmp); - } - let text = |v: f32| { - let (n, d) = rational(v); - format!("{n}/{d}") - }; - let mut edits: Vec<(usize, usize, String)> = Vec::new(); - for (key, value) in [(XMP_X, text(ppi.0)), (XMP_Y, text(ppi.1)), (XMP_UNIT, "2".to_owned())] { - edits.extend(xmp_values(xmp, key).into_iter().map(|(a, b)| (a, b, value.clone()))); - } - if edits.is_empty() { - return Cow::Borrowed(xmp); - } - edits.sort_by_key(|e| e.0); - let mut out = String::with_capacity(xmp.len() + 16); - let mut at = 0; - for (a, b, v) in edits { - // Overlapping ranges can only come from malformed markup; keep the first edit. - if a < at { - continue; - } - out.push_str(xmp.get(at..a).unwrap_or_default()); - out.push_str(&v); - at = b; - } - out.push_str(xmp.get(at..).unwrap_or_default()); - if out == xmp { Cow::Borrowed(xmp) } else { Cow::Owned(out) } -} - -/// `exif` without its embedded thumbnail, or `None` when it has none (or doesn't parse). -/// -/// EXIF keeps the thumbnail in IFD1, the IFD IFD0's next-IFD pointer names. Finder, Android's -/// gallery and upload pickers show that thumbnail instead of decoding the image, so a copied one -/// keeps showing the photo as it was before it was edited (#2147, #2373). The pointer is cleared; -/// nothing else moves, so every other offset in the block stays valid. -fn exif_without_thumbnail(exif: &[u8]) -> Option> { - let body = tiff_body(exif); - let prefix = exif.len() - body.len(); - let (_, first, count, entry_bytes, big_tiff) = ifd0_entries(body)?; - let next = prefix.checked_add(first)?.checked_add(count.checked_mul(entry_bytes)?)?; - let width = if big_tiff { 8 } else { 4 }; - // In bounds: `ifd0_entries` checked that the whole pointer is present. - if exif.get(next..next.checked_add(width)?)?.iter().all(|b| *b == 0) { - return None; - } - let mut out = exif.to_vec(); - out.get_mut(next..next + width)?.fill(0); - Some(out) -} - -/// The EXIF to write next to upright pixels at resolution `ppi`: Orientation 1 (see -/// [`upright_exif`]), no thumbnail of the source image (see [`exif_without_thumbnail`]) and, -/// when `ppi` is known, the resolution (see [`exif_with_resolution`]). -pub fn export_exif(exif: &[u8], ppi: Option<(f32, f32)>) -> Cow<'_, [u8]> { - export_exif_inner(exif, ppi, false) -} - -/// Export EXIF for a format without its own resolution field, adding missing X/Y resolution -/// entries to valid metadata when necessary. -pub(crate) fn export_exif_adding_missing_resolution(exif: &[u8], ppi: Option<(f32, f32)>) -> Cow<'_, [u8]> { - export_exif_inner(exif, ppi, true) -} - -/// Normalize EXIF orientation and thumbnails, optionally adding missing output resolution tags. -fn export_exif_inner(exif: &[u8], ppi: Option<(f32, f32)>, add_missing: bool) -> Cow<'_, [u8]> { - let up = upright_exif(exif); - let up = match exif_without_thumbnail(&up) { - Some(v) => Cow::Owned(v), - None => up, - }; - let Some(ppi) = ppi else { return up }; - let result = if add_missing { exif_with_resolution_adding_missing(&up, ppi) } else { exif_with_resolution(&up, ppi) }; - if let Cow::Owned(v) = result { - return Cow::Owned(v); - } - up -} - -/// The XMP to write next to upright pixels at resolution `ppi` (see [`upright_xmp`] and -/// [`xmp_with_resolution`]). -pub fn export_xmp(xmp: &str, ppi: Option<(f32, f32)>) -> Cow<'_, str> { - let up = upright_xmp(xmp); - let Some(ppi) = ppi else { return up }; - if let Cow::Owned(v) = xmp_with_resolution(&up, ppi) { - return Cow::Owned(v); - } - up -} - -/// The resolution a decoded file's metadata blocks record, in precedence order: XMP, then -/// EXIF (Photoshop's ResolutionInfo is format-specific and read by the decoder). +/// Resolution used by raster decode, preferring XMP over EXIF as before. pub(crate) fn from_metadata(exif: Option<&[u8]>, xmp: Option<&str>) -> Option<(f32, f32)> { - xmp.and_then(xmp_resolution).or_else(|| exif.and_then(exif_resolution)) + craft_metadata::edit::resolution(exif, xmp) } From 90e76d75720a33d45ec128c9e38dd697d97453f1 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:29:45 -0500 Subject: [PATCH 04/98] Use valid scan parameters in JPEG coverage --- crates/image-codecs/src/codecs/jpeg.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/image-codecs/src/codecs/jpeg.rs b/crates/image-codecs/src/codecs/jpeg.rs index e2e854622..23545bcb3 100644 --- a/crates/image-codecs/src/codecs/jpeg.rs +++ b/crates/image-codecs/src/codecs/jpeg.rs @@ -821,7 +821,10 @@ mod tests { fn lossless_and_twelve_bit_progressive_are_unsupported_not_damaged() { for marker in [0xC3, 0xC2] { let mut bytes = vec![0xFF, 0xD8, 0xFF, marker, 0, 11, 12, 0, 8, 0, 8, 1, 1, 0x11, 0]; - bytes.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, 0x3F, 0, 0x12, 0x34, 0xFF, 0xD9]); + // A progressive frame's first scan is a DC scan (Ss=Se=0); a + // sequential frame uses the full spectral range (Ss=0, Se=63). + let spectral_end = if marker == 0xC2 { 0 } else { 0x3F }; + bytes.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, spectral_end, 0, 0x12, 0x34, 0xFF, 0xD9]); let e = crate::decode(&bytes).unwrap_err(); assert!(matches!(e, CodecError::Unsupported { .. }), "{marker:#x}: {e:?}"); } From 0153632dfa8c739b96321f4e1009d55dc6deb03f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:32:07 -0500 Subject: [PATCH 05/98] Align codec EXIF thumbnail assertion --- crates/image-codecs/tests/it/resolution.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/tests/it/resolution.rs b/crates/image-codecs/tests/it/resolution.rs index eadecbaa3..14c682f8d 100644 --- a/crates/image-codecs/tests/it/resolution.rs +++ b/crates/image-codecs/tests/it/resolution.rs @@ -516,10 +516,13 @@ fn exports_drop_the_source_exif_thumbnail() { assert!(out.windows(6).any(|w| w == b"NIKON\0"), "the rest of the EXIF is kept"); near(exif_resolution(&out), ppi.map_or(300.0, |p| p.0)); } - // Only the pointer changes. + // Only the IFD1 pointer and thumbnail payload change: the pointer is unlinked and the + // stale JPEG bytes are blanked by the shared metadata editor. let out = export_exif(&exif, None); + let thumb_len = b"\xFF\xD8stale thumbnail\xFF\xD9".len(); let differing: Vec = (0..exif.len()).filter(|&i| exif[i] != out[i]).collect(); - assert!(differing.iter().all(|&i| (8 + 2 + 4 * 12..8 + 2 + 4 * 12 + 4).contains(&i)), "{differing:?}"); + assert!(differing.iter().all(|&i| (8 + 2 + 4 * 12..8 + 2 + 4 * 12 + 4).contains(&i) || i >= exif.len() - thumb_len), "{differing:?}"); + assert!(!out.windows(5).any(|w| w == b"stale")); // With a JPEG-style prefix too. let prefixed = [b"Exif\0\0".as_slice(), &exif].concat(); assert_eq!(next_ifd(e, &export_exif(&prefixed, None)[6..]), 0); From 5cbd15a0906e35bdbc01eb028c3a6670f2cffb55 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:48:03 -0500 Subject: [PATCH 06/98] Make progressive JPEG fixture valid --- crates/image-codecs/src/codecs/jpeg.rs | 28 +++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/image-codecs/src/codecs/jpeg.rs b/crates/image-codecs/src/codecs/jpeg.rs index 23545bcb3..c2d178a9d 100644 --- a/crates/image-codecs/src/codecs/jpeg.rs +++ b/crates/image-codecs/src/codecs/jpeg.rs @@ -820,11 +820,29 @@ mod tests { #[test] fn lossless_and_twelve_bit_progressive_are_unsupported_not_damaged() { for marker in [0xC3, 0xC2] { - let mut bytes = vec![0xFF, 0xD8, 0xFF, marker, 0, 11, 12, 0, 8, 0, 8, 1, 1, 0x11, 0]; - // A progressive frame's first scan is a DC scan (Ss=Se=0); a - // sequential frame uses the full spectral range (Ss=0, Se=63). - let spectral_end = if marker == 0xC2 { 0 } else { 0x3F }; - bytes.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, spectral_end, 0, 0x12, 0x34, 0xFF, 0xD9]); + let bytes = if marker == 0xC2 { + // Use a valid progressive DC scan so this exercises unsupported + // precision rather than failing on missing tables or scan syntax. + let mut bytes = vec![0xFF, 0xD8]; + let mut dqt = vec![0]; + dqt.extend([1u8; 64]); + segment(&mut bytes, 0xDB, &dqt); + segment(&mut bytes, marker, &[12, 0, 8, 0, 8, 1, 1, 0x11, 0]); + let mut dht = vec![0x00, 1]; + dht.extend([0u8; 15]); + dht.push(0); + dht.extend([0x10, 1]); + dht.extend([0u8; 15]); + dht.push(0); + segment(&mut bytes, 0xC4, &dht); + segment(&mut bytes, 0xDA, &[1, 1, 0, 0, 0, 0]); + bytes.extend([0x7F, 0xFF, 0xD9]); + bytes + } else { + let mut bytes = vec![0xFF, 0xD8, 0xFF, marker, 0, 11, 12, 0, 8, 0, 8, 1, 1, 0x11, 0]; + bytes.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, 0x3F, 0, 0x12, 0x34, 0xFF, 0xD9]); + bytes + }; let e = crate::decode(&bytes).unwrap_err(); assert!(matches!(e, CodecError::Unsupported { .. }), "{marker:#x}: {e:?}"); } From 60b0258d27e4a7e695a33c30a62f9c90e0fd5f2f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 09:24:59 -0500 Subject: [PATCH 07/98] Accept supported 12-bit progressive JPEG decoding --- crates/image-codecs/src/codecs/jpeg.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/image-codecs/src/codecs/jpeg.rs b/crates/image-codecs/src/codecs/jpeg.rs index c2d178a9d..3bfdcddb2 100644 --- a/crates/image-codecs/src/codecs/jpeg.rs +++ b/crates/image-codecs/src/codecs/jpeg.rs @@ -818,7 +818,7 @@ mod tests { } #[test] - fn lossless_and_twelve_bit_progressive_are_unsupported_not_damaged() { + fn lossless_is_unsupported_and_twelve_bit_progressive_decodes() { for marker in [0xC3, 0xC2] { let bytes = if marker == 0xC2 { // Use a valid progressive DC scan so this exercises unsupported @@ -843,8 +843,15 @@ mod tests { bytes.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, 1, 0, 0, 0x3F, 0, 0x12, 0x34, 0xFF, 0xD9]); bytes }; - let e = crate::decode(&bytes).unwrap_err(); - assert!(matches!(e, CodecError::Unsupported { .. }), "{marker:#x}: {e:?}"); + if marker == 0xC2 { + // craft-jpeg supports 12-bit progressive Huffman frames; keep the facade's + // routing contract aligned with the decoder rather than rejecting valid input. + let image = crate::decode(&bytes).unwrap(); + assert_eq!((image.dimensions(), image.sample_type(), image.layout()), ((8, 8), SampleType::U16, ChannelLayout::Gray)); + } else { + let error = crate::decode(&bytes).unwrap_err(); + assert!(matches!(error, CodecError::Unsupported { .. }), "{marker:#x}: {error:?}"); + } } } From 2db7c82705077e0bdffbd388900d511d4949ce27 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 08:40:24 -0500 Subject: [PATCH 08/98] Preserve raster facade encode contracts --- crates/image-codecs/src/codecs/xpm.rs | 10 ++++++++++ crates/image-codecs/src/lib.rs | 5 +---- crates/raster-codecs/src/dds.rs | 11 +++++++++++ crates/raster-codecs/src/fits.rs | 3 +++ crates/raster-codecs/src/iff.rs | 3 +++ crates/raster-codecs/src/pnm.rs | 5 +++++ 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/image-codecs/src/codecs/xpm.rs b/crates/image-codecs/src/codecs/xpm.rs index 2dbd867c9..c74b5f16b 100644 --- a/crates/image-codecs/src/codecs/xpm.rs +++ b/crates/image-codecs/src/codecs/xpm.rs @@ -114,6 +114,16 @@ pub(crate) fn named(name: &str) -> Option<[u8; 3]> { }) } +/// Exact, normalized X11 lookup for consumers that need table membership rather +/// than XPM's additional aliases and normalization rules. +pub(crate) fn exact_x11_name(name: &str) -> Option<[u8; 3]> { + if name.is_empty() || name.len() > 64 || !name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()) { + return None; + } + let index = x11_colors::X11_COLORS.binary_search_by_key(&name, |(entry, _)| *entry).ok()?; + Some(x11_colors::X11_COLORS.get(index)?.1) +} + fn color(name: &str) -> Option<[u8; 4]> { if name.eq_ignore_ascii_case("none") { return Some([0, 0, 0, 0]); diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 8085ca44e..e6a54756c 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -218,10 +218,7 @@ pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { /// The X11 color-name lookup used by XPM files, expressed as normalized sRGB. pub fn xpm_named_color_rgb(name: &str) -> Option<[f32; 3]> { - if name.is_empty() || name.len() > 64 || !name.bytes().all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit()) { - return None; - } - let rgb = codecs::xpm::named(name)?; + let rgb = codecs::xpm::exact_x11_name(name)?; Some(rgb.map(|channel| f32::from(channel) / 255.0)) } diff --git a/crates/raster-codecs/src/dds.rs b/crates/raster-codecs/src/dds.rs index 789028fb4..eb703a074 100644 --- a/crates/raster-codecs/src/dds.rs +++ b/crates/raster-codecs/src/dds.rs @@ -52,6 +52,17 @@ pub fn encode(image: &Image, options: DdsOptions) -> Result, CodecError> if image.sample_type() != SampleType::U8 { return Err(CodecError::encode("DDS", "DDS is written at 8 bits per channel")); } + // BC storage encodes RGBA blocks. The generic image API can plan a + // conversion from other layouts, but this compatibility facade accepts + // only layouts its former writer stored directly. + let allowed = if options.compression.is_block() { + image.layout() == crate::ChannelLayout::Rgba + } else { + matches!(image.layout(), crate::ChannelLayout::Gray | crate::ChannelLayout::GrayA | crate::ChannelLayout::Rgb | crate::ChannelLayout::Rgba) + }; + if !allowed { + return Err(CodecError::encode("DDS", format!("{:?} pixels can't be written as `{}`", image.layout(), options.compression.name()))); + } let compression = match options.compression { DdsCompression::None => craft_image_codecs::DdsCompression::None, DdsCompression::Bc1 => craft_image_codecs::DdsCompression::Bc1, diff --git a/crates/raster-codecs/src/fits.rs b/crates/raster-codecs/src/fits.rs index 4a662ab6e..5b82541c9 100644 --- a/crates/raster-codecs/src/fits.rs +++ b/crates/raster-codecs/src/fits.rs @@ -12,5 +12,8 @@ pub fn decode(bytes: &[u8], limits: &Limits) -> Result { bridge::format(Format::Fits, bytes, limits) } pub fn encode(image: &Image) -> Result, CodecError> { + if !matches!(image.layout(), crate::ChannelLayout::Gray | crate::ChannelLayout::Rgb) { + return Err(CodecError::encode("FITS", format!("FITS cannot store {:?}", image.layout()))); + } bridge::encode(image, Format::Fits, &Default::default()) } diff --git a/crates/raster-codecs/src/iff.rs b/crates/raster-codecs/src/iff.rs index 282617e21..b2d44b89a 100644 --- a/crates/raster-codecs/src/iff.rs +++ b/crates/raster-codecs/src/iff.rs @@ -9,5 +9,8 @@ pub fn decode(bytes: &[u8], limits: &Limits) -> Result { bridge::format(Format::Iff, bytes, limits) } pub fn encode(image: &Image) -> Result, CodecError> { + if image.layout() != crate::ChannelLayout::Rgb || image.sample_type() != crate::SampleType::U8 { + return Err(CodecError::encode("IFF", "ILBM writer stores 8-bit RGB pixels")); + } bridge::encode(image, Format::Iff, &Default::default()) } diff --git a/crates/raster-codecs/src/pnm.rs b/crates/raster-codecs/src/pnm.rs index 834e6508a..338938700 100644 --- a/crates/raster-codecs/src/pnm.rs +++ b/crates/raster-codecs/src/pnm.rs @@ -9,5 +9,10 @@ pub fn decode(bytes: &[u8], limits: &Limits) -> Result { bridge::format(Format::Pnm, bytes, limits) } pub fn encode(image: &Image) -> Result, CodecError> { + // PFM has only gray and RGB tuples. Preserve this facade's contract that + // unsupported layouts are refused instead of silently dropping alpha. + if image.sample_type() == crate::SampleType::F32 && !matches!(image.layout(), crate::ChannelLayout::Gray | crate::ChannelLayout::Rgb) { + return Err(CodecError::encode("PNM", format!("PFM cannot store {:?}", image.layout()))); + } bridge::encode(image, Format::Pnm, &Default::default()) } From 80b55dee5a4a33e5cedd38df16c10f2aebaeeb82 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 09:00:23 -0500 Subject: [PATCH 09/98] Allow alpha in IFF compatibility encoder --- crates/raster-codecs/src/iff.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/raster-codecs/src/iff.rs b/crates/raster-codecs/src/iff.rs index b2d44b89a..95e439a58 100644 --- a/crates/raster-codecs/src/iff.rs +++ b/crates/raster-codecs/src/iff.rs @@ -9,8 +9,8 @@ pub fn decode(bytes: &[u8], limits: &Limits) -> Result { bridge::format(Format::Iff, bytes, limits) } pub fn encode(image: &Image) -> Result, CodecError> { - if image.layout() != crate::ChannelLayout::Rgb || image.sample_type() != crate::SampleType::U8 { - return Err(CodecError::encode("IFF", "ILBM writer stores 8-bit RGB pixels")); + if !matches!(image.layout(), crate::ChannelLayout::Rgb | crate::ChannelLayout::Rgba) || image.sample_type() != crate::SampleType::U8 { + return Err(CodecError::encode("IFF", "ILBM writer stores 8-bit RGB pixels with optional alpha")); } bridge::encode(image, Format::Iff, &Default::default()) } From 81bd633c7f384e6f964176eb0baa4b271f1c7f8e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 08:51:23 -0500 Subject: [PATCH 10/98] Expose checked SVG preflight and viewport rewrite --- crates/svg/src/guard.rs | 16 +++++-- crates/svg/src/preflight.rs | 11 ++++- crates/svg/src/tree.rs | 89 ++++++++++++++++++++++++++++++++++--- 3 files changed, 104 insertions(+), 12 deletions(-) diff --git a/crates/svg/src/guard.rs b/crates/svg/src/guard.rs index 073bfe34c..f3573e443 100644 --- a/crates/svg/src/guard.rs +++ b/crates/svg/src/guard.rs @@ -178,6 +178,10 @@ pub fn too_wide(svg: &str, most: usize, most_work: usize) -> bool { pub struct Prescan<'s> { /// Whether the elements nest deeper than the cap. The pass stops there. pub too_deep: bool, + /// Deepest element nesting observed (markup entities count ten times). + pub depth: usize, + /// Deepest stack of unclosed source elements, excluding self-closing leaves. + pub open_depth: usize, /// The text with each data URL in an element's attribute values left out: a `data:` value's /// quotes kept, a `url(…)`'s parentheses and quotes. Borrowed when there is none, and the /// whole text when `too_deep`. An embedded picture is a data URL and most of the text when @@ -250,7 +254,7 @@ pub fn prescan(svg: &str, most: usize) -> Prescan<'_> { (b.len(), false) }; // (now, deepest) of the elements, and of the markup inside the document type. - let (mut elements, mut entities) = ((0usize, 0usize), (0usize, 0usize)); + let (mut elements, mut entities, mut open_depth) = ((0usize, 0usize), (0usize, 0usize), 0usize); let mut doctype_end = 0; let mut i = 0; while let Some(at) = b.get(i..).and_then(|rest| rest.iter().position(|c| *c == b'<')).map(|at| i + at) { @@ -287,8 +291,14 @@ pub fn prescan(svg: &str, most: usize) -> Prescan<'_> { end }; if elements.1.saturating_add(entities.1.saturating_mul(10)) > most { - return Prescan { too_deep: true, markers: Cow::Borrowed(svg) }; + return Prescan { + too_deep: true, + depth: elements.1.saturating_add(entities.1.saturating_mul(10)), + open_depth, + markers: Cow::Borrowed(svg), + }; } + open_depth = open_depth.max(elements.0); } let markers = if data_values.is_empty() { Cow::Borrowed(svg) @@ -303,7 +313,7 @@ pub fn prescan(svg: &str, most: usize) -> Prescan<'_> { text.push_str(svg.get(from..).unwrap_or_default()); Cow::Owned(text) }; - Prescan { too_deep: false, markers } + Prescan { too_deep: false, depth: elements.1.saturating_add(entities.1.saturating_mul(10)), open_depth, markers } } #[cfg(test)] diff --git a/crates/svg/src/preflight.rs b/crates/svg/src/preflight.rs index 3ab381ff3..a11a5d9b7 100644 --- a/crates/svg/src/preflight.rs +++ b/crates/svg/src/preflight.rs @@ -71,6 +71,11 @@ pub fn text<'a>(bytes: &'a [u8], limits: &Limits) -> Result, Error> /// Parses already-decoded SVG text and checks it against `limits`. pub fn parse_text<'a>(svg: &'a str, limits: &Limits) -> Result, Error> { + parse_text_with_depth(svg, limits).map(|(xml, _, _)| xml) +} + +/// Parses and validates SVG text, returning its source nesting depth as well as its XML tree. +pub fn parse_text_with_depth<'a>(svg: &'a str, limits: &Limits) -> Result<(roxmltree::Document<'a>, usize, usize), Error> { if svg.len() > limits.bytes { return Err(Error::Text(crate::svgz::Error::TooLarge(limits.bytes))); } @@ -83,9 +88,11 @@ pub fn parse_text<'a>(svg: &'a str, limits: &Limits) -> Result(bytes: &'a [u8], limits: &Limits) -> Result, Error> { let text = svgz::text(bytes, limits.bytes)?; - checked(&text, limits)?; + preflight(&text, limits)?; Ok(text) } @@ -130,12 +136,30 @@ pub fn parse(bytes: &[u8], limits: &Limits, options: &usvg::Options<'_>) -> Resu /// [`parse`] for SVG text the caller already holds. pub fn parse_text(svg: &str, limits: &Limits, options: &usvg::Options<'_>) -> Result { - let xml = checked(svg, limits)?; - usvg::Tree::from_xmltree(&xml, options).map_err(|e| Error::Parse(e.to_string())) + preflight(svg, limits)?.parse(options) } -/// Every check on SVG text, cheapest first; the XML it parses to on the way. -fn checked<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { +/// SVG text after shared byte, entity, attribute, XML and expansion-depth checks. +/// Keep it opaque so consumers use the same checked tree and don't reparse XML locally. +pub struct Preflight<'s> { + xml: roxmltree::Document<'s>, + depth: usize, +} + +impl Preflight<'_> { + /// Maximum source nesting depth before following references. + pub fn depth(&self) -> usize { + self.depth + } + + /// Converts this already-checked XML into the configured usvg tree. + pub fn parse(self, options: &usvg::Options<'_>) -> Result { + usvg::Tree::from_xmltree(&self.xml, options).map_err(|e| Error::Parse(e.to_string())) + } +} + +/// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion. +pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { let preflight_limits = crate::preflight::Limits { bytes: limits.bytes, depth: limits.depth, @@ -145,7 +169,7 @@ fn checked<'s>(svg: &'s str, limits: &Limits) -> Result, attribute_work: limits.attribute_work, xml_nodes: limits.xml_nodes, }; - let xml = crate::preflight::parse_text(svg, &preflight_limits).map_err(|e| match e { + let (xml, _depth, depth) = crate::preflight::parse_text_with_depth(svg, &preflight_limits).map_err(|e| match e { crate::preflight::Error::Text(e) => Error::Text(e), crate::preflight::Error::Entities => Error::Entities, crate::preflight::Error::EntityMarkup => Error::EntityMarkup, @@ -156,7 +180,58 @@ fn checked<'s>(svg: &'s str, limits: &Limits) -> Result, if height(&xml, limits.tree_depth)?.is_none() { return Err(Error::TooDeep(limits.tree_depth)); } - Ok(xml) + Ok(Preflight { xml, depth }) +} + +/// Dimensions of an SVG viewport in SVG user units. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Viewport { + pub width: f64, + pub height: f64, +} + +/// Wraps a validated SVG root in a new viewport while preserving the original root as content. +/// The original root must have finite, positive pixel or point dimensions. +pub fn rewrite_root_viewport(svg: &str, target: Viewport, limits: &Limits, output_limit: usize) -> Result { + if !target.width.is_finite() || !target.height.is_finite() || target.width <= 0.0 || target.height <= 0.0 { + return Err(Error::Viewport("target dimensions must be finite and positive".into())); + } + let checked = preflight(svg, limits)?; + let root = checked.xml.root_element(); + if root.tag_name().name() != "svg" { + return Err(Error::Viewport("root element is not svg".into())); + } + let root_dimension = |key: &str| { + let value = root.attribute(key)?; + let (number, scale) = value.strip_suffix("pt").map_or((value, 1.0), |n| (n, 4.0 / 3.0)); + number.parse::().ok().map(|n| n * scale).filter(|n| n.is_finite() && *n > 0.0) + }; + let (width, height) = root_dimension("width") + .zip(root_dimension("height")) + .ok_or_else(|| Error::Viewport("source has no finite pixel or point dimensions".into()))?; + let range = root.range(); + let preamble = svg.get(..range.start).ok_or_else(|| Error::Viewport("invalid XML preamble range".into()))?; + let inner = svg.get(range).ok_or_else(|| Error::Viewport("invalid root element range".into()))?; + let prefix = format!( + r#""#, + target.width, target.height, width, height + ); + let len = preamble + .len() + .checked_add(prefix.len()) + .and_then(|n| n.checked_add(inner.len())) + .and_then(|n| n.checked_add(6)) + .ok_or(Error::OutputTooLarge(output_limit))?; + if len > output_limit { + return Err(Error::OutputTooLarge(output_limit)); + } + let mut output = String::new(); + output.try_reserve_exact(len).map_err(|_| Error::OutputTooLarge(output_limit))?; + output.push_str(preamble); + output.push_str(&prefix); + output.push_str(inner); + output.push_str(""); + Ok(output) } /// Properties whose value an element's descendants use too: the paint server they name is From 330280203ce56ec31b329d2c1fd83394a3e5295d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 08:58:12 -0500 Subject: [PATCH 11/98] Reuse bounded preflight for embedded SVG --- crates/svg/src/tree.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index f38dbf8d9..088d8bd89 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -509,17 +509,18 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' } }; used.bytes.fetch_add(text.len(), Ordering::Relaxed); - let xml = match checked(&text, limits) { - Ok(xml) => xml, + let checked = match preflight(&text, limits) { + Ok(checked) => checked, + Err(Error::TooDeep(_)) => { + used.svg_limit.fetch_add(1, Ordering::Relaxed); + return None; + } Err(_) => { used.invalid_svg.fetch_add(1, Ordering::Relaxed); return None; } }; - let Some(xml_depth) = height(&xml, limits.tree_depth).ok().flatten() else { - used.svg_limit.fetch_add(1, Ordering::Relaxed); - return None; - }; + let xml_depth = checked.depth(); let total_depth = used.xml_depth.load(Ordering::Relaxed).checked_add(xml_depth); let Some(total_depth) = total_depth.filter(|depth| *depth <= limits.tree_depth) else { used.svg_limit.fetch_add(1, Ordering::Relaxed); @@ -528,7 +529,7 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' let previous_depth = used.xml_depth.swap(total_depth, Ordering::Relaxed); let depth = used.depth.fetch_add(1, Ordering::Relaxed).saturating_add(1); used.svg_depth.fetch_max(depth, Ordering::Relaxed); - let tree = usvg::Tree::from_xmltree(&xml, options); + let tree = checked.parse(options); used.xml_depth.store(previous_depth, Ordering::Relaxed); used.depth.fetch_sub(1, Ordering::Relaxed); match tree { From 3eae15e8ecd17c5d02d3182659eee8f21d3b74fd Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 09:02:07 -0500 Subject: [PATCH 12/98] Expose bounded unitless SVG viewport metadata --- crates/svg/src/tree.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 088d8bd89..fdabc2a50 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -183,6 +183,21 @@ pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Err Ok(Preflight { xml, depth }) } +/// Reads finite numeric root `width` and `height` values without interpreting physical units. +pub fn root_unitless_size(svg: &str, limits: &Limits) -> Result, Error> { + let checked = preflight(svg, limits)?; + let root = checked.xml.root_element(); + if root.tag_name().name() != "svg" { + return Ok(None); + } + let Some((width, height)) = + root.attribute("width").and_then(|value| value.parse::().ok()).zip(root.attribute("height").and_then(|value| value.parse::().ok())) + else { + return Ok(None); + }; + Ok(Some((width, height))) +} + /// Dimensions of an SVG viewport in SVG user units. #[derive(Clone, Copy, Debug, PartialEq)] pub struct Viewport { From 344dbef231b1791675a01b3ba71e2c41645fbf35 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 09:28:33 -0500 Subject: [PATCH 13/98] Test SVG preflight limits and classify bad embedded SVG --- crates/svg/src/preflight.rs | 26 ++++++++++++++++++++++++++ crates/svg/src/tree.rs | 6 +++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/crates/svg/src/preflight.rs b/crates/svg/src/preflight.rs index a11a5d9b7..463c3ce16 100644 --- a/crates/svg/src/preflight.rs +++ b/crates/svg/src/preflight.rs @@ -96,3 +96,29 @@ pub fn parse_text_with_depth<'a>(svg: &'a str, limits: &Limits) -> Result<(roxml let xml = roxmltree::Document::parse_with_options(svg, options).map_err(|e| Error::Xml(e.to_string()))?; Ok((xml, prescan.depth, prescan.open_depth)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parsed_tree_reports_source_nesting_depth() -> Result<(), Error> { + let limits = Limits { depth: 4, ..Default::default() }; + let (document, depth, open_depth) = parse_text_with_depth("", &limits)?; + assert!(document.root_element().has_tag_name("svg")); + assert_eq!(depth, 2); + assert_eq!(open_depth, 0); + let simple = parse_text("", &limits)?; + assert!(simple.root_element().has_tag_name("svg")); + Ok(()) + } + + #[test] + fn source_depth_and_xml_errors_keep_their_specific_kind() { + let shallow = Limits { depth: 1, ..Default::default() }; + assert!(matches!(parse_text_with_depth("", &shallow), Err(Error::TooDeep(1)))); + + let normal = Limits::default(); + assert!(matches!(parse_text_with_depth("", &normal), Err(Error::Xml(_)))); + } +} diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index fdabc2a50..20dcd61cd 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -517,11 +517,15 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' Ok(text) => text, // What a refused document unpacked before it was refused was work too: it uses up what // is left, so the next image can't repeat it. - Err(_) => { + Err(svgz::Error::TooLarge(_)) => { used.bytes.fetch_add(left, Ordering::Relaxed); used.svg_limit.fetch_add(1, Ordering::Relaxed); return None; } + Err(_) => { + used.invalid_svg.fetch_add(1, Ordering::Relaxed); + return None; + } }; used.bytes.fetch_add(text.len(), Ordering::Relaxed); let checked = match preflight(&text, limits) { From 35a16a63fd55506c64f0c2b2556af7497897640b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sat, 14 Nov 2026 23:56:16 +0000 Subject: [PATCH 14/98] fix(image-codecs): preserve AVIF EXIF and HLG levels --- crates/image-codecs/src/codecs/avif.rs | 45 ++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index d27112fa1..cef4e3a8d 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -25,12 +25,17 @@ fn pq_encode(nits: f32) -> f32 { #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] fn hlg_signal(rgb: [f32; 3]) -> [f32; 3] { + const K: [f32; 3] = [0.2627, 0.6780, 0.0593]; + const GAMMA: f32 = 1.2; const A: f32 = 0.17883277; const B: f32 = 0.28466892; const C: f32 = 0.559_910_7; - rgb.map(|v| { - let s = if v.is_finite() { v.clamp(0.0, 12.0) / 12.0 } else { 0.0 }; - if s <= 1.0 / 12.0 { (3.0 * s).sqrt() } else { A * (12.0 * s - B).ln() + C } + let display = rgb.map(|v| (if v.is_finite() { v.max(0.0) } else { 0.0 }) * (HDR_REFERENCE_WHITE_NITS / crate::cicp::HLG_PEAK_NITS)); + let y = K[0] * display[0] + K[1] * display[1] + K[2] * display[2]; + let inverse_ootf = if y > 0.0 { y.powf((1.0 - GAMMA) / GAMMA) } else { 0.0 }; + display.map(|v| { + let scene = (v * inverse_ootf).clamp(0.0, 1.0); + if scene <= 1.0 / 12.0 { (3.0 * scene).sqrt() } else { A * (12.0 * scene - B).ln() + C } }) } @@ -41,6 +46,9 @@ pub fn encode_avif_with_speed(img: &Image, quality: u8, speed: u8) -> Result Result Date: Sun, 15 Nov 2026 00:10:45 +0000 Subject: [PATCH 15/98] fix(image-codecs): place AVIF tests after encoders --- crates/image-codecs/src/codecs/avif.rs | 75 +++++++++++++++----------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index cef4e3a8d..33d86b011 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -92,37 +92,6 @@ pub fn encode_avif_with_speed(img: &Image, quality: u8, speed: u8) -> Result Date: Sun, 15 Nov 2026 00:40:25 +0000 Subject: [PATCH 16/98] test(avif): assert HLG color metadata without decoder --- crates/image-codecs/src/codecs/avif.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index 33d86b011..c8660f2ea 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -248,16 +248,13 @@ mod tests { #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] #[test] - fn hlg_encoded_output_decodes_to_reference_white_and_peak() { - let decode_gray = |linear: f32| { - let avif = super::encode_avif_hdr(1, 1, &[linear; 3], 100, 10, None, super::HdrTransfer::Hlg).unwrap(); - image::load_from_memory_with_format(&avif, image::ImageFormat::Avif).unwrap().to_rgb32f().get_pixel(0, 0)[0] - }; - - let white = decode_gray(1.0); - assert!((white - 0.75).abs() < 0.02, "decoded HLG SDR white: {white}"); - - let peak = decode_gray(crate::cicp::HLG_PEAK_NITS / super::HDR_REFERENCE_WHITE_NITS); - assert!((peak - 1.0).abs() < 0.02, "decoded HLG peak: {peak}"); + fn hlg_encoded_output_signals_bt2020_hlg() { + let avif = super::encode_avif_hdr(1, 1, &[1.0; 3], 100, 10, None, super::HdrTransfer::Hlg).unwrap(); + let nclx = avif.windows(4).position(|window| window == b"nclx").expect("the AVIF has nclx color metadata"); + let color = avif.get(nclx + 4..nclx + 11).expect("complete nclx color metadata"); + assert_eq!(&color[..2], &[0, 9], "BT.2020 primaries"); + assert_eq!(&color[2..4], &[0, 18], "BT.2100 HLG transfer"); + assert_eq!(&color[4..6], &[0, 9], "BT.2020 non-constant luminance matrix"); + assert!(!avif.windows(4).any(|window| window == b"clli"), "HLG does not signal PQ content-light metadata"); } } From c0ce0553840d9a57b2349321f98e9d6c26ecc712 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sat, 14 Nov 2026 20:04:36 -0500 Subject: [PATCH 17/98] image-codecs: bound PNG row streaming independently of full output --- crates/image-codecs/src/codecs/png.rs | 65 ++++++++++++++-- crates/image-codecs/tests/it/png_jpeg.rs | 94 ++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 7 deletions(-) diff --git a/crates/image-codecs/src/codecs/png.rs b/crates/image-codecs/src/codecs/png.rs index 4ceb75403..9965dfc30 100644 --- a/crates/image-codecs/src/codecs/png.rs +++ b/crates/image-codecs/src/codecs/png.rs @@ -141,12 +141,12 @@ impl> PngDecodeRows { /// Reads the header and refuses what [`decode`] refuses before any pixel is decoded: sizes /// past `limits`, and a file too small for the image it declares. pub fn new(bytes: B, limits: &Limits) -> Result { - Self::open(bytes, limits, true) + Self::open(bytes, limits, true, false) } /// [`PngDecodeRows::new`], optionally without reserving the whole image: a caller that hands /// the rows on as they come ([`decode_rows`]) never holds them all. - fn open(bytes: B, limits: &Limits, reserve: bool) -> Result { + fn open(bytes: B, limits: &Limits, reserve: bool, bounded_stream: bool) -> Result { let at = std::sync::Arc::new(std::sync::atomic::AtomicU64::new(0)); let bytes = SharedInput(std::sync::Arc::new(bytes)); let input = Tracked { cursor: Cursor::new(bytes.clone()), at: at.clone() }; @@ -187,7 +187,14 @@ impl> PngDecodeRows { let (width, height) = info.frame_control.map_or_else(|| info.size(), |f| (f.width, f.height)); let line = reader.output_line_size(width).ok_or_else(|| err("image too large"))?; // The output layout is known: charge its real bytes per pixel, not RGBA16's worst case. - limits.check(width, height, layout, sample)?; + if bounded_stream { + limits.check_size(width, height)?; + if line > limits.alloc_usize() { + return Err(CodecError::LimitExceeded("one PNG row exceeds max_alloc".into())); + } + } else { + limits.check(width, height, layout, sample)?; + } let height = usize::try_from(height).map_err(|_| err("image too large"))?; if width == 0 || height == 0 { return Err(err("the image is empty")); @@ -234,11 +241,35 @@ impl> PngDecodeRows { /// decode the whole image after all (an interlaced file) calls /// [`PngDecodeRows::reserve_output`] before [`PngDecodeRows::step`]. pub fn new_streaming(bytes: B, limits: &Limits) -> Result { - Self::open(bytes, limits, false) + Self::open(bytes, limits, false, false) + } + + /// Opens a bounded row stream without charging a full image that is never retained. + /// Dimension, pixel-count, plausibility, decoder and metadata limits remain unchanged; + /// `max_alloc` also bounds each row group. It is not a total process-memory limit. + /// Unlike [`Self::new_streaming`], a streamable image may exceed the full-image budget. + /// Non-streamable images still require that budget, as do [`Self::step`] and + /// [`Self::reserve_output`]. The caller must bound any output it retains in its sink. + pub fn new_bounded_streaming(bytes: B, limits: &Limits) -> Result { + let decoder = Self::open(bytes, limits, false, true)?; + if !decoder.streamable() { + decoder.check_full_output()?; + } + Ok(decoder) + } + + fn check_full_output(&self) -> Result<(), CodecError> { + let height = u32::try_from(self.height).map_err(|_| err("image too large"))?; + self.limits.check(self.width, height, self.layout, self.sample)?; + if self.size > self.limits.alloc_usize() { + return Err(CodecError::LimitExceeded("PNG output exceeds max_alloc".into())); + } + Ok(()) } /// Reserves the buffer [`PngDecodeRows::step`] decodes the whole image into. pub fn reserve_output(&mut self) -> Result<(), CodecError> { + self.check_full_output()?; let size = self.size; self.buf.try_reserve_exact(size).map_err(|_| CodecError::LimitExceeded(format!("cannot allocate {size} bytes for the image"))) } @@ -271,6 +302,7 @@ impl> PngDecodeRows { /// Decodes at least one row (the whole image if it is interlaced), then goes on while `more()` /// returns true. `Some` once the image is complete, after which the decoder is spent. pub fn step(&mut self, more: &mut dyn FnMut() -> bool) -> Result, CodecError> { + self.check_full_output()?; if self.done { return Err(err("the image was already decoded")); } @@ -338,7 +370,9 @@ impl> PngDecodeRows { /// `more()` returns true; `Some` with the extras once the file is read to its end, with the /// resolution a format that lacks one takes from its EXIF or XMP. A damaged file fails with /// the error [`decode`] gives, after however many rows were delivered. An orientation (an - /// `eXIf` after the image data may set one) is for the caller to apply. + /// `eXIf` after the image data may set one) is for the caller to apply. A new group size + /// must exceed any partial group retained from an earlier yield; otherwise this returns + /// an error without consuming another row. pub fn step_stream>( &mut self, group: u32, @@ -348,12 +382,29 @@ impl> PngDecodeRows { if self.done { return Err(err("the image was already decoded").into()); } - let group = usize::try_from(group.max(1)).unwrap_or(usize::MAX); + // Tail EXIF may report an orientation after rows have already been delivered; + // that belongs in the returned extras, not a rejection of metadata continuation. + if self.rows == 0 && !self.tail && !self.streamable() { + return Err(err("this PNG requires whole-image decoding").into()); + } + let group = usize::try_from(group.max(1)).unwrap_or(usize::MAX).min(self.height); + if self.rows - self.group_start >= group { + return Err(err("PNG row group is smaller than rows held from the previous step").into()); + } + let group_bytes = group.checked_mul(self.line).ok_or_else(|| err("PNG row group is too large"))?; + if group_bytes > self.limits.alloc_usize() { + return Err(CodecError::LimitExceeded("PNG row group exceeds max_alloc".into()).into()); + } if !self.tail { while self.rows < self.height { let held = self.rows - self.group_start; let at = held.checked_mul(self.line).ok_or_else(|| err("image too large"))?; let end = at.checked_add(self.line).ok_or_else(|| err("image too large"))?; + // A caller may change its group size between yields. Bound the actual held + // bytes as well as the requested group before growing the row buffer. + if end > self.limits.alloc_usize() { + return Err(CodecError::LimitExceeded("PNG row group exceeds max_alloc".into()).into()); + } if self.buf.len() < end { self.buf .try_reserve_exact(end - self.buf.len()) @@ -368,7 +419,7 @@ impl> PngDecodeRows { to_native_endian(row); } self.rows += 1; - if self.rows - self.group_start == group || self.rows == self.height { + if self.rows - self.group_start >= group || self.rows == self.height { let data = self.buf.get(..end).ok_or_else(|| err("invalid PNG row group"))?; sink(u32::try_from(self.group_start).map_err(|_| err("image too large"))?, data)?; self.group_start = self.rows; diff --git a/crates/image-codecs/tests/it/png_jpeg.rs b/crates/image-codecs/tests/it/png_jpeg.rs index 6b213ea83..07c57bf24 100644 --- a/crates/image-codecs/tests/it/png_jpeg.rs +++ b/crates/image-codecs/tests/it/png_jpeg.rs @@ -665,3 +665,97 @@ fn step_stream_gives_the_pixels_and_extras_of_decode() { fn dec_line(w: u32, layout: ChannelLayout, sample: SampleType) -> usize { w as usize * layout.channels() * sample.bytes() } + +#[test] +fn bounded_png_stream_does_not_charge_unretained_full_frame() { + let image = synth(256, 1024, ChannelLayout::Rgba, SampleType::U16, 77, 0.5); + let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap(); + let limits = Limits { max_alloc: 512 * 1024, ..Limits::default() }; + assert!(image.data().len() > limits.max_alloc as usize); + assert!(PngDecodeRows::new(bytes.as_slice(), &limits).is_err()); + assert!(PngDecodeRows::new_streaming(bytes.as_slice(), &limits).is_err()); + let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &limits).unwrap(); + assert!(decoder.streamable()); + // Opening a stream must not become a way to bypass whole-image allocation guards. + assert!(decoder.reserve_output().is_err()); + assert!(decoder.step(&mut || true).is_err()); + let mut next = 0; + let mut sink = |first: u32, rows: &[u8]| -> Result<(), CodecError> { + assert_eq!(first as usize, next); + assert!(rows.len() <= 3 * 256 * 8); + let start = next * 256 * 8; + assert_eq!(rows, &image.data()[start..start + rows.len()]); + next += rows.len() / (256 * 8); + Ok(()) + }; + while decoder.step_stream(3, &mut || false, &mut sink).unwrap().is_none() {} + assert_eq!(next, 1024); +} + +#[test] +fn bounded_png_stream_keeps_group_dimension_and_fallback_limits() { + let image = synth(256, 1024, ChannelLayout::Rgba, SampleType::U16, 81, 0.5); + let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap(); + let limits = Limits { max_alloc: 512 * 1024, ..Limits::default() }; + let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &limits).unwrap(); + assert!(decoder.step_stream::(1024, &mut || true, &mut |_, _| panic!("oversized group reached sink")).is_err()); + for strict in [ + Limits { max_width: 255, ..limits }, + Limits { max_height: 1023, ..limits }, + Limits { max_pixels: 256 * 1024 - 1, ..limits }, + Limits { max_alloc: 1024, ..limits }, + ] { + assert!(PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &strict).is_err()); + } + let adam7 = interlaced(&image); + assert!(PngDecodeRows::new_bounded_streaming(adam7.as_slice(), &limits).is_err()); +} + +#[test] +fn bounded_png_stream_rejects_shrinking_below_buffered_rows() { + let image = synth(16, 8, ChannelLayout::Rgba, SampleType::U16, 83, 0.5); + let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap(); + let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &Limits::default()).unwrap(); + for _ in 0..3 { + assert!(decoder.step_stream::(4, &mut || false, &mut |_, _| panic!("group is incomplete")).unwrap().is_none()); + } + assert!(decoder.step_stream::(1, &mut || true, &mut |_, _| panic!("changed group reached sink")).is_err()); + let mut output = Vec::new(); + decoder + .step_stream::(4, &mut || true, &mut |_, rows| { + output.extend_from_slice(rows); + Ok(()) + }) + .unwrap(); + assert_eq!(output, image.data()); +} + +#[test] +fn bounded_png_stream_preserves_late_exif_across_metadata_yields() { + let image = synth(16, 8, ChannelLayout::Rgba, SampleType::U16, 89, 0.5); + let bytes = encode(&image, Format::Png, &EncodeOptions::default()).unwrap(); + let mut exif = sample_exif(); + exif[18..22].copy_from_slice(&6u32.to_le_bytes()); + let mut chunks = vec![png_chunk(b"eXIf", &exif)]; + for i in 0..150 { + chunks.push(png_chunk(b"tEXt", format!("key{i}\0value").as_bytes())); + } + let bytes = with_chunks_before_iend(&bytes, &chunks); + let mut decoder = PngDecodeRows::new_bounded_streaming(bytes.as_slice(), &Limits::default()).unwrap(); + assert!(decoder.streamable()); + let mut output = Vec::new(); + let extras = loop { + if let Some(extras) = decoder + .step_stream::(2, &mut || false, &mut |_, rows| { + output.extend_from_slice(rows); + Ok(()) + }) + .unwrap() + { + break extras; + } + }; + assert_eq!(output, image.data()); + assert_eq!(extras.orientation, 6); + assert_eq!(extras.meta.text.len(), 150); +} From 30b78d048ad1bb151f51cbc99c72057a23137cd8 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 10:41:10 +0000 Subject: [PATCH 18/98] Preserve requested AVIF alpha quality --- crates/image-codecs/src/codecs/avif.rs | 28 ++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index c8660f2ea..cf37a11a1 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -44,8 +44,11 @@ fn hlg_signal(rgb: [f32; 3]) -> [f32; 3] { pub fn encode_avif_with_speed(img: &Image, quality: u8, speed: u8) -> Result, CodecError> { #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] { - let mut enc = - ravif::Encoder::new().with_quality(f32::from(quality.clamp(1, 100))).with_speed(speed.clamp(1, 10)).with_bit_depth(ravif::BitDepth::Ten); + let mut enc = ravif::Encoder::new() + .with_quality(f32::from(quality.clamp(1, 100))) + .with_alpha_quality(f32::from(quality.clamp(1, 100))) + .with_speed(speed.clamp(1, 10)) + .with_bit_depth(ravif::BitDepth::Ten); if let Some(exif) = img.meta.exif.as_deref() { enc = enc.with_exif(exif); } @@ -220,6 +223,27 @@ pub fn encode_avif_hdr( #[cfg(test)] mod tests { + #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] + #[test] + fn sdr_avif_uses_requested_alpha_quality() { + use crate::{ChannelLayout, Image}; + + let pixels: Vec = (0..1024).map(|i| ravif::RGBA8::new(100, 120, 140, ((i * 37) % 256) as u8)).collect(); + let samples: Vec = pixels.iter().flat_map(|p| [p.r, p.g, p.b, p.a]).collect(); + let image = Image::from_u8(32, 32, ChannelLayout::Rgba, samples).unwrap(); + let actual = super::encode_avif_with_speed(&image, 20, 10).unwrap(); + let expected = ravif::Encoder::new() + .with_quality(20.0) + .with_alpha_quality(20.0) + .with_speed(10) + .with_bit_depth(ravif::BitDepth::Ten) + .with_alpha_color_mode(ravif::AlphaColorMode::UnassociatedClean) + .encode_rgba(ravif::Img::new(&pixels, 32, 32)) + .unwrap() + .avif_file; + assert!(actual == expected, "encoded alpha must use the requested quality"); + } + #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] #[test] fn sdr_avif_keeps_exif_metadata() { From a8f2612c7c8ac00256ab4a9e1a0eb1897bc5a61f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 10:46:51 +0000 Subject: [PATCH 19/98] Test decoded HLG AVIF reference levels --- Cargo.lock | 1 + crates/image-codecs/Cargo.toml | 1 + crates/image-codecs/src/codecs/avif.rs | 27 ++++++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 9b4662232..8f3ea8e4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1861,6 +1861,7 @@ dependencies = [ "craft-heif", "craft-jpeg", "craft-metadata", + "craft-video", "exr", "flate2", "half", diff --git a/crates/image-codecs/Cargo.toml b/crates/image-codecs/Cargo.toml index 2dc9426ac..637779d99 100644 --- a/crates/image-codecs/Cargo.toml +++ b/crates/image-codecs/Cargo.toml @@ -47,4 +47,5 @@ proptest = { workspace = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +craft-video = { workspace = true, features = ["av1-decode"] } jxl-oxide = { version = "=0.12.6", default-features = false, features = ["rayon"] } diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index cf37a11a1..4912c75bf 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -281,4 +281,31 @@ mod tests { assert_eq!(&color[4..6], &[0, 9], "BT.2020 non-constant luminance matrix"); assert!(!avif.windows(4).any(|window| window == b"clli"), "HLG does not signal PQ content-light metadata"); } + + #[cfg(all(feature = "avif", not(target_arch = "wasm32")))] + #[test] + fn hlg_encoded_pixels_preserve_reference_white_and_peak() { + let mut rgb = Vec::new(); + for _ in 0..32 { + for x in 0..64 { + let level = if x < 32 { 1.0 } else { crate::cicp::HLG_PEAK_NITS / super::HDR_REFERENCE_WHITE_NITS }; + rgb.extend_from_slice(&[level; 3]); + } + } + let avif = super::encode_avif_hdr(64, 32, &rgb, 100, 10, None, super::HdrTransfer::Hlg).unwrap(); + let mut offset = 0; + let payload = loop { + let header = avif.get(offset..offset + 8).expect("complete AVIF box header"); + let size = u32::from_be_bytes(header[..4].try_into().unwrap()) as usize; + assert!(size >= 8 && offset + size <= avif.len(), "valid AVIF box size"); + if &header[4..] == b"mdat" { + break &avif[offset + 8..offset + size]; + } + offset += size; + }; + let frames = craft_video::av1::decode::Decoder::new().decode(payload).unwrap(); + let luma = &frames.first().expect("encoded AV1 frame").planes[0]; + assert!((i32::from(luma[16 * 64 + 16]) - 767).abs() <= 5, "diffuse white"); + assert!((i32::from(luma[16 * 64 + 48]) - 1023).abs() <= 5, "peak white"); + } } From 3401d0af8ec5d410d160866d7a5b594c65b7c3f0 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 05:58:00 -0500 Subject: [PATCH 20/98] pcraft-format: extract PhotoCraft brush tip envelope and samples Port PhotoCraft preset_store.rs at 19aec5eb44131de44361189d64910e7b3e19e57c. Preserve original primary author and later codec contributions. Co-authored-by: David Mann Co-authored-by: FinnNk --- Cargo.lock | 1 + crates/pcraft-format/Cargo.toml | 3 ++ crates/pcraft-format/README.md | 14 ++++- crates/pcraft-format/src/lib.rs | 3 ++ crates/pcraft-format/src/tip.rs | 91 +++++++++++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 crates/pcraft-format/src/tip.rs diff --git a/Cargo.lock b/Cargo.lock index 9a600099d..887c8ccd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2243,6 +2243,7 @@ dependencies = [ name = "craft-pcraft-format" version = "0.1.0" dependencies = [ + "flate2", "serde", "serde_json", "thiserror 2.0.21", diff --git a/crates/pcraft-format/Cargo.toml b/crates/pcraft-format/Cargo.toml index ce1d6873f..f54bb0c82 100644 --- a/crates/pcraft-format/Cargo.toml +++ b/crates/pcraft-format/Cargo.toml @@ -16,3 +16,6 @@ serde = { workspace = true } # the nesting against `Limits::max_depth` before parsing. serde_json = { workspace = true, features = ["unbounded_depth"] } thiserror = { workspace = true } + +# Same pure-Rust backend as the PhotoCraft source codec. MIT OR Apache-2.0. +flate2 = { version = "1.1.10", default-features = false, features = ["zlib-rs", "runtime_detection"] } diff --git a/crates/pcraft-format/README.md b/crates/pcraft-format/README.md index 5d8869b74..f5b0dfa73 100644 --- a/crates/pcraft-format/README.md +++ b/crates/pcraft-format/README.md @@ -5,7 +5,7 @@ references, the entry paths, the limits for reading a manifest, and the runner f migrations. The document model stays in PhotoCraft and is a type parameter here. The crate does not read ZIP archives, open files or decode tiles. No `unsafe`, no panics on malformed input, and it builds for `wasm32-unknown-unknown`. The dependencies are `serde`, `serde_json` -and `thiserror`. +`thiserror`, and `flate2`. ```toml craft-pcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "" } @@ -120,3 +120,15 @@ limits.max_manifest_bytes = 16 << 20; `.github/workflows/pcraft-format.yml` runs fmt, clippy and the tests on Linux and Windows, and a `wasm32-unknown-unknown` check. + +## Brush-tip envelope + +`tip::encode`, `tip::decode` and `tip::decode_with_max_side` share PhotoCraft's PCTIP1 +brush-tip codec: a six-byte magic, one-byte depth, little-endian width/height and zlib +samples. Eight-bit-compatible samples use 8 bits, otherwise 16-bit little-endian. +Dimensions, sample count, exact inflated length and caller preview side limits are checked. +Preset group schemas, tip hashes and storage policy remain application-owned. + +Ported from `storytold/photocraft` commit `19aec5eb44131de44361189d64910e7b3e19e57c`, +`crates/engine/src/preset_store.rs`. Original codec by Brandon Thomas (PhotoCraft #82), +with later bounds/backend/storage changes by David Mann and FinnNk, under MIT OR Apache-2.0. diff --git a/crates/pcraft-format/src/lib.rs b/crates/pcraft-format/src/lib.rs index 18868da2d..f12bc5213 100644 --- a/crates/pcraft-format/src/lib.rs +++ b/crates/pcraft-format/src/lib.rs @@ -36,6 +36,9 @@ mod limits; mod migrate; mod read; +/// Lossless brush-tip envelope used by PhotoCraft preset libraries. +pub mod tip; + use serde::{Deserialize, Serialize}; pub use limits::{ diff --git a/crates/pcraft-format/src/tip.rs b/crates/pcraft-format/src/tip.rs new file mode 100644 index 000000000..cfcc4b805 --- /dev/null +++ b/crates/pcraft-format/src/tip.rs @@ -0,0 +1,91 @@ +//! PhotoCraft's PCTIP1 brush-tip envelope and lossless grayscale sample serialization. +//! +//! Extracted from PhotoCraft preset_store.rs at 19aec5eb44131de44361189d64910e7b3e19e57c. +//! Original implementation: Brandon Thomas; subsequent codec changes: David Mann and FinnNk. +//! Group schemas, file naming, content hashes and storage policy remain with the application. +use std::io::Read; + +/// Largest accepted tip side. +pub const MAX_SIDE: u32 = 16384; +/// Largest accepted sample count. +pub const MAX_SAMPLES: u64 = 1 << 26; +const TIP_MAGIC: &[u8; 6] = b"PCTIP1"; +const TIP_HEADER: usize = 6 + 1 + 4 + 4; + +/// Decoded grayscale samples. Eight-bit samples expand losslessly by multiplication by 257. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tip { + pub width: u32, + pub height: u32, + pub data: Vec, +} + +/// `PCTIP1`, bits (8|16), width, height (u32 LE), then the zlib-deflated samples. +pub fn encode(width: u32, height: u32, data: &[u16]) -> Vec { + use std::io::Write; + let eight = data.iter().all(|v| v % 257 == 0); + let mut out = Vec::with_capacity(TIP_HEADER + data.len() / 4); + out.extend_from_slice(TIP_MAGIC); + out.push(if eight { 8 } else { 16 }); + out.extend_from_slice(&width.to_le_bytes()); + out.extend_from_slice(&height.to_le_bytes()); + // Level 3: zlib-rs level 1 writes static Huffman blocks only, 13-30% bigger than miniz's level 1. + let mut z = flate2::write::ZlibEncoder::new(out, flate2::Compression::new(3)); + let mut buf = Vec::with_capacity(8192); + let mut ok = true; + for chunk in data.chunks(4096) { + buf.clear(); + if eight { + buf.extend(chunk.iter().map(|v| (v / 257) as u8)); + } else { + for v in chunk { + buf.extend_from_slice(&v.to_le_bytes()); + } + } + ok &= z.write_all(&buf).is_ok(); + } + match z.finish() { + Ok(v) if ok => v, + _ => Vec::new(), + } +} + +/// Inverse of [`encode`]; any malformed, truncated or oversized input is an error. +pub fn decode(bytes: &[u8]) -> Result { + decode_with_max_side(bytes, MAX_SIDE) +} + +/// [`decode`] refusing (before inflating anything) a bitmap with a side over `max_side`. +pub fn decode_with_max_side(bytes: &[u8], max_side: u32) -> Result { + let head = bytes.get(..TIP_HEADER).ok_or("truncated tip header")?; + if &head[..6] != TIP_MAGIC { + return Err("not a PhotoCraft tip".into()); + } + let bits = head[6]; + let u32_at = |i: usize| u32::from_le_bytes([head[i], head[i + 1], head[i + 2], head[i + 3]]); + let (w, h) = (u32_at(7), u32_at(11)); + if w == 0 || h == 0 || w > MAX_SIDE.min(max_side) || h > MAX_SIDE.min(max_side) { + return Err(format!("bad tip size {w}×{h}")); + } + let n = u64::from(w) * u64::from(h); + if n > MAX_SAMPLES { + return Err(format!("tip too large ({w}×{h})")); + } + let bytes_per = match bits { + 8 => 1u64, + 16 => 2, + b => return Err(format!("unsupported tip depth {b}")), + }; + let want = n * bytes_per; + let mut raw = Vec::with_capacity(want as usize); + flate2::read::ZlibDecoder::new(&bytes[TIP_HEADER..]).take(want + 1).read_to_end(&mut raw).map_err(|e| format!("tip data: {e}"))?; + if raw.len() as u64 != want { + return Err(format!("tip data has {} bytes, expected {want}", raw.len())); + } + let data: Vec = if bits == 8 { + raw.iter().map(|&v| u16::from(v) * 257).collect() + } else { + raw.as_chunks::<2>().0.iter().map(|c| u16::from_le_bytes(*c)).collect() + }; + Ok(Tip { width: w, height: h, data }) +} From dd866b846c558eb2bd95edd52cf18e4a6b0e0c4d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 05:58:40 -0500 Subject: [PATCH 21/98] pcraft-format: validate tip inputs and zlib completion with parity tests --- crates/pcraft-format/src/tip.rs | 17 +++++++-- crates/pcraft-format/tests/tip.rs | 58 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 3 deletions(-) create mode 100644 crates/pcraft-format/tests/tip.rs diff --git a/crates/pcraft-format/src/tip.rs b/crates/pcraft-format/src/tip.rs index cfcc4b805..9b971f5d9 100644 --- a/crates/pcraft-format/src/tip.rs +++ b/crates/pcraft-format/src/tip.rs @@ -3,7 +3,6 @@ //! Extracted from PhotoCraft preset_store.rs at 19aec5eb44131de44361189d64910e7b3e19e57c. //! Original implementation: Brandon Thomas; subsequent codec changes: David Mann and FinnNk. //! Group schemas, file naming, content hashes and storage policy remain with the application. -use std::io::Read; /// Largest accepted tip side. pub const MAX_SIDE: u32 = 16384; @@ -21,8 +20,13 @@ pub struct Tip { } /// `PCTIP1`, bits (8|16), width, height (u32 LE), then the zlib-deflated samples. +/// Returns an empty vector for invalid dimensions/sample counts or compression failure. pub fn encode(width: u32, height: u32, data: &[u16]) -> Vec { use std::io::Write; + let count = u64::from(width) * u64::from(height); + if width == 0 || height == 0 || width > MAX_SIDE || height > MAX_SIDE || count > MAX_SAMPLES || count != data.len() as u64 { + return Vec::new(); + } let eight = data.iter().all(|v| v % 257 == 0); let mut out = Vec::with_capacity(TIP_HEADER + data.len() / 4); out.extend_from_slice(TIP_MAGIC); @@ -77,8 +81,15 @@ pub fn decode_with_max_side(bytes: &[u8], max_side: u32) -> Result b => return Err(format!("unsupported tip depth {b}")), }; let want = n * bytes_per; - let mut raw = Vec::with_capacity(want as usize); - flate2::read::ZlibDecoder::new(&bytes[TIP_HEADER..]).take(want + 1).read_to_end(&mut raw).map_err(|e| format!("tip data: {e}"))?; + // One byte beyond the declared size detects overlong output, without allocating for + // an attacker-controlled expansion. Require the zlib trailer, not merely enough samples. + let mut raw = Vec::with_capacity(want as usize + 1); + let status = flate2::Decompress::new(true) + .decompress_vec(&bytes[TIP_HEADER..], &mut raw, flate2::FlushDecompress::Finish) + .map_err(|e| format!("tip data: {e}"))?; + if status != flate2::Status::StreamEnd { + return Err("tip data: truncated or oversized zlib stream".into()); + } if raw.len() as u64 != want { return Err(format!("tip data has {} bytes, expected {want}", raw.len())); } diff --git a/crates/pcraft-format/tests/tip.rs b/crates/pcraft-format/tests/tip.rs new file mode 100644 index 000000000..e54b44ac8 --- /dev/null +++ b/crates/pcraft-format/tests/tip.rs @@ -0,0 +1,58 @@ +use craft_pcraft_format::tip::{MAX_SAMPLES, MAX_SIDE, decode, decode_with_max_side, encode}; +use std::io::Write; + +fn fixture(bits: u8, width: u32, height: u32, samples: &[u8]) -> Vec { + let mut out = b"PCTIP1".to_vec(); + out.push(bits); + out.extend(width.to_le_bytes()); + out.extend(height.to_le_bytes()); + let mut z = flate2::write::ZlibEncoder::new(out, flate2::Compression::new(3)); + z.write_all(samples).unwrap(); + z.finish().unwrap() +} + +#[test] +fn legacy_eight_and_sixteen_bit_fixtures_and_roundtrips() { + for (bits, raw, samples) in [(8, vec![0, 128, 255], vec![0, 32896, 65535]), (16, vec![1, 0, 52, 18, 255, 255], vec![1, 0x1234, 65535])] { + let legacy = fixture(bits, 3, 1, &raw); + let tip = decode(&legacy).unwrap(); + assert_eq!((tip.width, tip.height), (3, 1)); + assert_eq!(tip.data, samples); + let encoded = encode(3, 1, &samples); + assert_eq!(&encoded[..15], &legacy[..15]); + assert_eq!(decode(&encoded).unwrap(), tip); + } +} + +#[test] +fn checks_headers_limits_and_sample_lengths() { + let good = fixture(8, 1, 1, &[255]); + for n in 0..good.len() { + assert!(decode(&good[..n]).is_err(), "truncation at {n}"); + } + let mut bad = good.clone(); + bad[0] = b'X'; + assert!(decode(&bad).is_err()); + for (bits, w, h) in [(7, 1, 1), (8, 0, 1), (8, 1, 0), (8, MAX_SIDE + 1, 1), (8, MAX_SIDE, MAX_SIDE)] { + assert!(decode(&fixture(bits, w, h, &[])).is_err()); + } + assert!(MAX_SAMPLES < u64::from(MAX_SIDE) * u64::from(MAX_SIDE)); + for raw in [vec![], vec![0, 1], vec![0; 10000]] { + assert!(decode(&fixture(8, 1, 1, &raw)).is_err()); + } + assert!(decode(&fixture(16, 1, 1, &[1])).is_err()); + let mut checksum = good; + *checksum.last_mut().unwrap() ^= 1; + assert!(decode(&checksum).is_err()); + assert!(encode(0, 1, &[]).is_empty()); + assert!(encode(1, 1, &[]).is_empty()); + assert!(encode(MAX_SIDE + 1, 1, &[0]).is_empty()); +} + +#[test] +fn preview_side_limit_is_checked_before_inflation() { + let good = encode(2, 1, &[0, 65535]); + assert!(decode_with_max_side(&good, 1).is_err()); + assert!(decode_with_max_side(&good, 2).is_ok()); + assert!(decode_with_max_side(&good, 0).is_err()); +} From ec2b4976f52fe42c17f0954be08e7f72d9c8eb77 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 22:49:07 -0400 Subject: [PATCH 22/98] Import original PhotoCraft camera EXIF module Source: storytold/photocraft commit aa87012aab6d710b1e1d2ff1654a45e58c09d955 --- crates/metadata/src/camera_fields.rs | 240 +++++++++++++++++++++++++++ crates/metadata/src/lib.rs | 1 + 2 files changed, 241 insertions(+) create mode 100644 crates/metadata/src/camera_fields.rs diff --git a/crates/metadata/src/camera_fields.rs b/crates/metadata/src/camera_fields.rs new file mode 100644 index 000000000..445f51498 --- /dev/null +++ b/crates/metadata/src/camera_fields.rs @@ -0,0 +1,240 @@ +//! Minimal EXIF reader for the camera settings the photography commands use (exposure, aperture, +//! ISO, focal length, camera and lens names). Input is the raw TIFF-structured EXIF payload +//! (`II*\0` / `MM\0*`, as stored in `Document.metadata.exif`); malformed data yields `None` +//! fields, never a panic. Tag numbers are from the EXIF 2.32 specification (CIPA DC-008). + +/// Camera settings read from EXIF. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CameraInfo { + pub make: Option, + pub model: Option, + pub lens: Option, + /// Seconds. + pub exposure_time: Option, + pub f_number: Option, + pub iso: Option, + /// Millimetres. + pub focal_length: Option, + /// 35 mm-equivalent focal length in millimetres. + pub focal_length_35mm: Option, +} + +impl CameraInfo { + /// Linear exposure factor `t · ISO / N²` (relative brightness of the image for one scene). + pub fn exposure_factor(&self) -> Option { + let t = self.exposure_time?; + let n = self.f_number.unwrap_or(1.0).max(0.1); + Some(t * self.iso.unwrap_or(100.0) / 100.0 / (n * n)) + } +} + +struct Tiff<'a> { + b: &'a [u8], + le: bool, +} + +impl Tiff<'_> { + fn u16(&self, o: usize) -> Option { + let s = self.b.get(o..o + 2)?; + Some(if self.le { u16::from_le_bytes([s[0], s[1]]) } else { u16::from_be_bytes([s[0], s[1]]) }) + } + fn u32(&self, o: usize) -> Option { + let s = self.b.get(o..o + 4)?; + let a = [s[0], s[1], s[2], s[3]]; + Some(if self.le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) }) + } + /// Entries of the IFD at `off`: (tag, type, count, value offset field position). + fn ifd(&self, off: usize) -> Vec<(u16, u16, u32, usize)> { + let Some(n) = self.u16(off) else { return Vec::new() }; + (0..n as usize) + .filter_map(|k| { + let e = off + 2 + k * 12; + Some((self.u16(e)?, self.u16(e + 2)?, self.u32(e + 4)?, e + 8)) + }) + .collect() + } + fn data_at(&self, typ: u16, count: u32, field: usize) -> Option { + let size = match typ { + 1 | 2 | 6 | 7 => 1, + 3 | 8 => 2, + 4 | 9 | 11 => 4, + 5 | 10 | 12 => 8, + _ => return None, + } * count as usize; + if size <= 4 { Some(field) } else { self.u32(field).map(|v| v as usize) } + } + fn number(&self, typ: u16, count: u32, field: usize) -> Option { + let at = self.data_at(typ, count, field)?; + match typ { + 3 => self.u16(at).map(f64::from), + 4 => self.u32(at).map(f64::from), + 5 => { + let (n, d) = (self.u32(at)?, self.u32(at + 4)?); + (d != 0).then(|| n as f64 / d as f64) + } + 10 => { + let (n, d) = (self.u32(at)? as i32, self.u32(at + 4)? as i32); + (d != 0).then(|| n as f64 / d as f64) + } + _ => None, + } + } + fn text(&self, typ: u16, count: u32, field: usize) -> Option { + if typ != 2 { + return None; + } + let at = self.data_at(typ, count, field)?; + let s = self.b.get(at..at + count as usize)?; + let s = String::from_utf8_lossy(s).trim_end_matches('\0').trim().to_string(); + (!s.is_empty()).then_some(s) + } +} + +/// Reads [`CameraInfo`] from a TIFF-structured EXIF payload (an `Exif\0\0` prefix is skipped). +pub fn read(exif: &[u8]) -> CameraInfo { + let b = exif.strip_prefix(b"Exif\0\0").unwrap_or(exif); + let mut info = CameraInfo::default(); + let le = match b.get(..4) { + Some([b'I', b'I', 42, 0]) => true, + Some([b'M', b'M', 0, 42]) => false, + _ => return info, + }; + let t = Tiff { b, le }; + let Some(ifd0) = t.u32(4) else { return info }; + let mut queue = vec![ifd0 as usize]; + let mut seen = Vec::new(); + while let Some(off) = queue.pop() { + if seen.contains(&off) || seen.len() > 8 { + continue; + } + seen.push(off); + for (tag, typ, count, field) in t.ifd(off) { + match tag { + 0x010F => info.make = t.text(typ, count, field), + 0x0110 => info.model = t.text(typ, count, field), + 0x829A => info.exposure_time = t.number(typ, count, field), + 0x829D => info.f_number = t.number(typ, count, field), + 0x8827 => info.iso = t.number(typ, count, field), + 0x920A => info.focal_length = t.number(typ, count, field), + 0xA405 => info.focal_length_35mm = t.number(typ, count, field).filter(|v| *v > 0.0), + 0xA434 => info.lens = t.text(typ, count, field), + 0x8769 => { + if let Some(p) = t.u32(field) { + queue.push(p as usize); + } + } + _ => {} + } + } + } + info +} + +/// Builds a little-endian EXIF payload (used by tests and by synthetic fixtures). +pub fn build(info: &CameraInfo) -> Vec { + // IFD0: Make, Model, ExifIFD pointer; Exif IFD: the rest. Data area after the IFDs. + let mut ifd0: Vec<(u16, u16, u32, Vec)> = Vec::new(); + let mut exif: Vec<(u16, u16, u32, Vec)> = Vec::new(); + let text = |s: &str| { + let mut v = s.as_bytes().to_vec(); + v.push(0); + v + }; + let rational = |v: f64| { + let d = 10000u32; + let mut b = ((v * d as f64).round() as u32).to_le_bytes().to_vec(); + b.extend_from_slice(&d.to_le_bytes()); + b + }; + if let Some(s) = &info.make { + ifd0.push((0x010F, 2, s.len() as u32 + 1, text(s))); + } + if let Some(s) = &info.model { + ifd0.push((0x0110, 2, s.len() as u32 + 1, text(s))); + } + if let Some(v) = info.exposure_time { + exif.push((0x829A, 5, 1, rational(v))); + } + if let Some(v) = info.f_number { + exif.push((0x829D, 5, 1, rational(v))); + } + if let Some(v) = info.iso { + exif.push((0x8827, 3, 1, (v as u16).to_le_bytes().to_vec())); + } + if let Some(v) = info.focal_length { + exif.push((0x920A, 5, 1, rational(v))); + } + if let Some(v) = info.focal_length_35mm { + exif.push((0xA405, 3, 1, (v as u16).to_le_bytes().to_vec())); + } + if let Some(s) = &info.lens { + exif.push((0xA434, 2, s.len() as u32 + 1, text(s))); + } + let ifd_len = |n: usize| 2 + 12 * n + 4; + let ifd0_off = 8usize; + let n0 = ifd0.len() + 1; + let exif_off = ifd0_off + ifd_len(n0); + let mut data_off = exif_off + ifd_len(exif.len()); + let mut out = b"II*\0".to_vec(); + out.extend_from_slice(&(ifd0_off as u32).to_le_bytes()); + let mut data = Vec::new(); + let mut write_ifd = |out: &mut Vec, entries: &[(u16, u16, u32, Vec)], extra: Option| { + let n = entries.len() + usize::from(extra.is_some()); + out.extend_from_slice(&(n as u16).to_le_bytes()); + for (tag, typ, count, bytes) in entries { + out.extend_from_slice(&tag.to_le_bytes()); + out.extend_from_slice(&typ.to_le_bytes()); + out.extend_from_slice(&count.to_le_bytes()); + if bytes.len() <= 4 { + let mut f = bytes.clone(); + f.resize(4, 0); + out.extend_from_slice(&f); + } else { + out.extend_from_slice(&(data_off as u32).to_le_bytes()); + data.extend_from_slice(bytes); + data_off += bytes.len(); + } + } + if let Some(p) = extra { + out.extend_from_slice(&0x8769u16.to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(&p.to_le_bytes()); + } + out.extend_from_slice(&0u32.to_le_bytes()); + }; + write_ifd(&mut out, &ifd0, Some(exif_off as u32)); + write_ifd(&mut out, &exif, None); + out.extend_from_slice(&data); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn round_trips_and_survives_garbage() { + let info = CameraInfo { + make: Some("Generic".into()), + model: Some("Camera 1".into()), + lens: Some("24-70mm".into()), + exposure_time: Some(0.004), + f_number: Some(8.0), + iso: Some(200.0), + focal_length: Some(35.0), + focal_length_35mm: Some(52.0), + }; + let b = build(&info); + let r = read(&b); + assert_eq!(r, info); + assert!((r.exposure_factor().unwrap() - 0.004 * 2.0 / 64.0).abs() < 1e-9); + let mut prefixed = b"Exif\0\0".to_vec(); + prefixed.extend_from_slice(&b); + assert_eq!(read(&prefixed), info); + assert_eq!(read(b"nonsense"), CameraInfo::default()); + for cut in [5, 12, 30, b.len() - 3] { + let _ = read(&b[..cut]); + } + } +} diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs index 1bc9660b8..ff2c530fa 100644 --- a/crates/metadata/src/lib.rs +++ b/crates/metadata/src/lib.rs @@ -19,6 +19,7 @@ //! HEIF, PSD and MRW files are read by their own crates. Put the blocks they return into an //! [`Embedded`] and call [`from_embedded`]. +pub mod camera_fields; mod container; pub mod cr3; mod datetime; From ef3e23916fca2962c024e7515d62e0165d41502b Mon Sep 17 00:00:00 2001 From: Fabio Lussi Date: Sun, 11 Oct 2026 08:26:29 +0200 Subject: [PATCH 23/98] Carry PhotoCraft camera date and checked-bounds changes Source: storytold/photocraft commit 1ccdac4c940ce43e2647f4c07fac1d61725e1f74 --- crates/metadata/src/camera_fields.rs | 81 ++++++++++++++++++++++++---- 1 file changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/metadata/src/camera_fields.rs b/crates/metadata/src/camera_fields.rs index 445f51498..d18bcdec9 100644 --- a/crates/metadata/src/camera_fields.rs +++ b/crates/metadata/src/camera_fields.rs @@ -17,6 +17,8 @@ pub struct CameraInfo { pub focal_length: Option, /// 35 mm-equivalent focal length in millimetres. pub focal_length_35mm: Option, + /// DateTimeOriginal as stored (`YYYY:MM:DD HH:MM:SS`). + pub date_taken: Option, } impl CameraInfo { @@ -35,11 +37,11 @@ struct Tiff<'a> { impl Tiff<'_> { fn u16(&self, o: usize) -> Option { - let s = self.b.get(o..o + 2)?; + let s = self.b.get(o..o.checked_add(2)?)?; Some(if self.le { u16::from_le_bytes([s[0], s[1]]) } else { u16::from_be_bytes([s[0], s[1]]) }) } fn u32(&self, o: usize) -> Option { - let s = self.b.get(o..o + 4)?; + let s = self.b.get(o..o.checked_add(4)?)?; let a = [s[0], s[1], s[2], s[3]]; Some(if self.le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) }) } @@ -48,20 +50,25 @@ impl Tiff<'_> { let Some(n) = self.u16(off) else { return Vec::new() }; (0..n as usize) .filter_map(|k| { - let e = off + 2 + k * 12; - Some((self.u16(e)?, self.u16(e + 2)?, self.u32(e + 4)?, e + 8)) + // Checked: offsets come from the file, and usize is 32 bits on wasm. + let e = off.checked_add(2)?.checked_add(k * 12)?; + Some((self.u16(e)?, self.u16(e.checked_add(2)?)?, self.u32(e.checked_add(4)?)?, e.checked_add(8)?)) }) .collect() } fn data_at(&self, typ: u16, count: u32, field: usize) -> Option { let size = match typ { - 1 | 2 | 6 | 7 => 1, + 1 | 2 | 6 | 7 => 1usize, 3 | 8 => 2, 4 | 9 | 11 => 4, 5 | 10 | 12 => 8, _ => return None, - } * count as usize; - if size <= 4 { Some(field) } else { self.u32(field).map(|v| v as usize) } + } + .checked_mul(usize::try_from(count).ok()?)?; + let at = if size <= 4 { field } else { usize::try_from(self.u32(field)?).ok()? }; + // The whole value must lie inside the payload, not just its first element. + self.b.get(at..at.checked_add(size)?)?; + Some(at) } fn number(&self, typ: u16, count: u32, field: usize) -> Option { let at = self.data_at(typ, count, field)?; @@ -69,11 +76,11 @@ impl Tiff<'_> { 3 => self.u16(at).map(f64::from), 4 => self.u32(at).map(f64::from), 5 => { - let (n, d) = (self.u32(at)?, self.u32(at + 4)?); + let (n, d) = (self.u32(at)?, self.u32(at.checked_add(4)?)?); (d != 0).then(|| n as f64 / d as f64) } 10 => { - let (n, d) = (self.u32(at)? as i32, self.u32(at + 4)? as i32); + let (n, d) = (self.u32(at)? as i32, self.u32(at.checked_add(4)?)? as i32); (d != 0).then(|| n as f64 / d as f64) } _ => None, @@ -84,7 +91,7 @@ impl Tiff<'_> { return None; } let at = self.data_at(typ, count, field)?; - let s = self.b.get(at..at + count as usize)?; + let s = self.b.get(at..at.checked_add(usize::try_from(count).ok()?)?)?; let s = String::from_utf8_lossy(s).trim_end_matches('\0').trim().to_string(); (!s.is_empty()).then_some(s) } @@ -118,6 +125,7 @@ pub fn read(exif: &[u8]) -> CameraInfo { 0x920A => info.focal_length = t.number(typ, count, field), 0xA405 => info.focal_length_35mm = t.number(typ, count, field).filter(|v| *v > 0.0), 0xA434 => info.lens = t.text(typ, count, field), + 0x9003 => info.date_taken = t.text(typ, count, field), 0x8769 => { if let Some(p) = t.u32(field) { queue.push(p as usize); @@ -161,6 +169,9 @@ pub fn build(info: &CameraInfo) -> Vec { if let Some(v) = info.iso { exif.push((0x8827, 3, 1, (v as u16).to_le_bytes().to_vec())); } + if let Some(s) = &info.date_taken { + exif.push((0x9003, 2, s.len() as u32 + 1, text(s))); + } if let Some(v) = info.focal_length { exif.push((0x920A, 5, 1, rational(v))); } @@ -224,6 +235,7 @@ mod tests { iso: Some(200.0), focal_length: Some(35.0), focal_length_35mm: Some(52.0), + date_taken: Some("2024:02:17 12:40:19".into()), }; let b = build(&info); let r = read(&b); @@ -237,4 +249,53 @@ mod tests { let _ = read(&b[..cut]); } } + + /// Hostile payloads: huge counts, offsets near `u32::MAX`, self-referencing IFDs and random + /// bytes read as `None` fields instead of panicking or overflowing (usize is 32 bits on wasm). + #[test] + fn hostile_payloads_never_panic() { + let entry = |tag: u16, typ: u16, count: u32, value: u32| { + let mut e = tag.to_le_bytes().to_vec(); + e.extend_from_slice(&typ.to_le_bytes()); + e.extend_from_slice(&count.to_le_bytes()); + e.extend_from_slice(&value.to_le_bytes()); + e + }; + let mut b = b"II*\0".to_vec(); + b.extend_from_slice(&8u32.to_le_bytes()); + let entries = [ + entry(0x010F, 2, u32::MAX, 8), // Make: count far past the end + entry(0x0110, 2, 16, u32::MAX - 1), // Model: offset at the top of the range + entry(0x829A, 5, u32::MAX, 20), // ExposureTime: count overflows the size + entry(0x829D, 5, 1, u32::MAX - 3), // FNumber: rational straddles u32::MAX + entry(0x9003, 2, 0x4000_0000, 8), // DateTimeOriginal: size overflow on 32 bits + entry(0x8769, 4, 1, 8), // Exif IFD pointing back at IFD0 + entry(0x8769, 4, 1, u32::MAX), // ... and past the end + ]; + b.extend_from_slice(&(entries.len() as u16 + 3).to_le_bytes()); // claims more entries than exist + for e in entries { + b.extend_from_slice(&e); + } + assert_eq!(read(&b), CameraInfo::default()); + let mut be = b"MM\0*".to_vec(); + be.extend_from_slice(&u32::MAX.to_be_bytes()); + assert_eq!(read(&be), CameraInfo::default()); + // Deterministic noise behind a valid header, and every truncation of a real payload. + let mut x = 0x2545_F491u32; + for _ in 0..200 { + let mut n = b"II*\0\x08\0\0\0".to_vec(); + for _ in 0..96 { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + n.push(x as u8); + } + let _ = read(&n); + } + let good = + build(&CameraInfo { make: Some("Make".into()), date_taken: Some("2024:02:17 12:40:19".into()), exposure_time: Some(0.004), ..Default::default() }); + for cut in 0..good.len() { + let _ = read(&good[..cut]); + } + } } From 2c7891db0ec26a7d9bd45001e54c783c4de9a2cf Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 06:05:32 -0500 Subject: [PATCH 24/98] Adapt camera EXIF to shared TIFF and add bounded parity tests Retain original contributor imports as authored commits. Reuse shared spans and writer, preserve field semantics, add text budget and differential controls. --- crates/metadata/src/camera_fields.rs | 352 ++++++++++++------ .../src/camera_fields_legacy_tests.rs | 237 ++++++++++++ 2 files changed, 485 insertions(+), 104 deletions(-) create mode 100644 crates/metadata/src/camera_fields_legacy_tests.rs diff --git a/crates/metadata/src/camera_fields.rs b/crates/metadata/src/camera_fields.rs index d18bcdec9..ade835736 100644 --- a/crates/metadata/src/camera_fields.rs +++ b/crates/metadata/src/camera_fields.rs @@ -1,8 +1,12 @@ -//! Minimal EXIF reader for the camera settings the photography commands use (exposure, aperture, +//! Unnormalized camera-field EXIF view, reused from PhotoCraft (exposure, aperture, //! ISO, focal length, camera and lens names). Input is the raw TIFF-structured EXIF payload //! (`II*\0` / `MM\0*`, as stored in `Document.metadata.exif`); malformed data yields `None` //! fields, never a panic. Tag numbers are from the EXIF 2.32 specification (CIPA DC-008). +// Semantics originate in PhotoCraft, including Brandon Thomas and Fabio Lussi. +// Unlike read_exif this view does not normalize dates or derive missing fields. +#![cfg_attr(test, allow(clippy::unwrap_used, clippy::expect_used, clippy::panic))] + /// Camera settings read from EXIF. #[derive(Clone, Debug, Default, PartialEq)] pub struct CameraInfo { @@ -30,20 +34,22 @@ impl CameraInfo { } } +/// Text bytes decoded per EXIF read (real Make/Model/Lens/Date strings are a few dozen bytes). +const MAX_TEXT_BYTES: usize = 1 << 20; + struct Tiff<'a> { b: &'a [u8], - le: bool, + order: craft_tiff::ByteOrder, + /// Text bytes this read may still decode: duplicate tags can share one large blank value. + text_left: std::cell::Cell, } impl Tiff<'_> { fn u16(&self, o: usize) -> Option { - let s = self.b.get(o..o.checked_add(2)?)?; - Some(if self.le { u16::from_le_bytes([s[0], s[1]]) } else { u16::from_be_bytes([s[0], s[1]]) }) + self.order.read_u16(self.b, o) } fn u32(&self, o: usize) -> Option { - let s = self.b.get(o..o.checked_add(4)?)?; - let a = [s[0], s[1], s[2], s[3]]; - Some(if self.le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) }) + self.order.read_u32(self.b, o) } /// Entries of the IFD at `off`: (tag, type, count, value offset field position). fn ifd(&self, off: usize) -> Vec<(u16, u16, u32, usize)> { @@ -56,19 +62,14 @@ impl Tiff<'_> { }) .collect() } - fn data_at(&self, typ: u16, count: u32, field: usize) -> Option { - let size = match typ { - 1 | 2 | 6 | 7 => 1usize, - 3 | 8 => 2, - 4 | 9 | 11 => 4, - 5 | 10 | 12 => 8, - _ => return None, + fn data_at(&self, typ: u16, _count: u32, field: usize) -> Option { + // Keep this caller's historical classic-TIFF type policy; shared spans also + // support IFD/BigTIFF types which CameraInfo never accepted. + if !(1..=12).contains(&typ) { + return None; } - .checked_mul(usize::try_from(count).ok()?)?; - let at = if size <= 4 { field } else { usize::try_from(self.u32(field)?).ok()? }; - // The whole value must lie inside the payload, not just its first element. - self.b.get(at..at.checked_add(size)?)?; - Some(at) + let span = craft_tiff::entry_span(self.b, field.checked_sub(8)?, self.order, false, 0)?; + usize::try_from(span.start).ok() } fn number(&self, typ: u16, count: u32, field: usize) -> Option { let at = self.data_at(typ, count, field)?; @@ -87,16 +88,23 @@ impl Tiff<'_> { } } fn text(&self, typ: u16, count: u32, field: usize) -> Option { + #[cfg(test)] + TEXT_DECODES.with(|c| c.set(c.get() + 1)); if typ != 2 { return None; } let at = self.data_at(typ, count, field)?; - let s = self.b.get(at..at.checked_add(usize::try_from(count).ok()?)?)?; + let n = usize::try_from(count).ok()?; + self.text_left.set(self.text_left.get().checked_sub(n)?); + let s = self.b.get(at..at.checked_add(n)?)?; let s = String::from_utf8_lossy(s).trim_end_matches('\0').trim().to_string(); (!s.is_empty()).then_some(s) } } +#[cfg(test)] +thread_local!(static TEXT_DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }); + /// Reads [`CameraInfo`] from a TIFF-structured EXIF payload (an `Exif\0\0` prefix is skipped). pub fn read(exif: &[u8]) -> CameraInfo { let b = exif.strip_prefix(b"Exif\0\0").unwrap_or(exif); @@ -106,7 +114,11 @@ pub fn read(exif: &[u8]) -> CameraInfo { Some([b'M', b'M', 0, 42]) => false, _ => return info, }; - let t = Tiff { b, le }; + let t = Tiff { + b, + order: if le { craft_tiff::ByteOrder::Little } else { craft_tiff::ByteOrder::Big }, + text_left: std::cell::Cell::new(MAX_TEXT_BYTES), + }; let Some(ifd0) = t.u32(4) else { return info }; let mut queue = vec![ifd0 as usize]; let mut seen = Vec::new(); @@ -117,15 +129,17 @@ pub fn read(exif: &[u8]) -> CameraInfo { seen.push(off); for (tag, typ, count, field) in t.ifd(off) { match tag { - 0x010F => info.make = t.text(typ, count, field), - 0x0110 => info.model = t.text(typ, count, field), + // Duplicate TIFF text tags can point at the same large payload. Keep the first + // recognized value and do not repeatedly decode/copy later duplicates. + 0x010F if info.make.is_none() => info.make = t.text(typ, count, field), + 0x0110 if info.model.is_none() => info.model = t.text(typ, count, field), 0x829A => info.exposure_time = t.number(typ, count, field), 0x829D => info.f_number = t.number(typ, count, field), 0x8827 => info.iso = t.number(typ, count, field), 0x920A => info.focal_length = t.number(typ, count, field), 0xA405 => info.focal_length_35mm = t.number(typ, count, field).filter(|v| *v > 0.0), - 0xA434 => info.lens = t.text(typ, count, field), - 0x9003 => info.date_taken = t.text(typ, count, field), + 0xA434 if info.lens.is_none() => info.lens = t.text(typ, count, field), + 0x9003 if info.date_taken.is_none() => info.date_taken = t.text(typ, count, field), 0x8769 => { if let Some(p) = t.u32(field) { queue.push(p as usize); @@ -138,86 +152,47 @@ pub fn read(exif: &[u8]) -> CameraInfo { info } -/// Builds a little-endian EXIF payload (used by tests and by synthetic fixtures). -pub fn build(info: &CameraInfo) -> Vec { - // IFD0: Make, Model, ExifIFD pointer; Exif IFD: the rest. Data area after the IFDs. - let mut ifd0: Vec<(u16, u16, u32, Vec)> = Vec::new(); - let mut exif: Vec<(u16, u16, u32, Vec)> = Vec::new(); - let text = |s: &str| { - let mut v = s.as_bytes().to_vec(); - v.push(0); - v - }; - let rational = |v: f64| { - let d = 10000u32; - let mut b = ((v * d as f64).round() as u32).to_le_bytes().to_vec(); - b.extend_from_slice(&d.to_le_bytes()); - b - }; - if let Some(s) = &info.make { - ifd0.push((0x010F, 2, s.len() as u32 + 1, text(s))); +/// Build a classic TIFF EXIF block using the shared TIFF writer. +/// Raw date strings and numeric rounding match the camera-field view. Text is +/// bounded to the reader's 1 MiB aggregate budget; oversized inputs return an error. +/// Physical directory placement is not part of this API's contract. +pub fn try_build(info: &CameraInfo) -> craft_tiff::Result> { + use craft_tiff::{ByteOrder, IfdBuilder, TiffError, TiffWriter, Value}; + let total = + [&info.make, &info.model, &info.lens, &info.date_taken].into_iter().flatten().try_fold(0usize, |n, s| n.checked_add(s.len())?.checked_add(1)); + if total.is_none_or(|n| n > MAX_TEXT_BYTES) { + return Err(TiffError::Limit("camera EXIF text")); } - if let Some(s) = &info.model { - ifd0.push((0x0110, 2, s.len() as u32 + 1, text(s))); - } - if let Some(v) = info.exposure_time { - exif.push((0x829A, 5, 1, rational(v))); - } - if let Some(v) = info.f_number { - exif.push((0x829D, 5, 1, rational(v))); - } - if let Some(v) = info.iso { - exif.push((0x8827, 3, 1, (v as u16).to_le_bytes().to_vec())); - } - if let Some(s) = &info.date_taken { - exif.push((0x9003, 2, s.len() as u32 + 1, text(s))); - } - if let Some(v) = info.focal_length { - exif.push((0x920A, 5, 1, rational(v))); - } - if let Some(v) = info.focal_length_35mm { - exif.push((0xA405, 3, 1, (v as u16).to_le_bytes().to_vec())); - } - if let Some(s) = &info.lens { - exif.push((0xA434, 2, s.len() as u32 + 1, text(s))); - } - let ifd_len = |n: usize| 2 + 12 * n + 4; - let ifd0_off = 8usize; - let n0 = ifd0.len() + 1; - let exif_off = ifd0_off + ifd_len(n0); - let mut data_off = exif_off + ifd_len(exif.len()); - let mut out = b"II*\0".to_vec(); - out.extend_from_slice(&(ifd0_off as u32).to_le_bytes()); - let mut data = Vec::new(); - let mut write_ifd = |out: &mut Vec, entries: &[(u16, u16, u32, Vec)], extra: Option| { - let n = entries.len() + usize::from(extra.is_some()); - out.extend_from_slice(&(n as u16).to_le_bytes()); - for (tag, typ, count, bytes) in entries { - out.extend_from_slice(&tag.to_le_bytes()); - out.extend_from_slice(&typ.to_le_bytes()); - out.extend_from_slice(&count.to_le_bytes()); - if bytes.len() <= 4 { - let mut f = bytes.clone(); - f.resize(4, 0); - out.extend_from_slice(&f); - } else { - out.extend_from_slice(&(data_off as u32).to_le_bytes()); - data.extend_from_slice(bytes); - data_off += bytes.len(); - } + let mut root = IfdBuilder::new(); + let mut exif = IfdBuilder::new(); + for (tag, value) in [(0x010f, &info.make), (0x0110, &info.model)] { + if let Some(value) = value { + root.set(tag, Value::Ascii(value.clone())); } - if let Some(p) = extra { - out.extend_from_slice(&0x8769u16.to_le_bytes()); - out.extend_from_slice(&4u16.to_le_bytes()); - out.extend_from_slice(&1u32.to_le_bytes()); - out.extend_from_slice(&p.to_le_bytes()); + } + for (tag, value) in [(0x9003, &info.date_taken), (0xa434, &info.lens)] { + if let Some(value) = value { + exif.set(tag, Value::Ascii(value.clone())); } - out.extend_from_slice(&0u32.to_le_bytes()); - }; - write_ifd(&mut out, &ifd0, Some(exif_off as u32)); - write_ifd(&mut out, &exif, None); - out.extend_from_slice(&data); - out + } + for (tag, value) in [(0x829a, info.exposure_time), (0x829d, info.f_number), (0x920a, info.focal_length)] { + if let Some(value) = value { + exif.set(tag, Value::Rational(vec![((value * 10000.0).round() as u32, 10000)])); + } + } + for (tag, value) in [(0x8827, info.iso), (0xa405, info.focal_length_35mm)] { + if let Some(value) = value { + exif.set(tag, Value::Short(vec![value as u16])); + } + } + // Empty child IFDs are valid here; the reader yields absent fields. + root.set_child(0x8769, exif); + TiffWriter::new(ByteOrder::Little, false).write(&[root]) +} + +#[cfg(test)] +fn build(info: &CameraInfo) -> Vec { + try_build(info).unwrap_or_default() } #[cfg(test)] @@ -252,6 +227,72 @@ mod tests { /// Hostile payloads: huge counts, offsets near `u32::MAX`, self-referencing IFDs and random /// bytes read as `None` fields instead of panicking or overflowing (usize is 32 bits on wasm). + #[test] + fn blank_duplicate_text_tags_share_one_decode_budget() { + // Blank values never fill `make`, so every duplicate would decode the same 1 MiB again; + // the per-read budget stops after the first, so the later real value is not reached. + let blank = MAX_TEXT_BYTES; + let entry = |count: u32, off: u32| { + let mut e = Vec::new(); + e.extend_from_slice(&0x010Fu16.to_le_bytes()); + e.extend_from_slice(&2u16.to_le_bytes()); + e.extend_from_slice(&count.to_le_bytes()); + e.extend_from_slice(&off.to_le_bytes()); + e + }; + let mut b = b"II*\0".to_vec(); + b.extend_from_slice(&8u32.to_le_bytes()); + b.extend_from_slice(&3u16.to_le_bytes()); + b.extend(entry(blank as u32, 50)); + b.extend(entry(blank as u32, 50)); + b.extend(entry(2, u32::from_le_bytes(*b"M\0\0\0"))); + b.extend_from_slice(&0u32.to_le_bytes()); + assert_eq!(b.len(), 50); + b.extend(std::iter::repeat_n(b' ', blank)); + assert_eq!(read(&b).make, None); + } + + #[test] + fn duplicate_text_tags_keep_the_first_value_without_copying_large_duplicates() { + let value = vec![b'X'; 2 * 1024 * 1024]; + let mut b = b"II*\0".to_vec(); + b.extend_from_slice(&8u32.to_le_bytes()); + // IFD0: Make="M", a duplicate Make pointing at a 2 MiB payload, and the Exif IFD link. + b.extend_from_slice(&3u16.to_le_bytes()); + b.extend_from_slice(&0x010Fu16.to_le_bytes()); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&2u32.to_le_bytes()); + b.extend_from_slice(b"M\0\0\0"); + b.extend_from_slice(&0x010Fu16.to_le_bytes()); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&(value.len() as u32).to_le_bytes()); + b.extend_from_slice(&80u32.to_le_bytes()); + b.extend_from_slice(&0x8769u16.to_le_bytes()); + b.extend_from_slice(&4u16.to_le_bytes()); + b.extend_from_slice(&1u32.to_le_bytes()); + b.extend_from_slice(&50u32.to_le_bytes()); + b.extend_from_slice(&0u32.to_le_bytes()); + // Exif IFD at 50: DateTimeOriginal="D" and the same large duplicate payload at offset 80. + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&0x9003u16.to_le_bytes()); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&2u32.to_le_bytes()); + b.extend_from_slice(b"D\0\0\0"); + b.extend_from_slice(&0x9003u16.to_le_bytes()); + b.extend_from_slice(&2u16.to_le_bytes()); + b.extend_from_slice(&(value.len() as u32).to_le_bytes()); + b.extend_from_slice(&80u32.to_le_bytes()); + b.extend_from_slice(&0u32.to_le_bytes()); + b.extend_from_slice(&value); + + TEXT_DECODES.with(|c| c.set(0)); + let info = read(&b); + assert_eq!(info.make.as_deref(), Some("M")); + assert_eq!(info.date_taken.as_deref(), Some("D")); + // One decode per tag: duplicates must be skipped before `text` copies their payload. + assert_eq!(TEXT_DECODES.with(|c| c.get()), 2); + } + #[test] fn hostile_payloads_never_panic() { let entry = |tag: u16, typ: u16, count: u32, value: u32| { @@ -292,10 +333,113 @@ mod tests { } let _ = read(&n); } - let good = - build(&CameraInfo { make: Some("Make".into()), date_taken: Some("2024:02:17 12:40:19".into()), exposure_time: Some(0.004), ..Default::default() }); + let good = build(&CameraInfo { + make: Some("Make".into()), + date_taken: Some("2024:02:17 12:40:19".into()), + exposure_time: Some(0.004), + ..Default::default() + }); for cut in 0..good.len() { let _ = read(&good[..cut]); } } } + +// Keep the previous byte reader only as a test oracle during the shared-span cutover. +#[cfg(test)] +#[path = "camera_fields_legacy_tests.rs"] +mod legacy; + +#[cfg(test)] +mod shared_span_parity { + use super::*; + + fn compare(bytes: &[u8]) { + assert_eq!(format!("{:?}", read(bytes)), format!("{:?}", legacy::read(bytes))); + } + + #[test] + fn shared_writer_matches_legacy_fields_and_rejects_oversized_text() { + for value in [0.0, -1.0, 0.004, 2.8, 65535.0, f64::INFINITY, f64::NAN] { + let camera = CameraInfo { + exposure_time: Some(value), + f_number: Some(value), + iso: Some(value), + focal_length: Some(value), + focal_length_35mm: Some(value), + date_taken: Some("raw date".into()), + make: Some("M".into()), + model: Some("Model".into()), + lens: Some("Lens".into()), + }; + let old = legacy::CameraInfo { + exposure_time: camera.exposure_time, + f_number: camera.f_number, + iso: camera.iso, + focal_length: camera.focal_length, + focal_length_35mm: camera.focal_length_35mm, + date_taken: camera.date_taken.clone(), + make: camera.make.clone(), + model: camera.model.clone(), + lens: camera.lens.clone(), + }; + let bytes = try_build(&camera).unwrap(); + assert_eq!(format!("{:?}", read(&bytes)), format!("{:?}", legacy::read(&legacy::build(&old)))); + } + assert_eq!(read(&try_build(&CameraInfo::default()).unwrap()), CameraInfo::default()); + assert!(try_build(&CameraInfo { make: Some("x".repeat(MAX_TEXT_BYTES)), ..Default::default() }).is_err()); + assert!(try_build(&CameraInfo { make: Some("x".repeat(MAX_TEXT_BYTES - 1)), ..Default::default() }).is_ok()); + } + + #[test] + fn preserves_camera_selection_for_valid_truncated_and_mutated_payloads() { + let bytes = build(&CameraInfo { + make: Some("Maker".into()), + model: Some("Camera".into()), + lens: Some("Lens".into()), + exposure_time: Some(0.125), + f_number: Some(2.8), + iso: Some(200.0), + focal_length: Some(50.0), + focal_length_35mm: Some(75.0), + date_taken: Some("not normalized: original date".into()), + }); + compare(&bytes); + let mut prefixed = b"Exif\0\0".to_vec(); + prefixed.extend_from_slice(&bytes); + compare(&prefixed); + for end in 0..=bytes.len() { + compare(&bytes[..end]); + } + // Exercise offsets, tags, types, counts and payloads independently, not random + // external data. The complete shared span must remain bounded for every edit. + for index in 0..bytes.len() { + for value in [0, 0xff, 0x80] { + let mut changed = bytes.clone(); + changed[index] = value; + compare(&changed); + } + } + } + + #[test] + fn preserves_partial_entry_and_zero_count_semantics_in_both_orders() { + for little in [true, false] { + let u16b = |v: u16| if little { v.to_le_bytes() } else { v.to_be_bytes() }; + let u32b = |v: u32| if little { v.to_le_bytes() } else { v.to_be_bytes() }; + for count in [0u32, 1, 2, u32::MAX] { + let mut bytes = if little { b"II".to_vec() } else { b"MM".to_vec() }; + bytes.extend(u16b(42)); + bytes.extend(u32b(8)); + bytes.extend(u16b(1)); + bytes.extend(u16b(0x8827)); + bytes.extend(u16b(3)); + bytes.extend(u32b(count)); + bytes.extend(u16b(200)); + compare(&bytes); // two inline bytes, no full 12-byte entry/next pointer + bytes.extend(u16b(0)); + compare(&bytes); + } + } + } +} diff --git a/crates/metadata/src/camera_fields_legacy_tests.rs b/crates/metadata/src/camera_fields_legacy_tests.rs new file mode 100644 index 000000000..1b560f91f --- /dev/null +++ b/crates/metadata/src/camera_fields_legacy_tests.rs @@ -0,0 +1,237 @@ +// Frozen differential oracle: PhotoCraft main abb30dff, exif.rs. Test-only, not a production fallback. +#![allow(dead_code)] +// Minimal EXIF reader for the camera settings the photography commands use (exposure, aperture, +// ISO, focal length, camera and lens names). Input is the raw TIFF-structured EXIF payload +// (`II*\0` / `MM\0*`, as stored in `Document.metadata.exif`); malformed data yields `None` +// fields, never a panic. Tag numbers are from the EXIF 2.32 specification (CIPA DC-008). + +/// Camera settings read from EXIF. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CameraInfo { + pub make: Option, + pub model: Option, + pub lens: Option, + /// Seconds. + pub exposure_time: Option, + pub f_number: Option, + pub iso: Option, + /// Millimetres. + pub focal_length: Option, + /// 35 mm-equivalent focal length in millimetres. + pub focal_length_35mm: Option, + /// DateTimeOriginal as stored (`YYYY:MM:DD HH:MM:SS`). + pub date_taken: Option, +} + +impl CameraInfo { + /// Linear exposure factor `t · ISO / N²` (relative brightness of the image for one scene). + pub fn exposure_factor(&self) -> Option { + let t = self.exposure_time?; + let n = self.f_number.unwrap_or(1.0).max(0.1); + Some(t * self.iso.unwrap_or(100.0) / 100.0 / (n * n)) + } +} + +/// Text bytes decoded per EXIF read (real Make/Model/Lens/Date strings are a few dozen bytes). +const MAX_TEXT_BYTES: usize = 1 << 20; + +struct Tiff<'a> { + b: &'a [u8], + le: bool, + /// Text bytes this read may still decode: duplicate tags can share one large blank value. + text_left: std::cell::Cell, +} + +impl Tiff<'_> { + fn u16(&self, o: usize) -> Option { + let s = self.b.get(o..o.checked_add(2)?)?; + Some(if self.le { u16::from_le_bytes([s[0], s[1]]) } else { u16::from_be_bytes([s[0], s[1]]) }) + } + fn u32(&self, o: usize) -> Option { + let s = self.b.get(o..o.checked_add(4)?)?; + let a = [s[0], s[1], s[2], s[3]]; + Some(if self.le { u32::from_le_bytes(a) } else { u32::from_be_bytes(a) }) + } + /// Entries of the IFD at `off`: (tag, type, count, value offset field position). + fn ifd(&self, off: usize) -> Vec<(u16, u16, u32, usize)> { + let Some(n) = self.u16(off) else { return Vec::new() }; + (0..n as usize) + .filter_map(|k| { + // Checked: offsets come from the file, and usize is 32 bits on wasm. + let e = off.checked_add(2)?.checked_add(k * 12)?; + Some((self.u16(e)?, self.u16(e.checked_add(2)?)?, self.u32(e.checked_add(4)?)?, e.checked_add(8)?)) + }) + .collect() + } + fn data_at(&self, typ: u16, count: u32, field: usize) -> Option { + let size = match typ { + 1 | 2 | 6 | 7 => 1usize, + 3 | 8 => 2, + 4 | 9 | 11 => 4, + 5 | 10 | 12 => 8, + _ => return None, + } + .checked_mul(usize::try_from(count).ok()?)?; + let at = if size <= 4 { field } else { usize::try_from(self.u32(field)?).ok()? }; + // The whole value must lie inside the payload, not just its first element. + self.b.get(at..at.checked_add(size)?)?; + Some(at) + } + fn number(&self, typ: u16, count: u32, field: usize) -> Option { + let at = self.data_at(typ, count, field)?; + match typ { + 3 => self.u16(at).map(f64::from), + 4 => self.u32(at).map(f64::from), + 5 => { + let (n, d) = (self.u32(at)?, self.u32(at.checked_add(4)?)?); + (d != 0).then(|| n as f64 / d as f64) + } + 10 => { + let (n, d) = (self.u32(at)? as i32, self.u32(at.checked_add(4)?)? as i32); + (d != 0).then(|| n as f64 / d as f64) + } + _ => None, + } + } + fn text(&self, typ: u16, count: u32, field: usize) -> Option { + #[cfg(test)] + TEXT_DECODES.with(|c| c.set(c.get() + 1)); + if typ != 2 { + return None; + } + let at = self.data_at(typ, count, field)?; + let n = usize::try_from(count).ok()?; + self.text_left.set(self.text_left.get().checked_sub(n)?); + let s = self.b.get(at..at.checked_add(n)?)?; + let s = String::from_utf8_lossy(s).trim_end_matches('\0').trim().to_string(); + (!s.is_empty()).then_some(s) + } +} + +#[cfg(test)] +thread_local!(static TEXT_DECODES: std::cell::Cell = const { std::cell::Cell::new(0) }); + +/// Reads [`CameraInfo`] from a TIFF-structured EXIF payload (an `Exif\0\0` prefix is skipped). +pub fn read(exif: &[u8]) -> CameraInfo { + let b = exif.strip_prefix(b"Exif\0\0").unwrap_or(exif); + let mut info = CameraInfo::default(); + let le = match b.get(..4) { + Some([b'I', b'I', 42, 0]) => true, + Some([b'M', b'M', 0, 42]) => false, + _ => return info, + }; + let t = Tiff { b, le, text_left: std::cell::Cell::new(MAX_TEXT_BYTES) }; + let Some(ifd0) = t.u32(4) else { return info }; + let mut queue = vec![ifd0 as usize]; + let mut seen = Vec::new(); + while let Some(off) = queue.pop() { + if seen.contains(&off) || seen.len() > 8 { + continue; + } + seen.push(off); + for (tag, typ, count, field) in t.ifd(off) { + match tag { + // Duplicate TIFF text tags can point at the same large payload. Keep the first + // recognized value and do not repeatedly decode/copy later duplicates. + 0x010F if info.make.is_none() => info.make = t.text(typ, count, field), + 0x0110 if info.model.is_none() => info.model = t.text(typ, count, field), + 0x829A => info.exposure_time = t.number(typ, count, field), + 0x829D => info.f_number = t.number(typ, count, field), + 0x8827 => info.iso = t.number(typ, count, field), + 0x920A => info.focal_length = t.number(typ, count, field), + 0xA405 => info.focal_length_35mm = t.number(typ, count, field).filter(|v| *v > 0.0), + 0xA434 if info.lens.is_none() => info.lens = t.text(typ, count, field), + 0x9003 if info.date_taken.is_none() => info.date_taken = t.text(typ, count, field), + 0x8769 => { + if let Some(p) = t.u32(field) { + queue.push(p as usize); + } + } + _ => {} + } + } + } + info +} + +/// Builds a little-endian EXIF payload (used by tests and by synthetic fixtures). +pub fn build(info: &CameraInfo) -> Vec { + // IFD0: Make, Model, ExifIFD pointer; Exif IFD: the rest. Data area after the IFDs. + let mut ifd0: Vec<(u16, u16, u32, Vec)> = Vec::new(); + let mut exif: Vec<(u16, u16, u32, Vec)> = Vec::new(); + let text = |s: &str| { + let mut v = s.as_bytes().to_vec(); + v.push(0); + v + }; + let rational = |v: f64| { + let d = 10000u32; + let mut b = ((v * d as f64).round() as u32).to_le_bytes().to_vec(); + b.extend_from_slice(&d.to_le_bytes()); + b + }; + if let Some(s) = &info.make { + ifd0.push((0x010F, 2, s.len() as u32 + 1, text(s))); + } + if let Some(s) = &info.model { + ifd0.push((0x0110, 2, s.len() as u32 + 1, text(s))); + } + if let Some(v) = info.exposure_time { + exif.push((0x829A, 5, 1, rational(v))); + } + if let Some(v) = info.f_number { + exif.push((0x829D, 5, 1, rational(v))); + } + if let Some(v) = info.iso { + exif.push((0x8827, 3, 1, (v as u16).to_le_bytes().to_vec())); + } + if let Some(s) = &info.date_taken { + exif.push((0x9003, 2, s.len() as u32 + 1, text(s))); + } + if let Some(v) = info.focal_length { + exif.push((0x920A, 5, 1, rational(v))); + } + if let Some(v) = info.focal_length_35mm { + exif.push((0xA405, 3, 1, (v as u16).to_le_bytes().to_vec())); + } + if let Some(s) = &info.lens { + exif.push((0xA434, 2, s.len() as u32 + 1, text(s))); + } + let ifd_len = |n: usize| 2 + 12 * n + 4; + let ifd0_off = 8usize; + let n0 = ifd0.len() + 1; + let exif_off = ifd0_off + ifd_len(n0); + let mut data_off = exif_off + ifd_len(exif.len()); + let mut out = b"II*\0".to_vec(); + out.extend_from_slice(&(ifd0_off as u32).to_le_bytes()); + let mut data = Vec::new(); + let mut write_ifd = |out: &mut Vec, entries: &[(u16, u16, u32, Vec)], extra: Option| { + let n = entries.len() + usize::from(extra.is_some()); + out.extend_from_slice(&(n as u16).to_le_bytes()); + for (tag, typ, count, bytes) in entries { + out.extend_from_slice(&tag.to_le_bytes()); + out.extend_from_slice(&typ.to_le_bytes()); + out.extend_from_slice(&count.to_le_bytes()); + if bytes.len() <= 4 { + let mut f = bytes.clone(); + f.resize(4, 0); + out.extend_from_slice(&f); + } else { + out.extend_from_slice(&(data_off as u32).to_le_bytes()); + data.extend_from_slice(bytes); + data_off += bytes.len(); + } + } + if let Some(p) = extra { + out.extend_from_slice(&0x8769u16.to_le_bytes()); + out.extend_from_slice(&4u16.to_le_bytes()); + out.extend_from_slice(&1u32.to_le_bytes()); + out.extend_from_slice(&p.to_le_bytes()); + } + out.extend_from_slice(&0u32.to_le_bytes()); + }; + write_ifd(&mut out, &ifd0, Some(exif_off as u32)); + write_ifd(&mut out, &exif, None); + out.extend_from_slice(&data); + out +} From dab6a8ea211f1c38fc7974eff73baa2e2ad3520d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:13:51 +0000 Subject: [PATCH 25/98] Flush delayed AV1 frames in HLG regression --- crates/image-codecs/src/codecs/avif.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/image-codecs/src/codecs/avif.rs b/crates/image-codecs/src/codecs/avif.rs index 4912c75bf..042b5684b 100644 --- a/crates/image-codecs/src/codecs/avif.rs +++ b/crates/image-codecs/src/codecs/avif.rs @@ -303,7 +303,9 @@ mod tests { } offset += size; }; - let frames = craft_video::av1::decode::Decoder::new().decode(payload).unwrap(); + let mut decoder = craft_video::av1::decode::Decoder::new(); + let mut frames = decoder.decode(payload).unwrap(); + frames.extend(decoder.flush_result().unwrap()); let luma = &frames.first().expect("encoded AV1 frame").planes[0]; assert!((i32::from(luma[16 * 64 + 16]) - 767).abs() <= 5, "diffuse white"); assert!((i32::from(luma[16 * 64 + 48]) - 1023).abs() <= 5, "peak white"); From 9d088c77ea6746acc8d56b582c9c0c607b5a92a0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:25:41 +0000 Subject: [PATCH 26/98] Support buffered output from tiled AV1 encoder --- crates/video/src/av1/encode/README.md | 7 +- crates/video/src/av1/encode/large.rs | 130 ++++++++++++++++++++++++-- crates/video/src/av1/encode/mod.rs | 2 +- 3 files changed, 127 insertions(+), 12 deletions(-) diff --git a/crates/video/src/av1/encode/README.md b/crates/video/src/av1/encode/README.md index 2df88ba64..0fd60c50f 100644 --- a/crates/video/src/av1/encode/README.md +++ b/crates/video/src/av1/encode/README.md @@ -8,9 +8,10 @@ no `unsafe`, builds for `wasm32-unknown-unknown`). The optional `av1-large-encode` feature exposes `LargeEncoder`, a tiled rav1e-backed encoder for 8-bit 4:2:0 frames up to 8192 samples per side. Use it when the clean-room `Encoder`'s one-tile limits (4096 samples per side and 2304 superblocks) would reject the frame. It supports constant -quantizer rate control and emits one low-latency packet per input frame. The fallback is native -only; the clean-room encoder remains available without the feature and supports 8- and 10-bit -input. +quantizer rate control. `push` returns packets available so far, which can be empty while frames +are buffered; `finish` returns delayed packets. Each packet includes its input frame number for +matching durations. The fallback is native only; the clean-room encoder remains available without +the feature and supports 8- and 10-bit input. ## What is implemented diff --git a/crates/video/src/av1/encode/large.rs b/crates/video/src/av1/encode/large.rs index 109a536dc..8ed26d52b 100644 --- a/crates/video/src/av1/encode/large.rs +++ b/crates/video/src/av1/encode/large.rs @@ -12,6 +12,14 @@ pub struct LargeEncoder { width: usize, height: usize, seq_header: Option>, + finished: bool, +} + +/// An encoded frame and the number of the input frame it represents. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct BufferedPacket { + pub input_frame_number: u64, + pub packet: Packet, } impl LargeEncoder { @@ -50,11 +58,13 @@ impl LargeEncoder { ec.max_key_frame_interval = cfg.keyint.clamp(1, 120) as u64; ec.tiles = std::thread::available_parallelism().map_or(4, |n| n.get()).clamp(2, 16); let ctx = Config::new().with_encoder_config(ec).new_context().map_err(|e| Error::InvalidConfig(format!("rav1e: {e}")))?; - Ok(Self { ctx, width: cfg.width as usize, height: cfg.height as usize, seq_header: None }) + Ok(Self { ctx, width: cfg.width as usize, height: cfg.height as usize, seq_header: None, finished: false }) } - /// Encodes one frame. The frame planes must match the configured 4:2:0 dimensions. - pub fn encode_with_key(&mut self, frame: &Frame<'_>, force_key: bool) -> Result { + fn submit(&mut self, frame: &Frame<'_>, force_key: bool) -> Result<(), Error> { + if self.finished { + return Err(Error::InvalidFrame("AV1 encoder was already finished".into())); + } if frame.y_stride < self.width || frame.uv_stride < self.width / 2 || frame.y.len() < frame.y_stride.saturating_mul(self.height) @@ -70,14 +80,54 @@ impl LargeEncoder { let params = FrameParameters { frame_type_override: if force_key { FrameTypeOverride::Key } else { FrameTypeOverride::No }, ..Default::default() }; self.ctx.send_frame((input, params)).map_err(|e| Error::InvalidFrame(format!("rav1e: {e}")))?; + Ok(()) + } + + fn packet(&mut self, packet: rav1e::prelude::Packet) -> Result { + let (data, seq) = sample_data(&packet.data)?; + if self.seq_header.is_none() { + self.seq_header = seq; + } + Ok(BufferedPacket { input_frame_number: packet.input_frameno, packet: Packet { data, keyframe: packet.frame_type == FrameType::KEY } }) + } + + fn drain(&mut self) -> Result, Error> { + let mut packets = Vec::new(); + loop { + match self.ctx.receive_packet() { + Ok(packet) => packets.push(self.packet(packet)?), + Err(rav1e::prelude::EncoderStatus::Encoded) => {} + Err(rav1e::prelude::EncoderStatus::NeedMoreData) if !self.finished => return Ok(packets), + Err(rav1e::prelude::EncoderStatus::LimitReached) if self.finished => return Ok(packets), + Err(e) => return Err(Error::InvalidFrame(format!("rav1e: {e}"))), + } + } + } + + /// Submit a frame and return all packets available now, which may be empty while the encoder + /// buffers input. Call [`Self::finish`] after the last frame to retrieve delayed packets. + pub fn push(&mut self, frame: &Frame<'_>, force_key: bool) -> Result, Error> { + self.submit(frame, force_key)?; + self.drain() + } + + /// Finish the stream and return its remaining packets. Repeated calls return no packets. + pub fn finish(&mut self) -> Result, Error> { + if !self.finished { + self.ctx.send_frame(None).map_err(|e| Error::InvalidFrame(format!("rav1e: {e}")))?; + self.finished = true; + } + self.drain() + } + + /// Encodes one frame if a packet is immediately available. The input is accepted even if + /// rav1e needs more frames and this returns an error; new callers should use [`Self::push`]. + pub fn encode_with_key(&mut self, frame: &Frame<'_>, force_key: bool) -> Result { + self.submit(frame, force_key)?; loop { match self.ctx.receive_packet() { Ok(packet) => { - let (data, seq) = sample_data(&packet.data)?; - if self.seq_header.is_none() { - self.seq_header = seq; - } - return Ok(Packet { data, keyframe: packet.frame_type == FrameType::KEY }); + return Ok(self.packet(packet)?.packet); } Err(rav1e::prelude::EncoderStatus::Encoded) => continue, Err(rav1e::prelude::EncoderStatus::NeedMoreData | rav1e::prelude::EncoderStatus::LimitReached) => { @@ -153,3 +203,67 @@ fn sample_data(data: &[u8]) -> Result<(Vec, Option>), Error> { } Ok((sample, sequence)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn buffered_frames_are_returned_with_input_numbers_and_finish_drains() { + let cfg = EncoderConfig::new(64, 48, 24, 1); + let mut encoder = LargeEncoder::with_speed(&cfg, 10).unwrap(); + assert!(encoder.av1c().is_none()); + let y = vec![96; 64 * 48]; + let uv = vec![128; 32 * 24]; + let frame = Frame { y: &y, u: &uv, v: &uv, y_stride: 64, uv_stride: 32 }; + let mut packets = Vec::new(); + for i in 0..4 { + packets.extend(encoder.push(&frame, i == 2).unwrap()); + } + packets.extend(encoder.finish().unwrap()); + assert_eq!(packets.iter().map(|p| p.input_frame_number).collect::>(), [0, 1, 2, 3]); + assert!(packets[0].packet.keyframe); + assert!(packets[2].packet.keyframe, "requested key frame"); + assert!(encoder.av1c().is_some()); + #[cfg(feature = "av1-decode")] + { + let mut decoder = crate::av1::decode::Decoder::with_threads(1); + let mut frames = Vec::new(); + for packet in &packets { + frames.extend(decoder.decode(&packet.packet.data).unwrap()); + } + frames.extend(decoder.flush_result().unwrap()); + assert_eq!(frames.len(), 4); + assert!(frames.iter().all(|frame| (frame.width, frame.height) == (64, 48))); + } + assert!(encoder.finish().unwrap().is_empty()); + assert!(encoder.push(&frame, false).is_err()); + } + + #[test] + fn a_large_frame_can_be_flushed_even_when_the_first_push_is_buffered() { + let cfg = EncoderConfig::new(4112, 16, 30, 1); + let mut encoder = LargeEncoder::with_speed(&cfg, 10).unwrap(); + let y = vec![64; 4112 * 16]; + let uv = vec![128; 2056 * 8]; + let frame = Frame { y: &y, u: &uv, v: &uv, y_stride: 4112, uv_stride: 2056 }; + let mut packets = encoder.push(&frame, false).unwrap(); + packets.extend(encoder.finish().unwrap()); + assert_eq!(packets.len(), 1); + assert_eq!(packets[0].input_frame_number, 0); + assert!(packets[0].packet.keyframe); + assert!(!packets[0].packet.data.is_empty()); + assert!(encoder.av1c().is_some()); + } + + #[test] + fn invalid_frame_is_not_queued_and_empty_stream_finishes() { + let cfg = EncoderConfig::new(64, 48, 24, 1); + let mut encoder = LargeEncoder::with_speed(&cfg, 10).unwrap(); + let short = [0; 1]; + let frame = Frame { y: &short, u: &short, v: &short, y_stride: 64, uv_stride: 32 }; + assert!(encoder.push(&frame, false).is_err()); + assert!(encoder.finish().unwrap().is_empty()); + assert!(encoder.av1c().is_none()); + } +} diff --git a/crates/video/src/av1/encode/mod.rs b/crates/video/src/av1/encode/mod.rs index 3a954cef2..d6632a94d 100644 --- a/crates/video/src/av1/encode/mod.rs +++ b/crates/video/src/av1/encode/mod.rs @@ -23,7 +23,7 @@ mod writer; #[cfg(all(feature = "av1-large-encode", not(target_arch = "wasm32")))] mod large; #[cfg(all(feature = "av1-large-encode", not(target_arch = "wasm32")))] -pub use large::LargeEncoder; +pub use large::{BufferedPacket, LargeEncoder}; use bitw::obu; use craft_media::bitstream::BitWriter; From 7dbfafa356682da30911a89058bda1206524db0f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:26:24 +0000 Subject: [PATCH 27/98] Share bounded portable material archives --- Cargo.lock | 10 +++ Cargo.toml | 1 + crates/mcmat/Cargo.toml | 17 ++++ crates/mcmat/README.md | 17 ++++ crates/mcmat/src/lib.rs | 191 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 236 insertions(+) create mode 100644 crates/mcmat/Cargo.toml create mode 100644 crates/mcmat/README.md create mode 100644 crates/mcmat/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 9a600099d..1f46e7ff8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2056,6 +2056,16 @@ dependencies = [ "serde_json", ] +[[package]] +name = "craft-mcmat" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-media" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20da79..190160671 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ craft-dialogs = { path = "crates/dialogs" } craft-doc-binary = { path = "crates/doc-binary" } craft-docx = { path = "crates/docx" } craft-document-bundle = { path = "crates/document-bundle" } +craft-mcmat = { path = "crates/mcmat" } craft-designcraft-format = { path = "crates/designcraft-format" } craft-dxf = { path = "crates/dxf" } craft-drawingml = { path = "crates/drawingml" } diff --git a/crates/mcmat/Cargo.toml b/crates/mcmat/Cargo.toml new file mode 100644 index 000000000..ee3a18626 --- /dev/null +++ b/crates/mcmat/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-mcmat" +description = "Bounded portable material archive interchange for Craft applications" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-zip = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/mcmat/README.md b/crates/mcmat/README.md new file mode 100644 index 000000000..c14db09d7 --- /dev/null +++ b/crates/mcmat/README.md @@ -0,0 +1,17 @@ +# craft-mcmat + +A bounded reader and writer for portable MaterialCraft `.mcmat` ZIP packages. A package +contains `material.json`, named texture maps and a source package. Applications interpret +parameter values and cook or render maps themselves; this crate does not depend on an app's +model. It supports callers that read either version 1 or version 2 of the manifest, or both. + +`Material::open` takes caller-provided version, entry-count, map-count, per-entry and total +expanded-byte limits. Each subsequent `read` and `read_source` shares that budget, including +repeated references to the same ZIP member. Duplicate member names are refused. The manifest +preserves unknown fields at the top level and on maps; parameter, input and preset objects are +kept as JSON values so their unknown fields survive too. `write` preserves the caller's source +entry name and map names, and rejects duplicate archive member names through `craft-zip`. + +The source package is opaque. Version 1 commonly names `source.mcraft`; MaterialCraft's version 2 +writer names `source.sbs`. Callers that invoke another application must preserve the source's +format and extension rather than infer it from `.mcmat` alone. diff --git a/crates/mcmat/src/lib.rs b/crates/mcmat/src/lib.rs new file mode 100644 index 000000000..071331413 --- /dev/null +++ b/crates/mcmat/src/lib.rs @@ -0,0 +1,191 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] +//! Bounded interchange for MaterialCraft portable materials (`.mcmat`). +//! +//! The manifest and its version belong to the format, while interpreting parameter values, +//! cooking maps and opening the source package belong to the caller. Unknown fields survive +//! deserialization and serialization, including fields added by later versions. + +use std::collections::BTreeMap; + +use craft_zip::{Archive, Compression, Writer}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub const FORMAT: &str = "craft-material"; +pub const MANIFEST_ENTRY: &str = "material.json"; + +/// A portable manifest; typed parameter values remain JSON so applications can adapt their own model. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub format: String, + pub version: u32, + pub name: String, + pub graph: String, + pub size: [u32; 2], + pub maps: Vec, + #[serde(default)] + pub parameters: Vec, + #[serde(default)] + pub inputs: Vec, + #[serde(default)] + pub presets: Vec, + pub source: String, + #[serde(flatten)] + pub extra: BTreeMap, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct MapEntry { + pub usage: String, + pub file: String, + #[serde(flatten)] + pub extra: BTreeMap, +} + +/// All limits are supplied by the consumer, including the versions it understands. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub min_version: u32, + pub max_version: u32, + pub max_entries: u64, + pub max_maps: usize, + pub max_manifest_bytes: u64, + pub max_entry_bytes: u64, + pub max_total_bytes: u64, +} + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid portable material: {0}")] + Invalid(String), + #[error("unsupported portable material version {0}")] + Version(u32), + #[error("portable material limit: {0}")] + Limit(String), + #[error("portable material archive: {0}")] + Archive(#[from] craft_zip::Error), +} + +/// Manifest and original archive, retaining one shared expanded-byte budget across reads. +pub struct Material<'a> { + pub manifest: Manifest, + archive: Archive<'a>, + max_entry_bytes: u64, +} + +impl<'a> Material<'a> { + pub fn open(bytes: &'a [u8], limits: Limits) -> Result { + let mut archive = Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: limits.max_total_bytes })?; + if archive.has_duplicate_names() { + return Err(Error::Invalid("duplicate archive entries".into())); + } + let json = archive.read(MANIFEST_ENTRY, limits.max_manifest_bytes)?.ok_or_else(|| Error::Invalid("material.json is missing".into()))?; + let manifest: Manifest = serde_json::from_slice(&json).map_err(|e| Error::Invalid(format!("material.json: {e}")))?; + if manifest.format != FORMAT { + return Err(Error::Invalid(format!("format `{}`", manifest.format))); + } + if !(limits.min_version..=limits.max_version).contains(&manifest.version) { + return Err(Error::Version(manifest.version)); + } + if manifest.maps.len() > limits.max_maps { + return Err(Error::Limit(format!("{} maps exceeds {}", manifest.maps.len(), limits.max_maps))); + } + Ok(Self { manifest, archive, max_entry_bytes: limits.max_entry_bytes }) + } + + /// Read one named member, charging it to the same budget as previous reads. + pub fn read(&mut self, name: &str) -> Result, Error> { + self.archive.read(name, self.max_entry_bytes)?.ok_or_else(|| Error::Invalid(format!("{name} is missing"))) + } + + pub fn read_source(&mut self) -> Result, Error> { + self.read(&self.manifest.source.clone()) + } +} + +/// Write a manifest, its named maps, and the source. Duplicate member names are rejected. +pub fn write(manifest: &Manifest, maps: &[(String, Vec)], source: &[u8]) -> Result, Error> { + let json = serde_json::to_vec_pretty(manifest).map_err(|e| Error::Invalid(format!("material.json: {e}")))?; + let mut writer = Writer::new(); + writer.add(MANIFEST_ENTRY, &json, Compression::Deflated)?; + for (name, bytes) in maps { + writer.add(name, bytes, Compression::Stored)?; + } + writer.add(&manifest.source, source, Compression::Stored)?; + Ok(writer.finish()?) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits() -> Limits { + Limits { + min_version: 1, + max_version: 2, + max_entries: 100, + max_maps: 4, + max_manifest_bytes: 4096, + max_entry_bytes: 100, + max_total_bytes: 8192, + } + } + + fn sample(version: u32) -> Manifest { + serde_json::from_value(serde_json::json!({ + "format": "craft-material", "version": version, "name": "Stone", "graph": "Stone", "size": [64, 64], + "maps": [{"usage": "baseColor", "file": "maps/base.png", "colorSpace": "srgb", "futureMap": true}], + "parameters": [{"id": "rough", "value": {"t": "float", "v": 0.25}, "futureParam": 42}], + "inputs": [{"identifier": "curvature", "kind": "gray"}], "presets": [], + "source": if version == 1 { "source.mcraft" } else { "source.sbs" }, "futureField": {"keep": true} + })) + .unwrap() + } + + #[test] + fn both_protocol_versions_keep_unknown_fields_and_payloads() { + for version in [1, 2] { + let manifest = sample(version); + let bytes = write(&manifest, &[("maps/base.png".into(), vec![1, 2, 3])], b"source").unwrap(); + let mut material = Material::open(&bytes, limits()).unwrap(); + assert_eq!(material.manifest, manifest); + assert_eq!(material.read("maps/base.png").unwrap(), [1, 2, 3]); + assert_eq!(material.read_source().unwrap(), b"source"); + let json = serde_json::to_value(&material.manifest).unwrap(); + assert_eq!(json["futureField"]["keep"], true); + assert_eq!(json["maps"][0]["futureMap"], true); + assert_eq!(json["parameters"][0]["futureParam"], 42); + } + } + + #[test] + fn limits_apply_to_the_manifest_maps_and_aggregate_reads() { + let manifest = sample(2); + let bytes = write(&manifest, &[("maps/base.png".into(), vec![1; 8])], &[2; 8]).unwrap(); + let mut unsupported = limits(); + unsupported.max_version = 1; + assert!(matches!(Material::open(&bytes, unsupported), Err(Error::Version(2)))); + let mut no_maps = limits(); + no_maps.max_maps = 0; + assert!(matches!(Material::open(&bytes, no_maps), Err(Error::Limit(_)))); + let mut tiny_entry = limits(); + tiny_entry.max_entry_bytes = 7; + assert!(Material::open(&bytes, tiny_entry).unwrap().read("maps/base.png").is_err()); + let json_len = serde_json::to_vec_pretty(&manifest).unwrap().len() as u64; + let mut aggregate = limits(); + aggregate.max_total_bytes = json_len + 15; + let mut material = Material::open(&bytes, aggregate).unwrap(); + assert_eq!(material.read("maps/base.png").unwrap().len(), 8); + assert!(material.read_source().is_err()); + } + + #[test] + fn refuses_duplicate_names_on_write() { + let manifest = sample(2); + let duplicate = [("maps/base.png".to_string(), vec![1]), ("maps/base.png".to_string(), vec![2])]; + assert!(write(&manifest, &duplicate, b"source").is_err()); + } +} From 923ae7054952cfa577a90faa7c2774f1ce9dfb52 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:30:20 +0000 Subject: [PATCH 28/98] Reject duplicate manifest fields on write --- crates/mcmat/src/lib.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/mcmat/src/lib.rs b/crates/mcmat/src/lib.rs index 071331413..2e38924b1 100644 --- a/crates/mcmat/src/lib.rs +++ b/crates/mcmat/src/lib.rs @@ -108,6 +108,13 @@ impl<'a> Material<'a> { /// Write a manifest, its named maps, and the source. Duplicate member names are rejected. pub fn write(manifest: &Manifest, maps: &[(String, Vec)], source: &[u8]) -> Result, Error> { + // Flattened fields must not serialize a second copy of a known key. + const KNOWN: &[&str] = &["format", "version", "name", "graph", "size", "maps", "parameters", "inputs", "presets", "source"]; + if manifest.extra.keys().any(|key| KNOWN.contains(&key.as_str())) + || manifest.maps.iter().any(|map| map.extra.contains_key("usage") || map.extra.contains_key("file")) + { + return Err(Error::Invalid("an extra field duplicates a known field".into())); + } let json = serde_json::to_vec_pretty(manifest).map_err(|e| Error::Invalid(format!("material.json: {e}")))?; let mut writer = Writer::new(); writer.add(MANIFEST_ENTRY, &json, Compression::Deflated)?; @@ -182,6 +189,16 @@ mod tests { assert!(material.read_source().is_err()); } + #[test] + fn refuses_extra_fields_that_duplicate_known_keys() { + let mut manifest = sample(2); + manifest.extra.insert("version".into(), 3.into()); + assert!(write(&manifest, &[], b"source").is_err()); + manifest.extra.remove("version"); + manifest.maps[0].extra.insert("file".into(), "other".into()); + assert!(write(&manifest, &[], b"source").is_err()); + } + #[test] fn refuses_duplicate_names_on_write() { let manifest = sample(2); From 26980ff331efa6ec2cb2b9093dc5a154a8f66bdb Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:38:06 +0000 Subject: [PATCH 29/98] Bound portable material detection --- crates/mcmat/README.md | 2 +- crates/mcmat/src/lib.rs | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/mcmat/README.md b/crates/mcmat/README.md index c14db09d7..074a0ac44 100644 --- a/crates/mcmat/README.md +++ b/crates/mcmat/README.md @@ -5,7 +5,7 @@ contains `material.json`, named texture maps and a source package. Applications parameter values and cook or render maps themselves; this crate does not depend on an app's model. It supports callers that read either version 1 or version 2 of the manifest, or both. -`Material::open` takes caller-provided version, entry-count, map-count, per-entry and total +`is_material` checks for a manifest with a bounded ZIP index; `Material::open` takes caller-provided version, entry-count, map-count, per-entry and total expanded-byte limits. Each subsequent `read` and `read_source` shares that budget, including repeated references to the same ZIP member. Duplicate member names are refused. The manifest preserves unknown fields at the top level and on maps; parameter, input and preset objects are diff --git a/crates/mcmat/src/lib.rs b/crates/mcmat/src/lib.rs index 2e38924b1..75b8709ef 100644 --- a/crates/mcmat/src/lib.rs +++ b/crates/mcmat/src/lib.rs @@ -69,6 +69,12 @@ pub enum Error { Archive(#[from] craft_zip::Error), } +/// Whether a bounded ZIP index contains a material manifest. This does not validate the manifest. +pub fn is_material(bytes: &[u8], max_entries: u64) -> bool { + Archive::open(bytes, craft_zip::Limits { max_entries, max_total_bytes: 0 }) + .is_ok_and(|archive| !archive.has_duplicate_names() && archive.contains(MANIFEST_ENTRY)) +} + /// Manifest and original archive, retaining one shared expanded-byte budget across reads. pub struct Material<'a> { pub manifest: Manifest, @@ -157,6 +163,8 @@ mod tests { for version in [1, 2] { let manifest = sample(version); let bytes = write(&manifest, &[("maps/base.png".into(), vec![1, 2, 3])], b"source").unwrap(); + assert!(is_material(&bytes, 100)); + assert!(!is_material(&bytes, 1)); let mut material = Material::open(&bytes, limits()).unwrap(); assert_eq!(material.manifest, manifest); assert_eq!(material.read("maps/base.png").unwrap(), [1, 2, 3]); From ff61266d6337abc59285dd3a6369b23ef0b445a0 Mon Sep 17 00:00:00 2001 From: el-j Date: Sun, 15 Nov 2026 12:38:57 +0100 Subject: [PATCH 30/98] craft-audio: QOA (Quite OK Audio) decoder and encoder Adds the qoa module behind the qoa, qoa-decode and qoa-encode features: whole .qoa files, 16-bit, 1 to 8 channels. Streaming files are unsupported. An ffmpeg oracle test checks that ffmpeg decodes our files to exactly the samples our decoder returns. Closes #1541 --- crates/audio/Cargo.toml | 8 + crates/audio/src/lib.rs | 5 +- crates/audio/src/qoa.rs | 411 +++++++++++++++++++++++++++++++ crates/audio/tests/qoa_oracle.rs | 67 +++++ 4 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 crates/audio/src/qoa.rs create mode 100644 crates/audio/tests/qoa_oracle.rs diff --git a/crates/audio/Cargo.toml b/crates/audio/Cargo.toml index c411a0357..645246130 100644 --- a/crates/audio/Cargo.toml +++ b/crates/audio/Cargo.toml @@ -38,6 +38,10 @@ aac-decode = ["craft-media/bitstream"] # The AAC encoder (`aac::Encoder`), and AAC in M4A through `encode` and `FileEncoder`. aac-encode = ["craft-media/bitstream", "craft-media/isobmff"] opus-decode = [] +# QOA (Quite OK Audio) files: decoder and encoder in the `qoa` module, each on its own. +qoa = ["qoa-decode", "qoa-encode"] +qoa-decode = [] +qoa-encode = [] opus-encode = [] ac3-decode = ["craft-media/bitstream"] mpa-decode = ["craft-media/bitstream"] @@ -84,6 +88,10 @@ required-features = ["decode", "lossless-encode", "peaks"] name = "codecs" required-features = ["decode", "encode"] +[[test]] +name = "qoa_oracle" +required-features = ["qoa"] + [[example]] name = "info" required-features = ["decode"] diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 33dcdee3b..d825b4a78 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -27,7 +27,8 @@ //! - Codec modules for packet-level use, each with its own features: [`aac`] (`aac-decode`, //! `aac-encode`), [`alac`] (`alac-decode`, `alac-encode`), [`opus`] (`opus-decode`, and //! [`opus::encoder`] with `opus-encode`), [`ac3`] (`ac3-decode`), [`mpa`] (`mpa-decode`), [`mp3`] -//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder). +//! (`mp3-decode`, `mp3-encode`), [`flac`] (`flac-encode`: the packet encoder), [`qoa`] +//! (`qoa-decode`, `qoa-encode`: whole QOA files). //! //! The shared types below build with no features. @@ -49,6 +50,8 @@ pub mod opus; mod pcm; #[cfg(feature = "peaks")] pub mod peaks; +#[cfg(any(feature = "qoa-decode", feature = "qoa-encode"))] +pub mod qoa; #[cfg(feature = "decode-native")] mod read; #[cfg(any(feature = "aac-decode", feature = "alac-decode", feature = "opus-decode", feature = "ac3-decode", feature = "mpa-decode"))] diff --git a/crates/audio/src/qoa.rs b/crates/audio/src/qoa.rs new file mode 100644 index 000000000..a04430137 --- /dev/null +++ b/crates/audio/src/qoa.rs @@ -0,0 +1,411 @@ +//! QOA ("Quite OK Audio"): a decoder (feature `qoa-decode`) and an encoder (feature `qoa-encode`) +//! for whole `.qoa` files, 16-bit, 1 to 8 channels. +//! +//! Written from the published QOA specification (qoaformat.org). A file is an 8-byte header (the +//! magic `qoaf` and the number of samples per channel) followed by frames of up to 5120 samples +//! per channel. Each frame starts with its channel count, sample rate, sample count and size, then +//! the state of one 4-tap sign-sign LMS predictor per channel, then 64-bit slices of 20 samples: +//! a 4-bit scale factor and twenty 3-bit quantised residuals, the channels interleaved slice by +//! slice. +//! +//! Streaming files (a sample count of 0 in the header) are not supported. +//! Samples are `i16`, one `Vec` per channel. + +#[cfg(feature = "qoa-decode")] +const MAX_FRAME_SAMPLES: usize = 5120; +const SLICE_SAMPLES: usize = 20; +const LMS_LEN: usize = 4; +const MAGIC: &[u8; 4] = b"qoaf"; +/// The most channels a file may have. +pub const MAX_CHANNELS: usize = 8; + +/// `dequant[scale factor][quantised residual]`: `round(round((sf + 1)^2.75) * [0.75, -0.75, 2.5, +/// -2.5, 4.5, -4.5, 7, -7])`. +const DEQUANT: [[i32; 8]; 16] = [ + [1, -1, 3, -3, 5, -5, 7, -7], + [5, -5, 18, -18, 32, -32, 49, -49], + [16, -16, 53, -53, 95, -95, 147, -147], + [34, -34, 113, -113, 203, -203, 315, -315], + [63, -63, 210, -210, 378, -378, 588, -588], + [104, -104, 345, -345, 621, -621, 966, -966], + [158, -158, 528, -528, 950, -950, 1477, -1477], + [228, -228, 760, -760, 1368, -1368, 2128, -2128], + [316, -316, 1053, -1053, 1895, -1895, 2947, -2947], + [422, -422, 1405, -1405, 2529, -2529, 3934, -3934], + [548, -548, 1828, -1828, 3290, -3290, 5117, -5117], + [696, -696, 2320, -2320, 4176, -4176, 6496, -6496], + [868, -868, 2893, -2893, 5207, -5207, 8099, -8099], + [1064, -1064, 3548, -3548, 6386, -6386, 9933, -9933], + [1286, -1286, 4288, -4288, 7718, -7718, 12005, -12005], + [1536, -1536, 5120, -5120, 9216, -9216, 14336, -14336], +]; + +/// What went wrong reading or writing a QOA file. +#[non_exhaustive] +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + /// The input is not a QOA file, or is damaged or truncated. + #[error("malformed QOA: {0}")] + Malformed(&'static str), + /// A valid file this module does not handle (a streaming file). + #[error("unsupported QOA: {0}")] + Unsupported(&'static str), + /// The samples or settings given to the encoder cannot be written as QOA. + #[error("QOA cannot encode: {0}")] + InvalidInput(&'static str), +} + +/// A decoded file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Audio { + /// Sample rate in Hz (1 to 16 777 215). + pub sample_rate: u32, + /// One `Vec` of samples per channel, all the same length. + pub channels: Vec>, +} + +/// The predictor: the last four samples and four weights. +#[derive(Clone, Copy)] +struct Lms { + history: [i32; LMS_LEN], + weights: [i32; LMS_LEN], +} + +impl Lms { + fn new() -> Self { + Lms { history: [0; LMS_LEN], weights: [0, 0, -(1 << 13), 1 << 14] } + } + + fn predict(&self) -> i32 { + let mut p = 0i32; + for i in 0..LMS_LEN { + p = p.wrapping_add(self.weights[i].wrapping_mul(self.history[i])); + } + p >> 13 + } + + fn update(&mut self, sample: i32, residual: i32) { + let delta = residual >> 4; + for i in 0..LMS_LEN { + self.weights[i] += if self.history[i] < 0 { -delta } else { delta }; + } + self.history.copy_within(1.., 0); + self.history[LMS_LEN - 1] = sample; + } +} + +fn clamp16(v: i32) -> i32 { + v.clamp(-32768, 32767) +} + +#[cfg(feature = "qoa-decode")] +mod decode { + use super::{Audio, DEQUANT, Error, LMS_LEN, Lms, MAGIC, MAX_CHANNELS, MAX_FRAME_SAMPLES, SLICE_SAMPLES, clamp16}; + + fn be16(b: &[u8], i: usize) -> u16 { + u16::from_be_bytes([b[i], b[i + 1]]) + } + + /// Decode a whole `.qoa` file. + /// + /// The declared length is checked against the input before anything is allocated: a slice of + /// 8 bytes holds at most 20 samples, so a file cannot declare more samples than that. + pub fn decode(bytes: &[u8]) -> Result { + if bytes.len() < 8 || &bytes[..4] != MAGIC { + return Err(Error::Malformed("missing the qoaf magic")); + } + let total = u32::from_be_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]) as usize; + if total == 0 { + return Err(Error::Unsupported("streaming files (a sample count of 0)")); + } + if bytes.len() < 16 { + return Err(Error::Malformed("truncated frame header")); + } + let channels = usize::from(bytes[8]); + let sample_rate = u32::from_be_bytes([0, bytes[9], bytes[10], bytes[11]]); + if channels == 0 || channels > MAX_CHANNELS || sample_rate == 0 { + return Err(Error::Malformed("bad channel count or sample rate")); + } + // 8 bytes (20 samples) per channel at the very least, plus the frame headers. + let room = (bytes.len() - 8) / (8 * channels) * SLICE_SAMPLES; + if total > room { + return Err(Error::Malformed("the file is too short for its declared length")); + } + let mut out: Vec> = Vec::new(); + for _ in 0..channels { + let mut v = Vec::new(); + v.try_reserve_exact(total).map_err(|_| Error::Malformed("too many samples"))?; + out.push(v); + } + let mut pos = 8; + let mut done = 0usize; + while done < total { + let head = bytes.get(pos..pos + 8).ok_or(Error::Malformed("truncated frame header"))?; + let rate = u32::from_be_bytes([0, head[1], head[2], head[3]]); + if usize::from(head[0]) != channels || rate != sample_rate { + return Err(Error::Malformed("the channel count or sample rate changes between frames")); + } + let samples = usize::from(be16(head, 4)); + let size = usize::from(be16(head, 6)); + let slices = samples.div_ceil(SLICE_SAMPLES); + if samples == 0 || samples > MAX_FRAME_SAMPLES || samples > total - done { + return Err(Error::Malformed("bad frame sample count")); + } + if size != 8 + 16 * channels + 8 * slices * channels { + return Err(Error::Malformed("frame size does not match its sample count")); + } + let frame = bytes.get(pos..pos + size).ok_or(Error::Malformed("truncated frame"))?; + let mut lms = [Lms::new(); MAX_CHANNELS]; + for (c, state) in lms.iter_mut().take(channels).enumerate() { + let at = 8 + c * 16; + for i in 0..LMS_LEN { + state.history[i] = i32::from(be16(frame, at + 2 * i) as i16); + state.weights[i] = i32::from(be16(frame, at + 8 + 2 * i) as i16); + } + } + let mut at = 8 + 16 * channels; + for s in 0..slices { + let n = (samples - s * SLICE_SAMPLES).min(SLICE_SAMPLES); + for (c, state) in lms.iter_mut().take(channels).enumerate() { + let mut slice = u64::from_be_bytes([ + frame[at], + frame[at + 1], + frame[at + 2], + frame[at + 3], + frame[at + 4], + frame[at + 5], + frame[at + 6], + frame[at + 7], + ]); + at += 8; + let table = &DEQUANT[(slice >> 60) as usize & 15]; + for _ in 0..n { + let dequantised = table[(slice >> 57) as usize & 7]; + slice <<= 3; + let sample = clamp16(state.predict() + dequantised); + out[c].push(sample as i16); + state.update(sample, dequantised); + } + } + } + pos += size; + done += samples; + } + Ok(Audio { sample_rate, channels: out }) + } +} +#[cfg(feature = "qoa-decode")] +pub use decode::decode; + +#[cfg(feature = "qoa-encode")] +mod encode { + use super::{DEQUANT, Error, Lms, MAGIC, MAX_CHANNELS, SLICE_SAMPLES, clamp16}; + + /// The quantised residual (0..8) whose dequantised value is nearest `residual`. + fn quantise(table: &[i32; 8], residual: i32) -> usize { + let mut best = 0; + let mut best_err = i64::MAX; + for (q, &d) in table.iter().enumerate() { + let e = i64::from(residual - d).pow(2); + if e < best_err { + best = q; + best_err = e; + } + } + best + } + + /// Encode one slice of one channel with every scale factor and keep the one with the least + /// squared error. Returns the 64-bit slice; `lms` is advanced to the chosen result. + fn encode_slice(lms: &mut Lms, samples: &[i16]) -> u64 { + let mut best: Option<(i64, u64, Lms)> = None; + for (sf, table) in DEQUANT.iter().enumerate() { + let mut trial = *lms; + let mut slice = sf as u64; + let mut err = 0i64; + for k in 0..SLICE_SAMPLES { + let mut q = 0; + if let Some(&s) = samples.get(k) { + let predicted = trial.predict(); + q = quantise(table, i32::from(s) - predicted); + let sample = clamp16(predicted + table[q]); + err += i64::from(i32::from(s) - sample).pow(2); + trial.update(sample, table[q]); + } + slice = slice << 3 | q as u64; + } + // 4 + 60 bits are in `slice`; the scale factor belongs on top. + let slice = slice & ((1 << 60) - 1) | (sf as u64) << 60; + if best.as_ref().is_none_or(|b| err < b.0) { + best = Some((err, slice, trial)); + } + } + match best { + Some((_, slice, state)) => { + *lms = state; + slice + } + None => 0, + } + } + + /// Encode `channels` (one `Vec` of samples per channel, all the same length) as a `.qoa` file. + pub fn encode(sample_rate: u32, channels: &[Vec]) -> Result, Error> { + let n = channels.len(); + if n == 0 || n > MAX_CHANNELS { + return Err(Error::InvalidInput("QOA stores 1 to 8 channels")); + } + if sample_rate == 0 || sample_rate > 0x00ff_ffff { + return Err(Error::InvalidInput("the sample rate must be 1 to 16777215 Hz")); + } + let total = channels[0].len(); + if channels.iter().any(|c| c.len() != total) { + return Err(Error::InvalidInput("the channels differ in length")); + } + let total32 = u32::try_from(total).map_err(|_| Error::InvalidInput("too many samples"))?; + if total == 0 { + return Err(Error::InvalidInput("QOA cannot store an empty file")); + } + let mut out = Vec::with_capacity(8 + total * n / 2); + out.extend_from_slice(MAGIC); + out.extend_from_slice(&total32.to_be_bytes()); + let mut lms = [Lms::new(); MAX_CHANNELS]; + let mut start = 0; + while start < total { + let len = (total - start).min(5120); + let slices = len.div_ceil(SLICE_SAMPLES); + let size = 8 + 16 * n + 8 * slices * n; + out.push(n as u8); + out.extend_from_slice(&sample_rate.to_be_bytes()[1..]); + out.extend_from_slice(&(len as u16).to_be_bytes()); + out.extend_from_slice(&(size as u16).to_be_bytes()); + // The state a frame starts from is written as 16-bit values, so the predictor + // continues from exactly what a decoder reads. + for state in lms.iter_mut().take(n) { + for v in state.history.iter_mut().chain(state.weights.iter_mut()) { + *v = i32::from(*v as i16); + out.extend_from_slice(&(*v as i16).to_be_bytes()); + } + } + for s in 0..slices { + let from = start + s * SLICE_SAMPLES; + let to = (from + SLICE_SAMPLES).min(start + len); + for (c, state) in lms.iter_mut().take(n).enumerate() { + out.extend_from_slice(&encode_slice(state, &channels[c][from..to]).to_be_bytes()); + } + } + start += len; + } + Ok(out) + } +} +#[cfg(feature = "qoa-encode")] +pub use encode::encode; + +#[cfg(all(test, feature = "qoa-decode", feature = "qoa-encode"))] +mod tests { + use super::*; + + fn tone(frames: usize, ch: usize) -> Vec> { + (0..ch) + .map(|c| (0..frames).map(|i| (((i as f64) * (0.03 + 0.01 * c as f64)).sin() * 12000.0 + ((i * 7919) % 200) as f64) as i16).collect()) + .collect() + } + + fn snr(a: &[i16], b: &[i16]) -> f64 { + let (mut s, mut e) = (0f64, 0f64); + for (&x, &y) in a.iter().zip(b) { + s += f64::from(x).powi(2); + e += (f64::from(x) - f64::from(y)).powi(2); + } + 10.0 * (s / e.max(1.0)).log10() + } + + #[test] + fn dequant_table_matches_its_formula() { + for (sf, row) in DEQUANT.iter().enumerate() { + let scale = ((sf as f64 + 1.0).powf(2.75)).round(); + for (q, base) in [0.75, -0.75, 2.5, -2.5, 4.5, -4.5, 7.0, -7.0].into_iter().enumerate() { + assert_eq!(f64::from(row[q]), (scale * base).round(), "sf {sf} q {q}"); + } + } + } + + #[test] + fn round_trips_with_a_good_signal_to_noise_ratio() { + for (frames, ch) in [(1, 1), (19, 2), (20, 1), (21, 2), (5120, 1), (5121, 2), (12_345, 8)] { + let src = tone(frames, ch); + let bytes = encode(44_100, &src).unwrap(); + let back = decode(&bytes).unwrap(); + assert_eq!((back.sample_rate, back.channels.len(), back.channels[0].len()), (44_100, ch, frames)); + if frames > 1000 { + for (c, (a, b)) in src.iter().zip(&back.channels).enumerate() { + assert!(snr(a, b) > 25.0, "{frames}x{ch} channel {c}"); + } + } + } + } + + #[test] + fn silence_and_full_scale_extremes_survive() { + let silent = vec![vec![0i16; 3000]]; + // No residual value is zero, so silence decodes to a tiny residue, never exactly 0. + let quiet = decode(&encode(8000, &silent).unwrap()).unwrap(); + assert!(quiet.channels[0].iter().all(|&s| s.abs() <= 2), "silence stays within 2 LSB"); + let square: Vec = (0..4000).map(|i| if (i / 50) % 2 == 0 { i16::MAX } else { i16::MIN }).collect(); + let back = decode(&encode(48_000, std::slice::from_ref(&square)).unwrap()).unwrap(); + assert_eq!(back.channels[0].len(), square.len()); + } + + #[test] + fn the_file_has_the_documented_layout() { + let bytes = encode(48_000, &tone(100, 2)).unwrap(); + assert_eq!(&bytes[..4], b"qoaf"); + assert_eq!(u32::from_be_bytes(bytes[4..8].try_into().unwrap()), 100); + assert_eq!(bytes[8], 2); + assert_eq!(&bytes[9..12], &48_000u32.to_be_bytes()[1..]); + assert_eq!(u16::from_be_bytes([bytes[12], bytes[13]]), 100); + // 8 header + 2 channels * 16 state + 5 slices * 2 channels * 8 + assert_eq!(usize::from(u16::from_be_bytes([bytes[14], bytes[15]])), 8 + 32 + 80); + assert_eq!(bytes.len(), 8 + 8 + 32 + 80); + // The first frame starts from the specified predictor state. + assert_eq!(&bytes[16 + 8..16 + 16], &[0, 0, 0, 0, 0xe0, 0, 0x40, 0]); + } + + #[test] + fn invalid_input_to_the_encoder_is_refused() { + assert!(encode(44_100, &[]).is_err()); + assert!(encode(0, &[vec![0; 10]]).is_err()); + assert!(encode(1 << 24, &[vec![0; 10]]).is_err()); + assert!(encode(44_100, &[vec![0; 10], vec![0; 9]]).is_err()); + assert!(encode(44_100, &[vec![]]).is_err()); + assert!(encode(44_100, &vec![vec![0; 10]; 9]).is_err()); + } + + #[test] + fn hostile_input_is_an_error_not_a_panic() { + let good = encode(44_100, &tone(6000, 2)).unwrap(); + for len in 0..good.len().min(600) { + let _ = decode(&good[..len]); + } + assert!(decode(&good[..good.len() - 1]).is_err()); + // A header that declares more samples than the bytes can hold. + let mut huge = good.clone(); + huge[4..8].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(decode(&huge).is_err()); + // A streaming file, and a frame whose size field lies. + let mut streaming = good.clone(); + streaming[4..8].fill(0); + assert_eq!(decode(&streaming), Err(Error::Unsupported("streaming files (a sample count of 0)"))); + let mut lying = good.clone(); + lying[14] ^= 1; + assert!(decode(&lying).is_err()); + let mut zero_rate = good.clone(); + zero_rate[9..12].fill(0); + assert!(decode(&zero_rate).is_err()); + let mut changed = good; + let second = 8 + usize::from(u16::from_be_bytes([changed[14], changed[15]])); + changed[second] = 1; + assert!(decode(&changed).is_err()); + } +} diff --git a/crates/audio/tests/qoa_oracle.rs b/crates/audio/tests/qoa_oracle.rs new file mode 100644 index 000000000..cf0ed8c37 --- /dev/null +++ b/crates/audio/tests/qoa_oracle.rs @@ -0,0 +1,67 @@ +//! QOA against ffmpeg's decoder as the reference (oracle; skipped when ffmpeg is absent, a failure +//! with `CRAFT_MEDIA_REQUIRE_ORACLES=1`): ffmpeg decodes the files our encoder writes to exactly +//! the samples our decoder returns for them, over several sizes (partial slices, partial frames, +//! several frames), channel counts and rates. + +use craft_audio::qoa::{decode, encode}; +use std::process::Command; + +fn signal(channels: usize, frames: usize) -> Vec> { + let mut seed = 0x2545_F491u32; + (0..channels) + .map(|c| { + (0..frames) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let tone = ((i as f64) * (0.02 + 0.013 * c as f64)).sin() * 9000.0; + // Tones, noise bursts, silence and full-scale peaks. + match (i / 700) % 4 { + 0 => tone as i16, + 1 => (tone + f64::from(seed % 6000) - 3000.0) as i16, + 2 => 0, + _ => { + if i % 2 == 0 { + i16::MAX + } else { + i16::MIN + } + } + } + }) + .collect() + }) + .collect() +} + +#[test] +fn ffmpeg_decodes_our_files_to_the_same_samples() { + let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("qoa"); + for (rate, channels, frames) in + [(44_100u32, 1usize, 1usize), (8_000, 1, 19), (48_000, 2, 5_121), (22_050, 2, 12_345), (96_000, 1, 20_000), (44_100, 4, 6_000)] + { + let src = signal(channels, frames); + let bytes = encode(rate, &src).expect("encode"); + let ours = decode(&bytes).expect("decode"); + let file = dir.join(format!("ours_{rate}_{channels}_{frames}.qoa")); + std::fs::write(&file, &bytes).expect("write fixture"); + let out = Command::new(&ffmpeg) + .args(["-v", "error", "-y", "-i"]) + .arg(&file) + .args(["-f", "s16le", "-acodec", "pcm_s16le", "-"]) + .output() + .expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg failed on {}: {}", file.display(), String::from_utf8_lossy(&out.stderr)); + let theirs: Vec = out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect(); + let mut interleaved = Vec::with_capacity(frames * channels); + for i in 0..frames { + for c in 0..channels { + interleaved.push(ours.channels[c][i]); + } + } + assert_eq!(theirs.len(), interleaved.len(), "{rate} Hz, {channels} ch, {frames} frames: sample count"); + assert!(theirs == interleaved, "{rate} Hz, {channels} ch, {frames} frames: samples differ"); + } +} From 6dfb580c08aeacfb0f30f79900d392394a14e798 Mon Sep 17 00:00:00 2001 From: Trevor Brown Date: Sun, 15 Nov 2026 06:39:11 -0500 Subject: [PATCH 31/98] cms: share annotated CUBE export and integer-depth precision Port header and quantization behavior from PhotoCraft d62728aeae8b331f69ec6d283add3e59fc47924e, retaining compact write_cube behavior. Adaptation and shared tests by Dylan M. Taylor. --- crates/cms/src/lutfile.rs | 71 ++++++++++++++++++++++++++++++++++----- 1 file changed, 63 insertions(+), 8 deletions(-) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index 404b5a315..ed1207f64 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -518,21 +518,47 @@ pub fn parse_look(text: &str) -> Result { LutFile { title: tag("title").unwrap_or_default(), size, data, domain_min: [0.0; 3], domain_max: [1.0; 3] }.check() } +/// Export presentation for a `.cube` table. Compact output preserves the existing writer. +#[derive(Clone, Copy, Debug, Default)] +pub struct CubeWriteOptions<'a> { + /// Add the annotated export header, including the default domain, with this creator. + pub created_by: Option<&'a str>, + /// Clamp to 0..1 and round to 1/32768 steps, for integer-depth export compatibility. + pub quantize_15_bit: bool, +} + /// Writes a `.cube` file (what Photoshop embeds in a Color Lookup layer). pub fn write_cube(l: &LutFile) -> String { + write_cube_with_options(l, CubeWriteOptions::default()) +} + +/// Writes a table with optional annotated headers and integer-depth export quantization. +/// As with [`write_cube`], callers supply a valid table in red-fastest order. +pub fn write_cube_with_options(l: &LutFile, options: CubeWriteOptions<'_>) -> String { let mut s = String::new(); - if !l.title.is_empty() { + if let Some(creator) = options.created_by { + s.push_str(&format!("#Created by: {creator}\n")); + } + if !l.title.is_empty() || options.created_by.is_some() { s.push_str(&format!("TITLE \"{}\"\n", l.title.replace('"', "'"))); } - // The domain is what every row means: a table read with a non-default one must not be - // written back as if it expected inputs in 0..1. `parse_cube` accepts these anywhere. - if !l.domain_is_default() { - s.push_str(&format!("DOMAIN_MIN {} {} {}\n", l.domain_min[0], l.domain_min[1], l.domain_min[2])); - s.push_str(&format!("DOMAIN_MAX {} {} {}\n", l.domain_max[0], l.domain_max[1], l.domain_max[2])); + if options.created_by.is_some() { + s.push_str(&format!("\n#LUT size\nLUT_3D_SIZE {}\n\n#data domain\n", l.size)); + } + // The input domain must survive a write. Annotated exports also spell out the default. + if !l.domain_is_default() || options.created_by.is_some() { + let domain = |v: f32| if options.created_by.is_some() { format!("{v:?}") } else { v.to_string() }; + s.push_str(&format!("DOMAIN_MIN {} {} {}\n", domain(l.domain_min[0]), domain(l.domain_min[1]), domain(l.domain_min[2]))); + s.push_str(&format!("DOMAIN_MAX {} {} {}\n", domain(l.domain_max[0]), domain(l.domain_max[1]), domain(l.domain_max[2]))); + } + if options.created_by.is_some() { + s.push_str("\n#LUT data points\n"); + } else { + s.push_str(&format!("LUT_3D_SIZE {}\n", l.size)); } - s.push_str(&format!("LUT_3D_SIZE {}\n", l.size)); for c in l.data.as_chunks::<3>().0 { - s.push_str(&format!("{:.6} {:.6} {:.6}\n", c[0], c[1], c[2])); + let value = |v: f32| if options.quantize_15_bit { (v.clamp(0.0, 1.0) * 32768.0).round() / 32768.0 } else { v }; + s.push_str(&format!("{:.6} {:.6} {:.6}\n", value(c[0]), value(c[1]), value(c[2]))); } s } @@ -1299,4 +1325,33 @@ LUT_3D_SIZE 2 assert_eq!(back.domain_min, l.domain_min, "the written table must mean the same"); assert_eq!(back.domain_max, l.domain_max); } + #[test] + fn annotated_cube_export_preserves_header_and_integer_precision() { + let mut lut = LutFile::identity(2).unwrap(); + lut.title = "My \"Look\"".into(); + lut.data[..3].copy_from_slice(&[1.0 / 6.0, 5.0 / 6.0, 160.0 / 255.0]); + let output = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("PhotoCraft"), quantize_15_bit: true }); + assert!(output.starts_with("#Created by: PhotoCraft\nTITLE \"My 'Look'\"\n\n#LUT size\nLUT_3D_SIZE 2\n\n#data domain\nDOMAIN_MIN 0.0 0.0 0.0\nDOMAIN_MAX 1.0 1.0 1.0\n\n#LUT data points\n0.166656 0.833344 0.627441\n")); + assert_eq!(output.lines().count(), 11 + 8); + let parsed = parse_cube(&output).unwrap(); + assert_eq!(parsed.size, 2); + assert_eq!(&parsed.data[3..], &lut.data[3..]); + let float = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("PhotoCraft"), quantize_15_bit: false }); + assert!(float.contains("0.166667 0.833333 0.627451\n")); + } + + #[test] + fn annotated_domain_and_empty_title_roundtrip() { + let mut lut = LutFile::identity(2).unwrap(); + lut.title.clear(); + lut.domain_min = [-1.0, 0.25, 0.5]; + lut.domain_max = [2.0, 0.75, 1.5]; + let text = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("Exporter"), ..Default::default() }); + assert!(text.contains("TITLE \"\"\n")); + let back = parse_cube(&text).unwrap(); + assert_eq!(back.domain_min, lut.domain_min); + assert_eq!(back.domain_max, lut.domain_max); + assert_eq!(back.data, lut.data); + assert_eq!(write_cube(&lut), write_cube_with_options(&lut, CubeWriteOptions::default())); + } } From 02c639c3578db902d197134ceaf62cea3c9ba209 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:40:59 +0000 Subject: [PATCH 32/98] cms: share commented CUBE export layout --- crates/cms/src/lutfile.rs | 48 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index 404b5a315..c52c145f5 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -531,10 +531,32 @@ pub fn write_cube(l: &LutFile) -> String { s.push_str(&format!("DOMAIN_MAX {} {} {}\n", l.domain_max[0], l.domain_max[1], l.domain_max[2])); } s.push_str(&format!("LUT_3D_SIZE {}\n", l.size)); + write_cube_rows(&mut s, l); + s +} + +/// Writes a `.cube` file with a creator comment, section comments and an explicit input domain. +/// +/// This layout is useful for exported LUTs; [`write_cube`] keeps its compact layout for embedded +/// tables. Values retain six decimal places, and their order and quantisation belong to the +/// caller. A default domain is written as `0.0` and `1.0` for compatibility with export consumers. +pub fn write_cube_commented(l: &LutFile, creator: &str) -> String { + let mut s = format!("#Created by: {creator}\nTITLE \"{}\"\n\n#LUT size\nLUT_3D_SIZE {}\n\n#data domain\n", l.title.replace('"', "'"), l.size); + if l.domain_is_default() { + s.push_str("DOMAIN_MIN 0.0 0.0 0.0\nDOMAIN_MAX 1.0 1.0 1.0\n"); + } else { + s.push_str(&format!("DOMAIN_MIN {} {} {}\n", l.domain_min[0], l.domain_min[1], l.domain_min[2])); + s.push_str(&format!("DOMAIN_MAX {} {} {}\n", l.domain_max[0], l.domain_max[1], l.domain_max[2])); + } + s.push_str("\n#LUT data points\n"); + write_cube_rows(&mut s, l); + s +} + +fn write_cube_rows(s: &mut String, l: &LutFile) { for c in l.data.as_chunks::<3>().0 { s.push_str(&format!("{:.6} {:.6} {:.6}\n", c[0], c[1], c[2])); } - s } fn luma(c: [f32; 3]) -> f32 { @@ -602,6 +624,30 @@ pub fn builtin(id: &str) -> Option { mod tests { use super::*; + #[test] + fn commented_cube_preserves_export_header_and_rows() { + let mut lut = super::LutFile::identity(2).unwrap(); + lut.title = "A \"look\"".into(); + let text = super::write_cube_commented(&lut, "ExampleCraft"); + assert!(text.starts_with("#Created by: ExampleCraft\nTITLE \"A 'look'\"\n\n#LUT size\nLUT_3D_SIZE 2\n\n#data domain\nDOMAIN_MIN 0.0 0.0 0.0\nDOMAIN_MAX 1.0 1.0 1.0\n\n#LUT data points\n")); + let parsed = super::parse_cube(&text).unwrap(); + assert_eq!(parsed.data, lut.data); + assert_eq!(parsed.title, "A 'look'"); + assert_eq!(text.lines().count(), 11 + 8); + let rows = |s: &str| s.lines().filter(|line| line.starts_with(|c: char| c.is_ascii_digit())).map(str::to_owned).collect::>(); + assert_eq!(rows(&text), rows(&super::write_cube(&lut))); + } + + #[test] + fn commented_cube_keeps_nondefault_domain_and_sample_precision() { + let mut lut = super::LutFile::identity(2).unwrap(); + lut.domain_min = [0.125, 0.25, 0.375]; + lut.domain_max = [0.625, 0.75, 0.875]; + lut.data[0] = 0.833344; + let parsed = super::parse_cube(&super::write_cube_commented(&lut, "ExampleCraft")).unwrap(); + assert_eq!(parsed, lut); + } + #[test] fn cube_title_hashes_and_comment_quotes_are_distinct() { let rows = "LUT_1D_SIZE 2\n0 0 0 # low\n1 1 1 # high\n"; From c4ed8d3a84caf683132e51db94293e3aab3b111b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 06:49:49 -0500 Subject: [PATCH 33/98] cms: stream annotated export rows through the existing CUBE formatter --- crates/cms/src/lutfile.rs | 84 +++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 47 deletions(-) diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index ed1207f64..0e8767dc6 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -518,51 +518,48 @@ pub fn parse_look(text: &str) -> Result { LutFile { title: tag("title").unwrap_or_default(), size, data, domain_min: [0.0; 3], domain_max: [1.0; 3] }.check() } -/// Export presentation for a `.cube` table. Compact output preserves the existing writer. -#[derive(Clone, Copy, Debug, Default)] -pub struct CubeWriteOptions<'a> { - /// Add the annotated export header, including the default domain, with this creator. - pub created_by: Option<&'a str>, - /// Clamp to 0..1 and round to 1/32768 steps, for integer-depth export compatibility. - pub quantize_15_bit: bool, -} - /// Writes a `.cube` file (what Photoshop embeds in a Color Lookup layer). pub fn write_cube(l: &LutFile) -> String { - write_cube_with_options(l, CubeWriteOptions::default()) -} - -/// Writes a table with optional annotated headers and integer-depth export quantization. -/// As with [`write_cube`], callers supply a valid table in red-fastest order. -pub fn write_cube_with_options(l: &LutFile, options: CubeWriteOptions<'_>) -> String { let mut s = String::new(); - if let Some(creator) = options.created_by { - s.push_str(&format!("#Created by: {creator}\n")); - } - if !l.title.is_empty() || options.created_by.is_some() { + if !l.title.is_empty() { s.push_str(&format!("TITLE \"{}\"\n", l.title.replace('"', "'"))); } - if options.created_by.is_some() { - s.push_str(&format!("\n#LUT size\nLUT_3D_SIZE {}\n\n#data domain\n", l.size)); + // The domain is what every row means: a table read with a non-default one must not be + // written back as if it expected inputs in 0..1. `parse_cube` accepts these anywhere. + if !l.domain_is_default() { + s.push_str(&format!("DOMAIN_MIN {} {} {}\n", l.domain_min[0], l.domain_min[1], l.domain_min[2])); + s.push_str(&format!("DOMAIN_MAX {} {} {}\n", l.domain_max[0], l.domain_max[1], l.domain_max[2])); } - // The input domain must survive a write. Annotated exports also spell out the default. - if !l.domain_is_default() || options.created_by.is_some() { - let domain = |v: f32| if options.created_by.is_some() { format!("{v:?}") } else { v.to_string() }; - s.push_str(&format!("DOMAIN_MIN {} {} {}\n", domain(l.domain_min[0]), domain(l.domain_min[1]), domain(l.domain_min[2]))); - s.push_str(&format!("DOMAIN_MAX {} {} {}\n", domain(l.domain_max[0]), domain(l.domain_max[1]), domain(l.domain_max[2]))); - } - if options.created_by.is_some() { - s.push_str("\n#LUT data points\n"); - } else { - s.push_str(&format!("LUT_3D_SIZE {}\n", l.size)); - } - for c in l.data.as_chunks::<3>().0 { - let value = |v: f32| if options.quantize_15_bit { (v.clamp(0.0, 1.0) * 32768.0).round() / 32768.0 } else { v }; - s.push_str(&format!("{:.6} {:.6} {:.6}\n", value(c[0]), value(c[1]), value(c[2]))); + s.push_str(&format!("LUT_3D_SIZE {}\n", l.size)); + cube_rows(s, l.data.as_chunks::<3>().0.iter().copied()) +} + +fn cube_rows(mut s: String, rows: impl Iterator) -> String { + for c in rows { + s.push_str(&format!("{:.6} {:.6} {:.6}\n", c[0], c[1], c[2])); } s } +/// Annotated identity-domain CUBE export, streamed in red-fastest order without a table copy. +/// Callers supply exactly `size³` rows for a valid size (2..=[`MAX_SIZE`]), as with [`write_cube`]. +/// Integer-depth exports clamp/round to 1/32768 steps; floating exports clamp without rounding. +pub fn write_cube_export(creator: &str, title: &str, size: usize, integer_depth: bool, rows: impl Iterator) -> String { + let s = format!( + "#Created by: {creator}\nTITLE \"{}\"\n\n#LUT size\nLUT_3D_SIZE {size}\n\n#data domain\nDOMAIN_MIN 0.0 0.0 0.0\nDOMAIN_MAX 1.0 1.0 1.0\n\n#LUT data points\n", + title.replace('"', "'") + ); + cube_rows( + s, + rows.map(|c| { + c.map(|v| { + let v = v.clamp(0.0, 1.0); + if integer_depth { (v * 32768.0).round() / 32768.0 } else { v } + }) + }), + ) +} + fn luma(c: [f32; 3]) -> f32 { 0.299 * c[0] + 0.587 * c[1] + 0.114 * c[2] } @@ -1330,28 +1327,21 @@ LUT_3D_SIZE 2 let mut lut = LutFile::identity(2).unwrap(); lut.title = "My \"Look\"".into(); lut.data[..3].copy_from_slice(&[1.0 / 6.0, 5.0 / 6.0, 160.0 / 255.0]); - let output = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("PhotoCraft"), quantize_15_bit: true }); + let output = write_cube_export("PhotoCraft", &lut.title, lut.size, true, lut.data.as_chunks::<3>().0.iter().copied()); assert!(output.starts_with("#Created by: PhotoCraft\nTITLE \"My 'Look'\"\n\n#LUT size\nLUT_3D_SIZE 2\n\n#data domain\nDOMAIN_MIN 0.0 0.0 0.0\nDOMAIN_MAX 1.0 1.0 1.0\n\n#LUT data points\n0.166656 0.833344 0.627441\n")); assert_eq!(output.lines().count(), 11 + 8); let parsed = parse_cube(&output).unwrap(); assert_eq!(parsed.size, 2); assert_eq!(&parsed.data[3..], &lut.data[3..]); - let float = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("PhotoCraft"), quantize_15_bit: false }); + let float = write_cube_export("PhotoCraft", &lut.title, lut.size, false, lut.data.as_chunks::<3>().0.iter().copied()); assert!(float.contains("0.166667 0.833333 0.627451\n")); } #[test] - fn annotated_domain_and_empty_title_roundtrip() { - let mut lut = LutFile::identity(2).unwrap(); - lut.title.clear(); - lut.domain_min = [-1.0, 0.25, 0.5]; - lut.domain_max = [2.0, 0.75, 1.5]; - let text = write_cube_with_options(&lut, CubeWriteOptions { created_by: Some("Exporter"), ..Default::default() }); + fn annotated_empty_title_and_float_clamping() { + let text = write_cube_export("Exporter", "", 2, false, std::iter::repeat_n([-0.5, 0.25, 1.5], 8)); assert!(text.contains("TITLE \"\"\n")); let back = parse_cube(&text).unwrap(); - assert_eq!(back.domain_min, lut.domain_min); - assert_eq!(back.domain_max, lut.domain_max); - assert_eq!(back.data, lut.data); - assert_eq!(write_cube(&lut), write_cube_with_options(&lut, CubeWriteOptions::default())); + assert_eq!(back.data, [0.0, 0.25, 1.0].repeat(8)); } } From e67d770c09614f3b2a582b4d229910040b94d262 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:51:16 +0000 Subject: [PATCH 34/98] Add bounded JSON-RPC server envelopes --- Cargo.lock | 7 ++ Cargo.toml | 1 + crates/jsonrpc/Cargo.toml | 14 +++ crates/jsonrpc/README.md | 27 +++++ crates/jsonrpc/src/lib.rs | 186 +++++++++++++++++++++++++++++++++ crates/jsonrpc/tests/server.rs | 145 +++++++++++++++++++++++++ 6 files changed, 380 insertions(+) create mode 100644 crates/jsonrpc/Cargo.toml create mode 100644 crates/jsonrpc/README.md create mode 100644 crates/jsonrpc/src/lib.rs create mode 100644 crates/jsonrpc/tests/server.rs diff --git a/Cargo.lock b/Cargo.lock index 345553bd9..570259c92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1969,6 +1969,13 @@ dependencies = [ "zune-jpeg", ] +[[package]] +name = "craft-jsonrpc" +version = "0.1.0" +dependencies = [ + "serde_json", +] + [[package]] name = "craft-jxl" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20da79..5b19da360 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } +craft-jsonrpc = { path = "crates/jsonrpc" } craft-jxl = { path = "crates/jxl" } craft-kiwi = { path = "crates/kiwi" } craft-keymap = { path = "crates/keymap" } diff --git a/crates/jsonrpc/Cargo.toml b/crates/jsonrpc/Cargo.toml new file mode 100644 index 000000000..7456ececf --- /dev/null +++ b/crates/jsonrpc/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-jsonrpc" +description = "Bounded JSON-RPC 2.0 server envelopes without transport or application routing" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } diff --git a/crates/jsonrpc/README.md b/crates/jsonrpc/README.md new file mode 100644 index 000000000..0b78e5470 --- /dev/null +++ b/crates/jsonrpc/README.md @@ -0,0 +1,27 @@ +# craft-jsonrpc + +`craft-jsonrpc` provides bounded JSON-RPC 2.0 server envelopes independently of a transport or an +application's method router. It can read a bounded newline-delimited message, parse it from bytes, +check request IDs and batch sizes, dispatch requests and notifications separately, and build +response envelopes. It does not start a server, choose an MCP protocol version, or interpret tool +names and arguments. + +`Limits` defaults to 64 MiB per message and 4096 batch entries. `parse_json` checks the byte limit +before deserialization, and `process` rejects an oversized batch before calling any handler. +For newline-delimited transports, `read_line` bounds memory while reading; after an over-limit +line, the caller must close the stream or discard the rest of that line. +`serde_json`'s nesting limit remains in effect. Response sizes are the caller's responsibility. + +A valid request receives the handler's result or error with its original string, integer, or null +ID. An omitted ID is a notification: it calls the separate notification handler and gets no reply. +Malformed batch entries receive an invalid-request error; an empty batch gets one error response; +a batch containing only notifications has no response. Invalid parameter shapes receive an +invalid-params error for requests and no reply for notifications. The handler decides whether +object or array parameters are valid for each method. + +The caller may inspect the bounded parsed value before `process` to support an existing legacy +protocol. Parse failures and limit failures are returned explicitly so each transport can choose +its error and connection policy; `parse_error_response` builds the JSON-RPC parse error. + +The envelope behavior is extracted from StencilCraft's JSON-lines MCP/control server. App-specific +method routing, legacy control requests, and MCP lifecycle policy stay with the app. diff --git a/crates/jsonrpc/src/lib.rs b/crates/jsonrpc/src/lib.rs new file mode 100644 index 000000000..7389869c3 --- /dev/null +++ b/crates/jsonrpc/src/lib.rs @@ -0,0 +1,186 @@ +//! Bounded JSON-RPC 2.0 server envelopes without a transport or application router. +//! +//! Parse a bounded line with [`parse_json`], then pass the value to [`process`]. A host may +//! inspect the parsed value first for a legacy protocol. `process` validates envelopes, dispatches +//! requests and notifications separately, and constructs single or batch replies. The host owns +//! the transport, method semantics and response-size policy. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use serde_json::{Value, json}; +use std::io::BufRead; + +/// Bounds on a single JSON document and on the number of requests in a batch. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + pub max_message_bytes: usize, + pub max_batch_items: usize, +} + +impl Default for Limits { + fn default() -> Self { + Self { max_message_bytes: 64 * 1024 * 1024, max_batch_items: 4096 } + } +} + +/// Failure before dispatch. An over-limit batch is rejected before any callback is invoked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ParseError { + MessageTooLarge, + InvalidJson, + BatchTooLarge, +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::MessageTooLarge => "JSON-RPC message exceeds the byte limit", + Self::InvalidJson => "invalid JSON", + Self::BatchTooLarge => "JSON-RPC batch exceeds the item limit", + }) + } +} +impl std::error::Error for ParseError {} + +/// Error reading a newline-delimited message. After `TooLong`, close or otherwise discard the +/// remaining input line before reading another message. +#[derive(Debug)] +pub enum LineError { + Io(std::io::Error), + TooLong, +} + +impl std::fmt::Display for LineError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(error) => write!(f, "message read failed: {error}"), + Self::TooLong => f.write_str("message exceeds the byte limit"), + } + } +} +impl std::error::Error for LineError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(error) => Some(error), + Self::TooLong => None, + } + } +} + +/// Read at most `max_bytes` before a newline, without allocating for an oversized line. The +/// newline is consumed but omitted from the returned bytes. EOF with no bytes returns `None`; +/// an empty line returns `Some(Vec::new())`. +pub fn read_line(reader: &mut R, max_bytes: usize) -> Result>, LineError> { + let mut line = Vec::new(); + loop { + let bytes = reader.fill_buf().map_err(LineError::Io)?; + if bytes.is_empty() { + return Ok((!line.is_empty()).then_some(line)); + } + let newline = bytes.iter().position(|&byte| byte == b'\n'); + let len = newline.unwrap_or(bytes.len()); + if line.len().saturating_add(len) > max_bytes { + return Err(LineError::TooLong); + } + line.extend_from_slice(&bytes[..len]); + reader.consume(len + usize::from(newline.is_some())); + if newline.is_some() { + return Ok(Some(line)); + } + } +} + +/// Parse JSON after checking its byte length. `serde_json` also bounds nesting depth. +pub fn parse_json(bytes: &[u8], limits: Limits) -> Result { + if bytes.len() > limits.max_message_bytes { + return Err(ParseError::MessageTooLarge); + } + serde_json::from_slice(bytes).map_err(|_| ParseError::InvalidJson) +} + +/// A validated request or notification. A present null `id` is distinct from an omitted `id`. +#[derive(Clone, Copy, Debug)] +pub struct Request<'a> { + pub id: Option<&'a Value>, + pub method: &'a str, + pub params: Option<&'a Value>, +} + +/// A JSON-RPC error returned by a method handler. +#[derive(Clone, Debug, PartialEq)] +pub struct RpcError { + pub code: i64, + pub message: String, + pub data: Option, +} + +impl RpcError { + pub fn new(code: i64, message: impl Into) -> Self { + Self { code, message: message.into(), data: None } + } +} + +/// Build an error response, including optional error data. +pub fn error_response(id: Value, error: RpcError) -> Value { + let mut detail = json!({"code": error.code, "message": error.message}); + if let (Some(data), Some(object)) = (error.data, detail.as_object_mut()) { + object.insert("data".into(), data); + } + json!({"jsonrpc": "2.0", "id": id, "error": detail}) +} + +/// Build the standard parse-error response for input that is not JSON. +pub fn parse_error_response() -> Value { + error_response(Value::Null, RpcError::new(-32700, "Parse error")) +} + +fn invalid_request() -> Value { + error_response(Value::Null, RpcError::new(-32600, "Invalid Request")) +} + +fn one(value: &Value, on_request: &mut F, on_notification: &mut N) -> Option +where + F: FnMut(Request<'_>) -> Result, + N: FnMut(Request<'_>), +{ + let Some(object) = value.as_object() else { return Some(invalid_request()) }; + let id = object.get("id"); + let valid_id = id.is_none_or(|v| v.is_null() || v.is_string() || v.is_i64() || v.is_u64()); + let method = object.get("method").and_then(Value::as_str); + if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") || method.is_none() || !valid_id { + return Some(invalid_request()); + } + let Some(method) = method else { return Some(invalid_request()) }; + let request = Request { id, method, params: object.get("params") }; + if request.params.is_some_and(|v| !v.is_object() && !v.is_array()) { + return id.map(|id| error_response(id.clone(), RpcError::new(-32602, "Invalid params"))); + } + let Some(id) = id else { + on_notification(request); + return None; + }; + Some(match on_request(request) { + Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}), + Err(error) => error_response(id.clone(), error), + }) +} + +/// Process a parsed JSON-RPC message. Notifications never receive a response. Invalid batch +/// entries produce individual errors; an empty batch produces one invalid-request response. +/// A batch over the limit returns [`ParseError::BatchTooLarge`] without invoking either callback. +pub fn process(value: &Value, limits: Limits, mut on_request: F, mut on_notification: N) -> Result, ParseError> +where + F: FnMut(Request<'_>) -> Result, + N: FnMut(Request<'_>), +{ + let Some(batch) = value.as_array() else { return Ok(one(value, &mut on_request, &mut on_notification)) }; + if batch.len() > limits.max_batch_items { + return Err(ParseError::BatchTooLarge); + } + if batch.is_empty() { + return Ok(Some(invalid_request())); + } + let responses: Vec<_> = batch.iter().filter_map(|v| one(v, &mut on_request, &mut on_notification)).collect(); + Ok((!responses.is_empty()).then_some(Value::Array(responses))) +} diff --git a/crates/jsonrpc/tests/server.rs b/crates/jsonrpc/tests/server.rs new file mode 100644 index 000000000..2a03d57b4 --- /dev/null +++ b/crates/jsonrpc/tests/server.rs @@ -0,0 +1,145 @@ +use craft_jsonrpc::{Limits, ParseError, RpcError, parse_error_response, parse_json, process}; +use serde_json::{Value, json}; + +#[test] +fn batches_preserve_ids_order_and_suppress_notification_replies() { + let input = json!([ + {"jsonrpc": "2.0", "method": "notifications/initialized"}, + {"jsonrpc": "2.0", "id": "a", "method": "ping"}, + {"jsonrpc": "2.0", "id": 2, "method": "unknown"}, + 17, + {"jsonrpc": "2.0", "id": null, "method": "ping"} + ]); + let mut notifications = Vec::new(); + let result = process( + &input, + Limits::default(), + |request| match request.method { + "ping" => Ok(json!({})), + _ => Err(RpcError::new(-32601, "Method not found")), + }, + |request| notifications.push(request.method.to_owned()), + ) + .unwrap() + .unwrap(); + assert_eq!(notifications, ["notifications/initialized"]); + assert_eq!( + result, + json!([ + {"jsonrpc":"2.0", "id":"a", "result":{}}, + {"jsonrpc":"2.0", "id":2, "error":{"code":-32601,"message":"Method not found"}}, + {"jsonrpc":"2.0", "id":null, "error":{"code":-32600,"message":"Invalid Request"}}, + {"jsonrpc":"2.0", "id":null, "result":{}} + ]) + ); +} + +#[test] +fn notifications_never_receive_responses_including_all_notification_batches() { + let input = json!([ + {"jsonrpc":"2.0", "method":"tools/call", "params":{"name":"mutating"}}, + {"jsonrpc":"2.0", "method":"notifications/initialized"} + ]); + let mut requests = 0; + let mut notices = 0; + let result = process( + &input, + Limits::default(), + |_| { + requests += 1; + Ok(Value::Null) + }, + |_| notices += 1, + ) + .unwrap(); + assert!(result.is_none()); + assert_eq!(requests, 0); + assert_eq!(notices, 2); +} + +#[test] +fn invalid_envelopes_and_params_return_standard_errors() { + for id in [json!(1.5), json!(true), json!([]), json!({})] { + let input = json!({"jsonrpc":"2.0", "id":id, "method":"ping"}); + let reply = process(&input, Limits::default(), |_| panic!("invalid id dispatched"), |_| panic!("invalid id notified")).unwrap().unwrap(); + assert_eq!(reply["id"], Value::Null); + assert_eq!(reply["error"]["code"], -32600); + } + for input in [json!({}), json!({"jsonrpc":"1.0", "id":1, "method":"ping"}), json!({"jsonrpc":"2.0", "id":1}), json!([])] { + let reply = + process(&input, Limits::default(), |_| panic!("invalid request dispatched"), |_| panic!("invalid notification")).unwrap().unwrap(); + assert_eq!(reply["error"]["code"], -32600); + } + let input = json!({"jsonrpc":"2.0", "id":4, "method":"ping", "params":17}); + let reply = process(&input, Limits::default(), |_| panic!("invalid params dispatched"), |_| {}).unwrap().unwrap(); + assert_eq!(reply["id"], 4); + assert_eq!(reply["error"]["code"], -32602); + let notification = json!({"jsonrpc":"2.0", "method":"notice", "params":17}); + assert!(process(¬ification, Limits::default(), |_| panic!("invalid request"), |_| panic!("invalid notification")).unwrap().is_none()); +} + +#[test] +fn arrays_are_valid_params_and_errors_can_include_data() { + let input = json!({"jsonrpc":"2.0", "id":-7, "method":"test", "params":[1, 2]}); + let reply = process( + &input, + Limits::default(), + |request| { + assert_eq!(request.params, Some(&json!([1, 2]))); + let mut error = RpcError::new(-32000, "failure"); + error.data = Some(json!({"why":"example"})); + Err(error) + }, + |_| {}, + ) + .unwrap() + .unwrap(); + assert_eq!(reply, json!({"jsonrpc":"2.0", "id":-7, "error":{"code":-32000,"message":"failure","data":{"why":"example"}}})); +} + +#[test] +fn parse_and_batch_limits_apply_before_dispatch() { + let limits = Limits { max_message_bytes: 3, max_batch_items: 1 }; + assert_eq!(parse_json(b"null", limits), Err(ParseError::MessageTooLarge)); + assert_eq!(parse_json(b"123", limits).unwrap(), json!(123)); + assert_eq!(parse_json(b"{", limits), Err(ParseError::InvalidJson)); + assert_eq!(parse_error_response()["error"]["code"], -32700); + let input = json!([{"jsonrpc":"2.0", "id":1, "method":"ping"}, {"jsonrpc":"2.0", "method":"notice"}]); + assert_eq!( + process(&input, limits, |_| panic!("over-limit batch dispatched"), |_| panic!("over-limit batch notified")), + Err(ParseError::BatchTooLarge) + ); + let one = json!([{"jsonrpc":"2.0", "id":1, "method":"ping"}]); + assert_eq!(process(&one, limits, |_| Ok(json!({})), |_| {}).unwrap().unwrap().as_array().unwrap().len(), 1); +} + +#[test] +fn invalid_utf8_and_excessive_nesting_are_rejected() { + assert_eq!(parse_json(&[0xff], Limits::default()), Err(ParseError::InvalidJson)); + let nested = format!("{}0{}", "[".repeat(200), "]".repeat(200)); + assert_eq!(parse_json(nested.as_bytes(), Limits::default()), Err(ParseError::InvalidJson)); +} + +#[test] +fn bounded_line_read_handles_chunks_empty_lines_and_eof() { + use std::io::{BufReader, Cursor}; + let mut reader = BufReader::with_capacity(2, Cursor::new(b"abc\n\nxy")); + assert_eq!(craft_jsonrpc::read_line(&mut reader, 3).unwrap(), Some(b"abc".to_vec())); + assert_eq!(craft_jsonrpc::read_line(&mut reader, 3).unwrap(), Some(Vec::new())); + assert_eq!(craft_jsonrpc::read_line(&mut reader, 3).unwrap(), Some(b"xy".to_vec())); + assert_eq!(craft_jsonrpc::read_line(&mut reader, 3).unwrap(), None); + let mut too_long = BufReader::with_capacity(2, Cursor::new(b"abcd\n")); + assert!(matches!(craft_jsonrpc::read_line(&mut too_long, 3), Err(craft_jsonrpc::LineError::TooLong))); +} + +#[test] +fn bounded_line_read_propagates_io_errors() { + struct Broken; + impl std::io::Read for Broken { + fn read(&mut self, _: &mut [u8]) -> std::io::Result { + Err(std::io::Error::other("broken")) + } + } + let mut reader = std::io::BufReader::new(Broken); + assert!(matches!(craft_jsonrpc::read_line(&mut reader, 1), Err(craft_jsonrpc::LineError::Io(_)))); +} From 3ce92b332db8fa1f8d33214d299e88096263c007 Mon Sep 17 00:00:00 2001 From: el-j Date: Sun, 15 Nov 2026 12:53:20 +0100 Subject: [PATCH 35/98] craft-image-codecs: JPEG-LS (ITU-T T.87) reader and writer Adds Format::JpegLs: lossless and near-lossless gray and RGB, 2 to 16 bits. Reads scans that code one component each; writes lossless 8- and 16-bit gray and RGB. Line- and sample-interleaved scans, mapping tables and colour transforms are reported as unsupported. Closes #1558 --- Cargo.lock | 1 + crates/image-codecs/Cargo.toml | 1 + crates/image-codecs/README.md | 1 + crates/image-codecs/src/codecs.rs | 1 + crates/image-codecs/src/codecs/jpegls.rs | 870 ++++++++++++++++++ crates/image-codecs/src/format.rs | 14 +- crates/image-codecs/src/lib.rs | 4 +- crates/image-codecs/tests/it/detect_caps.rs | 16 + crates/image-codecs/tests/it/jpegls_oracle.rs | 103 +++ crates/image-codecs/tests/it/main.rs | 1 + 10 files changed, 1009 insertions(+), 3 deletions(-) create mode 100644 crates/image-codecs/src/codecs/jpegls.rs create mode 100644 crates/image-codecs/tests/it/jpegls_oracle.rs diff --git a/Cargo.lock b/Cargo.lock index 9a600099d..55ab615b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1919,6 +1919,7 @@ dependencies = [ "avif-serialize", "craft-heif", "craft-jpeg", + "craft-media-testkit", "craft-metadata", "exr", "flate2", diff --git a/crates/image-codecs/Cargo.toml b/crates/image-codecs/Cargo.toml index dc920f14b..5281fe2cf 100644 --- a/crates/image-codecs/Cargo.toml +++ b/crates/image-codecs/Cargo.toml @@ -46,6 +46,7 @@ exr = { version = "1.73", default-features = false, features = ["rayon"] } proptest = { workspace = true } image = { version = "0.25", default-features = false, features = ["png", "jpeg", "tiff"] } sha2 = { version = "0.10", default-features = false, features = ["std"] } +craft-media-testkit = { workspace = true } [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] jxl-oxide = { version = "=0.12.6", default-features = false, features = ["rayon"] } diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 6e9e11617..a67be77c5 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -56,6 +56,7 @@ craft-image-codecs = { git = "https://github.com/storytold/craft-libs", rev = "< | Cineon | yes | yes | U16 (10-bit code values read and written exactly through U16) | Gray, RGB | no | no | no | no | no | no | no | built in | | DICOM | yes | **no** | U8, U16 | Gray, RGB | no | no | no | no | no | no | n/a | built in | | Amiga IFF (ILBM/PBM) | yes | yes (24-bit ILBM) | U8 | RGB, RGBA | yes (1-bit mask plane on write) | no | no | no | no | no | no | built in | +| JPEG-LS (`.jls`) | yes (any NEAR; one scan per component) | yes (lossless) | U8, U16 | Gray, RGB | no | no | no | no | no | no | no | built in | | AVIF | **no** | only with feature `avif` | U8 | RGB, RGBA | yes | no | no | no | no | no | yes | `image` + `ravif` | | HEIF/HEIC | with feature `heif` | **no** | U8, U16 (10/12-bit decodes to U16) | RGB, RGBA | yes (auxiliary alpha) | yes (`colr` prof) | yes | yes | no | no | n/a | `heic-rs` | diff --git a/crates/image-codecs/src/codecs.rs b/crates/image-codecs/src/codecs.rs index 0288ae22b..af91aa536 100644 --- a/crates/image-codecs/src/codecs.rs +++ b/crates/image-codecs/src/codecs.rs @@ -13,6 +13,7 @@ pub(crate) mod heif; pub(crate) mod iff; pub(crate) mod jp2; pub(crate) mod jpeg; +pub(crate) mod jpegls; pub(crate) mod jxl; pub(crate) mod pcx; pub(crate) mod png; diff --git a/crates/image-codecs/src/codecs/jpegls.rs b/crates/image-codecs/src/codecs/jpegls.rs new file mode 100644 index 000000000..b405e71ed --- /dev/null +++ b/crates/image-codecs/src/codecs/jpegls.rs @@ -0,0 +1,870 @@ +//! JPEG-LS (ITU-T T.87 | ISO/IEC 14495-1, "LOCO-I"): lossless and near-lossless gray and RGB +//! images, 2 to 16 bits per sample. +//! +//! Read: baseline JPEG-LS with the default or `LSE`-preset coding parameters, any `NEAR`, and +//! scans that code one component each (`ILV` 0), which is also how this module writes. Line- and +//! sample-interleaved scans, colour transforms and mapping tables are reported as unsupported. +//! Write: lossless (`NEAR` 0), 8 or 16 bits, gray or RGB, one scan per component. +//! +//! Samples of 9 to 15 bits are scaled to 16 bits on read (the top bits are repeated into the low +//! ones), so a decoded image always spans the full range of its sample type. + +use crate::error::CodecError; +use crate::fidelity::Plan; +use crate::format::Format; +use crate::image::{ChannelLayout, Image, SampleType}; +use crate::options::Limits; + +const F: Format = Format::JpegLs; +const SOF55: u8 = 0xF7; +const LSE: u8 = 0xF8; +const RESET_DEFAULT: i32 = 64; +const MIN_C: i32 = -128; +const MAX_C: i32 = 127; +/// `J[RUNindex]`: the log2 of the run length a run-index step codes. +const J: [u32; 32] = [0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 9, 10, 11, 12, 13, 14, 15]; +/// Context slots: the regular contexts (365 of the 405 slots the index below can reach), then the two +/// run-interruption contexts. +const RI_BASE: usize = 405; +const CONTEXTS: usize = RI_BASE + 2; + +fn bad(reason: impl std::fmt::Display) -> CodecError { + CodecError::malformed(F, reason) +} + +/// Whether `b` is a JPEG-LS file: an SOI followed by marker segments up to an SOF55 frame header. +pub(crate) fn looks_like_jpegls(b: &[u8]) -> bool { + if !b.starts_with(&[0xFF, 0xD8]) { + return false; + } + let mut pos = 2; + for _ in 0..64 { + let (Some(&0xFF), Some(&marker)) = (b.get(pos), b.get(pos + 1)) else { return false }; + match marker { + SOF55 => return true, + 0xDA | 0xD9 => return false, + 0xC0..=0xCF | 0xDB..=0xDF => return false, + 0x01 | 0xD0..=0xD7 | 0xFF => pos += if marker == 0xFF { 1 } else { 2 }, + _ => { + let Some(len) = b.get(pos + 2..pos + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false }; + pos += 2 + len; + } + } + } + false +} + +/// The coding parameters of one scan, derived as T.87 section C.2 specifies. +#[derive(Clone, Copy)] +struct Params { + maxval: i32, + near: i32, + range: i32, + qbpp: u32, + limit: u32, + t1: i32, + t2: i32, + t3: i32, + reset: i32, +} + +fn ceil_log2(v: i32) -> u32 { + let mut n = 0; + while (1i64 << n) < i64::from(v) { + n += 1; + } + n +} + +impl Params { + /// `preset` holds the `LSE` values (`maxval`, `t1`, `t2`, `t3`, `reset`), 0 where absent. + fn new(precision: u32, near: i32, preset: [i32; 5]) -> Result { + let default_max = (1i32 << precision) - 1; + let maxval = if preset[0] == 0 { default_max } else { preset[0] }; + if maxval < 1 || maxval > default_max || near < 0 || near > (maxval / 2).min(255) { + return Err(bad("invalid MAXVAL or NEAR")); + } + let clamp = |v: i32, lo: i32, hi: i32| if v > hi || v < lo { lo } else { v }; + let (d1, d2, d3) = if maxval >= 128 { + let factor = (maxval.min(4095) + 128) / 256; + let t1 = clamp(factor + 2 + 3 * near, near + 1, maxval); + let t2 = clamp(factor * 4 + 3 + 5 * near, t1, maxval); + let t3 = clamp(factor * 17 + 4 + 7 * near, t2, maxval); + (t1, t2, t3) + } else { + let factor = 256 / (maxval + 1); + let t1 = clamp((3 / factor + 3 * near).max(2), near + 1, maxval); + let t2 = clamp((7 / factor + 5 * near).max(3), t1, maxval); + let t3 = clamp((21 / factor + 7 * near).max(4), t2, maxval); + (t1, t2, t3) + }; + let t1 = if preset[1] == 0 { d1 } else { preset[1] }; + let t2 = if preset[2] == 0 { d2 } else { preset[2] }; + let t3 = if preset[3] == 0 { d3 } else { preset[3] }; + let reset = if preset[4] == 0 { RESET_DEFAULT } else { preset[4] }; + if !(near < t1 && t1 <= t2 && t2 <= t3 && t3 <= maxval) || !(3..=255).contains(&reset) { + return Err(bad("invalid threshold or RESET preset")); + } + let range = (maxval + 2 * near) / (2 * near + 1) + 1; + let bpp = ceil_log2(maxval + 1).max(2); + Ok(Params { maxval, near, range, qbpp: ceil_log2(range), limit: 2 * (bpp + bpp.max(8)), t1, t2, t3, reset }) + } +} + +/// Adaptive state of one scan: contexts, bias and the run index. +struct State { + a: [i32; CONTEXTS], + b: [i32; CONTEXTS], + c: [i32; CONTEXTS], + n: [i32; CONTEXTS], + nn: [i32; 2], + run_index: usize, +} + +impl State { + fn new(p: &Params) -> State { + let a0 = ((p.range + 32) / 64).max(2); + State { a: [a0; CONTEXTS], b: [0; CONTEXTS], c: [0; CONTEXTS], n: [1; CONTEXTS], nn: [0; 2], run_index: 0 } + } +} + +fn quantise_gradient(d: i32, p: &Params) -> i32 { + if d <= -p.t3 { + -4 + } else if d <= -p.t2 { + -3 + } else if d <= -p.t1 { + -2 + } else if d < -p.near { + -1 + } else if d <= p.near { + 0 + } else if d < p.t1 { + 1 + } else if d < p.t2 { + 2 + } else if d < p.t3 { + 3 + } else { + 4 + } +} + +fn predict(ra: i32, rb: i32, rc: i32) -> i32 { + if rc >= ra.max(rb) { + ra.min(rb) + } else if rc <= ra.min(rb) { + ra.max(rb) + } else { + ra + rb - rc + } +} + +/// The error quantised for `NEAR` and reduced into the modulo range, as the encoder codes it. +fn reduce(mut e: i32, p: &Params) -> i32 { + if p.near > 0 { + e = if e > 0 { (e + p.near) / (2 * p.near + 1) } else { -(p.near - e) / (2 * p.near + 1) }; + } + if e < 0 { + e += p.range; + } + if e >= (p.range + 1) / 2 { + e -= p.range; + } + e +} + +/// The reconstructed sample for a decoded (sign-adjusted) error. +fn reconstruct(px: i32, sign: i32, e: i32, p: &Params) -> i32 { + let mut r = px + sign * e * (2 * p.near + 1); + if r < -p.near { + r += p.range * (2 * p.near + 1); + } else if r > p.maxval + p.near { + r -= p.range * (2 * p.near + 1); + } + r.clamp(0, p.maxval) +} + +fn golomb_k(n: i32, a: i32) -> u32 { + let mut k = 0; + while (i64::from(n) << k) < i64::from(a) { + k += 1; + } + k +} + +/// Bits of the entropy-coded segment, with T.87's stuffing: after an `0xFF` byte the next byte +/// carries only 7 bits. +struct BitReader<'a> { + data: &'a [u8], + pos: usize, + cur: u32, + left: u32, + prev_ff: bool, +} + +impl<'a> BitReader<'a> { + fn new(data: &'a [u8]) -> Self { + BitReader { data, pos: 0, cur: 0, left: 0, prev_ff: false } + } + + fn bit(&mut self) -> Result { + if self.left == 0 { + let byte = *self.data.get(self.pos).ok_or_else(|| bad("the entropy-coded data ends early"))?; + self.pos += 1; + if self.prev_ff { + if byte & 0x80 != 0 { + return Err(bad("a marker inside entropy-coded data")); + } + self.cur = u32::from(byte); + self.left = 7; + } else { + self.cur = u32::from(byte); + self.left = 8; + } + self.prev_ff = byte == 0xFF; + } + self.left -= 1; + Ok((self.cur >> self.left) & 1) + } + + fn bits(&mut self, n: u32) -> Result { + let mut v = 0; + for _ in 0..n { + v = v << 1 | self.bit()?; + } + Ok(v) + } + + /// A limited-length Golomb code: the mapped error. + fn golomb(&mut self, k: u32, limit: u32, qbpp: u32) -> Result { + let max_zeros = limit.checked_sub(qbpp + 1).ok_or_else(|| bad("invalid LIMIT"))?; + let mut zeros = 0; + while self.bit()? == 0 { + zeros += 1; + if zeros > max_zeros { + return Err(bad("a Golomb code is longer than LIMIT")); + } + } + let v = if zeros < max_zeros { (zeros << k) + self.bits(k)? } else { self.bits(qbpp)? + 1 }; + i32::try_from(v).map_err(|_| bad("mapped error out of range")) + } +} + +/// Decode one component scan of `w` x `h` samples into `out` (row-major). +fn decode_scan(data: &[u8], w: usize, h: usize, p: &Params, out: &mut [u16]) -> Result<(), CodecError> { + let mut r = BitReader::new(data); + let mut st = State::new(p); + let mut prev = vec![0i32; w + 2]; + let mut cur = vec![0i32; w + 2]; + for y in 0..h { + // Ra at the first sample is Rb; the slot at index 0 is Rc there, left over from the line above. + cur[0] = prev[1]; + if y > 0 { + prev[w + 1] = prev[w]; + } + let mut x = 0usize; + while x < w { + let (ra, rb, rc, rd) = (cur[x], prev[x + 1], prev[x], if y == 0 { 0 } else { prev[x + 2] }); + let (q1, q2, q3) = (quantise_gradient(rd - rb, p), quantise_gradient(rb - rc, p), quantise_gradient(rc - ra, p)); + if q1 == 0 && q2 == 0 && q3 == 0 { + x = decode_run(&mut r, &mut st, p, &prev, &mut cur, ra, x, w)?; + continue; + } + let (sign, q1, q2, q3) = if q1 < 0 || (q1 == 0 && (q2 < 0 || (q2 == 0 && q3 < 0))) { (-1, -q1, -q2, -q3) } else { (1, q1, q2, q3) }; + let q = (q1 * 81 + (q2 + 4) * 9 + (q3 + 4)) as usize; + let k = golomb_k(st.n[q], st.a[q]); + let m = r.golomb(k, p.limit, p.qbpp)?; + let swap = p.near == 0 && k == 0 && 2 * st.b[q] <= -st.n[q]; + let e = if swap { + if m & 1 == 1 { (m - 1) >> 1 } else { -(m >> 1) - 1 } + } else if m & 1 == 0 { + m >> 1 + } else { + -((m + 1) >> 1) + }; + let px = (predict(ra, rb, rc) + sign * st.c[q]).clamp(0, p.maxval); + let rx = reconstruct(px, sign, e, p); + update_regular(&mut st, q, e, p); + cur[x + 1] = rx; + x += 1; + } + let row = &mut out[y * w..(y + 1) * w]; + for (o, &v) in row.iter_mut().zip(&cur[1..=w]) { + *o = v as u16; + } + // The line just decoded becomes the line above; its slot 0 is the Rc of the next line's first sample. + std::mem::swap(&mut prev, &mut cur); + } + Ok(()) +} + +fn update_regular(st: &mut State, q: usize, e: i32, p: &Params) { + st.b[q] += e * (2 * p.near + 1); + st.a[q] += e.abs(); + if st.n[q] == p.reset { + st.a[q] >>= 1; + st.b[q] = if st.b[q] >= 0 { st.b[q] >> 1 } else { -((1 - st.b[q]) >> 1) }; + st.n[q] >>= 1; + } + st.n[q] += 1; + if st.b[q] <= -st.n[q] { + if st.c[q] > MIN_C { + st.c[q] -= 1; + } + st.b[q] += st.n[q]; + if st.b[q] <= -st.n[q] { + st.b[q] = -st.n[q] + 1; + } + } else if st.b[q] > 0 { + if st.c[q] < MAX_C { + st.c[q] += 1; + } + st.b[q] -= st.n[q]; + if st.b[q] > 0 { + st.b[q] = 0; + } + } +} + +/// Run-interruption variables update shared by both directions. +fn update_run(st: &mut State, q: usize, e: i32, em: i32, ri_type: i32, p: &Params) { + let i = q - RI_BASE; + if e < 0 { + st.nn[i] += 1; + } + st.a[q] += (em + 1 - ri_type) >> 1; + if st.n[q] == p.reset { + st.a[q] >>= 1; + st.n[q] >>= 1; + st.nn[i] >>= 1; + } + st.n[q] += 1; +} + +/// The run-interruption Golomb parameter and the mapping flag of an error. +fn run_k(st: &State, q: usize, ri_type: i32) -> u32 { + let temp = if ri_type == 1 { st.a[q] + (st.n[q] >> 1) } else { st.a[q] }; + golomb_k(st.n[q], temp) +} + +#[allow(clippy::too_many_arguments)] +fn decode_run( + r: &mut BitReader, + st: &mut State, + p: &Params, + prev: &[i32], + cur: &mut [i32], + ra: i32, + x: usize, + w: usize, +) -> Result { + let remaining = w - x; + let mut run = 0usize; + while r.bit()? == 1 { + let step = 1usize << J[st.run_index]; + let count = step.min(remaining - run); + run += count; + if count == step && st.run_index < 31 { + st.run_index += 1; + } + if run == remaining { + break; + } + } + if run < remaining { + let rest = r.bits(J[st.run_index])? as usize; + run += rest; + if run > remaining { + return Err(bad("a run passes the end of the line")); + } + } + for v in &mut cur[x + 1..x + 1 + run] { + *v = ra; + } + if run == remaining { + return Ok(w); + } + // The sample that ended the run: its own Rb, the sample above it. + let rb = prev[x + run + 1]; + let ri_type = i32::from((ra - rb).abs() <= p.near); + let q = RI_BASE + ri_type as usize; + let k = run_k(st, q, ri_type); + let limit = p.limit - J[st.run_index] - 1; + let em = r.golomb(k, limit, p.qbpp)?; + let tmp = em + ri_type; + let cond = k == 0 && 2 * st.nn[ri_type as usize] < st.n[q]; + let map = tmp & 1; + let mag = (tmp + map) / 2; + let e = if (cond && map == 1) || (!cond && map == 0) { mag } else { -mag }; + let (px, sign) = if ri_type == 1 { (ra, 1) } else { (rb, if ra > rb { -1 } else { 1 }) }; + // The decoder holds the sign-adjusted error; the sign flips the reconstruction. + let rx = reconstruct(px, sign, e, p); + update_run(st, q, e, em, ri_type, p); + if st.run_index > 0 { + st.run_index -= 1; + } + cur[x + run + 1] = rx; + Ok(x + run + 1) +} + +// ---- encoder ---- + +struct BitWriter { + out: Vec, + cur: u32, + used: u32, + cap: u32, +} + +impl BitWriter { + fn new() -> Self { + BitWriter { out: Vec::new(), cur: 0, used: 0, cap: 8 } + } + + fn put(&mut self, value: u32, n: u32) { + for i in (0..n).rev() { + // A run of more than 32 zero bits (a long escape code) has no bits of `value` to shift out. + let bit = if i < 32 { (value >> i) & 1 } else { 0 }; + self.cur = self.cur << 1 | bit; + self.used += 1; + if self.used == self.cap { + let byte = self.cur as u8; + self.out.push(byte); + self.cap = if byte == 0xFF { 7 } else { 8 }; + self.cur = 0; + self.used = 0; + } + } + } + + /// Pad the last byte with zero bits; a closing 0xFF is followed by a zero byte. + fn finish(mut self) -> Vec { + if self.used > 0 { + let pad = self.cap - self.used; + self.put(0, pad); + } + if self.out.last() == Some(&0xFF) { + self.out.push(0); + } + self.out + } + + fn golomb(&mut self, m: i32, k: u32, limit: u32, qbpp: u32) { + let m = m as u32; + let max_zeros = limit - qbpp - 1; + let q = m >> k; + if q < max_zeros { + self.put(0, q); + self.put(1, 1); + self.put(m & ((1 << k) - 1), k); + } else { + self.put(0, max_zeros); + self.put(1, 1); + self.put(m - 1, qbpp); + } + } +} + +fn encode_scan(plane: &[u16], w: usize, h: usize, p: &Params) -> Vec { + let mut wr = BitWriter::new(); + let mut st = State::new(p); + let mut prev = vec![0i32; w + 2]; + let mut cur = vec![0i32; w + 2]; + for y in 0..h { + cur[0] = prev[1]; + if y > 0 { + prev[w + 1] = prev[w]; + } + let line = &plane[y * w..(y + 1) * w]; + let mut x = 0usize; + while x < w { + let (ra, rb, rc, rd) = (cur[x], prev[x + 1], prev[x], if y == 0 { 0 } else { prev[x + 2] }); + let ix = i32::from(line[x]); + let (q1, q2, q3) = (quantise_gradient(rd - rb, p), quantise_gradient(rb - rc, p), quantise_gradient(rc - ra, p)); + if q1 == 0 && q2 == 0 && q3 == 0 { + x = encode_run(&mut wr, &mut st, p, line, &prev, &mut cur, ra, x); + continue; + } + let (sign, q1, q2, q3) = if q1 < 0 || (q1 == 0 && (q2 < 0 || (q2 == 0 && q3 < 0))) { (-1, -q1, -q2, -q3) } else { (1, q1, q2, q3) }; + let q = (q1 * 81 + (q2 + 4) * 9 + (q3 + 4)) as usize; + let px = (predict(ra, rb, rc) + sign * st.c[q]).clamp(0, p.maxval); + let e = reduce(sign * (ix - px), p); + let k = golomb_k(st.n[q], st.a[q]); + let swap = p.near == 0 && k == 0 && 2 * st.b[q] <= -st.n[q]; + let m = if swap { + if e >= 0 { 2 * e + 1 } else { -2 * (e + 1) } + } else if e >= 0 { + 2 * e + } else { + -2 * e - 1 + }; + wr.golomb(m, k, p.limit, p.qbpp); + cur[x + 1] = reconstruct(px, sign, e, p); + update_regular(&mut st, q, e, p); + x += 1; + } + std::mem::swap(&mut prev, &mut cur); + } + wr.finish() +} + +#[allow(clippy::too_many_arguments)] +fn encode_run(wr: &mut BitWriter, st: &mut State, p: &Params, line: &[u16], prev: &[i32], cur: &mut [i32], ra: i32, x: usize) -> usize { + let w = line.len(); + let mut run = 0usize; + while x + run < w && (i32::from(line[x + run]) - ra).abs() <= p.near { + cur[x + run + 1] = ra; + run += 1; + } + let mut count = run; + while count >= 1usize << J[st.run_index] { + wr.put(1, 1); + count -= 1usize << J[st.run_index]; + if st.run_index < 31 { + st.run_index += 1; + } + } + if x + run == w { + if count > 0 { + wr.put(1, 1); + } + return w; + } + wr.put(0, 1); + wr.put(count as u32, J[st.run_index]); + let ix = i32::from(line[x + run]); + let rb = prev[x + run + 1]; + let ri_type = i32::from((ra - rb).abs() <= p.near); + let q = RI_BASE + ri_type as usize; + let (px, sign) = if ri_type == 1 { (ra, 1) } else { (rb, if ra > rb { -1 } else { 1 }) }; + let e = reduce(sign * (ix - px), p); + let k = run_k(st, q, ri_type); + let n_neg = st.nn[ri_type as usize]; + let map = i32::from((k == 0 && e > 0 && 2 * n_neg < st.n[q]) || (e < 0 && 2 * n_neg >= st.n[q]) || (e < 0 && k != 0)); + let em = 2 * e.abs() - ri_type - map; + wr.golomb(em, k, p.limit - J[st.run_index] - 1, p.qbpp); + cur[x + run + 1] = reconstruct(px, sign, e, p); + update_run(st, q, e, em, ri_type, p); + if st.run_index > 0 { + st.run_index -= 1; + } + x + run + 1 +} + +/// Writes a lossless JPEG-LS file: 8 or 16 bits, gray or RGB, one scan per component. +pub(crate) fn encode(src: &Image, plan: Plan) -> Result, CodecError> { + let enc = |msg: String| CodecError::encode(F, msg); + let img = src.converted(plan.layout, plan.sample); + let nc = match img.layout() { + ChannelLayout::Gray => 1, + ChannelLayout::Rgb => 3, + l => return Err(enc(format!("no JPEG-LS layout for {l:?}"))), + }; + let precision: u32 = match img.sample_type() { + SampleType::U8 => 8, + SampleType::U16 => 16, + s => return Err(enc(format!("JPEG-LS stores 8 or 16 bits per sample, not {s:?}"))), + }; + let (w, h) = img.dimensions(); + let (Ok(w16), Ok(h16)) = (u16::try_from(w), u16::try_from(h)) else { + return Err(enc(format!("dimensions exceed 65535x65535: {w}x{h}"))); + }; + let (wu, hu) = (w as usize, h as usize); + let data = img.data(); + if data.len() != wu * hu * nc * (precision as usize / 8) { + return Err(enc("the image has the wrong number of samples".into())); + } + let params = Params::new(precision, 0, [0; 5]).map_err(|_| enc("invalid coding parameters".into()))?; + let mut out = vec![0xFF, 0xD8, 0xFF, SOF55]; + out.extend_from_slice(&((8 + 3 * nc) as u16).to_be_bytes()); + out.push(precision as u8); + out.extend_from_slice(&h16.to_be_bytes()); + out.extend_from_slice(&w16.to_be_bytes()); + out.push(nc as u8); + for c in 0..nc { + out.extend_from_slice(&[c as u8 + 1, 0x11, 0]); + } + let mut plane = vec![0u16; wu * hu]; + for c in 0..nc { + for (i, v) in plane.iter_mut().enumerate() { + let at = i * nc + c; + *v = if precision == 8 { u16::from(data[at]) } else { u16::from_ne_bytes([data[2 * at], data[2 * at + 1]]) }; + } + out.extend_from_slice(&[0xFF, 0xDA, 0, 8, 1, c as u8 + 1, 0, 0, 0, 0]); + out.extend_from_slice(&encode_scan(&plane, wu, hu, ¶ms)); + } + out.extend_from_slice(&[0xFF, 0xD9]); + Ok(out) +} + +struct Frame { + precision: u32, + width: usize, + height: usize, + components: Vec, +} + +fn be16(b: &[u8], i: usize) -> Result { + b.get(i..i + 2).map(|s| usize::from(u16::from_be_bytes([s[0], s[1]]))).ok_or_else(|| bad("truncated header")) +} + +pub(crate) fn decode(b: &[u8], limits: &Limits) -> Result { + if !looks_like_jpegls(b) { + return Err(bad("not a JPEG-LS file")); + } + let mut pos = 2; + let mut frame: Option = None; + let mut preset = [0i32; 5]; + let mut planes: Vec> = Vec::new(); + let mut scans_done = 0; + loop { + // Find the next marker. + while b.get(pos) != Some(&0xFF) { + if pos >= b.len() { + return finish(frame, planes, scans_done); + } + pos += 1; + } + while b.get(pos + 1) == Some(&0xFF) { + pos += 1; + } + let Some(&marker) = b.get(pos + 1) else { return finish(frame, planes, scans_done) }; + if marker == 0xD9 { + return finish(frame, planes, scans_done); + } + if marker == 0x00 || (0xD0..=0xD7).contains(&marker) || marker == 0x01 { + pos += 2; + continue; + } + let len = be16(b, pos + 2)?; + let seg = b.get(pos + 4..pos + 2 + len).ok_or_else(|| bad("truncated marker segment"))?; + match marker { + SOF55 => { + if frame.is_some() || seg.len() < 6 { + return Err(bad("bad or repeated frame header")); + } + let (precision, height, width, nf) = (u32::from(seg[0]), be16(seg, 1)?, be16(seg, 3)?, usize::from(seg[5])); + if !(2..=16).contains(&precision) || width == 0 || height == 0 || !matches!(nf, 1 | 3) || seg.len() < 6 + 3 * nf { + return Err(bad("unsupported frame header")); + } + if height == 0 { + return Err(CodecError::unsupported(F, "a frame height set later by a DNL marker")); + } + let layout = if nf == 1 { ChannelLayout::Gray } else { ChannelLayout::Rgb }; + let sample = if precision <= 8 { SampleType::U8 } else { SampleType::U16 }; + limits.check(width as u32, height as u32, layout, sample)?; + let components = (0..nf).map(|i| seg[6 + 3 * i]).collect(); + frame = Some(Frame { precision, width, height, components }); + } + LSE => { + if seg.first() == Some(&1) && seg.len() >= 11 { + for (i, p) in preset.iter_mut().enumerate() { + *p = be16(seg, 1 + 2 * i)? as i32; + } + } else if seg.first() != Some(&1) { + return Err(CodecError::unsupported(F, "mapping tables and colour transforms (LSE)")); + } + } + 0xDA => { + let f = frame.as_ref().ok_or_else(|| bad("a scan before the frame header"))?; + let ns = usize::from(*seg.first().ok_or_else(|| bad("empty scan header"))?); + if seg.len() < 1 + 2 * ns + 3 || ns != 1 { + return Err(CodecError::unsupported(F, "interleaved scans (ILV 1 and 2)")); + } + let (cs, tm, near, ilv) = (seg[1], seg[2], i32::from(seg[3]), seg[4]); + if tm != 0 { + return Err(CodecError::unsupported(F, "mapping tables")); + } + if ilv != 0 && f.components.len() > 1 { + return Err(CodecError::unsupported(F, "interleaved scans (ILV 1 and 2)")); + } + if seg[5] & 0x0f != 0 { + return Err(CodecError::unsupported(F, "a point transform")); + } + let idx = f.components.iter().position(|&c| c == cs).ok_or_else(|| bad("a scan names an unknown component"))?; + if planes.len() <= idx { + planes.resize(f.components.len(), Vec::new()); + } + if !planes[idx].is_empty() { + return Err(bad("a component is coded twice")); + } + let params = Params::new(f.precision, near, preset)?; + let start = pos + 2 + len; + // The scan ends at the next real marker: an 0xFF followed by a byte with its top bit set. + let mut end = start; + while end < b.len() { + if b[end] == 0xFF && b.get(end + 1).is_none_or(|&n| n & 0x80 != 0) { + break; + } + end += 1; + } + let mut samples = Vec::new(); + samples.try_reserve_exact(f.width * f.height).map_err(|_| bad("pixel buffer is too large"))?; + samples.resize(f.width * f.height, 0u16); + decode_scan(&b[start..end], f.width, f.height, ¶ms, &mut samples)?; + planes[idx] = samples; + scans_done += 1; + pos = end; + continue; + } + _ => {} + } + pos += 2 + len; + } +} + +fn finish(frame: Option, planes: Vec>, scans: usize) -> Result { + let f = frame.ok_or_else(|| bad("no frame header"))?; + if scans != f.components.len() || planes.iter().any(Vec::is_empty) || planes.len() != f.components.len() { + return Err(bad("the file ends before every component is coded")); + } + let (w, h, nc) = (f.width, f.height, f.components.len()); + let layout = if nc == 1 { ChannelLayout::Gray } else { ChannelLayout::Rgb }; + let p = f.precision; + let scale = |v: u16| -> u16 { + if p == 16 { + v + } else { + let v = u32::from(v); + ((v << (16 - p)) | (v >> (2 * p).saturating_sub(16).min(15))) as u16 + } + }; + if p <= 8 { + let to8 = |v: u16| -> u8 { if p == 8 { v as u8 } else { (scale(v) >> 8) as u8 } }; + let mut data = Vec::with_capacity(w * h * nc); + for i in 0..w * h { + for plane in &planes { + data.push(to8(plane[i])); + } + } + Image::from_u8(w as u32, h as u32, layout, data) + } else { + let mut data = Vec::with_capacity(w * h * nc); + for i in 0..w * h { + for plane in &planes { + data.push(scale(plane[i])); + } + } + Image::from_u16(w as u32, h as u32, layout, &data) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn noise_u16(n: usize, seed: &mut u32, mask: u32) -> Vec { + (0..n) + .map(|_| { + *seed ^= *seed << 13; + *seed ^= *seed >> 17; + *seed ^= *seed << 5; + (*seed & mask) as u16 + }) + .collect() + } + + fn image(w: u32, h: u32, layout: ChannelLayout, wide: bool, kind: u8) -> Image { + let n = w as usize * h as usize * layout.channels(); + let mut seed = 0x1234_5679u32; + let samples: Vec = match kind { + 0 => noise_u16(n, &mut seed, if wide { 0xFFFF } else { 0xFF }), + 1 => vec![if wide { 40_000 } else { 100 }; n], + _ => (0..n).map(|i| if wide { ((i / 3) * 37 % 65_536) as u16 } else { ((i / 3) % 251) as u16 }).collect(), + }; + if wide { + Image::from_u16(w, h, layout, &samples).unwrap() + } else { + Image::from_u8(w, h, layout, samples.iter().map(|&v| v as u8).collect()).unwrap() + } + } + + fn plan(img: &Image) -> Plan { + Plan { layout: img.layout(), sample: img.sample_type() } + } + + #[test] + fn round_trips_noise_flat_and_smooth_images() { + for layout in [ChannelLayout::Gray, ChannelLayout::Rgb] { + for wide in [false, true] { + for kind in 0..3 { + // Widths around the run-length steps, and single-pixel dimensions. + for (w, h) in [(1, 1), (1, 9), (2, 3), (7, 5), (33, 4), (300, 3), (1025, 2)] { + let img = image(w, h, layout, wide, kind); + let bytes = encode(&img, plan(&img)).unwrap(); + assert!(looks_like_jpegls(&bytes)); + let back = decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{layout:?} wide={wide} kind={kind} {w}x{h}: {e}")); + assert_eq!(back.data(), img.data(), "{layout:?} wide={wide} kind={kind} {w}x{h}"); + } + } + } + } + } + + #[test] + fn default_parameters_follow_the_standard() { + let p = Params::new(8, 0, [0; 5]).unwrap(); + assert_eq!((p.maxval, p.range, p.qbpp, p.limit, p.t1, p.t2, p.t3, p.reset), (255, 256, 8, 32, 3, 7, 21, 64)); + let p = Params::new(16, 0, [0; 5]).unwrap(); + assert_eq!((p.maxval, p.range, p.qbpp, p.limit, p.t1, p.t2, p.t3), (65535, 65536, 16, 64, 18, 67, 276)); + let p = Params::new(8, 3, [0; 5]).unwrap(); + assert_eq!((p.range, p.t1), (38, 12)); // RANGE = floor((255 + 2 * 3) / 7) + 1 + assert!(Params::new(8, 200, [0; 5]).is_err()); + } + + #[test] + fn detection_tells_jpeg_ls_from_jpeg() { + let img = image(4, 4, ChannelLayout::Gray, false, 0); + let ls = encode(&img, plan(&img)).unwrap(); + assert!(looks_like_jpegls(&ls)); + // A baseline JPEG starts SOI, APP0 / DQT, ... SOF0: not JPEG-LS. + assert!(!looks_like_jpegls(&[0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC0, 0, 11])); + assert!(!looks_like_jpegls(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11])); + // SOF55 after an application segment still is. + assert!(looks_like_jpegls(&[0xFF, 0xD8, 0xFF, 0xE8, 0, 4, 0, 0, 0xFF, 0xF7, 0, 11])); + assert!(!looks_like_jpegls(&[0xFF, 0xD8])); + } + + #[test] + fn interleaved_scans_and_tables_are_unsupported() { + let img = image(8, 4, ChannelLayout::Rgb, false, 2); + let good = encode(&img, plan(&img)).unwrap(); + // The scan header body is Ns, Cs, Tm, NEAR, ILV, point transform, after the marker and length. + let sos = good.windows(2).position(|w| w == [0xFF, 0xDA]).unwrap() + 4; + let mut line = good.clone(); + line[sos + 4] = 1; + assert!(matches!(decode(&line, &Limits::default()), Err(CodecError::Unsupported { .. }))); + let mut mapped = good.clone(); + mapped[sos + 2] = 1; + assert!(matches!(decode(&mapped, &Limits::default()), Err(CodecError::Unsupported { .. }))); + let mut transform = good; + transform[sos + 5] = 1; + assert!(matches!(decode(&transform, &Limits::default()), Err(CodecError::Unsupported { .. }))); + } + + #[test] + fn hostile_input_is_an_error_not_a_panic() { + for (w, h, layout, wide) in [(23, 9, ChannelLayout::Gray, false), (11, 7, ChannelLayout::Rgb, true)] { + let img = image(w, h, layout, wide, 0); + let good = encode(&img, plan(&img)).unwrap(); + for len in 0..good.len() { + let _ = decode(&good[..len], &Limits::default()); + } + // Flip every bit of the entropy-coded data and of the headers, one at a time. + for i in 0..good.len() { + for bit in 0..8 { + let mut c = good.clone(); + c[i] ^= 1 << bit; + let _ = decode(&c, &Limits::default()); + } + } + } + // A huge frame is stopped by the limits or the length check before any allocation. + let img = image(4, 4, ChannelLayout::Gray, false, 0); + let mut huge = encode(&img, plan(&img)).unwrap(); + let sof = huge.windows(2).position(|w| w == [0xFF, 0xF7]).unwrap(); + huge[sof + 5..sof + 9].copy_from_slice(&[0xFF, 0xFF, 0xFF, 0xFF]); + assert!(decode(&huge, &Limits::default()).is_err()); + } +} diff --git a/crates/image-codecs/src/format.rs b/crates/image-codecs/src/format.rs index ecaf945cd..810c1ad48 100644 --- a/crates/image-codecs/src/format.rs +++ b/crates/image-codecs/src/format.rs @@ -44,6 +44,8 @@ pub enum Format { Iff, /// JPEG 2000 (JP2 container or raw J2K codestream): read-only; see [`ASYMMETRIC_EXCEPTIONS`]. Jp2, + /// JPEG-LS (`.jls`): lossless and near-lossless gray and RGB. + JpegLs, } /// What a format can hold **and** what this crate reads/writes for it. @@ -113,7 +115,7 @@ const TIFF_LAYOUTS: &[ChannelLayout] = &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L:: impl Format { /// Every format known to the crate (enabled or not). - pub const ALL: [Format; 24] = [ + pub const ALL: [Format; 25] = [ Format::Png, Format::Jpeg, Format::Jp2, @@ -138,6 +140,7 @@ impl Format { Format::Cineon, Format::Dicom, Format::Iff, + Format::JpegLs, ]; pub fn caps(self) -> FormatCaps { @@ -176,6 +179,7 @@ impl Format { Format::Dicom => "DICOM", Format::Iff => "Amiga IFF", Format::Jp2 => "JPEG 2000", + Format::JpegLs => "JPEG-LS", } } @@ -207,6 +211,7 @@ impl Format { Format::Dicom => &["dcm", "dicom", "dc3"], Format::Iff => &["iff", "lbm", "ilbm"], Format::Jp2 => &["jp2", "j2k", "j2c", "jpf", "jpx"], + Format::JpegLs => &["jls"], } } @@ -236,13 +241,14 @@ impl Format { Format::Dicom => "application/dicom", Format::Iff => "image/x-iff", Format::Jp2 => "image/jp2", + Format::JpegLs => "image/jls", } } /// Maximum encodable dimensions, if the format has a hard limit. pub fn max_dimensions(self) -> Option<(u32, u32)> { match self { - Format::Jpeg | Format::Gif | Format::Iff => Some((65535, 65535)), + Format::Jpeg | Format::Gif | Format::Iff | Format::JpegLs => Some((65535, 65535)), Format::WebP => Some((16384, 16384)), Format::Ico => Some((256, 256)), _ => None, @@ -310,6 +316,7 @@ pub fn caps(format: Format) -> FormatCaps { Format::Cineon => FormatCaps { depths: &[S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base }, Format::Dicom => FormatCaps { read: true, write: false, depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base }, Format::Iff => FormatCaps { depths: &[S::U8], layouts: &[L::Rgb, L::Rgba], alpha: true, ..base }, + Format::JpegLs => FormatCaps { depths: &[S::U8, S::U16], layouts: &[L::Gray, L::Rgb], alpha: false, ..base }, Format::Jp2 => FormatCaps { read: true, write: false, layouts: &[L::Gray, L::GrayA, L::Rgb, L::Rgba, L::Cmyk, L::CmykA], icc: true, ..base }, } } @@ -321,6 +328,9 @@ pub fn detect(bytes: &[u8]) -> Option { if b.starts_with(b"\x89PNG\r\n\x1a\n") { return Some(Format::Png); } + if crate::codecs::jpegls::looks_like_jpegls(b) { + return Some(Format::JpegLs); + } if b.starts_with(&[0xFF, 0xD8, 0xFF]) { return Some(Format::Jpeg); } diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 21130e392..b60691e83 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -62,7 +62,7 @@ pub use crate::resolution::{exif_resolution, export_exif, export_xmp, photoshop_ pub use crate::source_settings::{WebpEncoding, jpeg_quality, webp_encoding}; pub use half::f16; -use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jxl, pcx, png, pnm, tiff, via_image, webp, xpm}; +use crate::codecs::{dds, dicom, dpx, exr, fits, heif, iff, jp2, jpeg, jpegls, jxl, pcx, png, pnm, tiff, via_image, webp, xpm}; /// Detect the format and decode with default [`Limits`]. pub fn decode(bytes: &[u8]) -> Result { @@ -101,6 +101,7 @@ pub fn decode_as_with(format: Format, bytes: &[u8], opts: &DecodeOptions) -> Res Format::Cineon => dpx::decode_cineon(bytes, l), Format::Dicom => dicom::decode(bytes, l), Format::Iff => iff::decode(bytes, l), + Format::JpegLs => jpegls::decode(bytes, l), Format::OpenExr => exr::decode(bytes, l), Format::Heif => heif::decode(bytes, l, opts.keep_orientation), Format::Jp2 => jp2::decode(bytes, l), @@ -457,6 +458,7 @@ pub fn encode(image: &Image, format: Format, opts: &EncodeOptions) -> Result dpx::encode_cineon(image, plan), Format::Dicom => Err(CodecError::unsupported(format, "DICOM is import-only; save as TIFF or PNG instead")), Format::Iff => iff::encode(image, plan), + Format::JpegLs => jpegls::encode(image, plan), Format::OpenExr => exr::encode(image, plan, opts), Format::Heif => Err(CodecError::unsupported(format, "encoding is not available in this build")), Format::Jp2 => Err(CodecError::unsupported(format, "JPEG 2000 is import-only; save as PNG or TIFF instead")), diff --git a/crates/image-codecs/tests/it/detect_caps.rs b/crates/image-codecs/tests/it/detect_caps.rs index 640772c83..390eaa41f 100644 --- a/crates/image-codecs/tests/it/detect_caps.rs +++ b/crates/image-codecs/tests/it/detect_caps.rs @@ -62,6 +62,7 @@ fn default_build_formats_all_symmetric() { Format::Dpx, Format::Cineon, Format::Iff, + Format::JpegLs, ]; for f in expected { assert!(caps(f).read && caps(f).write, "{f:?}"); @@ -218,6 +219,21 @@ fn detect_magic_iff() { assert_eq!(detect(b"FORM\0\0\0\x14AIFF"), None); assert!(decode_as(Format::Iff, b"FORM\0\0\0\x14ILBM").is_err()); } +#[test] +fn detect_magic_jpeg_ls() { + use craft_image_codecs::{ChannelLayout, EncodeOptions, Image, encode}; + let img = Image::from_u8(5, 3, ChannelLayout::Gray, (0..15).collect()).unwrap(); + let bytes = encode(&img, Format::JpegLs, &EncodeOptions::default()).unwrap(); + assert_eq!(detect(&bytes), Some(Format::JpegLs)); + assert_eq!(from_extension("scan.JLS"), Some(Format::JpegLs)); + assert_eq!(decode(&bytes).unwrap().data(), img.data()); + assert!(!ASYMMETRIC_EXCEPTIONS.iter().any(|(f, _)| *f == Format::JpegLs)); + assert_eq!(Format::JpegLs.max_dimensions(), Some((65535, 65535))); + // Plain JPEG still is JPEG: SOI, then a baseline frame header. + assert_eq!(detect(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11]), Some(Format::Jpeg)); + assert!(decode_as(Format::JpegLs, &[0xFF, 0xD8, 0xFF, 0xF7]).is_err()); +} + #[test] fn detect_magic_avif() { assert_eq!(detect(b"\0\0\0\x1cftypavif\0\0\0\0avifmif1miaf"), Some(Format::Avif)); diff --git a/crates/image-codecs/tests/it/jpegls_oracle.rs b/crates/image-codecs/tests/it/jpegls_oracle.rs new file mode 100644 index 000000000..aa98609a1 --- /dev/null +++ b/crates/image-codecs/tests/it/jpegls_oracle.rs @@ -0,0 +1,103 @@ +//! JPEG-LS against ffmpeg as the reference (oracle; skipped when ffmpeg is absent, a failure with +//! `CRAFT_MEDIA_REQUIRE_ORACLES=1`): +//! - files ffmpeg's encoder writes (8- and 16-bit gray: smooth, noisy and flat images, which +//! exercise the regular, run and escape-coded paths) decode to the pixels ffmpeg decodes; +//! - files our encoder writes (gray 8 and 16 bits, RGB 8 bits) decode in ffmpeg to our pixels (ffmpeg +//! cannot decode 16-bit RGB JPEG-LS, so that case is only round-tripped through our own decoder); +//! - ffmpeg writes RGB line-interleaved, which is reported as unsupported. + +use craft_image_codecs::{ChannelLayout, CodecError, EncodeOptions, Format, Image, SampleType, decode, encode}; +use std::path::Path; +use std::process::Command; + +const W: u32 = 66; +const H: u32 = 40; + +fn run(ffmpeg: &Path, args: &[&str]) -> Vec { + let out = Command::new(ffmpeg).args(["-v", "error", "-y"]).args(args).output().expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + out.stdout +} + +/// `(name, lavfi source)`: smooth, noisy and flat pictures of W x H. +fn sources() -> Vec<(&'static str, String)> { + let s = format!("{W}x{H}"); + vec![ + ("testsrc", format!("testsrc2=size={s}:rate=1")), + ("noise", format!("nullsrc=size={s}:rate=1,geq=lum='random(1)*255':cb=128:cr=128")), + ("flat", format!("color=c=0x336699:size={s}:rate=1")), + ("gradient", format!("gradients=size={s}:rate=1:seed=7")), + ] +} + +#[test] +fn ffmpeg_files_decode_to_ffmpegs_pixels() { + let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("jpegls"); + for (name, src) in sources() { + for (fmt, ext_bits) in [("gray", 8), ("gray16le", 16)] { + let file = dir.join(format!("ffmpeg_{name}_{ext_bits}.jls")); + let f = file.to_str().expect("path"); + run(&ffmpeg, &["-f", "lavfi", "-i", &src, "-frames:v", "1", "-pix_fmt", fmt, f]); + let theirs = run(&ffmpeg, &["-i", f, "-f", "rawvideo", "-pix_fmt", fmt, "-"]); + let ours = decode(&std::fs::read(&file).expect("read")).unwrap_or_else(|e| panic!("{name} {fmt}: {e}")); + assert_eq!((ours.width(), ours.height(), ours.layout()), (W, H, ChannelLayout::Gray), "{name} {fmt}"); + let want_sample = if ext_bits == 8 { SampleType::U8 } else { SampleType::U16 }; + assert_eq!(ours.sample_type(), want_sample); + let ours_bytes: Vec = if ext_bits == 8 { + ours.data().to_vec() + } else { + ours.data().as_chunks::<2>().0.iter().flat_map(|b| u16::from_ne_bytes(*b).to_le_bytes()).collect() + }; + assert!(ours_bytes == theirs, "{name} {fmt}: pixels differ from ffmpeg's decode"); + } + } +} + +#[test] +fn ffmpegs_rgb_is_line_interleaved_and_unsupported() { + let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("jpegls"); + let file = dir.join("ffmpeg_rgb.jls"); + run( + &ffmpeg, + &["-f", "lavfi", "-i", &format!("testsrc2=size={W}x{H}:rate=1"), "-frames:v", "1", "-pix_fmt", "rgb24", file.to_str().expect("path")], + ); + assert!(matches!(decode(&std::fs::read(&file).expect("read")), Err(CodecError::Unsupported { .. }))); +} + +#[test] +fn ffmpeg_decodes_our_files_to_our_pixels() { + let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("jpegls"); + for (name, src) in sources() { + for (layout, fmt8, fmt16) in [(ChannelLayout::Gray, "gray", "gray16le"), (ChannelLayout::Rgb, "rgb24", "rgb48le")] { + for bits in [8, 16] { + let fmt = if bits == 8 { fmt8 } else { fmt16 }; + // Source pixels from ffmpeg, so the test image is realistic. + let raw = run(&ffmpeg, &["-f", "lavfi", "-i", &src, "-frames:v", "1", "-pix_fmt", fmt, "-f", "rawvideo", "-"]); + let img = if bits == 8 { + Image::from_u8(W, H, layout, raw).expect("image") + } else { + let samples: Vec = raw.as_chunks::<2>().0.iter().map(|b| u16::from_le_bytes(*b)).collect(); + Image::from_u16(W, H, layout, &samples).expect("image") + }; + let bytes = encode(&img, Format::JpegLs, &EncodeOptions::default()).expect("encode"); + assert_eq!(decode(&bytes).expect("our own file decodes").data(), img.data(), "{name} {fmt}: round trip"); + let file = dir.join(format!("ours_{name}_{fmt}.jls")); + std::fs::write(&file, &bytes).expect("write"); + // ffmpeg's decoder does not implement 16-bit colour ("Not yet implemented"). + if layout == ChannelLayout::Rgb && bits == 16 { + continue; + } + let theirs = run(&ffmpeg, &["-i", file.to_str().expect("path"), "-f", "rawvideo", "-pix_fmt", fmt, "-"]); + let ours_bytes: Vec = if bits == 8 { + img.data().to_vec() + } else { + img.data().as_chunks::<2>().0.iter().flat_map(|b| u16::from_ne_bytes(*b).to_le_bytes()).collect() + }; + assert!(theirs == ours_bytes, "{name} {fmt}: ffmpeg decodes our file to other pixels"); + } + } + } +} diff --git a/crates/image-codecs/tests/it/main.rs b/crates/image-codecs/tests/it/main.rs index f465a0cbb..4eda74a26 100644 --- a/crates/image-codecs/tests/it/main.rs +++ b/crates/image-codecs/tests/it/main.rs @@ -14,6 +14,7 @@ mod exr_stream; mod fidelity; mod gif_frames; mod heif; +mod jpegls_oracle; mod limits; mod malformed; mod metadata; From 6d1c6188f31795e09761d049b9a9ff458f429e41 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:56:16 +0000 Subject: [PATCH 36/98] Expose validated single JSON-RPC request inspection --- crates/jsonrpc/src/lib.rs | 22 +++++++++++++--------- crates/jsonrpc/tests/server.rs | 12 ++++++++++++ 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/jsonrpc/src/lib.rs b/crates/jsonrpc/src/lib.rs index 7389869c3..8d2aa6c6e 100644 --- a/crates/jsonrpc/src/lib.rs +++ b/crates/jsonrpc/src/lib.rs @@ -107,6 +107,17 @@ pub struct Request<'a> { pub params: Option<&'a Value>, } +/// Inspect a single request envelope. This validates its JSON-RPC version, method and ID; +/// [`process`] additionally validates parameter shape before dispatching it. +pub fn request(value: &Value) -> Option> { + let object = value.as_object()?; + let id = object.get("id"); + if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") || !id.is_none_or(|v| v.is_null() || v.is_string() || v.is_i64() || v.is_u64()) { + return None; + } + Some(Request { id, method: object.get("method")?.as_str()?, params: object.get("params") }) +} + /// A JSON-RPC error returned by a method handler. #[derive(Clone, Debug, PartialEq)] pub struct RpcError { @@ -144,15 +155,8 @@ where F: FnMut(Request<'_>) -> Result, N: FnMut(Request<'_>), { - let Some(object) = value.as_object() else { return Some(invalid_request()) }; - let id = object.get("id"); - let valid_id = id.is_none_or(|v| v.is_null() || v.is_string() || v.is_i64() || v.is_u64()); - let method = object.get("method").and_then(Value::as_str); - if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") || method.is_none() || !valid_id { - return Some(invalid_request()); - } - let Some(method) = method else { return Some(invalid_request()) }; - let request = Request { id, method, params: object.get("params") }; + let Some(request) = request(value) else { return Some(invalid_request()) }; + let id = request.id; if request.params.is_some_and(|v| !v.is_object() && !v.is_array()) { return id.map(|id| error_response(id.clone(), RpcError::new(-32602, "Invalid params"))); } diff --git a/crates/jsonrpc/tests/server.rs b/crates/jsonrpc/tests/server.rs index 2a03d57b4..c2ff1d0d2 100644 --- a/crates/jsonrpc/tests/server.rs +++ b/crates/jsonrpc/tests/server.rs @@ -143,3 +143,15 @@ fn bounded_line_read_propagates_io_errors() { let mut reader = std::io::BufReader::new(Broken); assert!(matches!(craft_jsonrpc::read_line(&mut reader, 1), Err(craft_jsonrpc::LineError::Io(_)))); } + +#[test] +fn inspect_single_request_distinguishes_null_ids_and_invalid_envelopes() { + let request = json!({"jsonrpc":"2.0", "id":null, "method":"ping"}); + let parsed = craft_jsonrpc::request(&request).unwrap(); + assert_eq!(parsed.id, Some(&Value::Null)); + assert_eq!(parsed.method, "ping"); + let notification = json!({"jsonrpc":"2.0", "method":"ping"}); + assert!(craft_jsonrpc::request(¬ification).unwrap().id.is_none()); + assert!(craft_jsonrpc::request(&json!({"jsonrpc":"2.0", "id":false, "method":"ping"})).is_none()); + assert!(craft_jsonrpc::request(&json!({"method":"ping"})).is_none()); +} From d6c11b40b73f7b94f040524cf1fc248888bd3afc Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:00:28 +0000 Subject: [PATCH 37/98] Share bounded seekable ZIP indexing --- crates/zip/README.md | 9 +++ crates/zip/src/lib.rs | 2 + crates/zip/src/metadata_reader.rs | 114 ++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) create mode 100644 crates/zip/src/metadata_reader.rs diff --git a/crates/zip/README.md b/crates/zip/README.md index 079267b9f..6cffa7786 100644 --- a/crates/zip/README.md +++ b/crates/zip/README.md @@ -70,3 +70,12 @@ calls it first. `preflight`, `read_entry` and `Budget`: the ZIP half of the craft-archive proposal (#126, #129). `check_entry_count`: craft-opc's `guard`. Duplicate detection: craft-penpot's rule, made exact by walking the directory. The writer: craft-opc's `ZipOut`. + +## Seekable indexing + +`MetadataReader::new(reader, metadata_bytes)` wraps a `Read + Seek` source without loading its +payload into memory. During ZIP indexing it admits reads only from the last `metadata_bytes` +bytes and charges repeated reads against twice that number. Call `Indexing::finish` on the +returned handle after the ZIP index has been parsed to permit normal streaming payload reads. +The caller must still enforce entry-count, declared-size and overlap limits: the wrapper only +bounds metadata reads and does not validate the archive. diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 452351ee3..73eb9840a 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -23,6 +23,7 @@ struct ReadmeExamples; mod archive; mod guard; +mod metadata_reader; pub mod preflight; mod write; @@ -30,6 +31,7 @@ pub use archive::{Archive, Limits}; pub use guard::check_entry_count; #[cfg(any(test, feature = "test-support"))] pub use guard::zip64_entry_count_sample; +pub use metadata_reader::{Indexing, MetadataReader}; pub use write::{Compression, Writer}; /// The payload of a ZIP's first entry when it is a stored file named `mimetype` of at most 256 diff --git a/crates/zip/src/metadata_reader.rs b/crates/zip/src/metadata_reader.rs new file mode 100644 index 000000000..6bba5759d --- /dev/null +++ b/crates/zip/src/metadata_reader.rs @@ -0,0 +1,114 @@ +//! Bound the reads a ZIP indexer may perform before accessing entry payloads. + +use std::io::{self, Read, Seek, SeekFrom}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// A seekable reader whose indexing phase is restricted to a tail window and read budget. +/// +/// Keep the returned [`Indexing`] handle until the ZIP central directory has been parsed, then +/// call [`Indexing::finish`] to allow normal entry reads. Repeated reads during indexing count +/// towards the allowance, including reads after seeks. This bounds metadata scanning without +/// buffering the whole file or restricting the size of its payload entries. +pub struct MetadataReader { + inner: R, + indexing: Arc, + metadata_start: u64, + remaining: u64, + position: u64, +} + +/// Controls the indexing restriction for the associated reader. +#[derive(Clone)] +pub struct Indexing(Arc); + +impl Indexing { + /// Permit reads outside the metadata window once indexing is finished. + pub fn finish(&self) { + self.0.store(false, Ordering::Release); + } +} + +impl MetadataReader { + /// Limit indexing to the last `metadata_bytes` bytes and at most twice that many bytes read. + pub fn new(mut inner: R, metadata_bytes: u64) -> io::Result<(Self, Indexing)> { + let end = inner.seek(SeekFrom::End(0))?; + let indexing = Arc::new(AtomicBool::new(true)); + let handle = Indexing(indexing.clone()); + Ok(( + Self { inner, indexing, metadata_start: end.saturating_sub(metadata_bytes), remaining: metadata_bytes.saturating_mul(2), position: end }, + handle, + )) + } +} + +impl Read for MetadataReader { + fn read(&mut self, bytes: &mut [u8]) -> io::Result { + if bytes.is_empty() { + return Ok(0); + } + let indexing = self.indexing.load(Ordering::Acquire); + if indexing && (self.position < self.metadata_start || self.remaining == 0) { + return Err(io::Error::new(io::ErrorKind::InvalidData, "ZIP metadata exceeds the indexing limit")); + } + let length = if indexing { bytes.len().min(usize::try_from(self.remaining).unwrap_or(usize::MAX)) } else { bytes.len() }; + let read = self.inner.read(bytes.get_mut(..length).unwrap_or_default())?; + self.position = self.position.saturating_add(read as u64); + if indexing { + self.remaining -= read as u64; + } + Ok(read) + } +} + +impl Seek for MetadataReader { + fn seek(&mut self, from: SeekFrom) -> io::Result { + self.position = self.inner.seek(from)?; + Ok(self.position) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn indexing_cannot_read_payloads_outside_the_metadata_window() { + let (mut reader, indexing) = MetadataReader::new(Cursor::new(vec![7; 100]), 8).unwrap(); + reader.seek(SeekFrom::Start(91)).unwrap(); + assert!(reader.read(&mut [0; 1]).is_err()); + reader.seek(SeekFrom::Start(92)).unwrap(); + reader.read_exact(&mut [0; 8]).unwrap(); + indexing.finish(); + reader.seek(SeekFrom::Start(0)).unwrap(); + reader.read_exact(&mut [0; 100]).unwrap(); + } + + #[test] + fn zip_indexing_is_tail_bounded_then_payload_reads_stream() { + let mut writer = crate::Writer::new(); + writer.add("large.bin", &vec![42; 131_072], crate::Compression::Stored).unwrap(); + let bytes = writer.finish().unwrap(); + let (reader, indexing) = MetadataReader::new(Cursor::new(bytes), 16_384).unwrap(); + let config = zip::read::Config { archive_offset: zip::read::ArchiveOffset::Known(0) }; + let mut archive = zip::ZipArchive::with_config(config, reader).unwrap(); + assert!(archive.by_name("large.bin").is_err(), "indexing must not read the payload header"); + indexing.finish(); + let mut entry = archive.by_name("large.bin").unwrap(); + let mut payload = Vec::new(); + entry.read_to_end(&mut payload).unwrap(); + assert_eq!(payload, vec![42; 131_072]); + } + + #[test] + fn repeated_metadata_scans_share_one_read_allowance() { + let (mut reader, _indexing) = MetadataReader::new(Cursor::new(vec![7; 8]), 8).unwrap(); + for _ in 0..2 { + reader.seek(SeekFrom::Start(0)).unwrap(); + reader.read_exact(&mut [0; 8]).unwrap(); + } + reader.seek(SeekFrom::Start(0)).unwrap(); + assert!(reader.read(&mut [0; 1]).is_err()); + } +} From 576fdd5e1a36331d16ba08ee03376b1799864881 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:06:30 +0000 Subject: [PATCH 38/98] Support explicit JSON-RPC ID policies --- crates/jsonrpc/README.md | 8 ++++++-- crates/jsonrpc/src/lib.rs | 21 ++++++++++++++------- crates/jsonrpc/tests/server.rs | 27 +++++++++++++++++++++------ 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/crates/jsonrpc/README.md b/crates/jsonrpc/README.md index 0b78e5470..8711c25ff 100644 --- a/crates/jsonrpc/README.md +++ b/crates/jsonrpc/README.md @@ -12,8 +12,12 @@ For newline-delimited transports, `read_line` bounds memory while reading; after line, the caller must close the stream or discard the rest of that line. `serde_json`'s nesting limit remains in effect. Response sizes are the caller's responsibility. -A valid request receives the handler's result or error with its original string, integer, or null -ID. An omitted ID is a notification: it calls the separate notification handler and gets no reply. +A valid request receives the handler's result or error with its original string, numeric, or null +ID. The default accepts numeric IDs as required by [JSON-RPC 2.0](https://www.jsonrpc.org/specification#request_object), +which discourages fractional values. Set `integer_ids_only` and clear `allow_null_ids` for +protocols such as MCP that require integer or string IDs. A legacy server can retain its previous +null-ID behavior independently. An omitted ID is a notification: it calls the separate notification +handler and gets no reply. Malformed batch entries receive an invalid-request error; an empty batch gets one error response; a batch containing only notifications has no response. Invalid parameter shapes receive an invalid-params error for requests and no reply for notifications. The handler decides whether diff --git a/crates/jsonrpc/src/lib.rs b/crates/jsonrpc/src/lib.rs index 8d2aa6c6e..9f8128d1d 100644 --- a/crates/jsonrpc/src/lib.rs +++ b/crates/jsonrpc/src/lib.rs @@ -16,11 +16,15 @@ use std::io::BufRead; pub struct Limits { pub max_message_bytes: usize, pub max_batch_items: usize, + /// Reject fractional JSON numbers as IDs for protocols that require integer IDs. + pub integer_ids_only: bool, + /// Permit an explicit null request ID, as JSON-RPC does; some protocols prohibit it. + pub allow_null_ids: bool, } impl Default for Limits { fn default() -> Self { - Self { max_message_bytes: 64 * 1024 * 1024, max_batch_items: 4096 } + Self { max_message_bytes: 64 * 1024 * 1024, max_batch_items: 4096, integer_ids_only: false, allow_null_ids: true } } } @@ -109,10 +113,13 @@ pub struct Request<'a> { /// Inspect a single request envelope. This validates its JSON-RPC version, method and ID; /// [`process`] additionally validates parameter shape before dispatching it. -pub fn request(value: &Value) -> Option> { +pub fn request(value: &Value, limits: Limits) -> Option> { let object = value.as_object()?; let id = object.get("id"); - if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") || !id.is_none_or(|v| v.is_null() || v.is_string() || v.is_i64() || v.is_u64()) { + let valid_id = id.is_none_or(|v| { + (v.is_null() && limits.allow_null_ids) || v.is_string() || if limits.integer_ids_only { v.is_i64() || v.is_u64() } else { v.is_number() } + }); + if object.get("jsonrpc").and_then(Value::as_str) != Some("2.0") || !valid_id { return None; } Some(Request { id, method: object.get("method")?.as_str()?, params: object.get("params") }) @@ -150,12 +157,12 @@ fn invalid_request() -> Value { error_response(Value::Null, RpcError::new(-32600, "Invalid Request")) } -fn one(value: &Value, on_request: &mut F, on_notification: &mut N) -> Option +fn one(value: &Value, limits: Limits, on_request: &mut F, on_notification: &mut N) -> Option where F: FnMut(Request<'_>) -> Result, N: FnMut(Request<'_>), { - let Some(request) = request(value) else { return Some(invalid_request()) }; + let Some(request) = request(value, limits) else { return Some(invalid_request()) }; let id = request.id; if request.params.is_some_and(|v| !v.is_object() && !v.is_array()) { return id.map(|id| error_response(id.clone(), RpcError::new(-32602, "Invalid params"))); @@ -178,13 +185,13 @@ where F: FnMut(Request<'_>) -> Result, N: FnMut(Request<'_>), { - let Some(batch) = value.as_array() else { return Ok(one(value, &mut on_request, &mut on_notification)) }; + let Some(batch) = value.as_array() else { return Ok(one(value, limits, &mut on_request, &mut on_notification)) }; if batch.len() > limits.max_batch_items { return Err(ParseError::BatchTooLarge); } if batch.is_empty() { return Ok(Some(invalid_request())); } - let responses: Vec<_> = batch.iter().filter_map(|v| one(v, &mut on_request, &mut on_notification)).collect(); + let responses: Vec<_> = batch.iter().filter_map(|v| one(v, limits, &mut on_request, &mut on_notification)).collect(); Ok((!responses.is_empty()).then_some(Value::Array(responses))) } diff --git a/crates/jsonrpc/tests/server.rs b/crates/jsonrpc/tests/server.rs index c2ff1d0d2..a9186f4ae 100644 --- a/crates/jsonrpc/tests/server.rs +++ b/crates/jsonrpc/tests/server.rs @@ -59,7 +59,7 @@ fn notifications_never_receive_responses_including_all_notification_batches() { #[test] fn invalid_envelopes_and_params_return_standard_errors() { - for id in [json!(1.5), json!(true), json!([]), json!({})] { + for id in [json!(true), json!([]), json!({})] { let input = json!({"jsonrpc":"2.0", "id":id, "method":"ping"}); let reply = process(&input, Limits::default(), |_| panic!("invalid id dispatched"), |_| panic!("invalid id notified")).unwrap().unwrap(); assert_eq!(reply["id"], Value::Null); @@ -99,7 +99,7 @@ fn arrays_are_valid_params_and_errors_can_include_data() { #[test] fn parse_and_batch_limits_apply_before_dispatch() { - let limits = Limits { max_message_bytes: 3, max_batch_items: 1 }; + let limits = Limits { max_message_bytes: 3, max_batch_items: 1, ..Limits::default() }; assert_eq!(parse_json(b"null", limits), Err(ParseError::MessageTooLarge)); assert_eq!(parse_json(b"123", limits).unwrap(), json!(123)); assert_eq!(parse_json(b"{", limits), Err(ParseError::InvalidJson)); @@ -147,11 +147,26 @@ fn bounded_line_read_propagates_io_errors() { #[test] fn inspect_single_request_distinguishes_null_ids_and_invalid_envelopes() { let request = json!({"jsonrpc":"2.0", "id":null, "method":"ping"}); - let parsed = craft_jsonrpc::request(&request).unwrap(); + let parsed = craft_jsonrpc::request(&request, Limits::default()).unwrap(); assert_eq!(parsed.id, Some(&Value::Null)); assert_eq!(parsed.method, "ping"); let notification = json!({"jsonrpc":"2.0", "method":"ping"}); - assert!(craft_jsonrpc::request(¬ification).unwrap().id.is_none()); - assert!(craft_jsonrpc::request(&json!({"jsonrpc":"2.0", "id":false, "method":"ping"})).is_none()); - assert!(craft_jsonrpc::request(&json!({"method":"ping"})).is_none()); + assert!(craft_jsonrpc::request(¬ification, Limits::default()).unwrap().id.is_none()); + assert!(craft_jsonrpc::request(&json!({"jsonrpc":"2.0", "id":false, "method":"ping"}), Limits::default()).is_none()); + assert!(craft_jsonrpc::request(&json!({"method":"ping"}), Limits::default()).is_none()); +} + +#[test] +fn numeric_ids_follow_the_selected_protocol_policy() { + let input = json!({"jsonrpc":"2.0", "id":1.5, "method":"ping"}); + let reply = process(&input, Limits::default(), |_| Ok(json!({})), |_| {}).unwrap().unwrap(); + assert_eq!(reply["id"], json!(1.5)); + let strict = Limits { integer_ids_only: true, ..Limits::default() }; + let reply = process(&input, strict, |_| panic!("fractional ID dispatched"), |_| {}).unwrap().unwrap(); + assert_eq!(reply["id"], Value::Null); + assert_eq!(reply["error"]["code"], -32600); + let no_null = Limits { allow_null_ids: false, ..strict }; + let null_id = json!({"jsonrpc":"2.0", "id":null, "method":"ping"}); + let reply = process(&null_id, no_null, |_| panic!("null ID dispatched"), |_| {}).unwrap().unwrap(); + assert_eq!(reply["error"]["code"], -32600); } From 5ab7a6035c0ff409cc743978b17d73f414b58524 Mon Sep 17 00:00:00 2001 From: el-j Date: Sun, 15 Nov 2026 13:06:49 +0100 Subject: [PATCH 39/98] craft-video: FFV1 lossless video, version 3 range-coder decoder and encoder Adds the ffv1 feature (RFC 9043): YCbCr (any 2^n subsampling, gray, optional transparency) and RGB through the JPEG 2000 reversible colour transform, 8 to 16 bits, any slice raster, slice and configuration record CRCs. Not covered yet: the Golomb-Rice coder, versions 0 and 1, inter frames. Part of #1509 --- crates/video/Cargo.toml | 6 + crates/video/src/ffv1/README.md | 52 ++ crates/video/src/ffv1/mod.rs | 929 ++++++++++++++++++++++++++++++++ crates/video/src/ffv1/rc.rs | 229 ++++++++ crates/video/src/lib.rs | 4 + crates/video/tests/ffv1.rs | 304 +++++++++++ 6 files changed, 1524 insertions(+) create mode 100644 crates/video/src/ffv1/README.md create mode 100644 crates/video/src/ffv1/mod.rs create mode 100644 crates/video/src/ffv1/rc.rs create mode 100644 crates/video/tests/ffv1.rs diff --git a/crates/video/Cargo.toml b/crates/video/Cargo.toml index c3ebfd722..dae21e5b1 100644 --- a/crates/video/Cargo.toml +++ b/crates/video/Cargo.toml @@ -34,6 +34,8 @@ mpeg2-encode = ["mpeg2-decode"] # MPEG-4 Part 2 video (ISO/IEC 14496-2, the Xvid/DivX format): Simple profile encoder. mpeg4 = ["mpeg4-encode"] mpeg4-encode = ["craft-media/bitstream"] +# FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder. +ffv1 = [] # Apple ProRes (SMPTE RDD 36): decoder and encoder. prores = ["craft-media/bitstream"] # H.264 / AVC: decoder and encoder; `h264-decode` or `h264-encode` for one of them. @@ -143,6 +145,10 @@ required-features = ["mpeg4-encode"] name = "mpeg2" required-features = ["mpeg2-decode"] +[[test]] +name = "ffv1" +required-features = ["ffv1"] + [[test]] name = "prores" required-features = ["prores"] diff --git a/crates/video/src/ffv1/README.md b/crates/video/src/ffv1/README.md new file mode 100644 index 000000000..90312dbe5 --- /dev/null +++ b/crates/video/src/ffv1/README.md @@ -0,0 +1,52 @@ +# FFV1 + +Clean-room FFV1 lossless video codec, written from RFC 9043. Feature `ffv1`. + +## What is supported + +| | Decoder | Encoder | +|---|---|---| +| Version | 3 (the one with a configuration record) | 3 | +| Coder | range coder, default or custom state transition table | range coder, default table | +| Colour | YCbCr (any 2^n chroma subsampling, gray, optional transparency), RGB through the JPEG 2000 reversible colour transform (optional transparency) | the same, RGB up to 15 bits | +| Depth | 8 to 16 bits per sample | 8 to 16 bits (RGB 8 to 15) | +| Slices | any raster whose boundaries are multiples of the chroma subsampling | the same | +| Frames | keyframes | keyframes only (`intra` 1) | +| Error detection | slice and configuration record CRCs, checked | written | + +Not supported, and reported as `Error::Unsupported`: the Golomb-Rice coder (`coder_type` 0), +versions 0 and 1 (parameters inside each keyframe, no configuration record), inter frames +(`keyframe` 0), and slice rasters whose boundaries fall inside a chroma sample. + +## Using it + +The configuration record goes in the container (Matroska `CodecPrivate`, the AVI `strf` +extension, an MP4 `glbl` box); each frame is one packet. + +```text +let enc = Encoder::new(&template_frame, EncoderConfig::default())?; +let config = enc.config_record(); +let packet = enc.encode(&frame)?; + +let dec = Decoder::new(&config, width, height)?; +let frame = dec.decode(&packet)?; +``` + +## Notes on the format + +- The Parameters in the configuration record are range coded with the *default* state transition + table. A custom table (`coder_type` 2) applies to the slices only. +- Each Quantization Table starts from fresh states; a Quantization Table Set is five of them. +- A slice is one range-coded stream: for the first slice the `keyframe` bit, then the slice header, + then the samples. Cb and Cr share one set of context states; transparency has its own. +- The context is the sum of five quantised differences. With the default tables this encoder + writes 15125 contexts; FFmpeg refuses tables that multiply to more than 32768, so the encoder + stays below that although the RFC's limit (on half the product) is looser. +- The encoder ends a stream with the top of its final range, so bytes after it (a CRC in the + configuration record) cannot change what decodes. + +## Verification + +`tests/ffv1.rs`: round trips over every shape, hostile input (every truncation and bit flip), +and ffmpeg as the oracle in both directions (skipped without ffmpeg, a failure with +`CRAFT_MEDIA_REQUIRE_ORACLES=1`). diff --git a/crates/video/src/ffv1/mod.rs b/crates/video/src/ffv1/mod.rs new file mode 100644 index 000000000..4df60f9a1 --- /dev/null +++ b/crates/video/src/ffv1/mod.rs @@ -0,0 +1,929 @@ +//! FFV1 lossless video codec (RFC 9043): version 3, range coder, intra-only, YCbCr and RGB (JPEG 2000 +//! RCT), 8 to 16 bits per sample, any number of slices. +//! +//! - [`Decoder`]: built from the *configuration record* a container stores (Matroska `CodecPrivate`, +//! AVI `strf` extension, MP4 `glbl`) and the frame size; [`Decoder::decode`] turns one packet into a +//! [`Frame`]. Slice CRCs are checked. +//! - [`Encoder`]: [`Encoder::config_record`] gives the bytes for the container, [`Encoder::encode`] +//! one packet per frame; every frame is a keyframe. +//! +//! Not supported (reported as [`Error::Unsupported`]): the Golomb-Rice coder (`coder_type` 0), +//! versions 0 and 1 (no configuration record, parameters in every keyframe), inter frames +//! (`intra` 0 with `keyframe` 0), and slice rasters whose boundaries are not multiples of the +//! chroma subsampling. See `src/ffv1/README.md`. +//! +//! Implemented from RFC 9043. Frames are planar `u16`: for YCbCr, `[Y, Cb, Cr]` (plus an optional +//! transparency plane); for RGB, `[R, G, B]` (plus transparency), all at full size. + +mod rc; + +use crate::budget::{DECODE_MEMORY_BUDGET, PictureShape}; +use rc::{Decoder as Rac, Encoder as RacEnc, Transitions}; + +const CONTEXT_SIZE: usize = 32; +const MAX_CONTEXT_INPUTS: usize = 5; +const MAX_CONTEXTS: u64 = 32768; + +/// Codec errors. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The data ends early or its sizes do not add up. + #[error("truncated FFV1 data")] + Truncated, + /// The data is not valid FFV1. + #[error("invalid FFV1 data: {0}")] + Invalid(&'static str), + /// Valid FFV1 that this module does not decode. + #[error("unsupported FFV1 feature: {0}")] + Unsupported(&'static str), + /// A slice or the configuration record failed its CRC. + #[error("FFV1 CRC mismatch in {0}")] + Crc(&'static str), + /// The encoder was given a frame or settings it cannot encode. + #[error("invalid encoder input: {0}")] + Input(String), +} + +/// `Result` of this module. +pub type Result = std::result::Result; + +/// The colour space coded (RFC 9043 table 8). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Colorspace { + /// YCbCr, coded plane by plane without a pixel transform. + YCbCr, + /// RGB through the JPEG 2000 reversible colour transform, coded line by line. + Rgb, +} + +/// One frame of planar samples. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + /// Width in pixels. + pub width: u32, + /// Height in pixels. + pub height: u32, + /// Colour space of the planes. + pub colorspace: Colorspace, + /// Bits per sample, 8 to 16. + pub bit_depth: u8, + /// log2 of the horizontal and vertical chroma subsampling (YCbCr only; 0, 0 is 4:4:4). + pub chroma_shift: (u8, u8), + /// The planes, in the order the module doc gives. + pub planes: Vec>, +} + +impl Frame { + /// Whether the planes are Y, Cb, Cr (or R, G, B) rather than Y (gray) alone, or with transparency. + fn plane_count_ok(&self) -> bool { + matches!(self.planes.len(), 1..=4) + } + + fn dims(&self, plane: usize, chroma_planes: bool) -> (usize, usize) { + plane_dims(self.width as usize, self.height as usize, self.colorspace, self.chroma_shift, plane, chroma_planes) + } +} + +/// `(width, height)` of plane `p` for a full frame (or a slice) of `w` x `h`. +fn plane_dims(w: usize, h: usize, cs: Colorspace, shift: (u8, u8), p: usize, chroma_planes: bool) -> (usize, usize) { + if cs == Colorspace::YCbCr && chroma_planes && (p == 1 || p == 2) { (w.div_ceil(1 << shift.0), h.div_ceil(1 << shift.1)) } else { (w, h) } +} + +/// One Quantization Table Set (section 4.1). +#[derive(Clone)] +struct QuantSet { + tables: Vec<[i32; 256]>, + context_count: usize, +} + +/// The parsed "Parameters" of a stream. +#[derive(Clone)] +struct Params { + coder_type: u32, + transitions: Transitions, + colorspace: Colorspace, + bits: u32, + chroma_planes: bool, + shift: (u8, u8), + extra_plane: bool, + num_h: u32, + num_v: u32, + sets: Vec, + initial_states: Vec>>, + ec: bool, + intra: bool, +} + +impl Params { + fn plane_count(&self) -> usize { + 1 + if self.chroma_planes { 2 } else { 0 } + usize::from(self.extra_plane) + } + + /// Quantization Table Set index slot of plane `p` (section 3.6). + fn qset_slot(&self, p: usize) -> usize { + match p { + 0 => 0, + 1 | 2 if self.chroma_planes => 1, + _ => 2, + } + } +} + +fn crc32(data: &[u8]) -> u32 { + // The IEEE polynomial 0x04C11DB7, initial value 0, no reflection, no final inversion. + let mut crc = 0u32; + for &b in data { + crc ^= u32::from(b) << 24; + for _ in 0..8 { + crc = if crc & 0x8000_0000 != 0 { crc << 1 ^ 0x04C1_1DB7 } else { crc << 1 }; + } + } + crc +} + +// ---- Parameters ---- + +fn read_quant_set(rac: &mut Rac, t: &Transitions) -> Result { + let mut scale: u64 = 1; + let mut tables = Vec::with_capacity(MAX_CONTEXT_INPUTS); + for _ in 0..MAX_CONTEXT_INPUTS { + let mut table = [0i32; 256]; + // Each quantization table starts from fresh states. + let mut state = [128u8; CONTEXT_SIZE]; + let mut v: i64 = 0; + let mut k = 0usize; + while k < 128 { + let len = rac.symbol(&mut state, false, t) as usize + 1; + if k + len > 128 { + return Err(Error::Invalid("quantization table run passes its half")); + } + for _ in 0..len { + table[k] = i32::try_from(scale as i64 * v).map_err(|_| Error::Invalid("quantization table overflows"))?; + k += 1; + } + v += 1; + } + for k in 1..128 { + table[256 - k] = -table[k]; + } + table[128] = -table[127]; + tables.push(table); + scale = scale.checked_mul(2 * v as u64 - 1).filter(|&s| s <= 2 * MAX_CONTEXTS).ok_or(Error::Invalid("too many contexts"))?; + } + let context_count = scale.div_ceil(2); + if context_count > MAX_CONTEXTS { + return Err(Error::Invalid("too many contexts")); + } + Ok(QuantSet { tables, context_count: context_count as usize }) +} + +fn parse_params(rac: &mut Rac) -> Result { + let mut state = [128u8; CONTEXT_SIZE]; + // Until the custom table is known, the default table decodes the fields before it. + let t0 = Transitions::default_table(); + let version = rac.symbol(&mut state, false, &t0); + if version != 3 { + return Err(Error::Unsupported("only FFV1 version 3 has a configuration record; versions 0 and 1 are not supported")); + } + let _micro = rac.symbol(&mut state, false, &t0); + let coder_type = rac.symbol(&mut state, false, &t0) as u32; + let mut delta = [0i32; 256]; + if coder_type > 1 { + for d in delta.iter_mut().skip(1) { + *d = rac.symbol(&mut state, true, &t0); + } + } + let transitions = Transitions::new(&delta); + if coder_type == 0 { + return Err(Error::Unsupported("the Golomb-Rice coder (coder_type 0)")); + } + if coder_type > 2 { + return Err(Error::Invalid("unknown coder_type")); + } + let colorspace = match rac.symbol(&mut state, false, &t0) { + 0 => Colorspace::YCbCr, + 1 => Colorspace::Rgb, + _ => return Err(Error::Unsupported("colorspace_type")), + }; + let bits = rac.symbol(&mut state, false, &t0) as u32; + let bits = if bits == 0 { 8 } else { bits }; + let chroma_planes = rac.bit(&mut state[0], &t0); + let (h, v) = (rac.symbol(&mut state, false, &t0), rac.symbol(&mut state, false, &t0)); + let extra_plane = rac.bit(&mut state[0], &t0); + let num_h = rac.symbol(&mut state, false, &t0) as u32 + 1; + let num_v = rac.symbol(&mut state, false, &t0) as u32 + 1; + let set_count = rac.symbol(&mut state, false, &t0); + if !(8..=16).contains(&bits) || h > 3 || v > 3 || h < 0 || v < 0 || num_h > 1024 || num_v > 1024 || !(1..=8).contains(&set_count) { + return Err(Error::Invalid("parameter out of range")); + } + if colorspace == Colorspace::Rgb && (!chroma_planes || h != 0 || v != 0) { + return Err(Error::Unsupported("RGB with chroma_planes 0 or subsampling")); + } + let mut sets = Vec::new(); + for _ in 0..set_count { + // The Parameters are range coded with the default table; the custom one is for the slices. + sets.push(read_quant_set(rac, &t0)?); + } + let mut initial_states = Vec::new(); + for set in &sets { + if rac.bit(&mut state[0], &t0) { + let mut all = Vec::with_capacity(set.context_count); + let mut prev = [128u8; CONTEXT_SIZE]; + for _ in 0..set.context_count { + let mut cur = [0u8; CONTEXT_SIZE]; + for k in 0..CONTEXT_SIZE { + let d = rac.symbol(&mut state, true, &t0); + cur[k] = (i32::from(prev[k]) + d) as u8; + } + prev = cur; + all.push(cur); + } + initial_states.push(Some(all)); + } else { + initial_states.push(None); + } + } + let ec = rac.symbol(&mut state, false, &t0) != 0; + let intra = rac.symbol(&mut state, false, &t0) != 0; + Ok(Params { + coder_type, + transitions, + colorspace, + bits, + chroma_planes, + shift: (h as u8, v as u8), + extra_plane, + num_h, + num_v, + sets, + initial_states, + ec, + intra, + }) +} + +/// The default quantisation tables of this encoder: 11 levels for the two gradients next to the +/// sample and 5 for the other three, which makes 15125 contexts (FFmpeg refuses more than 32768). +fn default_quant_set() -> QuantSet { + // |difference| thresholds: the first |d| of each level above 0 (levels 0..=5 and 0..=2). + let first_of_level: [&[usize]; MAX_CONTEXT_INPUTS] = [&[1, 2, 4, 8, 16], &[1, 2, 4, 8, 16], &[1, 3], &[1, 3], &[1, 3]]; + let mut scale: i32 = 1; + let mut tables = Vec::new(); + let mut counts = [0usize; MAX_CONTEXT_INPUTS]; + for (j, firsts) in first_of_level.iter().enumerate() { + let mut table = [0i32; 256]; + for (k, slot) in table.iter_mut().take(128).enumerate() { + let level = firsts.iter().filter(|&&f| k >= f).count() as i32; + *slot = scale * level; + } + for k in 1..128 { + table[256 - k] = -table[k]; + } + table[128] = -table[127]; + counts[j] = firsts.len() + 1; + scale *= 2 * counts[j] as i32 - 1; + tables.push(table); + } + QuantSet { tables, context_count: (scale as usize).div_ceil(2) } +} + +/// Run lengths (`len - 1` per step) that describe the first half of a table, as written to the stream. +fn quant_runs(table: &[i32; 256]) -> Vec { + let mut runs = Vec::new(); + let mut k = 0usize; + while k < 128 { + let start = k; + while k < 128 && table[k] == table[start] { + k += 1; + } + runs.push((k - start) as i32 - 1); + } + runs +} + +fn write_params(enc: &mut RacEnc, p: &Params) { + let t = Transitions::default_table(); + let mut state = [128u8; CONTEXT_SIZE]; + enc.symbol(&mut state, 3, false, &t); // version + enc.symbol(&mut state, 4, false, &t); // micro_version: first stable variant + enc.symbol(&mut state, p.coder_type as i32, false, &t); + enc.symbol(&mut state, if p.colorspace == Colorspace::YCbCr { 0 } else { 1 }, false, &t); + enc.symbol(&mut state, p.bits as i32, false, &t); + enc.bit(p.chroma_planes, &mut state[0], &t); + enc.symbol(&mut state, i32::from(p.shift.0), false, &t); + enc.symbol(&mut state, i32::from(p.shift.1), false, &t); + enc.bit(p.extra_plane, &mut state[0], &t); + enc.symbol(&mut state, p.num_h as i32 - 1, false, &t); + enc.symbol(&mut state, p.num_v as i32 - 1, false, &t); + enc.symbol(&mut state, p.sets.len() as i32, false, &t); + for set in &p.sets { + for table in &set.tables { + let mut qstate = [128u8; CONTEXT_SIZE]; + for run in quant_runs(table) { + enc.symbol(&mut qstate, run, false, &t); + } + } + } + for _ in &p.sets { + enc.bit(false, &mut state[0], &t); // states_coded + } + enc.symbol(&mut state, i32::from(p.ec), false, &t); + enc.symbol(&mut state, i32::from(p.intra), false, &t); +} + +// ---- slices ---- + +/// A slice's place in the frame, in pixels. +#[derive(Clone, Copy)] +struct SliceRect { + x: usize, + y: usize, + w: usize, + h: usize, +} + +fn slice_rect(p: &Params, width: usize, height: usize, sx: usize, sy: usize, sw: usize, sh: usize) -> Result { + let (nh, nv) = (p.num_h as usize, p.num_v as usize); + if sx + sw > nh || sy + sh > nv || sw == 0 || sh == 0 { + return Err(Error::Invalid("slice outside the slice raster")); + } + let x0 = sx * width / nh; + let x1 = (sx + sw) * width / nh; + let y0 = sy * height / nv; + let y1 = (sy + sh) * height / nv; + if x1 <= x0 || y1 <= y0 { + return Err(Error::Invalid("empty slice")); + } + Ok(SliceRect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 }) +} + +/// The samples of one plane of a slice with the border of RFC 9043 section 3.1 around them: two +/// columns to the left, one to the right, two rows above. +struct Border { + buf: Vec, + stride: usize, + w: usize, +} + +impl Border { + fn new(w: usize, h: usize) -> Border { + let stride = w + 3; + Border { buf: vec![0; stride * (h + 2)], stride, w } + } + + fn at(&self, x: usize, y: usize) -> usize { + (y + 2) * self.stride + x + 2 + } + + /// Set up the left border of row `y` (the column left of it repeats the row above's first sample). + fn start_row(&mut self, y: usize) { + let row = (y + 2) * self.stride; + self.buf[row] = 0; + self.buf[row + 1] = if y > 0 { self.buf[(y + 1) * self.stride + 2] } else { 0 }; + } + + /// The right border of row `y` repeats its last sample. + fn end_row(&mut self, y: usize) { + let row = (y + 2) * self.stride; + self.buf[row + self.w + 2] = self.buf[row + self.w + 1]; + } + + /// `(context, predictor)` for sample `(x, y)`; `context` is signed (section 3.5). + fn context(&self, x: usize, y: usize, tables: &[[i32; 256]], sixteen_bit_ycbcr: bool) -> (i32, i32) { + let i = self.at(x, y); + let (b, s) = (&self.buf, self.stride); + let (l, ll, t, tl, tr, tt) = (b[i - 1], b[i - 2], b[i - s], b[i - s - 1], b[i - s + 1], b[i - 2 * s]); + let ctx = tables[0][((l - tl) & 255) as usize] + + tables[1][((tl - t) & 255) as usize] + + tables[2][((t - tr) & 255) as usize] + + tables[3][((ll - l) & 255) as usize] + + tables[4][((tt - t) & 255) as usize]; + let (pl, pt, ptl) = if sixteen_bit_ycbcr { + let s16 = |v: i32| if v >= 32768 { v - 65536 } else { v }; + (s16(l), s16(t), s16(tl)) + } else { + (l, t, tl) + }; + (ctx, median(pl, pt, pl + pt - ptl)) + } +} + +fn median(a: i32, b: i32, c: i32) -> i32 { + a.max(b).min(a.min(b).max(c)) +} + +/// The reversible colour transform bits: a plane's samples have `bits + 1` bits. +fn rct_swapped(p: &Params) -> bool { + p.colorspace == Colorspace::Rgb && (9..=15).contains(&p.bits) && !p.extra_plane +} + +fn coded_bits(p: &Params) -> u32 { + if p.colorspace == Colorspace::Rgb { p.bits + 1 } else { p.bits } +} + +/// One plane's samples inside a slice, and which class of states codes them. +struct PlaneCoder { + border: Border, + slot: usize, +} + +/// The adaptive states of one plane class: Y, Cb and Cr together, then transparency (section 3.6). +struct ClassStates { + set: usize, + states: Vec<[u8; CONTEXT_SIZE]>, +} + +fn make_coders(p: &Params, rect: SliceRect, set_idx: &[usize]) -> Result<(Vec, Vec)> { + let mut planes = Vec::new(); + for pl in 0..p.plane_count() { + let (w, h) = plane_dims(rect.w, rect.h, p.colorspace, p.shift, pl, p.chroma_planes); + planes.push(PlaneCoder { border: Border::new(w, h), slot: p.qset_slot(pl) }); + } + let mut classes = Vec::new(); + for slot in 0..=planes.iter().map(|c| c.slot).max().unwrap_or(0) { + let set = *set_idx.get(slot).ok_or(Error::Invalid("missing quant table set index"))?; + let q = p.sets.get(set).ok_or(Error::Invalid("quant table set index out of range"))?; + let states = match p.initial_states.get(set) { + Some(Some(init)) => init.clone(), + _ => vec![[128u8; CONTEXT_SIZE]; q.context_count], + }; + classes.push(ClassStates { set, states }); + } + Ok((planes, classes)) +} + +/// The number of Quantization Table Set indexes in a slice header (section 4.6.5, version 3). +fn set_index_count(p: &Params) -> usize { + 2 + usize::from(p.extra_plane) +} + +fn check_frame_size(p: &Params, w: u32, h: u32) -> Result<()> { + if w == 0 || h == 0 { + return Err(Error::Invalid("empty frame")); + } + let shape = PictureShape { + width: u64::from(w), + height: u64::from(h), + shift_x: u32::from(p.shift.0.min(1)), + shift_y: u32::from(p.shift.1.min(1)), + chroma_planes: if p.chroma_planes { 2 } else { 0 }, + bytes_per_sample: 2, + }; + // The decoded planes and, while a slice decodes, its bordered `i32` copy (2 x the samples at most). + if !shape.fits(5, DECODE_MEMORY_BUDGET) { + return Err(Error::Unsupported("frame size above the memory budget")); + } + if p.shift.0 > 1 || p.shift.1 > 1 { + // Subsampling beyond 2:1 is legal FFV1 (e.g. 4:1:1) but the budget model above is for 2:1. + let extra = (w as u64).checked_mul(h as u64).ok_or(Error::Invalid("frame too large"))?; + if extra > 1 << 28 { + return Err(Error::Unsupported("frame size above the memory budget")); + } + } + Ok(()) +} + +/// A decoder for one stream. +pub struct Decoder { + params: Params, + width: u32, + height: u32, +} + +impl Decoder { + /// Build a decoder from the container's configuration record and the frame size. + pub fn new(config_record: &[u8], width: u32, height: u32) -> Result { + if config_record.len() < 5 { + return Err(Error::Truncated); + } + let (body, _crc) = config_record.split_at(config_record.len() - 4); + if crc32(config_record) != 0 { + return Err(Error::Crc("the configuration record")); + } + let mut rac = Rac::new(body); + let params = parse_params(&mut rac)?; + if rac.consumed() > body.len() + 2 { + return Err(Error::Truncated); + } + check_frame_size(¶ms, width, height)?; + Ok(Decoder { params, width, height }) + } + + /// Decode one frame (a packet from the container). + pub fn decode(&self, packet: &[u8]) -> Result { + let p = &self.params; + let (w, h) = (self.width as usize, self.height as usize); + let planes = p.plane_count(); + let mut frame = Frame { + width: self.width, + height: self.height, + colorspace: p.colorspace, + bit_depth: p.bits as u8, + chroma_shift: if p.chroma_planes { p.shift } else { (0, 0) }, + planes: Vec::with_capacity(planes), + }; + for pl in 0..planes { + let (pw, ph) = plane_dims(w, h, p.colorspace, p.shift, pl, p.chroma_planes); + let mut v = Vec::new(); + v.try_reserve_exact(pw * ph).map_err(|_| Error::Unsupported("frame size above the memory budget"))?; + v.resize(pw * ph, 0u16); + frame.planes.push(v); + } + // Slices are found from the end of the packet through their footers (section 4.9). + let trailer = if p.ec { 8 } else { 3 }; + let slice_count = p.num_h as usize * p.num_v as usize; + let mut end = packet.len(); + let mut spans = Vec::with_capacity(slice_count); + for _ in 0..slice_count { + if end < trailer { + return Err(Error::Truncated); + } + let f = end - trailer; + let size = usize::from(packet[f]) << 16 | usize::from(packet[f + 1]) << 8 | usize::from(packet[f + 2]); + let start = f.checked_sub(size).ok_or(Error::Truncated)?; + spans.push((start, end)); + end = start; + } + spans.reverse(); + for (n, &(start, stop)) in spans.iter().enumerate() { + let slice = &packet[start..stop]; + if p.ec && crc32(slice) != 0 { + return Err(Error::Crc("a slice")); + } + self.decode_slice(slice, n == 0, &mut frame)?; + } + Ok(frame) + } + + fn decode_slice(&self, slice: &[u8], first: bool, frame: &mut Frame) -> Result<()> { + let p = &self.params; + let t = &p.transitions; + let data = &slice[..slice.len() - if p.ec { 8 } else { 3 }]; + let mut rac = Rac::new(data); + // The keyframe flag is the first symbol of the first slice. + // The whole slice, header included, is coded with the stream's state transition table. + let t0 = t; + if first { + let mut ks = 128u8; + if !rac.bit(&mut ks, t0) { + return Err(Error::Unsupported("inter frames (keyframe 0)")); + } + } + let mut hs = [128u8; CONTEXT_SIZE]; + let sx = rac.symbol(&mut hs, false, t0) as usize; + let sy = rac.symbol(&mut hs, false, t0) as usize; + let sw = rac.symbol(&mut hs, false, t0) as usize + 1; + let sh = rac.symbol(&mut hs, false, t0) as usize + 1; + let set_idx: Vec = (0..set_index_count(p)).map(|_| rac.symbol(&mut hs, false, t0).max(0) as usize).collect(); + let _picture_structure = rac.symbol(&mut hs, false, t0); + let _sar = (rac.symbol(&mut hs, false, t0), rac.symbol(&mut hs, false, t0)); + let rect = slice_rect(p, self.width as usize, self.height as usize, sx, sy, sw, sh)?; + let (sx_mask, sy_mask) = ((1usize << p.shift.0) - 1, (1usize << p.shift.1) - 1); + if p.colorspace == Colorspace::YCbCr && p.chroma_planes && (rect.x & sx_mask != 0 || rect.y & sy_mask != 0) { + return Err(Error::Unsupported("slice boundaries that are not multiples of the chroma subsampling")); + } + let (mut coders, mut classes) = make_coders(p, rect, &set_idx)?; + let bits = coded_bits(p); + let mask = (1i32 << bits) - 1; + let sixteen = p.colorspace == Colorspace::YCbCr && p.bits == 16; + let read = |rac: &mut Rac, c: &mut PlaneCoder, cls: &mut ClassStates, x: usize, y: usize| -> i32 { + let q = &p.sets[cls.set]; + let (ctx, pred) = c.border.context(x, y, &q.tables, sixteen); + let (idx, neg) = if ctx < 0 { ((-ctx) as usize, true) } else { (ctx as usize, false) }; + let mut d = rac.symbol(&mut cls.states[idx.min(q.context_count - 1)], true, t); + if neg { + d = -d; + } + let v = (pred + d) & mask; + let at = c.border.at(x, y); + c.border.buf[at] = v; + v + }; + match p.colorspace { + Colorspace::YCbCr => { + for (pl, c) in coders.iter_mut().enumerate() { + let (pw, ph) = plane_dims(rect.w, rect.h, p.colorspace, p.shift, pl, p.chroma_planes); + for y in 0..ph { + c.border.start_row(y); + for x in 0..pw { + read(&mut rac, c, &mut classes[c.slot], x, y); + } + c.border.end_row(y); + } + } + } + Colorspace::Rgb => { + for y in 0..rect.h { + for c in coders.iter_mut() { + c.border.start_row(y); + for x in 0..rect.w { + read(&mut rac, c, &mut classes[c.slot], x, y); + } + c.border.end_row(y); + } + } + } + } + self.store(&coders, rect, frame); + Ok(()) + } + + /// Copy a decoded slice into the frame, undoing the colour transform for RGB. + fn store(&self, coders: &[PlaneCoder], rect: SliceRect, frame: &mut Frame) { + let p = &self.params; + if p.colorspace == Colorspace::YCbCr { + for (pl, c) in coders.iter().enumerate() { + let (pw, ph) = plane_dims(rect.w, rect.h, p.colorspace, p.shift, pl, p.chroma_planes); + let (fw, _) = plane_dims(self.width as usize, self.height as usize, p.colorspace, p.shift, pl, p.chroma_planes); + let (ox, oy) = if p.chroma_planes && (pl == 1 || pl == 2) { (rect.x >> p.shift.0, rect.y >> p.shift.1) } else { (rect.x, rect.y) }; + for y in 0..ph { + for x in 0..pw { + frame.planes[pl][(oy + y) * fw + ox + x] = c.border.buf[c.border.at(x, y)] as u16; + } + } + } + return; + } + let swapped = rct_swapped(p); + let off = 1i32 << p.bits; + let fw = self.width as usize; + for y in 0..rect.h { + for x in 0..rect.w { + let get = |i: usize| coders[i].border.buf[coders[i].border.at(x, y)]; + let (yy, cb, cr) = (get(0), get(1) - off, get(2) - off); + let (r, g, b); + if swapped { + let bb = yy - ((cb + cr) >> 2); + r = cr + bb; + g = cb + bb; + b = bb; + } else { + let gg = yy - ((cb + cr) >> 2); + r = cr + gg; + g = gg; + b = cb + gg; + } + let at = (rect.y + y) * fw + rect.x + x; + frame.planes[0][at] = r as u16; + frame.planes[1][at] = g as u16; + frame.planes[2][at] = b as u16; + if coders.len() > 3 { + frame.planes[3][at] = get(3) as u16; + } + } + } + } +} + +/// Encoder settings. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct EncoderConfig { + /// Slices across the frame. + pub slices_h: u32, + /// Slices down the frame. + pub slices_v: u32, + /// Write a CRC in every slice (error detection). + pub slice_crc: bool, +} + +impl Default for EncoderConfig { + fn default() -> Self { + EncoderConfig { slices_h: 1, slices_v: 1, slice_crc: true } + } +} + +/// An encoder for frames of one fixed format. +pub struct Encoder { + params: Params, + width: u32, + height: u32, +} + +impl Encoder { + /// An encoder for frames shaped like `template` (its samples are not used). + pub fn new(template: &Frame, cfg: EncoderConfig) -> Result { + if !template.plane_count_ok() || template.width == 0 || template.height == 0 { + return Err(Error::Input("a frame has 1 to 4 planes and a nonzero size".into())); + } + if !(8..=16).contains(&template.bit_depth) { + return Err(Error::Input("bit depth must be 8 to 16".into())); + } + if cfg.slices_h == 0 + || cfg.slices_v == 0 + || cfg.slices_h > 1024 + || cfg.slices_v > 1024 + || cfg.slices_h > template.width + || cfg.slices_v > template.height + { + return Err(Error::Input("invalid slice raster".into())); + } + let n = template.planes.len(); + let (chroma_planes, extra_plane) = match (template.colorspace, n) { + (Colorspace::YCbCr, 1) => (false, false), + (Colorspace::YCbCr, 2) => (false, true), + (Colorspace::YCbCr, 3) => (true, false), + (Colorspace::YCbCr, _) => (true, true), + (Colorspace::Rgb, 3) => (true, false), + (Colorspace::Rgb, 4) => (true, true), + (Colorspace::Rgb, _) => return Err(Error::Input("RGB frames have 3 or 4 planes".into())), + }; + let shift = if template.colorspace == Colorspace::YCbCr && chroma_planes { template.chroma_shift } else { (0, 0) }; + if shift.0 > 3 || shift.1 > 3 { + return Err(Error::Input("chroma subsampling above 8:1".into())); + } + if template.colorspace == Colorspace::Rgb && template.bit_depth > 15 { + return Err(Error::Input("RGB above 15 bits needs 17-bit coded samples".into())); + } + let params = Params { + coder_type: 1, + transitions: Transitions::default_table(), + colorspace: template.colorspace, + bits: u32::from(template.bit_depth), + chroma_planes, + shift, + extra_plane, + num_h: cfg.slices_h, + num_v: cfg.slices_v, + sets: vec![default_quant_set()], + initial_states: vec![None], + ec: cfg.slice_crc, + intra: true, + }; + // Every slice boundary must be a multiple of the chroma subsampling, so the decoder accepts it. + for i in 0..=cfg.slices_h as usize { + if params.chroma_planes && (i * template.width as usize / cfg.slices_h as usize) & ((1 << shift.0) - 1) != 0 && i != cfg.slices_h as usize + { + return Err(Error::Input("slice boundaries must be multiples of the chroma subsampling".into())); + } + } + for i in 0..cfg.slices_v as usize { + if params.chroma_planes && (i * template.height as usize / cfg.slices_v as usize) & ((1 << shift.1) - 1) != 0 { + return Err(Error::Input("slice boundaries must be multiples of the chroma subsampling".into())); + } + } + Ok(Encoder { params, width: template.width, height: template.height }) + } + + /// The configuration record for the container (`CodecPrivate` and the like). + pub fn config_record(&self) -> Vec { + let mut enc = RacEnc::new(); + write_params(&mut enc, &self.params); + let mut out = enc.finish(); + let crc = crc32(&out); + out.extend_from_slice(&crc.to_be_bytes()); + out + } + + /// Encode one frame as a keyframe packet. + pub fn encode(&self, frame: &Frame) -> Result> { + let p = &self.params; + if frame.width != self.width || frame.height != self.height || frame.colorspace != p.colorspace || u32::from(frame.bit_depth) != p.bits { + return Err(Error::Input("the frame does not match the encoder's format".into())); + } + if frame.planes.len() != p.plane_count() { + return Err(Error::Input("the frame has the wrong number of planes".into())); + } + let shift = if p.chroma_planes { p.shift } else { (0, 0) }; + if p.colorspace == Colorspace::YCbCr && p.chroma_planes && frame.chroma_shift != shift { + return Err(Error::Input("the chroma subsampling does not match".into())); + } + for (pl, plane) in frame.planes.iter().enumerate() { + let (w, h) = frame.dims(pl, p.chroma_planes); + if plane.len() != w * h { + return Err(Error::Input(format!("plane {pl} has {} samples, expected {}", plane.len(), w * h))); + } + if plane.iter().any(|&v| u32::from(v) >> p.bits != 0) { + return Err(Error::Input(format!("plane {pl} has a sample above {} bits", p.bits))); + } + } + let mut out = Vec::new(); + for sy in 0..p.num_v as usize { + for sx in 0..p.num_h as usize { + let rect = slice_rect(p, self.width as usize, self.height as usize, sx, sy, 1, 1)?; + let mut slice = self.encode_slice(frame, rect, sx, sy, out.is_empty())?; + let size = slice.len(); + slice.extend_from_slice(&(size as u32).to_be_bytes()[1..]); + if p.ec { + slice.push(0); // error_status + let crc = crc32(&slice); + slice.extend_from_slice(&crc.to_be_bytes()); + } + out.extend_from_slice(&slice); + } + } + Ok(out) + } + + fn encode_slice(&self, frame: &Frame, rect: SliceRect, sx: usize, sy: usize, first: bool) -> Result> { + let p = &self.params; + let t = &p.transitions; + let t0 = t; + let mut rac = RacEnc::new(); + if first { + let mut ks = 128u8; + rac.bit(true, &mut ks, t0); + } + let mut hs = [128u8; CONTEXT_SIZE]; + for v in [sx as i32, sy as i32, 0, 0] { + rac.symbol(&mut hs, v, false, t0); + } + let set_idx = vec![0usize; set_index_count(p)]; + for &s in &set_idx { + rac.symbol(&mut hs, s as i32, false, t0); + } + for v in [0, 0, 0] { + rac.symbol(&mut hs, v, false, t0); // picture_structure, sar_num, sar_den + } + let (mut coders, mut classes) = make_coders(p, rect, &set_idx)?; + let bits = coded_bits(p); + let half = 1i32 << (bits - 1); + let mask = (1i32 << bits) - 1; + let sixteen = p.colorspace == Colorspace::YCbCr && p.bits == 16; + // The samples to code, one bordered plane's worth per coder. + let values = self.slice_values(frame, rect); + let write = |rac: &mut RacEnc, c: &mut PlaneCoder, cls: &mut ClassStates, x: usize, y: usize, sample: i32| { + let q = &p.sets[cls.set]; + let (ctx, pred) = c.border.context(x, y, &q.tables, sixteen); + let mut d = ((sample - pred + half) & mask) - half; + let (idx, neg) = if ctx < 0 { ((-ctx) as usize, true) } else { (ctx as usize, false) }; + if neg { + d = -d; + } + rac.symbol(&mut cls.states[idx.min(q.context_count - 1)], d, true, t); + let at = c.border.at(x, y); + c.border.buf[at] = sample; + }; + match p.colorspace { + Colorspace::YCbCr => { + for (pl, c) in coders.iter_mut().enumerate() { + let (pw, ph) = plane_dims(rect.w, rect.h, p.colorspace, p.shift, pl, p.chroma_planes); + for y in 0..ph { + c.border.start_row(y); + for x in 0..pw { + write(&mut rac, c, &mut classes[c.slot], x, y, values[pl][y * pw + x]); + } + c.border.end_row(y); + } + } + } + Colorspace::Rgb => { + for y in 0..rect.h { + for (pl, c) in coders.iter_mut().enumerate() { + c.border.start_row(y); + for x in 0..rect.w { + write(&mut rac, c, &mut classes[c.slot], x, y, values[pl][y * rect.w + x]); + } + c.border.end_row(y); + } + } + } + } + Ok(rac.finish()) + } + + /// The slice's samples per coded plane (after the colour transform for RGB). + fn slice_values(&self, frame: &Frame, rect: SliceRect) -> Vec> { + let p = &self.params; + let fw = self.width as usize; + if p.colorspace == Colorspace::YCbCr { + return (0..p.plane_count()) + .map(|pl| { + let (pw, ph) = plane_dims(rect.w, rect.h, p.colorspace, p.shift, pl, p.chroma_planes); + let (full_w, _) = plane_dims(fw, self.height as usize, p.colorspace, p.shift, pl, p.chroma_planes); + let (ox, oy) = + if p.chroma_planes && (pl == 1 || pl == 2) { (rect.x >> p.shift.0, rect.y >> p.shift.1) } else { (rect.x, rect.y) }; + let mut v = Vec::with_capacity(pw * ph); + for y in 0..ph { + for x in 0..pw { + v.push(i32::from(frame.planes[pl][(oy + y) * full_w + ox + x])); + } + } + v + }) + .collect(); + } + let swapped = rct_swapped(p); + let off = 1i32 << p.bits; + let mut planes = vec![Vec::with_capacity(rect.w * rect.h); p.plane_count()]; + for y in 0..rect.h { + for x in 0..rect.w { + let at = (rect.y + y) * fw + rect.x + x; + let (r, g, b) = (i32::from(frame.planes[0][at]), i32::from(frame.planes[1][at]), i32::from(frame.planes[2][at])); + let (yy, cb, cr) = if swapped { + let (cb, cr) = (g - b, r - b); + (b + ((cb + cr) >> 2), cb, cr) + } else { + let (cb, cr) = (b - g, r - g); + (g + ((cb + cr) >> 2), cb, cr) + }; + planes[0].push(yy); + planes[1].push(cb + off); + planes[2].push(cr + off); + if p.extra_plane { + planes[3].push(i32::from(frame.planes[3][at])); + } + } + } + planes + } +} diff --git a/crates/video/src/ffv1/rc.rs b/crates/video/src/ffv1/rc.rs new file mode 100644 index 000000000..fa2a16ee4 --- /dev/null +++ b/crates/video/src/ffv1/rc.rs @@ -0,0 +1,229 @@ +//! The FFV1 range coder (RFC 9043 section 3.8.1): binary decisions with adaptive 8-bit states, +//! and the signed/unsigned integer coding built on them. + +/// RFC 9043 Figure 24: the default state transition table for a coded 1. +const DEFAULT_ONE_STATE: [u8; 256] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, + 49, 50, 51, 52, 53, 54, 55, 56, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 75, 76, 77, 78, 79, 80, 81, 82, + 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + 114, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 133, 134, 135, 136, 137, 138, 139, 140, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 190, 191, 192, 194, 194, 195, + 196, 197, 198, 199, 200, 201, 202, 202, 204, 205, 206, 207, 208, 209, 209, 210, 211, 212, 213, 215, 215, 216, 217, 218, 219, 220, 220, 222, 223, + 224, 225, 226, 227, 227, 229, 229, 230, 231, 232, 234, 234, 235, 236, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 247, 248, 248, 0, 0, 0, + 0, 0, 0, 0, +]; + +/// The state machine: the next state after a coded 0 or 1. +#[derive(Clone)] +pub(crate) struct Transitions { + one: [u8; 256], + zero: [u8; 256], +} + +impl Transitions { + /// The default table plus `delta` (`state_transition_delta`, indexed 1 to 255). + pub(crate) fn new(delta: &[i32; 256]) -> Transitions { + let mut one = [0u8; 256]; + for (i, o) in one.iter_mut().enumerate() { + *o = (i32::from(DEFAULT_ONE_STATE[i]) + delta[i]) as u8; + } + let mut zero = [0u8; 256]; + for i in 1..256 { + zero[i] = (256 - i32::from(one[256 - i])) as u8; + } + Transitions { one, zero } + } + + pub(crate) fn default_table() -> Transitions { + Transitions::new(&[0; 256]) + } +} + +/// Reads binary decisions from `data`; bytes past its end read as 0 (the "Closed" mode of 3.8.1.1.1). +pub(crate) struct Decoder<'a> { + data: &'a [u8], + pos: usize, + low: u32, + range: u32, +} + +impl<'a> Decoder<'a> { + pub(crate) fn new(data: &'a [u8]) -> Decoder<'a> { + let b0 = u32::from(data.first().copied().unwrap_or(0)); + let b1 = u32::from(data.get(1).copied().unwrap_or(0)); + let mut low = b0 << 8 | b1; + let range = 0xFF00; + if low >= range { + low = range; + } + Decoder { data, pos: 2, low, range } + } + + /// Bytes the coder has read so far. + pub(crate) fn consumed(&self) -> usize { + self.pos + } + + fn refill(&mut self) { + if self.range < 256 { + self.range <<= 8; + self.low <<= 8; + self.low += u32::from(self.data.get(self.pos).copied().unwrap_or(0)); + self.pos += 1; + } + } + + pub(crate) fn bit(&mut self, state: &mut u8, t: &Transitions) -> bool { + let off = (self.range * u32::from(*state)) >> 8; + self.range -= off; + if self.low < self.range { + *state = t.zero[usize::from(*state)]; + self.refill(); + false + } else { + self.low -= self.range; + *state = t.one[usize::from(*state)]; + self.range = off; + self.refill(); + true + } + } + + /// RFC 9043 Figure 21: `get_symbol`. + pub(crate) fn symbol(&mut self, state: &mut [u8; 32], signed: bool, t: &Transitions) -> i32 { + if self.bit(&mut state[0], t) { + return 0; + } + let mut e = 0usize; + while self.bit(&mut state[1 + e.min(9)], t) { + e += 1; + if e > 24 { + // No valid stream codes a magnitude this large; stop instead of spinning on garbage. + return 0; + } + } + let mut a = 1i32; + for i in (0..e).rev() { + a = a * 2 + i32::from(self.bit(&mut state[22 + i.min(9)], t)); + } + if signed && self.bit(&mut state[11 + e.min(10)], t) { -a } else { a } + } +} + +/// Writes binary decisions; the inverse of [`Decoder`]. +pub(crate) struct Encoder { + out: Vec, + low: u32, + range: u32, +} + +impl Encoder { + pub(crate) fn new() -> Encoder { + Encoder { out: Vec::new(), low: 0, range: 0xFF00 } + } + + fn shift(&mut self) { + while self.range < 256 { + // Bits 15..8 of `low` are final once a carry (bit 16) is added to the bytes before them. + let carry = self.low >> 16; + let byte = ((self.low >> 8) & 0xFF) as u8; + if carry != 0 { + for b in self.out.iter_mut().rev() { + let (v, over) = b.overflowing_add(1); + *b = v; + if !over { + break; + } + } + } + self.out.push(byte); + self.low = (self.low & 0xFF) << 8; + self.range <<= 8; + } + } + + pub(crate) fn bit(&mut self, bit: bool, state: &mut u8, t: &Transitions) { + let off = (self.range * u32::from(*state)) >> 8; + if bit { + self.low += self.range - off; + self.range = off; + *state = t.one[usize::from(*state)]; + } else { + self.range -= off; + *state = t.zero[usize::from(*state)]; + } + self.shift(); + } + + pub(crate) fn symbol(&mut self, state: &mut [u8; 32], v: i32, signed: bool, t: &Transitions) { + if v == 0 { + self.bit(true, &mut state[0], t); + return; + } + self.bit(false, &mut state[0], t); + let a = v.unsigned_abs(); + let e = (31 - a.leading_zeros()) as usize; + for i in 0..e { + self.bit(true, &mut state[1 + i.min(9)], t); + } + self.bit(false, &mut state[1 + e.min(9)], t); + for i in (0..e).rev() { + self.bit((a >> i) & 1 == 1, &mut state[22 + i.min(9)], t); + } + if signed { + self.bit(v < 0, &mut state[11 + e.min(10)], t); + } + } + + /// Finish the stream with the two bytes of the *top* of the final range: whatever bytes follow + /// (zeros, or a CRC in a container's configuration record) still decode inside the range. + pub(crate) fn finish(mut self) -> Vec { + self.low += self.range - 1; + let carry = self.low >> 16; + if carry != 0 { + for b in self.out.iter_mut().rev() { + let (v, over) = b.overflowing_add(1); + *b = v; + if !over { + break; + } + } + } + self.out.push(((self.low >> 8) & 0xFF) as u8); + self.out.push((self.low & 0xFF) as u8); + self.out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn symbols_round_trip_through_the_range_coder() { + let t = Transitions::default_table(); + let values: Vec = (0..4000).map(|i| ((i * 7919) % 2001) - 1000).chain([0, 0, 0, 65535, -65535, 1, -1]).collect(); + let mut enc = Encoder::new(); + let mut st = [128u8; 32]; + for &v in &values { + enc.symbol(&mut st, v, true, &t); + } + let bytes = enc.finish(); + let mut dec = Decoder::new(&bytes); + let mut st = [128u8; 32]; + for &v in &values { + assert_eq!(dec.symbol(&mut st, true, &t), v); + } + } + + #[test] + fn transition_tables_follow_the_rfc() { + let t = Transitions::default_table(); + assert_eq!(t.one[8], 20); + assert_eq!(t.one[247], 248); + // zero_state_i = 256 - one_state_(256 - i) + assert_eq!(t.zero[128], (256 - i32::from(t.one[128])) as u8); + assert_eq!(t.zero[1], (256 - i32::from(t.one[255])) as u8); + } +} diff --git a/crates/video/src/lib.rs b/crates/video/src/lib.rs index 0d8625bbd..8202a93f8 100644 --- a/crates/video/src/lib.rs +++ b/crates/video/src/lib.rs @@ -10,6 +10,7 @@ //! | `vp8` | [`vp8`] | VP8 (RFC 6386): decoder (`vp8-decode`) and key-frame encoder (`vp8-encode`) | //! | `mpeg2` | [`mpeg2`] | MPEG-2 video (ITU-T H.262) and MPEG-1 video: decoder (`mpeg2-decode`) and MPEG-2 I/P/B encoder (`mpeg2-encode`) | //! | `mpeg4` | [`mpeg4`] | MPEG-4 Part 2 video (ISO/IEC 14496-2, Xvid/DivX): Simple profile encoder (`mpeg4-encode`) | +//! | `ffv1` | [`ffv1`] | FFV1 lossless video (RFC 9043): version 3, range coder, decoder and encoder | //! | `prores` | [`prores`] | Apple ProRes (SMPTE RDD 36): decoder and encoder | //! | `h264` | [`h264`] | ITU-T H.264 / AVC: decoder (`h264-decode`) and encoder (`h264-encode`) | //! | `hevc` | [`hevc`] | ITU-T H.265 / HEVC: decoder (`hevc-decode`) and encoder (`hevc-encode`) | @@ -41,6 +42,9 @@ pub mod mpeg2; #[cfg(feature = "mpeg4-encode")] pub mod mpeg4; +#[cfg(feature = "ffv1")] +pub mod ffv1; + #[cfg(feature = "prores")] pub mod prores; diff --git a/crates/video/tests/ffv1.rs b/crates/video/tests/ffv1.rs new file mode 100644 index 000000000..1d5c73c0b --- /dev/null +++ b/crates/video/tests/ffv1.rs @@ -0,0 +1,304 @@ +//! FFV1 tests: round trips, hostile input, and ffmpeg as the reference (oracle; skipped when ffmpeg +//! is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). +//! +//! - Our packets, in an AVI with our configuration record, decode in ffmpeg to our source pixels. +//! - ffmpeg's FFV1 files (version 3, range coder, several pixel formats and slice counts) decode +//! with our decoder to ffmpeg's pixels. + +use craft_media::avi::{AviWriter, MuxOptions, StreamSpec, VideoSpec}; +use craft_video::ffv1::{Colorspace, Decoder, Encoder, EncoderConfig, Error, Frame}; +use std::path::Path; +use std::process::Command; + +fn noise(seed: &mut u32, mask: u32) -> u16 { + *seed ^= *seed << 13; + *seed ^= *seed >> 17; + *seed ^= *seed << 5; + (*seed & mask) as u16 +} + +/// A frame of smooth gradients, noise bursts and flat areas. +fn frame(w: u32, h: u32, cs: Colorspace, bits: u8, shift: (u8, u8), planes: usize) -> Frame { + let max = (1u32 << bits) - 1; + let mut seed = 0x9E37_79B9u32; + let mut ps = Vec::new(); + for p in 0..planes { + let (pw, ph) = if cs == Colorspace::YCbCr && (p == 1 || p == 2) && planes >= 3 { + ((w as usize).div_ceil(1 << shift.0), (h as usize).div_ceil(1 << shift.1)) + } else { + (w as usize, h as usize) + }; + let mut v = Vec::with_capacity(pw * ph); + for y in 0..ph { + for x in 0..pw { + let s = match (y / 11 + p) % 3 { + 0 => ((x * 5 + y * 3 + p * 40) as u32 * max / 600) % (max + 1), + 1 => noise(&mut seed, max) as u32, + _ => max / 3, + }; + v.push(s as u16); + } + } + ps.push(v); + } + Frame { width: w, height: h, colorspace: cs, bit_depth: bits, chroma_shift: shift, planes: ps } +} + +fn round_trip(f: &Frame, cfg: EncoderConfig) -> Frame { + let enc = Encoder::new(f, cfg).expect("encoder"); + let config = enc.config_record(); + let packet = enc.encode(f).expect("encode"); + let dec = Decoder::new(&config, f.width, f.height).expect("decoder"); + dec.decode(&packet).expect("decode") +} + +#[test] +fn round_trips_every_shape() { + for (cs, bits, shift, planes) in [ + (Colorspace::YCbCr, 8u8, (0u8, 0u8), 1usize), + (Colorspace::YCbCr, 8, (1, 1), 3), + (Colorspace::YCbCr, 8, (1, 0), 3), + (Colorspace::YCbCr, 8, (0, 0), 4), + (Colorspace::YCbCr, 10, (1, 1), 3), + (Colorspace::YCbCr, 12, (0, 0), 3), + (Colorspace::YCbCr, 16, (1, 0), 3), + (Colorspace::YCbCr, 16, (0, 0), 1), + (Colorspace::Rgb, 8, (0, 0), 3), + (Colorspace::Rgb, 8, (0, 0), 4), + (Colorspace::Rgb, 10, (0, 0), 3), + (Colorspace::Rgb, 12, (0, 0), 4), + (Colorspace::Rgb, 15, (0, 0), 3), + ] { + for (w, h) in [(1, 1), (3, 5), (64, 48), (67, 41)] { + let f = frame(w, h, cs, bits, shift, planes); + for crc in [true, false] { + let back = round_trip(&f, EncoderConfig { slice_crc: crc, ..EncoderConfig::default() }); + assert_eq!(back.planes, f.planes, "{cs:?} {bits}-bit {shift:?} {planes} planes {w}x{h} crc={crc}"); + } + } + } +} + +#[test] +fn round_trips_with_several_slices() { + let f = frame(96, 64, Colorspace::YCbCr, 8, (1, 1), 3); + for (sh, sv) in [(2, 1), (1, 2), (3, 2), (4, 4)] { + let back = round_trip(&f, EncoderConfig { slices_h: sh, slices_v: sv, slice_crc: true }); + assert_eq!(back.planes, f.planes, "{sh}x{sv} slices"); + } + let rgb = frame(50, 30, Colorspace::Rgb, 8, (0, 0), 3); + assert_eq!(round_trip(&rgb, EncoderConfig { slices_h: 3, slices_v: 2, slice_crc: true }).planes, rgb.planes); +} + +#[test] +fn encoder_refuses_what_it_cannot_write() { + let f = frame(8, 8, Colorspace::YCbCr, 8, (1, 1), 3); + assert!(Encoder::new(&f, EncoderConfig { slices_h: 0, ..EncoderConfig::default() }).is_err()); + assert!(Encoder::new(&f, EncoderConfig { slices_h: 9, ..EncoderConfig::default() }).is_err()); + let mut bad = f.clone(); + bad.bit_depth = 7; + assert!(Encoder::new(&bad, EncoderConfig::default()).is_err()); + let mut rgb16 = frame(8, 8, Colorspace::Rgb, 8, (0, 0), 3); + rgb16.bit_depth = 16; + assert!(Encoder::new(&rgb16, EncoderConfig::default()).is_err()); + let enc = Encoder::new(&f, EncoderConfig::default()).expect("encoder"); + let mut wrong = f.clone(); + wrong.planes[1].pop(); + assert!(enc.encode(&wrong).is_err()); + let mut over = f; + over.planes[0][0] = 300; + assert!(enc.encode(&over).is_err()); +} + +#[test] +fn hostile_input_is_an_error_not_a_panic() { + let f = frame(24, 18, Colorspace::YCbCr, 8, (1, 1), 3); + let enc = Encoder::new(&f, EncoderConfig { slices_h: 2, slices_v: 1, slice_crc: true }).expect("encoder"); + let config = enc.config_record(); + let packet = enc.encode(&f).expect("encode"); + // Every truncation of the configuration record and of the packet. + for n in 0..config.len() { + let _ = Decoder::new(&config[..n], 24, 18); + } + let dec = Decoder::new(&config, 24, 18).expect("decoder"); + for n in 0..packet.len() { + let _ = dec.decode(&packet[..n]); + } + // Every single-bit flip: the CRCs catch them in a packet, and none panics. + for i in 0..packet.len() { + for bit in 0..8 { + let mut c = packet.clone(); + c[i] ^= 1 << bit; + assert!(dec.decode(&c).is_err(), "bit {bit} of byte {i} went unnoticed"); + } + } + for i in 0..config.len() { + for bit in 0..8 { + let mut c = config.clone(); + c[i] ^= 1 << bit; + assert!(matches!(Decoder::new(&c, 24, 18), Err(Error::Crc(_))), "config bit {bit} of byte {i}"); + } + } + // Without slice CRCs, flips may decode to wrong pixels but must not panic. + let enc = Encoder::new(&f, EncoderConfig { slice_crc: false, ..EncoderConfig::default() }).expect("encoder"); + let (config, packet) = (enc.config_record(), enc.encode(&f).expect("encode")); + let dec = Decoder::new(&config, 24, 18).expect("decoder"); + for i in 0..packet.len() { + for bit in 0..8 { + let mut c = packet.clone(); + c[i] ^= 1 << bit; + let _ = dec.decode(&c); + } + } + // A huge frame size is refused before any allocation. + assert!(Decoder::new(&config, u32::MAX, u32::MAX).is_err()); +} + +// ---- ffmpeg ---- + +fn ffmpeg_run(ff: &Path, args: &[&str]) -> Vec { + let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); + out.stdout +} + +fn raw_bytes(f: &Frame) -> Vec { + let mut out = Vec::new(); + for p in &f.planes { + for &v in p { + if f.bit_depth == 8 { + out.push(v as u8); + } else { + out.extend_from_slice(&v.to_le_bytes()); + } + } + } + out +} + +/// `(ffmpeg pix_fmt, colourspace, bits, chroma shift, planes)`; RGB planes are G, B, R in ffmpeg's gbrp. +type Format = (&'static str, Colorspace, u8, (u8, u8), usize); + +const FORMATS: &[Format] = &[ + ("gray", Colorspace::YCbCr, 8, (0, 0), 1), + ("yuv420p", Colorspace::YCbCr, 8, (1, 1), 3), + ("yuv422p", Colorspace::YCbCr, 8, (1, 0), 3), + ("yuv444p", Colorspace::YCbCr, 8, (0, 0), 3), + ("yuv420p10le", Colorspace::YCbCr, 10, (1, 1), 3), + ("yuv444p12le", Colorspace::YCbCr, 12, (0, 0), 3), + ("gray16le", Colorspace::YCbCr, 16, (0, 0), 1), + ("yuv444p16le", Colorspace::YCbCr, 16, (0, 0), 3), + ("gbrp", Colorspace::Rgb, 8, (0, 0), 3), + ("gbrp10le", Colorspace::Rgb, 10, (0, 0), 3), + ("gbrp12le", Colorspace::Rgb, 12, (0, 0), 3), + ("yuva444p", Colorspace::YCbCr, 8, (0, 0), 4), + ("gbrap", Colorspace::Rgb, 8, (0, 0), 4), + ("gbrap12le", Colorspace::Rgb, 12, (0, 0), 4), +]; + +/// Formats only ffmpeg writes: our encoder stops at 15-bit RGB (the transform needs 17 bits). +const DECODE_ONLY: &[Format] = &[("gbrp16le", Colorspace::Rgb, 16, (0, 0), 3), ("yuv411p", Colorspace::YCbCr, 8, (2, 0), 3)]; + +/// ffmpeg's planar RGB is G, B, R (then alpha); ours is R, G, B (then alpha). +fn gbr_order(planes: &[Vec]) -> Vec> { + let mut v = vec![planes[1].clone(), planes[2].clone(), planes[0].clone()]; + v.extend(planes[3..].iter().cloned()); + v +} + +fn avi_with(frame: &Frame, config: &[u8], packet: &[u8]) -> Vec { + let mut spec = VideoSpec::new(*b"FFV1", frame.width, frame.height, 25, 1); + spec.format.extra = config.to_vec(); + let mut w = AviWriter::new(std::io::Cursor::new(Vec::new()), vec![StreamSpec::Video(spec)], MuxOptions::default()).expect("writer"); + w.write_chunk(0, packet, true).expect("chunk"); + w.finish().expect("finish").into_inner() +} + +#[test] +fn ffmpeg_decodes_our_packets_to_our_pixels() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("ffv1"); + for &(pix_fmt, cs, bits, shift, planes) in FORMATS { + for (slices_h, slices_v) in [(1, 1), (2, 2), (3, 2)] { + let f = frame(96, 48, cs, bits, shift, planes); + let enc = Encoder::new(&f, EncoderConfig { slices_h, slices_v, slice_crc: true }).expect("encoder"); + let avi = avi_with(&f, &enc.config_record(), &enc.encode(&f).expect("encode")); + let path = dir.join(format!("ours_{pix_fmt}_{slices_h}x{slices_v}.avi")); + std::fs::write(&path, avi).expect("write"); + let theirs = ffmpeg_run(&ff, &["-i", path.to_str().expect("path"), "-f", "rawvideo", "-pix_fmt", pix_fmt, "-"]); + // gbrp is planar G, B, R; our RGB planes are R, G, B. + let mut want = f.clone(); + if cs == Colorspace::Rgb { + want.planes = gbr_order(&f.planes); + } + assert!(theirs == raw_bytes(&want), "{pix_fmt} {slices_h}x{slices_v}: ffmpeg's pixels differ from ours"); + } + } +} + +#[test] +fn our_decoder_reads_ffmpegs_files() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("ffv1"); + for &(pix_fmt, cs, bits, shift, planes) in FORMATS.iter().chain(DECODE_ONLY) { + for slices in ["1", "4", "6"] { + for (ctx, extra) in [("0", false), ("1", true)] { + let path = dir.join(format!("ffmpeg_{pix_fmt}_s{slices}_c{ctx}.avi")); + let lavfi = "testsrc2=size=96x48:rate=1"; + ffmpeg_run( + &ff, + &[ + "-f", + "lavfi", + "-i", + lavfi, + "-frames:v", + "1", + "-vf", + &format!("format={pix_fmt}"), + "-c:v", + "ffv1", + "-level", + "3", + "-coder", + "1", + "-context", + ctx, + "-slices", + slices, + "-slicecrc", + "1", + "-g", + "1", + path.to_str().expect("path"), + ], + ); + let _ = extra; + let file = std::fs::read(&path).expect("read"); + let avi = craft_media::avi::open(&file[..]).expect("open avi"); + let stream = &avi.streams[0]; + let config = stream.video.as_ref().expect("video").extra.clone(); + let chunk = stream.chunks[0]; + let packet = file[chunk.offset as usize..chunk.offset as usize + chunk.size as usize].to_vec(); + let dec = Decoder::new(&config, 96, 48).unwrap_or_else(|e| panic!("{pix_fmt} s{slices} c{ctx}: config: {e}")); + let got = dec.decode(&packet).unwrap_or_else(|e| panic!("{pix_fmt} s{slices} c{ctx}: decode: {e}")); + let theirs = ffmpeg_run(&ff, &["-i", path.to_str().expect("path"), "-f", "rawvideo", "-pix_fmt", pix_fmt, "-"]); + let mut want = got.clone(); + if cs == Colorspace::Rgb { + want.planes = gbr_order(&got.planes); + } + assert_eq!((got.colorspace, got.bit_depth, got.chroma_shift, got.planes.len()), (cs, bits, shift, planes), "{pix_fmt}"); + if theirs != raw_bytes(&want) { + let mut off = 0; + for (i, p) in want.planes.iter().enumerate() { + let n = p.len() * if bits == 8 { 1 } else { 2 }; + let same = theirs[off..off + n] == raw_bytes(&Frame { planes: vec![p.clone()], ..want.clone() })[..]; + eprintln!("{pix_fmt} s{slices} c{ctx}: plane {i} {}", if same { "equal" } else { "DIFFERENT" }); + off += n; + } + panic!("{pix_fmt} s{slices} c{ctx}: our pixels differ from ffmpeg's"); + } + } + } + } +} From 6ac9046bd457f09cdb5eb7b229e50584877b8f68 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:10:04 +0000 Subject: [PATCH 40/98] Share streaming TextureCraft tile payloads --- Cargo.lock | 8 + Cargo.toml | 1 + crates/tcraft/Cargo.toml | 15 ++ crates/tcraft/README.md | 14 ++ crates/tcraft/src/lib.rs | 342 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 380 insertions(+) create mode 100644 crates/tcraft/Cargo.toml create mode 100644 crates/tcraft/README.md create mode 100644 crates/tcraft/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index d984ac115..b0f05a3b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2686,6 +2686,14 @@ dependencies = [ "kurbo 0.13.1", ] +[[package]] +name = "craft-tcraft" +version = "0.1.0" +dependencies = [ + "craft-zip", + "thiserror 2.0.21", +] + [[package]] name = "craft-text" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20da79..d60c3469b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,6 +142,7 @@ craft-stencilcraft-document = { path = "crates/stencilcraft-document" } craft-svg-paths = { path = "crates/svg-paths" } craft-stream-codecs = { path = "crates/stream-codecs" } craft-text = { path = "crates/text" } +craft-tcraft = { path = "crates/tcraft" } craft-texture = { path = "crates/texture" } craft-tga = { path = "crates/tga" } craft-timeline = { path = "crates/timeline" } diff --git a/crates/tcraft/Cargo.toml b/crates/tcraft/Cargo.toml new file mode 100644 index 000000000..76eeeafb0 --- /dev/null +++ b/crates/tcraft/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-tcraft" +description = "Streaming binary payloads of TextureCraft project archives" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-zip = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/tcraft/README.md b/crates/tcraft/README.md new file mode 100644 index 000000000..e5831ef98 --- /dev/null +++ b/crates/tcraft/README.md @@ -0,0 +1,14 @@ +# craft-tcraft + +Streaming neutral `TCTX` binary payloads used inside TextureCraft `.tcraft` projects. Sparse +paint data stores 64×64 tiles of little-endian `u16` components; baked maps store dense +little-endian `f32` values. This crate does not yet define the project JSON schema or ZIP +envelope and does not depend on TextureCraft's app types. + +`read_tiles` validates the expected dimensions and components, bounds tile count, rejects +out-of-range and repeated tile positions, and delivers one owned tile at a time. `read_baked` +validates dimensions and declared length and decodes in bounded chunks. Both consume the same +caller-owned `craft_zip::Budget` used for the surrounding project and reject trailing data. +`write_tiles` and `write_baked` stream their bodies rather than copying an entire image; a caller +must discard an incomplete output if a write fails. Limits and ownership rules beyond this +binary format remain with the project reader. diff --git a/crates/tcraft/src/lib.rs b/crates/tcraft/src/lib.rs new file mode 100644 index 000000000..50022bb1e --- /dev/null +++ b/crates/tcraft/src/lib.rs @@ -0,0 +1,342 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] +//! Streaming `TCTX` tile and baked-map payloads in TextureCraft `.tcraft` archives. +//! +//! This codec handles the neutral binary payload, not the application's project model or ZIP +//! envelope. Reads use a caller-owned budget shared with other archive entries. No reader opens +//! a file or reserves memory based on a claimed payload before checking it. + +use std::collections::HashSet; +use std::io::{self, Read, Write}; + +use craft_zip::Budget; + +pub const TILE_SIDE: u32 = 64; +pub const TILE_AREA: usize = 64 * 64; +pub const HEADER_BYTES: u64 = 24; +pub const VERSION: u32 = 1; +const MAGIC: &[u8; 4] = b"TCTX"; +const CHUNK: usize = 1 << 16; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("invalid TCTX payload: {0}")] + Invalid(String), + #[error("TCTX version {0} is newer than supported")] + NewerVersion(u32), + #[error("TCTX limit: {0}")] + Limit(String), + #[error("TCTX I/O: {0}")] + Io(#[from] io::Error), +} + +/// Width, height and number of components expected by the enclosing project schema. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + pub width: u32, + pub height: u32, + pub components: usize, +} + +#[derive(Clone, Copy, Debug)] +struct Header { + layout: Layout, + count: u32, +} + +fn charge(budget: &mut Budget, n: u64) -> Result<(), Error> { + budget.charge(n).map_err(|e| Error::Limit(e.to_string())) +} + +fn read_header(reader: &mut impl Read, budget: &mut Budget) -> Result { + charge(budget, HEADER_BYTES)?; + let mut bytes = [0u8; HEADER_BYTES as usize]; + reader.read_exact(&mut bytes)?; + if bytes.get(..4) != Some(MAGIC.as_slice()) { + return Err(Error::Invalid("bad magic".into())); + } + let word = |i: usize| bytes.get(i * 4..i * 4 + 4).map(|b| u32::from_le_bytes([b[0], b[1], b[2], b[3]])).unwrap_or(0); + let version = word(1); + if version > VERSION { + return Err(Error::NewerVersion(version)); + } + Ok(Header { layout: Layout { width: word(2), height: word(3), components: word(4) as usize }, count: word(5) }) +} + +fn write_header(writer: &mut impl Write, layout: Layout, count: u32) -> Result<(), Error> { + let components = u32::try_from(layout.components).map_err(|_| Error::Limit("too many components".into()))?; + writer.write_all(MAGIC)?; + for value in [VERSION, layout.width, layout.height, components, count] { + writer.write_all(&value.to_le_bytes())?; + } + Ok(()) +} + +fn expect_end(reader: &mut impl Read) -> Result<(), Error> { + let mut byte = [0u8; 1]; + if reader.read(&mut byte)? != 0 { + return Err(Error::Invalid("trailing data".into())); + } + Ok(()) +} + +fn tile_dimensions(layout: Layout) -> Result<(u32, u32, usize), Error> { + if layout.width == 0 || layout.height == 0 || layout.components == 0 { + return Err(Error::Invalid("empty tile layout".into())); + } + let x = layout.width.div_ceil(TILE_SIDE); + let y = layout.height.div_ceil(TILE_SIDE); + let bytes = TILE_AREA.checked_mul(layout.components).and_then(|n| n.checked_mul(2)).ok_or_else(|| Error::Limit("tile is too large".into()))?; + Ok((x, y, bytes)) +} + +/// Decode tiles one at a time, transferring each tile to the callback. +/// Positions and duplicate tiles are checked before delivery; the entry must end exactly. +pub fn read_tiles( + reader: &mut impl Read, + expected: Layout, + budget: &mut Budget, + mut tile: impl FnMut(u32, u32, Vec) -> Result<(), Error>, +) -> Result<(), Error> { + let header = read_header(reader, budget)?; + if header.layout != expected { + return Err(Error::Invalid("tile layout differs from the project".into())); + } + let (tiles_x, tiles_y, bytes_per_tile) = tile_dimensions(expected)?; + if u64::from(header.count) > u64::from(tiles_x) * u64::from(tiles_y) { + return Err(Error::Invalid("more tiles than the image holds".into())); + } + let mut bytes = Vec::new(); + let mut seen = HashSet::new(); + for _ in 0..header.count { + let payload = u64::try_from(bytes_per_tile).map_err(|_| Error::Limit("tile is too large".into()))?.saturating_add(8); + charge(budget, payload)?; + if bytes.is_empty() { + bytes.try_reserve_exact(bytes_per_tile).map_err(|_| Error::Limit("tile allocation failed".into()))?; + bytes.resize(bytes_per_tile, 0); + } + let mut pos = [0u8; 8]; + reader.read_exact(&mut pos)?; + let x = u32::from_le_bytes([pos[0], pos[1], pos[2], pos[3]]); + let y = u32::from_le_bytes([pos[4], pos[5], pos[6], pos[7]]); + if x >= tiles_x || y >= tiles_y || seen.contains(&(x, y)) { + return Err(Error::Invalid(format!("bad or repeated tile ({x}, {y})"))); + } + seen.try_reserve(1).map_err(|_| Error::Limit("tile index allocation failed".into()))?; + seen.insert((x, y)); + reader.read_exact(&mut bytes)?; + let mut values = Vec::new(); + values.try_reserve_exact(bytes_per_tile / 2).map_err(|_| Error::Limit("tile allocation failed".into()))?; + values.extend(bytes.as_chunks::<2>().0.iter().map(|b| u16::from_le_bytes(*b))); + tile(x, y, values)?; + } + expect_end(reader) +} + +/// Write tiles in caller order, without buffering the image. A failure may leave a partial +/// entry; callers writing a project should discard the incomplete output. +pub fn write_tiles<'a>( + writer: &mut impl Write, + layout: Layout, + count: u32, + tiles: impl IntoIterator, +) -> Result<(), Error> { + let (tiles_x, tiles_y, bytes_per_tile) = tile_dimensions(layout)?; + if u64::from(count) > u64::from(tiles_x) * u64::from(tiles_y) { + return Err(Error::Invalid("more tiles than the image holds".into())); + } + write_header(writer, layout, count)?; + let mut seen = HashSet::new(); + let mut bytes = Vec::new(); + let mut written = 0u32; + for (x, y, values) in tiles { + if written >= count || x >= tiles_x || y >= tiles_y || seen.contains(&(x, y)) || values.len() != bytes_per_tile / 2 { + return Err(Error::Invalid("bad, repeated or incorrectly sized tile".into())); + } + seen.try_reserve(1).map_err(|_| Error::Limit("tile index allocation failed".into()))?; + seen.insert((x, y)); + if bytes.is_empty() { + bytes.try_reserve_exact(bytes_per_tile).map_err(|_| Error::Limit("tile allocation failed".into()))?; + } + bytes.clear(); + bytes.extend(values.iter().flat_map(|v| v.to_le_bytes())); + writer.write_all(&x.to_le_bytes())?; + writer.write_all(&y.to_le_bytes())?; + writer.write_all(&bytes)?; + written += 1; + } + if written != count { + return Err(Error::Invalid("tile count differs from its data".into())); + } + Ok(()) +} + +/// Read a dense f32 map, validating declared size and charging its shared budget first. +pub fn read_baked( + reader: &mut impl Read, + declared_bytes: u64, + components: usize, + max_texels: u64, + budget: &mut Budget, +) -> Result<(Layout, Vec), Error> { + let header = read_header(reader, budget)?; + let layout = header.layout; + let texels = u64::from(layout.width) * u64::from(layout.height); + if texels == 0 || texels > max_texels || layout.components != components || components == 0 { + return Err(Error::Invalid("bad baked-map dimensions or components".into())); + } + let count = texels.checked_mul(components as u64).ok_or_else(|| Error::Limit("too many map values".into()))?; + let payload = count.checked_mul(4).ok_or_else(|| Error::Limit("map is too large".into()))?; + if declared_bytes != HEADER_BYTES.checked_add(payload).ok_or_else(|| Error::Limit("map is too large".into()))? { + return Err(Error::Invalid("declared map size differs from its dimensions".into())); + } + charge(budget, payload)?; + let count = usize::try_from(count).map_err(|_| Error::Limit("too many map values".into()))?; + let mut values = Vec::new(); + let chunk_bytes = CHUNK.min(count) * 4; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(chunk_bytes).map_err(|_| Error::Limit("map chunk allocation failed".into()))?; + bytes.resize(chunk_bytes, 0); + while values.len() < count { + let n = CHUNK.min(count - values.len()); + let chunk = bytes.get_mut(..n * 4).ok_or_else(|| Error::Invalid("bad map chunk".into()))?; + reader.read_exact(chunk)?; + values.try_reserve(n).map_err(|_| Error::Limit("map allocation failed".into()))?; + values.extend(chunk.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b))); + } + expect_end(reader)?; + Ok((layout, values)) +} + +/// Write a dense f32 map in bounded chunks without constructing a second copy of its payload. +pub fn write_baked(writer: &mut impl Write, layout: Layout, values: &[f32]) -> Result<(), Error> { + let count = u64::from(layout.width).checked_mul(u64::from(layout.height)).and_then(|n| n.checked_mul(layout.components as u64)); + if layout.width == 0 || layout.height == 0 || layout.components == 0 || count != Some(values.len() as u64) { + return Err(Error::Invalid("map dimensions differ from its data".into())); + } + write_header(writer, layout, 0)?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(CHUNK * 4).map_err(|_| Error::Limit("map chunk allocation failed".into()))?; + for chunk in values.chunks(CHUNK) { + bytes.clear(); + bytes.extend(chunk.iter().flat_map(|v| v.to_le_bytes())); + writer.write_all(&bytes)?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn header(layout: Layout, count: u32) -> Vec { + let mut bytes = Vec::new(); + write_header(&mut bytes, layout, count).unwrap(); + bytes + } + + #[test] + fn baked_map_matches_the_native_little_endian_wire_layout() { + let layout = Layout { width: 2, height: 1, components: 1 }; + let mut expected = b"TCTX".to_vec(); + for value in [1u32, 2, 1, 1, 0] { + expected.extend_from_slice(&value.to_le_bytes()); + } + expected.extend_from_slice(&1.0f32.to_le_bytes()); + expected.extend_from_slice(&(-0.5f32).to_le_bytes()); + let mut actual = Vec::new(); + write_baked(&mut actual, layout, &[1.0, -0.5]).unwrap(); + assert_eq!(actual, expected); + let (decoded, values) = read_baked(&mut Cursor::new(&expected), expected.len() as u64, 1, 2, &mut Budget::new("project", 32)).unwrap(); + assert_eq!(decoded, layout); + assert_eq!(values, [1.0, -0.5]); + expected[4..8].copy_from_slice(&2u32.to_le_bytes()); + assert!(matches!(read_baked(&mut Cursor::new(&expected), 32, 1, 2, &mut Budget::new("project", 32)), Err(Error::NewerVersion(2)))); + } + + #[test] + fn dense_maps_round_trip_bit_exactly_in_bounded_chunks() { + let layout = Layout { width: 300, height: 300, components: 1 }; + let mut values = vec![0.25; 90_000]; + values[123] = f32::from_bits(0x7fc0_1234); + let mut bytes = Vec::new(); + write_baked(&mut bytes, layout, &values).unwrap(); + let mut budget = Budget::new("project", bytes.len() as u64); + let (decoded_layout, decoded) = read_baked(&mut Cursor::new(&bytes), bytes.len() as u64, 1, 100_000, &mut budget).unwrap(); + assert_eq!(decoded_layout, layout); + assert!(decoded.iter().zip(values.iter()).all(|(a, b)| a.to_bits() == b.to_bits())); + assert_eq!(budget.remaining(), 0); + } + + #[test] + fn dense_maps_refuse_false_lengths_limits_and_trailing_data() { + let layout = Layout { width: 2, height: 2, components: 1 }; + let mut bytes = Vec::new(); + write_baked(&mut bytes, layout, &[1.0; 4]).unwrap(); + let mut budget = Budget::new("project", 100); + assert!(read_baked(&mut Cursor::new(&bytes), 999, 1, 4, &mut budget).is_err()); + let mut budget = Budget::new("project", 39); + assert!(matches!(read_baked(&mut Cursor::new(&bytes), 40, 1, 4, &mut budget), Err(Error::Limit(_)))); + let mut extra = bytes.clone(); + extra.push(0); + let mut budget = Budget::new("project", 100); + assert!(read_baked(&mut Cursor::new(extra), 40, 1, 4, &mut budget).is_err()); + } + + #[test] + fn hostile_headers_and_tiles_are_rejected_without_trusting_counts() { + let layout = Layout { width: 64, height: 64, components: 1 }; + let mut bad_magic = header(layout, 0); + bad_magic[0] = b'X'; + assert!(read_tiles(&mut Cursor::new(bad_magic), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(header(layout, 2)), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + let other = Layout { width: 65, ..layout }; + assert!(read_tiles(&mut Cursor::new(header(layout, 0)), other, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + let mut outside = header(layout, 1); + outside.extend_from_slice(&1u32.to_le_bytes()); + outside.extend_from_slice(&0u32.to_le_bytes()); + assert!(read_tiles(&mut Cursor::new(outside), layout, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); + let mut truncated = header(layout, 1); + truncated.extend_from_slice(&[0; 8]); + assert!(read_tiles(&mut Cursor::new(truncated), layout, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); + let mut extra = header(layout, 0); + extra.push(1); + assert!(read_tiles(&mut Cursor::new(extra), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + } + + #[test] + fn sparse_tiles_stream_and_reject_duplicates_and_short_budget() { + let layout = Layout { width: 128, height: 64, components: 1 }; + let mut bytes = header(layout, 2); + for x in [0u32, 1] { + bytes.extend_from_slice(&x.to_le_bytes()); + bytes.extend_from_slice(&0u32.to_le_bytes()); + for _ in 0..TILE_AREA { + bytes.extend_from_slice(&(x as u16 + 1).to_le_bytes()); + } + } + let first = vec![1u16; TILE_AREA]; + let second = vec![2u16; TILE_AREA]; + let mut written = Vec::new(); + write_tiles(&mut written, layout, 2, [(0, 0, first.as_slice()), (1, 0, second.as_slice())]).unwrap(); + assert_eq!(written, bytes, "the streamed writer keeps the native wire layout"); + assert!(write_tiles(&mut Vec::new(), layout, 2, [(0, 0, first.as_slice())]).is_err()); + let mut seen = Vec::new(); + let mut budget = Budget::new("project", bytes.len() as u64); + read_tiles(&mut Cursor::new(&bytes), layout, &mut budget, |x, y, tile| { + seen.push((x, y, tile.first().copied())); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, [(0, 0, Some(1)), (1, 0, Some(2))]); + let mut duplicate = bytes.clone(); + let second = HEADER_BYTES as usize + 8 + TILE_AREA * 2; + duplicate[second..second + 4].copy_from_slice(&0u32.to_le_bytes()); + assert!(read_tiles(&mut Cursor::new(duplicate), layout, &mut Budget::new("project", u64::MAX), |_, _, _| Ok(())).is_err()); + assert!(matches!( + read_tiles(&mut Cursor::new(bytes), layout, &mut Budget::new("project", HEADER_BYTES), |_, _, _| Ok(())), + Err(Error::Limit(_)) + )); + } +} From 5f46c0ab25f067f6fdb863c7e1f9ff567712f394 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:23:23 +0000 Subject: [PATCH 41/98] Preserve project budget and streaming IO errors --- crates/tcraft/src/lib.rs | 51 ++++++++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/crates/tcraft/src/lib.rs b/crates/tcraft/src/lib.rs index 50022bb1e..4646d8717 100644 --- a/crates/tcraft/src/lib.rs +++ b/crates/tcraft/src/lib.rs @@ -24,6 +24,8 @@ pub enum Error { Invalid(String), #[error("TCTX version {0} is newer than supported")] NewerVersion(u32), + #[error("TCTX budget: {0}")] + Budget(#[from] craft_zip::Exceeded), #[error("TCTX limit: {0}")] Limit(String), #[error("TCTX I/O: {0}")] @@ -45,10 +47,10 @@ struct Header { } fn charge(budget: &mut Budget, n: u64) -> Result<(), Error> { - budget.charge(n).map_err(|e| Error::Limit(e.to_string())) + budget.charge(n).map_err(Error::from) } -fn read_header(reader: &mut impl Read, budget: &mut Budget) -> Result { +fn read_header(reader: &mut (impl Read + ?Sized), budget: &mut Budget) -> Result { charge(budget, HEADER_BYTES)?; let mut bytes = [0u8; HEADER_BYTES as usize]; reader.read_exact(&mut bytes)?; @@ -63,7 +65,7 @@ fn read_header(reader: &mut impl Read, budget: &mut Budget) -> Result Result<(), Error> { +fn write_header(writer: &mut (impl Write + ?Sized), layout: Layout, count: u32) -> Result<(), Error> { let components = u32::try_from(layout.components).map_err(|_| Error::Limit("too many components".into()))?; writer.write_all(MAGIC)?; for value in [VERSION, layout.width, layout.height, components, count] { @@ -72,7 +74,7 @@ fn write_header(writer: &mut impl Write, layout: Layout, count: u32) -> Result<( Ok(()) } -fn expect_end(reader: &mut impl Read) -> Result<(), Error> { +fn expect_end(reader: &mut (impl Read + ?Sized)) -> Result<(), Error> { let mut byte = [0u8; 1]; if reader.read(&mut byte)? != 0 { return Err(Error::Invalid("trailing data".into())); @@ -93,12 +95,17 @@ fn tile_dimensions(layout: Layout) -> Result<(u32, u32, usize), Error> { /// Decode tiles one at a time, transferring each tile to the callback. /// Positions and duplicate tiles are checked before delivery; the entry must end exactly. pub fn read_tiles( - reader: &mut impl Read, + reader: &mut (impl Read + ?Sized), expected: Layout, + max_texels: u64, budget: &mut Budget, mut tile: impl FnMut(u32, u32, Vec) -> Result<(), Error>, ) -> Result<(), Error> { let header = read_header(reader, budget)?; + let texels = u64::from(expected.width) * u64::from(expected.height); + if texels == 0 || texels > max_texels { + return Err(Error::Limit("tile image exceeds its texel limit".into())); + } if header.layout != expected { return Err(Error::Invalid("tile layout differs from the project".into())); } @@ -136,7 +143,7 @@ pub fn read_tiles( /// Write tiles in caller order, without buffering the image. A failure may leave a partial /// entry; callers writing a project should discard the incomplete output. pub fn write_tiles<'a>( - writer: &mut impl Write, + writer: &mut (impl Write + ?Sized), layout: Layout, count: u32, tiles: impl IntoIterator, @@ -173,7 +180,7 @@ pub fn write_tiles<'a>( /// Read a dense f32 map, validating declared size and charging its shared budget first. pub fn read_baked( - reader: &mut impl Read, + reader: &mut (impl Read + ?Sized), declared_bytes: u64, components: usize, max_texels: u64, @@ -188,7 +195,7 @@ pub fn read_baked( let count = texels.checked_mul(components as u64).ok_or_else(|| Error::Limit("too many map values".into()))?; let payload = count.checked_mul(4).ok_or_else(|| Error::Limit("map is too large".into()))?; if declared_bytes != HEADER_BYTES.checked_add(payload).ok_or_else(|| Error::Limit("map is too large".into()))? { - return Err(Error::Invalid("declared map size differs from its dimensions".into())); + return Err(Error::Invalid(format!("{declared_bytes} bytes for a {}×{}×{} map", layout.width, layout.height, layout.components))); } charge(budget, payload)?; let count = usize::try_from(count).map_err(|_| Error::Limit("too many map values".into()))?; @@ -209,7 +216,7 @@ pub fn read_baked( } /// Write a dense f32 map in bounded chunks without constructing a second copy of its payload. -pub fn write_baked(writer: &mut impl Write, layout: Layout, values: &[f32]) -> Result<(), Error> { +pub fn write_baked(writer: &mut (impl Write + ?Sized), layout: Layout, values: &[f32]) -> Result<(), Error> { let count = u64::from(layout.width).checked_mul(u64::from(layout.height)).and_then(|n| n.checked_mul(layout.components as u64)); if layout.width == 0 || layout.height == 0 || layout.components == 0 || count != Some(values.len() as u64) { return Err(Error::Invalid("map dimensions differ from its data".into())); @@ -277,7 +284,7 @@ mod tests { let mut budget = Budget::new("project", 100); assert!(read_baked(&mut Cursor::new(&bytes), 999, 1, 4, &mut budget).is_err()); let mut budget = Budget::new("project", 39); - assert!(matches!(read_baked(&mut Cursor::new(&bytes), 40, 1, 4, &mut budget), Err(Error::Limit(_)))); + assert!(matches!(read_baked(&mut Cursor::new(&bytes), 40, 1, 4, &mut budget), Err(Error::Budget(_)))); let mut extra = bytes.clone(); extra.push(0); let mut budget = Budget::new("project", 100); @@ -287,22 +294,26 @@ mod tests { #[test] fn hostile_headers_and_tiles_are_rejected_without_trusting_counts() { let layout = Layout { width: 64, height: 64, components: 1 }; + assert!(matches!( + read_tiles(&mut Cursor::new(header(layout, 0)), layout, 4095, &mut Budget::new("project", 100), |_, _, _| Ok(())), + Err(Error::Limit(_)) + )); let mut bad_magic = header(layout, 0); bad_magic[0] = b'X'; - assert!(read_tiles(&mut Cursor::new(bad_magic), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); - assert!(read_tiles(&mut Cursor::new(header(layout, 2)), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(bad_magic), layout, u64::MAX, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(header(layout, 2)), layout, u64::MAX, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); let other = Layout { width: 65, ..layout }; - assert!(read_tiles(&mut Cursor::new(header(layout, 0)), other, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(header(layout, 0)), other, u64::MAX, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); let mut outside = header(layout, 1); outside.extend_from_slice(&1u32.to_le_bytes()); outside.extend_from_slice(&0u32.to_le_bytes()); - assert!(read_tiles(&mut Cursor::new(outside), layout, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(outside), layout, u64::MAX, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); let mut truncated = header(layout, 1); truncated.extend_from_slice(&[0; 8]); - assert!(read_tiles(&mut Cursor::new(truncated), layout, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(truncated), layout, u64::MAX, &mut Budget::new("project", 10_000), |_, _, _| Ok(())).is_err()); let mut extra = header(layout, 0); extra.push(1); - assert!(read_tiles(&mut Cursor::new(extra), layout, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(extra), layout, u64::MAX, &mut Budget::new("project", 100), |_, _, _| Ok(())).is_err()); } #[test] @@ -324,7 +335,7 @@ mod tests { assert!(write_tiles(&mut Vec::new(), layout, 2, [(0, 0, first.as_slice())]).is_err()); let mut seen = Vec::new(); let mut budget = Budget::new("project", bytes.len() as u64); - read_tiles(&mut Cursor::new(&bytes), layout, &mut budget, |x, y, tile| { + read_tiles(&mut Cursor::new(&bytes), layout, u64::MAX, &mut budget, |x, y, tile| { seen.push((x, y, tile.first().copied())); Ok(()) }) @@ -333,10 +344,10 @@ mod tests { let mut duplicate = bytes.clone(); let second = HEADER_BYTES as usize + 8 + TILE_AREA * 2; duplicate[second..second + 4].copy_from_slice(&0u32.to_le_bytes()); - assert!(read_tiles(&mut Cursor::new(duplicate), layout, &mut Budget::new("project", u64::MAX), |_, _, _| Ok(())).is_err()); + assert!(read_tiles(&mut Cursor::new(duplicate), layout, u64::MAX, &mut Budget::new("project", u64::MAX), |_, _, _| Ok(())).is_err()); assert!(matches!( - read_tiles(&mut Cursor::new(bytes), layout, &mut Budget::new("project", HEADER_BYTES), |_, _, _| Ok(())), - Err(Error::Limit(_)) + read_tiles(&mut Cursor::new(bytes), layout, u64::MAX, &mut Budget::new("project", HEADER_BYTES), |_, _, _| Ok(())), + Err(Error::Budget(_)) )); } } From e2e2ce2c889f0fe777e09161c5a0f1c8582246b3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:34:42 +0000 Subject: [PATCH 42/98] Retain IJG and SIMD notices for shared JPEG encoding --- ATTRIBUTION.md | 1 + NOTICE | 123 ++++++++++++ crates/image-codecs/NOTICE | 122 ++++++++++++ crates/image-codecs/README.md | 8 + licenses/jpeg-encoder-0.6.1/IJG-NOTICE.txt | 72 +++++++ licenses/jpeg-encoder-0.6.1/LICENSE-APACHE | 201 ++++++++++++++++++++ licenses/jpeg-encoder-0.6.1/LICENSE-MIT | 25 +++ licenses/jpeg-encoder-0.6.1/SIMD-NOTICE.txt | 8 + licenses/jpeg-encoder-0.6.1/SOURCE.json | 29 +++ 9 files changed, 589 insertions(+) create mode 100644 crates/image-codecs/NOTICE create mode 100644 licenses/jpeg-encoder-0.6.1/IJG-NOTICE.txt create mode 100644 licenses/jpeg-encoder-0.6.1/LICENSE-APACHE create mode 100644 licenses/jpeg-encoder-0.6.1/LICENSE-MIT create mode 100644 licenses/jpeg-encoder-0.6.1/SIMD-NOTICE.txt create mode 100644 licenses/jpeg-encoder-0.6.1/SOURCE.json diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 77d3578cb..cc808c328 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -6,6 +6,7 @@ author, source and license. Library code and original assets are under the repos | File | Author | Source | License | |---|---|---|---| +| `licenses/jpeg-encoder-0.6.1/`, `crates/image-codecs/NOTICE` | jpeg-encoder contributors and the IJG, libjpeg-turbo and SIMD authors named in the retained notices | [jpeg-encoder 0.6.1](https://crates.io/crates/jpeg-encoder/0.6.1), used unmodified; source checksums in `SOURCE.json` | (MIT OR Apache-2.0) AND IJG | | `crates/smime/testdata/` | MailCraft contributors; RFC 9216 example certificates | [MailCraft at 240d1d42](https://github.com/UCS-Care-Group/mailcraft/tree/240d1d42fe394cf3274402b180663ba9b43ca180/crates/smime/testdata), restored unmodified; provenance in the directory README | MIT OR Apache-2.0 (source repository) | | `licenses/dwg/acadrust-MPL-2.0.txt`, `licenses/dwg/acadrust-THIRD_PARTY_NOTICES.md` | `acadrust` contributors; ACadSharp authors for the noted ported routines | [acadrust 0.6.6](https://github.com/hakanaktt/acadrust), used unmodified as `craft-dwg`'s optional native dependency | MPL-2.0; embedded ACadSharp notice is MIT; see both retained files | | `crates/cms/profiles/photocraft-coated-cmyk.icc` | PhotoCraft contributors; generated by `craft_cms::synth::coated_cmyk` | [PhotoCraft `crates/cms`](https://github.com/storytold/photocraft/tree/b3222ee8e7e6bcb4799c37f904c595f20f9baf5e/crates/cms) | CC0-1.0 | diff --git a/NOTICE b/NOTICE index 4fa5e4754..c3cdbabc8 100644 --- a/NOTICE +++ b/NOTICE @@ -296,3 +296,126 @@ implementation, https://github.com/macosforge/alac at c38887c5c5e64a4b31108733bd (LICENSE-APACHE). The files are modified: translated to Rust, with bounds checks on hostile input, wrapping integer arithmetic, the unrolled predictor and per-width matrix routines folded into one, and the average bit rate filled in. Those files are Apache-2.0 only. + +JPEG encoding dependency: jpeg-encoder 0.6.1 + +Source: https://crates.io/crates/jpeg-encoder/0.6.1 +Declared license: (MIT OR Apache-2.0) AND IJG. +This software is based in part on the work of the Independent JPEG Group. + +The MIT license and unaltered upstream IJG and SIMD notices follow. The Apache-2.0 +alternative is also retained in licenses/jpeg-encoder-0.6.1 at the repository root. +The dependency is used unmodified; no encoder implementation is vendored here. + +LICENSE-MIT + +Copyright (c) 2021 Volker Ströbel + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +Original src/fdct.rs opening notice + +/* + * Ported from mozjpeg to rust + * + * This file was part of the Independent JPEG Group's software: + * Copyright (C) 1991-1996, Thomas G. Lane. + * libjpeg-turbo Modifications: + * Copyright (C) 2015, 2020, D. R. Commander. + * + * Conditions of distribution and use: + * In plain English: + * + * 1. We don't promise that this software works. (But if you find any bugs, + * please let us know!) + * 2. You can use this software for whatever you want. You don't have to pay us. + * 3. You may not pretend that you wrote this software. If you use it in a + * program, you must acknowledge somewhere in your documentation that + * you've used the IJG code. + * + * In legalese: + * + * The authors make NO WARRANTY or representation, either express or implied, + * with respect to this software, its quality, accuracy, merchantability, or + * fitness for a particular purpose. This software is provided "AS IS", and you, + * its user, assume the entire risk as to its quality and accuracy. + * + * This software is copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding. + * All Rights Reserved except as specified below. + * + * Permission is hereby granted to use, copy, modify, and distribute this + * software (or portions thereof) for any purpose, without fee, subject to these + * conditions: + * (1) If any part of the source code for this software is distributed, then this + * README file must be included, with this copyright and no-warranty notice + * unaltered; and any additions, deletions, or changes to the original files + * must be clearly indicated in accompanying documentation. + * (2) If only executable code is distributed, then the accompanying + * documentation must state that "this software is based in part on the work of + * the Independent JPEG Group". + * (3) Permission for use of this software is granted only if the user accepts + * full responsibility for any undesirable consequences; the authors accept + * NO LIABILITY for damages of any kind. + * + * These conditions apply to any software derived from or based on the IJG code, + * not just to the unmodified library. If you use our work, you ought to + * acknowledge us. + * + * Permission is NOT granted for the use of any IJG author's name or company name + * in advertising or publicity relating to this software or products derived from + * it. This software may be referred to only as "the Independent JPEG Group's + * software". + * + * We specifically permit and encourage the use of this software as the basis of + * commercial products, provided that all warranty or liability claims are + * assumed by the product vendor. + * + * This file contains a slower but more accurate integer implementation of the + * forward DCT (Discrete Cosine Transform). + * + * A 2-D DCT can be done by 1-D DCT on each row followed by 1-D DCT + * on each column. Direct algorithms are also available, but they are + * much more complex and seem not to be any faster when reduced to code. + * + * This implementation is based on an algorithm described in + * C. Loeffler, A. Ligtenberg and G. Moschytz, "Practical Fast 1-D DCT + * Algorithms with 11 Multiplications", Proc. Int'l. Conf. on Acoustics, + * Speech, and Signal Processing 1989 (ICASSP '89), pp. 988-991. + * The primary algorithm described there uses 11 multiplies and 29 adds. + * We use their alternate method with 12 multiplies and 32 adds. + * The advantage of this method is that no data path contains more than one + * multiplication; this allows a very simple and accurate implementation in + * scaled fixed-point arithmetic, with a minimal number of shifts. + */ + +Original src/avx2/fdct.rs opening notice + +/* + * Ported from mozjpeg / jfdctint-avx2.asm to rust + * Copyright 2009 Pierre Ossman for Cendio AB + * Copyright (C) 2009, 2016, 2018, 2020, D. R. Commander. + * + * Based on the x86 SIMD extension for IJG JPEG library + * Copyright (C) 1999-2006, MIYASAKA Masaru. + */ diff --git a/crates/image-codecs/NOTICE b/crates/image-codecs/NOTICE new file mode 100644 index 000000000..83fde3dba --- /dev/null +++ b/crates/image-codecs/NOTICE @@ -0,0 +1,122 @@ +JPEG encoding dependency: jpeg-encoder 0.6.1 + +Source: https://crates.io/crates/jpeg-encoder/0.6.1 +Declared license: (MIT OR Apache-2.0) AND IJG. +This software is based in part on the work of the Independent JPEG Group. + +The MIT license and unaltered upstream IJG and SIMD notices follow. The Apache-2.0 +alternative is also retained in licenses/jpeg-encoder-0.6.1 at the repository root. +The dependency is used unmodified; no encoder implementation is vendored here. + +LICENSE-MIT + +Copyright (c) 2021 Volker Ströbel + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +Original src/fdct.rs opening notice + +/* + * Ported from mozjpeg to rust + * + * This file was part of the Independent JPEG Group's software: + * Copyright (C) 1991-1996, Thomas G. Lane. + * libjpeg-turbo Modifications: + * Copyright (C) 2015, 2020, D. R. Commander. + * + * Conditions of distribution and use: + * In plain English: + * + * 1. We don't promise that this software works. (But if you find any bugs, + * please let us know!) + * 2. You can use this software for whatever you want. You don't have to pay us. + * 3. You may not pretend that you wrote this software. If you use it in a + * program, you must acknowledge somewhere in your documentation that + * you've used the IJG code. + * + * In legalese: + * + * The authors make NO WARRANTY or representation, either express or implied, + * with respect to this software, its quality, accuracy, merchantability, or + * fitness for a particular purpose. This software is provided "AS IS", and you, + * its user, assume the entire risk as to its quality and accuracy. + * + * This software is copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding. + * All Rights Reserved except as specified below. + * + * Permission is hereby granted to use, copy, modify, and distribute this + * software (or portions thereof) for any purpose, without fee, subject to these + * conditions: + * (1) If any part of the source code for this software is distributed, then this + * README file must be included, with this copyright and no-warranty notice + * unaltered; and any additions, deletions, or changes to the original files + * must be clearly indicated in accompanying documentation. + * (2) If only executable code is distributed, then the accompanying + * documentation must state that "this software is based in part on the work of + * the Independent JPEG Group". + * (3) Permission for use of this software is granted only if the user accepts + * full responsibility for any undesirable consequences; the authors accept + * NO LIABILITY for damages of any kind. + * + * These conditions apply to any software derived from or based on the IJG code, + * not just to the unmodified library. If you use our work, you ought to + * acknowledge us. + * + * Permission is NOT granted for the use of any IJG author's name or company name + * in advertising or publicity relating to this software or products derived from + * it. This software may be referred to only as "the Independent JPEG Group's + * software". + * + * We specifically permit and encourage the use of this software as the basis of + * commercial products, provided that all warranty or liability claims are + * assumed by the product vendor. + * + * This file contains a slower but more accurate integer implementation of the + * forward DCT (Discrete Cosine Transform). + * + * A 2-D DCT can be done by 1-D DCT on each row followed by 1-D DCT + * on each column. Direct algorithms are also available, but they are + * much more complex and seem not to be any faster when reduced to code. + * + * This implementation is based on an algorithm described in + * C. Loeffler, A. Ligtenberg and G. Moschytz, "Practical Fast 1-D DCT + * Algorithms with 11 Multiplications", Proc. Int'l. Conf. on Acoustics, + * Speech, and Signal Processing 1989 (ICASSP '89), pp. 988-991. + * The primary algorithm described there uses 11 multiplies and 29 adds. + * We use their alternate method with 12 multiplies and 32 adds. + * The advantage of this method is that no data path contains more than one + * multiplication; this allows a very simple and accurate implementation in + * scaled fixed-point arithmetic, with a minimal number of shifts. + */ + +Original src/avx2/fdct.rs opening notice + +/* + * Ported from mozjpeg / jfdctint-avx2.asm to rust + * Copyright 2009 Pierre Ossman for Cendio AB + * Copyright (C) 2009, 2016, 2018, 2020, D. R. Commander. + * + * Based on the x86 SIMD extension for IJG JPEG library + * Copyright (C) 1999-2006, MIYASAKA Masaru. + */ diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 6e9e11617..27bb3c6ba 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -339,3 +339,11 @@ folder, copy the five files from that commit into `corpus/exr/`, or run at commit `74330157cb601573ae4523a95bb9132822fc3918` and copy its `corpus/exr/` here, as CI does. The images are not committed here; OpenEXR distributes them under its [license](https://github.com/AcademySoftwareFoundation/openexr/blob/69b2604fc76e370615438bdc8d2cd95b9349c12e/LICENSE.md). + +## Redistribution notices + +JPEG encoding uses `jpeg-encoder` 0.6.1, including work from the Independent JPEG Group. +Retain this crate's [NOTICE](NOTICE) with binary distributions using that encoder. It includes +its MIT license, the IJG acknowledgement and original IJG/SIMD notices. Both upstream license +alternatives and source checksums are retained in +[`licenses/jpeg-encoder-0.6.1`](../../licenses/jpeg-encoder-0.6.1) at the repository root. diff --git a/licenses/jpeg-encoder-0.6.1/IJG-NOTICE.txt b/licenses/jpeg-encoder-0.6.1/IJG-NOTICE.txt new file mode 100644 index 000000000..4fcb9b583 --- /dev/null +++ b/licenses/jpeg-encoder-0.6.1/IJG-NOTICE.txt @@ -0,0 +1,72 @@ +/* + * Ported from mozjpeg to rust + * + * This file was part of the Independent JPEG Group's software: + * Copyright (C) 1991-1996, Thomas G. Lane. + * libjpeg-turbo Modifications: + * Copyright (C) 2015, 2020, D. R. Commander. + * + * Conditions of distribution and use: + * In plain English: + * + * 1. We don't promise that this software works. (But if you find any bugs, + * please let us know!) + * 2. You can use this software for whatever you want. You don't have to pay us. + * 3. You may not pretend that you wrote this software. If you use it in a + * program, you must acknowledge somewhere in your documentation that + * you've used the IJG code. + * + * In legalese: + * + * The authors make NO WARRANTY or representation, either express or implied, + * with respect to this software, its quality, accuracy, merchantability, or + * fitness for a particular purpose. This software is provided "AS IS", and you, + * its user, assume the entire risk as to its quality and accuracy. + * + * This software is copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding. + * All Rights Reserved except as specified below. + * + * Permission is hereby granted to use, copy, modify, and distribute this + * software (or portions thereof) for any purpose, without fee, subject to these + * conditions: + * (1) If any part of the source code for this software is distributed, then this + * README file must be included, with this copyright and no-warranty notice + * unaltered; and any additions, deletions, or changes to the original files + * must be clearly indicated in accompanying documentation. + * (2) If only executable code is distributed, then the accompanying + * documentation must state that "this software is based in part on the work of + * the Independent JPEG Group". + * (3) Permission for use of this software is granted only if the user accepts + * full responsibility for any undesirable consequences; the authors accept + * NO LIABILITY for damages of any kind. + * + * These conditions apply to any software derived from or based on the IJG code, + * not just to the unmodified library. If you use our work, you ought to + * acknowledge us. + * + * Permission is NOT granted for the use of any IJG author's name or company name + * in advertising or publicity relating to this software or products derived from + * it. This software may be referred to only as "the Independent JPEG Group's + * software". + * + * We specifically permit and encourage the use of this software as the basis of + * commercial products, provided that all warranty or liability claims are + * assumed by the product vendor. + * + * This file contains a slower but more accurate integer implementation of the + * forward DCT (Discrete Cosine Transform). + * + * A 2-D DCT can be done by 1-D DCT on each row followed by 1-D DCT + * on each column. Direct algorithms are also available, but they are + * much more complex and seem not to be any faster when reduced to code. + * + * This implementation is based on an algorithm described in + * C. Loeffler, A. Ligtenberg and G. Moschytz, "Practical Fast 1-D DCT + * Algorithms with 11 Multiplications", Proc. Int'l. Conf. on Acoustics, + * Speech, and Signal Processing 1989 (ICASSP '89), pp. 988-991. + * The primary algorithm described there uses 11 multiplies and 29 adds. + * We use their alternate method with 12 multiplies and 32 adds. + * The advantage of this method is that no data path contains more than one + * multiplication; this allows a very simple and accurate implementation in + * scaled fixed-point arithmetic, with a minimal number of shifts. + */ diff --git a/licenses/jpeg-encoder-0.6.1/LICENSE-APACHE b/licenses/jpeg-encoder-0.6.1/LICENSE-APACHE new file mode 100644 index 000000000..dae696981 --- /dev/null +++ b/licenses/jpeg-encoder-0.6.1/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. \ No newline at end of file diff --git a/licenses/jpeg-encoder-0.6.1/LICENSE-MIT b/licenses/jpeg-encoder-0.6.1/LICENSE-MIT new file mode 100644 index 000000000..6a4eb8746 --- /dev/null +++ b/licenses/jpeg-encoder-0.6.1/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2021 Volker Ströbel + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/licenses/jpeg-encoder-0.6.1/SIMD-NOTICE.txt b/licenses/jpeg-encoder-0.6.1/SIMD-NOTICE.txt new file mode 100644 index 000000000..58e09bf43 --- /dev/null +++ b/licenses/jpeg-encoder-0.6.1/SIMD-NOTICE.txt @@ -0,0 +1,8 @@ +/* + * Ported from mozjpeg / jfdctint-avx2.asm to rust + * Copyright 2009 Pierre Ossman for Cendio AB + * Copyright (C) 2009, 2016, 2018, 2020, D. R. Commander. + * + * Based on the x86 SIMD extension for IJG JPEG library + * Copyright (C) 1999-2006, MIYASAKA Masaru. + */ diff --git a/licenses/jpeg-encoder-0.6.1/SOURCE.json b/licenses/jpeg-encoder-0.6.1/SOURCE.json new file mode 100644 index 000000000..8a7fb7ab2 --- /dev/null +++ b/licenses/jpeg-encoder-0.6.1/SOURCE.json @@ -0,0 +1,29 @@ +{ + "package": "jpeg-encoder", + "version": "0.6.1", + "source": "https://crates.io/crates/jpeg-encoder/0.6.1", + "registry_checksum": "b454d911ac55068f53495488d8ccd0646eaa540c033a28ee15b07838afafb01f", + "license": "(MIT OR Apache-2.0) AND IJG", + "files": { + "LICENSE-MIT": { + "source": "LICENSE-MIT", + "source_sha256": "4d022727ea392ebba2fe5e9a5f78c610a0eae506b491f3db29c78b6d430d6503", + "sha256": "4d022727ea392ebba2fe5e9a5f78c610a0eae506b491f3db29c78b6d430d6503" + }, + "LICENSE-APACHE": { + "source": "LICENSE-APACHE", + "source_sha256": "85a884980abd6032fc6b5439ba918ab16db9ac9051b5cf18db9c27c587df07c0", + "sha256": "85a884980abd6032fc6b5439ba918ab16db9ac9051b5cf18db9c27c587df07c0" + }, + "IJG-NOTICE.txt": { + "source": "src/fdct.rs", + "source_sha256": "0b59244bbd05b2593ded252ecf0d010a8a4c6ec1d4713b3caeb6fb724586e4d6", + "sha256": "432424ca69caaa82565c0491c11cdab311a2e7d39180970b8ba22017996bf678" + }, + "SIMD-NOTICE.txt": { + "source": "src/avx2/fdct.rs", + "source_sha256": "bc362322f568f92c3699f4ca2922b261c12b7dd997120b01fd3cab58d9d2f9ab", + "sha256": "ee36eeaa9df1f7f09c7046f428ce4a848fcf518394e63d3e2125ec261949e8ba" + } + } +} From b85e4b150b7e427b1fd3c88b4ed9e6aba055053c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:34:47 +0000 Subject: [PATCH 43/98] Add neutral project document schema --- Cargo.lock | 2 ++ crates/tcraft/Cargo.toml | 2 ++ crates/tcraft/src/lib.rs | 50 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index b0f05a3b0..761575903 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2691,6 +2691,8 @@ name = "craft-tcraft" version = "0.1.0" dependencies = [ "craft-zip", + "serde", + "serde_json", "thiserror 2.0.21", ] diff --git a/crates/tcraft/Cargo.toml b/crates/tcraft/Cargo.toml index 76eeeafb0..3ce35abfb 100644 --- a/crates/tcraft/Cargo.toml +++ b/crates/tcraft/Cargo.toml @@ -12,4 +12,6 @@ workspace = true [dependencies] craft-zip = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/tcraft/src/lib.rs b/crates/tcraft/src/lib.rs index 4646d8717..730ef3b0c 100644 --- a/crates/tcraft/src/lib.rs +++ b/crates/tcraft/src/lib.rs @@ -6,10 +6,12 @@ //! envelope. Reads use a caller-owned budget shared with other archive entries. No reader opens //! a file or reserves memory based on a claimed payload before checking it. -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::io::{self, Read, Write}; use craft_zip::Budget; +use serde::{Deserialize, Serialize}; +use serde_json::Value; pub const TILE_SIDE: u32 = 64; pub const TILE_AREA: usize = 64 * 64; @@ -32,6 +34,34 @@ pub enum Error { Io(#[from] io::Error), } +/// Neutral top-level project schema. Nested app-domain values and unknown fields remain JSON +/// so they round-trip without requiring TextureCraft's in-memory model. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ProjectDocument { + pub format_version: u32, + pub name: String, + pub mesh: String, + pub normal_format: Value, + pub texture_sets: Vec, + pub layers: BTreeMap, + pub next_id: u64, + pub display: Value, + #[serde(default)] + pub cameras: Vec, + #[serde(flatten)] + pub extra: BTreeMap, +} + +/// Read only the version before an application applies its own recovery policy. Serde's +/// structured parse rejects a duplicate or invalid version field, like the project reader. +pub fn project_version(bytes: &[u8]) -> Result { + #[derive(Deserialize)] + struct Version { + format_version: u32, + } + Ok(serde_json::from_slice::(bytes)?.format_version) +} + /// Width, height and number of components expected by the enclosing project schema. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Layout { @@ -243,6 +273,24 @@ mod tests { bytes } + #[test] + fn project_schema_keeps_nested_and_future_fields() { + let json = serde_json::json!({ + "format_version": 2, "name": "Stone", "mesh": "mesh.obj", "normal_format": "openGl", + "texture_sets": [{"name": "main", "futureSet": {"keep": true}}], + "layers": {"1": {"kind": "paint", "futureLayer": [1, 2]}}, "next_id": 2, + "display": {"futureDisplay": 4}, "mesh_file": "source.obj", "futureProject": [3] + }); + let bytes = serde_json::to_vec(&json).unwrap(); + assert_eq!(project_version(&bytes).unwrap(), 2); + let document: ProjectDocument = serde_json::from_slice(&bytes).unwrap(); + let mut expected = json; + expected["cameras"] = serde_json::json!([]); + assert_eq!(serde_json::to_value(document).unwrap(), expected); + assert!(project_version(br#"{"format_version":1,"format_version":2}"#).is_err()); + assert!(project_version(br#"{"format_version":-1}"#).is_err()); + } + #[test] fn baked_map_matches_the_native_little_endian_wire_layout() { let layout = Layout { width: 2, height: 1, components: 1 }; From 2af057527956387e30a2916b1219b7e75014690d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Oct 2026 22:49:07 -0400 Subject: [PATCH 44/98] psd: import annotation reader primitives from PhotoCraft --- crates/psd/src/annotations.rs | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 crates/psd/src/annotations.rs diff --git a/crates/psd/src/annotations.rs b/crates/psd/src/annotations.rs new file mode 100644 index 000000000..c982d3338 --- /dev/null +++ b/crates/psd/src/annotations.rs @@ -0,0 +1,43 @@ +// Reader primitives extracted from PhotoCraft annotations_map.rs at aa87012a. +struct Reader<'a> { + d: &'a [u8], + p: usize, +} + +impl<'a> Reader<'a> { + fn take(&mut self, n: usize) -> Option<&'a [u8]> { + let s = self.d.get(self.p..self.p.checked_add(n)?)?; + self.p += n; + Some(s) + } + fn u8(&mut self) -> Option { + self.take(1).map(|b| b[0]) + } + fn u16(&mut self) -> Option { + self.take(2).map(|b| u16::from_be_bytes([b[0], b[1]])) + } + fn u32(&mut self) -> Option { + self.take(4).map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + } + fn i32(&mut self) -> Option { + self.u32().map(|v| v as i32) + } + fn pascal(&mut self) -> Option { + let n = usize::from(self.u8()?); + let s = self.take(n)?; + if (n + 1) % 2 == 1 { + self.take(1)?; + } + Some(s.iter().map(|&c| c as char).collect()) + } +} + +fn pascal(out: &mut Vec, s: &str) { + let bytes: Vec = s.chars().map(|c| if (c as u32) < 256 { c as u32 as u8 } else { b'?' }).take(255).collect(); + out.push(bytes.len() as u8); + out.extend_from_slice(&bytes); + if (bytes.len() + 1) % 2 == 1 { + out.push(0); + } +} + From a26ee0616b24fcf7e6d009f1921dc0113694f8e8 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 07:43:18 -0500 Subject: [PATCH 45/98] mime: offer explicit optional Base64 terminal padding --- crates/mime/src/base64.rs | 25 +++++++++++++++----- crates/mime/src/base64_tests.rs | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/crates/mime/src/base64.rs b/crates/mime/src/base64.rs index a93a6fdfb..ebe077372 100644 --- a/crates/mime/src/base64.rs +++ b/crates/mime/src/base64.rs @@ -8,8 +8,9 @@ //! counts it; an unpadded one refuses `=`), a length that can be, and the unused bits of the last //! symbol zero (so every value has one spelling). A caller that reads mail, where line breaks and //! missing padding are everyday, cleans its input first, as it always has; the strictness is what -//! the signature, key and credential checks want. No input panics, and the output is never longer -//! than three quarters of the input. +//! the signature, key and credential checks want. [`Config::decode_optional_padding`] explicitly +//! permits missing terminal padding without relaxing alphabet, padding placement or unused bits. +//! No input panics, and the output is never longer than three quarters of the input. use std::fmt; @@ -108,9 +109,20 @@ impl Config { /// The bytes of `text`, which must be exactly base64 of this config's kind (see the module). pub fn decode(self, text: impl AsRef<[u8]>) -> Result, DecodeError> { - let text = text.as_ref(); - let (symbols, pad) = if self.padded { - if text.len() % 4 != 0 { + self.decode_inner(text.as_ref(), false) + } + + /// Decode this alphabet with zero, partial or complete terminal padding, regardless of the + /// config's encoding padding. Whitespace, junk, internal/excess padding and nonzero unused + /// bits still fail. Use this only when the calling format permits missing padding; [`Self::decode`] + /// remains strict for signatures and credentials that require one spelling. + pub fn decode_optional_padding(self, text: impl AsRef<[u8]>) -> Result, DecodeError> { + self.decode_inner(text.as_ref(), true) + } + + fn decode_inner(self, text: &[u8], optional_padding: bool) -> Result, DecodeError> { + let (symbols, pad) = if self.padded || optional_padding { + if !optional_padding && text.len() % 4 != 0 { return Err(DecodeError); } let pad = text.iter().rev().take(2).take_while(|c| **c == b'=').count(); @@ -120,7 +132,8 @@ impl Config { }; // A last group of one symbol holds under eight bits; with padding, `=` fills it to four. let rest = symbols.len() % 4; - if rest == 1 || (self.padded && pad != (4 - rest) % 4) { + let needed = (4 - rest) % 4; + if rest == 1 || (optional_padding && pad > needed) || (!optional_padding && self.padded && pad != needed) { return Err(DecodeError); } let mut out = Vec::with_capacity(symbols.len().saturating_mul(3) / 4); diff --git a/crates/mime/src/base64_tests.rs b/crates/mime/src/base64_tests.rs index e3d52794e..07fee89c7 100644 --- a/crates/mime/src/base64_tests.rs +++ b/crates/mime/src/base64_tests.rs @@ -79,3 +79,45 @@ proptest! { } } } + +/// Missing padding is opt-in; other canonical validation stays unchanged. +#[test] +fn optional_padding_keeps_alphabet_placement_and_tail_bit_checks() { + for cfg in CONFIGS { + for (input, expected) in [("", ""), ("Zg", "f"), ("Zg=", "f"), ("Zg==", "f"), ("Zm8", "fo"), ("Zm8=", "fo"), ("Zm9v", "foo")] { + assert_eq!(cfg.decode_optional_padding(input).unwrap(), expected.as_bytes()); + } + for bad in ["=", "====", "Z", "Z===", "Zg===", "Zm8==", "Zm9v=", "QQ==QUJD", "Zm=v", "Zh", "Zh=", "Zh==", "Zm9", "Zm9=", "Zg\n", " Zg", "é"] + { + assert!(cfg.decode_optional_padding(bad).is_err(), "{cfg:?}: {bad:?}"); + } + } + assert_eq!(STANDARD.decode_optional_padding("+/8").unwrap(), [251, 255]); + assert_eq!(URL_SAFE.decode_optional_padding("-_8=").unwrap(), [251, 255]); + assert!(STANDARD.decode_optional_padding("-_8=").is_err()); + assert!(URL_SAFE.decode_optional_padding("+/8=").is_err()); + assert!(STANDARD.decode("Zg=").is_err()); + assert!(STANDARD_NO_PAD.decode("Zg==").is_err()); +} + +proptest! { + #[test] + fn optional_padding_accepts_only_a_suffix_of_required_padding( + data in proptest::collection::vec(any::(), 0..400), + junk in proptest::collection::vec(any::(), 0..128), + ) { + for cfg in [STANDARD, URL_SAFE] { + let encoded = cfg.encode(&data); + let bare = encoded.trim_end_matches('='); + for length in bare.len()..=encoded.len() { + prop_assert_eq!(cfg.decode_optional_padding(&encoded[..length]).ok(), Some(data.clone())); + } + if let Ok(decoded) = cfg.decode_optional_padding(&junk) { + let canonical = cfg.encode(decoded); + let bare = canonical.trim_end_matches('='); + prop_assert!(junk.len() >= bare.len() && junk.len() <= canonical.len()); + prop_assert_eq!(canonical.as_bytes().get(..junk.len()), Some(junk.as_slice())); + } + } + } +} From 2c3171d73442f4b44aa3a8b0f1b4fc35dd497628 Mon Sep 17 00:00:00 2001 From: David Mann Date: Tue, 13 Oct 2026 02:37:02 +0200 Subject: [PATCH 46/98] psd: retain the non-advancing annotation length guard --- crates/psd/src/annotations.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/crates/psd/src/annotations.rs b/crates/psd/src/annotations.rs index c982d3338..b1d309c5a 100644 --- a/crates/psd/src/annotations.rs +++ b/crates/psd/src/annotations.rs @@ -41,3 +41,13 @@ fn pascal(out: &mut Vec, s: &str) { } } + +// Minimum-length guard imported from PhotoCraft b1b3dff8 (David Mann). +fn advancing_record_length(len: usize) -> Option<()> { + // The length counts itself and the type: anything shorter would put the cursor back on + // this entry, and a hostile count would then loop until memory runs out. + if len < 8 { + return None; + } + Some(()) +} From 22c9a0933d292d0875196201af09fb1455e8a710 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 07:43:59 -0500 Subject: [PATCH 47/98] psd: expose bounded annotation record views and serialization --- crates/psd/src/annotations.rs | 237 ++++++++++++++++++++++++---- crates/psd/src/annotations/tests.rs | 117 ++++++++++++++ crates/psd/src/lib.rs | 1 + 3 files changed, 325 insertions(+), 30 deletions(-) create mode 100644 crates/psd/src/annotations/tests.rs diff --git a/crates/psd/src/annotations.rs b/crates/psd/src/annotations.rs index b1d309c5a..57b466843 100644 --- a/crates/psd/src/annotations.rs +++ b/crates/psd/src/annotations.rs @@ -1,53 +1,230 @@ -// Reader primitives extracted from PhotoCraft annotations_map.rs at aa87012a. +//! Bounded, borrowed views and serialization of PSD global `Anno` records. +//! +//! Framing follows PhotoCraft's annotation reader (Brandon Thomas, aa87012a), with +//! David Mann's minimum-length guard and the per-entry span fix from PhotoCraft +//! #11138. Coordinates, colour conversion, text decoding and popup defaults are +//! deliberately caller policy. Unknown records can be carried through byte-exact. + +use crate::{PsdError, Result}; + +/// Explicit limits shared by readers and writers. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + /// Maximum complete block size. + pub bytes: usize, + /// Maximum annotation count. + pub records: usize, +} +impl Default for Limits { + fn default() -> Self { + Self { bytes: 16 * 1024 * 1024, records: 4096 } + } +} + struct Reader<'a> { d: &'a [u8], p: usize, } - impl<'a> Reader<'a> { - fn take(&mut self, n: usize) -> Option<&'a [u8]> { - let s = self.d.get(self.p..self.p.checked_add(n)?)?; - self.p += n; - Some(s) + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let end = self.p.checked_add(n).ok_or(PsdError::LimitExceeded("annotation offset"))?; + let s = self.d.get(self.p..end).ok_or(PsdError::UnexpectedEof { offset: self.p, needed: n })?; + self.p = end; + Ok(s) } - fn u8(&mut self) -> Option { - self.take(1).map(|b| b[0]) + fn u8(&mut self) -> Result { + Ok(self.take(1)?.first().copied().unwrap_or_default()) } - fn u16(&mut self) -> Option { - self.take(2).map(|b| u16::from_be_bytes([b[0], b[1]])) + fn u16(&mut self) -> Result { + Ok(u16::from_be_bytes(self.take(2)?.try_into().map_err(|_| PsdError::invalid("annotation u16"))?)) } - fn u32(&mut self) -> Option { - self.take(4).map(|b| u32::from_be_bytes([b[0], b[1], b[2], b[3]])) + fn u32(&mut self) -> Result { + Ok(u32::from_be_bytes(self.take(4)?.try_into().map_err(|_| PsdError::invalid("annotation u32"))?)) } - fn i32(&mut self) -> Option { - self.u32().map(|v| v as i32) + fn key(&mut self) -> Result<[u8; 4]> { + self.take(4)?.try_into().map_err(|_| PsdError::invalid("annotation key")) } - fn pascal(&mut self) -> Option { + fn rect(&mut self) -> Result<[i32; 4]> { + Ok([self.u32()? as i32, self.u32()? as i32, self.u32()? as i32, self.u32()? as i32]) + } + fn pascal(&mut self) -> Result<&'a [u8]> { let n = usize::from(self.u8()?); - let s = self.take(n)?; + let bytes = self.take(n)?; if (n + 1) % 2 == 1 { self.take(1)?; } - Some(s.iter().map(|&c| c as char).collect()) + Ok(bytes) } } -fn pascal(out: &mut Vec, s: &str) { - let bytes: Vec = s.chars().map(|c| if (c as u32) < 256 { c as u32 as u8 } else { b'?' }).take(255).collect(); - out.push(bytes.len() as u8); - out.extend_from_slice(&bytes); +/// A borrowed record, including its original length, padding and unknown fields. +#[derive(Clone, Copy, Debug)] +pub struct Record<'a> { + raw: &'a [u8], +} +impl<'a> Record<'a> { + /// Complete self-counting record, suitable for lossless passthrough. + pub fn raw(&self) -> &'a [u8] { + self.raw + } + /// Parse text records; other record kinds remain opaque. + pub fn text(&self) -> Result>> { + let mut r = Reader { d: self.raw, p: 4 }; + if r.key()? != *b"txtA" { + return Ok(None); + } + // Never read beyond this record, even if a text length points into its neighbour. + let open = r.u8()?; + let flags = r.u8()?; + let optional_blocks = r.u16()?; + let icon = r.rect()?; + let popup = r.rect()?; + let color_space = r.u16()?; + let color = [r.u16()?, r.u16()?, r.u16()?, r.u16()?]; + let author = r.pascal()?; + let name = r.pascal()?; + let modified = r.pascal()?; + let data_length = r.u32()?; + let data_key = r.key()?; + let size = usize::try_from(r.u32()?).map_err(|_| PsdError::LimitExceeded("annotation text"))?; + let data = r.take(size)?; + Ok(Some(Text { open, flags, optional_blocks, icon, popup, color_space, color, author, name, modified, data_length, data_key, data })) + } +} + +/// Wire-level text annotation. Text payload and Pascal strings remain bytes. +/// No text normalisation or application colour/coordinate policy is applied. +#[derive(Clone, Copy, Debug)] +pub struct Text<'a> { + /// Open state byte. + pub open: u8, + /// Flags byte. + pub flags: u8, + /// Optional block flags. + pub optional_blocks: u16, + /// Icon rectangle, top/left/bottom/right. + pub icon: [i32; 4], + /// Popup rectangle, top/left/bottom/right. + pub popup: [i32; 4], + /// PSD colour space. + pub color_space: u16, + /// Raw colour components. + pub color: [u16; 4], + /// Pascal author bytes, maximum 255. + pub author: &'a [u8], + /// Pascal name bytes, maximum 255. + pub name: &'a [u8], + /// Pascal modification-date bytes, maximum 255. + pub modified: &'a [u8], + /// Original self-counting data length, exposed for diagnostics. + /// The writer recomputes this field. + pub data_length: u32, + /// Usually `txtC`; retained without imposing application policy. + pub data_key: [u8; 4], + /// Payload, including any text byte-order mark. + pub data: &'a [u8], +} + +/// Parse complete framing before exposing records. Borrowed payloads do not allocate. +/// Trailing block bytes are accepted as in the existing PhotoCraft reader. +pub fn read(data: &[u8], limits: Limits) -> Result>> { + if data.len() > limits.bytes { + return Err(PsdError::LimitExceeded("annotation bytes")); + } + let mut r = Reader { d: data, p: 0 }; + r.u16()?; + r.u16()?; + let count = usize::try_from(r.u32()?).map_err(|_| PsdError::LimitExceeded("annotation count"))?; + if count > limits.records || count > data.len().saturating_sub(8) / 8 { + return Err(PsdError::LimitExceeded("annotation count")); + } + let mut out = Vec::new(); + out.try_reserve_exact(count).map_err(|_| PsdError::OutOfMemory("annotation records"))?; + for _ in 0..count { + let start = r.p; + let len = usize::try_from(r.u32()?).map_err(|_| PsdError::LimitExceeded("annotation length"))?; + // David Mann's zero-length guard: a self-counting record must advance. + if len < 8 { + return Err(PsdError::invalid("annotation record shorter than its header")); + } + r.p = start; + let record = Record { raw: r.take(len)? }; + record.text()?; // validate text spans, including data borrowed by later consumers + out.push(record); + } + Ok(out) +} + +fn append(out: &mut Vec, bytes: &[u8], limit: usize) -> Result<()> { + let len = out.len().checked_add(bytes.len()).ok_or(PsdError::LimitExceeded("annotation bytes"))?; + if len > limit { + return Err(PsdError::LimitExceeded("annotation bytes")); + } + out.try_reserve(bytes.len()).map_err(|_| PsdError::OutOfMemory("annotation bytes"))?; + out.extend_from_slice(bytes); + Ok(()) +} + +fn pascal(out: &mut Vec, bytes: &[u8], limit: usize) -> Result<()> { + let n = u8::try_from(bytes.len()).map_err(|_| PsdError::LimitExceeded("annotation Pascal string"))?; + append(out, &[n], limit)?; + append(out, bytes, limit)?; if (bytes.len() + 1) % 2 == 1 { - out.push(0); + append(out, &[0], limit)?; } + Ok(()) } - -// Minimum-length guard imported from PhotoCraft b1b3dff8 (David Mann). -fn advancing_record_length(len: usize) -> Option<()> { - // The length counts itself and the type: anything shorter would put the cursor back on - // this entry, and a hostile count would then loop until memory runs out. - if len < 8 { - return None; +/// Serialize one text record, computing both length fields. Caller supplies raw +/// text bytes and already-mapped colour/coordinates; nothing is truncated. +pub fn write_text(text: &Text<'_>, limits: Limits) -> Result> { + let mut out = Vec::new(); + append(&mut out, &[0; 4], limits.bytes)?; + append(&mut out, b"txtA", limits.bytes)?; + append(&mut out, &[text.open, text.flags], limits.bytes)?; + append(&mut out, &text.optional_blocks.to_be_bytes(), limits.bytes)?; + for value in text.icon.into_iter().chain(text.popup) { + append(&mut out, &value.to_be_bytes(), limits.bytes)?; } - Some(()) + append(&mut out, &text.color_space.to_be_bytes(), limits.bytes)?; + for value in text.color { + append(&mut out, &value.to_be_bytes(), limits.bytes)?; + } + for value in [text.author, text.name, text.modified] { + pascal(&mut out, value, limits.bytes)?; + } + let n = u32::try_from(text.data.len()).map_err(|_| PsdError::LimitExceeded("annotation text"))?; + let data_length = n.checked_add(12).ok_or(PsdError::LimitExceeded("annotation data length"))?; + append(&mut out, &data_length.to_be_bytes(), limits.bytes)?; + append(&mut out, &text.data_key, limits.bytes)?; + append(&mut out, &n.to_be_bytes(), limits.bytes)?; + append(&mut out, text.data, limits.bytes)?; + let len = u32::try_from(out.len()).map_err(|_| PsdError::LimitExceeded("annotation record length"))?; + out.get_mut(..4).ok_or_else(|| PsdError::invalid("annotation header"))?.copy_from_slice(&len.to_be_bytes()); + Ok(out) } + +/// Combine already-framed records into a version 2.1 block. Unknown record types +/// are preserved byte-exact; malformed text records are refused. +pub fn write(records: &[&[u8]], limits: Limits) -> Result> { + if records.len() > limits.records { + return Err(PsdError::LimitExceeded("annotation count")); + } + let count = u32::try_from(records.len()).map_err(|_| PsdError::LimitExceeded("annotation count"))?; + let mut out = Vec::new(); + append(&mut out, &[0, 2, 0, 1], limits.bytes)?; + append(&mut out, &count.to_be_bytes(), limits.bytes)?; + for raw in records { + let mut r = Reader { d: raw, p: 0 }; + let declared = usize::try_from(r.u32()?).map_err(|_| PsdError::LimitExceeded("annotation record length"))?; + if raw.len() < 8 || declared != raw.len() { + return Err(PsdError::invalid("annotation record length mismatch")); + } + Record { raw }.text()?; + append(&mut out, raw, limits.bytes)?; + } + Ok(out) +} + +#[cfg(test)] +mod tests; diff --git a/crates/psd/src/annotations/tests.rs b/crates/psd/src/annotations/tests.rs new file mode 100644 index 000000000..e207852d4 --- /dev/null +++ b/crates/psd/src/annotations/tests.rs @@ -0,0 +1,117 @@ +use super::*; + +fn note() -> Text<'static> { + Text { + open: 1, + flags: 0x1c, + optional_blocks: 1, + icon: [10, 20, 30, 36], + popup: [30, 40, 190, 280], + color_space: 0, + color: [65535, 1000, 3000, 0], + author: b"Writer", + name: b"", + modified: b"2026", + data_length: 0, + data_key: *b"txtC", + data: &[0xfe, 0xff, 0, 65, 0, 13, 0, 66], + } +} + +#[test] +fn text_and_unknown_sound_round_trip_without_copying_payloads() { + let limits = Limits::default(); + let text = write_text(¬e(), limits).unwrap(); + let sound = [0, 0, 0, 11, b's', b'n', b'd', b'A', 42, 3, 9]; + let block = write(&[&text, &sound], limits).unwrap(); + let parsed = read(&block, limits).unwrap(); + assert_eq!(parsed.len(), 2); + let view = parsed[0].text().unwrap().unwrap(); + assert_eq!(view.author, note().author); + assert_eq!(view.data, note().data); + assert_eq!(view.icon, note().icon); + assert_eq!(view.color, note().color); + assert_eq!(view.data_length, 20); + assert!(parsed[1].text().unwrap().is_none()); + assert_eq!(parsed[1].raw(), sound); + let slices: Vec<_> = parsed.iter().map(Record::raw).collect(); + assert_eq!(write(&slices, limits).unwrap(), block); + assert!(view.data.as_ptr() >= block.as_ptr()); + assert!(view.data.as_ptr() < block.as_ptr().wrapping_add(block.len())); +} + +#[test] +fn text_cannot_read_past_the_declared_entry_into_its_neighbour() { + // Regression from PhotoCraft #11138, without its app Note/Color policy. + let mut record = write_text(¬e(), Limits::default()).unwrap(); + let shortened = record.len() - note().data.len(); + record[..4].copy_from_slice(&(shortened as u32).to_be_bytes()); + let mut block = vec![0, 2, 0, 1, 0, 0, 0, 1]; + block.extend_from_slice(&record); + assert!(read(&block, Limits::default()).is_err()); + assert!(write(&[&record], Limits::default()).is_err()); +} + +#[test] +fn truncation_and_hostile_counts_lengths_fail_closed() { + let record = write_text(¬e(), Limits::default()).unwrap(); + let block = write(&[&record], Limits::default()).unwrap(); + for end in 0..block.len() { + assert!(read(&block[..end], Limits::default()).is_err(), "{end}"); + } + for len in [0u32, 1, 7, u32::MAX] { + let mut bad = block.clone(); + bad[8..12].copy_from_slice(&len.to_be_bytes()); + assert!(read(&bad, Limits::default()).is_err()); + } + let mut bad = block; + bad[4..8].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(read(&bad, Limits::default()).is_err()); +} + +#[test] +fn explicit_limits_and_pascal_length_are_not_silently_truncated() { + let record = write_text(¬e(), Limits::default()).unwrap(); + let block = write(&[&record], Limits::default()).unwrap(); + assert!(read(&block, Limits { records: 0, ..Limits::default() }).is_err()); + assert!(read(&block, Limits { bytes: block.len() - 1, ..Limits::default() }).is_err()); + assert!(write(&[&record], Limits { records: 0, ..Limits::default() }).is_err()); + assert!(write_text(¬e(), Limits { bytes: record.len() - 1, ..Limits::default() }).is_err()); + let long = [0; 256]; + let mut text = note(); + text.author = &long; + assert!(write_text(&text, Limits::default()).is_err()); +} + +#[test] +fn preserves_legacy_version_and_unused_data_header_acceptance() { + let record = write_text(¬e(), Limits::default()).unwrap(); + let mut block = write(&[&record], Limits::default()).unwrap(); + block[..4].copy_from_slice(&[0, 9, 0, 8]); + block.extend_from_slice(b"trailing"); + assert_eq!(read(&block, Limits::default()).unwrap()[0].raw(), record); +} + +#[test] +fn canonical_layout_matches_the_existing_photocraft_writer() { + // PhotoCraft annotations_map::write_note layout (Brandon Thomas, aa87012a). + // App RGB/position/text conversion happens before this shared wire writer. + let n = note(); + let mut expected = vec![0, 0, 0, 0]; + expected.extend_from_slice(b"txtA"); + expected.extend_from_slice(&[1, 0x1c, 0, 1]); + for value in [10i32, 20, 30, 36, 30, 40, 190, 280] { + expected.extend_from_slice(&value.to_be_bytes()); + } + for value in [0u16, 65535, 1000, 3000, 0] { + expected.extend_from_slice(&value.to_be_bytes()); + } + expected.extend_from_slice(b"\x06Writer\x00\x00\x00\x042026\x00"); + expected.extend_from_slice(&20u32.to_be_bytes()); + expected.extend_from_slice(b"txtC"); + expected.extend_from_slice(&8u32.to_be_bytes()); + expected.extend_from_slice(n.data); + let len = expected.len() as u32; + expected[..4].copy_from_slice(&len.to_be_bytes()); + assert_eq!(write_text(&n, Limits::default()).unwrap(), expected); +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 93acd5f39..8126437ff 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -54,6 +54,7 @@ pub mod aco; pub mod act; pub mod adjust; pub mod animation; +pub mod annotations; pub mod ase; pub mod asl; pub mod atn; From d7dc95d273fca95e30395c08525da06e47fbb16b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 07:52:56 -0500 Subject: [PATCH 48/98] fix(mime): use checked divisibility idiom for padding --- crates/mime/src/base64.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/mime/src/base64.rs b/crates/mime/src/base64.rs index ebe077372..856f33398 100644 --- a/crates/mime/src/base64.rs +++ b/crates/mime/src/base64.rs @@ -122,7 +122,7 @@ impl Config { fn decode_inner(self, text: &[u8], optional_padding: bool) -> Result, DecodeError> { let (symbols, pad) = if self.padded || optional_padding { - if !optional_padding && text.len() % 4 != 0 { + if !optional_padding && !text.len().is_multiple_of(4) { return Err(DecodeError); } let pad = text.iter().rev().take(2).take_while(|c| **c == b'=').count(); From 72ee990e036fc2305adf2180110b42d32d49716c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 49/98] Import binary plist parser changes from Craft Anything f64ae3bf --- crates/plist/LICENSE-APACHE | 177 ++++++++++++++++++++++++++ crates/plist/LICENSE-MIT | 21 ++++ crates/plist/src/binary.rs | 241 ++++++++++++++++++++++++++++++++++++ crates/plist/src/bytes.rs | 82 ++++++++++++ 4 files changed, 521 insertions(+) create mode 100644 crates/plist/LICENSE-APACHE create mode 100644 crates/plist/LICENSE-MIT create mode 100644 crates/plist/src/binary.rs create mode 100644 crates/plist/src/bytes.rs diff --git a/crates/plist/LICENSE-APACHE b/crates/plist/LICENSE-APACHE new file mode 100644 index 000000000..f433b1a53 --- /dev/null +++ b/crates/plist/LICENSE-APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/crates/plist/LICENSE-MIT b/crates/plist/LICENSE-MIT new file mode 100644 index 000000000..8e6399401 --- /dev/null +++ b/crates/plist/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ArtCraft Team and the Craft Anything contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs new file mode 100644 index 000000000..21ac84a86 --- /dev/null +++ b/crates/plist/src/binary.rs @@ -0,0 +1,241 @@ +use crate::{FormatError, bytes::Reader}; + +pub const MAX_DEPTH: usize = 256; +pub const MAX_NODES: usize = 2_000_000; +pub const MAX_EXPANDED_BYTES: usize = 64 << 20; + +#[derive(Debug, Clone, PartialEq)] +pub enum Value { + Dict(Vec<(String, Value)>), + Array(Vec), + String(String), + Integer(i128), + Real(f64), + Bool(bool), + /// Seconds since 2001-01-01T00:00:00Z (binary), or the ISO text (XML). + Date(DateValue), + Data(Vec), + /// A keyed-archive object reference. + Uid(u64), + Null, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum DateValue { + Seconds(f64), + Text(String), +} + +impl Value { + pub fn get(&self, key: &str) -> Option<&Value> { + match self { + Value::Dict(d) => d.iter().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + pub fn as_str(&self) -> Option<&str> { + match self { + Value::String(s) => Some(s), + _ => None, + } + } + + pub fn as_array(&self) -> Option<&[Value]> { + match self { + Value::Array(a) => Some(a), + _ => None, + } + } + + pub fn as_uid(&self) -> Option { + match self { + Value::Uid(u) => Some(*u), + _ => None, + } + } + + pub fn as_int(&self) -> Option { + match self { + Value::Integer(i) => Some(*i), + _ => None, + } + } + +} +struct Bin<'a> { + r: Reader<'a>, + offsets: Vec, + ref_size: usize, + nodes: usize, +} + +/// Parse a `bplist00` binary property list. +pub fn parse_binary(data: &[u8]) -> Result { + if !data.starts_with(b"bplist") { + return Err(FormatError::Malformed("not a binary plist".into())); + } + let len = data.len(); + if len < 8 + 32 { + return Err(FormatError::Truncated("binary plist trailer".into())); + } + let r = Reader::new(data); + let t = len - 32; + let off_size = usize::from(r.u8(t + 6).unwrap_or(0)); + let ref_size = usize::from(r.u8(t + 7).unwrap_or(0)); + let num = r.u64_be(t + 8).unwrap_or(0); + let top = r.u64_be(t + 16).unwrap_or(0); + let table = r.u64_be(t + 24).unwrap_or(0); + if !(1..=8).contains(&off_size) || !(1..=8).contains(&ref_size) { + return Err(FormatError::Malformed("bad offset or reference size".into())); + } + let num = usize::try_from(num).map_err(|_| FormatError::Malformed("object count".into()))?; + let table = usize::try_from(table).map_err(|_| FormatError::Malformed("offset table".into()))?; + if num == 0 || num > len { + return Err(FormatError::Malformed(format!("{num} objects in a {len}-byte file"))); + } + let need = num.checked_mul(off_size).and_then(|n| n.checked_add(table)).ok_or_else(|| FormatError::Malformed("offset table".into()))?; + if need > t || table < 8 { + return Err(FormatError::Truncated("offset table runs past the trailer".into())); + } + let mut offsets = Vec::with_capacity(num); + for i in 0..num { + let o = read_be(&r, table + i * off_size, off_size).ok_or_else(|| FormatError::Truncated("offset table".into()))?; + let o = usize::try_from(o).map_err(|_| FormatError::Malformed("object offset".into()))?; + if o >= table { + return Err(FormatError::Malformed(format!("object {i} lies outside the object area"))); + } + offsets.push(o); + } + let top = usize::try_from(top).map_err(|_| FormatError::Malformed("top object".into()))?; + let mut b = Bin { r, offsets, ref_size, nodes: 0 }; + let mut stack = Vec::new(); + b.object(top, &mut stack) +} + +fn read_be(r: &Reader<'_>, off: usize, n: usize) -> Option { + let s = r.slice(off, n)?; + Some(s.iter().fold(0u64, |acc, &b| (acc << 8) | u64::from(b))) +} + +impl Bin<'_> { + fn count(&self, at: usize, low: u8) -> Result<(usize, usize), FormatError> { + if low != 0xf { + return Ok((usize::from(low), at + 1)); + } + let m = self.r.u8(at + 1).ok_or_else(|| FormatError::Truncated("length".into()))?; + if m & 0xf0 != 0x10 { + return Err(FormatError::Malformed("bad extended length marker".into())); + } + let n = 1usize << (m & 0x0f).min(3); + let v = read_be(&self.r, at + 2, n).ok_or_else(|| FormatError::Truncated("length".into()))?; + let v = usize::try_from(v).map_err(|_| FormatError::Malformed("length".into()))?; + Ok((v, at + 2 + n)) + } + + fn reference(&self, at: usize, i: usize) -> Result { + let off = i.checked_mul(self.ref_size).and_then(|x| x.checked_add(at)).ok_or_else(|| FormatError::Malformed("reference".into()))?; + let v = read_be(&self.r, off, self.ref_size).ok_or_else(|| FormatError::Truncated("object reference".into()))?; + usize::try_from(v).map_err(|_| FormatError::Malformed("reference".into())) + } + + fn object(&mut self, idx: usize, stack: &mut Vec) -> Result { + self.nodes += 1; + if self.nodes > MAX_NODES { + return Err(FormatError::TooLarge(format!("more than {MAX_NODES} plist nodes"))); + } + if stack.len() >= MAX_DEPTH { + return Err(FormatError::Malformed("nesting too deep".into())); + } + if stack.contains(&idx) { + return Err(FormatError::Malformed(format!("object {idx} contains itself"))); + } + let at = *self.offsets.get(idx).ok_or_else(|| FormatError::Malformed(format!("reference to missing object {idx}")))?; + let m = self.r.u8(at).ok_or_else(|| FormatError::Truncated("object marker".into()))?; + let (hi, lo) = (m >> 4, m & 0x0f); + let v = match hi { + 0x0 => match lo { + 0x0 => Value::Null, + 0x8 => Value::Bool(false), + 0x9 => Value::Bool(true), + _ => Value::Null, + }, + 0x1 => { + let n = 1usize << lo.min(4); + let bytes = self.r.slice(at + 1, n).ok_or_else(|| FormatError::Truncated("integer".into()))?; + let v = match n { + 8 => i128::from(i64::from_be_bytes(bytes.try_into().map_err(|_| FormatError::Truncated("integer".into()))?)), + 16 => i128::from_be_bytes(bytes.try_into().map_err(|_| FormatError::Truncated("integer".into()))?), + _ => i128::from(bytes.iter().fold(0u64, |acc, &b| (acc << 8) | u64::from(b))), + }; + Value::Integer(v) + } + 0x2 => { + let v = match lo { + 2 => self.r.u32_be(at + 1).map(|b| f64::from(f32::from_bits(b))), + 3 => self.r.u64_be(at + 1).map(f64::from_bits), + _ => None, + }; + Value::Real(v.ok_or_else(|| FormatError::Truncated("real".into()))?) + } + 0x3 => Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)), + 0x4 => { + let (n, s) = self.count(at, lo)?; + Value::Data(self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("data".into()))?.to_vec()) + } + 0x5 => { + let (n, s) = self.count(at, lo)?; + Value::String(String::from_utf8_lossy(self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("string".into()))?).into_owned()) + } + 0x6 => { + let (n, s) = self.count(at, lo)?; + let bytes = self + .r + .slice(s, n.checked_mul(2).ok_or_else(|| FormatError::Malformed("string".into()))?) + .ok_or_else(|| FormatError::Truncated("string".into()))?; + let units: Vec = bytes.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + Value::String(String::from_utf16_lossy(&units)) + } + 0x8 => { + let n = usize::from(lo) + 1; + Value::Uid(read_be(&self.r, at + 1, n).ok_or_else(|| FormatError::Truncated("uid".into()))?) + } + 0xa | 0xc => { + let (n, s) = self.count(at, lo)?; + if n > self.offsets.len().saturating_mul(4).max(16) || s.saturating_add(n.saturating_mul(self.ref_size)) > self.r.len() { + return Err(FormatError::Malformed("array larger than the file".into())); + } + stack.push(idx); + let mut out = Vec::with_capacity(n); + for i in 0..n { + let child = self.reference(s, i)?; + out.push(self.object(child, stack)?); + } + stack.pop(); + Value::Array(out) + } + 0xd => { + let (n, s) = self.count(at, lo)?; + if s.saturating_add(n.saturating_mul(self.ref_size).saturating_mul(2)) > self.r.len() { + return Err(FormatError::Malformed("dictionary larger than the file".into())); + } + stack.push(idx); + let mut out = Vec::with_capacity(n); + for i in 0..n { + let k = self.reference(s, i)?; + let v = self.reference(s, i + n)?; + let key = match self.object(k, stack)? { + Value::String(s) => s, + other => format!("{:?}", other.to_json()), + }; + out.push((key, self.object(v, stack)?)); + } + stack.pop(); + Value::Dict(out) + } + _ => return Err(FormatError::Malformed(format!("unknown object marker {m:#04x}"))), + }; + Ok(v) + } +} + diff --git a/crates/plist/src/bytes.rs b/crates/plist/src/bytes.rs new file mode 100644 index 000000000..b9de989c3 --- /dev/null +++ b/crates/plist/src/bytes.rs @@ -0,0 +1,82 @@ +//! Checked little/big-endian reads over untrusted bytes. Every read returns `None` when out of range. + +#[derive(Debug, Clone, Copy)] +pub struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + pub fn len(&self) -> usize { + self.data.len() + } + + pub fn is_empty(&self) -> bool { + self.data.is_empty() + } + + pub fn data(&self) -> &'a [u8] { + self.data + } + + pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + let end = off.checked_add(len)?; + self.data.get(off..end) + } + + pub fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + + fn arr(&self, off: usize) -> Option<[u8; N]> { + self.slice(off, N)?.try_into().ok() + } + + pub fn u16_le(&self, off: usize) -> Option { + self.arr(off).map(u16::from_le_bytes) + } + + pub fn u32_le(&self, off: usize) -> Option { + self.arr(off).map(u32::from_le_bytes) + } + + pub fn u64_le(&self, off: usize) -> Option { + self.arr(off).map(u64::from_le_bytes) + } + + pub fn u16_be(&self, off: usize) -> Option { + self.arr(off).map(u16::from_be_bytes) + } + + pub fn u32_be(&self, off: usize) -> Option { + self.arr(off).map(u32::from_be_bytes) + } + + pub fn u64_be(&self, off: usize) -> Option { + self.arr(off).map(u64::from_be_bytes) + } + + /// A NUL-terminated string starting at `off`, at most `max` bytes, lossily decoded. + pub fn cstr(&self, off: usize, max: usize) -> Option { + let rest = self.data.get(off..)?; + let rest = rest.get(..max.min(rest.len()))?; + let end = rest.iter().position(|&b| b == 0).unwrap_or(rest.len()); + Some(String::from_utf8_lossy(rest.get(..end)?).into_owned()) + } + + /// Unsigned LEB128 at `off`: (value, bytes consumed). At most 10 bytes. + pub fn uleb128(&self, off: usize) -> Option<(u64, usize)> { + let mut result = 0u64; + for i in 0..10usize { + let b = self.u8(off.checked_add(i)?)?; + result |= u64::from(b & 0x7f).checked_shl(7 * i as u32)?; + if b & 0x80 == 0 { + return Some((result, i + 1)); + } + } + None + } +} From feea4e316df5c0a2ce6466f409a96bc58bde0bae Mon Sep 17 00:00:00 2001 From: BFlat Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 50/98] Import binary plist parser changes from Craft Anything 15de3661 --- crates/plist/src/binary.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs index 21ac84a86..9b792410a 100644 --- a/crates/plist/src/binary.rs +++ b/crates/plist/src/binary.rs @@ -102,13 +102,15 @@ pub fn parse_binary(data: &[u8]) -> Result { for i in 0..num { let o = read_be(&r, table + i * off_size, off_size).ok_or_else(|| FormatError::Truncated("offset table".into()))?; let o = usize::try_from(o).map_err(|_| FormatError::Malformed("object offset".into()))?; - if o >= table { + if o < 8 || o >= table { return Err(FormatError::Malformed(format!("object {i} lies outside the object area"))); } offsets.push(o); } let top = usize::try_from(top).map_err(|_| FormatError::Malformed("top object".into()))?; - let mut b = Bin { r, offsets, ref_size, nodes: 0 }; + // Object payloads and references must not borrow bytes from the offset table or trailer. + let objects = data.get(..table).ok_or_else(|| FormatError::Truncated("object area".into()))?; + let mut b = Bin { r: Reader::new(objects), offsets, ref_size, nodes: 0 }; let mut stack = Vec::new(); b.object(top, &mut stack) } @@ -127,7 +129,10 @@ impl Bin<'_> { if m & 0xf0 != 0x10 { return Err(FormatError::Malformed("bad extended length marker".into())); } - let n = 1usize << (m & 0x0f).min(3); + if m & 0x0f > 3 { + return Err(FormatError::Malformed("extended length width exceeds 64 bits".into())); + } + let n = 1usize << (m & 0x0f); let v = read_be(&self.r, at + 2, n).ok_or_else(|| FormatError::Truncated("length".into()))?; let v = usize::try_from(v).map_err(|_| FormatError::Malformed("length".into()))?; Ok((v, at + 2 + n)) @@ -161,7 +166,10 @@ impl Bin<'_> { _ => Value::Null, }, 0x1 => { - let n = 1usize << lo.min(4); + if lo > 4 { + return Err(FormatError::Malformed("integer width exceeds 128 bits".into())); + } + let n = 1usize << lo; let bytes = self.r.slice(at + 1, n).ok_or_else(|| FormatError::Truncated("integer".into()))?; let v = match n { 8 => i128::from(i64::from_be_bytes(bytes.try_into().map_err(|_| FormatError::Truncated("integer".into()))?)), @@ -197,6 +205,9 @@ impl Bin<'_> { Value::String(String::from_utf16_lossy(&units)) } 0x8 => { + if lo > 7 { + return Err(FormatError::Malformed("uid width exceeds 64 bits".into())); + } let n = usize::from(lo) + 1; Value::Uid(read_be(&self.r, at + 1, n).ok_or_else(|| FormatError::Truncated("uid".into()))?) } From d307449ee4c4ad7e09ceb2b8fafa1c3e3f66e169 Mon Sep 17 00:00:00 2001 From: RRFZ <247457381+Plastic-time@users.noreply.github.com> Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 51/98] Import binary plist parser changes from Craft Anything 0a8cc088 --- crates/plist/src/binary.rs | 56 +++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs index 9b792410a..4fb85fd93 100644 --- a/crates/plist/src/binary.rs +++ b/crates/plist/src/binary.rs @@ -68,10 +68,17 @@ struct Bin<'a> { offsets: Vec, ref_size: usize, nodes: usize, + /// Data and text bytes still allowed to be copied out. + bytes_left: usize, } /// Parse a `bplist00` binary property list. pub fn parse_binary(data: &[u8]) -> Result { + parse_binary_within(data, MAX_EXPANDED_BYTES.max(data.len().saturating_mul(3))) +} + +/// Parse a binary property list, producing at most `byte_budget` bytes of data and decoded text. +fn parse_binary_within(data: &[u8], byte_budget: usize) -> Result { if !data.starts_with(b"bplist") { return Err(FormatError::Malformed("not a binary plist".into())); } @@ -110,7 +117,7 @@ pub fn parse_binary(data: &[u8]) -> Result { let top = usize::try_from(top).map_err(|_| FormatError::Malformed("top object".into()))?; // Object payloads and references must not borrow bytes from the offset table or trailer. let objects = data.get(..table).ok_or_else(|| FormatError::Truncated("object area".into()))?; - let mut b = Bin { r: Reader::new(objects), offsets, ref_size, nodes: 0 }; + let mut b = Bin { r: Reader::new(objects), offsets, ref_size, nodes: 0, bytes_left: byte_budget }; let mut stack = Vec::new(); b.object(top, &mut stack) } @@ -120,6 +127,21 @@ fn read_be(r: &Reader<'_>, off: usize, n: usize) -> Option { Some(s.iter().fold(0u64, |acc, &b| (acc << 8) | u64::from(b))) } +/// UTF-8 size of `String::from_utf8_lossy(bytes)`: each damaged sequence becomes a U+FFFD. +fn lossy_utf8_len(bytes: &[u8]) -> usize { + bytes.utf8_chunks().fold(0usize, |n, c| { + let replacement = if c.invalid().is_empty() { 0 } else { char::REPLACEMENT_CHARACTER.len_utf8() }; + n.saturating_add(c.valid().len()).saturating_add(replacement) + }) +} + +/// UTF-8 size of `String::from_utf16_lossy` over big-endian code units: a lone surrogate becomes +/// a U+FFFD. +fn lossy_utf16_len(bytes: &[u8]) -> usize { + let units = bytes.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)); + char::decode_utf16(units).fold(0usize, |n, c| n.saturating_add(c.map_or(char::REPLACEMENT_CHARACTER.len_utf8(), char::len_utf8))) +} + impl Bin<'_> { fn count(&self, at: usize, low: u8) -> Result<(usize, usize), FormatError> { if low != 0xf { @@ -144,6 +166,22 @@ impl Bin<'_> { usize::try_from(v).map_err(|_| FormatError::Malformed("reference".into())) } + /// Refuse a container whose `children` would pass MAX_NODES, before space is reserved for them. + fn room_for_nodes(&self, children: usize) -> Result<(), FormatError> { + if children > MAX_NODES.saturating_sub(self.nodes) { + return Err(FormatError::TooLarge(format!("more than {MAX_NODES} plist nodes"))); + } + Ok(()) + } + + /// Spend `n` bytes of the data and text budget before copying them out. + fn spend(&mut self, n: usize) -> Result<(), FormatError> { + self.bytes_left = + self.bytes_left.checked_sub(n).ok_or_else(|| FormatError::TooLarge("repeated data and text expand past the plist byte budget".into()))?; + Ok(()) + } + + /// Read object `idx`. `stack` holds the containers on the current path, so a cycle is refused. fn object(&mut self, idx: usize, stack: &mut Vec) -> Result { self.nodes += 1; if self.nodes > MAX_NODES { @@ -189,11 +227,15 @@ impl Bin<'_> { 0x3 => Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)), 0x4 => { let (n, s) = self.count(at, lo)?; - Value::Data(self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("data".into()))?.to_vec()) + let bytes = self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("data".into()))?; + self.spend(bytes.len())?; + Value::Data(bytes.to_vec()) } 0x5 => { let (n, s) = self.count(at, lo)?; - Value::String(String::from_utf8_lossy(self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("string".into()))?).into_owned()) + let bytes = self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("string".into()))?; + self.spend(lossy_utf8_len(bytes))?; + Value::String(String::from_utf8_lossy(bytes).into_owned()) } 0x6 => { let (n, s) = self.count(at, lo)?; @@ -201,6 +243,7 @@ impl Bin<'_> { .r .slice(s, n.checked_mul(2).ok_or_else(|| FormatError::Malformed("string".into()))?) .ok_or_else(|| FormatError::Truncated("string".into()))?; + self.spend(lossy_utf16_len(bytes))?; let units: Vec = bytes.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); Value::String(String::from_utf16_lossy(&units)) } @@ -213,9 +256,13 @@ impl Bin<'_> { } 0xa | 0xc => { let (n, s) = self.count(at, lo)?; - if n > self.offsets.len().saturating_mul(4).max(16) || s.saturating_add(n.saturating_mul(self.ref_size)) > self.r.len() { + // Writers store equal values once, so an array may name one object many times: + // only its references have to fit in the file. MAX_NODES and the byte budget bound + // the expansion. + if s.saturating_add(n.saturating_mul(self.ref_size)) > self.r.len() { return Err(FormatError::Malformed("array larger than the file".into())); } + self.room_for_nodes(n)?; stack.push(idx); let mut out = Vec::with_capacity(n); for i in 0..n { @@ -230,6 +277,7 @@ impl Bin<'_> { if s.saturating_add(n.saturating_mul(self.ref_size).saturating_mul(2)) > self.r.len() { return Err(FormatError::Malformed("dictionary larger than the file".into())); } + self.room_for_nodes(n.saturating_mul(2))?; stack.push(idx); let mut out = Vec::with_capacity(n); for i in 0..n { From e20c7c1a6e4dcf2c430da3423f04aad30a4b24a2 Mon Sep 17 00:00:00 2001 From: Afloat Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 52/98] Import binary plist parser changes from Craft Anything 0c386e8a --- crates/plist/src/binary.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs index 4fb85fd93..26129ad83 100644 --- a/crates/plist/src/binary.rs +++ b/crates/plist/src/binary.rs @@ -82,6 +82,9 @@ fn parse_binary_within(data: &[u8], byte_budget: usize) -> Result { 0x0 => Value::Null, 0x8 => Value::Bool(false), 0x9 => Value::Bool(true), - _ => Value::Null, + _ => return Err(FormatError::Malformed(format!("unknown simple plist object {m:#04x}"))), }, 0x1 => { if lo > 4 { @@ -224,7 +227,7 @@ impl Bin<'_> { }; Value::Real(v.ok_or_else(|| FormatError::Truncated("real".into()))?) } - 0x3 => Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)), + 0x3 if lo == 3 => Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)), 0x4 => { let (n, s) = self.count(at, lo)?; let bytes = self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("data".into()))?; @@ -285,7 +288,7 @@ impl Bin<'_> { let v = self.reference(s, i + n)?; let key = match self.object(k, stack)? { Value::String(s) => s, - other => format!("{:?}", other.to_json()), + _ => return Err(FormatError::Unsupported("non-string dictionary keys cannot be represented in this plist view".into())), }; out.push((key, self.object(v, stack)?)); } From b2f888d788042c6df64ffa1aa9f8ddf537d48864 Mon Sep 17 00:00:00 2001 From: BFlat Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 53/98] Import plist regression from Craft Anything 15de3661 --- crates/plist/tests/plist_marker_widths.rs | 47 +++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 crates/plist/tests/plist_marker_widths.rs diff --git a/crates/plist/tests/plist_marker_widths.rs b/crates/plist/tests/plist_marker_widths.rs new file mode 100644 index 000000000..5873371e7 --- /dev/null +++ b/crates/plist/tests/plist_marker_widths.rs @@ -0,0 +1,47 @@ +use craft_plist::{Value, parse_binary}; + +fn single_object(object: &[u8]) -> Vec { + let mut data = b"bplist00".to_vec(); + data.extend_from_slice(object); + let table = data.len() as u64; + data.push(8); + data.extend_from_slice(&[0, 0, 0, 0, 0, 0, 1, 1]); + data.extend_from_slice(&1u64.to_be_bytes()); + data.extend_from_slice(&0u64.to_be_bytes()); + data.extend_from_slice(&table.to_be_bytes()); + data +} + +#[test] +fn unsupported_integer_widths_are_not_clamped() { + for marker in 0x15..=0x1f { + let mut object = vec![marker]; + object.extend_from_slice(&[0; 16]); + assert!(parse_binary(&single_object(&object)).is_err(), "accepted {marker:#x}"); + } +} + +#[test] +fn extended_lengths_and_uids_do_not_lose_high_bytes() { + let mut length = vec![0x4f, 0x14]; + length.extend_from_slice(&[0; 16]); + assert!(parse_binary(&single_object(&length)).is_err()); + for marker in 0x88..=0x8f { + let mut object = vec![marker, 1]; + object.resize(usize::from(marker & 15) + 2, 0); + assert!(parse_binary(&single_object(&object)).is_err(), "accepted {marker:#x}"); + } +} + +#[test] +fn supported_maximum_widths_keep_their_values() { + let mut integer = vec![0x14]; + integer.extend_from_slice(&i128::MAX.to_be_bytes()); + assert!(matches!(parse_binary(&single_object(&integer)), Ok(Value::Integer(i)) if i == i128::MAX)); + let mut uid = vec![0x87]; + uid.extend_from_slice(&u64::MAX.to_be_bytes()); + assert!(matches!(parse_binary(&single_object(&uid)), Ok(Value::Uid(i)) if i == u64::MAX)); + let mut empty = vec![0x4f, 0x13]; + empty.extend_from_slice(&0u64.to_be_bytes()); + assert!(matches!(parse_binary(&single_object(&empty)), Ok(Value::Data(d)) if d.is_empty())); +} From 0bc19896102774e5ffe12ed012fc6cb036e6b9d3 Mon Sep 17 00:00:00 2001 From: BFlat Date: Sun, 15 Nov 2026 07:55:30 -0500 Subject: [PATCH 54/98] Import plist regression from Craft Anything 15de3661 --- crates/plist/tests/plist_object_bounds.rs | 34 +++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 crates/plist/tests/plist_object_bounds.rs diff --git a/crates/plist/tests/plist_object_bounds.rs b/crates/plist/tests/plist_object_bounds.rs new file mode 100644 index 000000000..52d30cb80 --- /dev/null +++ b/crates/plist/tests/plist_object_bounds.rs @@ -0,0 +1,34 @@ +use craft_plist::{Value, parse_binary}; +use craft_anything_testkit::apple as fx; + +fn single_object(object: &[u8], offset: u8) -> Vec { + let mut data = b"bplist00".to_vec(); + data.extend_from_slice(object); + let table = data.len() as u64; + data.push(offset); + data.extend_from_slice(&[0, 0, 0, 0, 0, 0, 1, 1]); + data.extend_from_slice(&1u64.to_be_bytes()); + data.extend_from_slice(&0u64.to_be_bytes()); + data.extend_from_slice(&table.to_be_bytes()); + data +} + +#[test] +fn binary_values_cannot_consume_the_offset_table_or_trailer() { + for object in [&[0x54, b'x'][..], &[0x43, 0x42], &[0x13], &[0x87], &[0x62, 0, b'x']] { + assert!(parse_binary(&single_object(object, 8)).is_err(), "accepted object {object:02x?}"); + } +} + +#[test] +fn binary_offsets_cannot_point_inside_the_file_header() { + assert!(parse_binary(&single_object(&[0x00], 0)).is_err()); + assert!(matches!(parse_binary(&single_object(&[0x00], 8)), Ok(Value::Null))); +} + +#[test] +fn objects_ending_at_the_table_boundary_are_valid() { + assert!(matches!(parse_binary(&single_object(&[0x51, b'x'], 8)), Ok(Value::String(s)) if s == "x")); + let sample = fx::sample_plist(); + assert_eq!(parse_binary(&fx::to_bplist(&sample)).unwrap().get("Count").and_then(Value::as_int), Some(-42)); +} From 1ec7ab92b5a316d41642c388561fc5c2a2ad8492 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 07:57:29 -0500 Subject: [PATCH 55/98] Expose bounded cross-platform binary plist reader with explicit limits --- ATTRIBUTION.md | 4 + Cargo.lock | 7 ++ Cargo.toml | 1 + crates/plist/Cargo.toml | 14 ++++ crates/plist/README.md | 11 +++ crates/plist/src/binary.rs | 90 +++++++++++++++-------- crates/plist/src/bytes.rs | 45 ------------ crates/plist/src/lib.rs | 33 +++++++++ crates/plist/tests/limits.rs | 57 ++++++++++++++ crates/plist/tests/plist_object_bounds.rs | 3 - 10 files changed, 188 insertions(+), 77 deletions(-) create mode 100644 crates/plist/Cargo.toml create mode 100644 crates/plist/README.md create mode 100644 crates/plist/src/lib.rs create mode 100644 crates/plist/tests/limits.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 77d3578cb..c087ecda1 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -95,3 +95,7 @@ and the exact transfer revision are recorded in each vendored crate's SOURCE.jso | `crates/raw/docs/raf-corpus.sha256` (hashes only; the files are not bundled) | The photographers who contributed each file to raw.pixls.us | [raw.pixls.us](https://raw.pixls.us/) | CC0-1.0 | | `crates/drawingml/src/preset_text.txt` | DeckCraft contributors (Nikita Rakitin); generated by `crates/drawingml/tests/fixtures/make_preset_guides.py` from the guide formulas of ECMA-376 Part 1 `presetShapeDefinitions.xml` | [storytold/deckcraft#650](https://github.com/storytold/deckcraft/pull/650) | MIT OR Apache-2.0 (the formulas are the format's definition in ECMA-376) | | `crates/drawingml/tests/fixtures/preset_table.txt` | DeckCraft contributors (Nikita Rakitin); generated by `crates/drawingml/tests/fixtures/make_preset_table.py` from ECMA-376 Part 1 `presetShapeDefinitions.xml` | [storytold/deckcraft#650](https://github.com/storytold/deckcraft/pull/650) | MIT OR Apache-2.0 (the formulas are the format's definition in ECMA-376) | + +## Binary Apple property lists + +`crates/plist` is extracted from `storytold/craft-anything` (`crates/formats/src/plist.rs` and checked byte reads), at source main `88d1d9cac852aec1c7fd97886db451e005e49d2a`. MIT OR Apache-2.0; source license texts are retained in the crate. Primary import Git authors preserve Brandon Thomas, BFlat, RRFZ and Afloat. Regression imports preserve BFlat. XML, JSON projection, calendar formatting and app policy were not imported into the public API. diff --git a/Cargo.lock b/Cargo.lock index 4936ffe2b..529e1a2e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2357,6 +2357,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-plist" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-platform" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 77b20da79..c5a585d7b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ craft-proto = { path = "crates/proto" } acadrust = "=0.6.6" log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` +craft-plist = { path = "crates/plist" } craft-platform = { path = "crates/platform" } craft-aep = { path = "crates/aep" } craft-affinity = { path = "crates/affinity" } diff --git a/crates/plist/Cargo.toml b/crates/plist/Cargo.toml new file mode 100644 index 000000000..7ae95aae0 --- /dev/null +++ b/crates/plist/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-plist" +description = "Bounded cross-platform binary Apple property-list reader" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +thiserror.workspace = true diff --git a/crates/plist/README.md b/crates/plist/README.md new file mode 100644 index 000000000..dc5c52007 --- /dev/null +++ b/crates/plist/README.md @@ -0,0 +1,11 @@ +# craft-plist + +A cross-platform bounded binary (`bplist00`) reader, extracted from Craft Anything's existing implementation and hardening history. Typed dictionaries retain order and duplicate keys; `get` returns the first match. Strings retain the source reader's lossy decoding policy. Shared references expand under node and decoded-byte budgets, and cycles are rejected. + +`parse_binary_with_limits` accepts explicit input, object, expanded-node, nesting and decoded-byte limits. Recursion always has a hard ceiling of 256 even if a caller supplies a larger depth. Counts are checked before container reservation; allocations can return a resource error. Limits are per parse, not a promise of a precise process RSS cap. + +This first PR contains reading only. XML parsing/writing and keyed-archive interpretation are not implemented. JSON projection, date formatting, OS file access and preference/locale selection are not part of this crate. Existing updater/Photo streaming `plist` users remain unchanged. + +Initial consumers: PhotoCraft's AppleLanguages binary-reader path; later Craft Anything binary parsing and LightCraft XML presets (after XML support/parity). Photo must retain its 4 MiB input policy and first-language selection. + +Source: `storytold/craft-anything` at `88d1d9cac852aec1c7fd97886db451e005e49d2a`, `crates/formats/src/plist.rs` and checked byte reader. Source license files retained. Imported changes retain Brandon Thomas, BFlat, RRFZ and Afloat as primary Git authors; imported regression tests are BFlat's. No external specification implementation copied. diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs index 26129ad83..3e65f1a9e 100644 --- a/crates/plist/src/binary.rs +++ b/crates/plist/src/binary.rs @@ -1,8 +1,4 @@ -use crate::{FormatError, bytes::Reader}; - -pub const MAX_DEPTH: usize = 256; -pub const MAX_NODES: usize = 2_000_000; -pub const MAX_EXPANDED_BYTES: usize = 64 << 20; +use crate::{FormatError, Limits, bytes::Reader}; #[derive(Debug, Clone, PartialEq)] pub enum Value { @@ -61,7 +57,6 @@ impl Value { _ => None, } } - } struct Bin<'a> { r: Reader<'a>, @@ -70,19 +65,23 @@ struct Bin<'a> { nodes: usize, /// Data and text bytes still allowed to be copied out. bytes_left: usize, + limits: Limits, } /// Parse a `bplist00` binary property list. pub fn parse_binary(data: &[u8]) -> Result { - parse_binary_within(data, MAX_EXPANDED_BYTES.max(data.len().saturating_mul(3))) + parse_binary_with_limits(data, Limits::default()) } -/// Parse a binary property list, producing at most `byte_budget` bytes of data and decoded text. -fn parse_binary_within(data: &[u8], byte_budget: usize) -> Result { +/// Parse a binary property list, with caller-selected input, object, expansion and nesting limits. +pub fn parse_binary_with_limits(data: &[u8], limits: Limits) -> Result { + if data.len() > limits.max_input_bytes { + return Err(FormatError::TooLarge("input bytes".into())); + } if !data.starts_with(b"bplist") { return Err(FormatError::Malformed("not a binary plist".into())); } - if !data.starts_with(b"bplist0") { + if !data.starts_with(b"bplist00") { return Err(FormatError::Unsupported("unknown binary plist version".into())); } let len = data.len(); @@ -104,11 +103,14 @@ fn parse_binary_within(data: &[u8], byte_budget: usize) -> Result len { return Err(FormatError::Malformed(format!("{num} objects in a {len}-byte file"))); } + if num > limits.max_objects { + return Err(FormatError::TooLarge("object count".into())); + } let need = num.checked_mul(off_size).and_then(|n| n.checked_add(table)).ok_or_else(|| FormatError::Malformed("offset table".into()))?; if need > t || table < 8 { return Err(FormatError::Truncated("offset table runs past the trailer".into())); } - let mut offsets = Vec::with_capacity(num); + let mut offsets = reserved(num)?; for i in 0..num { let o = read_be(&r, table + i * off_size, off_size).ok_or_else(|| FormatError::Truncated("offset table".into()))?; let o = usize::try_from(o).map_err(|_| FormatError::Malformed("object offset".into()))?; @@ -120,8 +122,8 @@ fn parse_binary_within(data: &[u8], byte_budget: usize) -> Result { usize::try_from(v).map_err(|_| FormatError::Malformed("reference".into())) } - /// Refuse a container whose `children` would pass MAX_NODES, before space is reserved for them. + /// Refuse a container whose `children` would pass the node limit, before space is reserved for them. fn room_for_nodes(&self, children: usize) -> Result<(), FormatError> { - if children > MAX_NODES.saturating_sub(self.nodes) { - return Err(FormatError::TooLarge(format!("more than {MAX_NODES} plist nodes"))); + if children > self.limits.max_nodes.saturating_sub(self.nodes) { + return Err(FormatError::TooLarge("expanded node count".into())); } Ok(()) } @@ -187,10 +189,10 @@ impl Bin<'_> { /// Read object `idx`. `stack` holds the containers on the current path, so a cycle is refused. fn object(&mut self, idx: usize, stack: &mut Vec) -> Result { self.nodes += 1; - if self.nodes > MAX_NODES { - return Err(FormatError::TooLarge(format!("more than {MAX_NODES} plist nodes"))); + if self.nodes > self.limits.max_nodes { + return Err(FormatError::TooLarge("expanded node count".into())); } - if stack.len() >= MAX_DEPTH { + if stack.len() >= self.limits.max_depth.min(256) { return Err(FormatError::Malformed("nesting too deep".into())); } if stack.contains(&idx) { @@ -227,18 +229,32 @@ impl Bin<'_> { }; Value::Real(v.ok_or_else(|| FormatError::Truncated("real".into()))?) } - 0x3 if lo == 3 => Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)), + 0x3 if lo == 3 => { + Value::Date(DateValue::Seconds(self.r.u64_be(at + 1).map(f64::from_bits).ok_or_else(|| FormatError::Truncated("date".into()))?)) + } 0x4 => { let (n, s) = self.count(at, lo)?; let bytes = self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("data".into()))?; self.spend(bytes.len())?; - Value::Data(bytes.to_vec()) + { + let mut out = reserved(bytes.len())?; + out.extend_from_slice(bytes); + Value::Data(out) + } } 0x5 => { let (n, s) = self.count(at, lo)?; let bytes = self.r.slice(s, n).ok_or_else(|| FormatError::Truncated("string".into()))?; - self.spend(lossy_utf8_len(bytes))?; - Value::String(String::from_utf8_lossy(bytes).into_owned()) + let size = lossy_utf8_len(bytes); + self.spend(size)?; + let mut out = reserved_string(size)?; + for chunk in bytes.utf8_chunks() { + out.push_str(chunk.valid()); + if !chunk.invalid().is_empty() { + out.push(char::REPLACEMENT_CHARACTER); + } + } + Value::String(out) } 0x6 => { let (n, s) = self.count(at, lo)?; @@ -246,9 +262,14 @@ impl Bin<'_> { .r .slice(s, n.checked_mul(2).ok_or_else(|| FormatError::Malformed("string".into()))?) .ok_or_else(|| FormatError::Truncated("string".into()))?; - self.spend(lossy_utf16_len(bytes))?; - let units: Vec = bytes.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); - Value::String(String::from_utf16_lossy(&units)) + let size = lossy_utf16_len(bytes); + self.spend(size)?; + let mut out = reserved_string(size)?; + let units = bytes.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)); + for ch in char::decode_utf16(units) { + out.push(ch.unwrap_or(char::REPLACEMENT_CHARACTER)); + } + Value::String(out) } 0x8 => { if lo > 7 { @@ -260,14 +281,14 @@ impl Bin<'_> { 0xa | 0xc => { let (n, s) = self.count(at, lo)?; // Writers store equal values once, so an array may name one object many times: - // only its references have to fit in the file. MAX_NODES and the byte budget bound + // only its references have to fit in the file. the node limit and the byte budget bound // the expansion. if s.saturating_add(n.saturating_mul(self.ref_size)) > self.r.len() { return Err(FormatError::Malformed("array larger than the file".into())); } self.room_for_nodes(n)?; stack.push(idx); - let mut out = Vec::with_capacity(n); + let mut out = reserved(n)?; for i in 0..n { let child = self.reference(s, i)?; out.push(self.object(child, stack)?); @@ -282,7 +303,7 @@ impl Bin<'_> { } self.room_for_nodes(n.saturating_mul(2))?; stack.push(idx); - let mut out = Vec::with_capacity(n); + let mut out = reserved(n)?; for i in 0..n { let k = self.reference(s, i)?; let v = self.reference(s, i + n)?; @@ -301,3 +322,14 @@ impl Bin<'_> { } } +fn reserved(n: usize) -> Result, FormatError> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| FormatError::TooLarge("allocation".into()))?; + Ok(v) +} + +fn reserved_string(n: usize) -> Result { + let mut s = String::new(); + s.try_reserve_exact(n).map_err(|_| FormatError::TooLarge("string allocation".into()))?; + Ok(s) +} diff --git a/crates/plist/src/bytes.rs b/crates/plist/src/bytes.rs index b9de989c3..bdfecc668 100644 --- a/crates/plist/src/bytes.rs +++ b/crates/plist/src/bytes.rs @@ -14,14 +14,6 @@ impl<'a> Reader<'a> { self.data.len() } - pub fn is_empty(&self) -> bool { - self.data.is_empty() - } - - pub fn data(&self) -> &'a [u8] { - self.data - } - pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { let end = off.checked_add(len)?; self.data.get(off..end) @@ -35,22 +27,6 @@ impl<'a> Reader<'a> { self.slice(off, N)?.try_into().ok() } - pub fn u16_le(&self, off: usize) -> Option { - self.arr(off).map(u16::from_le_bytes) - } - - pub fn u32_le(&self, off: usize) -> Option { - self.arr(off).map(u32::from_le_bytes) - } - - pub fn u64_le(&self, off: usize) -> Option { - self.arr(off).map(u64::from_le_bytes) - } - - pub fn u16_be(&self, off: usize) -> Option { - self.arr(off).map(u16::from_be_bytes) - } - pub fn u32_be(&self, off: usize) -> Option { self.arr(off).map(u32::from_be_bytes) } @@ -58,25 +34,4 @@ impl<'a> Reader<'a> { pub fn u64_be(&self, off: usize) -> Option { self.arr(off).map(u64::from_be_bytes) } - - /// A NUL-terminated string starting at `off`, at most `max` bytes, lossily decoded. - pub fn cstr(&self, off: usize, max: usize) -> Option { - let rest = self.data.get(off..)?; - let rest = rest.get(..max.min(rest.len()))?; - let end = rest.iter().position(|&b| b == 0).unwrap_or(rest.len()); - Some(String::from_utf8_lossy(rest.get(..end)?).into_owned()) - } - - /// Unsigned LEB128 at `off`: (value, bytes consumed). At most 10 bytes. - pub fn uleb128(&self, off: usize) -> Option<(u64, usize)> { - let mut result = 0u64; - for i in 0..10usize { - let b = self.u8(off.checked_add(i)?)?; - result |= u64::from(b & 0x7f).checked_shl(7 * i as u32)?; - if b & 0x80 == 0 { - return Some((result, i + 1)); - } - } - None - } } diff --git a/crates/plist/src/lib.rs b/crates/plist/src/lib.rs new file mode 100644 index 000000000..3e4c6cd85 --- /dev/null +++ b/crates/plist/src/lib.rs @@ -0,0 +1,33 @@ +//! Bounded binary Apple property lists, independent of operating-system preferences. +//! XML and writing are intentionally not implemented in this first extraction. +//! File discovery, locale selection, bundle validation and application policy stay in callers. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +mod binary; +mod bytes; +pub use binary::{DateValue, Value, parse_binary, parse_binary_with_limits}; + +/// Caller-selected resource limits. Zero values refuse the corresponding resource. +#[derive(Clone, Copy, Debug)] +pub struct Limits { + pub max_input_bytes: usize, + pub max_objects: usize, + pub max_nodes: usize, + pub max_depth: usize, + pub max_expanded_bytes: usize, +} +impl Default for Limits { + fn default() -> Self { + Self { max_input_bytes: 16 << 20, max_objects: 100_000, max_nodes: 100_000, max_depth: 64, max_expanded_bytes: 64 << 20 } + } +} +#[derive(Debug, thiserror::Error)] +pub enum FormatError { + #[error("malformed property list: {0}")] + Malformed(String), + #[error("truncated property list: {0}")] + Truncated(String), + #[error("unsupported property list: {0}")] + Unsupported(String), + #[error("property list resource limit: {0}")] + TooLarge(String), +} diff --git a/crates/plist/tests/limits.rs b/crates/plist/tests/limits.rs new file mode 100644 index 000000000..0c4fefd7c --- /dev/null +++ b/crates/plist/tests/limits.rs @@ -0,0 +1,57 @@ +use craft_plist::{Limits, Value, parse_binary, parse_binary_with_limits}; +fn plist(objects: &[&[u8]]) -> Vec { + let mut b = b"bplist00".to_vec(); + let mut offsets = Vec::new(); + for obj in objects { + offsets.push(b.len() as u8); + b.extend_from_slice(obj); + } + let table = b.len() as u64; + b.extend_from_slice(&offsets); + b.extend_from_slice(&[0, 0, 0, 0, 0, 0, 1, 1]); + b.extend_from_slice(&(objects.len() as u64).to_be_bytes()); + b.extend_from_slice(&0u64.to_be_bytes()); + b.extend_from_slice(&table.to_be_bytes()); + b +} +#[test] +fn languages_and_every_truncation() { + let b = plist(&[&[0xd1, 1, 2], b"\x5eAppleLanguages", &[0xa1, 3], b"\x55ja-JP"]); + let p = parse_binary(&b).unwrap(); + assert_eq!(p.get("AppleLanguages").unwrap().as_array().unwrap()[0].as_str(), Some("ja-JP")); + for n in 0..b.len() { + assert!(parse_binary(&b[..n]).is_err(), "prefix {n}"); + } +} +#[test] +fn cycles_and_repeated_references() { + assert!(parse_binary(&plist(&[&[0xa1, 0]])).is_err()); + let b = plist(&[&[0xa3, 1, 1, 1], b"\x51x"]); + assert_eq!(parse_binary(&b).unwrap(), Value::Array(vec![Value::String("x".into()); 3])); + assert!(parse_binary_with_limits(&b, Limits { max_nodes: 3, ..Limits::default() }).is_err()); + assert!(parse_binary_with_limits(&b, Limits { max_expanded_bytes: 2, ..Limits::default() }).is_err()); +} +#[test] +fn explicit_limits_and_version() { + let b = plist(&[&[0xa1, 1], b"\x51x"]); + for l in [ + Limits { max_input_bytes: b.len() - 1, ..Limits::default() }, + Limits { max_objects: 1, ..Limits::default() }, + Limits { max_depth: 1, ..Limits::default() }, + Limits { max_nodes: 0, ..Limits::default() }, + ] { + assert!(parse_binary_with_limits(&b, l).is_err()); + } + let mut future = b; + future[7] = b'1'; + assert!(parse_binary(&future).is_err()); +} +#[test] +fn utf16_and_lossy_expansion_spend_budget() { + let b = plist(&[&[0x62, 0xd8, 0x3d, 0xde, 0x00]]); + assert_eq!(parse_binary(&b).unwrap().as_str(), Some("😀")); + assert!(parse_binary_with_limits(&b, Limits { max_expanded_bytes: 3, ..Limits::default() }).is_err()); + let bad = plist(&[&[0x51, 0xff]]); + assert_eq!(parse_binary(&bad).unwrap().as_str(), Some("�")); + assert!(parse_binary_with_limits(&bad, Limits { max_expanded_bytes: 2, ..Limits::default() }).is_err()); +} diff --git a/crates/plist/tests/plist_object_bounds.rs b/crates/plist/tests/plist_object_bounds.rs index 52d30cb80..d35fd91b1 100644 --- a/crates/plist/tests/plist_object_bounds.rs +++ b/crates/plist/tests/plist_object_bounds.rs @@ -1,5 +1,4 @@ use craft_plist::{Value, parse_binary}; -use craft_anything_testkit::apple as fx; fn single_object(object: &[u8], offset: u8) -> Vec { let mut data = b"bplist00".to_vec(); @@ -29,6 +28,4 @@ fn binary_offsets_cannot_point_inside_the_file_header() { #[test] fn objects_ending_at_the_table_boundary_are_valid() { assert!(matches!(parse_binary(&single_object(&[0x51, b'x'], 8)), Ok(Value::String(s)) if s == "x")); - let sample = fx::sample_plist(); - assert_eq!(parse_binary(&fx::to_bplist(&sample)).unwrap().get("Count").and_then(Value::as_int), Some(-42)); } From cc4fc29cdba60f70e3af3b5d975c60bd0085cb1b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 08:00:09 -0500 Subject: [PATCH 56/98] Add bounded selective plist array lookup for locale parity --- crates/plist/src/binary.rs | 62 ++++++++++++++++++++++++++++++++++-- crates/plist/src/lib.rs | 2 +- crates/plist/tests/limits.rs | 18 +++++++++++ 3 files changed, 78 insertions(+), 4 deletions(-) diff --git a/crates/plist/src/binary.rs b/crates/plist/src/binary.rs index 3e65f1a9e..3fe6aed92 100644 --- a/crates/plist/src/binary.rs +++ b/crates/plist/src/binary.rs @@ -75,6 +75,12 @@ pub fn parse_binary(data: &[u8]) -> Result { /// Parse a binary property list, with caller-selected input, object, expansion and nesting limits. pub fn parse_binary_with_limits(data: &[u8], limits: Limits) -> Result { + let (mut b, top) = reader(data, limits)?; + let mut stack = reserved(limits.max_depth.min(256))?; + b.object(top, &mut stack) +} + +fn reader(data: &[u8], limits: Limits) -> Result<(Bin<'_>, usize), FormatError> { if data.len() > limits.max_input_bytes { return Err(FormatError::TooLarge("input bytes".into())); } @@ -122,9 +128,8 @@ pub fn parse_binary_with_limits(data: &[u8], limits: Limits) -> Result, off: usize, n: usize) -> Option { @@ -333,3 +338,54 @@ fn reserved_string(n: usize) -> Result { s.try_reserve_exact(n).map_err(|_| FormatError::TooLarge("string allocation".into()))?; Ok(s) } + +/// Select the first string in one top-level dictionary array without expanding unrelated values. +/// First matching dictionary key wins. Missing/wrong-typed values return `None`; malformed +/// selected objects and resource exhaustion return errors. Unselected value graphs are not +/// validated. This permits preference readers to ignore unrelated future object kinds. +pub fn first_string_in_array(data: &[u8], key: &str, limits: Limits) -> Result, FormatError> { + let (mut b, top) = reader(data, limits)?; + let (kind, n, start) = b.header(top)?; + if kind != 0xd { + return Ok(None); + } + if n > limits.max_nodes { + return Err(FormatError::TooLarge("dictionary lookup entries".into())); + } + let bytes = n.checked_mul(2).and_then(|n| n.checked_mul(b.ref_size)).ok_or_else(|| FormatError::Malformed("dictionary references".into()))?; + b.r.slice(start, bytes).ok_or_else(|| FormatError::Truncated("dictionary references".into()))?; + for i in 0..n { + let k = b.reference(start, i)?; + if b.selected_string(k)?.as_deref() != Some(key) { + continue; + } + let value = b.reference(start, n + i)?; + let (kind, count, payload) = b.header(value)?; + if kind != 0xa || count == 0 { + return Ok(None); + } + let first = b.reference(payload, 0)?; + return b.selected_string(first); + } + Ok(None) +} +impl Bin<'_> { + fn header(&self, idx: usize) -> Result<(u8, usize, usize), FormatError> { + let at = *self.offsets.get(idx).ok_or_else(|| FormatError::Malformed("missing object".into()))?; + let marker = self.r.u8(at).ok_or_else(|| FormatError::Truncated("object marker".into()))?; + let kind = marker >> 4; + // Only containers and strings use an extended length. Unknown types are not traversed. + let (n, start) = if matches!(kind, 0x5 | 0x6 | 0xa | 0xd) { self.count(at, marker & 15)? } else { (usize::from(marker & 15), at + 1) }; + Ok((kind, n, start)) + } + fn selected_string(&mut self, idx: usize) -> Result, FormatError> { + let (kind, _, _) = self.header(idx)?; + if !matches!(kind, 0x5 | 0x6) { + return Ok(None); + } + match self.object(idx, &mut Vec::new())? { + Value::String(s) => Ok(Some(s)), + _ => Ok(None), + } + } +} diff --git a/crates/plist/src/lib.rs b/crates/plist/src/lib.rs index 3e4c6cd85..f3af79ddc 100644 --- a/crates/plist/src/lib.rs +++ b/crates/plist/src/lib.rs @@ -4,7 +4,7 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod binary; mod bytes; -pub use binary::{DateValue, Value, parse_binary, parse_binary_with_limits}; +pub use binary::{DateValue, Value, first_string_in_array, parse_binary, parse_binary_with_limits}; /// Caller-selected resource limits. Zero values refuse the corresponding resource. #[derive(Clone, Copy, Debug)] diff --git a/crates/plist/tests/limits.rs b/crates/plist/tests/limits.rs index 0c4fefd7c..dc1882698 100644 --- a/crates/plist/tests/limits.rs +++ b/crates/plist/tests/limits.rs @@ -55,3 +55,21 @@ fn utf16_and_lossy_expansion_spend_budget() { assert_eq!(parse_binary(&bad).unwrap().as_str(), Some("�")); assert!(parse_binary_with_limits(&bad, Limits { max_expanded_bytes: 2, ..Limits::default() }).is_err()); } + +#[test] +fn selective_lookup_ignores_unrelated_graphs_and_preserves_first_match() { + use craft_plist::first_string_in_array; + let b = plist(&[&[0xd2, 1, 2, 3, 4], b"\x55Other", b"\x5eAppleLanguages", &[0xaf], &[0xa2, 5, 6], b"\x55ja-JP", &[0xff]]); + assert!(parse_binary(&b).is_err()); + assert_eq!(first_string_in_array(&b, "AppleLanguages", Limits::default()).unwrap().as_deref(), Some("ja-JP")); + let duplicate = plist(&[&[0xd2, 1, 1, 2, 3], b"\x5eAppleLanguages", &[0xa1, 4], &[0xa1, 5], b"\x55zh-TW", b"\x55ko-KR"]); + assert_eq!(first_string_in_array(&duplicate, "AppleLanguages", Limits::default()).unwrap().as_deref(), Some("zh-TW")); + assert!(first_string_in_array(&b, "AppleLanguages", Limits { max_expanded_bytes: 1, ..Limits::default() }).is_err()); +} + +#[test] +fn selective_lookup_requires_an_array_not_a_set() { + use craft_plist::first_string_in_array; + let b = plist(&[&[0xd1, 1, 2], b"\x5eAppleLanguages", &[0xc1, 3], b"\x55ja-JP"]); + assert_eq!(first_string_in_array(&b, "AppleLanguages", Limits::default()).unwrap(), None); +} From ac7586ed9f4ed099b042b9e7867e0b468b9fb08c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 08:01:42 -0500 Subject: [PATCH 57/98] Merge pull request #1507 from storytold/dot/photo-ipp-shared print: reuse direct CUPS transport with explicit app parity API --- crates/print/src/cups/mod.rs | 955 +++++++++++++++++++++++++++++++++ crates/print/src/cups/tests.rs | 593 ++++++++++++++++++++ crates/print/src/lib.rs | 4 + 3 files changed, 1552 insertions(+) create mode 100644 crates/print/src/cups/mod.rs create mode 100644 crates/print/src/cups/tests.rs diff --git a/crates/print/src/cups/mod.rs b/crates/print/src/cups/mod.rs new file mode 100644 index 000000000..1a0a9360b --- /dev/null +++ b/crates/print/src/cups/mod.rs @@ -0,0 +1,955 @@ +//! Based on the direct CUPS transport from contributor PR #156 (ahmedhosnypro), +//! kept separate from the broader network-printing registry in PR #116. +//! IPP/1.1 straight to the local CUPS server: printer discovery (`CUPS-Get-Printers`, +//! `CUPS-Get-Default`) and job submission (`Print-Job`) over HTTP/1.1, for when `lpstat`/`lp` +//! are missing (a sandboxed install, where the print dialog would otherwise have no printers). +//! Implemented from the public IPP encoding (RFC 2910/8010) and semantics (RFC 8011) and +//! CUPS's IPP documentation; the scheduler is reached on its socket or selected TCP endpoint. +//! +//! # Blocking and authorization +//! The caller authorizes endpoint selection. The absolute exchange deadline covers all request +//! writes and reply reads **after connection**. Standard-library DNS resolution and Unix socket +//! connection are blocking OS calls outside that deadline. TCP connection attempts have their +//! own timeout. This is not a whole-operation deadline API: do not call it on a UI thread or +//! promise bounded completion for arbitrary named hosts/Unix listeners. No detached timeout +//! thread is spawned; callers needing cancellation must supply a separately bounded transport. + +#[cfg(all(test, unix))] +mod tests; + +use super::{Duplex, Error, Job, Printer, Scaling}; +use std::io::{Read, Write}; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; +#[cfg(unix)] +use std::os::unix::net::UnixStream; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::{Duration, Instant}; + +/// Where the local scheduler listens: its socket, else its loopback port. +const CUPS_SOCKET: &str = "/run/cups/cups.sock"; + +/// Caps on what the server may hand back: a reply is read into memory before it is parsed. +const MAX_BODY: usize = 1 << 20; +const MAX_HEADERS: usize = 8 << 10; +const MAX_PRINTERS: usize = 256; +const MAX_NAME: usize = 256; +/// A job's title on the wire; IPP values are u16-length, and no real title is this long. +const MAX_JOB_NAME: usize = 255; + +/// A wedged scheduler must not hang the dialog; a job may carry megabytes and is given longer. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(1); +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(2); +const JOB_TIMEOUT: Duration = Duration::from_secs(30); + +/// IPP operation codes: the standard Print-Job, and CUPS's discovery extensions. +const OPERATION_PRINT_JOB: u16 = 0x0002; +const OPERATION_CUPS_GET_DEFAULT: u16 = 0x4001; +const OPERATION_CUPS_GET_PRINTERS: u16 = 0x4002; + +/// IPP value tags and group delimiters (RFC 8010 §5.1); the ones this client sends or reads. +const GROUP_OPERATION: u8 = 0x01; +const GROUP_JOB: u8 = 0x02; +const GROUP_END: u8 = 0x03; +const GROUP_PRINTER: u8 = 0x04; +const TAG_INTEGER: u8 = 0x21; +const TAG_BOOLEAN: u8 = 0x22; +const TAG_NAME: u8 = 0x42; +const TAG_KEYWORD: u8 = 0x44; +const TAG_URI: u8 = 0x45; +const TAG_CHARSET: u8 = 0x47; +const TAG_LANGUAGE: u8 = 0x48; +const TAG_MIME: u8 = 0x49; + +static REQUEST_ID: AtomicU32 = AtomicU32::new(1); + +/// How the scheduler is reached: its unix socket, else its loopback port. +#[derive(Clone, Debug)] +pub enum Server { + Unix(PathBuf), + Tcp(SocketAddr), + /// A caller-authorized host. DNS is resolved once for each connection attempt. + Named(String, u16), +} + +impl Server { + /// Parse an explicitly selected CUPS_SERVER value, without connecting. No URLs, + /// credentials, control characters or HTTP authority delimiters are accepted. + pub fn parse(value: &str) -> Result { + if value.starts_with('/') { + return Ok(Self::Unix(value.into())); + } + if value.is_empty() || value.len() > 255 || value.bytes().any(|c| c.is_ascii_control() || c.is_ascii_whitespace() || b"/@?#\\".contains(&c)) { + return Err(Error::Spool("invalid CUPS server".into())); + } + let (host, port) = if let Some(rest) = value.strip_prefix('[') { + let (host, tail) = rest.split_once(']').ok_or_else(|| Error::Spool("invalid CUPS IPv6 host".into()))?; + if !host.contains(':') { + return Err(Error::Spool("bracketed CUPS host must be IPv6".into())); + } + let port = if tail.is_empty() { + 631 + } else { + tail.strip_prefix(':') + .and_then(|p| p.parse::().ok()) + .filter(|p| *p != 0) + .ok_or_else(|| Error::Spool("invalid CUPS port".into()))? + }; + (host, port) + } else if value.matches(':').count() > 1 { + // A bare IPv6 address is a host. A colon after its scope name may + // specify a port without cutting an address hextet in half. + if let Some((_, zone)) = value.split_once('%') + && zone.contains(':') + { + let (host, port) = value.rsplit_once(':').ok_or_else(|| Error::Spool("invalid CUPS endpoint".into()))?; + (host, port.parse::().ok().filter(|p| *p != 0).ok_or_else(|| Error::Spool("invalid CUPS port".into()))?) + } else { + (value, 631) + } + } else { + match value.rsplit_once(':') { + Some((host, port)) => (host, port.parse::().ok().filter(|p| *p != 0).ok_or_else(|| Error::Spool("invalid CUPS port".into()))?), + None => (value, 631), + } + }; + if host.is_empty() || host.contains(['[', ']']) { + return Err(Error::Spool("invalid CUPS host".into())); + } + if let Ok(ip) = host.parse::() { + return Ok(Self::Tcp(SocketAddr::new(ip, port))); + } + if host.contains(':') { + let (ip, zone) = host.split_once('%').ok_or_else(|| Error::Spool("invalid CUPS IPv6 host".into()))?; + if ip.parse::().is_err() + || zone.is_empty() + || !zone.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.')) + { + return Err(Error::Spool("invalid CUPS IPv6 scope".into())); + } + } else if host.contains('%') { + return Err(Error::Spool("invalid CUPS host".into())); + } + Ok(Self::Named(host.into(), port)) + } + + fn authority(&self) -> String { + match self { + Self::Unix(_) => "localhost".into(), + Self::Tcp(addr) => addr.to_string().replace('%', "%25"), + Self::Named(host, port) if host.contains(':') => format!("[{}]:{port}", host.replace('%', "%25")), + Self::Named(host, port) => format!("{host}:{port}"), + } + } +} + +/// Local scheduler candidates, honoring the user's explicit CUPS_SERVER setting. +/// No requests are made by this function; apps must authorize selection before use. +pub fn configured_servers() -> Result, Error> { + if let Some(value) = std::env::var_os("CUPS_SERVER").filter(|v| !v.is_empty()) { + return Ok(vec![Server::parse(&value.to_string_lossy())?]); + } + Ok(vec![ + Server::Unix(CUPS_SOCKET.into()), + Server::Unix("/var/run/cups/cups.sock".into()), + Server::Unix("/private/var/run/cupsd".into()), + Server::Tcp(([127, 0, 0, 1], 631).into()), + ]) +} + +fn endpoints() -> [Server; 2] { + [Server::Unix(PathBuf::from(CUPS_SOCKET)), Server::Tcp(([127, 0, 0, 1], 631).into())] +} + +/// Whether the local CUPS server is reachable: its socket, else its loopback port. A missing +/// scheduler is a normal `false`, not an error. +pub fn cups_reachable() -> bool { + endpoints().iter().any(reachable) +} + +/// Probe a caller-authorized endpoint without sending a print request. +/// DNS and Unix connect have the blocking limitations documented on this module. +pub fn reachable(server: &Server) -> bool { + connect(server).is_ok() +} + +fn connect(server: &Server) -> std::io::Result { + match server { + #[cfg(unix)] + Server::Unix(path) => UnixStream::connect(path).map(Stream::Unix), + #[cfg(not(unix))] + Server::Unix(_) => Err(std::io::Error::new(std::io::ErrorKind::Unsupported, "Unix sockets unavailable")), + Server::Named(host, port) => { + // Validate even directly constructed public variants before resolving or writing headers. + let endpoint = if host.contains(':') { format!("[{host}]:{port}") } else { format!("{host}:{port}") }; + let parsed = Server::parse(&endpoint).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + if matches!(parsed, Server::Unix(_)) { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "a named CUPS host cannot be a Unix path")); + } + let addr = (host.as_str(), *port) + .to_socket_addrs()? + .next() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::NotFound, "CUPS host has no address"))?; + TcpStream::connect_timeout(&addr, CONNECT_TIMEOUT).map(Stream::Tcp) + } + // A loopback connect either answers promptly or the server is not there. + Server::Tcp(addr) => TcpStream::connect_timeout(addr, CONNECT_TIMEOUT).map(|s| { + let _ = s.set_nodelay(true); + Stream::Tcp(s) + }), + } +} + +/// A connection to the scheduler. +enum Stream { + #[cfg(unix)] + Unix(UnixStream), + Tcp(TcpStream), +} + +impl Stream { + fn set_timeouts(&self, timeout: Duration) -> std::io::Result<()> { + let (read, write) = match self { + #[cfg(unix)] + Stream::Unix(s) => (s.set_read_timeout(Some(timeout)), s.set_write_timeout(Some(timeout))), + Stream::Tcp(s) => (s.set_read_timeout(Some(timeout)), s.set_write_timeout(Some(timeout))), + }; + read.and(write) + } +} + +impl Read for Stream { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + #[cfg(unix)] + Stream::Unix(s) => s.read(buf), + Stream::Tcp(s) => s.read(buf), + } + } +} + +impl Write for Stream { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + #[cfg(unix)] + Stream::Unix(s) => s.write(buf), + Stream::Tcp(s) => s.write(buf), + } + } + fn flush(&mut self) -> std::io::Result<()> { + match self { + #[cfg(unix)] + Stream::Unix(s) => s.flush(), + Stream::Tcp(s) => s.flush(), + } + } +} + +/// The server's queues (CUPS-Get-Printers) and its default destination (CUPS-Get-Default), for +/// the Print dialog when `lpstat` is missing. A default the queue list lacks is appended, so +/// the dialog always has a selection to restore; a default that cannot be named still leaves +/// the queues usable. +pub fn printers() -> Result, Error> { + printers_at(&endpoints()) +} + +pub fn printers_at(endpoints: &[Server]) -> Result, Error> { + let queues = discover(endpoints, OPERATION_CUPS_GET_PRINTERS)?; + let default = discover(endpoints, OPERATION_CUPS_GET_DEFAULT).ok().and_then(|names| names.into_iter().next()); + let mut printers: Vec = Vec::new(); + for name in queues { + if printers.len() >= MAX_PRINTERS { + break; + } + if !printers.iter().any(|p| p.name == name) { + printers.push(Printer { description: String::new(), default: default.as_deref() == Some(name.as_str()), name, two_sided: None }); + } + } + if let Some(name) = default + && !printers.iter().any(|p| p.name == name) + { + printers.push(Printer { description: String::new(), name, default: true, two_sided: None }); + } + Ok(printers) +} + +/// The printer names a discovery reply carries: every queue (CUPS-Get-Printers), or the default +/// destination (CUPS-Get-Default). +fn discover(endpoints: &[Server], operation: u16) -> Result, Error> { + let reply = post(endpoints, "/", &discovery_request(operation), DISCOVERY_TIMEOUT, true)?; + let parsed = parse_reply(&reply)?; + if status_ok(parsed.status) { Ok(parsed.names) } else { Err(refused(parsed.status)) } +} + +/// Submit the PDF as a Print-Job to the printer (the job's, else the server's default); the +/// reply is the job message, like `lp` prints it (`request id is Office-12`). The driver's own +/// job options (PPD keywords) are not sent on this path: they have no IPP equivalents without +/// reading the printer's capabilities, which is another request. +pub fn print_job(pdf: &[u8], job: &Job) -> Result { + print_job_at(&endpoints(), pdf, job) +} + +pub(crate) fn print_job_at(endpoints: &[Server], pdf: &[u8], job: &Job) -> Result { + let queue = match &job.printer { + Some(name) => name.clone(), + None => discover(endpoints, OPERATION_CUPS_GET_DEFAULT)? + .into_iter() + .next() + .ok_or_else(|| Error::Spool("the print server has no default printer; choose one in the dialog".into()))?, + }; + if !is_queue_name(&queue) { + return Err(Error::Spool(format!("not a printer name: {queue}"))); + } + let reply = post(endpoints, &format!("/printers/{queue}"), &print_job_request(&queue, pdf, job), JOB_TIMEOUT, false)?; + let parsed = parse_reply(&reply)?; + if !status_ok(parsed.status) { + return Err(refused(parsed.status)); + } + Ok(match parsed.job_id { + Some(id) => format!("request id is {queue}-{id}"), + None => format!("the job was accepted by {queue}"), + }) +} + +/// A queue class can route a job to any member printer. Keep the destination kind +/// explicit rather than guessing it from a name shared by a printer and a class. +#[derive(Clone, Debug)] +pub struct Destination { + pub name: String, + pub is_class: bool, +} + +/// List both printer queues and classes from caller-selected CUPS endpoints. +/// The caller owns endpoint selection/authorization; this API does not discover network hosts. +pub fn destinations_at(endpoints: &[Server]) -> Result, Error> { + let mut out = queues_at(endpoints, false)?; + out.extend(queues_at(endpoints, true)?); + if out.len() > MAX_PRINTERS { + return Err(Error::Spool("too many CUPS destinations".into())); + } + let default = default_at(endpoints)?; + for queue in &mut out { + queue.is_default = default.as_deref() == Some(queue.name.as_str()); + } + Ok(out) +} + +/// List either printer queues or classes, without resolving defaults. Applications can +/// explicitly choose whether an optional class/default query failure should hide otherwise +/// usable printer queues. No error is swallowed here except CUPS's absent/unsupported list. +pub fn queues_at(endpoints: &[Server], is_class: bool) -> Result, Error> { + let operation = if is_class { 0x4005 } else { OPERATION_CUPS_GET_PRINTERS }; + let reply = post(endpoints, "/", &discovery_request(operation), DISCOVERY_TIMEOUT, true)?; + let parsed = parse_reply(&reply)?; + if !status_ok(parsed.status) { + if parsed.status == 0x0406 || (is_class && parsed.status == 0x0501) { + return Ok(Vec::new()); + } + return Err(refused(parsed.status)); + } + let mut queues = parsed.queues; + for queue in &mut queues { + queue.is_class = is_class; + } + Ok(queues) +} + +/// The scheduler's default queue; not-found means no default has been configured. +pub fn default_at(endpoints: &[Server]) -> Result, Error> { + let reply = post(endpoints, "/", &discovery_request(OPERATION_CUPS_GET_DEFAULT), DISCOVERY_TIMEOUT, true)?; + let parsed = parse_reply(&reply)?; + if parsed.status == 0x0406 { + return Ok(None); + } + if !status_ok(parsed.status) { + return Err(refused(parsed.status)); + } + Ok(parsed.names.into_iter().next()) +} + +/// Metadata from one CUPS printer group, without losing which fields belong together. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Queue { + pub name: String, + pub info: String, + pub state: u32, + pub accepting_jobs: bool, + pub is_default: bool, + pub is_class: bool, +} +impl Default for Queue { + fn default() -> Self { + Self { name: String::new(), info: String::new(), state: 3, accepting_jobs: true, is_default: false, is_class: false } + } +} + +/// Submit once to an explicitly selected queue/class, returning the numeric server job id. +/// An absent id is an uncertain accepted outcome, not permission to retry the document. +pub fn submit_at(endpoints: &[Server], destination: &Destination, pdf: &[u8], job: &Job) -> Result { + submit_as_at(endpoints, destination, pdf, job, "") +} + +/// As [`submit_at`], with the user name supplied by the application, never inferred from +/// process credentials. Once any request bytes may have been sent, no other endpoint is tried. +pub fn submit_as_at(endpoints: &[Server], destination: &Destination, pdf: &[u8], job: &Job, user: &str) -> Result { + let path = destination_path(destination)?; + if user.len() > MAX_JOB_NAME || user.chars().any(char::is_control) { + return Err(Error::Spool("invalid CUPS user name".into())); + } + if job.copies == 0 || job.copies > 999 { + return Err(Error::Spool("copies must be between 1 and 999".into())); + } + let mut last = Error::Spool("no CUPS endpoint selected".into()); + for endpoint in endpoints { + let mut stream = match connect(endpoint) { + Ok(stream) => stream, + Err(error) => { + last = Error::Spool(format!("CUPS connection failed: {error}")); + continue; + } + }; + let authority = endpoint.authority(); + let request = print_job_request_for(&authority, &path, pdf, job, user); + // Never retry exchange errors: the scheduler may already have accepted the job. + let reply = exchange(&mut stream, &authority, &path, &request, JOB_TIMEOUT)?; + let parsed = parse_reply(&reply)?; + if !status_ok(parsed.status) { + return Err(refused(parsed.status)); + } + return parsed + .job_id + .filter(|id| *id != 0 && *id <= i32::MAX as u32) + .ok_or_else(|| Error::Spool("CUPS accepted the job without a usable job id; do not resubmit".into())); + } + Err(last) +} + +/// Encode one destination as a URI path segment, never as routing syntax. Names from +/// discovery may include spaces or Unicode. Reject separators, controls and dot segments +/// rather than relying on a server to keep an encoded traversal request harmless. +fn destination_path(destination: &Destination) -> Result { + let name = &destination.name; + if name.is_empty() + || name.len() > MAX_NAME + || matches!(name.as_str(), "." | "..") + || name.chars().any(|c| c.is_control() || matches!(c, '/' | '\\')) + { + return Err(Error::Spool("invalid CUPS destination name".into())); + } + let kind = if destination.is_class { "classes" } else { "printers" }; + let mut path = format!("/{kind}/"); + for byte in name.bytes() { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + path.push(char::from(byte)); + } else { + use std::fmt::Write as _; + write!(&mut path, "%{byte:02X}").map_err(|_| Error::Spool("could not encode CUPS destination".into()))?; + } + } + Ok(path) +} + +/// A CUPS queue name: ASCII letters, digits and `_`, `-`, `.` — what the scheduler accepts, and +/// what cannot break the printer URI's path. +fn is_queue_name(name: &str) -> bool { + !name.is_empty() && name.len() <= MAX_NAME && name.bytes().all(|b| b.is_ascii_alphanumeric() || matches!(b, b'_' | b'-' | b'.')) +} + +/// One IPP request and its reply body. Discovery may try each endpoint in turn; a job is given +/// to the first endpoint that accepts a connection, once: a retry could print it twice. +pub(crate) fn post(endpoints: &[Server], path: &str, body: &[u8], timeout: Duration, retry: bool) -> Result, Error> { + let mut failed: Option = None; + for server in endpoints { + let mut stream = match connect(server) { + Ok(stream) => stream, + Err(e) => { + failed = Some(Error::Spool(format!("the print server could not be reached: {e}"))); + continue; + } + }; + match exchange(&mut stream, &server.authority(), path, body, timeout) { + Ok(reply) => return Ok(reply), + Err(e) => { + failed = Some(e); + if !retry { + break; + } + } + } + } + Err(failed.unwrap_or_else(|| Error::Spool("the print server could not be reached".into()))) +} + +/// Send one IPP request and read the reply's body. +fn exchange(stream: &mut Stream, authority: &str, path: &str, body: &[u8], timeout: Duration) -> Result, Error> { + let mut stream = DeadlineReader { stream, end: Instant::now() + timeout }; + let head = format!( + "POST {path} HTTP/1.1\r\nHost: {authority}\r\nContent-Type: application/ipp\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + stream + .write_all(head.as_bytes()) + .and_then(|()| stream.write_all(body)) + .and_then(|()| stream.flush()) + .map_err(|e| Error::Spool(format!("the request could not be sent to the print server: {e}")))?; + let reply = read_reply(&mut stream)?; + validate_reply_identity(body, &reply)?; + Ok(reply) +} + +/// One absolute deadline shared by request writes and reply reads after connection. +struct DeadlineReader<'a> { + stream: &'a mut Stream, + end: Instant, +} +impl Write for DeadlineReader<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let left = self + .end + .checked_duration_since(Instant::now()) + .filter(|d| !d.is_zero()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::TimedOut, "CUPS exchange deadline exceeded"))?; + self.stream.set_timeouts(left)?; + self.stream.write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + self.stream.flush() + } +} + +fn validate_reply_identity(request: &[u8], reply: &[u8]) -> Result<(), Error> { + if reply.get(..2) != Some(&[1, 1]) || request.get(4..8).is_none() || request.get(4..8) != reply.get(4..8) { + return Err(Error::Spool("CUPS reply version or request id does not match; do not resubmit".into())); + } + Ok(()) +} + +impl Read for DeadlineReader<'_> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let left = self + .end + .checked_duration_since(Instant::now()) + .filter(|d| !d.is_zero()) + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::TimedOut, "CUPS reply deadline exceeded"))?; + #[cfg(not(target_os = "macos"))] + self.stream.set_timeouts(left)?; + #[cfg(target_os = "macos")] + if let Err(error) = self.stream.set_timeouts(left) { + // Darwin rejects timeout updates on a disconnected Unix socket even + // when a complete reply is still buffered. Never fall back to a + // blocking read: drain at most this buffer, or fail on WouldBlock. + // Every subsequent read rechecks the same absolute deadline above. + if error.raw_os_error() == Some(22) + && let Stream::Unix(stream) = self.stream + { + stream.set_nonblocking(true)?; + let result = stream.read(buf); + let restored = stream.set_nonblocking(false); + return result.and_then(|count| restored.map(|()| count)); + } + return Err(error); + } + self.stream.read(buf) + } +} + +fn read_reply(stream: &mut impl Read) -> Result, Error> { + let mut head = Vec::new(); + let mut byte = [0u8; 1]; + loop { + if head.len() >= MAX_HEADERS { + return Err(Error::Spool("the print server sent a header block that is too large".into())); + } + match stream.read(&mut byte) { + Ok(0) => return Err(Error::Spool("the print server closed the connection before sending its reply".into())), + Ok(_) => head.push(byte[0]), + Err(e) => return Err(Error::Spool(format!("the print server's reply could not be read: {e}"))), + } + if head.ends_with(b"\r\n\r\n") { + break; + } + } + let head = String::from_utf8_lossy(&head); + let Some(status_line) = head.lines().next() else { return Err(Error::Spool("the print server sent an unreadable reply".into())) }; + let code = status_line + .strip_prefix("HTTP/") + .and_then(|rest| rest.split_once(' ')) + .and_then(|(_, code)| code.split_whitespace().next()) + .and_then(|code| code.parse::().ok()) + .ok_or_else(|| Error::Spool("the print server sent an unreadable reply".into()))?; + match code { + 200..=299 => {} + 401 => return Err(Error::Spool("the print server requires authentication".into())), + 426 => return Err(Error::Spool("the print server only accepts encrypted connections".into())), + _ => return Err(Error::Spool(format!("the print server refused the request (HTTP {code})"))), + } + let mut content_length: Option = None; + let mut chunked = false; + for line in head.lines().skip(1) { + // The empty line ends the header block; any other line without a colon is malformed. + if line.is_empty() { + continue; + } + let Some((name, value)) = line.split_once(':') else { + return Err(Error::Spool("the print server sent an unreadable header".into())); + }; + match name.trim().to_ascii_lowercase().as_str() { + "content-length" => { + let len = value.trim().parse::().map_err(|_| Error::Spool("invalid Content-Length".into()))?; + if content_length.replace(len).is_some() { + return Err(Error::Spool("duplicate Content-Length".into())); + } + } + // Only `identity` is not a transfer encoding to be decoded. + "transfer-encoding" => { + if chunked || !value.trim().eq_ignore_ascii_case("chunked") { + return Err(Error::Spool("unsupported Transfer-Encoding".into())); + } + chunked = true; + } + _ => {} + } + } + if chunked { + if content_length.is_some() { + return Err(Error::Spool("ambiguous reply framing".into())); + } + return read_chunks(stream); + } + let Some(len) = content_length else { return Err(Error::Spool("the print server sent no Content-Length".into())) }; + if len > MAX_BODY { + return Err(Error::Spool(format!("the print server's reply is too large ({len} bytes)"))); + } + let mut body = vec![0u8; len]; + stream.read_exact(&mut body).map_err(|e| Error::Spool(format!("the print server's reply was cut short: {e}")))?; + Ok(body) +} + +fn read_chunks(stream: &mut impl Read) -> Result, Error> { + fn line(stream: &mut impl Read) -> Result, Error> { + let mut out = Vec::new(); + for _ in 0..MAX_HEADERS { + let mut byte = [0]; + stream.read_exact(&mut byte).map_err(|e| Error::Spool(format!("truncated chunk: {e}")))?; + out.push(byte[0]); + if out.ends_with(b"\r\n") { + out.truncate(out.len() - 2); + return Ok(out); + } + } + Err(Error::Spool("chunk line exceeds limit".into())) + } + let mut body = Vec::new(); + for _ in 0..100_000 { + let header = line(stream)?; + let size = header.split(|b| *b == b';').next().unwrap_or_default(); + if size.is_empty() || !size.iter().all(u8::is_ascii_hexdigit) { + return Err(Error::Spool("invalid chunk size".into())); + } + let len = std::str::from_utf8(size) + .ok() + .and_then(|s| usize::from_str_radix(s, 16).ok()) + .ok_or_else(|| Error::Spool("invalid chunk size".into()))?; + if len == 0 { + let mut total = 0usize; + loop { + let trailer = line(stream)?; + total = total.saturating_add(trailer.len() + 2); + if total > MAX_HEADERS { + return Err(Error::Spool("chunk trailers exceed limit".into())); + } + if trailer.is_empty() { + return Ok(body); + } + if !trailer.contains(&b':') { + return Err(Error::Spool("invalid chunk trailer".into())); + } + } + } + let end = body.len().checked_add(len).filter(|n| *n <= MAX_BODY).ok_or_else(|| Error::Spool("chunked reply exceeds limit".into()))?; + let start = body.len(); + body.resize(end, 0); + stream.read_exact(&mut body[start..]).map_err(|e| Error::Spool(format!("truncated chunk: {e}")))?; + let mut crlf = [0; 2]; + stream.read_exact(&mut crlf).map_err(|e| Error::Spool(format!("truncated chunk: {e}")))?; + if crlf != *b"\r\n" { + return Err(Error::Spool("invalid chunk terminator".into())); + } + } + Err(Error::Spool("too many chunks".into())) +} + +/// An IPP request being built: a version, an operation and a request id, then attributes, then +/// the document. +struct Request { + bytes: Vec, +} + +impl Request { + fn new(operation: u16) -> Self { + // IPP 1.1: version, operation, a request id (any nonzero value). + let mut bytes = vec![1, 1]; + bytes.extend(operation.to_be_bytes()); + bytes.extend(REQUEST_ID.fetch_add(1, Ordering::Relaxed).max(1).to_be_bytes()); + Request { bytes } + } + + fn group(&mut self, tag: u8) { + self.bytes.push(tag); + } + + fn end(&mut self) { + self.bytes.push(GROUP_END); + } + + fn attribute(&mut self, tag: u8, name: &str, value: &[u8]) { + let (Ok(name_len), Ok(value_len)) = (u16::try_from(name.len()), u16::try_from(value.len())) else { + return; + }; + self.bytes.push(tag); + self.bytes.extend(name_len.to_be_bytes()); + self.bytes.extend(name.as_bytes()); + self.bytes.extend(value_len.to_be_bytes()); + self.bytes.extend(value); + } + + /// An additional value of the previous attribute (a `1setOf`): the name is empty. + fn additional(&mut self, tag: u8, value: &str) { + self.attribute(tag, "", value.as_bytes()); + } +} + +/// A discovery request: the operation's required attributes, then the names to be answered with. +fn discovery_request(operation: u16) -> Vec { + let mut r = Request::new(operation); + r.group(GROUP_OPERATION); + r.attribute(TAG_CHARSET, "attributes-charset", b"utf-8"); + r.attribute(TAG_LANGUAGE, "attributes-natural-language", b"en"); + r.attribute(TAG_URI, "printer-uri", b"ipp://localhost/"); + r.attribute(TAG_KEYWORD, "requested-attributes", b"printer-name"); + r.additional(TAG_KEYWORD, "printer-info"); + r.additional(TAG_KEYWORD, "printer-state"); + r.additional(TAG_KEYWORD, "printer-is-accepting-jobs"); + r.end(); + r.bytes +} + +/// A Print-Job request for `queue`: the job's options as IPP attributes, the PDF as the document +/// data that follows the end of attributes. +fn print_job_request(queue: &str, pdf: &[u8], job: &Job) -> Vec { + print_job_request_path(&format!("/printers/{queue}"), pdf, job) +} + +fn print_job_request_path(path: &str, pdf: &[u8], job: &Job) -> Vec { + print_job_request_for("localhost", path, pdf, job, "") +} + +fn print_job_request_for(authority: &str, path: &str, pdf: &[u8], job: &Job, user: &str) -> Vec { + let mut r = Request::new(OPERATION_PRINT_JOB); + r.group(GROUP_OPERATION); + r.attribute(TAG_CHARSET, "attributes-charset", b"utf-8"); + r.attribute(TAG_LANGUAGE, "attributes-natural-language", b"en"); + r.attribute(TAG_URI, "printer-uri", format!("ipp://{authority}{path}").as_bytes()); + if !user.is_empty() { + r.attribute(TAG_NAME, "requesting-user-name", user.as_bytes()); + } + r.attribute(TAG_NAME, "job-name", truncate_utf8(&job.title, MAX_JOB_NAME).as_bytes()); + r.attribute(TAG_MIME, "document-format", b"application/pdf"); + r.group(GROUP_JOB); + r.attribute(TAG_INTEGER, "copies", &job.copies.clamp(1, 999).to_be_bytes()); + r.attribute(TAG_BOOLEAN, "collate", &[u8::from(job.collate)]); + if let Some(duplex) = job.duplex { + r.attribute( + TAG_KEYWORD, + "sides", + match duplex { + Duplex::Off => "one-sided", + Duplex::LongEdge => "two-sided-long-edge", + Duplex::ShortEdge => "two-sided-short-edge", + } + .as_bytes(), + ); + } + if job.grayscale { + r.attribute(TAG_KEYWORD, "print-color-mode", b"monochrome"); + } + // PWG 5100.13 §6.2.5: `none` keeps every sheet at its size, `auto-fit` shrinks only + // oversized ones — the same choices `lp`'s fit-to-page and print-scaling options make. + r.attribute( + TAG_KEYWORD, + "print-scaling", + match job.scaling { + Scaling::Exact => "none", + Scaling::ShrinkToFit => "auto-fit", + } + .as_bytes(), + ); + r.end(); + r.bytes.extend(pdf); + r.bytes +} + +/// The string cut to at most `max` bytes, at a character boundary. +fn truncate_utf8(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut end = max; + while !s.is_char_boundary(end) { + end -= 1; + } + // `end` is a checked boundary; nothing is the only name that cannot be cut. + s.get(..end).unwrap_or_default() +} + +/// The parsed parts of a reply: the IPP status, the printer names in its printer groups, and a +/// submission's job id. +struct Reply { + status: u16, + names: Vec, + queues: Vec, + job_id: Option, +} + +/// A cursor over a reply: `None` at the end or past a length, never an over-read. +struct Reader<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Reader<'a> { + fn take(&mut self, len: usize) -> Option<&'a [u8]> { + let end = self.pos.checked_add(len)?; + let taken = self.bytes.get(self.pos..end)?; + self.pos = end; + Some(taken) + } + + fn array(&mut self) -> Option<[u8; N]> { + <[u8; N]>::try_from(self.take(N)?).ok() + } + + fn u16(&mut self) -> Option { + self.array::<2>().map(u16::from_be_bytes) + } +} + +fn parse_reply(reply: &[u8]) -> Result { + let malformed = || Error::Spool("the print server sent a malformed reply".into()); + let mut r = Reader { bytes: reply, pos: 0 }; + let _version = r.array::<2>().ok_or_else(malformed)?; + let status = r.u16().ok_or_else(malformed)?; + let _request_id = r.array::<4>().ok_or_else(malformed)?; + let (mut names, mut job_id, mut group) = (Vec::new(), None, 0u8); + let mut queues = Vec::new(); + let mut queue = Queue::default(); + loop { + let Some(tag) = r.array::<1>().map(|t| t[0]) else { return Err(malformed()) }; + match tag { + GROUP_END => { + if !queue.name.is_empty() { + queues.push(queue); + } + break; + } + GROUP_OPERATION | GROUP_JOB | GROUP_PRINTER | 0x05 => { + if !queue.name.is_empty() { + if queues.len() >= MAX_PRINTERS { + return Err(malformed()); + } + queues.push(std::mem::take(&mut queue)); + } else { + queue = Queue::default(); + } + group = tag; + } + // An attribute: name and value, each u16-length. An empty name is an additional + // value of the previous one (the attributes read here are single-valued, so those + // are ignored). Group tags not in the registry and collections (the attributes + // requested here are never collections) are refused, not guessed. + 0x10..=0x33 | 0x35..=0x4A => { + let name_len = r.u16().ok_or_else(malformed)?; + if usize::from(name_len) > MAX_NAME { + return Err(malformed()); + } + let name_bytes = r.take(usize::from(name_len)).ok_or_else(malformed)?; + let name = std::str::from_utf8(name_bytes).map_err(|_| malformed())?; + let value_len = r.u16().ok_or_else(malformed)?; + let value = r.take(usize::from(value_len)).ok_or_else(malformed)?; + match (group, name, tag) { + (GROUP_PRINTER, "printer-name", TAG_NAME) => { + if let Some(printer) = std::str::from_utf8(value).ok().filter(|n| !n.is_empty() && n.len() <= MAX_NAME) { + if names.len() >= MAX_PRINTERS { + return Err(Error::Spool("too many CUPS destinations".into())); + } + if !queue.name.is_empty() { + return Err(malformed()); + } + queue.name = printer.to_string(); + names.push(printer.to_string()); + } + } + (GROUP_PRINTER, "printer-info", 0x41 | TAG_NAME) => { + // Human descriptions are text, not queue identifiers. Their u16 + // wire length and the whole-response MAX_BODY cap bound allocation. + queue.info = std::str::from_utf8(value).map_err(|_| malformed())?.to_owned(); + } + (GROUP_PRINTER, "printer-state", 0x23 | TAG_INTEGER) => { + queue.state = u32::from_be_bytes(value.try_into().map_err(|_| malformed())?); + } + (GROUP_PRINTER, "printer-is-accepting-jobs", TAG_BOOLEAN) => { + queue.accepting_jobs = match value { + [0] => false, + [1] => true, + _ => return Err(malformed()), + }; + } + (GROUP_JOB, "job-id", TAG_INTEGER) => { + if let Ok(id) = <[u8; 4]>::try_from(value) { + job_id = Some(u32::from_be_bytes(id)); + } + } + _ => {} + } + } + _ => return Err(malformed()), + } + } + Ok(Reply { status, names, queues, job_id }) +} + +/// Successful IPP status codes are `0x00xx` (RFC 8011 Appendix B). +fn status_ok(status: u16) -> bool { + status < 0x0100 +} + +/// The IPP status code's name, for the error the user sees. +fn status_name(status: u16) -> Option<&'static str> { + Some(match status { + 0x0400 => "bad request", + 0x0401 => "forbidden", + 0x0402 => "not authenticated", + 0x0403 => "not authorized", + 0x0404 => "not possible", + 0x0406 => "printer not found", + 0x0408 => "request too large", + 0x040A => "PDF is not a supported document format for this printer", + 0x0412 => "the document could not be accessed", + 0x0500 => "internal error", + 0x0501 => "operation not supported", + 0x0502 => "service unavailable", + 0x0503 => "IPP version not supported", + 0x0504 => "device error", + 0x0505 => "temporary error", + 0x0506 => "the printer is not accepting jobs", + 0x0507 => "the printer is busy", + _ => return None, + }) +} + +fn refused(status: u16) -> Error { + match status_name(status) { + Some(name) => Error::Spool(format!("the print server refused the request ({name})")), + None => Error::Spool(format!("the print server refused the request (IPP status 0x{status:04X})")), + } +} diff --git a/crates/print/src/cups/tests.rs b/crates/print/src/cups/tests.rs new file mode 100644 index 000000000..978246e56 --- /dev/null +++ b/crates/print/src/cups/tests.rs @@ -0,0 +1,593 @@ +//! The IPP client against a fake CUPS server (a unix socket or a loopback port): discovery, +//! Print-Job, and the malformed replies it must refuse without a panic. + +use super::*; +use crate as spool; +use crate::cups::{self as ipp, Server}; +use std::collections::VecDeque; +use std::io::{Read, Write}; +use std::sync::Mutex; +use std::sync::mpsc::{self, Receiver}; +use std::time::Duration; + +const TIMEOUT: Duration = Duration::from_secs(2); + +/// A fake CUPS server: each connection carries one request; the next canned reply goes back +/// (nothing, when the list is out — the client then sees a hang-up). Every request is recorded +/// for the assertions. +struct FakeCups { + server: Server, + requests: Receiver>, +} + +impl FakeCups { + fn on_unix(replies: Vec>) -> std::io::Result { + let dir = tempfile::tempdir()?; + let path = dir.path().join("cups.sock"); + let listener = std::os::unix::net::UnixListener::bind(&path)?; + // The socket must outlive this function; the folder is left for the process to reap. + std::mem::forget(dir); + let requests = listen_unix(listener, replies); + Ok(FakeCups { server: Server::Unix(path), requests }) + } + + fn on_tcp(replies: Vec>) -> std::io::Result { + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let server = Server::Tcp(listener.local_addr()?); + let requests = listen_tcp(listener, replies); + Ok(FakeCups { server, requests }) + } + + fn request(&self) -> Result, mpsc::RecvTimeoutError> { + self.requests.recv_timeout(Duration::from_secs(5)) + } + + /// The server as the one-endpoint list the client functions take. + fn endpoint(&self) -> &[Server] { + std::slice::from_ref(&self.server) + } +} + +fn listen_unix(listener: std::os::unix::net::UnixListener, replies: Vec>) -> Receiver> { + let (tx, rx) = mpsc::channel(); + let replies = Mutex::new(VecDeque::from(replies)); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + serve(stream, &replies, &tx); + } + }); + rx +} + +fn listen_tcp(listener: std::net::TcpListener, replies: Vec>) -> Receiver> { + let (tx, rx) = mpsc::channel(); + let replies = Mutex::new(VecDeque::from(replies)); + std::thread::spawn(move || { + for stream in listener.incoming().flatten() { + serve(stream, &replies, &tx); + } + }); + rx +} + +fn serve(mut stream: impl Read + Write, replies: &Mutex>>, requests: &mpsc::Sender>) { + let Some(request) = read_request(&mut stream) else { return }; + let Some(body) = ipp_body(&request) else { return }; + let request_id = body.get(4..8).map(|id| id.to_vec()); + let _ = requests.send(request); + // A list out of replies is a server that hangs up before answering. + let Ok(mut replies) = replies.lock() else { return }; + if let Some(mut reply) = replies.pop_front() { + if let Some(body) = reply.windows(4).position(|w| w == b"\r\n\r\n").map(|at| at + 4) + && reply.get(body..body + 2) == Some(&[1, 1]) + && let Some(id) = request_id + && let Some(destination) = reply.get_mut(body + 4..body + 8) + { + destination.copy_from_slice(&id); + } + let _ = stream.write_all(&reply); + let _ = stream.flush(); + } +} + +/// One HTTP request: the headers (bounded), then the `Content-Length` body it announces. +fn read_request(stream: &mut impl Read) -> Option> { + let mut buf = Vec::new(); + let mut byte = [0u8; 1]; + loop { + match stream.read(&mut byte) { + Ok(0) | Err(_) => return None, + Ok(_) => buf.push(byte[0]), + } + if buf.ends_with(b"\r\n\r\n") { + break; + } + if buf.len() > 1 << 16 { + return None; + } + } + let head = String::from_utf8_lossy(&buf); + let len: usize = head.lines().find_map(|l| l.strip_prefix("Content-Length:")).and_then(|v| v.trim().parse().ok())?; + let head_len = buf.len(); + buf.resize(head_len + len, 0); + stream.read_exact(&mut buf[head_len..]).ok()?; + Some(buf) +} + +/// An HTTP reply carrying `body`, as CUPS would send it. +fn http_reply(body: &[u8]) -> Vec { + let mut out = format!("HTTP/1.1 200 OK\r\nContent-Type: application/ipp\r\nContent-Length: {}\r\n\r\n", body.len()).into_bytes(); + out.extend(body); + out +} + +/// An IPP reply body: `status`, then the attribute groups as raw bytes, then end of attributes. +fn ipp_reply(status: u16, groups: &[&[u8]]) -> Vec { + let mut out = vec![1, 1]; + out.extend(status.to_be_bytes()); + out.extend(1u32.to_be_bytes()); + for &group in groups { + out.extend(group); + } + out.push(0x03); + out +} + +fn group(tag: u8) -> Vec { + vec![tag] +} + +fn printer_name(name: &str) -> Vec { + attr(0x42, "printer-name", name.as_bytes()) +} + +/// One attribute, framed the way the client frames them, so `has` finds it in a request. +fn attr(tag: u8, name: &str, value: &[u8]) -> Vec { + let mut out = vec![tag]; + out.extend((name.len() as u16).to_be_bytes()); + out.extend(name.as_bytes()); + out.extend((value.len() as u16).to_be_bytes()); + out.extend(value); + out +} + +fn has(bytes: &[u8], needle: &[u8]) -> bool { + !needle.is_empty() && bytes.windows(needle.len()).any(|w| w == needle) +} + +/// The IPP part of a recorded request: everything after the HTTP head. +fn ipp_body(request: &[u8]) -> Option<&[u8]> { + let at = request.windows(4).position(|w| w == b"\r\n\r\n")?; + request.get(at + 4..) +} + +fn entry(name: &str, default: bool) -> spool::Printer { + spool::Printer { description: String::new(), name: name.into(), default, two_sided: None } +} + +#[test] +fn discovery_finds_queues_and_the_default_over_a_unix_socket() { + let fake = FakeCups::on_unix(vec![ + http_reply(&ipp_reply( + 0x0000, + &[ + &group(0x01), + &printer_name("Office_Laser"), + &group(0x04), + &printer_name("Office_Laser"), + &group(0x04), + &printer_name("Label_Writer"), + &group(0x04), + &printer_name("Office_Laser"), + ], + )), + http_reply(&ipp_reply(0x0000, &[&group(0x01), &group(0x04), &printer_name("Office_Laser")])), + ]) + .unwrap(); + let printers = ipp::printers_at(fake.endpoint()).unwrap(); + assert_eq!(printers, [entry("Office_Laser", true), entry("Label_Writer", false)], "each queue once, the default marked"); + let taken = fake.request().unwrap(); + let queues = ipp_body(&taken).unwrap(); + assert_eq!(&queues[..4], &[1, 1, 0x40, 0x02], "IPP 1.1, CUPS-Get-Printers"); + assert!(has(queues, b"ipp://localhost/")); + assert!(has(queues, &attr(0x44, "requested-attributes", b"printer-name"))); + assert!(has(queues, b"printer-is-accepting-jobs"), "only the attributes the dialog needs are asked for"); + let taken = fake.request().unwrap(); + let default = ipp_body(&taken).unwrap(); + assert_eq!(&default[..4], &[1, 1, 0x40, 0x01], "CUPS-Get-Default"); +} + +#[test] +fn a_print_job_carries_the_pdf_and_the_options() { + let pdf: Vec = b"%PDF-1.7 fake print job\n".iter().copied().cycle().take(1 << 20).collect(); + let job = spool::Job { + printer: Some("Office".into()), + copies: 2, + collate: false, + duplex: Some(spool::Duplex::LongEdge), + grayscale: true, + scaling: spool::Scaling::ShrinkToFit, + title: "memo.pdf".into(), + ..Job::default() + }; + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply( + 0x0000, + &[&group(0x01), &group(0x02), &attr(0x21, "job-id", &12u32.to_be_bytes()), &attr(0x23, "job-state", &[3u8])], + ))]) + .unwrap(); + let message = ipp::print_job_at(fake.endpoint(), &pdf, &job).unwrap(); + assert_eq!(message, "request id is Office-12", "the same message lp prints"); + let taken = fake.request().unwrap(); + let request = ipp_body(&taken).unwrap(); + assert_eq!(&request[..4], &[1, 1, 0x00, 0x02], "IPP 1.1, Print-Job"); + assert!(has(request, b"ipp://localhost/printers/Office")); + let mut copies = group(0x02); + copies.extend(attr(0x21, "copies", &2u32.to_be_bytes())); + assert!(has(request, &copies), "copies are a job attribute"); + assert!(has(request, &attr(0x22, "collate", &[0]))); + assert!(has(request, &attr(0x44, "sides", b"two-sided-long-edge"))); + assert!(has(request, &attr(0x44, "print-color-mode", b"monochrome"))); + assert!(has(request, &attr(0x44, "print-scaling", b"auto-fit"))); + assert!(has(request, &attr(0x49, "document-format", b"application/pdf"))); + assert!(has(request, &attr(0x42, "job-name", b"memo.pdf"))); + assert!(request.ends_with(&pdf), "the whole job is the document data"); +} + +#[test] +fn the_default_printer_is_used_when_the_job_has_none() { + let fake = FakeCups::on_tcp(vec![ + http_reply(&ipp_reply(0x0000, &[&group(0x01), &group(0x04), &printer_name("Office")])), + http_reply(&ipp_reply(0x0000, &[&group(0x01), &group(0x02), &attr(0x21, "job-id", &7u32.to_be_bytes())])), + ]) + .unwrap(); + let message = ipp::print_job_at(fake.endpoint(), b"%PDF-1.7\n", &Job::default()).unwrap(); + assert_eq!(message, "request id is Office-7"); + let default = fake.request().unwrap(); + assert_eq!(&ipp_body(&default).unwrap()[..4], &[1, 1, 0x40, 0x01], "the default comes from CUPS-Get-Default"); + let taken = fake.request().unwrap(); + assert!(has(ipp_body(&taken).unwrap(), b"ipp://localhost/printers/Office"), "the resolved default is the target"); +} + +#[test] +fn a_default_outside_the_queue_list_still_gets_its_marker() { + let fake = FakeCups::on_unix(vec![ + http_reply(&ipp_reply(0x0000, &[&group(0x01), &group(0x04), &printer_name("Office_Laser")])), + http_reply(&ipp_reply(0x0000, &[&group(0x01), &group(0x04), &printer_name("Label_Writer")])), + ]) + .unwrap(); + let printers = ipp::printers_at(fake.endpoint()).unwrap(); + assert_eq!( + printers, + [entry("Office_Laser", false), entry("Label_Writer", true)], + "the default joins the list, so the dialog has a selection to restore" + ); +} + +#[test] +fn no_default_printer_is_an_error_when_the_job_names_none() { + let fake = FakeCups::on_unix(vec![http_reply(&ipp_reply(0x0000, &[&group(0x01)]))]).unwrap(); + let err = ipp::print_job_at(fake.endpoint(), b"%PDF-1.7\n", &Job::default()).unwrap_err(); + assert!(err.to_string().contains("no default printer"), "{err}"); +} + +#[test] +fn a_printer_name_that_cannot_be_a_uri_path_is_refused() { + let fake = FakeCups::on_tcp(vec![]).unwrap(); + let mut long = "x".repeat(257); + long.push('y'); + for name in ["Office (east)", "../../admin", "", "a b", "Brother/HL", "Копир", long.as_str()] { + let job = spool::Job { printer: Some(name.into()), ..Job::default() }; + assert_eq!( + ipp::print_job_at(fake.endpoint(), b"pdf", &job), + Err(spool::Error::Spool(format!("not a printer name: {name}"))), + "{name:?} cannot go in the printer URI's path" + ); + } +} + +#[test] +fn an_ipp_refusal_is_an_error_that_names_the_problem() { + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0x0506, &[&group(0x01)]))]).unwrap(); + let job = spool::Job { printer: Some("Office".into()), ..Job::default() }; + assert_eq!( + ipp::print_job_at(fake.endpoint(), b"%PDF-1.7\n", &job), + Err(spool::Error::Spool("the print server refused the request (the printer is not accepting jobs)".into())) + ); +} + +#[test] +fn malformed_replies_are_errors_never_a_panic() { + let bodies: Vec<(&str, Vec)> = vec![ + // Shorter than an IPP header. + ("shorter than an IPP header", vec![1, 1, 0]), + // A reply that ends inside an attribute: the name is announced, the value never comes. + ("cut inside an attribute", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0x42, 0x00, 0x20, b'p']), + // A value tag the registry does not know. + ("unknown value tag", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0xFF, 0x00, 0x00, 0x00, 0x00]), + // A collection, which the attributes asked for never are. + ("collection", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0x34, 0x00, 0x00, 0x00, 0x00]), + // A group tag that is no delimiter in the registry. + ("reserved delimiter", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x06]), + // A length within the cap that runs past the reply, and one past the cap. + ("length past the reply", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0x42, 0x00, 0xFE, b'p']), + ("length past the cap", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0x42, 0xFF, 0xFF, b'p']), + // No end of attributes before the body runs out. + ("no end of attributes", vec![1, 1, 0, 0, 0, 0, 0, 1, 0x04, 0x42, 0x00, 0x01, b'x']), + ]; + for (why, body) in bodies { + let fake = FakeCups::on_unix(vec![http_reply(&body)]).unwrap(); + let result = ipp::printers_at(fake.endpoint()); + assert!(result.is_err(), "{why}: the reply must be an error, not a panic or a success"); + } +} + +#[test] +fn http_replies_that_cannot_carry_an_answer_are_refused() { + let replies: Vec<(&str, Vec)> = vec![ + ("requires authentication", b"HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\n\r\n".to_vec()), + ("demands a TLS upgrade", b"HTTP/1.1 426 Upgrade Required\r\nContent-Length: 0\r\n\r\n".to_vec()), + ("chunked", b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n25\r\nthe body is chunked\r\n0\r\n\r\n".to_vec()), + ("no Content-Length", b"HTTP/1.1 200 OK\r\nContent-Type: application/ipp\r\n\r\n".to_vec()), + ("Content-Length past the cap", b"HTTP/1.1 200 OK\r\nContent-Length: 99999999\r\n\r\n".to_vec()), + ("unreadable status line", b"not HTTP at all\r\n\r\n".to_vec()), + ]; + for (why, reply) in replies { + let fake = FakeCups::on_unix(vec![reply]).unwrap(); + let result = ipp::post(&[fake.server], "/", b"ignored", TIMEOUT, true); + assert!(result.is_err(), "{why}: the reply must be an error, not a panic or a success"); + } + // A server that takes the request and hangs up. + let fake = FakeCups::on_unix(vec![]).unwrap(); + assert!(ipp::post(&[fake.server], "/", b"ignored", TIMEOUT, true).is_err(), "a hang-up is an error"); +} + +#[test] +fn a_server_that_answers_is_reachable_and_silence_is_not() { + let fake = FakeCups::on_unix(vec![]).unwrap(); + assert!(ipp::reachable(&fake.server), "a unix socket that accepts connections is reachable"); + let fake = FakeCups::on_tcp(vec![]).unwrap(); + assert!(ipp::reachable(&fake.server), "a loopback port that accepts connections is reachable"); + let absent = Server::Unix("/nonexistent/craft-print-test.sock".into()); + assert!(!ipp::reachable(&absent), "a socket nothing listens on is not reachable"); + let addr = { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + listener.local_addr().unwrap() + }; + assert!(!ipp::reachable(&Server::Tcp(addr)), "a port nothing listens on is not reachable"); +} + +#[test] +fn chunked_reply_is_bounded_and_rejects_ambiguous_framing() { + let good = b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n"; + assert_eq!(super::read_reply(&mut &good[..]).unwrap(), b"abcde"); + for bad in [ + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\nContent-Length: 0\r\n\r\n0\r\n\r\n".as_slice(), + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n100001\r\n", + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\na", + b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n", + ] { + assert!(super::read_reply(&mut &bad[..]).is_err()); + } +} + +#[test] +fn explicit_server_rejects_header_injection_and_malformed_port() { + for value in ["host\r\nInjected: yes", "host:0", "host:99999", "http://host", "user@host", ""] { + assert!(Server::parse(value).is_err(), "{value:?}"); + } + assert_eq!(Server::parse("[::1]:631").unwrap().authority(), "[::1]:631"); + assert_eq!(Server::parse("print.example:8631").unwrap().authority(), "print.example:8631"); +} + +#[test] +fn class_submission_preserves_authority_user_and_numeric_job_id() { + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0, &[&group(2), &attr(0x21, "job-id", &42u32.to_be_bytes())]))]).unwrap(); + let destination = super::Destination { name: "Studio".into(), is_class: true }; + assert_eq!(super::submit_as_at(fake.endpoint(), &destination, b"pdf", &Job::default(), "artist").unwrap(), 42); + let request = fake.request().unwrap(); + assert!(has(&request, b"POST /classes/Studio HTTP/1.1")); + assert!(has(&request, format!("Host: {}", fake.server.authority()).as_bytes())); + assert!(has(ipp_body(&request).unwrap(), format!("ipp://{}/classes/Studio", fake.server.authority()).as_bytes())); + assert!(has(ipp_body(&request).unwrap(), &attr(0x42, "requesting-user-name", b"artist"))); +} + +#[test] +fn uncertain_submission_never_tries_the_second_endpoint() { + let first = FakeCups::on_tcp(vec![]).unwrap(); + let second = FakeCups::on_tcp(vec![]).unwrap(); + let destination = super::Destination { name: "Studio".into(), is_class: false }; + assert!(super::submit_at(&[first.server.clone(), second.server.clone()], &destination, b"pdf", &Job::default()).is_err()); + let _ = first.request().unwrap(); + assert!(second.requests.recv_timeout(Duration::from_millis(100)).is_err()); +} + +#[test] +fn queue_metadata_stays_with_its_printer_group() { + let bytes = ipp_reply( + 0, + &[ + &group(4), + &attr(0x41, "printer-info", b"Studio printer"), + &printer_name("First"), + &attr(0x23, "printer-state", &5u32.to_be_bytes()), + &attr(0x22, "printer-is-accepting-jobs", &[0]), + &group(4), + &printer_name("Second"), + ], + ); + let reply = super::parse_reply(&bytes).unwrap(); + assert_eq!(reply.queues.len(), 2); + assert_eq!(reply.queues[0].info, "Studio printer"); + assert_eq!(reply.queues[0].state, 5); + assert!(!reply.queues[0].accepting_jobs); + assert!(reply.queues[1].info.is_empty()); + assert!(reply.queues[1].accepting_jobs); +} + +#[test] +fn reply_identity_and_metadata_request_are_checked() { + let request = super::discovery_request(0x4002); + assert!(has(&request, b"printer-info")); + let mut reply = vec![1, 1, 0, 0]; + reply.extend_from_slice(&request[4..8]); + reply.push(3); + assert!(super::validate_reply_identity(&request, &reply).is_ok()); + reply[7] ^= 1; + assert!(super::validate_reply_identity(&request, &reply).is_err()); + reply[4..8].copy_from_slice(&request[4..8]); + reply[0] = 9; + assert!(super::validate_reply_identity(&request, &reply).is_err()); +} + +#[test] +fn slow_reply_cannot_reset_the_absolute_deadline() { + let (client, mut peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let producer = std::thread::spawn(move || { + // Finite work and a joined handle: no abandoned timeout worker. + for _ in 0..20 { + if peer.write_all(b"x").is_err() { + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + }); + let start = std::time::Instant::now(); + let mut stream = super::Stream::Unix(client); + let mut reader = super::DeadlineReader { stream: &mut stream, end: start + Duration::from_millis(80) }; + let mut bytes = [0; 20]; + assert!(reader.read_exact(&mut bytes).is_err(), "trickle reads must not extend the absolute deadline"); + drop(stream); + producer.join().unwrap(); + assert!(start.elapsed() < Duration::from_secs(3)); +} + +#[test] +fn blocked_upload_obeys_the_same_absolute_deadline() { + let (client, _peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let start = std::time::Instant::now(); + let mut stream = super::Stream::Unix(client); + let mut writer = super::DeadlineReader { stream: &mut stream, end: start + Duration::from_millis(80) }; + let chunk = [0u8; 64 * 1024]; + let mut timed_out = false; + // A peer that never drains cannot turn each partial write into a fresh timeout budget. + for _ in 0..512 { + if writer.write_all(&chunk).is_err() { + timed_out = true; + break; + } + } + assert!(timed_out, "blocked upload must terminate without a reader"); + assert!(start.elapsed() < Duration::from_secs(3)); +} + +#[test] +fn recorded_request_body_requires_complete_http_headers() { + assert!(ipp_body(b"POST / HTTP/1.1\r\n").is_none()); + assert_eq!(ipp_body(b"POST / HTTP/1.1\r\n\r\nbody"), Some(b"body".as_slice())); +} + +#[test] +fn named_destinations_encode_one_path_segment_on_both_wire_fields() { + for is_class in [false, true] { + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0, &[&group(2), &attr(0x21, "job-id", &42u32.to_be_bytes())]))]).unwrap(); + let destination = super::Destination { name: "Office café%20?#".into(), is_class }; + assert_eq!(super::submit_as_at(fake.endpoint(), &destination, b"pdf", &Job::default(), "artist").unwrap(), 42); + let kind = if is_class { "classes" } else { "printers" }; + let path = format!("/{kind}/Office%20caf%C3%A9%2520%3F%23"); + let request = fake.request().unwrap(); + assert!(has(&request, format!("POST {path} HTTP/1.1").as_bytes())); + assert!(has(ipp_body(&request).unwrap(), format!("ipp://{}{path}", fake.server.authority()).as_bytes())); + } +} + +#[test] +fn destination_path_rejects_routing_syntax_before_connecting() { + for name in ["", ".", "..", "../admin", "a/b", "a\\b", "a\r\nb", "a\0b"] { + let destination = super::Destination { name: name.into(), is_class: false }; + assert!(super::destination_path(&destination).is_err(), "{name:?}"); + } + let destination = super::Destination { name: "x".repeat(257), is_class: false }; + assert!(super::destination_path(&destination).is_err()); +} + +#[test] +fn separate_queue_query_preserves_optional_discovery_errors() { + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0x0403, &[&group(1)]))]).unwrap(); + assert!(super::queues_at(fake.endpoint(), true).is_err(), "authorization refusal is not an empty class list"); + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0x0501, &[&group(1)]))]).unwrap(); + assert!(super::queues_at(fake.endpoint(), true).unwrap().is_empty(), "unsupported class operation is absent"); + let fake = FakeCups::on_tcp(vec![http_reply(&ipp_reply(0, &[&group(1), &group(4), &printer_name("Pool")]))]).unwrap(); + let queues = super::queues_at(fake.endpoint(), true).unwrap(); + assert_eq!(queues.len(), 1); + assert!(queues[0].is_class); + assert!(!queues[0].is_default, "default resolution is caller-owned for this API"); + let request = fake.request().unwrap(); + assert_eq!(&ipp_body(&request).unwrap()[..4], &[1, 1, 0x40, 0x05]); +} + +#[test] +fn ipv6_endpoint_keeps_scope_for_resolution_and_escapes_wire_authority() { + assert_eq!(Server::parse("[::1]").unwrap().authority(), "[::1]:631"); + assert_eq!(Server::parse("::1").unwrap().authority(), "[::1]:631"); + let scoped = Server::parse("[fe80::1%en0]:8631").unwrap(); + assert!(matches!(&scoped, Server::Named(host, 8631) if host == "fe80::1%en0")); + assert_eq!(scoped.authority(), "[fe80::1%25en0]:8631"); + assert_eq!(Server::parse("fe80::1%en0:8631").unwrap().authority(), scoped.authority()); + assert_eq!(Server::parse("fe80::1%en0").unwrap().authority(), "[fe80::1%25en0]:631"); + assert!(Server::parse("[host]:631").is_err()); + assert_eq!(super::connect(&Server::Named("/invalid".into(), 631)).err().unwrap().kind(), std::io::ErrorKind::InvalidInput); + for bad in ["[::1]junk", "[::1]:0", "[::1]:abc", "[fe80::1%]:631", "[fe80::1%x%y]:631", "host%zone"] { + assert!(Server::parse(bad).is_err(), "{bad}"); + } +} + +#[test] +fn long_printer_descriptions_are_not_limited_like_queue_names() { + let description = "Studio ".repeat(200); + let body = ipp_reply(0, &[&group(1), &group(4), &printer_name("Studio"), &attr(0x41, "printer-info", description.as_bytes())]); + let parsed = super::parse_reply(&body).unwrap(); + assert_eq!(parsed.queues[0].info, description); +} + +#[test] +fn destination_limit_is_explicit_and_never_silently_truncates() { + let mut body = vec![1, 1, 0, 0, 0, 0, 0, 1]; + for index in 0..super::MAX_PRINTERS { + body.push(4); + body.extend(printer_name(&format!("Printer{index}"))); + } + body.push(3); + assert_eq!(super::parse_reply(&body).unwrap().queues.len(), super::MAX_PRINTERS); + assert_eq!(body.pop(), Some(3)); + body.push(4); + body.extend(printer_name("OneTooMany")); + body.push(3); + assert!(super::parse_reply(&body).err().unwrap().to_string().contains("too many CUPS destinations")); +} + +#[test] +fn closed_unix_peer_keeps_its_buffered_reply_readable_with_a_deadline() { + let (client, mut peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let body = ipp_reply(0, &[&group(1), &group(4), &printer_name("Studio")]); + peer.write_all(&http_reply(&body)).unwrap(); + drop(peer); + let mut stream = super::Stream::Unix(client); + let mut reader = super::DeadlineReader { stream: &mut stream, end: std::time::Instant::now() + Duration::from_secs(1) }; + assert_eq!(super::read_reply(&mut reader).unwrap(), body); + let mut byte = [0]; + assert_eq!(reader.read(&mut byte).unwrap(), 0); + reader.end = std::time::Instant::now(); + assert_eq!(reader.read(&mut byte).unwrap_err().kind(), std::io::ErrorKind::TimedOut); +} + +#[test] +fn stalled_unix_peer_still_fails_within_the_absolute_read_deadline() { + let (client, _peer) = std::os::unix::net::UnixStream::pair().unwrap(); + let start = std::time::Instant::now(); + let mut stream = super::Stream::Unix(client); + let mut reader = super::DeadlineReader { stream: &mut stream, end: start + Duration::from_millis(80) }; + let mut byte = [0]; + let error = reader.read(&mut byte).unwrap_err(); + assert!(matches!(error.kind(), std::io::ErrorKind::TimedOut | std::io::ErrorKind::WouldBlock)); + assert!(start.elapsed() < Duration::from_secs(3)); +} diff --git a/crates/print/src/lib.rs b/crates/print/src/lib.rs index dcc9c53f9..bf50e3a3f 100644 --- a/crates/print/src/lib.rs +++ b/crates/print/src/lib.rs @@ -8,6 +8,10 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +/// Direct local-CUPS transport, independent of command-line spooler selection. +#[cfg(not(target_arch = "wasm32"))] +pub mod cups; + #[cfg(any(test, windows))] mod progress; #[cfg(any(test, windows))] From 4cf2a6f4cb26bc0503191049edccf85d4d0b21a8 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 13:06:02 +0000 Subject: [PATCH 58/98] Inspect seekable archive entry budgets and overlaps --- crates/zip/src/lib.rs | 2 + crates/zip/src/seekable.rs | 75 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 crates/zip/src/seekable.rs diff --git a/crates/zip/src/lib.rs b/crates/zip/src/lib.rs index 73eb9840a..0a348dc1f 100644 --- a/crates/zip/src/lib.rs +++ b/crates/zip/src/lib.rs @@ -25,6 +25,7 @@ mod archive; mod guard; mod metadata_reader; pub mod preflight; +mod seekable; mod write; pub use archive::{Archive, Limits}; @@ -32,6 +33,7 @@ pub use guard::check_entry_count; #[cfg(any(test, feature = "test-support"))] pub use guard::zip64_entry_count_sample; pub use metadata_reader::{Indexing, MetadataReader}; +pub use seekable::{InspectError, inspect_seekable}; pub use write::{Compression, Writer}; /// The payload of a ZIP's first entry when it is a stored file named `mimetype` of at most 256 diff --git a/crates/zip/src/seekable.rs b/crates/zip/src/seekable.rs new file mode 100644 index 000000000..2750a0123 --- /dev/null +++ b/crates/zip/src/seekable.rs @@ -0,0 +1,75 @@ +//! Checks for an already indexed, seekable ZIP archive. + +use std::io::{Read, Seek}; + +/// Why an indexed seekable archive exceeded its caller's limits. +#[derive(Debug, thiserror::Error)] +pub enum InspectError { + #[error("the archive has {actual} entries, more than {limit}")] + TooManyEntries { actual: usize, limit: usize }, + #[error("declared entry sizes exceed {limit} bytes")] + DeclaredBytesExceeded { limit: u64 }, + #[error("ZIP entries overlap")] + OverlappingEntries, + #[error("ZIP index: {0}")] + Zip(#[from] zip::result::ZipError), +} + +/// Check every declared payload, including entries the caller may ignore, and reject overlapping +/// entry headers and compressed ranges. This is a post-index check; it does not replace a pre-index count guard or +/// bound bytes actually decompressed by a caller. +pub fn inspect_seekable(archive: &mut zip::ZipArchive, max_entries: usize, max_declared_bytes: u64) -> Result<(), InspectError> { + if archive.len() > max_entries { + return Err(InspectError::TooManyEntries { actual: archive.len(), limit: max_entries }); + } + let mut remaining = max_declared_bytes; + for index in 0..archive.len() { + remaining = + remaining.checked_sub(archive.by_index_raw(index)?.size()).ok_or(InspectError::DeclaredBytesExceeded { limit: max_declared_bytes })?; + } + if archive.has_overlapping_entries()? { + return Err(InspectError::OverlappingEntries); + } + Ok(()) +} + +#[cfg(test)] +#[allow(clippy::unwrap_used, clippy::expect_used)] +mod tests { + use super::*; + use std::io::{Cursor, Write}; + + #[test] + fn counts_all_declared_entries_and_detects_aliases() { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + writer.start_file("first", options).unwrap(); + writer.write_all(b"payload").unwrap(); + writer.shallow_copy_file("first", "alias").unwrap(); + let bytes = writer.finish().unwrap().into_inner(); + let mut archive = zip::ZipArchive::new(Cursor::new(bytes)).unwrap(); + assert!(matches!(inspect_seekable(&mut archive, 1, 14), Err(InspectError::TooManyEntries { .. }))); + assert!(matches!(inspect_seekable(&mut archive, 2, 13), Err(InspectError::DeclaredBytesExceeded { .. }))); + assert!(matches!(inspect_seekable(&mut archive, 2, 14), Err(InspectError::OverlappingEntries))); + } + + #[test] + fn empty_aliases_still_overlap_at_the_header() { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("empty", zip::write::SimpleFileOptions::default()).unwrap(); + writer.shallow_copy_file("empty", "alias").unwrap(); + let mut archive = zip::ZipArchive::new(Cursor::new(writer.finish().unwrap().into_inner())).unwrap(); + assert!(matches!(inspect_seekable(&mut archive, 2, 0), Err(InspectError::OverlappingEntries))); + } + + #[test] + fn permits_exact_limits_and_empty_archive() { + let mut writer = zip::ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("a", zip::write::SimpleFileOptions::default()).unwrap(); + writer.write_all(b"123").unwrap(); + let mut archive = zip::ZipArchive::new(Cursor::new(writer.finish().unwrap().into_inner())).unwrap(); + assert!(inspect_seekable(&mut archive, 1, 3).is_ok()); + let empty = zip::ZipWriter::new(Cursor::new(Vec::new())).finish().unwrap().into_inner(); + assert!(inspect_seekable(&mut zip::ZipArchive::new(Cursor::new(empty)).unwrap(), 0, 0).is_ok()); + } +} From d2c1cac0324491bd67f6e3b583a5dffb18bf78eb Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 13:38:46 +0000 Subject: [PATCH 59/98] Reject colliding project extension fields --- crates/tcraft/src/lib.rs | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/tcraft/src/lib.rs b/crates/tcraft/src/lib.rs index 730ef3b0c..d7af3e8a6 100644 --- a/crates/tcraft/src/lib.rs +++ b/crates/tcraft/src/lib.rs @@ -36,7 +36,7 @@ pub enum Error { /// Neutral top-level project schema. Nested app-domain values and unknown fields remain JSON /// so they round-trip without requiring TextureCraft's in-memory model. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Deserialize)] pub struct ProjectDocument { pub format_version: u32, pub name: String, @@ -52,6 +52,30 @@ pub struct ProjectDocument { pub extra: BTreeMap, } +impl Serialize for ProjectDocument { + fn serialize(&self, serializer: S) -> Result { + use serde::ser::{Error as _, SerializeMap as _}; + const RESERVED: &[&str] = &["format_version", "name", "mesh", "normal_format", "texture_sets", "layers", "next_id", "display", "cameras"]; + if self.extra.keys().any(|key| RESERVED.contains(&key.as_str())) { + return Err(S::Error::custom("an extra project field duplicates a known field")); + } + let mut map = serializer.serialize_map(Some(9 + self.extra.len()))?; + map.serialize_entry("format_version", &self.format_version)?; + map.serialize_entry("name", &self.name)?; + map.serialize_entry("mesh", &self.mesh)?; + map.serialize_entry("normal_format", &self.normal_format)?; + map.serialize_entry("texture_sets", &self.texture_sets)?; + map.serialize_entry("layers", &self.layers)?; + map.serialize_entry("next_id", &self.next_id)?; + map.serialize_entry("display", &self.display)?; + map.serialize_entry("cameras", &self.cameras)?; + for (key, value) in &self.extra { + map.serialize_entry(key, value)?; + } + map.end() + } +} + /// Read only the version before an application applies its own recovery policy. Serde's /// structured parse rejects a duplicate or invalid version field, like the project reader. pub fn project_version(bytes: &[u8]) -> Result { @@ -286,7 +310,10 @@ mod tests { let document: ProjectDocument = serde_json::from_slice(&bytes).unwrap(); let mut expected = json; expected["cameras"] = serde_json::json!([]); - assert_eq!(serde_json::to_value(document).unwrap(), expected); + assert_eq!(serde_json::to_value(&document).unwrap(), expected); + let mut duplicate = document; + duplicate.extra.insert("mesh".into(), serde_json::json!("other.obj")); + assert!(serde_json::to_vec(&duplicate).is_err()); assert!(project_version(br#"{"format_version":1,"format_version":2}"#).is_err()); assert!(project_version(br#"{"format_version":-1}"#).is_err()); } From 533b37bd5627e723f249e00bfefbd0273ad12871 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 13:41:22 +0000 Subject: [PATCH 60/98] Add reusable MCP server session envelopes --- Cargo.lock | 8 ++ Cargo.toml | 1 + crates/mcp/Cargo.toml | 15 +++ crates/mcp/README.md | 17 ++++ crates/mcp/src/lib.rs | 207 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 248 insertions(+) create mode 100644 crates/mcp/Cargo.toml create mode 100644 crates/mcp/README.md create mode 100644 crates/mcp/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8e12a6995..d140d2ed3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2075,6 +2075,14 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-mcp" +version = "0.1.0" +dependencies = [ + "craft-jsonrpc", + "serde_json", +] + [[package]] name = "craft-media" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 5d0bbbc66..a0141c2cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,7 @@ craft-jmap = { path = "crates/jmap" } craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } craft-jsonrpc = { path = "crates/jsonrpc" } +craft-mcp = { path = "crates/mcp" } craft-jxl = { path = "crates/jxl" } craft-kiwi = { path = "crates/kiwi" } craft-keymap = { path = "crates/keymap" } diff --git a/crates/mcp/Cargo.toml b/crates/mcp/Cargo.toml new file mode 100644 index 000000000..9aeb746d9 --- /dev/null +++ b/crates/mcp/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-mcp" +description = "Transport-independent MCP server initialization and tool envelopes" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-jsonrpc = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/mcp/README.md b/crates/mcp/README.md new file mode 100644 index 000000000..e84a1a40c --- /dev/null +++ b/crates/mcp/README.md @@ -0,0 +1,17 @@ +# craft-mcp + +Transport-independent MCP 2025-03-26 server initialization and tool result helpers. +Use one `Session` per client connection with `craft-jsonrpc` for message parsing and +batch dispatch. The caller owns transport, tool definitions and execution, response +size limits, and shutdown. Pass the outer envelope's batch status to `Session::request`. + +`Policy::Compatible` retains existing permissive clients: tool calls can precede +initialization, initialization can occur in a batch, and client information needs only +to be an object. `Policy::Strict` requires initialization first (except ping), +rejects batched or repeated initialization, and requires client name and version. +Both respond with the sole supported version, 2025-03-26, even if the client offered +a different version. An initialized notification moves the session to `Ready`. +This crate does not enforce every MCP lifecycle or capability rule; applications +must validate any additional requirements they depend on. + +See the [MCP lifecycle specification](https://modelcontextprotocol.io/specification/2025-03-26/basic/lifecycle). diff --git a/crates/mcp/src/lib.rs b/crates/mcp/src/lib.rs new file mode 100644 index 000000000..96cfdc7ae --- /dev/null +++ b/crates/mcp/src/lib.rs @@ -0,0 +1,207 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +//! MCP 2025-03-26 server initialization and tool envelopes, independent of transport and tools. +//! The caller owns routing, transport and JSON-RPC batching. Compatibility mode preserves servers +//! that accepted calls before initialization and initialize in a batch; strict mode enforces the +//! initialization ordering and batch prohibition. Create one session per connection. + +use craft_jsonrpc::{Request, RpcError}; +use serde_json::{Value, json}; + +pub const PROTOCOL_VERSION: &str = "2025-03-26"; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Policy { + /// Preserve existing permissive servers, including minimal client information. + #[default] + Compatible, + /// Enforce the initialize request and client identity required by MCP. + Strict, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum State { + #[default] + New, + AwaitingInitialized, + Ready, +} + +/// State for one connection. Unknown requests are left to the caller's router. +pub struct Session { + name: String, + version: String, + policy: Policy, + state: State, + capabilities: Value, +} + +impl Session { + pub fn new(name: impl Into, version: impl Into, policy: Policy) -> Self { + Self { name: name.into(), version: version.into(), policy, state: State::New, capabilities: json!({"tools": {}}) } + } + + /// Set the server capabilities returned at initialization. The value must be an object. + pub fn with_capabilities(mut self, capabilities: Value) -> Result { + if !capabilities.is_object() { + return Err(RpcError::new(-32602, "Capabilities must be an object")); + } + self.capabilities = capabilities; + Ok(self) + } + + pub fn state(&self) -> State { + self.state + } + + /// Handles `initialize` and `ping`. Returns `None` for a request the app should route. + /// `in_batch` must describe the outer JSON-RPC envelope. + pub fn request(&mut self, request: Request<'_>, in_batch: bool) -> Option> { + if request.method == "ping" { + return Some(Ok(json!({}))); + } + if request.method == "initialize" { + if self.policy == Policy::Strict && (in_batch || self.state != State::New) { + return Some(Err(RpcError::new(-32600, "Invalid Request"))); + } + let Some(params) = request.params.and_then(Value::as_object) else { + return Some(Err(invalid_initialize())); + }; + let valid = params.get("protocolVersion").and_then(Value::as_str).is_some() + && params.get("capabilities").is_some_and(Value::is_object) + && params.get("clientInfo").is_some_and(Value::is_object); + let identity = params.get("clientInfo").and_then(Value::as_object); + let strict_identity = identity + .is_some_and(|info| info.get("name").and_then(Value::as_str).is_some() && info.get("version").and_then(Value::as_str).is_some()); + if !valid || (self.policy == Policy::Strict && !strict_identity) { + return Some(Err(invalid_initialize())); + } + self.state = State::AwaitingInitialized; + return Some(Ok( + json!({"protocolVersion": PROTOCOL_VERSION, "capabilities": self.capabilities, "serverInfo": {"name": self.name, "version": self.version}}), + )); + } + if self.policy == Policy::Strict && self.state == State::New { + return Some(Err(RpcError::new(-32600, "Server not initialized"))); + } + None + } + + /// Observe a JSON-RPC notification; notifications never receive replies. + pub fn notification(&mut self, request: Request<'_>) { + if request.method == "notifications/initialized" && self.state == State::AwaitingInitialized { + self.state = State::Ready; + } + } +} + +fn invalid_initialize() -> RpcError { + RpcError::new(-32602, "Invalid initialize parameters") +} + +/// Validate MCP tool-call parameters without owning the application's tool routing. +pub fn tool_call(params: &Value) -> Result<(&str, Value), RpcError> { + let name = params.get("name").and_then(Value::as_str).ok_or_else(|| RpcError::new(-32602, "Missing tool name"))?; + let arguments = params.get("arguments").cloned().unwrap_or_else(|| json!({})); + if !arguments.is_object() { + return Err(RpcError::new(-32602, "Tool arguments must be an object")); + } + Ok((name, arguments)) +} + +/// Wrap an application-provided tool list in the MCP result envelope. +pub fn tool_list(tools: Value) -> Value { + json!({"tools": tools}) +} + +/// Construct the text result currently used by Craft's MCP tools. +pub fn text_tool_result(text: &str, is_error: bool) -> Value { + json!({"content": [{"type": "text", "text": text}], "isError": is_error}) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn call<'a>(method: &'a str, params: Option<&'a Value>) -> Request<'a> { + Request { id: None, method, params } + } + + #[test] + fn compatible_negotiation_preserves_existing_clients() { + let mut s = Session::new("test", "1", Policy::Compatible); + assert!(s.request(call("tools/list", None), false).is_none()); + let p = json!({"protocolVersion":"old", "capabilities":{}, "clientInfo":{}}); + let r = s.request(call("initialize", Some(&p)), true).unwrap().unwrap(); + assert_eq!(r["protocolVersion"], PROTOCOL_VERSION); + assert_eq!(r["serverInfo"]["name"], "test"); + assert_eq!(s.state(), State::AwaitingInitialized); + s.notification(call("notifications/initialized", None)); + assert_eq!(s.state(), State::Ready); + } + + #[test] + fn strict_rejects_batch_and_calls_before_initialization() { + let mut s = Session::new("test", "1", Policy::Strict); + assert_eq!(s.request(call("tools/list", None), false).unwrap().unwrap_err().code, -32600); + assert!(s.request(call("ping", None), false).unwrap().is_ok()); + let p = json!({"protocolVersion":PROTOCOL_VERSION, "capabilities":{}, "clientInfo":{"name":"client", "version":"1"}}); + assert!(s.request(call("initialize", Some(&p)), true).unwrap().is_err()); + assert_eq!(s.state(), State::New); + assert!(s.request(call("initialize", Some(&p)), false).unwrap().is_ok()); + assert!(s.request(call("initialize", Some(&p)), false).unwrap().is_err()); + s.notification(call("notifications/initialized", None)); + assert_eq!(s.state(), State::Ready); + } + + #[test] + fn capabilities_are_configurable_and_must_be_an_object() { + assert!(Session::new("test", "1", Policy::Compatible).with_capabilities(json!([])).is_err()); + let mut session = Session::new("test", "1", Policy::Compatible).with_capabilities(json!({"resources": {"subscribe": true}})).unwrap(); + let params = json!({"protocolVersion": "2025-03-26", "capabilities": {}, "clientInfo": {}}); + let result = session.request(call("initialize", Some(¶ms)), false).unwrap().unwrap(); + assert_eq!(result["capabilities"], json!({"resources": {"subscribe": true}})); + } + + #[test] + fn malformed_initialize_does_not_change_state() { + let mut s = Session::new("test", "1", Policy::Strict); + for p in [json!({}), json!({"protocolVersion":"x", "capabilities":{}, "clientInfo":{}})] { + assert_eq!(s.request(call("initialize", Some(&p)), false).unwrap().unwrap_err().code, -32602); + assert_eq!(s.state(), State::New); + } + } + + #[test] + fn tool_envelopes_validate_arguments_and_preserve_results() { + let p = json!({"name":"open"}); + assert_eq!(tool_call(&p).unwrap(), ("open", json!({}))); + assert_eq!(tool_call(&json!({"name":"open", "arguments":[]})).unwrap_err().code, -32602); + assert_eq!(tool_call(&json!({})).unwrap_err().code, -32602); + assert_eq!(text_tool_result("bad", true), json!({"content":[{"type":"text", "text":"bad"}], "isError":true})); + } + + #[test] + fn jsonrpc_batch_preserves_ids_and_skips_notifications() { + use std::cell::RefCell; + let session = RefCell::new(Session::new("test", "1", Policy::Compatible)); + let input = json!([ + {"jsonrpc":"2.0", "id":1, "method":"initialize", "params":{"protocolVersion":"2025-03-26", "capabilities":{}, "clientInfo":{}}}, + {"jsonrpc":"2.0", "method":"notifications/initialized"}, + {"jsonrpc":"2.0", "id":"tools", "method":"tools/list"} + ]); + let reply = craft_jsonrpc::process( + &input, + craft_jsonrpc::Limits::default(), + |r| session.borrow_mut().request(r, true).unwrap_or_else(|| Ok(tool_list(json!([])))), + |r| session.borrow_mut().notification(r), + ) + .unwrap() + .unwrap(); + assert_eq!(reply.as_array().unwrap().len(), 2); + assert_eq!(reply[0]["id"], 1); + assert_eq!(reply[1]["id"], "tools"); + assert_eq!(reply[1]["result"]["tools"], json!([])); + assert_eq!(session.borrow().state(), State::Ready); + } +} From 09188c3bbfa443b089a3e022fad6abd7b464efda Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 08:47:29 -0500 Subject: [PATCH 61/98] Merge pull request #1565 from storytold/extract/psd-linked-views craft-psd: bounded header-only linked records for deferred consumers --- crates/psd/src/smart.rs | 163 +++++++++++++++++++++++++++++++++++----- 1 file changed, 143 insertions(+), 20 deletions(-) diff --git a/crates/psd/src/smart.rs b/crates/psd/src/smart.rs index a8f778ccd..a2195bf5d 100644 --- a/crates/psd/src/smart.rs +++ b/crates/psd/src/smart.rs @@ -470,6 +470,55 @@ fn linked_file(item: &[u8], at: usize) -> Result { Ok(out) } +/// Header of an embedded (`liFD`) linked-layer item, without reading its payload. +/// Offsets are relative to the item (not its enclosing eight-byte length field). +/// A deferred reader can supply just the header, then validate the payload range +/// against the full record length before reading those bytes. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct EmbeddedFileHeader { + /// Lossy UTF-8 identifier, matching Photoshop metadata display semantics. + pub uuid: String, + /// Lossy UTF-16 filename with trailing NULs removed. + pub name: String, + /// Start of the payload within the item. + pub data_offset: usize, + /// Declared payload length; the payload need not be resident. + pub data_len: usize, +} + +impl EmbeddedFileHeader { + /// Borrow the payload only when the supplied complete item contains it. + pub fn payload<'a>(&self, item: &'a [u8]) -> Option<&'a [u8]> { + item.get(self.data_offset..self.data_offset.checked_add(self.data_len)?) + } +} + +/// Read an embedded item's header from a possibly partial/deferred item buffer. +/// This deliberately does not require its declared payload to be present. A malformed +/// header or a non-embedded item is an error, not an empty embedded file. Callers walking +/// framed records may continue to the next record after such an error. +/// Ported from PhotoCraft's linked-item reader; record versions remain forward-compatible. +pub fn embedded_file_header(item: &[u8]) -> Result { + let mut r = Reader::new(item); + let kind = r.array::<4>()?; + let _version = r.u32()?; + let uuid = String::from_utf8_lossy(&read_pascal(&mut r, 1)?).into_owned(); + // A filename is metadata, never an unbounded allocation. Check before decoding. + if r.clone().u32()? > 32_768 { + return Err(PsdError::LimitExceeded("linked filename exceeds 32768 UTF-16 units")); + } + let name = UnicodeString::read(&mut r)?.to_string_lossy(); + r.skip(8)?; // file type and creator + let data_len = usize::try_from(r.u64()?).map_err(|_| PsdError::LimitExceeded("linked payload exceeds address space"))?; + if r.u8()? != 0 { + crate::descriptor::VersionedDescriptor::read(&mut r)?; + } + if kind != *b"liFD" { + return Err(PsdError::invalid("linked item is not embedded")); + } + Ok(EmbeddedFileHeader { uuid, name, data_offset: r.pos(), data_len }) +} + /// The four-character file type Photoshop records for an embedded file, from its first bytes: /// `8BPS` (`8BPB` for a PSB), `PNGf`, `JPEG`, `TIFF`, `GIFf`, or four spaces. pub fn file_type(bytes: &[u8]) -> [u8; 4] { @@ -519,7 +568,29 @@ pub struct LinkedRecord<'a> { pub raw: &'a [u8], } -impl LinkedRecord<'_> { +impl<'a> LinkedRecord<'a> { + /// The complete item, excluding framing and padding; absent for a damaged tail. + pub fn item(&self) -> Option<&'a [u8]> { + framed_record(self.raw).map(|(item, _)| item) + } + + /// Copy this complete record under a new ID, leaving the payload byte-identical. + /// Selection policy belongs to the caller; malformed tails cannot be copied. + pub fn copy_with_uuid(&self, new_uuid: &str) -> Option> { + let item = self.item()?; + let old_end = 9usize.checked_add(usize::from(*item.get(8)?))?; + let new_len = u8::try_from(new_uuid.len()).ok()?; + let mut copy = item.get(..8)?.to_vec(); + copy.push(new_len); + copy.extend_from_slice(new_uuid.as_bytes()); + copy.extend_from_slice(item.get(old_end..)?); + let mut out = Vec::with_capacity(copy.len() + 11); + out.put_u64(copy.len() as u64); + out.extend(copy); + out.resize(out.len().next_multiple_of(4), 0); + Some(out) + } + /// The unique id as text, when it is readable and valid UTF-8. pub fn uuid_str(&self) -> Option<&str> { std::str::from_utf8(self.uuid?).ok() @@ -545,17 +616,26 @@ fn record_uuid(item: &[u8]) -> Option<&[u8]> { /// The records of a linked layer block's data, in order, each with its unique id when its header /// is readable. A tail that is no complete record is returned as one record without an id. pub fn linked_records(data: &[u8]) -> Vec> { - let mut out = Vec::new(); + iter_linked_records(data).collect() +} + +/// Stream the same raw views as [`linked_records`] without allocating a record list. +/// Complete frames with malformed inner headers do not hide subsequent frames; +/// an incomplete tail is yielded once with no identifier. +pub fn iter_linked_records(data: &[u8]) -> impl Iterator> { let mut rest = data; - while !rest.is_empty() { + std::iter::from_fn(move || { + if rest.is_empty() { + return None; + } let Some((item, raw)) = framed_record(rest) else { - out.push(LinkedRecord { uuid: None, raw: rest }); - break; + let tail = rest; + rest = &[]; + return Some(LinkedRecord { uuid: None, raw: tail }); }; - out.push(LinkedRecord { uuid: record_uuid(item), raw }); rest = rest.get(raw.len()..).unwrap_or_default(); - } - out + Some(LinkedRecord { uuid: record_uuid(item), raw }) + }) } /// The unique ids of every record of a linked layer block's data that has a readable one. @@ -653,18 +733,7 @@ pub fn contains_linked_record(data: &[u8], record: &[u8]) -> bool { /// field and padding; the rest of the record (embedded, external or alias data) is kept. pub fn copy_linked_record(data: &[u8], uuid: &str, new_uuid: &str) -> Option> { let record = linked_records(data).into_iter().find(|r| r.uuid == Some(uuid.as_bytes()))?; - let (item, _) = framed_record(record.raw)?; - let old_end = 9usize.checked_add(usize::from(*item.get(8)?))?; - let new_len = u8::try_from(new_uuid.len()).ok()?; - let mut copy = item.get(..8)?.to_vec(); - copy.push(new_len); - copy.extend_from_slice(new_uuid.as_bytes()); - copy.extend_from_slice(item.get(old_end..)?); - let mut out = Vec::with_capacity(copy.len() + 11); - out.put_u64(copy.len() as u64); - out.extend(copy); - out.resize(out.len().next_multiple_of(4), 0); - Some(out) + record.copy_with_uuid(new_uuid) } impl PsdFile { @@ -1111,3 +1180,57 @@ mod tests { assert!(u64::from(w) * u64::from(h) <= MAX_BAKE_PIXELS); } } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod embedded_header_tests { + use super::*; + + #[test] + fn headers_do_not_require_resident_payload() { + let data = encode_embedded_file("id", "a.psd", b"payload"); + let record = iter_linked_records(&data).next().unwrap(); + let item = record.item().unwrap(); + let header = embedded_file_header(item).unwrap(); + assert_eq!(header.uuid, "id"); + assert_eq!(header.name, "a.psd"); + assert_eq!(header.payload(item), Some(&b"payload"[..])); + assert_eq!(embedded_file_header(&item[..header.data_offset]).unwrap(), header); + assert_eq!(header.payload(&item[..header.data_offset]), None); + for n in 0..header.data_offset { + assert!(embedded_file_header(&item[..n]).is_err()); + } + } + + #[test] + fn filename_limits_and_lossy_metadata_are_preserved() { + let data = encode_embedded_file("x", &"a".repeat(32767), b""); + let item = iter_linked_records(&data).next().unwrap().item().unwrap(); + assert_eq!(embedded_file_header(item).unwrap().name.len(), 32767); + let data = encode_embedded_file("x", &"a".repeat(32768), b""); + assert!(embedded_file_header(iter_linked_records(&data).next().unwrap().item().unwrap()).is_err()); + let data = encode_embedded_file("x", "y", b"payload"); + let mut item = iter_linked_records(&data).next().unwrap().item().unwrap().to_vec(); + item[9] = 0xff; + item[14..16].copy_from_slice(&0xd800u16.to_be_bytes()); + let h = embedded_file_header(&item).unwrap(); + assert_eq!(h.uuid, "\u{fffd}"); + assert_eq!(h.name, "\u{fffd}"); + item[0..4].copy_from_slice(b"liFE"); + assert!(embedded_file_header(&item).is_err()); + } + + #[test] + fn malformed_inner_records_do_not_hide_later_files() { + let mut data = vec![0; 8]; // A complete empty item, not a truncated frame. + data.extend(encode_embedded_file("wanted", "x", b"ok")); + data.extend([1, 2, 3]); // Incomplete tail retained, never interpreted as an item. + let records: Vec<_> = iter_linked_records(&data).collect(); + assert_eq!(records, linked_records(&data)); + assert!(embedded_file_header(records[0].item().unwrap()).is_err()); + let item = records[1].item().unwrap(); + assert_eq!(embedded_file_header(item).unwrap().payload(item), Some(&b"ok"[..])); + assert_eq!(records[2].item(), None); + assert_eq!(records[2].raw, &[1, 2, 3]); + } +} From b1ecb6776d2d6511405ef41e957c77a9e3bf6d42 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 14:04:03 +0000 Subject: [PATCH 62/98] Guard ZIP64 directory-size sentinel counts --- crates/zip/src/guard.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/zip/src/guard.rs b/crates/zip/src/guard.rs index dd3ebe451..553e1532f 100644 --- a/crates/zip/src/guard.rs +++ b/crates/zip/src/guard.rs @@ -46,7 +46,7 @@ pub fn check_entry_count(bytes: &[u8], max_entries: u64) -> Result<(), Error> { continue; } let (Some(on_disk), Some(total), Some(cd_offset)) = (u16_at(at + 8), u16_at(at + 10), u32_at(at + 16)) else { continue }; - let zip64 = total == u16::MAX || cd_offset == u32::MAX; + let zip64 = total == u16::MAX || u32_at(at + 12) == Some(u32::MAX) || cd_offset == u32::MAX; if zip64 && let Some(locator) = at.checked_sub(20) && bytes.get(locator..locator + 4) == Some(EOCD64_LOCATOR.as_slice()) @@ -155,6 +155,17 @@ mod tests { assert!(check(&prefixed, 20_000).is_err()); } + #[test] + fn zip64_directory_size_sentinel_alone_cannot_bypass_the_count_limit() { + let mut bytes = zip64_entry_count_sample(21); + let eocd = bytes.len() - 22; + // zip recognizes ZIP64 when only the directory size is a sentinel. + bytes[eocd + 8..eocd + 12].copy_from_slice(&[1, 0, 1, 0]); + bytes[eocd + 16..eocd + 20].copy_from_slice(&0u32.to_le_bytes()); + assert!(zip::ZipArchive::new(std::io::Cursor::new(&bytes)).is_ok()); + assert_eq!(check(&bytes, 20), Err(Error::TooManyEntries { declared: 21, limit: 20 })); + } + #[test] fn a_plain_end_record_is_checked_as_zip_reads_it() { assert!(check(&plain(3, b""), 20_000).is_ok()); From 2b724bb168a15a29748672522c61a45d2f28ef43 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:06:50 +0100 Subject: [PATCH 63/98] Link jpeg-encoder licence records by repository URL The relative link and root path do not resolve from a packaged craft-image-codecs. --- crates/image-codecs/NOTICE | 3 ++- crates/image-codecs/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/image-codecs/NOTICE b/crates/image-codecs/NOTICE index 83fde3dba..b3424a8b7 100644 --- a/crates/image-codecs/NOTICE +++ b/crates/image-codecs/NOTICE @@ -5,7 +5,8 @@ Declared license: (MIT OR Apache-2.0) AND IJG. This software is based in part on the work of the Independent JPEG Group. The MIT license and unaltered upstream IJG and SIMD notices follow. The Apache-2.0 -alternative is also retained in licenses/jpeg-encoder-0.6.1 at the repository root. +alternative is also retained in licenses/jpeg-encoder-0.6.1 at the repository root +(https://github.com/storytold/craft-libs/tree/main/licenses/jpeg-encoder-0.6.1). The dependency is used unmodified; no encoder implementation is vendored here. LICENSE-MIT diff --git a/crates/image-codecs/README.md b/crates/image-codecs/README.md index 27bb3c6ba..aaf7f54b4 100644 --- a/crates/image-codecs/README.md +++ b/crates/image-codecs/README.md @@ -346,4 +346,5 @@ JPEG encoding uses `jpeg-encoder` 0.6.1, including work from the Independent JPE Retain this crate's [NOTICE](NOTICE) with binary distributions using that encoder. It includes its MIT license, the IJG acknowledgement and original IJG/SIMD notices. Both upstream license alternatives and source checksums are retained in -[`licenses/jpeg-encoder-0.6.1`](../../licenses/jpeg-encoder-0.6.1) at the repository root. +[`licenses/jpeg-encoder-0.6.1`](https://github.com/storytold/craft-libs/tree/main/licenses/jpeg-encoder-0.6.1) +at the repository root. From 4f6e49b1ee3e450e5d5e105271465c13be19354f Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:13:06 +0100 Subject: [PATCH 64/98] Test SVG preflight depth and root viewport rewrite; accept px root sizes Correct the source-depth expectations (svg>g>path is 3 levels, 2 open), cover Preflight and every rewrite_root_viewport refusal, and read a root width or height written with px as the pixels the docs promise. --- crates/svg/src/preflight.rs | 8 ++-- crates/svg/src/tree.rs | 9 +++- crates/svg/src/tree/tests.rs | 82 ++++++++++++++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 5 deletions(-) diff --git a/crates/svg/src/preflight.rs b/crates/svg/src/preflight.rs index 463c3ce16..0c2d3a7b1 100644 --- a/crates/svg/src/preflight.rs +++ b/crates/svg/src/preflight.rs @@ -74,7 +74,8 @@ pub fn parse_text<'a>(svg: &'a str, limits: &Limits) -> Result(svg: &'a str, limits: &Limits) -> Result<(roxmltree::Document<'a>, usize, usize), Error> { if svg.len() > limits.bytes { return Err(Error::Text(crate::svgz::Error::TooLarge(limits.bytes))); @@ -106,8 +107,9 @@ mod tests { let limits = Limits { depth: 4, ..Default::default() }; let (document, depth, open_depth) = parse_text_with_depth("", &limits)?; assert!(document.root_element().has_tag_name("svg")); - assert_eq!(depth, 2); - assert_eq!(open_depth, 0); + // Every element is a level (svg, g, path); a self-closing leaf opens none (svg, g). + assert_eq!(depth, 3); + assert_eq!(open_depth, 2); let simple = parse_text("", &limits)?; assert!(simple.root_element().has_tag_name("svg")); Ok(()) diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 86a6d0ab1..37d956a48 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -167,7 +167,8 @@ pub struct Preflight<'s> { } impl Preflight<'_> { - /// Maximum source nesting depth before following references. + /// The deepest stack of open source elements, before references are followed; a + /// self-closing leaf opens no level (`` is 2). pub fn depth(&self) -> usize { self.depth } @@ -223,7 +224,11 @@ pub fn rewrite_root_viewport(svg: &str, target: Viewport, limits: &Limits, outpu } let root_dimension = |key: &str| { let value = root.attribute(key)?; - let (number, scale) = value.strip_suffix("pt").map_or((value, 1.0), |n| (n, 4.0 / 3.0)); + let value = value.trim(); + let (number, scale) = match value.strip_suffix("pt") { + Some(points) => (points, 4.0 / 3.0), + None => (value.strip_suffix("px").unwrap_or(value), 1.0), + }; number.parse::().ok().map(|n| n * scale).filter(|n| n.is_finite() && *n > 0.0) }; let (width, height) = root_dimension("width") diff --git a/crates/svg/src/tree/tests.rs b/crates/svg/src/tree/tests.rs index 2708ae0f4..99b0c1cab 100644 --- a/crates/svg/src/tree/tests.rs +++ b/crates/svg/src/tree/tests.rs @@ -584,3 +584,85 @@ mod invalid_image_accounting { assert_eq!(report.raster_limit_images, 1); } } + +#[test] +fn preflight_keeps_source_depth_and_parses_the_checked_tree() { + let text = svg(""); + let checked = preflight(&text, &Limits::default()).unwrap(); + // svg and g are open; the self-closing rect opens no level. + assert_eq!(checked.depth(), 2); + let tree = checked.parse(&options()).unwrap(); + assert_eq!(tree.size().width(), 10.0); + assert!(matches!(preflight(&svg(&groups(30)), &shallow()), Err(Error::TooDeep(24)))); +} + +#[test] +fn preflight_matches_parse_text_refusals() { + for text in [""] { + assert!(matches!(preflight(text, &Limits::default()), Err(Error::Xml(_)))); + assert!(matches!(parse_text(text, &Limits::default(), &options()), Err(Error::Xml(_)))); + } +} + +#[test] +fn rewrite_root_viewport_wraps_the_root_and_keeps_the_preamble() { + let source = concat!( + "\n]>\n", + "", + "" + ); + let out = rewrite_root_viewport(source, Viewport { width: 40.0, height: 30.0 }, &Limits::default(), 4096).unwrap(); + assert!(out.starts_with("\n]>\n")); + let tree = parse_text(&out, &Limits::default(), &options()).unwrap(); + assert_eq!((tree.size().width(), tree.size().height()), (40.0, 30.0)); +} + +#[test] +fn rewrite_root_viewport_reads_points_and_pixels() { + let points = rewrite_root_viewport( + "", + Viewport { width: 1.0, height: 1.0 }, + &Limits::default(), + 4096, + ) + .unwrap(); + assert!(points.contains("viewBox=\"0 0 96 48\"")); + let pixels = rewrite_root_viewport( + "", + Viewport { width: 1.0, height: 1.0 }, + &Limits::default(), + 4096, + ) + .unwrap(); + assert!(pixels.contains("viewBox=\"0 0 12 8\"")); +} + +#[test] +fn rewrite_root_viewport_refusals() { + let limits = Limits::default(); + let one = Viewport { width: 1.0, height: 1.0 }; + let sized = ""; + for target in [ + Viewport { width: f64::NAN, height: 1.0 }, + Viewport { width: 1.0, height: f64::INFINITY }, + Viewport { width: 0.0, height: 1.0 }, + Viewport { width: 1.0, height: -2.0 }, + ] { + assert!(matches!(rewrite_root_viewport(sized, target, &limits, 4096), Err(Error::Viewport(_)))); + } + assert!(matches!(rewrite_root_viewport("", one, &limits, 4096), Err(Error::Viewport(_)))); + for source in [ + "", + "", + "", + "", + "", + ] { + assert!(matches!(rewrite_root_viewport(source, one, &limits, 4096), Err(Error::Viewport(_))), "{source}"); + } + assert!(matches!(rewrite_root_viewport(" Date: Sun, 15 Nov 2026 15:18:25 +0100 Subject: [PATCH 65/98] craft-captions: ASS and SSA subtitles (Format::Ass) Reads the [Events] section of ASS v4+ and SSA v4 files and writes a minimal ASS v4+ file. Keeps line breaks, the Name field as the speaker and the \i, \b, \u override tags; other override tags and vector drawings are dropped. Closes #1550 --- Cargo.lock | 1 + crates/captions/Cargo.toml | 2 + crates/captions/README.md | 1 + crates/captions/src/ass.rs | 239 +++++++++++++++++++++++++++++++ crates/captions/src/lib.rs | 17 ++- crates/captions/tests/it/ass.rs | 113 +++++++++++++++ crates/captions/tests/it/main.rs | 1 + 7 files changed, 371 insertions(+), 3 deletions(-) create mode 100644 crates/captions/src/ass.rs create mode 100644 crates/captions/tests/it/ass.rs diff --git a/Cargo.lock b/Cargo.lock index 8e12a6995..55552f9f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1425,6 +1425,7 @@ version = "0.1.0" name = "craft-captions" version = "0.1.0" dependencies = [ + "craft-media-testkit", "craft-timeline", "craft-xml", "proptest", diff --git a/crates/captions/Cargo.toml b/crates/captions/Cargo.toml index df665336d..cd09f7e16 100644 --- a/crates/captions/Cargo.toml +++ b/crates/captions/Cargo.toml @@ -19,3 +19,5 @@ thiserror = { workspace = true } proptest = { workspace = true } # A second XML reader for the tests: checks of written TTML, and the comparison in tests/it/ttml_xml_views.rs. roxmltree = "0.21" +# ffmpeg as the reference for the ASS tests (tests/it/ass.rs). +craft-media-testkit = { workspace = true } diff --git a/crates/captions/README.md b/crates/captions/README.md index 0676d1d72..7380a3ce4 100644 --- a/crates/captions/README.md +++ b/crates/captions/README.md @@ -55,6 +55,7 @@ lists at most 20 characters per stream. | EBU STL `.stl` | yes | yes | EBU Tech 3264-E: GSI block + 128-byte TTI blocks, `STL25.01` (25 fps) or `STL30.01` (read/written as 29.97 fps, non-drop labels), ISO/IEC 6937 Latin text (accents as diacritic + letter), italics / underline codes as `` / ``, extension blocks for long subtitles, start-of-programme (TCP) subtracted. | | TTML `.ttml` (IMSC1) | yes | yes | Writes the IMSC1 Text profile (TTML2 namespaces) with the document's language as `xml:lang` (read back on import): `

` per cue, `
`, styled ``s for `` / `` / ``, `ttm:agent` speakers, exact frame times (`123f`) at the sequence rate. Reads any TTML (TTML1, TTML2, the 2006 `ttaf1` drafts): clock / frame / offset / tick times, `dur`, nested time containers (`body` / `div` begins offset and ends clip their contents; `seq` containers are read as `par` with a warning), timed spans, referenced styles. | | DFXP `.dfxp` | yes | yes | TTML1 with the DFXP presentation profile; clock times in milliseconds (frame-exact after `snap_to_frames`). | +| ASS / SSA `.ass`, `.ssa` | yes | yes (ASS v4+) | The `[Events]` section: `Dialogue:` lines through the file's `Format:` line, `Name` as the speaker, `\N`, `\n`, `\h` and the `\i`, `\b`, `\u` override tags kept (as ``, ``, ``); other override tags and drawing lines are dropped. Times are centiseconds. Written files have one `Default` style. | All cue times are exact `Tick`s. Milliseconds convert exactly (1 ms = 254 016 000 ticks); SCC frames map through `FrameRate::FPS_29_97`. `Document::snap_to_frames` snaps cues to a frame grid and diff --git a/crates/captions/src/ass.rs b/crates/captions/src/ass.rs new file mode 100644 index 000000000..3c907c829 --- /dev/null +++ b/crates/captions/src/ass.rs @@ -0,0 +1,239 @@ +//! Advanced SubStation Alpha and SubStation Alpha (`.ass`, `.ssa`). +//! +//! Reading takes the `[Events]` section: its `Format:` line gives the field order (ASS v4+ and SSA +//! v4 differ in the first field only), each `Dialogue:` line becomes a cue; `Comment:` lines and +//! the other sections are skipped. A cue gets the `Name` field as its speaker. The text keeps what +//! the neutral model can show: `\N` and `\n` become a line break and a space, `\h` a no-break +//! space, and the override tags `\i`, `\b` and `\u` become ``, `` and ``. Every other +//! override tag (colours, fonts, positions, karaoke, animation) is dropped, and a line that draws +//! vector shapes (`\p1`) is skipped with a warning. +//! +//! Writing gives a minimal ASS v4+ file: one `Default` style, one `Dialogue:` line per cue with +//! times to the centisecond. Braces in the text, which ASS reads as override blocks, are written +//! as the full-width `{` and `}`. + +use crate::time::{TICKS_PER_SECOND, Tick}; +use crate::{Cue, Document, Error, Result}; + +const TICKS_PER_CS: i64 = TICKS_PER_SECOND / 100; + +/// `H:MM:SS.cc`, the form SSA and ASS use (any number of fraction digits is read). +fn parse_time(s: &str) -> Option { + let s = s.trim(); + let (hms, frac) = s.split_once('.').unwrap_or((s, "")); + let mut parts = hms.split(':'); + let (h, m, sec) = (parts.next()?, parts.next()?, parts.next()?); + if parts.next().is_some() { + return None; + } + let num = |p: &str| -> Option { (!p.is_empty() && p.len() <= 9 && p.bytes().all(|b| b.is_ascii_digit())).then(|| p.parse().ok()).flatten() }; + let (h, m, sec) = (num(h)?, num(m)?, num(sec)?); + if m >= 60 || sec >= 60 || h > 9999 { + return None; + } + let mut ticks = ((h * 60 + m) * 60 + sec) * TICKS_PER_SECOND; + if !frac.is_empty() { + if frac.len() > 9 || !frac.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let d: i64 = frac.parse().ok()?; + ticks += (i128::from(d) * i128::from(TICKS_PER_SECOND) / 10i128.pow(frac.len() as u32)) as i64; + } + Some(Tick(ticks)) +} + +fn format_time(t: Tick) -> String { + let cs = (i128::from(t.0.max(0)) + i128::from(TICKS_PER_CS) / 2) / i128::from(TICKS_PER_CS); + let (h, m, s, c) = (cs / 360_000, (cs / 6000) % 60, (cs / 100) % 60, cs % 100); + format!("{h}:{m:02}:{s:02}.{c:02}") +} + +/// The Text field of a Dialogue line as cue text; `None` when the line draws shapes. +fn convert_text(raw: &str) -> Option { + let mut out = String::new(); + // Open ``, ``, `` in the order they were opened. + let mut open: Vec = Vec::new(); + let mut chars = raw.chars().peekable(); + let mut drawing = false; + let mut dropped_drawing = false; + while let Some(c) = chars.next() { + match c { + '{' => { + let mut block = String::new(); + let mut closed = false; + for b in chars.by_ref() { + if b == '}' { + closed = true; + break; + } + block.push(b); + } + if !closed { + // An unterminated block is literal text. + out.push('{'); + out.push_str(&block); + continue; + } + for tag in block.split('\\').skip(1) { + let tag = tag.trim_end(); + let (name, val) = tag.split_at(tag.find(|c: char| !c.is_ascii_alphabetic()).unwrap_or(tag.len())); + match name { + "i" | "b" | "u" => { + let letter = name.chars().next().unwrap_or('i'); + let on = val.trim() != "0"; + if on && !open.contains(&letter) { + open.push(letter); + out.push_str(&format!("<{letter}>")); + } else if !on && open.contains(&letter) { + // Close back to it, reopening what was opened after. + let at = open.iter().position(|&o| o == letter).unwrap_or(0); + let after: Vec = open.split_off(at + 1); + for o in after.iter().rev() { + out.push_str(&format!("")); + } + open.pop(); + out.push_str(&format!("")); + for o in after { + open.push(o); + out.push_str(&format!("<{o}>")); + } + } + } + "p" => drawing = val.trim().parse::().unwrap_or(0) != 0, + _ => {} + } + } + } + '\\' => match chars.peek() { + Some('N') => { + chars.next(); + out.push('\n'); + } + Some('n') => { + chars.next(); + out.push(' '); + } + Some('h') => { + chars.next(); + out.push('\u{a0}'); + } + _ => out.push('\\'), + }, + c => { + if drawing { + dropped_drawing = true; + } else { + out.push(c); + } + } + } + } + if dropped_drawing && out.trim().is_empty() { + return None; + } + while let Some(o) = open.pop() { + out.push_str(&format!("")); + } + Some(out) +} + +pub fn parse(text: &str) -> Result { + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + let mut doc = Document::default(); + let mut in_events = false; + let mut saw_events = false; + // Column of each field in a Dialogue line. + let mut cols: Vec = + ["layer", "start", "end", "style", "name", "marginl", "marginr", "marginv", "effect", "text"].map(String::from).to_vec(); + let mut drawings = 0usize; + for (n, line) in text.lines().enumerate() { + let line = line.trim(); + if line.starts_with('[') && line.ends_with(']') { + in_events = line.eq_ignore_ascii_case("[events]"); + saw_events |= in_events; + continue; + } + if !in_events { + continue; + } + let Some((key, rest)) = line.split_once(':') else { continue }; + let rest = rest.trim_start(); + match key.trim().to_ascii_lowercase().as_str() { + "format" => cols = rest.split(',').map(|f| f.trim().to_ascii_lowercase()).collect(), + "dialogue" => { + // The last column (Text) keeps its commas. + let fields: Vec<&str> = rest.splitn(cols.len(), ',').collect(); + let get = |name: &str| cols.iter().position(|c| c == name).and_then(|i| fields.get(i)).copied(); + let (Some(start), Some(end), Some(raw)) = (get("start").and_then(parse_time), get("end").and_then(parse_time), get("text")) else { + doc.warnings.push(format!("line {}: not a Dialogue line this reader understands", n + 1)); + continue; + }; + let Some(text) = convert_text(raw) else { + drawings += 1; + continue; + }; + let name = get("name").map(str::trim).filter(|s| !s.is_empty()); + doc.cues.push(Cue { start, end, text, speaker: name.map(str::to_string), ..Default::default() }); + } + _ => {} + } + } + if !saw_events { + return Err(Error::NotFormat("ASS")); + } + if drawings > 0 { + doc.warnings.push(format!("{drawings} vector drawing line(s) were skipped")); + } + Ok(doc) +} + +fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len() + 8); + let mut rest = text; + while !rest.is_empty() { + if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\i1}"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\i0}"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\b1}"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\b0}"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\u1}"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { + out.push_str("{\\u0}"); + rest = r; + } else if let Some(c) = rest.chars().next() { + match c { + '\n' => out.push_str("\\N"), + '{' => out.push('\u{ff5b}'), + '}' => out.push('\u{ff5d}'), + '\u{a0}' => out.push_str("\\h"), + c => out.push(c), + } + rest = &rest[c.len_utf8()..]; + } + } + out +} + +pub fn write(doc: &Document) -> String { + let mut out = String::from( + "[Script Info]\n; Written by craft-captions\nScriptType: v4.00+\nWrapStyle: 0\nScaledBorderAndShadow: yes\nPlayResX: 384\nPlayResY: 288\n\n\ + [V4+ Styles]\nFormat: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding\n\ + Style: Default,Arial,16,&H00FFFFFF,&H000000FF,&H00000000,&H00000000,0,0,0,0,100,100,0,0,1,1,0,2,10,10,10,1\n\n\ + [Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n", + ); + for c in &doc.cues { + // A comma in the Name field would shift the columns; the other fields are fixed text. + let name = c.speaker.as_deref().unwrap_or("").replace([',', '\n'], " "); + out.push_str(&format!("Dialogue: 0,{},{},Default,{},0,0,0,,{}\n", format_time(c.start), format_time(c.end), name, escape(&c.text))); + } + out +} diff --git a/crates/captions/src/lib.rs b/crates/captions/src/lib.rs index 0ba9f2ba5..e8a492722 100644 --- a/crates/captions/src/lib.rs +++ b/crates/captions/src/lib.rs @@ -4,7 +4,7 @@ //! - **Files:** [SubRip](srt) (`.srt`), [WebVTT](vtt) (`.vtt`), [Scenarist SCC](scc) (`.scc`, //! CEA-608 pop-on, roll-up and paint-on on read; pop-on on write), [MacCaption MCC](mcc) //! (`.mcc`, SMPTE ST 334-2 packets with CEA-608 and CEA-708), [EBU STL](stl) (`.stl`, EBU -//! Tech 3264) and [TTML](ttml) (`.ttml` IMSC1 Text profile, `.dfxp` DFXP) parse into a +//! Tech 3264), [ASS/SSA](ass) (`.ass`, `.ssa`) and [TTML](ttml) (`.ttml` IMSC1 Text profile, `.dfxp` DFXP) parse into a //! [`Document`] of [`Cue`]s and write back. Readers are forgiving (BOM, UTF-16, Windows-1252 //! fallback, CRLF/CR, missing SRT indexes, `,` or `.` milliseconds, missing hours, no blank //! line between cues); writers are strict. WebVTT cue identifiers, cue settings, `` @@ -32,6 +32,7 @@ #[doc = include_str!("../README.md")] struct ReadmeExamples; +pub mod ass; pub mod cea608; pub mod mcc; pub mod scc; @@ -69,10 +70,12 @@ pub enum Format { Ttml, /// DFXP (TTML1). Dfxp, + /// Advanced SubStation Alpha (also reads SubStation Alpha). + Ass, } impl Format { - pub const ALL: [Format; 7] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp]; + pub const ALL: [Format; 8] = [Format::Srt, Format::WebVtt, Format::Scc, Format::Mcc, Format::Stl, Format::Ttml, Format::Dfxp, Format::Ass]; pub fn name(self) -> &'static str { match self { Format::Srt => "SubRip (SRT)", @@ -82,6 +85,7 @@ impl Format { Format::Stl => "EBU STL", Format::Ttml => "TTML (IMSC1)", Format::Dfxp => "DFXP (TTML1)", + Format::Ass => "Advanced SubStation Alpha (ASS)", } } pub fn extension(self) -> &'static str { @@ -93,6 +97,7 @@ impl Format { Format::Stl => "stl", Format::Ttml => "ttml", Format::Dfxp => "dfxp", + Format::Ass => "ass", } } pub fn from_name(s: &str) -> Option { @@ -104,6 +109,7 @@ impl Format { "stl" | "ebu-stl" | "ebustl" | "ebu" => Some(Format::Stl), "ttml" | "imsc" | "imsc1" | "xml" => Some(Format::Ttml), "dfxp" => Some(Format::Dfxp), + "ass" | "ssa" => Some(Format::Ass), _ => None, } } @@ -251,6 +257,9 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option { if head.starts_with("File Format=MacCaption_MCC") { return Some(Format::Mcc); } + if head.starts_with("[Script Info]") { + return Some(Format::Ass); + } let ext = ext.map(|e| e.trim_start_matches('.').to_ascii_lowercase()); if (head.starts_with('<') && (head.contains("") || head.contains(":tt "))) && (head.contains("ns/ttml") || head.contains("ttaf1")) @@ -262,7 +271,7 @@ pub fn detect(bytes: &[u8], ext: Option<&str>) -> Option { match ext.as_deref() { Some("srt") if has_arrow_timing || head.is_empty() => Some(Format::Srt), Some("vtt") if has_arrow_timing => Some(Format::WebVtt), - Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml") => None, + Some("srt" | "vtt" | "scc" | "mcc" | "stl" | "ttml" | "dfxp" | "xml" | "ass" | "ssa") => None, _ if has_arrow_timing => Some(Format::Srt), _ => None, } @@ -296,6 +305,7 @@ fn parse_unbounded(bytes: &[u8], format: Format) -> Result { Format::Scc => scc::parse(&text), Format::Mcc => mcc::parse(&text), Format::Ttml | Format::Dfxp => ttml::parse(&text), + Format::Ass => ass::parse(&text), Format::Stl => stl::parse(bytes), } } @@ -333,6 +343,7 @@ pub fn write(doc: &Document, format: Format, opts: WriteOptions) -> Vec { Format::Stl => stl::write(doc, opts.rate, opts.program), Format::Ttml => ttml::write(doc, ttml::Flavor::Imsc1, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(), Format::Dfxp => ttml::write(doc, ttml::Flavor::Dfxp, opts.rate, doc.language.as_deref().unwrap_or("")).into_bytes(), + Format::Ass => ass::write(doc).into_bytes(), } } diff --git a/crates/captions/tests/it/ass.rs b/crates/captions/tests/it/ass.rs new file mode 100644 index 000000000..de5d43167 --- /dev/null +++ b/crates/captions/tests/it/ass.rs @@ -0,0 +1,113 @@ +//! SubStation Alpha / Advanced SubStation Alpha: reading, writing, and ffmpeg as the reference +//! (oracle; skipped when ffmpeg is absent, a failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). + +use craft_captions::{Cue, Document, Format, Tick, WriteOptions, detect, parse, write}; +use std::process::Command; + +const T: i64 = craft_captions::TICKS_PER_SECOND; + +fn cs(n: i64) -> Tick { + Tick(n * (T / 100)) +} + +const SAMPLE: &str = "\u{feff}[Script Info]\n; a comment\nTitle: Test\nScriptType: v4.00+\n\n[V4+ Styles]\nFormat: Name, Fontname\nStyle: Default,Arial\n\n[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nComment: 0,0:00:00.00,0:00:01.00,Default,,0,0,0,,not a cue\nDialogue: 0,0:00:01.50,0:00:03.25,Default,Bob,0,0,0,,Hello, {\\i1}world{\\i0}\\Nsecond {\\b1}line{\\b0}\nDialogue: 0,1:02:03.04,1:02:04.5,Default,,0,0,0,,{\\pos(10,20)\\c&H0000FF&}Red\\hspace\\nsoft\nDialogue: 0,0:00:09.00,0:00:10.00,Default,,0,0,0,,{\\p1}m 0 0 l 100 0 100 100{\\p0}\n"; + +#[test] +fn reads_dialogue_lines_with_commas_overrides_and_line_breaks() { + let doc = parse(SAMPLE.as_bytes(), Format::Ass).unwrap(); + assert_eq!(doc.cues.len(), 2); + assert_eq!(doc.cues[0].start, cs(150)); + assert_eq!(doc.cues[0].end, cs(325)); + assert_eq!(doc.cues[0].text, "Hello, world\nsecond line"); + assert_eq!(doc.cues[0].speaker.as_deref(), Some("Bob")); + // 1:02:03.04, a one-digit fraction, an override block dropped, \h and \n. + assert_eq!(doc.cues[1].start, Tick((3600 + 120 + 3) * T + 4 * (T / 100))); + assert_eq!(doc.cues[1].end, Tick((3600 + 120 + 4) * T + T / 2)); + assert_eq!(doc.cues[1].text, "Red\u{a0}space soft"); + assert!(doc.warnings.iter().any(|w| w.contains("vector drawing")), "{:?}", doc.warnings); +} + +#[test] +fn ssa_v4_files_read_through_their_own_format_line() { + let ssa = "[Script Info]\nScriptType: v4.00\n\n[V4 Styles]\nFormat: Name\n\n[Events]\nFormat: Marked, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nDialogue: Marked=0,0:00:01.00,0:00:02.00,*Default,,0000,0000,0000,,Old style\n"; + let doc = parse(ssa.as_bytes(), Format::Ass).unwrap(); + assert_eq!((doc.cues[0].start, doc.cues[0].end, doc.cues[0].text.as_str()), (cs(100), cs(200), "Old style")); +} + +#[test] +fn write_then_read_keeps_times_text_and_speaker() { + let cues = vec![ + Cue { start: cs(100), end: cs(250), text: "Line one\nline two".into(), speaker: Some("Ann".into()), ..Default::default() }, + Cue { + start: Tick(7200 * T + 99 * (T / 100)), + end: Tick(7203 * T), + text: "bold and under {braces}".into(), + ..Default::default() + }, + ]; + let doc = Document { cues, ..Default::default() }; + let bytes = write(&doc, Format::Ass, WriteOptions::default()); + assert_eq!(detect(&bytes, Some("ass")), Some(Format::Ass)); + let back = parse(&bytes, Format::Ass).unwrap(); + assert_eq!(back.cues[0], doc.cues[0]); + assert_eq!(back.cues[1].start, doc.cues[1].start); + // Braces are written full-width, as ASS reads `{` as the start of an override block. + assert_eq!(back.cues[1].text, "bold and under \u{ff5b}braces\u{ff5d}"); +} + +#[test] +fn files_without_an_events_section_are_not_ass() { + assert!(parse(b"[Script Info]\nTitle: x\n", Format::Ass).is_err()); + assert!(parse(b"hello", Format::Ass).is_err()); + assert_eq!(detect(b"hello", Some("ass")), None); +} + +#[test] +fn bad_lines_are_skipped_with_a_warning() { + let text = "[Events]\nFormat: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\nDialogue: 0,garbage,0:00:02.00,Default,,0,0,0,,x\nDialogue: 0,0:00:01.00,0:00:02.00,Default,,0,0,0,,ok\n"; + let doc = parse(text.as_bytes(), Format::Ass).unwrap(); + assert_eq!(doc.cues.len(), 1); + assert_eq!(doc.warnings.len(), 1); +} + +fn ffmpeg_run(ff: &std::path::Path, args: &[&str]) { + let out = Command::new(ff).args(["-v", "error", "-nostdin", "-y"]).args(args).output().expect("run ffmpeg"); + assert!(out.status.success(), "ffmpeg {args:?}: {}", String::from_utf8_lossy(&out.stderr)); +} + +const SRT: &str = "1\n00:00:01,500 --> 00:00:03,250\nHello, world\nsecond line\n\n2\n00:00:04,000 --> 00:00:05,990\nPlain, with a comma\n\n3\n01:00:00,010 --> 01:00:02,999\nUnder and italic both\n"; + +#[test] +fn we_read_what_ffmpeg_writes() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("captions_ass"); + let (srt, ass) = (dir.join("in.srt"), dir.join("ffmpeg.ass")); + std::fs::write(&srt, SRT).expect("write"); + ffmpeg_run(&ff, &["-i", srt.to_str().expect("path"), ass.to_str().expect("path")]); + let from_ass = parse(&std::fs::read(&ass).expect("read"), Format::Ass).expect("parse ass"); + let from_srt = parse(SRT.as_bytes(), Format::Srt).expect("parse srt"); + assert_eq!(from_ass.cues.len(), from_srt.cues.len()); + for (a, s) in from_ass.cues.iter().zip(&from_srt.cues) { + // ASS times are centiseconds; the SRT ones round to them. + assert_eq!(a.start, cs((s.start.0 + T / 200) / (T / 100)), "start of {:?}", s.text); + assert_eq!(a.end, cs((s.end.0 + T / 200) / (T / 100)), "end of {:?}", s.text); + assert_eq!(a.text, s.text); + } +} + +#[test] +fn ffmpeg_reads_what_we_write() { + let Some(ff) = craft_media_testkit::ffmpeg() else { return }; + let dir = craft_media_testkit::fixture_subdir("captions_ass"); + let doc = parse(SRT.as_bytes(), Format::Srt).expect("parse srt"); + let (ass, srt) = (dir.join("ours.ass"), dir.join("back.srt")); + std::fs::write(&ass, write(&doc, Format::Ass, WriteOptions::default())).expect("write"); + ffmpeg_run(&ff, &["-i", ass.to_str().expect("path"), srt.to_str().expect("path")]); + let theirs = parse(&std::fs::read(&srt).expect("read"), Format::Srt).expect("parse back"); + assert_eq!(theirs.cues.len(), doc.cues.len()); + for (t, d) in theirs.cues.iter().zip(&doc.cues) { + assert_eq!(t.start, cs((d.start.0 + T / 200) / (T / 100))); + assert_eq!(t.end, cs((d.end.0 + T / 200) / (T / 100))); + assert_eq!(t.text, d.text, "text of the cue at {:?}", d.start); + } +} diff --git a/crates/captions/tests/it/main.rs b/crates/captions/tests/it/main.rs index 16f8fcbb6..be03d2855 100644 --- a/crates/captions/tests/it/main.rs +++ b/crates/captions/tests/it/main.rs @@ -1,3 +1,4 @@ +mod ass; mod limits; mod literal_roundtrips; mod mcc_708_timing; From ee20ae63fdc314da17362b0e1dc429fa081a686b Mon Sep 17 00:00:00 2001 From: el-j Date: Sun, 15 Nov 2026 15:24:46 +0100 Subject: [PATCH 66/98] craft-audio: qoa_oracle test without index loops (CI clippy) --- crates/audio/tests/qoa_oracle.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/crates/audio/tests/qoa_oracle.rs b/crates/audio/tests/qoa_oracle.rs index cf0ed8c37..7b5ac2674 100644 --- a/crates/audio/tests/qoa_oracle.rs +++ b/crates/audio/tests/qoa_oracle.rs @@ -55,12 +55,7 @@ fn ffmpeg_decodes_our_files_to_the_same_samples() { .expect("run ffmpeg"); assert!(out.status.success(), "ffmpeg failed on {}: {}", file.display(), String::from_utf8_lossy(&out.stderr)); let theirs: Vec = out.stdout.as_chunks::<2>().0.iter().map(|b| i16::from_le_bytes(*b)).collect(); - let mut interleaved = Vec::with_capacity(frames * channels); - for i in 0..frames { - for c in 0..channels { - interleaved.push(ours.channels[c][i]); - } - } + let interleaved: Vec = (0..frames).flat_map(|i| ours.channels.iter().map(move |ch| ch[i])).collect(); assert_eq!(theirs.len(), interleaved.len(), "{rate} Hz, {channels} ch, {frames} frames: sample count"); assert!(theirs == interleaved, "{rate} Hz, {channels} ch, {frames} frames: samples differ"); } From 8322a1166c48967d557f549b37d827a07f359bcb Mon Sep 17 00:00:00 2001 From: Scott Macdonald Date: Sun, 15 Nov 2026 09:28:27 -0500 Subject: [PATCH 67/98] Port bounded Displace payload reader from PhotoCraft --- crates/psd/src/displace_map.rs | 37 ++++++++++++++++++++++++++++++++++ crates/psd/src/lib.rs | 1 + 2 files changed, 38 insertions(+) create mode 100644 crates/psd/src/displace_map.rs diff --git a/crates/psd/src/displace_map.rs b/crates/psd/src/displace_map.rs new file mode 100644 index 000000000..ce5193516 --- /dev/null +++ b/crates/psd/src/displace_map.rs @@ -0,0 +1,37 @@ +//! Bounded two-plane PSD Displace (`DspD`) payloads. +//! App filter settings, image conversion and external-path policy belong to the caller. +//! Ported from PhotoCraft `crates/io/src/displace_map.rs` (MIT/Apache-2.0). + +const HEADER: usize = 10; +const MODE_TWO_PLANES: u16 = 3; +/// Maximum supported decoded pixel count (8K by 8K). +pub const MAX_MAP_PIXELS: usize = 1 << 26; + +/// A parsed embedded map: size and the two 8-bit planes (borrowed from the blob). +#[derive(Debug, PartialEq)] +pub struct EmbeddedMap<'a> { + /// Width in pixels. + pub width: usize, + /// Height in pixels. + pub height: usize, + /// Horizontal displacement samples. + pub horizontal: &'a [u8], + /// Vertical displacement samples. + pub vertical: &'a [u8], +} + +/// Parses an embedded displacement map; `None` for any other mode, a zero or oversized map, or a +/// blob shorter than the header plus both planes. +pub fn parse_map(data: &[u8]) -> Option> { + let u32_at = |i: usize| Some(u32::from_le_bytes(data.get(i..i + 4)?.try_into().ok()?)); + let height = usize::try_from(u32_at(0)?).ok()?; + let width = usize::try_from(u32_at(4)?).ok()?; + let mode = u16::from_le_bytes(data.get(8..HEADER)?.try_into().ok()?); + let n = width.checked_mul(height)?; + if mode != MODE_TWO_PLANES || n == 0 || n > MAX_MAP_PIXELS { + return None; + } + let horizontal = data.get(HEADER..HEADER.checked_add(n)?)?; + let vertical = data.get(HEADER + n..HEADER.checked_add(n.checked_mul(2)?)?)?; + Some(EmbeddedMap { width, height, horizontal, vertical }) +} diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 8126437ff..acd3c8853 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -64,6 +64,7 @@ mod codepage; pub mod compression; pub mod csh; pub mod descriptor; +pub mod displace_map; pub mod engine_data; pub mod error; pub mod file; From 46cd009367b90ee4cd476cb70ffd28b1277b4df3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:29:01 -0500 Subject: [PATCH 68/98] Add exact-plane payload writer and hostile-input regression tests --- crates/psd/src/displace_map.rs | 56 ++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/crates/psd/src/displace_map.rs b/crates/psd/src/displace_map.rs index ce5193516..8b40eab3f 100644 --- a/crates/psd/src/displace_map.rs +++ b/crates/psd/src/displace_map.rs @@ -35,3 +35,59 @@ pub fn parse_map(data: &[u8]) -> Option> { let vertical = data.get(HEADER + n..HEADER.checked_add(n.checked_mul(2)?)?)?; Some(EmbeddedMap { width, height, horizontal, vertical }) } + +/// Writes a mode-3 payload from exact-sized 8-bit planes, without padding. +/// Returns `None` for empty/oversized dimensions, mismatched planes or allocation failure. +/// Callers retain their own (possibly lower) embedding and image-conversion budgets. +pub fn write_map(width: usize, height: usize, horizontal: &[u8], vertical: &[u8]) -> Option> { + let n = width.checked_mul(height)?; + if n == 0 || n > MAX_MAP_PIXELS || horizontal.len() != n || vertical.len() != n { + return None; + } + let width = u32::try_from(width).ok()?; + let height = u32::try_from(height).ok()?; + let len = HEADER.checked_add(n.checked_mul(2)?)?; + let mut out = Vec::new(); + out.try_reserve_exact(len).ok()?; + out.extend_from_slice(&height.to_le_bytes()); + out.extend_from_slice(&width.to_le_bytes()); + out.extend_from_slice(&MODE_TWO_PLANES.to_le_bytes()); + out.extend_from_slice(horizontal); + out.extend_from_slice(vertical); + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exact_planes_roundtrip_and_padding_is_ignored() { + let mut bytes = write_map(2, 1, &[0, 255], &[128, 64]).unwrap(); + assert_eq!(bytes, [1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 255, 128, 64]); + bytes.extend_from_slice(&[99, 88]); + let m = parse_map(&bytes).unwrap(); + assert_eq!((m.width, m.height), (2, 1)); + assert_eq!(m.horizontal, &[0, 255]); + assert_eq!(m.vertical, &[128, 64]); + } + + #[test] + fn truncated_unsupported_and_overflow_are_rejected() { + let bytes = write_map(2, 1, &[0, 255], &[128, 64]).unwrap(); + for end in 0..bytes.len() { + assert!(parse_map(&bytes[..end]).is_none()); + } + let mut invalid = bytes.clone(); + invalid[8] = 4; + assert!(parse_map(&invalid).is_none()); + invalid[..8].fill(255); + assert!(parse_map(&invalid).is_none()); + invalid[..8].fill(0); + assert!(parse_map(&invalid).is_none()); + assert!(write_map(usize::MAX, 2, &[], &[]).is_none()); + assert!(write_map(MAX_MAP_PIXELS + 1, 1, &[], &[]).is_none()); + assert!(write_map(2, 1, &[0], &[0, 0]).is_none()); + assert!(write_map(2, 1, &[0, 0], &[0, 0, 0]).is_none()); + } +} From f2106054e0f69b7df628c09235eadadd17906bf7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:30:28 -0500 Subject: [PATCH 69/98] Support bounded streaming displacement samples without plane copies --- crates/psd/src/displace_map.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/psd/src/displace_map.rs b/crates/psd/src/displace_map.rs index 8b40eab3f..a4df21445 100644 --- a/crates/psd/src/displace_map.rs +++ b/crates/psd/src/displace_map.rs @@ -40,8 +40,20 @@ pub fn parse_map(data: &[u8]) -> Option> { /// Returns `None` for empty/oversized dimensions, mismatched planes or allocation failure. /// Callers retain their own (possibly lower) embedding and image-conversion budgets. pub fn write_map(width: usize, height: usize, horizontal: &[u8], vertical: &[u8]) -> Option> { + write_map_iter(width, height, horizontal.iter().copied(), vertical.iter().copied()) +} + +/// Writes exact-sized sample iterators without allocating intermediate planes. +/// Iterators must terminate after exactly width times height samples; extra or missing +/// samples are rejected. At most that many samples plus one are consumed per plane. +pub fn write_map_iter( + width: usize, + height: usize, + mut horizontal: impl Iterator, + mut vertical: impl Iterator, +) -> Option> { let n = width.checked_mul(height)?; - if n == 0 || n > MAX_MAP_PIXELS || horizontal.len() != n || vertical.len() != n { + if n == 0 || n > MAX_MAP_PIXELS { return None; } let width = u32::try_from(width).ok()?; @@ -52,8 +64,14 @@ pub fn write_map(width: usize, height: usize, horizontal: &[u8], vertical: &[u8] out.extend_from_slice(&height.to_le_bytes()); out.extend_from_slice(&width.to_le_bytes()); out.extend_from_slice(&MODE_TWO_PLANES.to_le_bytes()); - out.extend_from_slice(horizontal); - out.extend_from_slice(vertical); + for samples in [&mut horizontal as &mut dyn Iterator, &mut vertical] { + for _ in 0..n { + out.push(samples.next()?); + } + if samples.next().is_some() { + return None; + } + } Some(out) } From 4d594a1130c78c88e9f9baf28891fc7d76162611 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 09:30:30 -0500 Subject: [PATCH 70/98] psd: share editable TySh binary envelope from PhotoCraft Preserve transform safety work and compatibility recovery; adapt to shared array geometry and add regression tests. Co-authored-by: David Mann Co-authored-by: Sean Lavery Co-authored-by: Dylan M. Taylor --- crates/psd/src/text.rs | 3 + crates/psd/src/text/envelope.rs | 142 ++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 crates/psd/src/text/envelope.rs diff --git a/crates/psd/src/text.rs b/crates/psd/src/text.rs index 6ba299bca..3a9e14952 100644 --- a/crates/psd/src/text.rs +++ b/crates/psd/src/text.rs @@ -150,3 +150,6 @@ mod tests { assert_eq!(t.style.size, Some(12.0)); } } + +/// Editable type-tool envelope with PhotoCraft-compatible recovery and geometry limits. +pub mod envelope; diff --git a/crates/psd/src/text/envelope.rs b/crates/psd/src/text/envelope.rs new file mode 100644 index 000000000..68664f7d7 --- /dev/null +++ b/crates/psd/src/text/envelope.rs @@ -0,0 +1,142 @@ +//! Editable PSD Type Tool Object Setting (`TySh`) envelope, extracted from PhotoCraft. +//! App-specific text styles, layout, units and editing policy stay with the consumer. +//! Parsing intentionally retains PhotoCraft's optional-warp and partial-bounds recovery. +//! Existing read-only [`super::TypeLayer`] behavior is unchanged. + +use crate::descriptor::{Descriptor, Id, Value as D, VersionedDescriptor}; + +/// Parsed `TySh` contents. +#[derive(Clone, Debug, PartialEq)] +pub struct TySh { + pub transform: [f64; 6], + pub text: Descriptor, + pub warp: Option, + /// The four trailing big-endian f32 values (left, top, right, bottom) after the warp + /// descriptor. Photoshop writes zeros for unwarped type and the text-space bounds (the + /// `bounds` descriptor's rectangle) for warped type. + pub bounds: [f32; 4], +} + +/// Whether a type transform can reach rendering. Pixel coordinates use i32 in the originating PhotoCraft consumer, so non-finite and unrepresentable affine terms are rejected (#4758). +pub fn transform_is_safe(t: &[f64; 6]) -> bool { + t.iter().all(|v| v.is_finite() && v.abs() <= f64::from(i32::MAX)) +} + +/// Parses a `TySh` block. +pub fn parse_tysh(data: &[u8]) -> Option { + let f = |at: usize| data.get(at..at + 8).map(|b| f64::from_be_bytes(b.try_into().unwrap_or([0; 8]))); + let mut m = [0.0; 6]; + for (i, v) in m.iter_mut().enumerate() { + *v = f(2 + i * 8)?; + } + if !transform_is_safe(&m) { + return None; + } + let (text, used) = VersionedDescriptor::parse_prefix(data.get(52..)?).ok()?; + let rest = data.get(52 + used..).unwrap_or(&[]); + let (warp, wused) = match rest.get(2..).and_then(|r| VersionedDescriptor::parse_prefix(r).ok()) { + Some((w, n)) => (Some(w.descriptor), n + 2), + None => (None, 0), + }; + let tail = rest.get(wused..).unwrap_or(&[]); + let mut bounds = [0f32; 4]; + for (i, b) in bounds.iter_mut().enumerate() { + if let Some(x) = tail.get(i * 4..i * 4 + 4) { + *b = f32::from_be_bytes(x.try_into().unwrap_or([0; 4])); + } + } + Some(TySh { transform: m, text: text.descriptor, warp, bounds }) +} + +/// `tysh` moved by (`dx`, `dy`): only the transform's translation changes. The text, its styles, +/// the warp and the trailing bounds (which are in text space) stay byte for byte, so a moved +/// Photoshop type layer keeps everything Photoshop wrote. `None` when the block is too short. +pub fn translate_tysh(tysh: &[u8], dx: f64, dy: f64) -> Option> { + let mut out = tysh.to_vec(); + // Version (2 bytes), then xx, xy, yx, yy, tx, ty as big-endian f64: tx at 34, ty at 42. + for (at, d) in [(34, dx), (42, dy)] { + let b = out.get_mut(at..at + 8)?; + let v = f64::from_be_bytes(b.try_into().ok()?) + d; + b.copy_from_slice(&v.to_be_bytes()); + } + Some(out) +} + +/// Serializes a `TySh` block. +pub fn write_tysh(t: &TySh) -> Vec { + let mut v = 1u16.to_be_bytes().to_vec(); + for x in t.transform { + v.extend_from_slice(&x.to_be_bytes()); + } + v.extend_from_slice(&50u16.to_be_bytes()); + v.extend(VersionedDescriptor::new(t.text.clone()).to_bytes()); + v.extend_from_slice(&1u16.to_be_bytes()); + v.extend(VersionedDescriptor::new(t.warp.clone().unwrap_or_else(default_warp)).to_bytes()); + for b in t.bounds { + v.extend_from_slice(&b.to_be_bytes()); + } + v +} + +fn default_warp() -> Descriptor { + Descriptor::new("warp") + .with("warpStyle", enumv("warpStyle", "warpNone")) + .with("warpValue", D::Double(0.0)) + .with("warpPerspective", D::Double(0.0)) + .with("warpPerspectiveOther", D::Double(0.0)) + .with("warpRotate", enumv("Ornt", "Hrzn")) +} + +fn enumv(t: &str, v: &str) -> D { + D::Enumerated { type_id: Id::new(t), value: Id::new(v) } +} + +#[cfg(test)] +mod tests { + use super::*; + fn sample() -> TySh { + TySh { + transform: [1.0, 0.0, 0.0, 1.0, 25.0, 30.0], + text: Descriptor::new("TxLr").with("EngineData", D::RawData(b"preserved".to_vec())), + warp: Some(default_warp()), + bounds: [-1.0, 2.0, 30.0, 40.0], + } + } + #[test] + fn roundtrip_preserves_descriptors_transform_and_bounds() { + let t = sample(); + let b = write_tysh(&t); + assert_eq!(parse_tysh(&b), Some(t)); + let shifted = translate_tysh(&b, 7.0, -3.0).unwrap(); + assert_eq!(&shifted[..34], &b[..34]); + assert_eq!(&shifted[50..], &b[50..]); + assert_eq!(parse_tysh(&shifted).unwrap().transform, [1.0, 0.0, 0.0, 1.0, 32.0, 27.0]); + } + #[test] + fn truncations_and_unsafe_transforms_preserve_recovery_contract() { + let mut t = sample(); + let bytes = write_tysh(&t); + let (_, n) = VersionedDescriptor::parse_prefix(&bytes[52..]).unwrap(); + for cut in 0..52 + n { + assert!(parse_tysh(&bytes[..cut]).is_none(), "{cut}"); + } + let short = parse_tysh(&bytes[..52 + n]).unwrap(); + assert!(short.warp.is_none()); + assert_eq!(short.bounds, [0.0; 4]); + for cut in 0..50 { + assert!(translate_tysh(&bytes[..cut], 1.0, 2.0).is_none()); + } + for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, f64::from(i32::MAX) + 1.0] { + for i in 0..6 { + t = sample(); + t.transform[i] = bad; + assert!(parse_tysh(&write_tysh(&t)).is_none()); + } + } + t = sample(); + t.warp = None; + assert!(parse_tysh(&write_tysh(&t)).unwrap().warp.is_some()); + let partial = parse_tysh(&bytes[..bytes.len() - 4]).unwrap(); + assert_eq!(partial.bounds, [-1.0, 2.0, 30.0, 0.0]); + } +} From d0feabc6c8c5998f76a68d2b6887660f23563e52 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:30:43 -0500 Subject: [PATCH 71/98] Exercise displacement dimension guards independently of mode --- crates/psd/src/displace_map.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/psd/src/displace_map.rs b/crates/psd/src/displace_map.rs index a4df21445..fbe8a7332 100644 --- a/crates/psd/src/displace_map.rs +++ b/crates/psd/src/displace_map.rs @@ -99,10 +99,15 @@ mod tests { let mut invalid = bytes.clone(); invalid[8] = 4; assert!(parse_map(&invalid).is_none()); + invalid[8] = 3; invalid[..8].fill(255); assert!(parse_map(&invalid).is_none()); invalid[..8].fill(0); assert!(parse_map(&invalid).is_none()); + invalid[..4].copy_from_slice(&1u32.to_le_bytes()); + invalid[4..8].copy_from_slice(&((MAX_MAP_PIXELS + 1) as u32).to_le_bytes()); + assert!(parse_map(&invalid).is_none()); + assert!(write_map_iter(2, 1, std::iter::repeat(0), std::iter::repeat(0)).is_none()); assert!(write_map(usize::MAX, 2, &[], &[]).is_none()); assert!(write_map(MAX_MAP_PIXELS + 1, 1, &[], &[]).is_none()); assert!(write_map(2, 1, &[0], &[0, 0]).is_none()); From 1e204c51dff0d49e528637b16d58f95cfa2b283a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:33:38 -0500 Subject: [PATCH 72/98] psd: bound TySh serialization and reject unsafe geometry --- crates/psd/src/text/envelope.rs | 186 ++++++++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 11 deletions(-) diff --git a/crates/psd/src/text/envelope.rs b/crates/psd/src/text/envelope.rs index 68664f7d7..e654ed390 100644 --- a/crates/psd/src/text/envelope.rs +++ b/crates/psd/src/text/envelope.rs @@ -3,7 +3,11 @@ //! Parsing intentionally retains PhotoCraft's optional-warp and partial-bounds recovery. //! Existing read-only [`super::TypeLayer`] behavior is unchanged. -use crate::descriptor::{Descriptor, Id, Value as D, VersionedDescriptor}; +use crate::descriptor::{Class, Descriptor, Id, ReferenceItem, UnicodeString, Value as D, VersionedDescriptor}; +use crate::error::{PsdError, Result}; + +/// Maximum encoded envelope size, bounding hostile input and serialized output. +pub const MAX_BYTES: usize = 8 * 1024 * 1024; /// Parsed `TySh` contents. #[derive(Clone, Debug, PartialEq)] @@ -24,6 +28,9 @@ pub fn transform_is_safe(t: &[f64; 6]) -> bool { /// Parses a `TySh` block. pub fn parse_tysh(data: &[u8]) -> Option { + if data.len() > MAX_BYTES { + return None; + } let f = |at: usize| data.get(at..at + 8).map(|b| f64::from_be_bytes(b.try_into().unwrap_or([0; 8]))); let mut m = [0.0; 6]; for (i, v) in m.iter_mut().enumerate() { @@ -45,6 +52,9 @@ pub fn parse_tysh(data: &[u8]) -> Option { *b = f32::from_be_bytes(x.try_into().unwrap_or([0; 4])); } } + if bounds.iter().any(|x| !x.is_finite()) { + return None; + } Some(TySh { transform: m, text: text.descriptor, warp, bounds }) } @@ -52,7 +62,21 @@ pub fn parse_tysh(data: &[u8]) -> Option { /// the warp and the trailing bounds (which are in text space) stay byte for byte, so a moved /// Photoshop type layer keeps everything Photoshop wrote. `None` when the block is too short. pub fn translate_tysh(tysh: &[u8], dx: f64, dy: f64) -> Option> { - let mut out = tysh.to_vec(); + if tysh.len() > MAX_BYTES || tysh.len() < 50 { + return None; + } + let mut transform = [0.0; 6]; + for (i, x) in transform.iter_mut().enumerate() { + *x = f64::from_be_bytes(tysh.get(2 + i * 8..10 + i * 8)?.try_into().ok()?); + } + transform[4] += dx; + transform[5] += dy; + if !transform_is_safe(&transform) { + return None; + } + let mut out = Vec::new(); + out.try_reserve_exact(tysh.len()).ok()?; + out.extend_from_slice(tysh); // Version (2 bytes), then xx, xy, yx, yy, tx, ty as big-endian f64: tx at 34, ty at 42. for (at, d) in [(34, dx), (42, dy)] { let b = out.get_mut(at..at + 8)?; @@ -63,19 +87,136 @@ pub fn translate_tysh(tysh: &[u8], dx: f64, dy: f64) -> Option> { } /// Serializes a `TySh` block. -pub fn write_tysh(t: &TySh) -> Vec { - let mut v = 1u16.to_be_bytes().to_vec(); +pub fn write_tysh(t: &TySh) -> Result> { + if !transform_is_safe(&t.transform) || t.bounds.iter().any(|x| !x.is_finite()) { + return Err(PsdError::LimitExceeded("TySh geometry")); + } + let default; + let warp = match &t.warp { + Some(w) => w, + None => { + default = default_warp(); + &default + } + }; + let mut budget = Budget(78); // versions, transform and trailing bounds + budget.descriptor(&t.text, 0)?; + budget.descriptor(warp, 0)?; + let mut v = Vec::new(); + v.try_reserve_exact(budget.0).map_err(|_| PsdError::OutOfMemory("TySh output"))?; + v.extend_from_slice(&1u16.to_be_bytes()); for x in t.transform { v.extend_from_slice(&x.to_be_bytes()); } v.extend_from_slice(&50u16.to_be_bytes()); - v.extend(VersionedDescriptor::new(t.text.clone()).to_bytes()); + v.extend_from_slice(&16u32.to_be_bytes()); + t.text.write(&mut v); v.extend_from_slice(&1u16.to_be_bytes()); - v.extend(VersionedDescriptor::new(t.warp.clone().unwrap_or_else(default_warp)).to_bytes()); + v.extend_from_slice(&16u32.to_be_bytes()); + warp.write(&mut v); for b in t.bounds { v.extend_from_slice(&b.to_be_bytes()); } - v + Ok(v) +} + +// Count before cloning or serializing: bound both recursive trees and total output. +struct Budget(usize); +impl Budget { + fn add(&mut self, n: usize) -> Result<()> { + self.0 = self.0.checked_add(n).filter(|n| *n <= MAX_BYTES).ok_or(PsdError::LimitExceeded("TySh bytes"))?; + Ok(()) + } + fn unicode(&mut self, s: &UnicodeString) -> Result<()> { + self.add(4)?; + self.add(s.0.len().checked_mul(2).ok_or(PsdError::LimitExceeded("TySh string"))?) + } + fn id(&mut self, id: &Id) -> Result<()> { + self.add(4)?; + self.add(id.as_bytes().len()) + } + fn class(&mut self, c: &Class) -> Result<()> { + self.unicode(&c.name)?; + self.id(&c.class_id) + } + fn descriptor(&mut self, d: &Descriptor, depth: usize) -> Result<()> { + if depth > 32 { + return Err(PsdError::LimitExceeded("TySh depth")); + } + self.unicode(&d.name)?; + self.id(&d.class_id)?; + self.add(4)?; + for (k, v) in &d.items { + self.id(k)?; + self.value(v, depth + 1)?; + } + Ok(()) + } + fn value(&mut self, v: &D, depth: usize) -> Result<()> { + if depth > 32 { + return Err(PsdError::LimitExceeded("TySh depth")); + } + self.add(4)?; + match v { + D::Descriptor(d) | D::GlobalObject(d) => self.descriptor(d, depth), + D::ObjectArray(o) => { + self.add(4)?; + self.descriptor(&o.body, depth) + } + D::List(xs) => { + self.add(4)?; + for x in xs { + self.value(x, depth + 1)?; + } + Ok(()) + } + D::Reference(xs) => { + self.add(4)?; + for x in xs { + self.add(4)?; + match x { + ReferenceItem::Property { class, key } => { + self.class(class)?; + self.id(key)?; + } + ReferenceItem::Class(c) => self.class(c)?, + ReferenceItem::Enumerated { class, type_id, value } => { + self.class(class)?; + self.id(type_id)?; + self.id(value)?; + } + ReferenceItem::Offset { class, .. } | ReferenceItem::Identifier { class, .. } | ReferenceItem::Index { class, .. } => { + self.class(class)?; + self.add(4)?; + } + ReferenceItem::Name { class, name } => { + self.class(class)?; + self.unicode(name)?; + } + } + } + Ok(()) + } + D::Text(x) => self.unicode(x), + D::Class(c) | D::GlobalClass(c) => self.class(c), + D::Enumerated { type_id, value } => { + self.id(type_id)?; + self.id(value) + } + D::UnitFloats { values, .. } => { + self.add(8)?; + self.add(values.len().checked_mul(8).ok_or(PsdError::LimitExceeded("TySh values"))?) + } + D::RawData(x) | D::Alias(x) | D::Path(x) => { + self.add(4)?; + self.add(x.len()) + } + D::Double(_) | D::LargeInteger(_) => self.add(8), + D::UnitFloat { .. } => self.add(12), + D::Integer(_) => self.add(4), + D::Boolean(_) => self.add(1), + } + } } fn default_warp() -> Descriptor { @@ -105,7 +246,7 @@ mod tests { #[test] fn roundtrip_preserves_descriptors_transform_and_bounds() { let t = sample(); - let b = write_tysh(&t); + let b = write_tysh(&t).unwrap(); assert_eq!(parse_tysh(&b), Some(t)); let shifted = translate_tysh(&b, 7.0, -3.0).unwrap(); assert_eq!(&shifted[..34], &b[..34]); @@ -115,7 +256,7 @@ mod tests { #[test] fn truncations_and_unsafe_transforms_preserve_recovery_contract() { let mut t = sample(); - let bytes = write_tysh(&t); + let bytes = write_tysh(&t).unwrap(); let (_, n) = VersionedDescriptor::parse_prefix(&bytes[52..]).unwrap(); for cut in 0..52 + n { assert!(parse_tysh(&bytes[..cut]).is_none(), "{cut}"); @@ -130,13 +271,36 @@ mod tests { for i in 0..6 { t = sample(); t.transform[i] = bad; - assert!(parse_tysh(&write_tysh(&t)).is_none()); + assert!(write_tysh(&t).is_err()); } } t = sample(); t.warp = None; - assert!(parse_tysh(&write_tysh(&t)).unwrap().warp.is_some()); + assert!(parse_tysh(&write_tysh(&t).unwrap()).unwrap().warp.is_some()); let partial = parse_tysh(&bytes[..bytes.len() - 4]).unwrap(); assert_eq!(partial.bounds, [-1.0, 2.0, 30.0, 0.0]); } + #[test] + fn hostile_output_and_translations_are_refused_before_copying() { + let bytes = write_tysh(&sample()).unwrap(); + for bad in [f64::NAN, f64::INFINITY, f64::MAX] { + assert!(translate_tysh(&bytes, bad, 0.0).is_none()); + } + let mut t = sample(); + t.bounds[0] = f32::NAN; + assert!(write_tysh(&t).is_err()); + t = sample(); + t.text = Descriptor::new("TxLr").with("data", D::RawData(vec![0; MAX_BYTES])); + assert!(write_tysh(&t).is_err()); + let oversized = vec![0; MAX_BYTES + 1]; + assert!(translate_tysh(&oversized, 0.0, 0.0).is_none()); + assert!(parse_tysh(&oversized).is_none()); + let mut nested = D::Integer(1); + for _ in 0..34 { + nested = D::List(vec![nested]); + } + t = sample(); + t.text = Descriptor::new("TxLr").with("nest", nested); + assert!(write_tysh(&t).is_err()); + } } From f483f9d6dd0ec9dcaf7697323cfab0b7f6fdc57b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 15 Nov 2026 09:39:55 -0500 Subject: [PATCH 73/98] cms: preserve original PDFCraft ICC v2 sRGB generator Extracted unchanged from storytold/pdfcraft e098c5c85, crates/preflight/src/icc.rs; original implementation 4d60af8b1, rename d3db99818. --- crates/cms/src/builtin_v2.rs | 128 +++++++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 crates/cms/src/builtin_v2.rs diff --git a/crates/cms/src/builtin_v2.rs b/crates/cms/src/builtin_v2.rs new file mode 100644 index 000000000..935dd105c --- /dev/null +++ b/crates/cms/src/builtin_v2.rs @@ -0,0 +1,128 @@ +//! A small sRGB ICC profile (version 2, display class, matrix/TRC), written from the published +//! sRGB primaries (IEC 61966-2-1) adapted to D50, with the sRGB tone curve sampled at 1024 +//! points. Used as the PDF/A output intent's destination profile. + +fn s15f16(v: f64) -> [u8; 4] { + ((v * 65536.0).round() as i32).to_be_bytes() +} + +fn xyz(x: f64, y: f64, z: f64) -> Vec { + let mut t = b"XYZ \0\0\0\0".to_vec(); + t.extend(s15f16(x)); + t.extend(s15f16(y)); + t.extend(s15f16(z)); + t +} + +fn desc(text: &str) -> Vec { + let mut t = b"desc\0\0\0\0".to_vec(); + let ascii: Vec = text.bytes().chain(std::iter::once(0)).collect(); + t.extend((ascii.len() as u32).to_be_bytes()); + t.extend(&ascii); + // No Unicode or ScriptCode descriptions. + t.extend([0u8; 4 + 4]); + t.extend([0u8; 2 + 1 + 67]); + t +} + +fn text(text: &str) -> Vec { + let mut t = b"text\0\0\0\0".to_vec(); + t.extend(text.bytes()); + t.push(0); + t +} + +fn curve() -> Vec { + const N: usize = 1024; + let mut t = b"curv\0\0\0\0".to_vec(); + t.extend((N as u32).to_be_bytes()); + for i in 0..N { + let v = i as f64 / (N - 1) as f64; + let lin = if v <= 0.04045 { v / 12.92 } else { ((v + 0.055) / 1.055).powf(2.4) }; + t.extend(((lin * 65535.0).round() as u16).to_be_bytes()); + } + t +} + +/// The profile's bytes. +pub fn srgb() -> Vec { + // sRGB primaries, Bradford-adapted to the D50 profile connection space. + let tags: Vec<(&[u8; 4], Vec)> = vec![ + (b"desc", desc("sRGB (PdfCraft)")), + (b"cprt", text("No copyright, use freely")), + (b"wtpt", xyz(0.9642, 1.0, 0.8249)), + (b"rXYZ", xyz(0.4361, 0.2225, 0.0139)), + (b"gXYZ", xyz(0.3851, 0.7169, 0.0971)), + (b"bXYZ", xyz(0.1431, 0.0606, 0.7141)), + (b"rTRC", curve()), + ]; + // gTRC and bTRC share rTRC's data. + let count = tags.len() + 2; + let table_len = 4 + count * 12; + let mut offset = 128 + table_len; + let mut table = (count as u32).to_be_bytes().to_vec(); + let mut data = Vec::new(); + let mut trc = (0, 0); + for (sig, body) in &tags { + let len = body.len(); + table.extend(*sig); + table.extend((offset as u32).to_be_bytes()); + table.extend((len as u32).to_be_bytes()); + if *sig == b"rTRC" { + trc = (offset, len); + } + data.extend(body); + // Tags start on 4-byte boundaries. + let pad = (4 - len % 4) % 4; + data.extend(std::iter::repeat_n(0u8, pad)); + offset += len + pad; + } + for sig in [b"gTRC", b"bTRC"] { + table.extend(sig); + table.extend((trc.0 as u32).to_be_bytes()); + table.extend((trc.1 as u32).to_be_bytes()); + } + let size = 128 + table.len() + data.len(); + let mut h = Vec::with_capacity(size); + h.extend((size as u32).to_be_bytes()); + h.extend(b"\0\0\0\0"); // preferred CMM + h.extend([2, 0x10, 0, 0]); // version 2.1 + h.extend(b"mntr"); + h.extend(b"RGB "); + h.extend(b"XYZ "); + h.extend([0u8; 12]); // date + h.extend(b"acsp"); + h.extend(b"\0\0\0\0"); // platform + h.extend([0u8; 4]); // flags + h.extend([0u8; 4]); // manufacturer + h.extend([0u8; 4]); // model + h.extend([0u8; 8]); // attributes + h.extend([0, 0, 0, 0]); // perceptual intent + h.extend(s15f16(0.9642)); + h.extend(s15f16(1.0)); + h.extend(s15f16(0.8249)); + h.extend([0u8; 4]); // creator + h.resize(128, 0); + h.extend(table); + h.extend(data); + h +} + +#[cfg(test)] +mod tests { + #[test] + fn profile_is_well_formed() { + let p = super::srgb(); + assert_eq!(u32::from_be_bytes(p[0..4].try_into().unwrap()) as usize, p.len()); + assert_eq!(&p[36..40], b"acsp"); + assert_eq!(&p[12..20], b"mntrRGB "); + let n = u32::from_be_bytes(p[128..132].try_into().unwrap()) as usize; + assert_eq!(n, 9); + for i in 0..n { + let e = 132 + i * 12; + let off = u32::from_be_bytes(p[e + 4..e + 8].try_into().unwrap()) as usize; + let len = u32::from_be_bytes(p[e + 8..e + 12].try_into().unwrap()) as usize; + assert!(off.is_multiple_of(4) && off + len <= p.len(), "tag {i}"); + } + } +} From 7bc5f2a4c8214b97394a5dcf60d01e9f5f5298c3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:40:37 -0500 Subject: [PATCH 74/98] cms: expose fixed v2 sRGB profile and verify shared parsing --- crates/cms/src/builtin.rs | 12 ++++++++++++ crates/cms/src/builtin_v2.rs | 17 ++++++++++++++++- crates/cms/src/lib.rs | 1 + 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/crates/cms/src/builtin.rs b/crates/cms/src/builtin.rs index ca7034286..e62faa907 100644 --- a/crates/cms/src/builtin.rs +++ b/crates/cms/src/builtin.rs @@ -450,3 +450,15 @@ mod tests { assert!(cicp_rgb(u16::MAX, u16::MAX, true).is_none()); } } + +/// Generates the compact ICC v2.1 sRGB matrix/TRC profile used by PDFCraft. +/// +/// Unlike [`Builtin::Srgb`]'s v4 profile, this uses a 1024-sample tone curve and +/// v2 `desc`/`text` tags for consumers that require a v2 output intent. The legacy +/// description `sRGB (PdfCraft)` is retained to preserve existing document bytes. +/// The generated profile contains no external assets; callers choose where it is +/// appropriate (including their own PDF/A conformance policy). +/// All sizes are fixed; no input-controlled allocation or recursion is involved. +pub fn srgb_v2() -> Vec { + crate::builtin_v2::srgb() +} diff --git a/crates/cms/src/builtin_v2.rs b/crates/cms/src/builtin_v2.rs index 935dd105c..4a9e2e25b 100644 --- a/crates/cms/src/builtin_v2.rs +++ b/crates/cms/src/builtin_v2.rs @@ -44,7 +44,7 @@ fn curve() -> Vec { t } -/// The profile's bytes. +/// Fixed-size profile bytes; all tag sizes and loop counts are constants. pub fn srgb() -> Vec { // sRGB primaries, Bradford-adapted to the D50 profile connection space. let tags: Vec<(&[u8; 4], Vec)> = vec![ @@ -110,6 +110,21 @@ pub fn srgb() -> Vec { #[cfg(test)] mod tests { + #[test] + fn parses_as_v2_with_shared_rgb_curves() { + let bytes = super::srgb(); + let p = crate::Profile::parse(&bytes).unwrap(); + assert_eq!(p.version, (2, 0x10)); + assert_eq!(p.color_space, crate::ColorSpace::Rgb); + assert_eq!(p.pcs, crate::Pcs::Xyz); + assert_eq!(p.description, "sRGB (PdfCraft)"); + let curves = p.trc.unwrap(); + assert_eq!(curves[0], curves[1]); + assert_eq!(curves[1], curves[2]); + assert!(matches!(&curves[0], crate::Curve::Table(v) if v.len() == 1024 && v[0] == 0.0 && v[1023] == 1.0)); + assert_eq!(bytes, super::srgb()); + } + #[test] fn profile_is_well_formed() { let p = super::srgb(); diff --git a/crates/cms/src/lib.rs b/crates/cms/src/lib.rs index 8d4f99f49..5e8a1e268 100644 --- a/crates/cms/src/lib.rs +++ b/crates/cms/src/lib.rs @@ -20,6 +20,7 @@ #![forbid(unsafe_code)] pub mod builtin; +mod builtin_v2; pub mod clut; pub mod curve; pub mod gamut; From 424a2dc2be7a90467bca0f62b2f161b6bed57bb0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 14:41:14 +0000 Subject: [PATCH 75/98] Share streaming gzip writer with stored segments --- Cargo.lock | 21 ++++--- Cargo.toml | 1 + crates/gzip/Cargo.toml | 14 +++++ crates/gzip/src/lib.rs | 119 ++++++++++++++++++++++++++++++++++++ crates/gzip/tests/writer.rs | 77 +++++++++++++++++++++++ 5 files changed, 225 insertions(+), 7 deletions(-) create mode 100644 crates/gzip/Cargo.toml create mode 100644 crates/gzip/src/lib.rs create mode 100644 crates/gzip/tests/writer.rs diff --git a/Cargo.lock b/Cargo.lock index c396f1978..222e4679d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1843,6 +1843,13 @@ dependencies = [ "image", ] +[[package]] +name = "craft-gzip" +version = "0.1.0" +dependencies = [ + "flate2", +] + [[package]] name = "craft-hdr" version = "0.1.0" @@ -2375,13 +2382,6 @@ dependencies = [ "thiserror 2.0.21", ] -[[package]] -name = "craft-plist" -version = "0.1.0" -dependencies = [ - "thiserror 2.0.21", -] - [[package]] name = "craft-platform" version = "0.1.0" @@ -2399,6 +2399,13 @@ dependencies = [ "zbus", ] +[[package]] +name = "craft-plist" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-plotter" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 4985d19d0..c3751dacb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ hayro-syntax = { version = "=0.7.2", default-features = false, features = ["std" bytemuck = "1" kurbo = "0.13" thiserror = "2" +craft-gzip = { path = "crates/gzip" } craft-aaf = { path = "crates/aaf" } craft-atomic = { path = "crates/atomic" } craft-captions = { path = "crates/captions" } diff --git a/crates/gzip/Cargo.toml b/crates/gzip/Cargo.toml new file mode 100644 index 000000000..3d4f04603 --- /dev/null +++ b/crates/gzip/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-gzip" +description = "Streaming gzip writer with stored and compressed segments" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } diff --git a/crates/gzip/src/lib.rs b/crates/gzip/src/lib.rs new file mode 100644 index 000000000..33f2a5ac1 --- /dev/null +++ b/crates/gzip/src/lib.rs @@ -0,0 +1,119 @@ +//! Streaming gzip output with optional stored segments. +//! +//! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors. +//! Extracted from VectorCraft, licensed under MIT OR Apache-2.0. +//! +//! `flush` does not emit a deflate block; call [`Gz::finish`] to complete and flush the +//! stream. After an I/O error, discard the writer: the output may be incomplete. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use flate2::{Compress, Compression, Crc, FlushCompress, Status}; +use std::io::{self, Write}; + +/// Bytes of compressed output gathered before they go to the writer. +const OUT_CHUNK: usize = 64 << 10; + +/// A gzip stream, written as [`flate2::write::GzEncoder`] at the default level writes it (the +/// same header and the same data; the deflate blocks are the same too where flate2 deflates with +/// miniz_oxide, not with zlib-rs, which writes them by how the data is fed), except that data which +/// is compressed already ([`Gz::packed`]) is stored as it is: the default level spends a second +/// or more per photo on it for next to nothing, and makes it slower to unpack. Where the level +/// changes the deflate blocks so far end with a sync flush and a fresh compressor writes on, +/// which makes one valid deflate stream that every reader unpacks. +pub struct Gz { + out: W, + deflate: Compress, + /// What `deflate` writes: stored blocks (packed data), else the default level. + stored: bool, + crc: Crc, + buf: Vec, + /// Bytes written (unpacked). + total: u64, +} + +impl Gz { + /// Write a gzip header and construct a writer. + pub fn new(mut out: W) -> io::Result { + // flate2's header for the default level: no name, no time, unknown OS. + out.write_all(&[0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 0, 255])?; + Ok(Self { + out, + deflate: Compress::new(Compression::default(), false), + stored: false, + crc: Crc::new(), + buf: Vec::with_capacity(OUT_CHUNK), + total: 0, + }) + } + + /// Run the compressor over `input` with `flush` until it has taken it all (and, flushing, + /// written everything out). + fn deflate(&mut self, mut input: &[u8], flush: FlushCompress) -> io::Result<()> { + loop { + if self.buf.len() == self.buf.capacity() { + self.out.write_all(&self.buf)?; + self.buf.clear(); + } + let (before_in, before_out) = (self.deflate.total_in(), self.deflate.total_out()); + let status = self.deflate.compress_vec(input, &mut self.buf, flush).map_err(io::Error::other)?; + let taken = usize::try_from(self.deflate.total_in() - before_in).unwrap_or(input.len()).min(input.len()); + input = input.get(taken..).unwrap_or_default(); + let made = self.deflate.total_out() - before_out; + let full = self.buf.len() == self.buf.capacity(); + match flush { + FlushCompress::None if input.is_empty() => return Ok(()), + FlushCompress::Finish if status == Status::StreamEnd => return Ok(()), + // A flush is done once it left room in the output. + FlushCompress::Sync if input.is_empty() && !full => return Ok(()), + _ => {} + } + if taken == 0 && made == 0 && !full { + return Err(io::Error::other("the compressor made no progress")); + } + } + } + + /// Return the number of uncompressed bytes supplied. + pub fn written(&self) -> u64 { + self.total + } + + /// What is written next is compressed already (stored as it is), or not (compressed at the + /// default level). + pub fn packed(&mut self, packed: bool) -> io::Result<()> { + if packed == self.stored { + return Ok(()); + } + self.deflate(&[], FlushCompress::Sync)?; + self.deflate = Compress::new(if packed { Compression::none() } else { Compression::default() }, false); + self.stored = packed; + Ok(()) + } + + /// Complete the gzip trailer and flush the underlying writer. + pub fn finish(mut self) -> io::Result<()> { + self.deflate(&[], FlushCompress::Finish)?; + self.out.write_all(&self.buf)?; + self.out.write_all(&self.crc.sum().to_le_bytes())?; + self.out.write_all(&self.crc.amount().to_le_bytes())?; + self.out.flush() + } +} + +impl Write for Gz { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.total = self.total.checked_add(buf.len() as u64).ok_or_else(|| io::Error::other("uncompressed byte count overflow"))?; + self.crc.update(buf); + self.deflate(buf, FlushCompress::None) + } + + fn flush(&mut self) -> io::Result<()> { + // Flushing would end a deflate block: the stream is flushed when it is finished. + Ok(()) + } +} diff --git a/crates/gzip/tests/writer.rs b/crates/gzip/tests/writer.rs new file mode 100644 index 000000000..38981ad0a --- /dev/null +++ b/crates/gzip/tests/writer.rs @@ -0,0 +1,77 @@ +use std::io::{self, Read, Write}; + +use craft_gzip::Gz; +use flate2::{Compression, read::GzDecoder, write::GzEncoder}; + +fn decode(bytes: &[u8]) -> Vec { + let mut result = Vec::new(); + GzDecoder::new(bytes).read_to_end(&mut result).unwrap(); + result +} + +#[test] +fn matches_flate2_without_mode_changes() { + let data = b"The quick brown fox jumps over the lazy dog. ".repeat(10000); + let mut expected = GzEncoder::new(Vec::new(), Compression::default()); + let mut actual = Vec::new(); + { + let mut gz = Gz::new(&mut actual).unwrap(); + for chunk in data.chunks(131) { + expected.write_all(chunk).unwrap(); + gz.write_all(chunk).unwrap(); + } + assert_eq!(gz.written(), data.len() as u64); + gz.finish().unwrap(); + } + assert_eq!(actual, expected.finish().unwrap()); + assert_eq!(decode(&actual), data); +} + +#[test] +fn mixed_segments_round_trip_and_stored_bytes_remain_uncompressed() { + let plain = b"repeat me ".repeat(20000); + let packed: Vec = (0..200_000).map(|i| (i * 31) as u8).collect(); + let mut bytes = Vec::new(); + { + let mut gz = Gz::new(&mut bytes).unwrap(); + gz.write_all(&plain).unwrap(); + gz.packed(true).unwrap(); + gz.packed(true).unwrap(); + gz.write_all(&packed).unwrap(); + gz.packed(false).unwrap(); + gz.write_all(&plain).unwrap(); + assert_eq!(gz.written(), (plain.len() * 2 + packed.len()) as u64); + gz.finish().unwrap(); + } + let expected = [plain.as_slice(), packed.as_slice(), plain.as_slice()].concat(); + assert_eq!(decode(&bytes), expected); + assert!(bytes.windows(1024).any(|window| window == &packed[..1024])); + assert!(bytes.len() < expected.len()); +} + +struct FailAfter { + remaining: usize, +} +impl Write for FailAfter { + fn write(&mut self, data: &[u8]) -> io::Result { + if self.remaining == 0 { + return Err(io::Error::other("injected failure")); + } + let n = self.remaining.min(data.len()); + self.remaining -= n; + Ok(n) + } + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("injected flush failure")) + } +} + +#[test] +fn propagates_header_body_and_flush_failures() { + assert!(Gz::new(FailAfter { remaining: 0 }).is_err()); + let mut body = Gz::new(FailAfter { remaining: 10 }).unwrap(); + body.packed(true).unwrap(); + assert!(body.write_all(&vec![42; 200_000]).is_err()); + let empty = Gz::new(FailAfter { remaining: usize::MAX }).unwrap(); + assert_eq!(empty.finish().unwrap_err().to_string(), "injected flush failure"); +} From cab96afd08c3b68073340d2162657ccc989bd751 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 09:43:27 -0500 Subject: [PATCH 76/98] cms: keep public v2 helper before test module --- crates/cms/src/builtin.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/crates/cms/src/builtin.rs b/crates/cms/src/builtin.rs index e62faa907..5a8b03228 100644 --- a/crates/cms/src/builtin.rs +++ b/crates/cms/src/builtin.rs @@ -424,6 +424,18 @@ fn shipped_cmyk(bytes: &[u8], b: Builtin) -> Profile { }) } +/// Generates the compact ICC v2.1 sRGB matrix/TRC profile used by PDFCraft. +/// +/// Unlike [`Builtin::Srgb`]'s v4 profile, this uses a 1024-sample tone curve and +/// v2 `desc`/`text` tags for consumers that require a v2 output intent. The legacy +/// description `sRGB (PdfCraft)` is retained to preserve existing document bytes. +/// The generated profile contains no external assets; callers choose where it is +/// appropriate (including their own PDF/A conformance policy). +/// All sizes are fixed; no input-controlled allocation or recursion is involved. +pub fn srgb_v2() -> Vec { + crate::builtin_v2::srgb() +} + #[cfg(test)] mod tests { use super::*; @@ -450,15 +462,3 @@ mod tests { assert!(cicp_rgb(u16::MAX, u16::MAX, true).is_none()); } } - -/// Generates the compact ICC v2.1 sRGB matrix/TRC profile used by PDFCraft. -/// -/// Unlike [`Builtin::Srgb`]'s v4 profile, this uses a 1024-sample tone curve and -/// v2 `desc`/`text` tags for consumers that require a v2 output intent. The legacy -/// description `sRGB (PdfCraft)` is retained to preserve existing document bytes. -/// The generated profile contains no external assets; callers choose where it is -/// appropriate (including their own PDF/A conformance policy). -/// All sizes are fixed; no input-controlled allocation or recursion is involved. -pub fn srgb_v2() -> Vec { - crate::builtin_v2::srgb() -} From 0cdda6e15d274847962bf1cb98a91d1a3a9b5cbf Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 14:44:38 +0000 Subject: [PATCH 77/98] Share VectorCraft streaming envelope helpers --- Cargo.lock | 15 +- crates/vectorcraft-format/Cargo.toml | 3 + crates/vectorcraft-format/src/lib.rs | 13 +- crates/vectorcraft-format/src/stream.rs | 242 ++++++++++++++++++++++ crates/vectorcraft-format/tests/stream.rs | 83 ++++++++ 5 files changed, 348 insertions(+), 8 deletions(-) create mode 100644 crates/vectorcraft-format/src/stream.rs create mode 100644 crates/vectorcraft-format/tests/stream.rs diff --git a/Cargo.lock b/Cargo.lock index c396f1978..759f68168 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2375,13 +2375,6 @@ dependencies = [ "thiserror 2.0.21", ] -[[package]] -name = "craft-plist" -version = "0.1.0" -dependencies = [ - "thiserror 2.0.21", -] - [[package]] name = "craft-platform" version = "0.1.0" @@ -2399,6 +2392,13 @@ dependencies = [ "zbus", ] +[[package]] +name = "craft-plist" +version = "0.1.0" +dependencies = [ + "thiserror 2.0.21", +] + [[package]] name = "craft-plotter" version = "0.1.0" @@ -2841,6 +2841,7 @@ name = "craft-vectorcraft-format" version = "0.1.0" dependencies = [ "flate2", + "proptest", "serde", "serde_json", "thiserror 2.0.21", diff --git a/crates/vectorcraft-format/Cargo.toml b/crates/vectorcraft-format/Cargo.toml index e85d90886..c10435289 100644 --- a/crates/vectorcraft-format/Cargo.toml +++ b/crates/vectorcraft-format/Cargo.toml @@ -15,3 +15,6 @@ flate2 = { version = "1", default-features = false, features = ["rust_backend"] serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/vectorcraft-format/src/lib.rs b/crates/vectorcraft-format/src/lib.rs index 051c31a70..b180c046c 100644 --- a/crates/vectorcraft-format/src/lib.rs +++ b/crates/vectorcraft-format/src/lib.rs @@ -34,6 +34,7 @@ //! [`sniff`] reads: a reader of a compressed file decompresses it with its own limits first. pub mod base64; +pub mod stream; use std::collections::{BTreeMap, HashMap}; use std::fmt; @@ -769,10 +770,20 @@ impl<'a> Tail<'a> { /// Write what follows the JSON: [`BLOB_MAGIC`] and the entries, or nothing when there are none. pub fn write_to(&self, mut w: impl Write) -> io::Result<()> { + self.write_to_with(&mut w, |_, _| Ok(())) + } + + /// Write the marker and parts, calling `before_part` before each payload. A compressed + /// writer can use the MIME type to switch between compressed and stored deflate blocks. + pub fn write_to_with(&self, mut w: W, mut before_part: impl FnMut(&mut W, &str) -> io::Result<()>) -> io::Result<()> { if self.parts.is_empty() { return Ok(()); } w.write_all(BLOB_MAGIC)?; - self.parts.iter().try_for_each(|p| w.write_all(p.bytes)) + for part in &self.parts { + before_part(&mut w, &part.mime)?; + w.write_all(part.bytes)?; + } + Ok(()) } } diff --git a/crates/vectorcraft-format/src/stream.rs b/crates/vectorcraft-format/src/stream.rs new file mode 100644 index 000000000..8f205a15d --- /dev/null +++ b/crates/vectorcraft-format/src/stream.rs @@ -0,0 +1,242 @@ +//! Streaming helpers for the native JSON envelope. +//! +//! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors. +//! Extracted from VectorCraft, licensed under MIT OR Apache-2.0. + +use serde::Serialize; +use std::collections::HashSet; +use std::io::{self, Write}; + +/// Indents JSON text written through it one level (two spaces after each newline), as a member +/// of the file's top-level object; newlines only occur between tokens, inside strings they are +/// escaped. +pub struct Indent { + pub inner: W, + pub on: bool, +} + +impl Write for Indent { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + if !self.on { + return self.inner.write_all(buf); + } + let mut lines = buf.split(|&b| b == b'\n'); + self.inner.write_all(lines.next().unwrap_or_default())?; + for line in lines { + self.inner.write_all(b"\n ")?; + self.inner.write_all(line)?; + } + Ok(()) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// The member that names an image. +const TAG: &[u8] = b"\"key\":"; + +/// Passes JSON text through, collecting the string values of every `"key": "…"` member at any +/// depth (what the images a document uses are named by). Quotes inside JSON strings are escaped, +/// so every match is a real member. +pub struct KeyScan { + pub inner: W, + pub keys: HashSet, + /// Looking for keys (else it only passes the text on). + pub active: bool, + /// How much of [`TAG`] the last bytes matched. + matched: usize, + /// Right after a tag: skipping whitespace to its value. + after_tag: bool, + /// The value string so far, quotes included. + value: Option>, + escaped: bool, +} + +impl KeyScan { + pub fn new(inner: W) -> Self { + Self { inner, keys: HashSet::new(), active: true, matched: 0, after_tag: false, value: None, escaped: false } + } + + fn scan(&mut self, buf: &[u8]) { + if !self.active { + return; + } + let mut i = 0; + while let Some(&b) = buf.get(i) { + // Nothing in progress: skip to the next quote. + if self.matched == 0 && !self.after_tag && self.value.is_none() { + match buf.get(i..).and_then(|r| r.iter().position(|&c| c == b'"')) { + Some(at) => i += at, + None => return, + } + self.matched = 1; + i += 1; + continue; + } + self.step(b); + i += 1; + } + } + + fn step(&mut self, b: u8) { + if let Some(v) = self.value.as_mut() { + v.push(b); + if self.escaped { + self.escaped = false; + } else if b == b'\\' { + self.escaped = true; + } else if b == b'"' + && let Some(v) = self.value.take() + && let Ok(s) = serde_json::from_slice::(&v) + { + self.keys.insert(s); + } + } else if self.after_tag && !b.is_ascii_whitespace() { + self.after_tag = false; + if b == b'"' { + self.value = Some(vec![b]); + } + } + if TAG.get(self.matched) == Some(&b) { + self.matched += 1; + if self.matched == TAG.len() { + self.matched = 0; + self.after_tag = true; + self.value = None; + self.escaped = false; + } + } else { + self.matched = usize::from(b == b'"'); + } + } +} + +impl Write for KeyScan { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_all(buf)?; + Ok(buf.len()) + } + + fn write_all(&mut self, buf: &[u8]) -> io::Result<()> { + self.scan(buf); + self.inner.write_all(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +/// The file's top-level object, written member by member (the document is streamed in). +pub struct Envelope { + pretty: bool, + first: bool, +} + +impl Envelope { + /// Start an object with optional indentation. + pub fn new(pretty: bool) -> Self { + Self { pretty, first: true } + } + + /// Serialize a complete member before writing it to the envelope. + pub fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> io::Result<()> { + let json = if self.pretty { serde_json::to_vec_pretty(value) } else { serde_json::to_vec(value) }.map_err(io::Error::other)?; + self.key(out, key)?; + Indent { inner: out, on: self.pretty }.write_all(&json) + } + + /// The start of member `key` (a plain name): its value follows, indented one level when + /// pretty (as serde_json would: see [`Indent`]). + pub fn key(&mut self, out: &mut impl Write, key: &str) -> io::Result<()> { + let sep: &[u8] = if std::mem::take(&mut self.first) { b"{" } else { b"," }; + let indent: &[u8] = if self.pretty { b"\n " } else { b"" }; + out.write_all(sep)?; + out.write_all(indent)?; + serde_json::to_writer(&mut *out, key).map_err(io::Error::other)?; + out.write_all(if self.pretty { b": " } else { b":" }) + } + + /// Close the object. + pub fn finish(self, out: &mut impl Write) -> io::Result<()> { + if self.first { + return out.write_all(b"{}"); + } + out.write_all(if self.pretty { b"\n}" } else { b"}" }) + } +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + + /// How the keys were found before saves were streamed: in the whole JSON text at once. + fn string_members(json: &[u8]) -> HashSet { + let mut out = HashSet::new(); + let mut rest = json; + while let Some(at) = rest.windows(TAG.len()).position(|w| w == TAG) { + rest = rest.get(at + TAG.len()..).unwrap_or_default(); + let value = rest.trim_ascii_start(); + if value.first() == Some(&b'"') + && let Some(Ok(s)) = serde_json::Deserializer::from_slice(value).into_iter::().next() + { + out.insert(s); + } + } + out + } + + /// The keys `KeyScan` finds in `json` written in pieces of `chunk` bytes. + fn scanned(json: &[u8], chunk: usize) -> HashSet { + let mut scan = KeyScan::new(Vec::new()); + for piece in json.chunks(chunk.max(1)) { + scan.write_all(piece).unwrap(); + } + assert_eq!(scan.inner, json, "the text passes through"); + scan.keys + } + + #[test] + fn finds_the_keys_the_whole_text_search_found() { + let v = serde_json::json!({"key": "a", "x": [{"key": "b \"q\" \\ é"}, {"monkey": "no"}, {"k": "key"}], "s": "\"key\": \"no\"", "key2": "n", "o": {"key": 3}}); + for json in [serde_json::to_vec(&v).unwrap(), serde_json::to_vec_pretty(&v).unwrap()] { + let want = string_members(&json); + assert_eq!(want, HashSet::from(["a".to_string(), "b \"q\" \\ é".to_string()])); + for chunk in [1, 2, 3, 7, 64, json.len()] { + assert_eq!(scanned(&json, chunk), want, "chunk {chunk}"); + } + } + } + + fn arb_json() -> impl Strategy { + let key = prop::sample::select(vec!["key", "k", "kkey", "key ", "x"]).prop_map(str::to_string); + let leaf = prop_oneof![ + Just(serde_json::Value::Null), + any::().prop_map(serde_json::Value::from), + "[a-z\"\\\\: ké\n]{0,8}".prop_map(serde_json::Value::from), + ]; + leaf.prop_recursive(4, 32, 6, move |inner| { + prop_oneof![ + prop::collection::vec(inner.clone(), 0..5).prop_map(serde_json::Value::from), + prop::collection::vec((key.clone(), inner), 0..5).prop_map(|m| serde_json::Value::Object(m.into_iter().collect())), + ] + }) + } + + proptest! { + #[test] + fn scanning_in_pieces_finds_what_the_whole_text_search_found(v in arb_json(), chunk in 1usize..40, pretty in any::()) { + let json = if pretty { serde_json::to_vec_pretty(&v) } else { serde_json::to_vec(&v) }.unwrap(); + prop_assert_eq!(scanned(&json, chunk), string_members(&json)); + } + } +} diff --git a/crates/vectorcraft-format/tests/stream.rs b/crates/vectorcraft-format/tests/stream.rs new file mode 100644 index 000000000..728f533ca --- /dev/null +++ b/crates/vectorcraft-format/tests/stream.rs @@ -0,0 +1,83 @@ +use craft_vectorcraft_format::stream::{Envelope, Indent, KeyScan}; +use std::io::{self, Write}; + +#[test] +fn envelope_streams_nested_json_and_escapes_keys() { + for pretty in [false, true] { + let mut bytes = Vec::new(); + let mut env = Envelope::new(pretty); + env.field(&mut bytes, "quoted\"key", &vec![1, 2]).unwrap(); + env.key(&mut bytes, "document").unwrap(); + let mut output = Indent { inner: &mut bytes, on: pretty }; + if pretty { + serde_json::to_writer_pretty(&mut output, &serde_json::json!({"nested": [3,4]})).unwrap(); + } else { + serde_json::to_writer(&mut output, &serde_json::json!({"nested": [3,4]})).unwrap(); + } + env.finish(&mut bytes).unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value, serde_json::json!({"quoted\"key": [1, 2], "document": {"nested": [3,4]}})); + let mut expected = Vec::new(); + let empty = Envelope::new(pretty); + empty.finish(&mut expected).unwrap(); + assert_eq!(expected, b"{}"); + } +} + +#[test] +fn scan_survives_every_chunk_boundary_and_forwards_data() { + let json = br#"{"key":"one","nested":[{"key":"two \\\"quoted\\\""},{"key":42}],"other":"x"}"#; + for chunk in 1..=json.len() { + let mut scan = KeyScan::new(Vec::new()); + for part in json.chunks(chunk) { + scan.write_all(part).unwrap(); + } + assert_eq!(scan.inner, json); + assert!(scan.keys.contains("one")); + assert_eq!(scan.keys.len(), 2); + } +} + +struct Broken; +impl Write for Broken { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::other("failure")) + } + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("failure")) + } +} +#[test] +fn forwards_writer_errors() { + assert!(Envelope::new(false).field(&mut Broken, "x", &1).is_err()); + assert!(KeyScan::new(Broken).write_all(b"{}").is_err()); + assert!(Indent { inner: Broken, on: true }.write_all(b"x\ny").is_err()); +} + +#[test] +fn tail_calls_hook_in_order_and_propagates_errors() { + use craft_vectorcraft_format::{BLOB_MAGIC, INLINE_MAX, Tail}; + let first = vec![1; INLINE_MAX]; + let second = vec![2; INLINE_MAX]; + let mut tail = Tail::new(4); + tail.image("image/png", &first, false); + tail.image("application/pdf", &second, false); + let mut out = Vec::new(); + let mut seen = Vec::new(); + tail.write_to_with(&mut out, |_, mime| { + seen.push(mime.to_owned()); + Ok(()) + }) + .unwrap(); + assert_eq!(seen, ["image/png", "application/pdf"]); + assert_eq!(out, [BLOB_MAGIC, &first, &second].concat()); + assert!(tail.write_to_with(Vec::new(), |_, _| Err(io::Error::other("failure"))).is_err()); + let mut called = false; + Tail::new(4) + .write_to_with(Vec::new(), |_, _| { + called = true; + Ok(()) + }) + .unwrap(); + assert!(!called); +} From d1ee17aca44f8425bc01d87969f54343b1e7d8ea Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 14:46:50 +0000 Subject: [PATCH 78/98] Own native JSON schema in streaming writer --- crates/vectorcraft-format/src/stream.rs | 54 ++++++++++++++++++++++- crates/vectorcraft-format/tests/stream.rs | 25 +++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) diff --git a/crates/vectorcraft-format/src/stream.rs b/crates/vectorcraft-format/src/stream.rs index 8f205a15d..71b6e90eb 100644 --- a/crates/vectorcraft-format/src/stream.rs +++ b/crates/vectorcraft-format/src/stream.rs @@ -4,9 +4,61 @@ //! Extracted from VectorCraft, licensed under MIT OR Apache-2.0. use serde::Serialize; -use std::collections::HashSet; +use std::collections::{BTreeMap, HashSet}; use std::io::{self, Write}; +/// Native file metadata for streaming a JSON envelope. Asset entries should be prepared using +/// the same [`crate::Tail`] whose parts will be written after the JSON. +pub struct FileHeader<'a> { + /// Native format version to write. + pub version: u32, + /// Application generator string. + pub generator: &'a str, + /// Optional saved PDF compatibility preference. + pub pdf_compatible: Option, + /// Optional inline preview. + pub preview: Option<&'a crate::Image>, + /// Embedded profiles keyed by profile name. + pub profiles: &'a BTreeMap<&'a str, crate::Image>, + /// Optional PDF entry. + pub pdf: Option<&'a crate::Image>, +} + +/// Write the native JSON envelope, streaming the document through `document`. The callback +/// returns the image entries after serializing the document, allowing it to collect image keys +/// without keeping a copy of the document JSON. It must write one complete JSON value. +pub fn write_file_json<'a>( + mut out: impl Write, + header: &FileHeader<'a>, + pretty: bool, + document: impl FnOnce(&mut dyn Write) -> io::Result>, +) -> io::Result<()> { + if !(crate::MIN_VERSION..=crate::VERSION).contains(&header.version) { + return Err(io::Error::new(io::ErrorKind::InvalidInput, "unsupported native file version")); + } + let mut env = Envelope::new(pretty); + let format = if header.version < 3 { crate::LEGACY_EXTENSION } else { crate::EXTENSION }; + env.field(&mut out, "format", &format)?; + env.field(&mut out, "version", &header.version)?; + env.field(&mut out, "generator", &header.generator)?; + if let Some(compatible) = header.pdf_compatible { + env.field(&mut out, "pdfCompatible", &compatible)?; + } + if let Some(preview) = header.preview { + env.field(&mut out, "preview", preview)?; + } + if !header.profiles.is_empty() { + env.field(&mut out, "profiles", header.profiles)?; + } + if let Some(pdf) = header.pdf { + env.field(&mut out, "pdf", pdf)?; + } + env.key(&mut out, "document")?; + let images = document(&mut Indent { inner: &mut out, on: pretty })?; + env.field(&mut out, "images", &images)?; + env.finish(&mut out) +} + /// Indents JSON text written through it one level (two spaces after each newline), as a member /// of the file's top-level object; newlines only occur between tokens, inside strings they are /// escaped. diff --git a/crates/vectorcraft-format/tests/stream.rs b/crates/vectorcraft-format/tests/stream.rs index 728f533ca..35347ca2e 100644 --- a/crates/vectorcraft-format/tests/stream.rs +++ b/crates/vectorcraft-format/tests/stream.rs @@ -81,3 +81,28 @@ fn tail_calls_hook_in_order_and_propagates_errors() { .unwrap(); assert!(!called); } + +#[test] +fn native_schema_order_and_versions() { + use craft_vectorcraft_format::stream::{FileHeader, write_file_json}; + let profiles = Default::default(); + for (version, name) in [(1, "drawcraft"), (4, "vectorcraft")] { + for pretty in [false, true] { + let header = FileHeader { version, generator: "test", pdf_compatible: Some(false), preview: None, profiles: &profiles, pdf: None }; + let mut out = Vec::new(); + write_file_json(&mut out, &header, pretty, |w| { + w.write_all(if pretty { b"{\n \"key\": \"image\"\n}" } else { b"{\"key\":\"image\"}" })?; + Ok(Default::default()) + }) + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&out).unwrap(); + assert_eq!(value["format"], name); + assert_eq!(value["pdfCompatible"], false); + let text = std::str::from_utf8(&out).unwrap(); + assert!(text.find("\"format\"").unwrap() < text.find("\"version\"").unwrap()); + assert!(text.find("\"document\"").unwrap() < text.find("\"images\"").unwrap()); + } + } + let header = FileHeader { version: 0, generator: "test", pdf_compatible: None, preview: None, profiles: &profiles, pdf: None }; + assert!(write_file_json(Vec::new(), &header, false, |_| Ok(Default::default())).is_err()); +} From b5bd10d0d89ab9bc8ff9f36131128d9cf3a01014 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 14:49:04 +0000 Subject: [PATCH 79/98] Stream envelope fields with bounded buffering --- crates/vectorcraft-format/src/stream.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/crates/vectorcraft-format/src/stream.rs b/crates/vectorcraft-format/src/stream.rs index 71b6e90eb..16bc77ee8 100644 --- a/crates/vectorcraft-format/src/stream.rs +++ b/crates/vectorcraft-format/src/stream.rs @@ -198,11 +198,15 @@ impl Envelope { Self { pretty, first: true } } - /// Serialize a complete member before writing it to the envelope. + /// Serialize a member directly to the output without buffering the full value. pub fn field(&mut self, out: &mut impl Write, key: &str, value: &impl Serialize) -> io::Result<()> { - let json = if self.pretty { serde_json::to_vec_pretty(value) } else { serde_json::to_vec(value) }.map_err(io::Error::other)?; self.key(out, key)?; - Indent { inner: out, on: self.pretty }.write_all(&json) + // serde_json emits small writes; keep compression and underlying I/O efficient while + // bounding temporary memory independently of the serialized table size. + let mut buffered = io::BufWriter::with_capacity(64 << 10, out); + let indented = Indent { inner: &mut buffered, on: self.pretty }; + if self.pretty { serde_json::to_writer_pretty(indented, value) } else { serde_json::to_writer(indented, value) }.map_err(io::Error::other)?; + buffered.into_inner().map(|_| ()).map_err(|e| e.into_error()) } /// The start of member `key` (a plain name): its value follows, indented one level when From cec6e24381bb8e15e0a5dcff67e6a33494d3ddd5 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sun, 15 Nov 2026 15:52:47 +0100 Subject: [PATCH 80/98] Check the rewritten SVG viewport output under the same limits --- crates/svg/src/tree.rs | 6 +++++- crates/svg/src/tree/tests.rs | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 37d956a48..9cf85c3bd 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -212,7 +212,8 @@ pub struct Viewport { } /// Wraps a validated SVG root in a new viewport while preserving the original root as content. -/// The original root must have finite, positive pixel or point dimensions. +/// The original root must have finite, positive pixel or point dimensions, and the wrapped +/// output must still pass [`preflight`] under `limits`. pub fn rewrite_root_viewport(svg: &str, target: Viewport, limits: &Limits, output_limit: usize) -> Result { if !target.width.is_finite() || !target.height.is_finite() || target.width <= 0.0 || target.height <= 0.0 { return Err(Error::Viewport("target dimensions must be finite and positive".into())); @@ -256,6 +257,9 @@ pub fn rewrite_root_viewport(svg: &str, target: Viewport, limits: &Limits, outpu output.push_str(&prefix); output.push_str(inner); output.push_str(""); + // The wrapper adds a level, so a source at the depth limit can come out over it: check the + // output under the limits `parse_text` applies rather than return text it would refuse. + preflight(&output, limits)?; Ok(output) } diff --git a/crates/svg/src/tree/tests.rs b/crates/svg/src/tree/tests.rs index 99b0c1cab..35cb13a2e 100644 --- a/crates/svg/src/tree/tests.rs +++ b/crates/svg/src/tree/tests.rs @@ -666,3 +666,22 @@ fn rewrite_root_viewport_refusals() { assert_eq!(rewrite_root_viewport(sized, one, &limits, fits.len()).unwrap(), fits); assert!(matches!(rewrite_root_viewport(sized, one, &limits, fits.len() - 1), Err(Error::OutputTooLarge(n)) if n == fits.len() - 1)); } + +#[test] +fn rewrite_root_viewport_output_stays_within_the_limits() { + let limits = shallow(); + let one = Viewport { width: 1.0, height: 1.0 }; + let mut refused_at_the_limit = false; + for n in 0..limits.depth { + let source = svg(&groups(n)); + if preflight(&source, &limits).is_err() { + break; + } + match rewrite_root_viewport(&source, one, &limits, 1 << 20) { + Ok(out) => assert!(parse_text(&out, &limits, &options()).is_ok(), "{n} groups"), + Err(Error::TooDeep(_)) => refused_at_the_limit = true, + Err(e) => panic!("{n} groups: {e:?}"), + } + } + assert!(refused_at_the_limit, "a source at the depth limit is refused, not wrapped past it"); +} From 38c26217b1f160cc27ac7453d9ad8fb6739c9830 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:01:28 +0000 Subject: [PATCH 81/98] Support fixed-precision SVG scene coordinates --- crates/svg/src/write.rs | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/crates/svg/src/write.rs b/crates/svg/src/write.rs index 469a4f8fa..66cb1d6a0 100644 --- a/crates/svg/src/write.rs +++ b/crates/svg/src/write.rs @@ -86,18 +86,36 @@ impl Attribute { pub fn translate_scale(x: f64, y: f64, scale_x: f64, scale_y: f64) -> Self { Self { name: "transform".into(), value: Value::TranslateScale { x, y, scale_x, scale_y } } } + /// A translate/scale transform with three decimal places for positions and four for scales. + pub fn translate_scale_fixed(x: f64, y: f64, scale_x: f64, scale_y: f64) -> Self { + Self::text("transform", format!("translate({} {}) scale({} {})", num(x, 3), num(y, 3), num(scale_x, 4), num(scale_y, 4))) + } } impl Scene { /// Serialize the scene as an SVG 1.1 document. pub fn write(&self) -> String { + self.write_impl(false) + } + + /// Serialize the root viewBox with three decimal places for stable fixed-width output. + pub fn write_fixed_view_box(&self) -> String { + self.write_impl(true) + } + + fn write_impl(&self, fixed: bool) -> String { let mut writer = SvgWriter::new(); + let view_box = if fixed { + format!("{} {} {} {}", num(self.view_box[0], 3), num(self.view_box[1], 3), num(self.view_box[2], 3), num(self.view_box[3], 3)) + } else { + view_box_data(self.view_box) + }; writer .root() .attr("version", "1.1") .attr("width", format!("{}mm", num(self.width_mm, 3))) .attr("height", format!("{}mm", num(self.height_mm, 3))) - .attr("viewBox", view_box_data(self.view_box)) + .attr("viewBox", view_box) .open(); write_elements(&mut writer, &self.elements, 1); writer.line(0); @@ -307,6 +325,14 @@ mod tests { p } + #[test] + fn scene_can_keep_fixed_view_box_without_changing_default_output() { + let scene = Scene { width_mm: 100.0, height_mm: 150.0, view_box: [0.0, 0.0, 100.0, 150.0], elements: vec![] }; + assert!(scene.write().contains("viewBox=\"0 0 100 150\"")); + assert!(scene.write_fixed_view_box().contains("viewBox=\"0.000 0.000 100.000 150.000\"")); + assert_eq!(Attribute::translate_scale_fixed(6.0, 46.0, 0.5, -0.5).value, Value::Text("translate(6.000 46.000) scale(0.5000 -0.5000)".into())); + } + #[test] fn numbers_have_fixed_decimals_and_are_never_non_finite() { assert_eq!(num(1.5, 3).to_string(), "1.500"); From 560b6f8268e2c5285c161be52dacfe877b08cf1b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:02:41 +0000 Subject: [PATCH 82/98] Compare streaming native output with previous framing --- crates/vectorcraft-format/src/stream.rs | 4 ++ crates/vectorcraft-format/tests/stream.rs | 74 +++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/crates/vectorcraft-format/src/stream.rs b/crates/vectorcraft-format/src/stream.rs index 16bc77ee8..2e8b0695e 100644 --- a/crates/vectorcraft-format/src/stream.rs +++ b/crates/vectorcraft-format/src/stream.rs @@ -1,5 +1,9 @@ //! Streaming helpers for the native JSON envelope. //! +//! Field values are emitted in bounded chunks. The decoded JSON and tail framing stay the same +//! as VectorCraft's previous writer, but compressed bytes can differ when a deflate backend's +//! output depends on input chunking. Gzip readers still validate the CRC and size trailer. +//! //! Copyright (c) 2026 ArtCraft Team and the VectorCraft contributors. //! Extracted from VectorCraft, licensed under MIT OR Apache-2.0. diff --git a/crates/vectorcraft-format/tests/stream.rs b/crates/vectorcraft-format/tests/stream.rs index 35347ca2e..98f4685e2 100644 --- a/crates/vectorcraft-format/tests/stream.rs +++ b/crates/vectorcraft-format/tests/stream.rs @@ -106,3 +106,77 @@ fn native_schema_order_and_versions() { let header = FileHeader { version: 0, generator: "test", pdf_compatible: None, preview: None, profiles: &profiles, pdf: None }; assert!(write_file_json(Vec::new(), &header, false, |_| Ok(Default::default())).is_err()); } + +// Oracle for the previous app writer: each member was first serialized to a Vec, then +// indented and appended. Compare decoded file bytes, not backend-dependent gzip bytes. +#[test] +fn streamed_native_file_matches_previous_framing() { + use craft_vectorcraft_format::{ + Tail, + stream::{FileHeader, write_file_json}, + }; + use flate2::{Compression, read::GzDecoder, write::GzEncoder}; + use std::collections::BTreeMap; + use std::io::Read; + + fn old_field(out: &mut Vec, first: &mut bool, pretty: bool, key: &str, value: &impl serde::Serialize) { + out.extend_from_slice(if std::mem::take(first) { b"{" } else { b"," }); + if pretty { + out.extend_from_slice(b"\n "); + } + out.extend_from_slice(format!("\"{key}\"{}", if pretty { ": " } else { ":" }).as_bytes()); + let json = if pretty { serde_json::to_vec_pretty(value) } else { serde_json::to_vec(value) }.unwrap(); + let mut indent = Indent { inner: out, on: pretty }; + indent.write_all(&json).unwrap(); + } + + let payload = vec![42; craft_vectorcraft_format::INLINE_MAX]; + for version in [1, 3, 4] { + for pretty in [false, true] { + let mut tail = Tail::new(version); + let image = tail.image("image/png", &payload, false); + let profile = tail.image("application/vnd.iccprofile", &payload, false); + let pdf = tail.image("application/pdf", &payload, false); + let profiles = BTreeMap::from([("profile", profile)]); + let images = BTreeMap::from([("image", image)]); + let document = serde_json::json!({"layers": [{"key": "image", "name": "A"}]}); + let header = FileHeader { + version, + generator: "VectorCraft test", + pdf_compatible: Some(true), + preview: None, + profiles: &profiles, + pdf: Some(&pdf), + }; + let mut actual = Vec::new(); + write_file_json(&mut actual, &header, pretty, |w| { + if pretty { serde_json::to_writer_pretty(w, &document) } else { serde_json::to_writer(w, &document) }.map_err(io::Error::other)?; + Ok(images.clone()) + }) + .unwrap(); + tail.write_to(&mut actual).unwrap(); + + let mut old = Vec::new(); + let mut first = true; + let format = if version < 3 { "drawcraft" } else { "vectorcraft" }; + old_field(&mut old, &mut first, pretty, "format", &format); + old_field(&mut old, &mut first, pretty, "version", &version); + old_field(&mut old, &mut first, pretty, "generator", &"VectorCraft test"); + old_field(&mut old, &mut first, pretty, "pdfCompatible", &true); + old_field(&mut old, &mut first, pretty, "profiles", &profiles); + old_field(&mut old, &mut first, pretty, "pdf", &pdf); + old_field(&mut old, &mut first, pretty, "document", &document); + old_field(&mut old, &mut first, pretty, "images", &images); + old.extend_from_slice(if pretty { b"\n}" } else { b"}" }); + tail.write_to(&mut old).unwrap(); + assert_eq!(actual, old, "version {version}, pretty {pretty}"); + for bytes in [&old, &actual] { + let mut gzip = GzEncoder::new(Vec::new(), Compression::default()); + gzip.write_all(bytes).unwrap(); + let mut decoded = Vec::new(); + GzDecoder::new(gzip.finish().unwrap().as_slice()).read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded, old); + } + } + } +} From 2bb5d0344f03a73833576b57847513ea529d3d2c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 10:10:41 -0500 Subject: [PATCH 83/98] docs(psd): document editable TySh envelope fields --- crates/psd/src/text/envelope.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/psd/src/text/envelope.rs b/crates/psd/src/text/envelope.rs index e654ed390..73345fa83 100644 --- a/crates/psd/src/text/envelope.rs +++ b/crates/psd/src/text/envelope.rs @@ -12,8 +12,11 @@ pub const MAX_BYTES: usize = 8 * 1024 * 1024; /// Parsed `TySh` contents. #[derive(Clone, Debug, PartialEq)] pub struct TySh { + /// Affine transform in PSD order: xx, xy, yx, yy, tx, ty. pub transform: [f64; 6], + /// Text descriptor, including text content and styling data. pub text: Descriptor, + /// Optional warp descriptor retained from the Type Tool block. pub warp: Option, /// The four trailing big-endian f32 values (left, top, right, bottom) after the warp /// descriptor. Photoshop writes zeros for unwarped type and the text-space bounds (the From 6c22881fc13314cd393846c3b3f23f59ab744a63 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:16:39 +0000 Subject: [PATCH 84/98] Share streaming neutral SVG geometry and output --- Cargo.lock | 1 + crates/svg/Cargo.toml | 1 + crates/svg/src/drawing_write.rs | 56 ++++ crates/svg/src/lib.rs | 3 + crates/svg/src/neutral_document.rs | 406 +++++++++++++++++++++++++ crates/svg/src/path_data.rs | 456 +++++++++++++++++++++++++++++ 6 files changed, 923 insertions(+) create mode 100644 crates/svg/src/drawing_write.rs create mode 100644 crates/svg/src/neutral_document.rs create mode 100644 crates/svg/src/path_data.rs diff --git a/Cargo.lock b/Cargo.lock index c396f1978..6c4aab5dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2696,6 +2696,7 @@ dependencies = [ "data-url", "flate2", "kurbo 0.13.1", + "quick-xml 0.41.0", "resvg 0.48.1", "roxmltree 0.21.2", "thiserror 2.0.21", diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml index 3615a719c..3b778da47 100644 --- a/crates/svg/Cargo.toml +++ b/crates/svg/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] craft-xml = { workspace = true } +quick-xml = { version = "0.41", default-features = false } craft-image-codecs = { workspace = true } # SVGZ (gzip), pure-Rust backend. flate2 = { version = "1", default-features = false, features = ["rust_backend"] } diff --git a/crates/svg/src/drawing_write.rs b/crates/svg/src/drawing_write.rs new file mode 100644 index 000000000..91b94d790 --- /dev/null +++ b/crates/svg/src/drawing_write.rs @@ -0,0 +1,56 @@ +//! SVG polyline and triangle path attributes for display-list exporters. +use crate::write::num; +use std::fmt::{self, Display}; + +/// A fixed-precision polyline attribute streamed into an SVG writer. +pub struct Points { + iter: I, + decimals: usize, +} +/// Format points without allocating an intermediate attribute string. Precision is capped at 32. +pub fn points + Clone>(iter: I, decimals: usize) -> Points { + Points { iter, decimals: decimals.min(32) } +} +impl + Clone> Display for Points { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for [x, y] in self.iter.clone() { + write!(f, "{},{} ", num(x, self.decimals), num(y, self.decimals))?; + } + Ok(()) + } +} +/// A fixed-precision sequence of closed triangle subpaths. +pub struct Triangles { + iter: I, + decimals: usize, +} +/// Format independent triangles without allocating an intermediate attribute string. Precision is capped at 32. +pub fn triangles + Clone>(iter: I, decimals: usize) -> Triangles { + Triangles { iter, decimals: decimals.min(32) } +} +impl + Clone> Display for Triangles { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for [[ax, ay], [bx, by], [cx, cy]] in self.iter.clone() { + write!( + f, + "M{} {}L{} {}L{} {}Z", + num(ax, self.decimals), + num(ay, self.decimals), + num(bx, self.decimals), + num(by, self.decimals), + num(cx, self.decimals), + num(cy, self.decimals) + )?; + } + Ok(()) + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn fixed_precision_attributes_keep_independent_triangles() { + assert_eq!(points([[1.0, 2.0], [3.0, 4.0]].into_iter(), 2).to_string(), "1.00,2.00 3.00,4.00 "); + assert_eq!(triangles([[[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]]].into_iter(), 1).to_string(), "M1.0 2.0L3.0 4.0L5.0 6.0Z"); + } +} diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index 982e4f82b..25bf880cd 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -18,7 +18,10 @@ #![forbid(unsafe_code)] pub mod color; +pub mod drawing_write; pub mod guard; +pub mod neutral_document; +pub mod path_data; pub mod paths; pub mod preflight; pub mod sniff; diff --git a/crates/svg/src/neutral_document.rs b/crates/svg/src/neutral_document.rs new file mode 100644 index 000000000..24294684a --- /dev/null +++ b/crates/svg/src/neutral_document.rs @@ -0,0 +1,406 @@ +//! A bounded SVG document walk that preserves geometric primitive kinds for CAD import. +//! +//! Coordinates remain in SVG user units. Each shape carries the composed transform into +//! millimetres with the page Y axis pointing up; consumers map shapes into their own model. +use crate::path_data::{self, Point, Subpath}; +use quick_xml::events::{BytesStart, Event as XmlEvent}; +use quick_xml::{Reader, XmlVersion}; +use std::collections::BTreeMap; + +const MAX_DEPTH: usize = 64; +const MAX_SHAPES: usize = 500_000; +const MAX_POINTS: usize = 8_000_000; +const MAX_POLY_POINTS: usize = 100_001; +const SKIP: &[&[u8]] = &[ + b"defs", + b"symbol", + b"clipPath", + b"mask", + b"pattern", + b"marker", + b"linearGradient", + b"radialGradient", + b"filter", + b"style", + b"text", + b"image", + b"title", + b"desc", + b"metadata", +]; +const CONTAINERS: &[&[u8]] = &[b"g", b"svg", b"a", b"switch"]; + +/// Input failure or resource bound. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +pub struct Error(pub String); +/// A shape's original geometry before its affine transform. +#[derive(Debug, Clone, PartialEq)] +pub enum Primitive { + Rect { x: f64, y: f64, width: f64, height: f64 }, + Circle { center: Point, radius: f64 }, + Ellipse { center: Point, rx: f64, ry: f64 }, + Line { a: Point, b: Point }, + Polyline { points: Vec, closed: bool }, + Path(Vec), +} +/// A primitive with its layer and SVG affine matrix `[a,b,c,d,e,f]`. +#[derive(Debug, Clone, PartialEq)] +pub struct Shape { + pub primitive: Primitive, + pub transform: [f64; 6], + pub layer: String, +} +/// The shapes, encountered layer names, and skipped-feature counts. `visit` streams shapes +/// to its sink, so its returned `shapes` is empty; `import` collects them. +#[derive(Debug, Default, PartialEq)] +pub struct Document { + pub shapes: Vec, + pub layers: Vec, + pub skipped: BTreeMap, +} +/// Features left out by this geometry-only import. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Skipped { + Text, + Image, + Use, + Style, + RoundedCorners, +} + +fn attr(e: &BytesStart<'_>, name: &[u8]) -> Option { + for a in e.attributes() { + let Ok(a) = a else { return None }; + if a.key.into_inner() == name { + return a.normalized_value(XmlVersion::Explicit1_0).ok().map(|v| v.into_owned()); + } + } + None +} +fn hidden(e: &BytesStart<'_>) -> bool { + attr(e, b"display").as_deref() == Some("none") || attr(e, b"style").is_some_and(|s| s.replace(' ', "").contains("display:none")) +} +fn length(s: &str) -> Option { + let s = s.trim(); + let cut = s.find(|c: char| !(c.is_ascii_digit() || matches!(c, '.' | '-' | '+' | 'e' | 'E'))).unwrap_or(s.len()); + let (n, unit) = s.split_at(cut); + let v: f64 = n.trim().parse().ok()?; + if !v.is_finite() { + return None; + } + Some( + v * match unit.trim() { + "" | "px" | "mm" => 1.0, + "cm" => 10.0, + "in" => 25.4, + "pt" => 25.4 / 72.0, + "pc" => 25.4 / 6.0, + _ => 25.4 / 96.0, + }, + ) +} +fn num(e: &BytesStart<'_>, name: &[u8], default: f64) -> f64 { + attr(e, name).as_deref().and_then(length).unwrap_or(default) +} +const ID: [f64; 6] = [1., 0., 0., 1., 0., 0.]; +fn mul(a: [f64; 6], b: [f64; 6]) -> [f64; 6] { + [ + a[0] * b[0] + a[2] * b[1], + a[1] * b[0] + a[3] * b[1], + a[0] * b[2] + a[2] * b[3], + a[1] * b[2] + a[3] * b[3], + a[0] * b[4] + a[2] * b[5] + a[4], + a[1] * b[4] + a[3] * b[5] + a[5], + ] +} +fn translate(x: f64, y: f64) -> [f64; 6] { + [1., 0., 0., 1., x, y] +} +fn scale(x: f64, y: f64) -> [f64; 6] { + [x, 0., 0., y, 0., 0.] +} +fn rotate(a: f64) -> [f64; 6] { + let (s, c) = a.to_radians().sin_cos(); + [c, s, -s, c, 0., 0.] +} +fn numbers(s: &str) -> impl Iterator + '_ { + path_data::Numbers::new(s) +} +fn transform(s: &str) -> Option<[f64; 6]> { + let mut out = None; + let mut rest = s.trim(); + while !rest.is_empty() { + let open = rest.find('(')?; + let name = rest[..open].trim(); + let close = rest[open..].find(')')? + open; + // No transform function accepts more than six arguments. + let args: Vec<_> = numbers(&rest[open + 1..close]).take(7).collect(); + let m = match (name, args.as_slice()) { + ("matrix", [a, b, c, d, e, f]) => [*a, *b, *c, *d, *e, *f], + ("translate", [x]) => translate(*x, 0.), + ("translate", [x, y]) => translate(*x, *y), + ("scale", [x]) => scale(*x, *x), + ("scale", [x, y]) => scale(*x, *y), + ("rotate", [a]) => rotate(*a), + ("rotate", [a, x, y]) => mul(mul(translate(*x, *y), rotate(*a)), translate(-x, -y)), + ("skewX", [a]) => [1., 0., a.to_radians().tan(), 1., 0., 0.], + ("skewY", [a]) => [1., a.to_radians().tan(), 0., 1., 0., 0.], + _ => return None, + }; + out = Some(out.map_or(m, |prev| mul(prev, m))); + rest = rest[close + 1..].trim_start_matches(|c: char| c == ',' || c.is_whitespace()); + } + out +} +fn viewport(e: &BytesStart<'_>) -> [f64; 6] { + let vb = attr(e, b"viewBox").and_then(|raw| { + let n: Vec<_> = numbers(&raw).take(5).collect(); + match n.as_slice() { + [x, y, w, h] if *w > 0. && *h > 0. => Some((*x, *y, *w, *h)), + _ => None, + } + }); + let root = |name: &[u8]| { + let raw = attr(e, name)?; + let mm = length(&raw)?; + let unit = raw.trim().trim_start_matches(|c: char| c.is_ascii_digit() || matches!(c, '.' | '-' | '+' | 'e' | 'E')).trim(); + Some(if matches!(unit, "" | "px") { mm * 25.4 / 96. } else { mm }) + }; + let unit = match (vb, root(b"width"), root(b"height")) { + (Some((_, _, w, _)), Some(x), _) if x > 0. => x / w, + (Some((_, _, _, h)), _, Some(y)) if y > 0. => y / h, + _ => 25.4 / 96., + }; + let (x, y, h) = vb.map_or((0., 0., 0.), |(x, y, _, h)| (x, y, h)); + [unit, 0., 0., -unit, -x * unit, (y + h) * unit] +} +fn layer_name(raw: &str) -> String { + let s: String = raw.trim().chars().map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '$') { c } else { '_' }).take(128).collect(); + let s = s.trim_matches('_'); + if s.is_empty() { "SVG".into() } else { s.into() } +} +/// A layer declaration or a drawable primitive in document order. +pub enum Event { + Layer(String), + Shape(Shape), +} + +struct Walker<'a> { + doc: Document, + stack: Vec<[f64; 6]>, + layer: String, + skip: usize, + points: usize, + shapes: usize, + sink: &'a mut dyn FnMut(Event) -> Result<(), Error>, +} +impl Walker<'_> { + fn current(&self) -> [f64; 6] { + self.stack.last().copied().unwrap_or(ID) + } + fn note(&mut self, what: Skipped) { + *self.doc.skipped.entry(what).or_default() += 1; + } + fn skipped(name: &[u8]) -> Option { + match name { + b"text" => Some(Skipped::Text), + b"image" => Some(Skipped::Image), + _ => None, + } + } + fn start(&mut self, name: &[u8], e: &BytesStart<'_>) -> Result<(), Error> { + if name == b"style" { + self.note(Skipped::Style); + } + if self.skip > 0 { + self.skip += 1; + return Ok(()); + } + if hidden(e) { + self.skip = 1; + return Ok(()); + } + if SKIP.contains(&name) { + if let Some(s) = Self::skipped(name) { + self.note(s); + } + self.skip = 1; + return Ok(()); + } + if !CONTAINERS.contains(&name) { + return self.element(name, e); + } + if name == b"svg" && self.stack.len() == 1 { + if let Some(root) = self.stack.first_mut() { + *root = viewport(e); + } + } + if name == b"g" + && attr(e, b"inkscape:groupmode").as_deref() == Some("layer") + && let Some(raw) = attr(e, b"inkscape:label").or_else(|| attr(e, b"id")) + { + let layer = layer_name(&raw); + if layer == "0" || self.doc.layers.contains(&layer) { + self.layer = layer; + } else if self.doc.layers.len() < 255 { + self.doc.layers.push(layer.clone()); + (self.sink)(Event::Layer(layer.clone()))?; + self.layer = layer; + } + } + if self.stack.len() >= MAX_DEPTH { + return Err(Error(format!("SVG: more than {MAX_DEPTH} nested groups"))); + } + let next = attr(e, b"transform").as_deref().and_then(transform).map_or(self.current(), |t| mul(self.current(), t)); + self.stack.push(next); + Ok(()) + } + fn element(&mut self, name: &[u8], e: &BytesStart<'_>) -> Result<(), Error> { + if self.skip > 0 { + return Ok(()); + } + if SKIP.contains(&name) || hidden(e) { + if !hidden(e) + && let Some(s) = Self::skipped(name) + { + self.note(s); + } + return Ok(()); + } + let p = |x, y| Point::new(num(e, x, 0.), num(e, y, 0.)); + let primitive = match name { + b"use" => { + self.note(Skipped::Use); + return Ok(()); + } + b"rect" => { + let (x, y, w, h) = (num(e, b"x", 0.), num(e, b"y", 0.), num(e, b"width", 0.), num(e, b"height", 0.)); + if w <= 0. || h <= 0. { + return Ok(()); + } + if num(e, b"rx", 0.) > 0. || num(e, b"ry", 0.) > 0. { + self.note(Skipped::RoundedCorners); + } + Primitive::Rect { x, y, width: w, height: h } + } + b"circle" => { + let r = num(e, b"r", 0.); + if r <= 0. { + return Ok(()); + } + Primitive::Circle { center: p(b"cx", b"cy"), radius: r } + } + b"ellipse" => { + let (rx, ry) = (num(e, b"rx", 0.), num(e, b"ry", 0.)); + if rx <= 0. || ry <= 0. { + return Ok(()); + } + Primitive::Ellipse { center: p(b"cx", b"cy"), rx, ry } + } + b"line" => Primitive::Line { a: p(b"x1", b"y1"), b: p(b"x2", b"y2") }, + b"polyline" | b"polygon" => { + let Some(raw) = attr(e, b"points") else { return Ok(()) }; + let n: Vec<_> = numbers(&raw).take(MAX_POLY_POINTS * 2).collect(); + let pts: Vec<_> = n.as_chunks::<2>().0.iter().map(|[x, y]| Point::new(*x, *y)).collect(); + if pts.len() < 2 { + return Ok(()); + } + Primitive::Polyline { points: pts, closed: name == b"polygon" } + } + b"path" => { + let Some(raw) = attr(e, b"d") else { return Ok(()) }; + Primitive::Path(path_data::parse_path(&raw).map_err(|e| Error(e.to_string()))?) + } + _ => return Ok(()), + }; + let cost = match &primitive { + Primitive::Polyline { points, .. } => points.len(), + Primitive::Path(paths) => paths.iter().map(|p| p.segs.len()).sum(), + _ => 1, + }; + self.points = self.points.checked_add(cost).filter(|n| *n <= MAX_POINTS).ok_or_else(|| Error("SVG: too much generated geometry".into()))?; + if self.shapes >= MAX_SHAPES { + return Err(Error(format!("SVG: more than {MAX_SHAPES} elements"))); + } + let matrix = attr(e, b"transform").as_deref().and_then(transform).map_or(self.current(), |t| mul(self.current(), t)); + self.shapes += 1; + (self.sink)(Event::Shape(Shape { primitive, transform: matrix, layer: self.layer.clone() })) + } +} +/// Walk supported SVG geometry without retaining a second copy of the document's shapes. +/// The sink receives each layer before its first shape and each shape in input order. +pub fn visit(text: &str, sink: &mut dyn FnMut(Event) -> Result<(), Error>) -> Result { + let mut walker = Walker { doc: Document::default(), stack: vec![ID], layer: "0".into(), skip: 0, points: 0, shapes: 0, sink }; + let mut reader = Reader::from_str(text); + reader.config_mut().trim_text(true); + loop { + match reader.read_event().map_err(|e| Error(format!("SVG: {e}")))? { + XmlEvent::Start(e) => walker.start(e.name().into_inner(), &e)?, + XmlEvent::Empty(e) => walker.element(e.name().into_inner(), &e)?, + XmlEvent::End(e) => { + if walker.skip > 0 { + walker.skip -= 1; + } else if CONTAINERS.contains(&e.name().into_inner()) && walker.stack.len() > 1 { + walker.stack.pop(); + } + } + XmlEvent::Eof => break, + _ => {} + } + } + Ok(walker.doc) +} +/// Read the supported geometry into an owned document. +pub fn import(text: &str) -> Result { + let mut shapes = Vec::new(); + let mut doc = visit(text, &mut |event| { + if let Event::Shape(shape) = event { + shapes.push(shape); + } + Ok(()) + })?; + doc.shapes = shapes; + Ok(doc) +} +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + #[test] + fn keeps_primitives_transform_layers_and_skips() { + let svg = r##"x"##; + let d = import(svg).unwrap(); + assert_eq!(d.layers, ["cut"]); + assert_eq!(d.shapes.len(), 3); + assert_eq!(d.shapes[0].primitive, Primitive::Circle { center: Point::new(3., 4.), radius: 2. }); + assert_eq!(d.shapes[0].transform, [1., 0., 0., -1., 2., 9.]); + assert!(matches!(d.shapes[1].primitive, Primitive::Ellipse { .. })); + assert!(matches!(&d.shapes[2].primitive,Primitive::Path(p) if matches!(p[0].segs[0],path_data::Segment::Arc { .. }))); + assert_eq!(d.skipped.get(&Skipped::Text), Some(&1)); + assert_eq!(d.skipped.get(&Skipped::Use), Some(&1)); + } + #[test] + fn streaming_delivers_layers_before_shapes_and_stops_on_error() { + let s = r#""#; + let mut events = Vec::new(); + let result = visit(s, &mut |event| { + match event { + Event::Layer(name) => events.push(name), + Event::Shape(_) => { + events.push("shape".into()); + return Err(Error("stop".into())); + } + } + Ok(()) + }); + assert!(result.is_err()); + assert_eq!(events, ["cut", "shape"]); + } + + #[test] + fn refuses_excessive_nesting() { + let s = format!("{}{}", "".repeat(100), "".repeat(100)); + assert!(import(&s).is_err()); + } +} diff --git a/crates/svg/src/path_data.rs b/crates/svg/src/path_data.rs new file mode 100644 index 000000000..96d0db771 --- /dev/null +++ b/crates/svg/src/path_data.rs @@ -0,0 +1,456 @@ +//! SVG path data as neutral line, cubic and elliptical arc segments. +//! +//! Relative commands and smooth controls are resolved, while arcs remain arcs. +//! The caller chooses how to map the result to its own geometry model. +use std::ops::{Add, Mul, Sub}; + +/// A point in the path's original user coordinate system. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct Point { + /// Horizontal coordinate. + pub x: f64, + /// Vertical coordinate. + pub y: f64, +} +impl Point { + /// The origin. + pub const ZERO: Self = Self { x: 0.0, y: 0.0 }; + /// Construct a point. + pub const fn new(x: f64, y: f64) -> Self { + Self { x, y } + } +} +impl Add for Point { + type Output = Self; + fn add(self, rhs: Self) -> Self { + Self::new(self.x + rhs.x, self.y + rhs.y) + } +} +impl Sub for Point { + type Output = Self; + fn sub(self, rhs: Self) -> Self { + Self::new(self.x - rhs.x, self.y - rhs.y) + } +} +impl Mul for Point { + type Output = Self; + fn mul(self, rhs: f64) -> Self { + Self::new(self.x * rhs, self.y * rhs) + } +} + +/// Invalid or oversized path data. +#[derive(Debug, thiserror::Error)] +#[error("{0}")] +pub struct Error(pub String); +type Result = std::result::Result; +const MAX_PATH_SEGMENTS: usize = 100_000; +const MAX_CURVE_POINTS: usize = 100_001; + +pub(crate) struct Numbers<'a> { + s: &'a [u8], + i: usize, +} + +impl<'a> Numbers<'a> { + pub(crate) fn new(s: &'a str) -> Self { + Self { s: s.as_bytes(), i: 0 } + } + fn skip_seps(&mut self) { + while matches!(self.s.get(self.i), Some(b' ' | b'\t' | b'\r' | b'\n' | b',')) { + self.i += 1; + } + } + + fn next_num(&mut self) -> Option { + self.skip_seps(); + let start = self.i; + if matches!(self.s.get(self.i), Some(b'+' | b'-')) { + self.i += 1; + } + let mut digits = 0; + let mut dot = false; + let mut exp = false; + while let Some(c) = self.s.get(self.i) { + match c { + b'0'..=b'9' => { + digits += 1; + self.i += 1; + } + b'.' if !dot && !exp => { + dot = true; + self.i += 1; + } + b'e' | b'E' if !exp && digits > 0 => { + exp = true; + self.i += 1; + if matches!(self.s.get(self.i), Some(b'+' | b'-')) { + self.i += 1; + } + } + _ => break, + } + } + if digits == 0 { + self.i = start; + return None; + } + std::str::from_utf8(&self.s[start..self.i]).ok()?.parse::().ok().filter(|v| v.is_finite()) + } +} + +impl Iterator for Numbers<'_> { + type Item = f64; + fn next(&mut self) -> Option { + self.next_num() + } +} + +/// One `M …` run of a `d` attribute. +#[derive(Debug, Clone, PartialEq)] +pub struct Subpath { + /// Start point. + pub start: Point, + /// Segments in drawing order. + pub segs: Vec, + /// Whether a close-path command closed this subpath. + pub closed: bool, +} + +/// A path segment; arc values retain the SVG endpoint parameterization. +#[derive(Debug, Clone, PartialEq)] +pub enum Segment { + Line(Point), + Cubic(Point, Point, Point), + Arc { rx: f64, ry: f64, rot: f64, large: bool, sweep: bool, end: Point }, +} + +/// Tokens of a `d` attribute: command letters and numbers, in order. +enum Tok { + Cmd(u8), + Num(f64), +} + +struct Tokenizer<'a> { + s: &'a [u8], + i: usize, + /// Start of the most recently returned token. + tok_start: usize, +} + +impl<'a> Tokenizer<'a> { + fn new(s: &'a str) -> Self { + Tokenizer { s: s.as_bytes(), i: 0, tok_start: 0 } + } + + /// Read an arc flag: a single `0` or `1` character, which needs no separator after it. + fn next_flag(&mut self) -> Option { + let mut nums = Numbers { s: self.s, i: self.i }; + nums.skip_seps(); + let v = match self.s.get(nums.i)? { + b'0' => 0.0, + b'1' => 1.0, + _ => return None, + }; + self.i = nums.i + 1; + Some(v) + } + + fn next_tok(&mut self) -> Option { + let mut nums = Numbers { s: self.s, i: self.i }; + nums.skip_seps(); + self.i = nums.i; + self.tok_start = self.i; + let c = self.s.get(self.i).copied()?; + if c.is_ascii_alphabetic() { + self.i += 1; + return Some(Tok::Cmd(c)); + } + let tok = nums.next_num().map(Tok::Num); + self.i = nums.i; + tok + } +} + +/// Split `d` into subpaths, resolving relative commands and the S/T control reflections. +pub fn parse_path(d: &str) -> Result> { + let mut tk = Tokenizer::new(d); + let mut lookahead: Option = None; + let mut cmd = 0u8; + let mut cur = Point::ZERO; + let mut start = Point::ZERO; + let mut segs: Vec = Vec::new(); + let mut out: Vec = Vec::new(); + let closed = false; + let mut prev_cubic: Option = None; + let mut prev_quad: Option = None; + let mut segments = 0usize; + + let next_num = |tk: &mut Tokenizer, look: &mut Option| -> Result { + match look.take() { + Some(Tok::Num(v)) => Ok(v), + Some(other @ Tok::Cmd(_)) => { + *look = Some(other); + Err(Error("SVG: path command is missing coordinates".into())) + } + None => match tk.next_tok() { + Some(Tok::Num(v)) => Ok(v), + Some(other @ Tok::Cmd(_)) => { + *look = Some(other); + Err(Error("SVG: path command is missing coordinates".into())) + } + None => Err(Error("SVG: path command is missing coordinates".into())), + }, + } + }; + + while let Some(tok) = lookahead.take().or_else(|| tk.next_tok()) { + let rel = match tok { + Tok::Cmd(c) => { + if c.eq_ignore_ascii_case(&b'z') { + if !segs.is_empty() { + out.push(Subpath { start, segs, closed: true }); + segs = Vec::new(); + } + cur = start; + cmd = 0; + continue; + } + cmd = c; + c.is_ascii_lowercase() + } + tok @ Tok::Num(_) => { + if cmd == 0 { + return Err(Error("SVG: path data does not start with a command".into())); + } + lookahead = Some(tok); + cmd.is_ascii_lowercase() + } + }; + let abs = cmd.to_ascii_uppercase(); + let count = match abs { + b'M' | b'L' | b'T' => 2, + b'H' | b'V' => 1, + b'C' => 6, + b'S' | b'Q' => 4, + b'A' => 7, + _ => return Err(Error(format!("SVG: unknown path command `{}`", cmd as char))), + }; + let mut args = [0.0f64; 7]; + for (idx, slot) in args.iter_mut().enumerate().take(count) { + if abs == b'A' && (idx == 3 || idx == 4) { + // Flags are single characters; re-scan a pending number token from its start. + if matches!(lookahead, Some(Tok::Num(_))) { + lookahead = None; + tk.i = tk.tok_start; + } + if lookahead.is_none() { + *slot = tk.next_flag().ok_or_else(|| Error("SVG: path command is missing coordinates".into()))?; + continue; + } + } + *slot = next_num(&mut tk, &mut lookahead)?; + } + let point = |x: f64, y: f64| if rel { cur + Point::new(x, y) } else { Point::new(x, y) }; + match abs { + b'M' => { + if !segs.is_empty() { + out.push(Subpath { start, segs, closed: false }); + segs = Vec::new(); + } + cur = point(args[0], args[1]); + start = cur; + cmd = if rel { b'l' } else { b'L' }; + prev_cubic = None; + prev_quad = None; + } + b'L' => { + cur = point(args[0], args[1]); + segs.push(Segment::Line(cur)); + prev_cubic = None; + prev_quad = None; + } + b'H' => { + cur = Point::new(if rel { cur.x + args[0] } else { args[0] }, cur.y); + segs.push(Segment::Line(cur)); + prev_cubic = None; + prev_quad = None; + } + b'V' => { + cur = Point::new(cur.x, if rel { cur.y + args[0] } else { args[0] }); + segs.push(Segment::Line(cur)); + prev_cubic = None; + prev_quad = None; + } + b'C' => { + let (c1, c2, p) = (point(args[0], args[1]), point(args[2], args[3]), point(args[4], args[5])); + segs.push(Segment::Cubic(c1, c2, p)); + prev_cubic = Some(c2); + prev_quad = None; + cur = p; + } + b'S' => { + let c1 = prev_cubic.map_or(cur, |c2| cur * 2.0 - c2); + let (c2, p) = (point(args[0], args[1]), point(args[2], args[3])); + segs.push(Segment::Cubic(c1, c2, p)); + prev_cubic = Some(c2); + prev_quad = None; + cur = p; + } + b'Q' => { + let (q, p) = (point(args[0], args[1]), point(args[2], args[3])); + segs.push(Segment::Cubic(cur + (q - cur) * (2.0 / 3.0), p + (q - p) * (2.0 / 3.0), p)); + prev_quad = Some(q); + prev_cubic = None; + cur = p; + } + b'T' => { + let reflect = prev_quad.map_or(cur, |q| cur * 2.0 - q); + let p = point(args[0], args[1]); + segs.push(Segment::Cubic(cur + (reflect - cur) * (2.0 / 3.0), p + (reflect - p) * (2.0 / 3.0), p)); + prev_quad = Some(reflect); + prev_cubic = None; + cur = p; + } + b'A' => { + let end = point(args[5], args[6]); + segs.push(Segment::Arc { rx: args[0].abs(), ry: args[1].abs(), rot: args[2], large: args[3] != 0.0, sweep: args[4] != 0.0, end }); + cur = end; + prev_cubic = None; + prev_quad = None; + } + _ => return Err(Error("SVG: unsupported path command".into())), + } + segments += 1; + if segments > MAX_PATH_SEGMENTS { + return Err(Error("SVG: path has too many segments".into())); + } + if segs.len() > MAX_CURVE_POINTS { + return Err(Error("SVG: path has too many segments".into())); + } + } + if !segs.is_empty() { + out.push(Subpath { start, segs, closed }); + } + Ok(out) +} +/// The centre and signed sweep of a circular arc from `p0` to `p1` in a Y-up frame, per the +/// SVG endpoint parameterisation (F.6.5). Radii that cannot span the chord are scaled up, +/// as the specification requires. +pub fn arc_center_circular(p0: Point, p1: Point, r: f64, large: bool, sweep: bool) -> Option<(Point, f64)> { + if r <= 0.0 || !r.is_finite() { + return None; + } + let d = p1 - p0; + let dist = d.x.hypot(d.y); + let r = if dist > 2.0 * r { dist / 2.0 } else { r }; + if dist <= f64::EPSILON { + return None; + } + let mid = (p0 + p1) * 0.5; + let h = (r * r - (dist * dist) / 4.0).max(0.0).sqrt(); + let n = Point::new(-d.y / dist, d.x / dist); + let sign = if large != sweep { 1.0 } else { -1.0 }; + let center = mid + n * (h * sign); + let a0 = (p0.y - center.y).atan2(p0.x - center.x); + let a1 = (p1.y - center.y).atan2(p1.x - center.x); + let ccw = (a1 - a0).rem_euclid(std::f64::consts::TAU); + Some((center, pick_sweep(ccw, large, sweep))) +} + +/// The signed sweep (CCW positive) for a chord sweep `ccw` in `[0, 2π)`: the centre is +/// already chosen so that the magnitude matches `large`, so only the direction is left. +fn pick_sweep(ccw: f64, large: bool, sweep: bool) -> f64 { + let tau = std::f64::consts::TAU; + let eps = f64::EPSILON; + if sweep { + if ccw <= eps { if large { tau } else { eps } } else { ccw } + } else if ccw >= tau - eps { + if large { -tau } else { -eps } + } else { + ccw - tau + } +} + +/// Endpoint → centre parameterisation for a rotated elliptical arc (SVG F.6.5), in the +/// frame the points are given in. Returns the centre, the start parameter and the signed +/// sweep. +pub fn elliptical_params(p0: Point, p1: Point, rx: f64, ry: f64, phi: f64, large: bool, sweep: bool) -> Option<(Point, f64, f64)> { + if rx <= 0.0 || ry <= 0.0 { + return None; + } + let (sphi, cphi) = phi.sin_cos(); + let dx = (p0.x - p1.x) / 2.0; + let dy = (p0.y - p1.y) / 2.0; + let x1 = cphi * dx + sphi * dy; + let y1 = -sphi * dx + cphi * dy; + let (rx, ry) = { + let lambda = x1 * x1 / (rx * rx) + y1 * y1 / (ry * ry); + if lambda > 1.0 { + let s = lambda.sqrt(); + (rx * s, ry * s) + } else { + (rx, ry) + } + }; + let num = rx * rx * ry * ry - rx * rx * y1 * y1 - ry * ry * x1 * x1; + let den = rx * rx * y1 * y1 + ry * ry * x1 * x1; + let coef = if sweep != large { 1.0 } else { -1.0 } * (num.max(0.0) / den.max(f64::MIN_POSITIVE)).sqrt(); + let cxp = coef * rx * y1 / ry; + let cyp = -coef * ry * x1 / rx; + let center = Point::new(cphi * cxp - sphi * cyp + (p0.x + p1.x) / 2.0, sphi * cxp + cphi * cyp + (p0.y + p1.y) / 2.0); + let angle = |ux: f64, uy: f64, vx: f64, vy: f64| { + let dot = ux * vx + uy * vy; + let det = ux * vy - uy * vx; + let denom = (ux.hypot(uy) * vx.hypot(vy)).max(f64::MIN_POSITIVE); + (dot / denom).clamp(-1.0, 1.0).acos().copysign(det) + }; + let (ux, uy) = ((x1 - cxp) / rx, (y1 - cyp) / ry); + let (vx, vy) = ((-x1 - cxp) / rx, (-y1 - cyp) / ry); + let theta = angle(1.0, 0.0, ux, uy); + let ccw = angle(ux, uy, vx, vy).rem_euclid(std::f64::consts::TAU); + Some((center, theta, pick_sweep(ccw, large, sweep))) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + + #[test] + fn resolves_relative_and_smooth_commands_without_flattening_arcs() { + let paths = parse_path("M1 2 l3 4 C5 6 7 8 9 10 s2 3 4 5 A6 7 30 0110 12 z").unwrap(); + assert_eq!(paths.len(), 1); + assert!(paths[0].closed); + assert_eq!(paths[0].start, Point::new(1.0, 2.0)); + assert_eq!(paths[0].segs[0], Segment::Line(Point::new(4.0, 6.0))); + assert_eq!(paths[0].segs[2], Segment::Cubic(Point::new(11.0, 12.0), Point::new(11.0, 13.0), Point::new(13.0, 15.0))); + assert_eq!(paths[0].segs[3], Segment::Arc { rx: 6.0, ry: 7.0, rot: 30.0, large: false, sweep: true, end: Point::new(10.0, 12.0) }); + } + + #[test] + fn smooth_quadratic_consumes_both_coordinates() { + let paths = parse_path("M0 0 Q3 3 6 0 T12 0").unwrap(); + assert_eq!(paths[0].segs[1], Segment::Cubic(Point::new(8.0, -2.0), Point::new(10.0, -2.0), Point::new(12.0, 0.0))); + } + + #[test] + fn arc_centers_keep_radius_correction_and_sweep() { + let (center, delta) = arc_center_circular(Point::new(0.0, 0.0), Point::new(10.0, 0.0), 2.0, false, true).unwrap(); + assert!((center.x - 5.0).abs() < 1e-10); + assert!(center.y.abs() < 1e-10); + assert!((delta - std::f64::consts::PI).abs() < 1e-10); + let (center, start, sweep) = elliptical_params(Point::new(0.0, 0.0), Point::new(10.0, 0.0), 2.0, 1.0, 0.0, false, true).unwrap(); + assert!((center.x - 5.0).abs() < 1e-10); + assert!(center.y.abs() < 1e-10 && start.is_finite()); + assert!((sweep - std::f64::consts::PI).abs() < 1e-10); + } + + #[test] + fn rejects_invalid_flags_and_limits_segments() { + assert!(parse_path("M0 0 A1 1 0 5 1 2 2").is_err()); + assert!(parse_path("M0 0 L").is_err()); + let excessive = format!("M0 0{}", " L1 1".repeat(MAX_PATH_SEGMENTS)); + assert!(parse_path(&excessive).is_err()); + } +} From 89cbc41650393266d30344df1af38da858920f78 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:22:39 +0000 Subject: [PATCH 85/98] Satisfy SVG reader clippy gate --- crates/svg/src/neutral_document.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/svg/src/neutral_document.rs b/crates/svg/src/neutral_document.rs index 24294684a..17f352058 100644 --- a/crates/svg/src/neutral_document.rs +++ b/crates/svg/src/neutral_document.rs @@ -231,10 +231,11 @@ impl Walker<'_> { if !CONTAINERS.contains(&name) { return self.element(name, e); } - if name == b"svg" && self.stack.len() == 1 { - if let Some(root) = self.stack.first_mut() { - *root = viewport(e); - } + if name == b"svg" + && self.stack.len() == 1 + && let Some(root) = self.stack.first_mut() + { + *root = viewport(e); } if name == b"g" && attr(e, b"inkscape:groupmode").as_deref() == Some("layer") From 2c9903ca1d892b5446acf3598d8b142f3e64842f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:29:27 +0000 Subject: [PATCH 86/98] Support configurable legacy MCP sessions and envelopes --- crates/jsonrpc/src/lib.rs | 58 ++++++++++++++++++++++++++++++ crates/mcp/src/lib.rs | 74 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 129 insertions(+), 3 deletions(-) diff --git a/crates/jsonrpc/src/lib.rs b/crates/jsonrpc/src/lib.rs index 9f8128d1d..cfb46ef79 100644 --- a/crates/jsonrpc/src/lib.rs +++ b/crates/jsonrpc/src/lib.rs @@ -195,3 +195,61 @@ where let responses: Vec<_> = batch.iter().filter_map(|v| one(v, limits, &mut on_request, &mut on_notification)).collect(); Ok((!responses.is_empty()).then_some(Value::Array(responses))) } +/// Process a permissive legacy JSON-RPC envelope. This mode accepts arbitrary IDs and +/// parameter shapes, ignores response envelopes, and retains legacy diagnostic messages. +/// The caller supplies dispatch and notification callbacks, and should bound transport input. +pub fn process_legacy(value: &Value, mut on_request: F, mut on_notification: N) -> Option +where + F: FnMut(Request<'_>) -> Result, + N: FnMut(Request<'_>), +{ + fn one(value: &Value, on_request: &mut F, on_notification: &mut N) -> Option + where + F: FnMut(Request<'_>) -> Result, + N: FnMut(Request<'_>), + { + let Some(object) = value.as_object() else { return Some(error_response(Value::Null, RpcError::new(-32600, "message must be an object"))) }; + let id = object.get("id"); + let Some(method) = object.get("method").and_then(Value::as_str) else { + if object.contains_key("result") || object.contains_key("error") { + return None; + } + return Some(error_response(id.cloned().unwrap_or(Value::Null), RpcError::new(-32600, "missing `method`"))); + }; + let request = Request { id, method, params: object.get("params") }; + let Some(id) = id else { + on_notification(request); + return None; + }; + Some(match on_request(request) { + Ok(result) => json!({"jsonrpc": "2.0", "id": id, "result": result}), + Err(error) => error_response(id.clone(), error), + }) + } + if let Some(batch) = value.as_array() { + if batch.is_empty() { + return Some(error_response(Value::Null, RpcError::new(-32600, "empty batch"))); + } + let replies: Vec<_> = batch.iter().filter_map(|v| one(v, &mut on_request, &mut on_notification)).collect(); + (!replies.is_empty()).then_some(Value::Array(replies)) + } else { + one(value, &mut on_request, &mut on_notification) + } +} + +#[cfg(test)] +mod legacy_tests { + use super::*; + #[test] + fn preserves_legacy_batch_routing_and_errors() { + let input = json!([{"id": [1], "method":"echo", "params":5}, {"method":"notice"}, {"result":1}, {}, 7]); + let seen = std::cell::Cell::new(false); + let result = process_legacy(&input, |r| Ok(r.params.cloned().unwrap_or(Value::Null)), |_| seen.set(true)).unwrap(); + assert!(seen.get()); + assert_eq!( + result, + json!([{"jsonrpc":"2.0","id":[1],"result":5}, {"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"missing `method`"}}, {"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"message must be an object"}}]) + ); + assert_eq!(process_legacy(&json!([]), |_| Ok(Value::Null), |_| {}).unwrap()["error"]["message"], "empty batch"); + } +} diff --git a/crates/mcp/src/lib.rs b/crates/mcp/src/lib.rs index 96cfdc7ae..6ca888fc0 100644 --- a/crates/mcp/src/lib.rs +++ b/crates/mcp/src/lib.rs @@ -34,11 +34,27 @@ pub struct Session { policy: Policy, state: State, capabilities: Value, + versions: Vec, + permissive_initialize: bool, + server_info: Option, + instructions: Option, + initialized: bool, } impl Session { pub fn new(name: impl Into, version: impl Into, policy: Policy) -> Self { - Self { name: name.into(), version: version.into(), policy, state: State::New, capabilities: json!({"tools": {}}) } + Self { + name: name.into(), + version: version.into(), + policy, + state: State::New, + capabilities: json!({"tools": {}}), + versions: vec![PROTOCOL_VERSION.into()], + permissive_initialize: false, + server_info: None, + instructions: None, + initialized: false, + } } /// Set the server capabilities returned at initialization. The value must be an object. @@ -50,6 +66,27 @@ impl Session { Ok(self) } + /// Configure a permissive legacy server: absent or malformed initialize parameters are + /// accepted, supported client versions are echoed, and unknown versions use the first entry. + pub fn with_legacy_initialize(mut self, versions: &[&str], server_info: Value) -> Self { + if !versions.is_empty() { + self.versions = versions.iter().map(|v| (*v).to_owned()).collect(); + } + self.permissive_initialize = true; + self.server_info = Some(server_info); + self + } + + /// Update connection-specific instructions returned by subsequent initialize requests. + pub fn set_instructions(&mut self, instructions: String) { + self.instructions = Some(instructions); + } + + /// Whether an initialized notification has been observed, including before initialize. + pub fn is_initialized(&self) -> bool { + self.initialized + } + pub fn state(&self) -> State { self.state } @@ -64,6 +101,18 @@ impl Session { if self.policy == Policy::Strict && (in_batch || self.state != State::New) { return Some(Err(RpcError::new(-32600, "Invalid Request"))); } + if self.permissive_initialize { + let asked = request.params.and_then(|p| p.get("protocolVersion")).and_then(Value::as_str); + let version = asked + .filter(|v| self.versions.iter().any(|supported| supported == v)) + .unwrap_or_else(|| self.versions.first().map(String::as_str).unwrap_or(PROTOCOL_VERSION)); + let mut result = json!({"protocolVersion": version, "capabilities": self.capabilities, "serverInfo": self.server_info}); + if let Some(instructions) = &self.instructions { + result["instructions"] = json!(instructions); + } + self.state = State::AwaitingInitialized; + return Some(Ok(result)); + } let Some(params) = request.params.and_then(Value::as_object) else { return Some(Err(invalid_initialize())); }; @@ -89,8 +138,11 @@ impl Session { /// Observe a JSON-RPC notification; notifications never receive replies. pub fn notification(&mut self, request: Request<'_>) { - if request.method == "notifications/initialized" && self.state == State::AwaitingInitialized { - self.state = State::Ready; + if request.method == "notifications/initialized" { + self.initialized = true; + if self.state == State::AwaitingInitialized { + self.state = State::Ready; + } } } } @@ -127,6 +179,22 @@ mod tests { Request { id: None, method, params } } + #[test] + fn legacy_initialization_negotiates_and_accepts_missing_parameters() { + let mut s = Session::new("test", "1", Policy::Compatible).with_legacy_initialize(&["new", "old"], json!({"name":"custom", "title":"Title"})); + s.notification(call("notifications/initialized", None)); + assert!(s.is_initialized()); + s.set_instructions("help".into()); + for (params, expected) in + [(None, "new"), (Some(json!({"protocolVersion":"old"})), "old"), (Some(json!({"protocolVersion":"unknown"})), "new")] + { + let result = s.request(call("initialize", params.as_ref()), false).unwrap().unwrap(); + assert_eq!(result["protocolVersion"], expected); + assert_eq!(result["serverInfo"]["title"], "Title"); + assert_eq!(result["instructions"], "help"); + } + } + #[test] fn compatible_negotiation_preserves_existing_clients() { let mut s = Session::new("test", "1", Policy::Compatible); From 451a390542fbde0519cbb8ade3a3325cf02d97be Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:37:41 +0000 Subject: [PATCH 87/98] Add opt-in legacy Base64 compatibility --- crates/mime/src/base64_compat.rs | 93 ++++++++++++++++++++++++++++++++ crates/mime/src/lib.rs | 1 + 2 files changed, 94 insertions(+) create mode 100644 crates/mime/src/base64_compat.rs diff --git a/crates/mime/src/base64_compat.rs b/crates/mime/src/base64_compat.rs new file mode 100644 index 000000000..452b276f9 --- /dev/null +++ b/crates/mime/src/base64_compat.rs @@ -0,0 +1,93 @@ +//! Compatibility decoding for legacy transports that accepted mixed Base64 alphabets and padding. +//! +//! This is deliberately permissive and must not be used for signatures, credentials or other +//! contexts requiring canonical Base64. Use [`crate::base64`] for strict RFC 4648 decoding. +use crate::base64::DecodeError; + +/// Decode standard or URL-safe Base64, accepting either alphabet even in the same input. +/// Spaces, CR, LF and `=` are ignored wherever they occur; an incomplete final sextet is +/// discarded. Other bytes are errors. Output allocation is bounded by the input length. +pub fn decode(input: impl AsRef<[u8]>) -> Result, DecodeError> { + let input = input.as_ref(); + let mut out = Vec::new(); + let capacity = (input.len() / 4) * 3 + (input.len() % 4) * 3 / 4; + out.try_reserve(capacity).map_err(|_| DecodeError)?; + let (mut acc, mut bits) = (0u32, 0u32); + for c in input { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' | b'-' => 62, + b'/' | b'_' => 63, + b'=' | b'\n' | b'\r' | b' ' => continue, + _ => return Err(DecodeError), + }; + acc = (acc << 6) | u32::from(v); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + Ok(out) +} +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + use super::*; + #[test] + fn accepts_mixed_alphabets_and_ignores_padding_anywhere() { + assert_eq!(decode("+_8=").unwrap(), [251, 255]); + assert_eq!(decode("Z=m 9\nv\r").unwrap(), b"foo"); + assert_eq!(decode("A").unwrap(), b""); + assert_eq!(decode("AB").unwrap(), [0]); + assert!(decode("Zm9v\t").is_err()); + assert!(decode("Zm9v!").is_err()); + assert!(crate::base64::STANDARD.decode("Z=m9v").is_err()); + } + // Differential against CADCraft's pre-cutover decoder, including malformed legacy inputs. + fn legacy(s: &str) -> Option> { + let mut out = Vec::new(); + let (mut acc, mut bits) = (0u32, 0u32); + for c in s.bytes() { + let value = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' | b'-' => 62, + b'/' | b'_' => 63, + b'=' | b'\n' | b'\r' | b' ' => continue, + _ => return None, + }; + acc = (acc << 6) | u32::from(value); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + } + } + Some(out) + } + #[test] + fn matches_the_legacy_decoder_on_seeded_mixed_inputs() { + let alphabet = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/-_= \r\n!\t"; + let mut seed = 0x1234_5678u32; + for n in 0..50_000 { + let mut text = String::new(); + for _ in 0..n % 45 { + seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + text.push(char::from(alphabet[(seed as usize) % alphabet.len()])); + } + assert_eq!(decode(&text).ok(), legacy(&text), "{text:?}"); + } + } + + #[test] + fn round_trip_rfc_vectors_with_the_shared_encoder() { + for data in [&b""[..], b"f", b"fo", b"foo", b"foobar", &[0, 255, 127]] { + assert_eq!(decode(crate::base64::STANDARD.encode(data)).unwrap(), data); + } + } +} diff --git a/crates/mime/src/lib.rs b/crates/mime/src/lib.rs index 72a930b8b..d3ff9f257 100644 --- a/crates/mime/src/lib.rs +++ b/crates/mime/src/lib.rs @@ -28,6 +28,7 @@ //! says) is the caller's: this crate has no notion of a mailbox or a document. pub mod base64; +pub mod base64_compat; pub mod charset; #[cfg(feature = "detect")] pub mod detect; From 32eff39b74661ad121a0aa08a36b69c01defaf58 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:39:23 +0000 Subject: [PATCH 88/98] Share PNG header and color chunk inspection --- crates/image-codecs/src/lib.rs | 1 + crates/image-codecs/src/png_probe.rs | 169 +++++++++++++++++++++++++++ 2 files changed, 170 insertions(+) create mode 100644 crates/image-codecs/src/png_probe.rs diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 6516b9ff2..d8659bf54 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -28,6 +28,7 @@ mod inspection; mod lab; mod options; pub mod orientation; +pub mod png_probe; pub mod resolution; pub mod source_settings; pub mod web; diff --git a/crates/image-codecs/src/png_probe.rs b/crates/image-codecs/src/png_probe.rs new file mode 100644 index 000000000..6ec610c14 --- /dev/null +++ b/crates/image-codecs/src/png_probe.rs @@ -0,0 +1,169 @@ +//! Header-only PNG inspection without inflating image data. +use crate::{CodecError, Format}; +fn err(message: &str) -> CodecError { + CodecError::malformed(Format::Png, message) +} + +/// Stored dimensions and EXIF orientation. This does not validate CRCs or compressed pixels. +pub fn header(b: &[u8]) -> Result<(u32, u32, u16), CodecError> { + if !b.starts_with(b"\x89PNG\r\n\x1a\n") { + return Err(err("bad signature")); + } + let be32 = |p: usize| b.get(p..p.checked_add(4)?).map(|v| u32::from_be_bytes([v[0], v[1], v[2], v[3]])); + if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") { + return Err(err("missing IHDR")); + } + let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?; + let (w, h) = (u32::from_be_bytes([ihdr[0], ihdr[1], ihdr[2], ihdr[3]]), u32::from_be_bytes([ihdr[4], ihdr[5], ihdr[6], ihdr[7]])); + let (depth, color, compression, filter, interlace) = (ihdr[8], ihdr[9], ihdr[10], ihdr[11], ihdr[12]); + if w > i32::MAX as u32 || h > i32::MAX as u32 { + return Err(err("dimensions out of range")); + } + if w == 0 || h == 0 { + return Err(err("zero dimension")); + } + let depth_ok = match color { + 0 => matches!(depth, 1 | 2 | 4 | 8 | 16), + 3 => matches!(depth, 1 | 2 | 4 | 8), + 2 | 4 | 6 => matches!(depth, 8 | 16), + _ => false, + }; + if !depth_ok || compression != 0 || filter != 0 || interlace > 1 { + return Err(err("invalid IHDR")); + } + let (mut p, mut palette, mut idat, mut after_idat, mut exif) = (8usize, false, false, false, None); + while let Some(len) = be32(p) { + let Some(kind) = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)) else { break }; + if idat && kind != b"IDAT" { + // a chunk follows the image data: the IDAT run is complete + after_idat = true; + } + if kind == b"IEND" { + break; + } + let start = p.saturating_add(8); + let data = (len <= i32::MAX as u32).then(|| start.checked_add(len as usize).and_then(|end| b.get(start..end))).flatten(); + let Some(data) = data else { + if after_idat { + break; // a damaged chunk after the image data: a decode ignores it too + } + return Err(err("truncated")); + }; + match kind { + b"PLTE" => palette = true, + b"IDAT" if after_idat => {} // a stray IDAT after other chunks: the first run is the image + b"IDAT" => { + if color == 3 && !palette { + return Err(err("indexed image without a palette")); + } + idat = true; + } + b"eXIf" if exif.is_none() => exif = Some(data.strip_prefix(b"Exif\0\0").unwrap_or(data)), + _ => {} + } + p = start.saturating_add(data.len()).saturating_add(4); + } + if !idat { + return Err(err("no image data")); + } + if !after_idat { + return Err(err("truncated image data")); + } + Ok((w, h, exif.map_or(1, crate::exif_orientation))) +} + +/// Container color declarations preceding image data. Raw values leave color policy to callers. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ColorChunks { + pub srgb: bool, + pub gamma: Option, + pub chromaticities: Option<[f64; 8]>, +} + +/// Read bounded color chunks before image data; malformed tails are ignored. +pub fn color_chunks(b: &[u8]) -> Option { + if !b.starts_with(b"\x89PNG\r\n\x1a\n") { + return None; + } + let mut result = ColorChunks::default(); + let mut p = 8usize; + while let Some(length) = b.get(p..p.checked_add(4)?) { + let len = u32::from_be_bytes(length.try_into().ok()?) as usize; + let start = p.checked_add(8)?; + let Some(kind) = b.get(p.checked_add(4)?..start) else { break }; + let Some(payload) = start.checked_add(len).and_then(|end| b.get(start..end)) else { break }; + match kind { + b"sRGB" => result.srgb = true, + b"gAMA" if payload.len() == 4 => { + let v = u32::from_be_bytes(payload.try_into().ok()?) as f32 / 100_000.0; + if v > 0.0 && v.is_finite() { + result.gamma = Some(v); + } + } + b"cHRM" if payload.len() == 32 => { + let mut vals = [0.0; 8]; + for (i, dst) in vals.iter_mut().enumerate() { + let raw = i32::from_be_bytes(payload[i * 4..i * 4 + 4].try_into().ok()?); + *dst = f64::from(raw) / 100_000.0; + } + result.chromaticities = Some(vals); + } + b"IDAT" | b"IEND" => break, + _ => {} + } + let Some(next) = start.checked_add(len).and_then(|end| end.checked_add(4)) else { break }; + p = next; + } + Some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chunk(out: &mut Vec, kind: &[u8; 4], data: &[u8]) { + out.extend_from_slice(&(data.len() as u32).to_be_bytes()); + out.extend_from_slice(kind); + out.extend_from_slice(data); + out.extend_from_slice(&[0; 4]); // Header inspection deliberately does not validate CRCs. + } + + fn png(color: u8) -> Vec { + let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&2u32.to_be_bytes()); + ihdr.extend_from_slice(&3u32.to_be_bytes()); + ihdr.extend_from_slice(&[8, color, 0, 0, 0]); + chunk(&mut bytes, b"IHDR", &ihdr); + bytes + } + + #[test] + fn header_checks_structure_and_reads_late_orientation_without_crc_validation() { + let mut bytes = png(2); + chunk(&mut bytes, b"IDAT", &[1]); + let exif = b"II\x2a\x00\x08\x00\x00\x00\x01\x00\x12\x01\x03\x00\x01\x00\x00\x00\x06\x00\x00\x00\x00\x00\x00\x00"; + chunk(&mut bytes, b"eXIf", exif); + chunk(&mut bytes, b"IEND", &[]); + assert_eq!(header(&bytes).ok(), Some((2, 3, 6))); + assert!(header(&bytes[..34]).is_err()); + let mut indexed = png(3); + chunk(&mut indexed, b"IDAT", &[1]); + chunk(&mut indexed, b"IEND", &[]); + assert!(header(&indexed).is_err()); + indexed[25] = 7; + assert!(header(&indexed).is_err()); + } + + #[test] + fn reads_color_chunks_only_before_image_data() { + let mut bytes = png(2); + chunk(&mut bytes, b"gAMA", &45455u32.to_be_bytes()); + chunk(&mut bytes, b"IDAT", &[1]); + chunk(&mut bytes, b"sRGB", &[0]); + let color = color_chunks(&bytes).unwrap(); + assert_eq!(color.gamma, Some(0.45455)); + assert!(!color.srgb); + assert!(color_chunks(b"not png").is_none()); + } +} From 8e394d43faae421541892945cc56f10e47ee3be7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:39:42 +0000 Subject: [PATCH 89/98] audio: share bounded range-backed WAV PCM access --- crates/audio/src/lib.rs | 3 + crates/audio/src/wav_range.rs | 218 ++++++++++++++++++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 crates/audio/src/wav_range.rs diff --git a/crates/audio/src/lib.rs b/crates/audio/src/lib.rs index 33dcdee3b..8681ce6d8 100644 --- a/crates/audio/src/lib.rs +++ b/crates/audio/src/lib.rs @@ -66,6 +66,9 @@ mod wav; feature = "vorbis-encode" ))] mod write; + +/// Range-backed PCM WAVE access and a 16-bit PCM writer. +pub mod wav_range; #[cfg(feature = "vorbis-encode")] pub use write::{OGG_BITRATES, encode_ogg}; diff --git a/crates/audio/src/wav_range.rs b/crates/audio/src/wav_range.rs new file mode 100644 index 000000000..1ced4fb78 --- /dev/null +++ b/crates/audio/src/wav_range.rs @@ -0,0 +1,218 @@ +//! Range-backed PCM WAVE reader and canonical 16-bit writer. +//! +//! Reads chunk headers and only the requested sample range; it never loads a whole file. + +use crate::{AudioError, Result}; +use craft_media::ByteSource; + +const MAX_CHUNKS: usize = 1 << 16; +const IXML_MAX: u64 = 1 << 20; +/// Maximum sample data returned by a single range read. +pub const MAX_RANGE_BYTES: u64 = 4 << 20; + +fn le16(b: &[u8], o: usize) -> u16 { + b.get(o..o.saturating_add(2)).and_then(|s| s.try_into().ok()).map_or(0, u16::from_le_bytes) +} +fn le32(b: &[u8], o: usize) -> u32 { + b.get(o..o.saturating_add(4)).and_then(|s| s.try_into().ok()).map_or(0, u32::from_le_bytes) +} +fn read_range(source: &impl ByteSource, off: u64, len: usize) -> Result> { + let mut bytes = vec![0; len]; + source.read_at(off, &mut bytes)?; + Ok(bytes) +} + +/// A PCM WAVE file held by an arbitrary random-access source. +pub struct WavRange { + source: S, + /// Number of complete sample frames. + pub frames: u64, + /// Channel count. + pub channels: u16, + /// Sample rate in Hz. + pub rate: u32, + /// Container bits per sample. + pub bits: u16, + /// Whether samples are floating point. + pub float: bool, + /// BWF time reference in samples, from bext or iXML. + pub time_reference: Option, + data_off: u64, +} + +impl WavRange { + /// Open with caller-selected bounds on channel count and sample rate. + pub fn open(source: S, max_channels: u32, max_rate: u32) -> Result { + let len = source.len(); + let mut pos: u64 = 12; + let mut fmt = None; + let mut data = None; + let mut bext_tr = None; + let mut ixml_tr = None; + let mut ds64_data = None; + for _ in 0..MAX_CHUNKS { + let Some(body) = pos.checked_add(8).filter(|&b| b <= len) else { break }; + let mut hdr = [0u8; 8]; + source.read_at(pos, &mut hdr)?; + let id = &hdr[0..4]; + let mut clen = u64::from(le32(&hdr, 4)); + let rest = len - body; + if id == b"data" { + if clen == 0xFFFF_FFFF { + clen = ds64_data.unwrap_or(rest); + } + clen = clen.min(rest); + } + if clen > rest { + break; + } + match id { + b"fmt " if clen >= 16 => { + let b = read_range(&source, body, clen.min(40) as usize)?; + let mut tag = le16(&b, 0); + if tag == 0xFFFE && clen >= 40 { + tag = le16(&b, 24); + } + fmt = Some((tag, le16(&b, 2), le32(&b, 4), le16(&b, 14))); + } + b"ds64" if clen >= 16 => { + let b = read_range(&source, body + 8, 8)?; + ds64_data = Some(u64::from(le32(&b, 0)) | u64::from(le32(&b, 4)) << 32); + } + b"data" => data = Some((body, clen)), + b"bext" if clen >= 346 => { + let b = read_range(&source, body + 338, 8)?; + bext_tr = Some(u64::from(le32(&b, 0)) | u64::from(le32(&b, 4)) << 32); + } + b"iXML" => { + let b = read_range(&source, body, clen.min(IXML_MAX) as usize)?; + ixml_tr = ixml_time_reference(&b); + } + _ => {} + } + pos = body.saturating_add(clen).saturating_add(clen & 1); + } + let (tag, channels, rate, bits) = fmt.ok_or_else(|| AudioError::Malformed("missing fmt chunk".into()))?; + let (data_off, dlen) = data.ok_or_else(|| AudioError::Malformed("missing data chunk".into()))?; + let float = tag == 3; + if !(tag == 1 || float) + || channels == 0 + || rate == 0 + || u32::from(channels) > max_channels + || rate > max_rate + || !matches!(bits, 8 | 16 | 24 | 32 | 64) + { + return Err(AudioError::Unsupported(format!("WAV format tag {tag}, {bits} bits"))); + } + let frames = dlen / (u64::from(channels) * u64::from(bits / 8)); + Ok(Self { source, frames, channels, rate, bits, float, time_reference: bext_tr.or(ixml_tr), data_off }) + } + /// Bytes in one whole frame. + pub fn frame_bytes(&self) -> usize { + usize::from(self.channels) * usize::from(self.bits / 8) + } + /// Read the inclusive range of frames, capped to prevent unbounded allocation. + pub fn read_frames(&self, first: u64, last: u64) -> Result> { + if first > last || last >= self.frames { + return Err(AudioError::Malformed("WAV frame range outside data".into())); + } + let n = (last - first + 1).saturating_mul(self.frame_bytes() as u64); + if n > MAX_RANGE_BYTES { + return Err(AudioError::TooLarge("WAV frame range".into())); + } + let off = self.data_off.saturating_add(first.saturating_mul(self.frame_bytes() as u64)); + read_range(&self.source, off, n as usize) + } + /// Decode a channel from a buffer returned by `read_frames`. + pub fn sample(&self, block: &[u8], frame: usize, ch: usize) -> f32 { + if ch >= usize::from(self.channels) { + return 0.0; + } + let bps = usize::from(self.bits / 8); + let Some(o) = frame.checked_mul(usize::from(self.channels)).and_then(|v| v.checked_add(ch)).and_then(|v| v.checked_mul(bps)) else { + return 0.0; + }; + let Some(b) = block.get(o..o.saturating_add(bps)) else { return 0.0 }; + match (self.bits, self.float, b) { + (8, _, &[a]) => (a as f32 - 128.0) / 128.0, + (16, _, &[a, b]) => i16::from_le_bytes([a, b]) as f32 / 32768.0, + (24, _, &[a, b, c]) => (i32::from_le_bytes([0, a, b, c]) >> 8) as f32 / 8_388_608.0, + (32, false, &[a, b, c, d]) => i32::from_le_bytes([a, b, c, d]) as f32 / 2_147_483_648.0, + (32, true, &[a, b, c, d]) => f32::from_le_bytes([a, b, c, d]), + (64, true, b) => f64::from_le_bytes(b.try_into().unwrap_or([0; 8])) as f32, + _ => 0.0, + } + } +} + +fn ixml_time_reference(x: &[u8]) -> Option { + let text = String::from_utf8_lossy(x); + let field = |tag: &str| -> Option { + let open = format!("<{tag}>"); + let a = text.find(&open)? + open.len(); + let b = a + text[a..].find('<')?; + text[a..b].trim().parse().ok() + }; + let low = field("BWF_TIME_REFERENCE_LOW")?; + Some(low | field("BWF_TIME_REFERENCE_HIGH").unwrap_or(0) << 32) +} + +/// Write interleaved samples as 16-bit PCM WAVE, using symmetric 32767 scaling. +#[cfg(feature = "pcm-encode")] +pub fn write_wav16(samples: &[f32], channels: u16, rate: u32) -> Vec { + let data_len = samples.len().saturating_mul(2); + let mut v = crate::pcm_wav_header(u32::try_from(data_len).unwrap_or(u32::MAX), channels, rate, 16); + v.reserve(data_len); + for s in samples { + v.extend_from_slice(&((s.clamp(-1.0, 1.0) * 32767.0).round() as i16).to_le_bytes()); + } + v +} + +#[cfg(all(test, feature = "pcm-encode"))] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + + struct Source { + bytes: Vec, + reads: Arc>>, + } + impl ByteSource for Source { + fn len(&self) -> u64 { + self.bytes.len() as u64 + } + fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { + self.reads.lock().unwrap().push((offset, buf.len())); + self.bytes.read_at(offset, buf) + } + } + #[test] + fn bounded_reads_and_pcm_values() { + let bytes = write_wav16(&[-1.0, -0.5, 0.0, 0.5, 1.0], 1, 48_000); + let reads = Arc::new(Mutex::new(Vec::new())); + let wav = WavRange::open(Source { bytes, reads: reads.clone() }, 64, 384_000).unwrap(); + assert_eq!((wav.frames, wav.channels, wav.rate, wav.bits), (5, 1, 48_000, 16)); + assert!(reads.lock().unwrap().iter().all(|(_, n)| *n <= 40)); + let data = wav.read_frames(1, 3).unwrap(); + assert_eq!(wav.sample(&data, 0, 0), -0.5); + assert_eq!(wav.sample(&data, 2, 0), 0.5); + assert!(wav.read_frames(0, 5).is_err()); + assert!(wav.read_frames(3, 1).is_err()); + assert_eq!(reads.lock().unwrap().last(), Some(&(46, 6))); + } + #[test] + fn ixml_fallback_accepts_flat_fields() { + let base = write_wav16(&[0.0], 1, 48_000); + let mut bytes = base[..36].to_vec(); + let xml = b"51"; + bytes.extend_from_slice(b"iXML"); + bytes.extend_from_slice(&(xml.len() as u32).to_le_bytes()); + bytes.extend_from_slice(xml); + if xml.len() % 2 != 0 { + bytes.push(0); + } + bytes.extend_from_slice(&base[36..]); + assert_eq!(WavRange::open(bytes.as_slice(), 64, 384_000).unwrap().time_reference, Some((1 << 32) + 5)); + } +} From 1bce1af0eb5e0fd512226ea1247bef958a87e751 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:39:50 +0000 Subject: [PATCH 90/98] Expose early PNG dimension preflight --- crates/image-codecs/src/png_probe.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/image-codecs/src/png_probe.rs b/crates/image-codecs/src/png_probe.rs index 6ec610c14..9cb40f402 100644 --- a/crates/image-codecs/src/png_probe.rs +++ b/crates/image-codecs/src/png_probe.rs @@ -4,6 +4,17 @@ fn err(message: &str) -> CodecError { CodecError::malformed(Format::Png, message) } +/// Dimensions declared in a PNG IHDR prefix, without checking the rest of the file. +/// Useful only for an early allocation limit; use [`header`] for structural inspection. +pub fn declared_dimensions(b: &[u8]) -> Option<(u32, u32)> { + if !b.starts_with(b"\x89PNG\r\n\x1a\n") || b.get(12..16) != Some(b"IHDR") { + return None; + } + let w = u32::from_be_bytes(b.get(16..20)?.try_into().ok()?); + let h = u32::from_be_bytes(b.get(20..24)?.try_into().ok()?); + Some((w, h)) +} + /// Stored dimensions and EXIF orientation. This does not validate CRCs or compressed pixels. pub fn header(b: &[u8]) -> Result<(u32, u32, u16), CodecError> { if !b.starts_with(b"\x89PNG\r\n\x1a\n") { From badb7904116d518ad84ff1da5a1d99d3cc5e584f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:40:02 +0000 Subject: [PATCH 91/98] audio: expose WAV signature detection --- crates/audio/src/wav_range.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/audio/src/wav_range.rs b/crates/audio/src/wav_range.rs index 1ced4fb78..d8aea0d54 100644 --- a/crates/audio/src/wav_range.rs +++ b/crates/audio/src/wav_range.rs @@ -5,6 +5,11 @@ use crate::{AudioError, Result}; use craft_media::ByteSource; +/// Whether a prefix identifies a RIFF or RF64 WAVE file. +pub fn sniff(b: &[u8]) -> bool { + b.len() >= 12 && (&b[0..4] == b"RIFF" || &b[0..4] == b"RF64") && &b[8..12] == b"WAVE" +} + const MAX_CHUNKS: usize = 1 << 16; const IXML_MAX: u64 = 1 << 20; /// Maximum sample data returned by a single range read. From e4cef773fcf04392df55cb3abf156da92a6f2ce1 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:40:50 +0000 Subject: [PATCH 92/98] Enforce PNG probe pixel limit before chunk scanning --- crates/image-codecs/src/png_probe.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/image-codecs/src/png_probe.rs b/crates/image-codecs/src/png_probe.rs index 9cb40f402..94e308add 100644 --- a/crates/image-codecs/src/png_probe.rs +++ b/crates/image-codecs/src/png_probe.rs @@ -17,6 +17,11 @@ pub fn declared_dimensions(b: &[u8]) -> Option<(u32, u32)> { /// Stored dimensions and EXIF orientation. This does not validate CRCs or compressed pixels. pub fn header(b: &[u8]) -> Result<(u32, u32, u16), CodecError> { + header_with_max_pixels(b, u64::MAX) +} + +/// As [`header`], refusing images over `max_pixels` before scanning their chunks. +pub fn header_with_max_pixels(b: &[u8], max_pixels: u64) -> Result<(u32, u32, u16), CodecError> { if !b.starts_with(b"\x89PNG\r\n\x1a\n") { return Err(err("bad signature")); } @@ -33,6 +38,9 @@ pub fn header(b: &[u8]) -> Result<(u32, u32, u16), CodecError> { if w == 0 || h == 0 { return Err(err("zero dimension")); } + if u64::from(w) * u64::from(h) > max_pixels { + return Err(CodecError::LimitExceeded("PNG pixel count".into())); + } let depth_ok = match color { 0 => matches!(depth, 1 | 2 | 4 | 8 | 16), 3 => matches!(depth, 1 | 2 | 4 | 8), From 188d935887cb1ea2b9f058e8063718b03e1f27ee Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:41:20 +0000 Subject: [PATCH 93/98] Cover bounded PNG header probing --- crates/image-codecs/src/png_probe.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/image-codecs/src/png_probe.rs b/crates/image-codecs/src/png_probe.rs index 94e308add..7edda454d 100644 --- a/crates/image-codecs/src/png_probe.rs +++ b/crates/image-codecs/src/png_probe.rs @@ -165,6 +165,8 @@ mod tests { chunk(&mut bytes, b"eXIf", exif); chunk(&mut bytes, b"IEND", &[]); assert_eq!(header(&bytes).ok(), Some((2, 3, 6))); + assert!(matches!(header_with_max_pixels(&bytes, 5), Err(CodecError::LimitExceeded(_)))); + assert_eq!(declared_dimensions(&bytes), Some((2, 3))); assert!(header(&bytes[..34]).is_err()); let mut indexed = png(3); chunk(&mut indexed, b"IDAT", &[1]); From aa398af9dd312523c8a9fcb3b2711b726a40abdd Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:48:01 +0000 Subject: [PATCH 94/98] audio: fix range test lint on workspace gates --- crates/audio/src/wav_range.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/audio/src/wav_range.rs b/crates/audio/src/wav_range.rs index d8aea0d54..33762caf2 100644 --- a/crates/audio/src/wav_range.rs +++ b/crates/audio/src/wav_range.rs @@ -188,7 +188,7 @@ mod tests { self.bytes.len() as u64 } fn read_at(&self, offset: u64, buf: &mut [u8]) -> std::io::Result<()> { - self.reads.lock().unwrap().push((offset, buf.len())); + self.reads.lock().unwrap_or_else(std::sync::PoisonError::into_inner).push((offset, buf.len())); self.bytes.read_at(offset, buf) } } @@ -214,7 +214,7 @@ mod tests { bytes.extend_from_slice(b"iXML"); bytes.extend_from_slice(&(xml.len() as u32).to_le_bytes()); bytes.extend_from_slice(xml); - if xml.len() % 2 != 0 { + if xml.len() & 1 != 0 { bytes.push(0); } bytes.extend_from_slice(&base[36..]); From a88aab72c4620d1956196ec51107c3ca9c241a52 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:58:28 +0000 Subject: [PATCH 95/98] aaf: bound duplicate property detection work --- crates/aaf/src/aaf/hostile_tests.rs | 16 ++++++++++++++++ crates/aaf/src/aaf/store.rs | 4 +++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/aaf/src/aaf/hostile_tests.rs b/crates/aaf/src/aaf/hostile_tests.rs index 853b0000c..527e50da8 100644 --- a/crates/aaf/src/aaf/hostile_tests.rs +++ b/crates/aaf/src/aaf/hostile_tests.rs @@ -108,3 +108,19 @@ fn objects_nested_too_deeply_are_an_error() { let e = store::read(&chain(250)).unwrap_err(); assert!(e.contains("nested too deeply"), "{e}"); } + +/// Many distinct property IDs still reject a duplicate without quadratic scanning. +#[test] +fn duplicate_after_many_distinct_properties_is_rejected() { + let mut props = vec![0x4C, 0x20]; + props.extend_from_slice(&8193u16.to_le_bytes()); + for id in (0..8192u16).chain(std::iter::once(7)) { + props.extend_from_slice(&id.to_le_bytes()); + props.extend_from_slice(&0x82u16.to_le_bytes()); + props.extend_from_slice(&0u16.to_le_bytes()); + } + let mut writer = Writer::new(Version::V3); + writer.stream(Writer::ROOT, "properties", props).unwrap(); + let e = store::read(&writer.finish()).unwrap_err(); + assert!(e.contains("duplicate property id"), "{e}"); +} diff --git a/crates/aaf/src/aaf/store.rs b/crates/aaf/src/aaf/store.rs index 69b14fcf2..859c0ea77 100644 --- a/crates/aaf/src/aaf/store.rs +++ b/crates/aaf/src/aaf/store.rs @@ -461,11 +461,13 @@ impl R<'_> { if at > props.len() { return Err(bad("truncated property index")); } + // A properties stream can name tens of thousands of ids: do not rescan all earlier ids. + let mut seen = std::collections::HashSet::with_capacity(n); for i in 0..n { let (Some(pid), Some(sf), Some(len)) = (rd_u16(&props, 4 + i * 6), rd_u16(&props, 6 + i * 6), rd_u16(&props, 8 + i * 6)) else { return Err(bad("truncated property index")); }; - if o.props.iter().any(|(previous, _)| *previous == pid) { + if !seen.insert(pid) { return Err(bad("duplicate property id")); } let v = props.get(at..at + len as usize).ok_or_else(|| bad("truncated property value"))?; From 6345459e1297ee85a7ca62b428a440691bd062c1 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 15:59:23 +0000 Subject: [PATCH 96/98] Share MTEXT Unicode and MIF escape decoding --- crates/dxf/src/lib.rs | 1 + crates/dxf/src/mtext.rs | 103 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 crates/dxf/src/mtext.rs diff --git a/crates/dxf/src/lib.rs b/crates/dxf/src/lib.rs index 4cf3e46e9..9c912613b 100644 --- a/crates/dxf/src/lib.rs +++ b/crates/dxf/src/lib.rs @@ -11,6 +11,7 @@ use std::io::Write as _; pub mod model; +pub mod mtext; pub mod paths; mod version; pub use version::Version; diff --git a/crates/dxf/src/mtext.rs b/crates/dxf/src/mtext.rs new file mode 100644 index 000000000..9b113aa6c --- /dev/null +++ b/crates/dxf/src/mtext.rs @@ -0,0 +1,103 @@ +//! MTEXT inline-escape handling independent of drawing and text layout. +//! +//! Unicode and MIF escapes are decoded to literal characters, with markup-significant +//! characters re-escaped so subsequent MTEXT parsers cannot treat them as commands. + +/// Whether an inline command takes an argument terminated by `;`. +pub fn takes_arg(code: char) -> bool { + matches!(code, 'f' | 'F' | 'H' | 'W' | 'Q' | 'T' | 'A' | 'C' | 'c' | 'p') +} + +/// Four hex digits at `chars[at..at + 4]`. +fn hex4(chars: &[char], at: usize) -> Option { + let digits = chars.get(at..at.checked_add(4)?)?; + digits.iter().try_fold(0u32, |acc, c| Some(acc * 16 + c.to_digit(16)?)) +} + +/// Push a decoded character so that the tokenizers read it as text, never as markup. +fn push_literal(out: &mut String, c: char) { + if matches!(c, '\\' | '{' | '}') { + out.push('\\'); + } + out.push(c); +} + +/// Decode `\U+hhhh` (one UTF-16 unit; a surrogate pair gives one character, a lone surrogate +/// U+FFFD) and `\M+nhhhh` (double-byte East Asian characters) to the characters they stand +/// for. Malformed escapes remain unchanged for the caller to interpret. +/// An escaped backslash (`\\U+0041`) is not an escape. +pub fn unescape_codes(s: &str) -> std::borrow::Cow<'_, str> { + if !s.contains("\\U+") && !s.contains("\\M+") { + return std::borrow::Cow::Borrowed(s); + } + let chars: Vec = s.chars().collect(); + let mut out = String::with_capacity(s.len()); + let mut i = 0; + while let Some(&c) = chars.get(i) { + if c != '\\' { + out.push(c); + i += 1; + continue; + } + let (n, plus) = (chars.get(i + 1).copied(), chars.get(i + 2).copied()); + if n == Some('U') && plus == Some('+') { + if let Some(unit) = hex4(&chars, i + 3) { + i += 7; + let low = |at: usize| { + (chars.get(at) == Some(&'\\') && chars.get(at + 1) == Some(&'U') && chars.get(at + 2) == Some(&'+')) + .then(|| hex4(&chars, at + 3)) + .flatten() + .filter(|u| (0xDC00..0xE000).contains(u)) + }; + let ch = if (0xD800..0xDC00).contains(&unit) { + match low(i) { + Some(lo) => { + i += 7; + char::from_u32(0x10000 + ((unit - 0xD800) << 10) + (lo - 0xDC00)) + } + None => None, + } + } else { + char::from_u32(unit) + }; + push_literal(&mut out, ch.unwrap_or('\u{fffd}')); + continue; + } + } else if n == Some('M') && plus == Some('+') && chars.get(i + 3).is_some_and(char::is_ascii_digit) && hex4(&chars, i + 4).is_some() { + push_literal(&mut out, crate::mif_escape_at(&chars, i).unwrap_or('\u{fffd}')); + i += 8; + continue; + } + // Any other backslash pair stays as it is (and an escaped backslash stays one). + out.push('\\'); + i += 1; + if let Some(&next) = chars.get(i) { + out.push(next); + i += 1; + } + } + std::borrow::Cow::Owned(out) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn unicode_and_mif_escapes_remain_literal_markup() { + assert_eq!(unescape_codes(r"a\U+0041\U+D83D\U+DE00"), "aA😀"); + assert_eq!(unescape_codes(r"\M+5BCFE"), "件"); + assert_eq!(unescape_codes(r"\U+005C\U+007B\U+007D"), r"\\\{\}"); + assert_eq!(unescape_codes(r"\\U+0041"), r"\\U+0041"); + assert_eq!(unescape_codes(r"\U+12"), r"\U+12"); + assert_eq!(unescape_codes(r"\U+D800"), "�"); + } + #[test] + fn parameterized_commands_are_classified_without_relaxing_others() { + for c in "fFHWQTACcp".chars() { + assert!(takes_arg(c)); + } + for c in ['S', 'P', 'Z', '\\'] { + assert!(!takes_arg(c)); + } + } +} From d3024d0cd5af26edecc04b0a68689436952bd116 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 11:05:39 -0500 Subject: [PATCH 97/98] fix(captions): preserve literal ASS escapes and normalize newlines --- crates/captions/src/ass.rs | 11 ++++++++--- crates/captions/tests/it/ass.rs | 31 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/crates/captions/src/ass.rs b/crates/captions/src/ass.rs index 3c907c829..2d0a1df55 100644 --- a/crates/captions/src/ass.rs +++ b/crates/captions/src/ass.rs @@ -191,7 +191,10 @@ fn escape(text: &str) -> String { let mut out = String::with_capacity(text.len() + 8); let mut rest = text; while !rest.is_empty() { - if let Some(r) = rest.strip_prefix("") { + if let Some(r) = rest.strip_prefix("\r\n") { + out.push_str("\\N"); + rest = r; + } else if let Some(r) = rest.strip_prefix("") { out.push_str("{\\i1}"); rest = r; } else if let Some(r) = rest.strip_prefix("") { @@ -211,7 +214,9 @@ fn escape(text: &str) -> String { rest = r; } else if let Some(c) = rest.chars().next() { match c { - '\n' => out.push_str("\\N"), + '\n' | '\r' => out.push_str("\\N"), + // Empty override block separates a literal slash from N/n/h. + '\\' => out.push_str("\\{}"), '{' => out.push('\u{ff5b}'), '}' => out.push('\u{ff5d}'), '\u{a0}' => out.push_str("\\h"), @@ -232,7 +237,7 @@ pub fn write(doc: &Document) -> String { ); for c in &doc.cues { // A comma in the Name field would shift the columns; the other fields are fixed text. - let name = c.speaker.as_deref().unwrap_or("").replace([',', '\n'], " "); + let name = c.speaker.as_deref().unwrap_or("").replace([',', '\n', '\r'], " "); out.push_str(&format!("Dialogue: 0,{},{},Default,{},0,0,0,,{}\n", format_time(c.start), format_time(c.end), name, escape(&c.text))); } out diff --git a/crates/captions/tests/it/ass.rs b/crates/captions/tests/it/ass.rs index de5d43167..fb6d4e01e 100644 --- a/crates/captions/tests/it/ass.rs +++ b/crates/captions/tests/it/ass.rs @@ -99,7 +99,8 @@ fn we_read_what_ffmpeg_writes() { fn ffmpeg_reads_what_we_write() { let Some(ff) = craft_media_testkit::ffmpeg() else { return }; let dir = craft_media_testkit::fixture_subdir("captions_ass"); - let doc = parse(SRT.as_bytes(), Format::Srt).expect("parse srt"); + let mut doc = parse(SRT.as_bytes(), Format::Srt).expect("parse srt"); + doc.cues.push(Cue { start: cs(400_000), end: cs(400_100), text: r"C:\New\notes\hello and \N \n \h".into(), ..Default::default() }); let (ass, srt) = (dir.join("ours.ass"), dir.join("back.srt")); std::fs::write(&ass, write(&doc, Format::Ass, WriteOptions::default())).expect("write"); ffmpeg_run(&ff, &["-i", ass.to_str().expect("path"), srt.to_str().expect("path")]); @@ -111,3 +112,31 @@ fn ffmpeg_reads_what_we_write() { assert_eq!(t.text, d.text, "text of the cue at {:?}", d.start); } } + +#[test] +fn literal_backslashes_survive_ass_round_trip() { + let text = r"C:\New\notes\hello and \N \n \h"; + let doc = Document { cues: vec![Cue { start: cs(100), end: cs(200), text: text.into(), ..Default::default() }], ..Default::default() }; + let bytes = write(&doc, Format::Ass, WriteOptions::default()); + assert_eq!(parse(&bytes, Format::Ass).unwrap().cues[0].text, text); +} + +#[test] +fn carriage_returns_cannot_create_dialogue_records() { + let doc = Document { + cues: vec![Cue { + start: cs(100), + end: cs(200), + text: "first\rsecond\r\nthird".into(), + speaker: Some("Ann\rOther".into()), + ..Default::default() + }], + ..Default::default() + }; + let bytes = write(&doc, Format::Ass, WriteOptions::default()); + assert!(!bytes.contains(&b'\r')); + let back = parse(&bytes, Format::Ass).unwrap(); + assert_eq!(back.cues.len(), 1); + assert_eq!(back.cues[0].text, "first\nsecond\nthird"); + assert_eq!(back.cues[0].speaker.as_deref(), Some("Ann Other")); +} From c9de09877ba38dba838ade37ba105032357709a9 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Sun, 15 Nov 2026 12:29:34 -0500 Subject: [PATCH 98/98] Share bounded BMP coverage inspection (#1610) --- crates/image-codecs/src/bmp_probe.rs | 145 +++++++++++++++++++++++++++ crates/image-codecs/src/lib.rs | 1 + 2 files changed, 146 insertions(+) create mode 100644 crates/image-codecs/src/bmp_probe.rs diff --git a/crates/image-codecs/src/bmp_probe.rs b/crates/image-codecs/src/bmp_probe.rs new file mode 100644 index 000000000..59c78ca1e --- /dev/null +++ b/crates/image-codecs/src/bmp_probe.rs @@ -0,0 +1,145 @@ +//! Header-only BMP dimensions and pixel-data coverage, without allocating a canvas. + +/// Coverage of the declared pixel array. Row byte counts include row padding; RLE coverage +/// counts the pixels drawn by commands and may be smaller than the canvas. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Coverage { + pub pixels: u64, + pub row_bytes: Option<(u64, u64)>, +} + +/// Inspect pixel data for the given dimensions without decoding a bitmap. The row tuple is +/// `(available, needed)`; callers choose their own sparse-canvas and allocation policy. +pub fn coverage(bytes: &[u8], w: u32, h: u32) -> Coverage { + match bmp_layout(bytes) { + BmpLayout::Rle { offset, rle4 } => Coverage { pixels: bmp_rle_pixels(bytes, offset, rle4, w, h), row_bytes: None }, + BmpLayout::Rows { offset, bits } => { + let needed = u64::from(w).saturating_mul(u64::from(bits)).div_ceil(32).saturating_mul(4).saturating_mul(u64::from(h)); + let held = bytes.len().saturating_sub(offset) as u64; + Coverage { pixels: u64::from(w) * u64::from(h), row_bytes: Some((held, needed)) } + } + } +} + +enum BmpLayout { + /// RLE8 or RLE4 commands. + Rle { offset: usize, rle4: bool }, + /// Rows of `bits` per pixel, each padded to 4 bytes. + Rows { offset: usize, bits: u16 }, +} + +/// The layout the decoder reads, from the file header and the bitmap header: a 12-byte core header +/// (bits per pixel at 24, never compressed) or a 40-byte or larger one (bits at 28, compression at +/// 30). Fields the decoder would have refused are read as 1 bit per pixel at offset 0, the most +/// lenient layout. +fn bmp_layout(bytes: &[u8]) -> BmpLayout { + let u16_at = |i: usize| bytes.get(i..i.checked_add(2)?).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes); + let u32_at = |i: usize| bytes.get(i..i.checked_add(4)?).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes); + let offset = u32_at(10).and_then(|o| usize::try_from(o).ok()).unwrap_or(0); + let (bits, compression) = match u32_at(14) { + Some(12) => (u16_at(24), Some(0)), + _ => (u16_at(28), u32_at(30)), + }; + match compression { + Some(1) => BmpLayout::Rle { offset, rle4: false }, + Some(2) => BmpLayout::Rle { offset, rle4: true }, + _ => BmpLayout::Rows { offset, bits: bits.filter(|&b| b > 0).unwrap_or(1) }, + } +} + +/// The pixels an RLE8 or RLE4 BMP's commands draw on a `w` × `h` canvas, read the way the decoder +/// paints them: a run is clipped at the end of its row, an end-of-row code moves to the next row, a +/// delta skips pixels (left blank), and an end-of-bitmap code, a truncated command or the last row +/// ends the picture. Every command takes at least 2 bytes, so the walk is linear in the file. +fn bmp_rle_pixels(bytes: &[u8], offset: usize, rle4: bool, w: u32, h: u32) -> u64 { + let (w, h) = (u64::from(w), u64::from(h)); + let (mut pos, mut x, mut y, mut covered) = (offset, 0u64, 0u64, 0u64); + while y < h { + let (Some(&count), Some(&value)) = (bytes.get(pos), bytes.get(pos.saturating_add(1))) else { + break; + }; + pos = pos.saturating_add(2); + match (count, value) { + // end of row + (0, 0) => (x, y) = (0, y.saturating_add(1)), + // end of bitmap + (0, 1) => break, + // delta: right and down + (0, 2) => { + let (Some(&dx), Some(&dy)) = (bytes.get(pos), bytes.get(pos.saturating_add(1))) else { + break; + }; + pos = pos.saturating_add(2); + (x, y) = (x.saturating_add(u64::from(dx)), y.saturating_add(u64::from(dy))); + } + // absolute: `value` pixels stored one per byte (RLE8) or nibble (RLE4), padded to 2 bytes + (0, n) => { + let len = if rle4 { usize::from(n).div_ceil(2) } else { usize::from(n) }; + pos = pos.saturating_add(len + (len & 1)); + if pos > bytes.len() { + break; + } + covered = covered.saturating_add(u64::from(n).min(w.saturating_sub(x))); + x = x.saturating_add(u64::from(n)); + } + // a run of `count` pixels + (n, _) => { + covered = covered.saturating_add(u64::from(n).min(w.saturating_sub(x))); + x = x.saturating_add(u64::from(n)); + } + } + } + covered +} + +/// Reads BMP `(width, height)` from the DIB header without allocating. +pub fn dimensions(bytes: &[u8]) -> Option<(u32, u32)> { + if bytes.get(0..2) != Some(b"BM") { + return None; + } + let dib = u32::from_le_bytes(bytes.get(14..18)?.try_into().ok()?); + let (w, h) = if dib == 12 { + (u16::from_le_bytes(bytes.get(18..20)?.try_into().ok()?) as u32, u16::from_le_bytes(bytes.get(20..22)?.try_into().ok()?) as u32) + } else { + let w = i32::from_le_bytes(bytes.get(18..22)?.try_into().ok()?); + let h = i32::from_le_bytes(bytes.get(22..26)?.try_into().ok()?); + (u32::try_from(w).ok()?, h.unsigned_abs()) + }; + Some((w, h)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bmp(compression: u32, pixels: &[u8]) -> Vec { + let mut b = vec![0; 54]; + b[..2].copy_from_slice(b"BM"); + b[10..14].copy_from_slice(&54u32.to_le_bytes()); + b[14..18].copy_from_slice(&40u32.to_le_bytes()); + b[18..22].copy_from_slice(&4i32.to_le_bytes()); + b[22..26].copy_from_slice(&2i32.to_le_bytes()); + b[28..30].copy_from_slice(&8u16.to_le_bytes()); + b[30..34].copy_from_slice(&compression.to_le_bytes()); + b.extend_from_slice(pixels); + b + } + + #[test] + fn dimensions_and_padded_rows() { + let b = bmp(0, &[0; 7]); + assert_eq!(dimensions(&b), Some((4, 2))); + assert_eq!(coverage(&b, 4, 2), Coverage { pixels: 8, row_bytes: Some((7, 8)) }); + assert_eq!(dimensions(&b[..20]), None); + } + + #[test] + fn rle_counts_clipped_runs_deltas_and_complete_absolute_blocks() { + let b = bmp(1, &[3, 7, 0, 0, 0, 2, 1, 0, 0, 3, 1, 2, 3, 0, 0, 1]); + assert_eq!(coverage(&b, 4, 2).pixels, 6); + let truncated = bmp(1, &[0, 4, 1, 2]); + assert_eq!(coverage(&truncated, 4, 2).pixels, 0); + let nibble = bmp(2, &[0, 3, 0x12, 0x30, 0, 1]); + assert_eq!(coverage(&nibble, 4, 2).pixels, 3); + } +} diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index 5a84e6368..adeed0825 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -16,6 +16,7 @@ mod allocation; pub mod apng; +pub mod bmp_probe; pub mod cicp; mod codecs; mod error;