From ce31f42e76a2605183385eb74e5baed2c4ef667f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:25:21 -0500 Subject: [PATCH] craft-media: QuickTime logs box, Matroska Colour in the muxer, laced remux test (#840) * craft-media: read and write the QuickTime logs box A video sample entry's `logs` child holds the CoreMedia log transfer-function identifier (`com.apple.rec2020.apple-log` for Apple Log). It is kept as `VideoParams::log_transfer_function`, whatever curve it names. The reader takes non-empty UTF-8 without NUL of at most 255 bytes; anything else, or a second `logs` box, reads as unknown. The writer refuses an identifier the reader would not accept. Ported from storytold/filmcraft#1764. * craft-media: the Matroska muxer writes a video track's Colour element `TrackSpec::color` takes the reader's `Color` and the muxer writes the fields that are set: matrix, range, transfer, primaries, chroma layout, light levels and mastering metadata (RFC 9559). VP9's own colour syntax carries only the matrix and range, so without the element a Rec. 2020, P3, PQ or HLG WebM reads as BT.709. Ported from storytold/effectcraft#2926, which wrote the element by hand in EffectCraft's WebM export. * craft-media: the laced remux test laces ffmpeg's Vorbis fixture again ffmpeg's Vorbis files declare a short DefaultDuration and store longer durations per block, so with the track's own default nothing laced and the test failed ("vorbis.mkv Xiph: nothing was laced", ffmpeg 9.0.2). Audio tracks now take their median packet duration as the default, and frames are written with their explicit durations, so a short last packet ends its lace. Ported from storytold/filmcraft#3622 (test-only part). --------- Co-authored-by: 20Coder <51251284+BitL8-ByteShort@users.noreply.github.com> Co-authored-by: JonRC Co-authored-by: Michael Butler --- crates/media/README.md | 8 +- crates/media/src/isobmff/codec.rs | 79 +++++++++++++++++++ crates/media/src/matroska/mux.rs | 105 ++++++++++++++++++++++++-- crates/media/tests/oracle_matroska.rs | 23 ++++-- 4 files changed, 202 insertions(+), 13 deletions(-) diff --git a/crates/media/README.md b/crates/media/README.md index 398ba277..c5c8ab2d 100644 --- a/crates/media/README.md +++ b/crates/media/README.md @@ -198,7 +198,10 @@ Boxes with 64-bit sizes and with size 0 (runs to the end of the file) are suppor | anything else | `Unknown { fourcc, raw }` | Video entries also parse `colr` (`nclx`/`nclc`/ICC), `pasp`, `clap`, `fiel`, `gama` and `btrt` into -`VideoParams`. Audio entries parse QuickTime sound description v0/v1/v2 into `AudioParams`. +`VideoParams`, and QuickTime's `logs` box into `VideoParams::log_transfer_function`: the CoreMedia log +transfer-function identifier (`com.apple.rec2020.apple-log` for Apple Log), kept as text whatever +curve it names. It is non-empty UTF-8 without NUL of at most 255 bytes; anything else, or a second +`logs` box, reads as unknown, and the writer refuses an identifier the reader would not accept. Audio entries parse QuickTime sound description v0/v1/v2 into `AudioParams`. #### Robustness The parser does not panic on malformed input; every error comes back as an `Error`. Table counts are @@ -409,6 +412,9 @@ after `cluster_duration_ns`), BlockGroups for frames whose duration differs from `default_duration_ns`, optional audio lacing (Xiph, EBML, fixed-size; for tracks with a `default_duration_ns`, at most 256 frames, ending at a timestamp gap), and Cues (video keyframes, or the first frame of each track per cluster). `MuxOptions::doc_type` = `"webm"` writes WebM. +`TrackSpec::color` writes a video track's `Colour` element (matrix, range, transfer, primaries, +chroma layout, light levels and mastering metadata: the fields that are set), which is where a WebM +file says it is Rec. 2020, P3, PQ or HLG. CodecPrivate is fixed when the writer is created, so a codec whose configuration arrives with the first frame (AV1 from `rav1e`) creates the writer after that frame. diff --git a/crates/media/src/isobmff/codec.rs b/crates/media/src/isobmff/codec.rs index 2a9f3fa9..302d7add 100644 --- a/crates/media/src/isobmff/codec.rs +++ b/crates/media/src/isobmff/codec.rs @@ -36,6 +36,10 @@ pub struct VideoParams { pub temporal_quality: u32, pub spatial_quality: u32, pub color: Option, + /// QuickTime `logs`: the CoreMedia log transfer-function identifier (for example + /// `com.apple.rec2020.apple-log`). Non-empty UTF-8 without NUL, at most 255 bytes; anything + /// else, or a second `logs` box, reads as `None`. + pub log_transfer_function: Option, /// `pasp`: (hSpacing, vSpacing). pub pixel_aspect: Option<(u32, u32)>, pub clean_aperture: Option, @@ -1013,6 +1017,7 @@ pub(crate) fn parse_sample_entry(format: FourCc, payload: &[u8], handler: FourCc let _ctab = c.i16()?; let mut codec = None; let mut bitrate = None; + let mut log_seen = false; for b in boxes(c.rest()).map_while(|b| b.ok()) { let p = b.payload; match &b.kind.0 { @@ -1022,6 +1027,12 @@ pub(crate) fn parse_sample_entry(format: FourCc, payload: &[u8], handler: FourCc b"vpcC" if f == b"vp09" => codec = Some(CodecConfig::Vp9(VpcConfig::parse(p)?)), b"apvC" if f == b"apv1" => codec = Some(CodecConfig::Apv(ApvConfig::parse(p)?)), b"colr" => v.color = parse_colr(p).ok(), + b"logs" => { + // Two identifiers are ambiguous: neither is believed. + v.log_transfer_function = + if log_seen { None } else { std::str::from_utf8(p).ok().filter(|s| valid_log_identifier(s)).map(str::to_owned) }; + log_seen = true; + } b"mdcv" => v.mastering_display = MasteringDisplay::parse(p), b"clli" if p.len() >= 4 => v.content_light = Some((u16::from_be_bytes([p[0], p[1]]), u16::from_be_bytes([p[2], p[3]]))), b"pasp" => { @@ -1419,6 +1430,10 @@ impl SampleEntry { /// Serialize as a complete sample entry box. For PCM the format is chosen from the brand. pub(crate) fn write(&self, b: &mut BoxBuf, qt: bool) -> Result<()> { + // An identifier the reader would not accept is refused, not truncated. + if self.video.as_ref().and_then(|v| v.log_transfer_function.as_deref()).is_some_and(|s| !valid_log_identifier(s)) { + return Err(Error::Invalid("invalid log transfer-function identifier".into())); + } if let CodecConfig::Unknown { fourcc, raw } = &self.codec { let m = b.start(&fourcc.0); b.bytes(raw); @@ -1604,6 +1619,11 @@ fn write_video_codec_config(b: &mut BoxBuf, codec: &CodecConfig) { } } +/// What a `logs` box may hold: a non-empty identifier of at most 255 bytes with no NUL. +fn valid_log_identifier(s: &str) -> bool { + !s.is_empty() && s.len() <= 255 && !s.as_bytes().contains(&0) +} + fn write_video_descriptive_boxes(b: &mut BoxBuf, v: &VideoParams) { if let Some(gamma) = v.gamma { b.leaf(b"gama", &gamma.to_be_bytes()); @@ -1614,6 +1634,9 @@ fn write_video_descriptive_boxes(b: &mut BoxBuf, v: &VideoParams) { if let Some(color) = &v.color { write_color_box(b, color); } + if let Some(log) = &v.log_transfer_function { + b.leaf(b"logs", log.as_bytes()); + } if let Some(display) = &v.mastering_display { b.leaf(b"mdcv", &display.to_bytes()); } @@ -1693,6 +1716,62 @@ mod tests { parse_sample_entry(bx.kind, bx.payload, FourCc(*handler), qt).unwrap() } + /// A QuickTime `logs` box keeps its identifier whatever curve it names; the reader does not + /// guess what an unknown one means. + #[test] + fn log_transfer_function_roundtrip() { + for id in ["com.apple.rec2020.apple-log", "com.apple.rec2020.apple-log2", "another.vendor.unknown-curve"] { + let mut e = SampleEntry::prores(FourCc(*b"apch"), 64, 64); + e.video.as_mut().unwrap().log_transfer_function = Some(id.into()); + let back = roundtrip(&e, true, b"vide"); + assert_eq!(back.video.as_ref().unwrap().log_transfer_function.as_deref(), Some(id)); + assert_eq!(back, e); + } + // Apple Log is a 35-byte box: the 27 identifier bytes with no terminator. + let mut e = SampleEntry::prores(FourCc(*b"apch"), 64, 64); + e.video.as_mut().unwrap().log_transfer_function = Some("com.apple.rec2020.apple-log".into()); + let mut b = BoxBuf::new(); + e.write(&mut b, true).unwrap(); + let logs = [&35u32.to_be_bytes()[..], b"logscom.apple.rec2020.apple-log"].concat(); + assert!(b.buf.windows(logs.len()).any(|w| w == logs)); + } + + /// The writer refuses an identifier the reader would not accept; it is never truncated. + #[test] + fn invalid_log_transfer_functions_are_not_written() { + for id in [String::new(), "a".repeat(256), "com.apple.rec2020.apple-log\0".into()] { + let mut e = SampleEntry::prores(FourCc(*b"apch"), 64, 64); + e.video.as_mut().unwrap().log_transfer_function = Some(id); + assert!(e.write(&mut BoxBuf::new(), true).is_err()); + } + let mut e = SampleEntry::prores(FourCc(*b"apch"), 64, 64); + e.video.as_mut().unwrap().log_transfer_function = Some("a".repeat(255)); + assert!(e.write(&mut BoxBuf::new(), true).is_ok()); + } + + /// An empty, oversized, non-UTF-8 or NUL-holding `logs` box, or two of them, reads as unknown. + #[test] + fn malformed_or_duplicate_logs_boxes_read_as_unknown() { + let read = |children: &[&[u8]]| { + let mut b = BoxBuf::new(); + SampleEntry::prores(FourCc(*b"apch"), 64, 64).write(&mut b, true).unwrap(); + for payload in children { + b.leaf(b"logs", payload); + } + let len = (b.buf.len() as u32).to_be_bytes(); + b.buf[..4].copy_from_slice(&len); + let bx = crate::isobmff::bytes::boxes(&b.buf).next().unwrap().unwrap(); + parse_sample_entry(bx.kind, bx.payload, FourCc(*b"vide"), true).unwrap().video.unwrap().log_transfer_function + }; + assert_eq!(read(&[b"vendor.curve"]).as_deref(), Some("vendor.curve")); + assert_eq!(read(&[b""]), None); + assert_eq!(read(&[&[b'a'; 256]]), None); + assert_eq!(read(&[b"bad\xFFutf8"]), None); + assert_eq!(read(&[b"nul\0inside"]), None); + assert_eq!(read(&[b"vendor.curve", b"vendor.curve"]), None); + assert_eq!(read(&[b"vendor.curve", b""]), None); + } + #[test] fn avcc_roundtrip() { let cfg = AvcConfig::new(vec![vec![0x67, 0x64, 0x00, 0x1F, 0xAC]], vec![vec![0x68, 0xEB]], 4); diff --git a/crates/media/src/matroska/mux.rs b/crates/media/src/matroska/mux.rs index 7d26a8f1..b0ae893d 100644 --- a/crates/media/src/matroska/mux.rs +++ b/crates/media/src/matroska/mux.rs @@ -7,13 +7,16 @@ use std::io::{Seek, SeekFrom, Write}; use crate::matroska::ebml::{write_id, write_size, write_unknown_size}; use crate::matroska::error::{Error, Result}; use crate::matroska::ids::{ - AUDIO, BIT_DEPTH, BLOCK, BLOCK_DURATION, BLOCK_GROUP, CHANNELS, CLUSTER, CODEC_DELAY, CODEC_ID, CODEC_PRIVATE, CUE_CLUSTER_POSITION, CUE_POINT, - CUE_TIME, CUE_TRACK, CUE_TRACK_POSITIONS, CUES, DEFAULT_DURATION, DOC_TYPE, DOC_TYPE_READ_VERSION, DOC_TYPE_VERSION, DURATION, EBML, - EBML_MAX_ID_LENGTH, EBML_MAX_SIZE_LENGTH, EBML_READ_VERSION, EBML_VERSION, FLAG_LACING, INFO, LANGUAGE, MUXING_APP, NAME, PIXEL_HEIGHT, - PIXEL_WIDTH, REFERENCE_BLOCK, SAMPLING_FREQUENCY, SEEK, SEEK_HEAD, SEEK_ID, SEEK_POSITION, SEEK_PRE_ROLL, SEGMENT, SIMPLE_BLOCK, TIMESTAMP, - TIMESTAMP_SCALE, TITLE, TRACK_ENTRY, TRACK_NUMBER, TRACK_TYPE, TRACK_UID, TRACKS, VIDEO, VOID, WRITING_APP, + AUDIO, BIT_DEPTH, BITS_PER_CHANNEL, BLOCK, BLOCK_DURATION, BLOCK_GROUP, CB_SUBSAMPLING_HORZ, CB_SUBSAMPLING_VERT, CHANNELS, CHROMA_SITING_HORZ, + CHROMA_SITING_VERT, CHROMA_SUBSAMPLING_HORZ, CHROMA_SUBSAMPLING_VERT, CLUSTER, CODEC_DELAY, CODEC_ID, CODEC_PRIVATE, COLOUR, + CUE_CLUSTER_POSITION, CUE_POINT, CUE_TIME, CUE_TRACK, CUE_TRACK_POSITIONS, CUES, DEFAULT_DURATION, DOC_TYPE, DOC_TYPE_READ_VERSION, + DOC_TYPE_VERSION, DURATION, EBML, EBML_MAX_ID_LENGTH, EBML_MAX_SIZE_LENGTH, EBML_READ_VERSION, EBML_VERSION, FLAG_LACING, INFO, LANGUAGE, + LUMINANCE_MAX, LUMINANCE_MIN, MASTERING_METADATA, MATRIX_COEFFICIENTS, MAX_CLL, MAX_FALL, MUXING_APP, NAME, PIXEL_HEIGHT, PIXEL_WIDTH, PRIMARIES, + PRIMARY_B_X, PRIMARY_B_Y, PRIMARY_G_X, PRIMARY_G_Y, PRIMARY_R_X, PRIMARY_R_Y, RANGE, REFERENCE_BLOCK, SAMPLING_FREQUENCY, SEEK, SEEK_HEAD, + SEEK_ID, SEEK_POSITION, SEEK_PRE_ROLL, SEGMENT, SIMPLE_BLOCK, TIMESTAMP, TIMESTAMP_SCALE, TITLE, TRACK_ENTRY, TRACK_NUMBER, TRACK_TYPE, + TRACK_UID, TRACKS, TRANSFER_CHARACTERISTICS, VIDEO, VOID, WHITE_POINT_X, WHITE_POINT_Y, WRITING_APP, }; -use crate::matroska::track::TrackKind; +use crate::matroska::track::{Color, TrackKind}; /// Lacing used for audio tracks. Only frames of a track with a DefaultDuration lace, and a lace /// ends where the timestamps stop advancing by that duration. @@ -37,6 +40,10 @@ pub struct TrackSpec { pub name: Option, /// Pixel width/height for video tracks. pub video_size: Option<(u32, u32)>, + /// The video track's `Colour` element (RFC 9559): the fields that are set are written. VP9's + /// own colour syntax carries only the matrix and range, so without it a Rec. 2020, P3, PQ or + /// HLG stream reads as BT.709. Ignored without a `video_size`. + pub color: Option, /// (sampling frequency, channels, bit depth) for audio tracks. pub audio: Option<(f64, u64, Option)>, pub codec_delay_ns: u64, @@ -53,6 +60,7 @@ impl TrackSpec { language: None, name: None, video_size: None, + color: None, audio: None, codec_delay_ns: 0, seek_pre_roll_ns: 0, @@ -106,6 +114,50 @@ pub(crate) fn el_float(out: &mut Vec, id: u32, v: f64) { el(out, id, &v.to_be_bytes()); } +/// The body of a `Colour` element: every field of `c` that is set. +fn colour(c: &Color) -> Vec { + let mut out = Vec::new(); + for (id, v) in [ + (MATRIX_COEFFICIENTS, c.matrix_coefficients), + (BITS_PER_CHANNEL, c.bits_per_channel), + (CHROMA_SUBSAMPLING_HORZ, c.chroma_subsampling_horz), + (CHROMA_SUBSAMPLING_VERT, c.chroma_subsampling_vert), + (CB_SUBSAMPLING_HORZ, c.cb_subsampling_horz), + (CB_SUBSAMPLING_VERT, c.cb_subsampling_vert), + (CHROMA_SITING_HORZ, c.chroma_siting_horz), + (CHROMA_SITING_VERT, c.chroma_siting_vert), + (RANGE, c.range), + (TRANSFER_CHARACTERISTICS, c.transfer_characteristics), + (PRIMARIES, c.primaries), + (MAX_CLL, c.max_cll), + (MAX_FALL, c.max_fall), + ] { + if let Some(v) = v { + el_uint(&mut out, id, v); + } + } + if let Some(m) = &c.mastering { + let [r, g, b] = m.primaries; + let mut md = Vec::new(); + for (id, v) in [ + (PRIMARY_R_X, r.0), + (PRIMARY_R_Y, r.1), + (PRIMARY_G_X, g.0), + (PRIMARY_G_Y, g.1), + (PRIMARY_B_X, b.0), + (PRIMARY_B_Y, b.1), + (WHITE_POINT_X, m.white_point.0), + (WHITE_POINT_Y, m.white_point.1), + (LUMINANCE_MAX, m.luminance_max), + (LUMINANCE_MIN, m.luminance_min), + ] { + el_float(&mut md, id, v); + } + el(&mut out, MASTERING_METADATA, &md); + } + out +} + pub(crate) fn el_str(out: &mut Vec, id: u32, s: &str) { el(out, id, s.as_bytes()); } @@ -233,6 +285,9 @@ impl MkvWriter { let mut v = Vec::new(); el_uint(&mut v, PIXEL_WIDTH, pw as u64); el_uint(&mut v, PIXEL_HEIGHT, ph as u64); + if let Some(c) = &t.color { + el(&mut v, COLOUR, &colour(c)); + } el(&mut e, VIDEO, &v); } if let Some((rate, ch, bits)) = t.audio { @@ -602,6 +657,44 @@ mod tests { } } + /// A video track's `Colour` element reads back as it was given, and is absent when not set. + #[test] + fn colour_element_roundtrip() { + use crate::matroska::track::MasteringMetadata; + let write = |color: Option| { + let mut v = TrackSpec::new(TrackKind::Video, "V_VP9"); + v.video_size = Some((16, 16)); + v.color = color; + let mut m = MkvWriter::new(Cursor::new(Vec::new()), vec![v], MuxOptions::default()).unwrap(); + m.write_frame(0, 0, true, &[0; 4], None).unwrap(); + let bytes = m.finish().unwrap().into_inner(); + crate::matroska::open(&bytes[..]).unwrap().tracks[0].video.clone().unwrap().color + }; + assert_eq!(write(None), None); + // Rec. 2020 PQ, broadcast range: the four fields a render knows + let pq = Color { matrix_coefficients: Some(9), range: Some(1), transfer_characteristics: Some(16), primaries: Some(9), ..Default::default() }; + assert_eq!(write(Some(pq.clone())), Some(pq.clone())); + let full = Color { + bits_per_channel: Some(10), + chroma_subsampling_horz: Some(1), + chroma_subsampling_vert: Some(1), + cb_subsampling_horz: Some(0), + cb_subsampling_vert: Some(0), + chroma_siting_horz: Some(1), + chroma_siting_vert: Some(2), + max_cll: Some(1000), + max_fall: Some(400), + mastering: Some(MasteringMetadata { + primaries: [(0.708, 0.292), (0.17, 0.797), (0.131, 0.046)], + white_point: (0.3127, 0.329), + luminance_max: 1000.0, + luminance_min: 0.0001, + }), + ..pq + }; + assert_eq!(write(Some(full.clone())), Some(full)); + } + #[test] fn block_duration_rounds_to_the_nearest_tick() { let mut v = TrackSpec::new(TrackKind::Video, "V_MJPEG"); diff --git a/crates/media/tests/oracle_matroska.rs b/crates/media/tests/oracle_matroska.rs index d4da588e..19f851f3 100644 --- a/crates/media/tests/oracle_matroska.rs +++ b/crates/media/tests/oracle_matroska.rs @@ -324,11 +324,20 @@ fn laced_remux() { .map(|(ti, t)| { let mut s = TrackSpec::new(t.kind, t.codec_id.clone()); s.codec_private = t.codec_private.clone(); - // audio without a DefaultDuration gets its mean packet spacing, so it can lace - s.default_duration_ns = t.default_duration_ns.or_else(|| { - let pts: Vec = packets.iter().filter(|p| p.track == ti).map(|p| p.pts_ns).collect(); - (t.kind == TrackKind::Audio && pts.len() > 1).then(|| ((pts[pts.len() - 1] - pts[0]) / (pts.len() as i64 - 1)) as u64) - }); + // BlockDuration overrides DefaultDuration: ffmpeg's Vorbis files declare a short + // default and store longer durations per block. Audio takes its median packet + // duration (the packet spacing when no packet has one), so its frames can lace. + s.default_duration_ns = if t.kind == TrackKind::Audio { + let audio: Vec<&Packet> = packets.iter().filter(|p| p.track == ti).collect(); + let mut durations: Vec = audio.iter().map(|p| p.duration_ns).filter(|&d| d > 0).collect(); + if durations.is_empty() { + durations = audio.windows(2).filter_map(|w| u64::try_from(w[1].pts_ns - w[0].pts_ns).ok()).filter(|&d| d > 0).collect(); + } + durations.sort_unstable(); + durations.get(durations.len() / 2).copied().or(t.default_duration_ns) + } else { + t.default_duration_ns + }; s.video_size = t.video.as_ref().map(|v| (v.pixel_width, v.pixel_height)); s.audio = t.audio.as_ref().map(|a| (a.sampling_frequency, a.channels, a.bit_depth)); s.codec_delay_ns = t.codec_delay_ns; @@ -338,7 +347,9 @@ fn laced_remux() { let opts = MuxOptions { lacing: mode, ..Default::default() }; let mut w = MkvWriter::new(std::io::Cursor::new(Vec::new()), specs, opts).unwrap(); for p in &packets { - w.write_frame(p.track, p.pts_ns, p.keyframe, &p.data, None).unwrap(); + // Keep explicit durations, a short last packet's too: one that differs from the + // track's default ends the lace. + w.write_frame(p.track, p.pts_ns, p.keyframe, &p.data, (p.duration_ns > 0).then_some(p.duration_ns)).unwrap(); } let out = w.finish().unwrap().into_inner(); let laced = {