Files
filmcraft/crates/codecs/src/hw_frame.rs
Dylan M. Taylor 504fe9fc5e Containers, MP3 encoder and video decoders come from craft-libs
Containers, MP3 encoder and video decoders come from craft-libs

* Containers, MP3 encoder and video decoders come from craft-libs

The AVI, MPEG-TS/PS, MP3 encoder, H.264, HEVC, VP9, AV1, MPEG-2 and APV
crates are removed. craft-media (avi, mpegts), craft-audio (mp3-encode)
and craft-video (h264-decode, hevc-decode, vp9-decode, av1-decode, mpeg2,
apv), which were cut from them, replace them. Every craft-libs crate is
pinned at craft-libs 50f21899.

* MXF comes from craft-libs; craft-libs at df469bb2

crates/mxf is removed for craft_media::mxf. Both writer entry points name
FilmCraft (company, product, version) as the old writer did, so the bytes
and the product UID are unchanged; the export and AAF tests check it.

* docs: the VP9 and AV1 hostile_lengths WASI run points at craft-video
2026-11-11 22:48:59 -05:00

308 lines
13 KiB
Rust

//! What a hardware decoder needs to know about a VP9 or AV1 stream so that it behaves exactly like
//! our software decoder for it (the frame-coded counterpart of [`crate::hw::NalStreamInfo`]):
//! codec profile, bit depth and chroma format from the container's `vpcC` / `av1C`, the picture
//! size and colour as the software decoder reports them (read from the bitstream the same way),
//! random-access samples, and in-band changes of the stream's parameters.
use filmcraft_color::{ColorInfo, Primaries, Transfer};
use filmcraft_isobmff::{CodecConfig, SampleEntry};
use crate::video::primaries_from_code;
use crate::{CodecError, Result};
/// The two frame-coded codecs hardware decoders take besides H.264 / HEVC.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum FrameCodec {
Vp9,
Av1,
}
/// Picture parameters read from a random-access sample.
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct PictureParams {
/// Picture size in luma samples (the decoded size).
pub size: (u32, u32),
pub bit_depth: u32,
/// `subsampling_x`, `subsampling_y`.
pub subsampling: (u8, u8),
/// Colour of every output picture, as the software decoder reports it.
pub color: ColorInfo,
}
/// A VP9 or AV1 stream, from its sample entry.
#[derive(Clone, Debug)]
pub struct FrameStreamInfo {
pub codec: FrameCodec,
/// VP9 `profile` (0..=3) / AV1 `seq_profile` (0..=2).
pub profile: u8,
pub bit_depth: u32,
pub subsampling: (u8, u8),
pub mono: bool,
/// Picture size: the sample entry's, checked against the bitstream's.
pub size: (u32, u32),
/// AV1: the `av1C` configuration OBUs (the sequence header the decoder is primed with).
pub config_obus: Vec<u8>,
/// AV1: the sequence header's payload; VP9: none.
sequence_header: Option<Vec<u8>>,
/// Colour of the stream when it is known before the first picture (AV1: from the sequence
/// header).
pub color: Option<ColorInfo>,
/// VP9: transfer and primaries from the container (not in the bitstream).
transfer: Option<Transfer>,
primaries: Option<Primaries>,
}
/// The OBUs of a low-overhead AV1 sample: `(type, payload)`, stopping at the first malformed one.
pub(crate) fn obus(data: &[u8]) -> Vec<(u8, &[u8])> {
let mut out = Vec::new();
let mut pos = 0usize;
while let Some(&h) = data.get(pos) {
let (obu_type, ext, has_size) = ((h >> 3) & 0xf, h & 4 != 0, h & 2 != 0);
let mut p = pos + 1 + usize::from(ext);
let size = if has_size {
let (mut v, mut shift, mut n) = (0usize, 0u32, 0usize);
loop {
let Some(&b) = data.get(p + n) else { return out };
v |= usize::from(b & 0x7f).checked_shl(shift).unwrap_or(0);
n += 1;
if b & 0x80 == 0 || n >= 8 {
break;
}
shift += 7;
}
p += n;
v
} else {
data.len().saturating_sub(p)
};
let Some(payload) = data.get(p..p.saturating_add(size)) else { return out };
out.push((obu_type, payload));
pos = p + size;
}
out
}
/// What an AV1 sequence header fixes about the pictures: profile, maximum size and the colour
/// configuration (bit depth, chroma format, colour description). `None` when it does not parse.
fn av1_format(seq: &[u8]) -> Option<(u8, u32, u32, craft_video::av1::decode::ColorConfig)> {
let h = craft_video::av1::decode::SequenceHeader::parse(seq).ok()?;
Some((h.profile, h.max_frame_width, h.max_frame_height, h.color))
}
/// VP9 colour as our decoder reports it: `color_space` and range from the bitstream, transfer and
/// primaries from the container when it says. (Shared with `Vp9Decoder`.)
pub(crate) fn vp9_color(width: u32, height: u32, color_space: u8, full_range: bool, transfer: Option<Transfer>, primaries: Option<Primaries>) -> ColorInfo {
use filmcraft_color::{Matrix, Range};
let mut color = ColorInfo { matrix: filmcraft_frame::default_matrix(width, height), ..ColorInfo::REC709 };
// color_space (7.2.2): 1 BT.601, 2 BT.709, 3 SMPTE-170, 4 SMPTE-240, 5 BT.2020, 7 sRGB.
match color_space {
1 | 3 => color.matrix = Matrix::Bt601,
2 | 4 => color.matrix = Matrix::Bt709,
5 => {
color.matrix = Matrix::Bt2020Ncl;
color.primaries = Primaries::Bt2020;
}
_ => {}
}
if let Some(t) = transfer {
color.transfer = t;
}
if let Some(p) = primaries {
color.primaries = p;
}
if full_range {
color.range = Range::Full;
}
color
}
/// AV1 colour as our decoder reports it. (Shared with `Av1Decoder`.)
pub(crate) fn av1_color(width: u32, height: u32, matrix_coefficients: u8, transfer: u8, primaries: u8, full_range: bool) -> ColorInfo {
let mut color = ColorInfo::REC709;
color.matrix = filmcraft_color::Matrix::from_code(matrix_coefficients).unwrap_or_else(|| filmcraft_frame::default_matrix(width, height));
if let Some(t) = Transfer::from_code(transfer) {
color.transfer = t;
}
color.primaries = match primaries {
9 => Primaries::Bt2020,
12 => Primaries::P3D65,
5 => Primaries::Bt601_625,
6 => Primaries::Bt601_525,
_ => Primaries::Bt709,
};
if full_range {
color.range = filmcraft_color::Range::Full;
}
color
}
impl FrameStreamInfo {
/// The stream info of a `vpcC` / `av1C` sample entry (`None` for other codecs).
pub fn from_entry(e: &SampleEntry) -> Option<Result<Self>> {
let size = e.video.as_ref().map_or((0, 0), |v| (u32::from(v.width), u32::from(v.height)));
match &e.codec {
CodecConfig::Vp9(c) => {
// vpcC chroma_subsampling: 0 / 1 4:2:0, 2 4:2:2, 3 4:4:4
let subsampling = match c.chroma_subsampling {
0 | 1 => (1, 1),
2 => (1, 0),
_ => (0, 0),
};
Some(Ok(Self {
codec: FrameCodec::Vp9,
profile: c.profile,
bit_depth: u32::from(c.bit_depth),
subsampling,
mono: false,
size,
config_obus: Vec::new(),
sequence_header: None,
color: None,
transfer: Transfer::from_code(c.transfer_characteristics),
primaries: primaries_from_code(c.colour_primaries),
}))
}
CodecConfig::Av1(c) => Some(Self::from_av1(c, size)),
_ => None,
}
}
fn from_av1(c: &filmcraft_isobmff::Av1Config, entry_size: (u32, u32)) -> Result<Self> {
let seq =
obus(&c.config_obus).into_iter().find(|(t, _)| *t == 1).map(|(_, p)| p).ok_or_else(|| CodecError::Decode("av1C has no sequence header".into()))?;
let h = craft_video::av1::decode::SequenceHeader::parse(seq).map_err(|e| CodecError::Decode(e.to_string()))?;
let size = (h.max_frame_width, h.max_frame_height);
if entry_size != (0, 0) && entry_size != size {
// the sample entry and the sequence header disagree: leave the stream to our decoder
return Err(CodecError::Decode(format!("sample entry says {}x{}, the sequence header {}x{}", entry_size.0, entry_size.1, size.0, size.1)));
}
let cc = &h.color;
Ok(Self {
codec: FrameCodec::Av1,
profile: h.profile,
bit_depth: u32::from(cc.bit_depth),
subsampling: (cc.subsampling_x, cc.subsampling_y),
mono: cc.mono_chrome,
size,
config_obus: c.config_obus.clone(),
sequence_header: Some(seq.to_vec()),
color: Some(av1_color(size.0, size.1, cc.matrix_coefficients, cc.transfer_characteristics, cc.color_primaries, cc.color_range)),
transfer: None,
primaries: None,
})
}
/// Whether `sample` has a sequence header OBU of its own (AV1).
pub fn carries_sequence_header(&self, sample: &[u8]) -> bool {
self.codec == FrameCodec::Av1 && obus(sample).iter().any(|(t, _)| *t == 1)
}
/// A description made by hand (tests of what a backend declines).
#[doc(hidden)]
pub fn for_tests(codec: FrameCodec, profile: u8, bit_depth: u32, subsampling: (u8, u8), size: (u32, u32)) -> Self {
Self {
codec,
profile,
bit_depth,
subsampling,
mono: false,
size,
config_obus: Vec::new(),
sequence_header: None,
color: None,
transfer: None,
primaries: None,
}
}
/// Whether decoding can start at `sample` (a VP9 key frame / an AV1 temporal unit whose first
/// frame is a shown key frame): the software decoders' own test.
pub fn is_random_access(&self, sample: &[u8]) -> bool {
match self.codec {
FrameCodec::Vp9 => craft_video::vp9::decode::is_keyframe(sample),
FrameCodec::Av1 => craft_video::av1::decode::is_key_frame_unit(sample),
}
}
/// The picture parameters a random-access `sample` declares (a VP9 key frame header; an AV1
/// sample carrying a sequence header), or `None`.
pub fn picture_params(&self, sample: &[u8]) -> Option<PictureParams> {
match self.codec {
FrameCodec::Vp9 => {
let k = craft_video::vp9::decode::keyframe_info(sample)?;
let color = vp9_color(k.width, k.height, k.color.color_space, k.color.full_range, self.transfer, self.primaries);
Some(PictureParams {
size: (k.width, k.height),
bit_depth: k.bit_depth,
subsampling: (u8::from(k.subsampling_x), u8::from(k.subsampling_y)),
color,
})
}
FrameCodec::Av1 => {
let seq = obus(sample).into_iter().find(|(t, _)| *t == 1).map(|(_, p)| p)?;
let h = craft_video::av1::decode::SequenceHeader::parse(seq).ok()?;
let cc = &h.color;
let size = (h.max_frame_width, h.max_frame_height);
let color = av1_color(size.0, size.1, cc.matrix_coefficients, cc.transfer_characteristics, cc.color_primaries, cc.color_range);
Some(PictureParams { size, bit_depth: u32::from(cc.bit_depth), subsampling: (cc.subsampling_x, cc.subsampling_y), color })
}
}
}
/// Whether `sample` declares parameters the hardware session was not set up for: an AV1
/// sequence header whose picture format (profile, size, colour configuration) differs from
/// `av1C`'s, a VP9 key frame of another size, bit depth or chroma format. The decoder then
/// hands the stream to our software decoder.
pub fn parameters_changed(&self, sample: &[u8]) -> bool {
match self.codec {
// Only what the picture format depends on counts: encoders such as SVT-AV1 put a
// provisional sequence header in `av1C` whose coding tool flags (CDEF, restoration,
// warped motion…) differ from the in-band one, and decoders follow the in-band one.
FrameCodec::Av1 => obus(sample).into_iter().any(|(t, p)| t == 1 && self.sequence_header.as_deref().is_none_or(|s| av1_format(s) != av1_format(p))),
FrameCodec::Vp9 => {
self.picture_params(sample).is_some_and(|p| p.size != self.size || p.bit_depth != self.bit_depth || p.subsampling != self.subsampling)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// An `av1C` sequence header from SVT-AV1 (1280x720 10-bit) and the in-band one of the same
/// stream: they differ only in coding tool flags.
const AV1C: [u8; 14] = [0x0a, 0x0c, 0x02, 0x00, 0x00, 0x2d, 0x6a, 0x67, 0xfd, 0x9e, 0x01, 0x7c, 0x20, 0x20];
const IN_BAND: [u8; 14] = [0x0a, 0x0c, 0x02, 0x00, 0x00, 0x2d, 0x6a, 0x67, 0xfd, 0x9e, 0x35, 0x7c, 0xe0, 0x20];
fn info() -> FrameStreamInfo {
let c = filmcraft_isobmff::Av1Config {
seq_profile: 0,
seq_level_idx_0: 0,
seq_tier_0: false,
high_bitdepth: true,
twelve_bit: false,
monochrome: false,
chroma_subsampling_x: true,
chroma_subsampling_y: true,
chroma_sample_position: 0,
initial_presentation_delay_minus_one: None,
config_obus: AV1C.to_vec(),
};
FrameStreamInfo::from_av1(&c, (0, 0)).unwrap()
}
#[test]
fn av1_sequence_headers_compare_by_picture_format() {
let i = info();
assert_eq!((i.size, i.bit_depth), ((1280, 720), 10));
assert!(!i.parameters_changed(&AV1C), "the same header");
// used to send every SVT-AV1 MP4 to the software decoder from its first frame
assert!(!i.parameters_changed(&IN_BAND), "only coding tool flags differ");
// a header that does not parse is a change
assert!(i.parameters_changed(&[0x0a, 0x02, 0xe0, 0x00]));
// no sequence header in the sample: nothing changed
assert!(!i.parameters_changed(&[0x12, 0x00]));
}
}