diff --git a/Cargo.lock b/Cargo.lock index 79691772a..feec44ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1483,6 +1483,7 @@ dependencies = [ "effectcraft-keyframe", "effectcraft-project", "effectcraft-time", + "log", "serde", "serde_json", "thiserror 2.0.21", diff --git a/crates/aep/Cargo.toml b/crates/aep/Cargo.toml index b1bf3f9fe..3deb09de2 100644 --- a/crates/aep/Cargo.toml +++ b/crates/aep/Cargo.toml @@ -15,6 +15,7 @@ effectcraft-color = { workspace = true } effectcraft-keyframe = { workspace = true } effectcraft-project = { workspace = true } effectcraft-effects = { workspace = true } +log = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } diff --git a/crates/aep/README.md b/crates/aep/README.md index cd963beec..39c8dd448 100644 --- a/crates/aep/README.md +++ b/crates/aep/README.md @@ -21,7 +21,9 @@ tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, ar The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded XML tokenizer, no dependency); `parse_any` picks the reader by content, so the typed view and the -importer take either form. The mapping is in `docs/aep-format.md`, section 1. +importer take either form. The mapping is in `docs/aep-format.md`, section 1. The importer, which +only reads, does not build the owned tree: it reads either form through the flat index +`TreeIndex`, an `.aep` in place and an `.aepx` decoded into one buffer first. Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs index 77511fcf1..fe99f10b1 100644 --- a/crates/aep/src/container.rs +++ b/crates/aep/src/container.rs @@ -23,6 +23,7 @@ mod effects; mod error; mod footage; mod header; +pub mod index; mod keys; mod masks; mod matte; @@ -36,6 +37,7 @@ mod values; mod version; mod write; +pub(crate) use aepx::{AepxDrafts, index_aepx}; pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; pub use arbitrary::{ CURVE_IDENTITY, CryptomatteData, CurvesData, ExtractorChannels, HueSaturationData, LevelsData, MeshData, curve_lut, identifier_channel_type, @@ -46,6 +48,7 @@ pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, Laye pub use error::AepError; pub use footage::{Footage, PsdLayer, SequenceInfo}; pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use index::{Node, NodeWalk, Nodes, TreeIndex}; pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; pub use matte::{MatteMode, TrackMatte}; diff --git a/crates/aep/src/container/aepx.rs b/crates/aep/src/container/aepx.rs index a45cbae1d..c4d2865ff 100644 --- a/crates/aep/src/container/aepx.rs +++ b/crates/aep/src/container/aepx.rs @@ -12,13 +12,23 @@ //! instructions. A document type declaration is refused (no entity expansion). It runs without //! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything //! it allocates is bounded by the size of the input. +//! +//! There are two readers. [`read_aepx`] builds the owned tree element by element +//! (`aepx/tree.rs`). The importer, which only reads, uses `index_aepx`: it decodes every body +//! (hex, text, the alias JSON, the numbers) into one buffer and records the chunks as entries of +//! the flat index ([`Draft`]s, in depth-first order, bodies as ranges of the buffer), which +//! [`crate::container::TreeIndex::from_drafts`] turns into an index that borrows the buffer. +//! The tests hold both to the same tree and the same errors. use std::borrow::Cow; -use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::chunk::{AepFile, LIST, NESTED_IDS}; use crate::container::error::AepError; +use crate::container::index::{Draft, Kind as EntryKind}; use crate::container::read::{Limits, parse}; +mod tree; + /// The root element of an `.aepx`. const ROOT: &[u8] = b"AfterEffectsProject"; /// The element that holds the XMP packet (the trailer of an `.aep`). @@ -68,17 +78,59 @@ pub fn read_aepx(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`read_aepx`]. pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + tree::read(data, limits) +} + +/// An `.aepx` read for the flat index: every body decoded into one buffer, and the chunks as +/// [`Draft`]s in depth-first order with their bodies as ranges of that buffer. +/// [`crate::container::TreeIndex::from_drafts`] makes the index, which borrows the buffer. +pub(crate) struct AepxDrafts { + /// The decoded bodies (hex, text, alias JSON, numbers) and the XMP packet. + pub(crate) buffer: Vec, + pub(crate) nodes: Vec, + /// The XMP packet: a range of `buffer` (empty without one). + pub(crate) trailer: (usize, usize), +} + +/// Reads an `.aepx` into [`AepxDrafts`]: the tree [`read_aepx_with_limits`] gives, with the same +/// errors, without one allocation per chunk. +/// +/// # Errors +/// See [`read_aepx_with_limits`]. When memory runs out, two errors come up in more cases than +/// there: [`AepError::TooLarge`] when the chunk list cannot grow (the owned-tree reader would +/// abort), and [`AepError::Xml`] "bdata is too large to hold in memory" also when the shared body +/// buffer cannot grow (the owned-tree reader reserves each value on its own). +pub(crate) fn index_aepx(data: &[u8], limits: Limits) -> Result { let text = match std::str::from_utf8(data) { Ok(t) => t.as_bytes(), Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), }; - Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() + // Every counted element takes at least 3 bytes (`` in a document cut short), so there are + // never more chunks than this. + let bound = (text.len() / 3).min(limits.max_chunks); + let mut nodes = Vec::new(); + let _ = nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + // Hex is most of an `.aepx` and decodes to half its length: one buffer of that size usually + // holds everything. A failed reservation is not an error here; the buffer grows as it goes. + let mut buffer = Vec::new(); + let _ = buffer.try_reserve_exact(text.len() / 2); + let reader = Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), root_closed: false, trailer: None, buffer, nodes, bound, attrs: Vec::new() }; + reader.run() } +/// The slots the chunk list starts with; it doubles from there (as the array of +/// [`crate::container::TreeIndex`] does). +const INITIAL_SLOTS: usize = 1024; + fn xml(offset: usize, what: &'static str) -> AepError { AepError::Xml { offset, what } } +/// The pad byte of a body of `len` bytes, as [`crate::container::Chunk::data`] sets it. +fn pad_of(len: usize) -> Option { + if len % 2 == 1 { Some(0) } else { None } +} + /// What an open element becomes when it closes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Kind { @@ -102,13 +154,14 @@ struct Frame { at: usize, /// The name as written, to match the end tag. name: (usize, usize), - /// The decoded `bdata` attribute. - bdata: Option>, + /// The chunk's entry in the list (for `Chunk`, `Utf8` and `Alias`). + entry: Option, + /// The buffer length when the element opened: its body starts here. + mark: usize, + /// The end of the decoded `bdata` attribute in the buffer. + bdata_end: Option, /// The attributes of a `fileReference`, in document order. attrs: Vec<(String, String)>, - /// Collected text (only for `string` and the XMP packet). - text: Vec, - children: Vec>, } struct Reader<'d> { @@ -117,13 +170,19 @@ struct Reader<'d> { limits: Limits, count: usize, stack: Vec, - /// The top-level chunks once the root element has closed. - top: Option>>, - trailer: Option>, + /// True once the root element has closed. + root_closed: bool, + trailer: Option<(usize, usize)>, + buffer: Vec, + nodes: Vec, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, + /// The attributes of the current start tag; kept to reuse its allocation. + attrs: Vec<((usize, usize), Cow<'d, [u8]>)>, } -impl Reader<'_> { - fn run(mut self) -> Result, AepError> { +impl<'d> Reader<'d> { + fn run(mut self) -> Result { if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { self.pos = 3; } @@ -137,11 +196,13 @@ impl Reader<'_> { if let Some(open) = self.stack.last() { return Err(xml(open.at, "an element is not closed before the end of the file")); } - let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; - Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + if !self.root_closed { + return Err(xml(self.data.len(), "no AfterEffectsProject root element")); + } + Ok(AepxDrafts { buffer: self.buffer, nodes: self.nodes, trailer: self.trailer.unwrap_or((0, 0)) }) } - fn rest(&self) -> &[u8] { + fn rest(&self) -> &'d [u8] { self.data.get(self.pos..).unwrap_or(&[]) } @@ -181,14 +242,13 @@ impl Reader<'_> { self.pos = start; let end = self.skip_past(b"]]>", "a CDATA section is not closed")?; let content = self.data.get(start..end).unwrap_or(&[]); - let at = start; - match self.stack.last_mut() { + match self.stack.last() { Some(frame) => { if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { - frame.text.extend_from_slice(content); + self.buffer.extend_from_slice(content); } } - None => return Err(xml(at, "CDATA outside the root element")), + None => return Err(xml(start, "CDATA outside the root element")), } } else if rest.starts_with(b" { self.pos = self.pos.saturating_add(n); } - fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + fn slice(&self, (a, b): (usize, usize)) -> &'d [u8] { self.data.get(a..b).unwrap_or(&[]) } - fn start_tag(&mut self) -> Result<(), AepError> { - let at = self.pos; - self.pos = self.pos.saturating_add(1); - let name = self.name()?; - let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); - let self_closing = loop { + /// Reads the attributes of a start tag into `attrs`; true when the tag closes itself. + fn attributes(&mut self, at: usize, attrs: &mut Vec<((usize, usize), Cow<'d, [u8]>)>) -> Result { + loop { self.skip_ws(); let rest = self.rest(); if rest.starts_with(b"/>") { self.pos = self.pos.saturating_add(2); - break true; + return Ok(true); } if rest.starts_with(b">") { self.pos = self.pos.saturating_add(1); - break false; + return Ok(false); } if rest.is_empty() { return Err(xml(at, "a start tag is not closed")); @@ -259,11 +316,27 @@ impl Reader<'_> { if raw.contains(&b'<') { return Err(xml(vstart, "`<` inside an attribute value")); } - let mut value = Vec::new(); - decode_entities(raw, vstart, &mut value)?; + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; self.pos = vend.saturating_add(1); attrs.push((key, value)); - }; + } + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = std::mem::take(&mut self.attrs); + attrs.clear(); + let self_closing = self.attributes(at, &mut attrs)?; let kind = self.kind_of(name, at)?; if kind != Kind::Root { self.count = self.count.saturating_add(1); @@ -275,14 +348,25 @@ impl Reader<'_> { return Err(AepError::TooDeep); } } - let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; - if matches!(kind, Kind::Chunk(_)) { - if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { - frame.bdata = Some(decode_hex(value, key.1)?); + let mark = self.buffer.len(); + let mut frame = Frame { kind, at, name, entry: None, mark, bdata_end: None, attrs: Vec::new() }; + match kind { + Kind::Chunk(id) => { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + decode_hex_into(value, key.1, MAX_BDATA, &mut self.buffer)?; + frame.bdata_end = Some(self.buffer.len()); + } + frame.entry = Some(self.open_entry(id)?); } - } else if kind == Kind::Alias { - frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + Kind::Utf8 => frame.entry = Some(self.open_entry(*b"Utf8")?), + Kind::Alias => { + frame.attrs = + attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + frame.entry = Some(self.open_entry(*b"alas")?); + } + Kind::Root | Kind::Xmp | Kind::Skip => {} } + self.attrs = attrs; self.stack.push(frame); if self_closing { self.close()?; @@ -290,12 +374,38 @@ impl Reader<'_> { Ok(()) } + /// Appends the entry of a chunk whose element just opened; its body is filled in when it + /// closes. The list grows by its length (at least 16) but never past [`Self::bound`]; an + /// allocation the target cannot make is an error, not an abort. + fn open_entry(&mut self, id: [u8; 4]) -> Result { + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + u32::try_from(at.checked_add(1).ok_or(AepError::TooManyChunks)?).map_err(|_| AepError::TooManyChunks)?; + if at == self.nodes.capacity() { + // Past the bound (which only a wrong guess could reach) it keeps doubling. + let room = self.bound.saturating_sub(at); + let more = if room == 0 { at.max(16) } else { at.max(16).min(room) }; + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(Draft::placeholder(id)); + Ok(at) + } + + /// Fills in the entry at `at` once its element has closed; its subtree is what follows it. + fn finish(&mut self, at: usize, id: [u8; 4], kind: EntryKind, list_type: [u8; 4], body: (usize, usize), pad: Option) -> Result<(), AepError> { + let end = u32::try_from(self.nodes.len()).map_err(|_| AepError::TooManyChunks)?; + if let Some(d) = self.nodes.get_mut(at) { + *d = Draft { start: body.0, len: body.1.saturating_sub(body.0), id, list_type, end, kind, pad }; + } + Ok(()) + } + /// Classifies an element by its name and where it is. fn kind_of(&self, name: (usize, usize), at: usize) -> Result { let n = self.slice(name); let parent = match self.stack.last() { None => { - if self.top.is_some() { + if self.root_closed { return Err(xml(at, "a second root element")); } if n != ROOT { @@ -339,39 +449,102 @@ impl Reader<'_> { self.close() } - /// Closes the innermost element and hands its chunk to the parent. + /// Closes the innermost element and fills in its chunk's entry. fn close(&mut self) -> Result<(), AepError> { let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; - let chunk = match frame.kind { - Kind::Root => { - self.top = Some(frame.children); - return Ok(()); - } - Kind::Skip => return Ok(()), + match frame.kind { + Kind::Root => self.root_closed = true, + Kind::Skip => {} Kind::Xmp => { + // Only the first packet is kept; a later one is dropped from the buffer (its + // children are skipped, so nothing else was written after it opened). if self.trailer.is_none() { - self.trailer = Some(frame.text); + self.trailer = Some((frame.mark, self.buffer.len())); + } else { + self.buffer.truncate(frame.mark); } - return Ok(()); } - Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), - Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), - Kind::Chunk(id) => chunk_of(id, frame)?, - }; - if let Some(parent) = self.stack.last_mut() { - parent.children.push(chunk); + Kind::Utf8 => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let end = self.buffer.len(); + self.finish(at, *b"Utf8", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Alias => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + self.buffer.extend_from_slice(alias_json(&frame.attrs).as_bytes()); + let end = self.buffer.len(); + self.finish(at, *b"alas", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Chunk(id) => self.close_chunk(id, &frame)?, } Ok(()) } + /// The chunk of a closed element named by a chunk id (what the owned reader's `chunk_of` + /// built): `bdata` wins over children, `numS` / `ppSn` hold their number, a nested id holds its + /// children (or nothing), any other name is a `LIST` of that type. + fn close_chunk(&mut self, id: [u8; 4], frame: &Frame) -> Result<(), AepError> { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let after = at.checked_add(1).ok_or(AepError::TooManyChunks)?; + if let Some(bdata_end) = frame.bdata_end { + // The children were read (and counted, and checked) but are not part of the chunk. + self.nodes.truncate(after); + self.buffer.truncate(bdata_end); + let body = (frame.mark, bdata_end); + let pad = pad_of(bdata_end.saturating_sub(frame.mark)); + if OPAQUE_LIST_TYPES.contains(&id) { + return self.finish(at, LIST, EntryKind::OpaqueList, id, body, pad); + } + return self.finish(at, id, EntryKind::Data, [0; 4], body, pad); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let missing = || xml(frame.at, "a numS or ppSn element without its number"); + // Exactly one child chunk, a `Utf8`: the entry after this one, whose subtree runs to the end. + let only = self.nodes.get(after).copied().filter(|c| usize::try_from(c.end).ok() == Some(self.nodes.len())).ok_or_else(missing)?; + if only.id != *b"Utf8" { + return Err(missing()); + } + let text = match only.kind { + EntryKind::Data | EntryKind::OpaqueList => only.start.checked_add(only.len).and_then(|end| self.buffer.get(only.start..end)).unwrap_or(&[]), + EntryKind::List | EntryKind::Nested => &[], + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body: [u8; 8]; + let bytes: &[u8] = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + body = u64::from(n as u32).to_be_bytes(); + body.get(4..).unwrap_or(&[]) + } else { + body = text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes(); + &body + }; + self.nodes.truncate(after); + self.buffer.truncate(frame.mark); + self.buffer.extend_from_slice(bytes); + let end = self.buffer.len(); + return self.finish(at, id, EntryKind::Data, [0; 4], (frame.mark, end), pad_of(bytes.len())); + } + let empty = (frame.mark, frame.mark); + if NESTED_IDS.contains(&id) { + if self.nodes.len() == after { + return self.finish(at, id, EntryKind::Data, [0; 4], empty, None); + } + return self.finish(at, id, EntryKind::Nested, [0; 4], empty, None); + } + self.finish(at, LIST, EntryKind::List, id, empty, None) + } + fn text(&mut self) -> Result<(), AepError> { let start = self.pos; let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); let end = start.saturating_add(len); self.pos = end; let raw = self.data.get(start..end).unwrap_or(&[]); - match self.stack.last_mut() { - Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + match self.stack.last() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut self.buffer), Some(_) => Ok(()), None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), None => Err(xml(start, "text outside the root element")), @@ -392,41 +565,6 @@ fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { Some(id) } -/// The chunk of a closed element named by a chunk id. -fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { - if let Some(bytes) = frame.bdata { - if OPAQUE_LIST_TYPES.contains(&id) { - let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; - return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); - } - return Ok(Chunk::data(id, bytes)); - } - // Numbers written as text: `0` and `150.000000000`. - if &id == b"numS" || &id == b"ppSn" { - let text = match frame.children.as_slice() { - [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), - _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), - }; - let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); - let body = if &id == b"numS" { - // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. - let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); - let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; - (n as u32).to_be_bytes().to_vec() - } else { - text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() - }; - return Ok(Chunk::data(id, body)); - } - if NESTED_IDS.contains(&id) { - if frame.children.is_empty() { - return Ok(Chunk::data(id, Vec::new())); - } - return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); - } - Ok(Chunk::list(id, frame.children)) -} - /// Appends `raw` with the predefined entities and character references decoded. fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let mut i = 0; @@ -464,23 +602,74 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), A Ok(()) } -/// Decodes a `bdata` value (hex digits, two per byte). +/// Decodes a `bdata` value (hex digits, two per byte) into a new `Vec` of exactly its size. fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { - if text.len() % 2 == 1 { - return Err(xml(offset, "bdata has an odd number of hex digits")); - } + decode_hex_at_most(text, offset, MAX_BDATA) +} + +/// The most bytes one `bdata` may decode to: what the 32-bit size field of a chunk can hold (a +/// larger body could not be written back as an `.aep` either). +const MAX_BDATA: usize = u32::MAX as usize; + +/// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it). +fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result, AepError> { + // Checked before anything is reserved, then sized once (collecting into `Option` has no + // size hint and regrows step by step), and without aborting when the memory is not there. + let pairs = check_hex(text, offset, max)?; + let mut out = Vec::new(); + out.try_reserve_exact(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, &mut out)?; + Ok(out) +} + +/// Appends the bytes of a `bdata` value (hex digits, two per byte, at most `max` bytes) to `out`. +/// On an error `out` is unchanged. +fn decode_hex_into(text: &[u8], offset: usize, max: usize, out: &mut Vec) -> Result<(), AepError> { + let pairs = check_hex(text, offset, max)?; + // Grown at most once per value (amortised over a shared buffer), and without aborting when the + // memory is not there. + out.try_reserve(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, out) +} + +/// Appends the bytes of a `bdata` value that [`check_hex`] accepted. On an error (a digit the +/// check let through, which it does not) `out` is unchanged. +fn decode_checked(text: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let digit = |b: u8| match b { b'0'..=b'9' => Some(b - b'0'), b'a'..=b'f' => Some(b - b'a' + 10), b'A'..=b'F' => Some(b - b'A' + 10), _ => None, }; - let (pairs, _) = text.as_chunks::<2>(); - pairs - .iter() - .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) - .collect::>>() - .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + let mark = out.len(); + for [hi, lo] in text.as_chunks::<2>().0 { + match (digit(*hi), digit(*lo)) { + (Some(h), Some(l)) => out.push((h << 4) | l), + _ => { + out.truncate(mark); + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + } + } + Ok(()) +} + +/// Checks a `bdata` value before anything is reserved for it: an even number of hex digits, at +/// most `max` bytes. Returns the number of bytes it decodes to. +fn check_hex(text: &[u8], offset: usize, max: usize) -> Result { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let pairs = text.len() / 2; + if pairs > max { + return Err(AepError::TooLarge); + } + // One cheap pass checks every digit first. Blocks without an early exit inside, so the check + // vectorises; it stops at the first bad block. + if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) { + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + Ok(pairs) } /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document @@ -528,6 +717,7 @@ fn json_str(out: &mut String, s: &str) { #[cfg(test)] mod tests { use super::*; + use crate::container::index::TreeIndex; fn doc(body: &str) -> String { format!("\n{body}\n") @@ -628,6 +818,92 @@ mod tests { assert!(matches!(read_aepx(b"\xff"), Err(AepError::Xml { .. }))); } + /// Attribute values decode entities only where they have them. + #[test] + fn attribute_values_decode_entities_only_where_they_have_them() { + // An entity inside `bdata` still decodes before the hex does. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x01, 0xab][..])); + // A bad entity in any attribute, even of an element that is skipped, is still an error. + for bad in ["", "", ""] { + assert!(matches!(read_aepx(doc(bad).as_bytes()), Err(AepError::Xml { .. })), "{bad}"); + } + // Without `&` the value is taken as written. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x0a, 0xff][..])); + } + + /// A large bdata with a bad digit is refused before it is sized. + #[test] + fn a_large_bdata_with_a_bad_digit_is_refused_before_it_is_sized() { + // 10 MB of hex that is bad from the first pair (and once more at the very end). + let mut text = b"zz".to_vec(); + text.extend(std::iter::repeat_n(*b"00", 5_000_000).flatten()); + assert_eq!(decode_hex(&text, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let mut late = vec![b'0'; 10_000_000]; + late.extend_from_slice(b"0g"); + assert_eq!(decode_hex(&late, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let x = doc(&format!("", std::str::from_utf8(&text).unwrap())); + assert!(matches!(read_aepx(x.as_bytes()), Err(AepError::Xml { .. }))); + } + + /// Bdata decodes up to the size limit and not past it. + #[test] + fn bdata_decodes_up_to_the_size_limit_and_not_past_it() { + assert_eq!(decode_hex_at_most(b"00ff10", 0, 3), Ok(vec![0x00, 0xff, 0x10])); + assert_eq!(decode_hex_at_most(b"00ff10", 0, 2), Err(AepError::TooLarge)); + assert_eq!(decode_hex_at_most(b"", 0, 0), Ok(Vec::new())); + assert_eq!(decode_hex_at_most(b"00", 0, 0), Err(AepError::TooLarge)); + // The odd-length error still comes first. + assert_eq!(decode_hex_at_most(b"001", 0, 0), Err(xml(0, "bdata has an odd number of hex digits"))); + // The real limit is what a chunk's 32-bit size field holds. + assert_eq!(MAX_BDATA as u64, u64::from(u32::MAX)); + } + + /// Decode hex matches the collecting version. + #[test] + fn decode_hex_matches_the_collecting_version() { + /// The collecting decoder this one replaced, as the reference. + fn old(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + } + let alphabet = b"0123456789abcdefABCDEFgG \0"; + let mut seed = 0x2545_f491_4f6c_dd1du64; + for len in 0..200usize { + for _ in 0..20 { + let text: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Mostly hex digits, now and then a byte that is not one. + let i = (seed % 1000) as usize; + alphabet[if i < 990 { i % 22 } else { 22 + i % 4 }] + }) + .collect(); + let new = decode_hex(&text, 7); + assert_eq!(new, old(&text, 7)); + if let Ok(v) = new { + assert_eq!(v.capacity(), v.len()); + } + } + } + } + #[test] fn limits_hold() { let deep = format!("{}{}", "".repeat(100_000), "".repeat(100_000)); @@ -637,4 +913,292 @@ mod tests { let many = format!("{}", "".repeat(10)); assert_eq!(read_aepx_with_limits(many.as_bytes(), Limits { max_depth: 64, max_chunks: 9 }), Err(AepError::TooManyChunks)); } + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number. + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { usize::try_from(self.next() % n as u64).unwrap() } + } + + /// One of `items`. + fn pick<'s>(&mut self, items: &[&'s str]) -> &'s str { + items[self.below(items.len())] + } + } + + /// `n` random bytes as hex, upper or lower case. + fn hex(rng: &mut Rng, n: usize) -> String { + let upper = rng.below(4) == 0; + (0..n).map(|_| if upper { format!("{:02X}", rng.below(256)) } else { format!("{:02x}", rng.below(256)) }).collect() + } + + /// Random `.aepx` content: every element form the reader knows (data, lists, nested ids, + /// strings, numbers, `btdk`, file references, skipped elements, XMP packets), `bdata` next to + /// children, bodies that read as chunk sequences, entities, CDATA, comments and stray text. + fn content(rng: &mut Rng, depth: usize, budget: &mut usize, out: &mut String) { + for _ in 0..rng.below(6) { + if *budget == 0 { + return; + } + *budget -= 1; + match rng.below(20) { + 0..=3 => { + let id = rng.pick(&["cdat", "tdmn", "head", "Utf8", "idta", "LIST", "a"]); + let n = rng.below(9); + out.push_str(&format!("<{id} bdata=\"{}\"/>", hex(rng, n))); + } + 4..=6 if depth < 10 => { + let id = rng.pick(&["Fold", "tdgp", "Layr", "Pin", "LIST", "btdk", "sspc"]); + let attr = if rng.below(5) == 0 { format!(" bdata=\"{}\"", hex(rng, 3)) } else { String::new() }; + out.push_str(&format!("<{id}{attr}>")); + content(rng, depth + 1, budget, out); + out.push_str(&format!("")); + } + 7 => { + let id = rng.pick(&["tdsn", "pdnm", "fnam"]); + match rng.below(4) { + 0 => out.push_str(&format!("<{id}/>")), + 1 => out.push_str(&format!("<{id}>name & A")), + // A body that is itself a chunk sequence (`Utf8`, 1 byte, pad): still data. + 2 => out.push_str(&format!("<{id} bdata=\"557466380000000161{}\"/>", if rng.below(2) == 0 { "00" } else { "" })), + _ => out.push_str(&format!("<{id}>ab")), + } + } + 8 => { + let text = rng.pick(&["plain", "A < B", "", "ab", "", "été", "xgoney", "\r\n"]); + out.push_str(&format!("{text}")); + } + 9 => { + let id = rng.pick(&["numS", "ppSn"]); + let inner = rng.pick(&[ + "0", + " 7 ", + "-2147483648", + "150.000000000", + "x", + "", + "12", + "", + "", + "34", + ]); + let attr = if rng.below(6) == 0 { " bdata=\"00000002\"" } else { "" }; + out.push_str(&format!("<{id}{attr}>{inner}")); + } + 10 => { + let n = rng.pick(&["", "41", "414243", "4c495354000000000000"]); + out.push_str(&format!("")); + } + 11 => { + let platform = rng.pick(&["Win", "1", "Mac"]); + out.push_str(&format!( + "", + rng.below(3) + )); + } + 12 if depth < 10 => { + out.push_str("text"); + content(rng, depth + 1, budget, out); + out.push_str(""); + } + 13 => { + let packet = rng.pick(&["x]]>", "plain & text", ""]); + out.push_str(&format!("{packet}")); + } + 14 => out.push_str(rng.pick(&["", "", "\n ", "stray"])), + _ => { + let n = rng.below(5); + out.push_str(&format!("", hex(rng, n))); + } + } + } + } + + /// The default limits and six tight ones (depth 0, 1 and 3; 0, 7 and 40 chunks). + fn limit_sets() -> [Limits; 7] { + [ + Limits { max_depth: 0, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 0 }, + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index reader and the owned-tree reader give the same `Result` (the tree, or the error), + /// and the index walks like the index of the owned tree. + fn same(bytes: &[u8], limits: Limits) { + let old = read_aepx_with_limits(bytes, limits); + let new = index_aepx(bytes, limits).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(new, old, "{}", String::from_utf8_lossy(bytes)); + if let Ok(file) = &old { + let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, limits).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let expected = TreeIndex::from_file(file).unwrap(); + assert_eq!(index.len(), expected.len()); + assert_eq!(index.trailer(), expected.trailer()); + let a: Vec = index.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + let b: Vec = expected.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + assert_eq!(a, b); + } + } + + /// The two readers give the same tree and errors on generated and damaged documents. + #[test] + fn the_two_readers_agree_on_generated_and_damaged_documents() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let noise = b"<>/=\"'&;#x0aZ \n![]-?"; + for _ in 0..600 { + let mut budget = 80; + let mut body = String::new(); + content(&mut rng, 0, &mut budget, &mut body); + let x = doc(&body); + for l in limit_sets() { + same(x.as_bytes(), l); + } + for _ in 0..4 { + let mut bad = x.clone().into_bytes(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(bad.len()); + bad[at] = noise[rng.below(noise.len())]; + } + for l in limit_sets() { + same(&bad, l); + } + } + let cut = rng.below(x.len()); + same(&x.as_bytes()[..cut], Limits::default()); + // Bytes that are not UTF-8, and a multi-byte character cut in half. + let mut bad = x.clone().into_bytes(); + let at = rng.below(bad.len()); + bad.insert(at, 0xff); + same(&bad, Limits::default()); + let open = format!("{}é", x.trim_end().trim_end_matches("")).into_bytes(); + same(&open[..open.len() - 1], Limits::default()); + } + } + + /// The flat chunk list never reserves more slots than the input or the chunk limit allows. + #[test] + fn the_chunk_list_is_bounded_by_the_input_and_the_limit() { + let many = format!("{}", "".repeat(10_000)); + let d = index_aepx(many.as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.nodes.len(), 10_000); + assert!(d.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", d.nodes.capacity()); + // The buffer is reserved once, at half the input, and holds nothing here. + assert!(d.buffer.is_empty() && d.buffer.capacity() <= many.len() / 2); + // At the limit the list never holds more slots than the limit. + let d = index_aepx(many.as_bytes(), Limits { max_depth: 64, max_chunks: 10_000 }).unwrap(); + assert!(d.nodes.capacity() <= 10_000); + let few = format!("{}", "".repeat(100)); + let d = index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 100 }).unwrap(); + assert!(d.nodes.capacity() <= 100, "{} slots for at most 100 chunks", d.nodes.capacity()); + assert_eq!(index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 99 }).err(), Some(AepError::TooManyChunks)); + // A tiny input cannot hold more than a third of its bytes in elements. + let tiny = doc(""); + assert!(index_aepx(tiny.as_bytes(), Limits::default()).unwrap().nodes.capacity() <= tiny.len() / 3); + } + + /// What a closed element drops is gone from the buffer and the chunk list, but was counted. + #[test] + fn dropped_content_leaves_nothing_behind_but_still_counts() { + // `bdata` wins over the children: they are read, counted and dropped. + let x = doc("xyzq"); + let d = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.nodes.len()), (&[1u8][..], 1)); + assert_eq!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 4 }).err(), Some(AepError::TooManyChunks)); + assert!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 5 }).is_ok()); + // A child's error is raised although the child would be dropped. + assert!(matches!(index_aepx(doc("").as_bytes(), Limits::default()), Err(AepError::Xml { .. }))); + // Only the first XMP packet stays in the buffer. + let packets = doc(&"packet".repeat(1000)); + let d = index_aepx(packets.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.trailer), (&b"packet"[..], (0, 6))); + // A number keeps its 4 or 8 bytes, not its text. + let d = index_aepx(doc(" 7 1.5").as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.buffer, [&[0, 0, 0, 7][..], &1.5f64.to_be_bytes()[..]].concat()); + assert_eq!(d.nodes.len(), 2); + } + + /// Bodies keep what the element says they are, even where reading the same bytes as an `.aep` + /// would decide otherwise: a nested id whose `bdata` is a chunk sequence stays data, an empty + /// `btdk` stays an opaque list, an empty list at the deepest level stays a list. + #[test] + fn bodies_keep_the_kind_the_element_gives_them() { + let x = doc(""); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let top: Vec<_> = index.chunks().collect(); + assert_eq!(top.len(), 3); + assert_eq!(top[0].bytes().map(<[u8]>::len), Some(11)); + assert!(top[0].children().next().is_none()); + assert!(top[1].is_opaque_list() && top[1].bytes() == Some(&[][..])); + assert!(top[2].is_opaque_list() && top[2].bytes().map(<[u8]>::len) == Some(12)); + // 64 levels: the innermost (empty) list is a list, not opaque. + let deep = format!("{}{}", "".repeat(64), "".repeat(64)); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(deep.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().all(|(_, n)| n.is_list(b"Fold") && !n.is_opaque_list())); + same(x.as_bytes(), Limits::default()); + same(deep.as_bytes(), Limits::default()); + } + + /// Hostile input is refused by both readers with the same errors. + #[test] + fn hostile_input_is_refused_alike_by_both_readers() { + let hostile = [ + format!("{}{}", "".repeat(100_000), "".repeat(100_000)), + format!("{}", "".repeat(1000)), + doc(&format!("", "00".repeat(100_000))), + doc(&format!("", "00".repeat(100_000))), + doc("4294967296"), + doc("1e400x"), + doc("�"), + doc("�"), + doc(&format!("{}", "".repeat(1000))), + format!("{}", "".repeat(10)), + " = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap(); + for l in limit_sets() { + let index = index_aepx(&bytes, l).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(index, read_aepx_with_limits(&bytes, l), "{}", path.display()); + } + } + } } diff --git a/crates/aep/src/container/aepx/tree.rs b/crates/aep/src/container/aepx/tree.rs new file mode 100644 index 000000000..ba7e20ed5 --- /dev/null +++ b/crates/aep/src/container/aepx/tree.rs @@ -0,0 +1,361 @@ +//! The `.aepx` reader of the owned chunk tree ([`super::read_aepx`]): it builds the tree element by +//! element. The importer uses the index reader in `aepx.rs` instead; the tests compare the two on +//! generated, damaged and corpus input (the same tree and the same errors). + +use std::borrow::Cow; + +use super::{FILE_REFERENCE, OPAQUE_LIST_TYPES, ROOT, STRING, XMP, alias_json, chunk_id, decode_entities, decode_hex, xml}; +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::error::AepError; +use crate::container::read::Limits; + +/// [`super::read_aepx_with_limits`]. +pub(super) fn read(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} diff --git a/crates/aep/src/container/arbitrary.rs b/crates/aep/src/container/arbitrary.rs index e4916cfe8..1fb8e8d86 100644 --- a/crates/aep/src/container/arbitrary.rs +++ b/crates/aep/src/container/arbitrary.rs @@ -7,9 +7,9 @@ //! Hue/Saturation (444-446) or a Levels value (512-516) changes only bytes of this `aRbp` (and stale //! UI bytes of `pard`); the visible parameters of the same names keep no record. -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_i32, be_u16, be_u32}; use crate::container::effects::Effect; +use crate::container::index::Node; impl<'f, 'a> Effect<'f, 'a> { /// The arbitrary data stored for the parameter with this match name (`-NNNN`): the @@ -17,13 +17,13 @@ impl<'f, 'a> Effect<'f, 'a> { /// none. pub fn arbitrary_data(&self, match_name: &str) -> Option<&'f [u8]> { let values = self.chunk().child_list(b"tdgp")?; - let kids = values.children(); - let at = kids.iter().position(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; - // The value record, then the data list (no match name of its own). - let rest = kids.get(at.checked_add(1)?..)?; - let mut it = rest.iter().take(3); + let mut kids = values.children(); + // Up to the parameter's match name, then the value record, then the data list (no match + // name of its own). + kids.find(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; + let mut it = kids.take(3); let _record = it.next().filter(|c| c.is_list(b"tdbs"))?; - it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Chunk::bytes) + it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Node::bytes) } } diff --git a/crates/aep/src/container/camera.rs b/crates/aep/src/container/camera.rs index 15623fd2c..0b72537ef 100644 --- a/crates/aep/src/container/camera.rs +++ b/crates/aep/src/container/camera.rs @@ -16,7 +16,6 @@ //! only when it is not the default 100. use crate::container::typed::{Layer, PropNode}; -use crate::container::values::{Tracked, ValueSource, read, read_opt}; /// The Position of a camera or light layer as stored. #[derive(Debug, Clone, Copy, PartialEq)] @@ -28,16 +27,6 @@ pub struct CameraPosition { pub auto_state: bool, } -impl CameraPosition { - /// The position as the batch-3 notes had After Effects report it: in the auto state of a - /// two-node camera (`two_node`) x and y are the composition centre; otherwise the stored - /// value. **Contradicted by 256 and 537 (session S16): After Effects reports and renders the - /// stored value**; use `stored` for what After Effects shows. - pub fn reported(&self, comp_width: u32, comp_height: u32, two_node: bool) -> [f64; 3] { - if self.auto_state && two_node { [f64::from(comp_width) / 2.0, f64::from(comp_height) / 2.0, self.stored[2]] } else { self.stored } - } -} - impl<'f, 'a> Layer<'f, 'a> { /// The Position record of a camera or light layer. pub fn camera_position(&self) -> Option { @@ -49,87 +38,9 @@ impl<'f, 'a> Layer<'f, 'a> { let derived = h.get(76) == Some(&1) && h.get(77) == Some(&1) && position.expression().is_none(); Some(CameraPosition { stored, auto_state: derived }) } - - /// The point of interest of a camera in pixels (its Anchor Point record); `None` when absent. - pub fn camera_point_of_interest(&self) -> Option<[f64; 3]> { - let Some(PropNode::Group(transform)) = self.properties()?.get("ADBE Transform Group") else { return None }; - let tracked = read(&transform, "ADBE Anchor Point", [0.0; 3], 1.0); - matches!(tracked.source, ValueSource::Stored | ValueSource::Keyframed).then_some(tracked.value) - } - - /// A light's intensity in percent; the default 100 when the property is absent. - pub fn light_intensity(&self) -> Option> { - let root = self.properties()?; - let Some(PropNode::Group(options)) = root.get("ADBE Light Options Group") else { return None }; - let t = read(&options, "ADBE Light Intensity", [100.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } } /// The camera zoom After Effects uses when `ADBE Camera Zoom` is absent: 2666.67 px for the 50 mm /// default camera in the 1920 px wide corpus composition. Whether it follows other composition /// widths was not sampled. pub const DEFAULT_CAMERA_ZOOM: f64 = 2_666.666_666_666_666_5; - -impl<'f, 'a> Layer<'f, 'a> { - fn camera_options(&self) -> Option> { - match self.properties()?.get("ADBE Camera Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - fn light_options(&self) -> Option> { - match self.properties()?.get("ADBE Light Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - /// A camera's zoom in pixels (`ADBE Camera Options Group/ADBE Camera Zoom`); absent = the - /// default, [`DEFAULT_CAMERA_ZOOM`] (341: 2000 is stored). The auto Position keeps `-2666.67` and - /// is not recomputed from the zoom. - pub fn camera_zoom(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Zoom", [DEFAULT_CAMERA_ZOOM], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// The depth of field switch (`ADBE Camera Depth of Field`, a checkbox; off by default, 342). - pub fn camera_depth_of_field(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Depth of Field", [0.0], 1.0); - Some(Tracked { value: t.value[0] != 0.0, source: t.source }) - } - - /// A camera's focus distance in pixels (`ADBE Camera Focus Distance`); `None` while absent (its - /// default is not known). - pub fn camera_focus_distance(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Focus Distance", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A camera's aperture (`ADBE Camera Aperture`); `None` while absent (its default is not known). - pub fn camera_aperture(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Aperture", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A light's colour as `[A, R, G, B]` fractions (`ADBE Light Options Group/ADBE Light Color`, - /// stored as doubles 0..255); white when absent (343 stores red). - pub fn light_color(&self) -> Option> { - Some(read(&self.light_options()?, "ADBE Light Color", [1.0; 4], 1.0 / 255.0)) - } - - /// A spot light's cone angle in degrees (`ADBE Light Cone Angle`); 90 when absent (344 stores 60). - /// A point light stores 90 with `tdsb` 3. - pub fn light_cone_angle(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Angle", [90.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A spot light's cone feather in percent (`ADBE Light Cone Feather 2`); 50 when absent (345 - /// stores 30). - pub fn light_cone_feather(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Feather 2", [50.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } -} diff --git a/crates/aep/src/container/effects.rs b/crates/aep/src/container/effects.rs index 2fa77d017..6ccfa3af6 100644 --- a/crates/aep/src/container/effects.rs +++ b/crates/aep/src/container/effects.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; -use crate::container::Chunk; use crate::container::bytes::{array, be_f32, be_f64, be_i32, be_u16, be_u32, cstr, text, until_nul}; +use crate::container::index::Node; use crate::container::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; /// The parameter type, the `u32` at byte 12 of `pard`. The numbering matches the plug-in @@ -132,7 +132,7 @@ pub enum LayerRef { #[derive(Debug, Clone)] pub struct Effect<'f, 'a> { match_name: String, - chunk: &'f Chunk<'a>, + chunk: Node<'f, 'a>, } /// Where the parameter definitions of an effect instance come from. @@ -178,13 +178,13 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The underlying `LIST sspc` chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } /// The instance name shown in the effect controls (`fnam`). pub fn name(&self) -> Option { - self.chunk.child(b"fnam")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text) + self.chunk.child(b"fnam")?.child(b"Utf8")?.bytes().map(text) } /// The name the user gave the instance: the `tdsn` of the instance's value group when it is @@ -192,7 +192,7 @@ impl<'f, 'a> Effect<'f, 'a> { /// `None` while the instance shows the effect's name. pub fn user_name(&self) -> Option { let values = self.chunk.child_list(b"tdgp")?; - let name = values.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; + let name = values.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (name != "-_0_/-" && !name.is_empty()).then_some(name) } @@ -250,7 +250,7 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The parameters described by `part` (a `LIST parT`) with the values of this instance. - fn params_from(&self, part: &'f Chunk<'a>, borrowed: bool) -> Vec> { + fn params_from(&self, part: Node<'f, 'a>, borrowed: bool) -> Vec> { let mut out: Vec> = Vec::new(); // The stored values by match name: the first entry of a name, as `PropGroup::get` finds it // (a group of that name means no stored value). Indexed once: a lookup per parameter @@ -280,7 +280,7 @@ impl<'f, 'a> Effect<'f, 'a> { }); } else if c.is(b"pdnm") && let Some(last) = out.last_mut() - && let Some(u) = c.children().iter().find(|u| u.is(b"Utf8")) + && let Some(u) = c.child(b"Utf8") && let Some(b) = u.bytes() // An empty `pdnm` names nothing (EffectCraft's exports write one: the effect // supplies its items). @@ -293,8 +293,9 @@ impl<'f, 'a> Effect<'f, 'a> { } } -fn has_pard(part: &Chunk<'_>) -> bool { - part.children().iter().any(|c| c.is(b"pard")) +/// True when a `LIST parT` holds parameter definitions (a `pard`). +fn has_pard(part: Node<'_, '_>) -> bool { + part.children().any(|c| c.is(b"pard")) } impl<'f, 'a> Project<'f, 'a> { @@ -302,18 +303,18 @@ impl<'f, 'a> Project<'f, 'a> { /// name and the `LIST parT` of the copied instance, in file order (the stack order of the /// types, 365/366). The `EfDC` count before them is little-endian and is not needed to read /// them. A type whose copy has no definitions is left out. - pub fn effect_definitions(&self) -> Vec<(String, &'f Chunk<'a>)> { - let mut out: Vec<(String, &'f Chunk<'a>)> = Vec::new(); + pub fn effect_definitions(&self) -> Vec<(String, Node<'f, 'a>)> { + let mut out: Vec<(String, Node<'f, 'a>)> = Vec::new(); let Some(defs) = self.effect_defs else { return out }; let mut seen: HashSet = HashSet::new(); - for df in defs.children().iter().filter(|c| c.is_list(b"EfDf")) { + for df in defs.children().filter(|c| c.is_list(b"EfDf")) { let mut name: Option = None; for c in df.children() { if c.is(b"tdmn") { name = c.bytes().map(|b| text(until_nul(b))); } else if c.is_list(b"sspc") && let Some(n) = name.take() - && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(p)) + && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(*p)) && seen.insert(n.clone()) { out.push((n, part)); @@ -328,8 +329,8 @@ impl<'f, 'a> Project<'f, 'a> { /// For a type without a project-level copy it also keeps the own `parT` of the first instance /// in the file (items, layers and effects in file order) that has one. pub fn effect_definition_index(&self) -> EffectDefinitions<'f, 'a> { - let by_name: HashMap> = self.effect_definitions().into_iter().collect(); - let mut instances: HashMap> = HashMap::new(); + let by_name: HashMap> = self.effect_definitions().into_iter().collect(); + let mut instances: HashMap> = HashMap::new(); for item in self.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { @@ -337,7 +338,7 @@ impl<'f, 'a> Project<'f, 'a> { if by_name.contains_key(&fx.match_name) || instances.contains_key(&fx.match_name) { continue; } - if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(p)) { + if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(*p)) { instances.insert(fx.match_name, part); } } @@ -363,9 +364,9 @@ impl<'f, 'a> Project<'f, 'a> { #[derive(Debug, Clone, Default)] pub struct EffectDefinitions<'f, 'a> { /// The project-level copies (`EfdG`). - by_name: HashMap>, + by_name: HashMap>, /// For a type without a project-level copy: the own `parT` of the first instance that has one. - instances: HashMap>, + instances: HashMap>, } impl<'f, 'a> EffectDefinitions<'f, 'a> { @@ -392,7 +393,7 @@ impl<'f, 'a> EffectDefinitions<'f, 'a> { return effect.params(); } match self.by_name.get(effect.match_name()).or_else(|| self.instances.get(effect.match_name())) { - Some(part) => effect.params_from(part, true), + Some(part) => effect.params_from(*part, true), None => Vec::new(), } } @@ -638,6 +639,7 @@ impl<'f, 'a> Layer<'f, 'a> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn tdmn(name: &str) -> Chunk<'static> { let mut b = name.as_bytes().to_vec(); @@ -670,11 +672,11 @@ mod tests { ("A".to_string(), Chunk::list(*b"tdbs", vec![Chunk::data(*b"cdat", vec![0u8; 8])])), ]; let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["A", "B", "C"]); let first = params.first().and_then(|p| p.stored).expect("A stored"); - assert!(first.chunk.children().is_empty(), "the first A (the second has a cdat)"); + assert!(first.chunk.children().next().is_none(), "the first A (the second has a cdat)"); assert!(params.get(1).is_some_and(|p| p.stored.is_none()), "a group of that name is no stored value"); assert!(params.get(2).is_some_and(|p| p.stored.is_none())); } @@ -687,7 +689,7 @@ mod tests { let names: Vec = (0..n).map(|i| format!("P{i}")).collect(); let stored: Vec<(String, Chunk<'static>)> = names.iter().map(|m| (m.clone(), tdbs())).collect(); let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.len(), n); assert!(params.iter().all(|p| p.stored.is_some())); @@ -707,19 +709,19 @@ mod tests { } let efdg = Chunk::list(*b"EfdG", defs); let fold = Chunk::list(*b"Fold", Vec::new()); - let project = Project { fold: &fold, effect_defs: Some(&efdg) }; + let project = Project { fold: Node::from(&fold), effect_defs: Some(Node::from(&efdg)) }; let all = project.effect_definitions(); assert_eq!(all.len(), n); let index = project.effect_definition_index(); let bare = Chunk::list(*b"sspc", vec![Chunk::list(*b"parT", Vec::new())]); for i in (0..n).step_by(50) { - let fx = Effect { match_name: format!("FX{i}"), chunk: &bare }; + let fx = Effect { match_name: format!("FX{i}"), chunk: Node::from(&bare) }; assert_eq!(index.source(&fx), DefinitionSource::Project); let params = index.params(&fx); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["R0"], "the first copy's definitions"); assert!(params.iter().all(|p| p.borrowed_definition)); } - let unknown = Effect { match_name: "nope".into(), chunk: &bare }; + let unknown = Effect { match_name: "nope".into(), chunk: Node::from(&bare) }; assert_eq!(index.source(&unknown), DefinitionSource::Missing); assert!(index.params(&unknown).is_empty()); assert_eq!(project.definition_source(&unknown), DefinitionSource::Missing); diff --git a/crates/aep/src/container/footage.rs b/crates/aep/src/container/footage.rs index 8fca7c27b..14aeb1116 100644 --- a/crates/aep/src/container/footage.rs +++ b/crates/aep/src/container/footage.rs @@ -2,8 +2,8 @@ //! of a layer's source. use crate::container::bytes::{be_f32, be_f64, be_i32, be_u32, cstr, le_u32, text, until_nul}; -use crate::container::chunk::Chunk; use crate::container::header::Rational; +use crate::container::index::Node; use crate::container::typed::{Item, ItemKind, Layer, Project}; /// The footage data of an item (`LIST Pin ` with `sspc`, `opti` and, for files, `Als2/alas`). @@ -101,21 +101,6 @@ impl PsdLayer { } impl Footage { - /// The number of frames of an image sequence (`duration * native_frame_rate`: 3/30 s at 30 fps is - /// 3); `None` for anything else. - pub fn sequence_frame_count(&self) -> Option { - if !self.is_sequence { - return None; - } - let d = self.duration?; - let frames = i128::from(d.num).checked_mul(i128::from(self.native_frame_rate))?; - let den = i128::from(d.den); - if den <= 0 || frames % den != 0 { - return None; - } - u32::try_from(frames / den).ok() - } - /// The solid's colour as fractions R, G, B. pub fn solid_rgb(&self) -> Option<[f32; 3]> { let [_, r, g, b] = self.solid_argb?; @@ -269,14 +254,13 @@ impl SequenceInfo { /// is unreadable (empty). A name that is not a plain file name (holding `/`, `\`, `:` or a control /// character such as NUL, or being `.` or `..`) makes the whole list unreadable (empty): the names /// are joined to the folder. -fn sequence_file_list(list: &Chunk<'_>) -> Vec { +fn sequence_file_list(list: Node<'_, '_>) -> Vec { // After Effects writes the count first (the one sample); a list without a readable count is // not read rather than guessed. let Some(count) = list.child(b"StVS").and_then(|c| c.bytes()).and_then(|b| le_u32(b, 0)).and_then(|n| usize::try_from(n).ok()) else { return Vec::new(); }; - let names: Vec = - list.children().iter().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).take(count).map(text).filter(|n| !n.is_empty()).collect(); + let names: Vec = list.children().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).take(count).map(text).filter(|n| !n.is_empty()).collect(); if names.iter().any(|n| n.contains(|c: char| matches!(c, '/' | '\\' | ':') || c.is_control()) || n == "." || n == "..") { return Vec::new(); } @@ -308,9 +292,8 @@ impl<'f, 'a> Item<'f, 'a> { } let pin = self.chunk.child_list(b"Pin ")?; let sspc = pin.child(b"sspc")?.bytes()?; - let kids = pin.children(); - let after = kids.iter().position(|c| c.is_list(b"Als2"))?; - let naming = || kids.iter().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); + let after = pin.children().position(|c| c.is_list(b"Als2"))?; + let naming = || pin.children().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); let (start, end, digits) = (be_u32(sspc, 172)?, be_u32(sspc, 176)?, be_u32(sspc, 180)?); // Force alphabetical order: the file list replaces the numbering. if let Some(list) = naming().find(|c| c.is_list(b"StVc")) { @@ -320,7 +303,7 @@ impl<'f, 'a> Item<'f, 'a> { } return Some(SequenceInfo { prefix: String::new(), suffix: String::new(), start, end, digits, files }); } - let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).map(text).take(2).collect(); + let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).map(text).take(2).collect(); let (prefix, suffix) = match names.as_slice() { [prefix, suffix] => (prefix.clone(), suffix.clone()), // An empty prefix is not stored (`0001.png` has only `.png`). @@ -350,12 +333,6 @@ impl<'f, 'a> Item<'f, 'a> { self.chunk.child(b"idta")?.bytes()?.get(58).copied() } - /// The ids of the compositions this item uses as precomp layers: the little-endian `u32` of - /// each `idpi` chunk. - pub fn used_item_ids(&self) -> Vec { - self.chunk.children().iter().filter(|c| c.is(b"idpi")).filter_map(|c| le_u32(c.bytes()?, 0)).collect() - } - /// The footage data of a footage item. pub fn footage(&self) -> Option { if self.kind()? != ItemKind::Footage { @@ -405,11 +382,6 @@ impl<'f, 'a> Project<'f, 'a> { self.item_by_id(layer.header()?.source_id?) } - /// The name a layer shows: its own name, or its source's name when it has none. - pub fn layer_display_name(&self, layer: &Layer<'f, 'a>) -> Option { - layer.name().or_else(|| self.layer_source(layer)?.display_name()) - } - /// The pixel size of a layer's source: a footage item's width and height, or a precomp's /// composition size. pub fn layer_source_size(&self, layer: &Layer<'f, 'a>) -> Option<(u32, u32)> { diff --git a/crates/aep/src/container/header.rs b/crates/aep/src/container/header.rs index 508353eb2..060d37840 100644 --- a/crates/aep/src/container/header.rs +++ b/crates/aep/src/container/header.rs @@ -237,12 +237,6 @@ impl LayerHeader { self.flag_bytes.get(byte).is_some_and(|b| b & mask != 0) } - /// Byte 37 bit `0x01`: the layer has a name set by the user (clear on a default-named camera or - /// light and on a precomp layer, whose name is the source's). - pub fn name_is_user_set(&self) -> bool { - self.flag(0, 0x01) - } - /// Byte 37 bit `0x02`: guide layer. pub fn is_guide(&self) -> bool { self.flag(0, 0x02) @@ -333,17 +327,6 @@ impl LayerHeader { Rational::new(i64::from(self.stretch), i64::from(self.stretch_den)) } - /// The time stretch in percent (`stretch / stretch_den * 100`); a zero denominator reads as 1. - pub fn stretch_percent(&self) -> f64 { - let den = if self.stretch_den == 0 { 1.0 } else { f64::from(self.stretch_den) }; - f64::from(self.stretch) / den * 100.0 - } - - /// The start time in seconds, `None` for a zero denominator. - pub fn start_seconds(&self) -> Option { - self.start.to_f64() - } - /// The absolute in point, `start + in * stretch`, exact and reduced. pub fn in_point(&self) -> Option { self.start.plus_scaled(&self.in_relative, &self.stretch_ratio()?) @@ -355,16 +338,6 @@ impl LayerHeader { pub fn out_point(&self) -> Option { self.start.plus_scaled(&self.out_relative, &self.stretch_ratio()?) } - - /// The absolute in point in seconds. - pub fn in_point_seconds(&self) -> Option { - self.in_point()?.to_f64() - } - - /// The absolute out point in seconds. - pub fn out_point_seconds(&self) -> Option { - self.out_point()?.to_f64() - } } impl<'f, 'a> Layer<'f, 'a> { @@ -464,64 +437,10 @@ impl CompHeader { self.time_base.frame_rate() } - /// The frame rate from the 16.16 fixed-point field (25 is `0x0019_0000`). - pub fn frame_rate_from_fixed(&self) -> f64 { - f64::from(self.frame_rate_fixed) / 65536.0 - } - /// The duration in seconds. pub fn duration_seconds(&self) -> Option { self.time_base.seconds(i32::try_from(self.duration_units).ok()?) } - - /// The work area start in seconds. - pub fn work_area_start_seconds(&self) -> Option { - self.time_base.seconds(i32::try_from(self.work_area_start_units).ok()?) - } - - /// The work area end in seconds; the composition duration when the work area is the whole - /// composition. - pub fn work_area_end_seconds(&self) -> Option { - match self.work_area_end_units { - Some(end) => self.time_base.seconds(i32::try_from(end).ok()?), - None => self.duration_seconds(), - } - } - - /// The background colour as fractions 0..1 (`byte / 255`). - pub fn background_rgb(&self) -> [f64; 3] { - self.background.map(|c| f64::from(c) / 255.0) - } - - /// The pixel aspect ratio as a number. - pub fn pixel_aspect_ratio(&self) -> Option { - self.pixel_aspect.to_f64() - } - - /// The display start time in seconds (the start timecode), `None` when absent or the - /// denominator is zero. - pub fn display_start_seconds(&self) -> Option { - self.display_start?.to_f64() - } - - /// The display start as a frame number (`display start * frame rate`, rounded); 100 at 25 fps - /// for a display start of 4 s. - pub fn display_start_frame(&self) -> Option { - let rate = self.frame_rate()?; - Some((self.display_start_seconds()? * rate).round() as i64) - } - - /// The frame rate as an exact fraction `timescale / frame_duration`, as typed: 29.97 fps is - /// `23976 / 800` (not `30000 / 1001`), 23.976 fps is `23976 / 1000`. - pub fn frame_rate_exact(&self) -> Option { - Rational::new(i64::from(self.time_base.timescale), i64::from(self.time_base.frame_duration)) - } - - /// The duration in whole frames (`duration / frame_duration`). After Effects snaps a duration - /// to a whole number of frames: 5 s at 29.97 fps becomes 150 frames (5.005 s). - pub fn duration_frames(&self) -> Option { - self.duration_units.checked_div(self.time_base.frame_duration) - } } impl<'f, 'a> Comp<'f, 'a> { diff --git a/crates/aep/src/container/index.rs b/crates/aep/src/container/index.rs new file mode 100644 index 000000000..695d1e9f3 --- /dev/null +++ b/crates/aep/src/container/index.rs @@ -0,0 +1,1015 @@ +//! A read-only, flat index of the chunk tree for readers that only look (the importer). +//! +//! [`TreeIndex`] holds every chunk of a file in one `Vec` in depth-first order (a chunk, then its +//! whole subtree, then its next sibling). Each entry keeps its id, list type, pad byte, the bytes of +//! its body (borrowed from the input) and the index one past its subtree, so: +//! - a chunk's subtree is the index range `at + 1 .. end`, +//! - its direct children are found by stepping from `at + 1` to each child's `end`, +//! - the whole tree is one array, grown a few times while it is built and freed at once; it never +//! holds more slots than the input has room for chunk headers or the chunk limit allows. +//! +//! [`crate::container::parse`] builds the owned [`Chunk`] tree instead (one `Vec` per list), which +//! the exporter and editors need. [`TreeIndex::to_file`] and [`Node::to_chunk`] convert the index +//! into that tree where an owned one is needed. +//! +//! [`TreeIndex::parse_with_limits`] walks the input exactly as the [`Chunk`] parser does: the same +//! chunks are counted against [`Limits::max_chunks`] in the same order, the same lists are kept +//! opaque at [`Limits::max_depth`] or when damaged, and the same errors come back. A test compares +//! the two on the corpus and on generated and damaged trees. Two errors are its own, and both are +//! stricter than the [`Chunk`] parser (which would abort or could not address the tree): +//! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and +//! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. +//! +//! An `.aepx` is indexed by its own reader (`container/aepx.rs`), which decodes every body into +//! one buffer and fills the entries as it reads; the index then borrows that buffer +//! (`TreeIndex::from_drafts`). It gives the tree [`crate::container::read_aepx`] gives. +//! +//! [`Node`] is the handle the typed view ([`crate::container::Project`] and the types below it) +//! reads through: a chunk of either tree. The view works the same on both. + +use std::borrow::Cow; +use std::fmt; + +use crate::container::bytes::be_u32; +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +use crate::container::error::AepError; +use crate::container::read::{Limits, container_end, id4}; + +/// What an entry's body is (the [`Body`] variants). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Data, + /// [`Body::Chunks`]. + Nested, + List, + OpaqueList, +} + +/// One chunk of a [`TreeIndex`]. +#[derive(Debug, Clone, Copy)] +struct Entry<'a> { + /// The body bytes of a data chunk, or the bytes after the list type of an opaque list; empty + /// for a list or nested chunk (their content is their children). + data: &'a [u8], + id: [u8; 4], + /// The list type of a `LIST` (parsed or opaque); zero otherwise. + list_type: [u8; 4], + /// One past the last entry of this chunk's subtree. + end: u32, + kind: Kind, + pad: Option, +} + +impl<'a> Entry<'a> { + /// An entry for a chunk whose body is not read yet. + fn placeholder(id: [u8; 4]) -> Self { + Self { data: &[], id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// An entry of an index whose bytes are still being written (the `.aepx` reader decodes into a +/// buffer that grows, so it keeps positions, not slices): the body is `start..start + len` of +/// that buffer. The fields are those of [`Entry`]; [`TreeIndex::from_drafts`] turns the list +/// into an index once the buffer is final. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Draft { + pub(crate) start: usize, + pub(crate) len: usize, + pub(crate) id: [u8; 4], + pub(crate) list_type: [u8; 4], + pub(crate) end: u32, + pub(crate) kind: Kind, + pub(crate) pad: Option, +} + +impl Draft { + /// A data chunk with no body yet. + pub(crate) fn placeholder(id: [u8; 4]) -> Self { + Self { start: 0, len: 0, id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// The chunk tree of a file as one flat, read-only array (see the module documentation). +#[derive(Clone)] +pub struct TreeIndex<'a> { + nodes: Vec>, + trailer: &'a [u8], +} + +impl fmt::Debug for TreeIndex<'_> { + /// The sizes only: the chunk count and the trailer length. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TreeIndex").field("chunks", &self.nodes.len()).field("trailer", &self.trailer.len()).finish() + } +} + +/// An index that does not fit a `u32` (more than 4 billion chunks) is refused as too many chunks. +fn index_u32(at: usize) -> Result { + u32::try_from(at).map_err(|_| AepError::TooManyChunks) +} + +impl<'a> TreeIndex<'a> { + /// Indexes an `.aep` with the default [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse`], for the same inputs. + pub fn parse(data: &'a [u8]) -> Result { + Self::parse_with_limits(data, Limits::default()) + } + + /// Indexes an `.aep` with explicit [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::container::parse_with_limits`], for the same inputs and limits; also + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks and + /// [`AepError::TooLarge`] when the array cannot be allocated. + pub fn parse_with_limits(data: &'a [u8], limits: Limits) -> Result { + let end = container_end(data)?; + // Every chunk header takes 8 bytes of its own and is counted once, so the tree never has + // more chunks than this. + let bound = (end.saturating_sub(12) / 8).min(limits.max_chunks); + let mut builder = Builder { data, limits, count: 0, nodes: Vec::new(), bound }; + // A small start; [`Builder::push`] doubles it as chunks arrive, never past the bound, so + // the array stays proportional to the chunk count (a file of a few large chunks gets a few + // slots, not one per 32 bytes). `try_reserve_exact` so it fails nothing here. + let _ = builder.nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + builder.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(Self { nodes: builder.nodes, trailer }) + } + + /// The index of the entries `drafts` (in depth-first order, see [`Draft`]) whose bodies and + /// trailer are ranges of `buffer`. A range outside `buffer` reads as empty. The list is turned + /// into entries; a [`Draft`] and an [`Entry`] have the same size and alignment, so the standard + /// library reuses the list's allocation for them. + pub(crate) fn from_drafts(buffer: &'a [u8], drafts: Vec, trailer: (usize, usize)) -> Self { + let slice = |start: usize, len: usize| start.checked_add(len).and_then(|end| buffer.get(start..end)).unwrap_or(&[]); + let nodes = + drafts.into_iter().map(|d| Entry { data: slice(d.start, d.len), id: d.id, list_type: d.list_type, end: d.end, kind: d.kind, pad: d.pad }).collect(); + let trailer = buffer.get(trailer.0..trailer.1).unwrap_or(&[]); + Self { nodes, trailer } + } + + /// Indexes an already parsed (or built) chunk tree. The index borrows the file's bytes. + /// + /// # Errors + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. + pub fn from_file(file: &'a AepFile<'_>) -> Result { + // Sized once, to the chunks the tree holds. + let mut nodes: Vec> = Vec::new(); + nodes.try_reserve_exact(file.walk().count()).map_err(|_| AepError::TooLarge)?; + // Each frame: the siblings still to visit and the entry whose subtree they fill. + let mut stack: Vec<(std::slice::Iter<'a, Chunk<'_>>, Option)> = vec![(file.chunks.iter(), None)]; + while let Some((iter, parent)) = stack.last_mut() { + let parent = *parent; + match iter.next() { + Some(chunk) => { + let at = nodes.len(); + // The entry after this one must still have a `u32` index. + let next = index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + let (kind, data, list_type) = match &chunk.body { + Body::Data(d) => (Kind::Data, &d[..], [0; 4]), + Body::Chunks(_) => (Kind::Nested, &[][..], [0; 4]), + Body::List { list_type, .. } => (Kind::List, &[][..], *list_type), + Body::OpaqueList { list_type, data } => (Kind::OpaqueList, &data[..], *list_type), + }; + nodes.push(Entry { data, id: chunk.id, list_type, end: next, kind, pad: chunk.pad }); + let kids = chunk.children(); + if !kids.is_empty() { + stack.push((kids.iter(), Some(at))); + } + } + None => { + stack.pop(); + if let Some(p) = parent { + let end = index_u32(nodes.len())?; + if let Some(e) = nodes.get_mut(p) { + e.end = end; + } + } + } + } + } + Ok(Self { nodes, trailer: &file.trailer[..] }) + } + + /// The number of chunks in the tree (at every depth). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// True when the container holds no chunks. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// The bytes after the container's declared end (see [`AepFile::trailer`]). + pub fn trailer(&self) -> &'a [u8] { + self.trailer + } + + /// The top-level chunks. + pub fn chunks(&self) -> Nodes<'_, 'a> { + Nodes(NodesRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX) }) + } + + /// The first top-level chunk with this id. + pub fn chunk(&self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(&self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } + + /// Visits every chunk depth-first with its nesting depth (the top level is 1), as + /// [`AepFile::walk`] does. + pub fn walk(&self) -> NodeWalk<'_, 'a> { + NodeWalk(WalkRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX), open: Vec::new() }) + } + + /// The owned chunk tree (the bodies stay borrowed from the input): what + /// [`crate::container::parse`] gives for the same file. + pub fn to_file(&self) -> AepFile<'a> { + AepFile { chunks: self.build(0, u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)), trailer: Cow::Borrowed(self.trailer) } + } + + /// The entry at `at`, `None` past the end. + fn entry(&self, at: u32) -> Option<&Entry<'a>> { + self.nodes.get(usize::try_from(at).ok()?) + } + + /// The end of the subtree at `at`; never before `at + 1`, so walks always move on. + fn end_of(&self, at: u32) -> u32 { + self.entry(at).map_or(u32::MAX, |e| e.end).max(at.saturating_add(1)) + } + + /// The owned chunks for the siblings that start at `next` and fill `next..end`, without + /// recursion. + fn build(&self, next: u32, end: u32) -> Vec> { + struct Frame<'a> { + /// The list or nested chunk being filled; `None` for the outermost range. + at: Option, + kids: Vec>, + next: u32, + end: u32, + } + let mut stack = vec![Frame { at: None, kids: Vec::new(), next, end }]; + loop { + let Some(top) = stack.last_mut() else { return Vec::new() }; + if top.next < top.end { + let at = top.next; + top.next = self.end_of(at); + let Some(e) = self.entry(at).copied() else { + top.next = top.end; + continue; + }; + match e.kind { + Kind::List | Kind::Nested => { + let sub_end = self.end_of(at); + stack.push(Frame { at: Some(at), kids: Vec::new(), next: at.saturating_add(1), end: sub_end }); + } + Kind::Data => top.kids.push(Chunk { id: e.id, body: Body::Data(Cow::Borrowed(e.data)), pad: e.pad }), + Kind::OpaqueList => { + top.kids.push(Chunk { id: e.id, body: Body::OpaqueList { list_type: e.list_type, data: Cow::Borrowed(e.data) }, pad: e.pad }); + } + } + } else { + let Some(frame) = stack.pop() else { return Vec::new() }; + let Some(at) = frame.at else { return frame.kids }; + let Some(e) = self.entry(at).copied() else { continue }; + let body = match e.kind { + Kind::Nested => Body::Chunks(frame.kids), + Kind::List | Kind::Data | Kind::OpaqueList => Body::List { list_type: e.list_type, children: frame.kids }, + }; + let chunk = Chunk { id: e.id, body, pad: e.pad }; + match stack.last_mut() { + Some(parent) => parent.kids.push(chunk), + None => return vec![chunk], + } + } + } + } +} + +/// The slots a [`TreeIndex`] starts with (32 KiB); it doubles from there. +const INITIAL_SLOTS: usize = 1024; + +/// Builds a [`TreeIndex`]; mirrors `read::Parser` step for step (see the module documentation). +struct Builder<'a> { + data: &'a [u8], + limits: Limits, + count: usize, + nodes: Vec>, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, +} + +impl<'a> Builder<'a> { + /// Appends an entry. A full array grows by its length (at least 16), but never past + /// [`Self::bound`]; an allocation the target cannot make is an error, not an abort. + fn push(&mut self, entry: Entry<'a>) -> Result<(), AepError> { + let len = self.nodes.len(); + if len == self.nodes.capacity() { + let more = len.max(16).min(self.bound.saturating_sub(len)).max(1); + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(entry); + Ok(()) + } + + /// Appends the sibling chunks that exactly fill `start..end`, each followed by its subtree. + fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let mut pos = start; + while pos < end { + let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; + let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; + let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let avail = end - body_start; + if size > avail { + return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); + } + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + self.push(Entry::placeholder(id))?; + let body_end = body_start + size; + self.body(at, id, body_start, body_end, depth)?; + let mut next = body_end; + let mut pad = None; + if size % 2 == 1 && next < end { + pad = Some(self.data.get(next).copied().ok_or(AepError::Malformed { offset: next, what: "missing pad byte" })?); + next += 1; + } + let subtree_end = index_u32(self.nodes.len())?; + if let Some(e) = self.nodes.get_mut(at) { + e.pad = pad; + e.end = subtree_end; + } + pos = next; + } + Ok(()) + } + + /// Like [`Self::sequence`] but a structural problem yields `false` and removes what the + /// attempt appended (the caller keeps bytes). The chunk count keeps the attempt's chunks, as + /// the [`Chunk`] parser's does. + fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result { + let mark = self.nodes.len(); + match self.sequence(start, end, depth) { + Ok(()) => Ok(true), + Err(e) if e.is_structural() => { + self.nodes.truncate(mark); + Ok(false) + } + Err(e) => Err(e), + } + } + + /// The input bytes `start..end` (empty when out of range). + fn slice(&self, start: usize, end: usize) -> &'a [u8] { + self.data.get(start..end).unwrap_or(&[]) + } + + /// Fills in the body of the entry at `at` once the parser knows what it is. + fn set(&mut self, at: usize, kind: Kind, data: &'a [u8], list_type: [u8; 4]) { + if let Some(e) = self.nodes.get_mut(at) { + e.kind = kind; + e.data = data; + e.list_type = list_type; + } + } + + /// Fills in the entry at `at` for the body `start..end` and appends its subtree. + fn body(&mut self, at: usize, id: [u8; 4], start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let len = end - start; + if id == LIST && len >= 4 { + let list_type = id4(self.data, start).ok_or(AepError::Malformed { offset: start, what: "short read" })?; + if depth < self.limits.max_depth && self.tentative(start + 4, end, depth + 1)? { + self.set(at, Kind::List, &[], list_type); + } else { + let data = self.slice(start + 4, end); + self.set(at, Kind::OpaqueList, data, list_type); + } + return Ok(()); + } + if NESTED_IDS.contains(&id) && len >= 8 && depth < self.limits.max_depth { + let mark = self.nodes.len(); + if self.tentative(start, end, depth + 1)? && self.nodes.len() > mark { + self.set(at, Kind::Nested, &[], [0; 4]); + return Ok(()); + } + } + let data = self.slice(start, end); + self.set(at, Kind::Data, data, [0; 4]); + Ok(()) + } +} + +/// A chunk of either tree: an owned [`Chunk`] or an entry of a [`TreeIndex`]. The typed view +/// reads through it, so it works the same on both. Cheap to copy. +#[derive(Clone, Copy)] +pub struct Node<'f, 'a>(NodeRepr<'f, 'a>); + +#[derive(Clone, Copy)] +enum NodeRepr<'f, 'a> { + Chunk(&'f Chunk<'a>), + Index(&'f TreeIndex<'a>, u32), +} + +impl<'f, 'a> From<&'f Chunk<'a>> for Node<'f, 'a> { + /// The handle for this value. + fn from(chunk: &'f Chunk<'a>) -> Self { + Node(NodeRepr::Chunk(chunk)) + } +} + +/// Prints the chunk as [`Chunk`]'s `Debug` does, whichever tree it is in, without copying the +/// subtree. +impl fmt::Debug for Node<'_, '_> { + /// The chunk as `Chunk`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + NodeRepr::Chunk(c) => fmt::Debug::fmt(c, f), + NodeRepr::Index(..) => f.debug_struct("Chunk").field("id", &self.id()).field("body", &BodyDebug(*self)).field("pad", &self.pad()).finish(), + } + } +} + +/// The body of an index [`Node`], printed as [`Body`]'s `Debug` prints it. +struct BodyDebug<'f, 'a>(Node<'f, 'a>); + +/// The children of a [`Node`], printed as a list. +struct ChildrenDebug<'f, 'a>(Node<'f, 'a>); + +impl fmt::Debug for BodyDebug<'_, '_> { + /// The body as `Body`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(e) = self.0.entry() else { return f.write_str("?") }; + match e.kind { + Kind::Data => f.debug_tuple("Data").field(&e.data).finish(), + Kind::Nested => f.debug_tuple("Chunks").field(&ChildrenDebug(self.0)).finish(), + Kind::List => f.debug_struct("List").field("list_type", &e.list_type).field("children", &ChildrenDebug(self.0)).finish(), + Kind::OpaqueList => f.debug_struct("OpaqueList").field("list_type", &e.list_type).field("data", &e.data).finish(), + } + } +} + +impl fmt::Debug for ChildrenDebug<'_, '_> { + /// The children as a list. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.children()).finish() + } +} + +impl<'f, 'a> Node<'f, 'a> { + /// The entry at `at`, `None` past the end. + fn entry(self) -> Option<&'f Entry<'a>> { + match self.0 { + NodeRepr::Index(tree, at) => tree.entry(at), + NodeRepr::Chunk(_) => None, + } + } + + /// The [`Chunk`] when this node is one of an owned tree; `None` for an index entry (see + /// [`Self::to_chunk`]). + pub fn as_chunk(self) -> Option<&'f Chunk<'a>> { + match self.0 { + NodeRepr::Chunk(c) => Some(c), + NodeRepr::Index(..) => None, + } + } + + /// The chunk and its subtree as an owned [`Chunk`] (bodies stay borrowed). + pub fn to_chunk(self) -> Chunk<'a> { + match self.0 { + NodeRepr::Chunk(c) => c.clone(), + NodeRepr::Index(tree, at) => tree.build(at, tree.end_of(at)).into_iter().next().unwrap_or_else(|| Chunk::data(self.id(), &[][..])), + } + } + + /// The 4-byte id (see [`Chunk::id`]). + pub fn id(self) -> [u8; 4] { + match self.0 { + NodeRepr::Chunk(c) => c.id, + NodeRepr::Index(..) => self.entry().map_or([0; 4], |e| e.id), + } + } + + /// The pad byte (see [`Chunk::pad`]). + pub fn pad(self) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.pad, + NodeRepr::Index(..) => self.entry().and_then(|e| e.pad), + } + } + + /// The raw bytes of a data chunk, or the bytes after the list type of an opaque list (see + /// [`Chunk::bytes`]). + pub fn bytes(self) -> Option<&'f [u8]> { + match self.0 { + NodeRepr::Chunk(c) => c.bytes(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::Data | Kind::OpaqueList)).map(|e| e.data), + } + } + + /// The child chunks of a list or nested chunk; none for data. + pub fn children(self) -> Nodes<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => Nodes(NodesRepr::Slice(c.children().iter())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + Nodes(NodesRepr::Index { tree, next: at.saturating_add(1), end }) + } + } + } + + /// The list type when this is a `LIST` (parsed or opaque). + pub fn list_type(self) -> Option<[u8; 4]> { + match self.0 { + NodeRepr::Chunk(c) => c.list_type(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::List | Kind::OpaqueList)).map(|e| e.list_type), + } + } + + /// True when this is a `LIST` of the given type. + pub fn is_list(self, list_type: &[u8; 4]) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_list(list_type), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type), + } + } + + /// True when this is a `LIST` whose content could not be read as chunks. + pub fn is_opaque_list(self) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_opaque_list(), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.kind == Kind::OpaqueList), + } + } + + /// True when this chunk has the given id. + pub fn is(self, id: &[u8; 4]) -> bool { + self.id() == *id + } + + /// The first child chunk with this id. + pub fn child(self, id: &[u8; 4]) -> Option { + self.children().find(|c| c.is(id)) + } + + /// The first child `LIST` of this type. + pub fn child_list(self, list_type: &[u8; 4]) -> Option { + self.children().find(|c| c.is_list(list_type)) + } + + /// Every chunk below this one depth-first (children before later siblings), with the depth + /// counted from 1 for the children of this chunk. + pub fn descendants(self) -> NodeWalk<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => NodeWalk(WalkRepr::Chunk(c.descendants())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + NodeWalk(WalkRepr::Index { tree, next: at.saturating_add(1), end, open: Vec::new() }) + } + } + } + + /// The first descendant `LIST` of this type, searching depth-first. + pub fn find_list(self, list_type: &[u8; 4]) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.find_list(list_type).map(Node::from), + // The subtree is a contiguous range in depth-first order: a plain scan. + NodeRepr::Index(tree, at) => { + let first = usize::try_from(at).ok()?.checked_add(1)?; + let end = usize::try_from(tree.end_of(at)).ok()?.min(tree.nodes.len()); + let range = tree.nodes.get(first..end)?; + let i = range.iter().position(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type)?; + let found = u32::try_from(first.checked_add(i)?).ok()?; + Some(Node(NodeRepr::Index(tree, found))) + } + } + } +} + +/// The children of a [`Node`] (or the top-level chunks of a [`TreeIndex`]), in file order. +#[derive(Clone)] +pub struct Nodes<'f, 'a>(NodesRepr<'f, 'a>); + +#[derive(Clone)] +enum NodesRepr<'f, 'a> { + Slice(std::slice::Iter<'f, Chunk<'a>>), + Index { tree: &'f TreeIndex<'a>, next: u32, end: u32 }, +} + +impl<'f, 'a> Iterator for Nodes<'f, 'a> { + type Item = Node<'f, 'a>; + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + NodesRepr::Slice(it) => it.next().map(Node::from), + NodesRepr::Index { tree, next, end } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + *next = tree.end_of(at); + Some(Node(NodeRepr::Index(tree, at))) + } + } + } +} + +/// A depth-first walk over [`Node`]s with their depth; see [`Node::descendants`] and +/// [`TreeIndex::walk`]. +pub struct NodeWalk<'f, 'a>(WalkRepr<'f, 'a>); + +enum WalkRepr<'f, 'a> { + Chunk(Walk<'f, 'a>), + /// `open` holds the subtree ends of the entries above the next one (its ancestors inside the + /// walked range); their count is the next entry's depth minus one. + Index { + tree: &'f TreeIndex<'a>, + next: u32, + end: u32, + open: Vec, + }, +} + +impl<'f, 'a> Iterator for NodeWalk<'f, 'a> { + type Item = (usize, Node<'f, 'a>); + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + WalkRepr::Chunk(w) => w.next().map(|(d, c)| (d, Node::from(c))), + WalkRepr::Index { tree, next, end, open } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + while open.last().is_some_and(|e| *e <= at) { + open.pop(); + } + let depth = open.len().saturating_add(1); + let sub_end = tree.end_of(at); + if sub_end > at.saturating_add(1) { + open.push(sub_end); + } + *next = at.saturating_add(1); + Some((depth, Node(NodeRepr::Index(tree, at)))) + } + } + } +} + +/// The top level of either tree: what the project settings and the version are read from. +#[derive(Clone, Copy)] +pub(crate) enum Root<'f, 'a> { + File(&'f AepFile<'a>), + Index(&'f TreeIndex<'a>), +} + +impl<'f, 'a> From<&'f AepFile<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(file: &'f AepFile<'a>) -> Self { + Root::File(file) + } +} + +impl<'f, 'a> From<&'f TreeIndex<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(index: &'f TreeIndex<'a>) -> Self { + Root::Index(index) + } +} + +impl<'f, 'a> Root<'f, 'a> { + /// The top-level chunks. + pub(crate) fn chunks(self) -> Nodes<'f, 'a> { + match self { + Root::File(f) => Nodes(NodesRepr::Slice(f.chunks.iter())), + Root::Index(t) => t.chunks(), + } + } + + /// The first top-level chunk with this id. + pub(crate) fn chunk(self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub(crate) fn list(self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::container::{parse_with_limits, write}; + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift64*). + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: u64) -> u64 { + if n == 0 { 0 } else { self.next() % n } + } + } + + /// A random tree: lists, nested-id chunks holding a `Utf8`, data of odd and even sizes, opaque + /// lists, empty lists, deep chains. + fn tree(rng: &mut Rng, depth: usize, budget: &mut usize) -> Vec> { + let n = rng.below(6); + let mut out = Vec::new(); + for _ in 0..n { + if *budget == 0 { + break; + } + *budget -= 1; + let pick = rng.below(10); + let chunk = if depth < 12 && pick < 4 { + let types: [&[u8; 4]; 4] = [b"Fold", b"tdgp", b"tdbs", b"Layr"]; + let t = *types[usize::try_from(rng.below(4)).expect("small")]; + Chunk::list(t, tree(rng, depth + 1, budget)) + } else if pick == 4 { + let ids: [&[u8; 4]; 3] = [b"tdsn", b"pdnm", b"fnam"]; + let id = *ids[usize::try_from(rng.below(3)).expect("small")]; + let text = vec![b'a'; usize::try_from(rng.below(5)).expect("small")]; + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text)]), pad: None } + } else if pick == 5 { + let raw = vec![0x5a; usize::try_from(rng.below(9)).expect("small")]; + Chunk { id: LIST, body: Body::OpaqueList { list_type: *b"btdk", data: raw.into() }, pad: None } + } else { + let ids: [&[u8; 4]; 4] = [b"cdat", b"tdmn", b"Utf8", b"idta"]; + let id = *ids[usize::try_from(rng.below(4)).expect("small")]; + Chunk::data(id, vec![u8::try_from(rng.below(256)).expect("byte"); usize::try_from(rng.below(13)).expect("small")]) + }; + out.push(chunk); + } + out + } + + /// Both readers on the same bytes and limits: the same result, errors included; and the + /// index's walk visits the same chunks at the same depths as the tree's. + fn same(bytes: &[u8], limits: Limits) { + let old = parse_with_limits(bytes, limits); + let new = TreeIndex::parse_with_limits(bytes, limits); + match (&old, &new) { + (Ok(file), Ok(index)) => { + assert_eq!(&index.to_file(), file, "the index converts to the parsed tree"); + let a: Vec<(usize, Chunk<'_>)> = file.walk().map(|(d, c)| (d, c.clone())).collect(); + let b: Vec<(usize, Chunk<'_>)> = index.walk().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(a, b, "the walks agree"); + assert_eq!(index.len(), a.len()); + // Indexing the parsed tree gives the same index. + let again = TreeIndex::from_file(file).expect("index"); + assert_eq!(again.to_file(), *file); + } + (Err(a), Err(b)) => assert_eq!(a, b, "the same error"), + _ => panic!("one reader failed: {:?} vs {:?}", old.as_ref().err(), new.as_ref().err()), + } + } + + /// The default limits and four tight ones (depth 1 and 3, 7 and 40 chunks). + fn limits() -> [Limits; 5] { + [ + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index matches the chunk parser on generated and damaged trees. + #[test] + fn the_index_matches_the_chunk_parser_on_generated_and_damaged_trees() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for _ in 0..400 { + let mut budget = 120; + let chunks = tree(&mut rng, 0, &mut budget); + let file = AepFile { chunks, trailer: Cow::Borrowed(b"x") }; + let bytes = write(&file).expect("write"); + for l in limits() { + same(&bytes, l); + } + // Damage: overwrite a few bytes past the container header (sizes, ids, list types). + for _ in 0..4 { + let mut bad = bytes.clone(); + for _ in 0..1 + rng.below(3) { + let at = 12 + usize::try_from(rng.below(u64::try_from(bad.len().saturating_sub(12)).expect("len"))).expect("index"); + if let Some(b) = bad.get_mut(at) { + *b = u8::try_from(rng.below(256)).expect("byte"); + } + } + for l in limits() { + same(&bad, l); + } + } + // Truncated inputs (the container size then no longer fits). + let cut = bytes.len().saturating_sub(usize::try_from(rng.below(8)).expect("small") + 1); + same(&bytes[..cut], Limits::default()); + } + } + + /// Header errors are the same. + #[test] + fn header_errors_are_the_same() { + for bytes in [&b""[..], b"RIFX", b"RIFX\0\0\0\x04Egg", b"RIFX\0\0\0\x04Fnord", b"RIFX\0\0\0\x08Egg!", b"RIFX\0\0\0\x03Egg!", b"RIFF\0\0\0\x04Egg!"] { + same(bytes, Limits::default()); + } + } + + /// The typed accessors give the same answers through either tree. + #[test] + fn nodes_answer_alike_on_both_trees() { + let mut rng = Rng(42); + for _ in 0..100 { + let mut budget = 200; + let file = AepFile { chunks: tree(&mut rng, 0, &mut budget), trailer: Cow::Borrowed(&[]) }; + let bytes = write(&file).expect("write"); + let parsed = crate::container::parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let a: Vec> = parsed.walk().map(|(_, c)| Node::from(c)).collect(); + let b: Vec> = index.walk().map(|(_, n)| n).collect(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(&b) { + assert_eq!((x.id(), x.pad(), x.bytes(), x.list_type(), x.is_opaque_list()), (y.id(), y.pad(), y.bytes(), y.list_type(), y.is_opaque_list())); + assert_eq!(x.children().map(Node::to_chunk).collect::>(), y.children().map(Node::to_chunk).collect::>()); + for t in [b"Fold", b"tdgp", b"tdbs", b"Layr", b"btdk"] { + assert_eq!(x.find_list(t).map(Node::to_chunk), y.find_list(t).map(Node::to_chunk)); + assert_eq!(x.child_list(t).map(Node::to_chunk), y.child_list(t).map(Node::to_chunk)); + assert_eq!(x.is_list(t), y.is_list(t)); + } + for id in [b"cdat", b"Utf8", b"tdsn"] { + assert_eq!(x.child(id).map(Node::to_chunk), y.child(id).map(Node::to_chunk)); + } + let dx: Vec<(usize, Chunk<'_>)> = x.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + let dy: Vec<(usize, Chunk<'_>)> = y.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(dx, dy); + assert_eq!(format!("{x:?}"), format!("{y:?}")); + } + } + } + + /// The index never reserves more than the input or the limit allows. + #[test] + fn the_index_never_reserves_more_than_the_input_or_the_limit_allows() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("index"); + assert_eq!(index.len(), 1000); + // The start (1024 slots) is cut to what the input can hold. + assert!(index.nodes.capacity() <= 1000, "{} slots for at most 1000 headers", index.nodes.capacity()); + // A short tail or a header that runs past the end does not raise the bound. + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(TreeIndex::parse(&big).is_err() && crate::container::parse(&big).is_err()); + // Under a tighter limit the parse fails as the chunk parser does, and the guess never + // exceeds the limit. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(TreeIndex::parse_with_limits(&data, tight).err(), Some(AepError::TooManyChunks)); + let small = Limits { max_depth: 64, max_chunks: 1000 }; + let index = TreeIndex::parse_with_limits(&data, small).expect("exactly at the limit"); + assert_eq!(index.len(), 1000); + // A limit below what the input holds bounds the reservation: 100 chunks, never more slots. + let mut wide = b"RIFX\0\0\0\0Egg!".to_vec(); + wide.extend_from_slice(b"LIST"); + wide.extend_from_slice(&u32::try_from(4 + 8 * 99).expect("small").to_be_bytes()); + wide.extend_from_slice(b"Fold"); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 99).flatten()); + let declared = u32::try_from(wide.len() - 8).expect("small"); + wide[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse_with_limits(&wide, Limits { max_depth: 64, max_chunks: 100 }).expect("100 chunks"); + assert_eq!(index.len(), 100); + assert!(index.nodes.capacity() <= 100); + // `from_file` is sized exactly. + let file = crate::container::parse(&wide).expect("parse"); + assert_eq!(TreeIndex::from_file(&file).expect("index").nodes.capacity(), 100); + } + + /// A file of `n` data chunks of `size` bytes each. + fn flat(n: usize, size: usize) -> Vec { + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + for _ in 0..n { + data.extend_from_slice(b"cdat"); + data.extend_from_slice(&u32::try_from(size).expect("small").to_be_bytes()); + data.extend(std::iter::repeat_n(0u8, size)); + } + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + data + } + + /// The index grows with the chunk count not the byte size. + #[test] + fn the_index_grows_with_the_chunk_count_not_the_byte_size() { + // 4 MB in 4 chunks: a few slots, not one per 32 bytes (that would be 131 072 slots, 4 MB). + let big = flat(4, 1 << 20); + let index = TreeIndex::parse(&big).expect("index"); + assert_eq!(index.len(), 4); + assert!(index.nodes.capacity() <= INITIAL_SLOTS, "{} slots for 4 chunks", index.nodes.capacity()); + // Many small chunks: at most twice the count (it doubles). + let many = flat(10_000, 2); + let index = TreeIndex::parse(&many).expect("index"); + assert_eq!(index.len(), 10_000); + assert!(index.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", index.nodes.capacity()); + same(&big, Limits::default()); + same(&many, Limits::default()); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Failed attempts count once and leave nothing behind. + #[test] + fn failed_attempts_count_once_and_leave_nothing_behind() { + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let headers = 1 + 50 + 1000; + let index = TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }).err(), Some(AepError::TooManyChunks)); + // Only the outermost list is left, opaque: every failed attempt was rolled back. + assert_eq!(index.len(), 1); + let top = index.chunks().next().expect("one top-level chunk"); + assert!(top.is_opaque_list() && top.bytes().map(<[u8]>::len) == Some(level.len() - 12)); + assert_eq!(write(&index.to_file()).expect("write"), data); + same(&data, Limits::default()); + } + + /// Depth beyond the limit stays opaque without recursion past it. + #[test] + fn depth_beyond_the_limit_stays_opaque_without_recursion_past_it() { + // 5000 nested lists: far deeper than the limit; everything below depth 64 stays bytes. + let mut level = b"LIST\0\0\0\x04tdgp".to_vec(); + for _ in 0..5000 { + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(level.len() + 4).expect("small").to_be_bytes()); + out.extend_from_slice(b"tdgp"); + out.extend_from_slice(&level); + level = out; + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("parse"); + assert_eq!(index.len(), 64); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().last().is_some_and(|(_, n)| n.is_opaque_list())); + same(&data, Limits::default()); + // Converting and indexing a deep owned tree uses no recursion either. + let file = index.to_file(); + assert_eq!(TreeIndex::from_file(&file).expect("index").len(), 64); + } +} diff --git a/crates/aep/src/container/keys.rs b/crates/aep/src/container/keys.rs index d78bc6693..06fcaecc5 100644 --- a/crates/aep/src/container/keys.rs +++ b/crates/aep/src/container/keys.rs @@ -193,19 +193,10 @@ impl<'f> Keyframe<'f> { } } - /// The value of a 3-component spatial keyframe (Position), x, y and z in pixels. - pub fn spatial_value(&self) -> Option<[f64; 3]> { - let v = self.value()?; - if !matches!(self.kind(), KeyframeKind::Spatial { components: 3 }) { - return None; - } - Some([*v.first()?, *v.get(1)?, *v.get(2)?]) - } - /// The ease of the key: the four slots at byte 16 of a one-component plain record or at byte 24 /// of a spatial record or a mask path record. For linear and hold keys all four are 0 (a bezier key stores what /// `setTemporalEase` was given). Plain records with several components keep four slots per - /// component, see [`Keyframe::ease_component`] and [`Keyframe::plain_ease_slots`]. + /// component, see [`Keyframe::ease_component`]. pub fn ease(&self) -> Option { match self.kind() { KeyframeKind::Plain { components: 1 } => ease_at(self.record, 16), @@ -214,16 +205,6 @@ impl<'f> Keyframe<'f> { } } - /// The `4 n` ease `f64` of a plain record as stored (from byte `8 + 8 n`). With one component - /// they are [`Keyframe::ease`]; with several they are type-major, see - /// [`Keyframe::ease_component`]. - pub fn plain_ease_slots(&self) -> Option> { - match self.kind() { - KeyframeKind::Plain { components } => doubles(self.record, components.checked_mul(8)?.checked_add(8)?, components.checked_mul(4)?), - _ => None, - } - } - /// The ease of one value component. A plain record with `n` components stores the `4 n` ease /// doubles **type-major**: `[in speed x n][in influence x n][out speed x n][out influence x n]` /// starting at byte `8 + 8 n`, and component `i` uses slot `i` of each block (verified for the @@ -260,16 +241,6 @@ impl<'f> Keyframe<'f> { (0..n).map(|i| self.ease_component(i)).collect() } - /// The Euclidean ARGB distance to the next key that After Effects writes at byte 16 of a colour - /// (4-component spatial) record whose interpolation is not linear: 255 sqrt(2) = 360.6245 for - /// red to blue; 0 on the last key and on linear keys. `None` for other records. - pub fn color_distance(&self) -> Option { - match self.kind() { - KeyframeKind::Spatial { components: 4 } => be_f64(self.record, 16), - _ => None, - } - } - /// The decoded spatial record (Position: 128 bytes, colour: 152 bytes). pub fn spatial(&self) -> Option { let KeyframeKind::Spatial { components: n } = self.kind() else { return None }; @@ -300,11 +271,6 @@ pub struct ExpressionFlags { } impl ExpressionFlags { - /// True when bytes 76, 77 and 120 are all 1. - pub fn all_set(&self) -> bool { - self.byte_76 == 1 && self.byte_77 == 1 && self.byte_120 == 1 - } - /// True when byte 119 is 1. pub fn disabled(&self) -> bool { self.byte_119 == 1 @@ -324,11 +290,6 @@ impl<'f, 'a> Property<'f, 'a> { be_u32(self.chunk.child(b"tdsb")?.bytes()?, 0) } - /// True when `tdb4` byte 5 has bit 0 set: the property is static, it has no keyframes. - pub fn is_static_flag_set(&self) -> Option { - Some(self.header()?.get(5)? & 1 != 0) - } - /// The expression source: the raw UTF-8 text of the `Utf8` child that follows the value in the /// property (no length prefix, no engine tag; the engine is project-wide). pub fn expression(&self) -> Option { diff --git a/crates/aep/src/container/masks.rs b/crates/aep/src/container/masks.rs index 7e77c69e9..7f00c07eb 100644 --- a/crates/aep/src/container/masks.rs +++ b/crates/aep/src/container/masks.rs @@ -6,9 +6,9 @@ //! property (`tdbs`, with a key list when animated) and `omks` with one `LIST shap` per keyframe (a //! single one when the path is static). -use crate::container::Chunk; use crate::container::bytes::{be_f32, be_u32, until_nul}; -use crate::container::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; +use crate::container::index::Node; +use crate::container::typed::{KeyList, Layer, PropGroup, PropNode, Property}; use crate::container::values::{Tracked, read}; /// The mask mode (`mkif` `u32` at byte 4). @@ -93,8 +93,10 @@ pub struct ShapePath { } impl ShapePath { - /// Decodes a `LIST shap`; `None` when `shph` or the point list is malformed. - pub fn from_chunk(shap: &Chunk<'_>) -> Option { + /// Decodes a `LIST shap` (a [`Chunk`](crate::container::Chunk) or a [`Node`]); `None` when + /// `shph` or the point list is malformed. + pub fn from_chunk<'f, 'a: 'f>(shap: impl Into>) -> Option { + let shap = shap.into(); let shph = shap.child(b"shph")?.bytes()?; let bbox = [be_f32(shph, 4)?, be_f32(shph, 8)?, be_f32(shph, 12)?, be_f32(shph, 16)?]; let closed_flag = *shph.get(20)?; @@ -187,22 +189,12 @@ fn mask_from_atom(atom: &PropGroup<'_, '_>, mkif: Option<&[u8]>) -> Mask { } } if let Some(omks) = oms.child_list(b"omks") { - mask.shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); + mask.shapes = omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); } } mask } -impl<'f, 'a> Project<'f, 'a> { - /// The vertices of every path shape of a mask in pixels of the layer: mask boxes are normalised - /// to the **layer** size (the size of its source), not the composition's. `None` when the layer - /// has no source with a size. - pub fn mask_vertices_px(&self, layer: &Layer<'f, 'a>, mask: &Mask) -> Option>> { - let (width, height) = self.layer_source_size(layer)?; - Some(mask.shapes.iter().map(|s| s.vertices((f64::from(width), f64::from(height)))).collect()) - } -} - /// One mask with the views of its records, for readers that need more than [`Mask`]: the /// keyframes and expressions of the mask properties (in [`MaskAtom::atom`]: `ADBE Mask Feather`, /// `ADBE Mask Opacity`, `ADBE Mask Offset`) and the path keyframes paired with their shapes. @@ -246,7 +238,7 @@ impl<'f, 'a> Layer<'f, 'a> { if let Some(PropNode::Other(oms)) = atom.get("ADBE Mask Shape") { path = oms.child_list(b"tdbs").and_then(Property::from_chunk); if let Some(omks) = oms.child_list(b"omks") { - path_shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); + path_shapes = omks.children().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); } } masks.push(MaskAtom { mask, atom, path, path_shapes }); diff --git a/crates/aep/src/container/read.rs b/crates/aep/src/container/read.rs index abae015ba..200b3d84a 100644 --- a/crates/aep/src/container/read.rs +++ b/crates/aep/src/container/read.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; +use crate::container::bytes::{array, be_u32}; use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; use crate::container::error::AepError; @@ -36,10 +37,20 @@ pub fn parse(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`parse`]; also fails with [`AepError::TooManyChunks`]. pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let end = container_end(data)?; + let mut parser = Parser { data, limits, count: 0 }; + let chunks = parser.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) +} + +/// Checks the `RIFX`/`Egg!` header and returns where the container ends (its chunks fill +/// `12..end`; the trailer follows). Shared with [`crate::container::index::TreeIndex::parse`]. +pub(super) fn container_end(data: &[u8]) -> Result { if id4(data, 0) != Some(*b"RIFX") { return Err(AepError::NotRifx); } - let declared = u64::from(be32(data, 4).ok_or(AepError::NotRifx)?); + let declared = u64::from(be_u32(data, 4).ok_or(AepError::NotRifx)?); let form = id4(data, 8).ok_or(AepError::NotRifx)?; if &form != b"Egg!" { return Err(AepError::BadFormType(form)); @@ -49,20 +60,12 @@ pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, Aep return Err(AepError::Truncated { declared, available }); } // `declared <= available < data.len()`, so the end is inside the input. - let end = usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge)?; - let mut parser = Parser { data, limits, count: 0 }; - let chunks = parser.sequence(12, end, 1)?; - let trailer = data.get(end..).unwrap_or(&[]); - Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) + usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge) } -fn be32(data: &[u8], at: usize) -> Option { - id4(data, at).map(u32::from_be_bytes) -} - -fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { - let b = data.get(at..at.checked_add(4)?)?; - Some([*b.first()?, *b.get(1)?, *b.get(2)?, *b.get(3)?]) +/// The 4-byte id at `at`, `None` when the input is too short. +pub(super) fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { + array::<4>(data, at) } struct Parser<'a> { @@ -74,13 +77,16 @@ struct Parser<'a> { impl<'a> Parser<'a> { /// Parses the sibling chunks that exactly fill `start..end`. fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result>, AepError> { + // Sized once: a well-formed sequence fills exactly this many slots. `try_reserve` so a + // limit raised past what the target can allocate fails nothing here (the Vec then grows). let mut out = Vec::new(); + let _ = out.try_reserve_exact(self.capacity_for(start, end)); let mut pos = start; while pos < end { let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; - let size = be32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; let avail = end - body_start; if size > avail { return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); @@ -103,6 +109,26 @@ impl<'a> Parser<'a> { Ok(out) } + /// The slots to reserve for the sequence `start..end`: its header count, bounded by the input + /// (one per 8 bytes at most) and by the chunks the limit still allows. + fn capacity_for(&self, start: usize, end: usize) -> usize { + self.count_siblings(start, end).min(self.limits.max_chunks.saturating_sub(self.count)) + } + + /// The number of chunk headers that tile `start..end` the way [`Self::sequence`] walks them, + /// stopping at the first that does not fit. Never more than `(end - start) / 8`. + fn count_siblings(&self, start: usize, end: usize) -> usize { + let mut n = 0usize; + let mut pos = start; + while let Some(body_start) = pos.checked_add(8).filter(|b| *b <= end) { + let Some(size) = be_u32(self.data, pos.saturating_add(4)) else { break }; + let Some(next) = usize::try_from(size).ok().and_then(|s| body_start.checked_add(s)).filter(|e| *e <= end) else { break }; + n = n.saturating_add(1); + pos = if size % 2 == 1 && next < end { next.saturating_add(1) } else { next }; + } + n + } + /// Like [`Self::sequence`] but a structural problem yields `None` (the caller keeps bytes). fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result>>, AepError> { match self.sequence(start, end, depth) { @@ -140,3 +166,113 @@ impl<'a> Parser<'a> { Ok(Body::Data(Cow::Borrowed(self.slice(start, end)))) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every children `Vec` in the tree, depth first. + fn child_vecs<'f, 'a>(chunks: &'f [Chunk<'a>], out: &mut Vec<&'f Vec>>) { + for c in chunks { + match &c.body { + Body::List { children, .. } | Body::Chunks(children) => { + out.push(children); + child_vecs(children, out); + } + Body::Data(_) | Body::OpaqueList { .. } => {} + } + } + } + + /// Every sibling list is allocated once at its exact length. + #[test] + fn every_sibling_list_is_allocated_once_at_its_exact_length() { + let inner: Vec> = (0..37u8).map(|i| Chunk::data(*b"cdat", vec![i; usize::from(i % 5)])).collect(); + let middle: Vec> = (0..9).map(|_| Chunk::list(*b"tdbs", inner.clone())).collect(); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0; 3]), Chunk::list(*b"Fold", middle)], trailer: Cow::Borrowed(&[]) }; + let bytes = crate::container::write(&file).expect("write"); + let parsed = parse(&bytes).expect("parse"); + assert_eq!(crate::container::write(&parsed).expect("rewrite"), bytes); + let mut vecs = vec![&parsed.chunks]; + child_vecs(&parsed.chunks, &mut vecs); + assert_eq!(vecs.len(), 11); + // `try_reserve_exact` gives exactly the requested capacity with std's allocator today; a + // Vec grown push by push would have rounded up to a power of two (37 -> 64, 9 -> 16). + for v in vecs { + assert_eq!(v.capacity(), v.len(), "a sibling Vec grew past its length"); + } + } + + /// The preallocation is bounded by the input and the chunk limit. + #[test] + fn the_preallocation_is_bounded_by_the_input_and_the_chunk_limit() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, data.len()), 1000); + assert!(parser.count_siblings(12, data.len()) <= (data.len() - 12) / 8); + // A header whose size runs past the end stops the count; a stray tail is not counted. + assert_eq!(parser.count_siblings(12, data.len() - 3), 999); + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + let parser = Parser { data: &big, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, big.len()), 0); + let f = parse(&data).expect("parse"); + assert_eq!((f.chunks.len(), f.chunks.capacity()), (1000, 1000)); + // Under a tighter limit the parse fails as before, without sizing for more than allowed. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(parse_with_limits(&data, tight), Err(AepError::TooManyChunks)); + // The reservation never exceeds what the chunk limit still allows. + let mut parser = Parser { data: &data, limits: tight, count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 10); + parser.count = 7; + assert_eq!(parser.capacity_for(12, data.len()), 3); + parser.count = 11; + assert_eq!(parser.capacity_for(12, data.len()), 0); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 1000); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Nested lists with broken tails parse each header once. + #[test] + fn nested_lists_with_broken_tails_parse_each_header_once() { + // A wide, well-formed list at the bottom, then 50 levels that each fail and fall back to + // opaque: every level re-parses its child before giving up. + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + // 1 + 50 + 1000 headers in the file: the parse counts each exactly once (the count spans + // every attempt), so the work stays linear in the input. + let headers = 1 + 50 + 1000; + let f = parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }), Err(AepError::TooManyChunks)); + // The outermost list keeps its bytes, and the file writes back unchanged. + let [top] = f.chunks.as_slice() else { panic!("one top-level chunk") }; + assert!(matches!(&top.body, Body::OpaqueList { list_type, data } if list_type == b"Fold" && data.len() + 12 == level.len())); + assert_eq!(crate::container::write(&f).expect("write"), data); + } +} diff --git a/crates/aep/src/container/settings.rs b/crates/aep/src/container/settings.rs index b44588232..a443df874 100644 --- a/crates/aep/src/container/settings.rs +++ b/crates/aep/src/container/settings.rs @@ -2,8 +2,10 @@ //! linear-light switches, the time display and the frame count start. Cases 620-634: one change //! each against the base project 620; every field below is the only byte (or chunk) that changes. +use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; use crate::container::footage::json_string; +use crate::container::index::{Node, Root, TreeIndex}; /// The time display style (`nhed` byte 8, feet + frames byte 11, film type byte 13). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -54,7 +56,17 @@ pub struct ProjectSettings { impl ProjectSettings { /// Reads the settings from the top-level chunks. pub fn read(file: &AepFile<'_>) -> ProjectSettings { - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); + Self::of(Root::from(file)) + } + + /// Reads the settings from the top-level chunks of an indexed file. + pub fn read_index(index: &TreeIndex<'_>) -> ProjectSettings { + Self::of(Root::from(index)) + } + + /// Reads the settings from the top-level chunks of either tree. + pub(crate) fn of(file: Root<'_, '_>) -> ProjectSettings { + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let byte = |i: usize| nhed.and_then(|b| b.get(i).copied()); let bits_per_channel = byte(15).and_then(|v| match v { 0 => Some(8), @@ -73,9 +85,9 @@ impl ProjectSettings { 1 => Some(1), _ => None, }); - let flag = |id: &[u8; 4]| file.chunk(id).and_then(|c| c.bytes()).and_then(|b| b.first().copied()) == Some(1); - let working_gamma_24 = file.chunk(b"dwga").and_then(|c| c.bytes()).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); - let scene_compensation = file.chunk(b"acer").and_then(|c| c.bytes()).and_then(|b| b.first().copied()).map(|v| v != 0); + let flag = |id: &[u8; 4]| file.chunk(id).and_then(Node::bytes).and_then(|b| b.first().copied()) == Some(1); + let working_gamma_24 = file.chunk(b"dwga").and_then(Node::bytes).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); + let scene_compensation = file.chunk(b"acer").and_then(Node::bytes).and_then(|b| b.first().copied()).map(|v| v != 0); ProjectSettings { bits_per_channel, working_space: working_space(file), @@ -90,9 +102,10 @@ impl ProjectSettings { } /// The working space: the `Utf8` right after `PwCs`. -fn working_space(file: &AepFile<'_>) -> Option { - let at = file.chunks.iter().position(|c| c.is(b"PwCs"))?; - let json = file.chunks.get(at.checked_add(1)?).filter(|c| c.is(b"Utf8"))?.bytes()?; +fn working_space(file: Root<'_, '_>) -> Option { + let mut chunks = file.chunks(); + chunks.find(|c| c.is(b"PwCs"))?; + let json = chunks.next().filter(|c| c.is(b"Utf8"))?.bytes()?; let trimmed: Vec = json.iter().copied().filter(|b| !b.is_ascii_whitespace() && *b != 0).collect(); if trimmed == b"{}" { return Some(WorkingSpace::None); @@ -130,7 +143,7 @@ fn base64_decode(text: &str) -> Option> { /// The profile description (ICC.1 tag `desc`): a v2 `desc` (ASCII) or a v4 `mluc` (its first /// record, UTF-16BE). `None` when the profile or the tag is cut short. fn icc_description(icc: &[u8]) -> Option { - let be32 = |at: usize| icc.get(at..at.checked_add(4)?).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes); + let be32 = |at: usize| be_u32(icc, at); let count = be32(128)?; let tag = (0..count.min(1024)).find_map(|i| { let at = 132usize.checked_add(usize::try_from(i).ok()?.checked_mul(12)?)?; @@ -141,13 +154,13 @@ fn icc_description(icc: &[u8]) -> Option { let body = icc.get(off..off.checked_add(len)?)?; let text = match body.get(..4)? { b"desc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(8..12)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 8)?).ok()?; let ascii = body.get(12..12usize.checked_add(n)?)?; String::from_utf8_lossy(ascii.split(|b| *b == 0).next().unwrap_or_default()).into_owned() } b"mluc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(20..24)?.try_into().ok()?)).ok()?; - let at = usize::try_from(u32::from_be_bytes(body.get(24..28)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 20)?).ok()?; + let at = usize::try_from(be_u32(body, 24)?).ok()?; let units: Vec = body.get(at..at.checked_add(n)?)?.as_chunks::<2>().0.iter().map(|p| u16::from_be_bytes(*p)).collect(); String::from_utf16_lossy(&units) } diff --git a/crates/aep/src/container/shapes.rs b/crates/aep/src/container/shapes.rs index ea89155a1..dee01bb22 100644 --- a/crates/aep/src/container/shapes.rs +++ b/crates/aep/src/container/shapes.rs @@ -10,7 +10,7 @@ //! points are in pixels relative to the layer origin. use crate::container::masks::ShapePath; -use crate::container::typed::{Layer, PropGroup, PropNode, Property}; +use crate::container::typed::{Layer, PropGroup, PropNode}; use crate::container::values::{Tracked, read, read_opt}; /// The stroke line cap (`ADBE Vector Stroke Line Cap`, stored as a double; butt is the default and @@ -140,7 +140,7 @@ fn number(group: &PropGroup<'_, '_>, name: &str) -> Option> { fn paths_of(group: &PropGroup<'_, '_>) -> Vec { let Some(PropNode::Other(oms)) = group.get("ADBE Vector Shape") else { return Vec::new() }; let Some(omks) = oms.child_list(b"omks") else { return Vec::new() }; - omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() + omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() } fn item(match_name: String, group: &PropGroup<'_, '_>) -> ShapeItem { @@ -262,12 +262,3 @@ impl<'f, 'a> Layer<'f, 'a> { .collect() } } - -impl<'f, 'a> Property<'f, 'a> { - /// The ARGB fractions of a static colour property (4 components stored as doubles 0..255). - pub fn color_argb(&self) -> Option<[f64; 4]> { - let v = self.static_value()?; - let [a, r, g, b] = *v.as_slice().first_chunk::<4>()?; - Some([a / 255.0, r / 255.0, g / 255.0, b / 255.0]) - } -} diff --git a/crates/aep/src/container/style_map.rs b/crates/aep/src/container/style_map.rs index fdb28c875..bc3169c9e 100644 --- a/crates/aep/src/container/style_map.rs +++ b/crates/aep/src/container/style_map.rs @@ -73,10 +73,6 @@ pub struct StyleMap { } impl StyleMap { - /// The switch group's match name (`dropShadow/enabled`). - pub fn switch_name(&self) -> String { - format!("{}/enabled", self.ae) - } /// A parameter's full match name (`dropShadow/opacity`). pub fn param_name(&self, p: &StyleParam) -> String { format!("{}/{}", self.ae, p.ae) diff --git a/crates/aep/src/container/textdoc.rs b/crates/aep/src/container/textdoc.rs index 566a48534..e673e63cf 100644 --- a/crates/aep/src/container/textdoc.rs +++ b/crates/aep/src/container/textdoc.rs @@ -5,8 +5,11 @@ //! as `(` + BOM `fe ff` + UTF-16BE + `)` with `\` escapes. [`parse_text_dict`] reads it (depth //! limited, never panics) and [`Layer::text_document`] pulls out the verified fields. -use crate::container::Chunk; +use std::borrow::Cow; +use std::sync::LazyLock; + use crate::container::bytes::{be_u32, text}; +use crate::container::index::Node; use crate::container::typed::{Comp, Layer, Property}; /// How deep dictionaries and arrays may nest before the parser gives up. @@ -73,15 +76,99 @@ impl TextValue { _ => None, } } +} - fn key_text(&self) -> String { +/// The text of a key the way the dictionary stores it: a `/name` as written, any other value in +/// its canonical form (`06` and `+6` are `6`, `1.0` is `1`, a string its decoded text). +fn owned_key(value: &V<'_>) -> String { + match value { + V::Name(s) | V::Token(s) => text(s), + V::Str(utf16) => decode_utf16be(utf16), + V::Int(i) => i.to_string(), + V::Float(f) => f.to_string(), + V::Bool(b) => b.to_string(), + V::Bytes(b) => text(b), + V::Dict(_) | V::Array(_) => String::new(), + } +} + +/// A value of the text dictionary as [`TextDocument::parse`] reads it: it borrows names, bare +/// words and strings from the body, so a document is read without a `String` per token. A key is +/// borrowed when it is a `/name` (every key After Effects writes); its text is the same as the +/// [`TextValue`] key's. [`V::into_owned`] gives the public [`TextValue`]. +#[derive(Debug, Clone)] +enum V<'b> { + Dict(Vec<(Cow<'b, str>, V<'b>)>), + Array(Vec>), + /// A string that started with the byte order mark: the UTF-16BE bytes after it, unescaped. + Str(Cow<'b, [u8]>), + Bytes(Cow<'b, [u8]>), + Name(&'b [u8]), + Int(i64), + Float(f64), + Bool(bool), + Token(&'b [u8]), +} + +impl<'b> V<'b> { + /// The value of a dictionary key (the last one when a key repeats). + fn get(&self, key: &str) -> Option<&V<'b>> { match self { - Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), - Self::Int(i) => i.to_string(), - Self::Float(f) => f.to_string(), - Self::Bool(b) => b.to_string(), - Self::Bytes(b) => text(b), - Self::Dict(_) | Self::Array(_) => String::new(), + Self::Dict(pairs) => pairs.iter().rev().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + fn at(&self, index: usize) -> Option<&V<'b>> { + match self { + Self::Array(items) => items.get(index), + _ => None, + } + } + + /// The decoded text of a string value. + fn string(&self) -> Option { + match self { + Self::Str(utf16) => Some(decode_utf16be(utf16)), + _ => None, + } + } + + fn as_f64(&self) -> Option { + match self { + Self::Int(i) => Some(*i as f64), + Self::Float(f) => Some(*f), + _ => None, + } + } + + fn as_i64(&self) -> Option { + match self { + Self::Int(i) => Some(*i), + _ => None, + } + } + + /// The key a value makes when it is written in a key position. + fn into_key(self) -> Cow<'b, str> { + match self { + Self::Name(s) => String::from_utf8_lossy(s), + other => Cow::Owned(owned_key(&other)), + } + } + + /// The public form. Recursion is bounded: a parsed value nests at most [`MAX_DEPTH`] + 1 deep. + fn into_owned(self) -> TextValue { + match self { + Self::Dict(pairs) => TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.into_owned(), v.into_owned())).collect()), + Self::Array(items) => TextValue::Array(items.into_iter().map(V::into_owned).collect()), + Self::Str(utf16) => TextValue::Str(decode_utf16be(&utf16)), + Self::Bytes(b) => TextValue::Bytes(b.into_owned()), + Self::Name(s) => TextValue::Name(text(s)), + Self::Int(i) => TextValue::Int(i), + Self::Float(f) => TextValue::Float(f), + Self::Bool(b) => TextValue::Bool(b), + Self::Token(s) => TextValue::Token(text(s)), } } } @@ -89,32 +176,44 @@ impl TextValue { struct Parser<'b> { b: &'b [u8], i: usize, + /// The pairs of the dictionaries being read, innermost last: a dictionary's pairs are moved + /// into a vector of their exact size when it closes (one allocation per dictionary instead of + /// one per doubling). + pairs: Vec<(Cow<'b, str>, V<'b>)>, + /// The same for the items of arrays. + items: Vec>, } +/// Whitespace between tokens. fn is_ws(c: u8) -> bool { matches!(c, b' ' | b'\t' | b'\r' | b'\n') } +/// A byte that ends a bare word. fn is_delimiter(c: u8) -> bool { is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') } -impl Parser<'_> { +impl<'b> Parser<'b> { + /// The next byte, `None` at the end. fn peek(&self) -> Option { self.b.get(self.i).copied() } + /// Moves past whitespace. fn skip_ws(&mut self) { while self.peek().is_some_and(is_ws) { self.i = self.i.saturating_add(1); } } + /// True when the unread bytes start with `s`. fn starts_with(&self, s: &[u8]) -> bool { self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) } - fn word(&mut self) -> &[u8] { + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &'b [u8] { let start = self.i; while self.peek().is_some_and(|c| !is_delimiter(c)) { self.i = self.i.saturating_add(1); @@ -122,79 +221,106 @@ impl Parser<'_> { self.b.get(start..self.i).unwrap_or(&[]) } - fn parse(&mut self, depth: usize) -> Option { + /// The body of a `( ... )` string after the `(`, with `\` escapes resolved: borrowed when it + /// has none. `None` when the string or an escape is not closed. + fn string_body(&mut self) -> Option> { + let rest = self.b.get(self.i..)?; + let stop = rest.iter().position(|c| matches!(c, b')' | b'\\'))?; + let head = rest.get(..stop)?; + self.i = self.i.saturating_add(stop); + if self.peek()? == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Borrowed(head)); + } + let mut out = head.to_vec(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Owned(out)); + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option> { self.skip_ws(); if depth > MAX_DEPTH { return None; } if self.starts_with(b"<<") { self.i = self.i.saturating_add(2); - let mut pairs = Vec::new(); + // On an error the pairs read so far stay on the stack: the error ends the whole parse. + let mark = self.pairs.len(); loop { self.skip_ws(); if self.starts_with(b">>") { self.i = self.i.saturating_add(2); - return Some(TextValue::Dict(pairs)); + return Some(V::Dict(self.pairs.drain(mark..).collect())); } let key = self.parse(depth.saturating_add(1))?; let value = self.parse(depth.saturating_add(1))?; - pairs.push((key.key_text(), value)); + self.pairs.push((key.into_key(), value)); } } match self.peek()? { b'[' => { self.i = self.i.saturating_add(1); - let mut items = Vec::new(); + let mark = self.items.len(); loop { self.skip_ws(); if self.peek()? == b']' { self.i = self.i.saturating_add(1); - return Some(TextValue::Array(items)); + return Some(V::Array(self.items.drain(mark..).collect())); } - items.push(self.parse(depth.saturating_add(1))?); + let item = self.parse(depth.saturating_add(1))?; + self.items.push(item); } } b'(' => { self.i = self.i.saturating_add(1); - let mut out = Vec::new(); - loop { - let c = self.peek()?; - if c == b'\\' { - out.push(*self.b.get(self.i.saturating_add(1))?); - self.i = self.i.saturating_add(2); - } else if c == b')' { - self.i = self.i.saturating_add(1); - break; - } else { - out.push(c); - self.i = self.i.saturating_add(1); - } - } - Some(match out.strip_prefix(&[0xfe, 0xff]) { - Some(utf16) => TextValue::Str(decode_utf16be(utf16)), - None => TextValue::Bytes(out), + let body = self.string_body()?; + Some(if body.starts_with(&[0xfe, 0xff]) { + V::Str(match body { + Cow::Borrowed(b) => Cow::Borrowed(b.get(2..).unwrap_or(&[])), + Cow::Owned(mut v) => { + let n = v.len().min(2); + v.drain(..n); + Cow::Owned(v) + } + }) + } else { + V::Bytes(body) }) } b'/' => { self.i = self.i.saturating_add(1); - Some(TextValue::Name(text(self.word()))) + Some(V::Name(self.word())) } _ => { let word = self.word(); if word.is_empty() { return None; } - let w = text(word); - Some(match w.as_str() { - "true" => TextValue::Bool(true), - "false" => TextValue::Bool(false), + // A word that is not UTF-8 reads as text with replacement characters, which is + // neither a boolean nor a number: a token. + let Ok(w) = std::str::from_utf8(word) else { return Some(V::Token(word)) }; + Some(match w { + "true" => V::Bool(true), + "false" => V::Bool(false), _ => match w.parse::() { - Ok(i) => TextValue::Int(i), + Ok(i) => V::Int(i), // `parse` also takes `nan`, `inf` and overflowing exponents; those are not // numbers a text document holds, so they stay tokens and read as missing. Err(_) => match w.parse::() { - Ok(f) if f.is_finite() => TextValue::Float(f), - _ => TextValue::Token(w), + Ok(f) if f.is_finite() => V::Float(f), + _ => V::Token(word), }, }, }) @@ -203,6 +329,7 @@ impl Parser<'_> { } } +/// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). fn decode_utf16be(bytes: &[u8]) -> String { let (pairs, rest) = bytes.as_chunks::<2>(); let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); @@ -213,11 +340,9 @@ fn decode_utf16be(bytes: &[u8]) -> String { s } -/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). -/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was -/// read before is returned. -pub fn parse_text_dict(body: &[u8]) -> TextValue { - let mut parser = Parser { b: body, i: 0 }; +/// The top-level `key value` pairs of a `btdk` body, borrowed (see [`parse_text_dict`]). +fn parse_dict(body: &[u8]) -> V<'_> { + let mut parser = Parser { b: body, i: 0, pairs: Vec::new(), items: Vec::new() }; let mut pairs = Vec::new(); loop { parser.skip_ws(); @@ -226,13 +351,20 @@ pub fn parse_text_dict(body: &[u8]) -> TextValue { } let Some(key) = parser.parse(0) else { break }; let Some(value) = parser.parse(0) else { break }; - pairs.push((key.key_text(), value)); + pairs.push((key.into_key(), value)); } - TextValue::Dict(pairs) + V::Dict(pairs) +} + +/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). +/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was +/// read before is returned. +pub fn parse_text_dict(body: &[u8]) -> TextValue { + parse_dict(body).into_owned() } /// The run style slots (`RUN/6/0[n]/0/0/6/`) that are decoded; every other slot is compared -/// with its default (`style_defaults`) and listed in +/// with its default (`STYLE_DEFAULTS`) and listed in /// [`TextDocument::undecoded_style_fields`] when it differs. pub const DECODED_STYLE_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12", "13", "18", "35", "36", "52", "53", "54", "56", "57", "58", "62", "63", "70"]; @@ -241,24 +373,27 @@ pub const DECODED_STYLE_SLOTS: &[&str] = pub const DECODED_PARAGRAPH_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "8", "9", "21", "29", "33"]; /// A number slot. -fn val(v: f64) -> TextValue { - TextValue::Float(v) +fn val(v: f64) -> V<'static> { + V::Float(v) } -fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { - TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) +/// A dictionary value with these keys, for the built-in defaults. +fn dict(pairs: Vec<(&'static str, V<'static>)>) -> V<'static> { + V::Dict(pairs.into_iter().map(|(k, v)| (Cow::Borrowed(k), v)).collect()) } -fn numbers(v: &[f64]) -> TextValue { - TextValue::Array(v.iter().map(|x| val(*x)).collect()) +/// An array of numbers, for the built-in defaults. +fn numbers(v: &[f64]) -> V<'static> { + V::Array(v.iter().map(|x| val(*x)).collect()) } /// The default of every run style slot that is not decoded and not neutral (zero, `false`, /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. -fn style_defaults() -> TextValue { - let paint = |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); - let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); +/// Built once. +static STYLE_DEFAULTS: LazyLock> = LazyLock::new(|| { + let paint = |a: f64| dict(vec![("99", V::Name(b"SimplePaint")), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &'static [u8]| dict(vec![("99", V::Name(name))]); dict(vec![ ("37", val(2.0)), ("41", val(2.0)), @@ -268,24 +403,24 @@ fn style_defaults() -> TextValue { ("46", val(7.0)), ("49", val(-1.0)), ("50", val(-1.0)), - ("55", class("SimpleBlender")), + ("55", class(b"SimpleBlender")), ("64", val(4.0)), ("71", val(4.0)), - ("77", TextValue::Bool(true)), - ("78", TextValue::Bool(true)), + ("77", V::Bool(true)), + ("78", V::Bool(true)), ("79", paint(0.0)), ("82", val(3.0)), ("83", val(3.0)), - ("86", class("SimpleCustomFeature")), + ("86", class(b"SimpleCustomFeature")), ("87", val(100.0)), - ("88", TextValue::Bool(true)), + ("88", V::Bool(true)), ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), - ("96", TextValue::Bool(true)), + ("96", V::Bool(true)), ]) -} +}); -/// The same for the paragraph slots (110, 243-245). -fn paragraph_defaults() -> TextValue { +/// The same for the paragraph slots (110, 243-245). Built once. +static PARAGRAPH_DEFAULTS: LazyLock> = LazyLock::new(|| { dict(vec![ ("6", val(1.0)), ("7", val(1.2)), @@ -293,30 +428,35 @@ fn paragraph_defaults() -> TextValue { ("11", val(2.0)), ("12", val(2.0)), ("14", val(36.0)), - ("15", TextValue::Bool(true)), + ("15", V::Bool(true)), ("16", val(0.5)), ("17", numbers(&[0.8, 1.0, 1.33])), ("18", numbers(&[0.0, 0.0, 0.0])), ("19", numbers(&[1.0, 1.0, 1.0])), ("20", val(6.0)), ("22", val(1.0)), - ("23", TextValue::Bool(true)), + ("23", V::Bool(true)), ("24", val(1.0)), - ("26", TextValue::Bool(true)), + ("26", V::Bool(true)), ("31", val(36.0)), ("35", val(1.0)), ("40", val(2.0)), ]) -} +}); -/// Equal values; numbers compare by value (`0` equals `0.0`). -fn same(a: &TextValue, b: &TextValue) -> bool { +/// Equal values; numbers compare by value (`0` equals `0.0`), names and words by their text, +/// strings by their decoded text. +fn same(a: &V<'_>, b: &V<'_>) -> bool { match (a, b) { - (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), - (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Dict(x), V::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (V::Array(x), V::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Str(x), V::Str(y)) => x == y || decode_utf16be(x) == decode_utf16be(y), + (V::Bytes(x), V::Bytes(y)) => x == y, + (V::Name(x), V::Name(y)) | (V::Token(x), V::Token(y)) => x == y || text(x) == text(y), + (V::Bool(x), V::Bool(y)) => x == y, _ => match (a.as_f64(), b.as_f64()) { (Some(p), Some(q)) => p == q, - _ => a == b, + _ => false, }, } } @@ -325,12 +465,12 @@ fn same(a: &TextValue, b: &TextValue) -> bool { /// listed when it holds something else (a neutral value carries nothing that could be lost). /// `/nil` is how an untouched document stores a slot without a value (244: paragraph `/27`, `/28`, /// `/36`). -fn is_neutral(v: &TextValue) -> bool { +fn is_neutral(v: &V<'_>) -> bool { match v { - TextValue::Name(n) => n == "nil", - TextValue::Bool(b) => !b, - TextValue::Array(a) => a.is_empty(), - TextValue::Dict(d) => d.is_empty(), + V::Name(n) => *n == b"nil", + V::Bool(b) => !b, + V::Array(a) => a.is_empty(), + V::Dict(d) => d.is_empty(), other => other.as_f64() == Some(0.0), } } @@ -355,13 +495,13 @@ pub const MAX_UNDECODED_FIELDS: usize = 64; /// Adds to `out` the slots of `actual` that are not decoded and differ from `sampled`, each once /// and at most [`MAX_UNDECODED_FIELDS`] in all. -fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { - let TextValue::Dict(pairs) = actual else { return }; +fn undecoded_fields(actual: &V<'_>, sampled: Option<&V<'_>>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let V::Dict(pairs) = actual else { return }; for (key, value) in pairs { if out.len() >= MAX_UNDECODED_FIELDS { return; } - if decoded.contains(&key.as_str()) || out.contains(key) { + if decoded.contains(&key.as_ref()) || out.iter().any(|o| o == key) { continue; } if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { @@ -372,7 +512,7 @@ fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[ None => !is_neutral(value), }; if differs { - out.push(key.clone()); + out.push(key.to_string()); } } } @@ -432,13 +572,13 @@ pub struct KernRun { /// The manual kerning list of a text run dictionary (`RUN/8/0`): entries `<< /0 << /0 amount >> /1 /// length >>` (554 kerns each of `HHHH` by 50; the closing `\r` has an entry without an amount). -fn kern_runs(run: &TextValue) -> Vec { - let Some(TextValue::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; +fn kern_runs(run: &V<'_>) -> Vec { + let Some(V::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; let runs: Vec = items .iter() .filter_map(|e| { let length = e.get("1").and_then(usize_of)?; - let amount = e.get("0").and_then(|v| v.get("0")).and_then(TextValue::as_f64).filter(|v| v.is_finite()); + let amount = e.get("0").and_then(|v| v.get("0")).and_then(V::as_f64).filter(|v| v.is_finite()); Some(KernRun { length, amount }) }) .collect(); @@ -644,24 +784,26 @@ pub struct TextBox { pub height: f64, } -fn colour_of(style: &TextValue, slot: &str) -> Option<[f64; 4]> { +/// The RGBA colour in a style slot (`slot/0/1`), `None` when it is not four numbers. +fn colour_of(style: &V<'_>, slot: &str) -> Option<[f64; 4]> { let c = style.get(slot)?.get("0")?.get("1")?; Some([c.at(0)?.as_f64()?, c.at(1)?.as_f64()?, c.at(2)?.as_f64()?, c.at(3)?.as_f64()?]) } -fn usize_of(v: &TextValue) -> Option { +/// A non-negative integer value as `usize`. +fn usize_of(v: &V<'_>) -> Option { usize::try_from(v.as_i64()?).ok() } /// A finite number slot of a style, `default` when it is missing or not a finite number. -fn num(style: &TextValue, slot: &str, default: f64) -> f64 { - style.get(slot).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(default) +fn num(style: &V<'_>, slot: &str, default: f64) -> f64 { + style.get(slot).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(default) } /// A boolean slot of a style, `default` when it is missing or not a boolean. -fn flag(style: &TextValue, slot: &str, default: bool) -> bool { +fn flag(style: &V<'_>, slot: &str, default: bool) -> bool { match style.get(slot) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, } } @@ -675,9 +817,9 @@ struct RunNotes { } /// The paragraph runs of a text run dictionary (`RUN/5/0`). -fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(paragraph_defaults()); +fn paragraph_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*PARAGRAPH_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -685,7 +827,7 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { continue; }; let para = e.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("5")); - let justification = match para.and_then(|p| p.get("0")).and_then(TextValue::as_i64) { + let justification = match para.and_then(|p| p.get("0")).and_then(V::as_i64) { Some(0) => Justification::Left, Some(1) => Justification::Right, Some(2) => Justification::Center, @@ -700,12 +842,12 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } }; if let Some(p) = para { - undecoded_fields(p, sampled.as_ref(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); + undecoded_fields(p, sampled, DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); } - let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); - let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(TextValue::as_i64).unwrap_or(default); + let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); + let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(V::as_i64).unwrap_or(default); let flag = |slot: &str, default: bool| match para.and_then(|p| p.get(slot)) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, }; out.push(ParagraphRun { @@ -727,9 +869,9 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } /// The style runs of a text run dictionary (`RUN/6/0`). -fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(style_defaults()); +fn style_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*STYLE_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -742,31 +884,31 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { out.push(StyleRun { length, ..prev }); continue; }; - let font_size = style.get("1").and_then(TextValue::as_f64); + let font_size = style.get("1").and_then(V::as_f64); let fill_argb = colour_of(style, "53"); let font_index = style.get("0").and_then(usize_of); if font_size.is_none() || fill_argb.is_none() || font_index.is_none() { notes.incomplete = notes.incomplete.saturating_add(1); } - undecoded_fields(style, sampled.as_ref(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); + undecoded_fields(style, sampled, DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); out.push(StyleRun { length, font_size: font_size.unwrap_or(prev.font_size), fill_argb: fill_argb.unwrap_or(prev.fill_argb), stroke_argb: colour_of(style, "54"), - tracking: style.get("8").and_then(TextValue::as_f64).unwrap_or(0.0), - faux_bold: matches!(style.get("2"), Some(TextValue::Bool(true))), - faux_italic: matches!(style.get("3"), Some(TextValue::Bool(true))), - caps: match style.get("12").and_then(TextValue::as_i64) { + tracking: style.get("8").and_then(V::as_f64).unwrap_or(0.0), + faux_bold: matches!(style.get("2"), Some(V::Bool(true))), + faux_italic: matches!(style.get("3"), Some(V::Bool(true))), + caps: match style.get("12").and_then(V::as_i64) { None | Some(0) => Caps::Normal, Some(1) => Caps::Small, Some(2) => Caps::All, Some(other) => Caps::Other(other), }, - auto_leading: matches!(style.get("4"), Some(TextValue::Bool(true))), - leading: style.get("5").and_then(TextValue::as_f64), + auto_leading: matches!(style.get("4"), Some(V::Bool(true))), + leading: style.get("5").and_then(V::as_f64), font_index: font_index.unwrap_or(prev.font_index), - baseline: match style.get("13").and_then(TextValue::as_i64) { + baseline: match style.get("13").and_then(V::as_i64) { None | Some(0) => FontBaseline::Normal, Some(1) => FontBaseline::Superscript, Some(2) => FontBaseline::Subscript, @@ -775,22 +917,22 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { horizontal_scale: num(style, "6", 1.0), vertical_scale: num(style, "7", 1.0), baseline_shift: num(style, "9", 0.0), - auto_kern: match style.get("11").and_then(TextValue::as_i64) { + auto_kern: match style.get("11").and_then(V::as_i64) { Some(0) => AutoKern::None, None | Some(1) => AutoKern::Metrics, Some(2) => AutoKern::Optical, Some(other) => AutoKern::Other(other), }, ligatures: flag(style, "18", false), - baseline_direction: style.get("35").and_then(TextValue::as_i64).unwrap_or(1), + baseline_direction: style.get("35").and_then(V::as_i64).unwrap_or(1), tsume: num(style, "36", 0.0), no_break: flag(style, "52", false), apply_fill: flag(style, "56", true), apply_stroke: flag(style, "57", false), stroke_over_fill: flag(style, "58", true), - line_join: style.get("62").and_then(TextValue::as_i64).unwrap_or(0), + line_join: style.get("62").and_then(V::as_i64).unwrap_or(0), stroke_width: num(style, "63", 1.0), - digit_set: style.get("70").and_then(TextValue::as_i64).unwrap_or(0), + digit_set: style.get("70").and_then(V::as_i64).unwrap_or(0), }); } out @@ -798,13 +940,13 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { /// The box of paragraph text from the frame dictionary `/0/8/0[0]/0`: key `/1/0` holds 16 x, y pairs /// that trace the rectangle, and `/2/0` is 1; both keys are absent for point text. -fn text_box(top: &TextValue) -> Option { +fn text_box(top: &V<'_>) -> Option { let frame = top.get("0")?.get("8")?.get("0")?.at(0)?.get("0")?; if frame.get("2")?.get("0")?.as_i64()? != 1 { return None; } - let TextValue::Array(coords) = frame.get("1")?.get("0")? else { return None }; - let numbers: Vec = coords.iter().map(TextValue::as_f64).collect::>()?; + let V::Array(coords) = frame.get("1")?.get("0")? else { return None }; + let numbers: Vec = coords.iter().map(V::as_f64).collect::>()?; let (pairs, _) = numbers.as_chunks::<2>(); if pairs.is_empty() { return None; @@ -818,18 +960,15 @@ fn text_box(top: &TextValue) -> Option { impl TextDocument { /// Decodes the body of a `btdk` list. pub fn parse(body: &[u8]) -> Option { - let top = parse_text_dict(body); + let top = parse_dict(body); let run = top.get("1")?.get("1")?.at(0)?.get("0")?; - let raw_text = run.get("0")?.as_str()?.to_string(); + let raw_text = run.get("0")?.string()?; let style = run.get("6")?.get("0")?.at(0)?.get("0")?.get("0")?.get("6")?; let fill_argb = colour_of(style, "53")?; let font_index = usize::try_from(style.get("0")?.as_i64()?).ok()?; let mut fonts = Vec::new(); - if let Some(TextValue::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { - fonts = list - .iter() - .map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(TextValue::as_str).unwrap_or_default().to_string()) - .collect(); + if let Some(V::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { + fonts = list.iter().map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(V::string).unwrap_or_default()).collect(); } let mut notes = RunNotes::default(); let paragraphs = paragraph_runs(run, &mut notes); @@ -857,7 +996,7 @@ impl TextDocument { .and_then(|v| v.get("0")) .and_then(|v| v.get("2")) .and_then(|v| v.get("1")) - .and_then(TextValue::as_i64), + .and_then(V::as_i64), }) } @@ -881,23 +1020,32 @@ impl TextDocument { } impl<'f, 'a> Layer<'f, 'a> { - /// The text document of a text layer (`LIST btdk` below `ADBE Text Properties`). + /// The Source Text record `LIST btds` (`ADBE Text Document`). It is a direct child of the + /// `ADBE Text Properties` group, which is a direct child of the layer's root `tdgp` + /// (`Layr/tdgp/tdgp/btds` in every sample, and where the exporter writes it), so only that + /// level is searched rather than the whole layer. + fn source_text_record(&self) -> Option> { + let root = self.chunk.child_list(b"tdgp")?; + root.children().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) + } + + /// The text document of a text layer: the first `LIST btdk` in the Source Text record `btds` + /// (the same documents [`Layer::text_document_count`] counts). pub fn text_document(&self) -> Option { - TextDocument::parse(self.chunk.find_list(b"btdk")?.bytes()?) + TextDocument::parse(self.source_text_record()?.find_list(b"btdk")?.bytes()?) } /// The Source Text property record: the `tdbs` in `btds[ADBE Text Document]` (a placeholder /// in every sample: no keys, no expression). Source Text keyframes and expressions were not /// sampled; a `list` (keys) or an expression in it marks them. pub fn source_text_property(&self) -> Option> { - let btds = self.chunk.descendants().map(|(_, c)| c).find(|c| c.is_list(b"btds"))?; - Property::from_chunk(btds.child_list(b"tdbs")?) + Property::from_chunk(self.source_text_record()?.child_list(b"tdbs")?) } - /// The number of text documents (`LIST btdk`) of the layer: 1 in every sample; more would be - /// one per Source Text keyframe (not sampled). + /// The number of text documents (`LIST btdk`) in the Source Text record: 1 in every sample; + /// more would be one per Source Text keyframe (not sampled). pub fn text_document_count(&self) -> usize { - self.chunk.descendants().filter(|(_, c)| c.is_list(b"btdk")).count() + self.source_text_record().map_or(0, |btds| btds.descendants().filter(|(_, c)| c.is_list(b"btdk")).count()) } } @@ -942,10 +1090,24 @@ pub enum CueType { Other(u32), } +impl<'f, 'a> Node<'f, 'a> { + /// The marker list `LIST mrst` of a marker holder: a layer (`LIST Layr`) or a composition's + /// `SecL` list. A layer's `mrst` is a direct child of its root `tdgp` (`ADBE Marker`, where + /// After Effects and the exporter write it), so only that place is looked at; most layers have + /// none, and searching their whole property tree for it was a large part of an import. Any + /// other holder is searched depth-first. + pub(crate) fn marker_list(self) -> Option> { + if self.is_list(b"Layr") { + return self.child_list(b"tdgp")?.child_list(b"mrst"); + } + self.find_list(b"mrst") + } +} + /// Reads the markers below `holder` (a layer or the composition's `SecL` list): the `mrst` list has a /// property whose 16-byte records give the times and one `Nmrd` list per marker with the strings. -fn markers_in(holder: &Chunk<'_>) -> Vec { - let Some(mrst) = holder.find_list(b"mrst") else { return Vec::new() }; +fn markers_in(holder: Node<'_, '_>) -> Vec { + let Some(mrst) = holder.marker_list() else { return Vec::new() }; let Some(prop) = mrst.child_list(b"tdbs").and_then(Property::from_chunk) else { return Vec::new() }; let Some(keys) = prop.keys() else { return Vec::new() }; let scale = prop.time_scale().unwrap_or(0); @@ -955,11 +1117,11 @@ fn markers_in(holder: &Chunk<'_>) -> Vec { .filter_map(|(k, nmrd)| { let time_units = k.time_units()?; // The five strings, in the order comment, chapter, URL, frame target, cue point name. - let mut strings = nmrd.children().iter().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); + let mut strings = nmrd.children().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); let mut next = || strings.next().unwrap_or_default(); let (comment, chapter, url, frame_target, cue_point_name) = (next(), next(), next(), next(), next()); let extras: Vec = strings.collect(); - let header = nmrd.child(b"NmHd").and_then(Chunk::bytes); + let header = nmrd.child(b"NmHd").and_then(Node::bytes); let duration_seconds = header.and_then(|h| { let (duration, per_second) = (be_u32(h, 8)?, be_u32(h, 12)?); (per_second != 0).then(|| f64::from(duration) / f64::from(per_second)) @@ -1002,9 +1164,631 @@ impl<'f, 'a> Comp<'f, 'a> { } } +/// The dictionary parser and the defaults as they were before they borrowed from the body (#3276), +/// kept as the reference the differential tests compare with. +#[cfg(test)] +mod oracle { + use super::{MAX_DEPTH, MAX_UNDECODED_FIELDS, TextValue}; + use crate::container::bytes::text; + + pub(super) fn same(a: &TextValue, b: &TextValue) -> bool { + match (a, b) { + (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + _ => match (a.as_f64(), b.as_f64()) { + (Some(p), Some(q)) => p == q, + _ => a == b, + }, + } + } + + pub(super) fn is_neutral(v: &TextValue) -> bool { + match v { + TextValue::Name(n) => n == "nil", + TextValue::Bool(b) => !b, + TextValue::Array(a) => a.is_empty(), + TextValue::Dict(d) => d.is_empty(), + other => other.as_f64() == Some(0.0), + } + } + + pub(super) fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let TextValue::Dict(pairs) = actual else { return }; + for (key, value) in pairs { + if out.len() >= MAX_UNDECODED_FIELDS { + return; + } + if decoded.contains(&key.as_str()) || out.contains(key) { + continue; + } + if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { + continue; + } + let differs = match sampled.and_then(|s| s.get(key)) { + Some(default) => !same(value, default), + None => !is_neutral(value), + }; + if differs { + out.push(key.clone()); + } + } + } + + impl TextValue { + /// The canonical text of a key, as the owning parser built it. + fn key_text(&self) -> String { + match self { + Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), + Self::Int(i) => i.to_string(), + Self::Float(f) => f.to_string(), + Self::Bool(b) => b.to_string(), + Self::Bytes(b) => text(b), + Self::Dict(_) | Self::Array(_) => String::new(), + } + } + } + + struct Parser<'b> { + b: &'b [u8], + i: usize, + } + + /// Whitespace between tokens. + fn is_ws(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\r' | b'\n') + } + + /// A byte that ends a bare word. + fn is_delimiter(c: u8) -> bool { + is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') + } + + impl Parser<'_> { + /// The next byte, `None` at the end. + fn peek(&self) -> Option { + self.b.get(self.i).copied() + } + + /// Moves past whitespace. + fn skip_ws(&mut self) { + while self.peek().is_some_and(is_ws) { + self.i = self.i.saturating_add(1); + } + } + + /// True when the unread bytes start with `s`. + fn starts_with(&self, s: &[u8]) -> bool { + self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) + } + + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &[u8] { + let start = self.i; + while self.peek().is_some_and(|c| !is_delimiter(c)) { + self.i = self.i.saturating_add(1); + } + self.b.get(start..self.i).unwrap_or(&[]) + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option { + self.skip_ws(); + if depth > MAX_DEPTH { + return None; + } + if self.starts_with(b"<<") { + self.i = self.i.saturating_add(2); + let mut pairs = Vec::new(); + loop { + self.skip_ws(); + if self.starts_with(b">>") { + self.i = self.i.saturating_add(2); + return Some(TextValue::Dict(pairs)); + } + let key = self.parse(depth.saturating_add(1))?; + let value = self.parse(depth.saturating_add(1))?; + pairs.push((key.key_text(), value)); + } + } + match self.peek()? { + b'[' => { + self.i = self.i.saturating_add(1); + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek()? == b']' { + self.i = self.i.saturating_add(1); + return Some(TextValue::Array(items)); + } + items.push(self.parse(depth.saturating_add(1))?); + } + } + b'(' => { + self.i = self.i.saturating_add(1); + let mut out = Vec::new(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + break; + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + Some(match out.strip_prefix(&[0xfe, 0xff]) { + Some(utf16) => TextValue::Str(decode_utf16be(utf16)), + None => TextValue::Bytes(out), + }) + } + b'/' => { + self.i = self.i.saturating_add(1); + Some(TextValue::Name(text(self.word()))) + } + _ => { + let word = self.word(); + if word.is_empty() { + return None; + } + let w = text(word); + Some(match w.as_str() { + "true" => TextValue::Bool(true), + "false" => TextValue::Bool(false), + _ => match w.parse::() { + Ok(i) => TextValue::Int(i), + // `parse` also takes `nan`, `inf` and overflowing exponents; those are not + // numbers a text document holds, so they stay tokens and read as missing. + Err(_) => match w.parse::() { + Ok(f) if f.is_finite() => TextValue::Float(f), + _ => TextValue::Token(w), + }, + }, + }) + } + } + } + } + + /// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). + fn decode_utf16be(bytes: &[u8]) -> String { + let (pairs, rest) = bytes.as_chunks::<2>(); + let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); + let mut s: String = char::decode_utf16(units).map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER)).collect(); + if !rest.is_empty() { + s.push(char::REPLACEMENT_CHARACTER); + } + s + } + + /// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). + /// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was + /// read before is returned. + pub(super) fn parse_text_dict(body: &[u8]) -> TextValue { + let mut parser = Parser { b: body, i: 0 }; + let mut pairs = Vec::new(); + loop { + parser.skip_ws(); + if parser.i >= body.len() { + break; + } + let Some(key) = parser.parse(0) else { break }; + let Some(value) = parser.parse(0) else { break }; + pairs.push((key.key_text(), value)); + } + TextValue::Dict(pairs) + } + + fn val(v: f64) -> TextValue { + TextValue::Float(v) + } + + /// A dictionary value with these keys, for the built-in defaults. + fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { + TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) + } + + /// An array of numbers, for the built-in defaults. + fn numbers(v: &[f64]) -> TextValue { + TextValue::Array(v.iter().map(|x| val(*x)).collect()) + } + + /// The default of every run style slot that is not decoded and not neutral (zero, `false`, + /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot + /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. + pub(super) fn style_defaults() -> TextValue { + let paint = |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); + dict(vec![ + ("37", val(2.0)), + ("41", val(2.0)), + ("43", dict(vec![("0", val(0.5))])), + ("44", val(2.0)), + ("45", val(2.0)), + ("46", val(7.0)), + ("49", val(-1.0)), + ("50", val(-1.0)), + ("55", class("SimpleBlender")), + ("64", val(4.0)), + ("71", val(4.0)), + ("77", TextValue::Bool(true)), + ("78", TextValue::Bool(true)), + ("79", paint(0.0)), + ("82", val(3.0)), + ("83", val(3.0)), + ("86", class("SimpleCustomFeature")), + ("87", val(100.0)), + ("88", TextValue::Bool(true)), + ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), + ("96", TextValue::Bool(true)), + ]) + } + + /// The same for the paragraph slots (110, 243-245). + pub(super) fn paragraph_defaults() -> TextValue { + dict(vec![ + ("6", val(1.0)), + ("7", val(1.2)), + ("10", val(6.0)), + ("11", val(2.0)), + ("12", val(2.0)), + ("14", val(36.0)), + ("15", TextValue::Bool(true)), + ("16", val(0.5)), + ("17", numbers(&[0.8, 1.0, 1.33])), + ("18", numbers(&[0.0, 0.0, 0.0])), + ("19", numbers(&[1.0, 1.0, 1.0])), + ("20", val(6.0)), + ("22", val(1.0)), + ("23", TextValue::Bool(true)), + ("24", val(1.0)), + ("26", TextValue::Bool(true)), + ("31", val(36.0)), + ("35", val(1.0)), + ("40", val(2.0)), + ]) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; + + /// A `LIST` of this type holding `kids`. + fn list(t: &[u8; 4], kids: Vec>) -> Chunk<'static> { + Chunk::list(*t, kids) + } + + /// A document with every kind of value, close to what After Effects writes: names, numbers, + /// a UTF-16 string with escapes, a plain string, nested dictionaries and arrays, an undecoded + /// style slot, a font list and a box frame. + const DOC: &[u8] = b"/0 << /1 << /0 [ << /0 << /0 << /0 (\xfe\xff\0M\0y\0r) >> /1 1 >> >> << /0 << /0 << /0 (\xfe\xff\0A\0\\)\0b) >> >> >> ] >> \ + /8 << /0 [ << /0 << /1 << /0 [ 0 0 10 0 10 5 0 5 ] >> /2 << /0 1 /1 2 >> >> >> ] >> >> \ + /1 << /1 [ << /0 << /0 (\xfe\xff\0H\0i\0\r) /5 << /0 [ << /0 << /0 << /5 << /0 2 /9 false /24 0 /34 7 /17 [ .8 1.0 1.33 ] >> >> >> /1 3 >> ] >> \ + /6 << /0 [ << /0 << /0 << /6 << /0 1 /1 36.5 /53 << /0 << /0 1 /1 [ 1 0.5 0 1 ] >> >> /79 << /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >> /68 1 /90 /x /91 (plain) /92 word >> >> >> /1 3 >> ] >> \ + /8 << /0 [ << /0 << /0 50 >> /1 2 >> << /1 1 >> ] >> >> >> ] >>"; + + /// A deterministic xorshift generator: the tests need varied input, not a dependency. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift). + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + usize::try_from(self.next() % (n as u64)).unwrap_or(0) + } + } + + /// The borrowing parser reads what the owning one read. + #[test] + fn the_borrowing_parser_reads_what_the_owning_one_read() { + // Every key form the owned keys canonicalise (`06`, `+6`, `-0`, `1.0`, `1e2`, strings, + // booleans, containers), names that are not UTF-8, escapes, unclosed strings and escapes, + // non-finite numbers, and bare words. + let parts: &[&[u8]] = &[ + b"<<", + b">>", + b"[", + b"]", + b"/0", + b"/1", + b"/nil", + b"/SimplePaint", + b"/\xff\xfe", + b"/\xc3\xa9", + b"/", + b"06", + b"+6", + b"-0", + b"1.0", + b"1.50", + b"1e2", + b"-2.5", + b".8", + b"inf", + b"nan", + b"1e999", + b"9223372036854775808", + b"true", + b"false", + b"(abc)", + b"(\xfe\xff\0A\0B)", + b"(\xfe\xff\0A\0)", + b"(a\\)b)", + b"(\\", + b"(\xfe\xff\\)\0)", + b"(\\\xfe\xff\0C)", + b"()", + b"(\xfe\xff)", + b"(unclosed", + b"word", + b"\xc3\x28", + b"\xff", + b" ", + b"\n", + b"\t\r", + b"<", + b">", + b")", + ]; + // A value: mostly well formed, with a random part in its place now and then. + /// Appends one generated value to `out`: mostly well formed, now and then a random part. + fn value(rng: &mut Rng, parts: &[&[u8]], depth: usize, out: &mut Vec) { + let pick = |rng: &mut Rng| parts.get(rng.below(parts.len())).copied().unwrap_or_default(); + match rng.below(if depth > 6 { 5 } else { 10 }) { + 0 => out.extend_from_slice(pick(rng)), + 1..=4 => out.extend_from_slice([&b"/0"[..], b"1", b"(\xfe\xff\0z)", b"0.5", b"/nil"].get(rng.below(5)).copied().unwrap_or_default()), + 5..=7 => { + out.extend_from_slice(b"<< "); + for _ in 0..rng.below(5) { + let key = if rng.below(4) == 0 { pick(rng) } else { [&b"/0"[..], b"/1", b"/6", b"/53"].get(rng.below(4)).copied().unwrap_or_default() }; + out.extend_from_slice(key); + out.push(b' '); + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.extend_from_slice(b">>"); + } + } + _ => { + out.extend_from_slice(b"[ "); + for _ in 0..rng.below(5) { + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.push(b']'); + } + } + } + } + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let (mut read, mut nested) = (0, 0); + for case in 0..40_000 { + let mut body = Vec::new(); + if case % 2 == 0 { + for _ in 0..rng.below(60) { + body.extend_from_slice(parts.get(rng.below(parts.len())).copied().unwrap_or_default()); + if rng.below(3) != 0 { + body.push(b' '); + } + } + } else { + for _ in 0..1 + rng.below(4) { + body.extend_from_slice(b"/0 "); + value(&mut rng, parts, 0, &mut body); + body.push(b'\n'); + } + } + let parsed = parse_text_dict(&body); + assert_eq!(parsed, oracle::parse_text_dict(&body), "case {case}: {}", String::from_utf8_lossy(&body)); + let _ = TextDocument::parse(&body); + if let TextValue::Dict(pairs) = &parsed { + read += usize::from(!pairs.is_empty()); + nested += usize::from(pairs.iter().any(|(_, v)| matches!(v, TextValue::Dict(d) if !d.is_empty()))); + } + } + // The input is not all rejected at the first token. + assert!(read > 25_000 && nested > 3_000, "{read} cases read something, {nested} a dictionary"); + // Every cut of a realistic document, and nesting past the limit. + for end in 0..=DOC.len() { + let body = DOC.get(..end).unwrap_or_default(); + assert_eq!(parse_text_dict(body), oracle::parse_text_dict(body), "cut at {end}"); + let _ = TextDocument::parse(body); + } + for depth in [63, 64, 65, 66, 200] { + let body = [b"/k ".to_vec(), b"<< /a ".repeat(depth), b"1".to_vec(), b" >>".repeat(depth), b" /z 2".to_vec()].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "depth {depth}"); + let body = [b"/k ".to_vec(), b"[ ".repeat(depth), b"]".repeat(depth)].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "array depth {depth}"); + } + // Dictionaries and arrays mixed (`<< /a [ << /a [ ...`), closed, and left open exactly + // at the limit. + for depth in 60..=68 { + let open: Vec = (0..depth).flat_map(|i| if i % 2 == 0 { &b"<< /a "[..] } else { &b"[ "[..] }).copied().collect(); + let close: Vec = (0..depth).rev().flat_map(|i| if i % 2 == 0 { &b" >>"[..] } else { &b" ]"[..] }).copied().collect(); + for body in [[b"/k ".to_vec(), open.clone(), b"1".to_vec(), close, b" /z 2".to_vec()].concat(), [b"/k ".to_vec(), open, b"1".to_vec()].concat()] { + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "mixed depth {depth}"); + let _ = TextDocument::parse(&body); + } + } + } + + /// A realistic document is read in full. + #[test] + fn a_realistic_document_is_read_in_full() { + let doc = TextDocument::parse(DOC).expect("a document"); + assert_eq!((doc.text.as_str(), doc.font_size, doc.fill_argb, doc.font.as_deref()), ("Hi", 36.5, [1.0, 0.5, 0.0, 1.0], Some("A)b"))); + assert_eq!(doc.fonts, ["Myr", "A)b"]); + assert_eq!(doc.paragraphs.first().map(|p| (p.length, p.justification, p.auto_hyphenate)), Some((3, Justification::Center, false))); + // `/68 1` and `/34 7` are neutral; `/90 /x`, `/91 (plain)` and `/92 word` differ from the + // (missing, neutral) default; `/79` equals its default. + assert_eq!(doc.undecoded_style_fields, ["90", "91", "92"]); + assert!(doc.undecoded_paragraph_fields.is_empty()); + assert_eq!(doc.kerning, [KernRun { length: 2, amount: Some(50.0) }, KernRun { length: 1, amount: None }]); + assert_eq!(doc.text_box, Some(TextBox { x: 0.0, y: 0.0, width: 10.0, height: 5.0 })); + assert_eq!(doc.line_orientation, Some(2)); + } + + /// The defaults are the sampled ones and built once. + #[test] + fn the_defaults_are_the_sampled_ones_and_built_once() { + assert_eq!(STYLE_DEFAULTS.clone().into_owned(), oracle::style_defaults()); + assert_eq!(PARAGRAPH_DEFAULTS.clone().into_owned(), oracle::paragraph_defaults()); + assert!(std::ptr::eq(&*STYLE_DEFAULTS, &*STYLE_DEFAULTS)); + } + + /// Names keys and strings are borrowed from the body. + #[test] + fn names_keys_and_strings_are_borrowed_from_the_body() { + let V::Dict(pairs) = parse_dict(b"/a /b /c (\xfe\xff\0x) /d (plain) /e word /f 1.5 /g (\\(esc) 07 1") else { panic!("a dict") }; + // Every `/name` key, name, string without escapes and bare word points into the body. + assert!(pairs.iter().take(6).all(|(k, _)| matches!(k, Cow::Borrowed(_)))); + assert!(matches!(pairs.first().map(|p| &p.1), Some(V::Name(b"b")))); + assert!(matches!(pairs.get(1).map(|p| &p.1), Some(V::Str(Cow::Borrowed(b"\0x"))))); + assert!(matches!(pairs.get(2).map(|p| &p.1), Some(V::Bytes(Cow::Borrowed(b"plain"))))); + assert!(matches!(pairs.get(3).map(|p| &p.1), Some(V::Token(b"word")))); + // An escaped string is copied; a number key is written in its canonical form. + assert!(matches!(pairs.get(5).map(|p| &p.1), Some(V::Bytes(Cow::Owned(b))) if b == b"(esc")); + assert_eq!(pairs.get(6).map(|p| p.0.as_ref()), Some("7")); + // Containers hold what was read. + let V::Dict(outer) = parse_dict(b"/0 << /a 1 /b 2 /c 3 /d 4 /e 5 >> /1 [ 1 2 3 4 5 6 7 8 9 ]") else { panic!("a dict") }; + assert!(matches!(outer.first().map(|p| &p.1), Some(V::Dict(d)) if d.len() == 5 && d.capacity() >= d.len())); + assert!(matches!(outer.get(1).map(|p| &p.1), Some(V::Array(a)) if a.len() == 9 && a.capacity() >= a.len())); + } + + /// Slot checks agree with the owned ones. + #[test] + fn slot_checks_agree_with_the_owned_ones() { + // Slots: defaults, decoded, neutral-listed and unknown ones. Values: each default as + // stored (also as `Int` where it is `2.0`), near misses, neutral values and every kind. + let slots = ["0", "6", "7", "17", "24", "34", "35", "37", "43", "55", "68", "77", "79", "86", "95", "96", "200", "x"]; + let values: &[&[u8]] = &[ + b"2", + b"2.0", + b"7", + b"1", + b"0", + b"-1", + b"0.5", + b"1.2", + b"36", + b"true", + b"false", + b"/nil", + b"/x", + b"/SimpleBlender", + b"[ ]", + b"<< >>", + b"[ .8 1 1.33 ]", + b"[ 0.8 1.0 ]", + b"[ 1 1 1 ]", + b"<< /0 .5 >>", + b"<< /0 0 /1 0 >>", + b"<< /1 0 /0 0 >>", + b"<< /99 /SimpleBlender >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 1 ] >> >>", + b"(plain)", + b"(\xfe\xff\0a)", + b"word", + b"/\xff", + ]; + let mut rng = Rng(0x2545_f491_4f6c_dd1d); + let mut listed = 0; + for case in 0..20_000 { + let mut body = b"/s << ".to_vec(); + for _ in 0..rng.below(12) { + body.push(b'/'); + body.extend_from_slice(slots.get(rng.below(slots.len())).copied().unwrap_or_default().as_bytes()); + body.push(b' '); + body.extend_from_slice(values.get(rng.below(values.len())).copied().unwrap_or_default()); + body.push(b' '); + } + body.extend_from_slice(b">>"); + let (new, old) = (parse_dict(&body), oracle::parse_text_dict(&body)); + let (Some(new), Some(old)) = (new.get("s"), old.get("s")) else { panic!("case {case}: a dict") }; + for (sampled, old_sampled, decoded, neutral) in [ + (&*STYLE_DEFAULTS, oracle::style_defaults(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES), + (&*PARAGRAPH_DEFAULTS, oracle::paragraph_defaults(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES), + ] { + let (mut a, mut b) = (Vec::new(), Vec::new()); + undecoded_fields(new, Some(sampled), decoded, neutral, &mut a); + oracle::undecoded_fields(old, Some(&old_sampled), decoded, neutral, &mut b); + assert_eq!(a, b, "case {case}: {}", String::from_utf8_lossy(&body)); + listed += a.len(); + } + // Every pair of values compares and reads as neutral alike. + if let (V::Dict(x), TextValue::Dict(y)) = (new, old) { + for ((_, p), (_, q)) in x.iter().zip(y) { + assert_eq!(is_neutral(p), oracle::is_neutral(q), "case {case}"); + for ((_, r), (_, t)) in x.iter().zip(y) { + assert_eq!(same(p, r), oracle::same(q, t), "case {case}"); + } + } + } + } + assert!(listed > 10_000, "{listed} slots listed"); + } + + /// Values compare by their text. + #[test] + fn values_compare_by_their_text() { + // Two names that are not UTF-8 read as the same text with replacement characters, as the + // owned values did. + assert!(same(&V::Name(b"\xff"), &V::Name(b"\xfe"))); + assert!(!same(&V::Name(b"a"), &V::Name(b"b"))); + assert!(same(&V::Str(Cow::Borrowed(b"\xd8\x00")), &V::Str(Cow::Owned(b"\xd8\x01".to_vec())))); + assert!(same(&V::Int(0), &V::Float(0.0)) && !same(&V::Int(0), &V::Bool(false)) && !same(&V::Name(b"x"), &V::Token(b"x"))); + assert!(is_neutral(&V::Name(b"nil")) && !is_neutral(&V::Token(b"nil")) && is_neutral(&V::Float(0.0)) && !is_neutral(&V::Str(Cow::Borrowed(b"")))); + } + + /// A layers marker list is the one in its root group. + #[test] + fn a_layers_marker_list_is_the_one_in_its_root_group() { + let marked = |tag: u8| list(b"mrst", vec![Chunk::data(*b"test", vec![tag])]); + // The stored place: `Layr/tdgp/mrst`. + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), marked(1)])]); + assert_eq!(Node::from(&layer).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[1u8][..])); + // A list deeper in the layer is not the layer's marker list (the old depth-first search + // found it); a layer without a marker list in its root group has no markers. + let deep = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(2)])])]); + assert!(Node::from(&deep).marker_list().is_none()); + assert!(Layer { chunk: Node::from(&deep) }.markers().is_empty()); + // Any other holder (a composition's `SecL`) is still searched depth-first. + let secl = list(b"SecL", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(3)])])]); + assert_eq!(Node::from(&secl).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[3u8][..])); + } + + /// The source text record is found in the text group. + #[test] + fn the_source_text_record_is_found_in_the_text_group() { + let doc = b"/1 << /1 [ << /0 << /0 (\xfe\xff\0H\0\r) /6 << /0 [ << /0 << /0 << /6 << /0 0 /1 50 /53 << /0 << /1 [ 1 1 0 0 ] >> >> >> >> >> /1 2 >> ] >> >> >> ] >>"; + // `btdk` is an opaque list (its body is not a chunk sequence), as the reader makes it. + let btdk = || Chunk { id: *b"LIST", body: crate::container::Body::OpaqueList { list_type: *b"btdk", data: doc.to_vec().into() }, pad: None }; + let btds_real = |n: usize| list(b"btds", std::iter::once(list(b"tdbs", vec![])).chain((0..n).map(|_| btdk())).collect()); + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), list(b"tdgp", vec![btds_real(2)])])]); + let view = Layer { chunk: Node::from(&layer) }; + assert_eq!(view.text_document().map(|d| d.text), Some("H".to_string())); + assert!(view.source_text_property().is_some()); + assert_eq!(view.text_document_count(), 2); + // A record anywhere else in the layer is not its Source Text. + let elsewhere = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![list(b"tdgp", vec![btds_real(1)])])])]); + let view = Layer { chunk: Node::from(&elsewhere) }; + assert!(view.text_document().is_none() && view.source_text_property().is_none()); + assert_eq!(view.text_document_count(), 0); + } #[test] fn non_finite_numbers_are_not_numbers() { @@ -1017,13 +1801,13 @@ mod tests { fn undecoded_fields_are_capped_and_linear() { // 300 000 distinct undecoded slots that differ from a sampled default: before, each was // checked against every slot recorded so far (4.5e10 compares) and all were listed. - let actual = TextValue::Dict((0..300_000).map(|i| (format!("{}", 9000 + i), TextValue::Int(1))).collect()); + let actual = V::Dict((0..300_000).map(|i| (Cow::Owned(format!("{}", 9000 + i)), V::Int(1))).collect()); let mut out = Vec::new(); undecoded_fields(&actual, None, &["0"], &[], &mut out); assert_eq!(out.len(), MAX_UNDECODED_FIELDS); assert_eq!(out.first().map(String::as_str), Some("9000")); // A slot listed once is not listed again. - let again = TextValue::Dict(vec![("9000".into(), TextValue::Int(1))]); + let again = V::Dict(vec![(Cow::Borrowed("9000"), V::Int(1))]); let mut short = vec!["9000".to_string()]; undecoded_fields(&again, None, &[], &[], &mut short); assert_eq!(short, ["9000"]); diff --git a/crates/aep/src/container/typed.rs b/crates/aep/src/container/typed.rs index a61c69090..7e7bfd0ec 100644 --- a/crates/aep/src/container/typed.rs +++ b/crates/aep/src/container/typed.rs @@ -4,42 +4,57 @@ //! //! Only facts that were established from our own one-change-per-file corpus and are checked by //! tests against it are decoded (see `docs/aep-format.md` for the evidence). Everything else -//! stays reachable through the raw [`Chunk`] tree. Views borrow the file and never fail loudly: -//! a chunk that does not have the expected shape yields `None` or is skipped. +//! stays reachable through the raw chunk tree ([`Node`]). Views borrow the file and never fail +//! loudly: a chunk that does not have the expected shape yields `None` or is skipped. +//! +//! A view reads either tree: the owned [`Chunk`](crate::container::Chunk) tree +//! ([`Project::new`]) or the flat [`TreeIndex`] the importer builds ([`Project::from_index`]); +//! both give the same answers. use crate::container::bytes::{be_f64, be_u16, be_u32, text, until_nul}; -use crate::container::chunk::{AepFile, Chunk}; +use crate::container::chunk::AepFile; +use crate::container::index::{Node, Root, TreeIndex}; /// The name a user gave a property or group: the `Utf8` inside its `tdsn` chunk. `None` for the /// sentinel `-_0_/-` ("no custom name") and for empty names. -pub(crate) fn user_name(chunk: &Chunk<'_>) -> Option { - let name = chunk.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; +pub(crate) fn user_name(chunk: Node<'_, '_>) -> Option { + let name = chunk.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (!name.is_empty() && name != "-_0_/-").then_some(name) } /// The content of a `Utf8` child as text (not trimmed). -pub(crate) fn utf8_child(chunk: &Chunk<'_>) -> Option { - chunk.child(b"Utf8").and_then(Chunk::bytes).map(text) +pub(crate) fn utf8_child(chunk: Node<'_, '_>) -> Option { + chunk.child(b"Utf8").and_then(Node::bytes).map(text) } /// The project: the top-level `LIST Fold`, which holds the project items. #[derive(Debug, Clone, Copy)] pub struct Project<'f, 'a> { - pub(crate) fold: &'f Chunk<'a>, + pub(crate) fold: Node<'f, 'a>, /// The top-level `LIST EfdG` (project-level effect definitions), when there is one. - pub(crate) effect_defs: Option<&'f Chunk<'a>>, + pub(crate) effect_defs: Option>, } impl<'f, 'a> Project<'f, 'a> { /// The project view of a file, or `None` when it has no `LIST Fold`. pub fn new(file: &'f AepFile<'a>) -> Option { - file.list(b"Fold").map(|fold| Self { fold, effect_defs: file.list(b"EfdG") }) + Self::of(Root::from(file)) + } + + /// The project view of an indexed file ([`TreeIndex`]), or `None` when it has no `LIST Fold`. + pub fn from_index(index: &'f TreeIndex<'a>) -> Option { + Self::of(Root::from(index)) + } + + /// The project view of either tree: its `LIST Fold` and `LIST EfdG`. + pub(crate) fn of(root: Root<'f, 'a>) -> Option { + root.list(b"Fold").map(|fold| Self { fold, effect_defs: root.list(b"EfdG") }) } /// The top-level items (the project panel's root; folders hold their children, see /// [`Item::children`]). pub fn items(&self) -> impl Iterator> + 'f { - self.fold.children().iter().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.fold.children().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// True when the `LIST Fold` could not be read as chunks (damaged): its items cannot be seen. @@ -78,12 +93,12 @@ pub enum ItemKind { /// A project item (`LIST Item`). #[derive(Debug, Clone, Copy)] pub struct Item<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Item<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -106,7 +121,7 @@ impl<'f, 'a> Item<'f, 'a> { /// The items inside a folder (`LIST Sfdr`). pub fn children(&self) -> impl Iterator> + 'f { - self.chunk.child_list(b"Sfdr").into_iter().flat_map(|sfdr| sfdr.children().iter()).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.chunk.child_list(b"Sfdr").into_iter().flat_map(Node::children).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// Whether the item has the `LIST Sfdr` of a folder's contents (After Effects writes an empty @@ -117,7 +132,7 @@ impl<'f, 'a> Item<'f, 'a> { /// True when the folder's `LIST Sfdr` is there but could not be read as chunks (damaged). pub fn folder_contents_unreadable(&self) -> bool { - self.chunk.child_list(b"Sfdr").is_some_and(Chunk::is_opaque_list) + self.chunk.child_list(b"Sfdr").is_some_and(Node::is_opaque_list) } /// The composition view when this item has composition data (`cdta`). @@ -135,6 +150,15 @@ pub struct TimeBase { } impl TimeBase { + /// The NTSC rates and the value After Effects stores for each (a float32 of exactly these + /// decimals): the one table the importer and the exporter use. + pub(crate) const NTSC: [(effectcraft_time::FrameRate, f32); 4] = [ + (effectcraft_time::FrameRate::FPS_23_976, 23.976), + (effectcraft_time::FrameRate::FPS_29_97, 29.97), + (effectcraft_time::FrameRate::FPS_59_94, 59.94), + (effectcraft_time::FrameRate::FPS_119_88, 119.88), + ]; + /// Frames per second (`timescale / frame_duration`), `None` for a zero duration. pub fn frame_rate(&self) -> Option { (self.frame_duration != 0).then(|| f64::from(self.timescale) / f64::from(self.frame_duration)) @@ -149,7 +173,7 @@ impl TimeBase { /// A composition item (an [`Item`] with `cdta`). #[derive(Debug, Clone, Copy)] pub struct Comp<'f, 'a> { - pub(crate) item: &'f Chunk<'a>, + pub(crate) item: Node<'f, 'a>, } impl<'f, 'a> Comp<'f, 'a> { @@ -163,19 +187,19 @@ impl<'f, 'a> Comp<'f, 'a> { /// The layers, top layer first (`LIST Layr`; the `DLay`/`SLay`/`CLay`/`SecL` lists next to /// them are not layers). pub fn layers(&self) -> impl Iterator> + 'f { - self.item.children().iter().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) + self.item.children().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) } } /// A layer (`LIST Layr`). #[derive(Debug, Clone, Copy)] pub struct Layer<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Layer<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -205,12 +229,12 @@ pub enum PropNode<'f, 'a> { /// A property (`LIST tdbs`). Property(Property<'f, 'a>), /// Any other list that follows a match name (observed: `otst`, the effect instance `sspc`). - Other(&'f Chunk<'a>), + Other(Node<'f, 'a>), } impl<'f, 'a> PropNode<'f, 'a> { /// The node a `LIST` after a match name is. - fn of(c: &'f Chunk<'a>) -> Self { + fn of(c: Node<'f, 'a>) -> Self { if c.is_list(b"tdgp") { PropNode::Group(PropGroup { chunk: c }) } else if c.is_list(b"tdbs") { @@ -233,17 +257,18 @@ pub struct PropEntry<'f, 'a> { /// `ADBE Group End` closes the group and names nothing. #[derive(Debug, Clone, Copy)] pub struct PropGroup<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> PropGroup<'f, 'a> { - /// A group view of a `LIST tdgp` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A group view of a `LIST tdgp` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdgp").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -264,7 +289,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes().map(match_name); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(name) = pending.take() { out.push(PropEntry { match_name: name, node: PropNode::of(c) }); @@ -285,9 +310,9 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes(); - } else if c.id == crate::container::chunk::LIST + } else if c.is(&crate::container::chunk::LIST) && let Some(body) = pending.take() - && match_name(body) == name + && match_name_is(body, name) { return Some(PropNode::of(c)); } @@ -314,21 +339,30 @@ fn match_name(body: &[u8]) -> String { text(until_nul(body)) } +/// `match_name(body) == name` without building the `String`: equal bytes match; otherwise only +/// a body that is not valid UTF-8 can still match (through its U+FFFD replacements), and only a +/// name that holds U+FFFD. +fn match_name_is(body: &[u8], name: &str) -> bool { + let bytes = until_nul(body); + bytes == name.as_bytes() || (name.contains('\u{FFFD}') && String::from_utf8_lossy(bytes) == name) +} + /// A property (`LIST tdbs`): `tdsb`, `tdsn`, the 124-byte header `tdb4`, then either the static /// value `cdat` or a keyframe `LIST list`. #[derive(Debug, Clone, Copy)] pub struct Property<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Property<'f, 'a> { - /// A property view of a `LIST tdbs` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A property view of a `LIST tdbs` chunk (a [`Chunk`](crate::container::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdbs").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -436,12 +470,13 @@ pub struct KeyLayout { impl<'f> KeyList<'f> { /// A view of a `LIST list` chunk. `None` when `lhd3` is not 52 bytes or `ldat` is not exactly /// `count * record_size` bytes (a list without `ldat` must have no records). - pub fn from_chunk<'a>(chunk: &'f Chunk<'a>) -> Option { + pub fn from_chunk<'a: 'f>(chunk: impl Into>) -> Option { + let chunk = chunk.into(); if !chunk.is_list(b"list") { return None; } let header = ListHeader::parse(chunk.child(b"lhd3")?.bytes()?)?; - let ldat: &[u8] = chunk.child(b"ldat").and_then(Chunk::bytes).unwrap_or(&[]); + let ldat: &[u8] = chunk.child(b"ldat").and_then(Node::bytes).unwrap_or(&[]); let expected = usize::try_from(header.count).ok()?.checked_mul(usize::try_from(header.record_size).ok()?)?; (ldat.len() == expected).then_some(Self { header, ldat, layout: None }) } @@ -500,6 +535,7 @@ impl<'f> Keyframe<'f> { #[cfg(test)] mod tests { use super::*; + use crate::container::Chunk; fn key_list(count: u32, record_size: u32, ldat: &[u8]) -> Chunk<'static> { let mut lhd3 = vec![0u8; 52]; @@ -536,4 +572,22 @@ mod tests { assert!(g.get("C").is_none()); assert_eq!(g.names(), ["A", "B", "A"]); } + + /// Match name is agrees with comparing the decoded name. + #[test] + fn match_name_is_agrees_with_comparing_the_decoded_name() { + let bodies: [&[u8]; 9] = + [b"ADBE Position\0\0\0", b"ADBE Position", b"ADBE Pos", b"", b"\0ADBE", b"ADBE \xff Bad\0", b"\xff", b"\xef\xbf\xbd", "Gr\u{fc}n\0".as_bytes()]; + let names = ["ADBE Position", "ADBE Pos", "", "ADBE \u{FFFD} Bad", "\u{FFFD}", "Gr\u{fc}n", "ADBE Position\0", "ADBE \u{FFFD}"]; + for body in bodies { + for name in names { + assert_eq!(match_name_is(body, name), match_name(body) == name, "{body:?} vs {name:?}"); + } + } + // Names read back from a group find their entries, a damaged one included. + let group = Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdmn", b"X\xffY\0".to_vec()), Chunk::list(*b"tdbs", Vec::new())]); + let g = PropGroup::from_chunk(&group).expect("group"); + let names = g.names(); + assert!(g.get(&names[0]).is_some()); + } } diff --git a/crates/aep/src/container/values.rs b/crates/aep/src/container/values.rs index a690751a2..ddd5caa75 100644 --- a/crates/aep/src/container/values.rs +++ b/crates/aep/src/container/values.rs @@ -8,6 +8,7 @@ //! 100 %. These defaults are verified for AV layers (solids); cameras and lights use other //! conventions (see the `camera` module). +use crate::container::bytes::be_f64; use crate::container::typed::{Layer, Project, PropGroup, PropNode}; /// Where a [`Tracked`] value came from. @@ -192,8 +193,7 @@ impl<'f, 'a> Layer<'f, 'a> { let otda = otst.child_list(b"otky")?.child(b"otda")?.bytes()?; let mut degrees = [0.0; 3]; for (i, slot) in degrees.iter_mut().enumerate() { - let b = otda.get(i * 8..i * 8 + 8)?; - *slot = f64::from_be_bytes(b.try_into().ok()?); + *slot = be_f64(otda, i * 8)?; } Some(Orientation { degrees, @@ -223,12 +223,10 @@ impl<'f, 'a> Layer<'f, 'a> { .child_list(b"otky") .map(|k| { k.children() - .iter() .filter(|c| c.is(b"otda")) .map(|c| { let b = c.bytes()?; - let at = |i: usize| b.get(i * 8..i * 8 + 8).and_then(|s| s.try_into().ok()).map(f64::from_be_bytes); - Some([at(0)?, at(1)?, at(2)?]) + Some([be_f64(b, 0)?, be_f64(b, 8)?, be_f64(b, 16)?]) }) .collect() }) diff --git a/crates/aep/src/container/version.rs b/crates/aep/src/container/version.rs index bf3fbf4c4..407c7046c 100644 --- a/crates/aep/src/container/version.rs +++ b/crates/aep/src/container/version.rs @@ -8,8 +8,9 @@ use std::fmt; -use crate::container::bytes::{be_u16, be_u32}; +use crate::container::bytes::be_u32; use crate::container::chunk::AepFile; +use crate::container::index::{Root, TreeIndex}; /// An After Effects version as packed in `head` / `svap`: `26.2.1x2` is major 26, minor 2, bug-fix /// 1, build 2 (the form of the scripting API's `app.version`). @@ -71,21 +72,36 @@ impl AepFile<'_> { /// version there, and the older version for File ▸ Save a Copy As 25.x / 24.x. `None` when the /// chunk is missing, short or holds no version. pub fn file_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + Root::from(self).file_version() } /// The version of the After Effects that wrote the file: `svap` bytes 0 to 3 (the same word in a /// down-saved copy as in a normal save). `None` when the chunk is missing (After Effects opens such /// a file), short or holds no version. pub fn saved_by_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) + Root::from(self).saved_by_version() + } +} + +impl TreeIndex<'_> { + /// [`AepFile::file_version`] of the indexed file. + pub fn file_version(&self) -> Option { + Root::from(self).file_version() } - /// The `u16` at `head` byte 0: 97 (`0x61`) in a 2026 file, 96 in a copy saved as 25.x, 95 as - /// 24.x. It steps with the major version of [`AepFile::file_version`]; what it counts is not - /// known, so nothing is derived from it. - pub fn head_revision(&self) -> Option { - be_u16(self.chunk(b"head")?.bytes()?, 0) + /// [`AepFile::saved_by_version`] of the indexed file. + pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl Root<'_, '_> { + pub(crate) fn file_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + } + + pub(crate) fn saved_by_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) } } diff --git a/crates/aep/src/container/write.rs b/crates/aep/src/container/write.rs index 3ccab50a4..9f401b3d4 100644 --- a/crates/aep/src/container/write.rs +++ b/crates/aep/src/container/write.rs @@ -17,6 +17,18 @@ const MAX_WRITE_DEPTH: usize = 256; /// [`AepError::TooDeep`] for an absurdly deep hand-built tree. pub fn write(file: &AepFile<'_>) -> Result, AepError> { let mut out = Vec::new(); + // Sized once to the exact output. A tree past the 32-bit container size fails at once (the + // same error the write below would give after serialising it). `try_reserve` so a hand-built + // tree that repeats one large borrowed body many times cannot abort on the allocation; the + // writer then grows as it goes, as before. + if let Some(len) = sequence_len(&file.chunks, 1) { + if u32::try_from(len.saturating_add(4)).is_err() { + return Err(AepError::TooLarge); + } + if let Some(total) = len.checked_add(12).and_then(|n| n.checked_add(file.trailer.len())) { + let _ = out.try_reserve_exact(total); + } + } out.extend_from_slice(b"RIFX"); out.extend_from_slice(&[0; 4]); out.extend_from_slice(b"Egg!"); @@ -56,6 +68,28 @@ fn write_sequence(out: &mut Vec, chunks: &[Chunk<'_>], depth: usize) -> Resu Ok(()) } +/// The number of bytes [`write_sequence`] produces for `chunks`; `None` past the depth limit or +/// on overflow (both make the write fail). +fn sequence_len(chunks: &[Chunk<'_>], depth: usize) -> Option { + if depth > MAX_WRITE_DEPTH { + return None; + } + let count = chunks.len(); + let mut total = 0usize; + for (i, chunk) in chunks.iter().enumerate() { + let body = match &chunk.body { + Body::Data(d) => d.len(), + Body::Chunks(c) => sequence_len(c, depth + 1)?, + Body::List { children, .. } => sequence_len(children, depth + 1)?.checked_add(4)?, + Body::OpaqueList { data, .. } => data.len().checked_add(4)?, + }; + // The same pad rule as `write_chunk`. + let pad = usize::from(body % 2 == 1 && (chunk.pad.is_some() || i + 1 != count)); + total = total.checked_add(8)?.checked_add(body)?.checked_add(pad)?; + } + Some(total) +} + fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize) -> Result<(), AepError> { out.extend_from_slice(&chunk.id); let size_at = out.len(); @@ -84,3 +118,62 @@ fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize } Ok(()) } + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::*; + + /// The output is sized once to its exact length. + #[test] + fn the_output_is_sized_once_to_its_exact_length() { + let odd = |n: u8| Chunk::data(*b"cdat", vec![n; usize::from(n)]); + let mut padded = odd(3); + padded.pad = Some(0x7f); + let trees = [ + AepFile { chunks: Vec::new(), trailer: Cow::Borrowed(&[]) }, + AepFile { + chunks: vec![ + odd(1), + Chunk::list(*b"Fold", vec![odd(2), odd(3), Chunk::list(*b"tdgp", vec![odd(5)])]), + Chunk { id: *b"LIST", body: Body::OpaqueList { list_type: *b"btdk", data: Cow::Owned(vec![1, 2, 3]) }, pad: None }, + Chunk { id: *b"tdmn", body: Body::Chunks(vec![odd(7), padded]), pad: None }, + odd(9), + ], + trailer: Cow::Borrowed(b""), + }, + ]; + for file in &trees { + let out = write(file).expect("write"); + assert_eq!(sequence_len(&file.chunks, 1).map(|n| n + 12 + file.trailer.len()), Some(out.len())); + // Exact with std's allocator today; grown push by push it would be rounded up. + assert_eq!(out.capacity(), out.len(), "the output grew past its size"); + } + } + + /// A tree past the container size fails before serialising. + #[cfg(target_pointer_width = "64")] + #[test] + fn a_tree_past_the_container_size_fails_before_serialising() { + // 4097 chunks borrowing one 1 MiB body: over 4 GiB of output, none of it written. + let body = vec![0u8; 1 << 20]; + let chunks: Vec> = (0..4097).map(|_| Chunk::data(*b"cdat", &body[..])).collect(); + let file = AepFile { chunks, trailer: Cow::Borrowed(&[]) }; + assert_eq!(write(&file), Err(AepError::TooLarge)); + // Just under the limit still sizes, and fails nothing up front. + assert!(sequence_len(&file.chunks[..4000], 1).is_some_and(|n| u32::try_from(n + 4).is_ok())); + } + + /// A tree past the depth limit has no size. + #[test] + fn a_tree_past_the_depth_limit_has_no_size() { + let mut deep = Chunk::data(*b"cdat", vec![0]); + for _ in 0..=MAX_WRITE_DEPTH { + deep = Chunk::list(*b"tdgp", vec![deep]); + } + let file = AepFile { chunks: vec![deep], trailer: Cow::Borrowed(&[]) }; + assert_eq!(sequence_len(&file.chunks, 1), None); + assert_eq!(write(&file), Err(AepError::TooDeep)); + } +} diff --git a/crates/aep/src/export.rs b/crates/aep/src/export.rs index 12f10aa0c..992dfecb3 100644 --- a/crates/aep/src/export.rs +++ b/crates/aep/src/export.rs @@ -26,6 +26,7 @@ mod footage; mod fx_defs; mod fxsynth; pub(crate) mod icc; +mod keys; mod markers; mod masks; mod plan; diff --git a/crates/aep/src/export/base.rs b/crates/aep/src/export/base.rs index d483cefb7..05aa15891 100644 --- a/crates/aep/src/export/base.rs +++ b/crates/aep/src/export/base.rs @@ -6,15 +6,20 @@ //! (read-backs t-no, t-zcdta, u-zsspc, u-noclrs, u-noftgi, u-nofrec, u-nocdrp). use crate::Chunk; -use crate::export::synth::{tdmn, tdsn}; +use crate::export::synth::{GROUP_END, tdmn, tdsn}; +use crate::export::tree::Fixed; -/// A zero buffer of `len` bytes with `fields` (offset, bytes) written into it. +/// A zero buffer of `len` bytes with `fields` (offset, bytes) written into it. The layouts are +/// fixed in code ([`Fixed`]). pub(crate) fn bytes(len: usize, fields: &[(usize, &[u8])]) -> Vec { - let mut b = vec![0u8; len]; + layout(len, fields).finish(|| format!("base::bytes ({len} bytes, fields at {:?})", fields.iter().map(|(at, v)| (*at, v.len())).collect::>())) +} + +/// The fixed layout of `len` zero bytes with `fields` written at their offsets. +fn layout(len: usize, fields: &[(usize, &[u8])]) -> Fixed { + let mut b = Fixed::zeroed(len); for (at, v) in fields { - if let Some(dst) = b.get_mut(*at..at.saturating_add(v.len())) { - dst.copy_from_slice(v); - } + b.put(*at, v); } b } @@ -52,12 +57,8 @@ pub(crate) fn project_chunks() -> Vec> { // colour depth (2: 32 bits); the save noise after the code page is left zero. let nhed = bytes(32, &[(7, &[5]), (9, &[1]), (12, &[0x1e, 16, 2, 2]), (18, &UTF8_CODE_PAGE)]); let nnhd = bytes(40, &[(7, &[5]), (9, &[1]), (15, &[0x1e]), (19, &[16, 2]), (24, &[2]), (26, &UTF8_CODE_PAGE)]); - let mut lhd3 = bytes(52, &[(0, &0x00d0_0bee_u32.to_be_bytes()), (12, &1u32.to_be_bytes()), (16, &RQ_ITEM_LEN.to_be_bytes())]); - for at in [20, 24, 28] { - if let Some(dst) = lhd3.get_mut(at..at + 4) { - dst.copy_from_slice(&1u32.to_be_bytes()); - } - } + let one = 1u32.to_be_bytes(); + let lhd3 = bytes(52, &[(0, &0x00d0_0bee_u32.to_be_bytes()), (12, &one), (16, &RQ_ITEM_LEN.to_be_bytes()), (20, &one), (24, &one), (28, &one)]); let wsnm: Vec = "Default".encode_utf16().flat_map(u16::to_be_bytes).collect(); vec![ Chunk::data(*b"svap", VERSION.to_vec()), @@ -157,30 +158,17 @@ fn view_layer(ty: &[u8; 4], name: &str, label: u8, kind: u8) -> Vec = + vec![(8, &one), (16, &base), (24, &scale), (28, &out), (32, &scale), (61, &label), (64, &name_copy), (108, &one), (128, &kind_word)]; + fields.extend_from_slice(flags); + let ldta = bytes(164, &fields); + let root = vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn(""), tdmn(GROUP_END)]; let layer = Chunk::list(*ty, vec![Chunk::data(*b"ldta", ldta), utf8(name), Chunk::list(*b"tdgp", root), guides()]); let mut out = vec![layer, Chunk::list(*b"Ewst", Vec::new())]; out.extend(crate::export::skeleton::f_records()); @@ -293,6 +281,16 @@ pub(crate) fn solid_item() -> Chunk<'static> { mod tests { use super::*; + /// A field out of range is an error. + #[test] + fn a_field_out_of_range_is_an_error() { + let ok = layout(4, &[(0, &[1]), (3, &[2])]); + assert!(ok.error().is_none()); + assert_eq!(ok.finish(String::new), [1, 0, 0, 2]); + assert!(layout(4, &[(3, &[1, 2])]).error().is_some()); + assert!(layout(4, &[(usize::MAX, &[1])]).error().is_some()); + } + /// The chunks After Effects refuses a file without, or crashes on (aep-export.md, "What After /// Effects needs"). #[test] diff --git a/crates/aep/src/export/camlight.rs b/crates/aep/src/export/camlight.rs index 597bf800a..a8958025f 100644 --- a/crates/aep/src/export/camlight.rs +++ b/crates/aep/src/export/camlight.rs @@ -13,7 +13,8 @@ use effectcraft_keyframe::Value; use effectcraft_project::{Layer, LightKind}; use crate::export::report::{Collector, LossCategory}; -use crate::export::tree::{R, child_data, kids_mut, list_mut, put_f64, tpl}; +use crate::export::synth::{GROUP_END, is_tdmn, tdmn}; +use crate::export::tree::{R, child_data, kids_mut, list_mut, put_f64, split_group, tpl}; /// A camera's options in the file's units (pixels, percent, a switch). #[derive(Clone, Debug, PartialEq)] @@ -183,50 +184,24 @@ const LIGHT_ORDER: &[&str] = &[ "ADBE Casts Shadows", ]; -pub(crate) fn tdmn_chunk(name: &str) -> Chunk<'static> { - let mut b = vec![0u8; 40]; - for (i, c) in name.bytes().take(39).enumerate() { - b[i] = c; - } - Chunk::data(*b"tdmn", b) -} - -fn tdmn_name(c: &Chunk<'_>) -> Option { - let b = c.bytes()?; - let end = b.iter().position(|x| *x == 0).unwrap_or(b.len()); - std::str::from_utf8(b.get(..end)?).ok().map(str::to_string) -} - /// Sets (or inserts) the records of an options group (`tdmn` named `group`) of a layer. fn put_group(layr: &mut Chunk<'static>, group: &str, order: &[&str], set: Vec<(&str, Chunk<'static>)>, remove: &[&str]) -> R<()> { let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(group)).ok_or_else(|| tpl("options group missing"))?; + let at = kids.iter().position(|c| is_tdmn(c, group)).ok_or_else(|| tpl("options group missing"))?; let g = kids.get_mut(at + 1).ok_or_else(|| tpl("options group without a list"))?; let gk = kids_mut(g)?; - let first = gk.iter().position(|c| c.is(b"tdmn")).unwrap_or(gk.len()); - let header: Vec> = gk.drain(..first).collect(); - let mut pairs: Vec<(String, Vec>)> = Vec::new(); - let mut it = std::mem::take(gk).into_iter(); - while let Some(m) = it.next() { - let name = tdmn_name(&m).unwrap_or_default(); - if name == "ADBE Group End" { - pairs.push((name, vec![m])); - continue; - } - let Some(node) = it.next() else { break }; - // An atom group (Light Env Atom) carries its own Group End inside the list. - pairs.push((name, vec![m, node])); - } + // An atom group (Light Env Atom) carries its own Group End inside its list. + let (header, mut pairs) = split_group(gk)?; pairs.retain(|(n, _)| !remove.contains(&n.as_str())); for (name, rec) in set { - let entry = vec![tdmn_chunk(name), rec]; + let entry = vec![tdmn(name), rec]; match pairs.iter_mut().find(|(n, _)| n == name) { Some(slot) => slot.1 = entry, None => pairs.push((name.to_string(), entry)), } } - let rank = |n: &str| if n == "ADBE Group End" { usize::MAX } else { order.iter().position(|g| *g == n).unwrap_or(order.len()) }; + let rank = |n: &str| if n == GROUP_END { usize::MAX } else { order.iter().position(|g| *g == n).unwrap_or(order.len()) }; pairs.sort_by_key(|(n, _)| rank(n)); gk.extend(header); gk.extend(pairs.into_iter().flat_map(|(_, v)| v)); diff --git a/crates/aep/src/export/emit.rs b/crates/aep/src/export/emit.rs index e7423efe9..f7b0ec6b8 100644 --- a/crates/aep/src/export/emit.rs +++ b/crates/aep/src/export/emit.rs @@ -6,8 +6,10 @@ use std::collections::HashMap; use crate::{AepFile, Chunk}; use effectcraft_project::{ItemId, LayerId}; +use crate::export::keys::fill_key_list; use crate::export::plan::{CompPlan, Entry, ItemPlan, KeyPlan, LayerPlan, LayerSrc, ProjectPlan, PropKind, PropPlan, SolidPlan, natural_key}; -use crate::export::template::Template; +use crate::export::synth::{is_tdmn, tdmn_name}; +use crate::export::template::{self, Template}; use crate::export::tree::{ R, child_data, data_mut, kids_mut, list_mut, put, put_bits, put_cstr, put_f32, put_f64, put_i32, put_u8, put_u16, put_u32, set_utf8, tpl, }; @@ -58,10 +60,14 @@ struct Cx<'t> { type Sorted = (String, u64, Vec>); pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { - let t = Template::load()?; + emit_with(template::shared()?, plan, extra) +} + +/// [`emit`] with the prototypes of `t`. +fn emit_with(t: &Template, plan: &ProjectPlan, extra: &[u8]) -> R> { let special = plan.root.iter().find(|e| e.name == "Solids" && matches!(e.plan, ItemPlan::Folder(_))).map(|e| e.ec); let mut cx = Cx { - t: &t, + t, ids: Ids { next: FIRST_ID }, solid_ids: HashMap::new(), comp_ids: HashMap::new(), @@ -77,16 +83,14 @@ pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { let mut root: Vec = Vec::new(); build_level(&plan.root, &mut cx, &ids_by_entry, &mut |name, key, chunks| root.push((name, key, chunks)))?; - // The Solids folder: root-level solids and the hidden solids of null layers. - let mut solids_children: Vec = Vec::new(); - for (name, key, chunks) in root.iter().filter(|(_, k, _)| *k & ROOT_SOLID != 0) { - solids_children.push((name.clone(), *key, chunks.clone())); - } - root.retain(|(_, k, c)| *k & ROOT_SOLID == 0 && !c.is_empty()); - for (i, (name, chunk)) in cx.extra_solids.iter().enumerate() { - let mut v = vec![chunk.clone()]; + // The Solids folder: root-level solids and the hidden solids of null layers (moved, not + // copied: the root keeps neither). + let (mut solids_children, mut root): (Vec, Vec) = root.into_iter().partition(|(_, k, _)| *k & ROOT_SOLID != 0); + root.retain(|(_, _, c)| !c.is_empty()); + for (i, (name, chunk)) in std::mem::take(&mut cx.extra_solids).into_iter().enumerate() { + let mut v = vec![chunk]; v.extend(cx.t.fset.iter().cloned()); - solids_children.push((name.clone(), i as u64, v)); + solids_children.push((name, i as u64, v)); } let mut folder = match cx.special_slot.take() { Some((item, inner)) => { @@ -125,26 +129,57 @@ pub(crate) fn emit(plan: &ProjectPlan, extra: &[u8]) -> R> { /// Marks a root entry that goes into the `Solids` folder. const ROOT_SOLID: u64 = 1 << 63; +/// Sorts by name in natural order, then by the tie-break key. Each name's sort key is built once, +/// not on every comparison; the sort is stable, as `sort_by` is. fn sort_named(v: &mut [Sorted]) { - v.sort_by(|a, b| natural_cmp(&a.0, &b.0).then(a.1.cmp(&b.1))); + v.sort_by_cached_key(|(name, key, _)| (natural_bytes(name), *key)); } -fn natural_cmp(a: &str, b: &str) -> std::cmp::Ordering { - let (ka, kb) = (natural_key(a), natural_key(b)); - for (x, y) in ka.iter().zip(kb.iter()) { - let c = match (x.0, y.0) { - (true, true) => x.2.cmp(&y.2).then_with(|| x.1.cmp(&y.1)), - (false, false) => x.1.cmp(&y.1), - (true, false) => std::cmp::Ordering::Less, - (false, true) => std::cmp::Ordering::Greater, - }; - if c != std::cmp::Ordering::Equal { - return c; +/// The [`natural_key`] of `name` as one byte string whose plain byte order is the natural order: +/// segment by segment, a number before text, numbers by value and then by their digit strings +/// byte by byte (`01` before `1`), text by its lowercase string, and a key that is a prefix of +/// the other first. +/// +/// - A number: `0x01`, the count of significant bytes of its value, those bytes big-endian, its +/// digits, `0x00` (digits are never `0x00`, so a digit string that is a prefix of another, as +/// `0` of `00`, sorts first). +/// - Text: `0x02`, its lowercase UTF-8 with each `0x00` written as `0x00 0x01`, then `0x00 0x00`. +/// +/// At most 5 bytes per byte of `name` (`NATURAL_BYTES_PER_NAME_BYTE`), so the cached keys stay +/// within a small multiple of the names they sort (one segment as a `(bool, String, u128)` takes +/// 48 bytes plus its string, which is why the segments themselves are not cached). +fn natural_bytes(name: &str) -> Vec { + let mut out = Vec::with_capacity(name.len().saturating_add(4)); + for (number, text, value) in natural_key(name) { + if number { + out.push(0x01); + let be = value.to_be_bytes(); + let zeros = be.iter().take_while(|b| **b == 0).count(); + let significant = be.get(zeros..).unwrap_or_default(); + out.push(significant.len() as u8); + out.extend_from_slice(significant); + out.extend_from_slice(text.as_bytes()); + out.push(0x00); + } else { + out.push(0x02); + for b in text.bytes() { + out.push(b); + if b == 0 { + out.push(0x01); + } + } + out.extend_from_slice(&[0x00, 0x00]); } } - ka.len().cmp(&kb.len()) + out } +/// The bound on [`natural_bytes`]: a run of `d` digits takes at most `3 + 2d` bytes (its value +/// never needs more bytes than digits), a text run of `b` bytes at most `3 + 2b` (lowercasing and +/// the NUL escape at most double a character), and every run is at least one byte long. +#[cfg(test)] +const NATURAL_BYTES_PER_NAME_BYTE: usize = 5; + fn assign_ids(entries: &[Entry], cx: &mut Cx<'_>, map: &mut HashMap) -> R<()> { // Iterative to bound the stack by the folder depth the plan already limits. let mut stack: Vec<&Entry> = entries.iter().rev().collect(); @@ -404,8 +439,7 @@ fn put_used_comps(cx: &Cx<'_>, item: &mut Chunk<'static>, cp: &CompPlan) -> R<() fn put_time_remap_max(layr: &mut Chunk<'static>, seconds: f64) -> R<()> { let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = - kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; let tdbs = kids.get_mut(at + 1).ok_or_else(|| tpl("Time Remapping without a record"))?; put_f64(child_data(tdbs, b"tduM")?, 0, seconds) } @@ -420,8 +454,7 @@ fn put_time_remap_max(layr: &mut Chunk<'static>, seconds: f64) -> R<()> { fn put_time_remap(cx: &Cx<'_>, layr: &mut Chunk<'static>, remap: &PropPlan) -> R<()> { let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = - kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Time Remapping")).ok_or_else(|| tpl("layer without Time Remapping"))?; let tdbs = kids.get_mut(at + 1).ok_or_else(|| tpl("Time Remapping without a record"))?; put_u32(child_data(tdbs, b"tdsb")?, 0, 1)?; { @@ -454,21 +487,12 @@ fn put_time_remap(cx: &Cx<'_>, layr: &mut Chunk<'static>, remap: &PropPlan) -> R .find(|k| matches!(&k.body, crate::Body::List { list_type, .. } if list_type == b"list")) .cloned() .ok_or_else(|| tpl("keyed prototype without a list"))?; - let n = u32::try_from(remap.keys.len()).map_err(|_| tpl("too many keys"))?; let size = 48usize; - { - let lhd3 = child_data(&mut list, b"lhd3")?; - crate::export::synth::put_key_counts(lhd3, n, 48)?; - } let mut ldat = Vec::with_capacity(size * remap.keys.len()); for k in &remap.keys { ldat.extend(plain_record(1, k, size)?); } - let ld = kids_mut(&mut list)?; - match ld.iter_mut().find(|c| c.is(b"ldat")) { - Some(c) => *c = Chunk::data(*b"ldat", ldat), - None => ld.push(Chunk::data(*b"ldat", ldat)), - } + fill_key_list(&mut list, ldat, remap.keys.len(), size)?; if let Some(slot) = rk.get_mut(cdat_at) { *slot = list; } @@ -616,10 +640,7 @@ fn put_masks(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan) -> R<()> { let parade = crate::export::masks::build_parade(&cx.t.masks, &lp.masks, lp.source_size)?; let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; kids.splice(at..at, parade); Ok(()) } @@ -633,10 +654,7 @@ fn put_effects(cx: &mut Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, layer let parade = crate::export::effects::build_parade(&cx.t.effects, &lp.fx, layer_id, layer_ids, lp.source_size, &mut cx.fx_use)?; let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; kids.splice(at..at, parade); Ok(()) } @@ -659,11 +677,6 @@ const GROUP_ORDER: [&str; 13] = [ "ADBE Group End", ]; -fn tdmn_name(c: &Chunk<'_>) -> Option { - let d = c.bytes()?; - Some(d.iter().take_while(|b| **b != 0).map(|b| char::from(*b)).collect()) -} - /// Inserts the properties of `lp` into the layer's Transform group, in the order the file uses. fn put_properties(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, cp: &CompPlan) -> R<()> { if lp.props.is_empty() { @@ -671,20 +684,19 @@ fn put_properties(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, cp: &C } let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Transform Group")) - .ok_or_else(|| tpl("layer without a Transform group"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Transform Group")).ok_or_else(|| tpl("layer without a Transform group"))?; let group = kids.get_mut(at + 1).ok_or_else(|| tpl("Transform group without a list"))?; let gk = kids_mut(group)?; // Split the group into its header (up to the first match name) and (tdmn, node) pairs. let first = gk.iter().position(|c| c.is(b"tdmn")).ok_or_else(|| tpl("empty Transform group"))?; let header: Vec> = gk.drain(..first).collect(); - let mut pairs: Vec<(String, Vec>)> = Vec::new(); + let mut pairs: Vec<(Vec, Vec>)> = Vec::new(); let mut it = std::mem::take(gk).into_iter(); while let Some(m) = it.next() { - let name = tdmn_name(&m).unwrap_or_default(); - if name == "ADBE Group End" { + // The match name as the shared decoder reads it (lossy UTF-8 up to the first NUL). Every + // name looked up below is ASCII, so a name matches exactly when its bytes do. + let name = tdmn_name(&m).unwrap_or_default().into_bytes(); + if name == b"ADBE Group End" { pairs.push((name, vec![m])); continue; } @@ -694,22 +706,19 @@ fn put_properties(cx: &Cx<'_>, layr: &mut Chunk<'static>, lp: &LayerPlan, cp: &C for p in &lp.props { let name = p.kind.match_name(); let built = build_prop(cx, p, lp, cp)?; - let tdmn = pairs.iter().find(|(n, _)| n == name).and_then(|(_, v)| v.first().cloned()).unwrap_or_else(|| { - cx.t.stat - .iter() - .chain(cx.t.keyd.iter()) - .find(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(name)) - .cloned() - .unwrap_or_else(|| Chunk::data(*b"tdmn", Vec::new())) + let tdmn = pairs.iter().find(|(n, _)| n == name.as_bytes()).and_then(|(_, v)| v.first().cloned()).unwrap_or_else(|| { + cx.t.stat.iter().chain(cx.t.keyd.iter()).find(|c| is_tdmn(c, name)).cloned().unwrap_or_else(|| Chunk::data(*b"tdmn", Vec::new())) }); let entry = vec![tdmn, built]; - if let Some(slot) = pairs.iter_mut().find(|(n, _)| n == name) { + if let Some(slot) = pairs.iter_mut().find(|(n, _)| n == name.as_bytes()) { slot.1 = entry; } else { - pairs.push((name.to_string(), entry)); + pairs.push((name.as_bytes().to_vec(), entry)); } } - let rank = |n: &str| if n == "ADBE Group End" { usize::MAX } else { GROUP_ORDER.iter().position(|g| *g == n).unwrap_or(GROUP_ORDER.len()) }; + let rank = |n: &[u8]| { + if n == b"ADBE Group End" { usize::MAX } else { GROUP_ORDER.iter().position(|g| g.as_bytes() == n).unwrap_or(GROUP_ORDER.len()) } + }; pairs.sort_by_key(|(n, _)| rank(n)); gk.extend(header); gk.extend(pairs.into_iter().flat_map(|(_, v)| v)); @@ -743,22 +752,12 @@ fn build_prop(cx: &Cx<'_>, p: &PropPlan, lp: &LayerPlan, cp: &CompPlan) -> R *c = Chunk::data(*b"ldat", ldat), - None => ld.push(Chunk::data(*b"ldat", ldat)), - } + fill_key_list(list_mut(&mut tdbs, b"list")?, ldat, p.keys.len(), size)?; } if let Some((text, _)) = &p.expr { let kids = kids_mut(&mut tdbs)?; @@ -818,3 +817,283 @@ pub(crate) fn plain_record(n: usize, k: &KeyPlan, size: usize) -> R> { } Ok(r) } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use effectcraft_color::Label; + use effectcraft_keyframe::{Keyframe, Value}; + use effectcraft_project::{Comp, Expression, ItemKind, LayerSource, Project, Property, Solid, build}; + use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; + + /// Seconds as ticks. + fn secs(s: f64) -> Tick { + Tick((s * TICKS_PER_SECOND as f64).round() as i64) + } + + /// Adds a layer to `comp`, changed by `edit` first. + fn add_layer(p: &mut Project, comp: ItemId, name: &str, source: LayerSource, edit: impl FnOnce(&mut effectcraft_project::Layer)) { + let mut c = p.comp(comp).cloned().expect("comp"); + let mut l = build::layer(p, &c, name, source, (640, 360), None); + l.name = name.to_string(); + edit(&mut l); + c.layers.insert(0, l); + if let Some(i) = p.item_mut(comp) { + i.kind = ItemKind::Comp(Arc::new(c)); + } + } + + /// Gives the property two keys (0 s and 1.5 s) holding its value. + fn two_keys(prop: Option<&mut Property>) { + let prop = prop.expect("property"); + let v = prop.value.clone(); + prop.keys = vec![Keyframe::new(secs(0.0), v.clone()), Keyframe::new(secs(1.5), v)]; + } + + /// A project that reaches every part of `emit`: folders (one named `Solids`), root and nested + /// solids with names that sort naturally, a precomposition, null layers (hidden solids), keyed + /// Transform properties, an expression and a keyed time remap. + fn varied(tag: &str) -> Project { + let mut p = Project::default(); + let solid = |w| ItemKind::Solid(Solid { color: [0.2, 0.4, 0.6], width: w, height: 360, pixel_aspect: 1.0 }); + let solids = p.add_item("Solids", Label::None, None, ItemKind::Folder); + let nested = p.add_item("Folder 10", Label::None, None, ItemKind::Folder); + p.add_item("folder 9", Label::None, Some(nested), ItemKind::Folder); + let mut ids = Vec::new(); + for name in ["Solid 10", "Solid 2", "solid 2", "Solid 02", "Solid", "Ölung 1", ""] { + ids.push(p.add_item(&format!("{name}{tag}"), Label::Red, None, solid(640))); + } + p.add_item("In Solids 3", Label::None, Some(solids), solid(320)); + p.add_item("In folder", Label::None, Some(nested), solid(100)); + let pre = p.add_item("Pre 1", Label::None, Some(nested), ItemKind::Comp(Arc::new(Comp::new(640, 360, FrameRate::new(25, 1), secs(3.0))))); + add_layer(&mut p, pre, "S", LayerSource::Solid { item: ids[0] }, |_| {}); + let main = p.add_item("Main", Label::None, None, ItemKind::Comp(Arc::new(Comp::new(1920, 1080, FrameRate::new(30, 1), secs(5.0))))); + add_layer(&mut p, main, "Null A", LayerSource::Null, |_| {}); + add_layer(&mut p, main, "Null B", LayerSource::Null, |l| two_keys(l.props.prop_mut("transform/position"))); + add_layer(&mut p, main, "Solid layer", LayerSource::Solid { item: ids[1] }, |l| { + two_keys(l.props.prop_mut("transform/opacity")); + two_keys(l.props.prop_mut("transform/anchor")); + if let Some(r) = l.props.prop_mut("transform/rotation") { + r.expr = Some(Expression { text: "time * 10".into(), enabled: true }); + } + }); + add_layer(&mut p, main, "Precomp", LayerSource::Comp { item: pre }, |l| { + let mut r = Property::new(9_000, "timeRemap", "Time Remap", Value::Scalar(0.0)); + r.keys = vec![Keyframe::new(secs(0.0), Value::Scalar(3.0)), Keyframe::new(secs(3.0), Value::Scalar(0.0))]; + l.props.children.insert(0, r.into()); + }); + p + } + + /// The export plan of `p`. + fn plan_of(p: &Project) -> ProjectPlan { + let mut col = crate::export::report::Collector::default(); + crate::export::plan::plan(p, &mut col) + } + + /// The template is built once and every later export uses the same one. + #[test] + fn the_template_is_built_once() { + let a = template::shared().expect("template"); + let b = template::shared().expect("template"); + assert!(std::ptr::eq(a, b)); + } + + /// Exports with the shared template give the bytes a freshly built template gives, also after + /// other exports have used it (nothing an export does reaches the template). + #[test] + fn the_shared_template_exports_what_a_fresh_one_does() { + let (a, b) = (plan_of(&varied("")), plan_of(&varied(" x"))); + let fresh = |plan: &ProjectPlan| emit_with(&Template::load().expect("template"), plan, b"trailer").expect("emit"); + let (want_a, want_b) = (fresh(&a), fresh(&b)); + assert!(want_a != want_b); + // The project exports without losses and the remap is written. + let r = crate::export::export(&varied(""), &crate::export::ExportOptions { allow_losses: false }).expect("lossless"); + assert_eq!(r.report.exported.time_remaps, 1); + // The null layers' hidden solids and the root solids went into the `Solids` folder. + let file = crate::parse(&want_a).expect("parse"); + let folder = file.walk().map(|(_, c)| c).filter(|c| c.is_list(b"Sfdr")).map(|c| c.children().iter().filter(|k| k.is_list(b"Item")).count()).max(); + assert_eq!(folder, Some(10)); + for _ in 0..3 { + assert!(emit(&a, b"trailer").expect("emit") == want_a); + assert!(emit(&b, b"trailer").expect("emit") == want_b); + } + } + + /// Exports running at the same time on several threads all give the single-threaded bytes. + /// This checks concurrent exports from a template that is already built: other tests in this + /// binary usually fill the shared template first, so the race on the first build is not + /// reliably exercised here (the `static OnceLock` makes that race safe by construction). + #[test] + fn concurrent_exports_give_the_same_bytes() { + let plans: Vec = ["", " x", " y", " z"].iter().map(|t| plan_of(&varied(t))).collect(); + let want: Vec> = plans.iter().map(|plan| emit_with(&Template::load().expect("template"), plan, &[]).expect("emit")).collect(); + let got: Vec>> = std::thread::scope(|s| { + let handles: Vec<_> = (0..8) + .map(|i| { + let plans = &plans; + s.spawn(move || (0..4).map(|k| emit(&plans[(i + k) % plans.len()], &[]).expect("emit")).collect::>()) + }) + .collect(); + handles.into_iter().map(|h| h.join().expect("thread")).collect() + }); + for (i, bytes) in got.iter().enumerate() { + for (k, b) in bytes.iter().enumerate() { + assert!(b == &want[(i + k) % want.len()], "thread {i}, export {k}"); + } + } + } + + /// The comparator `sort_named` used before its keys were cached: `natural_key` built on every + /// comparison. + fn old_sort_named(v: &mut [Sorted]) { + /// The natural name order the cached sort keys replaced, as the reference. + fn natural_cmp(a: &str, b: &str) -> std::cmp::Ordering { + let (ka, kb) = (natural_key(a), natural_key(b)); + for (x, y) in ka.iter().zip(kb.iter()) { + let c = match (x.0, y.0) { + (true, true) => x.2.cmp(&y.2).then_with(|| x.1.cmp(&y.1)), + (false, false) => x.1.cmp(&y.1), + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + }; + if c != std::cmp::Ordering::Equal { + return c; + } + } + ka.len().cmp(&kb.len()) + } + v.sort_by(|a, b| natural_cmp(&a.0, &b.0).then(a.1.cmp(&b.1))); + } + + /// The cached-key sort orders names exactly as the comparator it replaced, and keeps entries + /// with the same name and key in their order (each carries its position as a marker chunk). + #[test] + fn sort_named_orders_as_the_old_comparator() { + let parts = [ + "", + "0", + "00", + "1", + "01", + "2", + "10", + "a", + "A", + "b", + " ", + "-", + ".", + "é", + "É", + "ß", + "İ", + "\u{0}", + "\u{1}", + "a\u{0}", + "Ⱥ", + "K", + "ẞ", + "Solid ", + "solid ", + "x9", + "9x", + "340282366920938463463374607431768211455", + "340282366920938463463374607431768211456", + "999999999999999999999999999999999999999999", + ]; + // A small deterministic generator (splitmix64). + let mut state = 0x2545_f491_4f6c_dd1d_u64; + let mut next = move || { + state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + (z ^ (z >> 31)) as usize + }; + for round in 0..2000 { + let n = 1 + next() % 40; + let v: Vec = (0..n) + .map(|i| { + let name: String = (0..next() % 4).map(|_| parts[next() % parts.len()]).collect(); + let key = (next() % 3) as u64 | if next() % 2 == 0 { ROOT_SOLID } else { 0 }; + (name, key, vec![Chunk::data(*b"mark", (i as u32).to_le_bytes().to_vec())]) + }) + .collect(); + let (mut old, mut new) = (v.clone(), v); + old_sort_named(&mut old); + sort_named(&mut new); + assert_eq!(old, new, "round {round}"); + } + } + + /// The cached sort keys stay within [`NATURAL_BYTES_PER_NAME_BYTE`] bytes per name byte, also + /// for the worst name (255 bytes of alternating one-character text and digit runs: 255 + /// segments, which as `(bool, String, u128)` segments took about 14 KB), and the per-character + /// premise of the bound holds for every character. + #[test] + fn sort_keys_are_bounded_by_the_name() { + for c in (0..=0x10_ffff_u32).filter_map(char::from_u32) { + let lower: usize = c.to_lowercase().map(char::len_utf8).sum::() + usize::from(c == '\0'); + assert!(lower <= 2 * c.len_utf8(), "{c:?}"); + } + let worst: String = (0..255).map(|i| if i % 2 == 0 { 'a' } else { '1' }).collect(); + let others = ["", "1", "a", "\u{0}", "İ", "Ⱥ1", "99999999999999999999999999999999999999999999", &"\u{0}".repeat(255), &"Ⱥ".repeat(127)]; + for name in std::iter::once(worst.as_str()).chain(others) { + let k = natural_bytes(name); + assert!(k.len() <= NATURAL_BYTES_PER_NAME_BYTE * name.len(), "{name:?}: {} bytes", k.len()); + } + assert_eq!(natural_key(&worst).len(), 255); + assert!(natural_bytes(&worst).len() <= 1_275); + } + + /// The shared `is_tdmn` / `tdmn_name` the emitter uses agree with the Latin-1 decoding it used + /// before for every name it looks up, which holds because those names are ASCII. + #[test] + fn match_names_compare_as_the_latin1_decoding_did() { + /// The Latin-1 decoding the emitter used before. + fn old_tdmn_name(c: &Chunk<'_>) -> Option { + let d = c.bytes()?; + Some(d.iter().take_while(|b| **b != 0).map(|b| char::from(*b)).collect()) + } + let mut wanted: Vec<&str> = GROUP_ORDER.to_vec(); + wanted.extend(PropKind::ALL.iter().map(|k| k.match_name())); + wanted.extend(["ADBE Time Remapping", "ADBE Transform Group", "ADBE Group End"]); + assert!(wanted.iter().all(|n| n.is_ascii())); + let mut chunks: Vec> = wanted.iter().map(|n| Chunk::data(*b"tdmn", format!("{n}\0\0junk").into_bytes())).collect(); + chunks.extend([ + Chunk::data(*b"tdmn", Vec::new()), + Chunk::data(*b"tdmn", b"ADBE Position".to_vec()), + Chunk::data(*b"tdmn", b"ADBE Position_0\0".to_vec()), + Chunk::data(*b"tdmn", b"ADBE Opacit\xe9\0".to_vec()), + Chunk::data(*b"tdmn", b"\xc3\x84DBE Scale\0".to_vec()), + Chunk::data(*b"tdmn", b"\0ADBE Scale".to_vec()), + Chunk::list(*b"tdmn", Vec::new()), + ]); + for c in &chunks { + for n in &wanted { + assert_eq!(is_tdmn(c, n), old_tdmn_name(c).as_deref() == Some(*n), "{c:?} {n}"); + let key = tdmn_name(c).unwrap_or_default().into_bytes(); + assert_eq!(key == n.as_bytes(), old_tdmn_name(c).as_deref() == Some(*n), "{c:?} {n}"); + } + } + } + + /// `fill_key_list`, which the time remap and the Transform keys use, writes the counts and + /// replaces the records, or adds them when the list has none. + #[test] + fn fill_key_list_writes_counts_and_records() { + let lhd3 = || Chunk::data(*b"lhd3", vec![0; 52]); + let mut want_lhd3 = vec![0; 52]; + crate::export::synth::put_key_counts(&mut want_lhd3, 3, 48).expect("counts"); + for start in [vec![lhd3(), Chunk::data(*b"ldat", vec![9; 4])], vec![lhd3()]] { + let mut list = Chunk::list(*b"list", start); + fill_key_list(&mut list, vec![7; 144], 3, 48).expect("set"); + assert_eq!(list.children(), &[Chunk::data(*b"lhd3", want_lhd3.clone()), Chunk::data(*b"ldat", vec![7; 144])]); + } + let mut no_header = Chunk::list(*b"list", Vec::new()); + assert!(fill_key_list(&mut no_header, Vec::new(), 0, 48).is_err()); + } +} diff --git a/crates/aep/src/export/fxsynth.rs b/crates/aep/src/export/fxsynth.rs index 4fa02f0c5..611ef7310 100644 --- a/crates/aep/src/export/fxsynth.rs +++ b/crates/aep/src/export/fxsynth.rs @@ -11,7 +11,8 @@ use crate::Chunk; use crate::effect_map::{Conv, by_match_name}; use crate::export::fx_defs::{BUILT_IN, FxDef, PDef}; -use crate::export::synth::{KeyTemplate, NO_NAME, RecSpec, class, tdmn, tdsn}; +use crate::export::synth::{GROUP_END, KeyTemplate, NO_NAME, RecSpec, class, tdmn, tdsn, utf8_in}; +use crate::export::tree::Fixed; use crate::{CURVE_IDENTITY, CurvesData, MeshData}; /// The layer a prototype's layer parameters name when their default is the owning layer; the @@ -37,12 +38,6 @@ pub(crate) fn value_spec(code: u8, range: Option<(f64, f64)>) -> Option }) } -fn put(b: &mut [u8], at: usize, bytes: &[u8]) { - if let Some(dst) = b.get_mut(at..at + bytes.len()) { - dst.copy_from_slice(bytes); - } -} - fn fixed(v: f64) -> [u8; 4] { ((v * 65536.0).round().clamp(f64::from(i32::MIN), f64::from(i32::MAX)) as i32).to_be_bytes() } @@ -53,83 +48,88 @@ fn fixed(v: f64) -> [u8; 4] { /// display 124; fixed and integer: 124 / 128 / 132 / 136, default 140, display 144). Every other /// byte is zero (the switches at 48 are not needed: s12-e2-nofl). pub(crate) fn pard(p: &PDef, popup_items: Option) -> Vec { - let mut b = vec![0u8; 148]; - put(&mut b, 12, &u32::from(p.code).to_be_bytes()); + pard_layout(p, popup_items).finish(|| format!("pard (type {})", p.code)) +} + +/// The 148-byte `pard` layout of a parameter definition: type, name and defaults. +fn pard_layout(p: &PDef, popup_items: Option) -> Fixed { + let mut b = Fixed::zeroed(148); + b.put(12, &u32::from(p.code).to_be_bytes()); let d = |i: usize| p.default.get(i).copied().unwrap_or(0.0); // After Effects takes the default of a parameter without a value record from here (s8-e1: // the copies below are the ones it reads). match p.code { 10 => { - put(&mut b, 56, &d(0).to_be_bytes()); + b.put(56, &d(0).to_be_bytes()); // The default again as `f32` at 120, the display word (decimals) at 124. - put(&mut b, 120, &(d(0) as f32).to_be_bytes()); - put(&mut b, 124, &p.display.to_be_bytes()); + b.put(120, &(d(0) as f32).to_be_bytes()); + b.put(124, &p.display.to_be_bytes()); } - 2 => put(&mut b, 56, &fixed(d(0))), + 2 => b.put(56, &fixed(d(0))), // An angle: the default again at 60. 3 => { - put(&mut b, 56, &fixed(d(0))); - put(&mut b, 60, &fixed(d(0))); + b.put(56, &fixed(d(0))); + b.put(60, &fixed(d(0))); } - 1 => put(&mut b, 56, &(d(0).round() as i32).to_be_bytes()), + 1 => b.put(56, &(d(0).round() as i32).to_be_bytes()), // A checkbox: the default again as a byte at 60. 4 => { - put(&mut b, 56, &(d(0).round().max(0.0) as u32).to_be_bytes()); - put(&mut b, 60, &[u8::from(d(0) != 0.0)]); + b.put(56, &(d(0).round().max(0.0) as u32).to_be_bytes()); + b.put(60, &[u8::from(d(0) != 0.0)]); } // A popup: the default again as `u16` at 62 (the item count is at 60). 7 => { let sel = d(0).round().clamp(0.0, f64::from(u16::MAX)); - put(&mut b, 56, &(sel as u32).to_be_bytes()); - put(&mut b, 62, &(sel as u16).to_be_bytes()); + b.put(56, &(sel as u32).to_be_bytes()); + b.put(62, &(sel as u16).to_be_bytes()); } 6 => { // Fractions of the layer at 56 / 60 and the same in percent at 68 / 72. - put(&mut b, 56, &fixed(d(0))); - put(&mut b, 60, &fixed(d(1))); - put(&mut b, 68, &fixed(d(0) * 100.0)); - put(&mut b, 72, &fixed(d(1) * 100.0)); + b.put(56, &fixed(d(0))); + b.put(60, &fixed(d(1))); + b.put(68, &fixed(d(0) * 100.0)); + b.put(72, &fixed(d(1) * 100.0)); } 5 => { // ARGB at 56 and the same at 60. let c: Vec = (0..4).map(|i| d(i).round().clamp(0.0, 255.0) as u8).collect(); - put(&mut b, 56, &c); - put(&mut b, 60, &c); + b.put(56, &c); + b.put(60, &c); } _ => {} } if p.code == 7 { let n = popup_items.unwrap_or_else(|| u16::try_from(p.display).unwrap_or(0)); - put(&mut b, 60, &n.to_be_bytes()); + b.put(60, &n.to_be_bytes()); } match p.code { 10 => { if let Some(m) = p.valid_min { - put(&mut b, 104, &(m as f32).to_be_bytes()); + b.put(104, &(m as f32).to_be_bytes()); } if let Some(m) = p.valid_max { - put(&mut b, 108, &(m as f32).to_be_bytes()); + b.put(108, &(m as f32).to_be_bytes()); } if let Some((lo, hi)) = p.range { - put(&mut b, 112, &(lo as f32).to_be_bytes()); - put(&mut b, 116, &(hi as f32).to_be_bytes()); + b.put(112, &(lo as f32).to_be_bytes()); + b.put(116, &(hi as f32).to_be_bytes()); } } 1 | 2 => { let enc = |v: f64| if p.code == 2 { fixed(v) } else { (v.round() as i32).to_be_bytes() }; if let Some(m) = p.valid_min { - put(&mut b, 124, &enc(m)); + b.put(124, &enc(m)); } if let Some(m) = p.valid_max { - put(&mut b, 128, &enc(m)); + b.put(128, &enc(m)); } if let Some((lo, hi)) = p.range { - put(&mut b, 132, &enc(lo)); - put(&mut b, 136, &enc(hi)); + b.put(132, &enc(lo)); + b.put(136, &enc(hi)); } // The default again at 140, the display word (decimals, percent) at 144. - put(&mut b, 140, &enc(d(0))); - put(&mut b, 144, &p.display.to_be_bytes()); + b.put(140, &enc(d(0))); + b.put(144, &p.display.to_be_bytes()); } _ => {} } @@ -224,7 +224,7 @@ fn data_record(data: Vec) -> [Chunk<'static>; 2] { /// group (records only for layer parameters, groups, buttons and arbitrary data, the built-in /// group) and `pgui` (zero). pub(crate) fn default_sspc(fx: &FxDef) -> Chunk<'static> { - let fnam = Chunk { id: *b"fnam", body: crate::Body::Chunks(vec![Chunk::data(*b"Utf8", fx.name.as_bytes().to_vec())]), pad: None }; + let fnam = utf8_in(*b"fnam", fx.name); let count = u32::try_from(fx.params.len()).unwrap_or(u32::MAX); let mut part = vec![Chunk::data(*b"parn", count.to_be_bytes().to_vec())]; let mut values = vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn(NO_NAME)]; @@ -235,11 +235,11 @@ pub(crate) fn default_sspc(fx: &FxDef) -> Chunk<'static> { if matches!(p.code, 4 | 7 | 15) { // Checkboxes, popups and buttons have a `pdnm` (their label or items); its text may // be empty. - part.push(Chunk { id: *b"pdnm", body: crate::Body::Chunks(vec![Chunk::data(*b"Utf8", Vec::new())]), pad: None }); + part.push(utf8_in(*b"pdnm", "")); } if p.id == BUILT_IN { values.push(tdmn(&record)); - values.push(Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn(NO_NAME), tdmn("ADBE Group End")])); + values.push(Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn(NO_NAME), tdmn(GROUP_END)])); continue; } match p.code { @@ -265,6 +265,24 @@ pub(crate) fn default_sspc(fx: &FxDef) -> Chunk<'static> { _ => {} } } - values.push(tdmn("ADBE Group End")); + values.push(tdmn(GROUP_END)); Chunk::list(*b"sspc", vec![fnam, Chunk::list(*b"parT", part), Chunk::list(*b"tdgp", values), Chunk::data(*b"pgui", vec![0u8; 16])]) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::export::fx_defs::FX_DEFS; + + /// The `pard` of every parameter of the effect table fits its 148 bytes. + #[test] + fn every_parameter_definition_fits() { + for fx in FX_DEFS { + for (i, p) in fx.params.iter().enumerate() { + let b = pard_layout(p, popup_items(fx, i)); + assert!(b.error().is_none(), "{} {}", fx.match_name, i); + assert_eq!(b.finish(String::new).len(), 148); + } + } + } +} diff --git a/crates/aep/src/export/keys.rs b/crates/aep/src/export/keys.rs new file mode 100644 index 000000000..f484276fc --- /dev/null +++ b/crates/aep/src/export/keys.rs @@ -0,0 +1,198 @@ +//! Key lists shared by the exporters: the times of a property's keys in the file's units, the +//! spatial key record, and the rewrite of a key list (`LIST list`: `lhd3` + `ldat`, aep-format.md +//! 3.5). + +use effectcraft_keyframe::Keyframe; + +use crate::Chunk; +use crate::export::plan::KeyPlan; +use crate::export::report::{Collector, LossCategory}; +use crate::export::timebase::TimeBase; +use crate::export::tree::{R, child_data, kids_mut, list_mut, put, put_f64, put_u8, put_u16, put_u32, tpl}; + +/// The keys of a property at their times in the file's units, in time order. A key that does not +/// fall on the time base is reported as moved; keys that end up at the same time are merged (the +/// first one is kept) and reported once. `noun` names the key in the report ("key", "path key"). +pub(crate) fn timed_keys<'k>(keys: &'k [Keyframe], tb: &TimeBase, what: &str, noun: &str, col: &mut Collector) -> Vec<(i64, &'k Keyframe)> { + let mut timed: Vec<(i64, &Keyframe)> = Vec::with_capacity(keys.len()); + for k in keys { + let (u, inexact) = tb.units(k.time); + if inexact { + col.lose(LossCategory::RoundedTimes, format!("{what}: {noun} at {:.4} s moved to the time base", k.time.seconds())); + } + timed.push((u, k)); + } + timed.sort_by_key(|(u, _)| *u); + let before = timed.len(); + timed.dedup_by_key(|(u, _)| *u); + if timed.len() != before { + col.lose(LossCategory::ApproximatedKeyframes, format!("{what}: {noun}s at the same time were merged")); + } + timed +} + +/// A key of a point (flags `0x0007`, `u32@8` the automatic tangent switch; the spatial layout of +/// a 2D Position, 853) or a colour (flags `0x0001`, 854): the temporal ease of a point at 24..55, +/// then the `n` values at 56 and, for a point, the spatial in and out tangents after them. +pub(crate) fn spatial_record(k: &KeyPlan, n: usize, point: bool, size: usize) -> R> { + let mut r = vec![0u8; size]; + put(&mut r, 0, &k.time.to_be_bytes())?; + put_u8(&mut r, 4, k.in_interp)?; + put_u8(&mut r, 5, k.out_interp)?; + let base: u16 = if point { 0x0007 } else { 0x0001 }; + put_u16(&mut r, 6, base | (u16::from(k.label) << 6))?; + if point { + put_u32(&mut r, 8, u32::from(k.spatial_auto))?; + let (i, o) = (k.in_ease.first().copied().unwrap_or_default(), k.out_ease.first().copied().unwrap_or_default()); + put_f64(&mut r, 24, i.speed)?; + put_f64(&mut r, 32, i.influence)?; + put_f64(&mut r, 40, o.speed)?; + put_f64(&mut r, 48, o.influence)?; + } + for j in 0..n { + put_f64(&mut r, 56 + 8 * j, k.value.get(j).copied().unwrap_or(0.0))?; + if point { + put_f64(&mut r, 56 + 8 * n + 8 * j, k.spatial_in.get(j).copied().unwrap_or(0.0))?; + put_f64(&mut r, 56 + 16 * n + 8 * j, k.spatial_out.get(j).copied().unwrap_or(0.0))?; + } + } + Ok(r) +} + +/// Fills a key list (`LIST list`) with `count` records of `size` bytes: the count words of `lhd3` +/// and `ldat` (added when the list has none). +pub(crate) fn fill_key_list(list: &mut Chunk<'static>, records: Vec, count: usize, size: usize) -> R<()> { + let n = u32::try_from(count).map_err(|_| tpl("too many keys"))?; + let size = u32::try_from(size).map_err(|_| tpl("record size"))?; + crate::export::synth::put_key_counts(child_data(list, b"lhd3")?, n, size)?; + let kids = kids_mut(list)?; + match kids.iter_mut().find(|c| c.is(b"ldat")) { + Some(c) => *c = Chunk::data(*b"ldat", records), + None => kids.push(Chunk::data(*b"ldat", records)), + } + Ok(()) +} + +/// Rewrites the key list of a keyed property record (`LIST tdbs`), see [`fill_key_list`]. +pub(crate) fn set_key_list(tdbs: &mut Chunk<'static>, records: Vec, count: usize, size: usize) -> R<()> { + fill_key_list(list_mut(tdbs, b"list")?, records, count, size) +} + +/// Puts the key list `list` in the place of the static value (`cdat`) of a property record. +pub(crate) fn replace_cdat(tdbs: &mut Chunk<'static>, list: Chunk<'static>) -> R<()> { + let slot = kids_mut(tdbs)?.iter_mut().find(|c| c.is(b"cdat")).ok_or_else(|| tpl("record without cdat"))?; + *slot = list; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::export::synth::key_list; + + /// Keys are timed sorted merged and reported. + #[test] + fn keys_are_timed_sorted_merged_and_reported() { + use effectcraft_keyframe::Value; + use effectcraft_time::{TICKS_PER_SECOND, Tick}; + let tb = TimeBase { frame_duration: 1024, timescale: 25600 }; + // Out of order; 1/3 s is not on the time base; two keys meet at 1 s (the first one kept). + let keys = [ + Keyframe::new(Tick(TICKS_PER_SECOND), Value::Scalar(1.0)), + Keyframe::new(Tick(TICKS_PER_SECOND / 3), Value::Scalar(2.0)), + Keyframe::new(Tick(0), Value::Scalar(3.0)), + Keyframe::new(Tick(TICKS_PER_SECOND), Value::Scalar(4.0)), + ]; + let mut col = Collector::default(); + let timed = timed_keys(&keys, &tb, "L", "path key", &mut col); + let got: Vec<(i64, f64)> = timed.iter().map(|(u, k)| (*u, if let Value::Scalar(v) = k.value { v } else { f64::NAN })).collect(); + assert_eq!(got, [(0, 3.0), (8533, 2.0), (25600, 1.0)]); + // The report lists its categories in a fixed order; one merge report for the pair. + let report = col.finish(); + let lines: Vec<(LossCategory, &str)> = report.losses.iter().flat_map(|l| l.details.iter().map(move |d| (l.category, d.as_str()))).collect(); + assert_eq!( + lines, + [ + (LossCategory::ApproximatedKeyframes, "L: path keys at the same time were merged"), + (LossCategory::RoundedTimes, "L: path key at 0.3333 s moved to the time base"), + ] + ); + // No keys, no report. + let mut col = Collector::default(); + assert!(timed_keys(&[], &tb, "L", "key", &mut col).is_empty()); + assert!(col.finish().losses.is_empty()); + } + + /// Two keys at different times that round to the same time unit are merged too: the first + /// one is kept, and both the rounding and the merge are reported. + #[test] + fn keys_that_round_to_the_same_time_are_merged() { + use effectcraft_keyframe::Value; + use effectcraft_time::Tick; + let tb = TimeBase { frame_duration: 1024, timescale: 25600 }; + // One tick is far below one unit (1/25600 s): it rounds to unit 0, where the first key is. + let keys = [Keyframe::new(Tick(0), Value::Scalar(1.0)), Keyframe::new(Tick(1), Value::Scalar(2.0))]; + let mut col = Collector::default(); + let timed = timed_keys(&keys, &tb, "L", "key", &mut col); + let got: Vec<(i64, f64)> = timed.iter().map(|(u, k)| (*u, if let Value::Scalar(v) = k.value { v } else { f64::NAN })).collect(); + assert_eq!(got, [(0, 1.0)]); + let report = col.finish(); + let lines: Vec<(LossCategory, &str)> = report.losses.iter().flat_map(|l| l.details.iter().map(move |d| (l.category, d.as_str()))).collect(); + assert_eq!( + lines, + [ + (LossCategory::ApproximatedKeyframes, "L: keys at the same time were merged"), + (LossCategory::RoundedTimes, "L: key at 0.0000 s moved to the time base"), + ] + ); + } + + /// A key list is filled and replaces the value. + #[test] + fn a_key_list_is_filled_and_replaces_the_value() { + let mut list = Chunk::list(*b"list", vec![Chunk::data(*b"lhd3", vec![0; 52])]); + fill_key_list(&mut list, vec![7; 96], 2, 48).unwrap(); + let h = list.child(b"lhd3").and_then(|c| c.bytes()).unwrap(); + assert_eq!((h[11], h[19]), (2, 48)); + assert_eq!(list.child(b"ldat").and_then(|c| c.bytes()).map(<[u8]>::len), Some(96)); + // A second fill replaces the records instead of adding a second `ldat`. + fill_key_list(&mut list, vec![1; 48], 1, 48).unwrap(); + assert_eq!(list.children().iter().filter(|c| c.is(b"ldat")).count(), 1); + + let mut rec = Chunk::list(*b"tdbs", vec![Chunk::data(*b"tdb4", vec![0; 124]), Chunk::data(*b"cdat", vec![0; 8])]); + replace_cdat(&mut rec, key_list(48, 0, Vec::new())).unwrap(); + assert!(rec.child(b"cdat").is_none() && rec.child_list(b"list").is_some()); + // A record without a value is an error, not a quiet no-op. + assert!(replace_cdat(&mut rec, key_list(48, 0, Vec::new())).is_err()); + set_key_list(&mut rec, vec![0; 48], 1, 48).unwrap(); + assert!(set_key_list(&mut Chunk::list(*b"tdbs", Vec::new()), Vec::new(), 0, 48).is_err()); + } + + /// A spatial record holds the point layout. + #[test] + fn a_spatial_record_holds_the_point_layout() { + let k = KeyPlan { + time: 5, + value: vec![1.0, 2.0], + in_interp: 2, + out_interp: 1, + in_ease: Vec::new(), + out_ease: Vec::new(), + spatial_in: [3.0; 3], + spatial_out: [4.0; 3], + spatial_auto: true, + roving: false, + label: 1, + }; + let r = spatial_record(&k, 2, true, 56 + 24 * 2).unwrap(); + assert_eq!(&r[..12], &[0, 0, 0, 5, 2, 1, 0, 0x47, 0, 0, 0, 1]); + assert_eq!(&r[64..72], &2.0f64.to_be_bytes()); + assert_eq!(&r[96..104], &4.0f64.to_be_bytes()); + // A colour has no tangents and flags 0x0001. + let c = spatial_record(&k, 2, false, 56 + 24 * 2).unwrap(); + assert_eq!((c[7], c[11]), (0x41, 0)); + assert!(c[72..].iter().all(|b| *b == 0)); + // A record shorter than its layout is an error. + assert!(spatial_record(&k, 2, true, 64).is_err()); + } +} diff --git a/crates/aep/src/export/masks.rs b/crates/aep/src/export/masks.rs index 61d8634c4..9b235eb28 100644 --- a/crates/aep/src/export/masks.rs +++ b/crates/aep/src/export/masks.rs @@ -17,9 +17,11 @@ use crate::Chunk; use effectcraft_keyframe::{Ease, Interp, ShapePath, Value}; use effectcraft_project::{FeatherFalloff, GroupKind, Layer, MaskMode, MaskMotionBlur, PropGroup}; +use crate::export::keys::{set_key_list, timed_keys}; use crate::export::plan::{CompPlan, EaseSlot, KeyPlan, LIMIT, cut_name}; use crate::export::report::{Collector, LossCategory}; -use crate::export::tree::{R, child_data, child_mut, kids_mut, list_mut, put_f32, put_f64, put_i32, put_u8, put_u16, put_u32, tpl}; +use crate::export::synth::GROUP_END; +use crate::export::tree::{Fixed, R, child_data, child_mut, kids_mut, list_mut, put_f32, put_f64, put_i32, put_u8, put_u16, put_u32, split_group, tpl}; /// The most vertices one mask path may have (as the importer). pub(crate) const MAX_VERTICES: usize = 65_536; @@ -179,20 +181,7 @@ fn plan_mask(g: &PropGroup, size: (u32, u32), cp: &CompPlan, what: &str, col: &m } // The path and its keys. let mut path_keys: Vec = Vec::new(); - let mut keys: Vec<(i64, &effectcraft_keyframe::Keyframe)> = Vec::new(); - for k in &path_prop.keys { - let (u, inexact) = cp.tb.units(k.time); - if inexact { - col.lose(LossCategory::RoundedTimes, format!("{what}: path key at {:.4} s moved to the time base", k.time.seconds())); - } - keys.push((u, k)); - } - keys.sort_by_key(|(u, _)| *u); - let before = keys.len(); - keys.dedup_by_key(|(u, _)| *u); - if keys.len() != before { - col.lose(LossCategory::ApproximatedKeyframes, format!("{what}: path keys at the same time were merged")); - } + let keys = timed_keys(&path_prop.keys, &cp.tb, what, "path key", col); let side = |i: Interp| -> u8 { match i { Interp::Linear => 1, @@ -288,20 +277,7 @@ fn plan_mask(g: &PropGroup, size: (u32, u32), cp: &CompPlan, what: &str, col: &m /// Feather keys (plain template, two components, speed in pixels per second). fn plan_feather_keys(src: &[effectcraft_keyframe::Keyframe], cp: &CompPlan, what: &str, col: &mut Collector) -> Vec { - let mut keys: Vec<(i64, &effectcraft_keyframe::Keyframe)> = Vec::with_capacity(src.len()); - for k in src { - let (u, inexact) = cp.tb.units(k.time); - if inexact { - col.lose(LossCategory::RoundedTimes, format!("{what}: feather key at {:.4} s moved to the time base", k.time.seconds())); - } - keys.push((u, k)); - } - keys.sort_by_key(|(u, _)| *u); - let before = keys.len(); - keys.dedup_by_key(|(u, _)| *u); - if keys.len() != before { - col.lose(LossCategory::ApproximatedKeyframes, format!("{what}: feather keys at the same time were merged")); - } + let keys = timed_keys(src, &cp.tb, what, "feather key", col); let mut out = Vec::with_capacity(keys.len()); let beyond = std::cell::Cell::new(false); for (u, k) in &keys { @@ -381,11 +357,6 @@ pub(crate) struct MaskProtos { pub keyed_feather: Chunk<'static>, } -pub(crate) fn tdmn_name(c: &Chunk<'_>) -> Option { - let d = c.bytes()?; - Some(d.iter().take_while(|b| **b != 0).map(|b| char::from(*b)).collect()) -} - /// The limit of a mask's feather and expansion (`tduM` 32 000, 091). const MASK_LIMIT: f64 = 32_000.0; @@ -394,9 +365,25 @@ const MASK_LIMIT: f64 = 32_000.0; /// s13-m1), byte 3 `0x01` (`0x09` once the path is open), the box at 4..19 (written per shape), /// byte 20 1. pub(crate) fn shph() -> Vec { - let mut b = vec![0u8; 24]; - let _ = crate::export::tree::put(&mut b, 0, &[0xb3, 0xde, 0x02, 0x01]); - let _ = put_u8(&mut b, 20, 1); + shph_layout().finish(|| "shph".to_string()) +} + +/// The 24-byte `shph` layout of a closed path with a unit box. +fn shph_layout() -> Fixed { + let mut b = Fixed::zeroed(24); + b.put(0, &[0xb3, 0xde, 0x02, 0x01]); + b.put(20, &[1]); + b +} + +/// The two prototype records of a keyed path (093): 64 bytes each, flags `0x0001`, the `f64` at +/// 16 1 in the first (the rest is zero: its last bytes were uninitialised). +fn path_key_records() -> Fixed { + let mut b = Fixed::zeroed(128); + for at in [6, 64 + 6] { + b.put(at, &0x0001u16.to_be_bytes()); + } + b.put(16, &1.0f64.to_be_bytes()); b } @@ -413,15 +400,7 @@ impl MaskProtos { let path_rec = |keyed: bool| { let mut kids = vec![state(), tdsn(NO_NAME), Chunk::data(*b"tdb4", tdb4(&path, keyed, 25_600))]; if keyed { - // Path keys are 64-byte records: flags `0x0001`, the `f64` at 16 1 on the first - // key (093; the rest of the record is zero: its last bytes were uninitialised). - let mut first = vec![0u8; 64]; - let mut later = vec![0u8; 64]; - for r in [&mut first, &mut later] { - let _ = put_u16(r, 6, 0x0001); - } - let _ = put_f64(&mut first, 16, 1.0); - kids.push(key_list(64, 2, [first, later].concat())); + kids.push(key_list(64, 2, path_key_records().finish(|| "the keyed path records".to_string()))); } else { kids.push(Chunk::data(*b"cdat", vec![0u8; 4])); } @@ -444,7 +423,7 @@ impl MaskProtos { tdbs(&opacity, false, 25_600), tdmn("ADBE Mask Offset"), tdbs(&offset, false, 25_600), - tdmn("ADBE Group End"), + tdmn(GROUP_END), ], ); MaskProtos { @@ -453,7 +432,7 @@ impl MaskProtos { atom_name: tdmn("ADBE Mask Atom"), mkif: Chunk::data(*b"mkif", vec![0u8; 48]), atom, - group_end: tdmn("ADBE Group End"), + group_end: tdmn(GROUP_END), keyed_path: path_rec(true), keyed_feather: tdbs(&feather, true, 25_600), } @@ -485,27 +464,21 @@ fn build_atom(t: &MaskProtos, m: &MaskPlan, size: (u32, u32)) -> R> = kids.drain(..first).collect(); - let mut pairs: Vec<(String, [Chunk<'static>; 2])> = Vec::new(); - let mut end: Option> = None; - let mut it = std::mem::take(kids).into_iter(); - while let Some(c) = it.next() { - let name = tdmn_name(&c).unwrap_or_default(); - if name == "ADBE Group End" { - end = Some(c); - continue; - } - let node = it.next().ok_or_else(|| tpl("mask match name without a node"))?; - pairs.push((name, [c, node])); - } - let end = end.ok_or_else(|| tpl("mask atom without its end"))?; - let mut out = header; - for (name, [tdmn, mut node]) in pairs { + let (mut out, entries) = split_group(kids_mut(&mut atom)?)?; + let mut has_shape = false; + for (name, entry) in entries { + // The closing Group End (no node) is kept as it is. + let [tdmn, mut node]: [Chunk<'static>; 2] = match entry.try_into() { + Ok(pair) => pair, + Err(end) => { + out.extend(end); + continue; + } + }; let keep = match name.as_str() { "ADBE Mask Shape" => { build_shape(t, &mut node, m, size)?; + has_shape = true; true } "ADBE Mask Feather" => { @@ -540,7 +513,10 @@ fn build_atom(t: &MaskProtos, m: &MaskPlan, size: (u32, u32)) -> R, values: &[f64]) -> R<()> { Ok(()) } -/// Rewrites a key list (`lhd3` count words and `ldat`). -fn set_key_list(tdbs: &mut Chunk<'static>, records: Vec, count: usize, size: usize) -> R<()> { - let list = list_mut(tdbs, b"list")?; - let n = u32::try_from(count).map_err(|_| tpl("too many keys"))?; - { - let lhd3 = child_data(list, b"lhd3")?; - crate::export::synth::put_key_counts(lhd3, n, u32::try_from(size).map_err(|_| tpl("record size"))?)?; - } - let ld = kids_mut(list)?; - match ld.iter_mut().find(|c| c.is(b"ldat")) { - Some(c) => *c = Chunk::data(*b"ldat", records), - None => ld.push(Chunk::data(*b"ldat", records)), - } - Ok(()) -} - fn keyed_feather(t: &MaskProtos, keys: &[KeyPlan]) -> R> { let mut tdbs = t.keyed_feather.clone(); // The plain template with two components (208): 8 + 40 * 2 bytes. @@ -669,3 +629,37 @@ pub(crate) fn build_shap(proto: &Chunk<'static>, s: &ShapePlan) -> R MaskPlan { + let shape = ShapePlan { bbox: [0.0, 0.0, 1.0, 1.0], open: false, points: vec![[0.0, 0.0]; 3] }; + MaskPlan { + name: "M".into(), + mode: 1, + inverted: false, + shape, + path_keys: Vec::new(), + feather: None, + feather_keys: Vec::new(), + opacity: None, + expansion: None, + } + } + + /// An atom without its path is an error. + #[test] + fn an_atom_without_its_path_is_an_error() { + let mut t = MaskProtos::build(); + assert!(build_atom(&t, &mask(), (100, 100)).is_ok()); + // Without the Mask Shape pair the path of every mask would be dropped quietly. + let kids = kids_mut(&mut t.atom).unwrap(); + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Mask Shape")).unwrap(); + kids.drain(at..at + 2); + assert!(build_atom(&t, &mask(), (100, 100)).is_err()); + } +} diff --git a/crates/aep/src/export/report.rs b/crates/aep/src/export/report.rs index 7caba0e7d..22145956d 100644 --- a/crates/aep/src/export/report.rs +++ b/crates/aep/src/export/report.rs @@ -7,8 +7,9 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; -/// Most example lines kept per loss category (the count is always exact). -pub const MAX_DETAILS: usize = 20; +/// Most example lines kept per loss category (the count is always exact); the same as the import +/// report's. +pub const MAX_DETAILS: usize = crate::import::MAX_DETAILS; /// Why a piece of the project did not arrive in the `.aep`, or arrived changed. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] @@ -130,15 +131,9 @@ impl LossCategory { } } -/// One category of things that were not exported. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct Loss { - pub category: LossCategory, - /// How many occurrences (exact). - pub count: usize, - /// Up to [`MAX_DETAILS`] example lines. - pub details: Vec, -} +/// One category of things that were not exported: the import report's loss entry with the +/// export categories. +pub type Loss = crate::import::Loss; /// What was written to the file. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -222,7 +217,7 @@ pub struct ExportReport { impl ExportReport { /// The sum of all loss counts. pub fn total_losses(&self) -> usize { - self.losses.iter().map(|l| l.count).sum() + Loss::total(&self.losses) } /// True when nothing was lost. @@ -232,12 +227,12 @@ impl ExportReport { /// The loss count of a category. pub fn count(&self, category: LossCategory) -> usize { - self.losses.iter().find(|l| l.category == category).map_or(0, |l| l.count) + Loss::count_of(&self.losses, category) } /// `effects: 3, masks: 1` (empty when lossless). pub fn summary(&self) -> String { - self.losses.iter().map(|l| format!("{}: {}", l.category.label(), l.count)).collect::>().join(", ") + Loss::summary(&self.losses, LossCategory::label) } } @@ -251,11 +246,7 @@ pub(crate) struct Collector { impl Collector { /// Records one loss. pub fn lose(&mut self, category: LossCategory, detail: impl Into) { - let loss = self.losses.entry(category).or_insert_with(|| Loss { category, count: 0, details: Vec::new() }); - loss.count = loss.count.saturating_add(1); - if loss.details.len() < MAX_DETAILS { - loss.details.push(detail.into()); - } + Loss::record(&mut self.losses, category, move || detail.into()); } pub fn finish(self) -> ExportReport { @@ -266,3 +257,56 @@ impl Collector { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// A report with losses in two categories, as the collector finishes it. + fn collected() -> ExportReport { + let mut col = Collector::default(); + col.exported.layers = 2; + col.lose(LossCategory::RoundedTimes, "time 1"); + col.lose(LossCategory::Effects, String::from("fx 1")); + col.lose(LossCategory::Effects, "fx 2"); + let mut report = col.finish(); + report.limitations.truncate(1); + report.limitations[0] = "limit".into(); + report + } + + /// The JSON shape agents read: field names, category spelling, order and the loss entries. + #[test] + fn the_json_shape_is_pinned() { + let json = serde_json::to_string(&collected()).unwrap(); + assert_eq!( + json, + r#"{"exported":{"folders":0,"compositions":0,"solids":0,"layers":2,"solidLayers":0,"nullLayers":0,"adjustmentLayers":0,"properties":0,"keyframes":0,"expressions":0,"masks":0,"precompLayers":0,"effects":0,"footage":0,"footageLayers":0,"textLayers":0,"markers":0,"cameras":0,"lights":0,"shapeLayers":0,"timeRemaps":0,"textAnimators":0,"layerStyles":0},"losses":[{"category":"effects","count":2,"details":["fx 1","fx 2"],"unlisted":0},{"category":"rounded_times","count":1,"details":["time 1"],"unlisted":0}],"limitations":["limit"]}"# + ); + let back: ExportReport = serde_json::from_str(&json).unwrap(); + assert_eq!(back, collected()); + } + + /// The queries read the losses. + #[test] + fn the_queries_read_the_losses() { + let report = collected(); + assert_eq!((report.total_losses(), report.is_lossless()), (3, false)); + assert_eq!((report.count(LossCategory::Effects), report.count(LossCategory::Lights)), (2, 0)); + assert_eq!(report.summary(), "effects: 2, rounded times: 1"); + let empty = Collector::default().finish(); + assert_eq!((empty.total_losses(), empty.is_lossless(), empty.summary()), (0, true, String::new())); + } + + /// Details stop at the cap. + #[test] + fn details_stop_at_the_cap() { + let mut col = Collector::default(); + for i in 0..MAX_DETAILS + 5 { + col.lose(LossCategory::Masks, format!("mask {i}")); + } + let report = col.finish(); + assert_eq!(report.count(LossCategory::Masks), MAX_DETAILS + 5); + assert_eq!(report.losses[0].details.len(), MAX_DETAILS); + } +} diff --git a/crates/aep/src/export/shapes.rs b/crates/aep/src/export/shapes.rs index 74cd49d5b..981286604 100644 --- a/crates/aep/src/export/shapes.rs +++ b/crates/aep/src/export/shapes.rs @@ -24,10 +24,12 @@ use crate::Chunk; use effectcraft_keyframe::{Gradient, Interp, Keyframe, Value}; use effectcraft_project::{Node, PropGroup, Property}; +use crate::export::keys::{fill_key_list, replace_cdat, spatial_record, timed_keys}; use crate::export::masks::ShapePlan; use crate::export::plan::{CompPlan, EaseSlot, KeyPlan}; use crate::export::report::{Collector, LossCategory}; -use crate::export::tree::{R, child_data, kids_mut, list_mut, put, put_f64, put_u8, put_u16, put_u32, set_utf8, tpl}; +use crate::export::synth::{GROUP_END, is_tdmn, tdmn}; +use crate::export::tree::{R, child_data, kids_mut, list_mut, put_f64, put_u8, set_utf8, tpl}; /// The deepest group nesting written. const MAX_DEPTH: usize = 16; @@ -192,7 +194,7 @@ impl ShapeProtos { put_f64(child_data(&mut node, b"tdum")?, 0, lo)?; put_f64(child_data(&mut node, b"tduM")?, 0, hi)?; } - Ok([crate::export::camlight::tdmn_chunk(name), node]) + Ok([tdmn(name), node]) } } @@ -267,20 +269,7 @@ pub(crate) struct ShCx<'a> { /// The keys of `p` in the file's units. fn plan_keys(p: &Property, conv: Conv, layout: Layout, what: &str, cx: &mut ShCx<'_>) -> Vec { - let mut timed: Vec<(i64, &Keyframe)> = Vec::with_capacity(p.keys.len()); - for k in &p.keys { - let (u, inexact) = cx.cp.tb.units(k.time); - if inexact { - cx.col.lose(LossCategory::RoundedTimes, format!("{what}: key at {:.4} s moved to the time base", k.time.seconds())); - } - timed.push((u, k)); - } - timed.sort_by_key(|(u, _)| *u); - let before = timed.len(); - timed.dedup_by_key(|(u, _)| *u); - if timed.len() != before { - cx.col.lose(LossCategory::ApproximatedKeyframes, format!("{what}: keys at the same time were merged")); - } + let timed = timed_keys(&p.keys, &cx.cp.tb, what, "key", cx.col); let all: Vec = timed.iter().map(|(_, k)| (*k).clone()).collect(); let mut out = Vec::with_capacity(timed.len()); let beyond = std::cell::Cell::new(false); @@ -618,38 +607,6 @@ pub(crate) fn plan_contents(contents: &PropGroup, what: &str, depth: usize, cx: out } -fn tdmn_name(c: &Chunk<'_>) -> Option { - let b = c.bytes()?; - let end = b.iter().position(|x| *x == 0).unwrap_or(b.len()); - std::str::from_utf8(b.get(..end)?).ok().map(str::to_string) -} - -/// A key of a point (the spatial layout of a 2D Position, 853) or a colour (854). -fn spatial_key(k: &KeyPlan, n: usize, point: bool, size: usize) -> R> { - let mut r = vec![0u8; size]; - put(&mut r, 0, &k.time.to_be_bytes())?; - put_u8(&mut r, 4, k.in_interp)?; - put_u8(&mut r, 5, k.out_interp)?; - let base: u16 = if point { 0x0007 } else { 0x0001 }; - put_u16(&mut r, 6, base | (u16::from(k.label) << 6))?; - if point { - put_u32(&mut r, 8, u32::from(k.spatial_auto))?; - let (i, o) = (k.in_ease.first().copied().unwrap_or_default(), k.out_ease.first().copied().unwrap_or_default()); - put_f64(&mut r, 24, i.speed)?; - put_f64(&mut r, 32, i.influence)?; - put_f64(&mut r, 40, o.speed)?; - put_f64(&mut r, 48, o.influence)?; - } - for j in 0..n { - put_f64(&mut r, 56 + 8 * j, k.value.get(j).copied().unwrap_or(0.0))?; - if point { - put_f64(&mut r, 56 + 8 * n + 8 * j, k.spatial_in.get(j).copied().unwrap_or(0.0))?; - put_f64(&mut r, 56 + 16 * n + 8 * j, k.spatial_out.get(j).copied().unwrap_or(0.0))?; - } - } - Ok(r) -} - /// The value record of `name` holding `v`. fn value_record(t: &ShapeProtos, name: &'static str, v: &RecVal) -> R<[Chunk<'static>; 2]> { let [m, node] = t.record(name)?; @@ -681,32 +638,17 @@ pub(crate) fn build_record(m: Chunk<'static>, mut node: Chunk<'static>, layout: put_u8(tdb4, 79, 0)?; } } - let mut list = key_list.clone(); - let count = u32::try_from(v.keys.len()).map_err(|_| tpl("too many keys"))?; - { - let lhd3 = child_data(&mut list, b"lhd3")?; - crate::export::synth::put_key_counts(lhd3, count, u32::try_from(size).map_err(|_| tpl("record size"))?)?; - } let mut ldat = Vec::with_capacity(size * v.keys.len()); for k in &v.keys { ldat.extend(match layout { - Layout::Point => spatial_key(k, n, true, size)?, - Layout::Color => spatial_key(k, n, false, size)?, + Layout::Point => spatial_record(k, n, true, size)?, + Layout::Color => spatial_record(k, n, false, size)?, _ => crate::export::emit::plain_record(n, k, size)?, }); } - { - let lk = kids_mut(&mut list)?; - match lk.iter_mut().find(|c| c.is(b"ldat")) { - Some(c) => *c = Chunk::data(*b"ldat", ldat), - None => lk.push(Chunk::data(*b"ldat", ldat)), - } - } - let kids = kids_mut(&mut node)?; - let at = kids.iter().position(|c| c.is(b"cdat")).ok_or_else(|| tpl("shape record without cdat"))?; - if let Some(slot) = kids.get_mut(at) { - *slot = list; - } + let mut list = key_list.clone(); + fill_key_list(&mut list, ldat, v.keys.len(), size)?; + replace_cdat(&mut node, list)?; } if let Some((text, enabled)) = &v.expr { // As on the Transform properties (R15; 856). @@ -746,7 +688,7 @@ pub(crate) fn build_items(t: &ShapeProtos, items: &[ShItem], depth: usize) -> R< if depth > MAX_DEPTH { return Ok(out); } - let pos = |kids: &[Chunk<'static>], n: &str| kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(n)); + let pos = |kids: &[Chunk<'static>], n: &str| kids.iter().position(|c| is_tdmn(c, n)); for item in items { match item { ShItem::Off(inner) => { @@ -766,7 +708,7 @@ pub(crate) fn build_items(t: &ShapeProtos, items: &[ShItem], depth: usize) -> R< let vg = pos(kids, "ADBE Vectors Group").ok_or_else(|| tpl("group prototype without contents"))?; let contents = kids.get_mut(vg + 1).ok_or_else(|| tpl("group prototype without contents"))?; let ck = kids_mut(contents)?; - let end = ck.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Group End")).unwrap_or(ck.len()); + let end = ck.iter().position(|c| is_tdmn(c, GROUP_END)).unwrap_or(ck.len()); ck.splice(end..end, children); let tg = pos(kids, "ADBE Vector Transform Group").ok_or_else(|| tpl("group prototype without transform"))?; let tnode = kids.get_mut(tg + 1).ok_or_else(|| tpl("group prototype without transform"))?; @@ -815,13 +757,10 @@ pub(crate) fn put_contents(layr: &mut Chunk<'static>, t: &ShapeProtos, items: &[ let built = build_items(t, items, 0)?; let root = list_mut(layr, b"tdgp")?; let kids = kids_mut(root)?; - let at = kids - .iter() - .position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Root Vectors Group")) - .ok_or_else(|| tpl("shape layer without contents"))?; + let at = kids.iter().position(|c| is_tdmn(c, "ADBE Root Vectors Group")).ok_or_else(|| tpl("shape layer without contents"))?; let rv = kids.get_mut(at + 1).ok_or_else(|| tpl("shape layer without contents"))?; let rk = kids_mut(rv)?; - let end = rk.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some("ADBE Group End")).unwrap_or(rk.len()); + let end = rk.iter().position(|c| is_tdmn(c, GROUP_END)).unwrap_or(rk.len()); rk.splice(end..end, built); Ok(()) } diff --git a/crates/aep/src/export/skeleton.rs b/crates/aep/src/export/skeleton.rs index 037b4d865..38a2c17ab 100644 --- a/crates/aep/src/export/skeleton.rs +++ b/crates/aep/src/export/skeleton.rs @@ -4,14 +4,14 @@ //! e1-sparse). use crate::Chunk; -use crate::export::synth::{KeyTemplate, NO_NAME, RecSpec, class, tdbs, tdmn, tdsn}; +use crate::export::synth::{GROUP_END, KeyTemplate, NO_NAME, RecSpec, class, tdbs, tdmn, tdmn_name, tdsn}; /// A group: its match name and a `tdgp` with the state `state` (`tdsb`), no user name, `kids` and /// the closing `ADBE Group End`. pub(crate) fn group(name: &str, state: u32, kids: Vec>) -> [Chunk<'static>; 2] { let mut body = vec![Chunk::data(*b"tdsb", state.to_be_bytes().to_vec()), tdsn(NO_NAME)]; body.extend(kids); - body.push(tdmn("ADBE Group End")); + body.push(tdmn(GROUP_END)); [tdmn(name), Chunk::list(*b"tdgp", body)] } @@ -61,7 +61,7 @@ pub(crate) fn av_layer() -> Vec> { let mut root = vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn(""), tdmn("ADBE Time Remapping"), time_remap()]; root.extend(group("ADBE Transform Group", 1, crate::export::tree_spec::build(crate::export::tree_defs::AV_TRANSFORM))); root.extend(layer_styles()); - root.push(tdmn("ADBE Group End")); + root.push(tdmn(GROUP_END)); let layr = Chunk::list( *b"Layr", vec![Chunk::data(*b"ldta", vec![0; 164]), Chunk::data(*b"Utf8", Vec::new()), Chunk::list(*b"tdgp", root), crate::export::base::guides()], @@ -78,7 +78,7 @@ pub(crate) fn av_layer() -> Vec> { pub(crate) fn layer_block(nodes: &[crate::export::tree_spec::Node]) -> Vec> { let mut root = vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), tdsn("")]; root.extend(crate::export::tree_spec::build(nodes)); - root.push(tdmn("ADBE Group End")); + root.push(tdmn(GROUP_END)); let layr = Chunk::list( *b"Layr", vec![Chunk::data(*b"ldta", vec![0; 164]), Chunk::data(*b"Utf8", Vec::new()), Chunk::list(*b"tdgp", root), crate::export::base::guides()], @@ -125,7 +125,7 @@ pub(crate) fn text_layer(doc: &[u8]) -> Vec> { root.extend(group("ADBE Text Properties", 1, props)); root.extend(group("ADBE Transform Group", 1, crate::export::tree_spec::build(crate::export::tree_defs::AV_TRANSFORM))); root.extend(layer_styles()); - root.push(tdmn("ADBE Group End")); + root.push(tdmn(GROUP_END)); let layr = Chunk::list( *b"Layr", vec![Chunk::data(*b"ldta", vec![0; 164]), Chunk::data(*b"Utf8", Vec::new()), Chunk::list(*b"tdgp", root), crate::export::base::guides()], @@ -141,7 +141,7 @@ pub(crate) fn pairs(nodes: &[crate::export::tree_spec::Node]) -> Vec<(String, [C let mut out = Vec::new(); let mut it = crate::export::tree_spec::build(nodes).into_iter(); while let (Some(m), Some(n)) = (it.next(), it.next()) { - let name = String::from_utf8_lossy(m.bytes().unwrap_or_default().split(|b| *b == 0).next().unwrap_or_default()).into_owned(); + let name = tdmn_name(&m).unwrap_or_default(); out.push((name, [m, n])); } out diff --git a/crates/aep/src/export/styles.rs b/crates/aep/src/export/styles.rs index 4b8cb51f1..1533de7b9 100644 --- a/crates/aep/src/export/styles.rs +++ b/crates/aep/src/export/styles.rs @@ -23,8 +23,10 @@ use effectcraft_project::build::Ids; use effectcraft_project::styles::{self as st, STYLE_BLEND_MODES}; use effectcraft_project::{Comp, Layer, PropGroup, Property}; +use crate::export::keys::{fill_key_list, replace_cdat, timed_keys}; use crate::export::plan::{CompPlan, EaseSlot, KeyPlan}; use crate::export::report::{Collector, LossCategory}; +use crate::export::synth::{GROUP_END, is_tdmn}; use crate::export::tree::{R, child_data, kids_mut, put_f64, put_u8, put_u32, tpl}; /// A record's value. @@ -128,20 +130,7 @@ fn plan_record(spec: &StyleParam, full: String, prop: &Property, cp: &CompPlan, /// The keys of a scalar in the layer's time base: linear, hold and bezier sides with the ease in /// stored units per second (the plain template of 976). fn plan_number_keys(prop: &Property, cp: &CompPlan, what: &str, col: &mut Collector) -> Vec { - let mut timed: Vec<(i64, &effectcraft_keyframe::Keyframe)> = Vec::with_capacity(prop.keys.len()); - for k in &prop.keys { - let (u, inexact) = cp.tb.units(k.time); - if inexact { - col.lose(LossCategory::RoundedTimes, format!("{what}: key at {:.4} s moved to the time base", k.time.seconds())); - } - timed.push((u, k)); - } - timed.sort_by_key(|(u, _)| *u); - let before = timed.len(); - timed.dedup_by_key(|(u, _)| *u); - if timed.len() != before { - col.lose(LossCategory::ApproximatedKeyframes, format!("{what}: keys at the same time were merged")); - } + let timed = timed_keys(&prop.keys, &cp.tb, what, "key", col); let mut out = Vec::with_capacity(timed.len()); let beyond = std::cell::Cell::new(false); for (u, k) in timed { @@ -279,11 +268,6 @@ pub(crate) struct StyleProtos { keyed: Chunk<'static>, } -fn tdmn_name(c: &Chunk<'_>) -> Option { - let d = c.bytes()?; - Some(d.iter().take_while(|b| **b != 0).map(|b| char::from(*b)).collect()) -} - impl StyleProtos { /// The records built in code from the style table (`style_defs`); the keyed form of a scalar /// takes `tdb4` bytes 5, 68 and 79 and the key list of a keyed one-component record. @@ -322,28 +306,13 @@ impl StyleProtos { } } let mut list = self.keyed.child_list(b"list").cloned().ok_or_else(|| tpl("keyed style prototype without keys"))?; - let n = u32::try_from(keys.len()).map_err(|_| tpl("too many keys"))?; let size = 48usize; - { - let lhd3 = child_data(&mut list, b"lhd3")?; - crate::export::synth::put_key_counts(lhd3, n, 48)?; - } let mut ldat = Vec::with_capacity(size * keys.len()); for k in keys { ldat.extend(crate::export::emit::plain_record(1, k, size)?); } - { - let lk = kids_mut(&mut list)?; - match lk.iter_mut().find(|c| c.is(b"ldat")) { - Some(c) => *c = Chunk::data(*b"ldat", ldat), - None => lk.push(Chunk::data(*b"ldat", ldat)), - } - } - let kids = kids_mut(&mut rec)?; - let at = kids.iter().position(|c| c.is(b"cdat")).ok_or_else(|| tpl("style record without cdat"))?; - if let Some(slot) = kids.get_mut(at) { - *slot = list; - } + fill_key_list(&mut list, ldat, keys.len(), size)?; + replace_cdat(&mut rec, list)?; } } if let Some((text, enabled)) = &r.expr { @@ -362,7 +331,7 @@ impl StyleProtos { /// The node after the `tdmn` named `name` among `kids`. fn named_mut<'a>(kids: &'a mut [Chunk<'static>], name: &str) -> R<&'a mut Chunk<'static>> { - let i = kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(name)).ok_or_else(|| tpl("layer styles group incomplete"))?; + let i = kids.iter().position(|c| is_tdmn(c, name)).ok_or_else(|| tpl("layer styles group incomplete"))?; kids.get_mut(i + 1).ok_or_else(|| tpl("layer styles group incomplete")) } @@ -377,7 +346,7 @@ fn insert_records(protos: &StyleProtos, g: &mut Chunk<'static>, recs: &[Record], built.extend(protos.build(r)?); } let kids = kids_mut(g)?; - let at = kids.iter().position(|c| c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(before)).ok_or_else(|| tpl("layer styles group incomplete"))?; + let at = kids.iter().position(|c| is_tdmn(c, before)).ok_or_else(|| tpl("layer styles group incomplete"))?; kids.splice(at..at, built); Ok(()) } @@ -392,12 +361,12 @@ pub(crate) fn put_styles(protos: &StyleProtos, layr: &mut Chunk<'static>, plan: set_flags(bo, sm::FLAGS_ON)?; insert_records(protos, bo, &plan.global, sm::ADV_BLEND)?; let adv = named_mut(kids_mut(bo)?, sm::ADV_BLEND)?; - insert_records(protos, adv, &plan.adv, "ADBE Group End")?; + insert_records(protos, adv, &plan.adv, GROUP_END)?; } for (prefix, recs, on) in &plan.styles { let g = named_mut(kids_mut(ls)?, &format!("{prefix}/enabled"))?; set_flags(g, if *on { sm::FLAGS_ON } else { sm::FLAGS_OFF })?; - insert_records(protos, g, recs, "ADBE Group End")?; + insert_records(protos, g, recs, GROUP_END)?; } Ok(()) } diff --git a/crates/aep/src/export/synth.rs b/crates/aep/src/export/synth.rs index 07d75c194..8af515f24 100644 --- a/crates/aep/src/export/synth.rs +++ b/crates/aep/src/export/synth.rs @@ -4,9 +4,10 @@ //! evidence, or zero where After Effects was shown to accept zero (the read-backs listed in //! `docs/aep-export.md`, "Built in code"). +use crate::export::tree::{Fixed, R, put_u32}; use crate::{Body, Chunk}; -/// `tdmn`: a match name padded to 40 bytes. +/// `tdmn`: a match name padded to 40 bytes (at most 39 bytes of the name, so that a NUL remains). pub(crate) fn tdmn(name: &str) -> Chunk<'static> { let mut b = vec![0u8; 40]; for (slot, c) in b.iter_mut().zip(name.bytes().take(39)) { @@ -15,12 +16,34 @@ pub(crate) fn tdmn(name: &str) -> Chunk<'static> { Chunk::data(*b"tdmn", b) } +/// The match name that closes a property group. +pub(crate) const GROUP_END: &str = "ADBE Group End"; + +/// The match name in a `tdmn`: the bytes up to the first NUL as UTF-8, invalid sequences +/// replaced, as the importer reads it (`container::typed`). After Effects writes ASCII here +/// (aep-format.md 3.4; every `tdmn` of the corpus is ASCII); `None` when `c` is not a data chunk. +pub(crate) fn tdmn_name(c: &Chunk<'_>) -> Option { + let b = c.bytes()?; + let end = b.iter().position(|x| *x == 0).unwrap_or(b.len()); + Some(String::from_utf8_lossy(b.get(..end)?).into_owned()) +} + +/// Whether `c` is a `tdmn` naming `name`. +pub(crate) fn is_tdmn(c: &Chunk<'_>, name: &str) -> bool { + c.is(b"tdmn") && tdmn_name(c).as_deref() == Some(name) +} + /// The user name of a property that has none (`-_0_/-`, aep-format.md 3.4). pub(crate) const NO_NAME: &str = "-_0_/-"; /// `tdsn`: the user name, a nested chunk holding one `Utf8`. pub(crate) fn tdsn(name: &str) -> Chunk<'static> { - Chunk { id: *b"tdsn", body: Body::Chunks(vec![Chunk::data(*b"Utf8", name.as_bytes().to_vec())]), pad: None } + utf8_in(*b"tdsn", name) +} + +/// A nested chunk `id` holding one `Utf8` with `text` (`tdsn`, `fnam`, `pdnm`). +pub(crate) fn utf8_in(id: [u8; 4], text: &str) -> Chunk<'static> { + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text.as_bytes().to_vec())]), pad: None } } /// How a property's keys are stored (`tdb4` byte 7, aep-format.md 3.5). @@ -100,13 +123,13 @@ impl RecSpec { /// word and the constant doubles at 16..56 are written; After Effects reads the other bytes as /// zero (read-back e1-ztdb4d). pub(crate) fn tdb4(spec: &RecSpec, keyed: bool, time_scale: u32) -> Vec { - let mut b = vec![0u8; 124]; - let set = |b: &mut Vec, at: usize, bytes: &[u8]| { - if let Some(dst) = b.get_mut(at..at + bytes.len()) { - dst.copy_from_slice(bytes); - } - }; - set(&mut b, 0, &[0xdb, 0x99, 0x00, spec.dims]); + tdb4_layout(spec, keyed, time_scale).finish(|| "tdb4".to_string()) +} + +/// The 124-byte `tdb4` property header layout for a record spec. +fn tdb4_layout(spec: &RecSpec, keyed: bool, time_scale: u32) -> Fixed { + let mut b = Fixed::zeroed(124); + b.put(0, &[0xdb, 0x99, 0x00, spec.dims]); let spatial = match spec.template { KeyTemplate::Plain => 0, KeyTemplate::Spatial(t) => t, @@ -115,19 +138,19 @@ pub(crate) fn tdb4(spec: &RecSpec, keyed: bool, time_scale: u32) -> Vec { Some(b) => (b & !0x01) | u8::from(!keyed), None => (if spatial == 0 { 0 } else { spatial_byte5(spatial) }) | u8::from(!keyed), }; - set(&mut b, 5, &[byte5]); - set(&mut b, 7, &[spatial]); - set(&mut b, 8, &spec.class.to_be_bytes()); - set(&mut b, 12, &time_scale.to_be_bytes()); + b.put(5, &[byte5]); + b.put(7, &[spatial]); + b.put(8, &spec.class.to_be_bytes()); + b.put(12, &time_scale.to_be_bytes()); // Every record After Effects writes carries 0.0001 and four 1.0 here; without them a roving // key reads back at time zero instead of its retimed place (read-back s14-re-keyframes). - set(&mut b, 16, &1e-4f64.to_be_bytes()); + b.put(16, &1e-4f64.to_be_bytes()); for at in [24, 32, 40, 48] { - set(&mut b, at, &1.0f64.to_be_bytes()); + b.put(at, &1.0f64.to_be_bytes()); } - set(&mut b, 68, &[u8::from(keyed)]); + b.put(68, &[u8::from(keyed)]); // Set on a static spatial property, cleared once it is keyed (aep-format.md 3.4, byte 79). - set(&mut b, 79, &[u8::from(spatial != 0 && !keyed)]); + b.put(79, &[u8::from(spatial != 0 && !keyed)]); b } @@ -135,16 +158,21 @@ pub(crate) fn tdb4(spec: &RecSpec, keyed: bool, time_scale: u32) -> Vec { /// 3.5): word 0 `0x00d00bee`, word 2 the count, word 3 the blocks of four (at least one), word 4 /// the size, words 5 and 7 the capacity (the count rounded up to four, at least two), word 6 one. pub(crate) fn key_list(size: u32, count: u32, records: Vec) -> Chunk<'static> { - let mut lhd3 = vec![0u8; 52]; - // Cannot fail: the header is 52 bytes. - let _ = crate::export::tree::put_u32(&mut lhd3, 0, 0x00d0_0bee); - let _ = crate::export::tree::put_u32(&mut lhd3, 24, 1); - let _ = put_key_counts(&mut lhd3, count, size); + let lhd3 = key_list_header(size, count).finish(|| "lhd3".to_string()); Chunk::list(*b"list", vec![Chunk::data(*b"lhd3", lhd3), Chunk::data(*b"ldat", records)]) } +/// The 52-byte `lhd3` key list header for `count` records of `size` bytes. +fn key_list_header(size: u32, count: u32) -> Fixed { + let mut lhd3 = Fixed::zeroed(52); + lhd3.put(0, &0x00d0_0bee_u32.to_be_bytes()); + lhd3.put(24, &1u32.to_be_bytes()); + lhd3.apply(|d| put_key_counts(d, count, size)); + lhd3 +} + /// Sets the count words of a key list header for `count` keys of `size` bytes. -pub(crate) fn put_key_counts(lhd3: &mut [u8], count: u32, size: u32) -> crate::export::tree::R<()> { +pub(crate) fn put_key_counts(lhd3: &mut [u8], count: u32, size: u32) -> R<()> { let capacity = count.div_ceil(4).saturating_mul(4).max(2); put_counts(lhd3, count, count.div_ceil(4).max(1), size, capacity, capacity) } @@ -153,8 +181,7 @@ pub(crate) fn put_key_counts(lhd3: &mut [u8], count: u32, size: u32) -> crate::e /// record size, word 5 `w5`, word 7 the capacity. A key list has `w5` = the capacity; a path /// list counts vertices: `blocks` = `w5` = the vertex count, the capacity four times it (090: /// 12 points, 4, 4 and 16; aep-format.md 3.5). -pub(crate) fn put_counts(lhd3: &mut [u8], count: u32, blocks: u32, size: u32, w5: u32, capacity: u32) -> crate::export::tree::R<()> { - use crate::export::tree::put_u32; +pub(crate) fn put_counts(lhd3: &mut [u8], count: u32, blocks: u32, size: u32, w5: u32, capacity: u32) -> R<()> { put_u32(lhd3, 8, count)?; put_u32(lhd3, 12, blocks)?; put_u32(lhd3, 16, size)?; @@ -204,6 +231,38 @@ mod tests { assert_eq!(rec.child(b"cdat").and_then(|c| c.bytes()).map(<[u8]>::len), Some(40)); } + /// Every record header fits. + #[test] + fn every_record_header_fits() { + for dims in 0..=u8::MAX { + for template in [KeyTemplate::Plain, KeyTemplate::Spatial(1), KeyTemplate::Spatial(3), KeyTemplate::Spatial(u8::MAX)] { + for byte5 in [None, Some(0xff)] { + let spec = RecSpec { dims, template, class: u32::MAX, limits: None, byte5 }; + assert!(tdb4_layout(&spec, false, u32::MAX).error().is_none() && tdb4_layout(&spec, true, 0).error().is_none()); + } + } + } + for (size, count) in [(0, 0), (u32::MAX, u32::MAX), (48, 5)] { + let h = key_list_header(size, count); + assert!(h.error().is_none()); + assert_eq!(h.finish(String::new).len(), 52); + } + } + + /// A match name reads as the importer reads it. + #[test] + fn a_match_name_reads_as_the_importer_reads_it() { + let m = tdmn("ADBE Group End"); + assert_eq!(tdmn_name(&m).as_deref(), Some(GROUP_END)); + assert!(is_tdmn(&m, GROUP_END) && !is_tdmn(&m, "ADBE Group") && !is_tdmn(&tdsn(GROUP_END), GROUP_END)); + // At most 39 bytes, so that a NUL remains. + let long = "x".repeat(50); + assert_eq!(tdmn_name(&tdmn(&long)).map(|n| n.len()), Some(39)); + // Not ASCII (After Effects writes ASCII): UTF-8, invalid bytes replaced, as on import. + let odd = Chunk::data(*b"tdmn", vec![b'A', 0xc3, 0xa9, 0xff, 0, b'B']); + assert_eq!(tdmn_name(&odd).as_deref(), Some("A\u{e9}\u{fffd}")); + } + #[test] fn a_key_list_header_counts_and_rounds() { let l = key_list(48, 5, vec![0; 240]); diff --git a/crates/aep/src/export/template.rs b/crates/aep/src/export/template.rs index 91daba636..f26be4c84 100644 --- a/crates/aep/src/export/template.rs +++ b/crates/aep/src/export/template.rs @@ -81,6 +81,24 @@ fn transform_records(keyed: bool) -> Vec> { .collect() } +/// The template every export clones from: built by the first export of the process and kept, as +/// [`Template::load`] always builds the same prototypes and nothing changes them afterwards (the +/// exporter only reads them through `&Template` and clones what it edits). +/// +/// Concurrent exports are safe: the `static` requires `Template: Send + Sync`, which the compiler +/// checks. Exports that race on the first call each build a template; one is kept and the others +/// are dropped, and all of them are equal. A failed build is not kept, so the next export tries +/// again and reports the same error. +pub(crate) fn shared() -> R<&'static Template> { + static SHARED: std::sync::OnceLock