Files
effectcraft/crates/render/src/audio.rs
T

671 lines
31 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! Audio mixdown of a composition: every audible layer (footage with audio, precomps, and any
//! layer with a sound-generating effect such as Tone) summed with its Audio Levels (dB, per
//! channel), respecting in/out points, the Audio switch and solo. Layer audio effects
//! (Effect > Audio, see [`effectcraft_effects::audio_fx`]) run on each layer's audio before
//! its levels; Backwards plays the layer's [in, out) span reversed.
//!
//! Callers mix in short blocks (one video frame, an audio device buffer, or a second of export).
//! Effect parameters are evaluated once per block. Animated Audio Levels, and animated effects
//! that work sample by sample (Stereo Mixer), are evaluated on a grid of comp samples every
//! millisecond instead, so keyframed fades ramp smoothly whatever the block size. To make the
//! result independent of block boundaries, layer effects are run from a cleared state over a
//! pre-roll of earlier audio and only the block's tail is kept — so preview playback and export
//! produce identical samples.
//!
//! Time-stretched layers are resampled linearly, so pitch follows the speed. A reversed layer
//! (negative stretch, Layer > Time > Time-Reverse Layer) plays its audio backwards at the same
//! speed: comp sample `k` plays the source just before layer time `layer_time(k)`, so at -100 %
//! with the start at the out point (After Effects' layout) it is the exact mirror of the
//! layer's span. Backwards on a reversed layer plays it forwards again.
use effectcraft_effects::Params;
use effectcraft_effects::audio_fx;
use effectcraft_project::{Comp, GroupKind, ItemId, ItemKind, Layer, LayerSource, Node, Project};
use effectcraft_time::{TICKS_PER_SECOND, Tick};
use crate::{EvalCtx, ExprHost, FootageSource};
const MAX_DEPTH: usize = 16;
/// Fastest a layer's audio plays (source samples per comp sample): 1 % stretch, the smallest
/// Time Stretch accepts. The mix plays the clamped stretch (`Layer::effective_stretch`, #3247),
/// so a layer is never faster; a faster speed would be silent instead of reading unbounded
/// source: a last-resort guard.
const MAX_SPEED: f64 = 100.0;
/// Most source sample frames one read asks a footage item or precomp for; a longer read is made
/// in pieces, so memory stays bounded however fast a layer plays and however deep stretched
/// precomps nest. The work is not bounded: each 1 % precomp level still reads 100 times its
/// span.
pub(crate) const MAX_READ: usize = 1 << 18;
/// The most effect preroll (history before a block) mixed, in seconds, across a layer's effects.
const MAX_PREROLL_SECONDS: f64 = 30.0;
/// The longest footage a waveform summarises (a day).
const MAX_PEAK_SECONDS: i64 = 24 * 3600;
/// `frames` stereo sample frames of `comp`'s mix from comp time `start` at `rate` Hz,
/// interleaved (L R L R …).
pub fn mix_comp(project: &Project, footage: &dyn FootageSource, expr: Option<&dyn ExprHost>, comp: ItemId, start: Tick, frames: usize, rate: u32) -> Vec<f32> {
let mut out = vec![0.0f32; frames * 2];
mix_into(&AudioSources { project, footage, expr, rate }, comp, start, &mut out, 0);
out
}
/// Whether the comp has anything audible (footage with audio, a sound generator, or a precomp
/// that has).
pub fn comp_has_audio(project: &Project, comp: ItemId) -> bool {
// Breadth first, so each comp is looked at once and at its shallowest level: a comp reached
// first through a long chain (past MAX_DEPTH) is still found by a shorter route, and a cycle
// cannot hide a result (nothing is cached as "silent" while it is being explored).
let mut seen = std::collections::HashSet::from([comp]);
let mut level = vec![comp];
for _ in 0..MAX_DEPTH {
let mut next = vec![];
for id in level {
let Some(c) = project.comp(id) else { continue };
for l in c.layers.iter().filter(|l| l.switches.audio) {
match &l.source {
_ if generates(l) => return true,
LayerSource::Footage { item } if matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio && !f.missing) => {
return true;
}
LayerSource::Comp { item } if seen.insert(*item) => next.push(*item),
_ => {}
}
}
}
level = next;
}
false
}
/// Enabled audio effects of a layer (in stack order) with their effect ids.
fn audio_effect_groups(l: &Layer) -> impl Iterator<Item = (&str, &effectcraft_project::PropGroup)> {
l.effects().filter(|_| l.switches.effects).into_iter().flat_map(|fx| fx.groups()).filter(|g| g.enabled).filter_map(|g| match &g.kind {
GroupKind::Effect { effect } if audio_fx::is_audio_effect(effect) => Some((effect.as_str(), g)),
_ => None,
})
}
/// The layer makes sound on its own (Tone).
fn generates(l: &Layer) -> bool {
audio_effect_groups(l).any(|(id, _)| audio_fx::generates_audio(id))
}
/// Does the layer contribute to the mix (Audio switch on and something to hear)?
pub fn audible(project: &Project, l: &Layer) -> bool {
l.switches.audio
&& (generates(l)
|| match &l.source {
LayerSource::Footage { item } => matches!(project.item(*item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_audio),
LayerSource::Comp { .. } => true,
_ => false,
})
}
/// Linear gain of an Audio Levels value in dB; -96 dB and below is silence.
pub fn db_to_gain(db: f64) -> f32 {
if db <= -96.0 { 0.0 } else { 10f64.powf(db / 20.0) as f32 }
}
fn units_ceil(t: Tick, rate: u32) -> i64 {
let n = t.0 as i128 * rate as i128;
let d = TICKS_PER_SECOND as i128;
(n.div_euclid(d) + i128::from(n.rem_euclid(d) != 0)) as i64
}
/// Samples between evaluations of animated levels and per-sample effects: one millisecond, on
/// a grid of comp sample indices so the result does not depend on where a block starts.
fn param_step(rate: u32) -> i64 {
i64::from(rate / 1000).max(1)
}
/// Does any property of the group change over time?
fn animated(g: &effectcraft_project::PropGroup) -> bool {
g.children.iter().any(|c| match c {
Node::Prop(p) => p.is_animated() || p.has_expression(),
Node::Group(sub) => animated(sub),
})
}
/// The clock (seconds) a layer's audio effects run on at comp time `t` (oscillator phase): comp
/// time since the layer's start. It advances one sample per sample at any stretch, forwards or
/// reversed, so blocks join up; at 100 % it is the layer time.
fn fx_clock(l: &Layer, t: Tick) -> f64 {
Tick(t.0.saturating_sub(l.start_time.0)).seconds()
}
fn sample_tick(i: i64, rate: u32) -> Tick {
// Keep the requested sample when the rate does not divide the tick clock.
let t = Tick::from_units(i, i64::from(rate));
if t.to_units_floor(i64::from(rate)) < i { Tick(t.0.saturating_add(1)) } else { t }
}
struct MixEnv<'a> {
project: &'a Project,
footage: &'a dyn FootageSource,
expr: Option<&'a dyn ExprHost>,
comp_id: ItemId,
comp: &'a Comp,
rate: u32,
depth: usize,
}
impl MixEnv<'_> {
fn ctx(&self, t: Tick) -> EvalCtx<'_> {
let mut ctx = EvalCtx::new(self.project, self.comp_id, self.comp, t);
ctx.expr = self.expr;
ctx
}
/// The layer's own (pre-effect) audio for comp sample indices `i0 .. i0 + n`, silent
/// outside its [in, out) span. Comp sample index `k` maps to `in + out − 1 − k` when
/// `reversed` (the Backwards effect).
fn layer_raw(&self, l: &Layer, i0: i64, n: usize, reversed: bool) -> Vec<f32> {
let rate = self.rate;
let mut out = vec![0.0f32; n * 2];
let (in_i, out_i) = (units_ceil(l.in_point, rate), units_ceil(l.out_point, rate));
if reversed {
// Mirror the clipped range, fetch it forwards, reverse.
let a = i0.max(in_i);
let b = (i0 + n as i64).min(out_i);
if a >= b {
return out;
}
let (ma, mb) = (in_i + out_i - b, in_i + out_i - a);
let fwd = self.layer_raw(l, ma, (mb - ma) as usize, false);
for k in a..b {
let j = (in_i + out_i - 1 - k - ma) as usize;
let o = (k - i0) as usize;
out[o * 2] = fwd[j * 2];
out[o * 2 + 1] = fwd[j * 2 + 1];
}
return out;
}
let a = (in_i - i0).clamp(0, n as i64) as usize;
let b = (out_i - i0).clamp(0, n as i64) as usize;
if a >= b {
return out;
}
// Within ±1 % to ±9900 % (#3247): a stretch of 1e-300 % set in memory read about
// usize::MAX source samples for one block. A negative stretch plays backwards (#3239).
let speed = 100.0 / l.effective_stretch();
let aspeed = speed.abs();
// The clamped stretch keeps the speed within MAX_SPEED; NaN is checked anyway. The
// tolerance keeps exactly 1 % audible whatever the float noise in the stretch.
if aspeed.is_nan() || aspeed > MAX_SPEED * (1.0 + 1e-9) {
return out;
}
let backwards = speed < 0.0;
// Comp samples per piece, so that one piece reads at most MAX_READ source frames (its
// span, the fraction it starts past a source sample, and the sample after its end).
let piece = (((MAX_READ - 4) as f64 / aspeed) as usize).clamp(1, b - a);
let mut p0 = a;
while p0 < b {
let p1 = p0.saturating_add(piece).min(b);
// Forwards a piece reads the source from the layer time of its first sample. Backwards
// it reads from the layer time of its end and plays what it read last to first, so
// comp sample `k` plays the source just before `layer_time(k)` — at -100 % the exact
// mirror Backwards gives a forward layer.
let origin = sample_tick(i0.saturating_add((if backwards { p1 } else { p0 }) as i64), rate);
let Some(run) = self.source_run(l, l.layer_time(origin), p1 - p0, aspeed) else { return out };
for (i, f) in run.as_chunks::<2>().0.iter().take(p1 - p0).enumerate() {
let k = if backwards { (p1 - 1).saturating_sub(i) } else { p0 + i };
if let Some(o) = out.get_mut(k * 2..k * 2 + 2) {
o.copy_from_slice(f);
}
}
p0 = p1;
}
out
}
/// `m` sample frames of the layer's source audio from layer time `lt` on, played at `speed`
/// (≥ 0) source samples per comp sample: blended linearly between source samples, or copied
/// as they are at speed 1. `None` when the layer has no source audio.
///
/// Sources deliver samples on their sample grid (from the sample at or before the time asked
/// for), so a resampled run reads from that grid sample and starts `x` at `lt`'s exact
/// position past it: a run then continues where the previous block or piece left off.
fn source_run(&self, l: &Layer, lt: Tick, m: usize, speed: f64) -> Option<Vec<f32>> {
let rate = self.rate;
let same = (speed - 1.0).abs() < 1e-9;
let s0 = lt.to_units_floor(i64::from(rate));
let from = sample_tick(s0, rate);
// `lt`'s position past sample s0, in samples (0 ≤ frac < 1); copies ignore it.
let tps = TICKS_PER_SECOND as i128;
let frac = if same { 0.0 } else { (i128::from(lt.0) * i128::from(rate)).rem_euclid(tps) as f64 / tps as f64 };
// Resampled, one more source sample than the run covers: the last sample blends with it.
let src_n = ((frac + m as f64 * speed).ceil() as usize).saturating_add(usize::from(!same)).max(1);
let buf = match &l.source {
LayerSource::Footage { item } => match self.project.item(*item).map(|i| &i.kind) {
Some(ItemKind::Footage(f)) if f.has_audio => self.footage.audio(*item, f, from, src_n, rate).unwrap_or_else(|| vec![0.0; src_n * 2]),
_ => return None,
},
LayerSource::Comp { item } => {
if self.project.comp_contains(*item, self.comp_id) {
return None;
}
let mut sub = vec![0.0f32; src_n * 2];
mix_into(&AudioSources { project: self.project, footage: self.footage, expr: self.expr, rate }, *item, from, &mut sub, self.depth + 1);
sub
}
_ => return None,
};
let mut out = vec![0.0f32; m * 2];
for i in 0..m {
// The source sample at fractional index `x`, blended linearly with the next one unless the
// layer plays at its own speed (then the samples are copied as they are).
let x = frac + i as f64 * speed;
let j = if same { i } else { (x as usize).min(src_n - 1) };
if j * 2 + 1 >= buf.len() {
break;
}
let k = if same || (j + 1) * 2 + 1 >= buf.len() { j } else { j + 1 };
let f = if same { 0.0 } else { ((x - j as f64) as f32).clamp(0.0, 1.0) };
out[i * 2] = buf[j * 2] + (buf[k * 2] - buf[j * 2]) * f;
out[i * 2 + 1] = buf[j * 2 + 1] + (buf[k * 2 + 1] - buf[j * 2 + 1]) * f;
}
Some(out)
}
/// Effect parameters at comp time `t`.
fn params(&self, l: &Layer, g: &effectcraft_project::PropGroup, t: Tick) -> Params {
let ctx = self.ctx(t);
let mut p = Params::default();
for c in &g.children {
if let Node::Prop(pr) = c {
p.values.insert(pr.match_id.clone(), ctx.value(l, pr));
}
}
p
}
/// The layer's processed audio (effects applied, levels not) for comp sample indices
/// `i0 .. i0 + n`.
fn layer_audio(&self, l: &Layer, i0: i64, n: usize) -> Vec<f32> {
let t = sample_tick(i0, self.rate);
let fx: Vec<(&str, &effectcraft_project::PropGroup, Params)> = audio_effect_groups(l).map(|(id, g)| (id, g, self.params(l, g, t))).collect();
if fx.is_empty() {
return self.layer_raw(l, i0, n, false);
}
let reversed = fx.iter().any(|(id, _, _)| audio_fx::reverses(id));
let pre_s: f64 = fx.iter().map(|(id, _, p)| audio_fx::preroll(id, p)).sum::<f64>().clamp(0.0, MAX_PREROLL_SECONDS);
let pre = (pre_s * self.rate as f64).ceil() as usize;
let b0 = i0 - pre as i64;
let mut buf = self.layer_raw(l, b0, n + pre, reversed);
let start = fx_clock(l, sample_tick(b0, self.rate));
for (id, g, p) in &fx {
if audio_fx::per_sample(id) && animated(g) {
self.process_stepped(l, id, g, &mut buf, b0);
} else {
audio_fx::process(id, p, &mut buf, self.rate, start);
}
}
buf.drain(..pre * 2);
buf
}
/// Run per-sample effect `id` over `buf` (comp samples from `b0`) in pieces of the parameter
/// grid, each with the parameters at its first sample.
fn process_stepped(&self, l: &Layer, id: &str, g: &effectcraft_project::PropGroup, buf: &mut [f32], b0: i64) {
let step = param_step(self.rate);
let frames = buf.len() / 2;
let mut k = 0usize;
while k < frames {
let i = b0.saturating_add(k as i64);
let next = i.div_euclid(step).saturating_add(1).saturating_mul(step);
let end = usize::try_from(next.saturating_sub(b0)).unwrap_or(frames).clamp(k + 1, frames);
let t = sample_tick(i, self.rate);
if let Some(piece) = buf.get_mut(k * 2..end * 2) {
audio_fx::process(id, &self.params(l, g, t), piece, self.rate, fx_clock(l, t));
}
k = end;
}
}
/// Linear gains (left, right) of the layer's Audio Levels for comp samples `i0 .. i0 + n`.
/// Animated levels are evaluated on the parameter grid and interpolated in dB in between.
fn level_gains(&self, l: &Layer, i0: i64, n: usize) -> Vec<[f32; 2]> {
let Some(pr) = l.props.sub("audio").and_then(|g| g.get("levels")) else { return vec![[1.0, 1.0]; n] };
let db = |i: i64| self.ctx(sample_tick(i, self.rate)).value(l, pr).as_vec2();
let gain = |[a, b]: [f64; 2]| [db_to_gain(a), db_to_gain(b)];
if !pr.is_animated() && !pr.has_expression() {
return vec![gain(db(i0)); n];
}
let step = param_step(self.rate);
let end = i0.saturating_add(n as i64);
let mut out = Vec::with_capacity(n);
let mut k = i0.div_euclid(step).saturating_mul(step);
let mut lo = db(k);
while k < end {
let hi = db(k.saturating_add(step));
for i in k.max(i0)..k.saturating_add(step).min(end) {
let f = (i - k) as f64 / step as f64;
out.push(gain([lo[0] + (hi[0] - lo[0]) * f, lo[1] + (hi[1] - lo[1]) * f]));
}
k = k.saturating_add(step);
lo = hi;
}
out
}
}
/// Shared inputs for composition audio mixing, including nested compositions.
struct AudioSources<'a> {
project: &'a Project,
footage: &'a dyn FootageSource,
expr: Option<&'a dyn ExprHost>,
rate: u32,
}
fn mix_into(sources: &AudioSources<'_>, comp_id: ItemId, start: Tick, out: &mut [f32], depth: usize) {
let AudioSources { project, footage, expr, rate } = *sources;
let Some(comp) = project.comp(comp_id) else { return };
if depth >= MAX_DEPTH || rate == 0 {
return;
}
let env = MixEnv { project, footage, expr, comp_id, comp, rate, depth };
let frames = out.len() / 2;
let any_solo = comp.layers.iter().any(|l| l.switches.solo && audible(project, l));
let s0 = start.to_units_floor(rate as i64);
for l in &comp.layers {
// The stretch the layer's time uses (#3247): 0 and NaN play at 100 %, as `layer_time`;
// a negative one plays backwards (#3239).
if !audible(project, l) || (any_solo && !l.switches.solo) {
continue;
}
// Sample range of the block that falls inside the layer's [in, out).
let a = (units_ceil(l.in_point, rate) - s0).clamp(0, frames as i64) as usize;
let b = (units_ceil(l.out_point, rate) - s0).clamp(0, frames as i64) as usize;
if a >= b {
continue;
}
let gains = env.level_gains(l, s0 + a as i64, b - a);
if gains.iter().all(|g| *g == [0.0, 0.0]) {
continue;
}
let buf = env.layer_audio(l, s0 + a as i64, b - a);
for (i, (f, g)) in buf.as_chunks::<2>().0.iter().zip(&gains).enumerate() {
out[(a + i) * 2] += f[0] * g[0];
out[(a + i) * 2 + 1] += f[1] * g[1];
}
}
}
/// A min/max peak summary of interleaved stereo audio for waveform drawing: one entry per
/// `bin` sample frames, `[min L, max L, min R, max R]`.
pub fn peak_summary(samples: &[f32], bin: usize) -> Vec<[f32; 4]> {
let bin = bin.max(1);
samples
.chunks(bin * 2)
.map(|c| {
let mut s = [f32::MAX, f32::MIN, f32::MAX, f32::MIN];
for f in c.as_chunks::<2>().0 {
s[0] = s[0].min(f[0]);
s[1] = s[1].max(f[0]);
s[2] = s[2].min(f[1]);
s[3] = s[3].max(f[1]);
}
if s[0] > s[1] { [0.0; 4] } else { s }
})
.collect()
}
/// Peak summary of a footage item's audio from source time 0 over `duration`, `bins_per_sec`
/// entries per second (rendered at a low internal rate, fetched in one-second chunks).
pub fn footage_peaks(footage: &dyn FootageSource, item: ItemId, f: &effectcraft_project::Footage, duration: Tick, bins_per_sec: u32) -> Vec<[f32; 4]> {
let bins_per_sec = bins_per_sec.max(1);
// Enough samples per bin to catch peaks of audible content, without decoding at full rate.
let bin = (64usize).max(4000 / bins_per_sec as usize);
let rate = bin as u32 * bins_per_sec;
let chunk = bin * bins_per_sec as usize;
// The stated duration comes from the project: at most a day is summarised, and the summary
// ends where the audio does.
let total = units_ceil(duration, rate).clamp(0, MAX_PEAK_SECONDS.saturating_mul(rate as i64)) as usize;
let mut out = Vec::with_capacity(total / bin + 1);
let mut i = 0usize;
while i < total {
let n = chunk.min(total - i);
let Some(buf) = footage.audio(item, f, sample_tick(i as i64, rate), n, rate) else { break };
out.extend(peak_summary(&buf, bin));
i += n;
}
out
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use effectcraft_project::{Footage, FootageKind, build};
use effectcraft_time::FrameRate;
use super::*;
use crate::Image;
/// A 1 kHz-ish constant "tone": every sample is 0.25 (left) / -0.25 (right).
struct Dc;
impl FootageSource for Dc {
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<Image>> {
None
}
fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, _: u32) -> Option<Vec<f32>> {
Some((0..frames).flat_map(|_| if start >= Tick::ZERO { [0.25, -0.25] } else { [0.0, 0.0] }).collect())
}
}
#[test]
fn audio_detection_answers_each_comp_once() {
// Twenty levels, each comp holding two layers of the one below: 2^20 walks without a
// memo, and all silent. A cycle is answered too.
let mut p = Project::default();
let mk = |p: &mut Project, layers: Vec<Layer>| {
let mut c = Comp::new(8, 8, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0));
c.layers = layers.into();
p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(c.into()))
};
let base = Comp::new(8, 8, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0));
let mut below = mk(&mut p, vec![]);
for _ in 0..20 {
let a = build::layer(&mut p, &base, "a", LayerSource::Comp { item: below }, (0, 0), None);
let b = build::layer(&mut p, &base, "b", LayerSource::Comp { item: below }, (0, 0), None);
below = mk(&mut p, vec![a, b]);
}
let (tx, rx) = std::sync::mpsc::channel();
std::thread::spawn(move || tx.send(comp_has_audio(&p, below)).ok());
assert!(!rx.recv_timeout(std::time::Duration::from_secs(20)).expect("detection finishes"));
}
#[test]
fn audio_found_by_a_short_route_is_not_hidden_by_a_deep_one() {
let mut p = Project::default();
let base = Comp::new(8, 8, FrameRate::new(25, 1), Tick::from_seconds_f64(1.0));
let mk = |p: &mut Project, layers: Vec<Layer>| {
let mut c = base.clone();
c.layers = layers.into();
p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(c.into()))
};
let f = Footage { kind: FootageKind::Audio, has_audio: true, ..Default::default() };
let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f));
let sound = build::layer(&mut p, &base, "s", LayerSource::Footage { item: fid }, (0, 0), None);
let audible = mk(&mut p, vec![sound]);
// A chain of MAX_DEPTH comps ends in `audible` one level past what is searched; the root
// reaches `audible` directly as well, after the chain.
let mut chain = audible;
for _ in 0..MAX_DEPTH {
let l = build::layer(&mut p, &base, "c", LayerSource::Comp { item: chain }, (0, 0), None);
chain = mk(&mut p, vec![l]);
}
assert!(!comp_has_audio(&p, chain), "too deep to be found");
let via_chain = build::layer(&mut p, &base, "chain", LayerSource::Comp { item: chain }, (0, 0), None);
let direct = build::layer(&mut p, &base, "direct", LayerSource::Comp { item: audible }, (0, 0), None);
let root = mk(&mut p, vec![via_chain, direct]);
assert!(comp_has_audio(&p, root));
}
#[test]
fn mixes_layers_with_levels_and_in_out() {
let mut p = Project::default();
let f = Footage {
path: "x.wav".into(),
kind: FootageKind::Audio,
width: 0,
height: 0,
pixel_aspect: 1.0,
frame_rate: FrameRate::new(25, 1),
native_rate: None,
duration: Tick::from_seconds_f64(10.0),
has_video: false,
has_audio: true,
alpha: Default::default(),
premul_color: [0.0; 3],
loop_count: 1,
codec: String::new(),
missing: false,
sequence: vec![],
color_profile: None,
..Default::default()
};
let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f));
let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0));
let mut l1 = build::layer(&mut p, &comp, "a", LayerSource::Footage { item: fid }, (0, 0), None);
l1.in_point = Tick::from_seconds_f64(1.0);
let mut l2 = build::layer(&mut p, &comp, "b", LayerSource::Footage { item: fid }, (0, 0), None);
if let Some(g) = l2.props.sub_mut("audio")
&& let Some(pr) = g.get_mut("levels")
{
pr.value = effectcraft_project::Value::Vec2([-6.0206, -96.0]);
}
let mut comp = comp;
comp.layers = vec![l1, l2].into();
let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into()));
assert!(comp_has_audio(&p, cid));
let rate = 1000;
let m = mix_comp(&p, &Dc, None, cid, Tick::from_seconds_f64(0.5), 1000, rate);
// first half: only layer b (half gain left, muted right)
assert!((m[0] - 0.125).abs() < 1e-3 && m[1].abs() < 1e-6, "{} {}", m[0], m[1]);
// second half (≥ 1 s): a + b
let i = 700 * 2;
assert!((m[i] - 0.375).abs() < 1e-3 && (m[i + 1] + 0.25).abs() < 1e-3, "{} {}", m[i], m[i + 1]);
}
/// A 440 Hz tone, so a change in the mix shows in the samples.
struct Tone;
impl FootageSource for Tone {
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<Image>> {
None
}
fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option<Vec<f32>> {
let t0 = start.seconds();
Some(
(0..frames)
.flat_map(|k| {
let v = (std::f64::consts::TAU * 440.0 * (t0 + k as f64 / rate as f64)).sin() as f32 * 0.5;
[v, v]
})
.collect(),
)
}
}
#[test]
fn a_plain_precomp_mixes_the_same_audio_as_its_layers() {
let mut p = Project::default();
let f = Footage {
path: "tone.wav".into(),
kind: FootageKind::Audio,
has_audio: true,
frame_rate: FrameRate::new(25, 1),
duration: Tick::from_seconds_f64(10.0),
..Default::default()
};
let fid = p.add_item("tone.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f));
let base = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0));
let tone = build::layer(&mut p, &base, "tone", LayerSource::Footage { item: fid }, (0, 0), None);
let mut inner = base.clone();
inner.layers = vec![tone].into();
let inner_id = p.add_item("Inner", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(inner.into()));
let pre = build::layer(&mut p, &base, "pre", LayerSource::Comp { item: inner_id }, (64, 64), None);
let mut outer = base.clone();
outer.layers = vec![pre].into();
let outer_id = p.add_item("Outer", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(outer.into()));
let rate = 8000;
let t = Tick::from_seconds_f64(0.5);
let direct = mix_comp(&p, &Tone, None, inner_id, t, 4000, rate);
let nested = mix_comp(&p, &Tone, None, outer_id, t, 4000, rate);
assert_eq!(direct.len(), nested.len());
let worst = direct.iter().zip(&nested).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max);
assert!(worst < 1e-6, "a plain precomp changes its audio by {worst}");
}
/// The error of a tone mixed at `stretch` percent against the exact tone, at `rate` Hz.
fn stretch_error(stretch: f64, rate: u32) -> (f64, f64) {
let mut p = Project::default();
let f = Footage {
path: "tone.wav".into(),
kind: FootageKind::Audio,
has_audio: true,
frame_rate: FrameRate::new(25, 1),
duration: Tick::from_seconds_f64(10.0),
..Default::default()
};
let fid = p.add_item("tone.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f));
let base = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0));
let mut tone = build::layer(&mut p, &base, "tone", LayerSource::Footage { item: fid }, (0, 0), None);
tone.stretch = stretch;
// A reversed layer as After Effects lays it out: over [0, 2) s, starting at its out point.
let end = if stretch < 0.0 { 2.0 } else { 0.0 };
if stretch < 0.0 {
(tone.in_point, tone.out_point, tone.start_time) = (Tick::ZERO, Tick::from_seconds_f64(2.0), Tick::from_seconds_f64(2.0));
}
let mut comp = base.clone();
comp.layers = vec![tone].into();
let cid = p.add_item("C", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.into()));
let n = 2000usize;
let t = Tick::from_seconds_f64(0.5);
let m = mix_comp(&p, &Tone, None, cid, t, n, rate);
let speed = 100.0 / stretch;
let (mut worst, mut sq) = (0.0f64, 0.0f64);
for i in 0..n {
// The source plays `speed` times as fast as the comp: source time = speed * (comp time −
// start). Reversed, a comp sample plays the source sample just before that time.
let back = if stretch < 0.0 { speed / rate as f64 } else { 0.0 };
let src = speed * (t.seconds() + i as f64 / rate as f64 - end) + back;
let ideal = (std::f64::consts::TAU * 440.0 * src).sin() * 0.5;
let e = (m[i * 2] as f64 - ideal).abs();
worst = worst.max(e);
sq += e * e;
}
(worst, (sq / n as f64).sqrt())
}
#[test]
fn stretched_layers_resample_a_tone_with_small_error() {
// Blending the two samples around each time: about 0.0002 at 48 kHz and 0.007 at 8 kHz. Copying
// the nearest sample gave 0.025 and 0.15.
for stretch in [70.0, 33.0] {
let (worst, _) = stretch_error(stretch, 48_000);
assert!(worst < 1e-3, "stretch {stretch}%: error {worst} at 48 kHz");
let (worst, _) = stretch_error(stretch, 8_000);
assert!(worst < 1e-2, "stretch {stretch}%: error {worst} at 8 kHz");
}
}
/// A reversed layer plays its tone backwards with the same resampling quality as forwards
/// (it was silent, #3239); at -100 % the samples are copied exactly.
#[test]
fn reversed_layers_resample_a_tone_with_small_error() {
for stretch in [-70.0, -33.0] {
let (worst, _) = stretch_error(stretch, 48_000);
assert!(worst < 1e-3, "stretch {stretch}%: error {worst} at 48 kHz");
let (worst, _) = stretch_error(stretch, 8_000);
assert!(worst < 1e-2, "stretch {stretch}%: error {worst} at 8 kHz");
}
let (worst, _) = stretch_error(-100.0, 48_000);
assert!(worst < 1e-5, "stretch -100%: error {worst}");
}
}