mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-17 13:34:45 +00:00
Timeline waveform: draw what the mix plays for reversed layers and Backwards (#3238)
A time-reversed layer (negative stretch) is silent in the audio mix, but the Timeline drew its waveform; with the Backwards audio effect the mix plays the layer's [in, out) span reversed, but the waveform was drawn forwards. The render crate now exposes the mix's rules as stretch_is_mixed and plays_backwards, and the mix uses them. The waveform uses the same rules, decided once per layer: only the baselines for a layer the mix leaves silent, and comp time mirrored over [in, out) under Backwards, as the mix maps it. Columns are limited to [in, out) so an edge column draws no audio past the out point. Fixes #3238
This commit is contained in:
@@ -66,6 +66,17 @@ fn generates(l: &Layer) -> bool {
|
||||
audio_effect_groups(l).any(|(id, _)| audio_fx::generates_audio(id))
|
||||
}
|
||||
|
||||
/// Does the mix play the layer at its time stretch? Reversed layers (negative stretch) are
|
||||
/// silent for now; the Timeline waveform shares this rule so it draws what is heard.
|
||||
pub fn stretch_is_mixed(l: &Layer) -> bool {
|
||||
l.stretch > 0.0
|
||||
}
|
||||
|
||||
/// Does an enabled audio effect (Backwards) play the layer's [in, out) span reversed?
|
||||
pub fn plays_backwards(l: &Layer) -> bool {
|
||||
audio_effect_groups(l).any(|(id, _)| audio_fx::reverses(id))
|
||||
}
|
||||
|
||||
/// Does the layer contribute to the mix (Audio switch on and something to hear)?
|
||||
pub fn audible(project: &Project, l: &Layer) -> bool {
|
||||
l.switches.audio
|
||||
@@ -210,7 +221,7 @@ impl MixEnv<'_> {
|
||||
if fx.is_empty() {
|
||||
return self.layer_raw(l, i0, n, false);
|
||||
}
|
||||
let reversed = fx.iter().any(|(id, _, _)| audio_fx::reverses(id));
|
||||
let reversed = plays_backwards(l);
|
||||
let pre_s: f64 = fx.iter().map(|(id, _, p)| audio_fx::preroll(id, p)).sum();
|
||||
let pre = (pre_s * self.rate as f64).ceil() as usize;
|
||||
let b0 = i0 - pre as i64;
|
||||
@@ -291,7 +302,7 @@ fn mix_into(sources: &AudioSources<'_>, comp_id: ItemId, start: Tick, out: &mut
|
||||
let any_solo = comp.layers.iter().any(|l| l.switches.solo && audible(project, l));
|
||||
let s0 = start.to_units_floor(rate as i64);
|
||||
for l in &comp.layers {
|
||||
if !audible(project, l) || (any_solo && !l.switches.solo) || l.stretch <= 0.0 {
|
||||
if !audible(project, l) || (any_solo && !l.switches.solo) || !stretch_is_mixed(l) {
|
||||
continue;
|
||||
}
|
||||
// Sample range of the block that falls inside the layer's [in, out).
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use effectcraft_engine::project::{ItemId, ItemKind, Layer, LayerSource, Project, Property};
|
||||
use effectcraft_engine::render::audio::db_to_gain;
|
||||
use effectcraft_engine::render::audio::{self, db_to_gain};
|
||||
use effectcraft_engine::time::Tick;
|
||||
use egui::{Color32, Painter, Rect, Stroke, pos2};
|
||||
|
||||
@@ -39,6 +39,39 @@ pub fn audio_item(project: &Project, l: &Layer) -> Option<ItemId> {
|
||||
}
|
||||
}
|
||||
|
||||
/// How the layer plays its audio, as far as its waveform goes: `None` when it is silent
|
||||
/// (reversed, see [`audio::stretch_is_mixed`]), else whether the Backwards effect plays its
|
||||
/// [in, out) span mirrored ([`audio::plays_backwards`]). The Audio switch and solo are not
|
||||
/// followed: a muted layer still shows its waveform.
|
||||
fn played(layer: &Layer) -> Option<bool> {
|
||||
audio::stretch_is_mixed(layer).then(|| audio::plays_backwards(layer))
|
||||
}
|
||||
|
||||
/// The summary bins under the Timeline column for comp times `ta .. tb` (seconds, limited to
|
||||
/// the layer's [in, out) span), as the layer plays them: mirrored over that span when `mirror`
|
||||
/// (see [`played`]), so the first column shows the last samples.
|
||||
fn column_bins<'a>(layer: &Layer, s: &'a Summary, ta: f64, tb: f64, mirror: bool) -> Option<&'a [[f32; 4]]> {
|
||||
let (i, o) = (layer.in_point.seconds(), layer.out_point.seconds());
|
||||
let (ta, tb) = (ta.max(i).min(o), tb.max(i).min(o));
|
||||
if tb <= ta {
|
||||
return None;
|
||||
}
|
||||
let span = i + o;
|
||||
let src = |t: f64| {
|
||||
let t = if mirror { span - t } else { t };
|
||||
layer.layer_time(Tick::from_seconds_f64(t)).seconds() * s.bins_per_sec as f64
|
||||
};
|
||||
let (u, v) = (src(ta), src(tb));
|
||||
let n = s.peaks.len() as i64;
|
||||
let ba = u.min(v).floor() as i64;
|
||||
let bb = (u.max(v).ceil() as i64).max(ba.saturating_add(1));
|
||||
let (ba, bb) = (ba.clamp(0, n), bb.clamp(0, n));
|
||||
if ba >= bb {
|
||||
return None;
|
||||
}
|
||||
s.peaks.get(ba as usize..bb as usize)
|
||||
}
|
||||
|
||||
/// The summary for `item`, starting its computation on first request.
|
||||
pub fn summary(app: &EffectcraftApp, ctx: &egui::Context, item: ItemId) -> Option<Arc<Summary>> {
|
||||
let mut m = app.waveforms.map.lock().ok()?;
|
||||
@@ -88,27 +121,23 @@ fn level_gains(levels: Option<&Property>, t: Tick) -> [f32; 2] {
|
||||
}
|
||||
|
||||
/// Draw `layer`'s waveform in `rect` (left channel above right), columns mapped through the
|
||||
/// timeline's time map and the layer's time mapping, limited to its [in, out) span.
|
||||
/// timeline's time map and the layer's time mapping, limited to its [in, out) span and drawn as
|
||||
/// the layer plays (see [`played`]): only the baselines for a silent layer.
|
||||
pub(crate) fn draw(p: &Painter, rect: Rect, tm: &TMap, layer: &Layer, s: &Summary, color: Color32) {
|
||||
let mid = [rect.min.y + rect.height() * 0.25, rect.min.y + rect.height() * 0.75];
|
||||
let half = rect.height() * 0.25 - 1.0;
|
||||
let x0 = tm.x(layer.in_point.seconds()).max(rect.min.x).floor() as i32;
|
||||
let x1 = tm.x(layer.out_point.seconds()).min(rect.max.x).ceil() as i32;
|
||||
let n = s.peaks.len() as i64;
|
||||
let levels = layer.props.sub("audio").and_then(|g| g.get("levels"));
|
||||
for c in 0..2 {
|
||||
p.line_segment([pos2(x0 as f32, mid[c]), pos2(x1 as f32, mid[c])], Stroke::new(1.0, color.gamma_multiply(0.5)));
|
||||
}
|
||||
let Some(mirror) = played(layer) else { return };
|
||||
for x in x0..x1 {
|
||||
let (ta, tb) = (tm.t(x as f32), tm.t(x as f32 + 1.0));
|
||||
let src = |t: f64| layer.layer_time(Tick::from_seconds_f64(t)).seconds() * s.bins_per_sec as f64;
|
||||
let (ba, bb) = (src(ta).floor() as i64, src(tb).ceil() as i64);
|
||||
let (ba, bb) = (ba.min(bb).clamp(0, n), ba.max(bb).max(ba + 1).clamp(0, n));
|
||||
if ba >= bb {
|
||||
continue;
|
||||
}
|
||||
let Some(bins) = column_bins(layer, s, ta, tb, mirror) else { continue };
|
||||
let mut agg = [0.0f32; 4];
|
||||
for b in &s.peaks[ba as usize..bb as usize] {
|
||||
for b in bins {
|
||||
agg[0] = agg[0].min(b[0]);
|
||||
agg[1] = agg[1].max(b[1]);
|
||||
agg[2] = agg[2].min(b[2]);
|
||||
@@ -137,3 +166,131 @@ mod tests {
|
||||
assert_eq!(level_gains(None, Tick::ZERO), [1.0, 1.0]);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod played_tests {
|
||||
use super::*;
|
||||
use effectcraft_engine::color::Label;
|
||||
use effectcraft_engine::project::{Comp, Footage, FootageKind, GroupKind, build};
|
||||
use effectcraft_engine::render::audio::{footage_peaks, mix_comp};
|
||||
use effectcraft_engine::render::{FootageSource, Image};
|
||||
use effectcraft_engine::time::FrameRate;
|
||||
|
||||
/// Footage audio equal to its source time (seconds) on both channels over [0, 2) s.
|
||||
struct Ramp;
|
||||
impl FootageSource for Ramp {
|
||||
fn frame(&self, _: ItemId, _: &Footage, _: Tick) -> Option<Arc<Image>> {
|
||||
None
|
||||
}
|
||||
fn audio(&self, _: ItemId, _: &Footage, start: Tick, frames: usize, rate: u32) -> Option<Vec<f32>> {
|
||||
let t0 = start.seconds();
|
||||
Some(
|
||||
(0..frames)
|
||||
.flat_map(|k| {
|
||||
let t = t0 + k as f64 / rate as f64;
|
||||
let v = if (0.0..2.0).contains(&t) { t as f32 } else { 0.0 };
|
||||
[v, v]
|
||||
})
|
||||
.collect(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A project with a 2 s ramp clip and a comp holding one layer of it (start and in 0 s,
|
||||
/// out 1 s) changed by `edit`, and the clip's peak summary.
|
||||
fn setup(edit: &dyn Fn(&mut Layer)) -> (Project, ItemId, Layer, Summary) {
|
||||
let mut p = Project::default();
|
||||
let f = Footage {
|
||||
path: "ramp.wav".into(),
|
||||
kind: FootageKind::Audio,
|
||||
has_audio: true,
|
||||
frame_rate: FrameRate::new(25, 1),
|
||||
duration: Tick::from_seconds_f64(2.0),
|
||||
..Default::default()
|
||||
};
|
||||
let fid = p.add_item("ramp.wav", Label::SeaFoam, None, ItemKind::Footage(f.clone()));
|
||||
let mut comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0));
|
||||
let mut l = build::layer(&mut p, &comp, "a", LayerSource::Footage { item: fid }, (0, 0), None);
|
||||
(l.start_time, l.in_point, l.out_point) = (Tick::ZERO, Tick::ZERO, Tick::from_seconds_f64(1.0));
|
||||
edit(&mut l);
|
||||
comp.layers = vec![l.clone()].into();
|
||||
let cid = p.add_item("C", Label::Sandstone, None, ItemKind::Comp(comp.into()));
|
||||
let peaks = footage_peaks(&Ramp, fid, &f, f.duration, BINS_PER_SEC);
|
||||
(p, cid, l, Summary { bins_per_sec: BINS_PER_SEC, peaks })
|
||||
}
|
||||
|
||||
fn add_backwards(l: &mut Layer, enabled: bool) {
|
||||
let mut n = 1_000_000;
|
||||
let mut g = build::Ids(&mut n).group("backwards", "Backwards");
|
||||
g.kind = GroupKind::Effect { effect: "ec.audio.backwards".into() };
|
||||
g.enabled = enabled;
|
||||
l.props.group_mut("effects").expect("effects group").children.push(g.into());
|
||||
}
|
||||
|
||||
/// The loudest left-channel peak drawn in the column for comp times `ta .. tb`.
|
||||
fn drawn(l: &Layer, s: &Summary, ta: f64, tb: f64) -> Option<f32> {
|
||||
played(l).and_then(|mirror| column_bins(l, s, ta, tb, mirror)).map(|bins| bins.iter().fold(f32::MIN, |m, b| m.max(b[1])))
|
||||
}
|
||||
|
||||
/// The left-channel sample the mix plays at comp time `t`.
|
||||
fn heard(p: &Project, cid: ItemId, t: f64) -> f32 {
|
||||
mix_comp(p, &Ramp, None, cid, Tick::from_seconds_f64(t), 1, 8000).first().copied().unwrap_or(f32::NAN)
|
||||
}
|
||||
|
||||
/// #3238: a time-reversed layer (Time-Reverse Layer: stretch -100 %, start time at its out
|
||||
/// point) is silent in the mix, so the Timeline draws no waveform columns for it.
|
||||
#[test]
|
||||
fn a_reversed_layer_draws_no_waveform_as_the_mix_is_silent() {
|
||||
let (p, cid, l, s) = setup(&|l| {
|
||||
l.stretch = -100.0;
|
||||
l.start_time = Tick::from_seconds_f64(1.0);
|
||||
});
|
||||
for k in 0..200 {
|
||||
let (ta, tb) = (k as f64 / 200.0, (k + 1) as f64 / 200.0);
|
||||
assert_eq!(drawn(&l, &s, ta, tb), None, "column at {ta} s");
|
||||
}
|
||||
for t in [0.1, 0.5, 0.9] {
|
||||
assert_eq!(heard(&p, cid, t), 0.0, "the mix plays the reversed layer at {t} s");
|
||||
}
|
||||
}
|
||||
|
||||
/// #3238: with the Backwards audio effect the first column shows the clip's last samples
|
||||
/// and the last column its first, matching what the mix plays there.
|
||||
#[test]
|
||||
fn backwards_draws_the_waveform_mirrored_as_the_mix_plays_it() {
|
||||
let (p, cid, l, s) = setup(&|l| add_backwards(l, true));
|
||||
let first = drawn(&l, &s, 0.0, 0.005).expect("first column");
|
||||
let h = heard(&p, cid, 0.0);
|
||||
assert!(first > 0.98, "first column draws {first}, not the clip's last samples");
|
||||
assert!((first - h).abs() < 0.01, "first column {first}, heard {h}");
|
||||
let last = drawn(&l, &s, 0.995, 1.0).expect("last column");
|
||||
let h = heard(&p, cid, 0.995);
|
||||
assert!(last < 0.02, "last column draws {last}, not the clip's first samples");
|
||||
assert!((last - h).abs() < 0.01, "last column {last}, heard {h}");
|
||||
}
|
||||
|
||||
/// No Backwards effect, a disabled one, or the layer's Effects switch off: the mix plays
|
||||
/// forwards, and so the waveform is drawn.
|
||||
#[test]
|
||||
fn without_an_active_backwards_effect_the_waveform_runs_forwards() {
|
||||
let edits: [(&str, &dyn Fn(&mut Layer)); 3] = [
|
||||
("no effect", &|_| {}),
|
||||
("effect disabled", &|l| add_backwards(l, false)),
|
||||
("effects switch off", &|l| {
|
||||
add_backwards(l, true);
|
||||
l.switches.effects = false;
|
||||
}),
|
||||
];
|
||||
for (name, edit) in edits {
|
||||
let (p, cid, l, s) = setup(edit);
|
||||
let first = drawn(&l, &s, 0.0, 0.005).expect("first column");
|
||||
let h = heard(&p, cid, 0.004);
|
||||
assert!(first < 0.02, "{name}: first column draws {first}");
|
||||
assert!((first - h).abs() < 0.01, "{name}: first column {first}, heard {h}");
|
||||
// A column reaching past the out point draws only up to it.
|
||||
let edge = drawn(&l, &s, 0.998, 1.003).expect("edge column");
|
||||
assert!(edge < 1.001, "{name}: the edge column draws {edge}, past the out point");
|
||||
assert_eq!(drawn(&l, &s, 1.0, 1.005), None, "{name}: a column after the out point");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user