diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs index 189e0a27c..21db30890 100644 --- a/apps/filmcraft-cli/src/main.rs +++ b/apps/filmcraft-cli/src/main.rs @@ -272,6 +272,10 @@ impl filmcraft_export::FrameRenderer for GpuFrameRenderer { ) -> Option { self.0.render(project, seq, t, opts, sources) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + self.0.caps() + } } fn register_gpu_frame_renderer() { diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index 34eb7e249..ba1989568 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -439,6 +439,10 @@ impl filmcraft_export::FrameRenderer for GpuFrameRenderer { ) -> Option { self.0.render(project, seq, t, opts, sources) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + self.0.caps() + } } fn register_gpu_frame_renderer() { diff --git a/crates/engine/src/captions/tests.rs b/crates/engine/src/captions/tests.rs index fb2878b7f..747bd80a3 100644 --- a/crates/engine/src/captions/tests.rs +++ b/crates/engine/src/captions/tests.rs @@ -81,6 +81,7 @@ fn captions_import_edit_export_and_burn_in() { s.state.active_sequence.unwrap(), s.playhead(), filmcraft_render::RenderOptions { scale: 0.25, captions: true, ..Default::default() }, + filmcraft_render::plan::DeviceCaps::ALL, &s.media.provider(s.project.clone(), s.services.clone()), ); let cpu = filmcraft_render::plan::execute_cpu(&plan); diff --git a/crates/engine/src/multicam_tests.rs b/crates/engine/src/multicam_tests.rs index ffdf3412d..75fbbcf78 100644 --- a/crates/engine/src/multicam_tests.rs +++ b/crates/engine/src/multicam_tests.rs @@ -461,7 +461,14 @@ fn multicam_clip_structure_and_render() { let provider = s.media.provider(s.project.clone(), s.services.clone()); let edit_id = s.state.active_sequence.unwrap(); let t10 = FrameRate::FPS_24.tick_of(10); - let plan = filmcraft_render::plan::plan_frame(&s.project, edit_id, t10, filmcraft_render::RenderOptions::default(), &provider); + let plan = filmcraft_render::plan::plan_frame( + &s.project, + edit_id, + t10, + filmcraft_render::RenderOptions::default(), + filmcraft_render::plan::DeviceCaps::ALL, + &provider, + ); match &plan { filmcraft_render::plan::FramePlan::Layers { layers, .. } => { assert_eq!(layers.len(), 1); diff --git a/crates/export/src/pipeline.rs b/crates/export/src/pipeline.rs index b79a60eb1..8508b35cd 100644 --- a/crates/export/src/pipeline.rs +++ b/crates/export/src/pipeline.rs @@ -66,6 +66,8 @@ impl Geometry { struct RendererPool { /// Initial capacity, also the maximum number of frame tasks admitted at once. capacity: usize, + /// What every renderer of the pool runs: a frame is planned for whichever one it gets. + caps: filmcraft_render::plan::DeviceCaps, renderers: std::sync::Mutex>>, available: std::sync::Condvar, /// A renderer panicked: the pool is retired and the rest of the export renders on the CPU. @@ -75,8 +77,10 @@ struct RendererPool { impl RendererPool { /// `None` for an empty list (a pool nobody could take from). fn new(renderers: Vec>) -> Option { - (!renderers.is_empty()).then(|| Self { + let adjust_side = renderers.iter().map(|r| r.caps().adjust_side).min()?; + Some(Self { capacity: renderers.len(), + caps: filmcraft_render::plan::DeviceCaps { adjust_side }, renderers: std::sync::Mutex::new(renderers), available: std::sync::Condvar::new(), failed: std::sync::atomic::AtomicBool::new(false), @@ -248,10 +252,11 @@ impl Pipeline { let t = self.rate.tick_of(f); // The registered GPU renderer goes first; `None` (no adapter, a plan the GPU cannot // draw, any internal error) falls back to the CPU reference renderer below. A frame the - // planner hands back as one CPU image anyway (an adjustment layer, a complex transition, - // HDR) does not take a pooled renderer: it would render on the CPU inside the pool, as - // many frames at a time as there are renderers instead of one per export thread. - let pool = self.renderer.as_ref().filter(|_| !filmcraft_render::plan::is_cpu_frame(&self.project, self.seq, t)); + // planner hands back as one CPU image anyway (an adjustment layer the GPU can't run, a + // complex transition, HDR) does not take a pooled renderer: it would render on the CPU + // inside the pool, as many frames at a time as there are renderers instead of one per + // export thread. + let pool = self.renderer.as_ref().filter(|pool| !filmcraft_render::plan::is_cpu_frame(&self.project, self.seq, t, self.opts, pool.caps)); let gpu = match pool { Some(pool) => { let asked = web_time::Instant::now(); diff --git a/crates/export/src/registry.rs b/crates/export/src/registry.rs index 315078645..4a1198593 100644 --- a/crates/export/src/registry.rs +++ b/crates/export/src/registry.rs @@ -117,6 +117,11 @@ pub trait FrameRenderer: Send { opts: filmcraft_render::RenderOptions, sources: &dyn SourceProvider, ) -> Option; + + /// What the renderer's device runs. The export plans with it before taking a renderer + /// ([`filmcraft_render::plan::is_cpu_frame`]), so a frame the device can't composite renders + /// on the export's own thread instead of on the CPU inside the renderer. + fn caps(&self) -> filmcraft_render::plan::DeviceCaps; } /// Builds the frame renderer for one export, or `None` when this host cannot (no adapter). diff --git a/crates/export/src/tests.rs b/crates/export/src/tests.rs index 305e931d2..9ab3cd9b6 100644 --- a/crates/export/src/tests.rs +++ b/crates/export/src/tests.rs @@ -382,7 +382,7 @@ fn gpu_hook_scenarios() { // 2. Producing renderer: counters up and its picture reaches the export. crate::reset_frame_renderers_for_tests(); - crate::register_frame_renderer(|| Some(Box::new(FakeRenderer) as Box)); + crate::register_frame_renderer(|| Some(Box::new(FakeRenderer(filmcraft_render::plan::DeviceCaps::ALL)) as Box)); mk(&tmp("fake-gpu.png"), Some(true)); assert!(gpu_render_stats().frames >= 2, "gpu frames counted: {:?}", gpu_render_stats()); assert_eq!(gpu_render_stats().fallbacks, 0); @@ -453,16 +453,18 @@ fn gpu_hook_scenarios() { // at most one panic per pooled renderer (4): none is used again after it panicked assert!(PANICS.load(std::sync::atomic::Ordering::SeqCst) <= 4, "renderer reused after a panic: {}", PANICS.load(std::sync::atomic::Ordering::SeqCst)); - // 5. A frame the planner hands back as one CPU image (an adjustment layer over the clip from - // frame 12) is rendered by the CPU without taking a pooled renderer: it counts as a - // fallback, and the renderer's picture reaches only the frames before it. + // 5. A frame the planner hands back as one CPU image (an adjustment layer with an effect the + // GPU doesn't run, over the clip from frame 12) is rendered by the CPU without taking a + // pooled renderer: it counts as a fallback, and the renderer's picture reaches only the + // frames before it. crate::reset_frame_renderers_for_tests(); - crate::register_frame_renderer(|| Some(Box::new(FakeRenderer) as Box)); + crate::register_frame_renderer(|| Some(Box::new(FakeRenderer(filmcraft_render::plan::DeviceCaps::ALL)) as Box)); let (p, seq, m) = project(); let mut p = Arc::unwrap_or_clone(p); let r = FrameRate::FPS_24; let adj = p.add_item("adj", Label::Iris, ItemKind::AdjustmentLayer { width: 320, height: 180, rate: r, duration: Tick(TICKS_PER_SECOND) }, None); - let over = p.make_track_item(adj, TrackKind::Video, r.tick_of(12), TimeRange::new(Tick::ZERO, r.tick_of(12)), r).unwrap(); + let mut over = p.make_track_item(adj, TrackKind::Video, r.tick_of(12), TimeRange::new(Tick::ZERO, r.tick_of(12)), r).unwrap(); + over.effects.insert(0, filmcraft_project::find_effect("emboss").unwrap().instance()); let v2 = filmcraft_project::TrackId(p.alloc_id()); let q = p.sequence_mut(seq).unwrap(); q.video_tracks.push(filmcraft_project::Track::new(v2, TrackKind::Video, "Video 2".into())); @@ -482,6 +484,43 @@ fn gpu_hook_scenarios() { assert_eq!(frame("adj-off", f), frame("adj-auto", f), "frame {f}: an adjustment layer frame is the CPU's"); } assert_eq!((gpu_render_stats().frames, gpu_render_stats().fallbacks), (12, 12), "{:?}", gpu_render_stats()); + + // 6. An adjustment layer whose effect the GPU runs (Invert, over the clip from frame 12) is + // handed to a renderer whose device runs adjustment layers, and never to one without an + // effect stage: there those frames render on the export's thread, byte-equal to Off, and + // count as fallbacks. + let (p, seq, m) = project(); + let mut p = Arc::unwrap_or_clone(p); + let adj = p.add_item("adj", Label::Iris, ItemKind::AdjustmentLayer { width: 320, height: 180, rate: r, duration: Tick(TICKS_PER_SECOND) }, None); + let mut over = p.make_track_item(adj, TrackKind::Video, r.tick_of(12), TimeRange::new(Tick::ZERO, r.tick_of(12)), r).unwrap(); + over.effects.insert(0, filmcraft_project::find_effect("invert").unwrap().instance()); + let v2 = filmcraft_project::TrackId(p.alloc_id()); + let q = p.sequence_mut(seq).unwrap(); + q.video_tracks.push(filmcraft_project::Track::new(v2, TrackKind::Video, "Video 2".into())); + q.video_tracks[1].items.push(over); + let p = Arc::new(p); + let mk24 = |path: &String, gpu: crate::GpuRendering| { + let mut s = ExportSettings { format: Format::PngSequence, path: path.clone(), ..Default::default() }; + s.range = Some(TimeRange::new(Tick::ZERO, r.tick_of(24))); + s.gpu_rendering = gpu; + export(&p, seq, &s, &m, &Progress::default()).unwrap() + }; + mk24(&tmp("gpu-adj-off.png"), crate::GpuRendering::Off); + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| Some(Box::new(FakeRenderer(filmcraft_render::plan::DeviceCaps::NONE)) as Box)); + mk24(&tmp("gpu-adj-nofx.png"), crate::GpuRendering::Auto); + assert_ne!(frame("gpu-adj-off", "011"), frame("gpu-adj-nofx", "011"), "a frame without the adjustment layer goes to the renderer"); + for f in ["012", "023"] { + assert_eq!(frame("gpu-adj-off", f), frame("gpu-adj-nofx", f), "frame {f}: no effect stage, the CPU renders it"); + } + assert_eq!((gpu_render_stats().frames, gpu_render_stats().fallbacks), (12, 12), "{:?}", gpu_render_stats()); + crate::reset_frame_renderers_for_tests(); + crate::register_frame_renderer(|| Some(Box::new(FakeRenderer(filmcraft_render::plan::DeviceCaps::ALL)) as Box)); + mk24(&tmp("gpu-adj-fx.png"), crate::GpuRendering::Auto); + for f in ["012", "023"] { + assert_ne!(frame("gpu-adj-off", f), frame("gpu-adj-fx", f), "frame {f}: an effect stage runs the adjustment layer"); + } + assert_eq!((gpu_render_stats().frames, gpu_render_stats().fallbacks), (24, 0), "{:?}", gpu_render_stats()); crate::reset_frame_renderers_for_tests(); } @@ -501,9 +540,14 @@ impl FrameRenderer for PanickingRenderer { PANICS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); panic!("GPU renderer failure (test)"); } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + filmcraft_render::plan::DeviceCaps::ALL + } } -struct FakeRenderer; +/// Returns a blank picture for every frame it is given, on a device that runs these caps. +struct FakeRenderer(filmcraft_render::plan::DeviceCaps); impl FrameRenderer for FakeRenderer { fn render( @@ -516,6 +560,10 @@ impl FrameRenderer for FakeRenderer { ) -> Option { Some(filmcraft_render::Image::new(320, 180)) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + self.0 + } } /// The render threads may convert H.264 input ahead ([`VideoEncoder::prepare_ahead`]): the packets diff --git a/crates/export/tests/it/renderer_reentrancy.rs b/crates/export/tests/it/renderer_reentrancy.rs index 1b42606a0..915f27e9a 100644 --- a/crates/export/tests/it/renderer_reentrancy.rs +++ b/crates/export/tests/it/renderer_reentrancy.rs @@ -32,6 +32,10 @@ impl FrameRenderer for YieldingRenderer { RENDERED.fetch_add(1, Ordering::Relaxed); Some(filmcraft_render::render_sequence(project, seq, t, opts, sources)) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + filmcraft_render::plan::DeviceCaps::ALL + } } fn factory() -> Option> { diff --git a/crates/golden/tests/golden.rs b/crates/golden/tests/golden.rs index 68c20d468..7d4154a8f 100644 --- a/crates/golden/tests/golden.rs +++ b/crates/golden/tests/golden.rs @@ -773,7 +773,7 @@ fn gpu_matches_cpu_on_golden_scenes() { let s = make(); let t = RATE.tick_of(s.frame); let cpu = render_cpu(&s); - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, t, RenderOptions::default(), &s.sources); + let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, t, RenderOptions::default(), c.caps(), &s.sources); let kind = if matches!(plan, filmcraft_render::plan::FramePlan::Layers { .. }) { "layers" } else { "cpu image" }; c.composite(&plan); let (gw, gh, mut px) = c.read_output().expect("GPU readback"); @@ -847,7 +847,7 @@ fn gpu_blend_modes_match_cpu_render() { let s = b.at(12); let t = RATE.tick_of(s.frame); let cpu = render_cpu(&s); - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, t, RenderOptions::default(), &s.sources); + let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, t, RenderOptions::default(), filmcraft_render::plan::DeviceCaps::ALL, &s.sources); let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { failures.push(format!("{mode}: planned as a CPU image")); continue; @@ -884,7 +884,14 @@ fn gpu_transition_of_blended_clips_matches_cpu_render() { } b.transition(1, "cross_dissolve", ca, cb, 24, 12); let s = b.at(22); - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), RenderOptions::default(), &s.sources); + let plan = filmcraft_render::plan::plan_frame( + &s.project, + s.seq, + RATE.tick_of(s.frame), + RenderOptions::default(), + filmcraft_render::plan::DeviceCaps::ALL, + &s.sources, + ); let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("planned as a CPU image") }; assert!(layers.iter().all(|l| l.blend == filmcraft_render::Blend::Normal)); let mut c = filmcraft_gpu::GpuCompositor::new(&dev, &q); @@ -938,7 +945,14 @@ fn gpu_effect_chains_match_cpu_render() { b.fixed(v5, "motion", &[("scale", fl(25.0)), ("position", pt(60.0, 140.0))]); b.effect(v5, "emboss", &[]); let mut s = b.at(12); - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), RenderOptions::default(), &s.sources); + let plan = filmcraft_render::plan::plan_frame( + &s.project, + s.seq, + RATE.tick_of(s.frame), + RenderOptions::default(), + filmcraft_render::plan::DeviceCaps::ALL, + &s.sources, + ); let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("planned as a CPU image") }; assert_eq!(layers.len(), 5); for (i, l) in layers.iter().take(4).enumerate() { @@ -953,7 +967,7 @@ fn gpu_effect_chains_match_cpu_render() { let mut failures = Vec::new(); for frame in [0, 12, 30] { s.frame = frame; - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(frame), RenderOptions::default(), &s.sources); + let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(frame), RenderOptions::default(), c.caps(), &s.sources); c.composite(&plan); let (gw, gh, mut px) = c.read_output().expect("GPU readback"); px.as_chunks_mut::<4>().0.iter_mut().for_each(|p| p[3] = 255); @@ -971,7 +985,7 @@ fn gpu_effect_chains_match_cpu_render() { } // ½ playback resolution: the working images (and the effects' pixel radii) shrink with it let opts = RenderOptions { scale: 0.5, ..RenderOptions::default() }; - let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), opts, &s.sources); + let plan = filmcraft_render::plan::plan_frame(&s.project, s.seq, RATE.tick_of(s.frame), opts, c.caps(), &s.sources); let filmcraft_render::plan::FramePlan::Layers { layers, .. } = &plan else { panic!("½: planned as a CPU image") }; assert!(layers.iter().take(4).all(|l| l.fx.is_some())); c.composite(&plan); diff --git a/crates/gpu/src/export_renderer.rs b/crates/gpu/src/export_renderer.rs index cefed04b8..460803801 100644 --- a/crates/gpu/src/export_renderer.rs +++ b/crates/gpu/src/export_renderer.rs @@ -13,7 +13,7 @@ use crate::{GpuCompositor, prepare}; use filmcraft_project::{ItemId, Project}; -use filmcraft_render::plan::{FramePlan, plan_frame}; +use filmcraft_render::plan::{DeviceCaps, FramePlan, plan_frame}; use filmcraft_render::{Image, RenderOptions, SourceProvider}; use filmcraft_time::Tick; @@ -143,10 +143,16 @@ impl ExportRenderer { Some(Self { _instance: instance, device, queue, compositor }) } + /// What this renderer's device runs: frames are planned with it, here and by the export + /// before it takes this renderer. + pub fn caps(&self) -> DeviceCaps { + self.compositor.caps() + } + /// One frame as `render_sequence` would return it (premultiplied linear-light RGBA at the /// render size), or `None` when this frame must go to the CPU renderer. pub fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option { - let plan = plan_frame(project, seq, t, opts, sources); + let plan = plan_frame(project, seq, t, opts, self.compositor.caps(), sources); match &plan { // Already the CPU image: return it unchanged instead of rendering it twice. FramePlan::Image(img) => Some(img.clone()), @@ -181,8 +187,8 @@ mod tests { const EPS: f32 = 1e-4; - /// A solid colour generator at 320×180 plus a tone generator (mirrors the export tests' - /// fixture-free project helper). + /// A solid colour generator at 320×180 on V1 (V2 empty) plus a tone generator (mirrors the + /// export tests' fixture-free project helper). fn project(transform: bool) -> (Arc, ItemId, SourceMap) { let mut p = Project::new("x"); let g = GeneratorSource::new( @@ -215,7 +221,7 @@ mod tests { let red = add(&mut p, &g); let t = add(&mut p, &tone); let seq = - p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 2, 1, None); let r = filmcraft_time::FrameRate::FPS_24; let v = p.make_track_item(red, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap(); let a = p.make_track_item(t, TrackKind::Audio, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap(); @@ -286,6 +292,256 @@ mod tests { assert_parity(&cpu, &gpu, "solid"); } + /// Without an effect stage the renderer says so (`DeviceCaps::NONE`): the export renders a + /// frame with an adjustment layer on its own thread (`is_cpu_frame`) instead of taking this + /// renderer, and a frame planned for it holds no adjustment step to composite on the CPU here. + #[test] + fn adjustment_layers_without_an_effect_stage_go_back_to_the_cpu() { + let Some(mut r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu export adjustment): no adapter"); + return; + }; + let (p, seq, m) = project(false); + let mut p = Arc::unwrap_or_clone(p); + let rate = filmcraft_time::FrameRate::FPS_24; + let a = p.add_item("Adjustment Layer", Label::Lavender, ItemKind::AdjustmentLayer { width: 320, height: 180, rate, duration: rate.tick_of(24) }, None); + let mut ti = p.make_track_item(a, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, rate.tick_of(24)), rate).unwrap(); + let mut e = filmcraft_project::find_effect("proc_amp").unwrap().instance(); + e.param_mut("brightness").unwrap().value = filmcraft_project::ParamValue::Float(25.0); + ti.effects.insert(0, e); + p.sequence_mut(seq).unwrap().video_tracks[1].items.push(ti); + let (t, opts) = (Tick::ZERO, RenderOptions::default()); + let cpu = filmcraft_render::render_sequence(&p, seq, t, opts, &m); + if r.compositor.fx.is_some() { + assert!(!filmcraft_render::plan::is_cpu_frame(&p, seq, t, opts, r.caps()), "the effect stage runs the adjustment layer"); + let gpu = r.render(&p, seq, t, opts, &m).expect("GPU rendered the frame"); + assert_parity(&cpu, &gpu, "adjustment layer"); + assert_eq!((r.compositor.gpu_adjustments, r.compositor.cpu_adjustments), (1, 0)); + } + r.compositor.fx = None; + assert_eq!(r.caps(), DeviceCaps::NONE); + assert!(filmcraft_render::plan::is_cpu_frame(&p, seq, t, opts, r.caps()), "no effect stage: the export renders the frame without this renderer"); + let counts = (r.compositor.gpu_adjustments, r.compositor.cpu_adjustments); + // asked anyway, it hands back the CPU renderer's picture, planned as one image + let img = r.render(&p, seq, t, opts, &m).expect("the CPU image the frame was planned as"); + assert_eq!(img.px, cpu.px); + assert_eq!((r.compositor.gpu_adjustments, r.compositor.cpu_adjustments), counts, "no adjustment step reached the compositor"); + } + + /// `g` as a media item of `p`, its source in `m`. + fn media_item(p: &mut Project, m: &mut SourceMap, g: GeneratorSource) -> ItemId { + let info = g.info().clone(); + let media = MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info: info.clone(), + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + }; + let item = p.add_item(&info.name, Label::Iris, ItemKind::Media(Box::new(media)), None); + m.0.insert(item, Arc::new(g)); + item + } + + /// `item` on video track `track` of `seq` from frame `start` for `frames` frames, scaled to + /// the frame; returns the clip. + fn put(p: &mut Project, seq: ItemId, track: usize, item: ItemId, start: i64, frames: i64) -> filmcraft_project::ClipId { + let rate = filmcraft_time::FrameRate::FPS_24; + let mut ti = p.make_track_item(item, TrackKind::Video, rate.tick_of(start), TimeRange::new(Tick::ZERO, rate.tick_of(frames)), rate).unwrap(); + ti.scale_to_frame = true; + let id = ti.id; + let q = p.sequence_mut(seq).unwrap(); + q.video_tracks[track].items.push(ti); + q.video_tracks[track].sort(); + id + } + + fn set_param(p: &mut Project, seq: ItemId, clip: filmcraft_project::ClipId, effect: &str, param: &str, v: f64) { + let it = p.sequence_mut(seq).unwrap().find_item_mut(clip).unwrap().1; + it.effect_mut(effect).unwrap().params.get_mut(param).unwrap().value = filmcraft_project::ParamValue::Float(v); + } + + /// A full-frame adjustment layer on track `track` of `seq` for frames 0..48 with Proc Amp + /// (brightness 25) and Gaussian Blur (6): the kind a device with an effect stage runs as a + /// `PlanStep::Adjust`, the blur spreading the picture into any transparent part of the canvas. + fn adjustment_layer(p: &mut Project, seq: ItemId, track: usize) { + let rate = filmcraft_time::FrameRate::FPS_24; + let a = p.add_item("Adjustment Layer", Label::Lavender, ItemKind::AdjustmentLayer { width: 320, height: 180, rate, duration: rate.tick_of(48) }, None); + let clip = put(p, seq, track, a, 0, 48); + let it = p.sequence_mut(seq).unwrap().find_item_mut(clip).unwrap().1; + for (id, param, v) in [("gaussian_blur", "blurriness", 6.0), ("proc_amp", "brightness", 25.0)] { + let mut e = filmcraft_project::find_effect(id).unwrap().instance(); + e.param_mut(param).unwrap().value = filmcraft_project::ParamValue::Float(v); + it.effects.insert(0, e); + } + } + + /// The alpha of `gpu` against the CPU reference's, in 8-bit steps (`assert_parity` compares + /// the colour over black, which hides a wrong alpha). + fn assert_alpha_parity(cpu: &Image, gpu: &Image, what: &str) { + let mut d: Vec = + cpu.px.iter().skip(3).step_by(4).zip(gpu.px.iter().skip(3).step_by(4)).map(|(a, b)| ((a - b).abs() * 255.0).round() as u32).collect(); + d.sort_unstable(); + let p99 = d[d.len() * 99 / 100]; + let max = d.last().copied().unwrap_or(0); + eprintln!("{what}: GPU vs CPU alpha p99 {p99}, max {max}"); + assert!(p99 <= 2 && max <= 8, "{what}: alpha parity p99 {p99}, max {max}"); + } + + /// A Push under an adjustment layer exports on the GPU (the Push and then the adjustment + /// layer, both by the compositor) and matches `render_sequence` over black, forward and + /// reversed, at full and half resolution. + #[test] + fn push_under_an_adjustment_layer_exports_on_the_gpu() { + let Some(mut r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu push + adjustment export): no adapter"); + return; + }; + if r.caps() == DeviceCaps::NONE { + eprintln!("SKIPPED (gpu push + adjustment export): no effect stage on this device"); + return; + } + let mut p = Project::new("x"); + let mut m = SourceMap::default(); + let ocean = media_item(&mut p, &mut m, GeneratorSource::demo(filmcraft_media::DemoScene::OceanSunset)); + let blue = media_item( + &mut p, + &mut m, + GeneratorSource::new( + Generator::ColorMatte { color: [0.1, 0.4, 0.95, 1.0] }, + 320, + 180, + filmcraft_time::FrameRate::FPS_24, + Tick(4 * TICKS_PER_SECOND), + ), + ); + let seq = + p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 2, 0, None); + let a = put(&mut p, seq, 0, ocean, 0, 24); + let b = put(&mut p, seq, 0, blue, 24, 24); + adjustment_layer(&mut p, seq, 1); + let rate = filmcraft_time::FrameRate::FPS_24; + for reverse in [false, true] { + p.sequence_mut(seq).unwrap().video_tracks[0].transitions = vec![filmcraft_project::Transition { + id: filmcraft_project::TransitionId(1), + effect: filmcraft_project::find_effect("push").unwrap().instance(), + start: rate.tick_of(18), + duration: rate.tick_of(12), + from: Some(a), + to: Some(b), + align: Default::default(), + reverse, + }]; + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + for f in [20, 24, 27] { + let (t, what) = (rate.tick_of(f), format!("Push + adjustment, reverse {reverse}, scale {scale}, frame {f}")); + assert!(!filmcraft_render::plan::is_cpu_frame(&p, seq, t, opts, r.caps()), "{what}: takes a GPU renderer"); + let before = (r.compositor.gpu_transitions, r.compositor.gpu_adjustments, r.compositor.cpu_adjustments); + let gpu = r.render(&p, seq, t, opts, &m).expect("GPU frame"); + let after = (r.compositor.gpu_transitions, r.compositor.gpu_adjustments, r.compositor.cpu_adjustments); + assert_eq!(after, (before.0 + 1, before.1 + 1, before.2), "{what}: the Push and the adjustment layer drawn on the GPU"); + let cpu = filmcraft_render::render_sequence(&p, seq, t, opts, &m); + assert_parity(&cpu, &gpu, &what); + } + } + } + } + + /// Adjustment layers composited on the GPU match `render_sequence` (not the plan's own CPU + /// executor) over a partly transparent canvas, one over another, over an empty canvas, over a + /// nested sequence and inside one, at full and half resolution, alpha included. + #[test] + fn adjustment_layers_match_the_cpu_renderer() { + let Some(mut r) = ExportRenderer::new() else { + eprintln!("SKIPPED (gpu adjustment parity): no adapter"); + return; + }; + if r.caps() == DeviceCaps::NONE { + eprintln!("SKIPPED (gpu adjustment parity): no effect stage on this device"); + return; + } + let rate = filmcraft_time::FrameRate::FPS_24; + let settings = SequenceSettings { width: 320, height: 180, frame_rate: rate, ..Default::default() }; + // V1 of `seq`: a 160×88 matte at 70 % opacity in the middle, unscaled, so transparent + // around it and translucent inside, with its edges on whole pixels at both scales: the + // comparison measures the adjustment layer, not how a resampled edge is covered + let patch = |p: &mut Project, m: &mut SourceMap, seq: ItemId| { + let g = media_item(p, m, GeneratorSource::new(Generator::ColorMatte { color: [0.9, 0.5, 0.2, 1.0] }, 160, 88, rate, Tick(4 * TICKS_PER_SECOND))); + let c = put(p, seq, 0, g, 0, 48); + p.sequence_mut(seq).unwrap().find_item_mut(c).unwrap().1.scale_to_frame = false; + set_param(p, seq, c, "opacity", "opacity", 70.0); + }; + // (what, project, sequence, sources, adjustment steps in the frame) + let mut cases: Vec<(&str, Project, ItemId, SourceMap, u64)> = Vec::new(); + { + let (mut p, mut m) = (Project::new("x"), SourceMap::default()); + let seq = p.new_sequence("s", settings.clone(), 2, 0, None); + patch(&mut p, &mut m, seq); + adjustment_layer(&mut p, seq, 1); + cases.push(("a partly transparent canvas", p, seq, m, 1)); + } + { + let (mut p, mut m) = (Project::new("x"), SourceMap::default()); + let seq = p.new_sequence("s", settings.clone(), 3, 0, None); + patch(&mut p, &mut m, seq); + adjustment_layer(&mut p, seq, 1); + adjustment_layer(&mut p, seq, 2); + cases.push(("two adjustment layers, one over the other", p, seq, m, 2)); + } + { + // nothing below the adjustment layer: it adjusts an empty canvas + let (mut p, m) = (Project::new("x"), SourceMap::default()); + let seq = p.new_sequence("s", settings.clone(), 2, 0, None); + adjustment_layer(&mut p, seq, 1); + cases.push(("an empty canvas", p, seq, m, 1)); + } + { + // the matte inside a nested sequence on V1, drawn with the outer layers + let (mut p, mut m) = (Project::new("x"), SourceMap::default()); + let inner = p.new_sequence("inner", settings.clone(), 1, 0, None); + patch(&mut p, &mut m, inner); + let seq = p.new_sequence("s", settings.clone(), 2, 0, None); + put(&mut p, seq, 0, inner, 0, 48); + adjustment_layer(&mut p, seq, 1); + cases.push(("a nested sequence below", p, seq, m, 1)); + } + { + // an adjustment layer inside a nested sequence adjusts only the nest (the matte), not + // the red matte under the nest + let (mut p, mut m) = (Project::new("x"), SourceMap::default()); + let red = media_item( + &mut p, + &mut m, + GeneratorSource::new(Generator::ColorMatte { color: [0.9, 0.2, 0.1, 1.0] }, 320, 180, rate, Tick(4 * TICKS_PER_SECOND)), + ); + let inner = p.new_sequence("inner", settings.clone(), 2, 0, None); + patch(&mut p, &mut m, inner); + adjustment_layer(&mut p, inner, 1); + let seq = p.new_sequence("s", settings.clone(), 2, 0, None); + put(&mut p, seq, 0, red, 0, 48); + put(&mut p, seq, 1, inner, 0, 48); + cases.push(("an adjustment layer inside a nested sequence", p, seq, m, 0)); + } + for (what, p, seq, m, steps) in &cases { + for scale in [1.0, 0.5] { + let (t, opts) = (rate.tick_of(12), RenderOptions { scale, ..Default::default() }); + let what = format!("{what} at {scale}"); + assert!(!filmcraft_render::plan::is_cpu_frame(p, *seq, t, opts, r.caps()), "{what}: takes a GPU renderer"); + let before = (r.compositor.gpu_adjustments, r.compositor.cpu_adjustments); + let gpu = r.render(p, *seq, t, opts, m).expect("GPU frame"); + assert_eq!((r.compositor.gpu_adjustments, r.compositor.cpu_adjustments), (before.0 + steps, before.1), "{what}: adjustment steps on the GPU"); + let cpu = filmcraft_render::render_sequence(p, *seq, t, opts, m); + assert_parity(&cpu, &gpu, &what); + assert_alpha_parity(&cpu, &gpu, &what); + } + } + } + /// Frames inside a Push transition (forward and reversed), which plans as a GPU composite, /// export on the GPU and match the CPU; a transition it cannot draw stays a CPU frame. #[test] @@ -337,7 +593,7 @@ mod tests { }]; for f in [20, 24, 27] { let t = rate.tick_of(f); - assert!(!filmcraft_render::plan::is_cpu_frame(&p, seq, t), "{id} takes a GPU renderer"); + assert!(!filmcraft_render::plan::is_cpu_frame(&p, seq, t, opts, r.caps()), "{id} takes a GPU renderer"); let before = r.compositor.gpu_transitions; let gpu = r.render(&p, seq, t, opts, &m).expect("GPU transition frame"); assert_eq!(r.compositor.gpu_transitions, before + 1, "{id}: drawn on the GPU"); @@ -346,7 +602,7 @@ mod tests { } // Film Dissolve has no GPU stage: the whole frame stays a CPU image p.sequence_mut(seq).unwrap().video_tracks[0].transitions[0].effect = filmcraft_project::find_effect("film_dissolve").unwrap().instance(); - assert!(filmcraft_render::plan::is_cpu_frame(&p, seq, rate.tick_of(24))); + assert!(filmcraft_render::plan::is_cpu_frame(&p, seq, rate.tick_of(24), opts, r.caps())); } #[test] @@ -415,6 +671,10 @@ mod tests { fn render(&mut self, project: &Project, seq: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> Option { self.0.render(project, seq, t, opts, sources) } + + fn caps(&self) -> DeviceCaps { + self.0.caps() + } } let Some(r) = ExportRenderer::new() else { eprintln!("SKIPPED (gpu export parity): no adapter"); diff --git a/crates/gpu/src/fx.rs b/crates/gpu/src/fx.rs index fdedd7d17..7a45d078b 100644 --- a/crates/gpu/src/fx.rs +++ b/crates/gpu/src/fx.rs @@ -68,6 +68,10 @@ pub(crate) struct FxStage { source: wgpu::RenderPipeline, pool: HashMap<(u32, u32), Vec>, used: HashSet<(u32, u32)>, + /// Open [`begin_frame`](Self::begin_frame)s. + frames: u32, + /// Working textures created so far (a steady stream of frames creates none). + created: u64, } /// One compute pass of a job. @@ -403,18 +407,30 @@ impl FxStage { multiview_mask: None, cache: None, }); - Self { px: compute("fx_px"), run: compute("fx_run"), bgl, source, pool: HashMap::new(), used: HashSet::new() } + Self { px: compute("fx_px"), run: compute("fx_run"), bgl, source, pool: HashMap::new(), used: HashSet::new(), frames: 0, created: 0 } } /// Start a frame: pooled textures of sizes no layer uses by [`end_frame`](Self::end_frame) - /// are released then. + /// are released then. Frames nest (a composite of several steps is one frame around the + /// layers of each step): the pool is trimmed when the outermost one ends. pub(crate) fn begin_frame(&mut self) { - self.used.clear(); + if self.frames == 0 { + self.used.clear(); + } + self.frames = self.frames.saturating_add(1); } pub(crate) fn end_frame(&mut self) { - let used = &self.used; - self.pool.retain(|k, _| used.contains(k)); + self.frames = self.frames.saturating_sub(1); + if self.frames == 0 { + let used = &self.used; + self.pool.retain(|k, _| used.contains(k)); + } + } + + #[cfg(test)] + pub(crate) fn created(&self) -> u64 { + self.created } /// No working textures held. @@ -442,6 +458,7 @@ impl FxStage { }); let view = t.create_view(&Default::default()); v.push((t, view)); + self.created = self.created.saturating_add(1); } v.iter().take(n).cloned().collect() } diff --git a/crates/gpu/src/fx_tests.rs b/crates/gpu/src/fx_tests.rs index 6ffe65015..be3e8d9b3 100644 --- a/crates/gpu/src/fx_tests.rs +++ b/crates/gpu/src/fx_tests.rs @@ -620,6 +620,10 @@ fn hostile_parameters_are_bounded() { assert_eq!(gpu.len(), w * h * 4); // a working image larger than any texture is refused, not attempted assert!(c.effect_image(&frame, &LayerFx { size: (1 << 20, 4), decimation: 1, turns: 0, ops: vec![FxOp::BlackWhite] }).is_none()); + // and the refusal ends its frame: the next frame, which uses no effect, releases the working + // images the first one pooled + c.composite(&FramePlan::Layers { width: w, height: h, layers: vec![PlanLayer::new(Arc::new(frame), Affine::IDENTITY, 1.0, Blend::Normal)] }); + assert!(c.fx.as_ref().is_some_and(|f| f.is_idle()), "working images kept after a refused effect image"); } #[test] diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 215c57cbb..16d40479b 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -36,7 +36,7 @@ use rayon::prelude::*; use filmcraft_color::{Matrix, Range, Transfer}; use filmcraft_frame::{Chroma, PixelData, VideoFrame}; use filmcraft_render::Blend; -use filmcraft_render::plan::{FramePlan, PlanLayer, PlanStep}; +use filmcraft_render::plan::{DeviceCaps, FramePlan, PlanLayer, PlanStep}; pub mod adapter; pub mod export_renderer; @@ -92,6 +92,11 @@ pub struct GpuCompositor { transition_inputs: [Option<(wgpu::Texture, wgpu::TextureView, (u32, u32))>; 2], pub gpu_transitions: u64, pub cpu_transitions: u64, + /// Adjustment layers ([`PlanStep::Adjust`]) run by the effect stage, and frames with one that + /// were composited on the CPU here instead: a plan made for another device ([`Self::caps`]), or + /// the effect stage declining it. Planning with this device's caps keeps the second at zero. + pub gpu_adjustments: u64, + pub cpu_adjustments: u64, /// Total bytes uploaded (stats). pub uploaded_bytes: u64, } @@ -264,6 +269,7 @@ pub fn prepare(plan: &FramePlan) -> PreparedPlan { .flat_map(|s| match s { PlanStep::Layers(layers) => vec![prepare_layers(layers)], PlanStep::Transition { inputs, .. } => inputs.iter().map(|ls| prepare_layers(ls)).collect(), + PlanStep::Adjust { .. } => Vec::new(), }) .collect(), ..Default::default() @@ -426,6 +432,8 @@ impl GpuCompositor { transition_inputs: [None, None], gpu_transitions: 0, cpu_transitions: 0, + gpu_adjustments: 0, + cpu_adjustments: 0, uploaded_bytes: 0, } } @@ -744,7 +752,7 @@ impl GpuCompositor { [PlanLayer { frame: Arc::new(frame), matrix: filmcraft_geom::Affine::IDENTITY, opacity: 1.0, blend: Blend::Normal, fx: None, input: None }]; (img.w as u32, img.h as u32, &owned[..]) } - FramePlan::Composite { width, height, steps } => return self.composite_steps(*width, *height, steps, prep), + FramePlan::Composite { width, height, steps } => return self.composite_steps(*width, *height, steps, prep, resolve), }; let (w, h) = (w.max(1), h.max(1)); let accum_view = Self::target(&self.device, &mut self.accum, w, h, ACCUM_FORMAT, wgpu::TextureUsages::COPY_SRC); @@ -946,15 +954,39 @@ impl GpuCompositor { (if resolve { out_view } else { accum_view }, (w, h)) } - fn composite_steps(&mut self, width: usize, height: usize, steps: &[PlanStep], prep: Option<&PreparedPlan>) -> (wgpu::TextureView, (u32, u32)) { + fn composite_steps( + &mut self, + width: usize, + height: usize, + steps: &[PlanStep], + prep: Option<&PreparedPlan>, + resolve: bool, + ) -> (wgpu::TextureView, (u32, u32)) { + let cpu = |c: &mut Self| { + let plan = FramePlan::Composite { width, height, steps: steps.to_vec() }; + c.composite_prepared(&FramePlan::Image(filmcraft_render::plan::execute_cpu(&plan)), None) + }; if steps.iter().any(|s| matches!(s, PlanStep::Transition { effect, .. } if effect.effect != "push")) { self.cpu_transitions += 1; - let plan = FramePlan::Composite { width, height, steps: steps.to_vec() }; - return self.composite_prepared(&FramePlan::Image(filmcraft_render::plan::execute_cpu(&plan)), None); + return cpu(self); + } + // plan_frame makes Adjust steps only for this device's caps, so the monitor and the export + // never get here; a plan made for another device still gets the right picture + if steps.iter().any(|s| matches!(s, PlanStep::Adjust { .. })) && !self.caps().adjusts(width, height) { + self.cpu_adjustments += 1; + return cpu(self); + } + // one frame for the effect stage's pool: the adjustment steps' working textures stay + // pooled with those of the layers between them + if let Some(f) = self.fx.as_mut() { + f.begin_frame(); } let empty = FramePlan::Layers { width, height, layers: Vec::new() }; let (canvas, dims) = self.render_plan(&empty, None, true, false); let mut children = prep.map(|p| p.children.iter()); + // counted once the frame is done on the GPU: a frame that falls back to the CPU half way + // throws away the steps drawn so far + let (mut transitions, mut adjustments) = (0u64, 0u64); for step in steps { match step { PlanStep::Layers(layers) => { @@ -972,17 +1004,97 @@ impl GpuCompositor { views[index] = view; } self.transition.draw((&self.device, &self.queue), &canvas, [&views[0], &views[1]], dims, effect, *progress); - self.gpu_transitions += 1; + transitions += 1; + } + PlanStep::Adjust { ops } => { + if self.adjust(&canvas, dims, ops).is_none() { + if let Some(f) = self.fx.as_mut() { + f.end_frame(); + } + self.cpu_adjustments += 1; + return cpu(self); + } + adjustments += 1; } } } - self.render_plan(&empty, None, false, true) + self.gpu_transitions += transitions; + self.gpu_adjustments += adjustments; + let out = self.render_plan(&empty, None, false, resolve); + if let Some(f) = self.fx.as_mut() { + f.end_frame(); + } + out + } + + /// What this device runs, for planning its frames ([`filmcraft_render::plan::plan_frame`]): + /// adjustment layers up to its texture limit when it has an effect stage. + pub fn caps(&self) -> DeviceCaps { + DeviceCaps { adjust_side: if self.fx.is_some() { self.device.limits().max_texture_dimension_2d } else { 0 } } + } + + /// The adjustment layer `ops` on the accumulator (`canvas`, of size `w`×`h`): it is drawn + /// into the effect stage's working image as a linear premultiplied RGBA source, the effects + /// run there, and the result replaces the accumulator. None when the stage can't run them + /// (nothing is drawn then). + fn adjust(&mut self, canvas: &wgpu::TextureView, (w, h): (u32, u32), ops: &[filmcraft_render::gpufx::FxOp]) -> Option<()> { + let rgba = SrcInfo { + kind: 1, + code_scale: 1.0, + code_levels: 1.0, + alpha: 0.0, + chroma: (w, h), + size: (w, h), + color: filmcraft_color::ColorInfo::SRGB_FULL, + input: None, + }; + let identity = filmcraft_geom::Affine::IDENTITY; + let mut su = Self::uniforms(&rgba, &identity, 1.0, Blend::Normal, (w, h)); + // (the effect source draw's integer decimation: the canvas is the working image) + su[26] = 1.0; + let sbuf = self.uniform_buffer(&su); + let sbg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("adjust-source"), + layout: &self.layer_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: sbuf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(canvas) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + wgpu::BindGroupEntry { binding: 5, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + ], + }); + let lfx = filmcraft_render::plan::LayerFx { size: (w, h), decimation: 1, turns: 0, ops: ops.to_vec() }; + let job = self.fx.as_mut()?.job(&self.device, &self.queue, &lfx, sbg, &self.dummy)?; + let buf = self.uniform_buffer(&Self::uniforms(&rgba, &identity, 1.0, Blend::Normal, (w, h))); + let bg = self.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("adjust-result"), + layout: &self.layer_bgl, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: buf.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: wgpu::BindingResource::TextureView(&job.result) }, + wgpu::BindGroupEntry { binding: 2, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + wgpu::BindGroupEntry { binding: 3, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + wgpu::BindGroupEntry { binding: 5, resource: wgpu::BindingResource::TextureView(&self.dummy) }, + ], + }); + let mut enc = self.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("filmcraft-adjust") }); + self.fx.as_ref()?.record(&mut enc, &job); + { + // cleared, then drawn "over" nothing: the adjusted picture replaces the canvas + let mut pass = accum_pass(&mut enc, canvas, wgpu::LoadOp::Clear(wgpu::Color::TRANSPARENT)); + pass.set_pipeline(&self.layer_pipeline); + pass.set_bind_group(0, &bg, &[]); + pass.draw(0..6, 0..1); + } + self.queue.submit([enc.finish()]); + Some(()) } /// Export rendering: `plan` composited into the linear float accumulator and read back as - /// premultiplied RGBA f32 (the sRGB resolve is skipped). Layer plans and transition - /// composites (`plan_frame` composites only transitions the GPU draws); None on a failed - /// read-back. + /// premultiplied RGBA f32 (the sRGB resolve is skipped). Layer plans and composites + /// (`plan_frame` composites only the transitions and adjustment layers the device draws); + /// None on a failed read-back. pub fn render_export_prepared(&mut self, plan: &FramePlan, prep: Option<&PreparedPlan>) -> Option<(u32, u32, Vec)> { let t = std::time::Instant::now(); self.render_plan(plan, prep, true, false); @@ -1016,7 +1128,10 @@ impl GpuCompositor { }); let stage = self.fx.as_mut()?; stage.begin_frame(); - let job = stage.job(&self.device, &self.queue, fx, sbg, &self.dummy)?; + let Some(job) = stage.job(&self.device, &self.queue, fx, sbg, &self.dummy) else { + stage.end_frame(); + return None; + }; let row = (w * 16).div_ceil(256) * 256; let buf = self.device.create_buffer(&wgpu::BufferDescriptor { label: Some("fx-readback"), diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 52911f114..79a34f7e0 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -555,6 +555,108 @@ fn hdr_input_gpu_matches_cpu() { } } +#[test] +fn gpu_adjustment_step_matches_cpu_between_layers() { + use filmcraft_project::ParamValue; + use filmcraft_render::gpufx::FxOp; + let Some((dev, q)) = device() else { + eprintln!("no GPU adapter; skipping"); + return; + }; + let (w, h) = (128usize, 72usize); + // a soft picture at 70 % opacity: the canvas under the adjustment layer is partly transparent + let below = { + let px = (0..w * h) + .flat_map(|i| { + let (x, y) = ((i % w) as f32 / w as f32, (i / w) as f32 / h as f32); + [x, y, 0.5 * (1.0 - x), 1.0] + }) + .collect(); + PlanLayer::new(Arc::new(VideoFrame::rgba_f32(w as u32, h as u32, px)), Affine::IDENTITY, 0.7, Blend::Normal) + }; + // an opaque blue box above the adjustment layer, which must stay untouched + let above = { + let px = (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + if (40..70).contains(&x) && (20..40).contains(&y) { [0.0, 0.0, 1.0, 1.0] } else { [0.0; 4] } + }) + .collect(); + PlanLayer::new(Arc::new(VideoFrame::rgba_f32(w as u32, h as u32, px)), Affine::IDENTITY, 1.0, Blend::Normal) + }; + let effect = |id: &str, params: &[(&str, f64)]| { + let mut e = filmcraft_project::find_effect(id).unwrap().instance(); + for (p, v) in params { + e.param_mut(p).unwrap().value = ParamValue::Float(*v); + } + e + }; + let cx = filmcraft_render::effects::FxCtx { + t: filmcraft_time::Tick::ZERO, + px_scale: 1.0, + seconds: 0.0, + timecode: "", + clip_name: "", + project: None, + env: None, + frame_rate: 24.0, + working: filmcraft_color::WorkingSpace::Rec709, + }; + let chain = [ + effect("proc_amp", &[("brightness", 20.0)]), + effect("gaussian_blur", &[("blurriness", 8.0)]), + effect("lumetri", &[("contrast", 20.0), ("exposure", -0.25), ("vignette_amount", -3.0)]), + ]; + let ops: Vec = chain.iter().map(|e| FxOp::eval(e, &cx, w, h).filter(FxOp::gpu_ok).expect("GPU op")).collect(); + let plan = + FramePlan::Composite { width: w, height: h, steps: vec![PlanStep::Layers(vec![below]), PlanStep::Adjust { ops }, PlanStep::Layers(vec![above])] }; + let reference = execute_cpu(&plan); + // with the effect stage the step runs on the GPU; without one (a device that fails its + // validation) the frame is composited on the CPU, with the same picture + for stage in [true, false] { + let mut c = GpuCompositor::new(&dev, &q); + if stage && c.fx.is_none() { + eprintln!("no effect stage on this device; skipping its half"); + continue; + } + if !stage { + c.fx = None; + } + // the Program monitor's path (sRGB over black) + c.composite_prepared(&plan, Some(&prepare(&plan))); + if stage { + // the next frame reuses the step's working textures instead of making them again + let created = c.fx.as_ref().map(fx::FxStage::created); + c.composite_prepared(&plan, Some(&prepare(&plan))); + assert_eq!(c.fx.as_ref().map(fx::FxStage::created), created, "the second frame made new working textures"); + } + let (_, _, gpu) = c.read_output().expect("readback"); + let mut diffs: Vec = reference + .over_black_rgba8() + .as_chunks::<4>() + .0 + .iter() + .zip(gpu.as_chunks::<4>().0) + .map(|(a, b)| (0..3).map(|k| a[k].abs_diff(b[k]) as u32).max().unwrap()) + .collect(); + diffs.sort_unstable(); + let p99 = diffs[diffs.len() * 99 / 100]; + let mean = diffs.iter().sum::() as f64 / diffs.len() as f64; + eprintln!("adjustment step, stage {stage}: p99 {p99}, mean {mean:.3}"); + assert!(p99 <= 6 && mean < 1.5, "adjustment step, stage {stage}: p99 {p99}, mean {mean}"); + // the export's path (linear premultiplied f32, no resolve) + let (ew, eh, px) = c.render_export_prepared(&plan, Some(&prepare(&plan))).expect("export render"); + assert_eq!((ew as usize, eh as usize), (w, h)); + let worst = px.iter().zip(&reference.px).map(|(a, b)| (a - b).abs()).fold(0.0f32, f32::max); + eprintln!("adjustment step, stage {stage}, export: max |GPU - CPU| {worst}"); + assert!(worst < 0.02, "stage {stage}: export render of the adjustment step differs by {worst}"); + let i = (30 * w + 55) * 4; + assert_eq!(px.get(i..i + 4), Some(&[0.0, 0.0, 1.0, 1.0][..]), "the layer above the adjustment layer is untouched"); + let want = if stage { (3, 0) } else { (0, 2) }; + assert_eq!((c.gpu_adjustments, c.cpu_adjustments), want, "stage {stage}"); + } +} + /// Regression for #3418: several threads creating instances and devices, using them and dropping /// them at once deadlocked inside the NVIDIA driver (Vulkan loader critical section in /// `vkDestroyDevice` against other devices' `create_texture`/`submit`, plus WGL threads started by diff --git a/crates/gpu/tests/export_pipeline.rs b/crates/gpu/tests/export_pipeline.rs index ba41bbb5f..70c32d33a 100644 --- a/crates/gpu/tests/export_pipeline.rs +++ b/crates/gpu/tests/export_pipeline.rs @@ -26,9 +26,15 @@ impl FrameRenderer for GpuFrameRenderer { ) -> Option { self.0.render(project, seq, t, opts, sources) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + self.0.caps() + } } -fn project(width: u32, height: u32) -> (Arc, filmcraft_project::ItemId, SourceMap) { +/// The colour matte on V1, and with `adjusted` an adjustment layer over it on V2 (Proc Amp and +/// Gaussian Blur, opaque, Normal, full frame: the kind the GPU runs as a `PlanStep::Adjust`). +fn project_with(width: u32, height: u32, adjusted: bool) -> (Arc, filmcraft_project::ItemId, SourceMap) { let mut p = filmcraft_project::Project::new("x"); let g = GeneratorSource::new( Generator::ColorMatte { color: [0.2, 0.5, 0.9, 1.0] }, @@ -54,7 +60,8 @@ fn project(width: u32, height: u32) -> (Arc, filmcra })), None, ); - let seq = p.new_sequence("s", SequenceSettings { width, height, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, 1, 1, None); + let tracks = if adjusted { 2 } else { 1 }; + let seq = p.new_sequence("s", SequenceSettings { width, height, frame_rate: filmcraft_time::FrameRate::FPS_24, ..Default::default() }, tracks, 1, None); let v = p .make_track_item( red, @@ -65,6 +72,18 @@ fn project(width: u32, height: u32) -> (Arc, filmcra ) .unwrap(); p.sequence_mut(seq).unwrap().video_tracks[0].items.push(v); + if adjusted { + let rate = filmcraft_time::FrameRate::FPS_24; + let a = p.add_item("Adjustment Layer", Label::Lavender, ItemKind::AdjustmentLayer { width, height, rate, duration: rate.tick_of(2) }, None); + let mut ti = + p.make_track_item(a, TrackKind::Video, filmcraft_time::Tick::ZERO, TimeRange::new(filmcraft_time::Tick::ZERO, rate.tick_of(2)), rate).unwrap(); + for (id, param, v) in [("proc_amp", "brightness", 25.0), ("gaussian_blur", "blurriness", 6.0)] { + let mut e = filmcraft_project::find_effect(id).unwrap().instance(); + e.param_mut(param).unwrap().value = filmcraft_project::ParamValue::Float(v); + ti.effects.insert(0, e); + } + p.sequence_mut(seq).unwrap().video_tracks[1].items.push(ti); + } let mut m = SourceMap::default(); m.0.insert(red, Arc::new(g)); (Arc::new(p), seq, m) @@ -76,8 +95,12 @@ fn png(dir: &str, name: &str) -> Vec { std::fs::read(std::env::temp_dir().join(format!("fc-gpu-export-{dir}{index:03}.png"))).unwrap() } -fn export_png(dir: &str, gpu: GpuRendering, (w, h): (u32, u32)) { - let (p, seq, m) = project(w, h); +fn export_png(dir: &str, gpu: GpuRendering, size: (u32, u32)) { + export_project_png(dir, gpu, size, false); +} + +fn export_project_png(dir: &str, gpu: GpuRendering, (w, h): (u32, u32), adjusted: bool) { + let (p, seq, m) = project_with(w, h, adjusted); let mut s = ExportSettings { format: Format::PngSequence, path: std::env::temp_dir().join(format!("fc-gpu-export-{dir}")).to_string_lossy().into_owned(), @@ -131,4 +154,18 @@ fn gpu_export_matches_cpu_and_counts() { } let GpuRenderStats { frames, .. } = filmcraft_export::gpu_render_stats(); assert!(frames >= 8, "gpu frames counted across the parity exports: {frames}"); + + // An adjustment layer is composited by the GPU too (a `PlanStep::Adjust`), not handed to the + // CPU renderer, and matches the CPU export. + let composited = filmcraft_gpu::export_renderer::timing::get().frames; + let GpuRenderStats { fallbacks, .. } = filmcraft_export::gpu_render_stats(); + export_project_png("adj-cpu", GpuRendering::Off, (320, 180), true); + export_project_png("adj-gpu", GpuRendering::Auto, (320, 180), true); + for f in ["000", "001"] { + assert_parity(&png("adj-cpu", &format!("img{f}.png")), &png("adj-gpu", &format!("img{f}.png")), &format!("adjustment layer frame {f}")); + } + assert_ne!(png("adj-cpu", "img000.png"), png("off-cpu", "img000.png"), "the adjustment layer changes the picture"); + let gpu = filmcraft_gpu::export_renderer::timing::get().frames - composited; + assert_eq!(gpu, 2, "both frames with the adjustment layer composited on the GPU"); + assert_eq!(filmcraft_export::gpu_render_stats().fallbacks, fallbacks, "no frame with the adjustment layer fell back to the CPU"); } diff --git a/crates/render/src/adjustment_tests.rs b/crates/render/src/adjustment_tests.rs index 6b5cf8183..85b8214e2 100644 --- a/crates/render/src/adjustment_tests.rs +++ b/crates/render/src/adjustment_tests.rs @@ -318,7 +318,7 @@ fn a_transition_with_an_adjustment_layer_is_not_planned_as_layers() { dissolve(&mut s, 1, Some(b), Some(ac), 18); let q = s.p.sequence(s.seq).unwrap(); // frame 20 is still on the blue clip, but the dissolve into the adjustment layer shows - assert!(!crate::plan::layered_at(&s.p, q, R.tick_of(20))); + assert!(!crate::plan::layered_at(&s.p, q, R.tick_of(20), 1.0, crate::plan::DeviceCaps::ALL)); // halfway (frame 24): half blue, half the inverted red (cyan) assert!(near(at(&s, 24), [0.0, 0.5, 1.0], 1e-3), "{:?}", at(&s, 24)); } @@ -363,3 +363,238 @@ fn basic_3d_keeps_its_perspective_at_every_render_scale() { } } } + +fn set(s: &mut Scene, c: ClipId, effect: &str, param: &str, v: ParamValue) { + clip(s, c).effect_mut(effect).unwrap().params.get_mut(param).unwrap().value = v; +} + +/// The largest channel difference between two images of the same size. +fn max_diff(a: &Image, b: &Image) -> f32 { + assert_eq!((a.w, a.h), (b.w, b.h)); + a.px.iter().zip(&b.px).map(|(x, y)| (x - y).abs()).fold(0.0, f32::max) +} + +/// An opaque, Normal, full-frame adjustment layer whose effects all run on the GPU is planned as +/// a `PlanStep::Adjust` between the layers below and above it, and the plan draws what the CPU +/// renderer draws. +#[test] +fn a_gpu_adjustment_layer_plans_as_an_adjust_step() { + let mut s = scene(3); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + let blue = matte(&mut s.p, &mut s.map, [0.0, 0.0, 1.0, 1.0]); + place(&mut s.p, s.seq, 0, ocean); + let adj = adjustment(&mut s.p, s.seq, 1); + fx(&mut s, adj, "proc_amp"); + set(&mut s, adj, "proc_amp", "brightness", ParamValue::Float(20.0)); + fx(&mut s, adj, "gaussian_blur"); + set(&mut s, adj, "gaussian_blur", "blurriness", ParamValue::Float(8.0)); + fx(&mut s, adj, "lumetri"); + set(&mut s, adj, "lumetri", "contrast", ParamValue::Float(20.0)); + set(&mut s, adj, "lumetri", "exposure", ParamValue::Float(-0.25)); + // a small blue layer above the adjustment layer, which must stay untouched + let top = place(&mut s.p, s.seq, 2, blue); + set(&mut s, top, "motion", "scale", ParamValue::Float(25.0)); + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + let plan = crate::plan::plan_frame(&s.p, s.seq, R.tick_of(12), opts, crate::plan::DeviceCaps::ALL, &s.map); + let crate::plan::FramePlan::Composite { steps, .. } = &plan else { panic!("at {scale}: not planned as steps") }; + assert!( + steps.iter().any(|st| matches!(st, crate::plan::PlanStep::Adjust { ops } if ops.len() == 3)), + "at {scale}: no Adjust step with the three effects" + ); + let got = crate::plan::execute_cpu(&plan); + let want = render_sequence(&s.p, s.seq, R.tick_of(12), opts, &s.map); + let d = max_diff(&got, &want); + assert!(d < 1e-4, "at {scale}: plan vs CPU renderer max diff {d}"); + let (cx, cy) = (got.w / 2, got.h / 2); + assert!(near(got.get(cx, cy), [0.0, 0.0, 1.0], 1e-4), "at {scale}: the layer above is untouched: {:?}", got.get(cx, cy)); + } +} + +/// Every adjustment layer the GPU step doesn't cover yet keeps the whole frame on the CPU: an +/// effect without a GPU implementation, one whose parameters the GPU doesn't run (a shrinking +/// Transform, a Transform's shutter motion blur), an opacity below 100 %, a blend mode, a Motion +/// that leaves part of the frame out. `is_cpu_frame` says so before planning, so the export +/// doesn't take a GPU renderer for the frame. +#[test] +fn an_adjustment_layer_the_gpu_cant_run_plans_the_frame_on_the_cpu() { + let base = || { + let mut s = scene(2); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + place(&mut s.p, s.seq, 0, ocean); + let adj = adjustment(&mut s.p, s.seq, 1); + fx(&mut s, adj, "invert"); + (s, adj) + }; + let multiply = filmcraft_project::effect::BLEND_MODES.iter().position(|b| *b == "Multiply").unwrap() as u32; + let cases: Vec<(&str, Box)> = vec![ + ("an effect without a GPU implementation", Box::new(|s, a| fx(s, a, "emboss"))), + ( + "a Transform that shrinks the frame (the CPU's mip path)", + Box::new(|s, a| { + fx(s, a, "transform"); + set(s, a, "transform", "scale_height", ParamValue::Float(40.0)); + }), + ), + ( + "a Transform with a shutter angle (motion blur)", + Box::new(|s, a| { + fx(s, a, "transform"); + set(s, a, "transform", "shutter_override", ParamValue::Bool(true)); + set(s, a, "transform", "shutter_angle", ParamValue::Float(180.0)); + }), + ), + ("opacity 50 %", Box::new(|s, a| set(s, a, "opacity", "opacity", ParamValue::Float(50.0)))), + ("Multiply", Box::new(move |s, a| set(s, a, "opacity", "blend", ParamValue::Choice(multiply)))), + ("Motion at 50 %", Box::new(|s, a| set(s, a, "motion", "scale", ParamValue::Float(50.0)))), + ]; + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + let (s, _) = base(); + assert!(!matches!(crate::plan::plan_frame(&s.p, s.seq, R.tick_of(12), opts, crate::plan::DeviceCaps::ALL, &s.map), crate::plan::FramePlan::Image(_))); + assert!(!crate::plan::is_cpu_frame(&s.p, s.seq, R.tick_of(12), opts, crate::plan::DeviceCaps::ALL), "at {scale}: the plain layer goes to the GPU"); + for (what, change) in &cases { + let (mut s, adj) = base(); + change(&mut s, adj); + let plan = crate::plan::plan_frame(&s.p, s.seq, R.tick_of(12), opts, crate::plan::DeviceCaps::ALL, &s.map); + assert!(matches!(plan, crate::plan::FramePlan::Image(_)), "{what} at {scale}: should be rendered on the CPU"); + assert!( + crate::plan::is_cpu_frame(&s.p, s.seq, R.tick_of(12), opts, crate::plan::DeviceCaps::ALL), + "{what} at {scale}: the export should not hand it to the GPU" + ); + } + } +} + +/// An adjustment layer inside a nested sequence adjusts only that sequence: the nested sequence +/// is not flattened into the outer plan, where it would adjust the outer tracks below too. +#[test] +fn a_nested_adjustment_layer_does_not_adjust_the_outer_tracks() { + let mut s = scene(2); + let red = matte(&mut s.p, &mut s.map, [1.0, 0.0, 0.0, 1.0]); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + let inner = s.p.new_sequence("inner", SequenceSettings { width: 320, height: 180, frame_rate: R, ..Default::default() }, 2, 0, None); + // inner: a small red square and an adjustment layer (invert) over it + let sq = place(&mut s.p, inner, 0, red); + s.p.sequence_mut(inner).unwrap().find_item_mut(sq).unwrap().1.effect_mut("motion").unwrap().params.get_mut("scale").unwrap().value = + ParamValue::Float(25.0); + let a = s.p.add_item("Adjustment Layer", Label::Lavender, ItemKind::AdjustmentLayer { width: 320, height: 180, rate: R, duration: R.tick_of(48) }, None); + let ac = place(&mut s.p, inner, 1, a); + s.p.sequence_mut(inner).unwrap().find_item_mut(ac).unwrap().1.effects.insert(0, find_effect("invert").unwrap().instance()); + // outer: ocean on V1, the nested sequence at 100 % on V2 + place(&mut s.p, s.seq, 0, ocean); + place(&mut s.p, s.seq, 1, inner); + let t = R.tick_of(12); + let want = render_sequence(&s.p, s.seq, t, RenderOptions::default(), &s.map); + let got = crate::plan::execute_cpu(&crate::plan::plan_frame(&s.p, s.seq, t, RenderOptions::default(), crate::plan::DeviceCaps::ALL, &s.map)); + let d = max_diff(&got, &want); + assert!(d < 1e-4, "planned vs CPU renderer max diff {d}"); +} + +/// A Push on the tracks below an adjustment layer plans as the Push step and then the adjustment +/// step, and the plan draws what the CPU renderer draws: forward and reversed, at full and half +/// resolution. +#[test] +fn a_push_under_an_adjustment_layer_plans_the_push_then_the_adjustment() { + let mut s = scene(2); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + let blue = matte(&mut s.p, &mut s.map, [0.1, 0.3, 0.9, 1.0]); + // V1: the ocean for frames 0..24, blue for 24..48 + let a = place(&mut s.p, s.seq, 0, ocean); + clip(&mut s, a).duration = R.tick_of(24); + let mut tb = s.p.make_track_item(blue, TrackKind::Video, R.tick_of(24), TimeRange::new(Tick::ZERO, R.tick_of(24)), R).unwrap(); + tb.scale_to_frame = true; + let b = tb.id; + s.p.sequence_mut(s.seq).unwrap().video_tracks[0].items.push(tb); + let adj = adjustment(&mut s.p, s.seq, 1); + fx(&mut s, adj, "proc_amp"); + set(&mut s, adj, "proc_amp", "brightness", ParamValue::Float(20.0)); + fx(&mut s, adj, "gaussian_blur"); + set(&mut s, adj, "gaussian_blur", "blurriness", ParamValue::Float(6.0)); + for reverse in [false, true] { + s.p.sequence_mut(s.seq).unwrap().video_tracks[0].transitions = vec![filmcraft_project::Transition { + id: filmcraft_project::TransitionId(1), + effect: find_effect("push").unwrap().instance(), + start: R.tick_of(18), + duration: R.tick_of(12), + from: Some(a), + to: Some(b), + align: Default::default(), + reverse, + }]; + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + for f in [20, 24, 27] { + let what = format!("reverse {reverse}, scale {scale}, frame {f}"); + let plan = crate::plan::plan_frame(&s.p, s.seq, R.tick_of(f), opts, crate::plan::DeviceCaps::ALL, &s.map); + let crate::plan::FramePlan::Composite { steps, .. } = &plan else { panic!("{what}: not planned as steps") }; + assert!( + matches!(steps.as_slice(), [crate::plan::PlanStep::Transition { .. }, crate::plan::PlanStep::Adjust { ops }] if ops.len() == 2), + "{what}: not the Push and then the adjustment layer" + ); + let d = max_diff(&crate::plan::execute_cpu(&plan), &render_sequence(&s.p, s.seq, R.tick_of(f), opts, &s.map)); + assert!(d < 1e-4, "{what}: plan vs CPU renderer max diff {d}"); + } + } + } +} + +/// The device that composites the plan decides whether an adjustment layer is a step of it: one +/// without an effect stage, or whose largest texture is smaller than the frame, gets the frame as +/// one CPU image, and `is_cpu_frame` predicts that for each, so the export doesn't take a renderer +/// for it. The picture is the CPU renderer's either way. +#[test] +fn the_device_caps_decide_whether_an_adjustment_layer_is_a_step() { + let mut s = scene(2); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + place(&mut s.p, s.seq, 0, ocean); + let adj = adjustment(&mut s.p, s.seq, 1); + fx(&mut s, adj, "invert"); + let t = R.tick_of(12); + for scale in [1.0, 0.5] { + let opts = RenderOptions { scale, ..Default::default() }; + let (w, h) = output_size(s.p.sequence(s.seq).unwrap(), scale); + let side = w.max(h) as u32; + let cases = [ + ("no effect stage", crate::plan::DeviceCaps::NONE, false), + ("a texture one pixel too small", crate::plan::DeviceCaps { adjust_side: side - 1 }, false), + ("a texture just large enough", crate::plan::DeviceCaps { adjust_side: side }, true), + ("every step", crate::plan::DeviceCaps::ALL, true), + ]; + for (what, caps, step) in cases { + let plan = crate::plan::plan_frame(&s.p, s.seq, t, opts, caps, &s.map); + let planned = + matches!(&plan, crate::plan::FramePlan::Composite { steps, .. } if steps.iter().any(|st| matches!(st, crate::plan::PlanStep::Adjust { .. }))); + assert_eq!(planned, step, "{what} at {scale}: adjustment step"); + assert_eq!(matches!(plan, crate::plan::FramePlan::Image(_)), !step, "{what} at {scale}: CPU image"); + assert_eq!(crate::plan::is_cpu_frame(&s.p, s.seq, t, opts, caps), !step, "{what} at {scale}: is_cpu_frame agrees with plan_frame"); + let d = max_diff(&crate::plan::execute_cpu(&plan), &render_sequence(&s.p, s.seq, t, opts, &s.map)); + assert!(d < 1e-4, "{what} at {scale}: plan vs CPU renderer max diff {d}"); + } + } +} + +/// An adjustment layer that changes nothing at that time (switched off, or with no effect on it) +/// is not a step of the plan and doesn't send the frame to the CPU; switched off, not even on a +/// device without an effect stage. +#[test] +fn an_adjustment_layer_that_does_nothing_is_not_a_step() { + let t = R.tick_of(12); + for (what, enabled, effect, caps) in + [("switched off", false, true, crate::plan::DeviceCaps::NONE), ("with no effect", true, false, crate::plan::DeviceCaps::ALL)] + { + let mut s = scene(2); + let ocean = media(&mut s.p, &mut s.map, GeneratorSource::demo(DemoScene::OceanSunset)); + place(&mut s.p, s.seq, 0, ocean); + let adj = adjustment(&mut s.p, s.seq, 1); + if effect { + fx(&mut s, adj, "invert"); + } + clip(&mut s, adj).enabled = enabled; + let plan = crate::plan::plan_frame(&s.p, s.seq, t, RenderOptions::default(), caps, &s.map); + assert!(matches!(plan, crate::plan::FramePlan::Layers { .. }), "{what}: planned as layers"); + assert!(!crate::plan::is_cpu_frame(&s.p, s.seq, t, RenderOptions::default(), caps), "{what}: not a CPU frame"); + let d = max_diff(&crate::plan::execute_cpu(&plan), &render(&s)); + assert!(d < 1e-4, "{what}: plan vs CPU renderer max diff {d}"); + } +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 92ef327d6..269f9c6d7 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -379,23 +379,32 @@ fn adjustment_layer(project: &Project, seq: &Sequence, item: &TrackItem, t: Tick /// Coverage of an adjustment layer's (Motion-transformed) frame in output pixels, or `None` when /// it covers the whole output. fn adjustment_region(seq: &Sequence, item: &TrackItem, project: &Project, mt: Tick, scale: f32, w: usize, h: usize) -> Option { + let (fw, fh, m) = adjustment_frame(seq, item, project, mt, scale); + if covers_output(fw, fh, &m, w, h) { + return None; + } + Some(Image::filled(fw, fh, [1.0; 4]).transformed(w, h, &m)) +} + +/// Whether an adjustment layer's (Motion-transformed) frame covers the whole `w`×`h` output +/// ([`adjustment_region`] is `None`), without building its coverage. +pub(crate) fn adjustment_covers_output(seq: &Sequence, item: &TrackItem, project: &Project, mt: Tick, scale: f32, w: usize, h: usize) -> bool { + let (fw, fh, m) = adjustment_frame(seq, item, project, mt, scale); + covers_output(fw, fh, &m, w, h) +} + +/// An adjustment layer's frame size at `scale` and its Motion map into output pixels. +fn adjustment_frame(seq: &Sequence, item: &TrackItem, project: &Project, mt: Tick, scale: f32) -> (usize, usize, Affine) { let size = source_size(project, item.item).unwrap_or((seq.settings.width, seq.settings.height)); let motion = motion_matrix(seq, item, size, source_par(project, item.item), mt); let s = scale as f64; let (fw, fh) = (((size.0 as f64 * s).round() as usize).max(1), ((size.1 as f64 * s).round() as usize).max(1)); let m = Affine::scale(s, s).then_apply(&motion).then_apply(&Affine::scale(1.0 / s, 1.0 / s)); - let full = fw == w - && fh == h - && (m.a - 1.0).abs() < 1e-9 - && (m.d - 1.0).abs() < 1e-9 - && m.b.abs() < 1e-12 - && m.c.abs() < 1e-12 - && m.e.abs() < 1e-9 - && m.f.abs() < 1e-9; - if full { - return None; - } - Some(Image::filled(fw, fh, [1.0; 4]).transformed(w, h, &m)) + (fw, fh, m) +} + +fn covers_output(fw: usize, fh: usize, m: &Affine, w: usize, h: usize) -> bool { + fw == w && fh == h && (m.a - 1.0).abs() < 1e-9 && (m.d - 1.0).abs() < 1e-9 && m.b.abs() < 1e-12 && m.c.abs() < 1e-12 && m.e.abs() < 1e-9 && m.f.abs() < 1e-9 } /// Rendered captions of the visible caption tracks at `t` for a `w`×`h` output. diff --git a/crates/render/src/nest_tests.rs b/crates/render/src/nest_tests.rs index 7390a4bab..9191004ab 100644 --- a/crates/render/src/nest_tests.rs +++ b/crates/render/src/nest_tests.rs @@ -69,7 +69,7 @@ impl Rig { /// The same frame through the frame plan (what the GPU compositor is given). fn planned(&self, seq: ItemId, t: Tick) -> Image { - plan::execute_cpu(&plan::plan_frame(&self.p, seq, t, RenderOptions::default(), &self.map)) + plan::execute_cpu(&plan::plan_frame(&self.p, seq, t, RenderOptions::default(), plan::DeviceCaps::ALL, &self.map)) } } @@ -265,7 +265,7 @@ fn captions_inside_a_nest_are_part_of_its_picture() { /// How many layers the plan of `seq` at `t` has (None when it fell back to one CPU image). fn layer_count(r: &Rig, seq: ItemId, t: Tick) -> Option { - match plan::plan_frame(&r.p, seq, t, RenderOptions::default(), &r.map) { + match plan::plan_frame(&r.p, seq, t, RenderOptions::default(), plan::DeviceCaps::ALL, &r.map) { plan::FramePlan::Layers { layers, .. } => Some(layers.len()), _ => None, } @@ -378,7 +378,7 @@ fn transformed_plain_nests_preserve_the_group_picture() { for preview in [1.0, 0.5] { let opts = RenderOptions { scale: preview, ..Default::default() }; let reference = render_sequence(&r.p, outer, t, opts, &r.map); - let frame_plan = plan::plan_frame(&r.p, outer, t, opts, &r.map); + let frame_plan = plan::plan_frame(&r.p, outer, t, opts, plan::DeviceCaps::ALL, &r.map); let planned = plan::execute_cpu(&frame_plan); let difference = worst(&planned, &reference); assert!(difference < 0.001, "scale {scale}, rotation {rotation}, preview {preview}: {difference}"); diff --git a/crates/render/src/par_tests.rs b/crates/render/src/par_tests.rs index cb57c6c60..a9a2e8dd4 100644 --- a/crates/render/src/par_tests.rs +++ b/crates/render/src/par_tests.rs @@ -142,7 +142,7 @@ fn the_gpu_plan_places_an_anamorphic_clip_like_the_cpu() { for scale in [1.0, 0.5] { let opts = RenderOptions { scale, ..Default::default() }; let want = render_sequence(&p, seq, Tick::ZERO, opts, &map); - let got = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, opts, &map)); + let got = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, opts, plan::DeviceCaps::ALL, &map)); assert_eq!((got.w, got.h), (want.w, want.h)); assert_row(&row(&got), &row(&want)); } @@ -246,7 +246,7 @@ fn hostile_pixel_aspect_ratios_render_square_without_panicking() { // a sequence's ratio (a damaged project): square, so a 4:3-pixel clip fills it let r = std::panic::catch_unwind(|| { let (p, seq, _, map) = project((144, 108), (4, 3), (192, 108), bad, true); - let plan = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, RenderOptions::default(), &map)); + let plan = plan::execute_cpu(&plan::plan_frame(&p, seq, Tick::ZERO, RenderOptions::default(), plan::DeviceCaps::ALL, &map)); (row(&render(&p, seq, &map)), row(&plan)) }); let (cpu, gpu) = r.unwrap_or_default(); diff --git a/crates/render/src/plan.rs b/crates/render/src/plan.rs index 1d0cafcb6..4e1c356bf 100644 --- a/crates/render/src/plan.rs +++ b/crates/render/src/plan.rs @@ -7,14 +7,15 @@ //! them as a [`LayerFx`]: their parameters evaluated at the frame's time, run by the GPU on the //! clip's working image before Motion places it, in the CPU's order (effects → Motion → Opacity / //! blend). Anything the shaders don't cover yet — other standard effects, masks, adjustment -//! layers, nested sequences, transitions other than dissolves, dips and Push — is rendered on the CPU -//! for that layer (or the whole frame) and handed over as a pre-composited image, so the GPU path -//! is always exact with respect to the CPU reference. +//! layers other than [`PlanStep::Adjust`]'s, nested sequences, transitions other than dissolves, +//! dips and Push — is rendered on the CPU for that layer (or the whole frame) and handed over as a +//! pre-composited image, so the GPU path is always exact with respect to the CPU reference. //! //! A Push transition is planned as ordered [`PlanStep`] operations on isolated clip inputs: each //! side is drawn into its own image, the two are pushed together, and the result is composited (Normal) over -//! the canvas of the lower tracks, which is preserved for the blends above it. [`execute_cpu`] -//! runs the same steps on the CPU and is the reference. +//! the canvas of the lower tracks, which is preserved for the blends above it. An adjustment layer +//! is a [`PlanStep::Adjust`] on the canvas of the tracks below it. [`execute_cpu`] runs the same +//! steps on the CPU and is the reference. use std::sync::Arc; @@ -99,12 +100,45 @@ pub enum PlanStep { progress: f32, scale: f32, }, + /// An adjustment layer: `ops` run on the whole canvas of the tracks below, and the result + /// replaces it (full frame, opacity 100 %, Normal, no masks: anything else plans the frame on + /// the CPU). + Adjust { + ops: Vec, + }, +} + +/// What the device that composites a plan can run. [`plan_frame`] plans a frame with a step the +/// device can't run as one CPU image, on the thread that plans it, instead of leaving the +/// compositor to fall back to the CPU while it draws (the UI thread for a monitor, a renderer slot +/// for an export). The monitor and the export can composite on different GPUs, so each passes its +/// own; a cache of plans keys them by it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceCaps { + /// Largest output side whose adjustment layers ([`PlanStep::Adjust`]) the device runs: its + /// texture limit with an effect stage, 0 without one. + pub adjust_side: u32, +} + +impl DeviceCaps { + /// A device without an effect stage: a frame with an adjustment layer plans on the CPU. + pub const NONE: Self = Self { adjust_side: 0 }; + /// A device that runs every step ([`execute_cpu`] does). + pub const ALL: Self = Self { adjust_side: u32::MAX }; + + /// Whether the device runs the adjustment layers of a `w`×`h` frame. + pub fn adjusts(self, w: usize, h: usize) -> bool { + w.max(h) <= self.adjust_side as usize + } } #[derive(Default)] struct PlanStack { steps: Vec, exhausted: bool, + /// Something at this time only the CPU renderer draws (an adjustment layer the GPU can't run): + /// the whole frame goes to it, as when `exhausted`. + cpu_only: bool, /// Layers held in `steps`, kept as they are pushed so the cap check stays O(1). layers: usize, } @@ -140,8 +174,15 @@ impl PlanStack { self.steps.push(PlanStep::Transition { inputs, effect, progress, scale }); } + fn push_adjust(&mut self, ops: Vec) { + if self.exhausted || ops.is_empty() { + return; + } + self.steps.push(PlanStep::Adjust { ops }); + } + fn into_layers(self, width: usize, height: usize) -> Option> { - if self.exhausted { + if self.exhausted || self.cpu_only { return None; } match self.finish(width, height) { @@ -175,6 +216,7 @@ impl FramePlan { .map(|s| match s { PlanStep::Layers(ls) => layers(ls), PlanStep::Transition { inputs, .. } => inputs.iter().map(|ls| layers(ls)).sum(), + PlanStep::Adjust { .. } => 0, }) .sum(), } @@ -191,6 +233,7 @@ impl FramePlan { .map(|s| match s { PlanStep::Layers(ls) => layer_side(ls), PlanStep::Transition { inputs, .. } => inputs.iter().map(|ls| layer_side(ls)).max().unwrap_or(0), + PlanStep::Adjust { .. } => 0, }) .fold((*width).max(*height), usize::max), } @@ -230,16 +273,16 @@ fn gpu_chain<'a>(project: &Project, item: &'a TrackItem, opts: RenderOptions) -> Some(chain) } -/// Plan the frame at timeline `t`. -pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider) -> FramePlan { +/// Plan the frame at timeline `t` for a device that runs `caps`. +pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOptions, caps: DeviceCaps, sources: &dyn SourceProvider) -> FramePlan { let Some(seq) = project.sequence(seq_id) else { return FramePlan::Image(crate::Image::new(1, 1)) }; let (w, h) = output_size(seq, opts.scale); - if whole_frame_on_cpu(project, seq, t) { + if whole_frame_on_cpu(project, seq, t, opts.scale, caps) { return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources)); } let mut layers = PlanStack::default(); - push_tracks(project, seq, t, opts, sources, 0, &mut layers); - if layers.exhausted { + push_tracks(Scene { project, seq, t, opts, caps, sources, nest: 0 }, &mut layers); + if layers.exhausted || layers.cpu_only { return FramePlan::Image(crate::render_sequence(project, seq_id, t, opts, sources)); } if opts.captions { @@ -258,25 +301,69 @@ pub fn plan_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOption layers.finish(w, h) } -/// Whether [`plan_frame`] hands the frame at `t` back as one CPU image ([`FramePlan::Image`]). A -/// GPU renderer gains nothing on such a frame, so a caller with only a few of them (the export's -/// pool) can render it on the CPU without taking one. -pub fn is_cpu_frame(project: &Project, seq_id: ItemId, t: Tick) -> bool { - project.sequence(seq_id).is_none_or(|seq| whole_frame_on_cpu(project, seq, t)) +/// Whether [`plan_frame`] hands the frame at `t` back as one CPU image ([`FramePlan::Image`]) for +/// a device that runs `caps`. A GPU renderer gains nothing on such a frame, so a caller with only a +/// few of them (the export's pool) can render it on the CPU without taking one. +pub fn is_cpu_frame(project: &Project, seq_id: ItemId, t: Tick, opts: RenderOptions, caps: DeviceCaps) -> bool { + project.sequence(seq_id).is_none_or(|seq| whole_frame_on_cpu(project, seq, t, opts.scale, caps)) } /// The frames [`plan_frame`] does not plan: an HDR / wide-gamut sequence composites and converts -/// on the CPU, and so does a frame with an adjustment layer or a complex transition anywhere at -/// `t` (whole-frame fallback). -fn whole_frame_on_cpu(project: &Project, seq: &Sequence, t: Tick) -> bool { - !seq.settings.color.is_plain() || !layered_at(project, seq, t) +/// on the CPU, and so does a frame with an adjustment layer the device (`caps`) can't run or a +/// complex transition anywhere at `t` (whole-frame fallback). +fn whole_frame_on_cpu(project: &Project, seq: &Sequence, t: Tick, scale: f32, caps: DeviceCaps) -> bool { + !seq.settings.color.is_plain() || !layered_at(project, seq, t, scale, caps) } -/// Whether the frame of `seq` at `t` can be planned as layers: no adjustment layer and no -/// transition the compositor cannot mix itself is showing (a transition into or out of an -/// adjustment layer counts, also on the side of the cut where the other clip is). -pub(crate) fn layered_at(project: &Project, seq: &Sequence, t: Tick) -> bool { - let adjustment = |it: &TrackItem| project.item(it.item).is_some_and(|p| matches!(p.kind, ItemKind::AdjustmentLayer { .. })); +fn is_adjustment(project: &Project, it: &TrackItem) -> bool { + project.item(it.item).is_some_and(|p| matches!(p.kind, ItemKind::AdjustmentLayer { .. })) +} + +/// The ops of an adjustment layer that can be a [`PlanStep::Adjust`] at `t` and render `scale` on +/// a device that runs `caps`: the device adjusts a frame of this size, and the layer is opaque, +/// Normal, unmasked, covering the whole frame, with every enabled standard effect unmasked and +/// evaluating to an op the GPU runs (at the frame's size, as `adjustment_layer` applies them on the +/// CPU). None sends the frame to the CPU. [`layered_at`] and [`push_adjustment`] both ask this, so +/// [`is_cpu_frame`] knows which frames plan as one CPU image. +fn adjustment_ops(project: &Project, seq: &Sequence, it: &TrackItem, t: Tick, scale: f32, caps: DeviceCaps) -> Option> { + let mt = it.effect_time_at(t); + let (op, bl) = crate::opacity_blend(it, mt); + let (w, h) = output_size(seq, scale); + if !caps.adjusts(w, h) + || op < 1.0 - 1e-6 + || bl != Blend::Normal + || it.has_opacity_masks() + || !crate::adjustment_covers_output(seq, it, project, mt, scale, w, h) + { + return None; + } + let cx = crate::effects::FxCtx { + t: mt, + px_scale: scale, + seconds: (t - it.start).seconds(), + timecode: "", + clip_name: &it.name, + project: Some(project), + env: None, + frame_rate: seq.settings.frame_rate.as_f64(), + working: seq.settings.color.working, + }; + it.effects + .iter() + .filter(|e| e.enabled && e.def().is_some_and(|d| !d.intrinsic)) + .map(|e| { + let gpu = e.masks.iter().all(|m| m.mode == filmcraft_project::MaskMode::None) && crate::gpufx::GPU_EFFECTS.contains(&e.effect.as_str()); + if gpu { FxOp::eval(e, &cx, w, h).filter(FxOp::gpu_ok) } else { None } + }) + .collect() +} + +/// Whether the frame of `seq` at `t` and render `scale` can be planned as layers for a device that +/// runs `caps`: no adjustment layer it can't run ([`adjustment_ops`]) and no transition the +/// compositor cannot mix itself is showing (a transition into or out of an adjustment layer counts, +/// also on the side of the cut where the other clip is). +pub(crate) fn layered_at(project: &Project, seq: &Sequence, t: Tick, scale: f32, caps: DeviceCaps) -> bool { + let adjustment = |it: &TrackItem| is_adjustment(project, it); for tr in &seq.video_tracks { if !tr.enabled { continue; @@ -290,7 +377,9 @@ pub(crate) fn layered_at(project: &Project, seq: &Sequence, t: Tick) -> bool { } } if let Some(it) = tr.item_at(t) + && it.enabled && adjustment(it) + && adjustment_ops(project, seq, it, t, scale, caps).is_none() { return false; } @@ -298,11 +387,21 @@ pub(crate) fn layered_at(project: &Project, seq: &Sequence, t: Tick) -> bool { true } -/// Push the layers of every video track of `seq` at `t`, bottom track first (`seq` must be -/// [`layered_at`] `t`). `nest` counts the nested sequences already followed to reach `seq`. -fn push_tracks(project: &Project, seq: &Sequence, t: Tick, opts: RenderOptions, sources: &dyn SourceProvider, nest: u32, layers: &mut PlanStack) { +/// Plan the adjustment layer `item` of `seq` at `t` as a [`PlanStep::Adjust`]. One +/// [`adjustment_ops`] declines sends the frame to the CPU ([`layered_at`] has declined the frame +/// already then). +fn push_adjustment(sc: Scene, item: &TrackItem, out: &mut PlanStack) { + match adjustment_ops(sc.project, sc.seq, item, sc.t, sc.opts.scale, sc.caps) { + Some(ops) => out.push_adjust(ops), + None => out.cpu_only = true, + } +} + +/// Push the layers of every video track of the scene's sequence at its time, bottom track first +/// (the sequence must be [`layered_at`] that time). +fn push_tracks(sc: Scene, layers: &mut PlanStack) { + let Scene { project, seq, t, opts, .. } = sc; let (w, h) = output_size(seq, opts.scale); - let sc = Scene { project, seq, t, opts, sources, nest }; for tr in &seq.video_tracks { if layers.exhausted { return; @@ -361,6 +460,12 @@ fn push_tracks(project: &Project, seq: &Sequence, t: Tick, opts: RenderOptions, if !item.enabled { continue; } + if is_adjustment(project, item) { + if opts.effects { + push_adjustment(sc, item, layers); + } + continue; + } push_item(sc, item, 1.0, None, layers); } } @@ -369,11 +474,13 @@ fn push_tracks(project: &Project, seq: &Sequence, t: Tick, opts: RenderOptions, /// it is in, instead of being rendered to an image on the CPU first. Compositing its layers one /// by one over what is below gives the same picture as compositing them together first only when /// they all blend Normal, and the frame must be one that plans as layers in the same colour -/// pipeline. -fn nest_is_plain(project: &Project, seq: &Sequence, nested: &Sequence, ft: Tick) -> bool { +/// pipeline. An adjustment layer inside it would adjust the outer tracks below too, so it keeps +/// the nested sequence on the CPU. +fn nest_is_plain(project: &Project, seq: &Sequence, nested: &Sequence, ft: Tick, scale: f32, caps: DeviceCaps) -> bool { nested.settings.color == seq.settings.color && (nested.settings.width, nested.settings.height) == (seq.settings.width, seq.settings.height) - && layered_at(project, nested, ft) + && layered_at(project, nested, ft, scale, caps) + && !nested.video_tracks.iter().any(|tr| tr.enabled && tr.item_at(ft).is_some_and(|i| i.enabled && is_adjustment(project, i))) && !nested.video_tracks.iter().any(|tr| tr.enabled && tr.transitions.iter().any(|t| t.range().contains(ft) && !simple_transition(&t.effect.effect))) && nested.video_tracks.iter().filter(|tr| tr.enabled).all(|tr| { tr.transitions.iter().any(|x| x.range().contains(ft)) @@ -381,14 +488,15 @@ fn nest_is_plain(project: &Project, seq: &Sequence, nested: &Sequence, ft: Tick) }) } -/// What a layer is pushed against: the sequence at time `t`, how it is rendered and how many -/// nests were followed to reach it. +/// What a layer is pushed against: the sequence at time `t`, how it is rendered, what the +/// compositing device runs and how many nests were followed to reach it. #[derive(Clone, Copy)] struct Scene<'a> { project: &'a Project, seq: &'a Sequence, t: Tick, opts: RenderOptions, + caps: DeviceCaps, sources: &'a dyn SourceProvider, nest: u32, } @@ -398,7 +506,7 @@ struct Scene<'a> { /// `nest` counts the multi-camera clips already followed to reach `item` (see /// [`crate::MAX_NEST_DEPTH`]). fn push_item(sc: Scene, item: &TrackItem, extra_opacity: f32, blend: Option, out: &mut PlanStack) { - let Scene { project, seq, t, opts, sources, nest } = sc; + let Scene { project, seq, t, opts, caps, sources, nest } = sc; if out.exhausted { return; } @@ -437,9 +545,9 @@ fn push_item(sc: Scene, item: &TrackItem, extra_opacity: f32, blend: Option= 1.0 - 1e-6 && near_identity(&motion_matrix(seq, item, (nested.settings.width, nested.settings.height), Some(nested.settings.par), mt)) - && nest_is_plain(project, seq, nested, ft) + && nest_is_plain(project, seq, nested, ft, opts.scale, caps) { - push_tracks(project, nested, ft, opts, sources, nest + 1, out); + push_tracks(Scene { seq: nested, t: ft, nest: nest + 1, ..sc }, out); if out.exhausted { return; } @@ -588,6 +696,11 @@ pub fn execute_cpu(plan: &FramePlan) -> crate::Image { let mixed = crate::transitions::apply_scaled(effect, &a, &b, *progress, *scale); crate::blend::composite(&mut canvas, &mixed, 1.0, Blend::Normal); } + PlanStep::Adjust { ops } => { + for op in ops { + op.apply(&mut canvas); + } + } } } canvas @@ -681,7 +794,7 @@ mod tests { scale: 1.0, }]; let expected = execute_cpu(&FramePlan::Composite { width: 1, height: 1, steps: steps.clone() }); - let layers = PlanStack { steps, exhausted: false, layers: 1 }.into_layers(1, 1).unwrap(); + let layers = PlanStack { steps, exhausted: false, cpu_only: false, layers: 1 }.into_layers(1, 1).unwrap(); let got = execute_cpu(&FramePlan::Layers { width: 1, height: 1, layers }); assert_eq!(got.px, expected.px); assert!(got.px.iter().any(|v| *v != 0.0)); diff --git a/crates/render/src/region_tests.rs b/crates/render/src/region_tests.rs index 10f8831e2..904fdddf6 100644 --- a/crates/render/src/region_tests.rs +++ b/crates/render/src/region_tests.rs @@ -358,7 +358,7 @@ fn hdr_media_is_planned_for_the_gpu_with_its_colour_management() { let f = Arc::new(VideoFrame { color, ..(*camera(W, H)).clone() }); let (p, seq, map) = scene(&[Spec::new(f)]); let opts = RenderOptions::default(); - let plan = plan::plan_frame(&p, seq, Tick::ZERO, opts, &map); + let plan = plan::plan_frame(&p, seq, Tick::ZERO, opts, plan::DeviceCaps::ALL, &map); let plan::FramePlan::Layers { layers, .. } = &plan else { panic!("{name}: not a layer plan") }; assert_eq!(layers.len(), 1, "{name}"); assert!(matches!(layers[0].frame.data, PixelData::Yuv8 { .. }), "{name}: converted on the CPU"); diff --git a/crates/render/src/tests.rs b/crates/render/src/tests.rs index 414fe6572..3730018ab 100644 --- a/crates/render/src/tests.rs +++ b/crates/render/src/tests.rs @@ -212,7 +212,7 @@ fn plan_matches_reference_renderer() { for f in [0, 12, 20, 40] { let opts = RenderOptions { scale: 0.5, ..Default::default() }; let reference = render_sequence(&p, seq, r.tick_of(f), opts, &map); - let plan = plan::plan_frame(&p, seq, r.tick_of(f), opts, &map); + let plan = plan::plan_frame(&p, seq, r.tick_of(f), opts, plan::DeviceCaps::ALL, &map); assert!(matches!(plan, plan::FramePlan::Layers { .. }), "simple frame should be GPU-drawable"); let got = plan::execute_cpu(&plan); assert_eq!((got.w, got.h), (reference.w, reference.h)); @@ -251,7 +251,7 @@ fn push_plan_preserves_reference_track_order_and_reverse() { tr.to = if missing { None } else { Some(b) }; for f in [18, 21, 24, 29] { let opts = RenderOptions { scale, ..Default::default() }; - let plan = plan::plan_frame(&p, seq, rate.tick_of(f), opts, &map); + let plan = plan::plan_frame(&p, seq, rate.tick_of(f), opts, plan::DeviceCaps::ALL, &map); assert!(matches!(plan, plan::FramePlan::Composite { .. })); let got = plan::execute_cpu(&plan); let reference = render_sequence(&p, seq, rate.tick_of(f), opts, &map); @@ -282,9 +282,9 @@ fn morph_cut_frames_render_on_the_cpu_with_the_morph() { reverse: false, }); let opts = RenderOptions { scale: 0.5, ..Default::default() }; - assert!(plan::is_cpu_frame(&p, seq, rate.tick_of(24))); - assert!(!plan::is_cpu_frame(&p, seq, rate.tick_of(10)), "outside the transition the frame stays on the GPU"); - let plan = plan::plan_frame(&p, seq, rate.tick_of(24), opts, &map); + assert!(plan::is_cpu_frame(&p, seq, rate.tick_of(24), opts, plan::DeviceCaps::ALL)); + assert!(!plan::is_cpu_frame(&p, seq, rate.tick_of(10), opts, plan::DeviceCaps::ALL), "outside the transition the frame stays on the GPU"); + let plan = plan::plan_frame(&p, seq, rate.tick_of(24), opts, plan::DeviceCaps::ALL, &map); assert!(matches!(plan, plan::FramePlan::Image(_))); assert_eq!(plan::execute_cpu(&plan).px, render_sequence(&p, seq, rate.tick_of(24), opts, &map).px); } @@ -417,7 +417,7 @@ fn draft_reduced_resolution_plans_carry_decimated_yuv() { for draft in [false, true] { for (scale, small_w) in [(1.0f32, 320u32), (0.5, 160), (0.25, 80)] { let opts = RenderOptions { scale, ..Default::default() }; - let plan = filmcraft_media::cancel::with_draft(draft, || plan::plan_frame(&p, seq, t, opts, &map)); + let plan = filmcraft_media::cancel::with_draft(draft, || plan::plan_frame(&p, seq, t, opts, plan::DeviceCaps::ALL, &map)); let plan::FramePlan::Layers { layers, .. } = &plan else { panic!("GPU-drawable") }; // exact playback hands over the decoded picture; draft playback planes at the drawn size assert_eq!(layers[0].frame.width, if draft { small_w } else { 320 }, "scale {scale}, draft {draft}"); @@ -445,7 +445,7 @@ fn a_multicam_clip_showing_itself_plans_and_renders_to_an_end() { p.sequence_mut(a).unwrap().video_tracks[0].items.push(it); let opts = RenderOptions { scale: 0.1, ..Default::default() }; let sources = SourceMap::default(); - let img = plan::execute_cpu(&plan::plan_frame(&p, a, Tick::ZERO, opts, &sources)); + let img = plan::execute_cpu(&plan::plan_frame(&p, a, Tick::ZERO, opts, plan::DeviceCaps::ALL, &sources)); assert!(img.w > 0 && img.h > 0); let img = render_sequence(&p, a, Tick::ZERO, opts, &sources); assert!(img.w > 0 && img.h > 0); @@ -510,7 +510,7 @@ fn a_reversed_clip_shows_exactly_its_source_range() { let shown = |p: &Project, f: i64, planned: bool| -> Vec { src.asked.lock().unwrap().clear(); if planned { - plan::execute_cpu(&plan::plan_frame(p, seq, r.tick_of(f), opts, &map)); + plan::execute_cpu(&plan::plan_frame(p, seq, r.tick_of(f), opts, plan::DeviceCaps::ALL, &map)); } else { render_sequence(p, seq, r.tick_of(f), opts, &map); } diff --git a/crates/ui-egui/examples/bench/playback.rs b/crates/ui-egui/examples/bench/playback.rs index 4d08f22d8..6dde3a10d 100644 --- a/crates/ui-egui/examples/bench/playback.rs +++ b/crates/ui-egui/examples/bench/playback.rs @@ -137,7 +137,10 @@ impl Display { } fn target(&self, seq: ItemId) -> Target { - if self.gpu.is_some() { Target::SequencePlan(seq) } else { Target::Sequence(seq) } + match &self.gpu { + Some((c, _)) => Target::SequencePlan(seq, c.caps()), + None => Target::Sequence(seq), + } } /// Show the exact frame, or the nearest earlier cached one; returns whether it was exact. diff --git a/crates/ui-egui/examples/bench/sections.rs b/crates/ui-egui/examples/bench/sections.rs index 391402a67..ada15935c 100644 --- a/crates/ui-egui/examples/bench/sections.rs +++ b/crates/ui-egui/examples/bench/sections.rs @@ -29,6 +29,10 @@ fn gpu_export_frame_renderer_factory() -> Option Option { self.0.render(project, seq, t, opts, sources) } + + fn caps(&self) -> filmcraft_render::plan::DeviceCaps { + self.0.caps() + } } filmcraft_gpu::ExportRenderer::new().map(|r| Box::new(R(r)) as Box) } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 6995bf62c..12fea26d1 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -26,8 +26,10 @@ pub enum Target { Sequence(ItemId), /// A single project item at a media time (Source monitor, thumbnails). Item(ItemId), - /// A GPU frame plan of a sequence (decoded layers + transforms), composited on the GPU. - SequencePlan(ItemId), + /// A GPU frame plan of a sequence (decoded layers + transforms), planned for and composited on + /// a device that runs these caps. A plan made for other caps is another frame: it may hold + /// steps this device would have to composite on the CPU, on the UI thread. + SequencePlan(ItemId, filmcraft_render::plan::DeviceCaps), /// The Multi-Camera view's angle grid of a multi-camera source at its time: one page (grid /// side, 0 = automatic; page) of the shown angles, each at the job's scale. MulticamGrid(ItemId, u8, u16), @@ -586,7 +588,7 @@ impl FrameServer { /// Whether the exact frame (image or plan) is ready. pub fn is_ready(&self, k: &FrameKey) -> bool { match k.target { - Target::SequencePlan(_) => self.shared.plans.lock().unwrap_or_else(PoisonError::into_inner).map.contains_key(k), + Target::SequencePlan(..) => self.shared.plans.lock().unwrap_or_else(PoisonError::into_inner).map.contains_key(k), _ => self.shared.done.lock().unwrap_or_else(PoisonError::into_inner).map.contains_key(k), } } @@ -607,7 +609,7 @@ impl FrameServer { pub fn recent(&self, key: FrameKey, dir: i64, max_back: i64, lag: u64) -> Option { let keys = recent_keys(key, dir, max_back, lag); match key.target { - Target::SequencePlan(_) => { + Target::SequencePlan(..) => { let g = self.shared.plans.lock().unwrap_or_else(|e| e.into_inner()); keys.into_iter().find(|k| g.map.contains_key(k)) } @@ -990,8 +992,8 @@ fn run_job(sh: &Shared, job: &Job, pool: &Arc, services: &Arc, services: &Arc) -> J JobProvider { inner: pool.provider(job.project.clone(), services.clone()) } } -fn plan_job(job: &Job, seq: ItemId, pool: &Arc, services: &Arc, previews: &PreviewStore) -> (filmcraft_render::plan::FramePlan, bool) { +fn plan_job( + job: &Job, + seq: ItemId, + caps: filmcraft_render::plan::DeviceCaps, + pool: &Arc, + services: &Arc, + previews: &PreviewStore, +) -> (filmcraft_render::plan::FramePlan, bool) { let opts = filmcraft_render::RenderOptions { scale: job.scale, captions: true, ..Default::default() }; if let Some(frame) = preview_frame(job, seq, pool, previews) && let Some(q) = job.project.sequence(seq) @@ -1092,12 +1101,12 @@ fn plan_job(job: &Job, seq: ItemId, pool: &Arc, services: &Arc, services: &Arc, previews: &PreviewStore) -> (Rgba, bool) { - if let Target::Sequence(seq) | Target::SequencePlan(seq) = job.key.target + if let Target::Sequence(seq) | Target::SequencePlan(seq, _) = job.key.target && let Some(f) = preview_frame(job, seq, pool, previews) && let Some(q) = job.project.sequence(seq) { @@ -1118,7 +1127,7 @@ fn render_job(job: &Job, pool: &Arc, services: &Arc, pr let provider = provider(job, pool, services); let opts = filmcraft_render::RenderOptions { scale: job.scale, captions: true, ..Default::default() }; let img = match job.key.target { - Target::Sequence(seq) | Target::SequencePlan(seq) | Target::SequenceLive(seq) => { + Target::Sequence(seq) | Target::SequencePlan(seq, _) | Target::SequenceLive(seq) => { let (w, h, px) = filmcraft_render::render_sequence_rgba8(&job.project, seq, job.time, opts, &provider); return (Rgba { w, h, px }, false); } @@ -1153,7 +1162,7 @@ mod tests { assert!(!background_work(1, item)); // the scopes ask for the program frame at 40 while playing: never background assert!(!background_work(40, Target::Sequence(ItemId(1)))); - assert!(!background_work(50, Target::SequencePlan(ItemId(1)))); + assert!(!background_work(50, Target::SequencePlan(ItemId(1), filmcraft_render::plan::DeviceCaps::NONE))); } #[test] @@ -1303,4 +1312,68 @@ mod tests { assert!(job.cancel.load(Ordering::Relaxed), "the job ran without its cancel flag"); assert!(!server.is_ready(&job.key), "a cancelled job's frame is not cached"); } + /// A sequence plan is made for the caps in its key: the same frame asked for a device without + /// an effect stage and for one with it gives two cached plans, the CPU image and the plan with + /// the adjustment step, so a plan made for one device is never shown on the other. + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn a_sequence_plan_is_made_for_the_caps_in_its_key() { + use super::{FrameKey, FrameServer}; + use filmcraft_media::generators::GeneratorSource; + use filmcraft_media::{Generator, MediaSource}; + use filmcraft_project::{ItemKind, Label, MediaClip, MediaRef, Project, SequenceSettings, TrackKind}; + use filmcraft_render::plan::{DeviceCaps, FramePlan, PlanStep}; + use filmcraft_time::{FrameRate, TICKS_PER_SECOND, Tick, TimeRange}; + use std::sync::Arc; + use std::time::{Duration, Instant}; + + let r = FrameRate::FPS_24; + let mut p = Project::new("Frames"); + let g = GeneratorSource::new(Generator::ColorMatte { color: [0.9, 0.2, 0.1, 1.0] }, 320, 180, r, Tick(2 * TICKS_PER_SECOND)); + let info = g.info().clone(); + let matte = p.add_item( + &info.name.clone(), + Label::Iris, + ItemKind::Media(Box::new(MediaClip { + media: MediaRef::Generator(g.generator.clone()), + info, + interpret: Default::default(), + mark_in: None, + mark_out: None, + markers: vec![], + offline: false, + proxy: None, + identity: None, + })), + None, + ); + let adj = + p.add_item("Adjustment Layer", Label::Lavender, ItemKind::AdjustmentLayer { width: 320, height: 180, rate: r, duration: r.tick_of(24) }, None); + let seq = p.new_sequence("s", SequenceSettings { width: 320, height: 180, frame_rate: r, ..Default::default() }, 2, 0, None); + for (track, item) in [(0, matte), (1, adj)] { + let mut ti = p.make_track_item(item, TrackKind::Video, Tick::ZERO, TimeRange::new(Tick::ZERO, r.tick_of(24)), r).unwrap(); + if item == adj { + ti.effects.insert(0, filmcraft_project::find_effect("invert").unwrap().instance()); + } + p.sequence_mut(seq).unwrap().video_tracks[track].items.push(ti); + } + let p = Arc::new(p); + let s = filmcraft_engine::Session::default(); + let server = FrameServer::new(s.media.clone(), s.services.clone(), s.previews.clone(), 2); + let key = |caps| FrameKey { target: Target::SequencePlan(seq, caps), frame: 0, size: 1000, revision: 0, draft: false }; + for caps in [DeviceCaps::NONE, DeviceCaps::ALL] { + server.request(key(caps), Tick::ZERO, 1.0, &p, 0); + } + let t0 = Instant::now(); + while !(server.is_ready(&key(DeviceCaps::NONE)) && server.is_ready(&key(DeviceCaps::ALL))) && t0.elapsed() < Duration::from_secs(30) { + std::thread::sleep(Duration::from_millis(5)); + } + let none = server.get_plan(&key(DeviceCaps::NONE)).expect("the plan for a device without an effect stage"); + assert!(matches!(none.plan, FramePlan::Image(_)), "no effect stage: the worker renders the frame on the CPU"); + let all = server.get_plan(&key(DeviceCaps::ALL)).expect("the plan for a device with an effect stage"); + assert!( + matches!(&all.plan, FramePlan::Composite { steps, .. } if steps.iter().any(|st| matches!(st, PlanStep::Adjust { .. }))), + "an effect stage: the adjustment layer is a step of the plan" + ); + } } diff --git a/crates/ui-egui/src/panels/monitor.rs b/crates/ui-egui/src/panels/monitor.rs index 6a4a803f7..e3de8677d 100644 --- a/crates/ui-egui/src/panels/monitor.rs +++ b/crates/ui-egui/src/panels/monitor.rs @@ -73,9 +73,9 @@ fn draw_picture(app: &mut FilmcraftApp, ui: &mut egui::Ui, which: Which, mv: &cr // render previews are opaque, so the grid and Display Mode ▸ Alpha take live renders let live = grid || display == DisplayMode::Alpha; let cpu_target = target; - let target = match (use_gpu, target) { - (true, Target::Sequence(s)) => Target::SequencePlan(s), - (false, Target::Sequence(s)) if live => Target::SequenceLive(s), + let target = match (app.gpu.as_ref().filter(|_| use_gpu), target) { + (Some(g), Target::Sequence(s)) => Target::SequencePlan(s, g.compositor.caps()), + (None, Target::Sequence(s)) if live => Target::SequenceLive(s), (_, t) => t, }; // Draft decoding (opt-in): only while playing at 1/2 resolution or lower. diff --git a/crates/ui-egui/src/perf.rs b/crates/ui-egui/src/perf.rs index ce10ce8ef..91689bbb1 100644 --- a/crates/ui-egui/src/perf.rs +++ b/crates/ui-egui/src/perf.rs @@ -36,6 +36,8 @@ pub fn stats(app: &FilmcraftApp) -> Value { "uploadedBytes": g.compositor.uploaded_bytes, "transitions": g.compositor.gpu_transitions, "cpuTransitions": g.compositor.cpu_transitions, + "adjustments": g.compositor.gpu_adjustments, + "cpuAdjustments": g.compositor.cpu_adjustments, "submitMs": g.last_ms, }) }); diff --git a/docs/architecture.md b/docs/architecture.md index 896e11e80..2213e1686 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -321,7 +321,14 @@ file ──► codecs (MP4/MOV, MKV, audio) demux + decode, GOP-aware see the effects' parameters evaluated at that time: the source is drawn into an `Rgba32Float` working image at the size the CPU decodes it, each effect runs as compute passes with the CPU reference's math (effects, then Motion, then Opacity / blend, as on the CPU), and the result is - placed like any layer. Other standard effects, effect and opacity masks, adjustment layers, + placed like any layer. An adjustment layer that covers the whole frame at 100 % opacity in + Normal, unmasked and with only effects from that set, is a step of the plan: its effects run the + same way on the composite of the tracks below it, and the result replaces it before the tracks + above are drawn. The plan is made for the device that composites it (`plan::DeviceCaps`, part + of the monitor's frame key and of the export's renderer pool): on one without an effect stage, + or for a frame larger than its textures, the adjustment layer sends the frame to the CPU + renderer on the thread that plans it, not on the UI thread or inside a renderer. Other standard + effects, effect and opacity masks, the other adjustment layers, nested sequences (except a plain one, whose own layers go into the plan: same frame size and colour settings as its parent, nothing on the clip that changes the picture, every layer inside blended normally) and non-dissolve transitions are rendered on the CPU for that layer or frame diff --git a/docs/hardware-parity.md b/docs/hardware-parity.md index 7ab82a1aa..2856b2a68 100644 --- a/docs/hardware-parity.md +++ b/docs/hardware-parity.md @@ -1,6 +1,6 @@ # Hardware parity -> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-10 · **Change:** major (new file: hardware decode / encode per platform and codec, GPU effects, devices; re-measured after NVDEC #549 and GPU export #421) · **Target:** Adobe Premiere Pro 2026 (26.5.2) +> **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-23 · **Change:** minor (adjustment layers on the GPU compositor) · **Target:** Adobe Premiere Pro 2026 (26.5.2) Each hardware feature Premiere uses, per platform, against ours. Implementation detail lives in [`crates/platform/README.md`](../crates/platform/README.md) (decode / encode backends, the one @@ -53,6 +53,7 @@ against 793 s with our software encoder (SSIM 0.990). NVENC HEVC round trip 48.9 | Accelerated video effects | ~82 of ~92 effects marked Accelerated (reference list `plan/premiere/10-effects-list.md`) | **35 effect ids** (`render::gpufx::GPU_EFFECTS`): colour, levels, blurs (Bokeh Blur included), sharpen, crop, transform, flips, vignette, video limiter, ASC CDL, channel mix, Lumetri basic / creative / vignette | ~40% | | Lumetri curves, wheels, HSL secondary | GPU | CPU | 0% | | Keys (Ultra Key), masks, distortions, transitions | GPU | CPU (masks have WGSL parity tests, not in the live path) | 10% | +| Adjustment layers | GPU when every effect on them is accelerated | **GPU** for a full-frame one at 100 % opacity, Normal, unmasked, with only GPU effects (`PlanStep::Adjust`): 4.0–5.0× faster GPU exports under one; any other one renders the whole frame on the CPU | 60% | | Export rendering on the GPU | yes | **yes** (#421; Export ▸ GPU Rendering, Auto): 2.4× (1080p) to 3.2× (4K) faster on effects-heavy timelines | 80% | | Thumbnails on the GPU (26.0) | yes | CPU | 0% | | Software adapter fallback | warns | silently renders everything on the CPU (#632) | — | @@ -84,4 +85,5 @@ one encoder ~4–8 h ([target-app-parity.md](target-app-parity.md#calibration)). | Date | Change | Summary | |---|---|---| +| 2026-10-23 | minor | Adjustment layers row: full-frame, opaque, Normal ones with GPU effects run on the GPU compositor | | 2026-10-10 | major | Created from ROADMAP.md's performance-and-hardware row and `crates/platform/README.md`; added Linux NVDEC (#549), GPU export and Lumetri on the GPU (#421), Premiere's per-platform hardware list, devices and I/O | diff --git a/docs/performance.md b/docs/performance.md index aad506817..4b60db233 100644 --- a/docs/performance.md +++ b/docs/performance.md @@ -254,6 +254,28 @@ of a batch (a render-preview segment, a proxy), which the next part reuses. A pr 332 MB idle and at **258 MB** with the trim. The memory does not come back at once: macOS returns freed pages over a few seconds (892 MB right after the export in both cases, 726 MB after 2 s). +## Results (adjustment layers on the GPU compositor, main → after) + +An adjustment layer anywhere at a time sent the whole frame to the CPU renderer (`plan::layered_at`), +so Export ▸ GPU Rendering did nothing for the frames under one. A full-frame adjustment layer at +100 % opacity, Normal, unmasked and with only GPU effects is now a `PlanStep::Adjust`: its effects +run in the GPU effect stage on the composite of the tracks below. Windows 11 Pro 26300, Xeon +E5-2680 v4 (14 cores / 28 threads), RTX 5060 8 GB (driver 617.14), 2026-10-23, release builds of +main 20f02b9a and of the change with `cargo build --release -p filmcraft -p filmcraft-cli`, two +alternating rounds, `filmcraft-cli … run` of `file.exportMedia` (H.264, `gpuRendering` and +`hardwareEncoding` Auto, NVENC), 300 frames of a 1920×1080 30 fps sequence, seconds: + +| scene | main | **after** | +|---|---|---| +| the CityNight demo footage under an adjustment layer (Lumetri: contrast, exposure, vignette; Gaussian Blur 4) | 35.5, 35.8 | **9.4, 8.6** | +| a real edit: 1080p H.264 camera clips with dips to black, an adjustment layer with Lumetri over 89 % of it, about 150 titles and shapes above | 26.1, 24.4 | **5.2, 5.0** | + +4.0× and 5.0× faster (means of the two rounds). GPU 3D utilisation (`nvidia-smi dmon`) went from +1–4 % to 18–31 %. The two builds' files decode to the same picture within encoder noise (ffmpeg `psnr` +between them: average 61.4 / 54.0 dB, lowest frame 58.8 / 52.6 dB). The whole 82 s real edit (2,472 +frames) exports in 43.7 s; with the adjustment layer the render stage is still most of that (decoded +pictures and the composite still go through the CPU, and titles are rasterised there). + ## Results (GPU2: standard effects on the GPU compositor, #30, before → after) Before = this change with clips that carry standard effects sent back to the CPU layer path in