diff --git a/crates/gpu/src/context.rs b/crates/gpu/src/context.rs index fb7edc812..e2ab46bde 100644 --- a/crates/gpu/src/context.rs +++ b/crates/gpu/src/context.rs @@ -128,6 +128,11 @@ const POOL_BUDGET: usize = 256 << 20; /// miss (a new allocation), never a wrong pixel. const STALE_FRAMES: u64 = 2; +/// The pool owner of uploaded textures: never registered by an [`Enc`] (their ids start at 1), +/// so an upload takes only from the shared free list ([`Pool::take`], textures no open encoder +/// can read) and gives its texture back through the guarded [`Pool::release`]. +const UPLOAD_OWNER: u64 = 0; + /// Readback staging buffers kept for reuse across sizes (bytes). const STAGING_BUDGET: u64 = 256 << 20; @@ -693,6 +698,10 @@ pub struct GpuContext { dummy_tex: wgpu::TextureView, dummy_buf: wgpu::Buffer, uploads: Mutex, + /// Uploaded footage pictures, shared by the layers that show them (see [`crate::upload`]). + pictures: Mutex, + /// Reused staging buffers of large uploads (see [`crate::upload::Staging`]). + upload_staging: crate::upload::Staging, upload_budget: std::sync::atomic::AtomicUsize, upload_full_warned: std::sync::atomic::AtomicBool, pool: Arc>, @@ -1017,6 +1026,8 @@ impl GpuContext { dummy, dummy_buf, uploads: Mutex::new(Uploads::default()), + pictures: Default::default(), + upload_staging: Default::default(), upload_budget: std::sync::atomic::AtomicUsize::new(UPLOAD_BUDGET), upload_full_warned: Default::default(), pool, @@ -1132,11 +1143,11 @@ impl GpuContext { /// Bytes held in long-lived or still-referenced GPU caches/leases that submitting and waiting /// on `owner` (`closing_enc`) cannot free. fn kept_bytes(&self, owner: u64, closing_enc: Option) -> u64 { - let uploads = self.uploads.lock().map(|u| u.bytes as u64).unwrap_or(0); + // (uploaded layer and footage textures are pooled: the pool's leases count them) let pool = self.pool.lock().map(|p| p.kept_after_flush(owner, closing_enc) as u64).unwrap_or(0); let staging: u64 = self.staging.lock().map(|s| s.iter().map(|(k, v)| k * v.len() as u64).sum()).unwrap_or(0); let zeros: u64 = self.zeros.lock().map(|z| z.keys().map(|&(w, h)| w as u64 * h as u64 * 16).sum()).unwrap_or(0); - uploads + pool + staging + zeros + pool + staging + self.upload_staging.bytes() + zeros } #[cfg(test)] @@ -1549,6 +1560,8 @@ impl GpuContext { dummy: self.dummy.clone(), dummy_buf: self.dummy_buf.clone(), uploads: Mutex::new(Uploads::default()), + pictures: Default::default(), + upload_staging: Default::default(), upload_budget: std::sync::atomic::AtomicUsize::new(UPLOAD_BUDGET), upload_full_warned: Default::default(), pool: self.pool.clone(), @@ -1728,16 +1741,54 @@ impl GpuContext { return None; } self.transfers.count(true, img.data.len() * 16); - let g = self.image(img.width, img.height); - self.queue.write_texture( - wgpu::TexelCopyTextureInfo { texture: &g.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, - bytemuck::cast_slice(&img.data), - wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(img.width * 16), rows_per_image: Some(img.height) }, - wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 }, - ); + // A texture from the pool's shared free list: no open encoder can still read it, and the + // copy below runs before anything recorded after this call. + let (g, _) = self.pooled(img.width, img.height, UPLOAD_OWNER); + if !self.upload_staging.write(&self.device, &self.queue, img, &g.texture) { + self.queue.write_texture( + wgpu::TexelCopyTextureInfo { texture: &g.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + bytemuck::cast_slice(&img.data), + wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(img.width * 16), rows_per_image: Some(img.height) }, + wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 }, + ); + } Some(g) } + /// Upload a decoded footage picture once while the footage source keeps it: every layer + /// and effect input showing the same frame shares the texture (`Renderer::footage_picture`). + pub(crate) fn upload_picture(&self, img: &Arc) -> Option { + if let Some(t) = self.pictures.lock().ok().and_then(|mut p| p.get(img)) { + return Some(t); + } + let tex = self.upload_image(img)?; + if let Ok(mut p) = self.pictures.lock() { + p.insert(img, tex.clone(), crate::upload::PICTURE_BUDGET.min(self.upload_budget() / 4)); + } + Some(tex) + } + + /// Entries in the footage picture cache. + #[cfg(test)] + pub(crate) fn picture_entries(&self) -> usize { + self.pictures.lock().map(|p| p.len()).unwrap_or_default() + } + + /// The upload staging buffers (tests). + #[cfg(test)] + pub(crate) fn upload_staging(&self) -> &crate::upload::Staging { + &self.upload_staging + } + + /// A render is done: drop the footage picture textures and the upload staging buffers + /// (the next render's pictures upload again; their textures come from the pool). + pub fn trim_uploads(&self) { + if let Ok(mut p) = self.pictures.lock() { + p.clear(); + } + self.upload_staging.clear(); + } + /// Entries in the layer upload cache. #[cfg(test)] pub(crate) fn upload_entries(&self) -> usize { @@ -1799,6 +1850,10 @@ impl GpuContext { u.map.clear(); u.bytes = 0; } + if let Ok(mut p) = self.pictures.lock() { + p.clear(); + } + self.upload_staging.clear(); if let Ok(mut p) = self.pool.lock() { p.clear(); } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 3ede5c6b9..db6061487 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -84,6 +84,7 @@ mod memcap; mod ops; mod particles; mod readback; +mod upload; mod walk; mod yuv; @@ -439,6 +440,7 @@ impl Accelerator for Gpu { target: "effectcraft_gpu", "gpu pool: {hits} working textures reused, {misses} allocated ({pct}% reuse) so far" ); + self.ctx.trim_uploads(); self.ctx.trim_pool(); } diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index 37fcd42d3..9ab9fd49d 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -28,8 +28,10 @@ use crate::Gpu; mod adjust; mod adv3d; +mod bench_share; mod deferred; mod deferred_jobs; +mod footage_share; mod fx_color; mod fx_depth; mod fx_distort; diff --git a/crates/gpu/src/tests/bench_share.rs b/crates/gpu/src/tests/bench_share.rs new file mode 100644 index 000000000..580c7f4e6 --- /dev/null +++ b/crates/gpu/src/tests/bench_share.rs @@ -0,0 +1,83 @@ +//! Warm-frame benchmark of #3333 (run by hand): three layers of the same 1080p footage frame, +//! 32 bpc, a 30-frame sequence already decoded (as the media pool serves a warm frame), with a +//! layer cache as the Render Queue has. Prints the median and mean ms per frame and a hash of +//! every frame (to compare the pixels of two builds). +//! +//! `cargo test -p effectcraft-gpu --release --lib bench_share -- --ignored --nocapture` + +use std::collections::HashMap; +use std::sync::Mutex; + +use super::*; + +struct Decoded { + frames: Mutex>>, +} + +impl FootageSource for Decoded { + fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { + let i = f.frame_rate.frame_at(t).clamp(0, 29); + let mut m = self.frames.lock().unwrap(); + Some(m.entry(i).or_insert_with(|| Arc::new(pattern(item.0 as u32 + i as u32, f.width, f.height))).clone()) + } +} + +#[test] +#[ignore] +fn bench_share() { + let Some(g) = gpu() else { return }; + let (w, h) = (1920, 1080); + let mut p = Project::default(); + p.settings.bit_depth = BitDepth::Bpc32; + let comp = Comp::new(w, h, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let f = Footage { + path: "seq.exr".into(), + kind: FootageKind::Sequence, + width: w, + height: h, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(1.0), + has_video: true, + alpha: AlphaMode::Premultiplied, + ..Default::default() + }; + let fid = p.add_item("Sequence", Label::Lavender, None, ItemKind::Footage(f)); + for i in 0..3 { + let mut l = build::layer(&mut p, &comp, "Sequence", LayerSource::Footage { item: fid }, (w, h), None); + if i > 0 { + l.blend_mode = BlendMode::Add; + set(&mut l, "transform/opacity", Value::Scalar(25.0)); + } + p.comp_mut(cid).unwrap().layers.insert(0, l); + } + let src = Decoded { frames: Mutex::new(HashMap::new()) }; + let frames = 30; + let mut runs = vec![]; + let mut hashes = vec![]; + for pass in 0..4 { + let cache = effectcraft_render::cache::LayerCache::new(1 << 30); + let mut r = Renderer::new(&p, &src, RenderOpts { backend: Backend::Gpu, ..Default::default() }); + r.accel = Some(g); + r.cache = Some(&cache); + for k in 0..frames { + let t0 = std::time::Instant::now(); + let img = g.render(&r, cid, Tick::from_seconds_f64(k as f64 / 30.0)).expect("the GPU renders the frame"); + let ms = t0.elapsed().as_secs_f64() * 1e3; + if pass > 0 { + runs.push(ms); + } + if pass == 1 { + use std::hash::{Hash, Hasher}; + let mut hs = std::collections::hash_map::DefaultHasher::new(); + img.data.iter().for_each(|px| px.map(f32::to_bits).hash(&mut hs)); + hashes.push(hs.finish()); + } + } + } + runs.sort_by(f64::total_cmp); + let mean = runs.iter().sum::() / runs.len() as f64; + eprintln!("bench_share: {} frames, median {:.1} ms, mean {:.1} ms, min {:.1} ms", runs.len(), runs[runs.len() / 2], mean, runs[0]); + eprintln!("bench_share frame hashes: {hashes:016x?}"); +} diff --git a/crates/gpu/src/tests/footage_share.rs b/crates/gpu/src/tests/footage_share.rs new file mode 100644 index 000000000..1a31c3981 --- /dev/null +++ b/crates/gpu/src/tests/footage_share.rs @@ -0,0 +1,233 @@ +//! #3333: a footage frame uploads once per frame and its texture is shared by every layer and +//! effect input that shows it; the frame is bit-identical to drawing each layer from its own copy. + +use std::collections::HashMap; +use std::sync::Mutex; + +use super::*; +use crate::walk::SHARED_PICTURES; + +/// Keeps decoded frames as the media pool does: the same frame (at the same size) of the same +/// footage is the same `Arc` while kept. Counts the frames it decodes. +#[derive(Default)] +struct Kept { + frames: Mutex), Arc>>, +} + +impl Kept { + fn get(&self, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option> { + let i = f.frame_rate.frame_at(t); + let mut m = self.frames.lock().unwrap(); + let img = m.entry((item, f.width, f.height, i, size)).or_insert_with(|| { + let full = pattern(item.0 as u32 + i as u32 * 7 + f.width, f.width, f.height); + Arc::new(match size { + Some((w, h)) => effectcraft_raster::resample(&full, w, h), + None => full, + }) + }); + Some(img.clone()) + } +} + +impl FootageSource for Kept { + fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { + self.get(item, f, t, None) + } + fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, width: u32, height: u32) -> Option> { + self.get(item, f, t, Some((width, height))) + } +} + +/// One footage item (a sequence: a new picture every frame) shown by `layers` layers, each +/// placed differently (the first fills the comp). +fn shared_scene(depth: BitDepth, layers: usize) -> (Scene, ItemId) { + let mut s = Scene::new(depth); + let f = Footage { + path: "seq.exr".into(), + kind: FootageKind::Sequence, + width: 97, + height: 61, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + duration: Tick::from_seconds_f64(2.0), + has_video: true, + alpha: AlphaMode::Premultiplied, + ..Default::default() + }; + let fid = s.p.add_item("Sequence", Label::Lavender, None, ItemKind::Footage(f)); + for i in 0..layers { + let mut l = build::layer(&mut s.p, &s.comp, "Sequence", LayerSource::Footage { item: fid }, (97, 61), None); + if i > 0 { + set(&mut l, "transform/rotation", Value::Scalar(11.0 * i as f64)); + set(&mut l, "transform/scale", Value::Vec3([70.0 + 9.0 * i as f64, 64.0, 100.0])); + set(&mut l, "transform/position", v3(30.0 + 13.0 * i as f64, 25.0 + 4.0 * i as f64)); + set(&mut l, "transform/opacity", Value::Scalar(80.0)); + l.blend_mode = if i % 2 == 0 { BlendMode::Screen } else { BlendMode::Add }; + } + s.push(l); + } + (s, fid) +} + +/// The GPU frame with shared pictures on and off, and the number of uploads of each. +fn render_both(g: &crate::Gpu, s: &Scene, src: &Kept, opts: RenderOpts, t: Tick) -> (Image, Image, u64, u64) { + let mut r = Renderer::new(&s.p, src, RenderOpts { backend: Backend::Gpu, ..opts }); + r.accel = Some(g); + let before = g.ctx.transfer_stats().uploads; + let on = g.render(&r, s.cid, t).expect("the GPU renders the frame"); + let uploads = g.ctx.transfer_stats().uploads - before; + SHARED_PICTURES.store(false, std::sync::atomic::Ordering::Relaxed); + let before = g.ctx.transfer_stats().uploads; + let off = g.render(&r, s.cid, t); + let copied = g.ctx.transfer_stats().uploads - before; + SHARED_PICTURES.store(true, std::sync::atomic::Ordering::Relaxed); + (on, off.expect("the GPU renders the frame from per-layer copies"), uploads, copied) +} + +fn check_cpu(label: &str, s: &Scene, src: &Kept, opts: RenderOpts, t: Tick, gpu_img: &Image) { + let cpu = Renderer::new(&s.p, src, RenderOpts { backend: Backend::Cpu, ..opts }).comp_frame_cpu(s.cid, t); + let mut d = diff(&cpu, gpu_img, tolerance(s.p.settings.bit_depth)); + d.quantized = s.p.settings.bit_depth != BitDepth::Bpc32; + check(label, Some(d), 0.0); +} + +/// Three layers showing the same frame upload it once; a fourth that runs an effect on it +/// (resident on the GPU) reads the same texture as its input. Every frame of the sequence +/// uploads its own picture once. +#[test] +fn three_layers_of_one_footage_frame_upload_once() { + let Some(g) = gpu() else { return }; + let src = Kept::default(); + let (mut s, fid) = shared_scene(BitDepth::Bpc32, 3); + for k in 0..3 { + let t = Tick::from_seconds_f64(k as f64 / 30.0); + let (on, off, uploads, copied) = render_both(g, &s, &src, opts(), t); + assert_eq!(uploads, 1, "frame {k}: three layers of one footage frame uploaded {uploads} times"); + assert_eq!(copied, 3, "frame {k}: per-layer copies upload once per layer"); + assert!(on.data == off.data, "frame {k}: the shared picture changed the frame"); + check_cpu(&format!("shared picture frame {k}"), &s, &src, opts(), t, &on); + } + // An effect input: the blur reads the shared texture. Alone, the blurred layer uploads + // its picture (and whatever the blur itself uploads); the three plain layers showing the + // same frame add no upload. + let mut fx = build::layer(&mut s.p, &s.comp, "Blurred", LayerSource::Footage { item: fid }, (97, 61), None); + set(&mut fx, "transform/position", v3(20.0, 20.0)); + s.effect(&mut fx, "ec.blur.gaussian", &[("blurriness", n(3.0))]); + let (mut alone, _) = shared_scene(BitDepth::Bpc32, 0); + alone.p = s.p.clone(); + alone.p.comp_mut(alone.cid).unwrap().layers = vec![fx.clone()].into(); + let (_, _, blur_uploads, _) = render_both(g, &alone, &src, opts(), Tick::from_seconds_f64(0.4)); + s.push(fx); + let t = Tick::from_seconds_f64(0.5); + let (on, off, uploads, _) = render_both(g, &s, &src, opts(), t); + assert_eq!(uploads, blur_uploads, "three layers and an effect input of one footage frame uploaded {uploads} times, the effect layer alone {blur_uploads}"); + assert!(on.data == off.data, "the shared picture changed the frame with an effect input"); + check_cpu("shared picture with an effect input", &s, &src, opts(), t, &on); +} + +/// The frame is bit-identical with shared pictures on and off wherever they apply or not: every +/// bit depth, colour management and linear blending, half resolution, a track matte, Preserve +/// Transparency, motion blur, masks, layer styles and a proxy. +#[test] +fn shared_pictures_do_not_change_the_frame() { + let Some(g) = gpu() else { return }; + let src = Kept::default(); + let t = Tick::from_seconds_f64(0.2); + for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] { + let mut cases: Vec<(String, Scene, RenderOpts)> = vec![]; + cases.push(("plain".into(), shared_scene(depth, 3).0, opts())); + cases.push(("half resolution".into(), shared_scene(depth, 3).0, RenderOpts { scale: 0.5, ..opts() })); + for (ws, blend_linear) in [(Some(ColorSpace::Rec709), false), (None, true)] { + let (mut s, _) = shared_scene(depth, 3); + s.p.settings.working_space = ws; + s.p.settings.blend_linear = blend_linear; + cases.push((format!("colour {ws:?} linear blending {blend_linear}"), s, opts())); + } + { + let (mut s, _) = shared_scene(depth, 3); + let comp = s.p.comp_mut(s.cid).unwrap(); + comp.enable_motion_blur = true; + let matte = comp.layers[0].id; + comp.layers[1].track_matte = Some(TrackMatte { layer: matte, kind: MatteKind::Luma }); + comp.layers[2].preserve_transparency = true; + comp.layers[2].switches.motion_blur = true; + let pos = comp.layers[2].props.prop_mut("transform/position").unwrap(); + pos.keys = vec![Keyframe::new(Tick::ZERO, v3(10.0, 20.0)), Keyframe::new(Tick::from_seconds_f64(0.5), v3(90.0, 45.0))]; + s.comp.enable_motion_blur = true; + cases.push(("matte, preserve transparency, motion blur".into(), s, opts())); + } + { + let (mut s, _) = shared_scene(depth, 3); + let mut next = s.p.next_id; + let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::ellipse([40.0, 30.0], 60.0, 40.0), MaskMode::Add, [255, 255, 0]); + let comp = s.p.comp_mut(s.cid).unwrap(); + comp.layers[0].props.sub_mut("masks").unwrap().children.push(m.into()); + effectcraft_project::styles::add_style(&mut comp.layers[1], &mut Ids(&mut next), "dropShadow", &s.comp.global_light, true).unwrap(); + s.p.next_id = next; + cases.push(("mask and layer style".into(), s, opts())); + } + { + let (mut s, fid) = shared_scene(depth, 3); + let proxy = Footage { path: "proxy.png".into(), kind: FootageKind::Still, width: 48, height: 30, has_video: true, ..Default::default() }; + s.p.item_mut(fid).unwrap().proxy = Some(Box::new(effectcraft_project::Proxy { footage: proxy, enabled: true })); + cases.push(("proxy".into(), s, opts())); + } + for (label, s, o) in cases { + let (on, off, _, _) = render_both(g, &s, &src, o, t); + assert!(on.data == off.data, "{label} {depth:?}: the shared picture changed the frame"); + check_cpu(&format!("{label} {depth:?}"), &s, &src, o, t, &on); + } + } +} + +/// The picture cache keeps no texture of a picture the footage source dropped. +#[test] +fn picture_cache_drops_textures_of_dropped_pictures() { + let Some(g) = gpu() else { return }; + let before = g.ctx.picture_entries(); + for i in 0..6 { + let img = Arc::new(pattern(i, 64, 64)); + let up = g.ctx.upload_picture(&img).expect("upload"); + let again = g.ctx.upload_picture(&img).expect("upload"); + assert!(up.texture == again.texture, "the same picture uploaded twice"); + drop((up, again, img)); + } + let kept = g.ctx.picture_entries(); + assert!(kept <= before + 1, "the picture cache kept {kept} textures of dropped pictures (had {before})"); +} + +/// Large uploads copy through reused staging buffers (rows padded to the copy alignment) with +/// the same pixels as `Queue::write_texture`; the buffers stay within their budget, and a +/// clear while one is in flight forgets it. +#[test] +fn staging_uploads_keep_pixels_and_reuse_buffers() { + let Some(g) = gpu() else { return }; + let wait = || { + let _ = g.ctx.poll_readbacks(wgpu::PollType::wait_indefinitely()); + }; + // 300 px: 4800-byte rows, padded to 4864. + for (w, h) in [(300u32, 300u32), (512, 600)] { + let before = g.ctx.upload_staging().copies(); + for i in 0..4 { + let img = pattern(40 + i, w, h); + let up = g.ctx.upload_image(&img).expect("upload"); + let mut e = crate::context::Enc::new(&g.ctx); + let back = e.download(&up).expect("readback"); + assert!(back.data.iter().flatten().map(|v| v.to_bits()).eq(img.data.iter().flatten().map(|v| v.to_bits())), "{w}×{h} upload {i} changed pixels"); + drop((up, e)); + wait(); + } + assert!(g.ctx.upload_staging().copies() - before >= 2, "{w}×{h}: no upload went through a reused staging buffer"); + assert!(g.ctx.upload_staging().bytes() <= 192 << 20); + } + // A clear with a copy in flight: nothing kept, and later uploads still work. + let img = pattern(7, 300, 300); + let up = g.ctx.upload_image(&img).expect("upload"); + g.ctx.upload_staging().clear(); + wait(); + assert_eq!(g.ctx.upload_staging().bytes(), 0, "a cleared staging kept its buffers"); + let mut e = crate::context::Enc::new(&g.ctx); + assert!(e.download(&up).expect("readback").data == img.data); + let again = g.ctx.upload_image(&img).expect("upload after a clear"); + assert!(e.download(&again).expect("readback").data == img.data); +} diff --git a/crates/gpu/src/upload.rs b/crates/gpu/src/upload.rs new file mode 100644 index 000000000..9825fb335 --- /dev/null +++ b/crates/gpu/src/upload.rs @@ -0,0 +1,264 @@ +//! Footage uploads (#3333): each decoded footage picture uploads once and its texture is shared +//! by every layer and effect input that shows it ([`Pictures`]), and large uploads copy through +//! staging buffers that are kept mapped and reused ([`Staging`]) instead of one that +//! `Queue::write_texture` allocates and frees per upload. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, PoisonError, Weak}; + +use effectcraft_raster::Image; + +use crate::context::GpuImage; + +/// Bytes of picture textures kept after their frame (a sequence's next frame takes the oldest +/// one's texture back from the pool), at most a quarter of the context's upload budget. The +/// newest picture is always kept, whatever its size. +pub(crate) const PICTURE_BUDGET: usize = 256 << 20; + +/// Uploads from this size go through [`Staging`]; smaller ones through `Queue::write_texture`. +const STAGING_MIN: u64 = 1 << 20; + +/// Bytes of staging buffers that may exist at once (mapped and ready, or in flight). +const STAGING_BUDGET: u64 = 192 << 20; + +struct Picture { + img: Weak, + tex: GpuImage, + bytes: usize, + last_use: u64, +} + +/// Uploaded footage pictures by the identity of the decoded frame (its `Arc`): the footage +/// source serves one `Arc` per (file, frame, size, colour curve) while it keeps the frame, and +/// a new one when the file changed, so the same `Arc` means the same pixels. The `Weak` keeps +/// the allocation, so a key cannot be reused by another picture while its entry exists. +#[derive(Default)] +pub(crate) struct Pictures { + map: HashMap, + bytes: usize, + clock: u64, +} + +impl Pictures { + /// The texture of `img` if it was uploaded and is still kept. + pub(crate) fn get(&mut self, img: &Arc) -> Option { + self.clock += 1; + let now = self.clock; + let e = self.map.get_mut(&(Arc::as_ptr(img) as usize))?; + if !e.img.upgrade().is_some_and(|i| Arc::ptr_eq(&i, img)) { + return None; + } + e.last_use = now; + Some(e.tex.clone()) + } + + /// Keep `tex`, the upload of `img`. Pictures the footage source no longer holds go at once + /// (nothing can ask for them again), then the least recently used over [`PICTURE_BUDGET`], + /// never `img` itself. A dropped texture goes back to the pool once no open encoder can + /// still read it. + pub(crate) fn insert(&mut self, img: &Arc, tex: GpuImage, budget: usize) { + let key = Arc::as_ptr(img) as usize; + let bytes = img.data.len().saturating_mul(16); + self.clock += 1; + let entry = Picture { img: Arc::downgrade(img), tex, bytes, last_use: self.clock }; + if let Some(old) = self.map.insert(key, entry) { + self.bytes = self.bytes.saturating_sub(old.bytes); + } + self.bytes = self.bytes.saturating_add(bytes); + let dead: Vec = self.map.iter().filter(|(_, e)| e.img.strong_count() == 0).map(|(k, _)| *k).collect(); + for k in dead { + if let Some(e) = self.map.remove(&k) { + self.bytes = self.bytes.saturating_sub(e.bytes); + } + } + while self.bytes > budget { + let Some(k) = self.map.iter().filter(|(k, _)| **k != key).min_by_key(|(_, e)| e.last_use).map(|(k, _)| *k) else { break }; + if let Some(e) = self.map.remove(&k) { + self.bytes = self.bytes.saturating_sub(e.bytes); + } + } + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.map.len() + } + + pub(crate) fn clear(&mut self) { + self.map.clear(); + self.bytes = 0; + } +} + +#[derive(Default)] +struct StagingState { + /// Buffers mapped for writing and not in use, each mapped whole. + ready: Vec, + /// Bytes of every staging buffer that exists (ready, being mapped or in flight). + total: u64, + /// Raised by [`Staging::clear`]: a buffer of an earlier generation is dropped when its + /// mapping completes. + generation: u64, + /// Uploads copied through a staging buffer (diagnostics, tests). + copies: u64, +} + +/// Reused upload staging buffers (`MAP_WRITE | COPY_SRC`). A buffer is written while mapped, +/// unmapped, copied into the texture by a command buffer submitted at once, and mapped again +/// when that submission is done (`CommandEncoder::map_buffer_on_submit`; the callback runs when +/// the device is polled, as every frame does). A new buffer is mapped asynchronously too (never +/// `mapped_at_creation`, which fails hard on drivers that report a failed allocation as an +/// internal error, #591), so the upload that made it, and any upload while no ready buffer fits +/// or the budget is spent, goes through `Queue::write_texture` instead. +#[derive(Default)] +pub(crate) struct Staging { + state: Arc>, +} + +/// `bytes_per_row` of a staging copy: rows padded to `COPY_BYTES_PER_ROW_ALIGNMENT`. +fn padded_row(width: u32) -> Option { + let row = u64::from(width).checked_mul(16)?; + let a = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT); + row.div_ceil(a).checked_mul(a) +} + +impl Staging { + fn lock(&self) -> std::sync::MutexGuard<'_, StagingState> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Bytes of staging buffers that exist (see [`STAGING_BUDGET`]). + pub(crate) fn bytes(&self) -> u64 { + self.lock().total + } + + /// Uploads copied through a staging buffer so far. + #[cfg(test)] + pub(crate) fn copies(&self) -> u64 { + self.lock().copies + } + + /// Forget every staging buffer (after a failed frame, #1319): those in flight are dropped + /// when their mapping completes. + pub(crate) fn clear(&self) { + let mut s = self.lock(); + s.ready.clear(); + // (buffers still in flight are no longer counted: they drop when their mapping ends) + s.total = 0; + s.generation = s.generation.wrapping_add(1); + } + + /// The callback of a write mapping of `buffer`: once mapped it is ready for the next + /// upload; it is dropped (and its bytes uncounted) when mapping failed, and dropped when + /// the staging was cleared meanwhile (whose [`Staging::clear`] stopped counting it). + fn on_mapped(&self, buffer: &wgpu::Buffer, generation: u64) -> impl FnOnce(Result<(), wgpu::BufferAsyncError>) + wgpu::WasmNotSend + 'static { + let state = Arc::downgrade(&self.state); + let buffer = buffer.clone(); + move |res: Result<(), wgpu::BufferAsyncError>| { + let Some(state) = state.upgrade() else { return }; + let mut s = state.lock().unwrap_or_else(PoisonError::into_inner); + if s.generation != generation { + return; + } + if res.is_ok() { + s.ready.push(buffer); + } else { + s.total = s.total.saturating_sub(buffer.size()); + } + } + } + + /// Copy `img` into `texture` (its size) through a ready staging buffer, recorded and + /// submitted at once: before anything an open encoder records later, as + /// `Queue::write_texture` would. `false`: not done (small, no ready buffer fits, or the + /// mapping failed); the caller writes it through the queue. Starts mapping a new buffer for + /// later uploads when none fits and the budget allows. + pub(crate) fn write(&self, device: &wgpu::Device, queue: &wgpu::Queue, img: &Image, texture: &wgpu::Texture) -> bool { + let (w, h) = (img.width, img.height); + let row = w as usize * 16; + let Some(stride) = padded_row(w) else { return false }; + let (Some(need), Ok(stride32)) = (stride.checked_mul(u64::from(h)), u32::try_from(stride)) else { return false }; + if need < STAGING_MIN || img.data.len() != w as usize * h as usize { + return false; + } + let (buffer, generation) = { + let mut s = self.lock(); + let generation = s.generation; + // The smallest ready buffer that holds it. + let best = s.ready.iter().enumerate().filter(|(_, b)| b.size() >= need).min_by_key(|(_, b)| b.size()).map(|(i, _)| i); + match best { + Some(i) => (Some(s.ready.swap_remove(i)), generation), + None => { + // Ready buffers too small for it make room (a size that no longer occurs). + while s.total.saturating_add(need) > STAGING_BUDGET + && let Some(b) = s.ready.pop() + { + s.total = s.total.saturating_sub(b.size()); + } + if s.total.saturating_add(need) <= STAGING_BUDGET { + s.total += need; + drop(s); + let b = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("upload staging"), + size: need, + usage: wgpu::BufferUsages::MAP_WRITE | wgpu::BufferUsages::COPY_SRC, + mapped_at_creation: false, + }); + let done = self.on_mapped(&b, generation); + b.map_async(wgpu::MapMode::Write, .., done); + } + (None, generation) + } + } + }; + let Some(buffer) = buffer else { return false }; + self.lock().copies += 1; + { + let Ok(mut view) = buffer.get_mapped_range_mut(..need) else { + // Not mapped after all: give it up (its bytes too). + let mut s = self.lock(); + if s.generation == generation { + s.total = s.total.saturating_sub(buffer.size()); + } + return false; + }; + let src: &[u8] = bytemuck::cast_slice(&img.data); + let stride = stride as usize; + if stride == row { + view.slice(..src.len()).copy_from_slice(src); + } else { + for (y, line) in src.chunks_exact(row).enumerate() { + let at = y * stride; + view.slice(at..at + row).copy_from_slice(line); + } + } + } + buffer.unmap(); + let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("upload") }); + enc.copy_buffer_to_texture( + wgpu::TexelCopyBufferInfo { + buffer: &buffer, + layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(stride32), rows_per_image: Some(h) }, + }, + wgpu::TexelCopyTextureInfo { texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All }, + wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 }, + ); + let done = self.on_mapped(&buffer, generation); + enc.map_buffer_on_submit(&buffer, wgpu::MapMode::Write, .., done); + queue.submit([enc.finish()]); + true + } +} + +#[cfg(test)] +mod tests { + use super::padded_row; + + #[test] + fn staging_rows_are_padded_to_the_copy_alignment() { + assert_eq!(padded_row(1920), Some(30720)); + assert_eq!(padded_row(97), Some(1792)); + assert_eq!(padded_row(16), Some(256)); + assert_eq!(padded_row(1), Some(256)); + } +} diff --git a/crates/gpu/src/walk.rs b/crates/gpu/src/walk.rs index f688a6375..471c91339 100644 --- a/crates/gpu/src/walk.rs +++ b/crates/gpu/src/walk.rs @@ -241,6 +241,9 @@ fn draw_layer_on(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas if let Some(done) = draw_resident_fx(e, r, ctx, layer, canvas, opacity) { return done; } + if let Some(done) = draw_picture(e, r, ctx, layer, canvas, opacity) { + return done; + } let Some(st) = r.styled_layer(ctx, layer) else { return Some(()) }; if st.plain { return composite_layer(e, r, ctx, layer, &st.body, canvas, opacity); @@ -286,14 +289,28 @@ fn draw_resident_fx(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, can { return None; } - let input = r.layer_input(ctx, layer, 0)?; - if input.img.is_empty() { - return Some(Some(())); - } - let Some(src) = e.g.upload_buf(&input) else { return Some(None) }; + // The decoded footage picture itself when it is the layer's input (uploaded once per frame + // and shared with every layer that shows it), else the cached input buffer. + let (src, offset, scale) = match picture(r, ctx, layer) { + Some(p) => { + if p.img.is_empty() { + return Some(Some(())); + } + let Some(src) = e.g.upload_picture(&p.img) else { return Some(None) }; + (src, p.effect_offset, p.scale) + } + None => { + let input = r.layer_input(ctx, layer, 0)?; + if input.img.is_empty() { + return Some(Some(())); + } + let Some(src) = e.g.upload_buf(&input) else { return Some(None) }; + (src, input.offset, input.scale) + } + }; let pipe = r.pipe(); // `layer_input` is in effect space: run the stack there, then back to layer space. - let mut fx = crate::effects::GpuFx::new(e, src, input.offset, input.scale, pipe.levels); + let mut fx = crate::effects::GpuFx::new(e, src, offset, scale, pipe.levels); let origin = r.run_layer_effects_on(ctx, layer, &mut fx); let Some((mut img, mut offset, scale)) = fx.finish() else { return Some(None) }; offset = [offset[0] - origin[0] * scale, offset[1] - origin[1] * scale]; @@ -308,6 +325,52 @@ fn draw_resident_fx(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, can Some(place_src(e, r, ctx, layer, &shape, img, canvas, layer.blend_mode, opacity, pipe.levels).map(|c| *canvas = c)) } +/// Shared footage pictures ([`picture`]); tests turn it off to compare. +pub(crate) static SHARED_PICTURES: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true); + +/// The layer's source as the decoded footage picture (`Renderer::footage_picture`), unless +/// sharing is off or per-layer timings are being recorded (those time the layer-cache path). +fn picture(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> Option { + if !SHARED_PICTURES.load(std::sync::atomic::Ordering::Relaxed) || r.profile.is_some() { + return None; + } + r.footage_picture(ctx, layer) +} + +/// A footage layer without effects or layer styles whose source is the decoded picture as it +/// is (`Renderer::footage_picture`) and needs no conversion to the blending space: placed +/// straight from the picture's texture, uploaded once per frame and shared by every layer that +/// shows the same frame, instead of from a copy per layer in the layer cache, uploaded per +/// layer. The pixels are the same: the layer-cache buffer is that picture, copied. `None`: not +/// this kind of layer; `Some(None)`: the GPU failed. +fn draw_picture(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut GpuImage, opacity: f32) -> Option> { + if (layer.switches.effects && layer.effects().is_some_and(|f| !f.children.is_empty())) || styles::active(ctx, layer) || r.pipe().to_blend().is_some() { + return None; + } + let p = picture(r, ctx, layer)?; + if p.img.is_empty() { + return Some(Some(())); + } + let Some(src) = e.g.upload_picture(&p.img) else { return Some(None) }; + // The placement only reads the buffer's size, offset and scale. + let shape = Buf { img: Image { width: p.img.width, height: p.img.height, data: Vec::new() }, offset: [0.0; 2], scale: p.scale }; + Some(composite_src(e, r, ctx, layer, &shape, src, canvas, opacity)) +} + +/// [`composite_layer`] of a GPU image `src` shaped like `shape`. +#[allow(clippy::too_many_arguments)] +fn composite_src(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, shape: &Buf, src: GpuImage, canvas: &mut GpuImage, opacity: f32) -> Option<()> { + if opacity <= 0.0 { + return Some(()); + } + if r.track_matte(ctx, layer).is_none() && !layer.preserve_transparency { + *canvas = place_src(e, r, ctx, layer, shape, src, canvas, layer.blend_mode, opacity, r.pipe().levels)?; + return Some(()); + } + let iso = place_src(e, r, ctx, layer, shape, src, &e.zeros(canvas.width, canvas.height), BlendMode::Normal, 1.0, None)?; + composite_iso(e, r, ctx, layer, iso, canvas, opacity) +} + /// A 2D precomp layer, its nested comp rendered resident on the GPU (`walk::render` one level /// down) instead of by the CPU compositor and uploaded every frame (`Renderer::layer_source` /// renders nested comps on the CPU: depth > 0). Then the layer's effects (resident), its diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8a3d7b8ae..4bbd4e198 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -230,6 +230,11 @@ impl LayerCache { /// used. (A render that started before a drop may publish its buffer after the drop's /// purge, Reload Footage clearing the cache while the buffer was being written: the check /// here, not the order of the two, keeps it from being served.) + /// Whether a buffer is kept in memory under `key` (a peek: no hit, no miss, no disk read). + pub(crate) fn holds(&self, key: u64) -> bool { + self.kept(key, false).is_some() + } + fn kept(&self, key: u64, hit: bool) -> Option<(Arc, Arc)> { let found = { let mut g = self.inner.lock().ok()?; diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index e09794de2..0f3243e60 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -61,7 +61,7 @@ pub use layers::{Placement, StyledLayer}; pub(crate) use opts::Collapse3d; pub use opts::{LayerTiming, RenderOpts}; use opts::{MAX_THROUGH, Through}; -pub use pipeline::{Collapsed, MediaOverride, PropertyOverride}; +pub use pipeline::{Collapsed, FootagePicture, MediaOverride, PropertyOverride}; pub struct Renderer<'a> { pub project: &'a Project, diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index 6c79ba7bb..1d9efc795 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -19,6 +19,24 @@ pub(crate) struct EffectStack { pub origin: [f64; 2], } +/// A footage layer's source as the decoded picture itself (see [`Renderer::footage_picture`]): +/// its buffer in layer space is `img` at offset `[0, 0]` and `scale`; in effect space (what +/// [`Renderer::layer_input`] gives) at `effect_offset`. +#[derive(Clone, Debug)] +pub struct FootagePicture { + pub img: Arc, + pub scale: f64, + pub effect_offset: [f64; 2], +} + +/// How a decoded footage frame becomes a layer buffer ([`Renderer::footage_buf`]). +enum FootageLayout { + /// Its own pixels, at this buffer scale. + Native { scale: f64 }, + /// Resampled to `width`×`height` at this buffer scale. + Resample { width: u32, height: u32, scale: f64 }, +} + /// A property value an Essential Properties override gives a layer of a collapsed precomp. #[derive(Clone, Debug)] pub struct PropertyOverride { @@ -861,12 +879,7 @@ impl<'a> Renderer<'a> { return Some(buf); } } - let lt = ctx.source_time(layer); - // The proxy is decoded instead, at its own size, and fills the footage's frame. - let proxy = self.proxy_for(item); - let pf = proxy.unwrap_or(f); - let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none()); - let img = self.footage_frame(ctx, layer, item, pf, lt, size)?; + let (img, pf) = self.footage_picture_of(ctx, layer, item, f)?; Some(self.footage_buf(&img, pf, f.width, f.height)) } // (the font files go to this render's log, which `ctx` may not carry: see @@ -898,14 +911,9 @@ impl<'a> Renderer<'a> { /// A decoded frame of `f` as a layer buffer covering `w`×`h` layer pixels (the footage's /// nominal size; proxies are scaled up to it), converted from its colour profile. fn footage_buf(&self, img: &Image, f: &Footage, w: u32, h: u32) -> Buf { - let s = self.opts.scale; - let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; - // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = match self.reduced_size(w, h) { - // The source resampled it already (`FootageSource::frame_at_size`). - Some(size) if (img.width, img.height) == size => Buf { img: img.clone(), offset: [0.0; 2], scale: s }, - Some((rw, rh)) if k <= 1.0 + 1e-9 => Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s }, - _ => Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k }, + let mut buf = match self.footage_layout(img, w, h) { + FootageLayout::Native { scale } => Buf { img: img.clone(), offset: [0.0; 2], scale }, + FootageLayout::Resample { width, height, scale } => Buf { img: effectcraft_raster::resample(img, width, height), offset: [0.0; 2], scale }, }; if let Some(c) = self.pipe.media_in(f) { color::convert(&mut buf.img, &c); @@ -913,6 +921,79 @@ impl<'a> Renderer<'a> { buf } + /// How [`Renderer::footage_buf`] lays out a decoded frame `img` of footage `w`×`h` layer + /// pixels: its own pixels, or resampled. + fn footage_layout(&self, img: &Image, w: u32, h: u32) -> FootageLayout { + let s = self.opts.scale; + let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; + // Keep native pixels when downsampling is small; resample otherwise. + match self.reduced_size(w, h) { + // The source resampled it already (`FootageSource::frame_at_size`). + Some(size) if (img.width, img.height) == size => FootageLayout::Native { scale: s }, + Some((width, height)) if k <= 1.0 + 1e-9 => FootageLayout::Resample { width, height, scale: s }, + _ => FootageLayout::Native { scale: 1.0 / k }, + } + } + + /// A footage layer's decoded frame at the context time (the proxy's when one is used) and + /// the footage it was decoded from: what [`Renderer::source`] makes the layer's buffer of. + fn footage_picture_of(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &'a Footage) -> Option<(Arc, &'a Footage)> { + let lt = ctx.source_time(layer); + // The proxy is decoded instead, at its own size, and fills the footage's frame. + let proxy = self.proxy_for(item); + let pf = proxy.unwrap_or(f); + let size = self.reduced_size(f.width, f.height).filter(|_| proxy.is_none()); + let img = self.footage_frame(ctx, layer, item, pf, lt, size)?; + Some((img, pf)) + } + + /// The decoded footage picture a layer's masked source is, bit for bit, with no copy: a + /// footage layer (not continuously rasterised vector footage) without masks whose frame + /// needs no colour conversion, no resampling and no quantisation (32 bpc). The buffer + /// [`Renderer::layer_input`] gives for no effects is this picture at `effect_offset` and + /// `scale`. + /// Layers showing the same frame get the same `Arc` (the footage source's cached frame), so + /// an accelerator uploads it once and shares the texture. `None`: the layer's buffer is + /// made the usual way (any of the above applies, or the frame is missing), and also when the + /// frame is made per call rather than served by the footage source (frame blending, + /// separate fields, Invert Alpha, Interpret As Linear Light, Cineon conversion: a new + /// picture each time, nothing to share) or when the layer cache already holds the layer's + /// buffer (served from memory as before, without asking the footage source). + pub fn footage_picture(&self, ctx: &EvalCtx, layer: &Layer) -> Option { + let LayerSource::Footage { item } = &layer.source else { return None }; + if self.pipe.levels.is_some() || layer.masks().is_some_and(|m| !m.children.is_empty()) || frame_blend_mode(ctx, layer) != FrameBlend::Off { + return None; + } + let item = ctx.media_item(layer, *item); + let ItemKind::Footage(f) = &self.project.item(item)?.kind else { return None }; + if !f.has_video || (layer.switches.collapse && is_vector_footage(f)) { + return None; + } + let pf = self.proxy_for(item).unwrap_or(f); + if self.pipe.media_in(pf).is_some() + || pf.fields != effectcraft_project::FieldOrder::Off + || pf.invert_alpha + || pf.linear_light_applies() + || pf.cineon_applies() + { + return None; + } + if let Some(cache) = self.cache { + let (scale, draft, blur) = (self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer)); + let content = self.content_key(ctx, layer, cache::layer_key(ctx, layer, scale, draft, blur)); + let input = self.content_key(ctx, layer, cache::input_key(ctx, layer, scale, draft, blur, 0)); + if [content, input].into_iter().flatten().any(|k| cache.holds(k)) { + return None; + } + } + let (img, _) = self.footage_picture_of(ctx, layer, item, f)?; + let FootageLayout::Native { scale } = self.footage_layout(&img, f.width, f.height) else { return None }; + // `layer_input`'s rebase into effect space (`Buf::rebase` of a buffer at the origin). + let mut at = Buf { img: Image { width: 0, height: 0, data: Vec::new() }, offset: [0.0; 2], scale }; + at.rebase(ctx.effect_bounds(layer).1); + Some(FootagePicture { img, scale, effect_offset: at.offset }) + } + /// The size of `w` × `h` layer pixels in this render when it is reduced enough that footage /// is resampled to it (below 3/4 scale). fn reduced_size(&self, w: u32, h: u32) -> Option<(u32, u32)> {