Merge pull request #3355 from arnav-hiwarkar/fix/3345-gpu-preview-lanes

Preview: the window stays smooth while lanes render on the GPU (#3345)

crates/ui-egui/src/frames.rs

@@ -54,6 +54,14 @@ const GPU_BUDGET: usize = 1 << 30;
 const MIN_GPU_BUDGET: usize = 64 << 20;
 const GPU_PRESSURE_COOLDOWN: std::time::Duration = std::time::Duration::from_secs(5);
 const GPU_RECOVERY_INTERVAL: std::time::Duration = std::time::Duration::from_secs(30);
+/// The window's frame gaps that adapt [`Queue::gpu_limit`] while a preview plays: a longer one
+/// is a stall, shorter ones for a while are smooth.
+const UI_STALL: std::time::Duration = std::time::Duration::from_millis(100);
+const UI_SMOOTH: std::time::Duration = std::time::Duration::from_millis(50);
+/// How long the window must stay smooth before one more lane composites on the GPU, and before it
+/// tries again a limit that stalled.
+const GPU_LANE_STEP: std::time::Duration = std::time::Duration::from_millis(500);
+const GPU_LANE_RETRY: std::time::Duration = std::time::Duration::from_secs(10);
 
 mod f16;
 pub use f16::{F16_PIXEL_BYTES, F16Frame};
@@ -639,7 +647,7 @@ impl Default for Frames {
             })
             .collect();
         #[cfg(not(target_arch = "wasm32"))]
-        let queue = Queue { idle_lanes: (0..lanes.len()).collect(), lanes: lanes.len(), ..Default::default() };
+        let queue = Queue { idle_lanes: (0..lanes.len()).collect(), lanes: lanes.len(), gpu_limit: 1, ..Default::default() };
         #[cfg(target_arch = "wasm32")]
         let queue = Queue::default();
         Frames {
@@ -1282,6 +1290,8 @@ impl Frames {
         #[cfg(not(target_arch = "wasm32"))]
         let moved = q.playhead.is_some() != playhead.is_some();
         q.playhead = playhead;
+        // (called once per frame of the window while a preview plays)
+        let more_gpu = q.ui_frame(playhead.map(|_| web_time::Instant::now()));
         if let Some(p) = playhead
             && let Ok(mut inf) = self.inflight.lock()
         {
@@ -1293,6 +1303,13 @@ impl Frames {
                 !gone
             });
         }
+        // A lane more may composite: one that stopped at the limit starts again.
+        #[cfg(not(target_arch = "wasm32"))]
+        if more_gpu && !moved {
+            self.start_lanes(&mut q, 1);
+        }
+        #[cfg(target_arch = "wasm32")]
+        let _ = more_gpu;
         #[cfg(not(target_arch = "wasm32"))]
         if moved {
             self.start_lanes(&mut q, usize::MAX);
@@ -1610,6 +1627,17 @@ impl Drop for Draining {
     }
 }
 
+/// A lane's hold on one of the [`Queue::gpu_limit`] GPU slots (see [`Queue::gpu_lanes`]), given back when dropped,
+/// even by a panic.
+struct GpuSlot<'a>(&'a Mutex<Queue>);
+
+impl Drop for GpuSlot<'_> {
+    fn drop(&mut self) {
+        let mut q = self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
+        q.gpu_lanes = q.gpu_lanes.saturating_sub(1);
+    }
+}
+
 impl Worker {
     /// Render queued jobs, most important first, until the queue is empty.
     ///
@@ -1636,19 +1664,25 @@ impl Worker {
     /// idle again.
     fn run_next(&self, lane: Option<usize>) -> bool {
         let Worker { queue, content_keys, .. } = self;
-        let job = {
+        // (asked before the queue's lock: it takes the cache's)
+        let gpu = lane.is_some() && self.gpu_ready();
+        let (job, mut slot) = {
             let Ok(mut q) = queue.lock() else { return false };
-            let Some(i) = q.next(lane.is_some()) else {
+            let Some(i) = q.next(lane.is_some(), gpu) else {
                 q.idle_lanes.extend(lane);
                 return false;
             };
             let job = q.jobs.swap_remove(i);
+            let slot = (gpu && q.gpu_job(&job)).then(|| {
+                q.gpu_lanes += 1;
+                GpuSlot(queue)
+            });
             q.running.insert(job.key);
             if job.urgent {
                 q.urgent_running += 1;
                 q.urgent_cancels.push((job.key, job.cancel.clone()));
             }
-            job
+            (job, slot)
         };
         let t0 = web_time::Instant::now();
         // (the drop generation before the frame reads anything: File ▸ Reload Footage meanwhile
@@ -1657,6 +1691,10 @@ impl Worker {
             job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)), dc.drop_generation()));
         let from_disk = disk.as_ref().and_then(|(dc, k, _)| self.disk_frame(dc, *k, &job));
         let disk_hit = from_disk.is_some();
+        if disk_hit {
+            // Nothing for the GPU to do: another lane may composite meanwhile.
+            slot = None;
+        }
         // `files`: the source files the frame was made of (see `render::reads`).
         let (ci, files) = match from_disk {
             Some(hit) => hit,
@@ -1683,6 +1721,8 @@ impl Worker {
         // and compression. Persistence still runs on this worker, using the same snapshot.
         let since = disk.as_ref().map(|(dc, _, since)| DiskSince::of(dc, *since));
         self.finish(&job, ci.clone(), Some(files.clone()), since, t0);
+        // The GPU's part is done; storing it on disk is the CPU's.
+        drop(slot);
         if !disk_hit
             && let Some((dc, k, since)) = &disk
             && put_disk_frame(dc, *k, &ci, &files, self.virtual_id.as_deref(), *since)
@@ -1951,6 +1991,24 @@ struct Queue {
     /// caching left the same gaps, and held up the viewer's frame when it shared a piece.
     lanes: usize,
     idle_lanes: Vec<usize>,
+    /// Lanes compositing a frame on the GPU, at most [`Queue::gpu_limit`]. The GPU is the
+    /// viewer's device: each lane records, submits and reads back a whole frame on it, and the
+    /// window can't draw until that work is done. With a lane per core on a GPU that can't keep
+    /// up, the window froze for hundreds of milliseconds at a time while a preview played, and
+    /// the preview jumped ahead when it came back (#3345). The other lanes go on with the frames
+    /// that don't need the GPU (the disk cache's, or all of them while the GPU is out of memory
+    /// or retired); the pool and the viewer's frame are not counted.
+    gpu_lanes: usize,
+    /// How many lanes may composite on the GPU at a time (at least 1, see [`Queue::ui_frame`]).
+    gpu_limit: usize,
+    /// The limit that last stalled the window: tried again only after [`GPU_LANE_RETRY`].
+    gpu_stalled_at: usize,
+    /// While a preview plays: the window's last frame, and since when it has been smooth.
+    ui_last: Option<web_time::Instant>,
+    ui_smooth_since: Option<web_time::Instant>,
+    /// Tests: as if there were a GPU compositor (no adapter is there).
+    #[cfg(test)]
+    fake_gpu: bool,
 }
 
 impl Queue {
@@ -1959,9 +2017,63 @@ impl Queue {
         self.lanes > 0 && !j.urgent && (j.background || self.playhead.is_some())
     }
 
-    /// The job a lane (`lane`) or the pool renders next: the [`Queue::best`] of its own.
-    fn next(&self, lane: bool) -> Option<usize> {
-        self.best_of(|j| self.on_lane(j) == lane)
+    /// The job a lane (`lane`) or the pool renders next: the [`Queue::best`] of its own. `gpu`:
+    /// the GPU composites (a lane's frames), so a lane takes no GPU frame while
+    /// [`Queue::gpu_limit`] lanes are compositing.
+    fn next(&self, lane: bool, gpu: bool) -> Option<usize> {
+        let gpu_full = gpu && self.gpu_lanes >= self.gpu_limit.max(1);
+        self.best_of(|j| self.on_lane(j) == lane && !(gpu_full && self.gpu_job(j)))
+    }
+
+    /// The window drew a frame at `now` (`None`: no preview plays). A window that stalls with
+    /// lanes compositing gets fewer of them: the GPU can't keep up with them and the window. One
+    /// that stays smooth while every allowed lane composites gets one more, up to all of them: a
+    /// GPU that keeps up renders the preview on every lane. A limit that stalled is tried again
+    /// only after a longer smooth spell, so the window doesn't stall over and over. `true` when
+    /// one more lane may composite.
+    fn ui_frame(&mut self, now: Option<web_time::Instant>) -> bool {
+        let Some(now) = now else {
+            self.ui_last = None;
+            return false;
+        };
+        let limit = self.gpu_limit.max(1);
+        let gap = self.ui_last.map(|last| now.saturating_duration_since(last));
+        self.ui_last = Some(now);
+        let Some(gap) = gap else {
+            self.ui_smooth_since = Some(now);
+            return false;
+        };
+        if gap > UI_STALL && self.gpu_lanes > 0 {
+            self.gpu_stalled_at = limit;
+            self.gpu_limit = (limit / 2).max(1);
+            self.ui_smooth_since = Some(now);
+        } else if gap > UI_SMOOTH {
+            self.ui_smooth_since = Some(now);
+        } else {
+            let since = *self.ui_smooth_since.get_or_insert(now);
+            let wait = if limit + 1 >= self.gpu_stalled_at && self.gpu_stalled_at > 0 { GPU_LANE_RETRY } else { GPU_LANE_STEP };
+            if self.gpu_lanes >= limit && limit < self.lanes && now.saturating_duration_since(since) >= wait {
+                self.gpu_limit = limit + 1;
+                self.ui_smooth_since = Some(now);
+                return true;
+            }
+        }
+        false
+    }
+
+    /// Whether `j` is composited on the GPU when the GPU is ready: there is a GPU compositor and
+    /// the frame's renderer uses it (as `Renderer::active_accel` decides: Software Only keeps
+    /// every frame on the CPU).
+    fn gpu_job(&self, j: &Job) -> bool {
+        let adapter = j.src.gpu.is_some();
+        #[cfg(test)]
+        let adapter = adapter || self.fake_gpu;
+        adapter
+            && match j.opts.backend {
+                effectcraft_engine::render::Backend::Cpu => false,
+                effectcraft_engine::render::Backend::Gpu => true,
+                effectcraft_engine::render::Backend::Auto => j.src.project.settings.gpu_acceleration,
+            }
     }
 
     /// The job that matters most: urgent first (newest), then, while a preview plays, the

crates/ui-egui/src/frames/tests.rs

@@ -610,15 +610,21 @@ struct ParallelClip {
     started: std::sync::atomic::AtomicUsize,
     /// Frames started by the time frame 1 was done.
     started_at_first: std::sync::atomic::AtomicUsize,
+    /// Frames decoding now, and the most at a time.
+    active: std::sync::atomic::AtomicUsize,
+    most_active: std::sync::atomic::AtomicUsize,
 }
 
 impl FootageSource for ParallelClip {
     fn frame(&self, _: ItemId, _: &effectcraft_engine::project::Footage, t: Tick) -> Option<Arc<effectcraft_engine::render::Image>> {
         self.started.fetch_add(1, Ordering::SeqCst);
+        let active = self.active.fetch_add(1, Ordering::SeqCst) + 1;
+        self.most_active.fetch_max(active, Ordering::SeqCst);
         (0..16).into_par_iter().for_each(|_| (0..4).into_par_iter().for_each(|_| std::thread::sleep(std::time::Duration::from_millis(1))));
         if effectcraft_engine::time::FrameRate::FPS_30.frame_at(t) == 1 {
             self.started_at_first.store(self.started.load(Ordering::SeqCst), Ordering::SeqCst);
         }
+        self.active.fetch_sub(1, Ordering::SeqCst);
         Some(Arc::new(effectcraft_engine::render::Image::new(4, 4)))
     }
 }
@@ -729,6 +735,166 @@ fn queued_frames_render_when_a_preview_starts_or_stops() {
     assert_eq!(f.cached_frames(&series).len(), 119);
 }
 
+/// Waits for the lanes to be idle (after their last frame they give back their GPU slot) and
+/// checks every GPU slot was given back.
+fn wait_for_idle_lanes(f: &Frames) {
+    for _ in 0..2000 {
+        let q = f.queue.lock().unwrap();
+        if q.idle_lanes.len() == f.lanes.len() {
+            break;
+        }
+        drop(q);
+        std::thread::sleep(std::time::Duration::from_millis(5));
+    }
+    let q = f.queue.lock().unwrap();
+    assert_eq!(q.idle_lanes.len(), f.lanes.len(), "every lane is idle again");
+    assert_eq!(q.gpu_lanes, 0, "every lane gave its GPU slot back");
+}
+
+/// Plays a preview of `frames` frames of a [`parallel_clip`] as the viewer renders them (the
+/// project's renderer: `backend` Auto) or with `backend`, as if there were a GPU compositor
+/// (`gpu`), `retired` or with the project on Mercury Software Only (`!gpu_acceleration`), and
+/// waits for them: the frame cache and the most frames that rendered at a time.
+fn play_preview(gpu: bool, retired: bool, backend: effectcraft_engine::render::Backend, gpu_acceleration: bool) -> (Frames, usize) {
+    use effectcraft_engine::time::FrameRate;
+    let (f, footage, mut src, cid, series) = parallel_clip();
+    Arc::make_mut(&mut src.project).settings.gpu_acceleration = gpu_acceleration;
+    f.queue.lock().unwrap().fake_gpu = gpu;
+    if retired {
+        f.retire_gpu();
+    }
+    let opts = RenderOpts { backend, ..RenderOpts::default() };
+    f.set_playhead(Some(Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false }));
+    let frames = 24;
+    for frame in 1..=frames {
+        f.request(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), opts);
+    }
+    wait_for_renders(&f);
+    assert_eq!(f.cached_frames(&series).len(), frames as usize);
+    wait_for_idle_lanes(&f);
+    let most = footage.most_active.load(Ordering::SeqCst);
+    (f, most)
+}
+
+/// Every lane composited a playing preview's frame on the GPU at the same time, on the device
+/// the window draws with: the window froze for hundreds of milliseconds and the preview jumped
+/// ahead (#3345). A preview starts on one GPU lane (see [`Queue::ui_frame`]).
+#[test]
+fn a_preview_composites_on_a_few_lanes_at_a_time() {
+    use effectcraft_engine::render::Backend;
+    for backend in [Backend::Auto, Backend::Gpu] {
+        let (_, most) = play_preview(true, false, backend, true);
+        assert!(most <= 1, "{most} lanes composited on the GPU at once ({backend:?})");
+    }
+}
+
+/// Frames that render on the CPU use every lane: with no GPU compositor, with the GPU retired,
+/// and with Mercury Software Only (the project's renderer, or the CPU backend).
+#[test]
+fn frames_on_the_cpu_render_on_every_lane() {
+    use effectcraft_engine::render::Backend;
+    for (gpu, retired, backend, gpu_acceleration) in
+        [(false, false, Backend::Auto, true), (true, true, Backend::Auto, true), (true, false, Backend::Auto, false), (true, false, Backend::Cpu, true)]
+    {
+        let (f, most) = play_preview(gpu, retired, backend, gpu_acceleration);
+        if f.lanes.len() > 2 {
+            assert!(most > 2, "frames on the CPU rendered on {most} lanes at a time ({gpu} {retired} {backend:?} {gpu_acceleration})");
+        }
+    }
+}
+
+/// A queue of `lanes` lanes, `busy` of them compositing on the GPU.
+fn gpu_queue(lanes: usize, busy: usize) -> Queue {
+    Queue { lanes, gpu_lanes: busy, gpu_limit: 1, ..Default::default() }
+}
+
+/// Window frames every `gap` ms from `t` for `ms` ms; the time after the last.
+fn ui_frames(q: &mut Queue, t: web_time::Instant, gap: u64, ms: u64) -> web_time::Instant {
+    let mut t = t;
+    for _ in 0..ms / gap {
+        t += std::time::Duration::from_millis(gap);
+        q.ui_frame(Some(t));
+    }
+    t
+}
+
+/// A GPU that keeps up with the window gets every lane, one more each half second the window
+/// stays smooth with every allowed lane compositing.
+#[test]
+fn a_smooth_window_gives_the_gpu_every_lane() {
+    let mut q = gpu_queue(8, 0);
+    let t = web_time::Instant::now();
+    q.ui_frame(Some(t));
+    // Nothing compositing: the limit isn't what holds the preview back.
+    let t = ui_frames(&mut q, t, 16, 2000);
+    assert_eq!(q.gpu_limit, 1);
+    let mut t = t;
+    for _ in 0..20 {
+        q.gpu_lanes = q.gpu_limit;
+        t = ui_frames(&mut q, t, 16, 520);
+    }
+    assert_eq!(q.gpu_limit, 8, "no more than the lanes");
+}
+
+/// A window that stalls while lanes composite halves the limit, and the limit that stalled is
+/// tried again only after a long smooth spell; a window that is only slow keeps the limit.
+#[test]
+fn a_stalling_window_takes_lanes_from_the_gpu() {
+    let mut q = gpu_queue(8, 4);
+    q.gpu_limit = 4;
+    let t = web_time::Instant::now();
+    q.ui_frame(Some(t));
+    let t = ui_frames(&mut q, t, 300, 300);
+    assert_eq!((q.gpu_limit, q.gpu_stalled_at), (2, 4));
+    let t = ui_frames(&mut q, t, 80, 5000);
+    assert_eq!(q.gpu_limit, 2, "a slow (not stalling) window keeps the limit");
+    q.gpu_lanes = 2;
+    let t = ui_frames(&mut q, t, 16, 600);
+    assert_eq!(q.gpu_limit, 3);
+    q.gpu_lanes = 3;
+    let t = ui_frames(&mut q, t, 16, 2000);
+    assert_eq!(q.gpu_limit, 3, "4 stalled: it waits longer");
+    ui_frames(&mut q, t, 16, 10_000);
+    assert_eq!(q.gpu_limit, 4);
+    // A stall with nothing compositing isn't the GPU's.
+    let mut q = gpu_queue(8, 0);
+    q.gpu_limit = 4;
+    let t = web_time::Instant::now();
+    q.ui_frame(Some(t));
+    ui_frames(&mut q, t, 500, 500);
+    assert_eq!(q.gpu_limit, 4);
+    // Never below one lane; a stopped preview's pause is no stall.
+    let mut q = gpu_queue(8, 1);
+    let t = web_time::Instant::now();
+    q.ui_frame(Some(t));
+    ui_frames(&mut q, t, 400, 400);
+    assert_eq!(q.gpu_limit, 1);
+    q.gpu_limit = 2;
+    q.ui_frame(None);
+    q.ui_frame(Some(t + std::time::Duration::from_secs(5)));
+    assert_eq!(q.gpu_limit, 2);
+}
+
+/// A preview that stops while lanes composite: their frames finish, the queued ones move to the
+/// pool and render, the viewer's frame is not held up, and every GPU slot is given back.
+#[test]
+fn a_preview_stopping_mid_render_gives_the_gpu_slots_back() {
+    use effectcraft_engine::time::FrameRate;
+    let (f, _, src, cid, series) = parallel_clip();
+    f.queue.lock().unwrap().fake_gpu = true;
+    let opts = RenderOpts { backend: effectcraft_engine::render::Backend::Auto, ..RenderOpts::default() };
+    f.set_playhead(Some(Playhead { series, cur: 0, start: 0, end: 299, step: 1, looping: false }));
+    for frame in 1..=24 {
+        f.request(&src, FrameKey { frame, ..series }, cid, FrameRate::FPS_30.tick_of(frame), opts);
+    }
+    std::thread::sleep(std::time::Duration::from_millis(20));
+    f.set_playhead(None);
+    f.request_urgent(&src, FrameKey { frame: 100, ..series }, cid, FrameRate::FPS_30.tick_of(100), opts);
+    wait_for_renders(&f);
+    assert_eq!(f.cached_frames(&series).len(), 25);
+    wait_for_idle_lanes(&f);
+}
+
 /// The cache is full. A zoom (or preview-resolution) change left the frames of the old
 /// size to the plain least-recently-shown order, so the frames of the zoom the viewer is back
 /// at went before the ones of the abandoned zoom. The other size's frames go first, then the