Merge upstream/main into cc-ball-action-params-3197

This commit is contained in:
vangel76
2026-11-09 19:19:43 +01:00
271 changed files with 16887 additions and 2524 deletions
+1 -1
View File
@@ -801,7 +801,7 @@ fn read_buffer(e: &mut Enc, buf: &wgpu::Buffer, size: u64) -> Option<Vec<u8>> {
});
e.record(&[]).copy_buffer_to_buffer(buf, 0, &rb, 0, size);
e.submit();
let out = e.g.read_buffer(&rb, row, row, 1).map_err(|e| log::error!("gpu 3d readback: {e}")).ok()?;
let out = e.g.read_buffer_after(&rb, e.last_submission(), row, row, 1).map_err(|e| log::error!("gpu 3d readback: {e}")).ok()?;
Some(out)
}
+227 -41
View File
@@ -128,6 +128,11 @@ const POOL_BUDGET: usize = 256 << 20;
/// miss (a new allocation), never a wrong pixel.
const STALE_FRAMES: u64 = 2;
/// The pool owner of uploaded textures: never registered by an [`Enc`] (their ids start at 1),
/// so an upload takes only from the shared free list ([`Pool::take`], textures no open encoder
/// can read) and gives its texture back through the guarded [`Pool::release`].
const UPLOAD_OWNER: u64 = 0;
/// Readback staging buffers kept for reuse across sizes (bytes).
const STAGING_BUDGET: u64 = 256 << 20;
@@ -693,6 +698,10 @@ pub struct GpuContext {
dummy_tex: wgpu::TextureView,
dummy_buf: wgpu::Buffer,
uploads: Mutex<Uploads>,
/// Uploaded footage pictures, shared by the layers that show them (see [`crate::upload`]).
pictures: Mutex<crate::upload::Pictures>,
/// Reused staging buffers of large uploads (see [`crate::upload::Staging`]).
upload_staging: crate::upload::Staging,
upload_budget: std::sync::atomic::AtomicUsize,
upload_full_warned: std::sync::atomic::AtomicBool,
pool: Arc<Mutex<Pool>>,
@@ -1017,6 +1026,8 @@ impl GpuContext {
dummy,
dummy_buf,
uploads: Mutex::new(Uploads::default()),
pictures: Default::default(),
upload_staging: Default::default(),
upload_budget: std::sync::atomic::AtomicUsize::new(UPLOAD_BUDGET),
upload_full_warned: Default::default(),
pool,
@@ -1124,6 +1135,24 @@ impl GpuContext {
}
}
/// The memory budget (the memory cap: a conservative share of the device, see `memcap`),
/// the allocator's peak, one frame's share of it and what the upload cache may still add.
/// `None` where any is unknown: no allocator report (Metal, WebGPU, GL) or the cap is off.
pub fn frame_memory(&self) -> Option<effectcraft_render::FrameMemory> {
let budget = self.memcap.cap();
let now = self.allocated_bytes();
self.memcap.note(now);
// The allocator's peak (sampled while frames render) holds a frame's working textures
// at their most; what is gone again after the frame (and at least what the pool keeps
// for the next) is one frame's share. Frames that had to wait for memory: no room.
let peak = self.memcap.peak();
let frame = if self.memcap.pressed() { budget } else { peak.saturating_sub(now).max(self.pool_retained_bytes() as u64) };
// The upload cache may still fill up to its budget.
let uploads = self.uploads.lock().map(|u| u.bytes).unwrap_or(0);
let reserve = self.upload_budget().saturating_sub(uploads) as u64;
(budget > 0 && peak > 0 && frame > 0).then_some(effectcraft_render::FrameMemory { budget, peak, frame, reserve })
}
/// Bytes held by wgpu's device allocator (`0` on backends that do not report).
pub(crate) fn allocated_bytes(&self) -> u64 {
self.device.generate_allocator_report().map_or(0, |r| r.total_allocated_bytes)
@@ -1132,11 +1161,11 @@ impl GpuContext {
/// Bytes held in long-lived or still-referenced GPU caches/leases that submitting and waiting
/// on `owner` (`closing_enc`) cannot free.
fn kept_bytes(&self, owner: u64, closing_enc: Option<u64>) -> u64 {
let uploads = self.uploads.lock().map(|u| u.bytes as u64).unwrap_or(0);
// (uploaded layer and footage textures are pooled: the pool's leases count them)
let pool = self.pool.lock().map(|p| p.kept_after_flush(owner, closing_enc) as u64).unwrap_or(0);
let staging: u64 = self.staging.lock().map(|s| s.iter().map(|(k, v)| k * v.len() as u64).sum()).unwrap_or(0);
let zeros: u64 = self.zeros.lock().map(|z| z.keys().map(|&(w, h)| w as u64 * h as u64 * 16).sum()).unwrap_or(0);
uploads + pool + staging + zeros
pool + staging + self.upload_staging.bytes() + zeros
}
#[cfg(test)]
@@ -1198,18 +1227,99 @@ impl GpuContext {
/// never consumes the readback deadline or holds the mutex across the whole timeout.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn read_buffer(&self, buffer: &wgpu::Buffer, row: usize, stride: usize, height: usize) -> Result<Vec<u8>, String> {
self.read_buffer_after(buffer, None, row, stride, height)
}
/// [`Self::read_buffer`] waiting only for `submission` (the one that wrote `buffer`), not
/// for every submission so far: another frame's GPU work submitted meanwhile doesn't hold
/// this readback up (#3335). `None`: wait for all submitted work.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn read_buffer_after(
&self,
buffer: &wgpu::Buffer,
submission: Option<wgpu::SubmissionIndex>,
row: usize,
stride: usize,
height: usize,
) -> Result<Vec<u8>, String> {
self.read_mapped(buffer, submission, |view| crate::readback::packed(view, row, stride, height))
}
/// Map `buffer`, wait for the mapping (only for `submission` when given) and hand the
/// mapped bytes to `read`, on this thread, before unmapping: `read` copies the rows
/// straight into their destination, once (#3335). The copy happens outside `poll_lock`, so
/// it doesn't hold up other threads' readbacks the way copying inside the map callback
/// (which runs during a device poll) would.
#[cfg(not(target_arch = "wasm32"))]
pub(crate) fn read_mapped<R>(
&self,
buffer: &wgpu::Buffer,
submission: Option<wgpu::SubmissionIndex>,
read: impl FnOnce(&[u8]) -> Result<R, String>,
) -> Result<R, String> {
let (tx, rx) = std::sync::mpsc::channel();
self.map_readback(buffer, row, stride, height, move |r| {
let ticket = self.readbacks.start(move |r| {
let _ = tx.send(r);
})?;
if !ticket.active() {
return Err("GPU readback cancelled".into());
}
let b = buffer.clone();
buffer.map_async(wgpu::MapMode::Read, .., move |r| {
// Timed out or cancelled: nobody reads the mapping.
if !ticket.active() {
b.unmap();
return;
}
// Mapped: the waiting thread reads the rows and unmaps (an empty vector = mapped).
ticket.complete(r.map(|_| Vec::new()).map_err(|e| format!("GPU buffer mapping: {e}")));
});
// On any failure, unmap (harmless when not mapped; cancels a pending map) so a mapped
// buffer can never be handed out again.
let fail = |e: String| {
buffer.unmap();
e
};
self.wait_mapped(&rx, submission).map_err(fail)?.map_err(fail)?;
let result = {
let view = buffer.get_mapped_range(..).map_err(|e| format!("GPU mapped range: {e}"));
view.and_then(|view| read(&view))
};
buffer.unmap();
result
}
/// Poll until `rx` reports the mapping (see [`Self::read_mapped`]): polling and receiving
/// share one deadline, started once `poll_lock` is first acquired and polled in
/// [`Self::POLL_SLICE`] slices so lock contention never consumes the readback deadline or
/// holds the mutex across the whole timeout. While another thread holds `poll_lock` it is
/// polling the device, which delivers this mapping too: wait for it instead of queueing.
#[cfg(not(target_arch = "wasm32"))]
fn wait_mapped(
&self,
rx: &std::sync::mpsc::Receiver<Result<Vec<u8>, String>>,
submission: Option<wgpu::SubmissionIndex>,
) -> Result<Result<Vec<u8>, String>, String> {
let mut deadline = None;
loop {
self.readbacks.check()?;
if let Ok(result) = rx.try_recv() {
return result;
return Ok(result);
}
let (remaining, slice, poll_res) = {
let _guard = self.poll_lock.lock().unwrap_or_else(|e| e.into_inner());
let _guard = match self.poll_lock.try_lock() {
Ok(g) => g,
Err(std::sync::TryLockError::Poisoned(e)) => e.into_inner(),
Err(std::sync::TryLockError::WouldBlock) => {
match rx.recv_timeout(std::time::Duration::from_millis(1)) {
Ok(result) => return Ok(result),
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(e) => return Err(format!("GPU readback completion channel: {e}")),
}
// The ticket's own deadline still bounds a wait that never gets the lock.
continue;
}
};
self.readbacks.check()?;
let d = *deadline.get_or_insert_with(|| std::time::Instant::now() + crate::readback::TIMEOUT);
let remaining = d.saturating_duration_since(std::time::Instant::now());
@@ -1217,12 +1327,12 @@ impl GpuContext {
(remaining, std::time::Duration::ZERO, Ok(wgpu::PollStatus::Poll))
} else {
let slice = remaining.min(Self::POLL_SLICE);
let res = self.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(slice) });
let res = self.device.poll(wgpu::PollType::Wait { submission_index: submission.clone(), timeout: Some(slice) });
(remaining, slice, res)
}
};
if let Ok(result) = rx.try_recv() {
return result;
return Ok(result);
}
if remaining.is_zero() {
let error = "GPU readback receive timed out; context retired".to_string();
@@ -1241,7 +1351,7 @@ impl GpuContext {
}
// The callback may be running on another thread that polled the same device.
match rx.recv_timeout(remaining.min(std::time::Duration::from_millis(1))) {
Ok(result) => return result,
Ok(result) => return Ok(result),
Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {}
Err(e) => return Err(format!("GPU readback completion channel: {e}")),
}
@@ -1253,6 +1363,28 @@ impl GpuContext {
Err("synchronous GPU readback is unavailable in the browser".into())
}
#[cfg(target_arch = "wasm32")]
pub(crate) fn read_buffer_after(
&self,
_buffer: &wgpu::Buffer,
_submission: Option<wgpu::SubmissionIndex>,
_row: usize,
_stride: usize,
_height: usize,
) -> Result<Vec<u8>, String> {
Err("synchronous GPU readback is unavailable in the browser".into())
}
#[cfg(target_arch = "wasm32")]
pub(crate) fn read_mapped<R>(
&self,
_buffer: &wgpu::Buffer,
_submission: Option<wgpu::SubmissionIndex>,
_read: impl FnOnce(&[u8]) -> Result<R, String>,
) -> Result<R, String> {
Err("synchronous GPU readback is unavailable in the browser".into())
}
/// Switch to deferred readbacks (see [`crate::deferred`]): readbacks are never waited for.
pub fn set_deferred(&mut self, on: bool) {
self.readbacks.cancel_pending("GPU readback mode changed".into());
@@ -1446,6 +1578,8 @@ impl GpuContext {
dummy: self.dummy.clone(),
dummy_buf: self.dummy_buf.clone(),
uploads: Mutex::new(Uploads::default()),
pictures: Default::default(),
upload_staging: Default::default(),
upload_budget: std::sync::atomic::AtomicUsize::new(UPLOAD_BUDGET),
upload_full_warned: Default::default(),
pool: self.pool.clone(),
@@ -1625,16 +1759,54 @@ impl GpuContext {
return None;
}
self.transfers.count(true, img.data.len() * 16);
let g = self.image(img.width, img.height);
self.queue.write_texture(
wgpu::TexelCopyTextureInfo { texture: &g.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
bytemuck::cast_slice(&img.data),
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(img.width * 16), rows_per_image: Some(img.height) },
wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 },
);
// A texture from the pool's shared free list: no open encoder can still read it, and the
// copy below runs before anything recorded after this call.
let (g, _) = self.pooled(img.width, img.height, UPLOAD_OWNER);
if !self.upload_staging.write(&self.device, &self.queue, img, &g.texture) {
self.queue.write_texture(
wgpu::TexelCopyTextureInfo { texture: &g.texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
bytemuck::cast_slice(&img.data),
wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(img.width * 16), rows_per_image: Some(img.height) },
wgpu::Extent3d { width: img.width, height: img.height, depth_or_array_layers: 1 },
);
}
Some(g)
}
/// Upload a decoded footage picture once while the footage source keeps it: every layer
/// and effect input showing the same frame shares the texture (`Renderer::footage_picture`).
pub(crate) fn upload_picture(&self, img: &Arc<Image>) -> Option<GpuImage> {
if let Some(t) = self.pictures.lock().ok().and_then(|mut p| p.get(img)) {
return Some(t);
}
let tex = self.upload_image(img)?;
if let Ok(mut p) = self.pictures.lock() {
p.insert(img, tex.clone(), crate::upload::PICTURE_BUDGET.min(self.upload_budget() / 4));
}
Some(tex)
}
/// Entries in the footage picture cache.
#[cfg(test)]
pub(crate) fn picture_entries(&self) -> usize {
self.pictures.lock().map(|p| p.len()).unwrap_or_default()
}
/// The upload staging buffers (tests).
#[cfg(test)]
pub(crate) fn upload_staging(&self) -> &crate::upload::Staging {
&self.upload_staging
}
/// A render is done: drop the footage picture textures and the upload staging buffers
/// (the next render's pictures upload again; their textures come from the pool).
pub fn trim_uploads(&self) {
if let Ok(mut p) = self.pictures.lock() {
p.clear();
}
self.upload_staging.clear();
}
/// Entries in the layer upload cache.
#[cfg(test)]
pub(crate) fn upload_entries(&self) -> usize {
@@ -1696,6 +1868,10 @@ impl GpuContext {
u.map.clear();
u.bytes = 0;
}
if let Ok(mut p) = self.pictures.lock() {
p.clear();
}
self.upload_staging.clear();
if let Ok(mut p) = self.pool.lock() {
p.clear();
}
@@ -1952,6 +2128,11 @@ impl<'g> Enc<'g> {
dst
}
/// The last submission of this encoder's work (what a readback after it waits for).
pub(crate) fn last_submission(&self) -> Option<wgpu::SubmissionIndex> {
self.last_submit.clone()
}
/// Submit recorded work.
pub fn submit(&mut self) {
if let Some((enc, id)) = self.enc.take() {
@@ -1970,6 +2151,7 @@ impl<'g> Enc<'g> {
return;
}
let allocated = self.g.allocated_bytes();
self.g.memcap.note(allocated);
if allocated <= self.g.memcap.cap() {
let _ = self.g.memcap.should_wait(allocated, 0);
return;
@@ -1994,13 +2176,13 @@ impl<'g> Enc<'g> {
let layout = crate::readback::Layout::new(img.width, img.height, 16, self.g.device.limits().max_buffer_size)
.map_err(|e| log::error!("gpu image readback: {e}"))
.ok()?;
let mut data = Vec::new();
data.try_reserve_exact(layout.len / 16).map_err(|e| log::error!("gpu image allocation: {e}")).ok()?;
data.resize(layout.len / 16, [0.0; 4]);
let mut out = Image { width: img.width, height: img.height, data };
let dst: &mut [u8] = bytemuck::cast_slice_mut(&mut out.data);
self.read_texture_into(&img.texture, [0, 0, img.width, img.height], 16, dst)?;
Some(out)
// The mapped rows go straight into the image's pixels: one copy (#3335).
let data: Vec<[f32; 4]> = self.read_texture_with(&img.texture, [0, 0, img.width, img.height], 16, crate::readback::rows::<[f32; 4]>)?;
if data.len() != layout.len / 16 {
log::error!("gpu image readback size mismatch");
return None;
}
Some(Image { width: img.width, height: img.height, data })
}
/// Tightly packed texel bytes of a texture (`bpp` bytes per pixel).
@@ -2013,27 +2195,32 @@ impl<'g> Enc<'g> {
pub fn read_rect(&mut self, texture: &wgpu::Texture, x: u32, y: u32, w: u32, h: u32, bpp: u32) -> Option<Vec<u8>> {
let layout =
crate::readback::Layout::new(w, h, bpp, self.g.device.limits().max_buffer_size).map_err(|e| log::error!("gpu texture readback: {e}")).ok()?;
let mut out = Vec::new();
out.try_reserve_exact(layout.len).map_err(|e| log::error!("gpu readback allocation: {e}")).ok()?;
out.resize(layout.len, 0);
self.read_texture_into(texture, [x, y, w, h], bpp, &mut out)?;
let out: Vec<u8> = self.read_texture_with(texture, [x, y, w, h], bpp, crate::readback::rows::<u8>)?;
if out.len() != layout.len {
log::error!("gpu texture readback size mismatch");
return None;
}
Some(out)
}
/// [`Enc::read_rect`] into `out` (`w × bpp × h` bytes), through a reused staging buffer.
/// `rect` is `[x, y, w, h]`.
fn read_texture_into(&mut self, texture: &wgpu::Texture, rect: [u32; 4], bpp: u32, out: &mut [u8]) -> Option<()> {
/// Read `rect` (`[x, y, w, h]`) of `texture` back through a reused staging buffer: `read`
/// gets the mapped bytes, the row length, the padded row stride and the row count, and
/// copies the rows into their destination itself (one copy, #3335). Waits only for this
/// encoder's own submission, so frames rendering on other threads don't delay it.
fn read_texture_with<R>(
&mut self,
texture: &wgpu::Texture,
rect: [u32; 4],
bpp: u32,
read: impl FnOnce(&[u8], usize, usize, usize) -> Result<R, String>,
) -> Option<R> {
let [x, y, w, h] = rect;
if !self.g.can_wait() {
return None;
}
self.g.transfers.count(false, out.len());
let layout =
crate::readback::Layout::new(w, h, bpp, self.g.device.limits().max_buffer_size).map_err(|e| log::error!("gpu texture readback: {e}")).ok()?;
if out.len() != layout.len {
log::error!("gpu readback destination size mismatch");
return None;
}
self.g.transfers.count(false, layout.len);
let row = layout.row as usize;
let padded = layout.stride;
let size = layout.size;
@@ -2051,12 +2238,11 @@ impl<'g> Enc<'g> {
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
self.submit();
let bytes = self.g.read_buffer(&buf, row, padded as usize, h as usize).map_err(|e| log::error!("gpu texture readback: {e}")).ok()?;
if bytes.len() != out.len() {
log::error!("gpu texture readback size mismatch");
return None;
}
out.copy_from_slice(&bytes);
let out = self
.g
.read_mapped(&buf, self.last_submit.clone(), |view| read(view, row, padded as usize, h as usize))
.map_err(|e| log::error!("gpu texture readback: {e}"))
.ok()?;
// Keep a few staging buffers per size (frames read back at the same size every time).
// Layers whose bounds change every frame read back at a new size each time: drop the
// other sizes once over budget, or mapped host memory grows for the whole render (#2081).
@@ -2070,7 +2256,7 @@ impl<'g> Enc<'g> {
v.push(buf);
}
}
Some(())
Some(out)
}
/// [`Enc::read_texture`] without waiting for the GPU (the browser's main thread can't): `done`
+5 -10
View File
@@ -292,19 +292,14 @@ fn extractor(e: &mut Enc, ctx: &EffectCtx, b: GBuf, a: Option<&AuxChannels>) ->
let names = [pr.s("red"), pr.s("green"), pr.s("blue"), pr.s("alpha")];
let (bp, wp) = (pr.f("blackPoint") as f32, pr.f("whitePoint") as f32);
let unmult = pr.b("unMult");
let own = |n: &str| match n.trim().to_ascii_uppercase().as_str() {
"R" | "RED" => Some(0u32),
"G" | "GREEN" => Some(1),
"B" | "BLUE" => Some(2),
"A" | "ALPHA" => Some(3),
_ => None,
};
let src_ch: Vec<Option<&[f32]>> = names.iter().map(|n| a.and_then(|a| if n.trim().is_empty() { None } else { a.get(n.trim()) })).collect();
if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(n) == Some(k as u32)) {
// `(copy)`: the input's own channel of the same position, never an aux channel.
let own = |k: usize, n: &str| effectcraft_effects::extractor_own_channel(k, n).and_then(|c| u32::try_from(c).ok());
let src_ch: Vec<Option<&[f32]>> = names.iter().map(|n| a.and_then(|a| effectcraft_effects::extractor_aux_channel(a, n))).collect();
if src_ch.iter().all(Option::is_none) && bp == 0.0 && wp == 1.0 && !unmult && names.iter().enumerate().all(|(k, n)| own(k, n) == Some(k as u32)) {
return Some(b);
}
// Per output channel: 0..3 = the layer's own channel, 4 = aux, 5 = default (0, alpha 1).
let sources: [u32; 4] = std::array::from_fn(|k| if src_ch[k].is_some() { 4 } else { own(names[k]).unwrap_or(5) });
let sources: [u32; 4] = std::array::from_fn(|k| if src_ch[k].is_some() { 4 } else { own(k, names[k]).unwrap_or(5) });
let plane = match a {
Some(a) if src_ch.iter().any(Option::is_some) => {
let tex = aux_texture(e, a, [src_ch[0], src_ch[1], src_ch[2], src_ch[3]])?;
+7
View File
@@ -84,6 +84,7 @@ mod memcap;
mod ops;
mod particles;
mod readback;
mod upload;
mod walk;
mod yuv;
@@ -432,6 +433,11 @@ impl Accelerator for Gpu {
Some(&self.auto)
}
fn frame_memory(&self) -> Option<effectcraft_render::FrameMemory> {
self.ctx.check_health().ok()?;
self.ctx.frame_memory()
}
fn render_end(&self) {
let (hits, misses) = self.ctx.pool_stats();
let pct = hits.saturating_mul(100) / hits.saturating_add(misses).max(1);
@@ -439,6 +445,7 @@ impl Accelerator for Gpu {
target: "effectcraft_gpu",
"gpu pool: {hits} working textures reused, {misses} allocated ({pct}% reuse) so far"
);
self.ctx.trim_uploads();
self.ctx.trim_pool();
}
+19
View File
@@ -40,6 +40,8 @@ pub(crate) struct MemCap {
floor: AtomicU64,
/// Set after the first over-cap sample where non-reclaimable memory alone exceeds the cap.
warned: AtomicBool,
/// The most the allocator held in any sample (what frames in flight are sized by, #3335).
peak: AtomicU64,
#[cfg(test)]
timeout_ns: AtomicU64,
}
@@ -59,6 +61,7 @@ impl MemCap {
waits: AtomicU64::new(0),
floor: AtomicU64::new(0),
warned: AtomicBool::new(false),
peak: AtomicU64::new(0),
#[cfg(test)]
timeout_ns: AtomicU64::new(crate::readback::TIMEOUT.as_nanos() as u64),
}
@@ -74,6 +77,7 @@ impl MemCap {
waits: AtomicU64::new(0),
floor: AtomicU64::new(0),
warned: AtomicBool::new(false),
peak: AtomicU64::new(0),
timeout_ns: AtomicU64::new(self.timeout_ns.load(Relaxed)),
}
}
@@ -82,6 +86,21 @@ impl MemCap {
self.cap
}
/// Remember an allocator sample: the peak only grows.
pub(crate) fn note(&self, allocated: u64) {
self.peak.fetch_max(allocated, Relaxed);
}
/// The most the allocator held in any sample so far.
pub(crate) fn peak(&self) -> u64 {
self.peak.load(Relaxed)
}
/// Whether frames have had to wait for memory (the device ran over the cap).
pub(crate) fn pressed(&self) -> bool {
self.waits.load(Relaxed) > 0
}
/// Whether to sample the allocator on this call.
pub(crate) fn due(&self) -> bool {
self.cap != 0 && self.calls.fetch_add(1, Relaxed).is_multiple_of(EVERY)
+41 -5
View File
@@ -367,20 +367,32 @@ thread_local! {
/// Validate every row before publishing tightly packed bytes.
pub(crate) fn packed(view: &[u8], row: usize, stride: usize, height: usize) -> Result<Vec<u8>, Error> {
if row == 0 || height == 0 || stride < row {
rows(view, row, stride, height)
}
/// The `height` rows of `row` bytes each, `stride` bytes apart in the mapped `view`, as tightly
/// packed `T`s: one copy, straight from the mapped rows into a vector reserved once (no
/// zero-filled buffer first, no intermediate byte vector). Every row is validated before any
/// is copied. A row that is not aligned for `T` is decoded value by value (same bytes).
pub(crate) fn rows<T: bytemuck::Pod>(view: &[u8], row: usize, stride: usize, height: usize) -> Result<Vec<T>, Error> {
let unit = std::mem::size_of::<T>().max(1);
if row == 0 || height == 0 || stride < row || !row.is_multiple_of(unit) {
return Err("invalid GPU readback layout".into());
}
let size = row.checked_mul(height).ok_or("GPU readback output size overflow")?;
let size = row.checked_mul(height).ok_or("GPU readback output size overflow")? / unit;
let end = (height - 1).checked_mul(stride).and_then(|s| s.checked_add(row)).ok_or("GPU readback mapped size overflow")?;
if view.len() < end {
return Err(format!("short GPU readback: {} bytes, needs {end}", view.len()));
}
let mut out = Vec::new();
let mut out: Vec<T> = Vec::new();
out.try_reserve_exact(size).map_err(|e| format!("allocating GPU readback output: {e}"))?;
for y in 0..height {
let start = y.checked_mul(stride).ok_or("GPU readback row overflow")?;
let bytes = view.get(start..start + row).ok_or("short GPU readback row")?;
out.extend_from_slice(bytes);
let bytes = start.checked_add(row).and_then(|e| view.get(start..e)).ok_or("short GPU readback row")?;
match bytemuck::try_cast_slice::<u8, T>(bytes) {
Ok(values) => out.extend_from_slice(values),
Err(_) => out.extend(bytes.chunks_exact(unit).map(bytemuck::pod_read_unaligned::<T>)),
}
}
Ok(out)
}
@@ -658,4 +670,28 @@ mod tests {
assert!(packed(&[0; 7], 4, 4, 2).is_err());
assert_eq!(packed(&[1, 2, 0, 0, 3, 4], 2, 4, 2).unwrap(), vec![1, 2, 3, 4]);
}
/// #3335: rows go from the mapped buffer straight into the destination type, skipping the
/// row padding, whether or not the mapped bytes are aligned for it.
#[test]
fn rows_copy_once_into_the_destination_type() {
// Two rows of two RGBA f32 pixels, padded to a 48-byte stride.
let px = |i: f32| [i, i + 0.25, i + 0.5, 1.0];
let want = vec![px(1.0), px(2.0), px(3.0), px(4.0)];
let mut view = Vec::new();
for r in want.chunks(2) {
view.extend_from_slice(bytemuck::cast_slice(r));
view.extend_from_slice(&[0xAB; 16]);
}
assert_eq!(rows::<[f32; 4]>(&view, 32, 48, 2).unwrap(), want);
// The same bytes one byte off any f32 alignment decode the same.
let mut shifted = vec![0u8];
shifted.extend_from_slice(&view);
assert_eq!(rows::<[f32; 4]>(&shifted[1..], 32, 48, 2).unwrap(), want);
// A row that isn't whole pixels, a short last row and a stride under the row fail.
assert!(rows::<[f32; 4]>(&view, 30, 48, 2).is_err());
assert!(rows::<[f32; 4]>(&view[..view.len() - 17], 32, 48, 2).is_err());
assert!(rows::<[f32; 4]>(&view, 32, 16, 2).is_err());
assert_eq!(rows::<u8>(&[1, 2, 9, 3, 4], 2, 3, 2).unwrap(), vec![1, 2, 3, 4]);
}
}
+45
View File
@@ -28,8 +28,10 @@ use crate::Gpu;
mod adjust;
mod adv3d;
mod bench_share;
mod deferred;
mod deferred_jobs;
mod footage_share;
mod fx_color;
mod fx_depth;
mod fx_distort;
@@ -1247,6 +1249,49 @@ fn readback_staging_stays_bounded_across_sizes() {
assert!(held <= (256 << 20) + 2 * 1024 * 16 * 376, "staging buffers hold {held} bytes");
}
/// #3335: a readback copies the mapped rows straight into the image (no zero-filled buffer,
/// no intermediate byte vector), skipping each row's padding: odd widths, sub-rectangles and
/// readbacks from several threads at once (each waiting for its own submission) return the
/// uploaded pixels exactly.
#[test]
fn single_copy_readback_returns_the_exact_pixels() {
let Some(g) = gpu() else { return };
if !g.ctx.can_wait() {
return;
}
let image = |w: u32, h: u32, seed: f32| {
let mut img = Image::new(w, h);
for (i, p) in img.data.iter_mut().enumerate() {
let v = i as f32 * 0.37 + seed;
*p = [v, -v, v * 1e-3, 1.0 / (1.0 + v)];
}
img
};
// 37 × 16 bytes = 592 per row: padded to 768 in the staging buffer.
let src = image(37, 5, 0.5);
let up = g.ctx.upload_image(&src).expect("upload");
let mut e = crate::context::Enc::new(&g.ctx);
let back = e.download(&up).expect("download");
assert_eq!((back.width, back.height), (37, 5));
assert_eq!(back.data, src.data);
let rect = e.read_rect(&up.texture, 3, 1, 5, 2, 16).expect("read_rect");
let want: Vec<[f32; 4]> = (1..3).flat_map(|y| (3..8).map(move |x| (x, y))).map(|(x, y)| src.data[src.idx(x, y)]).collect();
assert_eq!(rect, bytemuck::cast_slice::<[f32; 4], u8>(&want));
// Frames on several threads, each with its own encoder and submission.
std::thread::scope(|s| {
for t in 0..4 {
s.spawn(move || {
for k in 0..3 {
let src = image(61 + t, 7 + k, (t * 10 + k) as f32);
let up = g.ctx.upload_image(&src).expect("upload");
let back = crate::context::Enc::new(&g.ctx).download(&up).expect("download");
assert_eq!(back.data, src.data, "thread {t} frame {k}");
}
});
}
});
}
/// A static adjustment layer with a time-dependent effect (Noise) keeps its footprint key
/// across frames (the footprint is the source through the masks, not the effects), so the GPU
/// uploads nothing after the first frame; matches the CPU.
+2
View File
@@ -123,6 +123,7 @@ pub(crate) fn scene(p: &Project, cid: ItemId) -> Scene {
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect();
adv::scene_of(&r, &ctx, &run, (ctx.comp.width, ctx.comp.height))
@@ -315,6 +316,7 @@ pub(crate) fn run_of(p: &Project, cid: ItemId) -> (EvalCtx<'_>, Vec<&effectcraft
footage: None,
property_overrides: &[],
media_overrides: &[],
reads: None,
};
let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect();
(ctx, run)
+83
View File
@@ -0,0 +1,83 @@
//! Warm-frame benchmark of #3333 (run by hand): three layers of the same 1080p footage frame,
//! 32 bpc, a 30-frame sequence already decoded (as the media pool serves a warm frame), with a
//! layer cache as the Render Queue has. Prints the median and mean ms per frame and a hash of
//! every frame (to compare the pixels of two builds).
//!
//! `cargo test -p effectcraft-gpu --release --lib bench_share -- --ignored --nocapture`
use std::collections::HashMap;
use std::sync::Mutex;
use super::*;
struct Decoded {
frames: Mutex<HashMap<i64, Arc<Image>>>,
}
impl FootageSource for Decoded {
fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
let i = f.frame_rate.frame_at(t).clamp(0, 29);
let mut m = self.frames.lock().unwrap();
Some(m.entry(i).or_insert_with(|| Arc::new(pattern(item.0 as u32 + i as u32, f.width, f.height))).clone())
}
}
#[test]
#[ignore]
fn bench_share() {
let Some(g) = gpu() else { return };
let (w, h) = (1920, 1080);
let mut p = Project::default();
p.settings.bit_depth = BitDepth::Bpc32;
let comp = Comp::new(w, h, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
let cid = p.add_item("Comp", Label::Sandstone, None, ItemKind::Comp(comp.clone().into()));
let f = Footage {
path: "seq.exr".into(),
kind: FootageKind::Sequence,
width: w,
height: h,
pixel_aspect: 1.0,
frame_rate: FrameRate::FPS_30,
duration: Tick::from_seconds_f64(1.0),
has_video: true,
alpha: AlphaMode::Premultiplied,
..Default::default()
};
let fid = p.add_item("Sequence", Label::Lavender, None, ItemKind::Footage(f));
for i in 0..3 {
let mut l = build::layer(&mut p, &comp, "Sequence", LayerSource::Footage { item: fid }, (w, h), None);
if i > 0 {
l.blend_mode = BlendMode::Add;
set(&mut l, "transform/opacity", Value::Scalar(25.0));
}
p.comp_mut(cid).unwrap().layers.insert(0, l);
}
let src = Decoded { frames: Mutex::new(HashMap::new()) };
let frames = 30;
let mut runs = vec![];
let mut hashes = vec![];
for pass in 0..4 {
let cache = effectcraft_render::cache::LayerCache::new(1 << 30);
let mut r = Renderer::new(&p, &src, RenderOpts { backend: Backend::Gpu, ..Default::default() });
r.accel = Some(g);
r.cache = Some(&cache);
for k in 0..frames {
let t0 = std::time::Instant::now();
let img = g.render(&r, cid, Tick::from_seconds_f64(k as f64 / 30.0)).expect("the GPU renders the frame");
let ms = t0.elapsed().as_secs_f64() * 1e3;
if pass > 0 {
runs.push(ms);
}
if pass == 1 {
use std::hash::{Hash, Hasher};
let mut hs = std::collections::hash_map::DefaultHasher::new();
img.data.iter().for_each(|px| px.map(f32::to_bits).hash(&mut hs));
hashes.push(hs.finish());
}
}
}
runs.sort_by(f64::total_cmp);
let mean = runs.iter().sum::<f64>() / runs.len() as f64;
eprintln!("bench_share: {} frames, median {:.1} ms, mean {:.1} ms, min {:.1} ms", runs.len(), runs[runs.len() / 2], mean, runs[0]);
eprintln!("bench_share frame hashes: {hashes:016x?}");
}
+233
View File
@@ -0,0 +1,233 @@
//! #3333: a footage frame uploads once per frame and its texture is shared by every layer and
//! effect input that shows it; the frame is bit-identical to drawing each layer from its own copy.
use std::collections::HashMap;
use std::sync::Mutex;
use super::*;
use crate::walk::SHARED_PICTURES;
/// Keeps decoded frames as the media pool does: the same frame (at the same size) of the same
/// footage is the same `Arc` while kept. Counts the frames it decodes.
#[derive(Default)]
struct Kept {
frames: Mutex<HashMap<(ItemId, u32, u32, i64, Option<(u32, u32)>), Arc<Image>>>,
}
impl Kept {
fn get(&self, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>) -> Option<Arc<Image>> {
let i = f.frame_rate.frame_at(t);
let mut m = self.frames.lock().unwrap();
let img = m.entry((item, f.width, f.height, i, size)).or_insert_with(|| {
let full = pattern(item.0 as u32 + i as u32 * 7 + f.width, f.width, f.height);
Arc::new(match size {
Some((w, h)) => effectcraft_raster::resample(&full, w, h),
None => full,
})
});
Some(img.clone())
}
}
impl FootageSource for Kept {
fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option<Arc<Image>> {
self.get(item, f, t, None)
}
fn frame_at_size(&self, item: ItemId, f: &Footage, t: Tick, width: u32, height: u32) -> Option<Arc<Image>> {
self.get(item, f, t, Some((width, height)))
}
}
/// One footage item (a sequence: a new picture every frame) shown by `layers` layers, each
/// placed differently (the first fills the comp).
fn shared_scene(depth: BitDepth, layers: usize) -> (Scene, ItemId) {
let mut s = Scene::new(depth);
let f = Footage {
path: "seq.exr".into(),
kind: FootageKind::Sequence,
width: 97,
height: 61,
pixel_aspect: 1.0,
frame_rate: FrameRate::FPS_30,
duration: Tick::from_seconds_f64(2.0),
has_video: true,
alpha: AlphaMode::Premultiplied,
..Default::default()
};
let fid = s.p.add_item("Sequence", Label::Lavender, None, ItemKind::Footage(f));
for i in 0..layers {
let mut l = build::layer(&mut s.p, &s.comp, "Sequence", LayerSource::Footage { item: fid }, (97, 61), None);
if i > 0 {
set(&mut l, "transform/rotation", Value::Scalar(11.0 * i as f64));
set(&mut l, "transform/scale", Value::Vec3([70.0 + 9.0 * i as f64, 64.0, 100.0]));
set(&mut l, "transform/position", v3(30.0 + 13.0 * i as f64, 25.0 + 4.0 * i as f64));
set(&mut l, "transform/opacity", Value::Scalar(80.0));
l.blend_mode = if i % 2 == 0 { BlendMode::Screen } else { BlendMode::Add };
}
s.push(l);
}
(s, fid)
}
/// The GPU frame with shared pictures on and off, and the number of uploads of each.
fn render_both(g: &crate::Gpu, s: &Scene, src: &Kept, opts: RenderOpts, t: Tick) -> (Image, Image, u64, u64) {
let mut r = Renderer::new(&s.p, src, RenderOpts { backend: Backend::Gpu, ..opts });
r.accel = Some(g);
let before = g.ctx.transfer_stats().uploads;
let on = g.render(&r, s.cid, t).expect("the GPU renders the frame");
let uploads = g.ctx.transfer_stats().uploads - before;
SHARED_PICTURES.store(false, std::sync::atomic::Ordering::Relaxed);
let before = g.ctx.transfer_stats().uploads;
let off = g.render(&r, s.cid, t);
let copied = g.ctx.transfer_stats().uploads - before;
SHARED_PICTURES.store(true, std::sync::atomic::Ordering::Relaxed);
(on, off.expect("the GPU renders the frame from per-layer copies"), uploads, copied)
}
fn check_cpu(label: &str, s: &Scene, src: &Kept, opts: RenderOpts, t: Tick, gpu_img: &Image) {
let cpu = Renderer::new(&s.p, src, RenderOpts { backend: Backend::Cpu, ..opts }).comp_frame_cpu(s.cid, t);
let mut d = diff(&cpu, gpu_img, tolerance(s.p.settings.bit_depth));
d.quantized = s.p.settings.bit_depth != BitDepth::Bpc32;
check(label, Some(d), 0.0);
}
/// Three layers showing the same frame upload it once; a fourth that runs an effect on it
/// (resident on the GPU) reads the same texture as its input. Every frame of the sequence
/// uploads its own picture once.
#[test]
fn three_layers_of_one_footage_frame_upload_once() {
let Some(g) = gpu() else { return };
let src = Kept::default();
let (mut s, fid) = shared_scene(BitDepth::Bpc32, 3);
for k in 0..3 {
let t = Tick::from_seconds_f64(k as f64 / 30.0);
let (on, off, uploads, copied) = render_both(g, &s, &src, opts(), t);
assert_eq!(uploads, 1, "frame {k}: three layers of one footage frame uploaded {uploads} times");
assert_eq!(copied, 3, "frame {k}: per-layer copies upload once per layer");
assert!(on.data == off.data, "frame {k}: the shared picture changed the frame");
check_cpu(&format!("shared picture frame {k}"), &s, &src, opts(), t, &on);
}
// An effect input: the blur reads the shared texture. Alone, the blurred layer uploads
// its picture (and whatever the blur itself uploads); the three plain layers showing the
// same frame add no upload.
let mut fx = build::layer(&mut s.p, &s.comp, "Blurred", LayerSource::Footage { item: fid }, (97, 61), None);
set(&mut fx, "transform/position", v3(20.0, 20.0));
s.effect(&mut fx, "ec.blur.gaussian", &[("blurriness", n(3.0))]);
let (mut alone, _) = shared_scene(BitDepth::Bpc32, 0);
alone.p = s.p.clone();
alone.p.comp_mut(alone.cid).unwrap().layers = vec![fx.clone()].into();
let (_, _, blur_uploads, _) = render_both(g, &alone, &src, opts(), Tick::from_seconds_f64(0.4));
s.push(fx);
let t = Tick::from_seconds_f64(0.5);
let (on, off, uploads, _) = render_both(g, &s, &src, opts(), t);
assert_eq!(uploads, blur_uploads, "three layers and an effect input of one footage frame uploaded {uploads} times, the effect layer alone {blur_uploads}");
assert!(on.data == off.data, "the shared picture changed the frame with an effect input");
check_cpu("shared picture with an effect input", &s, &src, opts(), t, &on);
}
/// The frame is bit-identical with shared pictures on and off wherever they apply or not: every
/// bit depth, colour management and linear blending, half resolution, a track matte, Preserve
/// Transparency, motion blur, masks, layer styles and a proxy.
#[test]
fn shared_pictures_do_not_change_the_frame() {
let Some(g) = gpu() else { return };
let src = Kept::default();
let t = Tick::from_seconds_f64(0.2);
for depth in [BitDepth::Bpc8, BitDepth::Bpc16, BitDepth::Bpc32] {
let mut cases: Vec<(String, Scene, RenderOpts)> = vec![];
cases.push(("plain".into(), shared_scene(depth, 3).0, opts()));
cases.push(("half resolution".into(), shared_scene(depth, 3).0, RenderOpts { scale: 0.5, ..opts() }));
for (ws, blend_linear) in [(Some(ColorSpace::Rec709), false), (None, true)] {
let (mut s, _) = shared_scene(depth, 3);
s.p.settings.working_space = ws;
s.p.settings.blend_linear = blend_linear;
cases.push((format!("colour {ws:?} linear blending {blend_linear}"), s, opts()));
}
{
let (mut s, _) = shared_scene(depth, 3);
let comp = s.p.comp_mut(s.cid).unwrap();
comp.enable_motion_blur = true;
let matte = comp.layers[0].id;
comp.layers[1].track_matte = Some(TrackMatte { layer: matte, kind: MatteKind::Luma });
comp.layers[2].preserve_transparency = true;
comp.layers[2].switches.motion_blur = true;
let pos = comp.layers[2].props.prop_mut("transform/position").unwrap();
pos.keys = vec![Keyframe::new(Tick::ZERO, v3(10.0, 20.0)), Keyframe::new(Tick::from_seconds_f64(0.5), v3(90.0, 45.0))];
s.comp.enable_motion_blur = true;
cases.push(("matte, preserve transparency, motion blur".into(), s, opts()));
}
{
let (mut s, _) = shared_scene(depth, 3);
let mut next = s.p.next_id;
let m = build::mask(&mut Ids(&mut next), "Mask 1", ShapePath::ellipse([40.0, 30.0], 60.0, 40.0), MaskMode::Add, [255, 255, 0]);
let comp = s.p.comp_mut(s.cid).unwrap();
comp.layers[0].props.sub_mut("masks").unwrap().children.push(m.into());
effectcraft_project::styles::add_style(&mut comp.layers[1], &mut Ids(&mut next), "dropShadow", &s.comp.global_light, true).unwrap();
s.p.next_id = next;
cases.push(("mask and layer style".into(), s, opts()));
}
{
let (mut s, fid) = shared_scene(depth, 3);
let proxy = Footage { path: "proxy.png".into(), kind: FootageKind::Still, width: 48, height: 30, has_video: true, ..Default::default() };
s.p.item_mut(fid).unwrap().proxy = Some(Box::new(effectcraft_project::Proxy { footage: proxy, enabled: true }));
cases.push(("proxy".into(), s, opts()));
}
for (label, s, o) in cases {
let (on, off, _, _) = render_both(g, &s, &src, o, t);
assert!(on.data == off.data, "{label} {depth:?}: the shared picture changed the frame");
check_cpu(&format!("{label} {depth:?}"), &s, &src, o, t, &on);
}
}
}
/// The picture cache keeps no texture of a picture the footage source dropped.
#[test]
fn picture_cache_drops_textures_of_dropped_pictures() {
let Some(g) = gpu() else { return };
let before = g.ctx.picture_entries();
for i in 0..6 {
let img = Arc::new(pattern(i, 64, 64));
let up = g.ctx.upload_picture(&img).expect("upload");
let again = g.ctx.upload_picture(&img).expect("upload");
assert!(up.texture == again.texture, "the same picture uploaded twice");
drop((up, again, img));
}
let kept = g.ctx.picture_entries();
assert!(kept <= before + 1, "the picture cache kept {kept} textures of dropped pictures (had {before})");
}
/// Large uploads copy through reused staging buffers (rows padded to the copy alignment) with
/// the same pixels as `Queue::write_texture`; the buffers stay within their budget, and a
/// clear while one is in flight forgets it.
#[test]
fn staging_uploads_keep_pixels_and_reuse_buffers() {
let Some(g) = gpu() else { return };
let wait = || {
let _ = g.ctx.poll_readbacks(wgpu::PollType::wait_indefinitely());
};
// 300 px: 4800-byte rows, padded to 4864.
for (w, h) in [(300u32, 300u32), (512, 600)] {
let before = g.ctx.upload_staging().copies();
for i in 0..4 {
let img = pattern(40 + i, w, h);
let up = g.ctx.upload_image(&img).expect("upload");
let mut e = crate::context::Enc::new(&g.ctx);
let back = e.download(&up).expect("readback");
assert!(back.data.iter().flatten().map(|v| v.to_bits()).eq(img.data.iter().flatten().map(|v| v.to_bits())), "{w}×{h} upload {i} changed pixels");
drop((up, e));
wait();
}
assert!(g.ctx.upload_staging().copies() - before >= 2, "{w}×{h}: no upload went through a reused staging buffer");
assert!(g.ctx.upload_staging().bytes() <= 192 << 20);
}
// A clear with a copy in flight: nothing kept, and later uploads still work.
let img = pattern(7, 300, 300);
let up = g.ctx.upload_image(&img).expect("upload");
g.ctx.upload_staging().clear();
wait();
assert_eq!(g.ctx.upload_staging().bytes(), 0, "a cleared staging kept its buffers");
let mut e = crate::context::Enc::new(&g.ctx);
assert!(e.download(&up).expect("readback").data == img.data);
let again = g.ctx.upload_image(&img).expect("upload after a clear");
assert!(e.download(&again).expect("readback").data == img.data);
}
+3
View File
@@ -288,4 +288,7 @@ fn extractor() {
);
case("ec.3d.extractor", &[("red", Value::Str("A".into())), ("blackPoint", n(0.2)), ("whitePoint", n(0.8)), ("clip", on())]);
case("ec.3d.extractor", &[("green", Value::Str("".into())), ("alpha", Value::Str("nothing".into()))]);
// `(copy)` reads the input's own channel, not the footage's R plane of aux_scene.
let copy = || Value::Str(effectcraft_effects::EXTRACTOR_COPY.into());
case("ec.3d.extractor", &[("red", copy()), ("green", Value::Str("Coverage".into())), ("alpha", copy())]);
}
+264
View File
@@ -0,0 +1,264 @@
//! Footage uploads (#3333): each decoded footage picture uploads once and its texture is shared
//! by every layer and effect input that shows it ([`Pictures`]), and large uploads copy through
//! staging buffers that are kept mapped and reused ([`Staging`]) instead of one that
//! `Queue::write_texture` allocates and frees per upload.
use std::collections::HashMap;
use std::sync::{Arc, Mutex, PoisonError, Weak};
use effectcraft_raster::Image;
use crate::context::GpuImage;
/// Bytes of picture textures kept after their frame (a sequence's next frame takes the oldest
/// one's texture back from the pool), at most a quarter of the context's upload budget. The
/// newest picture is always kept, whatever its size.
pub(crate) const PICTURE_BUDGET: usize = 256 << 20;
/// Uploads from this size go through [`Staging`]; smaller ones through `Queue::write_texture`.
const STAGING_MIN: u64 = 1 << 20;
/// Bytes of staging buffers that may exist at once (mapped and ready, or in flight).
const STAGING_BUDGET: u64 = 192 << 20;
struct Picture {
img: Weak<Image>,
tex: GpuImage,
bytes: usize,
last_use: u64,
}
/// Uploaded footage pictures by the identity of the decoded frame (its `Arc`): the footage
/// source serves one `Arc` per (file, frame, size, colour curve) while it keeps the frame, and
/// a new one when the file changed, so the same `Arc` means the same pixels. The `Weak` keeps
/// the allocation, so a key cannot be reused by another picture while its entry exists.
#[derive(Default)]
pub(crate) struct Pictures {
map: HashMap<usize, Picture>,
bytes: usize,
clock: u64,
}
impl Pictures {
/// The texture of `img` if it was uploaded and is still kept.
pub(crate) fn get(&mut self, img: &Arc<Image>) -> Option<GpuImage> {
self.clock += 1;
let now = self.clock;
let e = self.map.get_mut(&(Arc::as_ptr(img) as usize))?;
if !e.img.upgrade().is_some_and(|i| Arc::ptr_eq(&i, img)) {
return None;
}
e.last_use = now;
Some(e.tex.clone())
}
/// Keep `tex`, the upload of `img`. Pictures the footage source no longer holds go at once
/// (nothing can ask for them again), then the least recently used over [`PICTURE_BUDGET`],
/// never `img` itself. A dropped texture goes back to the pool once no open encoder can
/// still read it.
pub(crate) fn insert(&mut self, img: &Arc<Image>, tex: GpuImage, budget: usize) {
let key = Arc::as_ptr(img) as usize;
let bytes = img.data.len().saturating_mul(16);
self.clock += 1;
let entry = Picture { img: Arc::downgrade(img), tex, bytes, last_use: self.clock };
if let Some(old) = self.map.insert(key, entry) {
self.bytes = self.bytes.saturating_sub(old.bytes);
}
self.bytes = self.bytes.saturating_add(bytes);
let dead: Vec<usize> = self.map.iter().filter(|(_, e)| e.img.strong_count() == 0).map(|(k, _)| *k).collect();
for k in dead {
if let Some(e) = self.map.remove(&k) {
self.bytes = self.bytes.saturating_sub(e.bytes);
}
}
while self.bytes > budget {
let Some(k) = self.map.iter().filter(|(k, _)| **k != key).min_by_key(|(_, e)| e.last_use).map(|(k, _)| *k) else { break };
if let Some(e) = self.map.remove(&k) {
self.bytes = self.bytes.saturating_sub(e.bytes);
}
}
}
#[cfg(test)]
pub(crate) fn len(&self) -> usize {
self.map.len()
}
pub(crate) fn clear(&mut self) {
self.map.clear();
self.bytes = 0;
}
}
#[derive(Default)]
struct StagingState {
/// Buffers mapped for writing and not in use, each mapped whole.
ready: Vec<wgpu::Buffer>,
/// Bytes of every staging buffer that exists (ready, being mapped or in flight).
total: u64,
/// Raised by [`Staging::clear`]: a buffer of an earlier generation is dropped when its
/// mapping completes.
generation: u64,
/// Uploads copied through a staging buffer (diagnostics, tests).
copies: u64,
}
/// Reused upload staging buffers (`MAP_WRITE | COPY_SRC`). A buffer is written while mapped,
/// unmapped, copied into the texture by a command buffer submitted at once, and mapped again
/// when that submission is done (`CommandEncoder::map_buffer_on_submit`; the callback runs when
/// the device is polled, as every frame does). A new buffer is mapped asynchronously too (never
/// `mapped_at_creation`, which fails hard on drivers that report a failed allocation as an
/// internal error, #591), so the upload that made it, and any upload while no ready buffer fits
/// or the budget is spent, goes through `Queue::write_texture` instead.
#[derive(Default)]
pub(crate) struct Staging {
state: Arc<Mutex<StagingState>>,
}
/// `bytes_per_row` of a staging copy: rows padded to `COPY_BYTES_PER_ROW_ALIGNMENT`.
fn padded_row(width: u32) -> Option<u64> {
let row = u64::from(width).checked_mul(16)?;
let a = u64::from(wgpu::COPY_BYTES_PER_ROW_ALIGNMENT);
row.div_ceil(a).checked_mul(a)
}
impl Staging {
fn lock(&self) -> std::sync::MutexGuard<'_, StagingState> {
self.state.lock().unwrap_or_else(PoisonError::into_inner)
}
/// Bytes of staging buffers that exist (see [`STAGING_BUDGET`]).
pub(crate) fn bytes(&self) -> u64 {
self.lock().total
}
/// Uploads copied through a staging buffer so far.
#[cfg(test)]
pub(crate) fn copies(&self) -> u64 {
self.lock().copies
}
/// Forget every staging buffer (after a failed frame, #1319): those in flight are dropped
/// when their mapping completes.
pub(crate) fn clear(&self) {
let mut s = self.lock();
s.ready.clear();
// (buffers still in flight are no longer counted: they drop when their mapping ends)
s.total = 0;
s.generation = s.generation.wrapping_add(1);
}
/// The callback of a write mapping of `buffer`: once mapped it is ready for the next
/// upload; it is dropped (and its bytes uncounted) when mapping failed, and dropped when
/// the staging was cleared meanwhile (whose [`Staging::clear`] stopped counting it).
fn on_mapped(&self, buffer: &wgpu::Buffer, generation: u64) -> impl FnOnce(Result<(), wgpu::BufferAsyncError>) + wgpu::WasmNotSend + 'static {
let state = Arc::downgrade(&self.state);
let buffer = buffer.clone();
move |res: Result<(), wgpu::BufferAsyncError>| {
let Some(state) = state.upgrade() else { return };
let mut s = state.lock().unwrap_or_else(PoisonError::into_inner);
if s.generation != generation {
return;
}
if res.is_ok() {
s.ready.push(buffer);
} else {
s.total = s.total.saturating_sub(buffer.size());
}
}
}
/// Copy `img` into `texture` (its size) through a ready staging buffer, recorded and
/// submitted at once: before anything an open encoder records later, as
/// `Queue::write_texture` would. `false`: not done (small, no ready buffer fits, or the
/// mapping failed); the caller writes it through the queue. Starts mapping a new buffer for
/// later uploads when none fits and the budget allows.
pub(crate) fn write(&self, device: &wgpu::Device, queue: &wgpu::Queue, img: &Image, texture: &wgpu::Texture) -> bool {
let (w, h) = (img.width, img.height);
let row = w as usize * 16;
let Some(stride) = padded_row(w) else { return false };
let (Some(need), Ok(stride32)) = (stride.checked_mul(u64::from(h)), u32::try_from(stride)) else { return false };
if need < STAGING_MIN || img.data.len() != w as usize * h as usize {
return false;
}
let (buffer, generation) = {
let mut s = self.lock();
let generation = s.generation;
// The smallest ready buffer that holds it.
let best = s.ready.iter().enumerate().filter(|(_, b)| b.size() >= need).min_by_key(|(_, b)| b.size()).map(|(i, _)| i);
match best {
Some(i) => (Some(s.ready.swap_remove(i)), generation),
None => {
// Ready buffers too small for it make room (a size that no longer occurs).
while s.total.saturating_add(need) > STAGING_BUDGET
&& let Some(b) = s.ready.pop()
{
s.total = s.total.saturating_sub(b.size());
}
if s.total.saturating_add(need) <= STAGING_BUDGET {
s.total += need;
drop(s);
let b = device.create_buffer(&wgpu::BufferDescriptor {
label: Some("upload staging"),
size: need,
usage: wgpu::BufferUsages::MAP_WRITE | wgpu::BufferUsages::COPY_SRC,
mapped_at_creation: false,
});
let done = self.on_mapped(&b, generation);
b.map_async(wgpu::MapMode::Write, .., done);
}
(None, generation)
}
}
};
let Some(buffer) = buffer else { return false };
self.lock().copies += 1;
{
let Ok(mut view) = buffer.get_mapped_range_mut(..need) else {
// Not mapped after all: give it up (its bytes too).
let mut s = self.lock();
if s.generation == generation {
s.total = s.total.saturating_sub(buffer.size());
}
return false;
};
let src: &[u8] = bytemuck::cast_slice(&img.data);
let stride = stride as usize;
if stride == row {
view.slice(..src.len()).copy_from_slice(src);
} else {
for (y, line) in src.chunks_exact(row).enumerate() {
let at = y * stride;
view.slice(at..at + row).copy_from_slice(line);
}
}
}
buffer.unmap();
let mut enc = device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: Some("upload") });
enc.copy_buffer_to_texture(
wgpu::TexelCopyBufferInfo {
buffer: &buffer,
layout: wgpu::TexelCopyBufferLayout { offset: 0, bytes_per_row: Some(stride32), rows_per_image: Some(h) },
},
wgpu::TexelCopyTextureInfo { texture, mip_level: 0, origin: wgpu::Origin3d::ZERO, aspect: wgpu::TextureAspect::All },
wgpu::Extent3d { width: w, height: h, depth_or_array_layers: 1 },
);
let done = self.on_mapped(&buffer, generation);
enc.map_buffer_on_submit(&buffer, wgpu::MapMode::Write, .., done);
queue.submit([enc.finish()]);
true
}
}
#[cfg(test)]
mod tests {
use super::padded_row;
#[test]
fn staging_rows_are_padded_to_the_copy_alignment() {
assert_eq!(padded_row(1920), Some(30720));
assert_eq!(padded_row(97), Some(1792));
assert_eq!(padded_row(16), Some(256));
assert_eq!(padded_row(1), Some(256));
}
}
+69 -6
View File
@@ -241,6 +241,9 @@ fn draw_layer_on(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas
if let Some(done) = draw_resident_fx(e, r, ctx, layer, canvas, opacity) {
return done;
}
if let Some(done) = draw_picture(e, r, ctx, layer, canvas, opacity) {
return done;
}
let Some(st) = r.styled_layer(ctx, layer) else { return Some(()) };
if st.plain {
return composite_layer(e, r, ctx, layer, &st.body, canvas, opacity);
@@ -286,14 +289,28 @@ fn draw_resident_fx(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, can
{
return None;
}
let input = r.layer_input(ctx, layer, 0)?;
if input.img.is_empty() {
return Some(Some(()));
}
let Some(src) = e.g.upload_buf(&input) else { return Some(None) };
// The decoded footage picture itself when it is the layer's input (uploaded once per frame
// and shared with every layer that shows it), else the cached input buffer.
let (src, offset, scale) = match picture(r, ctx, layer) {
Some(p) => {
if p.img.is_empty() {
return Some(Some(()));
}
let Some(src) = e.g.upload_picture(&p.img) else { return Some(None) };
(src, p.effect_offset, p.scale)
}
None => {
let input = r.layer_input(ctx, layer, 0)?;
if input.img.is_empty() {
return Some(Some(()));
}
let Some(src) = e.g.upload_buf(&input) else { return Some(None) };
(src, input.offset, input.scale)
}
};
let pipe = r.pipe();
// `layer_input` is in effect space: run the stack there, then back to layer space.
let mut fx = crate::effects::GpuFx::new(e, src, input.offset, input.scale, pipe.levels);
let mut fx = crate::effects::GpuFx::new(e, src, offset, scale, pipe.levels);
let origin = r.run_layer_effects_on(ctx, layer, &mut fx);
let Some((mut img, mut offset, scale)) = fx.finish() else { return Some(None) };
offset = [offset[0] - origin[0] * scale, offset[1] - origin[1] * scale];
@@ -308,6 +325,52 @@ fn draw_resident_fx(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, can
Some(place_src(e, r, ctx, layer, &shape, img, canvas, layer.blend_mode, opacity, pipe.levels).map(|c| *canvas = c))
}
/// Shared footage pictures ([`picture`]); tests turn it off to compare.
pub(crate) static SHARED_PICTURES: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(true);
/// The layer's source as the decoded footage picture (`Renderer::footage_picture`), unless
/// sharing is off or per-layer timings are being recorded (those time the layer-cache path).
fn picture(r: &Renderer, ctx: &EvalCtx, layer: &Layer) -> Option<effectcraft_render::FootagePicture> {
if !SHARED_PICTURES.load(std::sync::atomic::Ordering::Relaxed) || r.profile.is_some() {
return None;
}
r.footage_picture(ctx, layer)
}
/// A footage layer without effects or layer styles whose source is the decoded picture as it
/// is (`Renderer::footage_picture`) and needs no conversion to the blending space: placed
/// straight from the picture's texture, uploaded once per frame and shared by every layer that
/// shows the same frame, instead of from a copy per layer in the layer cache, uploaded per
/// layer. The pixels are the same: the layer-cache buffer is that picture, copied. `None`: not
/// this kind of layer; `Some(None)`: the GPU failed.
fn draw_picture(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, canvas: &mut GpuImage, opacity: f32) -> Option<Option<()>> {
if (layer.switches.effects && layer.effects().is_some_and(|f| !f.children.is_empty())) || styles::active(ctx, layer) || r.pipe().to_blend().is_some() {
return None;
}
let p = picture(r, ctx, layer)?;
if p.img.is_empty() {
return Some(Some(()));
}
let Some(src) = e.g.upload_picture(&p.img) else { return Some(None) };
// The placement only reads the buffer's size, offset and scale.
let shape = Buf { img: Image { width: p.img.width, height: p.img.height, data: Vec::new() }, offset: [0.0; 2], scale: p.scale };
Some(composite_src(e, r, ctx, layer, &shape, src, canvas, opacity))
}
/// [`composite_layer`] of a GPU image `src` shaped like `shape`.
#[allow(clippy::too_many_arguments)]
fn composite_src(e: &mut Enc, r: &Renderer, ctx: &EvalCtx, layer: &Layer, shape: &Buf, src: GpuImage, canvas: &mut GpuImage, opacity: f32) -> Option<()> {
if opacity <= 0.0 {
return Some(());
}
if r.track_matte(ctx, layer).is_none() && !layer.preserve_transparency {
*canvas = place_src(e, r, ctx, layer, shape, src, canvas, layer.blend_mode, opacity, r.pipe().levels)?;
return Some(());
}
let iso = place_src(e, r, ctx, layer, shape, src, &e.zeros(canvas.width, canvas.height), BlendMode::Normal, 1.0, None)?;
composite_iso(e, r, ctx, layer, iso, canvas, opacity)
}
/// A 2D precomp layer, its nested comp rendered resident on the GPU (`walk::render` one level
/// down) instead of by the CPU compositor and uploaded every frame (`Renderer::layer_source`
/// renders nested comps on the CPU: depth > 0). Then the layer's effects (resident), its
+1 -1
View File
@@ -147,7 +147,7 @@ pub(crate) fn record(e: &mut Enc, yp: &YuvPipe, img: &GpuImage, bg: [f32; 3]) ->
pub(crate) fn convert(e: &mut Enc, yp: &YuvPipe, img: &GpuImage, bg: [f32; 3]) -> Option<Vec<u8>> {
let stg = record(e, yp, img, bg)?;
let n = planes_len(img.width, img.height) as usize;
e.g.read_buffer(&stg, n, n, 1).map_err(|err| log::error!(target: "effectcraft_gpu", "yuv420 readback: {err}")).ok()
e.g.read_buffer_after(&stg, e.last_submission(), n, n, 1).map_err(|err| log::error!(target: "effectcraft_gpu", "yuv420 readback: {err}")).ok()
}
#[cfg(test)]