Roto Brush, Object Matte: decode frozen mattes only at the layer's full-resolution grid

crates/effects/src/object_matte.rs

@@ -301,7 +301,8 @@ pub fn frozen_alpha(params: &Params, layer_size: [f64; 2], fps: f64, f: i64, w:
     let fr = fc.frames.get(&f)?;
     let [fw, fh] = fc.size;
     let expected = fw.checked_mul(fh)?;
-    if fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
+    // Only the layer's own full-resolution grid (see Roto Brush's frozen mattes).
+    if fc.size != roto::frozen_grid(layer_size) || fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
         return None;
     }
     let a = rle::decode_bounded(&fr.a, expected)?;
@@ -719,6 +720,12 @@ mod tests {
         p.values.insert("objectPoint".into(), Value::Vec2([40.0, 40.0]));
         p.values.insert(FROZEN.into(), Value::Str(fc2.to_json()));
         assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
+        // A compact plane claiming a size other than the layer's grid is not decoded.
+        let mut fc3 = fc.clone();
+        fc3.size = [4000, 4000];
+        fc3.frames.insert(2, roto::freeze_frame(&vec![1.0; 4000 * 4000], &[]));
+        p.values.insert(FROZEN.into(), Value::Str(fc3.to_json()));
+        assert_eq!(frozen_alpha(&p, [100.0, 80.0], 25.0, 2, 100, 80, None), None);
     }
 
     /// Mattes frozen with a model stay usable in a session where that model is not loaded (a

crates/effects/src/roto.rs

@@ -372,10 +372,22 @@ pub fn purge() {
     c.bytes = 0;
 }
 
+/// The pixel size of the layer-space grid at `scale` (`round(size · scale)`, at least 1).
+fn grid_size(layer_size: [f64; 2], scale: f64) -> (u32, u32) {
+    (((layer_size[0] * scale).round() as u32).max(1), ((layer_size[1] * scale).round() as u32).max(1))
+}
+
+/// The full-resolution layer grid frozen mattes are stored at. A frozen cache claiming any
+/// other size did not come from freezing this layer and is not decoded.
+pub fn frozen_grid(layer_size: [f64; 2]) -> [usize; 2] {
+    let (w, h) = grid_size(layer_size, 1.0);
+    [w as usize, h as usize]
+}
+
 /// The layer-space grid (`round(size · scale)` pixels) of a buffer.
 pub fn layer_grid(buf: &Buf, layer_size: [f64; 2]) -> Image {
     let s = if buf.scale > 0.0 { buf.scale } else { 1.0 };
-    let (w, h) = (((layer_size[0] * s).round() as u32).max(1), ((layer_size[1] * s).round() as u32).max(1));
+    let (w, h) = grid_size(layer_size, s);
     let (ox, oy) = (buf.offset[0].round() as i64, buf.offset[1].round() as i64);
     let mut img = Image::new(w, h);
     img.rows_mut().for_each(|(y, row)| {
@@ -1197,7 +1209,9 @@ fn matte_for(ctx: &EffectCtx, f: i64, img: &Arc<Image>, scale: f64) -> Option<(V
     {
         let [fw, fh] = fc.size;
         let expected = fw.checked_mul(fh)?;
-        if fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
+        // Only the layer's own full-resolution grid: a project cannot make a tiny layer decode
+        // (and convert to floats) planes as large as the RLE allows.
+        if fc.size != frozen_grid(ctx.layer_size) || fw == 0 || fh == 0 || expected > rle::MAX_PLANE_BYTES {
             return None;
         }
         let a = rle::decode_bounded(&fr.a, expected)?;
@@ -1890,6 +1904,18 @@ mod tests {
         }
     }
 
+    /// Frozen mattes are only decoded at the size the freeze pass stores: the layer grid at
+    /// full resolution.
+    #[test]
+    fn frozen_grid_is_the_full_resolution_layer_grid() {
+        for size in [[100.0, 80.0], [99.6, 0.2], [1.0, 1.0]] {
+            let buf = Buf { img: Image::new(3, 3), offset: [0.0; 2], scale: 1.0 };
+            let g = layer_grid(&buf, size);
+            assert_eq!(frozen_grid(size), [g.width as usize, g.height as usize]);
+        }
+        assert_eq!(frozen_grid([100.0, 80.0]), [100, 80]);
+    }
+
     #[test]
     fn saved_tracking_decoders_validate_dimensions_and_plane_lengths() {
         let valid = rle::encode(&[255, 0, 255, 0]);