VP9 encoder and WebAssembly plug-ins: refuse short planes, limit memories and tables (#2995)

* VP9 encoder and WebAssembly plug-ins: refuse short planes, limit memories and tables

encode_frame sized and filled superblock-aligned coding planes from the
configured dimensions (up to 65536 x 65536) even when the planes passed in
held a few bytes, making up the missing samples; it now encodes nothing for
planes shorter than the frame. The plug-in sandbox capped each linear
memory but not how many memories or tables a module declares, or their
elements; one memory, four tables and 65,536 elements per table are
allowed.

* address review: checked size arithmetic in the encoder's short-plane guard

crates/plugin/src/wasm.rs

@@ -13,6 +13,8 @@ const FUEL_BASE: u64 = 50_000_000;
 const FUEL_PER_PIXEL: u64 = 20_000;
 /// The most linear memory a plug-in instance may have (a 4K float frame is ~130 MB).
 const MAX_MEMORY: usize = 1 << 30;
+/// The most elements one table of a plug-in may hold.
+const MAX_TABLE_ELEMENTS: usize = 1 << 16;
 /// The longest manifest read.
 const MAX_MANIFEST: i32 = 1 << 20;
 
@@ -84,7 +86,10 @@ fn err(e: impl std::fmt::Display) -> String {
 
 /// Instantiate `module` in a fresh, limited store and read its memory and allocator.
 fn begin(engine: &Engine, module: &Module) -> Result<Pending, String> {
-    let mut store = Store::new(engine, StoreLimitsBuilder::new().memory_size(MAX_MEMORY).instances(1).build());
+    // One memory and a few small tables: a module declaring many of them would otherwise make
+    // the host allocate and initialise each before an export or fuel check could refuse it.
+    let limits = StoreLimitsBuilder::new().memory_size(MAX_MEMORY).memories(1).tables(4).table_elements(MAX_TABLE_ELEMENTS).instances(1).build();
+    let mut store = Store::new(engine, limits);
     store.limiter(|l| l);
     store.set_fuel(FUEL_BASE).map_err(err)?;
     let linker = Linker::<StoreLimits>::new(engine);

crates/vp9enc/src/lib.rs

@@ -231,9 +231,19 @@ impl Vp9Encoder {
         self.rc.debt += bits - per_frame;
     }
 
-    /// Encode one frame; `force_key` starts a new key frame.
+    /// Encode one frame; `force_key` starts a new key frame. Planes shorter than the configured
+    /// size encode nothing (an empty frame): the coding planes are not sized and filled from
+    /// samples that are not there.
     pub fn encode_frame(&mut self, y: &[u8], u: &[u8], v: &[u8], force_key: bool) -> EncodedFrame {
         let (w, h) = (self.cfg.width.max(1) as usize, self.cfg.height.max(1) as usize);
+        let empty = || EncodedFrame { data: vec![], key: false, q_idx: 0, filter_level: 0 };
+        // Checked: on 32-bit targets (wasm32) a huge size would wrap to a small one.
+        let (Some(luma), Some(chroma)) = (w.checked_mul(h), w.div_ceil(2).checked_mul(h.div_ceil(2))) else {
+            return empty();
+        };
+        if y.len() < luma || u.len() < chroma || v.len() < chroma {
+            return empty();
+        }
         let key = self.is_key_due(force_key);
         let qidx = self.pick_qidx(key);
         let lossless = qidx == 0;
@@ -480,4 +490,14 @@ mod tests {
         let enc = Vp9Encoder::new(EncoderConfig { width: 65_536, height: 1, ..Default::default() });
         assert_eq!((enc.cfg.width, enc.cfg.height), (65_536, 1));
     }
+
+    #[test]
+    fn planes_shorter_than_the_frame_encode_nothing() {
+        // 65536×65536 declared, a few bytes given: no 4 GiB coding planes are made.
+        let mut enc = Vp9Encoder::new(EncoderConfig { width: 65_536, height: 65_536, ..Default::default() });
+        let f = enc.encode_frame(&[1, 2, 3], &[], &[], true);
+        assert!(f.data.is_empty());
+        let mut enc = Vp9Encoder::new(EncoderConfig { width: 16, height: 16, ..Default::default() });
+        assert!(!enc.encode_frame(&[128; 256], &[128; 64], &[128; 64], true).data.is_empty());
+    }
 }