diff --git a/crates/audio/src/opus/encoder/bands.rs b/crates/audio/src/opus/encoder/bands.rs index 604aab595..697332eb2 100644 --- a/crates/audio/src/opus/encoder/bands.rs +++ b/crates/audio/src/opus/encoder/bands.rs @@ -8,7 +8,8 @@ use crate::opus::encoder::range::{BITRES, RangeEncoder, ilog}; use crate::opus::encoder::rate::{Mode, QTHETA_OFFSET, get_pulses}; -use crate::opus::encoder::tables::{EBANDS, NB_EBANDS}; +use crate::opus::encoder::tables::EBANDS; +use crate::opus::encoder::{CeltFrame, rate::Allocation}; pub const SPREAD_NORMAL: usize = 2; @@ -55,7 +56,7 @@ fn exp_rotation1_inv(x: &mut [f32], len: usize, stride: usize, c: f32, s: f32) { if len <= stride { return; } - if len >= 2 * stride + 1 { + if len > 2 * stride { for i in 0..=len - 2 * stride - 1 { let x1 = x[i]; let x2 = x[i + stride]; @@ -96,12 +97,24 @@ pub fn exp_rotation_inv(x: &mut [f32], len: usize, k: usize, spread: usize) { } } +/// Relative slack for the pulse-candidate filter in [`pvq_search`]: far above the f32 rounding of +/// the filter score (a few 1e-7), far below the spread of real scores. +const PVQ_SCORE_SLACK: f32 = 1e-5; + /// Greedy PVQ search: the integer vector `y` with `sum |y| = k` closest in direction to `x`. pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { let n = x.len(); y[..n].fill(0); - let abs: Vec = x.iter().map(|v| v.abs()).collect(); - let sum: f32 = abs.iter().sum(); + // `abs`, `dy` (2 y + 1, infinite in the padding) and the per-pass scores `score`, each padded to + // whole 8-lane chunks. + let np = n.next_multiple_of(8); + let mut buf = vec![0f32; 3 * np]; + let (abs, rest) = buf.split_at_mut(np); + let (dy, score) = rest.split_at_mut(np); + for (a, v) in abs.iter_mut().zip(x) { + *a = v.abs(); + } + let sum: f32 = abs[..n].iter().sum(); if !sum.is_finite() || sum <= 1e-15 { y[0] = k as i32; return; @@ -113,30 +126,69 @@ pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { // Pre-project onto the pyramid (rounding down) and place the rest greedily. let r = (k as f32 - 1.0) / sum; for j in 0..n { - let p = (abs[j] * r).floor() as i32; + // Non-negative and at most k, so the truncating cast is the floor. + let p = (abs[j] * r) as i32; y[j] = p; placed += p as usize; xy += abs[j] * p as f32; yy += (p * p) as f32; } } + for j in 0..np { + dy[j] = if j < n { 2.0 * y[j] as f32 + 1.0 } else { f32::INFINITY }; + } while placed < k { + // Vectorisable scores (xy + |x_j|)^2 / (yy + 2 y_j + 1) and their maximum. + for ((sc, &a), &d) in score.iter_mut().zip(abs.iter()).zip(dy.iter()) { + let t = xy + a; + *sc = t * t / (yy + d); + } + let mut lane_max = [0f32; 8]; + for c in score.as_chunks::<8>().0 { + for i in 0..8 { + lane_max[i] = if c[i] > lane_max[i] { c[i] } else { lane_max[i] }; + } + } + let top = lane_max.iter().fold(0f32, |m, &v| if v > m { v } else { m }); + // The exact cross-multiplied test below, in index order with the first maximum winning, only + // matters for entries within rounding distance of the top score: every other entry is beaten + // by the top one whenever it is reached and never beats it afterwards. Running the test on + // those alone therefore picks the same pulse as running it on all of them. + let filter = top > 1e-30 && top.is_finite(); + let lo = top * (1.0 - PVQ_SCORE_SLACK); + // Usually exactly one entry qualifies; its index falls out of a lane-wise count. + let mut count = [0u32; 8]; + let mut index = [0u32; 8]; + for (ci, c) in score.as_chunks::<8>().0.iter().enumerate() { + for i in 0..8 { + let hit = u32::from(c[i] >= lo); + count[i] += hit; + index[i] += hit * (ci * 8 + i) as u32; + } + } let mut best = 0usize; - let mut best_num = -1f32; - let mut best_den = 1f32; - for j in 0..n { - let num = xy + abs[j]; - let num = num * num; - let den = yy + 2.0 * y[j] as f32 + 1.0; - if num * best_den > best_num * den { - best_num = num; - best_den = den; - best = j; + if filter && count.iter().sum::() == 1 { + best = index.iter().sum::() as usize; + } else { + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + if !filter || score[j] >= lo { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } } } xy += abs[best]; yy += 2.0 * y[best] as f32 + 1.0; y[best] += 1; + dy[best] = 2.0 * y[best] as f32 + 1.0; placed += 1; } for j in 0..n { @@ -146,25 +198,45 @@ pub fn pvq_search(x: &[f32], k: usize, y: &mut [i32]) { } } -/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps -/// back to `y`. -pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec) { - let n = y.len(); - let w = k + 1; - v.clear(); - v.resize((n + 1) * w, 0); +/// Largest band width and pulse count the encoder codes (176 coefficients, `get_pulses(40)` = 128). +const PVQ_MAX_N: usize = 176; +const PVQ_MAX_K: usize = 128; + +/// Row-major `V(n, k)` for `n <= max_n`, `k <= max_k` with row stride `max_k + 1`, saturated at +/// `u32::MAX` (a codebook that large is never coded, see `rate::fits_in32`). +fn build_pvq_table(max_n: usize, max_k: usize) -> Vec { + let w = max_k + 1; + let mut v = vec![0u32; (max_n + 1) * w]; v[0] = 1; - for nn in 1..=n { + for nn in 1..=max_n { v[nn * w] = 1; - for kk in 1..=k { - let a = v[(nn - 1) * w + kk]; - let b = v[nn * w + kk - 1]; - let c = v[(nn - 1) * w + kk - 1]; - v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + for kk in 1..=max_k { + let sum = v[(nn - 1) * w + kk] as u64 + v[nn * w + kk - 1] as u64 + v[(nn - 1) * w + kk - 1] as u64; + v[nn * w + kk] = sum.min(u32::MAX as u64) as u32; } } - let total = v[n * w + k]; - let vv = |nn: usize, kx: usize| v[nn * w + kx]; + v +} + +fn pvq_table() -> &'static [u32] { + static TABLE: std::sync::OnceLock> = std::sync::OnceLock::new(); + TABLE.get_or_init(|| build_pvq_table(PVQ_MAX_N, PVQ_MAX_K)) +} + +/// Encodes the PVQ codeword `y` (`sum |y| = k`) as the index the RFC 6716 §4.3.4.2 decoder maps +/// back to `y`. +pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder) { + let n = y.len(); + // Shapes outside the coded range (never produced by the band layout) get a local table. + let local; + let (v, w) = if n <= PVQ_MAX_N && k <= PVQ_MAX_K { + (pvq_table(), PVQ_MAX_K + 1) + } else { + local = build_pvq_table(n, k); + (&local[..], k + 1) + }; + let row = |nn: usize| &v[nn * w..(nn + 1) * w]; + let total = row(n)[k] as u64; let mut index = 0u64; let mut kk = k; for j in 0..n { @@ -173,10 +245,11 @@ pub fn encode_pulses(y: &[i32], k: usize, enc: &mut RangeEncoder, v: &mut Vec { pub i: usize, pub spread: usize, pub remaining_bits: i32, - pvq_v: Vec, iy: Vec, tmp: Vec, } @@ -217,7 +289,7 @@ impl BandEnc<'_, '_> { let q = qn as u32; let half = q >> 1; let ft = (half + 1) * (half + 1); - let (fl, fs) = if it <= half { (it * (it + 1) >> 1, it + 1) } else { (ft - ((q + 1 - it) * (q + 2 - it) >> 1), q + 1 - it) }; + let (fl, fs) = if it <= half { ((it * (it + 1)) >> 1, it + 1) } else { (ft - (((q + 1 - it) * (q + 2 - it)) >> 1), q + 1 - it) }; self.enc.encode(fl, fl + fs, ft); itheta = ((it * 16384) / q) as i32; } @@ -287,36 +359,29 @@ impl BandEnc<'_, '_> { iy.clear(); iy.resize(n, 0); pvq_search(&tmp, k, &mut iy); - encode_pulses(&iy, k, self.enc, &mut self.pvq_v); + encode_pulses(&iy, k, self.enc); self.iy = iy; self.tmp = tmp; } } } -/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds `c` channels -/// of `n` normalised MDCT coefficients each. -#[allow(clippy::too_many_arguments)] -pub fn quant_all_bands( - m: &Mode, - start: usize, - end: usize, - x: &[f32], - n_total: usize, - c: usize, - pulses: &[i32; NB_EBANDS], - spread: usize, - dual_stereo: bool, - intensity: usize, - total_bits: i32, - mut balance: i32, - enc: &mut RangeEncoder, - lm: usize, - coded_bands: usize, -) { +/// Shape-coding controls, independent of the selected band allocation. +pub struct BandCoding { + pub spread: usize, + pub total_bits: i32, +} + +/// Codes all band shapes (`quant_all_bands`, encoder side, long blocks). `x` holds +/// `frame.channels` channels of `n_total` normalised MDCT coefficients each. +pub fn quant_all_bands(m: &Mode, frame: CeltFrame, x: &[f32], n_total: usize, allocation: &Allocation, coding: BandCoding, enc: &mut RangeEncoder) { + let CeltFrame { start, end, channels: c, lm } = frame; + let Allocation { pulses, dual_stereo, intensity, coded_bands, balance, .. } = allocation; + let (dual_stereo, intensity, coded_bands, mut balance) = (*dual_stereo, *intensity, *coded_bands, *balance); + let BandCoding { spread, total_bits } = coding; let mm = 1usize << lm; debug_assert!(c == 1 || dual_stereo || intensity <= start, "stereo frames are coded as dual stereo"); - let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, pvq_v: Vec::new(), iy: Vec::new(), tmp: Vec::new() }; + let mut ctx = BandEnc { m, enc, i: 0, spread, remaining_bits: 0, iy: Vec::new(), tmp: Vec::new() }; for i in start..end { ctx.i = i; let xo = mm * EBANDS[i] as usize; @@ -362,7 +427,7 @@ mod tests { x[i + stride] = c * x2 + s * x1; x[i] = c * x1 + ms * x2; } - if len >= 2 * stride + 1 { + if len > 2 * stride { for i in (0..=len - 2 * stride - 1).rev() { let x1 = x[i]; let x2 = x[i + stride]; @@ -466,12 +531,11 @@ mod tests { #[test] fn pvq_index_roundtrip_exhaustive() { - let mut v = Vec::new(); for n in 1..6usize { for k in 1..6usize { for y in all_codewords(n, k) { let mut e = RangeEncoder::new(); - encode_pulses(&y, k, &mut e, &mut v); + encode_pulses(&y, k, &mut e); let buf = e.done(16); let mut d = RangeDecoder::new(&buf); assert_eq!(decode_pulses(n, k, &mut d), y, "n={n} k={k}"); @@ -480,6 +544,165 @@ mod tests { } } + /// The per-call table encoder this module used before the shared table. + fn encode_pulses_reference(y: &[i32], k: usize, enc: &mut RangeEncoder) { + let n = y.len(); + let w = k + 1; + let mut v = vec![0u64; (n + 1) * w]; + v[0] = 1; + for nn in 1..=n { + v[nn * w] = 1; + for kk in 1..=k { + let a = v[(nn - 1) * w + kk]; + let b = v[nn * w + kk - 1]; + let c = v[(nn - 1) * w + kk - 1]; + v[nn * w + kk] = a.saturating_add(b).saturating_add(c); + } + } + let total = v[n * w + k]; + let vv = |nn: usize, kx: usize| v[nn * w + kx]; + let mut index = 0u64; + let mut kk = k; + for j in 0..n { + if kk == 0 { + break; + } + let rem = n - j; + let t = y[j].unsigned_abs() as usize; + let mut base: u64 = (0..kk - t).map(|m| vv(rem - 1, m)).sum(); + if y[j] < 0 { + base += (vv(rem - 1, kk) + vv(rem, kk)) / 2; + } + index += base; + kk -= t; + } + if total >= 2 { + enc.uint(index as u32, total as u32); + } + } + + #[test] + fn pvq_table_encode_matches_per_call_table() { + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move |m: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % m + }; + let mut checked = 0; + for n in (1..=PVQ_MAX_N).step_by(5).chain([PVQ_MAX_N]) { + for k in (1..=PVQ_MAX_K).step_by(3).chain([PVQ_MAX_K]) { + if crate::opus::encoder::rate::pvq_v(n, k) > u32::MAX as u128 { + continue; + } + for _ in 0..4 { + let mut y = vec![0i32; n]; + for _ in 0..k { + let at = next(n as u64) as usize; + y[at] += if next(2) == 0 { 1 } else { -1 }; + } + // Opposite pulses cancel; top up so that sum |y| = k again. + let mut have: usize = y.iter().map(|v| v.unsigned_abs() as usize).sum(); + while have < k { + let at = next(n as u64) as usize; + y[at] += if y[at] < 0 { -1 } else { 1 }; + have += 1; + } + let (mut a, mut b) = (RangeEncoder::new(), RangeEncoder::new()); + encode_pulses(&y, k, &mut a); + encode_pulses_reference(&y, k, &mut b); + assert_eq!(a.done(16), b.done(16), "n={n} k={k}"); + checked += 1; + } + } + } + assert!(checked > 100, "{checked}"); + } + + /// The scalar greedy search the vectorised one replaced. + fn pvq_search_ref(x: &[f32], k: usize, y: &mut [i32]) { + let n = x.len(); + y[..n].fill(0); + let abs: Vec = x.iter().map(|v| v.abs()).collect(); + let sum: f32 = abs.iter().sum(); + if !sum.is_finite() || sum <= 1e-15 { + y[0] = k as i32; + return; + } + let mut placed = 0usize; + let mut xy = 0f32; + let mut yy = 0f32; + if k > n / 2 { + let r = (k as f32 - 1.0) / sum; + for j in 0..n { + let p = (abs[j] * r).floor() as i32; + y[j] = p; + placed += p as usize; + xy += abs[j] * p as f32; + yy += (p * p) as f32; + } + } + while placed < k { + let mut best = 0usize; + let mut best_num = -1f32; + let mut best_den = 1f32; + for j in 0..n { + let num = xy + abs[j]; + let num = num * num; + let den = yy + 2.0 * y[j] as f32 + 1.0; + if num * best_den > best_num * den { + best_num = num; + best_den = den; + best = j; + } + } + xy += abs[best]; + yy += 2.0 * y[best] as f32 + 1.0; + y[best] += 1; + placed += 1; + } + for j in 0..n { + if x[j] < 0.0 { + y[j] = -y[j]; + } + } + } + + #[test] + fn pvq_search_matches_scalar_reference() { + let mut state = 0x9e3779b97f4a7c15u64; + let mut rnd = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + (state >> 40) as f32 / (1u64 << 24) as f32 * 2.0 - 1.0 + }; + let mut a = vec![0i32; 200]; + let mut b = vec![0i32; 200]; + let mut cases = 0; + for round in 0..6000 { + let n = 1 + (round * 37 + (rnd().abs() * 176.0) as usize) % 176; + let k = 1 + (rnd().abs() * 127.0) as usize; + let scale = [1.0f32, 1e-9, 1e6, 3e-3][round % 4]; + let x: Vec = (0..n) + .map(|_| match round % 6 { + 0 => rnd(), + 1 => if rnd() > 0.8 { rnd() } else { 0.0 }, + 2 => (rnd() * 4.0).round() / 4.0, + 3 => if rnd() > 0.0 { 1.0 } else { -1.0 }, + 4 => rnd().powi(5), + _ => if rnd() > 0.95 { 3.0 * rnd() } else { 1e-4 * rnd() }, + } * scale) + .collect(); + pvq_search(&x, k, &mut a); + pvq_search_ref(&x, k, &mut b); + assert_eq!(a[..n], b[..n], "n={n} k={k} round={round}"); + cases += 1; + } + assert_eq!(cases, 6000); + } + #[test] fn pvq_search_finds_good_direction() { let x: Vec = (0..32).map(|i| ((i as f32) * 0.7).sin()).collect(); diff --git a/crates/audio/src/opus/encoder/energy.rs b/crates/audio/src/opus/encoder/energy.rs index 8d4f357d0..fb5cc5a4b 100644 --- a/crates/audio/src/opus/encoder/energy.rs +++ b/crates/audio/src/opus/encoder/energy.rs @@ -4,8 +4,9 @@ //! The quantised energies are updated with exactly the decoder's arithmetic so the prediction //! state never drifts between encoder and decoder. +use crate::opus::encoder::CeltFrame; use crate::opus::encoder::range::RangeEncoder; -use crate::opus::encoder::rate::MAX_FINE_BITS; +use crate::opus::encoder::rate::{Allocation, MAX_FINE_BITS}; use crate::opus::encoder::tables::{BETA_COEF, BETA_INTRA, E_PROB_MODEL, NB_EBANDS, PRED_COEF, SMALL_ENERGY_ICDF}; const LAPLACE_MINP: u32 = 1; @@ -56,18 +57,15 @@ pub fn laplace_encode(enc: &mut RangeEncoder, value: i32, mut fs: u32, decay: u3 /// `quant_coarse_energy` for one prediction mode. `e` holds the target log2 energies (relative /// to `E_MEANS`), `old_e` the quantised energies, updated in place. `budget` is the frame size in /// bits. -#[allow(clippy::too_many_arguments)] pub fn quant_coarse_energy( - start: usize, - end: usize, + frame: CeltFrame, e: &[f32; 2 * NB_EBANDS], old_e: &mut [f32; 2 * NB_EBANDS], intra: bool, enc: &mut RangeEncoder, - c: usize, - lm: usize, budget: i32, ) { + let CeltFrame { start, end, channels: c, lm } = frame; let prob = &E_PROB_MODEL[lm][intra as usize]; let (coef, beta) = if intra { (0.0, BETA_INTRA) } else { (PRED_COEF[lm], BETA_COEF[lm]) }; let mut prev = [0f32; 2]; @@ -112,14 +110,13 @@ pub fn quant_coarse_energy( /// `quant_fine_energy`: raw bits refining each band's quantised energy. pub fn quant_fine_energy( - start: usize, - end: usize, + frame: CeltFrame, e: &[f32; 2 * NB_EBANDS], old_e: &mut [f32; 2 * NB_EBANDS], fine_quant: &[i32; NB_EBANDS], enc: &mut RangeEncoder, - c: usize, ) { + let CeltFrame { start, end, channels: c, .. } = frame; for i in start..end { let fq = fine_quant[i]; if fq <= 0 { @@ -139,18 +136,16 @@ pub fn quant_fine_energy( /// `quant_energy_finalise`: spends the bits left at the end of the frame on one more bit of /// energy resolution per band, in priority order. -#[allow(clippy::too_many_arguments)] pub fn quant_energy_finalise( - start: usize, - end: usize, + frame: CeltFrame, e: &[f32; 2 * NB_EBANDS], old_e: &mut [f32; 2 * NB_EBANDS], - fine_quant: &[i32; NB_EBANDS], - fine_priority: &[i32; NB_EBANDS], + allocation: &Allocation, mut bits_left: i32, enc: &mut RangeEncoder, - c: usize, ) { + let CeltFrame { start, end, channels: c, .. } = frame; + let Allocation { fine_quant, fine_priority, .. } = allocation; for prio in 0..2 { let mut i = start; while i < end && bits_left >= c as i32 { diff --git a/crates/audio/src/opus/encoder/mdct.rs b/crates/audio/src/opus/encoder/mdct.rs index 6a42cce96..fa5624514 100644 --- a/crates/audio/src/opus/encoder/mdct.rs +++ b/crates/audio/src/opus/encoder/mdct.rs @@ -24,6 +24,10 @@ impl Cpx { fn add(self, o: Cpx) -> Cpx { Cpx { re: self.re + o.re, im: self.im + o.im } } + #[inline] + fn sub(self, o: Cpx) -> Cpx { + Cpx { re: self.re - o.re, im: self.im - o.im } + } } /// Mixed-radix (2, 3, 4, 5) forward complex FFT, unscaled (decimation in time, recursive). @@ -71,18 +75,64 @@ impl Fft { self.rec(&mut out[q * m..(q + 1) * m], &inp[q * in_stride..], in_stride * p, m, fi + 1, tw_stride * p); } } - let big = self.n; - let mut t = [Cpx::default(); 5]; - for k in 0..m { - for (q, tq) in t.iter_mut().enumerate().take(p) { - *tq = out[q * m + k].mul(self.twiddle[(q * k * tw_stride) % big]); + // The twiddle index q * k * tw_stride is always below n, so no modulo is needed. + let tw = &self.twiddle; + match p { + 2 => { + for k in 0..m { + let a0 = out[k]; + let a1 = out[m + k].mul(tw[k * tw_stride]); + out[k] = a0.add(a1); + out[m + k] = a0.sub(a1); + } } - for q2 in 0..p { - let mut acc = Cpx::default(); - for (q, tq) in t.iter().enumerate().take(p) { - acc = acc.add(tq.mul(self.twiddle[((q * q2) % p) * (big / p)])); + 4 => { + for k in 0..m { + let t0 = out[k]; + let t1 = out[m + k].mul(tw[k * tw_stride]); + let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]); + let t3 = out[3 * m + k].mul(tw[3 * k * tw_stride]); + let (s0, d0) = (t0.add(t2), t0.sub(t2)); + let (s1, d1) = (t1.add(t3), t1.sub(t3)); + out[k] = s0.add(s1); + out[2 * m + k] = s0.sub(s1); + // The odd outputs combine d0 with -j * d1 = (d1.im, -d1.re) and its negative. + out[m + k] = Cpx { re: d0.re + d1.im, im: d0.im - d1.re }; + out[3 * m + k] = Cpx { re: d0.re - d1.im, im: d0.im + d1.re }; + } + } + 3 => { + // Imaginary part of exp(-2 pi i / 3) = -sqrt(3) / 2. + let h = tw[self.n / 3].im; + for k in 0..m { + let t0 = out[k]; + let t1 = out[m + k].mul(tw[k * tw_stride]); + let t2 = out[2 * m + k].mul(tw[2 * k * tw_stride]); + let (s, d) = (t1.add(t2), t1.sub(t2)); + let a = Cpx { re: t0.re - 0.5 * s.re, im: t0.im - 0.5 * s.im }; + out[k] = t0.add(s); + // j * h * d = (-h * d.im, h * d.re) + out[m + k] = Cpx { re: a.re - h * d.im, im: a.im + h * d.re }; + out[2 * m + k] = Cpx { re: a.re + h * d.im, im: a.im - h * d.re }; + } + } + _ => { + // Generic DFT butterfly (radix 5, once per transform). + let roots: [Cpx; 5] = std::array::from_fn(|q| if q < p { tw[q * (self.n / p)] } else { Cpx::default() }); + let mut t = [Cpx::default(); 5]; + for k in 0..m { + t[0] = out[k]; + for q in 1..p { + t[q] = out[q * m + k].mul(tw[q * k * tw_stride]); + } + for q2 in 0..p { + let mut acc = t[0]; + for q in 1..p { + acc = acc.add(t[q].mul(roots[(q * q2) % p])); + } + out[q2 * m + k] = acc; + } } - out[q2 * m + k] = acc; } } } @@ -199,6 +249,31 @@ mod tests { .collect() } + #[test] + fn fft_matches_dft() { + // Every radix combination, including the 480-point size of the 20 ms frame. + for &n in &[2usize, 3, 4, 5, 6, 8, 12, 15, 16, 20, 24, 30, 48, 60, 120, 240, 480] { + let fft = Fft::new(n); + let x: Vec = (0..n).map(|i| Cpx { re: ((i * 37 % 101) as f32 - 50.0) / 7.0, im: ((i * 53 % 89) as f32 - 44.0) / 9.0 }).collect(); + let mut y = x.clone(); + fft.process(&mut y, &mut Vec::new()); + for k in 0..n { + let (mut re, mut im) = (0f64, 0f64); + for (j, v) in x.iter().enumerate() { + let ph = -2.0 * std::f64::consts::PI * ((j * k) % n) as f64 / n as f64; + re += v.re as f64 * ph.cos() - v.im as f64 * ph.sin(); + im += v.re as f64 * ph.sin() + v.im as f64 * ph.cos(); + } + assert!( + (y[k].re as f64 - re).abs() < 2e-3 && (y[k].im as f64 - im).abs() < 2e-3, + "n={n} k={k}: ({}, {}) vs ({re}, {im})", + y[k].re, + y[k].im + ); + } + } + } + #[test] fn matches_direct_formula() { for &n in &[240usize, 960] { diff --git a/crates/audio/src/opus/encoder/mod.rs b/crates/audio/src/opus/encoder/mod.rs index cbb0d7984..a635cce40 100644 --- a/crates/audio/src/opus/encoder/mod.rs +++ b/crates/audio/src/opus/encoder/mod.rs @@ -10,8 +10,9 @@ //! above in the same range-coded frame; constant packet size. //! - **CELT** (fullband): MDCT transform coding; constant packet size. //! -//! From EffectCraft's `effectcraft-opusenc` (storytold/effectcraft@c618429). Public items keep their -//! names, so EffectCraft can re-export this module. +//! From EffectCraft's `effectcraft-opusenc` (storytold/effectcraft@c618429), with its later speed-ups +//! and CELT stage refactor up to storytold/effectcraft@d6c9ad66. Public items keep their names, so +//! EffectCraft can re-export this module. //! //! ``` //! # #[cfg(feature = "opus-encode")] { @@ -26,6 +27,7 @@ //! # } //! ``` +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] // The bit-allocation code mirrors the normative integer arithmetic; keep C-like shift expressions. #![allow(clippy::precedence, clippy::int_plus_one)] @@ -40,15 +42,24 @@ mod tables; use std::sync::OnceLock; -use bands::{SPREAD_NORMAL, quant_all_bands}; +use bands::{BandCoding, SPREAD_NORMAL, quant_all_bands}; use energy::{quant_coarse_energy, quant_energy_finalise, quant_fine_energy}; use mdct::Mdct; use range::{BITRES, RangeEncoder}; -use rate::{Mode as CeltMode, compute_allocation}; +use rate::{AllocationInput, Mode as CeltMode, compute_allocation}; use resample::Downsampler; use silk::SilkEncoder; use tables::{E_MEANS, EBANDS, NB_EBANDS, OVERLAP, PREEMPH, SPREAD_ICDF, TF_SELECT_TABLE, TRIM_ICDF}; +/// Band range, channel count and transform size shared by CELT coding stages. +#[derive(Clone, Copy)] +struct CeltFrame { + start: usize, + end: usize, + channels: usize, + lm: usize, +} + /// Samples per channel in one 20 ms frame at 48 kHz. const N: usize = 960; /// LM of a 20 ms frame (8 short blocks). @@ -338,7 +349,7 @@ impl Encoder { Mode::Celt => { let (log_e, silence) = self.celt_analysis(pcm); let len = self.frame_bytes; - packet.extend_from_slice(&self.encode_celt(RangeEncoder::new(), len, 0, NB_EBANDS, &log_e, silence)); + packet.extend_from_slice(&self.encode_celt(RangeEncoder::new(), len, 0..NB_EBANDS, &log_e, silence)); } Mode::Silk => { let x = self.silk_input(pcm); @@ -368,7 +379,7 @@ impl Encoder { } let (log_e, _) = self.celt_analysis(pcm); let end = if self.bandwidth == Bandwidth::SuperWideband { 19 } else { NB_EBANDS }; - packet.extend_from_slice(&self.encode_celt(enc, len, HYBRID_START_BAND, end, &log_e, false)); + packet.extend_from_slice(&self.encode_celt(enc, len, HYBRID_START_BAND..end, &log_e, false)); } } Ok(packet) @@ -430,10 +441,18 @@ impl Encoder { /// Codes one CELT frame (RFC 6716 §4.3, in bitstream order) of `len` bytes into `enc` /// (which already holds the SILK frame in hybrid mode), coding bands `start..end`, and /// finalises the frame. - #[allow(clippy::too_many_arguments)] - fn encode_celt(&mut self, mut enc: RangeEncoder, len: usize, start: usize, end: usize, log_e: &[f32; 2 * NB_EBANDS], silence: bool) -> Vec { + fn encode_celt( + &mut self, + mut enc: RangeEncoder, + len: usize, + bands: std::ops::Range, + log_e: &[f32; 2 * NB_EBANDS], + silence: bool, + ) -> Vec { let m = celt_mode(); let c = self.channels; + let frame = CeltFrame { start: bands.start, end: bands.end, channels: c, lm: LM }; + let CeltFrame { start, end, .. } = frame; if c == 1 { for i in 0..NB_EBANDS { self.old_band_e[i] = self.old_band_e[i].max(self.old_band_e[NB_EBANDS + i]); @@ -462,10 +481,10 @@ impl Encoder { let mut enc_intra = enc.clone(); let mut old_intra = self.old_band_e; enc_intra.bit_logp(true, 3); - quant_coarse_energy(start, end, log_e, &mut old_intra, true, &mut enc_intra, c, LM, total_bits_raw); + quant_coarse_energy(frame, log_e, &mut old_intra, true, &mut enc_intra, total_bits_raw); enc.bit_logp(false, 3); let mut old_inter = self.old_band_e; - quant_coarse_energy(start, end, log_e, &mut old_inter, false, &mut enc, c, LM, total_bits_raw); + quant_coarse_energy(frame, log_e, &mut old_inter, false, &mut enc, total_bits_raw); if enc_intra.tell_frac() < enc.tell_frac() { enc = enc_intra; self.old_band_e = old_intra; @@ -473,14 +492,14 @@ impl Encoder { self.old_band_e = old_inter; } } else { - quant_coarse_energy(start, end, log_e, &mut self.old_band_e, false, &mut enc, c, LM, total_bits_raw); + quant_coarse_energy(frame, log_e, &mut self.old_band_e, false, &mut enc, total_bits_raw); } // TF resolution: no changes. { let mut budget = total_bits_raw; let mut tell = enc.tell(); let mut logp = 4; - let tf_select_rsv = LM > 0 && tell + logp + 1 <= budget; + let tf_select_rsv = LM > 0 && tell + logp < budget; budget -= tf_select_rsv as i32; for _ in start..end { if tell + logp <= budget { @@ -512,27 +531,11 @@ impl Encoder { enc.icdf(alloc_trim as usize, &TRIM_ICDF, 7); } let bits = (total_bits_raw << BITRES) - enc.tell_frac() - 1; - let alloc = compute_allocation(m, start, end, &offsets, &cap, alloc_trim, bits, c, LM, &mut enc); - quant_fine_energy(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc, c); - quant_all_bands( - m, - start, - end, - &self.freq, - N, - c, - &alloc.pulses, - SPREAD_NORMAL, - alloc.dual_stereo, - alloc.intensity, - total_bits_raw << BITRES, - alloc.balance, - &mut enc, - LM, - alloc.coded_bands, - ); + let alloc = compute_allocation(m, frame, AllocationInput { offsets: &offsets, cap: &cap, trim: alloc_trim, bits }, &mut enc); + quant_fine_energy(frame, log_e, &mut self.old_band_e, &alloc.fine_quant, &mut enc); + quant_all_bands(m, frame, &self.freq, N, &alloc, BandCoding { spread: SPREAD_NORMAL, total_bits: total_bits_raw << BITRES }, &mut enc); let bits_left = total_bits_raw - enc.tell(); - quant_energy_finalise(start, end, log_e, &mut self.old_band_e, &alloc.fine_quant, &alloc.fine_priority, bits_left, &mut enc, c); + quant_energy_finalise(frame, log_e, &mut self.old_band_e, &alloc, bits_left, &mut enc); debug_assert!(enc.tell() <= total_bits_raw, "frame overflow: {} > {}", enc.tell(), total_bits_raw); if c == 1 { let (a, b) = self.old_band_e.split_at_mut(NB_EBANDS); diff --git a/crates/audio/src/opus/encoder/rate.rs b/crates/audio/src/opus/encoder/rate.rs index c360f9da7..58349bbd7 100644 --- a/crates/audio/src/opus/encoder/rate.rs +++ b/crates/audio/src/opus/encoder/rate.rs @@ -4,6 +4,7 @@ //! spec-derived `filmcraft-opus` decoder (first-party, MIT OR Apache-2.0) step by step; the only //! difference is that the encoder *writes* the band-skip, intensity and dual-stereo symbols. +use crate::opus::encoder::CeltFrame; use crate::opus::encoder::range::{BITRES, RangeEncoder, ilog}; use crate::opus::encoder::tables::{ALLOC, EBANDS, LOG2_FRAC_TABLE, MAX_LM, NB_EBANDS}; @@ -127,7 +128,7 @@ impl Mode { let mut cap = [0i32; NB_EBANDS]; for (i, cp) in cap.iter_mut().enumerate() { let n = (EBANDS[i + 1] - EBANDS[i]) << lm; - *cp = (self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n >> 2; + *cp = ((self.caps[NB_EBANDS * (2 * lm + c - 1) + i] as i32 + 64) * c as i32 * n) >> 2; } cap } @@ -200,7 +201,7 @@ fn compute_caps(m: &Mode) -> Vec { let mut n0 = EBANDS[j + 1] - EBANDS[j]; let max_bits = if n0 << i == 1 { - c * (1 + MAX_FINE_BITS) << BITRES + (c * (1 + MAX_FINE_BITS)) << BITRES } else { let mut lm0 = 0; if n0 > 2 { @@ -240,7 +241,7 @@ fn compute_caps(m: &Mode) -> Vec { let num = mb + ndof * offset; let den = (ndof - 1) << BITRES; let qb = ((num + (den >> 1)) / den).min(MAX_FINE_BITS); - mb += c * qb << BITRES; + mb += (c * qb) << BITRES; mb }; let w = c * ((EBANDS[j + 1] - EBANDS[j]) << i); @@ -263,21 +264,19 @@ pub struct Allocation { pub fine_priority: [i32; NB_EBANDS], } +/// Per-band limits, boosts and available budget for the allocation search. +pub struct AllocationInput<'a> { + pub offsets: &'a [i32; NB_EBANDS], + pub cap: &'a [i32; NB_EBANDS], + pub trim: i32, + pub bits: i32, +} + /// `clt_compute_allocation`, encoder side. Stereo frames are always coded as dual (L/R) stereo /// with no intensity bands, and no band is skipped voluntarily. -#[allow(clippy::too_many_arguments)] -pub fn compute_allocation( - m: &Mode, - start: usize, - end: usize, - offsets: &[i32; NB_EBANDS], - cap: &[i32; NB_EBANDS], - alloc_trim: i32, - total: i32, - c: usize, - lm: usize, - enc: &mut RangeEncoder, -) -> Allocation { +pub fn compute_allocation(m: &Mode, frame: CeltFrame, input: AllocationInput<'_>, enc: &mut RangeEncoder) -> Allocation { + let CeltFrame { start, end, channels: c, lm } = frame; + let AllocationInput { offsets, cap, trim: alloc_trim, bits: total } = input; let ci = c as i32; let lmi = lm as i32; let mut total = total.max(0); @@ -302,8 +301,8 @@ pub fn compute_allocation( let mut trim_offset = [0i32; NB_EBANDS]; for j in start..end { let w = EBANDS[j + 1] - EBANDS[j]; - thresh[j] = (ci << BITRES).max((3 * w << lmi << BITRES) >> 4); - trim_offset[j] = ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32)) >> 6; + thresh[j] = (ci << BITRES).max(((3 * w) << lmi << BITRES) >> 4); + trim_offset[j] = (ci * w * (alloc_trim - 5 - lmi) * (end as i32 - j as i32 - 1) * (1 << (lmi + BITRES as i32))) >> 6; if w << lmi == 1 { trim_offset[j] -= ci << BITRES; } @@ -317,7 +316,7 @@ pub fn compute_allocation( let mid = (lo + hi) >> 1; for j in (start..end).rev() { let n = EBANDS[j + 1] - EBANDS[j]; - let mut bitsj = ci * n * (ALLOC[mid as usize][j] as i32) << lmi >> 2; + let mut bitsj = (ci * n * (ALLOC[mid as usize][j] as i32)) << lmi >> 2; if bitsj > 0 { bitsj = (bitsj + trim_offset[j]).max(0); } @@ -342,8 +341,8 @@ pub fn compute_allocation( lo -= 1; for j in start..end { let n = EBANDS[j + 1] - EBANDS[j]; - let mut bits1j = ci * n * (ALLOC[lo as usize][j] as i32) << lmi >> 2; - let mut bits2j = if hi >= nb_alloc { cap[j] } else { ci * n * (ALLOC[hi as usize][j] as i32) << lmi >> 2 }; + let mut bits1j = (ci * n * (ALLOC[lo as usize][j] as i32)) << lmi >> 2; + let mut bits2j = if hi >= nb_alloc { cap[j] } else { (ci * n * (ALLOC[hi as usize][j] as i32)) << lmi >> 2 }; if bits1j > 0 { bits1j = (bits1j + trim_offset[j]).max(0); } @@ -361,27 +360,28 @@ pub fn compute_allocation( bits1[j] = bits1j; bits2[j] = bits2j; } - interp_bits2pulses(m, start, end, skip_start, &bits1, &bits2, &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv, c, lm, enc) + let interpolation = + AllocationInterpolation { skip_start, bits1: &bits1, bits2: &bits2, thresh: &thresh, cap, total, skip_rsv, intensity_rsv, dual_stereo_rsv }; + interp_bits2pulses(m, frame, interpolation, enc) } -#[allow(clippy::too_many_arguments)] -fn interp_bits2pulses( - m: &Mode, - start: usize, - end: usize, +/// Bracketing allocation vectors and signalling reserves for the interpolation stage. +struct AllocationInterpolation<'a> { skip_start: usize, - bits1: &[i32; NB_EBANDS], - bits2: &[i32; NB_EBANDS], - thresh: &[i32; NB_EBANDS], - cap: &[i32; NB_EBANDS], - mut total: i32, + bits1: &'a [i32; NB_EBANDS], + bits2: &'a [i32; NB_EBANDS], + thresh: &'a [i32; NB_EBANDS], + cap: &'a [i32; NB_EBANDS], + total: i32, skip_rsv: i32, - mut intensity_rsv: i32, - mut dual_stereo_rsv: i32, - c: usize, - lm: usize, - enc: &mut RangeEncoder, -) -> Allocation { + intensity_rsv: i32, + dual_stereo_rsv: i32, +} + +fn interp_bits2pulses(m: &Mode, frame: CeltFrame, interpolation: AllocationInterpolation<'_>, enc: &mut RangeEncoder) -> Allocation { + let CeltFrame { start, end, channels: c, lm } = frame; + let AllocationInterpolation { skip_start, bits1, bits2, thresh, cap, mut total, skip_rsv, mut intensity_rsv, mut dual_stereo_rsv } = + interpolation; let ci = c as i32; let alloc_floor = ci << BITRES; let stereo = (c > 1) as i32; @@ -393,7 +393,7 @@ fn interp_bits2pulses( let mut psum = 0; let mut done = false; for j in (start..end).rev() { - let tmp = bits1[j] + (mid * bits2[j] >> ALLOC_STEPS); + let tmp = bits1[j] + ((mid * bits2[j]) >> ALLOC_STEPS); if tmp >= thresh[j] || done { done = true; psum += tmp.min(cap[j]); @@ -411,7 +411,7 @@ fn interp_bits2pulses( let mut psum = 0; let mut done = false; for j in (start..end).rev() { - let mut tmp = bits1[j] + (lo * bits2[j] >> ALLOC_STEPS); + let mut tmp = bits1[j] + ((lo * bits2[j]) >> ALLOC_STEPS); if tmp < thresh[j] && !done { tmp = if tmp >= alloc_floor { alloc_floor } else { 0 }; } else { @@ -500,9 +500,9 @@ fn interp_bits2pulses( if n == 2 { offset += (den << BITRES) >> 2; } - if bits[j] + offset < den * 2 << BITRES { + if bits[j] + offset < (den * 2) << BITRES { offset += nclogn >> 2; - } else if bits[j] + offset < den * 3 << BITRES { + } else if bits[j] + offset < (den * 3) << BITRES { offset += nclogn >> 3; } let mut eb = (bits[j] + offset + (den << (BITRES - 1))).max(0); @@ -513,7 +513,7 @@ fn interp_bits2pulses( eb = eb.min(MAX_FINE_BITS); ebits[j] = eb; fine_priority[j] = (eb * (den << BITRES) >= bits[j] + offset) as i32; - bits[j] -= ci * eb << BITRES; + bits[j] -= (ci * eb) << BITRES; } else { excess = (bit - (ci << BITRES)).max(0); bits[j] = bit - excess; @@ -523,7 +523,7 @@ fn interp_bits2pulses( if excess > 0 { let extra_fine = (excess >> (stereo + BITRES as i32)).min(MAX_FINE_BITS - ebits[j]); ebits[j] += extra_fine; - let extra_bits = extra_fine * ci << BITRES; + let extra_bits = (extra_fine * ci) << BITRES; fine_priority[j] = (extra_bits >= excess - balance) as i32; excess -= extra_bits; } diff --git a/crates/audio/src/opus/encoder/resample.rs b/crates/audio/src/opus/encoder/resample.rs index 9045cb86d..1acc9a4e7 100644 --- a/crates/audio/src/opus/encoder/resample.rs +++ b/crates/audio/src/opus/encoder/resample.rs @@ -45,6 +45,25 @@ fn design(fc: f64, beta: f64, delay: usize) -> Vec { h.iter().map(|&v| v as f32).collect() } +/// Dot product with eight independent partial sums, which the compiler can vectorise (a single +/// running sum is bound by the latency of the float addition). The sums are combined pairwise. +#[inline] +fn dot8(a: &[f32], b: &[f32]) -> f32 { + let mut s = [0f32; 8]; + let (ca, ra) = a.as_chunks::<8>(); + let (cb, rb) = b.as_chunks::<8>(); + for (x, y) in ca.iter().zip(cb) { + for l in 0..8 { + s[l] += x[l] * y[l]; + } + } + let mut acc = ((s[0] + s[4]) + (s[1] + s[5])) + ((s[2] + s[6]) + (s[3] + s[7])); + for (x, y) in ra.iter().zip(rb) { + acc += x * y; + } + acc +} + /// Decimates one channel of 48 kHz audio by 3, 4 or 6. #[derive(Clone)] pub struct Downsampler { @@ -77,7 +96,7 @@ impl Downsampler { let t = taps - 1 + self.factor * j; let win = &self.buf[t + 1 - taps..=t]; // h is symmetric, so the window can be taken in forward order. - out.push(win.iter().zip(&self.h).map(|(a, b)| a * b).sum()); + out.push(dot8(win, &self.h)); } let n = self.buf.len(); self.buf.copy_within(n - (taps - 1).., 0); @@ -97,6 +116,16 @@ mod tests { (x.iter().map(|&v| (v as f64).powi(2)).sum::() / x.len() as f64).sqrt() } + #[test] + fn dot8_matches_sequential_sum() { + for len in [0usize, 1, 7, 8, 9, 63, 167, 251] { + let a: Vec = (0..len).map(|i| ((i * 37 % 101) as f32 - 50.0) / 50.0).collect(); + let b: Vec = (0..len).map(|i| ((i * 53 % 89) as f32 - 44.0) / 44.0).collect(); + let exact: f64 = a.iter().zip(&b).map(|(&x, &y)| x as f64 * y as f64).sum(); + assert!((dot8(&a, &b) as f64 - exact).abs() < 1e-4, "len {len}"); + } + } + #[test] fn passes_band_and_rejects_aliases() { for &(fs, pass, stop) in &[(16_000u32, 6_000.0, 8_600.0), (12_000, 4_500.0, 6_500.0), (8_000, 3_000.0, 4_500.0)] { diff --git a/crates/audio/src/opus/encoder/silk/analysis.rs b/crates/audio/src/opus/encoder/silk/analysis.rs index 439331076..d80052046 100644 --- a/crates/audio/src/opus/encoder/silk/analysis.rs +++ b/crates/audio/src/opus/encoder/silk/analysis.rs @@ -69,28 +69,55 @@ pub fn levinson(r: &[f64], order: usize) -> Vec { } /// Evaluates the symmetric polynomial `p` (length `d + 1`, `d` even) as the cosine series -/// `e^{jωd/2} P(e^{jω})`. -fn cos_series(p: &[f64], w: f64) -> f64 { +/// `e^{jωd/2} P(e^{jω})`, with `cos_m(m)` giving `cos(m ω)`. +#[inline] +fn cos_series_with(p: &[f64], cos_m: impl Fn(usize) -> f64) -> f64 { let h = (p.len() - 1) / 2; let mut s = p[h]; for k in 0..h { - s += 2.0 * p[k] * ((h - k) as f64 * w).cos(); + s += 2.0 * p[k] * cos_m(h - k); } s } +fn cos_series(p: &[f64], w: f64) -> f64 { + cos_series_with(p, |m| (m as f64 * w).cos()) +} + +/// Number of grid intervals the root search scans over `[0, π]`. +const GRID: usize = 1024; +/// Largest multiple of ω in the series (half the LPC order, at most 16 / 2). +const MAX_H: usize = 8; +/// Bisection steps per root: the bracket shrinks to 3e-9 rad, far below one Q15 NLSF step (9.6e-5 rad). +const BISECT_STEPS: usize = 20; + +/// `cos(m · π g / GRID)` for every grid point `g` and `m` in `0..=MAX_H`, computed with the same +/// expression as [`cos_series`], so the table values are bit-identical to calling `cos` there. +fn grid_cos() -> &'static [[f64; MAX_H + 1]] { + static TABLE: std::sync::OnceLock> = std::sync::OnceLock::new(); + TABLE.get_or_init(|| { + (0..=GRID) + .map(|g| { + let w = std::f64::consts::PI * g as f64 / GRID as f64; + std::array::from_fn(|m| (m as f64 * w).cos()) + }) + .collect() + }) +} + fn roots(p: &[f64], out: &mut Vec) { - const GRID: usize = 1024; + let grid = grid_cos(); + let at_grid = |g: usize| cos_series_with(p, |m| grid[g][m]); let mut w0 = 0.0; - let mut f0 = cos_series(p, w0); + let mut f0 = at_grid(0); for g in 1..=GRID { let w1 = std::f64::consts::PI * g as f64 / GRID as f64; - let f1 = cos_series(p, w1); + let f1 = at_grid(g); if f0 == 0.0 { out.push(w0); } else if f0 * f1 < 0.0 { let (mut lo, mut hi, mut flo) = (w0, w1, f0); - for _ in 0..40 { + for _ in 0..BISECT_STEPS { let mid = 0.5 * (lo + hi); let fm = cos_series(p, mid); if fm * flo <= 0.0 { @@ -111,6 +138,9 @@ fn roots(p: &[f64], out: &mut Vec) { /// when the root search fails (unstable or degenerate filter). pub fn a2nlsf(a: &[f64]) -> Option> { let d = a.len(); + if d > 2 * MAX_H { + return None; + } let mut c = vec![0f64; d + 2]; c[0] = 1.0; for k in 0..d { @@ -199,9 +229,20 @@ pub fn pitch_search(x: &[f32], start: usize, len: usize, fs_khz: usize) -> Pitch let score = full[(lag - min_lag) as usize]; // Contour: per-subframe lag offsets around the chosen lag. let cb: &[[i32; 4]] = if fs_khz == 8 { &CB_NB_20 } else { &CB_WB_20 }; + // The contour search revisits the same (subframe, lag) pairs for neighbouring base lags and + // codebook entries, so each correlation is computed once. + let span = (18 * fs - min_lag + 1) as usize; + let memo = std::cell::RefCell::new(vec![f64::NAN; 4 * span]); let sub_score = |k: usize, l: i32| -> f64 { let l = l.clamp(min_lag, 18 * fs); - ncorr(x, start + k * sub..start + (k + 1) * sub, l as usize) + let slot = k * span + (l - min_lag) as usize; + let cached = memo.borrow()[slot]; + if !cached.is_nan() { + return cached; + } + let v = ncorr(x, start + k * sub..start + (k + 1) * sub, l as usize); + memo.borrow_mut()[slot] = v; + v }; let mut best = (f64::MIN, lag, 0usize); for l0 in (lag - 2).max(min_lag)..=(lag + 2).min(max_lag) { @@ -319,4 +360,59 @@ mod tests { } } } + + /// The root search as it was before the grid table: `cos` called for every evaluation. + fn roots_reference(p: &[f64], out: &mut Vec) { + let mut w0 = 0.0; + let mut f0 = cos_series(p, w0); + for g in 1..=GRID { + let w1 = std::f64::consts::PI * g as f64 / GRID as f64; + let f1 = cos_series(p, w1); + if f0 == 0.0 { + out.push(w0); + } else if f0 * f1 < 0.0 { + let (mut lo, mut hi, mut flo) = (w0, w1, f0); + for _ in 0..BISECT_STEPS { + let mid = 0.5 * (lo + hi); + let fm = cos_series(p, mid); + if fm * flo <= 0.0 { + hi = mid; + } else { + lo = mid; + flo = fm; + } + } + out.push(0.5 * (lo + hi)); + } + w0 = w1; + f0 = f1; + } + } + + #[test] + fn grid_table_roots_are_bit_identical_to_direct_cos() { + let mut seed = 7u64; + let mut rnd = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 11) as f64 / (1u64 << 53) as f64 * 2.0 - 1.0 + }; + let mut found = 0; + for len in [11usize, 17] { + for _ in 0..300 { + let mut p = vec![0f64; len]; + for k in 0..len / 2 + 1 { + p[k] = rnd(); + p[len - 1 - k] = p[k]; + } + let (mut a, mut b) = (Vec::new(), Vec::new()); + roots(&p, &mut a); + roots_reference(&p, &mut b); + assert_eq!(a.iter().map(|v| v.to_bits()).collect::>(), b.iter().map(|v| v.to_bits()).collect::>()); + found += a.len(); + } + } + assert!(found > 1000, "{found}"); + } } diff --git a/crates/audio/src/opus/encoder/silk/mod.rs b/crates/audio/src/opus/encoder/silk/mod.rs index d82e537e0..0fda4f59a 100644 --- a/crates/audio/src/opus/encoder/silk/mod.rs +++ b/crates/audio/src/opus/encoder/silk/mod.rs @@ -129,6 +129,48 @@ struct Coded { pulses: Vec, } +/// Noise-feedback history (decoded − input), most recent first, as a double-length ring: each +/// sample is stored at `pos` and `pos + MAX_LPC_ORDER`, so `buf[pos..pos + MAX_LPC_ORDER]` is +/// always the contiguous window and a push costs two stores instead of a shift. +struct NfRing { + buf: [f64; 2 * MAX_LPC_ORDER], + /// Index of the most recent sample, always below `MAX_LPC_ORDER`. + pos: usize, +} + +impl NfRing { + fn new(hist: &[f64; MAX_LPC_ORDER]) -> Self { + let mut buf = [0.0; 2 * MAX_LPC_ORDER]; + buf[..MAX_LPC_ORDER].copy_from_slice(hist); + buf[MAX_LPC_ORDER..].copy_from_slice(hist); + NfRing { buf, pos: 0 } + } + + /// The history, most recent first. + fn window(&self) -> &[f64] { + self.buf.get(self.pos..self.pos + MAX_LPC_ORDER).unwrap_or(&[]) + } + + /// Makes `v` the most recent sample and drops the oldest. + fn push(&mut self, v: f64) { + self.pos = self.pos.checked_sub(1).unwrap_or(MAX_LPC_ORDER - 1); + for i in [self.pos, self.pos + MAX_LPC_ORDER] { + if let Some(slot) = self.buf.get_mut(i) { + *slot = v; + } + } + } + + /// The history as a plain array, most recent first. + fn to_array(&self) -> [f64; MAX_LPC_ORDER] { + let mut out = [0.0; MAX_LPC_ORDER]; + for (o, v) in out.iter_mut().zip(self.window()) { + *o = *v; + } + out + } +} + #[derive(Clone)] struct Channel { fs_khz: usize, @@ -237,7 +279,7 @@ impl Channel { let sub = self.sub; let order = self.order; let st = &mut self.dec; - let nf = &mut self.nf; + let mut ring = NfRing::new(&self.nf); let off = if p.signal_type == 0 { offset + INACTIVE_OFFSET } else { offset }; let gamma = nf_gamma(off); let mut nf_c = [0f64; MAX_LPC_ORDER]; @@ -355,7 +397,7 @@ impl Channel { } let lpc_q14 = lpc_pred.saturating_mul(16); // Target excitation (Q14) for this sample. - let fb: f64 = (0..order).map(|j| nf_c[j] * nf[j]).sum(); + let fb: f64 = nf_c.iter().zip(ring.window()).take(order).map(|(c, e)| c * e).sum(); let tgt = (x[n] as f64 + fb).clamp(-32768.0, 32767.0) * inv_g - lpc_q14 as f64 - ltp_q14 as f64; let sgn = if seed_v < 0 { -1.0 } else { 1.0 }; let u = sgn * tgt; @@ -394,11 +436,11 @@ impl Channel { let v = res.saturating_add(lpc_q14); s_lpc[MAX_LPC_ORDER + i] = v; xq[n] = sat16(rshift_round(smulww(v, gain_q10), 8)) as i16; - nf.copy_within(0..MAX_LPC_ORDER - 1, 1); - nf[0] = xq[n] as f64 - x[n] as f64; + ring.push(xq[n] as f64 - x[n] as f64); } s_lpc.copy_within(sub..sub + MAX_LPC_ORDER, 0); } + self.nf = ring.to_array(); st.s_lpc_q14.copy_from_slice(&s_lpc[..MAX_LPC_ORDER]); st.out_buf[..l].copy_from_slice(&xq); (Coded { gain_ix, gains_q16, seed, pulses }, xq) @@ -485,6 +527,14 @@ struct Prep { pred_q13: [i32; 2], } +/// One rate-loop trial: the frame coded at gain offset `offset`. +struct Trial { + offset: f64, + bits: i32, + state: SilkEncoder, + coder: RangeEncoder, +} + /// SILK encoder for one mono or stereo (mid/side) stream. #[derive(Clone)] pub struct SilkEncoder { @@ -635,11 +685,20 @@ impl SilkEncoder { pub fn encode(&mut self, input: &[Vec], target_bits: f64, max_bits: i32, enc: &mut RangeEncoder) { let prep = self.prepare(input); let start_bits = enc.tell(); - let trial = |s: &Self, o: f64| -> i32 { - let mut t = s.clone(); - let mut e = enc.clone(); - t.code(&prep, o, &mut e); - e.tell() - start_bits + // Every trial codes the frame into a clone of the encoder and the range coder. The result + // is kept per offset: the hard-limit check below hits the memo for the chosen offset, and + // the final step adopts that trial instead of coding the frame once more. + let mut trials: Vec = Vec::new(); + let mut trial = |s: &Self, o: f64| -> i32 { + if let Some(t) = trials.iter().find(|t| t.offset == o) { + return t.bits; + } + let mut state = s.clone(); + let mut coder = enc.clone(); + state.code(&prep, o, &mut coder); + let bits = coder.tell() - start_bits; + trials.push(Trial { offset: o, bits, state, coder }); + bits }; // Bisection on the gain offset: the smallest offset (finest quantisation) within budget. let target = target_bits.min(max_bits as f64); @@ -701,8 +760,20 @@ impl SilkEncoder { while trial(self, o) > max_bits && o < 12.0 { o += 1.0; } + match trials.iter().position(|t| t.offset == o) { + Some(i) => { + let t = trials.swap_remove(i); + *self = t.state; + *enc = t.coder; + } + None => self.code(&prep, o, enc), + } + self.end_frame(&prep, o); + } + + /// Advances the input history and the frame counter after frame `prep` was coded at `o`. + fn end_frame(&mut self, prep: &Prep, o: f64) { self.last_offset = o; - self.code(&prep, o, enc); for ch in 0..self.channels { self.ch[ch].push_history(&prep.x[ch]); } @@ -712,3 +783,80 @@ impl SilkEncoder { self.frame += 1; } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Deterministic speech-like test input: two harmonics with a moving pitch, a syllable + /// envelope, noise and short silences, at the internal rate and 16-bit scale. + fn signal(len: usize, fs_khz: usize, seed: u32) -> Vec { + let mut s = seed; + let fs = (fs_khz * 1000) as f64; + (0..len) + .map(|i| { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let noise = (s >> 8) as f64 / (1u32 << 24) as f64 - 0.5; + let t = i as f64 / fs; + let f0 = 140.0 + 40.0 * (t * 1.3).sin(); + let env = ((t * 4.0).sin().max(0.0)).powf(0.7); + let voiced = (std::f64::consts::TAU * f0 * t).sin() + 0.5 * (std::f64::consts::TAU * 2.0 * f0 * t).sin(); + (8000.0 * env * voiced + 600.0 * noise) as f32 + }) + .collect() + } + + #[test] + fn nf_ring_matches_a_shifted_history() { + let mut hist = [0f64; MAX_LPC_ORDER]; + for (i, h) in hist.iter_mut().enumerate() { + *h = i as f64 * 0.5 - 3.0; + } + let mut ring = NfRing::new(&hist); + assert_eq!(ring.window(), &hist[..]); + for n in 0..100 { + let v = (n as f64 * 0.37).sin() * 100.0; + hist.copy_within(0..MAX_LPC_ORDER - 1, 1); + hist[0] = v; + ring.push(v); + assert_eq!(ring.window(), &hist[..], "after push {n}"); + } + assert_eq!(ring.to_array(), hist); + } + + /// The rate loop adopts the memoised trial for the chosen offset. A second encoder that codes + /// every frame afresh at that offset (what the loop did before the memo) must produce the + /// same bytes, frame after frame, so the adopted encoder and range-coder states are exact. + #[test] + fn adopting_the_memoised_trial_equals_coding_the_frame_again() { + for (fs_khz, channels, target_bits, max_bits) in + [(8, 1, 200.0, 1000), (12, 1, 320.0, 1000), (16, 1, 480.0, 1000), (16, 2, 640.0, 1000), (16, 1, 900.0, 400), (8, 2, 120.0, 160)] + { + let mut memo = SilkEncoder::new(fs_khz, channels); + let mut fresh = memo.clone(); + let l = memo.frame_len(); + let frames = 40; + let left = signal(l * frames, fs_khz, 7); + let right: Vec = signal(l * frames + 37, fs_khz, 11)[37..].iter().map(|v| v * 0.6).collect(); + for f in 0..frames { + let mut input = vec![left[f * l..(f + 1) * l].to_vec()]; + if channels == 2 { + input.push(right[f * l..(f + 1) * l].to_vec()); + } + let mut enc = RangeEncoder::new(); + memo.encode(&input, target_bits, max_bits, &mut enc); + let o = memo.last_offset; + let prep = fresh.prepare(&input); + let mut enc_fresh = RangeEncoder::new(); + fresh.code(&prep, o, &mut enc_fresh); + fresh.end_frame(&prep, o); + assert_eq!(enc.tell(), enc_fresh.tell(), "{fs_khz} kHz {channels} ch frame {f}"); + assert_eq!(enc.done(1275), enc_fresh.done(1275), "{fs_khz} kHz {channels} ch frame {f}"); + for c in 0..channels { + assert_eq!(memo.ch[c].nf, fresh.ch[c].nf, "noise feedback, channel {c}, frame {f}"); + assert_eq!(memo.ch[c].dec.s_lpc_q14, fresh.ch[c].dec.s_lpc_q14, "LPC state, channel {c}, frame {f}"); + } + } + } + } +}