From bab4f88c22e525ac69474a8293b155fe70fccab4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 21:44:51 -0400 Subject: [PATCH 1/9] M12.6: camera tracker solve Structure from motion for the 3D Camera Tracker in effectcraft-track (camtrack): long KLT feature tracks with re-detection, parallax keyframes, two-view initialisation (normalised 8-point essential matrix in RANSAC or planar homography decomposition), incremental resection + triangulation, sparse Levenberg-Marquardt bundle adjustment with the Schur complement and Huber loss, focal length search (Fixed / Variable Zoom / Specify Angle of View), tripod pan solve, Auto Detect, ground plane / target / world mapping. Synthetic-scene tests (path, focal, error, tripod, flat, rendered frames). Co-Authored-By: Claude Opus 5.5 --- crates/track/Cargo.toml | 2 +- crates/track/src/camtrack/bundle.rs | 416 ++++++++++++ crates/track/src/camtrack/features.rs | 115 ++++ crates/track/src/camtrack/geometry.rs | 432 +++++++++++++ crates/track/src/camtrack/linalg.rs | 476 ++++++++++++++ crates/track/src/camtrack/mod.rs | 376 +++++++++++ crates/track/src/camtrack/solver.rs | 894 ++++++++++++++++++++++++++ crates/track/src/camtrack/tests.rs | 360 +++++++++++ crates/track/src/lib.rs | 1 + 9 files changed, 3071 insertions(+), 1 deletion(-) create mode 100644 crates/track/src/camtrack/bundle.rs create mode 100644 crates/track/src/camtrack/features.rs create mode 100644 crates/track/src/camtrack/geometry.rs create mode 100644 crates/track/src/camtrack/linalg.rs create mode 100644 crates/track/src/camtrack/mod.rs create mode 100644 crates/track/src/camtrack/solver.rs create mode 100644 crates/track/src/camtrack/tests.rs diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index ba449c92b..9bd46e26c 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "effectcraft-track" -description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, and corner-pin / transform solves" +description = "Motion tracking: feature/search region point tracking (pyramid normalized cross-correlation + Lucas-Kanade refinement), confidence, corner-pin / transform solves, stabilization and 3D camera solving (structure from motion)" version.workspace = true edition.workspace = true license.workspace = true diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs new file mode 100644 index 000000000..84e8a5ad6 --- /dev/null +++ b/crates/track/src/camtrack/bundle.rs @@ -0,0 +1,416 @@ +//! Sparse bundle adjustment: Levenberg–Marquardt over cameras and points with the **Schur +//! complement** eliminating the points (Triggs, McLauchlan, Hartley & Fitzgibbon, "Bundle +//! Adjustment — A Modern Synthesis", 2000, §6; H&Z Appendix 6). Residuals are reprojection +//! errors in pixels with a **Huber** robust loss (applied by iteratively reweighted least +//! squares, Triggs et al. §3.3). +//! +//! Cameras are `(R, C, f)`; rotations are updated multiplicatively (`R ← exp([δ]×) R`), focal +//! lengths in log space. The focal length is held fixed, shared by all cameras, or per camera +//! (with a smoothness prior between consecutive cameras). Tripod solves hold the centres fixed. + +use rayon::prelude::*; + +use super::geometry::Pose; +use super::linalg::*; + +/// A camera in the adjustment. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaCam { + pub pose: Pose, + /// Focal length (pixels). + pub f: f64, +} + +/// One observation: camera `cam` sees point `pt` at `uv` (pixels relative to the principal +/// point). +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct BaObs { + pub cam: u32, + pub pt: u32, + pub uv: [f64; 2], +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum FocalMode { + Fixed, + Shared, + /// Per camera, with prior weight `k` on the log-focal difference of consecutive cameras. + PerCamera(f64), +} + +#[derive(Clone, Debug)] +pub struct BaOpts { + pub focal: FocalMode, + /// Motion only: the points stay fixed. + pub fix_points: bool, + /// The camera centres stay fixed (tripod). + pub fix_centers: bool, + /// Cameras that stay fixed (gauge), by index; missing entries are free. + pub fixed: Vec, + /// Huber threshold (pixels). + pub huber: f64, + pub max_iter: usize, +} + +impl Default for BaOpts { + fn default() -> Self { + BaOpts { focal: FocalMode::Fixed, fix_points: false, fix_centers: false, fixed: vec![], huber: 2.0, max_iter: 30 } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct BaReport { + pub initial_cost: f64, + pub cost: f64, + pub iterations: usize, +} + +const BEHIND: f64 = 1e6; + +fn huber(e2: f64, k: f64) -> f64 { + if e2 <= k * k { e2 } else { 2.0 * k * e2.sqrt() - k * k } +} + +/// Residual (pixels) of one observation, `None` behind the camera. +pub fn residual(c: &BaCam, x: V3, uv: [f64; 2]) -> Option<[f64; 2]> { + let p = c.pose.to_cam(x); + if p[2] <= 1e-9 { + return None; + } + Some([c.f * p[0] / p[2] - uv[0], c.f * p[1] / p[2] - uv[1]]) +} + +/// Total robust cost. +pub fn cost(cams: &[BaCam], pts: &[V3], obs: &[BaObs], o: &BaOpts) -> f64 { + let c: f64 = obs + .par_iter() + .map(|ob| match residual(&cams[ob.cam as usize], pts[ob.pt as usize], ob.uv) { + Some(r) => huber(r[0] * r[0] + r[1] * r[1], o.huber), + None => BEHIND, + }) + .sum(); + c + prior_cost(cams, o) +} + +fn prior_cost(cams: &[BaCam], o: &BaOpts) -> f64 { + match o.focal { + FocalMode::PerCamera(k) => cams.windows(2).map(|w| (k * (w[0].f.ln() - w[1].f.ln())).powi(2)).sum(), + _ => 0.0, + } +} + +/// Parameter layout. +struct Layout { + /// First pose parameter of each camera (`usize::MAX` when its pose is fixed). + start: Vec, + /// Per-camera focal parameter (`usize::MAX` when none). Gauge-fixed cameras keep a free + /// focal length. + fidx: Vec, + /// Index of the shared focal parameter. + shared: Option, + n: usize, + rot_only: bool, +} + +fn layout(ncam: usize, o: &BaOpts) -> Layout { + let per_cam_f = matches!(o.focal, FocalMode::PerCamera(_)); + let d = if o.fix_centers { 3 } else { 6 }; + let mut start = vec![usize::MAX; ncam]; + let mut fidx = vec![usize::MAX; ncam]; + let mut n = 0; + for i in 0..ncam { + if !o.fixed.get(i).copied().unwrap_or(false) { + start[i] = n; + n += d; + } + if per_cam_f { + fidx[i] = n; + n += 1; + } + } + let shared = matches!(o.focal, FocalMode::Shared).then(|| { + n += 1; + n - 1 + }); + Layout { start, fidx, shared, n, rot_only: o.fix_centers } +} + +/// Camera-side Jacobian entries of one observation: (parameter index, d r / d param). +fn cam_jac(l: &Layout, ci: usize, c: &BaCam, xc: V3, a: &[[f64; 3]; 2], p: [f64; 2], out: &mut Vec<(usize, [f64; 2])>) { + out.clear(); + let s = l.start[ci]; + if s != usize::MAX { + // dXc/dδ = −[Xc]× + let sk = skew(xc); + for k in 0..3 { + let g = [-sk[0][k], -sk[1][k], -sk[2][k]]; + out.push((s + k, [dot(a[0], g), dot(a[1], g)])); + } + if !l.rot_only { + // dXc/dC = −R + for k in 0..3 { + let g = [-c.pose.r[0][k], -c.pose.r[1][k], -c.pose.r[2][k]]; + out.push((s + 3 + k, [dot(a[0], g), dot(a[1], g)])); + } + } + } + if l.fidx[ci] != usize::MAX { + out.push((l.fidx[ci], p)); + } + if let Some(g) = l.shared { + out.push((g, p)); + } +} + +/// Run Levenberg–Marquardt. `cams` and `pts` are updated in place. +pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> BaReport { + let l = layout(cams.len(), o); + let npts = pts.len(); + // Observations by point. + let mut by_pt: Vec> = vec![vec![]; npts]; + for (i, ob) in obs.iter().enumerate() { + by_pt[ob.pt as usize].push(i as u32); + } + let mut cur = cost(cams, pts, obs, o); + let initial = cur; + let mut lambda = 1e-4; + let mut iters = 0; + if l.n == 0 && o.fix_points { + return BaReport { initial_cost: initial, cost: cur, iterations: 0 }; + } + let n = l.n; + let parallel = n <= 700; + for _ in 0..o.max_iter { + iters += 1; + // ---- Normal equations (Gauss–Newton with IRLS weights). + // Per point: V (3×3), bp (3), W entries (cam-side index → 3). + struct PtSys { + v: [[f64; 3]; 3], + bp: V3, + w: Vec<(usize, V3)>, + } + let build_point = |j: usize, s_acc: &mut Vec, bc: &mut Vec| -> PtSys { + let mut ps = PtSys { v: [[0.0; 3]; 3], bp: [0.0; 3], w: vec![] }; + let mut jc: Vec<(usize, [f64; 2])> = Vec::with_capacity(8); + for &oi in &by_pt[j] { + let ob = &obs[oi as usize]; + let ci = ob.cam as usize; + let c = &cams[ci]; + let xc = c.pose.to_cam(pts[j]); + if xc[2] <= 1e-9 { + continue; + } + let iz = 1.0 / xc[2]; + let p = [c.f * xc[0] * iz, c.f * xc[1] * iz]; + let r = [p[0] - ob.uv[0], p[1] - ob.uv[1]]; + let e = (r[0] * r[0] + r[1] * r[1]).sqrt(); + let w = if e <= o.huber { 1.0 } else { o.huber / e }; + let a = [[c.f * iz, 0.0, -c.f * xc[0] * iz * iz], [0.0, c.f * iz, -c.f * xc[1] * iz * iz]]; + cam_jac(&l, ci, c, xc, &a, p, &mut jc); + // Point Jacobian: A R. + let jx = if o.fix_points { + [[0.0; 3]; 2] + } else { + let r0 = mtv(&c.pose.r, a[0]); + let r1 = mtv(&c.pose.r, a[1]); + [r0, r1] + }; + for (ia, ja) in &jc { + bc[*ia] -= w * (ja[0] * r[0] + ja[1] * r[1]); + for (ib, jb) in &jc { + s_acc[ia * n + ib] += w * (ja[0] * jb[0] + ja[1] * jb[1]); + } + } + if !o.fix_points { + for a1 in 0..3 { + ps.bp[a1] -= w * (jx[0][a1] * r[0] + jx[1][a1] * r[1]); + for b1 in 0..3 { + ps.v[a1][b1] += w * (jx[0][a1] * jx[0][b1] + jx[1][a1] * jx[1][b1]); + } + } + for (ia, ja) in &jc { + let wv = + [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; + match ps.w.iter_mut().find(|e| e.0 == *ia) { + Some(e) => e.1 = add(e.1, wv), + None => ps.w.push((*ia, wv)), + } + } + } + } + ps + }; + let (mut s_mat, mut bc, systems): (Vec, Vec, Vec) = if parallel && npts > 64 { + let chunks = rayon::current_num_threads().clamp(1, 16); + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut s, mut b) = (vec![0.0; n * n], vec![0.0; n]); + let v = (t * per..((t + 1) * per).min(npts)).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, v) + }) + .collect(); + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let mut sys = Vec::with_capacity(npts); + for (ps, pb, pv) in parts { + for (x, y) in s.iter_mut().zip(ps) { + *x += y; + } + for (x, y) in b.iter_mut().zip(pb) { + *x += y; + } + sys.extend(pv); + } + (s, b, sys) + } else { + let mut s = vec![0.0; n * n]; + let mut b = vec![0.0; n]; + let sys = (0..npts).map(|j| build_point(j, &mut s, &mut b)).collect(); + (s, b, sys) + }; + // Focal prior. + if let FocalMode::PerCamera(k) = o.focal { + for i in 0..cams.len().saturating_sub(1) { + let r = k * (cams[i].f.ln() - cams[i + 1].f.ln()); + let (a, b) = (l.fidx[i], l.fidx[i + 1]); + let ia = (a != usize::MAX).then_some(a); + let ib = (b != usize::MAX).then_some(b); + for (idx, g) in [(ia, k), (ib, -k)] { + let Some(x) = idx else { continue }; + bc[x] -= g * r; + for (idy, h) in [(ia, k), (ib, -k)] { + if let Some(y) = idy { + s_mat[x * n + y] += g * h; + } + } + } + } + } + let base_s = s_mat.clone(); + let base_bc = bc.clone(); + let mean_diag = ((0..n).map(|i| base_s[i * n + i]).sum::() / n.max(1) as f64).max(1e-12); + // ---- Try steps with increasing damping. + let mut accepted = false; + for _try in 0..12 { + let mut s = base_s.clone(); + let mut rhs = base_bc.clone(); + for i in 0..n { + let d = s[i * n + i]; + s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); + } + // Schur complement over the points. + let mut vinv: Vec> = vec![None; npts]; + if !o.fix_points { + for (j, ps) in systems.iter().enumerate() { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + let Some(vi) = inv3(&v) else { continue }; + vinv[j] = Some(vi); + let vb = mv(&vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w { + s[ia * n + ib] -= dot(wv[a], *wb); + } + } + } + } + let mut dc = rhs; + if n > 0 && !cholesky_solve(&mut s, n, &mut dc) { + lambda *= 10.0; + continue; + } + // Apply. + let mut nc: Vec = cams.to_vec(); + for (i, c) in nc.iter_mut().enumerate() { + let st = l.start[i]; + if st != usize::MAX { + let dr = [dc[st], dc[st + 1], dc[st + 2]]; + c.pose.r = mm(&rodrigues(dr), &c.pose.r); + if !l.rot_only { + c.pose.c = add(c.pose.c, [dc[st + 3], dc[st + 4], dc[st + 5]]); + } + } + if l.fidx[i] != usize::MAX { + c.f *= dc[l.fidx[i]].clamp(-0.5, 0.5).exp(); + } + if let Some(g) = l.shared { + c.f *= dc[g].clamp(-0.5, 0.5).exp(); + } + } + let mut np: Vec = pts.to_vec(); + if !o.fix_points { + np.par_iter_mut().enumerate().for_each(|(j, x)| { + let Some(vi) = &vinv[j] else { return }; + let ps = &systems[j]; + let mut b = ps.bp; + for (ia, w) in &ps.w { + b = sub(b, scale(*w, dc[*ia])); + } + *x = add(*x, mv(vi, b)); + }); + } + let c2 = cost(&nc, &np, obs, o); + if c2.is_finite() && c2 < cur { + cams.copy_from_slice(&nc); + pts.copy_from_slice(&np); + let gain = cur - c2; + cur = c2; + lambda = (lambda / 3.0).max(1e-9); + accepted = gain > 1e-10 * cur.max(1e-12); + if !accepted { + // Converged (tiny improvement). + return BaReport { initial_cost: initial, cost: cur, iterations: iters }; + } + break; + } + lambda *= 4.0; + } + if !accepted { + break; + } + } + BaReport { initial_cost: initial, cost: cur, iterations: iters } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::camtrack::geometry::Rng; + + #[test] + fn bundle_adjustment_converges_from_noise() { + let mut rng = Rng(99); + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + let truth: Vec = (0..80).map(|_| [u() * 6.0, u() * 4.0, 6.0 + u() * 4.0]).collect(); + let f = 900.0; + let cams_true: Vec = (0..6) + .map(|i| BaCam { pose: Pose { r: rodrigues([0.0, -0.05 * i as f64, 0.01 * i as f64]), c: [0.3 * i as f64, 0.05 * i as f64, 0.0] }, f }) + .collect(); + let mut obs = vec![]; + for (ci, c) in cams_true.iter().enumerate() { + for (pi, x) in truth.iter().enumerate() { + let r = residual(c, *x, [0.0, 0.0]).unwrap(); + obs.push(BaObs { cam: ci as u32, pt: pi as u32, uv: r }); + } + } + let mut cams = cams_true.clone(); + for (i, c) in cams.iter_mut().enumerate().skip(2) { + c.pose.r = mm(&rodrigues([0.01, -0.01 * i as f64 * 0.3, 0.005]), &c.pose.r); + c.pose.c = add(c.pose.c, [0.02, -0.03, 0.01]); + } + let mut pts: Vec = truth.iter().map(|x| add(*x, [u() * 0.1, u() * 0.1, u() * 0.2])).collect(); + let mut fixed = vec![false; 6]; + fixed[0] = true; + fixed[1] = true; + let rep = adjust(&mut cams, &mut pts, &obs, &BaOpts { fixed, max_iter: 50, ..Default::default() }); + let rms = (rep.cost / obs.len() as f64).sqrt(); + assert!(rms < 1e-4, "{rep:?}"); + } +} diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs new file mode 100644 index 000000000..311ec711d --- /dev/null +++ b/crates/track/src/camtrack/features.rs @@ -0,0 +1,115 @@ +//! Step 1 of the camera analysis: long feature tracks through the clip. +//! +//! Every frame, the active features are followed from the previous frame with pyramidal +//! Lucas–Kanade and a forward–backward check ([`crate::klt::track`]); a feature that fails ends +//! its track. When fewer than ~85 % of the wanted features remain, new Shi–Tomasi corners are +//! detected away from the active ones and start new tracks. Tracks shorter than three frames are +//! dropped at the end. + +use crate::Frame; +use crate::klt::{CornerOpts, GrayPyramid, LkOpts, analysis_factor, good_features, track}; + +use super::{CameraTracks, Track2D}; + +/// Analysis options. +#[derive(Clone, Copy, Debug, Default)] +pub struct AnalyzeOpts { + /// Detailed Analysis: twice the resolution and more features. + pub detailed: bool, +} + +/// Incremental feature tracking: push the clip's frames in order. +pub struct TrackAnalyzer { + opts: AnalyzeOpts, + prev: Option, + /// (track index, level-0 plane position) of the live features. + active: Vec<(usize, [f64; 2])>, + tracks: Vec, + frame: u32, + size: [f64; 2], + factor: f64, +} + +impl TrackAnalyzer { + pub fn new(size: [f64; 2], opts: AnalyzeOpts) -> TrackAnalyzer { + TrackAnalyzer { opts, prev: None, active: vec![], tracks: vec![], frame: 0, size, factor: 1.0 } + } + + pub fn frames(&self) -> u32 { + self.frame + } + + /// Features currently followed. + pub fn active(&self) -> usize { + self.active.len() + } + + fn params(&self) -> (u32, usize) { + if self.opts.detailed { (1280, 900) } else { (720, 500) } + } + + /// Analyse the next frame. + pub fn push(&mut self, frame: &Frame) { + let (max_side, want) = self.params(); + let factor = analysis_factor(frame.img.width, frame.img.height, max_side); + self.factor = factor as f64; + let pyr = GrayPyramid::from_image(frame.img, frame.offset, factor, 4); + let k = self.frame; + match &self.prev { + Some(prev) if prev.width() == pyr.width() && prev.height() == pyr.height() && prev.offset == pyr.offset => { + let pts: Vec<[f64; 2]> = self.active.iter().map(|a| a.1).collect(); + let q = track(prev, &pyr, &pts, None, &LkOpts { radius: 6, iterations: 30, epsilon: 0.005, fb_max: 0.5, ..Default::default() }); + let mut next = Vec::with_capacity(self.active.len()); + for ((ti, _), q) in self.active.iter().zip(q) { + if let Some(q) = q { + let l = pyr.to_layer(q); + self.tracks[*ti].pts.push([l[0] as f32, l[1] as f32]); + next.push((*ti, q)); + } + } + self.active = next; + } + _ => self.active.clear(), + } + // Re-detect where there are no features. + if (self.active.len() as f64) < want as f64 * 0.85 { + let md = (pyr.width().max(pyr.height()) as f64 / (want as f64).sqrt() * 0.6).clamp(5.0, 40.0); + let cell = md; + let (gw, gh) = ((pyr.width() as f64 / cell).ceil() as usize + 1, (pyr.height() as f64 / cell).ceil() as usize + 1); + let mut occ = vec![false; gw * gh]; + for (_, p) in &self.active { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + for yy in cy.saturating_sub(1)..=(cy + 1).min(gh - 1) { + for xx in cx.saturating_sub(1)..=(cx + 1).min(gw - 1) { + occ[yy * gw + xx] = true; + } + } + } + let free = |p: [f64; 2]| { + let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); + !occ.get(cy * gw + cx).copied().unwrap_or(true) + }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + for p in good_features(&pyr, &opts, Some(&free)) { + let l = pyr.to_layer(p); + let id = self.tracks.len() as u32; + self.tracks.push(Track2D { id, start: k, pts: vec![[l[0] as f32, l[1] as f32]] }); + self.active.push((id as usize, p)); + } + } + self.prev = Some(pyr); + self.frame += 1; + } + + pub fn finish(self, start: f64, frame_duration: f64) -> CameraTracks { + let mut tracks: Vec = self.tracks.into_iter().filter(|t| t.pts.len() >= 3).collect(); + // Round to 1/100 pixel (smaller JSON) and renumber densely. + for (i, t) in tracks.iter_mut().enumerate() { + t.id = i as u32; + for p in t.pts.iter_mut() { + *p = [(p[0] * 100.0).round() / 100.0, (p[1] * 100.0).round() / 100.0]; + } + } + CameraTracks { version: 1, size: self.size, start, frame_duration, frames: self.frame, factor: self.factor, detailed: self.opts.detailed, tracks } + } +} diff --git a/crates/track/src/camtrack/geometry.rs b/crates/track/src/camtrack/geometry.rs new file mode 100644 index 000000000..6956ac2b5 --- /dev/null +++ b/crates/track/src/camtrack/geometry.rs @@ -0,0 +1,432 @@ +//! Two-view and multi-view geometry for the camera solver, from Hartley & Zisserman (H&Z), +//! *Multiple View Geometry in Computer Vision* (2nd ed.) and Ma et al., *An Invitation to 3-D +//! Vision* (2004). +//! +//! All image points here are **normalised** (calibrated) coordinates `x = (u − c) / f`. +//! A camera is `(R, C)` with `X_cam = R (X − C)`. + +use super::linalg::*; + +/// A deterministic xorshift generator for RANSAC samples. +pub struct Rng(pub u64); +impl Rng { + pub fn next_u64(&mut self) -> u64 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.0 = x; + x + } + pub fn below(&mut self, n: usize) -> usize { + (self.next_u64() % n.max(1) as u64) as usize + } + /// `k` distinct indices below `n`. + pub fn sample(&mut self, n: usize, k: usize, out: &mut Vec) { + out.clear(); + let mut guard = 0; + while out.len() < k && guard < 1000 { + guard += 1; + let c = self.below(n); + if !out.contains(&c) { + out.push(c); + } + } + } +} + +/// Hartley normalisation (H&Z §4.4.4): centroid to the origin, mean distance √2. +fn normalizer(p: &[[f64; 2]]) -> M3 { + let n = p.len().max(1) as f64; + let c = [p.iter().map(|q| q[0]).sum::() / n, p.iter().map(|q| q[1]).sum::() / n]; + let d = p.iter().map(|q| (q[0] - c[0]).hypot(q[1] - c[1])).sum::() / n; + let s = if d > 1e-12 { std::f64::consts::SQRT_2 / d } else { 1.0 }; + [[s, 0.0, -s * c[0]], [0.0, s, -s * c[1]], [0.0, 0.0, 1.0]] +} + +fn h2(m: &M3, p: [f64; 2]) -> V3 { + mv(m, [p[0], p[1], 1.0]) +} + +/// The essential matrix from ≥ 8 correspondences (`x2ᵀ E x1 = 0`) by the normalised 8-point +/// algorithm (H&Z Alg. 11.1), with the essential constraint (two equal singular values, one +/// zero) enforced (H&Z §9.6.1 / Result 9.18). +pub fn essential_8pt(x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let (t1, t2) = (normalizer(&x1[..n]), normalizer(&x2[..n])); + let mut a = Vec::with_capacity(n * 9); + for i in 0..n { + let p = h2(&t1, x1[i]); + let q = h2(&t2, x2[i]); + a.extend_from_slice(&[q[0] * p[0], q[0] * p[1], q[0], q[1] * p[0], q[1] * p[1], q[1], p[0], p[1], 1.0]); + } + let e = null_vector(&a, n, 9); + let en = [[e[0], e[1], e[2]], [e[3], e[4], e[5]], [e[6], e[7], e[8]]]; + // Denormalise: E = T2ᵀ Ê T1. + let e = mm(&mt(&t2), &mm(&en, &t1)); + let (u, s, v) = svd3(&e); + let m = (s[0] + s[1]) * 0.5; + if m <= 0.0 || !m.is_finite() { + return None; + } + let d = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 0.0]]; + Some(mm(&u, &mm(&d, &mt(&v)))) +} + +/// Squared Sampson distance of a correspondence to `x2ᵀ E x1 = 0` (H&Z §11.4.3), in the units of +/// the coordinates. +pub fn sampson2(e: &M3, p: [f64; 2], q: [f64; 2]) -> f64 { + let x1 = [p[0], p[1], 1.0]; + let x2 = [q[0], q[1], 1.0]; + let ex1 = mv(e, x1); + let etx2 = mtv(e, x2); + let num = dot(x2, ex1); + let den = ex1[0] * ex1[0] + ex1[1] * ex1[1] + etx2[0] * etx2[0] + etx2[1] * etx2[1]; + if den < 1e-300 { f64::INFINITY } else { num * num / den } +} + +/// RANSAC essential matrix (8-point samples, Sampson distance below `thr`), refitted on the +/// consensus set. Returns the matrix and the inlier mask. +pub fn ransac_essential(x1: &[[f64; 2]], x2: &[[f64; 2]], thr: f64, iterations: usize, seed: u64) -> Option<(M3, Vec)> { + let n = x1.len().min(x2.len()); + if n < 8 { + return None; + } + let t2 = thr * thr; + let mut rng = Rng(seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1); + let mut best: Option<(usize, f64, M3)> = None; + let mut idx = vec![]; + let mut max_iter = iterations; + let mut it = 0; + let (mut s1, mut s2) = (vec![[0.0; 2]; 8], vec![[0.0; 2]; 8]); + while it < max_iter { + it += 1; + rng.sample(n, 8, &mut idx); + if idx.len() < 8 { + break; + } + for (k, &i) in idx.iter().enumerate() { + s1[k] = x1[i]; + s2[k] = x2[i]; + } + let Some(e) = essential_8pt(&s1, &s2) else { continue }; + let (mut c, mut err) = (0, 0.0); + for i in 0..n { + let d = sampson2(&e, x1[i], x2[i]); + if d < t2 { + c += 1; + err += d; + } + } + if best.as_ref().is_none_or(|b| c > b.0 || (c == b.0 && err < b.1)) { + best = Some((c, err, e)); + // Adaptive iteration count for 99 % confidence. + let w = c as f64 / n as f64; + let p_good = w.powi(8); + if p_good > 1e-9 { + let need = ((1.0 - 0.99f64).ln() / (1.0 - p_good).max(1e-12).ln()).ceil(); + if need.is_finite() { + max_iter = max_iter.min((need as usize).max(30)); + } + } + } + } + let (_, _, mut e) = best?; + // Refit on the inliers (twice: the consensus may grow). + let mut inl = vec![false; n]; + for _ in 0..2 { + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + let (a, b): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..n).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + match essential_8pt(&a, &b) { + Some(e2) => e = e2, + None => break, + } + } + for i in 0..n { + inl[i] = sampson2(&e, x1[i], x2[i]) < t2; + } + Some((e, inl)) +} + +/// The four `(R, t)` with `x2 ~ R x1 + t` (|t| = 1) of an essential matrix (H&Z Result 9.19). +pub fn decompose_essential(e: &M3) -> [(M3, V3); 4] { + let (mut u, _, mut v) = svd3(e); + if det3(&u) < 0.0 { + for row in u.iter_mut() { + row[2] = -row[2]; + } + } + if det3(&v) < 0.0 { + for row in v.iter_mut() { + row[2] = -row[2]; + } + } + let w = [[0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]]; + let r1 = mm(&u, &mm(&w, &mt(&v))); + let r2 = mm(&u, &mm(&mt(&w), &mt(&v))); + let t = col(&u, 2); + [(r1, t), (r1, scale(t, -1.0)), (r2, t), (r2, scale(t, -1.0))] +} + +/// A camera `(R, C)`. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Pose { + pub r: M3, + pub c: V3, +} + +impl Pose { + pub const IDENTITY: Pose = Pose { r: I3, c: [0.0; 3] }; + /// From `x_cam = R X + t`. + pub fn from_rt(r: M3, t: V3) -> Pose { + Pose { r, c: scale(mtv(&r, t), -1.0) } + } + pub fn t(&self) -> V3 { + scale(mv(&self.r, self.c), -1.0) + } + pub fn to_cam(&self, x: V3) -> V3 { + mv(&self.r, sub(x, self.c)) + } +} + +/// Linear (DLT) triangulation from normalised observations (H&Z §12.2), minimising the +/// algebraic error over all views. +pub fn triangulate(views: &[(Pose, [f64; 2])]) -> Option { + if views.len() < 2 { + return None; + } + let mut ata = [0.0f64; 16]; + for (p, x) in views { + let t = p.t(); + let rows = [[p.r[0][0], p.r[0][1], p.r[0][2], t[0]], [p.r[1][0], p.r[1][1], p.r[1][2], t[1]], [p.r[2][0], p.r[2][1], p.r[2][2], t[2]]]; + let a0: Vec = (0..4).map(|j| x[0] * rows[2][j] - rows[0][j]).collect(); + let a1: Vec = (0..4).map(|j| x[1] * rows[2][j] - rows[1][j]).collect(); + for a in [a0, a1] { + let nrm = a.iter().map(|v| v * v).sum::().sqrt().max(1e-300); + for i in 0..4 { + for j in 0..4 { + ata[i * 4 + j] += a[i] * a[j] / (nrm * nrm); + } + } + } + } + let (_, vecs) = sym_eigen(&ata, 4); + let h = [vecs[0], vecs[4], vecs[8], vecs[12]]; + if h[3].abs() < 1e-12 { + return None; + } + let x = [h[0] / h[3], h[1] / h[3], h[2] / h[3]]; + x.iter().all(|v| v.is_finite()).then_some(x) +} + +/// Angle (radians) between the rays from two camera centres to `x`. +pub fn ray_angle(a: V3, b: V3, x: V3) -> f64 { + let (u, v) = (normalize(sub(x, a)), normalize(sub(x, b))); + dot(u, v).clamp(-1.0, 1.0).acos() +} + +/// Squared reprojection error (normalised units) of `x` in `pose` against observation `o` +/// (`None` behind the camera). +pub fn reproj2(pose: &Pose, x: V3, o: [f64; 2]) -> Option { + let c = pose.to_cam(x); + if c[2] <= 1e-9 { + return None; + } + let (u, v) = (c[0] / c[2] - o[0], c[1] / c[2] - o[1]); + Some(u * u + v * v) +} + +/// Pick the essential decomposition with the most points in front of both cameras; returns +/// the second camera (the first is the identity) and that count. +pub fn select_pose(cands: &[(M3, V3)], x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Option<(Pose, usize)> { + let mut best: Option<(Pose, usize)> = None; + for (r, t) in cands { + let p2 = Pose::from_rt(*r, *t); + let mut c = 0; + for i in 0..x1.len() { + if let Some(x) = triangulate(&[(Pose::IDENTITY, x1[i]), (p2, x2[i])]) + && x[2] > 0.0 + && p2.to_cam(x)[2] > 0.0 + { + c += 1; + } + } + if best.as_ref().is_none_or(|b| c > b.1) { + best = Some((p2, c)); + } + } + best +} + +/// Decompose a calibrated planar homography `H ~ R + t nᵀ / d` (normalised coordinates, +/// `x2 ~ H x1`) into its motions `(R, t/d, n)` (Ma et al. §5.3.3). Up to four solutions; those +/// that put the plane behind the first camera are dropped. +pub fn decompose_homography(h: &M3, x1: &[[f64; 2]], x2: &[[f64; 2]]) -> Vec<(M3, V3, V3)> { + let (_, s, _) = svd3(h); + if s[1] <= 1e-12 { + return vec![]; + } + let mut hn = mscale(h, 1.0 / s[1]); + // Sign: x2ᵀ H x1 > 0 for the correspondences (positive depth). + let pos = x1.iter().zip(x2).filter(|(p, q)| dot([q[0], q[1], 1.0], mv(&hn, [p[0], p[1], 1.0])) > 0.0).count(); + if pos * 2 < x1.len() { + hn = mscale(&hn, -1.0); + } + let hth = mm(&mt(&hn), &hn); + let a: Vec = hth.iter().flatten().copied().collect(); + let (vals, vecs) = sym_eigen(&a, 3); + // Descending: σ1² ≥ σ2² = 1 ≥ σ3². + let v1 = [vecs[2], vecs[5], vecs[8]]; + let v2 = [vecs[1], vecs[4], vecs[7]]; + let v3 = [vecs[0], vecs[3], vecs[6]]; + let (s1, s3) = (vals[2].max(1.0), vals[0].clamp(0.0, 1.0)); + let mut out = vec![]; + if s1 - s3 < 1e-10 { + // H is a rotation: no translation (pure rotation or plane at infinity). + return vec![(nearest_rotation(&hn), [0.0; 3], [0.0, 0.0, 1.0])]; + } + let k = (s1 - s3).sqrt(); + let (a1, a3) = ((1.0 - s3).sqrt(), (s1 - 1.0).sqrt()); + let u1 = scale(add(scale(v1, a1), scale(v3, a3)), 1.0 / k); + let u2 = scale(sub(scale(v1, a1), scale(v3, a3)), 1.0 / k); + for u in [u1, u2] { + let uu = from_cols(v2, u, cross(v2, u)); + let hv2 = mv(&hn, v2); + let hu = mv(&hn, u); + let ww = from_cols(hv2, hu, cross(hv2, hu)); + let r = mm(&ww, &mt(&uu)); + let n = cross(v2, u); + let t = mv(&madd(&hn, &r, -1.0), n); + for sgn in [1.0, -1.0] { + let (n2, t2) = (scale(n, sgn), scale(t, sgn)); + // The plane must be in front of the first camera: nᵀ x1 > 0 (with n pointing away). + let front = x1.iter().filter(|p| dot(n2, [p[0], p[1], 1.0]) > 0.0).count(); + if front * 2 >= x1.len() { + out.push((r, t2, n2)); + } + } + } + out +} + +/// Least-squares plane through points: (centroid, unit normal, RMS in-plane spread). +pub fn fit_plane(p: &[V3]) -> Option<(V3, V3, f64)> { + if p.len() < 3 { + return None; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, vecs) = sym_eigen(&cov, 3); + if vals[1] <= 1e-18 { + return None; + } + let n = normalize([vecs[0], vecs[3], vecs[6]]); + Some((c, n, (vals[1] + vals[2]).max(0.0).sqrt())) +} + +/// The rotation `R` minimising `Σ |b − R a|²` over unit rays (orthogonal Procrustes / Kabsch). +pub fn procrustes(a: &[V3], b: &[V3]) -> Option { + if a.len() < 2 { + return None; + } + let mut m = [[0.0; 3]; 3]; + for (x, y) in a.iter().zip(b) { + m = madd(&m, &outer(*y, *x), 1.0); + } + let (u, _, v) = svd3(&m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + Some(r) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn scene() -> (Vec, Pose) { + let mut rng = Rng(12345); + let pts: Vec = (0..60) + .map(|_| { + let f = |r: &mut Rng| (r.next_u64() % 10_000) as f64 / 10_000.0 - 0.5; + [f(&mut rng) * 4.0, f(&mut rng) * 3.0, 5.0 + f(&mut rng) * 3.0] + }) + .collect(); + let p2 = Pose { r: rodrigues([0.02, -0.15, 0.03]), c: [0.6, 0.1, -0.2] }; + (pts, p2) + } + + fn proj(p: &Pose, x: V3) -> [f64; 2] { + let c = p.to_cam(x); + [c[0] / c[2], c[1] / c[2]] + } + + #[test] + fn essential_recovers_relative_pose() { + let (pts, p2) = scene(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let mut x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // Outliers. + x2[3] = [0.3, -0.2]; + x2[17] = [-0.1, 0.25]; + let (e, inl) = ransac_essential(&x1, &x2, 1e-3, 500, 7).unwrap(); + assert!(!inl[3] && !inl[17]); + assert!(inl.iter().filter(|b| **b).count() >= 57); + let (pose, n) = select_pose(&decompose_essential(&e), &x1, &x2).unwrap(); + assert!(n >= 57); + let r_err = norm(log_rot(&mm(&pose.r, &mt(&p2.r)))); + assert!(r_err < 1e-6, "{r_err}"); + let dir = normalize(p2.c); + assert!(norm(sub(normalize(pose.c), dir)) < 1e-5, "{:?} {:?}", pose.c, dir); + } + + #[test] + fn homography_decomposition_contains_the_motion() { + // Plane z = 4 (normal (0,0,1) in camera 1), seen after a motion. + let p2 = Pose { r: rodrigues([0.05, 0.2, -0.04]), c: [0.8, -0.3, 0.4] }; + let pts: Vec = (0..30).map(|i| [((i * 37) % 17) as f64 / 8.0 - 1.0, ((i * 11) % 13) as f64 / 6.0 - 1.0, 4.0]).collect(); + let x1: Vec<[f64; 2]> = pts.iter().map(|x| proj(&Pose::IDENTITY, *x)).collect(); + let x2: Vec<[f64; 2]> = pts.iter().map(|x| proj(&p2, *x)).collect(); + // H = R + t nᵀ / d with t = −R C, n = (0,0,1), d = 4. + let t = p2.t(); + let h = madd(&p2.r, &outer(t, [0.0, 0.0, 0.25]), 1.0); + let sols = decompose_homography(&mscale(&h, 3.7), &x1, &x2); + assert!(!sols.is_empty()); + let ok = sols + .iter() + .any(|(r, td, n)| norm(log_rot(&mm(r, &mt(&p2.r)))) < 1e-6 && norm(sub(*td, scale(t, 0.25))) < 1e-6 && norm(sub(*n, [0.0, 0.0, 1.0])) < 1e-6); + assert!(ok, "{sols:?}"); + } + + #[test] + fn triangulation_and_plane() { + let (pts, p2) = scene(); + for x in &pts[..5] { + let y = triangulate(&[(Pose::IDENTITY, proj(&Pose::IDENTITY, *x)), (p2, proj(&p2, *x))]).unwrap(); + assert!(norm(sub(*x, y)) < 1e-8); + } + let plane: Vec = (0..10).map(|i| [i as f64, (i * i % 7) as f64, 2.0 * i as f64 + 1.0]).collect(); + let (_, n, _) = fit_plane(&plane).unwrap(); + // All points satisfy z = 2x + 1: normal ∝ (2, 0, −1). + assert!(norm(cross(n, normalize([2.0, 0.0, -1.0]))) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs new file mode 100644 index 000000000..a7067f262 --- /dev/null +++ b/crates/track/src/camtrack/linalg.rs @@ -0,0 +1,476 @@ +//! Small dense linear algebra for the camera solver, written from the textbook methods: +//! +//! - 3-vectors / 3 × 3 matrices and rotations (Rodrigues' formula and its inverse); +//! - the **one-sided Jacobi SVD** (Hestenes 1958; Golub & Van Loan, *Matrix Computations*, +//! §8.6.4), accurate for the small, badly scaled systems of the 8-point algorithm; +//! - the **cyclic Jacobi eigenvalue method** for symmetric matrices (Golub & Van Loan §8.5); +//! - **Cholesky** factorisation for the bundle adjustment's normal equations; +//! - the closed-form **similarity alignment** of two point sets (Umeyama, "Least-squares +//! estimation of transformation parameters between two point patterns", PAMI 1991). + +pub type V3 = [f64; 3]; +pub type M3 = [[f64; 3]; 3]; + +pub const I3: M3 = [[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]]; + +pub fn add(a: V3, b: V3) -> V3 { + [a[0] + b[0], a[1] + b[1], a[2] + b[2]] +} +pub fn sub(a: V3, b: V3) -> V3 { + [a[0] - b[0], a[1] - b[1], a[2] - b[2]] +} +pub fn scale(a: V3, k: f64) -> V3 { + [a[0] * k, a[1] * k, a[2] * k] +} +pub fn dot(a: V3, b: V3) -> f64 { + a[0] * b[0] + a[1] * b[1] + a[2] * b[2] +} +pub fn cross(a: V3, b: V3) -> V3 { + [a[1] * b[2] - a[2] * b[1], a[2] * b[0] - a[0] * b[2], a[0] * b[1] - a[1] * b[0]] +} +pub fn norm(a: V3) -> f64 { + dot(a, a).sqrt() +} +pub fn normalize(a: V3) -> V3 { + let n = norm(a); + if n > 0.0 { scale(a, 1.0 / n) } else { a } +} +pub fn mv(m: &M3, v: V3) -> V3 { + [dot(m[0], v), dot(m[1], v), dot(m[2], v)] +} +/// `mᵀ v`. +pub fn mtv(m: &M3, v: V3) -> V3 { + [m[0][0] * v[0] + m[1][0] * v[1] + m[2][0] * v[2], m[0][1] * v[0] + m[1][1] * v[1] + m[2][1] * v[2], m[0][2] * v[0] + m[1][2] * v[1] + m[2][2] * v[2]] +} +pub fn mm(a: &M3, b: &M3) -> M3 { + let mut o = [[0.0; 3]; 3]; + for (i, row) in o.iter_mut().enumerate() { + for (j, v) in row.iter_mut().enumerate() { + *v = a[i][0] * b[0][j] + a[i][1] * b[1][j] + a[i][2] * b[2][j]; + } + } + o +} +pub fn mt(a: &M3) -> M3 { + [[a[0][0], a[1][0], a[2][0]], [a[0][1], a[1][1], a[2][1]], [a[0][2], a[1][2], a[2][2]]] +} +pub fn madd(a: &M3, b: &M3, k: f64) -> M3 { + let mut o = *a; + for i in 0..3 { + for j in 0..3 { + o[i][j] += k * b[i][j]; + } + } + o +} +pub fn mscale(a: &M3, k: f64) -> M3 { + madd(&[[0.0; 3]; 3], a, k) +} +pub fn det3(m: &M3) -> f64 { + m[0][0] * (m[1][1] * m[2][2] - m[1][2] * m[2][1]) - m[0][1] * (m[1][0] * m[2][2] - m[1][2] * m[2][0]) + m[0][2] * (m[1][0] * m[2][1] - m[1][1] * m[2][0]) +} +pub fn skew(v: V3) -> M3 { + [[0.0, -v[2], v[1]], [v[2], 0.0, -v[0]], [-v[1], v[0], 0.0]] +} +pub fn outer(a: V3, b: V3) -> M3 { + [[a[0] * b[0], a[0] * b[1], a[0] * b[2]], [a[1] * b[0], a[1] * b[1], a[1] * b[2]], [a[2] * b[0], a[2] * b[1], a[2] * b[2]]] +} +pub fn col(m: &M3, j: usize) -> V3 { + [m[0][j], m[1][j], m[2][j]] +} +pub fn from_cols(a: V3, b: V3, c: V3) -> M3 { + [[a[0], b[0], c[0]], [a[1], b[1], c[1]], [a[2], b[2], c[2]]] +} +pub fn inv3(m: &M3) -> Option { + let d = det3(m); + if d.abs() < 1e-300 { + return None; + } + let c = |r0: usize, c0: usize, r1: usize, c1: usize| m[r0][c0] * m[r1][c1] - m[r0][c1] * m[r1][c0]; + let adj = [[c(1, 1, 2, 2), -c(0, 1, 2, 2), c(0, 1, 1, 2)], [-c(1, 0, 2, 2), c(0, 0, 2, 2), -c(0, 0, 1, 2)], [c(1, 0, 2, 1), -c(0, 0, 2, 1), c(0, 0, 1, 1)]]; + Some(mscale(&adj, 1.0 / d)) +} + +/// Rotation matrix of an axis-angle vector (Rodrigues' formula). +pub fn rodrigues(w: V3) -> M3 { + let th = norm(w); + let k = skew(w); + if th < 1e-12 { + return madd(&I3, &k, 1.0); + } + let a = th.sin() / th; + let b = (1.0 - th.cos()) / (th * th); + madd(&madd(&I3, &k, a), &mm(&k, &k), b) +} + +/// Axis-angle vector of a rotation matrix (inverse of [`rodrigues`]). +pub fn log_rot(r: &M3) -> V3 { + let c = ((r[0][0] + r[1][1] + r[2][2] - 1.0) * 0.5).clamp(-1.0, 1.0); + let th = c.acos(); + let v = [r[2][1] - r[1][2], r[0][2] - r[2][0], r[1][0] - r[0][1]]; + if th < 1e-9 { + return scale(v, 0.5); + } + if std::f64::consts::PI - th < 1e-6 { + // Near π: the axis is the dominant column of R + I. + let b = madd(r, &I3, 1.0); + let j = (0..3).max_by(|&a, &bb| b[a][a].total_cmp(&b[bb][bb])).unwrap_or(0); + let axis = normalize(col(&b, j)); + return scale(axis, th); + } + scale(v, th / (2.0 * th.sin())) +} + +/// The rotation closest to `m` (Frobenius norm), via the SVD. +pub fn nearest_rotation(m: &M3) -> M3 { + let (u, _, v) = svd3(m); + let mut r = mm(&u, &mt(&v)); + if det3(&r) < 0.0 { + let mut u2 = u; + for row in u2.iter_mut() { + row[2] = -row[2]; + } + r = mm(&u2, &mt(&v)); + } + r +} + +/// Spherical interpolation between two rotations. +pub fn slerp(a: &M3, b: &M3, t: f64) -> M3 { + let d = log_rot(&mm(b, &mt(a))); + mm(&rodrigues(scale(d, t)), a) +} + +/// Thin SVD `A = U diag(s) Vᵀ` of an `m × n` row-major matrix (one-sided Jacobi). `U` is +/// `max(m, n) × n` (rows beyond `m` belong to zero padding), `s` is sorted descending and `V` +/// is `n × n` (columns are the right singular vectors). +pub fn svd(a: &[f64], m: usize, n: usize) -> (Vec, Vec, Vec) { + let rows = m.max(n); + let mut u = vec![0.0; rows * n]; + u[..m * n].copy_from_slice(&a[..m * n]); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _sweep in 0..60 { + let mut off = 0.0f64; + for p in 0..n { + for q in p + 1..n { + let (mut alpha, mut beta, mut gamma) = (0.0, 0.0, 0.0); + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + alpha += x * x; + beta += y * y; + gamma += x * y; + } + if gamma == 0.0 { + continue; + } + let c0 = gamma.abs() / (alpha * beta).sqrt().max(1e-300); + off = off.max(c0); + if c0 < 1e-15 { + continue; + } + let zeta = (beta - alpha) / (2.0 * gamma); + let t = zeta.signum() / (zeta.abs() + (1.0 + zeta * zeta).sqrt()); + let t = if zeta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (1.0 + t * t).sqrt(); + let s = c * t; + for r in 0..rows { + let (x, y) = (u[r * n + p], u[r * n + q]); + u[r * n + p] = c * x - s * y; + u[r * n + q] = s * x + c * y; + } + for r in 0..n { + let (x, y) = (v[r * n + p], v[r * n + q]); + v[r * n + p] = c * x - s * y; + v[r * n + q] = s * x + c * y; + } + } + } + if off < 1e-15 { + break; + } + } + let mut s: Vec = (0..n).map(|j| (0..rows).map(|r| u[r * n + j] * u[r * n + j]).sum::().sqrt()).collect(); + for (j, sj) in s.iter().enumerate() { + if *sj > 1e-300 { + for r in 0..rows { + u[r * n + j] /= sj; + } + } + } + // Sort descending. + let mut order: Vec = (0..n).collect(); + order.sort_by(|&a, &b| s[b].total_cmp(&s[a])); + let (mut u2, mut v2) = (vec![0.0; rows * n], vec![0.0; n * n]); + for (k, &j) in order.iter().enumerate() { + for r in 0..rows { + u2[r * n + k] = u[r * n + j]; + } + for r in 0..n { + v2[r * n + k] = v[r * n + j]; + } + } + s = order.iter().map(|&j| s[j]).collect(); + (u2, s, v2) +} + +/// SVD of a 3 × 3 matrix: (U, singular values descending, V) with `m = U diag(s) Vᵀ`. +pub fn svd3(m: &M3) -> (M3, V3, M3) { + let a: Vec = m.iter().flatten().copied().collect(); + let (u, s, v) = svd(&a, 3, 3); + let g = |x: &[f64]| [[x[0], x[1], x[2]], [x[3], x[4], x[5]], [x[6], x[7], x[8]]]; + let mut um = g(&u); + // Columns of U for zero singular values are undefined: complete them to an orthonormal basis. + if s[2] < 1e-300 { + let c = cross(col(&um, 0), col(&um, 1)); + for (r, row) in um.iter_mut().enumerate() { + row[2] = c[r]; + } + } + (um, [s[0], s[1], s[2]], g(&v)) +} + +/// The unit vector `x` minimising `|A x|` (`m × n` row-major): the right singular vector of +/// the smallest singular value. +pub fn null_vector(a: &[f64], m: usize, n: usize) -> Vec { + let (_, _, v) = svd(a, m, n); + (0..n).map(|r| v[r * n + n - 1]).collect() +} + +/// Eigen-decomposition of a symmetric `n × n` matrix (cyclic Jacobi): eigenvalues ascending +/// and the matching eigenvectors as the columns of the returned row-major matrix. +pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { + let mut a = a.to_vec(); + let mut v = vec![0.0; n * n]; + for i in 0..n { + v[i * n + i] = 1.0; + } + for _ in 0..100 { + let mut off = 0.0; + for p in 0..n { + for q in p + 1..n { + off += a[p * n + q] * a[p * n + q]; + } + } + if off < 1e-30 { + break; + } + for p in 0..n { + for q in p + 1..n { + let apq = a[p * n + q]; + if apq.abs() < 1e-300 { + continue; + } + let theta = (a[q * n + q] - a[p * n + p]) / (2.0 * apq); + let t = theta.signum() / (theta.abs() + (theta * theta + 1.0).sqrt()); + let t = if theta == 0.0 { 1.0 } else { t }; + let c = 1.0 / (t * t + 1.0).sqrt(); + let s = t * c; + for k in 0..n { + let (akp, akq) = (a[k * n + p], a[k * n + q]); + a[k * n + p] = c * akp - s * akq; + a[k * n + q] = s * akp + c * akq; + } + for k in 0..n { + let (apk, aqk) = (a[p * n + k], a[q * n + k]); + a[p * n + k] = c * apk - s * aqk; + a[q * n + k] = s * apk + c * aqk; + } + for k in 0..n { + let (vkp, vkq) = (v[k * n + p], v[k * n + q]); + v[k * n + p] = c * vkp - s * vkq; + v[k * n + q] = s * vkp + c * vkq; + } + } + } + } + let mut order: Vec = (0..n).collect(); + order.sort_by(|&x, &y| a[x * n + x].total_cmp(&a[y * n + y])); + let vals = order.iter().map(|&j| a[j * n + j]).collect(); + let mut vecs = vec![0.0; n * n]; + for (k, &j) in order.iter().enumerate() { + for r in 0..n { + vecs[r * n + k] = v[r * n + j]; + } + } + (vals, vecs) +} + +/// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by +/// Cholesky factorisation. Returns false when `A` is not positive definite. +pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { + for j in 0..n { + let mut d = a[j * n + j]; + for k in 0..j { + d -= a[j * n + k] * a[j * n + k]; + } + if d <= 0.0 || !d.is_finite() { + return false; + } + let d = d.sqrt(); + a[j * n + j] = d; + let (head, tail) = a.split_at_mut((j + 1) * n); + let rowj = &head[j * n..j * n + j]; + // Rows below j are independent. + use rayon::prelude::*; + let work = |row: &mut [f64]| { + let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); + row[j] = s / d; + }; + if n - j > 192 { + tail.par_chunks_mut(n).for_each(work); + } else { + tail.chunks_mut(n).for_each(work); + } + } + // Forward: L y = b. + for i in 0..n { + let mut s = b[i]; + for k in 0..i { + s -= a[i * n + k] * b[k]; + } + b[i] = s / a[i * n + i]; + } + // Back: Lᵀ x = y. + for i in (0..n).rev() { + let mut s = b[i]; + for k in i + 1..n { + s -= a[k * n + i] * b[k]; + } + b[i] = s / a[i * n + i]; + } + b.iter().all(|v| v.is_finite()) +} + +/// A similarity transform `x ↦ s R x + t`. +#[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct Similarity { + pub s: f64, + pub r: M3, + pub t: V3, +} + +impl Default for Similarity { + fn default() -> Self { + Similarity { s: 1.0, r: I3, t: [0.0; 3] } + } +} + +impl Similarity { + pub fn apply(&self, x: V3) -> V3 { + add(scale(mv(&self.r, x), self.s), self.t) + } +} + +/// The similarity minimising `Σ |dst − (s R src + t)|²` (Umeyama 1991). +pub fn umeyama(src: &[V3], dst: &[V3]) -> Option { + let n = src.len().min(dst.len()); + if n < 3 { + return None; + } + let k = 1.0 / n as f64; + let ms = scale(src[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let md = scale(dst[..n].iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [[0.0; 3]; 3]; + let mut var = 0.0; + for i in 0..n { + let (a, b) = (sub(src[i], ms), sub(dst[i], md)); + cov = madd(&cov, &outer(b, a), k); + var += dot(a, a) * k; + } + if var < 1e-300 { + return None; + } + let (u, d, v) = svd3(&cov); + let mut sgn = [1.0, 1.0, 1.0]; + if det3(&u) * det3(&v) < 0.0 { + sgn[2] = -1.0; + } + let ud = from_cols(scale(col(&u, 0), sgn[0]), scale(col(&u, 1), sgn[1]), scale(col(&u, 2), sgn[2])); + let r = mm(&ud, &mt(&v)); + let s = (d[0] * sgn[0] + d[1] * sgn[1] + d[2] * sgn[2]) / var; + let t = sub(md, scale(mv(&r, ms), s)); + Some(Similarity { s, r, t }) +} + +/// Euler angles (degrees) `[x, y, z]` with `m = Rz(z) Ry(y) Rx(x)` (After Effects' Orientation +/// order: X, then Y, then Z applied to the vector). +pub fn euler_xyz(m: &M3) -> V3 { + let sy = (-m[2][0]).clamp(-1.0, 1.0); + let y = sy.asin(); + let (x, z) = if sy.abs() < 0.999_999 { (m[2][1].atan2(m[2][2]), m[1][0].atan2(m[0][0])) } else { ((-m[1][2]).atan2(m[1][1]), 0.0) }; + [x.to_degrees(), y.to_degrees(), z.to_degrees()] +} + +/// `Rz(z) Ry(y) Rx(x)` from degrees. +pub fn from_euler_xyz(e: V3) -> M3 { + let (sx, cx) = e[0].to_radians().sin_cos(); + let (sy, cy) = e[1].to_radians().sin_cos(); + let (sz, cz) = e[2].to_radians().sin_cos(); + let rx = [[1.0, 0.0, 0.0], [0.0, cx, -sx], [0.0, sx, cx]]; + let ry = [[cy, 0.0, sy], [0.0, 1.0, 0.0], [-sy, 0.0, cy]]; + let rz = [[cz, -sz, 0.0], [sz, cz, 0.0], [0.0, 0.0, 1.0]]; + mm(&rz, &mm(&ry, &rx)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn svd_reconstructs() { + let a = [3.0, 1.0, 2.0, -1.0, 4.0, 0.5, 2.0, 2.0, 7.0, 0.1, -3.0, 1.0]; + let (u, s, v) = svd(&a, 4, 3); + for r in 0..4 { + for c in 0..3 { + let x: f64 = (0..3).map(|k| u[r * 3 + k] * s[k] * v[c * 3 + k]).sum(); + assert!((x - a[r * 3 + c]).abs() < 1e-10); + } + } + assert!(s[0] >= s[1] && s[1] >= s[2]); + } + + #[test] + fn rotations_round_trip() { + let w = [0.3, -1.2, 0.7]; + let r = rodrigues(w); + let w2 = log_rot(&r); + assert!(norm(sub(w, w2)) < 1e-12); + let e = euler_xyz(&r); + let r2 = from_euler_xyz(e); + for i in 0..3 { + for j in 0..3 { + assert!((r[i][j] - r2[i][j]).abs() < 1e-12); + } + } + assert!((det3(&r) - 1.0).abs() < 1e-12); + } + + #[test] + fn eigen_and_cholesky() { + let a = [4.0, 1.0, 0.5, 1.0, 3.0, 0.2, 0.5, 0.2, 2.0]; + let (vals, vecs) = sym_eigen(&a, 3); + for k in 0..3 { + let v = [vecs[k], vecs[3 + k], vecs[6 + k]]; + let av = [dot([a[0], a[1], a[2]], v), dot([a[3], a[4], a[5]], v), dot([a[6], a[7], a[8]], v)]; + assert!(norm(sub(av, scale(v, vals[k]))) < 1e-10); + } + let mut m = a; + let mut b = [1.0, 2.0, 3.0]; + assert!(cholesky_solve(&mut m, 3, &mut b)); + let r = [dot([a[0], a[1], a[2]], b), dot([a[3], a[4], a[5]], b), dot([a[6], a[7], a[8]], b)]; + assert!(norm(sub(r, [1.0, 2.0, 3.0])) < 1e-12); + } + + #[test] + fn umeyama_recovers_a_similarity() { + let t = Similarity { s: 2.5, r: rodrigues([0.1, 0.4, -0.3]), t: [1.0, -2.0, 5.0] }; + let src = [[0.0, 0.0, 0.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0], [1.0, 2.0, 3.0]]; + let dst: Vec = src.iter().map(|p| t.apply(*p)).collect(); + let e = umeyama(&src, &dst).unwrap(); + assert!((e.s - 2.5).abs() < 1e-10); + assert!(norm(sub(e.t, t.t)) < 1e-9); + } +} diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs new file mode 100644 index 000000000..056d7ad03 --- /dev/null +++ b/crates/track/src/camtrack/mod.rs @@ -0,0 +1,376 @@ +//! The 3D Camera Tracker's analysis: structure from motion on a single clip, clean-room from the +//! textbook pipeline (Hartley & Zisserman, *Multiple View Geometry in Computer Vision*, 2nd ed., +//! 2004; Triggs et al., "Bundle Adjustment — A Modern Synthesis", 2000): +//! +//! 1. **Feature tracks** ([`features::TrackAnalyzer`]): Shi–Tomasi corners followed through the +//! clip with pyramidal Lucas–Kanade and a forward–backward check ([`crate::klt`]); lost +//! features end their track and new ones are detected where the frame has none, so tracks +//! are as long as possible. +//! 2. **Solve** ([`solve`]): keyframes are selected by parallax; a two-view reconstruction +//! initialises the scene from either the **essential matrix** (normalised 8-point algorithm +//! inside RANSAC, H&Z §11.2 and §9.6) or, for mostly flat scenes, a **homography +//! decomposition** (Ma, Soatto, Košecká & Sastry, *An Invitation to 3-D Vision*, 2004, §5.3); +//! the other keyframes are added incrementally (**resection** by robust pose refinement, +//! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse +//! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it +//! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal +//! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment +//! as one shared parameter; *Variable Zoom* starts from the same estimate and adjusts a focal +//! length per frame, with a weak smoothness prior between neighbouring frames. +//! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame +//! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle +//! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the +//! tripod solution when it explains the tracks as well as the general one. +//! +//! The camera model has square pixels, the principal point at the layer centre and no lens +//! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera +//! is `(R, C, f)` with `X_cam = R (X − C)` and pixel `u = f x / z + w / 2`. +//! +//! The solve is stored in a canonical frame: the first solved frame's camera is at the origin +//! looking along +z, scaled so the median depth of the points it sees is 1. [`CameraSolve::world`] +//! maps it into composition space for a camera zoom, optionally re-based on a ground plane. + +pub mod bundle; +pub mod features; +pub mod geometry; +pub mod linalg; +mod solver; +#[cfg(test)] +mod tests; + +use serde::{Deserialize, Serialize}; + +pub use features::{AnalyzeOpts, TrackAnalyzer}; +use linalg::{M3, Similarity, V3}; +pub use solver::{SolveError, solve}; + +/// One feature track: positions (layer pixels) on consecutive frames from `start`. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Track2D { + pub id: u32, + pub start: u32, + pub pts: Vec<[f32; 2]>, +} + +impl Track2D { + pub fn end(&self) -> u32 { + self.start + self.pts.len() as u32 + } + /// Position on frame `k` when the track covers it. + pub fn at(&self, k: u32) -> Option<[f64; 2]> { + let i = k.checked_sub(self.start)? as usize; + self.pts.get(i).map(|p| [p[0] as f64, p[1] as f64]) + } +} + +/// The tracked features of a clip (step 1 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraTracks { + pub version: u32, + /// Layer size (pixels). + pub size: [f64; 2], + /// Layer time (seconds) of frame 0, and the frame duration. + pub start: f64, + pub frame_duration: f64, + pub frames: u32, + /// Analysis downsampling factor (image pixels per analysis pixel): the tracking noise scale. + pub factor: f64, + pub detailed: bool, + pub tracks: Vec, +} + +impl CameraTracks { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames == 0 + } + pub fn track(&self, id: u32) -> Option<&Track2D> { + self.tracks.binary_search_by_key(&id, |t| t.id).ok().map(|i| &self.tracks[i]) + } +} + +/// 3D Camera Tracker ▸ Shot Type. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ShotType { + #[default] + FixedAngle, + VariableZoom, + SpecifyAngle, +} + +impl ShotType { + pub const ALL: [ShotType; 3] = [ShotType::FixedAngle, ShotType::VariableZoom, ShotType::SpecifyAngle]; + pub fn label(self) -> &'static str { + match self { + ShotType::FixedAngle => "Fixed Angle of View", + ShotType::VariableZoom => "Variable Zoom", + ShotType::SpecifyAngle => "Specify Angle of View", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "fixed" | "fixedangle" | "fixedangleofview" => Some(ShotType::FixedAngle), + "variable" | "variablezoom" | "zoom" => Some(ShotType::VariableZoom), + "specify" | "specifyangle" | "specifyangleofview" => Some(ShotType::SpecifyAngle), + _ => None, + } + } +} + +/// 3D Camera Tracker ▸ Advanced ▸ Solve Method (and Method Used). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SolveMethod { + #[default] + Auto, + Typical, + MostlyFlat, + TripodPan, +} + +impl SolveMethod { + pub const ALL: [SolveMethod; 4] = [SolveMethod::Auto, SolveMethod::Typical, SolveMethod::MostlyFlat, SolveMethod::TripodPan]; + pub fn label(self) -> &'static str { + match self { + SolveMethod::Auto => "Auto Detect", + SolveMethod::Typical => "Typical", + SolveMethod::MostlyFlat => "Mostly Flat Scene", + SolveMethod::TripodPan => "Tripod Pan", + } + } + pub fn from_name(s: &str) -> Option { + let k = s.to_ascii_lowercase().replace([' ', '-', '_'], ""); + match k.as_str() { + "auto" | "autodetect" => Some(SolveMethod::Auto), + "typical" => Some(SolveMethod::Typical), + "flat" | "mostlyflat" | "mostlyflatscene" => Some(SolveMethod::MostlyFlat), + "tripod" | "tripodpan" => Some(SolveMethod::TripodPan), + _ => None, + } + } +} + +/// What a solve is made with. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolveSettings { + pub shot: ShotType, + /// Horizontal angle of view (degrees) for [`ShotType::SpecifyAngle`]. + pub hfov: f64, + pub method: SolveMethod, + /// Track ids the user deleted. + pub deleted: Vec, +} + +/// The solved camera of one frame. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedFrame { + /// World → camera rotation (axis-angle). + pub rot: V3, + /// Camera centre. + pub center: V3, + /// Focal length (layer pixels). + pub focal: f64, + pub solved: bool, +} + +impl SolvedFrame { + pub fn r(&self) -> M3 { + linalg::rodrigues(self.rot) + } + /// Camera-space position of a world point. + pub fn to_cam(&self, x: V3) -> V3 { + linalg::mv(&self.r(), linalg::sub(x, self.center)) + } + /// Project a world point to layer pixels (`None` behind the camera). + pub fn project(&self, size: [f64; 2], x: V3) -> Option<[f64; 2]> { + let c = self.to_cam(x); + (c[2] > 1e-9).then(|| [self.focal * c[0] / c[2] + size[0] * 0.5, self.focal * c[1] / c[2] + size[1] * 0.5]) + } +} + +/// A solved 3D track point. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct SolvedPoint { + /// The feature track it was made from. + pub id: u32, + pub pos: V3, + /// Average reprojection error (pixels). + pub error: f32, + /// Frames `[first, last]` the point was tracked on. + pub first: u32, + pub last: u32, +} + +/// A ground plane and origin (canonical solve coordinates). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Ground { + pub origin: V3, + /// Unit normal on the cameras' side ("up"). + pub normal: V3, + /// Unit in-plane direction that becomes world +x. + pub x_axis: V3, +} + +/// A plane through solved points: After Effects' target. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct Target { + pub center: V3, + /// Unit normal facing the camera. + pub normal: V3, + /// Diameter in canonical units. + pub size: f64, +} + +/// The camera solve (step 2 of the analysis). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", default)] +pub struct CameraSolve { + pub version: u32, + pub size: [f64; 2], + pub start: f64, + pub frame_duration: f64, + pub shot: ShotType, + /// The model that was used (Auto Detect resolves to one of the others). + pub method_used: SolveMethod, + /// Average reprojection error (pixels) over all solved observations. + pub average_error: f64, + pub frames: Vec, + pub points: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub ground: Option, +} + +impl CameraSolve { + pub fn to_json(&self) -> String { + serde_json::to_string(self).unwrap_or_default() + } + pub fn from_json(s: &str) -> Option { + if s.trim().is_empty() { + return None; + } + serde_json::from_str(s).ok() + } + pub fn is_empty(&self) -> bool { + self.frames.is_empty() + } + /// Frame index at a layer time (held at the ends). + pub fn frame_at(&self, layer_time: f64) -> Option { + if self.frames.is_empty() || self.frame_duration <= 0.0 { + return None; + } + let i = ((layer_time - self.start) / self.frame_duration).round(); + Some(i.clamp(0.0, (self.frames.len() - 1) as f64) as usize) + } + pub fn point(&self, id: u32) -> Option<&SolvedPoint> { + self.points.iter().find(|p| p.id == id) + } + /// Points tracked on frame `k`. + pub fn visible(&self, k: usize) -> impl Iterator { + let k = k as u32; + self.points.iter().filter(move |p| p.first <= k && k <= p.last) + } + /// Median focal length of the solved frames (layer pixels). + pub fn focal(&self) -> f64 { + let mut f: Vec = self.frames.iter().filter(|c| c.solved).map(|c| c.focal).collect(); + if f.is_empty() { + return 0.0; + } + f.sort_by(f64::total_cmp); + f[f.len() / 2] + } + /// Horizontal angle of view (degrees) of focal length `f`. + pub fn hfov(&self, f: f64) -> f64 { + 2.0 * (self.size[0] * 0.5 / f.max(1e-9)).atan().to_degrees() + } + + /// The plane through points `ids` seen from frame `k` (needs three points; with more the + /// plane is a least-squares fit). + pub fn target(&self, ids: &[u32], k: usize) -> Option { + let pts: Vec = ids.iter().filter_map(|i| self.point(*i)).map(|p| p.pos).collect(); + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + geometry::fit_plane(&pts).map(|(c, mut n, spread)| { + if linalg::dot(n, linalg::sub(cam.center, c)) < 0.0 { + n = linalg::scale(n, -1.0); + } + // After Effects sizes the target from the camera distance when the points are close. + let depth = linalg::norm(linalg::sub(cam.center, c)); + Target { center: c, normal: n, size: (spread * 2.0).max(depth * 0.12) } + }) + } + + /// A ground plane through `ids` (origin at their centroid, normal towards frame `k`'s + /// camera, +x along that camera's right vector projected on the plane). + pub fn ground_from(&self, ids: &[u32], k: usize) -> Option { + let t = self.target(ids, k)?; + let cam = self.frames.get(k).filter(|c| c.solved).or_else(|| self.frames.iter().find(|c| c.solved))?; + let right = linalg::mtv(&cam.r(), [1.0, 0.0, 0.0]); + let mut x = linalg::sub(right, linalg::scale(t.normal, linalg::dot(right, t.normal))); + if linalg::norm(x) < 1e-6 { + let fwd = linalg::mtv(&cam.r(), [0.0, 0.0, 1.0]); + x = linalg::cross(fwd, t.normal); + } + Some(Ground { origin: t.center, normal: t.normal, x_axis: linalg::normalize(x) }) + } + + /// Canonical solve coordinates → composition world, for a camera whose zoom on the first + /// solved frame is `zoom0` (comp pixels) in a `comp` sized composition. Without a ground + /// plane, the first solved frame's camera becomes After Effects' default camera (at + /// `(w/2, h/2, −zoom)` looking down +z, so points at the median depth land near z = 0). With + /// one, the ground plane becomes the X-Z plane through the origin, normal along −y (up). + pub fn world(&self, comp: [f64; 2], zoom0: f64) -> Similarity { + match &self.ground { + Some(g) => { + // Rows: canonical directions that map to world x, y, z. + let y = linalg::scale(g.normal, -1.0); + let x = g.x_axis; + let z = linalg::cross(x, y); + let r = [x, y, z]; + let t = linalg::scale(linalg::mv(&r, g.origin), -zoom0); + Similarity { s: zoom0, r, t } + } + None => Similarity { s: zoom0, r: linalg::I3, t: [comp[0] * 0.5, comp[1] * 0.5, -zoom0] }, + } + } + + /// The first solved frame. + pub fn first_solved(&self) -> Option { + self.frames.iter().position(|c| c.solved) + } +} + +/// Deterministic colour of a track point (After Effects draws each target in its own colour). +pub fn point_color(id: u32) -> [f32; 3] { + let mut h = id.wrapping_mul(0x9E37_79B9) ^ 0x5bd1_e995; + h ^= h >> 15; + let hue = (h % 360) as f32 / 60.0; + let x = 1.0 - (hue % 2.0 - 1.0).abs(); + let (r, g, b) = match hue as u32 { + 0 => (1.0, x, 0.0), + 1 => (x, 1.0, 0.0), + 2 => (0.0, 1.0, x), + 3 => (0.0, x, 1.0), + 4 => (x, 0.0, 1.0), + _ => (1.0, 0.0, x), + }; + // Bright, slightly desaturated. + [0.25 + 0.75 * r, 0.25 + 0.75 * g, 0.25 + 0.75 * b] +} diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs new file mode 100644 index 000000000..da3a92908 --- /dev/null +++ b/crates/track/src/camtrack/solver.rs @@ -0,0 +1,894 @@ +//! The camera solve pipeline (see the module docs of [`super`]). + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::atomic::{AtomicBool, Ordering}; + +use rayon::prelude::*; + +use super::bundle::{self, BaCam, BaObs, BaOpts, FocalMode}; +use super::geometry::{self, Pose}; +use super::linalg::*; +use super::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, SolvedFrame, SolvedPoint}; +use crate::fit::{Model, ransac}; + +fn debug() -> bool { + std::env::var_os("EC_CAMTRACK_DEBUG").is_some() +} + +/// Why a solve failed. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SolveError(pub String); + +impl std::fmt::Display for SolveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +fn err(m: &str) -> Result { + Err(SolveError(m.to_string())) +} + +/// Tracks in solver form: pixels relative to the principal point. +struct Data { + n: usize, + w: f64, + /// (track id, first frame, positions). + tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, + /// Per frame: (track index, position). + by_frame: Vec>, + /// Tracking noise scale (pixels). + noise: f64, +} + +impl Data { + fn new(t: &CameraTracks, deleted: &[u32]) -> Data { + let n = t.frames as usize; + let c = [t.size[0] * 0.5, t.size[1] * 0.5]; + let del: HashSet = deleted.iter().copied().collect(); + let mut tracks = vec![]; + let mut by_frame = vec![vec![]; n]; + for tr in t.tracks.iter().filter(|tr| !del.contains(&tr.id) && tr.pts.len() >= 3) { + let ti = tracks.len() as u32; + let pts: Vec<[f64; 2]> = tr.pts.iter().map(|p| [p[0] as f64 - c[0], p[1] as f64 - c[1]]).collect(); + for (k, p) in pts.iter().enumerate() { + if let Some(v) = by_frame.get_mut(tr.start as usize + k) { + v.push((ti, *p)); + } + } + tracks.push((tr.id, tr.start as usize, pts)); + } + Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + } + fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { + let (_, s, p) = &self.tracks[ti as usize]; + k.checked_sub(*s).and_then(|i| p.get(i).copied()) + } + fn inlier_thr(&self) -> f64 { + 4.0 * self.noise + } + fn huber(&self) -> f64 { + 1.0 * self.noise + } +} + +/// Parallax-based keyframes: a new keyframe when the median feature motion since the last +/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared +/// features drop below 24. The first and last frames with features are keyframes. +fn keyframes(d: &Data) -> Vec { + let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; + let mut kf = vec![first]; + let mut last = first; + let mut j = first + 1; + while j < d.n { + let base = &d.by_frame[last]; + let mut disp = vec![]; + for (ti, p) in base { + if let Some(q) = d.at(*ti, j) { + disp.push((q[0] - p[0]).hypot(q[1] - p[1])); + } + } + if disp.len() < 24 && j > last + 1 { + // Too few shared features: the previous frame becomes a keyframe. + last = j - 1; + kf.push(last); + continue; + } + disp.sort_by(f64::total_cmp); + let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); + let ratio = disp.len() as f64 / base.len().max(1) as f64; + if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + last = j; + kf.push(j); + } + j += 1; + } + if let Some(lastf) = (0..d.n).rev().find(|k| d.by_frame[*k].len() >= 8) + && *kf.last().expect("one") < lastf + { + kf.push(lastf); + } + kf +} + +/// A (partial) reconstruction. +#[derive(Clone, Default)] +struct Recon { + cams: BTreeMap, + pts: HashMap, + /// Rejected observations (track index, frame). + outliers: HashSet<(u32, usize)>, +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum Kind { + General, + Tripod, +} + +#[derive(Clone, Copy, PartialEq, Debug)] +struct Cfg { + kind: Kind, + /// Focal mode of the global adjustments. + focal: FocalMode, + /// Registration estimates a focal length per frame. + per_frame_f: bool, +} + +const F_PRIOR: f64 = 30.0; + +impl Recon { + /// Observations among `frames` (inliers, with a 3D point). + fn observations(&self, d: &Data, frames: &[usize]) -> (Vec, Vec, Vec, Vec) { + let cams: Vec = frames.iter().map(|k| self.cams[k]).collect(); + let mut pidx: HashMap = HashMap::new(); + let mut pts = vec![]; + let mut ids = vec![]; + let mut obs = vec![]; + for (ci, k) in frames.iter().enumerate() { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + let pi = *pidx.entry(*ti).or_insert_with(|| { + pts.push(*x); + ids.push(*ti); + (pts.len() - 1) as u32 + }); + obs.push(BaObs { cam: ci as u32, pt: pi, uv: *uv }); + } + } + (cams, pts, obs, ids) + } + + /// Bundle-adjust the cameras of `frames` (the first stays fixed) and their points. + fn adjust(&mut self, d: &Data, frames: &[usize], cfg: &Cfg, focal: FocalMode, iters: usize) -> f64 { + let (mut cams, mut pts, obs, ids) = self.observations(d, frames); + if obs.is_empty() { + return f64::INFINITY; + } + let mut fixed = vec![false; cams.len()]; + fixed[0] = true; + let rep = bundle::adjust( + &mut cams, + &mut pts, + &obs, + &BaOpts { focal, fix_points: false, fix_centers: cfg.kind == Kind::Tripod, fixed, huber: d.huber(), max_iter: iters }, + ); + for (k, c) in frames.iter().zip(&cams) { + self.cams.insert(*k, *c); + } + for (ti, x) in ids.iter().zip(&pts) { + self.pts.insert(*ti, *x); + } + rep.cost + } + + /// Mark observations above the inlier threshold as outliers; drop points left with fewer + /// than two observations among the registered frames. + fn reject(&mut self, d: &Data) { + let thr2 = d.inlier_thr().powi(2); + let mut count: HashMap = HashMap::new(); + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + match bundle::residual(c, *x, *uv) { + Some(r) if r[0] * r[0] + r[1] * r[1] <= thr2 => *count.entry(*ti).or_default() += 1, + _ => { + self.outliers.insert((*ti, *k)); + } + } + } + } + self.pts.retain(|ti, _| count.get(ti).copied().unwrap_or(0) >= 2); + } + + /// Triangulate tracks seen on ≥ 2 registered frames that have no point yet. + fn triangulate(&mut self, d: &Data, kind: Kind, min_angle_deg: f64, check: f64) { + let mut seen: HashMap> = HashMap::new(); + for k in self.cams.keys() { + for (ti, _) in &d.by_frame[*k] { + if !self.pts.contains_key(ti) { + seen.entry(*ti).or_default().push(*k); + } + } + } + let thr2 = (d.inlier_thr() * check).powi(2); + let new: Vec<(u32, V3)> = seen + .par_iter() + .filter(|(_, ks)| ks.len() >= 2) + .filter_map(|(ti, ks)| { + let views: Vec<(Pose, [f64; 2], &BaCam)> = ks + .iter() + .map(|k| { + let c = &self.cams[k]; + let uv = d.at(*ti, *k).expect("seen"); + (c.pose, [uv[0] / c.f, uv[1] / c.f], c) + }) + .collect(); + let x = match kind { + Kind::Tripod => normalize(views.iter().fold([0.0; 3], |a, (p, o, _)| add(a, normalize(mtv(&p.r, [o[0], o[1], 1.0]))))), + Kind::General => { + let v: Vec<(Pose, [f64; 2])> = views.iter().map(|(p, o, _)| (*p, *o)).collect(); + let x = geometry::triangulate(&v)?; + // Enough parallax between the most separated views. + let (a, b) = (views[0].0.c, views[views.len() - 1].0.c); + if geometry::ray_angle(a, b, x).to_degrees() < min_angle_deg { + return None; + } + x + } + }; + for (k, (_, _, c)) in ks.iter().zip(&views) { + let uv = d.at(*ti, *k).expect("seen"); + let r = bundle::residual(c, x, uv)?; + if r[0] * r[0] + r[1] * r[1] > thr2 { + return None; + } + } + Some((*ti, x)) + }) + .collect(); + self.pts.extend(new); + } + + /// Resect frame `k` from the existing points starting at `init` (pose, and the focal + /// length when `per_frame_f`). Returns false when too few points agree. + fn register(&mut self, d: &Data, k: usize, init: BaCam, cfg: &Cfg) -> bool { + let mut pts = vec![]; + let mut obs = vec![]; + let mut ids = vec![]; + for (ti, uv) in &d.by_frame[k] { + if let Some(x) = self.pts.get(ti) { + obs.push(BaObs { cam: 0, pt: pts.len() as u32, uv: *uv }); + pts.push(*x); + ids.push(*ti); + } + } + if obs.len() < 6 { + return false; + } + let thr2 = d.inlier_thr().powi(2); + let opts = |huber: f64| BaOpts { + focal: if cfg.per_frame_f { FocalMode::PerCamera(0.0) } else { FocalMode::Fixed }, + fix_points: true, + fix_centers: cfg.kind == Kind::Tripod, + fixed: vec![false], + huber, + max_iter: 40, + }; + let mut best: Option<(usize, BaCam)> = None; + // Robust start (large Huber threshold), then tighten; from the prediction and, failing + // that, from the nearest registered camera. + let nearest = self.cams.iter().min_by_key(|(c, _)| c.abs_diff(k)).map(|(_, c)| *c); + for start in std::iter::once(init).chain(nearest) { + if best.as_ref().is_some_and(|b| b.0 as f64 >= 0.8 * obs.len() as f64) { + break; + } + let mut cams = vec![start]; + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.inlier_thr() * 4.0)); + bundle::adjust(&mut cams, &mut pts.clone(), &obs, &opts(d.huber())); + let inl = obs.iter().filter(|o| bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).count(); + if best.as_ref().is_none_or(|b| inl > b.0) { + best = Some((inl, cams[0])); + } + } + let Some((inl, cam)) = best else { return false }; + if inl < 6 || (inl as f64) < 0.5 * obs.len() as f64 { + return false; + } + // Refine on the inliers only. + let keep: Vec = + obs.iter().filter(|o| bundle::residual(&cam, pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2)).copied().collect(); + let mut cams = vec![cam]; + bundle::adjust(&mut cams, &mut pts.clone(), &keep, &opts(d.huber())); + for o in &obs { + let ok = bundle::residual(&cams[0], pts[o.pt as usize], o.uv).is_some_and(|r| r[0] * r[0] + r[1] * r[1] <= thr2); + if !ok { + self.outliers.insert((ids[o.pt as usize], k)); + } + } + self.cams.insert(k, cams[0]); + true + } + + /// How well the reconstruction explains every observation on its frames: the mean of the + /// reprojection error truncated at the inlier threshold, an observation without a point + /// counting as the threshold. + fn score(&self, d: &Data) -> f64 { + let thr = d.inlier_thr(); + let mut s = 0.0; + let mut n = 0usize; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + n += 1; + s += match self.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + Some(r) => r[0].hypot(r[1]).min(thr), + None => thr, + }; + } + } + if n > 0 { s / n as f64 } else { f64::INFINITY } + } + + /// Average reprojection error (pixels) and observation count over the inliers. + fn error(&self, d: &Data) -> (f64, usize) { + let mut s = 0.0; + let mut n = 0; + for (k, c) in &self.cams { + for (ti, uv) in &d.by_frame[*k] { + let Some(x) = self.pts.get(ti) else { continue }; + if self.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(r) = bundle::residual(c, *x, *uv) { + s += r[0].hypot(r[1]); + n += 1; + } + } + } + (if n > 0 { s / n as f64 } else { f64::INFINITY }, n) + } +} + +/// Interpolated / extrapolated starting camera for frame `k` from the registered ones. +fn predict(r: &Recon, k: usize) -> Option { + let before = r.cams.range(..k).next_back().map(|(a, b)| (*a, *b)); + let after = r.cams.range(k + 1..).next().map(|(a, b)| (*a, *b)); + match (before, after) { + (Some((ka, a)), Some((kb, b))) => { + let t = (k - ka) as f64 / (kb - ka) as f64; + Some(BaCam { pose: Pose { r: slerp(&a.pose.r, &b.pose.r, t), c: add(scale(a.pose.c, 1.0 - t), scale(b.pose.c, t)) }, f: a.f * (b.f / a.f).powf(t) }) + } + (Some((ka, a)), None) => { + // Constant velocity from the two previous cameras. + match r.cams.range(..ka).next_back() { + Some((kp, p)) => { + let t = (k - ka) as f64 / (ka - kp) as f64; + let dr = log_rot(&mm(&a.pose.r, &mt(&p.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &a.pose.r); + let c = add(a.pose.c, scale(sub(a.pose.c, p.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: a.f }) + } + None => Some(a), + } + } + (None, Some((kb, b))) => match r.cams.range(kb + 1..).next() { + Some((kn, nx)) => { + let t = (kb - k) as f64 / (kn - kb) as f64; + let dr = log_rot(&mm(&b.pose.r, &mt(&nx.pose.r))); + let rr = mm(&rodrigues(scale(dr, t.min(2.0))), &b.pose.r); + let c = add(b.pose.c, scale(sub(b.pose.c, nx.pose.c), t.min(2.0))); + Some(BaCam { pose: Pose { r: rr, c }, f: b.f }) + } + None => Some(b), + }, + (None, None) => None, + } +} + +/// Shared tracks of frames `a` and `b`: (track index, pixels in a, pixels in b). +fn shared(d: &Data, a: usize, b: usize) -> Vec<(u32, [f64; 2], [f64; 2])> { + d.by_frame[a].iter().filter_map(|(ti, p)| d.at(*ti, b).map(|q| (*ti, *p, q))).collect() +} + +/// Two-view initialisations of frames `a`, `b` at focal length `f`: the second camera and the +/// triangulated points (the first camera is the identity). +fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> Vec<(Pose, HashMap)> { + let sh = shared(d, a, b); + if sh.len() < 12 { + return vec![]; + } + let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); + let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); + let thr = 2.0 * d.noise / f; + let mut cands: Vec = vec![]; + if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); + if let Some((pose, _)) = geometry::select_pose(&geometry::decompose_essential(&e), &p1, &p2) { + cands.push(pose); + } + } + if use_h { + let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + let h = fit.h.0; + // Normalised: K⁻¹ H K with K = diag(f, f, 1). + let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; + let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| fit.inliers[*i]).map(|i| (x1[i], x2[i])).unzip(); + for (r, td, _) in geometry::decompose_homography(&hn, &p1, &p2) { + if norm(td) > 1e-6 { + cands.push(Pose::from_rt(r, normalize(td))); + } + } + } + } + let thr2 = (d.inlier_thr() / f).powi(2); + cands + .into_iter() + .filter_map(|pose| { + let mut pts = HashMap::new(); + for (i, s) in sh.iter().enumerate() { + let Some(x) = geometry::triangulate(&[(Pose::IDENTITY, x1[i]), (pose, x2[i])]) else { continue }; + let ok = geometry::reproj2(&Pose::IDENTITY, x, x1[i]).is_some_and(|e| e < thr2) + && geometry::reproj2(&pose, x, x2[i]).is_some_and(|e| e < thr2) + && geometry::ray_angle([0.0; 3], pose.c, x).to_degrees() > 0.3; + if ok { + pts.insert(s.0, x); + } + } + (pts.len() >= 10 && pts.len() * 3 >= sh.len()).then_some((pose, pts)) + }) + .collect() +} + +/// Grow a reconstruction over keyframes `kfs` from an initial pair (`kfs[0]`, `kfs[j]`). +fn grow(d: &Data, kfs: &[usize], j: usize, init: &(Pose, HashMap), f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let (a, b) = (kfs[0], kfs[j]); + let mut r = Recon::default(); + r.cams.insert(a, BaCam { pose: Pose::IDENTITY, f }); + r.cams.insert(b, BaCam { pose: init.0, f }); + r.pts = init.1.clone(); + let inner = |r: &mut Recon, frames: &[usize]| { + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, frames, cfg, focal, 10); + r.reject(d); + }; + inner(&mut r, &[a, b]); + let order: Vec = kfs[1..j].iter().chain(&kfs[j + 1..]).copied().collect(); + let mut done = 0; + for k in order { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let start = predict(&r, k)?; + if !r.register(d, k, start, cfg) { + // A failed keyframe ends the reconstruction (a later frame is too far from the known + // points); the caller resects the rest from what is known. + break; + } + r.triangulate(d, cfg.kind, 1.0, 1.0); + done += 1; + let frames: Vec = r.cams.keys().copied().collect(); + if done % 3 == 0 || frames.len() <= 6 { + inner(&mut r, &frames); + } + } + let frames: Vec = r.cams.keys().copied().collect(); + inner(&mut r, &frames); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Tripod reconstruction over keyframes `kfs` at focal length `f`. +fn grow_tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let mut r = Recon::default(); + r.cams.insert(kfs[0], BaCam { pose: Pose::IDENTITY, f }); + let ray = |p: [f64; 2], f: f64| normalize([p[0] / f, p[1] / f, 1.0]); + for w in kfs.windows(2) { + if cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return None; + } + let (a, b) = (w[0], w[1]); + let sh = shared(d, a, b); + if sh.len() < 6 { + break; + } + let ra: Vec = sh.iter().map(|s| ray(s.1, f)).collect(); + let rb: Vec = sh.iter().map(|s| ray(s.2, f)).collect(); + let mut rel = geometry::procrustes(&ra, &rb)?; + // One round of outlier trimming. + let errs: Vec = ra.iter().zip(&rb).map(|(x, y)| norm(sub(*y, mv(&rel, *x)))).collect(); + let mut sorted = errs.clone(); + sorted.sort_by(f64::total_cmp); + let lim = (sorted[sorted.len() / 2] * 3.0).max(d.inlier_thr() / f); + let (ia, ib): (Vec, Vec) = ra.iter().zip(&rb).zip(&errs).filter(|(_, e)| **e <= lim).map(|((x, y), _)| (*x, *y)).unzip(); + if ia.len() >= 4 { + rel = geometry::procrustes(&ia, &ib)?; + } + let pa = r.cams[&a].pose.r; + r.cams.insert(b, BaCam { pose: Pose { r: mm(&rel, &pa), c: [0.0; 3] }, f }); + } + r.triangulate(d, Kind::Tripod, 0.0, 20.0); + let frames: Vec = r.cams.keys().copied().collect(); + let focal = if cfg.per_frame_f && frames.len() > 2 { FocalMode::PerCamera(F_PRIOR) } else { FocalMode::Fixed }; + r.adjust(d, &frames, cfg, focal, 15); + r.reject(d); + r.triangulate(d, Kind::Tripod, 0.0, 1.0); + r.adjust(d, &frames, cfg, focal, 15); + (r.cams.len() >= 2 && r.pts.len() >= 8).then_some(r) +} + +/// Focal length (pixels) of a horizontal angle of view. +fn focal_of(w: f64, hfov: f64) -> f64 { + w * 0.5 / (hfov.clamp(1.0, 170.0).to_radians() * 0.5).tan() +} + +struct Attempt { + recon: Recon, + err: f64, + method: SolveMethod, +} + +/// Choose the initial pair: the first keyframe and the later keyframe (among the next few) +/// that keeps the most points while having enough parallax. +fn best_init(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Option<(usize, (Pose, HashMap), bool)> { + let (use_e, use_h) = match method { + SolveMethod::Typical => (true, false), + SolveMethod::MostlyFlat => (false, true), + _ => (true, true), + }; + let mut best: Option<(f64, usize, (Pose, HashMap), bool)> = None; + for j in 1..kfs.len().min(6) { + let cands = init_pair(d, kfs[0], kfs[j], f, use_e, false); + let hc = if use_h { init_pair(d, kfs[0], kfs[j], f, false, true) } else { vec![] }; + for (c, is_h) in cands.into_iter().map(|c| (c, false)).chain(hc.into_iter().map(|c| (c, true))) { + // Score: points × median triangulation angle (capped). + let mut ang: Vec = c.1.values().map(|x| geometry::ray_angle([0.0; 3], c.0.c, *x).to_degrees()).collect(); + ang.sort_by(f64::total_cmp); + let med = ang.get(ang.len() / 2).copied().unwrap_or(0.0); + let score = c.1.len() as f64 * med.min(4.0); + if best.as_ref().is_none_or(|b| score > b.0) { + best = Some((score, j, c, is_h)); + } + } + } + best.map(|b| (b.1, b.2, b.3)) +} + +/// All initial candidates for the pair chosen by [`best_init`] (E and H decompositions). +fn init_candidates(d: &Data, kfs: &[usize], f: f64, method: SolveMethod) -> Vec<(usize, (Pose, HashMap), bool)> { + let Some((j, first, is_h)) = best_init(d, kfs, f, method) else { return vec![] }; + let mut out = vec![(j, first, is_h)]; + if matches!(method, SolveMethod::Auto | SolveMethod::MostlyFlat) { + for c in init_pair(d, kfs[0], kfs[j], f, false, true) { + if out.iter().all(|o| norm(log_rot(&mm(&o.1.0.r, &mt(&c.0.r)))) > 1e-3 || norm(sub(o.1.0.c, c.0.c)) > 1e-3) { + out.push((j, c, true)); + } + } + } + out +} + +/// A general (translating camera) reconstruction of keyframes `kfs` at focal length `f`. +fn general(d: &Data, kfs: &[usize], f: f64, method: SolveMethod, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let cands = init_candidates(d, kfs, f, method); + let res: Vec = cands + .par_iter() + .filter_map(|(j, init, is_h)| { + let r = grow(d, kfs, *j, init, f, cfg, cancel)?; + let (_, n) = r.error(d); + // Prefer reconstructions covering more keyframes. + let cover = r.cams.len() as f64 / kfs.len() as f64; + (n > 0).then(|| Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: if *is_h { SolveMethod::MostlyFlat } else { SolveMethod::Typical } }) + }) + .collect(); + res.into_iter().min_by(|a, b| a.err.total_cmp(&b.err)) +} + +fn tripod(d: &Data, kfs: &[usize], f: f64, cfg: &Cfg, cancel: Option<&AtomicBool>) -> Option { + let r = grow_tripod(d, kfs, f, cfg, cancel)?; + let cover = r.cams.len() as f64 / kfs.len() as f64; + Some(Attempt { err: r.score(d) / cover.max(0.05), recon: r, method: SolveMethod::TripodPan }) +} + +/// Search the focal length minimising `eval(f)` (a reprojection error): a log-spaced scan of +/// horizontal angles of view, then golden-section refinement around the best. +fn search_focal(w: f64, eval: &(dyn Fn(f64) -> f64 + Sync)) -> f64 { + let fovs = [8.0, 12.0, 16.0, 20.0, 25.0, 30.0, 36.0, 43.0, 51.0, 60.0, 70.0, 82.0, 96.0, 112.0]; + let fs: Vec = fovs.iter().map(|a| focal_of(w, *a)).collect(); + let errs: Vec = fs.par_iter().map(|f| eval(*f)).collect(); + if debug() { + eprintln!("focal scan: {:?}", fs.iter().zip(&errs).map(|(f, e)| format!("{f:.0}:{e:.3}")).collect::>()); + } + let i = (0..fs.len()).min_by(|a, b| errs[*a].total_cmp(&errs[*b])).unwrap_or(5); + let (mut lo, mut hi) = (fs[(i + 1).min(fs.len() - 1)].ln(), fs[i.saturating_sub(1)].ln()); + if lo > hi { + std::mem::swap(&mut lo, &mut hi); + } + let g = 0.618_033_988_75; + let (mut x1, mut x2) = (hi - g * (hi - lo), lo + g * (hi - lo)); + let (mut e1, mut e2) = rayon::join(|| eval(x1.exp()), || eval(x2.exp())); + for _ in 0..7 { + if e1 < e2 { + hi = x2; + x2 = x1; + e2 = e1; + x1 = hi - g * (hi - lo); + e1 = eval(x1.exp()); + } else { + lo = x1; + x1 = x2; + e1 = e2; + x2 = lo + g * (hi - lo); + e2 = eval(x2.exp()); + } + } + let best = if e1 < e2 { (x1, e1) } else { (x2, e2) }; + if best.1 <= errs[i] { best.0.exp() } else { fs[i] } +} + +/// Solve the camera of a tracked clip. +pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBool>) -> Result { + let d = Data::new(tracks, &s.deleted); + if d.n < 2 { + return err("the clip must be at least two frames long"); + } + let kfs = keyframes(&d); + if kfs.len() < 2 { + return err("not enough features were tracked to solve the camera"); + } + let cancelled = || cancel.is_some_and(|c| c.load(Ordering::Relaxed)); + let fixed_f = (s.shot == ShotType::SpecifyAngle).then(|| focal_of(d.w, if s.hfov > 0.0 { s.hfov } else { 40.0 })); + let variable = s.shot == ShotType::VariableZoom; + let gcfg = Cfg { kind: Kind::General, focal: FocalMode::Fixed, per_frame_f: false }; + let tcfg = Cfg { kind: Kind::Tripod, ..gcfg }; + // The focal-length search runs on the first keyframes only. + let mini: Vec = kfs.iter().copied().take(10).collect(); + let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); + let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + + let mut results: Vec<(Attempt, f64)> = vec![]; + if try_general { + let f = match fixed_f { + Some(f) => f, + None => { + search_focal(d.w, &|f| general(&d, &mini, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)) + } + }; + if cancelled() { + return err("cancelled"); + } + if let Some(a) = general(&d, &kfs, f, s.method, &Cfg { per_frame_f: variable, ..gcfg }, cancel) { + results.push((a, f)); + } + } + if try_tripod && !cancelled() { + let f = match fixed_f { + Some(f) => f, + None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), + }; + if let Some(a) = tripod(&d, &kfs, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel) { + results.push((a, f)); + } + } + if cancelled() { + return err("cancelled"); + } + // Auto Detect keeps the tripod model when it explains the tracks about as well. + let pick = { + let g = results.iter().position(|r| r.0.method != SolveMethod::TripodPan); + let t = results.iter().position(|r| r.0.method == SolveMethod::TripodPan); + match (g, t) { + (Some(g), Some(t)) => { + let (eg, et) = (results[g].0.err, results[t].0.err); + // A general solve whose camera hardly moves relative to the scene depth is a pan. + let rg = &results[g].0.recon; + let cs: Vec = rg.cams.values().map(|c| c.pose.c).collect(); + let base = cs.iter().flat_map(|a| cs.iter().map(move |b| norm(sub(*a, *b)))).fold(0.0, f64::max); + let c0 = rg.cams.values().next().expect("camera"); + let mut depth: Vec = rg.pts.values().map(|x| norm(sub(*x, c0.pose.c))).collect(); + depth.sort_by(f64::total_cmp); + let med = depth.get(depth.len() / 2).copied().unwrap_or(1.0); + if debug() { + eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); + } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + } + (Some(g), None) => g, + (None, Some(t)) => t, + (None, None) => return err("the camera could not be solved (not enough parallax or features)"), + } + }; + if debug() { + for (a, f) in &results { + eprintln!("candidate {:?}: err {:.4} f {f:.1} cams {} pts {}", a.method, a.err, a.recon.cams.len(), a.recon.pts.len()); + } + } + let (att, _) = results.swap_remove(pick); + let kind = if att.method == SolveMethod::TripodPan { Kind::Tripod } else { Kind::General }; + let cfg = Cfg { kind, focal: FocalMode::Fixed, per_frame_f: variable }; + let mut r = att.recon; + let focal = match (fixed_f, variable) { + (Some(_), _) => FocalMode::Fixed, + (None, true) => FocalMode::PerCamera(F_PRIOR), + (None, false) => FocalMode::Shared, + }; + // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. + let kf_frames: Vec = r.cams.keys().copied().collect(); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.triangulate(&d, kind, 1.0, 1.0); + r.adjust(&d, &kf_frames, &cfg, focal, 40); + if cancelled() { + return err("cancelled"); + } + // Every other frame: resect from the points (nearest registered camera first). + let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); + todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); + let base = r.clone(); + let resected: Vec<(usize, Option)> = todo + .par_iter() + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let ok = predict(&local, *k).is_some_and(|start| local.register(&d, *k, start, &cfg)); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in resected { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + if cancelled() { + return err("cancelled"); + } + // Points for the tracks that only lived between keyframes, then the final adjustment over + // every frame (when that stays affordable). + r.triangulate(&d, kind, 1.0, 1.0); + let all: Vec = r.cams.keys().copied().collect(); + r.reject(&d); + if all.len() <= 160 { + r.adjust(&d, &all, &cfg, focal, 25); + r.reject(&d); + } else { + // Points only (cameras fixed) would be cheap but the keyframe solution is already + // adjusted; refine each non-keyframe once more against the final points. + let base = r.clone(); + let refined: Vec<(usize, Option)> = all + .par_iter() + .filter(|k| !kf_frames.contains(k)) + .map(|k| { + let mut local = Recon { cams: base.cams.clone(), pts: base.pts.clone(), outliers: HashSet::new() }; + let start = local.cams[k]; + let ok = local.register(&d, *k, start, &cfg); + (*k, if ok { local.cams.get(k).copied() } else { None }) + }) + .collect(); + for (k, c) in refined.into_iter() { + if let Some(c) = c { + r.cams.insert(k, c); + } + } + r.reject(&d); + } + // Method Used: the chosen model; Auto Detect calls a general solve "Mostly Flat Scene" when + // its points are close to one plane. + let method = match (s.method, att.method) { + (_, SolveMethod::TripodPan) => SolveMethod::TripodPan, + (SolveMethod::Auto, _) => { + let p: Vec = r.pts.values().copied().collect(); + let flat = cov_ratio(&p) < 0.08; + if flat { SolveMethod::MostlyFlat } else { SolveMethod::Typical } + } + (m, _) => m, + }; + Ok(finish(&d, tracks, s, r, method, kind)) +} + +/// Thickness of a point cloud: √(smallest / middle eigenvalue of its covariance). +fn cov_ratio(p: &[V3]) -> f64 { + if p.len() < 3 { + return 1.0; + } + let k = 1.0 / p.len() as f64; + let c = scale(p.iter().fold([0.0; 3], |a, b| add(a, *b)), k); + let mut cov = [0.0; 9]; + for q in p { + let d = sub(*q, c); + for i in 0..3 { + for j in 0..3 { + cov[i * 3 + j] += d[i] * d[j] * k; + } + } + } + let (vals, _) = sym_eigen(&cov, 3); + (vals[0].max(0.0) / vals[1].max(1e-300)).sqrt() +} + +/// Canonical frame, per-frame output (unsolved frames interpolated), points and errors. +fn finish(d: &Data, tracks: &CameraTracks, s: &SolveSettings, r: Recon, method: SolveMethod, kind: Kind) -> CameraSolve { + let (k0, c0) = r.cams.iter().next().map(|(k, c)| (*k, *c)).expect("solved"); + // Scale: median depth of the points seen by the first solved camera is 1. + let mut depths: Vec = d.by_frame[k0].iter().filter_map(|(ti, _)| r.pts.get(ti)).map(|x| c0.pose.to_cam(*x)[2]).filter(|z| *z > 0.0).collect(); + depths.sort_by(f64::total_cmp); + let sc = match kind { + Kind::Tripod => 1.0, + Kind::General => depths.get(depths.len() / 2).copied().filter(|v| *v > 1e-12).unwrap_or(1.0), + }; + let r0 = c0.pose.r; + let cc = c0.pose.c; + let to_world = |x: V3| scale(mv(&r0, sub(x, cc)), 1.0 / sc); + let mut frames = vec![SolvedFrame::default(); d.n]; + for (k, c) in &r.cams { + let rr = mm(&c.pose.r, &mt(&r0)); + frames[*k] = SolvedFrame { rot: log_rot(&rr), center: to_world(c.pose.c), focal: c.f, solved: true }; + } + // Hold / interpolate unsolved frames so the camera path stays continuous. + let solved: Vec = (0..d.n).filter(|k| frames[*k].solved).collect(); + for k in 0..d.n { + if frames[k].solved { + continue; + } + let a = solved.iter().rev().find(|x| **x < k).copied(); + let b = solved.iter().find(|x| **x > k).copied(); + frames[k] = match (a, b) { + (Some(a), Some(b)) => { + let t = (k - a) as f64 / (b - a) as f64; + let (fa, fb) = (frames[a], frames[b]); + let rr = slerp(&fa.r(), &fb.r(), t); + SolvedFrame { + rot: log_rot(&rr), + center: add(scale(fa.center, 1.0 - t), scale(fb.center, t)), + focal: fa.focal + (fb.focal - fa.focal) * t, + solved: false, + } + } + (Some(a), None) => SolvedFrame { solved: false, ..frames[a] }, + (None, Some(b)) => SolvedFrame { solved: false, ..frames[b] }, + (None, None) => frames[k], + }; + } + let mut points = vec![]; + let (mut esum, mut en) = (0.0, 0usize); + let mut ids: Vec<&u32> = r.pts.keys().collect(); + ids.sort(); + for ti in ids { + let x = r.pts[ti]; + let (id, start, pts) = &d.tracks[*ti as usize]; + let (mut s1, mut n1) = (0.0, 0); + for (i, uv) in pts.iter().enumerate() { + let k = start + i; + if r.outliers.contains(&(*ti, k)) { + continue; + } + if let Some(c) = r.cams.get(&k) + && let Some(res) = bundle::residual(c, x, *uv) + { + s1 += res[0].hypot(res[1]); + n1 += 1; + } + } + if n1 < 2 { + continue; + } + esum += s1; + en += n1; + points.push(SolvedPoint { id: *id, pos: to_world(x), error: (s1 / n1 as f64) as f32, first: *start as u32, last: (start + pts.len() - 1) as u32 }); + } + CameraSolve { + version: 1, + size: tracks.size, + start: tracks.start, + frame_duration: tracks.frame_duration, + shot: s.shot, + method_used: method, + average_error: if en > 0 { esum / en as f64 } else { 0.0 }, + frames, + points, + ground: None, + } +} diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs new file mode 100644 index 000000000..258161f98 --- /dev/null +++ b/crates/track/src/camtrack/tests.rs @@ -0,0 +1,360 @@ +//! Synthetic scenes: random 3D points (or a plane) seen by a known moving camera with a known +//! focal length, projected directly (with sub-pixel noise) into feature tracks. + +use super::geometry::Rng; +use super::linalg::*; +use super::*; + +const W: f64 = 1280.0; +const H: f64 = 720.0; + +struct Truth { + /// Per frame: (R world→cam, centre, focal). + cams: Vec<(M3, V3, f64)>, +} + +fn noise(rng: &mut Rng) -> f64 { + // Sum of uniforms ≈ Gaussian, σ ≈ 0.1 px. + let u: f64 = (0..4).map(|_| (rng.next_u64() % 10_000) as f64 / 10_000.0 - 0.5).sum(); + u * 0.17 +} + +/// Project `pts` through the cameras into tracks (a track ends when its point leaves the frame; +/// re-entering starts a new track). +fn tracks_of(pts: &[V3], t: &Truth, seed: u64, noise_on: bool) -> CameraTracks { + let mut rng = Rng(seed); + let mut tracks: Vec = vec![]; + for x in pts { + let mut cur: Option = None; + for (k, (r, c, f)) in t.cams.iter().enumerate() { + let p = mv(r, sub(*x, *c)); + let uv = (p[2] > 0.1).then(|| [f * p[0] / p[2] + W / 2.0, f * p[1] / p[2] + H / 2.0]); + match uv.filter(|u| u[0] >= 2.0 && u[1] >= 2.0 && u[0] < W - 2.0 && u[1] < H - 2.0) { + Some(u) => { + let (nx, ny) = if noise_on { (noise(&mut rng), noise(&mut rng)) } else { (0.0, 0.0) }; + let q = [(u[0] + nx) as f32, (u[1] + ny) as f32]; + match &mut cur { + Some(tr) => tr.pts.push(q), + None => cur = Some(Track2D { id: 0, start: k as u32, pts: vec![q] }), + } + } + None => { + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + } + } + if let Some(tr) = cur.take() { + tracks.push(tr); + } + } + tracks.retain(|t| t.pts.len() >= 3); + for (i, tr) in tracks.iter_mut().enumerate() { + tr.id = i as u32; + } + CameraTracks { version: 1, size: [W, H], start: 0.0, frame_duration: 1.0 / 30.0, frames: t.cams.len() as u32, factor: 1.0, detailed: false, tracks } +} + +fn random_points(n: usize, seed: u64, center: V3, ext: V3) -> Vec { + let mut rng = Rng(seed); + (0..n) + .map(|_| { + let mut u = || (rng.next_u64() % 100_000) as f64 / 100_000.0 - 0.5; + [center[0] + u() * ext[0], center[1] + u() * ext[1], center[2] + u() * ext[2]] + }) + .collect() +} + +/// A camera dollying sideways and forward while panning to keep the scene in view. +fn moving_camera(frames: usize, f: impl Fn(usize) -> f64) -> Truth { + let cams = (0..frames) + .map(|k| { + let t = k as f64 / (frames - 1) as f64; + let c = [-3.0 + 6.0 * t, -0.6 + 0.8 * t + 0.5 * (std::f64::consts::PI * t).sin(), -1.0 + 2.0 * t - 1.5 * (t - 0.5) * (t - 0.5)]; + // Look at the scene centre (0, 0, 10). + let fwd = normalize(sub([0.3 * t, 0.2, 10.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + let r = [right, down, fwd]; + // A little roll. + let roll = rodrigues([0.0, 0.0, 0.05 * (t * 6.0).sin()]); + (mm(&roll, &r), c, f(k)) + }) + .collect(); + Truth { cams } +} + +/// Align the solved centres to the true ones (similarity) and return the RMS path error +/// relative to the path length, plus the worst rotation error (degrees). +fn path_error(s: &CameraSolve, t: &Truth) -> (f64, f64) { + let solved: Vec = (0..s.frames.len()).filter(|k| s.frames[*k].solved).collect(); + let src: Vec = solved.iter().map(|k| s.frames[*k].center).collect(); + let dst: Vec = solved.iter().map(|k| t.cams[*k].1).collect(); + let sim = umeyama(&src, &dst).expect("alignment"); + let len = norm(sub(dst[0], dst[dst.len() - 1])).max(1e-9); + let rms = (src.iter().zip(&dst).map(|(a, b)| dot(sub(sim.apply(*a), *b), sub(sim.apply(*a), *b))).sum::() / src.len() as f64).sqrt(); + // Rotation: R_true ≈ R_solved · simᵀ. + let mut worst: f64 = 0.0; + for k in &solved { + let rs = mm(&s.frames[*k].r(), &mt(&sim.r)); + let e = norm(log_rot(&mm(&rs, &mt(&t.cams[*k].0)))).to_degrees(); + worst = worst.max(e); + } + (rms / len, worst) +} + +fn settings(shot: ShotType, method: SolveMethod) -> SolveSettings { + SolveSettings { shot, hfov: 0.0, method, deleted: vec![] } +} + +#[test] +fn fixed_angle_solve_recovers_path_and_focal() { + let f = 1100.0; + let t = moving_camera(40, |_| f); + let pts = random_points(400, 1, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 2, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), None).expect("solve"); + eprintln!("fixed: {:?} err {:.3} px focal {:.1} in {:?}", s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.frames.iter().all(|c| c.solved)); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); + let (pe, re) = path_error(&s, &t); + assert!(pe < 0.01, "path {pe}"); + assert!(re < 0.5, "rotation {re}"); + // Canonical frame: first camera at the origin. + assert!(norm(s.frames[0].center) < 1e-9 && norm(s.frames[0].rot) < 1e-9); + // Serde round trip. + let j = s.to_json(); + assert_eq!(CameraSolve::from_json(&j).unwrap().points.len(), s.points.len()); +} + +#[test] +fn auto_detect_picks_a_general_model_for_a_translating_camera() { + let f = 900.0; + let t = moving_camera(30, |_| f); + let pts = random_points(300, 3, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 4, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + assert_eq!(s.method_used, SolveMethod::Typical); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "focal {}", s.focal()); +} + +#[test] +fn specified_angle_of_view_is_kept() { + let f = 1000.0; + let t = moving_camera(25, |_| f); + let pts = random_points(300, 5, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 6, true); + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: vec![] }, None).unwrap(); + assert!((s.focal() - f).abs() < 1e-6); + assert!(s.average_error < 0.5); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); +} + +#[test] +fn variable_zoom_recovers_per_frame_focal() { + let t = moving_camera(40, |k| 950.0 + 6.0 * k as f64); + let pts = random_points(450, 7, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 8, true); + let s = solve(&tr, &settings(ShotType::VariableZoom, SolveMethod::Typical), None).expect("solve"); + eprintln!("variable: err {:.3}", s.average_error); + assert!(s.average_error < 0.5); + for (k, c) in s.frames.iter().enumerate() { + let f = t.cams[k].2; + assert!((c.focal - f).abs() / f < 0.02, "frame {k}: {} vs {f}", c.focal); + } +} + +#[test] +fn tripod_pan_solve() { + let f = 1200.0; + let cams = (0..40) + .map(|k| { + let a = -0.4 + 0.8 * k as f64 / 39.0; + (mm(&rodrigues([0.0, 0.0, 0.02]), &mm(&rodrigues([0.05 * a, 0.0, 0.0]), &rodrigues([0.0, a, 0.0]))), [0.0; 3], f) + }) + .collect(); + let t = Truth { cams }; + // Points all around at varying distances (rotation only: depth is unobservable). + let pts = random_points(600, 9, [0.0, 0.0, 0.0], [60.0, 20.0, 60.0]).into_iter().filter(|p| p[2] > 3.0).collect::>(); + let tr = tracks_of(&pts, &t, 10, true); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("tripod: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::TripodPan); + assert!(s.average_error < 0.5); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + for (k, c) in s.frames.iter().enumerate() { + assert!(norm(c.center) < 1e-9); + // Relative rotation to frame 0 matches. + let rel_s = mm(&c.r(), &mt(&s.frames[0].r())); + let rel_t = mm(&t.cams[k].0, &mt(&t.cams[0].0)); + assert!(norm(log_rot(&mm(&rel_s, &mt(&rel_t)))).to_degrees() < 0.2); + } +} + +#[test] +fn mostly_flat_scene_and_ground_plane() { + let f = 1000.0; + // A textured floor y = 2 (below the camera), seen by a camera moving forward and sideways. + let cams = (0..40) + .map(|k| { + let t = k as f64 / 39.0; + let c = [-2.0 + 4.0 * t, -0.3 * (std::f64::consts::PI * t).sin(), 2.0 * t]; + let fwd = normalize([0.1 - 0.2 * t, 0.45, 1.0]); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + let down = cross(fwd, right); + ([right, down, fwd], c, f) + }) + .collect(); + let t = Truth { cams }; + let mut rng = Rng(11); + let pts: Vec = (0..500).map(|_| [((rng.next_u64() % 2000) as f64 / 100.0) - 10.0, 2.0, 2.0 + (rng.next_u64() % 2000) as f64 / 100.0]).collect(); + let tr = tracks_of(&pts, &t, 12, true); + for method in [SolveMethod::MostlyFlat, SolveMethod::Auto] { + let s = solve(&tr, &settings(ShotType::FixedAngle, method), None).expect("solve"); + eprintln!("flat {method:?}: {:?} err {:.3} focal {:.1}", s.method_used, s.average_error, s.focal()); + assert_eq!(s.method_used, SolveMethod::MostlyFlat); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.02, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.01, "{pe}"); + } + // Ground plane from the solved (coplanar) points: every point lies on it. + let hfov = 2.0 * (W * 0.5 / f).atan().to_degrees(); + let mut s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::MostlyFlat, deleted: vec![] }, None).expect("solve"); + let ids: Vec = s.visible(0).map(|p| p.id).take(12).collect(); + let g = s.ground_from(&ids, 0).expect("ground"); + let spread = s.points.iter().map(|p| dot(sub(p.pos, g.origin), g.normal).abs()).fold(0.0, f64::max); + let depth = norm(sub(s.frames[0].center, g.origin)); + assert!(spread < 0.01 * depth, "{spread}"); + // Up points towards the camera. + assert!(dot(sub(s.frames[0].center, g.origin), g.normal) > 0.0); + s.ground = Some(g); + let w = s.world([W, H], 1000.0); + // In the world, ground points have y ≈ 0 and the cameras are above (y < 0). + for p in s.points.iter().take(20) { + assert!(w.apply(p.pos)[1].abs() < 0.01 * 1000.0 * depth, "{:?}", w.apply(p.pos)); + } + assert!(w.apply(s.frames[0].center)[1] < 0.0); + assert!(norm(w.apply(g.origin)) < 1e-6); +} + +#[test] +fn deleted_tracks_are_ignored_and_cancel_stops() { + let t = moving_camera(20, |_| 1000.0); + let pts = random_points(250, 13, [0.0, 0.0, 10.0], [14.0, 8.0, 8.0]); + let tr = tracks_of(&pts, &t, 14, true); + let deleted: Vec = (0..40).collect(); + let hfov = 2.0 * (W * 0.5 / 1000.0f64).atan().to_degrees(); + let s = solve(&tr, &SolveSettings { shot: ShotType::SpecifyAngle, hfov, method: SolveMethod::Typical, deleted: deleted.clone() }, None).unwrap(); + assert!(s.points.iter().all(|p| !deleted.contains(&p.id))); + let stop = std::sync::atomic::AtomicBool::new(true); + let r = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Typical), Some(&stop)); + assert!(r.is_err()); +} + +/// End to end: rendered frames of a textured scene through the KLT feature tracker. +#[test] +fn rendered_scene_end_to_end() { + use effectcraft_raster::Image; + let (w, h) = (480u32, 270u32); + let f = 420.0; + // Three textured planes at different depths (a floor, a back wall and a box face). + fn tex(u: f64, v: f64) -> f32 { + fn hsh(i: i64, j: i64) -> f64 { + let mut x = (i.wrapping_mul(73856093) ^ j.wrapping_mul(19349663)) as u64; + x ^= x >> 13; + x = x.wrapping_mul(0x5bd1e995); + x ^= x >> 15; + (x % 1000) as f64 / 1000.0 + } + fn noise(x: f64, y: f64) -> f64 { + let (i, j) = (x.floor(), y.floor()); + let (tx, ty) = (x - i, y - j); + let (sx, sy) = (tx * tx * (3.0 - 2.0 * tx), ty * ty * (3.0 - 2.0 * ty)); + let (i, j) = (i as i64, j as i64); + let a = hsh(i, j) + (hsh(i + 1, j) - hsh(i, j)) * sx; + let b = hsh(i, j + 1) + (hsh(i + 1, j + 1) - hsh(i, j + 1)) * sx; + a + (b - a) * sy + } + (0.15 + 0.5 * noise(u * 4.0, v * 4.0) + 0.3 * noise(u * 11.0 + 7.0, v * 11.0 + 3.0)) as f32 + } + let frames = 24; + let t = Truth { + cams: (0..frames) + .map(|k| { + let s = k as f64 / (frames - 1) as f64; + let c = [-1.0 + 2.0 * s, -0.2, 0.5 * s]; + let fwd = normalize(sub([0.0, 0.3, 8.0], c)); + let right = normalize(cross([0.0, 1.0, 0.0], fwd)); + ([right, cross(fwd, right), fwd], c, f) + }) + .collect(), + }; + // Planes: (point, normal, u axis, v axis). + let planes: [(V3, V3, V3, V3); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0]), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0]), + ([-1.0, 0.0, 6.0], [0.3, 0.0, -1.0], [1.0, 0.0, 0.3], [0.0, 1.0, 0.0]), + ]; + let render = |k: usize| { + let (r, c, f) = t.cams[k]; + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + let d = mtv(&r, [(x as f64 + 0.5 - w as f64 / 2.0) / f, (y as f64 + 0.5 - h as f64 / 2.0) / f, 1.0]); + let mut best = (f64::INFINITY, 0.0f32); + for (i, (p0, n, ua, va)) in planes.iter().enumerate() { + let den = dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = dot(*n, sub(*p0, c)) / den; + let hit = add(c, scale(d, tt)); + // The box face is limited in extent. + if i == 2 && (dot(sub(hit, *p0), *ua).abs() > 1.2 || dot(sub(hit, *p0), *va).abs() > 1.0) { + continue; + } + if tt > 0.1 && tt < best.0 { + best = (tt, tex(dot(sub(hit, *p0), *ua) + 10.0 * i as f64, dot(sub(hit, *p0), *va))); + } + } + let v = best.1; + img.set(x, y, [v, v, v, 1.0]); + } + } + img + }; + let mut an = TrackAnalyzer::new([w as f64, h as f64], AnalyzeOpts::default()); + for k in 0..frames { + let img = render(k); + an.push(&crate::Frame::new(&img)); + } + let tr = an.finish(0.0, 1.0 / 24.0); + assert!(tr.tracks.len() > 100, "{}", tr.tracks.len()); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("rendered: {:?} err {:.3} focal {:.1} points {}", s.method_used, s.average_error, s.focal(), s.points.len()); + assert!(s.average_error < 0.5, "{}", s.average_error); + assert!((s.focal() - f).abs() / f < 0.05, "{}", s.focal()); + let (pe, _) = path_error(&s, &t); + assert!(pe < 0.03, "{pe}"); +} + +/// A long clip (300 frames, ~900 points): `cargo test -p effectcraft-track --release -- --ignored`. +#[test] +#[ignore] +fn perf_long_clip() { + let f = 1100.0; + let t = moving_camera(300, |_| f); + let pts = random_points(900, 21, [0.0, 0.0, 10.0], [16.0, 9.0, 8.0]); + let tr = tracks_of(&pts, &t, 22, true); + let t0 = std::time::Instant::now(); + let s = solve(&tr, &settings(ShotType::FixedAngle, SolveMethod::Auto), None).expect("solve"); + eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); + assert!(s.average_error < 0.5); +} diff --git a/crates/track/src/lib.rs b/crates/track/src/lib.rs index d7b239f5f..bd4e21dce 100644 --- a/crates/track/src/lib.rs +++ b/crates/track/src/lib.rs @@ -18,6 +18,7 @@ //! Coordinates are *layer* pixels: the frame image's pixel `(i, j)` covers `[i, i+1) × [j, j+1)` //! after subtracting the frame's `offset` (see [`Frame`]). +pub mod camtrack; pub mod fit; pub mod klt; pub mod mask; From abd39f5fd87472d5c6343c2298774416beeecb2b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:44:06 -0400 Subject: [PATCH 2/9] =?UTF-8?q?CMP-7:=20Essential=20Graphics=20controls,?= =?UTF-8?q?=20master=20properties,=20.ectemplate=20templates,=20Responsive?= =?UTF-8?q?=20Design=20=E2=80=94=20Time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comps expose controls (properties typed as text/color/slider/checkbox/point/angle/dropdown, Media Replacement, groups, comments); precomp layers get an Essential Properties group whose edits override per instance (Push to Comp / Revert); templates export/import as an open ZIP (manifest.json + project subset + media + poster). Stretched precomps keep protected regions at their original speed. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/essential.rs | 773 ++++++++++++++++++++++++ crates/engine/src/commands/file.rs | 1 + crates/engine/src/commands/file_more.rs | 1 + crates/engine/src/commands/mod.rs | 2 + crates/engine/src/lib.rs | 11 + crates/engine/src/menus.rs | 7 +- crates/engine/src/tests_essential.rs | 289 +++++++++ crates/engine/src/tests_model3d.rs | 1 + crates/engine/src/tests_stubs.rs | 1 + crates/engine/src/tests_track.rs | 2 + crates/gpu/src/tests.rs | 1 + crates/lottie/src/import.rs | 1 + crates/lottie/src/lib.rs | 2 +- crates/lottie/src/tests.rs | 1 + crates/media/src/pool.rs | 2 +- crates/media/src/probe.rs | 3 + crates/project/src/essential.rs | 538 +++++++++++++++++ crates/project/src/lib.rs | 114 +++- crates/project/src/props.rs | 7 + crates/render/src/audio.rs | 1 + crates/render/src/eval.rs | 76 +++ crates/render/src/lib.rs | 30 +- crates/render/src/tests_audio_fx.rs | 1 + crates/render/src/tests_color.rs | 1 + crates/render/src/tests_frame_blend.rs | 1 + 25 files changed, 1862 insertions(+), 5 deletions(-) create mode 100644 crates/engine/src/commands/essential.rs create mode 100644 crates/engine/src/tests_essential.rs create mode 100644 crates/project/src/essential.rs diff --git a/crates/engine/src/commands/essential.rs b/crates/engine/src/commands/essential.rs new file mode 100644 index 000000000..65e93d4e7 --- /dev/null +++ b/crates/engine/src/commands/essential.rs @@ -0,0 +1,773 @@ +//! Essential Graphics (Window ▸ Essential Graphics) and master properties. +//! +//! - **Authoring**: a comp's controls ([`EssentialGraphics`]): Add Property to Essential +//! Graphics, Media Replacement for footage layers, groups, comments, renaming, reordering, the +//! template name and the panel's Primary composition. +//! - **Instances**: every precomp layer of such a comp shows the controls under **Essential +//! Properties**; changing one overrides it for that instance only (`essential.set`, or any +//! property edit of the master property); Push to Comp writes the override into the source +//! comp, Revert drops it. +//! - **Templates**: `essential.exportTemplate` writes an open `.ectemplate` (a ZIP holding +//! `manifest.json` with the exposed controls, `project.ecproj` with the comp and everything it +//! uses, the media files and a poster frame); `essential.importTemplate` merges one into the +//! project and adds an instance to the active comp. + +use effectcraft_keyframe::Value as KV; +use effectcraft_project::essential::{self, ControlType, EgControl, EgKind, EssentialGraphics}; +use effectcraft_project::{GroupKind, ItemId, ItemKind, LayerId, LayerSource, ParamUi, Project, Uid}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, comp_id, has_comp, layer_p, selected_leaf_props, str_p}; +use crate::{EngineError, Result, Session, cmd, query}; + +/// Format id written into template manifests. +pub const TEMPLATE_FORMAT: &str = "effectcraft-template"; +pub const TEMPLATE_VERSION: u32 = 1; + +fn control_p(p: &Value, cmd: &str) -> Result { + p.get("control").and_then(Value::as_u64).ok_or_else(|| bad(cmd, "missing `control` (control id)")) +} + +/// Comp whose Essential Graphics a command edits: `comp`, else the panel's Primary comp, else +/// the active comp. +fn eg_comp(s: &Session, p: &Value) -> Result { + if p.get("comp").is_some() { + return comp_id(s, p); + } + if let Some(c) = s.state.essential_primary.filter(|c| s.project.comp(*c).is_some()) { + return Ok(c); + } + comp_id(s, p) +} + +fn eg_of(proj: &mut Project, cid: ItemId) -> Result<&mut EssentialGraphics> { + let name = proj.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let c = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + Ok(c.essential.get_or_insert_with(|| EssentialGraphics { name, controls: vec![] })) +} + +/// Insert `ctl` into `group` (or the top level) at `index` (or the end). +fn insert(eg: &mut EssentialGraphics, ctl: EgControl, group: Option, index: Option, cmd: &str) -> Result<()> { + let list = eg.list_mut(group).ok_or_else(|| bad(cmd, "`group` is not a group control"))?; + let i = index.unwrap_or(list.len()).min(list.len()); + list.insert(i, ctl); + Ok(()) +} + +fn set_primary(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + s.state.essential_primary = Some(cid); + s.bump(); + Ok(json!({"comp": cid.0})) +} + +fn set_name(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let name = str_p(p, "name").ok_or_else(|| bad("essential.setName", "missing `name`"))?.to_string(); + s.edit("Essential Graphics Name", None, |proj, _| { + eg_of(proj, cid)?.name = name.clone(); + Ok(()) + })?; + Ok(json!({"comp": cid.0, "name": name})) +} + +/// Add Property to Essential Graphics: `{layer, path|prop}` or the selected properties. +fn add_property(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.addProperty"; + let targets: Vec<(ItemId, LayerId, Uid)> = if p.get("path").is_some() || p.get("prop").is_some() { + vec![super::prop::prop_ref(s, p, C)?] + } else { + let cid = comp_id(s, p)?; + selected_leaf_props(s).into_iter().map(|(l, u)| (cid, l, u)).collect() + }; + if targets.is_empty() { + return Err(bad(C, "select a property in the timeline (or give `layer` and `path`)")); + } + let group = p.get("group").and_then(Value::as_u64); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + let rename = str_p(p, "name").map(str::to_string); + let mut added = vec![]; + for (cid, lid, uid) in &targets { + let l = s.project.comp(*cid).and_then(|c| c.layer(*lid)).ok_or(EngineError::NoComp)?; + let pr = l.props.find(*uid).ok_or_else(|| bad(C, "no such property"))?; + if control_type(pr).is_none() { + return Err(bad(C, format!("`{}` can't be added to Essential Graphics (unsupported property type)", pr.name))); + } + if s.project.comp(*cid).and_then(|c| c.essential.as_ref()).is_some_and(|e| e.control_for(*lid, *uid).is_some()) { + continue; + } + added.push((*cid, *lid, *uid, rename.clone().unwrap_or_else(|| pr.name.clone()))); + } + let ids = s.edit("Add Property to Essential Graphics", None, |proj, _| { + let mut ids = vec![]; + for (k, (cid, lid, uid, name)) in added.iter().enumerate() { + let id = proj.alloc(); + let eg = eg_of(proj, *cid)?; + insert(eg, EgControl { id, name: name.clone(), kind: EgKind::Property { layer: *lid, prop: *uid } }, group, index.map(|i| i + k), C)?; + ids.push(id); + } + Ok(ids) + })?; + Ok(json!({"controls": ids})) +} + +fn control_type(pr: &effectcraft_project::Property) -> Option { + essential::control_type(pr) +} + +/// Media Replacement for a footage layer. +fn add_media(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.addMedia"; + let (cid, lid) = layer_p(s, p, C)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + if !matches!(l.source, LayerSource::Footage { .. }) { + return Err(bad(C, "Media Replacement needs a footage layer")); + } + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("{} (Media)", l.name)); + let group = p.get("group").and_then(Value::as_u64); + let id = s.edit("Add Media Replacement", None, |proj, _| { + let id = proj.alloc(); + let eg = eg_of(proj, cid)?; + if eg.flat().iter().any(|c| matches!(c.kind, EgKind::Media { layer } if layer == lid)) { + return Err(bad(C, "this layer already has a Media Replacement control")); + } + insert(eg, EgControl { id, name, kind: EgKind::Media { layer: lid } }, group, None, C)?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn add_group(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let name = str_p(p, "name").unwrap_or("Group").to_string(); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + let id = s.edit("Add Essential Graphics Group", None, |proj, _| { + let id = proj.alloc(); + insert(eg_of(proj, cid)?, EgControl { id, name, kind: EgKind::Group { children: vec![] } }, None, index, "essential.addGroup")?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn add_comment(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let text = str_p(p, "text").unwrap_or("Comment").to_string(); + let group = p.get("group").and_then(Value::as_u64); + let id = s.edit("Add Essential Graphics Comment", None, |proj, _| { + let id = proj.alloc(); + insert(eg_of(proj, cid)?, EgControl { id, name: "Comment".into(), kind: EgKind::Comment { text } }, group, None, "essential.addComment")?; + Ok(id) + })?; + Ok(json!({"control": id})) +} + +fn rename(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.rename"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + let name = str_p(p, "name").or(str_p(p, "text")).ok_or_else(|| bad(C, "missing `name`"))?.to_string(); + s.edit("Rename Essential Graphics Control", None, |proj, _| { + let c = eg_of(proj, cid)?.find_mut(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + match &mut c.kind { + // A comment's name is its text. + EgKind::Comment { text } => *text = name.clone(), + _ => c.name = name.clone(), + } + Ok(()) + })?; + Ok(json!({"control": id, "name": name})) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.remove"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + s.edit("Remove Essential Graphics Control", None, |proj, _| { + eg_of(proj, cid)?.remove(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + Ok(()) + })?; + Ok(json!({"control": id})) +} + +fn move_control(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.move"; + let cid = eg_comp(s, p)?; + let id = control_p(p, C)?; + let group = p.get("group").and_then(Value::as_u64); + let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); + if group == Some(id) { + return Err(bad(C, "a group can't contain itself")); + } + s.edit("Move Essential Graphics Control", None, |proj, _| { + let eg = eg_of(proj, cid)?; + let ctl = eg.remove(id).ok_or_else(|| bad(C, format!("no control {id}")))?; + if matches!(ctl.kind, EgKind::Group { .. }) && group.is_some() { + return Err(bad(C, "groups can't be nested")); + } + insert(eg, ctl, group, index, C) + })?; + Ok(json!({"control": id})) +} + +/// Dropdown Menu Control ▸ Edit: the menu items of a dropdown (popup) property. +fn edit_dropdown(s: &mut Session, p: &Value) -> Result { + const C: &str = "effect.editDropdown"; + let items: Vec = match p.get("items") { + Some(Value::Array(a)) => a.iter().filter_map(|v| v.as_str().map(str::to_string)).filter(|x| !x.is_empty()).collect(), + _ => return Err(bad(C, "missing `items` (array of strings)")), + }; + if items.is_empty() { + return Err(bad(C, "a dropdown needs at least one item")); + } + let q = if p.get("path").is_none() && p.get("prop").is_none() { + // The first Dropdown Menu Control of the layer. + let (cid, lid) = layer_p(s, p, C)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let fx = l.effects().and_then(|fx| fx.groups().find(|g| matches!(&g.kind, GroupKind::Effect { effect } if effect == "ec.control.dropdown"))); + let uid = fx.and_then(|g| g.get("menu")).map(|pr| pr.uid).ok_or_else(|| bad(C, "the layer has no Dropdown Menu Control"))?; + let mut q = p.clone(); + q["prop"] = json!(uid); + q["layer"] = json!(lid.0); + q + } else { + p.clone() + }; + let (cid, lid, uid) = super::prop::prop_ref(s, &q, C)?; + let n = items.len() as u32; + s.edit("Edit Dropdown", None, |proj, _| { + let pr = super::layer_mut(proj, cid, lid)?.props.find_mut(uid).ok_or_else(|| bad(C, "no such property"))?; + if !matches!(pr.ui, ParamUi::Popup { .. }) { + return Err(bad(C, "not a dropdown property")); + } + pr.ui = ParamUi::Popup { options: items.clone() }; + let clamp = |v: &mut KV| { + if let KV::Enum(i) = v { + *i = (*i).min(n - 1); + } + }; + clamp(&mut pr.value); + for k in &mut pr.keys { + clamp(&mut k.value); + } + Ok(()) + })?; + Ok(json!({"prop": uid, "items": items})) +} + +fn solo_supported(s: &mut Session, p: &Value) -> Result { + let on = b_p(p, "on").unwrap_or(!s.state.essential_solo); + s.state.essential_solo = on; + s.bump(); + Ok(json!({"on": on})) +} + +/// Properties of a comp's layers that Essential Graphics can expose (Solo Supported +/// Properties): `[{layer, uid, path, name, type}]`. +pub fn supported_properties(project: &Project, cid: ItemId) -> Vec { + let Some(c) = project.comp(cid) else { return vec![] }; + let mut out = vec![]; + for l in &c.layers { + l.props.walk("", &mut |_, pr| { + if pr.three_d_only && !l.is_3d() { + return; + } + if let Some(t) = essential::control_type(pr) { + let path = super::match_path_of(&l.props, pr.uid).unwrap_or_default(); + if path.starts_with(essential::GROUP) { + return; + } + out.push(json!({"layer": l.id.0, "layerName": l.name, "uid": pr.uid, "path": path, "name": l.props.name_path_of(pr.uid), "type": t})); + } + }); + } + out +} + +fn control_json(project: &Project, cid: ItemId, c: &EgControl) -> Value { + match &c.kind { + EgKind::Property { layer, prop } => { + let src = essential::source_prop(project, cid, c); + json!({ + "id": c.id, "name": c.name, "kind": "property", "layer": layer.0, "prop": prop, + "layerName": src.map(|(l, _)| l.name.clone()), + "path": src.and_then(|(l, _)| super::match_path_of(&l.props, *prop)), + "type": src.and_then(|(_, pr)| essential::control_type(pr)), + "value": src.map(|(_, pr)| pr.value_at(Tick::ZERO).to_json()), + "missing": src.is_none(), + }) + } + EgKind::Media { layer } => json!({ + "id": c.id, "name": c.name, "kind": "media", "type": "media", "layer": layer.0, + "item": essential::source_media(project, cid, c).map(|i| i.0), + }), + EgKind::Comment { text } => json!({"id": c.id, "name": c.name, "kind": "comment", "text": text}), + EgKind::Group { children } => json!({ + "id": c.id, "name": c.name, "kind": "group", + "children": children.iter().map(|x| control_json(project, cid, x)).collect::>(), + }), + } +} + +fn list(s: &mut Session, p: &Value) -> Result { + let cid = eg_comp(s, p)?; + let eg = s.project.comp(cid).and_then(|c| c.essential.clone()).unwrap_or_default(); + let name = if eg.name.is_empty() { s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default() } else { eg.name.clone() }; + Ok(json!({ + "comp": cid.0, + "primary": s.state.essential_primary.map(|c| c.0), + "name": name, + "solo": s.state.essential_solo, + "controls": eg.controls.iter().map(|c| control_json(&s.project, cid, c)).collect::>(), + "supported": if b_p(p, "supported").unwrap_or(false) { json!(supported_properties(&s.project, cid)) } else { Value::Null }, + })) +} + +// ------------------------------------------------------------------------------ instances + +/// The precomp layer (an instance) and its source comp. +fn instance(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, ItemId)> { + let (cid, lid) = layer_p(s, p, cmd)?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + match l.source { + LayerSource::Comp { item } if essential::group(l).is_some() => Ok((cid, lid, item)), + _ => Err(bad(cmd, "the layer is not a precomp with Essential Properties")), + } +} + +/// The master property (instance child) of a control. +fn master_uid(s: &Session, cid: ItemId, lid: LayerId, control: u64) -> Option { + let l = s.project.comp(cid)?.layer(lid)?; + let g = essential::group(l)?; + let m = essential::match_id(control); + let mut found = None; + g.walk("", &mut |_, pr| { + if pr.match_id == m { + found = Some(pr.uid); + } + }); + found +} + +fn instance_json(s: &mut Session, p: &Value) -> Result { + let (cid, lid, src) = instance(s, p, "essential.instance")?; + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).ok_or(EngineError::NoComp)?; + let over = essential::overridden(l); + let eg = s.project.comp(src).and_then(|c| c.essential.clone()).unwrap_or_default(); + let t = s.time(); + let g = essential::group(l).cloned(); + let mut out = vec![]; + for c in eg.flat() { + let m = essential::match_id(c.id); + let mut pr = None; + if let Some(g) = &g { + g.walk("", &mut |_, x| { + if x.match_id == m { + pr = Some(x.clone()); + } + }); + } + let kind = match c.kind { + EgKind::Property { .. } => "property", + EgKind::Media { .. } => "media", + EgKind::Comment { .. } => "comment", + EgKind::Group { .. } => "group", + }; + out.push(json!({ + "control": c.id, "name": c.name, "kind": kind, + "uid": pr.as_ref().map(|x| x.uid), + "path": pr.as_ref().and_then(|x| super::match_path_of(&l.props, x.uid)), + "value": pr.as_ref().map(|x| x.value_at(l.layer_time(t)).to_json()), + "type": pr.as_ref().and_then(essential::control_type).or((kind == "media").then_some(ControlType::Media)), + "overridden": pr.as_ref().is_some_and(|x| over.contains(&x.uid)), + })); + } + Ok(json!({"layer": lid.0, "comp": src.0, "name": eg.name, "controls": out})) +} + +/// Set an instance's value for a control (`value`, or `item` for Media Replacement). +fn set_override(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.set"; + let (cid, lid, _) = instance(s, p, C)?; + let control = control_p(p, C)?; + let uid = master_uid(s, cid, lid, control).ok_or_else(|| bad(C, format!("no control {control} on this instance")))?; + if let Some(item) = p.get("item") { + let item = match item { + Value::Number(n) => ItemId(n.as_u64().unwrap_or(0)), + Value::String(name) => s.project.find_by_name(name).map(|i| i.id).ok_or_else(|| bad(C, format!("no item `{name}`")))?, + _ => return Err(bad(C, "`item` is an item id or name")), + }; + if !matches!(s.project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(f)) if f.has_video) { + return Err(bad(C, "replacement media must be footage with video")); + } + s.edit("Replace Media", None, |proj, _| { + let pr = super::layer_mut(proj, cid, lid)?.props.find_mut(uid).ok_or_else(|| bad(C, "no such property"))?; + pr.value = KV::Scalar(item.0 as f64); + Ok(()) + })?; + return Ok(json!({"control": control, "item": item.0})); + } + let mut q = json!({"comp": cid.0, "layer": lid.0, "prop": uid, "value": p.get("value").cloned().ok_or_else(|| bad(C, "missing `value` (or `item`)"))?}); + if let Some(m) = p.get("merge") { + q["merge"] = m.clone(); + } + s.execute("prop.set", q) +} + +/// Controls of an instance to act on: `control` or every overridden one. +fn targets(s: &Session, cid: ItemId, lid: LayerId, p: &Value) -> Vec<(u64, Uid)> { + let Some(l) = s.project.comp(cid).and_then(|c| c.layer(lid)) else { return vec![] }; + let over = essential::overridden(l); + let Some(g) = essential::group(l) else { return vec![] }; + let want = p.get("control").and_then(Value::as_u64); + let mut out = vec![]; + g.walk("", &mut |_, pr| { + if let Some(c) = essential::control_of(&pr.match_id) + && want.is_none_or(|w| w == c) + && (want.is_some() || over.contains(&pr.uid)) + { + out.push((c, pr.uid)); + } + }); + out +} + +fn revert(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.revert"; + let (cid, lid, _) = instance(s, p, C)?; + let ts = targets(s, cid, lid, p); + if ts.is_empty() { + return Ok(json!({"reverted": []})); + } + s.edit("Revert Essential Property", None, |proj, _| { + let l = super::layer_mut(proj, cid, lid)?; + let g = l.props.sub_mut(essential::GROUP).ok_or_else(|| bad(C, "no Essential Properties"))?; + if let GroupKind::Essential { overridden } = &mut g.kind { + overridden.retain(|u| !ts.iter().any(|(_, x)| x == u)); + } + for (_, uid) in &ts { + if let Some(pr) = g.find_mut(*uid) { + // The sync below brings back the source value. + pr.keys.clear(); + pr.expr = None; + } + } + Ok(()) + })?; + // Edits that only clear an override must not be taken for a new override: values are + // refreshed by the sync inside `edit`, which ran after the override list was trimmed. + Ok(json!({"reverted": ts.iter().map(|(c, _)| c).collect::>()})) +} + +/// Push to Comp: write the instance's override values into the source comp's properties (and +/// replaced media into the source layer), then drop the overrides. +fn push_to_comp(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.pushToComp"; + let (cid, lid, src) = instance(s, p, C)?; + let ts = targets(s, cid, lid, p); + let eg = s.project.comp(src).and_then(|c| c.essential.clone()).unwrap_or_default(); + let l = s.project.comp(cid).and_then(|c| c.layer(lid)).cloned().ok_or(EngineError::NoComp)?; + let mut writes = vec![]; + for (control, uid) in &ts { + let (Some(ctl), Some(pr)) = (eg.find(*control), l.props.find(*uid)) else { continue }; + writes.push((ctl.kind.clone(), pr.clone())); + } + s.edit("Push Override Values to Source", None, |proj, _| { + let sc = proj.comp_mut(src).ok_or(EngineError::NoComp)?; + for (kind, pr) in &writes { + match kind { + EgKind::Property { layer, prop } => { + let Some(inner) = sc.layer_mut(*layer) else { continue }; + // Instance keys are in the precomp layer's time (= the source comp's time); + // the source property's keys are in the inner layer's time. + let keys: Vec<_> = pr + .keys + .iter() + .map(|k| { + let mut k = k.clone(); + k.time = inner.layer_time(k.time); + k + }) + .collect(); + if let Some(target) = inner.props.find_mut(*prop) { + target.value = pr.value.clone(); + target.keys = keys; + } + } + EgKind::Media { layer } => { + if let Some(inner) = sc.layer_mut(*layer) { + inner.source = LayerSource::Footage { item: ItemId(pr.value.as_f64().max(0.0) as u64) }; + } + } + _ => {} + } + } + let g = super::layer_mut(proj, cid, lid)?.props.sub_mut(essential::GROUP).ok_or_else(|| bad(C, "no Essential Properties"))?; + if let GroupKind::Essential { overridden } = &mut g.kind { + overridden.retain(|u| !ts.iter().any(|(_, x)| x == u)); + } + for (_, uid) in &ts { + if let Some(pr) = g.find_mut(*uid) { + pr.keys.clear(); + pr.expr = None; + } + } + Ok(()) + })?; + Ok(json!({"pushed": ts.iter().map(|(c, _)| c).collect::>()})) +} + +// ------------------------------------------------------------------------------ templates + +fn file_name(path: &str) -> String { + std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()) +} + +/// The manifest's control list (flattened, with the group each belongs to). +fn manifest_controls(project: &Project, cid: ItemId, eg: &EssentialGraphics) -> Vec { + fn go(project: &Project, cid: ItemId, v: &[EgControl], group: Option, out: &mut Vec) { + for c in v { + let mut j = control_json(project, cid, c); + if let Some(o) = j.as_object_mut() { + o.remove("children"); + o.insert("group".into(), json!(group)); + } + out.push(j); + if let EgKind::Group { children } = &c.kind { + go(project, cid, children, Some(c.id), out); + } + } + } + let mut out = vec![]; + go(project, cid, &eg.controls, None, &mut out); + out +} + +fn export_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.exportTemplate"; + let cid = eg_comp(s, p)?; + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path` (.ectemplate)"))?.to_string(); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let eg = comp.essential.clone().filter(|e| !e.controls.is_empty()).ok_or_else(|| bad(C, "the composition has no Essential Graphics controls"))?; + let comp_name = s.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| if eg.name.is_empty() { comp_name.clone() } else { eg.name.clone() }); + // The project subset: the comp and everything it uses, at the root. + let deps = essential::dependencies(&s.project, cid); + let mut sub = Project { settings: s.project.settings.clone(), next_id: s.project.next_id, ..Project::default() }; + let mut entries: Vec<(String, Vec)> = vec![]; + let mut media = vec![]; + for id in &deps { + let Some(mut it) = s.project.item(*id).cloned() else { continue }; + it.parent = None; + it.proxy = None; + if let ItemKind::Footage(f) = &mut it.kind + && f.data.is_none() + { + let files: Vec = if f.sequence.is_empty() { vec![f.path.clone()] } else { f.sequence.clone() }; + let mut stored = vec![]; + for (k, file) in files.iter().enumerate() { + if let Ok(bytes) = s.services.read_file(file) { + let entry = format!("media/{}/{}", id.0, file_name(file)); + entries.push((entry.clone(), bytes)); + if k == 0 { + f.path = entry.clone(); + } + stored.push(entry); + } + } + if !stored.is_empty() { + if !f.sequence.is_empty() { + f.sequence = stored.clone(); + } + media.push(json!({"item": id.0, "files": stored})); + } + } + sub.items.insert(*id, it); + } + let manifest = json!({ + "format": TEMPLATE_FORMAT, + "version": TEMPLATE_VERSION, + "generator": format!("EffectCraft {}", env!("CARGO_PKG_VERSION")), + "name": name, + "comp": cid.0, + "compName": comp_name, + "width": comp.width, + "height": comp.height, + "frameRate": comp.frame_rate.as_f64(), + "duration": comp.duration.seconds(), + "controls": manifest_controls(&s.project, cid, &eg), + "media": media, + "poster": "poster.png", + }); + // A poster frame (longest side ≤ 320 px). + if let Ok((w, h, rgba)) = s.render_rgba8(cid, comp.poster_time, 320) + && let Ok(png) = super::comp_more::encode_png(&rgba, w, h) + { + entries.push(("poster.png".into(), png)); + } + entries.insert(0, ("project.ecproj".into(), sub.to_json().into_bytes())); + entries.insert(0, ("manifest.json".into(), serde_json::to_vec_pretty(&manifest).unwrap_or_default())); + let bytes = effectcraft_lottie::zip::store(&entries); + s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; + s.toast(format!("Exported template “{name}” to {path}")); + Ok(json!({"path": path, "name": name, "bytes": bytes.len(), "controls": eg.flat().len(), "items": deps.len()})) +} + +/// Read a template's manifest and project (with media paths still archive-relative). +pub fn read_template(bytes: &[u8]) -> std::result::Result<(Value, Project, Vec<(String, Vec)>), String> { + let entries = effectcraft_lottie::zip::read_stored(bytes); + let get = |n: &str| entries.iter().find(|(k, _)| k == n).map(|(_, v)| v.clone()); + let manifest: Value = + serde_json::from_slice(&get("manifest.json").ok_or("not an EffectCraft template (no manifest.json)")?).map_err(|e| format!("manifest.json: {e}"))?; + if manifest.get("format").and_then(Value::as_str) != Some(TEMPLATE_FORMAT) { + return Err("not an EffectCraft template".into()); + } + if manifest.get("version").and_then(Value::as_u64).unwrap_or(0) > TEMPLATE_VERSION as u64 { + return Err("the template was made by a newer EffectCraft".into()); + } + let text = String::from_utf8(get("project.ecproj").ok_or("template has no project.ecproj")?).map_err(|e| e.to_string())?; + let project = Project::from_json(&text).map_err(|e| e.to_string())?; + Ok((manifest, project, entries)) +} + +fn import_template(s: &mut Session, p: &Value) -> Result { + const C: &str = "essential.importTemplate"; + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path` (.ectemplate)"))?.to_string(); + let bytes = s.services.read_file(&path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let (manifest, mut sub, entries) = read_template(&bytes).map_err(|e| bad(C, e))?; + let name = manifest.get("name").and_then(Value::as_str).unwrap_or("Template").to_string(); + let tcomp = manifest.get("comp").and_then(Value::as_u64).map(ItemId).ok_or_else(|| bad(C, "manifest has no `comp`"))?; + // Media next to the template: `/ Media//`. + let pp = std::path::Path::new(&path); + let dir = pp.parent().map(|d| d.to_path_buf()).unwrap_or_default(); + let stem = pp.file_stem().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| "Template".into()); + let media_dir = dir.join(format!("{stem} Media")); + let mut written = std::collections::HashMap::new(); + for (k, v) in entries.iter().filter(|(k, _)| k.starts_with("media/")) { + let rel = k.trim_start_matches("media/"); + let out = media_dir.join(rel); + if let Some(parent) = out.parent() { + let _ = std::fs::create_dir_all(parent); + } + let out_s = out.to_string_lossy().to_string(); + if s.services.write_file(&out_s, v).is_ok() { + written.insert(k.clone(), out_s); + } + } + for it in sub.items.values_mut() { + if let ItemKind::Footage(f) = &mut it.kind { + if let Some(w) = written.get(&f.path) { + f.path = w.clone(); + } + for x in &mut f.sequence { + if let Some(w) = written.get(x) { + *x = w.clone(); + } + } + } + } + let target = if b_p(p, "addToComp") == Some(false) { None } else { comp_id(s, p).ok() }; + let base = s.project.next_id; + essential::offset_ids(&mut sub, base); + let new_comp = ItemId(tcomp.0 + base); + let ids = s.edit("Import Template", None, |proj, st| { + let folder = proj.add_item(&name, effectcraft_color::Label::Yellow, None, ItemKind::Folder); + let mut ids = vec![]; + for (id, mut it) in std::mem::take(&mut sub.items) { + if it.parent.is_none() { + it.parent = Some(folder); + } + ids.push(id.0); + proj.items.insert(id, it); + } + proj.next_id = proj.next_id.max(sub.next_id); + proj.fix_next_id(); + st.project_selection = vec![new_comp]; + Ok(ids) + })?; + let mut layer = Value::Null; + if let Some(cid) = target + && s.project.comp(cid).is_some() + { + let r = s.execute("layer.addItem", json!({"comp": cid.0, "item": new_comp.0, "time": 0.0}))?; + layer = r["layer"].clone(); + } + s.toast(format!("Imported template “{name}”")); + Ok(json!({"comp": new_comp.0, "items": ids, "layer": layer, "name": name})) +} + +/// Read a template's manifest without importing it (Essential Graphics / agents). +fn template_info(s: &mut Session, p: &Value) -> Result { + let path = str_p(p, "path").ok_or_else(|| bad("essential.templateInfo", "missing `path`"))?; + let bytes = s.services.read_file(path).map_err(|e| EngineError::Other(format!("cannot read {path}: {e}")))?; + let (manifest, _, _) = read_template(&bytes).map_err(|e| bad("essential.templateInfo", e))?; + Ok(manifest) +} + +fn has_instance(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let ok = s.active_comp().is_some_and(|c| s.state.selected_layers.iter().any(|l| c.layer(*l).is_some_and(|l| essential::group(l).is_some()))); + if ok { Ok(()) } else { Err("select a precomp layer with Essential Properties".into()) } +} + +fn has_eg(s: &Session) -> std::result::Result<(), String> { + let cid = s.state.essential_primary.or(s.state.active_comp).ok_or("no composition is open")?; + if s.project.comp(cid).and_then(|c| c.essential.as_ref()).is_some_and(|e| !e.controls.is_empty()) { + Ok(()) + } else { + Err("add properties to Essential Graphics first".into()) + } +} + +fn has_selected_props(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + if s.state.selected_props.is_empty() { Err("select a property in the timeline".into()) } else { Ok(()) } +} + +pub fn specs() -> Vec { + vec![ + cmd!("essential.setPrimary", "Primary Composition", [], None, "{comp}", has_comp, set_primary), + cmd!("essential.setName", "Essential Graphics Name", [], None, "{comp?, name}", has_comp, set_name), + cmd!( + "essential.addProperty", + "Add Property to Essential Graphics", + ["Animation"], + None, + "{comp?, layer?, path?|prop?: uid (default: the selected properties), name?, group?: control id, index?}", + has_selected_props, + add_property + ), + cmd!("essential.addMedia", "Add Media Replacement", [], None, "{layer, name?, group?}", has_comp, add_media), + cmd!("essential.addGroup", "Add Group", [], None, "{comp?, name?, index?}", has_comp, add_group), + cmd!("essential.addComment", "Add Comment", [], None, "{comp?, text?, group?}", has_comp, add_comment), + cmd!("essential.rename", "Rename Control", [], None, "{comp?, control, name}", has_comp, rename), + cmd!("essential.remove", "Remove Control", [], None, "{comp?, control}", has_comp, remove), + cmd!("essential.move", "Move Control", [], None, "{comp?, control, group?: id (null = top level), index?}", has_comp, move_control), + cmd!("essential.soloSupported", "Solo Supported Properties", [], None, "{on?}", has_comp, solo_supported), + cmd!( + "essential.exportTemplate", + "Essential Graphics Template...", + ["File", "Export"], + None, + "{comp?, path (.ectemplate), name?}", + has_eg, + export_template + ), + cmd!( + "essential.importTemplate", + "Essential Graphics Template...", + ["File", "Import"], + None, + "{path (.ectemplate), comp?, addToComp?: bool}", + super::always, + import_template + ), + cmd!("essential.set", "Set Essential Property", [], None, "{layer, control, value | item (media)}", has_instance, set_override), + cmd!("essential.pushToComp", "Push Override Values to Source", [], None, "{layer, control? (default: all overridden)}", has_instance, push_to_comp), + cmd!("essential.revert", "Revert", [], None, "{layer, control? (default: all overridden)}", has_instance, revert), + cmd!("effect.editDropdown", "Edit Dropdown Menu", [], None, "{layer, path?|prop?, items: [string]}", has_comp, edit_dropdown), + query!("essential.list", "Essential Graphics controls", "{comp?, supported?: bool}", list), + query!("essential.instance", "Essential Properties of a precomp layer", "{layer}", instance_json), + query!("essential.templateInfo", "Read a template's manifest", "{path}", template_info), + ] +} diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 11e8a3875..9e81db54d 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -111,6 +111,7 @@ fn import(s: &mut Session, p: &Value) -> Result { FootageKind::Still | FootageKind::Sequence => Label::Lavender, FootageKind::Audio => Label::SeaFoam, FootageKind::Video | FootageKind::Model => Label::Aqua, + FootageKind::Data => Label::Sandstone, }; let id = proj.add_item(&name, label, None, ItemKind::Footage(f)); ids.push(id.0); diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index 87c1e2dff..f5f4d3709 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -107,6 +107,7 @@ fn placeholder_footage(p: &Value) -> Footage { missing: true, sequence: vec![], color_profile: None, + ..Default::default() } } diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 1a1447232..7a053c774 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -8,6 +8,7 @@ mod comp; mod comp_more; mod edit; mod effect; +pub mod essential; mod file; mod file_more; mod frontend; @@ -136,6 +137,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(puppet::specs()); v.extend(liquify::specs()); v.extend(settings::specs()); + v.extend(essential::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 61a6f9c38..d0a4ba1a8 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -197,6 +197,13 @@ pub struct EditorState { /// Composition viewer display options (Show Channel, exposure, snapshot, Fast Previews). #[serde(default)] pub viewer: commands::viewer_cmds::ViewOptions, + /// Essential Graphics panel ▸ Primary composition (`None` = the active comp). + #[serde(default)] + pub essential_primary: Option, + /// Essential Graphics panel ▸ Solo Supported Properties (the timeline shows only properties + /// Essential Graphics can expose). + #[serde(default)] + pub essential_solo: bool, } fn one_view() -> u8 { @@ -368,6 +375,8 @@ impl Session { let r = f(&mut p, &mut st)?; // Layer styles: one Global Light per comp, whichever layer edited it. effectcraft_project::styles::sync_global_light(&before, &mut p); + // Essential Properties of precomp layers follow their comps' Essential Graphics. + effectcraft_project::essential::sync_project(&before, &mut p); // Warp Stabilizer analyses made from other frames are cleared (and queued again). for w in warp::invalidate(&before, &mut p) { if !self.warp_pending.contains(&w) { @@ -576,6 +585,8 @@ mod tests_anim_tools; #[cfg(test)] mod tests_effects; #[cfg(test)] +mod tests_essential; +#[cfg(test)] mod tests_fidelity; #[cfg(test)] mod tests_lottie; diff --git a/crates/engine/src/menus.rs b/crates/engine/src/menus.rs index 38252a884..6177e637d 100644 --- a/crates/engine/src/menus.rs +++ b/crates/engine/src/menus.rs @@ -17,7 +17,8 @@ //! //! An entry's shortcut is the command's default shortcut unless the line gives one (needed for //! entries with bound parameters). Adobe-service entries (Team Projects, Libraries, Bridge, Media -//! Encoder, Essential Graphics, Behance, Creative Cloud…) are intentionally absent. +//! Encoder, Behance, Creative Cloud…) are intentionally absent; Essential Graphics templates use +//! EffectCraft's own open `.ectemplate` format. use std::sync::OnceLock; @@ -337,9 +338,11 @@ File Placeholder... | file.importPlaceholder Solid... | file.importSolid Lottie... | file.importLottie + Essential Graphics Template... | essential.importTemplate Export Add to Render Queue | renderQueue.add Lottie JSON... | file.exportLottie + Essential Graphics Template... | essential.exportTemplate --- Find | app.find --- @@ -816,6 +819,7 @@ Animation Remove All Text Animators | text.removeAllAnimators --- Add Expression | prop.setExpression + Add Property to Essential Graphics | essential.addProperty Separate Dimensions | prop.separateDimensions Track Camera | track.camera Warp Stabilizer VFX | track.warpStabilizer @@ -931,6 +935,7 @@ Window Character | window.panel {"panel":"character"} | Cmd+6 Content-Aware Fill | window.unavailablePanel {"panel":"contentAwareFill"} Effects & Presets | window.panel {"panel":"effectsPresets"} | Cmd+5 + Essential Graphics | window.panel {"panel":"essentialGraphics"} Info | window.panel {"panel":"info"} | Cmd+2 Lumetri Scopes | window.unavailablePanel {"panel":"lumetriScopes"} Mask Interpolation | window.panel {"panel":"maskInterpolation"} diff --git a/crates/engine/src/tests_essential.rs b/crates/engine/src/tests_essential.rs new file mode 100644 index 000000000..334738576 --- /dev/null +++ b/crates/engine/src/tests_essential.rs @@ -0,0 +1,289 @@ +//! Essential Graphics: authoring controls, master properties (per-instance overrides, Push to +//! Comp, Revert), `.ectemplate` export/import, and Responsive Design — Time. + +use effectcraft_project::essential; +use effectcraft_project::{ItemId, LayerId}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use crate::Session; +use crate::render::RenderOpts; + +fn out_dir(name: &str) -> std::path::PathBuf { + let d = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out").join(name); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// A "Card" comp (64×64): a white solid with a Fill effect (red) and a text layer; its Fill +/// colour, opacity and text are exposed. Returns (session, card comp, solid layer). +fn card() -> (Session, ItemId, u64, Value) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Card", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let card = s.active_comp_id().unwrap(); + let solid = s.execute("layer.newSolid", json!({"name": "BG", "color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": solid, "effect": "Fill"})).unwrap(); + let text = s.execute("layer.newText", json!({"text": "Hello"})).unwrap()["layer"].as_u64().unwrap(); + let color = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#1/color", "name": "Card Color"})).unwrap(); + let opacity = s.execute("essential.addProperty", json!({"layer": solid, "path": "transform/opacity"})).unwrap(); + let g = s.execute("essential.addGroup", json!({"name": "Words"})).unwrap()["control"].as_u64().unwrap(); + let title = s.execute("essential.addProperty", json!({"layer": text, "path": "text/sourceText", "group": g})).unwrap(); + s.execute("essential.addComment", json!({"text": "Pick a colour"})).unwrap(); + s.execute("essential.setName", json!({"name": "Lower Card"})).unwrap(); + let ids = json!({ + "color": color["controls"][0], "opacity": opacity["controls"][0], "group": g, "title": title["controls"][0], + }); + (s, card, solid, ids) +} + +fn center(s: &Session, comp: ItemId) -> [f32; 4] { + let img = s.render(comp, Tick::ZERO, RenderOpts::default()); + img.get(img.width as i64 / 2, img.height as i64 / 2) +} + +#[test] +fn controls_are_listed_typed_and_editable() { + let (mut s, card, solid, ids) = card(); + let l = s.execute("essential.list", json!({"supported": true})).unwrap(); + assert_eq!(l["name"], "Lower Card"); + let top: Vec<&str> = l["controls"].as_array().unwrap().iter().map(|c| c["kind"].as_str().unwrap()).collect(); + assert_eq!(top, ["property", "property", "group", "comment"]); + assert_eq!(l["controls"][0]["name"], "Card Color"); + assert_eq!(l["controls"][0]["type"], "color"); + assert_eq!(l["controls"][1]["type"], "slider"); + assert_eq!(l["controls"][2]["children"][0]["type"], "text"); + assert!(l["supported"].as_array().unwrap().iter().any(|p| p["path"] == "transform/position" && p["type"] == "point")); + assert!(l["supported"].as_array().unwrap().iter().any(|p| p["path"] == "transform/rotation" && p["type"] == "angle")); + // Rename, move into the group, remove; all undoable. + let opacity = ids["opacity"].as_u64().unwrap(); + s.execute("essential.rename", json!({"control": opacity, "name": "Fade"})).unwrap(); + s.execute("essential.move", json!({"control": opacity, "group": ids["group"], "index": 0})).unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + assert_eq!(l["controls"][1]["children"][0]["name"], "Fade"); + s.execute("edit.undo", json!({})).unwrap(); + s.execute("edit.undo", json!({})).unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + assert_eq!(l["controls"][1]["name"], "Opacity"); + s.execute("essential.remove", json!({"control": opacity})).unwrap(); + assert_eq!(s.execute("essential.list", json!({})).unwrap()["controls"].as_array().unwrap().len(), 3); + // Unsupported properties are refused; adding the same property twice adds nothing. + s.execute("layer.addMask", json!({"layer": solid})).unwrap(); + assert!(s.execute("essential.addProperty", json!({"layer": solid, "path": "masks/#1/path"})).is_err()); + let again = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#1/color"})).unwrap(); + assert_eq!(again["controls"], json!([])); + // Dropdown Menu Control: its items are editable and it becomes a dropdown control. + s.execute("effect.apply", json!({"layer": solid, "effect": "Dropdown Menu Control"})).unwrap(); + s.execute("effect.editDropdown", json!({"layer": solid, "items": ["Small", "Medium", "Large", "Huge"]})).unwrap(); + let d = s.execute("essential.addProperty", json!({"layer": solid, "path": "effects/#2/menu"})).unwrap()["controls"][0].as_u64().unwrap(); + let l = s.execute("essential.list", json!({})).unwrap(); + let c = l["controls"].as_array().unwrap().iter().find(|c| c["id"] == d).unwrap().clone(); + assert_eq!(c["type"], "dropdown"); + let comp = s.project.comp(card).unwrap(); + let menu = comp.layer(LayerId(solid)).unwrap().props.prop("effects/#2/menu").unwrap(); + assert!(matches!(&menu.ui, effectcraft_project::ParamUi::Popup { options } if options.len() == 4 && options[3] == "Huge")); +} + +#[test] +fn master_properties_override_per_instance_push_and_revert() { + let (mut s, card, solid, ids) = card(); + s.execute("comp.new", json!({"name": "Main", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let main = s.active_comp_id().unwrap(); + let a = s.execute("layer.addItem", json!({"item": card.0})).unwrap()["layer"].as_u64().unwrap(); + let b = s.execute("layer.addItem", json!({"item": card.0})).unwrap()["layer"].as_u64().unwrap(); + // Both instances show the controls under Essential Properties. + let inst = s.execute("essential.instance", json!({"layer": a})).unwrap(); + let names: Vec<&str> = inst["controls"].as_array().unwrap().iter().map(|c| c["name"].as_str().unwrap()).collect(); + assert_eq!(names, ["Card Color", "Opacity", "Words", "Source Text", "Comment"]); + assert_eq!(inst["controls"][0]["value"], json!([1.0, 0.0, 0.0, 1.0])); + // Override the colour of the top instance (a = layer #2, b on top is #1). + s.execute("layer.select", json!({"layers": [b]})).unwrap(); + s.execute("essential.set", json!({"layer": b, "control": ids["color"], "value": [0.0, 0.0, 1.0, 1.0]})).unwrap(); + let px = center(&s, main); + assert!(px[2] > 0.99 && px[0] < 0.01, "instance b is blue: {px:?}"); + let inst_b = s.execute("essential.instance", json!({"layer": b})).unwrap(); + assert_eq!(inst_b["controls"][0]["overridden"], true); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][0]["overridden"], false); + // The source comp is untouched; hiding b shows a still red. + assert!(center(&s, card)[0] > 0.99); + s.execute("layer.setSwitch", json!({"layer": b, "switch": "video", "value": false})).unwrap(); + let px = center(&s, main); + assert!(px[0] > 0.99 && px[2] < 0.01, "instance a stays red: {px:?}"); + s.execute("edit.undo", json!({})).unwrap(); + // Editing the master property in the timeline (prop.set) also overrides. + let path = inst_b["controls"][1]["path"].as_str().unwrap().to_string(); + assert!(path.starts_with("essential/"), "{path}"); + s.execute("prop.set", json!({"layer": a, "path": path, "value": 50})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][1]["overridden"], true); + // A change to the source shows in instances that don't override it. + s.execute("prop.set", json!({"comp": card.0, "layer": solid, "path": "transform/opacity", "value": 80})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][1]["value"], json!(80.0)); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][1]["value"], json!(50.0)); + // Revert a's opacity. + s.execute("essential.revert", json!({"layer": a, "control": ids["opacity"]})).unwrap(); + let inst = s.execute("essential.instance", json!({"layer": a})).unwrap(); + assert_eq!(inst["controls"][1]["overridden"], false); + assert_eq!(inst["controls"][1]["value"], json!(80.0)); + // Push b's colour to the source comp: the override goes away and every instance is blue. + s.execute("essential.pushToComp", json!({"layer": b})).unwrap(); + let src = s.project.comp(card).unwrap().layer(LayerId(solid)).unwrap().props.prop("effects/#1/color").unwrap().value.clone(); + assert_eq!(src.to_json(), json!([0.0, 0.0, 1.0, 1.0])); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][0]["overridden"], false); + assert_eq!(s.execute("essential.instance", json!({"layer": a})).unwrap()["controls"][0]["value"], json!([0.0, 0.0, 1.0, 1.0])); + // Undo restores the override and the red source. + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(s.execute("essential.instance", json!({"layer": b})).unwrap()["controls"][0]["overridden"], true); + let src = s.project.comp(card).unwrap().layer(LayerId(solid)).unwrap().props.prop("effects/#1/color").unwrap().value.clone(); + assert_eq!(src.to_json(), json!([1.0, 0.0, 0.0, 1.0])); + // Removing a control removes it from instances. + s.execute("essential.remove", json!({"comp": card.0, "control": ids["color"]})).unwrap(); + let l = s.project.comp(main).unwrap().layer(LayerId(b)).unwrap().clone(); + assert!(essential::group(&l).unwrap().children.iter().all(|n| n.name() != "Card Color")); +} + +#[test] +fn template_export_import_round_trip_with_overrides() { + let dir = out_dir("ectemplate"); + let (mut s, card, _solid, ids) = card(); + let path = dir.join("Lower Card.ectemplate").to_string_lossy().to_string(); + let r = s.execute("essential.exportTemplate", json!({"comp": card.0, "path": path})).unwrap(); + assert_eq!(r["name"], "Lower Card"); + let info = s.execute("essential.templateInfo", json!({"path": path})).unwrap(); + assert_eq!(info["format"], "effectcraft-template"); + assert_eq!(info["width"], 64); + let kinds: Vec<&str> = info["controls"].as_array().unwrap().iter().map(|c| c["kind"].as_str().unwrap()).collect(); + assert_eq!(kinds, ["property", "property", "group", "property", "comment"]); + assert_eq!(info["controls"][3]["group"], ids["group"]); + let entries = effectcraft_lottie::zip::read_stored(&std::fs::read(&path).unwrap()); + let names: Vec<&str> = entries.iter().map(|(n, _)| n.as_str()).collect(); + assert!(names.contains(&"manifest.json") && names.contains(&"project.ecproj") && names.contains(&"poster.png"), "{names:?}"); + + // A fresh project imports it and gets an instance in its active comp. + let mut t = Session::default(); + t.execute("comp.new", json!({"name": "Show", "width": 64, "height": 64, "frameRate": 30, "duration": 2})).unwrap(); + let show = t.active_comp_id().unwrap(); + t.execute("layer.newSolid", json!({"name": "Back", "color": "#00ff00"})).unwrap(); + let r = t.execute("essential.importTemplate", json!({"path": path})).unwrap(); + let tcomp = ItemId(r["comp"].as_u64().unwrap()); + let inst = r["layer"].as_u64().unwrap(); + assert_eq!(t.project.item(tcomp).unwrap().name, "Card"); + assert!(t.project.items.values().any(|i| i.is_folder() && i.name == "Lower Card")); + // Ids were remapped consistently: the template comp's controls point at its own layers. + let eg = t.project.comp(tcomp).unwrap().essential.clone().unwrap(); + for c in eg.flat() { + if let essential::EgKind::Property { .. } = c.kind { + assert!(essential::source_prop(&t.project, tcomp, c).is_some(), "{c:?}"); + } + } + let px = center(&t, show); + assert!(px[0] > 0.99 && px[1] < 0.01, "the card renders red over green: {px:?}"); + // Override on the imported instance, then save/load keeps it. + let color_ctl = eg.flat().iter().find(|c| c.name == "Card Color").unwrap().id; + t.execute("essential.set", json!({"layer": inst, "control": color_ctl, "value": "#ffff00"})).unwrap(); + let px = center(&t, show); + assert!(px[0] > 0.99 && px[1] > 0.99 && px[2] < 0.01, "yellow: {px:?}"); + let saved = effectcraft_project::Project::from_json(&t.project.to_json()).unwrap(); + let mut u = Session::default(); + u.replace_project(saved, None); + u.open_comp(show); + let px = center(&u, show); + assert!(px[0] > 0.99 && px[1] > 0.99 && px[2] < 0.01, "override survives save/load: {px:?}"); + // Importing again keeps both copies apart (fresh ids). + let r2 = t.execute("essential.importTemplate", json!({"path": path, "addToComp": false})).unwrap(); + assert_ne!(r2["comp"], r["comp"]); + assert!(r2["layer"].is_null()); + // Not a template. + std::fs::write(dir.join("bad.ectemplate"), b"nope").unwrap(); + assert!(t.execute("essential.importTemplate", json!({"path": dir.join("bad.ectemplate").to_string_lossy()})).is_err()); +} + +#[test] +fn media_replacement_swaps_footage_per_instance() { + use std::sync::Arc; + // Two "footage" items rendered by a fake source: item → solid colour. + struct Colors; + impl crate::render::FootageSource for Colors { + fn frame(&self, _: ItemId, f: &effectcraft_project::Footage, _: Tick) -> Option> { + let c = if f.path.contains("red") { [1.0, 0.0, 0.0, 1.0] } else { [0.0, 0.0, 1.0, 1.0] }; + Some(Arc::new(effectcraft_raster::Image::filled(16, 16, c))) + } + } + let mut s = Session { footage: Arc::new(Colors), ..Default::default() }; + let foot = |p: &str| effectcraft_project::Footage { + path: p.into(), + kind: effectcraft_project::FootageKind::Still, + width: 16, + height: 16, + has_video: true, + ..Default::default() + }; + let (red, blue) = s + .edit("x", None, |proj, _| { + let r = proj.add_item("red.png", Default::default(), None, effectcraft_project::ItemKind::Footage(foot("red.png"))); + let b = proj.add_item("blue.png", Default::default(), None, effectcraft_project::ItemKind::Footage(foot("blue.png"))); + Ok((r, b)) + }) + .unwrap(); + s.execute("comp.new", json!({"name": "Slot", "width": 16, "height": 16, "frameRate": 30, "duration": 1})).unwrap(); + let slot = s.active_comp_id().unwrap(); + let fl = s.execute("layer.addItem", json!({"item": red.0})).unwrap()["layer"].as_u64().unwrap(); + let m = s.execute("essential.addMedia", json!({"layer": fl})).unwrap()["control"].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "Main", "width": 16, "height": 16, "frameRate": 30, "duration": 1})).unwrap(); + let main = s.active_comp_id().unwrap(); + let inst = s.execute("layer.addItem", json!({"item": slot.0})).unwrap()["layer"].as_u64().unwrap(); + assert!(center(&s, main)[0] > 0.99); + s.execute("essential.set", json!({"layer": inst, "control": m, "item": blue.0})).unwrap(); + let px = center(&s, main); + assert!(px[2] > 0.99 && px[0] < 0.01, "{px:?}"); + assert!(center(&s, slot)[0] > 0.99, "the source keeps its media"); + assert_eq!(s.execute("essential.instance", json!({"layer": inst})).unwrap()["controls"][0]["type"], "media"); + s.execute("essential.revert", json!({"layer": inst})).unwrap(); + assert!(center(&s, main)[0] > 0.99); +} + +/// Responsive Design — Time: stretching a precomp keeps its protected regions at their +/// original speed; the unprotected parts absorb the stretch. +#[test] +fn protected_regions_keep_speed_under_time_stretch() { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Inner", "width": 32, "height": 32, "frameRate": 30, "duration": 4})).unwrap(); + let inner = s.active_comp_id().unwrap(); + // Intro 0–1 s and outro 3–4 s. + s.execute("time.set", json!({"time": 1.0})).unwrap(); + s.execute("comp.responsiveTime", json!({"op": "intro"})).unwrap(); + s.execute("time.set", json!({"time": 3.0})).unwrap(); + s.execute("comp.responsiveTime", json!({"op": "outro"})).unwrap(); + let comp = s.project.comp(inner).unwrap().clone(); + assert_eq!(crate::render::eval::protected_regions(&comp), vec![(0.0, 1.0), (3.0, 4.0)]); + s.execute("comp.new", json!({"name": "Outer", "width": 32, "height": 32, "frameRate": 30, "duration": 10})).unwrap(); + let outer = s.active_comp_id().unwrap(); + let l = s.execute("layer.addItem", json!({"item": inner.0})).unwrap()["layer"].as_u64().unwrap(); + s.execute("layer.select", json!({"layers": [l]})).unwrap(); + s.execute("layer.timeStretch", json!({"percent": 200})).unwrap(); + let st = |s: &Session, t: f64| { + let c = s.project.comp(outer).unwrap(); + let layer = c.layer(LayerId(l)).unwrap(); + crate::render::EvalCtx::new(&s.project, outer, c, Tick::from_seconds_f64(t)).source_time(layer).seconds() + }; + // 8 s total: intro 1 s at speed 1, middle 2 s stretched to 6 s, outro 1 s at speed 1. + let near = |a: f64, b: f64| (a - b).abs() < 1e-6; + assert!(near(st(&s, 0.5), 0.5)); + assert!(near(st(&s, 1.0), 1.0)); + assert!(near(st(&s, 4.0), 2.0)); + assert!(near(st(&s, 7.0), 3.0)); + assert!(near(st(&s, 7.5), 3.5), "outro plays at full speed: {}", st(&s, 7.5)); + // Protected regions play at the original speed: one comp second = one source second. + assert!(near(st(&s, 0.8) - st(&s, 0.3), 0.5)); + assert!(near(st(&s, 7.9) - st(&s, 7.4), 0.5)); + assert!(near(st(&s, 5.0) - st(&s, 4.0), 1.0 / 3.0), "the middle is slowed down 3×"); + // Compressing to 75 %: the middle plays faster. + s.execute("layer.timeStretch", json!({"percent": 75})).unwrap(); + assert!(near(st(&s, 0.5), 0.5)); + assert!(near(st(&s, 1.0 + 0.5), 1.0 + 0.5 / 0.5), "{}", st(&s, 1.5)); + // Without protected regions the plain stretch applies. + s.execute("comp.open", json!({"comp": inner.0})).ok(); + let mut p = (*s.project).clone(); + p.comp_mut(inner).unwrap().markers.clear(); + s.project = std::sync::Arc::new(p); + assert!(near(st(&s, 1.5), 2.0)); +} diff --git a/crates/engine/src/tests_model3d.rs b/crates/engine/src/tests_model3d.rs index 51f9173d0..02e4e52a2 100644 --- a/crates/engine/src/tests_model3d.rs +++ b/crates/engine/src/tests_model3d.rs @@ -52,6 +52,7 @@ impl Importer for ModelImporter { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }) } } diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858caa..5cb702469 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -257,6 +257,7 @@ impl Importer for MockImporter { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }) } } diff --git a/crates/engine/src/tests_track.rs b/crates/engine/src/tests_track.rs index adcf7df0f..8c7037b69 100644 --- a/crates/engine/src/tests_track.rs +++ b/crates/engine/src/tests_track.rs @@ -101,6 +101,7 @@ pub(crate) fn setup(make: impl Fn(u32) -> Image + Send + Sync + 'static) -> (Ses missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); @@ -410,6 +411,7 @@ fn analyze_1080p_single_point_speed() { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = p.add_item("hd", Label::Aqua, None, ItemKind::Footage(footage)); let comp = p.comp(cid).unwrap().clone(); diff --git a/crates/gpu/src/tests.rs b/crates/gpu/src/tests.rs index ca2fb1b0c..698b3fc58 100644 --- a/crates/gpu/src/tests.rs +++ b/crates/gpu/src/tests.rs @@ -100,6 +100,7 @@ impl Scene { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = self.p.add_item("Pattern", Label::Lavender, None, ItemKind::Footage(f)); build::layer(&mut self.p, &self.comp, "Pattern", LayerSource::Footage { item: fid }, (w, h), None) diff --git a/crates/lottie/src/import.rs b/crates/lottie/src/import.rs index c87dc1879..0fb1bfebc 100644 --- a/crates/lottie/src/import.rs +++ b/crates/lottie/src/import.rs @@ -171,6 +171,7 @@ pub fn import( missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let item = project.add_item(&nm, Label::Lavender, Some(folder), ItemKind::Footage(foot)); cx.items.push(item); diff --git a/crates/lottie/src/lib.rs b/crates/lottie/src/lib.rs index 55c2d01c1..198ed7cfb 100644 --- a/crates/lottie/src/lib.rs +++ b/crates/lottie/src/lib.rs @@ -24,7 +24,7 @@ mod export; mod import; mod shapes; mod text; -mod zip; +pub mod zip; use effectcraft_color::BlendMode; use effectcraft_time::{FrameRate, TICKS_PER_SECOND, Tick}; diff --git a/crates/lottie/src/tests.rs b/crates/lottie/src/tests.rs index 5140133c8..6bb37d833 100644 --- a/crates/lottie/src/tests.rs +++ b/crates/lottie/src/tests.rs @@ -592,6 +592,7 @@ fn images_embed_as_base64_and_come_back() { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }), ); let c = comp(&p, cid); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index b6632fbac..478ddc094 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -386,7 +386,7 @@ impl Inner { let i = Self::frame_index(footage.frame_rate, t, footage.sequence.len() as i64, footage.loop_count); Loc { key: key(&footage.sequence[i as usize], 0), media_t: None } } - FootageKind::Still | FootageKind::Sequence | FootageKind::Model => Loc { key: key(&footage.path, 0), media_t: None }, + FootageKind::Still | FootageKind::Sequence | FootageKind::Model | FootageKind::Data => Loc { key: key(&footage.path, 0), media_t: None }, FootageKind::Video | FootageKind::Audio => { let rate = footage.frame_rate; let n = Self::frame_count(rate, footage.duration); diff --git a/crates/media/src/probe.rs b/crates/media/src/probe.rs index e97544dbe..9f1c13c3b 100644 --- a/crates/media/src/probe.rs +++ b/crates/media/src/probe.rs @@ -64,6 +64,7 @@ pub fn probe_model(path: &str, bytes: &[u8], resolve: &dyn Fn(&str) -> Option`. Changing a child marks it *overridden* for that instance only; +//! [`sync_project`] keeps the groups in step with the controls (and refreshes the values of +//! controls that are not overridden), [`with_overrides`] builds the project an instance renders. + +use serde::{Deserialize, Serialize}; + +use crate::props::{GroupKind, Node, ParamUi, PropGroup, Property, Uid}; +use crate::{ItemId, ItemKind, Layer, LayerId, LayerSource, Project, Value}; + +/// Match id of the Essential Properties group on precomp layers. +pub const GROUP: &str = "essential"; + +/// A composition's Essential Graphics definition. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct EssentialGraphics { + /// Template name (Essential Graphics panel ▸ name field; the exported template's title). + #[serde(default)] + pub name: String, + #[serde(default)] + pub controls: Vec, +} + +/// One entry of the Essential Graphics panel. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct EgControl { + pub id: u64, + /// Display name (renamable; defaults to the property's name). + pub name: String, + pub kind: EgKind, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum EgKind { + /// A property of a layer of the comp. + Property { layer: LayerId, prop: Uid }, + /// Media Replacement: the footage of a footage layer can be swapped per instance. + Media { layer: LayerId }, + /// A text note shown in the panel. + Comment { text: String }, + /// A named, collapsible group of controls. + Group { + #[serde(default)] + children: Vec, + }, +} + +/// The kind of control a property becomes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ControlType { + Text, + Color, + Slider, + Checkbox, + Point, + Angle, + Dropdown, + Media, +} + +impl ControlType { + pub fn label(self) -> &'static str { + match self { + ControlType::Text => "Text", + ControlType::Color => "Color", + ControlType::Slider => "Slider", + ControlType::Checkbox => "Checkbox", + ControlType::Point => "Point", + ControlType::Angle => "Angle", + ControlType::Dropdown => "Dropdown", + ControlType::Media => "Media", + } + } +} + +/// The control a property is shown as, `None` when Essential Graphics can't expose it (paths, +/// gradients, layer pickers, hidden data). +pub fn control_type(p: &Property) -> Option { + if matches!(p.ui, ParamUi::Hidden) { + return None; + } + Some(match (&p.value, &p.ui) { + (Value::Text(_) | Value::Str(_), _) => ControlType::Text, + (Value::Color(_), _) => ControlType::Color, + (Value::Bool(_), _) | (_, ParamUi::Checkbox) => ControlType::Checkbox, + (Value::Enum(_), _) | (_, ParamUi::Popup { .. }) => ControlType::Dropdown, + (_, ParamUi::Angle) => ControlType::Angle, + (Value::Vec2(_) | Value::Vec3(_), _) => ControlType::Point, + (Value::Scalar(_), _) => ControlType::Slider, + _ => return None, + }) +} + +impl EssentialGraphics { + /// Every control, depth first (groups before their children). + pub fn flat(&self) -> Vec<&EgControl> { + fn go<'a>(v: &'a [EgControl], out: &mut Vec<&'a EgControl>) { + for c in v { + out.push(c); + if let EgKind::Group { children } = &c.kind { + go(children, out); + } + } + } + let mut out = vec![]; + go(&self.controls, &mut out); + out + } + pub fn find(&self, id: u64) -> Option<&EgControl> { + self.flat().into_iter().find(|c| c.id == id) + } + pub fn find_mut(&mut self, id: u64) -> Option<&mut EgControl> { + fn go(v: &mut [EgControl], id: u64) -> Option<&mut EgControl> { + for c in v { + if c.id == id { + return Some(c); + } + if let EgKind::Group { children } = &mut c.kind + && let Some(x) = go(children, id) + { + return Some(x); + } + } + None + } + go(&mut self.controls, id) + } + /// Remove a control (anywhere) and return it. + pub fn remove(&mut self, id: u64) -> Option { + fn go(v: &mut Vec, id: u64) -> Option { + if let Some(i) = v.iter().position(|c| c.id == id) { + return Some(v.remove(i)); + } + v.iter_mut().find_map(|c| if let EgKind::Group { children } = &mut c.kind { go(children, id) } else { None }) + } + go(&mut self.controls, id) + } + /// The list a control is inserted into: a group's children or the top level. + pub fn list_mut(&mut self, group: Option) -> Option<&mut Vec> { + match group { + None => Some(&mut self.controls), + Some(g) => match &mut self.find_mut(g)?.kind { + EgKind::Group { children } => Some(children), + _ => None, + }, + } + } + /// The control exposing property `prop` of `layer`, if any. + pub fn control_for(&self, layer: LayerId, prop: Uid) -> Option<&EgControl> { + self.flat().into_iter().find(|c| matches!(c.kind, EgKind::Property { layer: l, prop: p } if l == layer && p == prop)) + } + /// Controls an instance can override (properties and media; not comments or groups). + pub fn has_instance_controls(&self) -> bool { + self.flat().iter().any(|c| matches!(c.kind, EgKind::Property { .. } | EgKind::Media { .. })) + } +} + +/// Match id of an instance property / group for a control. +pub fn match_id(control: u64) -> String { + format!("eg{control}") +} + +/// Control id of an Essential Properties child (`eg42` → 42). +pub fn control_of(match_id: &str) -> Option { + match_id.strip_prefix("eg")?.parse().ok() +} + +/// The Essential Properties group of a layer. +pub fn group(layer: &Layer) -> Option<&PropGroup> { + layer.props.sub(GROUP) +} + +/// Uids of the overridden master properties of a layer. +pub fn overridden(layer: &Layer) -> Vec { + match group(layer).map(|g| &g.kind) { + Some(GroupKind::Essential { overridden }) => overridden.clone(), + _ => vec![], + } +} + +/// The source property a control exposes, in `project`. +pub fn source_prop<'a>(project: &'a Project, comp: ItemId, c: &EgControl) -> Option<(&'a Layer, &'a Property)> { + let EgKind::Property { layer, prop } = c.kind else { return None }; + let l = project.comp(comp)?.layer(layer)?; + Some((l, l.props.find(prop)?)) +} + +/// The footage item a media control shows by default (the layer's own source). +pub fn source_media(project: &Project, comp: ItemId, c: &EgControl) -> Option { + let EgKind::Media { layer } = c.kind else { return None }; + match project.comp(comp)?.layer(layer)?.source { + LayerSource::Footage { item } => Some(item), + _ => None, + } +} + +/// Build the Essential Properties group a layer nesting `comp` should have, reusing uids and +/// override values from `old`. Uids for new nodes come from `next`. +fn build_group(project: &Project, comp: ItemId, eg: &EssentialGraphics, old: Option<&PropGroup>, next: &mut u64) -> PropGroup { + let over = match old.map(|g| &g.kind) { + Some(GroupKind::Essential { overridden }) => overridden.clone(), + _ => vec![], + }; + let mut alloc = || { + let v = *next; + *next += 1; + v + }; + fn find_old<'a>(old: Option<&'a PropGroup>, m: &str) -> Option<&'a Node> { + let g = old?; + g.children.iter().find(|c| c.match_id() == m).or_else(|| g.groups().find_map(|sub| find_old(Some(sub), m))) + } + fn build( + project: &Project, + comp: ItemId, + list: &[EgControl], + old: Option<&PropGroup>, + over: &[Uid], + alloc: &mut dyn FnMut() -> u64, + keep: &mut Vec, + ) -> Vec { + let mut out = vec![]; + for c in list { + let m = match_id(c.id); + let prev = find_old(old, &m); + match &c.kind { + EgKind::Comment { .. } => {} + EgKind::Group { children } => { + let uid = prev.map(Node::uid).unwrap_or_else(&mut *alloc); + let mut g = PropGroup::new(uid, &m, &c.name); + g.children = build(project, comp, children, old, over, alloc, keep); + out.push(Node::Group(g)); + } + EgKind::Property { .. } => { + let Some((_, src)) = source_prop(project, comp, c) else { continue }; + let prev = prev.and_then(Node::as_prop); + let uid = prev.map(|p| p.uid).unwrap_or_else(&mut *alloc); + let mut p = Property { uid, match_id: m, name: c.name.clone(), keys: vec![], expr: None, ..src.clone() }; + // Instance properties follow the source's look but are always editable. + p.static_only = false; + if let Some(prev) = prev.filter(|pp| over.contains(&pp.uid)) { + p.value = prev.value.clone(); + p.keys = prev.keys.clone(); + p.expr = prev.expr.clone(); + keep.push(uid); + } else { + p.value = src.value_at(effectcraft_time::Tick::ZERO); + } + out.push(Node::Prop(p)); + } + EgKind::Media { .. } => { + let Some(item) = source_media(project, comp, c) else { continue }; + let prev = prev.and_then(Node::as_prop); + let uid = prev.map(|p| p.uid).unwrap_or_else(&mut *alloc); + let mut p = Property::new(uid, &m, &c.name, Value::Scalar(item.0 as f64)).with_ui(ParamUi::Hidden); + p.static_only = true; + if let Some(prev) = prev.filter(|pp| over.contains(&pp.uid)) { + p.value = prev.value.clone(); + keep.push(uid); + } + out.push(Node::Prop(p)); + } + } + } + out + } + let uid = old.map(|g| g.uid).unwrap_or_else(&mut alloc); + let mut keep = vec![]; + let children = build(project, comp, &eg.controls, old, &over, &mut alloc, &mut keep); + PropGroup { uid, match_id: GROUP.into(), name: "Essential Properties".into(), kind: GroupKind::Essential { overridden: keep }, enabled: true, children } +} + +/// Master properties the user changed in this edit (`before` → `after`) become overridden. +fn mark_overrides(before: &Project, after: &mut Project) { + let ids: Vec = after.comps().map(|(i, _)| *i).collect(); + for cid in ids { + let Some(comp) = after.comp(cid) else { continue }; + let Some(old_comp) = before.comp(cid) else { continue }; + let mut changes: Vec<(LayerId, Vec)> = vec![]; + for l in &comp.layers { + let Some(g) = group(l) else { continue }; + let Some(ol) = old_comp.layer(l.id) else { continue }; + let Some(og) = group(ol) else { continue }; + // Properties overridden before this edit stay as the edit left them (Revert and + // Push to Comp clear overrides by editing them). + let mut over = overridden(l); + over.extend(overridden(ol)); + let mut add = vec![]; + g.walk("", &mut |_, p| { + if over.contains(&p.uid) { + return; + } + if let Some(op) = og.find(p.uid) + && (op.value != p.value || op.keys != p.keys || op.expr != p.expr) + { + add.push(p.uid); + } + }); + if !add.is_empty() { + changes.push((l.id, add)); + } + } + if changes.is_empty() { + continue; + } + let Some(c) = after.comp_mut(cid) else { continue }; + for (lid, add) in changes { + if let Some(g) = c.layer_mut(lid).and_then(|l| l.props.sub_mut(GROUP)) + && let GroupKind::Essential { overridden } = &mut g.kind + { + overridden.extend(add); + } + } + } +} + +/// After an edit: mark changed master properties as overridden, then add, update or remove the +/// Essential Properties groups of every precomp layer so they match their comp's controls. +pub fn sync_project(before: &Project, after: &mut Project) { + mark_overrides(before, after); + let ids: Vec = after.comps().map(|(i, _)| *i).collect(); + let mut next = after.next_id; + for cid in ids { + let Some(comp) = after.comp(cid) else { continue }; + let mut updates: Vec<(usize, Option)> = vec![]; + for (i, l) in comp.layers.iter().enumerate() { + let want = match l.source { + LayerSource::Comp { item } => after.comp(item).and_then(|c| c.essential.as_ref()).filter(|e| e.has_instance_controls()).map(|e| (item, e)), + _ => None, + }; + let old = group(l); + match want { + Some((item, eg)) => { + let g = build_group(after, item, eg, old, &mut next); + if old != Some(&g) { + updates.push((i, Some(g))); + } + } + None if old.is_some() => updates.push((i, None)), + None => {} + } + } + if updates.is_empty() { + continue; + } + let Some(c) = after.comp_mut(cid) else { continue }; + for (i, g) in updates { + let l = &mut c.layers[i]; + let pos = l.props.children.iter().position(|n| n.match_id() == GROUP); + match (pos, g) { + (Some(p), Some(g)) => l.props.children[p] = Node::Group(g), + (None, Some(g)) => l.props.children.insert(0, Node::Group(g)), + (Some(p), None) => { + l.props.children.remove(p); + } + (None, None) => {} + } + } + } + after.next_id = after.next_id.max(next); +} + +/// One instance override: the control and the value it takes in this instance. +#[derive(Clone, Debug, PartialEq)] +pub struct Override { + pub control: u64, + pub value: Value, +} + +/// The project as an instance of `comp` renders it: each overridden control's source property +/// set to the instance value (keyframes dropped; its expression, if any, still applies) and +/// replaced media swapped in. `None` when nothing applies. +pub fn with_overrides(project: &Project, comp: ItemId, overrides: &[Override]) -> Option { + let eg = project.comp(comp)?.essential.clone()?; + let mut p = project.clone(); + let c = p.comp_mut(comp)?; + let mut any = false; + for o in overrides { + let Some(ctl) = eg.find(o.control) else { continue }; + match ctl.kind { + EgKind::Property { layer, prop } => { + if let Some(pr) = c.layer_mut(layer).and_then(|l| l.props.find_mut(prop)) { + pr.keys.clear(); + pr.value = o.value.clone(); + any = true; + } + } + EgKind::Media { layer } => { + let item = ItemId(o.value.as_f64().max(0.0) as u64); + // Replacement media must be footage. + if !matches!(project.item(item).map(|i| &i.kind), Some(ItemKind::Footage(_))) { + continue; + } + if let Some(l) = c.layer_mut(layer) { + l.source = LayerSource::Footage { item }; + any = true; + } + } + _ => {} + } + } + any.then_some(p) +} + +// ------------------------------------------------------------------------------ id remapping + +/// Shift every id (items, layers, property uids, Essential Graphics controls) of `p` by +/// `offset`, keeping all references consistent: used when merging a template project into +/// another project. +pub fn offset_ids(p: &mut Project, offset: u64) { + let items = std::mem::take(&mut p.items); + for (_, mut it) in items { + it.id = ItemId(it.id.0 + offset); + it.parent = it.parent.map(|f| ItemId(f.0 + offset)); + if let ItemKind::Comp(c) = &mut it.kind { + let c = std::sync::Arc::make_mut(c); + for l in &mut c.layers { + offset_layer(l, offset); + } + if let Some(eg) = &mut c.essential { + fn go(v: &mut [EgControl], o: u64) { + for c in v { + c.id += o; + match &mut c.kind { + EgKind::Property { layer, prop } => { + *layer = LayerId(layer.0 + o); + *prop += o; + } + EgKind::Media { layer } => *layer = LayerId(layer.0 + o), + EgKind::Group { children } => go(children, o), + EgKind::Comment { .. } => {} + } + } + } + go(&mut eg.controls, offset); + } + } + p.items.insert(it.id, it); + } + p.render_queue.clear(); + p.next_id += offset; +} + +fn offset_layer(l: &mut Layer, o: u64) { + l.id = LayerId(l.id.0 + o); + l.parent = l.parent.map(|x| LayerId(x.0 + o)); + if let Some(tm) = &mut l.track_matte { + tm.layer = LayerId(tm.layer.0 + o); + } + match &mut l.source { + LayerSource::Footage { item } | LayerSource::Comp { item } | LayerSource::Solid { item } | LayerSource::Model { item } => *item = ItemId(item.0 + o), + _ => {} + } + fn group(g: &mut PropGroup, o: u64, essential_parent: bool) { + g.uid += o; + if let GroupKind::Essential { overridden } = &mut g.kind { + for u in overridden.iter_mut() { + *u += o; + } + } + let is_essential = matches!(g.kind, GroupKind::Essential { .. }) || essential_parent; + for c in &mut g.children { + match c { + Node::Prop(p) => { + p.uid += o; + let fix = |v: &mut Value| { + if let Value::Layer(Some(id)) = v { + *id += o; + } + }; + fix(&mut p.value); + for k in &mut p.keys { + fix(&mut k.value); + } + if is_essential && control_of(&p.match_id).is_some() { + p.match_id = match_id(control_of(&p.match_id).unwrap_or(0) + o); + // Media replacement values are item ids. + if matches!(p.ui, ParamUi::Hidden) + && let Value::Scalar(x) = &mut p.value + { + *x += o as f64; + } + } + } + Node::Group(sub) => { + if is_essential && let Some(id) = control_of(&sub.match_id) { + sub.match_id = match_id(id + o); + } + group(sub, o, is_essential); + } + } + } + } + group(&mut l.props, o, false); +} + +/// The items `comp` needs (itself, nested comps, footage, solids), transitively. +pub fn dependencies(project: &Project, comp: ItemId) -> Vec { + let mut out = vec![]; + let mut stack = vec![comp]; + while let Some(id) = stack.pop() { + if out.contains(&id) { + continue; + } + let Some(it) = project.item(id) else { continue }; + out.push(id); + if let ItemKind::Comp(c) = &it.kind { + for l in &c.layers { + if let Some(i) = l.source.item() { + stack.push(i); + } + // Media replacement candidates referenced by instances inside. + if let Some(g) = group(l) { + g.walk("", &mut |_, p| { + if matches!(p.ui, ParamUi::Hidden) + && control_of(&p.match_id).is_some() + && let Value::Scalar(x) = p.value + { + stack.push(ItemId(x as u64)); + } + }); + } + } + } + } + out.sort(); + out +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 4a00c4133..0e9c6947a 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -5,6 +5,7 @@ //! change. pub mod build; +pub mod essential; pub mod props; pub mod render_queue; pub mod styles; @@ -214,6 +215,9 @@ pub struct Comp { /// Viewer guides (View ▸ Add Guide…), in comp pixels. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub guides: Vec, + /// Essential Graphics: the controls this comp exposes to instances and templates. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub essential: Option, } /// A viewer guide line: vertical guides sit at an x position, horizontal ones at a y position. @@ -251,6 +255,7 @@ impl Comp { poster_time: Tick::ZERO, guides: vec![], global_light: styles::GlobalLight::default(), + essential: None, } } pub fn layer(&self, id: LayerId) -> Option<&Layer> { @@ -590,6 +595,38 @@ pub enum FootageKind { Audio, /// A 3D model (glTF 2.0 / OBJ), placed as a model layer. Model, + /// A data file (JSON, CSV, TSV) for data-driven animation: read by expressions through + /// `footage(name).sourceData` / `sourceText` / `dataValue`; it has no pixels. + Data, +} + +/// Interpret Footage ▸ Fields and Pulldown ▸ Separate Fields. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum FieldOrder { + /// Progressive frames. + #[default] + Off, + /// Interlaced, upper (odd) field first: each field becomes a frame at twice the rate. + UpperFirst, + LowerFirst, +} + +impl FieldOrder { + pub fn label(self) -> &'static str { + match self { + FieldOrder::Off => "Off", + FieldOrder::UpperFirst => "Upper Field First", + FieldOrder::LowerFirst => "Lower Field First", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "off" | "none" | "progressive" => Some(FieldOrder::Off), + "upper" | "upperfirst" | "upperfieldfirst" => Some(FieldOrder::UpperFirst), + "lower" | "lowerfirst" | "lowerfieldfirst" => Some(FieldOrder::LowerFirst), + _ => None, + } + } } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -629,6 +666,77 @@ pub struct Footage { /// Colour profile (from the file's metadata, or Interpret Footage). `None` = sRGB. #[serde(default, skip_serializing_if = "Option::is_none")] pub color_profile: Option, + /// Interpret Footage ▸ Fields and Pulldown ▸ Separate Fields. + #[serde(default, skip_serializing_if = "is_default")] + pub fields: FieldOrder, + /// Interpret Footage ▸ Alpha ▸ Invert Alpha. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub invert_alpha: bool, + /// Interpret Footage ▸ Color ▸ Interpret As Linear Light: the file's values are linear. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub linear_light: bool, + /// Data footage ([`FootageKind::Data`]): the file's text (JSON, CSV or TSV), kept in the + /// project so expressions read it everywhere (and on the web). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, +} + +fn is_default(v: &T) -> bool { + *v == T::default() +} + +impl Default for Footage { + fn default() -> Self { + Footage { + path: String::new(), + kind: FootageKind::Still, + width: 0, + height: 0, + pixel_aspect: 1.0, + frame_rate: FrameRate::FPS_30, + native_rate: None, + duration: Tick::ZERO, + has_video: false, + has_audio: false, + alpha: AlphaMode::Straight, + premul_color: [0.0; 3], + loop_count: 1, + codec: String::new(), + missing: false, + sequence: vec![], + color_profile: None, + fields: FieldOrder::Off, + invert_alpha: false, + linear_light: false, + data: None, + } + } +} + +impl Footage { + /// The data format of data footage: `json`, `csv` or `tsv` (from the file extension). + pub fn data_format(&self) -> Option<&'static str> { + if self.kind != FootageKind::Data { + return None; + } + let ext = std::path::Path::new(&self.path).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + Some(match ext.as_str() { + "csv" => "csv", + "tsv" | "tab" => "tsv", + _ => "json", + }) + } +} + +/// A proxy: a stand-in file used instead of a footage item or composition (File ▸ Set Proxy / +/// Create Proxy), when its Use Proxy switch is on and the render's Proxy Use allows it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Proxy { + /// The proxy's footage (path, size, rate…; File ▸ Interpret Footage ▸ Proxy edits it). + pub footage: Footage, + /// Use Proxy (the Project panel's proxy indicator: filled = in use, hollow = set but off). + #[serde(default = "yes")] + pub enabled: bool, } fn one() -> u32 { @@ -663,6 +771,9 @@ pub struct Item { #[serde(default)] pub parent: Option, pub kind: ItemKind, + /// The item's proxy (footage and compositions). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy: Option>, } impl Item { @@ -676,6 +787,7 @@ impl Item { FootageKind::Sequence => "Image Sequence", FootageKind::Audio => "Audio", FootageKind::Model => "3D Model", + FootageKind::Data => "Data", }, ItemKind::Solid(_) => "Solid", } @@ -753,7 +865,7 @@ impl Project { } pub fn add_item(&mut self, name: &str, label: Label, parent: Option, kind: ItemKind) -> ItemId { let id = ItemId(self.alloc()); - self.items.insert(id, Item { id, name: name.into(), label, comment: String::new(), parent, kind }); + self.items.insert(id, Item { id, name: name.into(), label, comment: String::new(), parent, kind, proxy: None }); id } /// Children of a folder (None = root), folders first then by name. diff --git a/crates/project/src/props.rs b/crates/project/src/props.rs index c1a3e3b93..c81a96b05 100644 --- a/crates/project/src/props.rs +++ b/crates/project/src/props.rs @@ -181,6 +181,13 @@ pub enum GroupKind { }, /// A motion tracker (Motion Trackers ▸ Tracker n) with its Tracker panel settings. Tracker { settings: Box }, + /// The Essential Properties of a precomp layer (master properties): one child per control + /// of the nested comp's Essential Graphics; `overridden` lists the children (uids) whose + /// values this instance overrides. + Essential { + #[serde(default)] + overridden: Vec, + }, } /// Mask motion blur (sub-frame samples of an animated mask path). diff --git a/crates/render/src/audio.rs b/crates/render/src/audio.rs index 4884411f6..98bb96668 100644 --- a/crates/render/src/audio.rs +++ b/crates/render/src/audio.rs @@ -330,6 +330,7 @@ mod tests { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = p.add_item("x.wav", effectcraft_color::Label::SeaFoam, None, ItemKind::Footage(f)); let comp = Comp::new(64, 64, FrameRate::new(25, 1), Tick::from_seconds_f64(2.0)); diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index eefe77c3f..7bf67774f 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -100,6 +100,15 @@ impl<'a> EvalCtx<'a> { if let Some(tr) = layer.props.get("timeRemap") { return Tick::from_seconds_f64(self.value(layer, tr).as_f64()); } + // Responsive Design — Time: a time-stretched precomp keeps its protected regions at + // their original speed. + if let LayerSource::Comp { item } = layer.source + && (layer.stretch - 100.0).abs() > 1e-9 + && let Some(nc) = self.project.comp(item) + && let Some(t) = responsive_source_time(nc, layer.stretch / 100.0, (self.time - layer.start_time).seconds()) + { + return t; + } layer.layer_time(self.time) } pub fn layer(&self, id: LayerId) -> Option<&'a Layer> { @@ -185,6 +194,73 @@ impl<'a> EvalCtx<'a> { } } +/// The protected regions of a comp (Responsive Design — Time markers), merged, sorted and +/// clamped to the comp, in seconds. +pub fn protected_regions(comp: &Comp) -> Vec<(f64, f64)> { + let d = comp.duration.seconds(); + let mut v: Vec<(f64, f64)> = comp + .markers + .iter() + .filter(|m| m.protected && m.duration > Tick::ZERO) + .map(|m| (m.time.seconds().clamp(0.0, d), (m.time + m.duration).seconds().clamp(0.0, d))) + .filter(|(a, b)| b > a) + .collect(); + v.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut out: Vec<(f64, f64)> = vec![]; + for (a, b) in v { + match out.last_mut() { + Some(last) if a <= last.1 => last.1 = last.1.max(b), + _ => out.push((a, b)), + } + } + out +} + +/// Source time of a precomp time-stretched by `stretch` (1 = 100%), `elapsed` comp seconds +/// after the layer's start, with Responsive Design — Time: protected regions play at their +/// original speed and the unprotected parts absorb the whole stretch. `None` when the comp has +/// no protected regions or the stretch can't be absorbed (reversed, or shorter than the +/// protected regions), so the plain stretch applies. +pub fn responsive_source_time(comp: &Comp, stretch: f64, elapsed: f64) -> Option { + let regions = protected_regions(comp); + if regions.is_empty() || stretch <= 0.0 { + return None; + } + let d = comp.duration.seconds(); + let lp: f64 = regions.iter().map(|(a, b)| b - a).sum(); + let lu = d - lp; + let k = if lu > 1e-9 { (d * stretch - lp) / lu } else { 1.0 }; + if k <= 1e-9 { + return None; + } + if elapsed < 0.0 { + return Some(Tick::from_seconds_f64(elapsed / k)); + } + // Segments in source order: (start, end, protected). + let mut segs = vec![]; + let mut at = 0.0; + for (a, b) in ®ions { + if *a > at { + segs.push((at, *a, false)); + } + segs.push((*a, *b, true)); + at = *b; + } + if d > at { + segs.push((at, d, false)); + } + let mut pos = 0.0; + for (a, b, prot) in segs { + let speed = if prot { 1.0 } else { k }; + let len = (b - a) * speed; + if elapsed < pos + len { + return Some(Tick::from_seconds_f64(a + (elapsed - pos) / speed)); + } + pos += len; + } + Some(Tick::from_seconds_f64(d + (elapsed - pos) / k)) +} + /// Size of a layer's source in layer pixels. pub fn source_size(project: &Project, layer: &Layer) -> (u32, u32) { match &layer.source { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 64e2684b3..5e7425bb7 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -154,6 +154,22 @@ impl EffectHost for FxHost<'_, '_, '_> { } } +/// The Essential Properties overrides of a precomp layer, evaluated at the context time. +pub fn essential_overrides(ctx: &EvalCtx, layer: &Layer) -> Vec { + use effectcraft_project::essential; + let over = essential::overridden(layer); + let Some(g) = essential::group(layer).filter(|_| !over.is_empty()) else { return vec![] }; + let mut out = vec![]; + g.walk("", &mut |_, p| { + if over.contains(&p.uid) + && let Some(control) = essential::control_of(&p.match_id) + { + out.push(essential::Override { control, value: ctx.value(layer, p) }); + } + }); + out +} + /// No footage available (renders footage layers as transparent). pub struct NoFootage; impl FootageSource for NoFootage { @@ -332,6 +348,10 @@ impl<'a> Renderer<'a> { if styles::active(ctx, layer) { return None; } + // Instance overrides (Essential Properties) need the nested comp rendered on its own. + if !effectcraft_project::essential::overridden(layer).is_empty() { + return None; + } Some(*item) } @@ -655,7 +675,15 @@ impl<'a> Renderer<'a> { if self.project.comp_contains(*item, ctx.comp_id) { return None; } - let sub = self.nested(); + // Essential Properties overrides render the nested comp with this instance's + // values. + let ov = essential_overrides(ctx, layer); + let tmp = if ov.is_empty() { None } else { effectcraft_project::essential::with_overrides(self.project, *item, &ov) }; + let base = self.nested(); + let sub = match &tmp { + Some(p) => Renderer { project: p, ..base }, + None => base, + }; let lt = ctx.source_time(layer); // Frame blending between the nested comp's frames (time-stretched/remapped). let mode = frame_blend_mode(ctx, layer); diff --git a/crates/render/src/tests_audio_fx.rs b/crates/render/src/tests_audio_fx.rs index a249955f6..4cb4e5e73 100644 --- a/crates/render/src/tests_audio_fx.rs +++ b/crates/render/src/tests_audio_fx.rs @@ -51,6 +51,7 @@ fn audio_footage() -> Footage { missing: false, sequence: vec![], color_profile: None, + ..Default::default() } } diff --git a/crates/render/src/tests_color.rs b/crates/render/src/tests_color.rs index 87f9dd1ae..0430a72b6 100644 --- a/crates/render/src/tests_color.rs +++ b/crates/render/src/tests_color.rs @@ -172,6 +172,7 @@ fn footage(profile: Option) -> Footage { missing: false, sequence: vec![], color_profile: profile, + ..Default::default() } } diff --git a/crates/render/src/tests_frame_blend.rs b/crates/render/src/tests_frame_blend.rs index c172b81c8..6cfbdb11b 100644 --- a/crates/render/src/tests_frame_blend.rs +++ b/crates/render/src/tests_frame_blend.rs @@ -80,6 +80,7 @@ fn setup(mode: FrameBlend, size: (u32, u32)) -> (Project, ItemId) { missing: false, sequence: vec![], color_profile: None, + ..Default::default() }; let fid = p.add_item("clip", Label::Aqua, None, ItemKind::Footage(f)); let mut l = build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, size, None); From 6d0d7de92220e705f5c1150d07ec575ddb8b1b2b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 22:58:04 -0400 Subject: [PATCH 3/9] ANM-5: sampleImage, data footage (JSON/CSV/TSV) with footage().sourceData, thisProject, error list, Expression Language menu sampleImage renders the sampled layer through the evaluating renderer's footage source (EvalCtx::footage), cached per thread and frame. File > Import reads .json/.csv/.tsv as data footage kept in the project. expr.errors collects failing expressions for the error bar; expr.languageMenu lists the categorised Expression Language menu. Marker keys expose protectedRegion and cue points; colorDepth, timeToFeetAndFrames, comp.activeCamera. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/expr_tools.rs | 375 +++++++++++++++++++ crates/engine/src/commands/file.rs | 32 +- crates/engine/src/commands/layer.rs | 6 +- crates/engine/src/commands/layer_time.rs | 2 +- crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/prop.rs | 3 +- crates/engine/src/commands/puppet.rs | 2 +- crates/engine/src/commands/three_d.rs | 4 +- crates/engine/src/tracking.rs | 2 +- crates/expr/src/host.rs | 130 ++++++- crates/expr/src/lib.rs | 3 +- crates/expr/src/prelude.js | 100 ++++- crates/expr/src/runtime.rs | 13 + crates/expr/src/sample.rs | 119 ++++++ crates/expr/src/tests.rs | 78 +++- crates/gpu/src/tests_adv3d.rs | 2 +- crates/host/src/lib.rs | 78 +++- crates/render/src/eval.rs | 5 +- crates/render/src/lib.rs | 2 +- crates/render/src/three_d/adv/tests.rs | 2 +- crates/ui-egui/src/panels/effect_controls.rs | 2 +- crates/ui-egui/src/panels/layer_panel.rs | 2 +- crates/ui-egui/src/panels/properties.rs | 2 +- crates/ui-egui/src/panels/text_panels.rs | 4 +- crates/ui-egui/src/panels/timeline.rs | 2 +- crates/ui-egui/src/panels/viewer.rs | 4 +- 26 files changed, 951 insertions(+), 25 deletions(-) create mode 100644 crates/engine/src/commands/expr_tools.rs create mode 100644 crates/expr/src/sample.rs diff --git a/crates/engine/src/commands/expr_tools.rs b/crates/engine/src/commands/expr_tools.rs new file mode 100644 index 000000000..868be63a7 --- /dev/null +++ b/crates/engine/src/commands/expr_tools.rs @@ -0,0 +1,375 @@ +//! Expression tools: the error list behind the expression error bar, and the Expression +//! Language menu (the ⓕ button next to an expression), written from the public Expression +//! Language Reference. + +use effectcraft_project::{ItemId, Layer, Property}; +use effectcraft_render::EvalCtx; +use effectcraft_time::Tick; +use serde::Serialize; +use serde_json::{Value, json}; + +use super::{CommandSpec, comp_id, f_p}; +use crate::{EngineError, Result, Session, query}; + +/// One failing expression. +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct ExprError { + pub comp: u64, + pub layer: u64, + #[serde(rename = "layerIndex")] + pub layer_index: usize, + #[serde(rename = "layerName")] + pub layer_name: String, + pub prop: u64, + /// Match path (`transform/opacity`). + pub path: String, + /// Property name path as shown in the timeline (`Transform/Opacity`). + pub name: String, + pub message: String, + /// The expression is disabled (it failed to compile). + pub disabled: bool, +} + +impl ExprError { + /// The error bar's text: `Error at line 1 in property 'Opacity' of layer 1 ('Solid') in comp 'Main'. …` + pub fn describe(&self, comp_name: &str) -> String { + let prop = self.name.rsplit('/').next().unwrap_or(&self.name); + format!("{} — property '{}' of layer {} ('{}') in comp '{}'", self.message, prop, self.layer_index, self.layer_name, comp_name) + } +} + +/// Every failing expression of comp `cid` at comp time `t`: evaluation errors of enabled +/// expressions and the syntax errors that disabled others. +pub fn errors(s: &Session, cid: ItemId, t: Tick) -> Vec { + let Some(comp) = s.project.comp(cid) else { return vec![] }; + let ctx = EvalCtx { footage: Some(s.footage.as_ref()), expr: s.expr.as_deref(), ..EvalCtx::new(&s.project, cid, comp, t) }; + let mut out = vec![]; + for (i, l) in comp.layers.iter().enumerate() { + let mut props: Vec<&Property> = vec![]; + l.props.walk("", &mut |_, p| { + if p.expr.as_ref().is_some_and(|e| !e.text.trim().is_empty()) { + props.push(p); + } + }); + for p in props { + let e = p.expr.as_ref().expect("filtered"); + let msg = if e.enabled { + match &ctx.expr { + Some(h) => h.eval(&ctx, l, p, &p.value_at(l.layer_time(t))).err(), + None => None, + } + } else { + s.expr_check.and_then(|check| check(&e.text).err()) + }; + if let Some(message) = msg { + out.push(error_of(cid, i, l, p, message, !e.enabled)); + } + } + } + out +} + +fn error_of(cid: ItemId, i: usize, l: &Layer, p: &Property, message: String, disabled: bool) -> ExprError { + ExprError { + comp: cid.0, + layer: l.id.0, + layer_index: i + 1, + layer_name: l.name.clone(), + prop: p.uid, + path: super::match_path_of(&l.props, p.uid).unwrap_or_default(), + name: l.props.name_path_of(p.uid).unwrap_or_else(|| p.name.clone()), + message, + disabled, + } +} + +fn errors_cmd(s: &mut Session, p: &Value) -> Result { + let cid = comp_id(s, p)?; + let t = f_p(p, "time").map(Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let name = s.project.item(cid).map(|i| i.name.clone()).ok_or(EngineError::NoComp)?; + let list = errors(s, cid, t); + let text: Vec = list.iter().map(|e| e.describe(&name)).collect(); + Ok(json!({"comp": cid.0, "time": t.seconds(), "count": list.len(), "errors": list, "text": text})) +} + +/// The Expression Language menu: (category, [(label, inserted text)]). Submenus are nested +/// with `▸` in the category name. +pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> { + vec![ + ( + "Global", + vec![ + ("comp(name)", "comp(\"name\")"), + ("footage(name)", "footage(\"name\")"), + ("thisComp", "thisComp"), + ("thisLayer", "thisLayer"), + ("thisProperty", "thisProperty"), + ("thisProject", "thisProject"), + ("time", "time"), + ("colorDepth", "colorDepth"), + ("posterizeTime(framesPerSecond)", "posterizeTime(framesPerSecond)"), + ("value", "value"), + ], + ), + ( + "Vector Math", + vec![ + ("add(vec1, vec2)", "add(vec1, vec2)"), + ("sub(vec1, vec2)", "sub(vec1, vec2)"), + ("mul(vec, amount)", "mul(vec, amount)"), + ("div(vec, amount)", "div(vec, amount)"), + ("clamp(value, limit1, limit2)", "clamp(value, limit1, limit2)"), + ("dot(vec1, vec2)", "dot(vec1, vec2)"), + ("cross(vec1, vec2)", "cross(vec1, vec2)"), + ("normalize(vec)", "normalize(vec)"), + ("length(vec)", "length(vec)"), + ("length(point1, point2)", "length(point1, point2)"), + ("lookAt(fromPoint, atPoint)", "lookAt(fromPoint, atPoint)"), + ], + ), + ( + "Random Numbers", + vec![ + ("seedRandom(offset, timeless=false)", "seedRandom(offset, timeless=false)"), + ("random()", "random()"), + ("random(maxValOrArray)", "random(maxValOrArray)"), + ("random(minValOrArray, maxValOrArray)", "random(minValOrArray, maxValOrArray)"), + ("gaussRandom()", "gaussRandom()"), + ("gaussRandom(maxValOrArray)", "gaussRandom(maxValOrArray)"), + ("gaussRandom(minValOrArray, maxValOrArray)", "gaussRandom(minValOrArray, maxValOrArray)"), + ("noise(valOrArray)", "noise(valOrArray)"), + ], + ), + ( + "Interpolation", + vec![ + ("linear(t, value1, value2)", "linear(t, value1, value2)"), + ("linear(t, tMin, tMax, value1, value2)", "linear(t, tMin, tMax, value1, value2)"), + ("ease(t, value1, value2)", "ease(t, value1, value2)"), + ("ease(t, tMin, tMax, value1, value2)", "ease(t, tMin, tMax, value1, value2)"), + ("easeIn(t, value1, value2)", "easeIn(t, value1, value2)"), + ("easeIn(t, tMin, tMax, value1, value2)", "easeIn(t, tMin, tMax, value1, value2)"), + ("easeOut(t, value1, value2)", "easeOut(t, value1, value2)"), + ("easeOut(t, tMin, tMax, value1, value2)", "easeOut(t, tMin, tMax, value1, value2)"), + ], + ), + ( + "Color Conversion", + vec![ + ("rgbToHsl(rgbaArray)", "rgbToHsl(rgbaArray)"), + ("hslToRgb(hslaArray)", "hslToRgb(hslaArray)"), + ("hexToRgb(hexString)", "hexToRgb(hexString)"), + ], + ), + ("Other Math", vec![("degreesToRadians(degrees)", "degreesToRadians(degrees)"), ("radiansToDegrees(radians)", "radiansToDegrees(radians)")]), + ( + "JavaScript Math", + vec![ + ("Math.cos(value)", "Math.cos(value)"), + ("Math.sin(value)", "Math.sin(value)"), + ("Math.atan2(y, x)", "Math.atan2(y, x)"), + ("Math.sqrt(value)", "Math.sqrt(value)"), + ("Math.abs(value)", "Math.abs(value)"), + ("Math.round(value)", "Math.round(value)"), + ("Math.floor(value)", "Math.floor(value)"), + ("Math.min(value1, value2)", "Math.min(value1, value2)"), + ("Math.max(value1, value2)", "Math.max(value1, value2)"), + ("Math.PI", "Math.PI"), + ], + ), + ( + "Comp", + vec![ + ("layer(index)", "layer(index)"), + ("layer(name)", "layer(\"name\")"), + ("layer(otherLayer, relIndex)", "layer(otherLayer, relIndex)"), + ("marker", "marker"), + ("numLayers", "numLayers"), + ("activeCamera", "activeCamera"), + ("width", "width"), + ("height", "height"), + ("duration", "duration"), + ("frameDuration", "frameDuration"), + ("pixelAspect", "pixelAspect"), + ("name", "name"), + ("bgColor", "bgColor"), + ("displayStartTime", "displayStartTime"), + ], + ), + ( + "Footage", + vec![ + ("width", "width"), + ("height", "height"), + ("duration", "duration"), + ("frameDuration", "frameDuration"), + ("pixelAspect", "pixelAspect"), + ("name", "name"), + ("sourceText", "sourceText"), + ("sourceData", "sourceData"), + ("dataValue(dataPath)", "dataValue(dataPath)"), + ("dataKeyCount", "dataKeyCount"), + ], + ), + ( + "Layer ▸ Sub-objects", + vec![ + ("source", "source"), + ("effect(name)", "effect(\"name\")"), + ("effect(index)", "effect(index)"), + ("mask(name)", "mask(\"name\")"), + ("mask(index)", "mask(index)"), + ], + ), + ( + "Layer ▸ General", + vec![ + ("width", "width"), + ("height", "height"), + ("index", "index"), + ("parent", "parent"), + ("hasParent", "hasParent"), + ("inPoint", "inPoint"), + ("outPoint", "outPoint"), + ("startTime", "startTime"), + ("hasVideo", "hasVideo"), + ("enabled", "enabled"), + ("active", "active"), + ], + ), + ( + "Layer ▸ Properties", + vec![ + ("anchorPoint", "anchorPoint"), + ("position", "position"), + ("scale", "scale"), + ("rotation", "rotation"), + ("opacity", "opacity"), + ("marker", "marker"), + ("name", "name"), + ], + ), + ( + "Layer ▸ Space Transforms", + vec![ + ("toComp(point, t = time)", "toComp(point, t = time)"), + ("fromComp(point, t = time)", "fromComp(point, t = time)"), + ("toWorld(point, t = time)", "toWorld(point, t = time)"), + ("fromWorld(point, t = time)", "fromWorld(point, t = time)"), + ("toCompVec(vec, t = time)", "toCompVec(vec, t = time)"), + ("fromCompVec(vec, t = time)", "fromCompVec(vec, t = time)"), + ("toWorldVec(vec, t = time)", "toWorldVec(vec, t = time)"), + ("fromWorldVec(vec, t = time)", "fromWorldVec(vec, t = time)"), + ], + ), + ( + "Layer ▸ Other", + vec![ + ("sourceRectAtTime(t = time, includeExtents = false)", "sourceRectAtTime(t = time, includeExtents = false)"), + ("sampleImage(point, radius = [.5, .5], postEffect = true, t = time)", "sampleImage(point, radius = [.5, .5], postEffect = true, t = time)"), + ], + ), + ( + "Property", + vec![ + ("value", "value"), + ("valueAtTime(t)", "valueAtTime(t)"), + ("velocity", "velocity"), + ("velocityAtTime(t)", "velocityAtTime(t)"), + ("speed", "speed"), + ("speedAtTime(t)", "speedAtTime(t)"), + ("wiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)", "wiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)"), + ("temporalWiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)", "temporalWiggle(freq, amp, octaves = 1, amp_mult = .5, t = time)"), + ("smooth(width = .2, samples = 5, t = time)", "smooth(width = .2, samples = 5, t = time)"), + ("loopIn(type = \"cycle\", numKeyframes = 0)", "loopIn(type = \"cycle\", numKeyframes = 0)"), + ("loopOut(type = \"cycle\", numKeyframes = 0)", "loopOut(type = \"cycle\", numKeyframes = 0)"), + ("loopInDuration(type = \"cycle\", duration = 0)", "loopInDuration(type = \"cycle\", duration = 0)"), + ("loopOutDuration(type = \"cycle\", duration = 0)", "loopOutDuration(type = \"cycle\", duration = 0)"), + ("key(index)", "key(index)"), + ("key(markerName)", "key(markerName)"), + ("nearestKey(t)", "nearestKey(t)"), + ("numKeys", "numKeys"), + ("name", "name"), + ("propertyIndex", "propertyIndex"), + ], + ), + ( + "Path Property", + vec![ + ("points(t = time)", "points(t = time)"), + ("inTangents(t = time)", "inTangents(t = time)"), + ("outTangents(t = time)", "outTangents(t = time)"), + ("isClosed()", "isClosed()"), + ( + "createPath(points, inTangents = [], outTangents = [], is_closed = true)", + "createPath(points, inTangents = [], outTangents = [], is_closed = true)", + ), + ], + ), + ( + "Text ▸ Style", + vec![ + ("text.sourceText.style", "text.sourceText.style"), + ("getStyleAt(charIndex, t = time)", "getStyleAt(charIndex, t = time)"), + ("createStyle()", "createStyle()"), + ("setFontSize(value)", "setFontSize(value)"), + ("setFillColor(value)", "setFillColor(value)"), + ("setText(value)", "setText(value)"), + ], + ), + ("Text ▸ Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), + ("Key", vec![("value", "value"), ("time", "time"), ("index", "index")]), + ( + "Marker Key", + vec![ + ("duration", "duration"), + ("comment", "comment"), + ("chapter", "chapter"), + ("url", "url"), + ("frameTarget", "frameTarget"), + ("eventCuePoint", "eventCuePoint"), + ("cuePointName", "cuePointName"), + ("parameters", "parameters"), + ("protectedRegion", "protectedRegion"), + ], + ), + ( + "Time Conversion", + vec![ + ( + "timeToFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + "timeToFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + ), + ("framesToTime(frames, fps = 1.0 / thisComp.frameDuration)", "framesToTime(frames, fps = 1.0 / thisComp.frameDuration)"), + ( + "timeToTimecode(t = time + thisComp.displayStartTime, timecodeBase = 30, isDuration = false)", + "timeToTimecode(t = time + thisComp.displayStartTime, timecodeBase = 30, isDuration = false)", + ), + ( + "timeToFeetAndFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, framesPerFoot = 16, isDuration = false)", + "timeToFeetAndFrames(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, framesPerFoot = 16, isDuration = false)", + ), + ( + "timeToCurrentFormat(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + "timeToCurrentFormat(t = time + thisComp.displayStartTime, fps = 1.0 / thisComp.frameDuration, isDuration = false)", + ), + ], + ), + ] +} + +fn menu_cmd(_s: &mut Session, _p: &Value) -> Result { + Ok(json!( + language_menu() + .into_iter() + .map(|(c, items)| json!({"category": c, "items": items.into_iter().map(|(l, t)| json!({"label": l, "text": t})).collect::>()})) + .collect::>() + )) +} + +pub fn specs() -> Vec { + vec![ + query!("expr.errors", "Expression errors (the error bar)", "{comp?, time? (s)}", errors_cmd), + query!("expr.languageMenu", "Expression Language menu", "{}", menu_cmd), + ] +} diff --git a/crates/engine/src/commands/file.rs b/crates/engine/src/commands/file.rs index 9e81db54d..972e00eac 100644 --- a/crates/engine/src/commands/file.rs +++ b/crates/engine/src/commands/file.rs @@ -84,12 +84,23 @@ fn import(s: &mut Session, p: &Value) -> Result { Some(Value::String(x)) => vec![x.clone()], _ => return Err(bad("file.import", "missing `paths`")), }; - let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; let mut ids = vec![]; let mut errors = vec![]; let mut probed = vec![]; let seq_rate = effectcraft_time::FrameRate::from_f64(s.prefs.import.sequence_fps); for path in &paths { + // Data files (JSON, CSV, TSV) for data-driven animation: kept as text in the project. + if let Some(f) = data_footage(s, path) { + match f { + Ok(f) => probed.push((path.clone(), f)), + Err(e) => errors.push(format!("{path}: {e}")), + } + continue; + } + let Some(importer) = s.importer.clone() else { + errors.push(format!("{path}: media import is not available in this build")); + continue; + }; match importer.probe(path) { Ok(mut f) => { // Settings ▸ Import ▸ Sequence Footage frames per second. @@ -122,6 +133,25 @@ fn import(s: &mut Session, p: &Value) -> Result { Ok(json!({"items": ids, "errors": errors})) } +/// Extensions imported as data footage. +pub const DATA_EXTENSIONS: &[&str] = &["json", "csv", "tsv"]; + +/// A data footage item for `path` (JSON / CSV / TSV), `None` for other files. +pub(crate) fn data_footage(s: &Session, path: &str) -> Option> { + let ext = std::path::Path::new(path).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase(); + if !DATA_EXTENSIONS.contains(&ext.as_str()) { + return None; + } + Some((|| { + let bytes = s.services.read_file(path).map_err(|e| e.to_string())?; + let text = String::from_utf8(bytes).map_err(|_| "data files must be UTF-8 text".to_string())?; + if ext == "json" { + serde_json::from_str::(text.trim_start_matches('\u{feff}')).map_err(|e| format!("invalid JSON: {e}"))?; + } + Ok(effectcraft_project::Footage { path: path.to_string(), kind: FootageKind::Data, codec: ext.to_uppercase(), data: Some(text), ..Default::default() }) + })()) +} + fn project_settings(s: &mut Session, p: &Value) -> Result { s.edit("Project Settings", None, |proj, _| { if let Some(b) = str_p(p, "bitDepth") { diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index ac7282001..e57b1c0a6 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -239,6 +239,9 @@ fn add_item(s: &mut Session, p: &Value) -> Result { let d = (s.prefs.import.still_footage == "seconds").then(|| Tick::from_seconds_f64(s.prefs.import.still_seconds)); (LayerSource::Footage { item }, (f.width, f.height), d) } + ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Data => { + return Err(bad("layer.addItem", "data files can't be layers; read them in expressions with footage(\"name\").sourceData")); + } ItemKind::Footage(f) if f.kind == effectcraft_project::FootageKind::Model => { return super::model3d::new_model(s, &serde_json::json!({"comp": cid.0, "item": item.0, "time": f_p(p, "time")})); } @@ -505,7 +508,8 @@ fn set_parent(s: &mut Session, p: &Value) -> Result { // Like AE's pick-whip, parenting keeps the layer where it is: its transform is re-expressed // in the new parent's space (Position, Rotation, Scale), unless `compensate: false`. let compensate = b_p(p, "compensate").unwrap_or(true); - let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: s.time(), expr: s.expr.as_deref() }; + let ectx = + effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: s.time(), expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; let space = |id: Option| -> effectcraft_geom::Mat3 { id.and_then(|i| comp.layer(i)).filter(|l| !l.is_3d()).map(|l| ectx.layer_to_comp(l).0).unwrap_or(effectcraft_geom::Mat3::IDENTITY) }; diff --git a/crates/engine/src/commands/layer_time.rs b/crates/engine/src/commands/layer_time.rs index d8090352f..7a0eb86b0 100644 --- a/crates/engine/src/commands/layer_time.rs +++ b/crates/engine/src/commands/layer_time.rs @@ -126,7 +126,7 @@ fn freeze(s: &mut Session, p: &Value, last: bool) -> Result { return Err(bad("layer.freezeFrame", "freezing needs a footage or composition layer")); } // What each layer shows now (honouring an existing remap) — evaluated before the edit. - let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: cti, expr: s.expr.as_deref() }; + let ectx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp, time: cti, expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; let now: Vec<(effectcraft_project::LayerId, f64)> = comp .layers .iter() diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 7a053c774..e1b3f15f7 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -9,6 +9,7 @@ mod comp_more; mod edit; mod effect; pub mod essential; +pub mod expr_tools; mod file; mod file_more; mod frontend; @@ -138,6 +139,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(liquify::specs()); v.extend(settings::specs()); v.extend(essential::specs()); + v.extend(expr_tools::specs()); v.extend(stubs::specs()); v }) diff --git a/crates/engine/src/commands/prop.rs b/crates/engine/src/commands/prop.rs index 05dc37fad..2f34898c6 100644 --- a/crates/engine/src/commands/prop.rs +++ b/crates/engine/src/commands/prop.rs @@ -557,7 +557,8 @@ fn convert_expr_to_keys(s: &mut Session, p: &Value) -> Result { let fd = comp.frame_duration(); let mut t = layer.in_point; while t < layer.out_point { - let ctx = effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp: &comp, time: t, expr: s.expr.as_deref() }; + let ctx = + effectcraft_render::EvalCtx { project: &s.project, comp_id: cid, comp: &comp, time: t, expr: s.expr.as_deref(), footage: Some(s.footage.as_ref()) }; keys.push(Keyframe::new(layer.layer_time(t), ctx.value(&layer, &pr))); t += fd; } diff --git a/crates/engine/src/commands/puppet.rs b/crates/engine/src/commands/puppet.rs index 00117dadd..4ada1dcd0 100644 --- a/crates/engine/src/commands/puppet.rs +++ b/crates/engine/src/commands/puppet.rs @@ -29,7 +29,7 @@ pub fn puppet_eval(s: &Session, cid: ItemId, lid: LayerId, fx_uid: Uid, t: Tick) let fx = layer.effects()?; let index = fx.groups().position(|g| g.uid == fx_uid)?; let g = fx.groups().nth(index)?; - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let mut r = Renderer::new(&s.project, &*s.footage, RenderOpts::default()); r.expr = s.expr.as_deref(); r.cache = Some(&s.layer_cache); diff --git a/crates/engine/src/commands/three_d.rs b/crates/engine/src/commands/three_d.rs index aea321059..55060d17a 100644 --- a/crates/engine/src/commands/three_d.rs +++ b/crates/engine/src/commands/three_d.rs @@ -393,7 +393,7 @@ fn get_3d(s: &mut Session, p: &Value) -> Result { let (w, h) = (comp.width as f64, comp.height as f64); let vs = s.state.views3d.get(&cid).cloned().unwrap_or_default(); let t = s.state.times.get(&cid).copied().unwrap_or(Tick::ZERO); - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let cam = vs.override_camera(w, h).unwrap_or_else(|| camera::active_camera(&ctx)); let f = cam.forward(); let lights: Vec = comp @@ -535,7 +535,7 @@ fn auto_orient(s: &mut Session, p: &Value) -> Result { /// through each layer's world matrix; cameras and lights contribute their position). fn layer_points(s: &Session, cid: ItemId, ids: &[LayerId], t: Tick) -> Vec { let Some(comp) = s.project.comp(cid) else { return vec![] }; - let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref() }; + let ctx = EvalCtx { project: &s.project, comp_id: cid, comp, time: t, expr: s.expr.as_deref(), footage: None }; let mut out = vec![]; for l in comp.layers.iter().filter(|l| ids.contains(&l.id)) { let m = ctx.world_matrix(l); diff --git a/crates/engine/src/tracking.rs b/crates/engine/src/tracking.rs index c406edb91..6a22fd81c 100644 --- a/crates/engine/src/tracking.rs +++ b/crates/engine/src/tracking.rs @@ -206,7 +206,7 @@ pub(crate) fn source_frame( t: Tick, expr: Option<&dyn ExprHost>, ) -> Option<(Arc, [f64; 2])> { - let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr }; + let ctx = EvalCtx { project, comp_id: cid, comp, time: t, expr, footage: None }; if let effectcraft_project::LayerSource::Footage { item } = &layer.source && let Some(effectcraft_project::ItemKind::Footage(f)) = project.item(*item).map(|i| &i.kind) && f.has_video diff --git a/crates/expr/src/host.rs b/crates/expr/src/host.rs index 4b92e224e..700937903 100644 --- a/crates/expr/src/host.rs +++ b/crates/expr/src/host.rs @@ -52,6 +52,93 @@ pub enum Req { /// A Source Text value as JSON `{doc, text, runs: [{len, style}], paras}` (styles as /// `layer.setText` keys), or null for other properties. Doc { comp: u64, layer: u64, path: String, t: Secs, pre: bool }, + /// `footage(name)`: `[name, width, height, duration, frameDuration, pixelAspect, kind, + /// sourceText|null, dataFormat|null]` or null. + Footage { name: String }, + /// A data footage item's parsed data as JSON `{data, rows, header}` (`rows`/`header` for + /// CSV/TSV), or null. + FootageData { name: String }, + /// `sampleImage`: the average straight `[r, g, b, a]` of the layer's pixels in the box + /// `point ± radius` (layer space) at comp time `t`, after masks and effects when `post`. + Sample { comp: u64, layer: u64, x: Secs, y: Secs, rx: Secs, ry: Secs, post: bool, t: Secs }, + /// `thisProject`: `[bitsPerChannel, linearBlending, workingSpace]`. + Project, +} + +/// Parse CSV/TSV text (RFC 4180 quoting) into rows of cells. +pub fn parse_delimited(text: &str, sep: char) -> Vec> { + let mut rows = vec![]; + let mut row = vec![]; + let mut cell = String::new(); + let mut quoted = false; + let mut chars = text.chars().peekable(); + while let Some(c) = chars.next() { + if quoted { + match c { + '"' if chars.peek() == Some(&'"') => { + chars.next(); + cell.push('"'); + } + '"' => quoted = false, + _ => cell.push(c), + } + continue; + } + match c { + '"' if cell.is_empty() => quoted = true, + c if c == sep => row.push(std::mem::take(&mut cell)), + '\r' => {} + '\n' => { + row.push(std::mem::take(&mut cell)); + rows.push(std::mem::take(&mut row)); + } + _ => cell.push(c), + } + } + if !cell.is_empty() || !row.is_empty() { + row.push(cell); + rows.push(row); + } + rows.retain(|r| !(r.len() == 1 && r[0].trim().is_empty())); + rows +} + +/// A CSV/TSV cell as JSON: numbers become numbers. +fn cell_json(c: &str) -> serde_json::Value { + let t = c.trim(); + match t.parse::() { + Ok(x) if x.is_finite() && !t.is_empty() => serde_json::json!(x), + _ => serde_json::json!(c), + } +} + +/// The data of a data footage item as `{data, rows, header}` JSON (see [`Req::FootageData`]). +pub fn footage_data_json(f: &effectcraft_project::Footage) -> Option { + let text = f.data.as_deref()?; + let text = text.strip_prefix('\u{feff}').unwrap_or(text); + Some(match f.data_format()? { + "json" => { + let v: serde_json::Value = serde_json::from_str(text).ok()?; + serde_json::json!({"data": v, "rows": null, "header": null}).to_string() + } + fmt => { + let rows = parse_delimited(text, if fmt == "tsv" { '\t' } else { ',' }); + let (header, body) = match rows.split_first() { + Some((h, b)) => (h.clone(), b.to_vec()), + None => (vec![], vec![]), + }; + let objects: Vec = body + .iter() + .map(|r| { + serde_json::Value::Object( + header.iter().enumerate().map(|(i, k)| (k.clone(), r.get(i).map(|c| cell_json(c)).unwrap_or(serde_json::Value::Null))).collect(), + ) + }) + .collect(); + let cells: Vec> = body.iter().map(|r| r.iter().map(|c| cell_json(c)).collect()).collect(); + serde_json::json!({"data": objects, "rows": cells, "header": header}).to_string() + } + }) } /// A text document as the style API's JSON. @@ -192,7 +279,7 @@ impl<'a> Resolver<'a> { } fn ctx_at(&self, c: u64, t: f64) -> Option> { let comp = self.comp(c)?; - Some(EvalCtx { project: self.ctx.project, comp_id: ItemId(c), comp, time: Tick::from_seconds_f64(t), expr: self.ctx.expr }) + Some(EvalCtx { project: self.ctx.project, comp_id: ItemId(c), comp, time: Tick::from_seconds_f64(t), expr: self.ctx.expr, footage: self.ctx.footage }) } fn prop(&self, c: u64, l: u64, path: &str) -> Option<(&'a Layer, &'a Property)> { let layer = self.layer(c, l)?; @@ -345,6 +432,36 @@ impl<'a> Resolver<'a> { }; Resp::nums(r) } + Req::Footage { name } => { + let it = project.items.values().find(|i| &i.name == name && matches!(i.kind, ItemKind::Footage(_)))?; + let ItemKind::Footage(f) = &it.kind else { return None }; + Resp::List(vec![ + Resp::Str(it.name.clone()), + Resp::Num(f.width as f64), + Resp::Num(f.height as f64), + Resp::Num(f.duration.seconds()), + Resp::Num(f.frame_rate.frame_duration().seconds()), + Resp::Num(f.pixel_aspect), + Resp::Str(it.type_name().into()), + f.data.clone().map_or(Resp::Null, Resp::Str), + f.data_format().map_or(Resp::Null, |x| Resp::Str(x.into())), + ]) + } + Req::FootageData { name } => { + let it = project.items.values().find(|i| &i.name == name && matches!(i.kind, ItemKind::Footage(_)))?; + let ItemKind::Footage(f) = &it.kind else { return None }; + Resp::Str(footage_data_json(f)?) + } + Req::Project => Resp::List(vec![ + Resp::Num(project.settings.bit_depth.bits() as f64), + Resp::Bool(project.settings.blend_linear || project.settings.linearize), + Resp::Str(project.settings.working_space.map(|w| format!("{w:?}")).unwrap_or_default()), + ]), + Req::Sample { comp, layer, x, y, rx, ry, post, t } => { + let l = self.layer(*comp, *layer)?; + let f = |b: &Secs| f64::from_bits(*b); + Resp::nums(crate::sample::sample_image(self.ctx, ItemId(*comp), l, [f(x), f(y)], [f(rx), f(ry)], *post, f(t))?.map(|v| v as f64)) + } Req::Markers { comp, layer } => { let marks: Vec<(f64, &effectcraft_project::Marker)> = match layer { Some(l) => { @@ -363,6 +480,17 @@ impl<'a> Resolver<'a> { Resp::Str(m.comment.clone()), Resp::Str(m.chapter.clone()), Resp::Str(m.url.clone()), + Resp::Str(m.frame_target.clone()), + Resp::Bool(m.cue_point.as_ref().is_some_and(|c| !c.navigation)), + Resp::Str(m.cue_point.as_ref().map(|c| c.name.clone()).unwrap_or_default()), + Resp::List( + m.cue_point + .iter() + .flat_map(|c| c.params.iter()) + .map(|(k, v)| Resp::List(vec![Resp::Str(k.clone()), Resp::Str(v.clone())])) + .collect(), + ), + Resp::Bool(m.protected), ]) }) .collect(), diff --git a/crates/expr/src/lib.rs b/crates/expr/src/lib.rs index 323e3c7a9..88c1b031b 100644 --- a/crates/expr/src/lib.rs +++ b/crates/expr/src/lib.rs @@ -28,6 +28,7 @@ pub mod host; mod noise; pub mod rewrite; mod runtime; +mod sample; use effectcraft_keyframe::Value; use effectcraft_project::{ItemId, Layer, LayerId, Project, Property}; @@ -112,7 +113,7 @@ fn evaluate_out( if depth >= MAX_DEPTH { return Err("Error: expressions reference each other too deeply (circular reference?)".into()); } - let ctx = EvalCtx { project: ctx.project, comp_id: ctx.comp_id, comp: ctx.comp, time: ctx.time, expr: Some(host) }; + let ctx = EvalCtx { project: ctx.project, comp_id: ctx.comp_id, comp: ctx.comp, time: ctx.time, expr: Some(host), footage: ctx.footage }; let comp = ctx.comp_id.0; let path = layer.props.path_of(prop.uid).unwrap_or_else(|| format!("@{}", prop.uid)); let resolver = Resolver { ctx, own: Own { comp, layer, prop, path: path.clone() } }; diff --git a/crates/expr/src/prelude.js b/crates/expr/src/prelude.js index 67ddf4585..1655f75d4 100644 --- a/crates/expr/src/prelude.js +++ b/crates/expr/src/prelude.js @@ -18,6 +18,7 @@ var selectorValue = [100, 100, 100]; // Request ops (must match runtime.rs). var __COMP = 0, __COMPNAME = 1, __LAYER = 2, __LINFO = 3, __CHILD = 4, __PINFO = 5, __VALUE = 6, __XFORM = 7, __RECT = 8, __MARKERS = 9, __DOC = 10; +var __FOOT = 11, __FDATA = 12, __SAMPLE = 13, __PROJ = 14; function __begin(t, v, c, l, path, uid, idx, fd, ti, tt, sv) { time = t; @@ -29,6 +30,7 @@ function __begin(t, v, c, l, path, uid, idx, fd, ti, tt, sv) { __rc = 0; __seed = 0; __timeless = false; + __projectInfo = null; } function __finish(r) { @@ -269,6 +271,18 @@ function timeToTimecode(t, base, isDuration) { return (neg ? '-' : '') + h + ':' + __pad(mm) + ':' + __pad(ss) + ':' + __pad(ff); } +function timeToFeetAndFrames(t, fps, framesPerFoot, isDuration) { + if (t === undefined) t = time + thisComp.displayStartTime; + if (fps === undefined) fps = 1 / __S.fd; + if (framesPerFoot === undefined) framesPerFoot = 16; + var neg = t < 0; + var f = Math.floor(Math.abs(t) * fps + 1e-6); + var feet = Math.floor(f / framesPerFoot), fr = f % framesPerFoot; + var s = String(fr); + while (s.length < 2) s = '0' + s; + return (neg ? '-' : '') + feet + '+' + s; +} + function timeToCurrentFormat(t, fps) { if (fps === undefined) fps = Math.round(1 / __S.fd); return timeToTimecode(t, fps); @@ -371,6 +385,14 @@ Comp.prototype = { get displayStartTime() { return this.__info[7]; }, get bgColor() { return this.__info[8]; }, get marker() { return new Markers(this.__c, -1); }, + // The topmost enabled camera active at the current time. + get activeCamera() { + for (var i = 1; i <= this.numLayers; i++) { + var l = this.layer(i); + if (l.__info[10] === 'Camera' && l.active) return l; + } + __err('Composition "' + this.name + '" has no active camera'); + }, layer: function (a, rel) { if (a !== null && typeof a === 'object' && a.__isLayer) return this.layer(a.index + (rel || 0)); var id = __h(__LAYER, this.__c, a); @@ -386,6 +408,68 @@ function comp(name) { return new Comp(id); } +// Footage (data-driven animation) and the project ------------------------------------------------ + +function Footage(info) { this.__i = info; this.__d = null; } +Footage.prototype = { + __isFootage: true, + get name() { return this.__i[0]; }, + get width() { return this.__i[1]; }, + get height() { return this.__i[2]; }, + get duration() { return this.__i[3]; }, + get frameDuration() { return this.__i[4]; }, + get pixelAspect() { return this.__i[5]; }, + get ntscDropFrame() { return false; }, + // The data file's text (JSON / CSV / TSV), or "" for media. + get sourceText() { return this.__i[7] === null ? '' : this.__i[7]; }, + get __data() { + if (this.__d !== null) return this.__d; + var j = __h(__FDATA, this.__i[0]); + if (j === null) __err('Footage "' + this.name + '" has no data (or it could not be parsed)'); + return (this.__d = JSON.parse(j)); + }, + // JSON: the parsed document; CSV/TSV: one object per row keyed by the header row. + get sourceData() { return this.__data.data; }, + // CSV/TSV cell by [column, row] (0-based, rows after the header). + dataValue: function (idx) { + idx = __v(idx); + var rows = this.__data.rows; + if (rows === null) { + // JSON: a path of keys / indexes. + var v = this.__data.data; + for (var i = 0; i < idx.length; i++) { + if (v === null || v === undefined) __err('dataValue: no value at ' + JSON.stringify(idx)); + v = v[idx[i]]; + } + return v; + } + var r = rows[idx[1]]; + if (r === undefined || r[idx[0]] === undefined) __err('dataValue: no cell at column ' + idx[0] + ', row ' + idx[1]); + return r[idx[0]]; + }, + get dataKeyCount() { var r = this.__data.rows; return r === null ? 0 : r.length; }, + // The column names (CSV/TSV header row). + get dataKeyNames() { return this.__data.header || []; }, + toString: function () { return '[object Footage]'; }, +}; + +function footage(name) { + var info = __h(__FOOT, String(name)); + if (info === null) __err('Footage named "' + name + '" does not exist'); + return new Footage(info); +} + +var __projectInfo = null; +// The project's bits per channel (8, 16 or 32). +__global('colorDepth', function () { return thisProject.bitsPerChannel; }); +var thisProject = { + get __info() { return __projectInfo || (__projectInfo = __h(__PROJ)); }, + get fullPath() { return ''; }, + get bitsPerChannel() { return this.__info[0]; }, + get linearBlending() { return this.__info[1]; }, + toString: function () { return '[object Project]'; }, +}; + // Layer ----------------------------------------------------------------------------------------- function Layer(c, l) { this.__c = c; this.__l = l; this.__i = null; } @@ -464,7 +548,14 @@ Layer.prototype = { var r = __h(__RECT, this.__c, this.__l, t === undefined ? time : t, !!includeExtents); return { top: r[0], left: r[1], width: r[2], height: r[3] }; }, - sampleImage: function () { return [0, 0, 0, 0]; }, + // Average straight RGBA of the layer's pixels in `point ± radius` (layer space). + sampleImage: function (point, radius, postEffect, t) { + point = __v(point); + radius = radius === undefined ? [0.5, 0.5] : __v(radius); + if (!Array.isArray(radius)) radius = [radius, radius]; + return __h(__SAMPLE, this.__c, this.__l, point[0], point[1], radius[0], radius.length > 1 ? radius[1] : radius[0], + postEffect === undefined ? true : !!__v(postEffect), t === undefined ? time : __v(t)); + }, toString: function () { return '[object Layer]'; }, }; @@ -817,7 +908,12 @@ Markers.prototype = { } if (!(n >= 1 && n <= list.length)) __err('Marker index ' + n + ' out of range'); var m = list[n - 1]; - return { time: m[0], duration: m[1], comment: m[2], chapter: m[3], url: m[4], index: n }; + var params = {}; + for (var k = 0; k < m[8].length; k++) params[m[8][k][0]] = m[8][k][1]; + return { + time: m[0], duration: m[1], comment: m[2], chapter: m[3], url: m[4], index: n, + frameTarget: m[5], eventCuePoint: m[6], cuePointName: m[7], parameters: params, protectedRegion: m[9], + }; }, nearestKey: function (t) { var list = this.__list; diff --git a/crates/expr/src/runtime.rs b/crates/expr/src/runtime.rs index af7ed9b66..d61388398 100644 --- a/crates/expr/src/runtime.rs +++ b/crates/expr/src/runtime.rs @@ -145,6 +145,19 @@ fn native_host(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult Req::Footage { name: string(args, 1, ctx)? }, + 12 => Req::FootageData { name: string(args, 1, ctx)? }, + 13 => Req::Sample { + comp: id(args, 1, ctx)?, + layer: id(args, 2, ctx)?, + x: secs(num(args, 3, ctx)?), + y: secs(num(args, 4, ctx)?), + rx: secs(num(args, 5, ctx)?), + ry: secs(num(args, 6, ctx)?), + post: arg(args, 7).to_boolean(), + t: secs(num(args, 8, ctx)?), + }, + 14 => Req::Project, _ => return Err(JsNativeError::typ().with_message("unknown host request").into()), }; let found = FRAMES.with(|f| { diff --git a/crates/expr/src/sample.rs b/crates/expr/src/sample.rs new file mode 100644 index 000000000..bff5a2149 --- /dev/null +++ b/crates/expr/src/sample.rs @@ -0,0 +1,119 @@ +//! `sampleImage`: read a layer's rendered pixels from an expression. +//! +//! The layer is rendered at full resolution through the renderer that is evaluating the +//! expression (its footage source; [`EvalCtx::footage`]): source → masks → effects for +//! `postEffect = true`, the bare source otherwise, in layer space (before the transform). Buffers +//! are cached per thread for the frame, so many expressions sampling the same layer at the same +//! time render it once. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::Arc; + +use effectcraft_project::{ItemId, Layer}; +use effectcraft_render::{EvalCtx, NoFootage, RenderOpts, Renderer}; +use effectcraft_time::Tick; + +/// A rendered layer buffer: straight-from-the-renderer premultiplied pixels with the layer +/// space → buffer pixel mapping. +#[derive(Clone)] +struct Sampled { + img: Arc, + offset: [f64; 2], + scale: f64, +} + +type Key = (usize, u64, u64, u64, bool, u64); + +thread_local! { + static CACHE: RefCell>> = RefCell::new(HashMap::new()); + /// Nested sampleImage renders (a sampled layer whose expressions sample again). + static DEPTH: RefCell = const { RefCell::new(0) }; +} + +const MAX_CACHE: usize = 32; +const MAX_DEPTH: usize = 2; + +/// A fingerprint of the layer's definition (so edits never reuse stale pixels). +fn fingerprint(layer: &Layer) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + format!("{layer:?}").hash(&mut h); + h.finish() +} + +fn render(ctx: EvalCtx, comp: ItemId, layer: &Layer, post: bool, t: f64) -> Option { + let depth = DEPTH.with(|d| *d.borrow()); + if depth >= MAX_DEPTH { + return None; + } + let key: Key = (ctx.project as *const _ as usize, comp.0, layer.id.0, t.to_bits(), post, fingerprint(layer)); + if let Some(hit) = CACHE.with(|c| c.borrow().get(&key).cloned()) { + return hit; + } + DEPTH.with(|d| *d.borrow_mut() += 1); + struct Pop; + impl Drop for Pop { + fn drop(&mut self) { + DEPTH.with(|d| *d.borrow_mut() -= 1); + } + } + let _pop = Pop; + let footage: &dyn effectcraft_render::FootageSource = ctx.footage.unwrap_or(&NoFootage); + let mut r = Renderer::new(ctx.project, footage, RenderOpts { motion_blur: false, ..Default::default() }); + r.expr = ctx.expr; + let ectx = r.eval_ctx(comp, Tick::from_seconds_f64(t))?; + let out = if post { + r.content_buf(&ectx, layer).map(|b| Sampled { img: Arc::new(b.img.clone()), offset: b.offset, scale: b.scale }) + } else { + r.layer_source(&ectx, layer).map(|b| Sampled { img: Arc::new(b.img), offset: b.offset, scale: b.scale }) + }; + CACHE.with(|c| { + let mut c = c.borrow_mut(); + if c.len() >= MAX_CACHE { + c.clear(); + } + c.insert(key, out.clone()); + }); + out +} + +/// The average colour (straight RGBA, 0..1) of `layer`'s pixels in the box `point ± radius` +/// (layer pixels) at comp time `t`. Transparent outside the layer. +pub fn sample_image(ctx: EvalCtx, comp: ItemId, layer: &Layer, point: [f64; 2], radius: [f64; 2], post: bool, t: f64) -> Option<[f32; 4]> { + if !layer.source.is_av() { + return Some([0.0; 4]); + } + let Some(b) = render(ctx, comp, layer, post, t) else { return Some([0.0; 4]) }; + let to_px = |x: f64, y: f64| (x * b.scale + b.offset[0], y * b.scale + b.offset[1]); + let (rx, ry) = (radius[0].abs() * b.scale, radius[1].abs() * b.scale); + let (cx, cy) = to_px(point[0], point[1]); + // Pixels whose centres (at +0.5) fall in the box; at least the one under the point. + let (x0, x1, y0, y1) = if rx < 0.5 && ry < 0.5 { + let (px, py) = (cx.floor() as i64, cy.floor() as i64); + (px, px, py, py) + } else { + let lo = |c: f64, r: f64| (c - r - 0.5).ceil() as i64; + let hi = |c: f64, r: f64| (c + r - 0.5).floor() as i64; + (lo(cx, rx), hi(cx, rx).max(lo(cx, rx)), lo(cy, ry), hi(cy, ry).max(lo(cy, ry))) + }; + let mut acc = [0.0f64; 4]; + let mut n = 0.0; + for y in y0..=y1.min(y0 + 4096) { + for x in x0..=x1.min(x0 + 4096) { + let p = b.img.get(x, y); + for k in 0..4 { + acc[k] += p[k] as f64; + } + n += 1.0; + } + } + if n == 0.0 { + return Some([0.0; 4]); + } + let a = acc[3] / n; + if a <= 1e-9 { + return Some([0.0; 4]); + } + Some([(acc[0] / n / a) as f32, (acc[1] / n / a) as f32, (acc[2] / n / a) as f32, a as f32]) +} diff --git a/crates/expr/src/tests.rs b/crates/expr/src/tests.rs index fd02acef8..d75972f76 100644 --- a/crates/expr/src/tests.rs +++ b/crates/expr/src/tests.rs @@ -1019,7 +1019,7 @@ fn text_expression_selector_values() { let sel_of = |p: &Project| { let c = p.comp(cid).unwrap(); let l = c.layer(lid).unwrap(); - let ctx = EvalCtx { project: p, comp_id: cid, comp: c, time: Tick::ZERO, expr: Some(&Expressions) }; + let ctx = EvalCtx { project: p, comp_id: cid, comp: c, time: Tick::ZERO, expr: Some(&Expressions), footage: None }; let doc = effectcraft_render::text::source_text(&ctx, l).unwrap(); let lay = effectcraft_text::layout_doc(&doc); let anim = l.props.group("text/animators/#1").unwrap(); @@ -1051,3 +1051,79 @@ fn text_expression_selector_values() { let v = eval_property(&p, cid, lid, "text/animators/#1/selectors/#2/amount", 0.0).unwrap(); assert_eq!(v, Value::Vec3([100.0; 3])); } + +/// `sampleImage` reads the layer's rendered pixels (layer space, straight colour) after masks +/// and effects, or the bare source with `postEffect = false`; the radius averages a box. +#[test] +fn sample_image_reads_rendered_pixels() { + let f = fx(); + let code = |c: &str| format!("var c = {c}; c[0] * 1000 + c[1] * 100 + c[2] * 10 + c[3]"); + // B: 40×40 blue solid. + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([20, 20])"), 0.0), 11.0); + // Half the box is outside the layer: alpha 0.5, colour still pure blue (straight). + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([0, 20], [1, 1])"), 0.0), 10.5); + // A: red with a mask covering 5..15: outside the mask it's transparent after masks, red before. + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([2, 2])"), 0.0), 0.0); + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([2, 2], [0.5, 0.5], false)"), 0.0), 1001.0); + assert_close!(f.num(&code("thisComp.layer(\"A\").sampleImage([10, 10])"), 0.0), 1001.0); + // The result has four values; outside the layer it's transparent. + assert_close!(f.num("thisComp.layer(\"B\").sampleImage([20, 20]).length", 0.0), 4.0); + assert_close!(f.num(&code("thisComp.layer(\"B\").sampleImage([-50, -50])"), 0.0), 0.0); +} + +fn data_item(p: &mut Project, name: &str, text: &str) { + let f = effectcraft_project::Footage { + path: format!("/data/{name}"), + kind: effectcraft_project::FootageKind::Data, + data: Some(text.into()), + ..Default::default() + }; + p.add_item(name, Label::Sandstone, None, ItemKind::Footage(f)); +} + +/// Data-driven animation: `footage(name)` with `sourceData` / `sourceText` / `dataValue` for +/// JSON, CSV and TSV items. +#[test] +fn data_footage_json_csv_tsv() { + let mut f = fx(); + data_item(&mut f.p, "stats.json", r#"{"title": "Q3", "items": [{"v": 3}, {"v": 7.5}]}"#); + data_item(&mut f.p, "scores.csv", "name,score\n\"Smith, J\",12\nLee,30.5\n"); + data_item(&mut f.p, "table.tsv", "a\tb\n1\t2\n3\t4\n"); + assert_close!(f.num("footage(\"stats.json\").sourceData.items[1].v", 0.0), 7.5); + assert_eq!(f.text("footage(\"stats.json\").sourceData.title", 0.0), "Q3"); + assert_close!(f.num("footage(\"stats.json\").dataValue([\"items\", 0, \"v\"])", 0.0), 3.0); + assert_close!(f.num("footage(\"scores.csv\").sourceData[1].score", 0.0), 30.5); + assert_eq!(f.text("footage(\"scores.csv\").sourceData[0].name", 0.0), "Smith, J"); + assert_close!(f.num("footage(\"scores.csv\").dataValue([1, 0])", 0.0), 12.0); + assert_close!(f.num("footage(\"scores.csv\").dataKeyCount", 0.0), 2.0); + assert_eq!(f.text("footage(\"scores.csv\").dataKeyNames.join('|')", 0.0), "name|score"); + assert_close!(f.num("footage(\"table.tsv\").dataValue([1, 1])", 0.0), 4.0); + assert_close!(f.num("footage(\"table.tsv\").sourceText.length", 0.0), 12.0); + assert!(f.err("footage(\"missing.json\").sourceData").contains("does not exist")); + // Animated from data: the value follows the CSV row picked by time. + assert_close!(f.num("footage(\"scores.csv\").dataValue([1, Math.floor(time)])", 1.2), 30.5); +} + +#[test] +fn project_and_time_globals() { + let mut f = fx(); + assert_close!(f.num("thisProject.bitsPerChannel", 0.0), 8.0); + assert_close!(f.num("colorDepth", 0.0), 8.0); + assert_close!(f.num("thisProject.linearBlending ? 1 : 0", 0.0), 0.0); + assert_eq!(f.text("timeToFeetAndFrames(2)", 0.0), "3+12"); + assert_close!(f.num("posterizeTime(4); time", 0.6), 0.5); + // Path API. + assert_close!(f.num("createPath([[0, 0], [10, 0], [10, 10]], [], [], true).points()[2][1]", 0.0), 10.0); + assert_close!(f.num("createPath([[0, 0], [10, 0]], [[1, 1], [2, 2]], [], false).inTangents()[1][0]", 0.0), 2.0); + assert_close!(f.num("mask(\"Mask 1\").maskPath.points().length", 0.0), 4.0); + assert_close!(f.num("mask(\"Mask 1\").maskPath.isClosed() ? 1 : 0", 0.0), 1.0); + // Marker keys carry protected regions and cue points. + { + let a = f.a; + let l = f.layer_mut(a); + l.markers[0].protected = true; + l.markers[0].cue_point = Some(effectcraft_project::CuePoint { name: "go".into(), navigation: false, params: vec![("k".into(), "v".into())] }); + } + assert_close!(f.num("thisLayer.marker.key(1).protectedRegion ? 1 : 0", 0.0), 1.0); + assert_eq!(f.text("thisComp.layer(\"A\").marker.key(1).cuePointName + thisComp.layer(\"A\").marker.key(1).parameters.k", 0.0), "gov"); +} diff --git a/crates/gpu/src/tests_adv3d.rs b/crates/gpu/src/tests_adv3d.rs index 067e4c8a8..f0e0f08bc 100644 --- a/crates/gpu/src/tests_adv3d.rs +++ b/crates/gpu/src/tests_adv3d.rs @@ -91,7 +91,7 @@ fn scene_project() -> (Project, ItemId) { fn scene(p: &Project, cid: ItemId) -> Scene { let r = Renderer::new(p, &NoFootage, RenderOpts::default()); - let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let ctx = EvalCtx { project: p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let run: Vec<&effectcraft_project::Layer> = ctx.comp.layers.iter().rev().filter(|l| l.is_3d() && l.has_video()).collect(); adv::scene_of(&r, &ctx, &run, (ctx.comp.width, ctx.comp.height)) } diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index d8906a1a3..dccf1aacf 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -183,7 +183,8 @@ mod tests { let cid = s.active_comp_id().unwrap(); let comp = s.project.comp(cid).unwrap(); let layer = comp.layer(effectcraft_engine::project::LayerId(l)).unwrap(); - let ctx = effectcraft_engine::render::EvalCtx { project: &s.project, comp_id: cid, comp, time: Default::default(), expr: s.expr.as_deref() }; + let ctx = + effectcraft_engine::render::EvalCtx { project: &s.project, comp_id: cid, comp, time: Default::default(), expr: s.expr.as_deref(), footage: None }; ctx.value(layer, layer.props.prop(path).unwrap()) } @@ -221,6 +222,81 @@ mod tests { assert_eq!(value(&s, a, "transform/rotation").as_f64(), 12.0); } + /// The expression error bar's list: evaluation errors and syntax errors, with the layer and + /// property they belong to. + #[test] + fn expression_errors_are_collected() { + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + let a = s.execute("layer.newSolid", json!({"name": "Box", "color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "50"})).unwrap(); + assert_eq!(s.execute("expr.errors", json!({})).unwrap()["count"], 0); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "thisComp.layer(\"Nope\").rotation"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "1 +* ("})).unwrap(); + let r = s.execute("expr.errors", json!({})).unwrap(); + assert_eq!(r["count"], 2, "{r}"); + let errs = r["errors"].as_array().unwrap(); + let rot = errs.iter().find(|e| e["path"] == "transform/rotation").unwrap(); + assert!(rot["message"].as_str().unwrap().contains("Nope"), "{rot}"); + assert_eq!(rot["layerIndex"], 1); + assert_eq!(rot["layerName"], "Box"); + assert_eq!(rot["disabled"], false); + let op = errs.iter().find(|e| e["path"] == "transform/opacity").unwrap(); + assert_eq!(op["disabled"], true); + assert!(r["text"][0].as_str().unwrap().contains("of layer 1 ('Box') in comp 'C'"), "{r}"); + // A user-disabled (but valid) expression is not an error. + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "45"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "enabled": false})).unwrap(); + assert_eq!(s.execute("expr.errors", json!({})).unwrap()["count"], 1); + // The Expression Language menu is categorised. + let m = s.execute("expr.languageMenu", json!({})).unwrap(); + assert!(m.as_array().unwrap().iter().any(|c| c["category"] == "Footage" && c["items"].as_array().unwrap().iter().any(|i| i["text"] == "sourceData"))); + } + + /// File ▸ Import of JSON/CSV data files as data footage, read by expressions. + #[test] + fn imported_data_files_drive_expressions() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-data"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("levels.csv"), "label,value\nlow,10\nhigh,90\n").unwrap(); + std::fs::write(dir.join("cfg.json"), r#"{"angle": 33}"#).unwrap(); + std::fs::write(dir.join("broken.json"), "{nope").unwrap(); + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + let paths: Vec = ["levels.csv", "cfg.json", "broken.json"].iter().map(|n| dir.join(n).to_string_lossy().to_string()).collect(); + let r = s.execute("file.import", json!({"paths": paths})).unwrap(); + assert_eq!(r["items"].as_array().unwrap().len(), 2, "{r}"); + assert_eq!(r["errors"].as_array().unwrap().len(), 1, "{r}"); + let csv = r["items"][0].as_u64().unwrap(); + assert_eq!(s.project.item(effectcraft_engine::project::ItemId(csv)).unwrap().type_name(), "Data"); + assert!(s.execute("layer.addItem", json!({"item": csv})).is_err(), "data files are not layers"); + let a = s.execute("layer.newSolid", json!({"color": "#ffffff"})).unwrap()["layer"].as_u64().unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/opacity", "expression": "footage(\"levels.csv\").sourceData[1].value"})).unwrap(); + s.execute("prop.setExpression", json!({"layer": a, "path": "transform/rotation", "expression": "footage(\"cfg.json\").sourceData.angle"})).unwrap(); + assert_eq!(value(&s, a, "transform/opacity").as_f64(), 90.0); + assert_eq!(value(&s, a, "transform/rotation").as_f64(), 33.0); + } + + /// sampleImage through the renderer: a box's Fill colour follows the solid under it. + #[test] + fn sample_image_drives_a_colour() { + let mut s = super::session(); + s.execute("comp.new", json!({"name": "C", "width": 100, "height": 100, "frameRate": 30, "duration": 2})).unwrap(); + s.execute("layer.newSolid", json!({"name": "Bg", "color": "#00ff00"})).unwrap(); + let a = s.execute("layer.newSolid", json!({"name": "Fg", "color": "#ffffff", "width": 20, "height": 20})).unwrap()["layer"].as_u64().unwrap(); + s.execute("effect.apply", json!({"layer": a, "effect": "Fill"})).unwrap(); + s.execute( + "prop.setExpression", + json!({"layer": a, "path": "effects/#1/color", "expression": "thisComp.layer(\"Bg\").sampleImage(thisComp.layer(\"Bg\").fromComp(toComp(anchorPoint)))"}), + ) + .unwrap(); + let cid = s.active_comp_id().unwrap(); + let img = s.render(cid, s.time(), effectcraft_engine::render::RenderOpts::default()); + let px = img.get(50, 50); + assert!(px[1] > 0.99 && px[0] < 0.01, "the box is filled with the sampled green: {px:?}"); + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index 7bf67774f..aff4d93c5 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -36,11 +36,14 @@ pub struct EvalCtx<'a> { /// Comp time. pub time: Tick, pub expr: Option<&'a dyn ExprHost>, + /// Footage frames for expressions that read rendered pixels (`sampleImage`): the renderer's + /// own source while rendering; `None` renders footage as transparent. + pub footage: Option<&'a dyn crate::FootageSource>, } impl<'a> EvalCtx<'a> { pub fn new(project: &'a Project, comp_id: ItemId, comp: &'a Comp, time: Tick) -> EvalCtx<'a> { - EvalCtx { project, comp_id, comp, time, expr: None } + EvalCtx { project, comp_id, comp, time, expr: None, footage: None } } pub fn at(&self, time: Tick) -> EvalCtx<'a> { EvalCtx { time, ..*self } diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 5e7425bb7..6699d68d9 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -451,7 +451,7 @@ impl<'a> Renderer<'a> { } fn ctx(&self, comp_id: ItemId, comp: &'a Comp, t: Tick) -> EvalCtx<'a> { - EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr } + EvalCtx { project: self.project, comp_id, comp, time: t, expr: self.expr, footage: Some(self.footage) } } /// The accelerator to use for this render, if any (see [`Backend`]). diff --git a/crates/render/src/three_d/adv/tests.rs b/crates/render/src/three_d/adv/tests.rs index 47ad6178d..b1d944132 100644 --- a/crates/render/src/three_d/adv/tests.rs +++ b/crates/render/src/three_d/adv/tests.rs @@ -303,7 +303,7 @@ fn extruded_text_renders_as_a_mesh() { set(l, "geometryOptions/bevelStyle", Value::Enum(1)); set(l, "transform/rotationY", Value::Scalar(40.0)); }); - let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None }; + let ctx = crate::EvalCtx { project: &p, comp_id: cid, comp: p.comp(cid).unwrap(), time: Tick::ZERO, expr: None, footage: None }; let l = &ctx.comp.layers[0]; let params = scene::extrusion_params(&ctx, l).unwrap(); let meshes = scene::extruded_meshes(&ctx, l, ¶ms); diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f7..70d562376 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -793,7 +793,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let comp_name = app.session.project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time: app.session.time(), expr: snap_expr.as_deref(), footage: None }; let hdr = Rect::from_min_size(rect.min, vec2(rect.width(), 24.0)); p.text(pos2(hdr.min.x + 10.0, hdr.center().y), Align2::LEFT_CENTER, format!("{} • {}", comp_name, layer.name), Tokens::ui(11.5), t.text_dim); p.line_segment([hdr.left_bottom(), hdr.right_bottom()], Stroke::new(1.0, t.separator)); diff --git a/crates/ui-egui/src/panels/layer_panel.rs b/crates/ui-egui/src/panels/layer_panel.rs index e53e268e7..5149241e8 100644 --- a/crates/ui-egui/src/panels/layer_panel.rs +++ b/crates/ui-egui/src/panels/layer_panel.rs @@ -119,7 +119,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let tex = match cached { Some(c) if c.key == key => Some(c), _ => { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; let mut r = Renderer::new(&app.session.project, &*app.session.footage, RenderOpts { scale, ..Default::default() }); r.expr = app.session.expr.as_deref(); r.cache = Some(&app.session.layer_cache); diff --git a/crates/ui-egui/src/panels/properties.rs b/crates/ui-egui/src/panels/properties.rs index fbd4e5e99..c57956854 100644 --- a/crates/ui-egui/src/panels/properties.rs +++ b/crates/ui-egui/src/panels/properties.rs @@ -69,7 +69,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let project = app.session.project.clone(); let expr = app.session.expr.clone(); let now = app.session.time(); - let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref() }; + let ectx = EvalCtx { project: &project, comp_id: cid, comp: &comp, time: now, expr: expr.as_deref(), footage: None }; let x0 = rect.min.x + PAD; let w = rect.width() - 2.0 * PAD; // Vertical scrolling: content height is known after drawing, so clamp with last frame's. diff --git a/crates/ui-egui/src/panels/text_panels.rs b/crates/ui-egui/src/panels/text_panels.rs index edb1132ff..31f98bb0b 100644 --- a/crates/ui-egui/src/panels/text_panels.rs +++ b/crates/ui-egui/src/panels/text_panels.rs @@ -56,7 +56,7 @@ pub fn text_target(app: &EffectcraftApp) -> Option { .iter() .filter_map(|id| comp.layer(*id)) .find(|l| matches!(l.source, effectcraft_engine::project::LayerSource::Text))?; - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp, time: app.session.time(), expr: app.session.expr.as_deref(), footage: None }; effectcraft_engine::render::text::source_text(&ectx, layer).map(|d| TextTarget { layer: layer.id.0, doc: d, range: None }) } @@ -453,7 +453,7 @@ pub fn align_layers(app: &mut EffectcraftApp, op: &str) { let time = app.session.time(); let mut moves = vec![]; { - let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref() }; + let ectx = EvalCtx { project: &app.session.project, comp_id: cid, comp: &comp, time, expr: app.session.expr.as_deref(), footage: None }; for l in comp.layers.iter().filter(|l| app.session.state.selected_layers.contains(&l.id)) { let Some(b) = effectcraft_engine::render::content_bounds(&ectx, l) else { continue }; let (m, _) = ectx.layer_to_comp(l); diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index c9a42698a..cd3bf6f92 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -687,7 +687,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.ui.timeline.scroll_y = app.ui.timeline.scroll_y.min(max_scroll); let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; let lp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(rows_rect.min, pos2(graph_x0 - 1.0, rows_rect.max.y)))); let gp = p.with_clip_rect(rows_rect.intersect(Rect::from_min_max(pos2(graph_x0, rows_rect.min.y), rows_rect.max))); let mut y = rows_rect.min.y - app.ui.timeline.scroll_y; diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb85..76ef92a9e 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -394,7 +394,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { } let snap_project = app.session.project.clone(); let snap_expr = app.session.expr.clone(); - let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref() }; + let ectx = EvalCtx { project: &snap_project, comp_id: cid, comp: &comp, time, expr: snap_expr.as_deref(), footage: None }; // The frame (or snapshot) through Show Channel and exposure; ROI frames cover the region. vt::draw_frame(app, &ctx, &painter, comp_rect, cid, &ectx); painter.rect_stroke(comp_rect, 0.0, Stroke::new(1.0, Color32::from_black_alpha(160)), StrokeKind::Outside); @@ -891,6 +891,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { comp: app.session.project.comp(cid).unwrap_or(&comp), time, expr: app.session.expr.as_deref(), + footage: None, }; if let Some(m) = l2c(&e2, &l).0.inverse() { *i2 = m; @@ -1396,6 +1397,7 @@ fn create_shape(app: &mut EffectcraftApp, tool: Tool, a: [f64; 2], b: [f64; 2], comp: &comp, time: app.session.time(), expr: None, + footage: None, }; let inv = l2c(&ectx, &l).0.inverse().unwrap_or(Mat3::IDENTITY); let p0 = inv.apply(gv2(cx - w / 2.0, cy - h / 2.0)); From cba8d7e737db9438c9b0e46fd882875df5e7946a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:02:39 -0400 Subject: [PATCH 4/9] M12.6: 3D Camera Tracker effect + UI Effect > Perspective > 3D Camera Tracker (Shot Type, Horizontal Angle of View, Show/Render Track Points, Track Point Size, Target Size, Advanced: Solve Method, Method Used, Average Error, Detailed Analysis, Auto-delete Points Across Time, Hide Warning Banner); tracks and solve stored in the effect with keys (frame edits re-track, setting / deleted-point edits only re-solve). Background job (Analyzing in background (step 1 of 2) / Solving camera banner, cancel, one undo step). Commands: track.camera (replaces the stub), camera.analyze/cancel/solveStatus/points/selectPoints/ createFromSolve/create/setGroundPlane/deletePoints; the one-node 3D Tracker Camera keyed per frame and text / solids / nulls / shadow catcher + light on the target plane, Create Multiple. Viewer: depth-sized coloured targets, hover target plane, click / shift / marquee selection, AE right-click menu, Delete; Effect Controls Analyze / Cancel / Create Camera; Tracker panel Track Camera. Solver: adaptive noise thresholds, denser keyframes, quick tripod check, faster Cholesky and parallel Schur complement. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + ROADMAP.md | 4 +- crates/effects/src/camera_tracker.rs | 291 ++++++++ crates/effects/src/lib.rs | 4 + crates/engine/src/camera_track.rs | 640 ++++++++++++++++++ crates/engine/src/commands/camera_cmds.rs | 574 ++++++++++++++++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/commands/stubs.rs | 1 - crates/engine/src/lib.rs | 22 + crates/engine/src/tests_camera_track.rs | 307 +++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/track/Cargo.toml | 1 + crates/track/src/camtrack/bundle.rs | 90 ++- crates/track/src/camtrack/features.rs | 2 +- crates/track/src/camtrack/linalg.rs | 30 +- crates/track/src/camtrack/mod.rs | 8 +- crates/track/src/camtrack/solver.rs | 83 ++- crates/track/src/camtrack/tests.rs | 78 +++ crates/ui-egui/src/lib.rs | 5 + .../ui-egui/src/panels/camera_tracker_ui.rs | 438 ++++++++++++ crates/ui-egui/src/panels/effect_controls.rs | 2 + crates/ui-egui/src/panels/mod.rs | 1 + crates/ui-egui/src/panels/tracker.rs | 7 +- crates/ui-egui/src/panels/viewer.rs | 3 + crates/ui-egui/tests/ui_camera_tracker.rs | 175 +++++ docs/agents.md | 28 + docs/architecture.md | 16 + docs/parity.md | 10 +- 28 files changed, 2771 insertions(+), 54 deletions(-) create mode 100644 crates/effects/src/camera_tracker.rs create mode 100644 crates/engine/src/camera_track.rs create mode 100644 crates/engine/src/commands/camera_cmds.rs create mode 100644 crates/engine/src/tests_camera_track.rs create mode 100644 crates/ui-egui/src/panels/camera_tracker_ui.rs create mode 100644 crates/ui-egui/tests/ui_camera_tracker.rs diff --git a/Cargo.lock b/Cargo.lock index 46f34e631..ce792ecbb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "rayon", "serde", "serde_json", + "web-time", ] [[package]] diff --git a/ROADMAP.md b/ROADMAP.md index a3ed979cf..7780c1641 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -28,10 +28,10 @@ once. Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): - **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 257 of 298 effects, 16 of them on the GPU. + professional daily-use features, 21% of the long tail); 258 of 298 effects, 16 of them on the GPU. - **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** by the audit's conservative per-feature figures. -- Hardest remaining pieces: Roto Brush, the 3D camera tracker, Advanced 3D, a scripting object model. +- Hardest remaining pieces: Roto Brush, Advanced 3D, a scripting object model (the 3D Camera Tracker landed). Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/crates/effects/src/camera_tracker.rs b/crates/effects/src/camera_tracker.rs new file mode 100644 index 000000000..dd97b585e --- /dev/null +++ b/crates/effects/src/camera_tracker.rs @@ -0,0 +1,291 @@ +//! 3D Camera Tracker (Effect ▸ Perspective ▸ 3D Camera Tracker; Animation ▸ Track Camera). +//! +//! The analysis runs as a background job in the engine (`camera.analyze`): step 1 tracks +//! features through the clip ([`effectcraft_track::camtrack::TrackAnalyzer`]), step 2 solves +//! the camera ([`effectcraft_track::camtrack::solve`]). Both results are stored as JSON in the +//! instance's hidden parameters — **Tracks** (with **Tracks Key**: what the frames were made +//! from, so source / In-Out / time edits invalidate them) and **Solve** (with **Solve Key**: the +//! Shot Type, Angle of View, Solve Method and deleted points it was solved with, so changing +//! those re-solves without re-tracking). **Method Used** and **Average Error** report the solve. +//! +//! The effect passes its input through; with **Render Track Points** on it draws the solved +//! points (as the viewer shows them) into the frame. The viewer overlay, target and the Create +//! commands live in the engine and the UI. + +use std::collections::HashMap; +use std::sync::{Arc, Mutex, OnceLock}; + +use effectcraft_keyframe::Value; +use effectcraft_project::ParamUi; +use effectcraft_raster::{Image, Px}; +use effectcraft_track::camtrack::{CameraSolve, CameraTracks, ShotType, SolveMethod, SolveSettings, linalg, point_color}; + +use crate::warp_stab::param; +use crate::{Buf, EffectCtx, EffectSpec, Params, num, p, popup, slider}; + +/// The effect's id. +pub const ID: &str = "ec.perspective.cameratracker"; + +/// Hidden parameters (see the module docs). +pub const TRACKS: &str = "tracks"; +pub const TRACKS_KEY: &str = "tracksKey"; +pub const SOLVE: &str = "solve"; +pub const SOLVE_KEY: &str = "solveKey"; +/// Deleted track point ids (JSON array). +pub const DELETED: &str = "deleted"; +/// The "3D Tracker Camera" layer the instance created (layer id, 0 = none). +pub const CAMERA: &str = "camera"; +pub const METHOD_USED: &str = "advanced/methodUsed"; +pub const AVERAGE_ERROR: &str = "advanced/averageError"; + +pub const SHOT_TYPES: [&str; 3] = ["Fixed Angle of View", "Variable Zoom", "Specify Angle of View"]; +pub const SHOW_POINTS: [&str; 2] = ["2D Source", "3D Solved"]; +pub const SOLVE_METHODS: [&str; 4] = ["Auto Detect", "Typical", "Mostly Flat Scene", "Tripod Pan"]; + +pub fn specs() -> Vec { + vec![EffectSpec { + id: ID, + name: "3D Camera Tracker", + category: "Perspective", + params: vec![ + p("shotType", "Shot Type", Value::Enum(0), popup(&SHOT_TYPES)), + p("horizontalAngleOfView", "Horizontal Angle of View", num(40.0), slider(1.0, 170.0, 10.0, 120.0, 1)), + p("showTrackPoints", "Show Track Points", Value::Enum(1), popup(&SHOW_POINTS)), + p("renderTrackPoints", "Render Track Points", Value::Bool(false), ParamUi::Checkbox), + p("trackPointSize", "Track Point Size", num(100.0), slider(0.0, 1000.0, 0.0, 300.0, 0)), + p("targetSize", "Target Size", num(100.0), slider(1.0, 1000.0, 10.0, 300.0, 0)), + p("advanced/solveMethod", "Solve Method", Value::Enum(0), popup(&SOLVE_METHODS)), + p(METHOD_USED, "Method Used", Value::Str(String::new()), ParamUi::Hidden), + p(AVERAGE_ERROR, "Average Error", num(0.0), ParamUi::Hidden), + p("advanced/detailedAnalysis", "Detailed Analysis", Value::Bool(false), ParamUi::Checkbox), + p("advanced/autoDeletePoints", "Auto-delete Points Across Time", Value::Bool(true), ParamUi::Checkbox), + p("advanced/hideWarningBanner", "Hide Warning Banner", Value::Bool(false), ParamUi::Checkbox), + p(TRACKS, "Tracks", Value::Str(String::new()), ParamUi::Hidden), + p(TRACKS_KEY, "Tracks Key", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE, "Solve", Value::Str(String::new()), ParamUi::Hidden), + p(SOLVE_KEY, "Solve Key", Value::Str(String::new()), ParamUi::Hidden), + p(DELETED, "Deleted Points", Value::Str(String::new()), ParamUi::Hidden), + p(CAMERA, "Camera Layer", num(0.0), ParamUi::Hidden), + ], + render, + gpu: false, + float: true, + }] +} + +fn f(params: &Params, id: &str) -> f64 { + param(params, id).map(Value::as_f64).unwrap_or(0.0) +} +fn e(params: &Params, id: &str) -> u32 { + param(params, id).map(Value::as_enum).unwrap_or(0) +} +fn b(params: &Params, id: &str) -> bool { + param(params, id).map(Value::as_bool).unwrap_or(false) +} +fn s<'a>(params: &'a Params, id: &str) -> &'a str { + match param(params, id) { + Some(Value::Str(s)) => s, + _ => "", + } +} + +/// Deleted point ids of an instance (stored as a JSON array of numbers). +pub fn deleted(params: &Params) -> Vec { + s(params, DELETED).trim().trim_start_matches('[').trim_end_matches(']').split(',').filter_map(|v| v.trim().parse().ok()).collect() +} + +/// The solve settings of an instance. +pub fn settings(params: &Params) -> SolveSettings { + let mut del = deleted(params); + del.sort_unstable(); + del.dedup(); + SolveSettings { + shot: ShotType::ALL.get(e(params, "shotType") as usize).copied().unwrap_or_default(), + hfov: f(params, "horizontalAngleOfView"), + method: SolveMethod::ALL.get(e(params, "advanced/solveMethod") as usize).copied().unwrap_or_default(), + deleted: del, + } +} + +/// What a solve depends on besides the tracks. +pub fn solve_key(st: &SolveSettings) -> String { + let hfov = if st.shot == ShotType::SpecifyAngle { format!("{:.4}", st.hfov) } else { String::new() }; + format!("{:?}|{hfov}|{:?}|{:?}", st.shot, st.method, st.deleted) +} + +fn fnv(bytes: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for b in bytes { + h ^= *b as u64; + h = h.wrapping_mul(0x0100_0000_01b3); + } + h +} + +type Cache = Mutex>>; + +fn cached(c: &'static OnceLock>, json: &str, parse: impl Fn(&str) -> Option) -> Option> { + let key = fnv(json.as_bytes()); + let c = c.get_or_init(Default::default); + if let Some(a) = c.lock().unwrap_or_else(|e| e.into_inner()).get(&key) { + return Some(a.clone()); + } + let a = Arc::new(parse(json)?); + let mut m = c.lock().unwrap_or_else(|e| e.into_inner()); + if m.len() > 8 { + m.clear(); + } + m.insert(key, a.clone()); + Some(a) +} + +/// The stored feature tracks of an instance (`None` when not analysed). +pub fn tracks(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, TRACKS); + if j.is_empty() { + return None; + } + cached(&C, j, CameraTracks::from_json).filter(|t| !t.is_empty()) +} + +/// The stored camera solve of an instance (`None` when not solved). +pub fn solve(params: &Params) -> Option> { + static C: OnceLock> = OnceLock::new(); + let j = s(params, SOLVE); + if j.is_empty() { + return None; + } + cached(&C, j, CameraSolve::from_json).filter(|t| !t.is_empty()) +} + +/// Screen size (layer pixels) of a target drawn for a point at depth `z`, relative to the +/// median depth of the frame's points. +pub fn target_radius(size_pct: f64, z: f64, median_z: f64) -> f64 { + (6.0 * size_pct / 100.0 * (median_z / z.max(1e-9))).clamp(1.5, 40.0) +} + +/// The solved points visible on frame `k`: (id, layer pixel position, camera depth). +pub fn projected(solve: &CameraSolve, k: usize) -> Vec<(u32, [f64; 2], f64)> { + let Some(cam) = solve.frames.get(k) else { return vec![] }; + solve + .visible(k) + .filter_map(|p| { + let c = cam.to_cam(p.pos); + cam.project(solve.size, p.pos).map(|q| (p.id, q, c[2])) + }) + .collect() +} + +/// Median of the depths. +pub fn median_depth(v: &[(u32, [f64; 2], f64)]) -> f64 { + let mut z: Vec = v.iter().map(|x| x.2).collect(); + if z.is_empty() { + return 1.0; + } + z.sort_by(f64::total_cmp); + z[z.len() / 2] +} + +fn render(ctx: &EffectCtx, mut buf: Buf) -> Buf { + if !b(ctx.params, "renderTrackPoints") { + return buf; + } + let Some(sol) = solve(ctx.params) else { return buf }; + if (sol.size[0] - ctx.layer_size[0]).abs() > 0.5 || (sol.size[1] - ctx.layer_size[1]).abs() > 0.5 { + return buf; + } + let Some(k) = sol.frame_at(ctx.time) else { return buf }; + let pts = projected(&sol, k); + let med = median_depth(&pts); + let size = f(ctx.params, "trackPointSize"); + let sc = if buf.scale > 0.0 { buf.scale } else { 1.0 }; + for (id, q, z) in pts { + let c = point_color(id); + let r = target_radius(size, z, med) * sc; + let at = [q[0] * sc + buf.offset[0], q[1] * sc + buf.offset[1]]; + ring(&mut buf.img, at, r, (r * 0.25).max(1.0), [c[0], c[1], c[2], 1.0]); + disc(&mut buf.img, at, (r * 0.2).max(0.8), [c[0], c[1], c[2], 1.0]); + } + buf +} + +fn lerp_px(a: Px, b: Px, t: f32) -> Px { + [a[0] + (b[0] - a[0]) * t, a[1] + (b[1] - a[1]) * t, a[2] + (b[2] - a[2]) * t, a[3] + (b[3] - a[3]) * t] +} + +fn cover(img: &mut Image, c: [f64; 2], r_out: f64, cov: impl Fn(f64) -> f64, col: Px) { + let (x0, x1) = ((c[0] - r_out - 1.0).floor().max(0.0) as i64, (c[0] + r_out + 1.0).ceil() as i64); + let (y0, y1) = ((c[1] - r_out - 1.0).floor().max(0.0) as i64, (c[1] + r_out + 1.0).ceil() as i64); + for y in y0..y1.min(img.height as i64) { + for x in x0..x1.min(img.width as i64) { + let d = (x as f64 + 0.5 - c[0]).hypot(y as f64 + 0.5 - c[1]); + let a = cov(d).clamp(0.0, 1.0) as f32; + if a > 0.0 { + let i = img.idx(x as u32, y as u32); + img.data[i] = lerp_px(img.data[i], col, a); + } + } + } +} + +fn disc(img: &mut Image, c: [f64; 2], r: f64, col: Px) { + cover(img, c, r, |d| r + 0.5 - d, col); +} + +fn ring(img: &mut Image, c: [f64; 2], r: f64, w: f64, col: Px) { + cover(img, c, r + w, |d| (w * 0.5 + 0.5 - (d - r).abs()).min(1.0), col); +} + +/// Points of a target disc (canonical coordinates) for drawing: `n` points on the circle of +/// radius `size / 2` in the plane through `center` with normal `normal`. +pub fn target_circle(center: linalg::V3, normal: linalg::V3, radius: f64, n: usize) -> Vec { + let a = if normal[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + let u = linalg::normalize(linalg::cross(normal, a)); + let v = linalg::cross(normal, u); + (0..n) + .map(|i| { + let t = i as f64 / n as f64 * std::f64::consts::TAU; + linalg::add(center, linalg::add(linalg::scale(u, radius * t.cos()), linalg::scale(v, radius * t.sin()))) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::EffectEnv; + use effectcraft_track::camtrack::{SolvedFrame, SolvedPoint}; + + fn params_with(solve: &CameraSolve, render: bool) -> Params { + let mut p = Params::default(); + p.values.insert(SOLVE.into(), Value::Str(solve.to_json())); + p.values.insert("renderTrackPoints".into(), Value::Bool(render)); + p.values.insert("trackPointSize".into(), Value::Scalar(100.0)); + p + } + + #[test] + fn render_track_points_draws_targets_and_passes_through_otherwise() { + let sol = CameraSolve { + version: 1, + size: [64.0, 48.0], + start: 0.0, + frame_duration: 1.0 / 30.0, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 50.0, solved: true }], + points: vec![SolvedPoint { id: 3, pos: [0.0, 0.0, 1.0], error: 0.1, first: 0, last: 0 }], + ..Default::default() + }; + let img = Image::new(64, 48); + for on in [false, true] { + let params = params_with(&sol, on); + let ctx = EffectCtx { params: ¶ms, time: 0.0, layer_size: [64.0, 48.0], seed: 0, adjustment: false, env: EffectEnv::default() }; + let out = render(&ctx, Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 }); + let px = out.img.data[out.img.idx(32, 24)]; + assert_eq!(px[3] > 0.5, on); + } + let st = settings(&Params::default()); + assert_eq!(st.shot, ShotType::FixedAngle); + assert!(solve_key(&st).contains("FixedAngle")); + } +} diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 6d11e19a1..9c2aefb50 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -8,6 +8,7 @@ pub mod audio_fx; mod blur2; +pub mod camera_tracker; mod channel; mod channel2; mod color2; @@ -354,6 +355,7 @@ pub fn registry() -> &'static [EffectSpec] { v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(warp_stab::specs()); + v.extend(camera_tracker::specs()); v.sort_by(|a, b| a.category.cmp(b.category).then(a.name.cmp(b.name))); for s in v.iter_mut() { if GPU_EFFECTS.contains(&s.id) { @@ -494,6 +496,8 @@ pub const TIME_DEPENDENT: &[&str] = &[ "ec.time.pixelmotionblur", // Each frame gets its own stabilizing warp. warp_stab::ID, + // Render Track Points draws each frame's solved points. + camera_tracker::ID, ]; /// See [`TIME_DEPENDENT`]. diff --git a/crates/engine/src/camera_track.rs b/crates/engine/src/camera_track.rs new file mode 100644 index 000000000..1b984aac2 --- /dev/null +++ b/crates/engine/src/camera_track.rs @@ -0,0 +1,640 @@ +//! 3D Camera Tracker runtime: the analysis job (`camera.analyze`, `track.camera`), keeping +//! analyses valid, and turning a solve into composition layers (`camera.createFromSolve`). +//! +//! The job renders the layer's input to the effect (`Renderer::layer_input`) for every frame +//! between the layer's In and Out points on a background thread and tracks features through +//! them (step 1 of 2, "Analyzing in background"), then solves the camera (step 2, "Solving +//! camera"). When only the solve settings changed (Shot Type, Angle of View, Solve Method, +//! deleted points) the stored tracks are re-solved without re-tracking. Results are written into +//! the effect's hidden parameters as one undo step; edits that change the layer's frames clear +//! them and queue the effect again ([`invalidate`], run after every edit), like After Effects. +//! +//! Composition space: the solve's canonical frame maps to comp world by +//! [`CameraSolve::world`] with the zoom of the first solved frame (focal length × the layer's +//! scale in the comp), so the "3D Tracker Camera" reproduces the footage's view and created +//! layers sit on the selected points. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use effectcraft_effects::camera_tracker::{self as ct, AVERAGE_ERROR, METHOD_USED, SOLVE, SOLVE_KEY, TRACKS, TRACKS_KEY}; +use effectcraft_keyframe::{Keyframe, Value}; +use effectcraft_project::{AutoOrient, GroupKind, ItemId, ItemKind, Layer, LayerId, Project, PropGroup, Uid}; +use effectcraft_render::{EvalCtx, ExprHost, FootageSource, LayerCache, RenderOpts, Renderer}; +use effectcraft_time::Tick; +use effectcraft_track::camtrack::linalg::{self, Similarity, V3}; +use effectcraft_track::camtrack::{AnalyzeOpts, CameraSolve, CameraTracks, SolveSettings, Target, TrackAnalyzer}; +use serde::Serialize; + +use crate::tracking::lock; +use crate::warp::{analysis_times, signature}; +use crate::{Event, Session}; + +/// Live analysis progress (serde for agents / the control channel). +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CameraProgress { + /// 1 = tracking features, 2 = solving the camera. + pub step: u8, + pub done: u64, + pub total: u64, + pub elapsed: f64, + pub fps: f64, + pub finished: bool, + pub cancelled: bool, + pub error: Option, + /// Re-solving stored tracks (no tracking step). + pub solve_only: bool, +} + +impl CameraProgress { + /// The viewer banner text, as After Effects words it. + pub fn banner(&self) -> String { + match self.step { + 2 => "Solving camera".into(), + _ => { + let pc = (100 * self.done).checked_div(self.total).unwrap_or(0); + format!("Analyzing in background (step 1 of 2): {pc}%") + } + } + } +} + +struct CamResult { + /// New tracks (None for a re-solve). + tracks: Option, + solve: Result, +} + +#[derive(Default)] +pub struct CameraShared { + pub cancel: AtomicBool, + pub state: Mutex, + result: Mutex>, +} + +/// A running (or finished, not yet polled) camera analysis. +pub struct CameraJob { + pub shared: Arc, + thread: Option>, + pub comp: ItemId, + pub layer: LayerId, + pub effect: Uid, + /// The layer signature the tracks are made from, and the solve settings key. + key: String, + solve_key: String, +} + +impl CameraJob { + pub fn is_finished(&self) -> bool { + lock(&self.shared.state).finished + } + pub fn progress(&self) -> CameraProgress { + lock(&self.shared.state).clone() + } +} + +struct CamWork { + project: Arc, + footage: Arc, + expr: Option>, + cache: Arc, + comp: ItemId, + layer: LayerId, + effect_index: usize, + size: [f64; 2], + detailed: bool, + times: Vec, + /// Stored tracks to re-solve (skips step 1). + tracks: Option>, + settings: SolveSettings, +} + +pub(crate) fn is_tracker(g: &PropGroup) -> bool { + matches!(&g.kind, GroupKind::Effect { effect } if effect == ct::ID) +} + +fn str_of(g: &PropGroup, m: &str) -> String { + match g.prop(m).map(|p| &p.value) { + Some(Value::Str(s)) => s.clone(), + _ => String::new(), + } +} + +/// What a tracker's tracks are made from: the layer's frames and Detailed Analysis. +fn tracks_key(project: &Project, cid: ItemId, l: &Layer, g: &PropGroup) -> String { + let detailed = effectcraft_effects::warp_stab::param(&static_params(g), "advanced/detailedAnalysis").is_some_and(Value::as_bool); + format!("{}{}", signature(project, cid, l), if detailed { "+detailed" } else { "" }) +} + +/// Parameters of an instance from static values (the tracker's settings are not animated). +pub fn static_params(g: &PropGroup) -> effectcraft_effects::Params { + effectcraft_effects::flatten_params(g, &mut |p| p.value.clone()) +} + +fn clear(g: &mut PropGroup, all: bool) { + let mut ids = vec![SOLVE, SOLVE_KEY, METHOD_USED]; + if all { + ids.extend([TRACKS, TRACKS_KEY]); + } + for m in ids { + if let Some(p) = g.prop_mut(m) { + p.value = Value::Str(String::new()); + p.keys.clear(); + } + } + if let Some(p) = g.prop_mut(AVERAGE_ERROR) { + p.value = Value::Scalar(0.0); + } +} + +/// Clear analyses that no longer match, after an edit: tracks made from other frames (and their +/// solve), and solves made with other settings. Returns the trackers that need (re-)analysis: +/// cleared ones and newly applied, unanalysed ones. +pub(crate) fn invalidate(before: &Project, after: &mut Project) -> Vec<(ItemId, LayerId, Uid)> { + let mut todo = vec![]; + let mut clears = vec![]; + for (cid, it) in &after.items { + let ItemKind::Comp(c) = &it.kind else { continue }; + for l in &c.layers { + let Some(fx) = l.effects() else { continue }; + if !fx.groups().any(is_tracker) { + continue; + } + let old_layer = before.comp(*cid).and_then(|bc| bc.layer(l.id)); + for g in fx.groups().filter(|g| is_tracker(g)) { + let tracked = !str_of(g, TRACKS).is_empty(); + if tracked && str_of(g, TRACKS_KEY) != tracks_key(after, *cid, l, g) { + clears.push((*cid, l.id, g.uid, true)); + todo.push((*cid, l.id, g.uid)); + } else if tracked && str_of(g, SOLVE_KEY) != ct::solve_key(&ct::settings(&static_params(g))) { + clears.push((*cid, l.id, g.uid, false)); + todo.push((*cid, l.id, g.uid)); + } else if !tracked && old_layer.and_then(|ol| ol.props.find_group(g.uid)).is_none() { + todo.push((*cid, l.id, g.uid)); + } + } + } + } + for (cid, lid, uid, all) in clears { + if let Some(g) = after.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)) { + clear(g, all); + } + } + todo +} + +fn input_frame(r: &Renderer, ctx: &EvalCtx, layer: &Layer, effect_index: usize) -> Option<(Arc, [f64; 2])> { + let buf = r.layer_input(ctx, layer, effect_index)?; + if (buf.scale - 1.0).abs() > 1e-9 && buf.scale > 0.0 { + let w = (buf.img.width as f64 / buf.scale).round().max(1.0) as u32; + let h = (buf.img.height as f64 / buf.scale).round().max(1.0) as u32; + let off = [buf.offset[0] / buf.scale, buf.offset[1] / buf.scale]; + return Some((Arc::new(effectcraft_raster::resample(&buf.img, w, h)), off)); + } + Some((Arc::new(buf.img.clone()), buf.offset)) +} + +fn run_work(w: CamWork, shared: &CameraShared) { + let t0 = web_time::Instant::now(); + let fail = |msg: &str| { + let mut s = lock(&shared.state); + s.error = Some(msg.to_string()); + s.finished = true; + }; + let cancelled = || { + let mut s = lock(&shared.state); + s.cancelled = true; + s.finished = true; + }; + let Some(comp) = w.project.comp(w.comp) else { return fail("composition missing") }; + let Some(layer) = comp.layer(w.layer) else { return fail("layer missing") }; + let lt0 = layer.layer_time(w.times[0]).seconds(); + let fd = if w.times.len() > 1 { layer.layer_time(w.times[1]).seconds() - lt0 } else { comp.frame_duration().seconds() }; + let (tracks, fresh) = match w.tracks.clone() { + Some(t) => (t, false), + None => { + let mut r = Renderer::new(&w.project, w.footage.as_ref(), RenderOpts { scale: 1.0, ..Default::default() }); + r.expr = w.expr.as_deref(); + r.cache = Some(&w.cache); + let r = &r; + let ctx0 = EvalCtx::new(&w.project, w.comp, comp, Tick::ZERO); + let ctx0 = EvalCtx { expr: w.expr.as_deref(), ..ctx0 }; + let frame_at = |t: Tick| input_frame(r, &ctx0.at(t), layer, w.effect_index); + { + let mut s = lock(&shared.state); + s.step = 1; + s.total = w.times.len() as u64; + } + let mut an = TrackAnalyzer::new(w.size, AnalyzeOpts { detailed: w.detailed }); + let mut next = frame_at(w.times[0]); + for k in 0..w.times.len() { + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + let Some(cur) = next.take() else { return fail(&format!("no frame at {:.3} s", w.times[k].seconds())) }; + let ((), nf) = rayon::join( + || an.push(&effectcraft_track::Frame { img: &cur.0, offset: cur.1 }), + || if k + 1 < w.times.len() { frame_at(w.times[k + 1]) } else { None }, + ); + next = nf; + let mut s = lock(&shared.state); + s.done = k as u64 + 1; + s.elapsed = t0.elapsed().as_secs_f64(); + s.fps = if s.elapsed > 0.0 { s.done as f64 / s.elapsed } else { 0.0 }; + } + (Arc::new(an.finish(lt0, fd)), true) + } + }; + lock(&shared.state).step = 2; + let solve = effectcraft_track::camtrack::solve(&tracks, &w.settings, Some(&shared.cancel)).map_err(|e| e.0); + if shared.cancel.load(Ordering::Relaxed) { + return cancelled(); + } + *lock(&shared.result) = Some(CamResult { tracks: fresh.then(|| (*tracks).clone()), solve }); + let mut s = lock(&shared.state); + s.elapsed = t0.elapsed().as_secs_f64(); + s.finished = true; +} + +impl Session { + pub fn is_camera_analyzing(&self) -> bool { + self.camera_job.as_ref().is_some_and(|j| !j.is_finished()) + } + + /// Progress of the running (or just finished) analysis. + pub fn camera_progress(&self) -> Option { + self.camera_job.as_ref().map(|j| j.progress()) + } + + /// The tracker being analysed: (comp, layer, effect uid). + pub fn camera_target(&self) -> Option<(ItemId, LayerId, Uid)> { + self.camera_job.as_ref().map(|j| (j.comp, j.layer, j.effect)) + } + + /// Cancel the running analysis (nothing is written). + pub fn stop_camera(&mut self) -> bool { + match &self.camera_job { + Some(j) if !j.is_finished() => { + j.shared.cancel.store(true, Ordering::Relaxed); + true + } + _ => false, + } + } + + /// Start analysing 3D Camera Tracker `effect` on `layer` (re-solving the stored tracks when + /// they are still valid). `wait`: block until done (wasm32 always does). Returns the number + /// of frames. + pub fn start_camera(&mut self, comp: ItemId, layer: LayerId, effect: Uid, wait: bool) -> Result { + if self.is_camera_analyzing() { + return Err("a 3D Camera Tracker analysis is already running".into()); + } + self.poll_camera_job(); + self.camera_pending.retain(|x| *x != (comp, layer, effect)); + let c = self.project.comp(comp).ok_or("no composition")?; + let l = c.layer(layer).ok_or("no layer")?; + let fx = l.effects().ok_or("the layer has no effects")?; + let index = fx.groups().position(|g| g.uid == effect).ok_or("no such effect")?; + let g = fx.groups().nth(index).filter(|g| is_tracker(g)).ok_or("not a 3D Camera Tracker")?; + if l.switches.adjustment { + return Err("3D Camera Tracker needs footage, not an adjustment layer".into()); + } + let times = analysis_times(c, l); + if times.len() < 2 { + return Err("the layer must be at least two frames long".into()); + } + let (w, h) = effectcraft_render::source_size(&self.project, l); + let size = if w == 0 { [c.width as f64, c.height as f64] } else { [w as f64, h as f64] }; + let params = static_params(g); + let key = tracks_key(&self.project, comp, l, g); + let settings = ct::settings(¶ms); + let solve_key = ct::solve_key(&settings); + let tracks = ct::tracks(¶ms).filter(|_| str_of(g, TRACKS_KEY) == key); + let n = times.len(); + let work = CamWork { + project: self.project.clone(), + footage: self.footage.clone(), + expr: self.expr.clone(), + cache: self.layer_cache.clone(), + comp, + layer, + effect_index: index, + size, + detailed: params.values.iter().any(|(k, v)| k.ends_with("detailedAnalysis") && v.as_bool()), + times, + tracks: tracks.clone(), + settings, + }; + let wait = wait || cfg!(target_arch = "wasm32"); + let shared = Arc::new(CameraShared::default()); + { + let mut st = lock(&shared.state); + st.solve_only = tracks.is_some(); + st.step = if tracks.is_some() { 2 } else { 1 }; + st.total = n as u64; + } + let thread = if wait { + run_work(work, &shared); + None + } else { + let sh = shared.clone(); + Some(std::thread::Builder::new().name("camera-track".into()).spawn(move || run_work(work, &sh)).map_err(|e| e.to_string())?) + }; + self.camera_job = Some(CameraJob { shared, thread, comp, layer, effect, key, solve_key }); + self.events.push(Event::ProjectChanged { revision: self.revision }); + self.poll_camera_job(); + Ok(n) + } + + /// Write a finished analysis into its effect (one undo step) and drop the job. + fn poll_camera_job(&mut self) -> bool { + let Some(job) = &self.camera_job else { return false }; + if !job.is_finished() { + return false; + } + let mut job = self.camera_job.take().expect("job"); + if let Some(t) = job.thread.take() { + let _ = t.join(); + } + let st = job.progress(); + let result = lock(&job.shared.result).take(); + match (result, &st.error) { + (Some(res), None) if !st.cancelled => { + let (cid, lid, uid) = (job.comp, job.layer, job.effect); + let (key, skey) = (job.key.clone(), job.solve_key.clone()); + let tracks_json = res.tracks.as_ref().map(CameraTracks::to_json); + let (solve_json, method, avg, msg, failed) = match &res.solve { + Ok(s) => ( + s.to_json(), + s.method_used.label().to_string(), + s.average_error, + format!( + "3D Camera Tracker: solved {} frame(s), {} points, average error {:.2} px ({:.1} s)", + s.frames.len(), + s.points.len(), + s.average_error, + st.elapsed + ), + false, + ), + Err(e) => (String::new(), String::new(), 0.0, format!("3D Camera Tracker: {e}"), true), + }; + let r = self.edit(if st.solve_only { "3D Camera Tracker Solve" } else { "3D Camera Tracker Analysis" }, None, |p, _| { + let g = p + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| crate::EngineError::Other("the 3D Camera Tracker was removed during analysis".into()))?; + let mut set = |m: &str, v: Value| { + if let Some(pr) = g.prop_mut(m) { + pr.value = v; + } + }; + if let Some(t) = &tracks_json { + set(TRACKS, Value::Str(t.clone())); + set(TRACKS_KEY, Value::Str(key.clone())); + } + set(SOLVE, Value::Str(solve_json.clone())); + set(SOLVE_KEY, Value::Str(skey.clone())); + set(METHOD_USED, Value::Str(method.clone())); + set(AVERAGE_ERROR, Value::Scalar(avg)); + Ok(()) + }); + let (msg, failed) = match r { + Ok(()) => (msg, failed), + Err(e) => (format!("3D Camera Tracker: {e}"), true), + }; + self.events.push(Event::Toast { message: msg, error: failed }); + } + _ => { + let msg = match &st.error { + Some(e) => format!("3D Camera Tracker analysis failed: {e}"), + None => "3D Camera Tracker analysis cancelled".into(), + }; + self.events.push(Event::Toast { message: msg, error: st.error.is_some() }); + self.bump(); + } + } + true + } + + /// Frontends call this every frame: finishes analyses and (with `auto`) starts queued + /// (re-)analyses in the background, like After Effects. + pub fn poll_camera(&mut self, auto: bool) -> bool { + let changed = self.poll_camera_job(); + if auto && self.camera_job.is_none() { + while let Some((c, l, e)) = self.camera_pending.first().copied() { + self.camera_pending.remove(0); + if self.start_camera(c, l, e, false).is_ok() { + return true; + } + } + } + changed + } +} + +// ---------------------------------------------------------------- solve → composition + +/// A tracker's solve placed in its composition. +pub struct Placed { + pub solve: Arc, + /// Canonical → comp world. + pub world: Similarity, + /// Comp pixels per layer pixel. + pub scale: f64, + /// Comp times of the analysed frames. + pub times: Vec, + pub comp_size: [f64; 2], +} + +impl Placed { + /// Comp-world position of a canonical point. + pub fn to_world(&self, x: V3) -> V3 { + self.world.apply(x) + } + /// Canonical position of a comp-world point. + pub fn from_world(&self, x: V3) -> V3 { + let r = &self.world; + linalg::scale(linalg::mtv(&r.r, linalg::sub(x, r.t)), 1.0 / r.s) + } + /// The 3D Tracker Camera on frame `k`: (position, orientation degrees, zoom). + pub fn camera(&self, k: usize) -> (V3, V3, f64) { + let c = &self.solve.frames[k]; + let m = linalg::mm(&self.world.r, &linalg::mt(&c.r())); + (self.to_world(c.center), linalg::euler_xyz(&m), c.focal * self.scale) + } + /// Analysed frame at comp time `t`. + pub fn frame_at(&self, t: Tick) -> usize { + self.times.iter().enumerate().min_by_key(|(_, x)| (x.0 - t.0).abs()).map(|(i, _)| i).unwrap_or(0).min(self.solve.frames.len().saturating_sub(1)) + } +} + +/// Comp pixels per layer pixel of `layer` (its 2D transform at its In point). +fn layer_scale(project: &Project, cid: ItemId, layer: &Layer) -> f64 { + let Some(comp) = project.comp(cid) else { return 1.0 }; + let ctx = EvalCtx::new(project, cid, comp, layer.in_point.max(Tick::ZERO)); + let m = ctx.world_matrix(layer).0; + let s = m[0][0].hypot(m[1][0]); + if s.is_finite() && s > 1e-9 { s } else { 1.0 } +} + +/// The solve of tracker `uid` on `lid`, placed in the comp. +pub fn placed(project: &Project, cid: ItemId, lid: LayerId, uid: Uid) -> Option { + let comp = project.comp(cid)?; + let layer = comp.layer(lid)?; + let g = layer.props.find_group(uid).filter(|g| is_tracker(g))?; + let solve = ct::solve(&static_params(g))?; + let scale = layer_scale(project, cid, layer); + let k0 = solve.first_solved()?; + let comp_size = [comp.width as f64, comp.height as f64]; + let world = solve.world(comp_size, solve.frames[k0].focal * scale); + let mut times = analysis_times(comp, layer); + times.truncate(solve.frames.len()); + Some(Placed { solve, world, scale, times, comp_size }) +} + +/// What to create from a target (`camera.createFromSolve`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum CreateKind { + Text, + Solid, + Null, + ShadowCatcher, + /// Only the camera. + Camera, +} + +impl CreateKind { + pub fn from_name(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '-', '_'], "").as_str() { + "text" => Some(CreateKind::Text), + "solid" => Some(CreateKind::Solid), + "null" => Some(CreateKind::Null), + "shadowcatcher" | "shadow" => Some(CreateKind::ShadowCatcher), + "camera" => Some(CreateKind::Camera), + _ => None, + } + } +} + +/// Rotation (layer → world) of a layer lying on a plane with unit `normal` (towards the camera), +/// its x axis along `right` projected on the plane: the layer's front (−z) faces the camera. +pub fn plane_orientation(normal: V3, right: V3) -> V3 { + let z = linalg::scale(normal, -1.0); + let mut x = linalg::sub(right, linalg::scale(z, linalg::dot(right, z))); + if linalg::norm(x) < 1e-9 { + x = if z[0].abs() < 0.9 { [1.0, 0.0, 0.0] } else { [0.0, 1.0, 0.0] }; + x = linalg::sub(x, linalg::scale(z, linalg::dot(x, z))); + } + let x = linalg::normalize(x); + let y = linalg::cross(z, x); + let m = linalg::from_cols(x, y, z); + norm_angles(linalg::euler_xyz(&m)) +} + +fn norm_angles(a: V3) -> V3 { + a.map(|v| { + let r = v.rem_euclid(360.0); + if (r - 360.0).abs() < 1e-9 { 0.0 } else { r } + }) +} + +/// Key `path` on `l` at every `(time, value)`. +pub(crate) fn key_all(l: &mut Layer, path: &str, keys: &[(Tick, Value)]) { + if let Some(pr) = l.props.prop_mut(path) { + pr.keys = keys.iter().map(|(t, v)| Keyframe::new(*t, v.clone())).collect(); + if let Some((_, v)) = keys.first() { + pr.value = v.clone(); + } + } +} + +/// Write the solved camera path into camera layer `l` (one-node, keyed on every analysed frame; +/// Zoom keyed for Variable Zoom). +pub(crate) fn write_camera(l: &mut Layer, pl: &Placed) { + l.auto_orient = AutoOrient::Off; + let n = pl.times.len().min(pl.solve.frames.len()); + let mut pos = Vec::with_capacity(n); + let mut ori = Vec::with_capacity(n); + let mut zoom = Vec::with_capacity(n); + let mut prev: Option = None; + for k in 0..n { + let (p, o, z) = pl.camera(k); + // Unwrap the angles so per-frame keys interpolate the short way. + let o = match prev { + None => norm_angles(o), + Some(q) => [0, 1, 2].map(|i| q[i] + (o[i] - q[i] + 180.0).rem_euclid(360.0) - 180.0), + }; + prev = Some(o); + let t = l.layer_time(pl.times[k]); + pos.push((t, Value::Vec3(p))); + ori.push((t, Value::Vec3(o))); + zoom.push((t, Value::Scalar(z))); + } + key_all(l, "transform/position", &pos); + key_all(l, "transform/orientation", &ori); + let variable = pl.solve.frames.iter().any(|f| (f.focal - pl.solve.frames[0].focal).abs() > 1e-6 * f.focal); + if variable { + key_all(l, "cameraOptions/zoom", &zoom); + } else if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/zoom") + { + pr.keys.clear(); + pr.value = z.clone(); + } + if let Some((_, z)) = zoom.first() + && let Some(pr) = l.props.prop_mut("cameraOptions/focusDistance") + { + pr.value = z.clone(); + } + // A one-node camera has no Point of Interest. + if let Some(tr) = l.props.sub_mut("transform") { + tr.children.retain(|c| !matches!(c, effectcraft_project::Node::Prop(p) if p.match_id == "poi")); + } + for rot in ["transform/rotationX", "transform/rotationY", "transform/rotation"] { + if let Some(pr) = l.props.prop_mut(rot) { + pr.keys.clear(); + pr.value = Value::Scalar(0.0); + } + } +} + +/// The target for `ids` on frame `k`: when fewer than three points are given, the nearest +/// solved neighbours (in 3D) complete the plane. +pub fn target_for(solve: &CameraSolve, ids: &[u32], k: usize) -> Option { + let mut use_ids: Vec = ids.iter().copied().filter(|i| solve.point(*i).is_some()).collect(); + if use_ids.is_empty() { + return None; + } + if use_ids.len() < 3 { + let c = solve.point(use_ids[0])?.pos; + let mut others: Vec<(f64, u32)> = solve.visible(k).filter(|p| !use_ids.contains(&p.id)).map(|p| (linalg::norm(linalg::sub(p.pos, c)), p.id)).collect(); + others.sort_by(|a, b| a.0.total_cmp(&b.0)); + for (_, id) in others.into_iter().take(6usize.saturating_sub(use_ids.len())) { + use_ids.push(id); + } + let mut t = solve.target(&use_ids, k)?; + // The target sits on the chosen point(s). + let pts: Vec = ids.iter().filter_map(|i| solve.point(*i)).map(|p| p.pos).collect(); + t.center = linalg::scale(pts.iter().fold([0.0; 3], |a, b| linalg::add(a, *b)), 1.0 / pts.len() as f64); + return Some(t); + } + solve.target(&use_ids, k) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plane_orientation_faces_the_normal() { + let n = linalg::normalize([0.2, -0.9, -0.3]); + let o = plane_orientation(n, [1.0, 0.0, 0.0]); + let m = linalg::from_euler_xyz(o); + // Layer −z (its front) is the normal. + let front = linalg::mv(&m, [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, n)) < 1e-9); + } +} diff --git a/crates/engine/src/commands/camera_cmds.rs b/crates/engine/src/commands/camera_cmds.rs new file mode 100644 index 000000000..5d454aed5 --- /dev/null +++ b/crates/engine/src/commands/camera_cmds.rs @@ -0,0 +1,574 @@ +//! 3D Camera Tracker commands: Animation ▸ Track Camera (`track.camera`), the effect's Analyze / +//! Cancel / Create Camera buttons, the viewer's track point selection and right-click menu +//! (Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … and Camera, +//! Create Multiple …, Delete Selected Points) and the status / points queries. + +use effectcraft_color::Label; +use effectcraft_effects::camera_tracker::{self as ct, CAMERA, DELETED, SOLVE}; +use effectcraft_keyframe::{Justify, TextDoc, Value as KV}; +use effectcraft_project::build; +use effectcraft_project::{ItemId, ItemKind, Layer, LayerId, LayerSource, LightKind, Project, Solid, Uid}; +use effectcraft_track::camtrack::linalg::{self, V3}; +use effectcraft_track::camtrack::{ShotType, SolveMethod, Target, point_color}; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, f_p, has_comp, layer_p, str_p}; +use crate::camera_track::{CreateKind, Placed, is_tracker, placed, plane_orientation, static_params, target_for, write_camera}; +use crate::{EngineError, Result, Session, cmd, query}; + +pub fn specs() -> Vec { + vec![ + cmd!( + "track.camera", + "Track Camera", + ["Animation"], + None, + "{layer?, shotType?: fixed|variable|specify, aov?: degrees, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}", + can_track, + track_camera + ), + cmd!("camera.analyze", "Analyze", [], None, "{layer?, effect?: uid|name|index, wait?: block until done}", has_comp, analyze), + cmd!("camera.cancel", "Cancel", [], None, "{}", is_analyzing, |s, _| Ok(json!({"stopped": s.stop_camera()}))), + query!("camera.solveStatus", "3D Camera Tracker Status", "{layer?, effect?}", status), + query!("camera.points", "3D Camera Tracker Points", "{layer?, effect?, time?: comp seconds} → visible solved points", points), + cmd!("camera.selectPoints", "Select Track Points", [], None, "{points: [id], add?, toggle?}", has_comp, select_points), + cmd!( + "camera.createFromSolve", + "Create from Camera Solve", + [], + None, + "{kind: text|solid|null|shadowCatcher|camera, points?: [id] (default: selected), target?: {center, normal, size?} (comp world), multiple?, layer?, effect?}", + has_comp, + create_from_solve + ), + cmd!("camera.create", "Create Camera", [], None, "{layer?, effect?}", has_comp, |s, p| create_from_solve(s, &with_kind(p, "camera"))), + cmd!("camera.setGroundPlane", "Set Ground Plane and Origin", [], None, "{points?: [id] (default: selected), layer?, effect?}", has_comp, set_ground), + cmd!("camera.deletePoints", "Delete Selected Points", [], None, "{points?: [id] (default: selected), layer?, effect?, wait?}", has_comp, delete_points), + ] +} + +fn with_kind(p: &Value, kind: &str) -> Value { + let mut q = p.clone(); + if !q.is_object() { + q = json!({}); + } + q["kind"] = json!(kind); + q +} + +fn can_track(s: &Session) -> std::result::Result<(), String> { + has_comp(s)?; + let c = s.active_comp().ok_or("no composition is open")?; + match s.state.selected_layers.first().and_then(|l| c.layer(*l)) { + Some(l) if l.has_video() && !l.switches.adjustment && !l.is_camera() && !l.is_light() => Ok(()), + Some(_) => Err("Track Camera needs a footage or composition layer".into()), + None => Err("select a layer to track".into()), + } +} + +fn is_analyzing(s: &Session) -> std::result::Result<(), String> { + if s.is_camera_analyzing() { Ok(()) } else { Err("no 3D Camera Tracker analysis is running".into()) } +} + +/// (comp, layer, effect uid) of a 3D Camera Tracker: `layer` / `effect` (uid, name or 1-based +/// index), else a selected one, else the first in the comp. +fn ct_p(s: &Session, p: &Value, cmd: &str) -> Result<(ItemId, LayerId, Uid)> { + let cid = super::comp_id(s, p)?; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let candidates: Vec = match p.get("layer") { + Some(v) => vec![super::resolve_layer(comp, v).ok_or_else(|| bad(cmd, format!("no layer {v}")))?], + None => { + let mut v: Vec = s.state.selected_props.iter().map(|(l, _)| *l).collect(); + v.extend(s.state.selected_layers.iter().copied()); + v.extend(comp.layers.iter().map(|l| l.id)); + v + } + }; + for lid in candidates { + let Some(fx) = comp.layer(lid).and_then(Layer::effects) else { continue }; + let g = match p.get("effect") { + Some(Value::Number(n)) => { + let n = n.as_u64().unwrap_or(0); + fx.groups().find(|g| g.uid == n).or_else(|| fx.groups().nth((n as usize).saturating_sub(1))) + } + Some(Value::String(name)) => fx.groups().find(|g| &g.name == name), + _ => s + .state + .selected_props + .iter() + .find_map(|(l, u)| (*l == lid).then(|| fx.groups().find(|g| g.uid == *u && is_tracker(g))).flatten()) + .or_else(|| fx.groups().find(|g| is_tracker(g))), + }; + if let Some(g) = g.filter(|g| is_tracker(g)) { + return Ok((cid, lid, g.uid)); + } + if p.get("layer").is_some() { + break; + } + } + Err(bad(cmd, "no 3D Camera Tracker found (apply Animation ▸ Track Camera first)")) +} + +fn shot_p(p: &Value) -> Result> { + match p.get("shotType") { + None => Ok(None), + Some(Value::Number(n)) => ShotType::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "shotType: 0..2")), + Some(Value::String(v)) => ShotType::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "shotType: fixed|variable|specify")), + Some(_) => Err(bad("track.camera", "shotType: fixed|variable|specify")), + } +} + +fn method_p(p: &Value) -> Result> { + match p.get("solveMethod") { + None => Ok(None), + Some(Value::Number(n)) => { + SolveMethod::ALL.get(n.as_u64().unwrap_or(99) as usize).copied().map(Some).ok_or_else(|| bad("track.camera", "solveMethod: 0..3")) + } + Some(Value::String(v)) => SolveMethod::from_name(v).map(Some).ok_or_else(|| bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + Some(_) => Err(bad("track.camera", "solveMethod: auto|typical|flat|tripod")), + } +} + +/// Animation ▸ Track Camera / the Tracker panel button: apply the 3D Camera Tracker to the layer +/// (reusing one it already has), set the given options and start the analysis. +fn track_camera(s: &mut Session, p: &Value) -> Result { + let (cid, lid) = layer_p(s, p, "track.camera")?; + let shot = shot_p(p)?; + let method = method_p(p)?; + let aov = f_p(p, "aov").or_else(|| f_p(p, "angleOfView")); + if aov.is_some_and(|a| !(1.0..=170.0).contains(&a)) { + return Err(bad("track.camera", "aov: 1..170 degrees")); + } + let detailed = b_p(p, "detailed"); + let existing = s.project.comp(cid).and_then(|c| c.layer(lid)).and_then(Layer::effects).and_then(|fx| fx.groups().find(|g| is_tracker(g)).map(|g| g.uid)); + let uid = match existing { + Some(u) => u, + None => { + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [lid.0], "comp": cid.0}))?; + r["effects"].get(0).and_then(Value::as_u64).ok_or_else(|| EngineError::Other("the effect could not be applied".into()))? + } + }; + if shot.is_some() || method.is_some() || aov.is_some() || detailed.is_some() { + s.edit("3D Camera Tracker Settings", None, |proj, _| { + let g = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + let mut set = |m: &str, v: KV| { + if let Some(pr) = g.prop_mut(m) { + pr.keys.clear(); + pr.value = v; + } + }; + if let Some(sh) = shot { + set("shotType", KV::Enum(ShotType::ALL.iter().position(|x| *x == sh).unwrap_or(0) as u32)); + } + if let Some(a) = aov { + set("horizontalAngleOfView", KV::Scalar(a)); + if shot.is_none() { + set("shotType", KV::Enum(2)); + } + } + if let Some(m) = method { + set("advanced/solveMethod", KV::Enum(SolveMethod::ALL.iter().position(|x| *x == m).unwrap_or(0) as u32)); + } + if let Some(d) = detailed { + set("advanced/detailedAnalysis", KV::Bool(d)); + } + Ok(()) + })?; + } + // Select the effect so the viewer shows its track points. + s.state.selected_props = vec![(lid, uid)]; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn analyze(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.analyze")?; + let wait = b_p(p, "wait").unwrap_or(false); + let n = s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + let mut out = json!({"layer": lid.0, "effect": uid, "frames": n, "running": s.is_camera_analyzing(), "progress": s.camera_progress()}); + if wait { + out["status"] = status(s, &json!({"layer": lid.0, "effect": uid, "comp": cid.0}))?; + } + Ok(out) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.poll_camera(false); + let mut out = json!({"running": s.is_camera_analyzing(), "progress": s.camera_progress(), "banner": s.camera_progress().map(|p| p.banner())}); + let Ok((cid, lid, uid)) = ct_p(s, p, "camera.solveStatus") else { return Ok(out) }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let g = layer.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + out["layer"] = json!(lid.0); + out["effect"] = json!(uid); + out["pending"] = json!(s.camera_pending.contains(&(cid, lid, uid))); + let tracks = ct::tracks(¶ms); + out["analyzed"] = json!(tracks.is_some()); + if let Some(t) = &tracks { + out["tracks"] = json!(t.tracks.len()); + out["frames"] = json!(t.frames); + } + let settings = ct::settings(¶ms); + out["shotType"] = json!(settings.shot.label()); + out["solveMethod"] = json!(settings.method.label()); + out["deleted"] = json!(settings.deleted.len()); + let solve = ct::solve(¶ms); + out["solved"] = json!(solve.is_some()); + if let Some(sv) = &solve { + let f = sv.focal(); + out["methodUsed"] = json!(sv.method_used.label()); + out["averageError"] = json!(sv.average_error); + out["points"] = json!(sv.points.len()); + out["solvedFrames"] = json!(sv.frames.iter().filter(|c| c.solved).count()); + out["focalLength"] = json!(f); + out["horizontalAngleOfView"] = json!(sv.hfov(f)); + out["groundPlane"] = json!(sv.ground.is_some()); + if let Some(pl) = placed(&s.project, cid, lid, uid) { + let k = pl.frame_at(s.time_of(cid)); + let (pos, ori, zoom) = pl.camera(k); + out["frame"] = json!(k); + out["camera"] = json!({"position": pos, "orientation": ori, "zoom": zoom}); + out["visible"] = json!(sv.visible(k).count()); + } + } + let cam = params.get(CAMERA).map(|v| v.as_f64() as u64).unwrap_or(0); + out["cameraLayer"] = if cam > 0 && comp.layer(LayerId(cam)).is_some_and(Layer::is_camera) { json!(cam) } else { Value::Null }; + out["selected"] = json!(s.state.camera_points); + Ok(out) +} + +/// The visible solved points at the current (or given) time, with their comp positions. +fn points(s: &mut Session, p: &Value) -> Result { + let (cid, lid, uid) = ct_p(s, p, "camera.points")?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad("camera.points", "the camera is not solved yet"))?; + let t = f_p(p, "time").map(effectcraft_time::Tick::from_seconds_f64).unwrap_or_else(|| s.time_of(cid)); + let k = pl.frame_at(t); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let layer = comp.layer(lid).ok_or(EngineError::NoComp)?; + let ctx = effectcraft_render::EvalCtx::new(&s.project, cid, comp, t); + let m = ctx.world_matrix(layer); + let list: Vec = ct::projected(&pl.solve, k) + .into_iter() + .map(|(id, q, z)| { + let c = m.apply(effectcraft_geom::vec3(q[0], q[1], 0.0)); + let sp = pl.solve.point(id).copied().unwrap_or_default(); + let col = point_color(id); + json!({"id": id, "comp": [c.x, c.y], "layer": q, "depth": z, "world": pl.to_world(sp.pos), "error": sp.error, "color": col, "selected": s.state.camera_points.contains(&id)}) + }) + .collect(); + Ok(json!({"frame": k, "points": list})) +} + +fn ids_p(s: &Session, p: &Value) -> Vec { + match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + Some(Value::Number(n)) => n.as_u64().map(|v| vec![v as u32]).unwrap_or_default(), + _ => s.state.camera_points.clone(), + } +} + +fn select_points(s: &mut Session, p: &Value) -> Result { + let ids: Vec = match p.get("points") { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(|v| v as u32).collect(), + _ => vec![], + }; + if b_p(p, "toggle").unwrap_or(false) { + for id in ids { + match s.state.camera_points.iter().position(|x| *x == id) { + Some(i) => { + s.state.camera_points.remove(i); + } + None => s.state.camera_points.push(id), + } + } + } else if b_p(p, "add").unwrap_or(false) { + for id in ids { + if !s.state.camera_points.contains(&id) { + s.state.camera_points.push(id); + } + } + } else { + s.state.camera_points = ids; + } + s.bump(); + Ok(json!({"selected": s.state.camera_points})) +} + +/// The target of a create / ground command in canonical solve coordinates. +fn target_p(s: &Session, p: &Value, pl: &Placed, k: usize, cmd: &str) -> Result { + if let Some(t) = p.get("target") { + let v3 = |k: &str| -> Option { + let a = t.get(k)?.as_array()?; + Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2).and_then(Value::as_f64).unwrap_or(0.0)]) + }; + let (c, n) = (v3("center").ok_or_else(|| bad(cmd, "target.center: [x, y, z]"))?, v3("normal").unwrap_or([0.0, -1.0, 0.0])); + let size = t.get("size").and_then(Value::as_f64).unwrap_or(100.0); + let nc = linalg::normalize(linalg::mtv(&pl.world.r, n)); + return Ok(Target { center: pl.from_world(c), normal: nc, size: size / pl.world.s }); + } + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points` / `target`)")); + } + target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane")) +} + +fn next_name(c: &effectcraft_project::Comp, base: &str) -> String { + let n = (1..).find(|i| !c.layers.iter().any(|l| l.name == format!("{base} {i}"))).unwrap_or(1); + format!("{base} {n}") +} + +fn place(l: &mut Layer, pos: V3, ori: V3, src: &Layer) { + l.switches.three_d = true; + l.in_point = src.in_point; + l.out_point = src.out_point; + if let Some(pr) = l.props.prop_mut("transform/position") { + pr.value = KV::Vec3(pos); + } + if let Some(pr) = l.props.prop_mut("transform/orientation") { + pr.value = KV::Vec3(ori); + } +} + +/// Insert `l` above the tracked layer. +fn insert_above(proj: &mut Project, cid: ItemId, above: LayerId, l: Layer) -> Result { + let comp = proj.comp_mut(cid).ok_or(EngineError::NoComp)?; + let at = comp.layers.iter().position(|x| x.id == above).unwrap_or(0); + let id = l.id; + comp.layers.insert(at, l); + Ok(id) +} + +fn add_solid(proj: &mut Project, comp: &effectcraft_project::Comp, name: &str, size: u32, color: [f32; 3]) -> Layer { + let folder = proj.folder_named("Solids").unwrap_or_else(|| proj.add_item("Solids", Label::Yellow, None, ItemKind::Folder)); + let sid = proj.add_item(name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: size, height: size, pixel_aspect: 1.0 })); + build::layer(proj, comp, name, LayerSource::Solid { item: sid }, (size, size), None) +} + +/// Create Text / Solid / Null / Shadow Catcher (and Light) on the target, plus the 3D Tracker +/// Camera when the tracker has none yet — one undo step. +fn create_from_solve(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.createFromSolve"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let kind = CreateKind::from_name(str_p(p, "kind").unwrap_or("null")).ok_or_else(|| bad(cmd, "kind: text|solid|null|shadowCatcher|camera"))?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet (analysis still running or failed)"))?; + let k = pl.frame_at(s.time_of(cid)); + let multiple = b_p(p, "multiple").unwrap_or(false); + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let src = comp.layer(lid).ok_or(EngineError::NoComp)?.clone(); + let g = src.props.find_group(uid).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let size_pct = effectcraft_effects::warp_stab::param(¶ms, "targetSize").map(|v| v.as_f64()).unwrap_or(100.0) / 100.0; + let existing_cam = params.get(CAMERA).map(|v| v.as_f64() as u64).filter(|id| *id > 0 && comp.layer(LayerId(*id)).is_some_and(Layer::is_camera)); + // Targets: one, or one per selected point (Create Multiple …) sharing the selection's plane. + let targets: Vec = if kind == CreateKind::Camera { + vec![] + } else { + let t = target_p(s, p, &pl, k, cmd)?; + if multiple { + let ids = ids_p(s, p); + ids.iter().filter_map(|i| pl.solve.point(*i)).map(|sp| Target { center: sp.pos, ..t }).collect() + } else { + vec![t] + } + }; + // Camera right vector on this frame (world), to align created layers with the view. + let cam = &pl.solve.frames[k]; + let right = linalg::mv(&pl.world.r, linalg::mtv(&cam.r(), [1.0, 0.0, 0.0])); + let label = match (kind, multiple) { + (CreateKind::Text, false) => "Create Text and Camera", + (CreateKind::Solid, false) => "Create Solid and Camera", + (CreateKind::Null, false) => "Create Null and Camera", + (CreateKind::ShadowCatcher, _) => "Create Shadow Catcher, Camera and Light", + (CreateKind::Text, true) => "Create Multiple Text Layers and Camera", + (CreateKind::Solid, true) => "Create Multiple Solids and Camera", + (CreateKind::Null, true) => "Create Multiple Nulls and Camera", + (CreateKind::Camera, _) => "Create Camera", + }; + let (camera, created) = s.edit(label, None, |proj, st| { + let mut created = vec![]; + // The camera: reuse the tracker's, else create "3D Tracker Camera" at the top. + let camera = match existing_cam { + Some(id) if kind != CreateKind::Camera => LayerId(id), + Some(id) => { + let l = proj.comp_mut(cid).and_then(|c| c.layer_mut(LayerId(id))).ok_or(EngineError::NoComp)?; + write_camera(l, &pl); + LayerId(id) + } + None => { + let mut l = build::layer(proj, &comp, "3D Tracker Camera", LayerSource::Camera, (comp.width, comp.height), None); + l.in_point = src.in_point; + l.out_point = src.out_point; + write_camera(&mut l, &pl); + let id = l.id; + proj.comp_mut(cid).ok_or(EngineError::NoComp)?.layers.insert(0, l); + if let Some(pr) = proj.comp_mut(cid).and_then(|c| c.layer_mut(lid)).and_then(|l| l.props.find_group_mut(uid)).and_then(|g| g.prop_mut(CAMERA)) { + pr.value = KV::Scalar(id.0 as f64); + } + id + } + }; + for t in &targets { + let center = pl.to_world(t.center); + let normal = linalg::normalize(linalg::mv(&pl.world.r, t.normal)); + let ori = plane_orientation(normal, right); + let size = (t.size * pl.world.s * size_pct).max(1.0); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + match kind { + CreateKind::Text => { + let mut l = build::layer(proj, &cur, "Text", LayerSource::Text, (comp.width, comp.height), None); + let doc = TextDoc { text: "Text".into(), justify: Justify::Center, size: (size * 0.35).clamp(4.0, 2000.0), ..Default::default() }; + if let Some(pr) = l.props.prop_mut("text/sourceText") { + pr.value = KV::Text(Box::new(doc)); + } + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Solid => { + let name = next_name(&cur, "Track Solid"); + let c = point_color((created.len() as u32).wrapping_add(7)); + let mut l = add_solid(proj, &cur, &name, size.round().max(1.0) as u32, c); + place(&mut l, center, ori, &src); + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::Null => { + let name = next_name(&cur, "Track Null"); + let mut l = build::layer(proj, &cur, &name, LayerSource::Null, (comp.width, comp.height), None); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("transform/scale") { + pr.value = KV::Vec3([size, size, size]); + } + created.push(insert_above(proj, cid, lid, l)?); + } + CreateKind::ShadowCatcher => { + let name = next_name(&cur, "Shadow Catcher"); + let mut l = add_solid(proj, &cur, &name, (size * 4.0).round().max(1.0) as u32, [1.0, 1.0, 1.0]); + place(&mut l, center, ori, &src); + if let Some(pr) = l.props.prop_mut("materialOptions/acceptsShadows") { + pr.value = KV::Enum(2); + } + if let Some(pr) = l.props.prop_mut("materialOptions/castsShadows") { + pr.value = KV::Enum(0); + } + let sc = created.len(); + created.push(insert_above(proj, cid, lid, l)?); + let cur = proj.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let lname = next_name(&cur, "Light"); + let mut light = build::layer(proj, &cur, &lname, LayerSource::Light { kind: LightKind::Point }, (comp.width, comp.height), None); + light.in_point = src.in_point; + light.out_point = src.out_point; + let lp = linalg::add(center, linalg::add(linalg::scale(normal, size * 2.0), linalg::scale(right, size))); + if let Some(pr) = light.props.prop_mut("transform/position") { + pr.value = KV::Vec3(lp); + } + if let Some(pr) = light.props.prop_mut("lightOptions/castsShadows") { + pr.value = KV::Bool(true); + } + let _ = sc; + created.push(insert_above(proj, cid, created[created.len() - 1], light)?); + } + CreateKind::Camera => {} + } + } + st.selected_layers = if created.is_empty() { vec![camera] } else { created.clone() }; + st.selected_props.clear(); + Ok((camera, created)) + })?; + let t0 = targets.first().map( + |t| json!({"center": pl.to_world(t.center), "normal": linalg::normalize(linalg::mv(&pl.world.r, t.normal)), "size": t.size * pl.world.s * size_pct}), + ); + Ok(json!({"camera": camera.0, "layers": created.iter().map(|l| l.0).collect::>(), "target": t0})) +} + +fn set_ground(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.setGroundPlane"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let pl = placed(&s.project, cid, lid, uid).ok_or_else(|| bad(cmd, "the camera is not solved yet"))?; + let k = pl.frame_at(s.time_of(cid)); + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let t = target_for(&pl.solve, &ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + let mut sv = (*pl.solve).clone(); + let mut g = sv.ground_from(&ids, k).ok_or_else(|| bad(cmd, "those points do not define a plane"))?; + g.origin = t.center; + sv.ground = Some(g); + let json = sv.to_json(); + s.edit("Set Ground Plane and Origin", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(SOLVE)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + Ok(json!({"origin": g.origin, "normal": g.normal})) +} + +/// Delete points (and, with Auto-delete Points Across Time, the points of other tracks on the +/// same 3D feature), then re-solve. +fn delete_points(s: &mut Session, p: &Value) -> Result { + let cmd = "camera.deletePoints"; + let (cid, lid, uid) = ct_p(s, p, cmd)?; + let ids = ids_p(s, p); + if ids.is_empty() { + return Err(bad(cmd, "select track points (or give `points`)")); + } + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?; + let g = comp.layer(lid).and_then(|l| l.props.find_group(uid)).ok_or(EngineError::NoComp)?; + let params = static_params(g); + let mut del = ct::deleted(¶ms); + let auto = effectcraft_effects::warp_stab::param(¶ms, "advanced/autoDeletePoints").is_some_and(KV::as_bool); + let mut add = ids.clone(); + if auto && let Some(sv) = ct::solve(¶ms) { + // Same feature tracked again at other times: solved within 1 % of its depth. + let cam0 = sv.first_solved().map(|k| sv.frames[k].center).unwrap_or([0.0; 3]); + for id in &ids { + let Some(pt) = sv.point(*id) else { continue }; + let r = linalg::norm(linalg::sub(pt.pos, cam0)) * 0.01; + for q in &sv.points { + let disjoint = q.last < pt.first || q.first > pt.last; + if q.id != *id && disjoint && linalg::norm(linalg::sub(q.pos, pt.pos)) < r { + add.push(q.id); + } + } + } + } + for id in add { + if !del.contains(&id) { + del.push(id); + } + } + del.sort_unstable(); + let json = serde_json::to_string(&del).unwrap_or_default(); + s.edit("Delete Track Points", None, |proj, _| { + let pr = proj + .comp_mut(cid) + .and_then(|c| c.layer_mut(lid)) + .and_then(|l| l.props.find_group_mut(uid)) + .and_then(|g| g.prop_mut(DELETED)) + .ok_or_else(|| EngineError::Other("no 3D Camera Tracker".into()))?; + pr.value = KV::Str(json); + Ok(()) + })?; + s.state.camera_points.retain(|x| !del.contains(x)); + // Re-solve now (After Effects re-solves after deleting points). + let wait = b_p(p, "wait").unwrap_or(false); + if !s.is_camera_analyzing() { + s.start_camera(cid, lid, uid, wait).map_err(EngineError::Other)?; + } + Ok(json!({"deleted": del, "running": s.is_camera_analyzing()})) +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 4537752f6..04034281e 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -4,6 +4,7 @@ mod anim; pub mod anim_tools; mod animation; +mod camera_cmds; mod comp; mod comp_more; mod edit; @@ -129,6 +130,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(track::specs()); v.extend(mask_interp::specs()); v.extend(warp_cmds::specs()); + v.extend(camera_cmds::specs()); v.extend(paint::specs()); v.extend(puppet::specs()); v.extend(settings::specs()); diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index f670ffd76..74de3aecf 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -59,7 +59,6 @@ pub fn specs() -> Vec { stub!("keys.audioToKeyframes", "Convert Audio to Keyframes", ["Animation", "Keyframe Assistant"], "{}"), stub!("keys.rpfCameraImport", "RPF Camera Import", ["Animation", "Keyframe Assistant"], "{}"), stub!("text.animatorFontAxes", "Variable Font Axes", ["Animation", "Animate Text"], "{}"), - stub!("track.camera", "Track Camera", ["Animation"], "{}"), // Workspaces and panels that don't exist yet. stub!("window.unavailablePanel", "Panel", [], "{panel}"), ] diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index ba34488ab..fbd2b391e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -9,6 +9,7 @@ //! (compositions are `Arc`s, so untouched comps are shared). pub mod autosave; +pub mod camera_track; pub mod commands; pub mod config; pub mod demo; @@ -41,6 +42,7 @@ pub use effectcraft_project as project; pub use effectcraft_render as render; pub use effectcraft_text as text; pub use effectcraft_time as time; +pub use effectcraft_track as track; pub use render_queue::{ExportJob, ExportResult, Exporter, JobState}; #[derive(Debug, thiserror::Error)] @@ -170,6 +172,9 @@ pub struct EditorState { /// Tracker panel ▸ Method in mask mode (Track Mask). #[serde(default)] pub mask_track_method: mask_track::MaskMethod, + /// 3D Camera Tracker: the selected track points (ids) in the viewer. + #[serde(default)] + pub camera_points: Vec, /// Mask Interpolation panel options. #[serde(default)] pub mask_interp: commands::mask_interp::MaskInterpOptions, @@ -263,6 +268,10 @@ pub struct Session { pub warp_job: Option, /// Warp Stabilizers waiting for (re-)analysis: (comp, layer, effect uid). pub warp_pending: Vec<(ItemId, LayerId, Uid)>, + /// The running (or finished, not yet polled) 3D Camera Tracker analysis. + pub camera_job: Option, + /// 3D Camera Trackers waiting for (re-)analysis: (comp, layer, effect uid). + pub camera_pending: Vec<(ItemId, LayerId, Uid)>, pub events: Vec, /// Commands executed: (id, params). pub journal: Vec<(String, Value)>, @@ -308,6 +317,8 @@ impl Default for Session { mask_job: None, warp_job: None, warp_pending: vec![], + camera_job: None, + camera_pending: vec![], events: vec![], journal: vec![], layer_cache: Arc::new(LayerCache::default()), @@ -374,6 +385,12 @@ impl Session { self.warp_pending.push(w); } } + // So are 3D Camera Tracker tracks (and solves made with other settings). + for w in camera_track::invalidate(&before, &mut p) { + if !self.camera_pending.contains(&w) { + self.camera_pending.push(w); + } + } let same = merge.is_some() && merge.map(str::to_string) == self.history.merge_key; if !same { self.history.undo.push((label.to_string(), before)); @@ -551,6 +568,9 @@ impl Session { self.stop_warp(); self.warp_job = None; self.warp_pending.clear(); + self.stop_camera(); + self.camera_job = None; + self.camera_pending.clear(); p.fix_next_id(); self.project = Arc::new(p); self.history = History::default(); @@ -574,6 +594,8 @@ mod tests_3d; #[cfg(test)] mod tests_anim_tools; #[cfg(test)] +mod tests_camera_track; +#[cfg(test)] mod tests_effects; #[cfg(test)] mod tests_fidelity; diff --git a/crates/engine/src/tests_camera_track.rs b/crates/engine/src/tests_camera_track.rs new file mode 100644 index 000000000..5bd5539c1 --- /dev/null +++ b/crates/engine/src/tests_camera_track.rs @@ -0,0 +1,307 @@ +//! The 3D Camera Tracker end to end through the commands, on synthetic footage rendered in-test: +//! textured planes (a floor, a back wall and a box face) seen through a known moving camera. + +use effectcraft_effects::camera_tracker as ct; +use effectcraft_geom::vec3; +use effectcraft_project::LayerId; +use effectcraft_raster::Image; +use effectcraft_render::EvalCtx; +use effectcraft_render::three_d::camera::active_camera; +use effectcraft_track::camtrack::linalg::{self, M3, V3}; +use serde_json::{Value, json}; + +use crate::Session; +use crate::tests_track::{H, W, frame_time, setup, texture}; + +const F: f64 = 260.0; + +/// The true camera on frame `f`: (world → camera rotation, centre). +fn true_camera(f: u32) -> (M3, V3) { + let t = f as f64 / 24.0; + let c = [-1.0 + 2.0 * t, -0.3 + 0.2 * (t * 3.0).sin(), 0.5 * t]; + let fwd = linalg::normalize(linalg::sub([0.2 * t, 0.4, 8.0], c)); + let right = linalg::normalize(linalg::cross([0.0, 1.0, 0.0], fwd)); + ([right, linalg::cross(fwd, right), fwd], c) +} + +/// Planes: (point, normal, u axis, v axis, half extent or 0 = infinite). +const PLANES: [(V3, V3, V3, V3, f64); 3] = [ + ([0.0, 1.5, 0.0], [0.0, -1.0, 0.0], [1.0, 0.0, 0.0], [0.0, 0.0, 1.0], 0.0), + ([0.0, 0.0, 10.0], [0.0, 0.0, -1.0], [1.0, 0.0, 0.0], [0.0, 1.0, 0.0], 0.0), + ([-1.2, 0.2, 6.0], [0.3, 0.0, -1.0], [0.957_826, 0.0, 0.287_348], [0.0, 1.0, 0.0], 1.1), +]; + +fn scene_frame(f: u32) -> Image { + let (r, c) = true_camera(f); + let mut img = Image::new(W, H); + for y in 0..H { + for x in 0..W { + let d = linalg::mtv(&r, [(x as f64 + 0.5 - W as f64 / 2.0) / F, (y as f64 + 0.5 - H as f64 / 2.0) / F, 1.0]); + let mut best = (f64::INFINITY, 0.2f32); + for (i, (p0, n, ua, va, ext)) in PLANES.iter().enumerate() { + let den = linalg::dot(*n, d); + if den.abs() < 1e-9 { + continue; + } + let tt = linalg::dot(*n, linalg::sub(*p0, c)) / den; + if tt <= 0.1 || tt >= best.0 { + continue; + } + let hit = linalg::add(c, linalg::scale(d, tt)); + let (u, v) = (linalg::dot(linalg::sub(hit, *p0), *ua), linalg::dot(linalg::sub(hit, *p0), *va)); + if *ext > 0.0 && (u.abs() > *ext || v.abs() > *ext) { + continue; + } + best = (tt, texture(u * 40.0, v * 40.0, 11 + i as u32)); + } + img.set(x, y, [best.1, best.1, best.1, 1.0]); + } + } + img +} + +/// The comp with the clip tracked (blocking) with `params`. +fn tracked(params: Value) -> (Session, LayerId) { + let (mut s, clip, _) = setup(scene_frame); + s.execute("layer.select", json!({"layers": [clip.0]})).unwrap(); + assert!(s.is_enabled("track.camera")); + let mut p = params; + p["wait"] = json!(true); + let r = s.execute_checked("track.camera", p).unwrap(); + assert_eq!(r["frames"], 25, "{r}"); + assert!(s.camera_job.is_none()); + (s, clip) +} + +fn status(s: &mut Session) -> Value { + s.execute("camera.solveStatus", json!({})).unwrap() +} + +#[test] +fn track_camera_solves_and_creates_layers_on_the_plane() { + let (mut s, clip) = tracked(json!({})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + let err = st["averageError"].as_f64().unwrap(); + assert!(err < 0.5, "average error {err}"); + let f = st["focalLength"].as_f64().unwrap(); + assert!((f - F).abs() / F < 0.05, "focal {f}"); + assert_ne!(st["methodUsed"], "Tripod Pan"); + // Method Used / Average Error are reported in the effect. + let g = s.active_comp().unwrap().layer(clip).unwrap().effects().unwrap().groups().next().unwrap().clone(); + assert!(matches!(g.prop(ct::METHOD_USED).map(|p| &p.value), Some(effectcraft_keyframe::Value::Str(m)) if !m.is_empty())); + + // Pick floor points on frame 12 (their 3D positions lie on y = 1.5 in the true scene: the + // ones seen below the horizon at the bottom of the frame). + s.execute("time.set", json!({"time": frame_time(12).seconds()})).unwrap(); + let pts = s.execute("camera.points", json!({})).unwrap(); + let list = pts["points"].as_array().unwrap(); + assert!(list.len() > 30, "{}", list.len()); + let floor: Vec = list.iter().filter(|p| p["comp"][1].as_f64().unwrap() > H as f64 * 0.8).map(|p| p["id"].as_u64().unwrap()).take(8).collect(); + assert!(floor.len() >= 3, "{floor:?}"); + s.execute("camera.selectPoints", json!({"points": floor})).unwrap(); + assert_eq!(s.state.camera_points.len(), floor.len()); + + // Create Solid and Camera: one undo step, camera + solid. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let cam = LayerId(r["camera"].as_u64().unwrap()); + let solid = LayerId(r["layers"][0].as_u64().unwrap()); + let comp = s.active_comp().unwrap().clone(); + assert!(comp.layer(cam).unwrap().is_camera()); + assert_eq!(comp.layer(cam).unwrap().name, "3D Tracker Camera"); + assert_eq!(comp.layers[0].id, cam); + let sl = comp.layer(solid).unwrap(); + assert!(sl.switches.three_d); + assert!(sl.name.starts_with("Track Solid")); + // The camera is keyed on every frame. + assert_eq!(comp.layer(cam).unwrap().props.prop("transform/position").unwrap().keys.len(), 25); + + // Every solved point projects through the created camera exactly where the solve puts it, + // and onto its tracked position (the footage fills the comp, so layer pixels are comp + // pixels): within 1 px for 95 % of the observations (the rest are the solve's outliers: + // features on occlusion edges), 0.5 px on average. + let cid = s.active_comp_id().unwrap(); + let uid = g.uid; + let pl = crate::camera_track::placed(&s.project, cid, clip, uid).unwrap(); + let tracks = ct::tracks(&crate::camera_track::static_params(&g)).unwrap(); + let mut errs = vec![]; + for k in [0u32, 6, 12, 18, 24] { + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(k)); + let camst = active_camera(&ctx); + for sp in pl.solve.visible(k as usize) { + let Some(obs) = tracks.track(sp.id).and_then(|t| t.at(k)) else { continue }; + let w = pl.to_world(sp.pos); + let q = camst.project(W as f64, H as f64, vec3(w[0], w[1], w[2])).unwrap(); + let own = pl.solve.frames[k as usize].project(pl.solve.size, sp.pos).unwrap(); + assert!((q.x - own[0]).hypot(q.y - own[1]) < 0.01, "camera layer disagrees with the solve"); + errs.push((q.x - obs[0]).hypot(q.y - obs[1])); + } + } + assert!(errs.len() > 100, "{}", errs.len()); + errs.sort_by(f64::total_cmp); + let p95 = errs[errs.len() * 95 / 100]; + let mean = errs.iter().sum::() / errs.len() as f64; + assert!(p95 < 1.0 && mean < 0.5, "p95 {p95} px, mean {mean} px"); + + // The solid lies on the selected points' plane. + let tr = sl.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + let ori = tr.get("orientation").unwrap().value.as_vec3(); + let normal = linalg::mv(&linalg::from_euler_xyz(ori), [0.0, 0.0, -1.0]); + let size = r["target"]["size"].as_f64().unwrap(); + for id in &s.state.camera_points { + let w = pl.to_world(pl.solve.point(*id).unwrap().pos); + let d = linalg::dot(linalg::sub(w, pos), normal).abs(); + assert!(d < 0.05 * size.max(1.0) + 1.0, "point {id} is {d} px off the plane"); + } + // The solid faces the camera. + let ctx = EvalCtx::new(&s.project, cid, &comp, frame_time(12)); + let eye = active_camera(&ctx).eye; + assert!(linalg::dot(normal, linalg::sub([eye.x, eye.y, eye.z], pos)) > 0.0); + + // Undo removes both layers; redo brings them back. + let before = comp.layers.len(); + assert!(s.undo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before - 2); + assert!(s.redo()); + assert_eq!(s.active_comp().unwrap().layers.len(), before); + + // A second create reuses the camera: Create Multiple Nulls and Camera. + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null", "multiple": true})).unwrap(); + assert_eq!(r["camera"].as_u64().unwrap(), cam.0); + assert_eq!(r["layers"].as_array().unwrap().len(), floor.len()); + // Text and Shadow Catcher (+ light). + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "text"})).unwrap(); + assert_eq!(r["layers"].as_array().unwrap().len(), 1); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "shadowCatcher"})).unwrap(); + let ids: Vec = r["layers"].as_array().unwrap().iter().map(|v| LayerId(v.as_u64().unwrap())).collect(); + let comp = s.active_comp().unwrap(); + assert!(comp.layer(ids[0]).unwrap().name.starts_with("Shadow Catcher")); + assert_eq!(comp.layer(ids[0]).unwrap().props.prop("materialOptions/acceptsShadows").unwrap().value.as_enum(), 2); + assert!(comp.layer(ids[1]).unwrap().is_light()); + assert!(s.execute_checked("camera.createFromSolve", json!({"kind": "teapot"})).is_err()); + + // Set Ground Plane and Origin: the target becomes the origin; a null created there sits at + // (0, 0, 0) lying on the X-Z plane (its front facing up, −y). + s.execute_checked("camera.setGroundPlane", json!({})).unwrap(); + assert_eq!(status(&mut s)["groundPlane"], true); + let r = s.execute_checked("camera.createFromSolve", json!({"kind": "null"})).unwrap(); + let null = LayerId(r["layers"][0].as_u64().unwrap()); + let tr = s.active_comp().unwrap().layer(null).unwrap().props.sub("transform").unwrap().clone(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(pos) < 1e-6, "{pos:?}"); + let up = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(up, [0.0, -1.0, 0.0])) < 1e-6, "{up:?}"); + + // An explicit target (comp world) places the layer there, its front facing the normal. + let r = s + .execute_checked("camera.createFromSolve", json!({"kind": "solid", "target": {"center": [10.0, 0.0, 20.0], "normal": [0.0, 0.0, -1.0], "size": 80.0}})) + .unwrap(); + let l = s.active_comp().unwrap().layer(LayerId(r["layers"][0].as_u64().unwrap())).unwrap().clone(); + let tr = l.props.sub("transform").unwrap(); + let pos = tr.get("position").unwrap().value.as_vec3(); + assert!(linalg::norm(linalg::sub(pos, [10.0, 0.0, 20.0])) < 1e-6, "{pos:?}"); + let front = linalg::mv(&linalg::from_euler_xyz(tr.get("orientation").unwrap().value.as_vec3()), [0.0, 0.0, -1.0]); + assert!(linalg::norm(linalg::sub(front, [0.0, 0.0, -1.0])) < 1e-6, "{front:?}"); + + // Serde keeps the analysis. + let back = effectcraft_project::Project::from_json(&s.project.to_json()).unwrap(); + let g2 = back.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap(); + let sv = ct::solve(&crate::camera_track::static_params(g2)).unwrap(); + assert!(sv.ground.is_some()); + assert_eq!(sv.points.len(), pl.solve.points.len()); +} + +#[test] +fn invalidation_resolve_cancel_and_undo() { + let (mut s, clip) = tracked(json!({"shotType": "specify", "aov": 2.0 * (W as f64 / 2.0 / F).atan().to_degrees()})); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["shotType"], "Specify Angle of View"); + assert!((st["focalLength"].as_f64().unwrap() - F).abs() < 0.01 * F); + let cid = s.active_comp_id().unwrap(); + let uid = st["effect"].as_u64().unwrap(); + // Undo removes the analysis (one step); redo restores it. + assert!(s.undo()); + assert_eq!(status(&mut s)["analyzed"], false); + assert!(s.redo()); + assert_eq!(status(&mut s)["solved"], true); + + // Changing the Solve Method keeps the tracks and only re-solves. + s.execute_checked("prop.set", json!({"layer": clip.0, "path": "effects/#1/advanced/solveMethod", "value": 1})).unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], true); + assert_eq!(st["solved"], false); + assert_eq!(st["pending"], true); + s.execute_checked("camera.analyze", json!({"wait": true})).unwrap(); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert_eq!(st["methodUsed"], "Typical"); + + // Deleting points re-solves without them. + let pts = s.execute("camera.points", json!({})).unwrap(); + let del: Vec = pts["points"].as_array().unwrap().iter().take(3).map(|p| p["id"].as_u64().unwrap()).collect(); + s.execute_checked("camera.deletePoints", json!({"points": del, "wait": true})).unwrap(); + s.poll_camera(false); + let st = status(&mut s); + assert_eq!(st["solved"], true, "{st}"); + assert!(st["deleted"].as_u64().unwrap() >= 3); + let g = s.project.comp(cid).unwrap().layer(clip).unwrap().props.find_group(uid).unwrap().clone(); + let sv = ct::solve(&crate::camera_track::static_params(&g)).unwrap(); + assert!(sv.points.iter().all(|p| !del.contains(&(p.id as u64)))); + + // Trimming the layer changes its frames: tracks and solve are cleared and queued. + s.edit("trim", None, |p, _| { + p.comp_mut(cid).unwrap().layer_mut(clip).unwrap().in_point = frame_time(2); + Ok(()) + }) + .unwrap(); + let st = status(&mut s); + assert_eq!(st["analyzed"], false); + assert_eq!(st["pending"], true); + + // A background analysis can be cancelled: nothing is written. + s.execute_checked("camera.analyze", json!({})).unwrap(); + assert!(s.is_camera_analyzing() || s.camera_job.is_some()); + let banner = status(&mut s)["banner"].as_str().map(str::to_string); + if s.is_camera_analyzing() { + assert!(banner.unwrap().starts_with("Analyzing in background (step 1 of 2)")); + } + s.execute("camera.cancel", json!({})).ok(); + let t0 = std::time::Instant::now(); + while s.camera_job.is_some() { + assert!(t0.elapsed().as_secs() < 120); + std::thread::sleep(std::time::Duration::from_millis(5)); + s.poll_camera(false); + } + // (When the job finished before the cancel landed it wrote a valid analysis.) + let st = status(&mut s); + assert_eq!(st["running"], false); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "shotType": "bogus"})).is_err()); + assert!(s.execute_checked("track.camera", json!({"layer": clip.0, "aov": 500})).is_err()); +} + +#[test] +fn effect_parameters_match_after_effects() { + let spec = effectcraft_effects::registry().iter().find(|e| e.id == ct::ID).unwrap(); + assert_eq!(spec.name, "3D Camera Tracker"); + assert_eq!(spec.category, "Perspective"); + let names: Vec<&str> = spec.params.iter().filter(|p| !matches!(p.ui, effectcraft_project::ParamUi::Hidden)).map(|p| p.name).collect(); + for n in [ + "Shot Type", + "Horizontal Angle of View", + "Show Track Points", + "Render Track Points", + "Track Point Size", + "Target Size", + "Solve Method", + "Detailed Analysis", + "Auto-delete Points Across Time", + "Hide Warning Banner", + ] { + assert!(names.contains(&n), "{n}"); + } + // The Animation menu entry is a real command now. + let c = crate::find_command("track.camera").unwrap(); + assert_eq!(c.label, "Track Camera"); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 141858caa..bde98add3 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -331,7 +331,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "file.createProxy"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/track/Cargo.toml b/crates/track/Cargo.toml index 9bd46e26c..4800e06eb 100644 --- a/crates/track/Cargo.toml +++ b/crates/track/Cargo.toml @@ -14,3 +14,4 @@ effectcraft-raster = { workspace = true } rayon = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +web-time = { workspace = true } diff --git a/crates/track/src/camtrack/bundle.rs b/crates/track/src/camtrack/bundle.rs index 84e8a5ad6..72cb2f174 100644 --- a/crates/track/src/camtrack/bundle.rs +++ b/crates/track/src/camtrack/bundle.rs @@ -173,6 +173,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> } let mut cur = cost(cams, pts, obs, o); let initial = cur; + let t0 = web_time::Instant::now(); + let mut tries = 0usize; + let dbg = std::env::var_os("EC_CAMTRACK_DEBUG").is_some() && l.n > 200; let mut lambda = 1e-4; let mut iters = 0; if l.n == 0 && o.fix_points { @@ -231,7 +234,10 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> for (ia, ja) in &jc { let wv = [w * (ja[0] * jx[0][0] + ja[1] * jx[1][0]), w * (ja[0] * jx[0][1] + ja[1] * jx[1][1]), w * (ja[0] * jx[0][2] + ja[1] * jx[1][2])]; - match ps.w.iter_mut().find(|e| e.0 == *ia) { + // Each observation of a point is on another camera: only the shared focal + // length repeats. + let found = if Some(*ia) == l.shared { ps.w.iter_mut().find(|e| e.0 == *ia) } else { None }; + match found { Some(e) => e.1 = add(e.1, wv), None => ps.w.push((*ia, wv)), } @@ -294,31 +300,67 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> // ---- Try steps with increasing damping. let mut accepted = false; for _try in 0..12 { + tries += 1; let mut s = base_s.clone(); let mut rhs = base_bc.clone(); for i in 0..n { let d = s[i * n + i]; s[i * n + i] = d + lambda * d.max(1e-6 * mean_diag); } - // Schur complement over the points. - let mut vinv: Vec> = vec![None; npts]; + // Schur complement over the points: S −= W V⁻¹ Wᵀ, rhs −= W V⁻¹ bp (in parallel + // chunks of points when the reduced system is small enough to copy per thread). + let vinv: Vec> = if o.fix_points { + vec![None; npts] + } else { + systems + .par_iter() + .map(|ps| { + let mut v = ps.v; + let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; + for k in 0..3 { + v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; + } + inv3(&v) + }) + .collect() + }; if !o.fix_points { - for (j, ps) in systems.iter().enumerate() { - let mut v = ps.v; - let vd = (v[0][0] + v[1][1] + v[2][2]) / 3.0; - for k in 0..3 { - v[k][k] += lambda * v[k][k].max(1e-6 * vd.max(1e-12)) + 1e-12; - } - let Some(vi) = inv3(&v) else { continue }; - vinv[j] = Some(vi); - let vb = mv(&vi, ps.bp); - let wv: Vec = ps.w.iter().map(|(_, w)| mv(&vi, *w)).collect(); - for (a, (ia, wa)) in ps.w.iter().enumerate() { - rhs[*ia] -= dot(*wa, vb); - for (ib, wb) in &ps.w { - s[ia * n + ib] -= dot(wv[a], *wb); + let reduce = |range: std::ops::Range, s: &mut [f64], rhs: &mut [f64]| { + for j in range { + let (Some(vi), ps) = (&vinv[j], &systems[j]) else { continue }; + let vb = mv(vi, ps.bp); + let wv: Vec = ps.w.iter().map(|(_, w)| mv(vi, *w)).collect(); + for (a, (ia, wa)) in ps.w.iter().enumerate() { + rhs[*ia] -= dot(*wa, vb); + for (ib, wb) in &ps.w[a..] { + let d = dot(wv[a], *wb); + s[ia * n + ib] -= d; + if ia != ib { + s[ib * n + ia] -= d; + } + } } } + }; + let chunks = rayon::current_num_threads().clamp(1, 8); + if n <= 1200 && npts >= 64 && chunks > 1 { + let per = npts.div_ceil(chunks); + let parts: Vec<(Vec, Vec)> = (0..chunks) + .into_par_iter() + .map(|t| { + let (mut ds, mut dr) = (vec![0.0; n * n], vec![0.0; n]); + reduce(t * per..((t + 1) * per).min(npts), &mut ds, &mut dr); + (ds, dr) + }) + .collect(); + for (ds, dr) in parts { + s.par_iter_mut().zip(ds.par_iter()).for_each(|(x, y)| *x += y); + for (x, y) in rhs.iter_mut().zip(dr) { + *x += y; + } + } + } else { + reduce(0..npts, &mut s, &mut rhs); } } let mut dc = rhs; @@ -363,9 +405,18 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> let gain = cur - c2; cur = c2; lambda = (lambda / 3.0).max(1e-9); - accepted = gain > 1e-10 * cur.max(1e-12); + accepted = gain > 1e-6 * cur.max(1e-12); if !accepted { // Converged (tiny improvement). + if dbg { + eprintln!( + "ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", + l.n, + npts, + obs.len(), + t0.elapsed().as_secs_f64() + ); + } return BaReport { initial_cost: initial, cost: cur, iterations: iters }; } break; @@ -376,6 +427,9 @@ pub fn adjust(cams: &mut [BaCam], pts: &mut [V3], obs: &[BaObs], o: &BaOpts) -> break; } } + if dbg { + eprintln!("ba n={} pts={} obs={} iters={iters} tries={tries} {:.2}s cost {initial:.1} -> {cur:.1}", l.n, npts, obs.len(), t0.elapsed().as_secs_f64()); + } BaReport { initial_cost: initial, cost: cur, iterations: iters } } diff --git a/crates/track/src/camtrack/features.rs b/crates/track/src/camtrack/features.rs index 311ec711d..30b019e54 100644 --- a/crates/track/src/camtrack/features.rs +++ b/crates/track/src/camtrack/features.rs @@ -89,7 +89,7 @@ impl TrackAnalyzer { let (cx, cy) = ((p[0] / cell) as usize, (p[1] / cell) as usize); !occ.get(cy * gw + cx).copied().unwrap_or(true) }; - let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.005, window: 2, border: 8 }; + let opts = CornerOpts { max_features: want - self.active.len().min(want), min_distance: md, quality: 0.001, window: 2, border: 8 }; for p in good_features(&pyr, &opts, Some(&free)) { let l = pyr.to_layer(p); let id = self.tracks.len() as u32; diff --git a/crates/track/src/camtrack/linalg.rs b/crates/track/src/camtrack/linalg.rs index a7067f262..b719cc8e5 100644 --- a/crates/track/src/camtrack/linalg.rs +++ b/crates/track/src/camtrack/linalg.rs @@ -298,6 +298,23 @@ pub fn sym_eigen(a: &[f64], n: usize) -> (Vec, Vec) { (vals, vecs) } +/// Dot product, four lanes (lets the compiler vectorise). +fn dot_n(a: &[f64], b: &[f64]) -> f64 { + let mut acc = [0.0f64; 4]; + let (ca, cb) = (a.chunks_exact(4), b.chunks_exact(4)); + let (ra, rb) = (ca.remainder(), cb.remainder()); + for (x, y) in ca.zip(cb) { + for k in 0..4 { + acc[k] += x[k] * y[k]; + } + } + let mut s = acc[0] + acc[1] + acc[2] + acc[3]; + for (x, y) in ra.iter().zip(rb) { + s += x * y; + } + s +} + /// Solve `A x = b` for a symmetric positive definite `A` (`n × n`, row-major) in place by /// Cholesky factorisation. Returns false when `A` is not positive definite. pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { @@ -313,16 +330,9 @@ pub fn cholesky_solve(a: &mut [f64], n: usize, b: &mut [f64]) -> bool { a[j * n + j] = d; let (head, tail) = a.split_at_mut((j + 1) * n); let rowj = &head[j * n..j * n + j]; - // Rows below j are independent. - use rayon::prelude::*; - let work = |row: &mut [f64]| { - let s: f64 = row[j] - row[..j].iter().zip(rowj).map(|(a, b)| a * b).sum::(); - row[j] = s / d; - }; - if n - j > 192 { - tail.par_chunks_mut(n).for_each(work); - } else { - tail.chunks_mut(n).for_each(work); + // Rows below j (sequential: per-column parallel dispatch costs more than it saves). + for row in tail.chunks_mut(n) { + row[j] = (row[j] - dot_n(&row[..j], rowj)) / d; } } // Forward: L y = b. diff --git a/crates/track/src/camtrack/mod.rs b/crates/track/src/camtrack/mod.rs index 056d7ad03..76c0637f6 100644 --- a/crates/track/src/camtrack/mod.rs +++ b/crates/track/src/camtrack/mod.rs @@ -13,7 +13,10 @@ //! the other keyframes are added incrementally (**resection** by robust pose refinement, //! **triangulation** of new tracks by the linear method of H&Z §12.2); a sparse //! **Levenberg–Marquardt bundle adjustment** with the Schur complement over the points -//! ([`bundle`]) refines everything; the frames between keyframes are resected last. +//! ([`bundle`]) refines everything; the frames between keyframes are resected last. Inlier +//! thresholds start from the analysis resolution and are re-estimated from the residuals +//! (median absolute residual) once the keyframes are solved, so features on moving objects, +//! occlusion edges and screen-locked overlays drop out. //! 3. **Focal length**: *Specify Angle of View* fixes it; *Fixed Angle of View* searches it //! (the reprojection error of the keyframe reconstruction over a log-spaced range of focal //! lengths, refined by golden-section search) and then adjusts it in the bundle adjustment @@ -22,7 +25,8 @@ //! 4. **Tripod pan** shots (no camera translation) are solved as a pure rotation: frame-to-frame //! rotations from the tracked rays (orthogonal Procrustes) refined by the same bundle //! adjustment with the camera centres fixed. *Auto Detect* solves both ways and keeps the -//! tripod solution when it explains the tracks as well as the general one. +//! tripod solution when it explains the tracks as well as the general one (a quick rotation fit +//! at the general solution's focal length skips the tripod search when it clearly fails). //! //! The camera model has square pixels, the principal point at the layer centre and no lens //! distortion. Camera space is x right, y down, z forward (After Effects' convention); a camera diff --git a/crates/track/src/camtrack/solver.rs b/crates/track/src/camtrack/solver.rs index da3a92908..85edd2bd8 100644 --- a/crates/track/src/camtrack/solver.rs +++ b/crates/track/src/camtrack/solver.rs @@ -37,8 +37,9 @@ struct Data { tracks: Vec<(u32, usize, Vec<[f64; 2]>)>, /// Per frame: (track index, position). by_frame: Vec>, - /// Tracking noise scale (pixels). - noise: f64, + /// Tracking noise scale (pixels): starts from the analysis resolution and is re-estimated + /// from the residuals once the keyframes are solved. + noise_bits: std::sync::atomic::AtomicU64, } impl Data { @@ -58,23 +59,31 @@ impl Data { } tracks.push((tr.id, tr.start as usize, pts)); } - Data { n, w: t.size[0], tracks, by_frame, noise: t.factor.max(1.0) } + let noise = (0.5 * t.factor).max(0.5); + Data { n, w: t.size[0], tracks, by_frame, noise_bits: std::sync::atomic::AtomicU64::new(noise.to_bits()) } } fn at(&self, ti: u32, k: usize) -> Option<[f64; 2]> { let (_, s, p) = &self.tracks[ti as usize]; k.checked_sub(*s).and_then(|i| p.get(i).copied()) } + fn noise(&self) -> f64 { + f64::from_bits(self.noise_bits.load(Ordering::Relaxed)) + } + fn set_noise(&self, v: f64) { + self.noise_bits.store(v.to_bits(), Ordering::Relaxed); + } fn inlier_thr(&self) -> f64 { - 4.0 * self.noise + 4.0 * self.noise() } fn huber(&self) -> f64 { - 1.0 * self.noise + self.noise() } } /// Parallax-based keyframes: a new keyframe when the median feature motion since the last -/// exceeds 3 % of the width, when fewer than 60 % of its features survive, or before the shared -/// features drop below 24. The first and last frames with features are keyframes. +/// exceeds 1.5 % of the width, when fewer than 70 % of its features survive, after 12 frames, or +/// before the shared features drop below 24. The first and last frames with features are +/// keyframes. fn keyframes(d: &Data) -> Vec { let Some(first) = (0..d.n).find(|k| d.by_frame[*k].len() >= 8) else { return vec![] }; let mut kf = vec![first]; @@ -97,7 +106,7 @@ fn keyframes(d: &Data) -> Vec { disp.sort_by(f64::total_cmp); let med = disp.get(disp.len() / 2).copied().unwrap_or(0.0); let ratio = disp.len() as f64 / base.len().max(1) as f64; - if med > 0.03 * d.w || ratio < 0.6 || disp.len() < 24 { + if med > 0.015 * d.w || ratio < 0.7 || disp.len() < 24 || j - last >= 12 { last = j; kf.push(j); } @@ -405,7 +414,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } let x1: Vec<[f64; 2]> = sh.iter().map(|s| [s.1[0] / f, s.1[1] / f]).collect(); let x2: Vec<[f64; 2]> = sh.iter().map(|s| [s.2[0] / f, s.2[1] / f]).collect(); - let thr = 2.0 * d.noise / f; + let thr = 2.0 * d.noise() / f; let mut cands: Vec = vec![]; if use_e && let Some((e, inl)) = geometry::ransac_essential(&x1, &x2, thr, 400, (a * 7919 + b) as u64) { let (p1, p2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = (0..x1.len()).filter(|i| inl[*i]).map(|i| (x1[i], x2[i])).unzip(); @@ -415,7 +424,7 @@ fn init_pair(d: &Data, a: usize, b: usize, f: f64, use_e: bool, use_h: bool) -> } if use_h { let (s1, s2): (Vec<[f64; 2]>, Vec<[f64; 2]>) = sh.iter().map(|s| (s.1, s.2)).unzip(); - if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise, 500, (a * 31 + b) as u64) { + if let Some(fit) = ransac(Model::Homography, &s1, &s2, 2.0 * d.noise(), 500, (a * 31 + b) as u64) { let h = fit.h.0; // Normalised: K⁻¹ H K with K = diag(f, f, 1). let hn = [[h[0][0], h[0][1], h[0][2] / f], [h[1][0], h[1][1], h[1][2] / f], [h[2][0] * f, h[2][1] * f, h[2][2]]]; @@ -651,6 +660,12 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo let try_general = matches!(s.method, SolveMethod::Auto | SolveMethod::Typical | SolveMethod::MostlyFlat); let try_tripod = matches!(s.method, SolveMethod::Auto | SolveMethod::TripodPan); + let t0 = web_time::Instant::now(); + let lap = |what: &str| { + if debug() { + eprintln!("{what}: {:.2} s", t0.elapsed().as_secs_f64()); + } + }; let mut results: Vec<(Attempt, f64)> = vec![]; if try_general { let f = match fixed_f { @@ -666,7 +681,20 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo results.push((a, f)); } } - if try_tripod && !cancelled() { + lap("general"); + // Auto Detect: a quick tripod check at the general solve's focal length; when a rotation + // clearly cannot explain the tracks, the tripod search is skipped. + let tripod_plausible = match (s.method, results.first()) { + (SolveMethod::Auto, Some((g, f))) => { + let quick = tripod(&d, &mini, *f, &tcfg, cancel).map(|a| a.err).unwrap_or(f64::INFINITY); + if debug() { + eprintln!("quick tripod check: {quick:.3} vs general {:.3}", g.err); + } + quick <= 3.0 * g.err + d.noise() + } + _ => true, + }; + if try_tripod && tripod_plausible && !cancelled() { let f = match fixed_f { Some(f) => f, None => search_focal(d.w, &|f| tripod(&d, &mini, f, &Cfg { per_frame_f: variable, ..tcfg }, cancel).map(|a| a.err).unwrap_or(f64::INFINITY)), @@ -696,7 +724,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if debug() { eprintln!("auto: general {eg:.4} tripod {et:.4} baseline/depth {:.4}", base / med); } - if et <= (eg * 1.25).max(eg + 0.05 * d.noise) || base / med < 0.01 { t } else { g } + if et <= (eg * 1.25).max(eg + 0.05 * d.noise()) || base / med < 0.01 { t } else { g } } (Some(g), None) => g, (None, Some(t)) => t, @@ -717,15 +745,44 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo (None, true) => FocalMode::PerCamera(F_PRIOR), (None, false) => FocalMode::Shared, }; + lap("tripod"); // Keyframes: adjust with the focal length free, reject, re-triangulate, adjust again. let kf_frames: Vec = r.cams.keys().copied().collect(); r.adjust(&d, &kf_frames, &cfg, focal, 40); r.reject(&d); + // Re-estimate the tracking noise from the inlier residuals (robustly, from their median: + // for 2D Gaussian noise of deviation σ the median distance is 1.1774 σ), then tighten the + // thresholds: observations of moving objects and occlusion edges become outliers. + let mut res: Vec = vec![]; + for (k, c) in &r.cams { + for (ti, uv) in &d.by_frame[*k] { + if r.outliers.contains(&(*ti, *k)) { + continue; + } + if let Some(e) = r.pts.get(ti).and_then(|x| bundle::residual(c, *x, *uv)) { + res.push(e[0].hypot(e[1])); + } + } + } + if res.len() > 20 { + res.sort_by(f64::total_cmp); + let sigma = res[res.len() / 2] / 1.1774; + let floor = 0.05 * tracks.factor.max(1.0); + d.set_noise((1.5 * sigma).clamp(floor, d.noise())); + if debug() { + eprintln!("noise: σ {sigma:.3} → {:.3}", d.noise()); + } + r.outliers.clear(); + r.reject(&d); + } r.triangulate(&d, kind, 1.0, 1.0); r.adjust(&d, &kf_frames, &cfg, focal, 40); + r.reject(&d); + r.adjust(&d, &kf_frames, &cfg, focal, 20); if cancelled() { return err("cancelled"); } + lap("keyframe adjustment"); // Every other frame: resect from the points (nearest registered camera first). let mut todo: Vec = (0..d.n).filter(|k| !r.cams.contains_key(k)).collect(); todo.sort_by_key(|k| r.cams.keys().map(|c| c.abs_diff(*k)).min().unwrap_or(usize::MAX)); @@ -746,6 +803,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo if cancelled() { return err("cancelled"); } + lap("resection"); // Points for the tracks that only lived between keyframes, then the final adjustment over // every frame (when that stays affordable). r.triangulate(&d, kind, 1.0, 1.0); @@ -786,6 +844,7 @@ pub fn solve(tracks: &CameraTracks, s: &SolveSettings, cancel: Option<&AtomicBoo } (m, _) => m, }; + lap("final adjustment"); Ok(finish(&d, tracks, s, r, method, kind)) } diff --git a/crates/track/src/camtrack/tests.rs b/crates/track/src/camtrack/tests.rs index 258161f98..4b5a83460 100644 --- a/crates/track/src/camtrack/tests.rs +++ b/crates/track/src/camtrack/tests.rs @@ -358,3 +358,81 @@ fn perf_long_clip() { eprintln!("300 frames, {} tracks: {:?} err {:.3} focal {:.1} in {:?}", tr.tracks.len(), s.method_used, s.average_error, s.focal(), t0.elapsed()); assert!(s.average_error < 0.5); } + +/// Solve tracks saved from the app (`EC_CAMTRACK_FILE=tracks.json cargo test -p effectcraft-track +/// --release solve_saved_tracks -- --ignored --nocapture`): a debugging aid. +#[test] +#[ignore] +fn solve_saved_tracks() { + let Some(path) = std::env::var_os("EC_CAMTRACK_FILE") else { return }; + let mut tr = CameraTracks::from_json(&std::fs::read_to_string(path).unwrap()).unwrap(); + if let Some(maxy) = std::env::var("EC_CAMTRACK_MAXY").ok().and_then(|v| v.parse::().ok()) { + tr.tracks.retain(|t| t.pts.iter().all(|p| p[1] < maxy)); + } + let shot = if std::env::var_os("EC_CAMTRACK_VARIABLE").is_some() { ShotType::VariableZoom } else { ShotType::FixedAngle }; + let t0 = std::time::Instant::now(); + // With the true cameras (`[{pos, poi, zoom}]` per frame, two-node), how consistent are the + // tracks themselves? + if let Some(tp) = std::env::var_os("EC_CAMTRACK_TRUTH") { + let v: Vec = serde_json::from_str(&std::fs::read_to_string(tp).unwrap()).unwrap(); + let g = |x: &serde_json::Value| -> V3 { [x[0].as_f64().unwrap(), x[1].as_f64().unwrap(), x[2].as_f64().unwrap()] }; + let cams: Vec<(super::geometry::Pose, f64)> = v + .iter() + .map(|c| { + let (pos, poi) = (g(&c["pos"]), g(&c["poi"])); + let fwd = normalize(sub(poi, pos)); + let d0 = [0.0, 1.0, 0.0]; + let down = normalize(sub(d0, scale(fwd, dot(d0, fwd)))); + let right = cross(down, fwd); + (super::geometry::Pose { r: [right, down, fwd], c: pos }, c["zoom"].as_f64().unwrap()) + }) + .collect(); + let (w, h) = (tr.size[0], tr.size[1]); + let mut means = vec![]; + let mut drop_ids = vec![]; + for t in &tr.tracks { + let views: Vec<(super::geometry::Pose, [f64; 2])> = t + .pts + .iter() + .enumerate() + .map(|(i, p)| { + let (pose, z) = cams[t.start as usize + i]; + (pose, [(p[0] as f64 - w / 2.0) / z, (p[1] as f64 - h / 2.0) / z]) + }) + .collect(); + let Some(x) = super::geometry::triangulate(&views) else { continue }; + let e: Vec = views.iter().filter_map(|(p, o)| super::geometry::reproj2(p, x, *o).map(|e| e.sqrt() * cams[0].1)).collect(); + means.push((e.iter().sum::() / e.len().max(1) as f64, t.pts[0])); + if std::env::var_os("EC_CAMTRACK_TRUTH_FILTER").is_some() && means.last().unwrap().0 > 1.0 { + drop_ids.push(t.id); + } + } + tr.tracks.retain(|t| !drop_ids.contains(&t.id)); + means.sort_by(|a, b| a.0.total_cmp(&b.0)); + eprintln!( + "true-camera track errors (px): median {:.2}; quartiles {:?}", + means[means.len() / 2].0, + [means[means.len() / 4].0, means[means.len() * 3 / 4].0] + ); + for m in means.iter().step_by(6) { + eprintln!(" {:.2} at {:?}", m.0, m.1); + } + } + let mut st = settings(shot, SolveMethod::Auto); + if let Some(a) = std::env::var("EC_CAMTRACK_AOV").ok().and_then(|v| v.parse::().ok()) { + st.shot = ShotType::SpecifyAngle; + st.hfov = a; + } + let s = solve(&tr, &st, None).expect("solve"); + eprintln!( + "{} frames, {} tracks: {:?} err {:.3} focal {:.1} (hfov {:.1}) points {} in {:?}", + tr.frames, + tr.tracks.len(), + s.method_used, + s.average_error, + s.focal(), + s.hfov(s.focal()), + s.points.len(), + t0.elapsed() + ); +} diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..d581ef35c 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -727,6 +727,11 @@ impl EffectcraftApp { self.session.poll_warp(true); ctx.request_repaint_after(std::time::Duration::from_millis(100)); } + // 3D Camera Tracker: likewise (tracking, then solving). + if self.session.camera_job.is_some() || !self.session.camera_pending.is_empty() { + self.session.poll_camera(true); + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } self.apply_prefs(&ctx); self.handle_events(&ctx); self.tick_autosave(&ctx); diff --git a/crates/ui-egui/src/panels/camera_tracker_ui.rs b/crates/ui-egui/src/panels/camera_tracker_ui.rs new file mode 100644 index 000000000..41fb7204f --- /dev/null +++ b/crates/ui-egui/src/panels/camera_tracker_ui.rs @@ -0,0 +1,438 @@ +//! 3D Camera Tracker in the Composition viewer and Effect Controls: +//! +//! - while the effect is selected, the solved 3D track points are drawn as coloured targets +//! sized by depth (or the 2D source tracks with Show Track Points ▸ 2D Source); +//! - hovering between three points shows the target plane they span (a disc in perspective); +//! click a point (Shift adds / toggles) or drag a marquee to select points, click inside a +//! triangle to select its three points; the selection's target is drawn; +//! - right-click: Set Ground Plane and Origin, Create Text / Solid / Null / Shadow Catcher … +//! and Camera, Create Multiple …, Delete Selected Points; Delete removes the selection; +//! - the "Analyzing in background (step 1 of 2)" / "Solving camera" banner; +//! - the Effect Controls block with Analyze / Cancel / Create Camera, Method Used and Average +//! Error. +//! +//! Everything is dispatched to the `camera.*` commands. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::geom::{Mat3, vec2 as gv2}; +use effectcraft_engine::keyframe::Value; +use effectcraft_engine::project::{Comp, Layer, LayerId, PropGroup, Uid}; +use effectcraft_engine::render::EvalCtx; +use effectcraft_engine::track::camtrack::{CameraSolve, CameraTracks, Target, point_color}; +use egui::{Align2, Color32, Pos2, Rect, Sense, Stroke, pos2, vec2}; +use serde_json::json; + +use super::viewer::ViewerMap; +use crate::theme::Tokens; +use crate::{EffectcraftApp, widgets}; + +/// The tracker whose points the viewer shows. +struct Active { + layer: LayerId, + uid: Uid, + solve: Option>, + tracks: Option>, + show_3d: bool, + point_size: f64, + target_size: f64, + /// Layer pixels → comp pixels. + l2c: Mat3, + frame: usize, +} + +/// A drawn point: (id, screen position, screen radius). +type Hit = (u32, Pos2, f32); + +fn tracker_group(l: &Layer, uid: Uid) -> Option<&PropGroup> { + l.props.find_group(uid).filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) +} + +/// The selected 3D Camera Tracker effect on an active layer (After Effects shows the points +/// while the effect is selected). +fn active(s: &Session, comp: &Comp, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) -> Option { + let time = ectx.time; + for (lid, uid) in &s.state.selected_props { + let Some(l) = comp.layer(*lid).filter(|l| l.is_active_at(time)) else { continue }; + let Some(g) = tracker_group(l, *uid) else { continue }; + let params = effectcraft_engine::camera_track::static_params(g); + let val = |id: &str| effectcraft_engine::effects::warp_stab::param(¶ms, id).cloned(); + let solve = ct::solve(¶ms); + let tracks = ct::tracks(¶ms); + let lt = l.layer_time(time).seconds(); + let frame = solve.as_ref().and_then(|sv| sv.frame_at(lt)).or_else(|| { + let t = tracks.as_ref()?; + (t.frame_duration > 0.0).then(|| ((lt - t.start) / t.frame_duration).round().clamp(0.0, t.frames.saturating_sub(1) as f64) as usize) + }); + return Some(Active { + layer: *lid, + uid: *uid, + solve, + tracks, + show_3d: val("showTrackPoints").map(|v| v.as_enum()).unwrap_or(1) == 1, + point_size: val("trackPointSize").map(|v| v.as_f64()).unwrap_or(100.0), + target_size: val("targetSize").map(|v| v.as_f64()).unwrap_or(100.0), + l2c: l2c(ectx, l), + frame: frame.unwrap_or(0), + }); + } + None +} + +fn to_screen(a: &Active, map: &ViewerMap, q: [f64; 2]) -> Pos2 { + let c = a.l2c.apply(gv2(q[0], q[1])); + map.to_screen([c.x, c.y]) +} + +fn col(id: u32) -> Color32 { + let c = point_color(id); + Color32::from_rgb((c[0] * 255.0) as u8, (c[1] * 255.0) as u8, (c[2] * 255.0) as u8) +} + +/// Comp-pixel scale of the layer → comp mapping. +fn l2c_scale(m: &Mat3) -> f32 { + m.0[0][0].hypot(m.0[1][0]) as f32 +} + +/// Draw the target disc (concentric rings in perspective) of `t` on frame `k`. +fn draw_target(p: &egui::Painter, a: &Active, map: &ViewerMap, sv: &CameraSolve, t: &Target) { + let Some(cam) = sv.frames.get(a.frame) else { return }; + let r = t.size * 0.5 * a.target_size / 100.0; + for (i, frac) in [1.0, 0.66, 0.33].into_iter().enumerate() { + let ring = ct::target_circle(t.center, t.normal, r * frac, 48); + let pts: Vec = ring.iter().filter_map(|x| cam.project(sv.size, *x)).map(|q| to_screen(a, map, q)).collect(); + if pts.len() < 3 { + return; + } + let fill = if i % 2 == 0 { Color32::from_rgba_unmultiplied(0xd0, 0x30, 0x30, 70) } else { Color32::from_rgba_unmultiplied(0xff, 0xff, 0xff, 50) }; + p.add(egui::Shape::convex_polygon(pts.clone(), fill, Stroke::new(1.0, Color32::from_rgb(0xe0, 0x40, 0x40)))); + } + if let Some(c) = cam.project(sv.size, t.center) { + let c = to_screen(a, map, c); + p.circle_filled(c, 3.0, Color32::from_rgb(0xff, 0xd0, 0x40)); + } +} + +/// The three drawn points forming a triangle around `hp` (nearest first). +fn triangle_at(hits: &[Hit], hp: Pos2) -> Option<[u32; 3]> { + let mut near: Vec<&Hit> = hits.iter().collect(); + near.sort_by(|x, y| x.1.distance_sq(hp).total_cmp(&y.1.distance_sq(hp))); + let near: Vec<&Hit> = near.into_iter().take(8).collect(); + let inside = |a: Pos2, b: Pos2, c: Pos2| { + let s = |p: Pos2, q: Pos2| (q.x - p.x) * (hp.y - p.y) - (q.y - p.y) * (hp.x - p.x); + let (d1, d2, d3) = (s(a, b), s(b, c), s(c, a)); + !((d1 < 0.0 || d2 < 0.0 || d3 < 0.0) && (d1 > 0.0 || d2 > 0.0 || d3 > 0.0)) + }; + for i in 0..near.len() { + for j in i + 1..near.len() { + for k in j + 1..near.len() { + let (a, b, c) = (near[i], near[j], near[k]); + // Skip slivers. + let area = ((b.1 - a.1).x * (c.1 - a.1).y - (b.1 - a.1).y * (c.1 - a.1).x).abs(); + if area > 30.0 && inside(a.1, b.1, c.1) { + return Some([a.0, b.0, c.0]); + } + } + } + } + None +} + +fn run(app: &mut EffectcraftApp, ctx: &egui::Context, cmd: &str, p: serde_json::Value) { + if let Err(e) = crate::menus::invoke(app, ctx, cmd, p) { + app.ui.status = e; + } +} + +/// The viewer overlay and its interaction (call after the viewer's own gestures so it takes +/// the clicks while a 3D Camera Tracker is selected). +pub fn viewer_hook( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + painter: &egui::Painter, + map: &ViewerMap, + ectx: &EvalCtx, + l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3, +) { + let ctx = ui.ctx().clone(); + let comp = ectx.comp; + banner(app, painter, map, ectx, l2c); + if !app.ui.viewer.show_layer_controls { + return; + } + let Some(a) = active(&app.session, comp, ectx, l2c) else { return }; + let selected = app.session.state.camera_points.clone(); + let lscale = l2c_scale(&a.l2c) * map.zoom; + let mut hits: Vec = vec![]; + match (&a.solve, a.show_3d) { + (Some(sv), true) => { + let pts = ct::projected(sv, a.frame); + let med = ct::median_depth(&pts); + for (id, q, z) in pts { + let sp = to_screen(&a, map, q); + if !map.area.expand(20.0).contains(sp) { + continue; + } + let r = (ct::target_radius(a.point_size, z, med) as f32 * lscale).clamp(2.0, 30.0); + let c = col(id); + let sel = selected.contains(&id); + if sel { + painter.circle_filled(sp, r, c); + painter.circle_stroke(sp, r + 1.5, Stroke::new(1.5, Color32::WHITE)); + } else { + painter.circle_stroke(sp, r, Stroke::new((r * 0.3).clamp(1.0, 3.0), c)); + painter.circle_filled(sp, (r * 0.25).max(1.0), c); + } + hits.push((id, sp, r)); + } + } + _ => { + // 2D source tracks (also shown before the solve finishes). + if let Some(tr) = &a.tracks { + for t in &tr.tracks { + let Some(q) = t.at(a.frame as u32) else { continue }; + let sp = to_screen(&a, map, q); + let c = col(t.id); + let r = (3.0 * a.point_size as f32 / 100.0).clamp(1.5, 12.0); + painter.line_segment([sp - vec2(r, r), sp + vec2(r, r)], Stroke::new(1.2, c)); + painter.line_segment([sp - vec2(r, -r), sp + vec2(r, -r)], Stroke::new(1.2, c)); + hits.push((t.id, sp, r)); + } + } + } + } + let Some(sv) = a.solve.clone() else { return }; + if app.ui.tool != crate::state::Tool::Selection { + return; + } + // Interaction over the image. + let resp = ui.interact(map.area, egui::Id::new("viewer-camera-tracker"), Sense::click_and_drag()); + app.auto.add("viewer.cameraTracker", map.area, "3D Camera Tracker points"); + for (id, sp, r) in hits.iter().take(400) { + app.auto.add(&format!("viewer.cameraTracker.point.{id}"), Rect::from_center_size(*sp, vec2(r * 2.0 + 4.0, r * 2.0 + 4.0)), "track point"); + } + let hover = resp.hover_pos(); + let point_at = |p: Pos2| hits.iter().filter(|h| h.1.distance(p) <= h.2 + 3.0).min_by(|x, y| x.1.distance(p).total_cmp(&y.1.distance(p))).map(|h| h.0); + let tri = hover.filter(|p| point_at(*p).is_none()).and_then(|p| triangle_at(&hits, p)); + let mid = egui::Id::new("viewer-camera-tracker-marquee"); + let marquee: Option = ui.data(|d| d.get_temp(mid)); + // Hover target / the selection's target. + if marquee.is_none() { + if let Some(t3) = tri + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &t3, a.frame) + { + let ps: Vec = t3.iter().filter_map(|i| hits.iter().find(|h| h.0 == *i)).map(|h| h.1).collect(); + if ps.len() == 3 { + painter.add(egui::Shape::closed_line(ps, Stroke::new(1.0, Color32::from_white_alpha(200)))); + } + draw_target(painter, &a, map, &sv, &t); + } else if !selected.is_empty() + && let Some(t) = effectcraft_engine::camera_track::target_for(&sv, &selected, a.frame) + { + draw_target(painter, &a, map, &sv, &t); + } + } + let mods = ui.input(|i| i.modifiers); + if resp.drag_started() + && let Some(p) = ui.input(|i| i.pointer.press_origin()) + { + ui.data_mut(|d| d.insert_temp(mid, p)); + } + if let (Some(start), Some(p)) = (marquee, resp.interact_pointer_pos()) { + let r = Rect::from_two_pos(start, p); + let t = app.tokens; + painter.rect_filled(r, 0.0, t.accent.gamma_multiply(0.12)); + painter.rect_stroke(r, 0.0, Stroke::new(1.0, t.accent), egui::StrokeKind::Middle); + if resp.drag_stopped() { + let ids: Vec = hits.iter().filter(|h| r.contains(h.1)).map(|h| h.0).collect(); + ui.data_mut(|d| d.remove::(mid)); + run(app, &ctx, "camera.selectPoints", json!({"points": ids, "add": mods.shift})); + } + } else if resp.drag_stopped() { + ui.data_mut(|d| d.remove::(mid)); + } + if resp.clicked() + && let Some(p) = resp.interact_pointer_pos() + { + match (point_at(p), triangle_at(&hits, p)) { + (Some(id), _) => run(app, &ctx, "camera.selectPoints", json!({"points": [id], "toggle": mods.shift})), + (None, Some(t3)) => run(app, &ctx, "camera.selectPoints", json!({"points": t3, "add": mods.shift})), + (None, None) => run(app, &ctx, "camera.selectPoints", json!({"points": []})), + } + } + // Right-click: the points under the cursor (selection, else the hovered triangle). + let cid = egui::Id::new("viewer-camera-tracker-menu-points"); + if resp.secondary_clicked() { + let pts: Vec = if !selected.is_empty() { + selected.clone() + } else if let Some(p) = resp.interact_pointer_pos() { + point_at(p).map(|i| vec![i]).or_else(|| triangle_at(&hits, p).map(|t| t.to_vec())).unwrap_or_default() + } else { + vec![] + }; + ui.data_mut(|d| d.insert_temp(cid, pts)); + } + let base = json!({"layer": a.layer.0, "effect": a.uid}); + resp.context_menu(|ui| { + let pts: Vec = ui.data(|d| d.get_temp(cid)).unwrap_or_default(); + let with = |kind: Option<&str>, multiple: bool| { + let mut q = base.clone(); + q["points"] = json!(pts); + if let Some(k) = kind { + q["kind"] = json!(k); + } + if multiple { + q["multiple"] = json!(true); + } + q + }; + let some = !pts.is_empty(); + let many = pts.len() > 1; + let mut pick: Option<(&str, serde_json::Value)> = None; + if ui.add_enabled(some, egui::Button::new("Set Ground Plane and Origin")).clicked() { + pick = Some(("camera.setGroundPlane", with(None, false))); + } + ui.separator(); + for (label, kind) in [ + ("Create Text and Camera", "text"), + ("Create Solid and Camera", "solid"), + ("Create Null and Camera", "null"), + ("Create Shadow Catcher, Camera and Light", "shadowCatcher"), + ] { + if ui.add_enabled(some, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), false))); + } + } + ui.separator(); + for (label, kind) in + [("Create Multiple Text Layers and Camera", "text"), ("Create Multiple Solids and Camera", "solid"), ("Create Multiple Nulls and Camera", "null")] + { + if ui.add_enabled(many, egui::Button::new(label)).clicked() { + pick = Some(("camera.createFromSolve", with(Some(kind), true))); + } + } + ui.separator(); + if ui.add_enabled(some, egui::Button::new("Delete Selected Points")).clicked() { + pick = Some(("camera.deletePoints", with(None, false))); + } + if let Some((cmd, q)) = pick { + ui.close(); + run(app, &ctx, cmd, q); + } + }); + if resp.hovered() + && !selected.is_empty() + && ui.input(|i| i.key_pressed(egui::Key::Delete) || i.key_pressed(egui::Key::Backspace)) + && !ctx.egui_wants_keyboard_input() + { + run(app, &ctx, "camera.deletePoints", json!({"layer": a.layer.0, "effect": a.uid, "points": selected})); + } +} + +/// "Analyzing in background (step 1 of 2)" / "Solving camera" over the analysed layer, and a +/// warning when the solve failed (unless Hide Warning Banner). +fn banner(app: &mut EffectcraftApp, painter: &egui::Painter, map: &ViewerMap, ectx: &EvalCtx, l2c: &dyn Fn(&EvalCtx, &Layer) -> Mat3) { + let comp = ectx.comp; + let time = ectx.time; + let cid = ectx.comp_id; + let draw = |app: &mut EffectcraftApp, layer: &Layer, text: String, color: Color32, id: &str| { + let m = l2c(ectx, layer); + let (w, h) = effectcraft_engine::render::source_size(&app.session.project, layer); + let c = m.apply(gv2(w as f64 / 2.0, h as f64 / 2.0)); + let at = map.to_screen([c.x, c.y]); + let galley = painter.layout_no_wrap(text.clone(), Tokens::medium(13.0), Color32::WHITE); + let r = Rect::from_center_size(at, galley.size() + vec2(28.0, 14.0)).intersect(map.area); + painter.rect_filled(r, 3.0, color); + painter.galley(pos2(r.center().x - galley.size().x / 2.0, r.center().y - galley.size().y / 2.0), galley, Color32::WHITE); + app.auto.add(id, r, &text); + }; + if let Some((cc, cl, _)) = app.session.camera_target() + && cc == cid + && let Some(pr) = app.session.camera_progress() + && let Some(layer) = comp.layer(cl) + && layer.is_active_at(time) + { + draw(app, layer, pr.banner(), Color32::from_rgba_unmultiplied(0x1d, 0x4f, 0x9c, 0xe0), "viewer.cameraBanner"); + return; + } + // A failed solve: tracks without a solve while nothing is running or queued. + for l in comp.layers.iter().filter(|l| l.is_active_at(time)) { + let Some(fx) = l.effects() else { continue }; + for g in fx.groups().filter(|g| matches!(&g.kind, effectcraft_engine::project::GroupKind::Effect { effect } if effect == ct::ID)) { + let params = effectcraft_engine::camera_track::static_params(g); + let hide = effectcraft_engine::effects::warp_stab::param(¶ms, "advanced/hideWarningBanner").is_some_and(Value::as_bool); + let solve_key = matches!(g.prop(ct::SOLVE_KEY).map(|p| &p.value), Some(Value::Str(s)) if !s.is_empty()); + let failed = solve_key && ct::tracks(¶ms).is_some() && ct::solve(¶ms).is_none(); + if failed && !hide && !app.session.camera_pending.iter().any(|(_, ll, u)| *ll == l.id && *u == g.uid) { + draw(app, l, "Unable to solve camera".into(), Color32::from_rgba_unmultiplied(0x9c, 0x2a, 0x1d, 0xe0), "viewer.cameraWarning"); + return; + } + } + } +} + +/// Effect Controls: Analyze / Cancel, Create Camera and the solve report. +#[allow(clippy::too_many_arguments)] +pub fn editor( + app: &mut EffectcraftApp, + ui: &mut egui::Ui, + p: &egui::Painter, + layer: &Layer, + g: &PropGroup, + r: Rect, + actions: &mut Vec<(String, serde_json::Value)>, +) { + let t = app.tokens; + let running = app.session.camera_target().is_some_and(|(_, l, u)| l == layer.id && u == g.uid) && app.session.is_camera_analyzing(); + let busy = app.session.is_camera_analyzing(); + let me = json!({"layer": layer.id.0, "effect": g.uid}); + let b1 = Rect::from_min_size(pos2(r.min.x + 24.0, r.min.y + 4.0), vec2(84.0, 22.0)); + let b2 = Rect::from_min_size(pos2(b1.max.x + 8.0, b1.min.y), vec2(84.0, 22.0)); + let b3 = Rect::from_min_size(pos2(b2.max.x + 8.0, b1.min.y), vec2(110.0, 22.0)); + if widgets::text_button(ui, b1, "Analyze", false, &t, egui::Id::new(("ct-analyze", g.uid))).clicked() && !busy { + actions.push(("camera.analyze".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.analyze", g.uid), b1, "Analyze"); + if running { + if widgets::text_button(ui, b2, "Cancel", false, &t, egui::Id::new(("ct-cancel", g.uid))).clicked() { + actions.push(("camera.cancel".into(), json!({}))); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.cancel", g.uid), b2, "Cancel"); + } + let params = effectcraft_engine::camera_track::static_params(g); + let solve = ct::solve(¶ms); + if solve.is_some() { + if widgets::text_button(ui, b3, "Create Camera", false, &t, egui::Id::new(("ct-camera", g.uid))).clicked() { + actions.push(("camera.create".into(), me.clone())); + } + app.auto.add(&format!("effectControls.cameraTracker.{}.createCamera", g.uid), b3, "Create Camera"); + } + let status = match app.session.camera_progress() { + Some(pr) if running => pr.banner(), + _ => match &solve { + Some(sv) => format!("Method Used: {} Average Error: {:.2} pixels", sv.method_used.label(), sv.average_error), + None if ct::tracks(¶ms).is_some() && app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => { + "Waiting to solve".into() + } + None if ct::tracks(¶ms).is_some() => "Unable to solve camera: try another Solve Method or Shot Type".into(), + None if app.session.camera_pending.iter().any(|(_, l, u)| *l == layer.id && *u == g.uid) => "Waiting to analyze".into(), + None => "Not analyzed: click Analyze".into(), + }, + }; + p.text(pos2(r.min.x + 24.0, r.min.y + 38.0), Align2::LEFT_CENTER, status, Tokens::ui(11.5), t.text_dim); + if let Some(sv) = &solve { + let f = sv.focal(); + p.text( + pos2(r.min.x + 24.0, r.min.y + 56.0), + Align2::LEFT_CENTER, + format!("{} solved points · Horizontal Angle of View {:.1}°", sv.points.len(), sv.hfov(f)), + Tokens::ui(11.5), + t.text_faint, + ); + } +} + +/// Height of [`editor`]. +pub const EDITOR_HEIGHT: f32 = 68.0; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index 0a19d57f7..11640d88c 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -545,6 +545,7 @@ fn editor_height(effect: &str, width: f32) -> f32 { match effect { "ec.color.curves" => 34.0 + curves_size(width) + 26.0, effectcraft_engine::effects::warp_stab::ID => 50.0, + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::EDITOR_HEIGHT, "ec.color.levels" | "ec.color.levelsic" => 34.0 + 80.0 + 58.0, "ec.color.autolevels" | "ec.color.autocontrast" | "ec.color.autocolor" => 24.0 + 80.0 + 12.0, _ => 0.0, @@ -920,6 +921,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { match effect.as_str() { "ec.color.curves" => curves_editor(app, ui, &bp, &layer, g, &ectx, er, &mut actions), effectcraft_engine::effects::warp_stab::ID => warp_editor(app, ui, &bp, &layer, g, er, &mut actions), + effectcraft_engine::effects::camera_tracker::ID => super::camera_tracker_ui::editor(app, ui, &bp, &layer, g, er, &mut actions), "ec.color.levels" | "ec.color.levelsic" => levels_editor(app, ui, &bp, &layer, g, effect, &ectx, er, &mut actions), _ => histogram_only(app, ui, &bp, g, er), } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..ead1fe99a 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -1,6 +1,7 @@ //! Panel bodies. pub mod anim_tools; +pub mod camera_tracker_ui; pub mod comp_settings; pub mod dialogs; pub mod dialogs_3d; diff --git a/crates/ui-egui/src/panels/tracker.rs b/crates/ui-egui/src/panels/tracker.rs index d1f456b26..364e114e5 100644 --- a/crates/ui-egui/src/panels/tracker.rs +++ b/crates/ui-egui/src/panels/tracker.rs @@ -130,11 +130,14 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { let cur = current(app); let can_track = has_comp && source.is_some(); - // Track Camera / Warp Stabilizer (not available) and Track Motion / Stabilize Motion. + // Track Camera / Warp Stabilizer and Track Motion / Stabilize Motion. let r1 = Rect::from_min_size(pos2(x0, y), vec2(half, 22.0)); let r2 = Rect::from_min_size(pos2(x0 + half + 6.0, y), vec2(half, 22.0)); let src_p = source.map(|l| json!({"layer": l.0})).unwrap_or_else(|| json!({})); - button(app, ui, r1, "Track Camera", false, "tracker.trackCamera"); + let can_camera = app.session.is_enabled("track.camera") && !app.session.is_camera_analyzing(); + if button(app, ui, r1, "Track Camera", can_camera, "tracker.trackCamera") { + run(app, &ctx, "track.camera", json!({})); + } let can_warp = app.session.is_enabled("track.warpStabilizer") && !app.session.is_warp_analyzing(); if button(app, ui, r2, "Warp Stabilizer", can_warp, "tracker.warpStabilizer") { run(app, &ctx, "track.warpStabilizer", json!({})); diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index d847edb85..c9dd70dbf 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -1144,6 +1144,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.session.open_comp(*item); } } + // 3D Camera Tracker points, targets and their menu (on top of the viewer's gestures while + // the effect is selected), and its analysis banner. + super::camera_tracker_ui::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); // Effect point controls and crosshair/eyedropper picks (on top of the viewer's gestures). crate::panels::effect_controls::viewer_hook(app, ui, &painter, &map, &ectx, &|c, l| l2c(c, l).0); diff --git a/crates/ui-egui/tests/ui_camera_tracker.rs b/crates/ui-egui/tests/ui_camera_tracker.rs new file mode 100644 index 000000000..45374fba4 --- /dev/null +++ b/crates/ui-egui/tests/ui_camera_tracker.rs @@ -0,0 +1,175 @@ +//! Headless checks for the 3D Camera Tracker UI (egui_kittest, UI logic only): the viewer's track +//! points and selection, the Effect Controls buttons and the Tracker panel's Track Camera button. +//! The solve is a synthetic one written into the effect (the analysis itself is tested in the +//! engine and track crates). + +use effectcraft_engine::Session; +use effectcraft_engine::effects::camera_tracker as ct; +use effectcraft_engine::track::camtrack::{CameraSolve, ShotType, SolveMethod, SolvedFrame, SolvedPoint}; +use effectcraft_ui_egui::EffectcraftApp; +use effectcraft_ui_egui::dock::PanelKind; +use egui_kittest::Harness; +use serde_json::json; + +const W: f64 = 640.0; +const H: f64 = 360.0; + +fn app() -> (EffectcraftApp, u64, u64) { + let mut s = Session::default(); + s.execute("comp.new", json!({"name": "Shot", "width": W, "height": H, "duration": 2})).unwrap(); + let plate = s.execute("layer.newSolid", json!({"name": "Plate", "color": "#406080"})).unwrap()["layer"].as_u64().unwrap(); + let r = s.execute("effect.apply", json!({"effect": ct::ID, "layers": [plate]})).unwrap(); + let uid = r["effects"][0].as_u64().unwrap(); + // A synthetic solve: a static camera (focal 600) looking at a 6 × 4 grid of points on z = 1. + let frames = 60; + let solve = CameraSolve { + version: 1, + size: [W, H], + start: 0.0, + frame_duration: 1.0 / 30.0, + shot: ShotType::FixedAngle, + method_used: SolveMethod::Typical, + average_error: 0.21, + frames: vec![SolvedFrame { rot: [0.0; 3], center: [0.0; 3], focal: 600.0, solved: true }; frames], + points: (0..24) + .map(|i| SolvedPoint { + id: i, + pos: [((i % 6) as f64 - 2.5) * 0.15, ((i / 6) as f64 - 1.5) * 0.12, 1.0 + 0.01 * (i % 3) as f64], + error: 0.2, + first: 0, + last: frames as u32 - 1, + }) + .collect(), + ground: None, + }; + s.execute("prop.set", json!({"layer": plate, "path": "effects/#1/solve", "value": solve.to_json()})).unwrap(); + s.camera_pending.clear(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + s.execute("layer.select", json!({"layers": [plate]})).unwrap(); + s.state.selected_props = vec![(effectcraft_engine::project::LayerId(plate), uid)]; + (EffectcraftApp::new(s), plate, uid) +} + +fn ids(h: &Harness<'_, EffectcraftApp>) -> Vec { + h.state().auto.previous.iter().chain(h.state().auto.elements.iter()).map(|e| e.id.clone()).collect() +} + +fn rect(h: &Harness<'_, EffectcraftApp>, id: &str) -> egui::Rect { + let e = h.state().auto.find(id).unwrap_or_else(|| panic!("no {id}")); + egui::Rect::from_min_size(egui::pos2(e.rect[0], e.rect[1]), egui::vec2(e.rect[2], e.rect[3])) +} + +fn click(h: &mut Harness<'_, EffectcraftApp>, at: egui::Pos2, modifiers: egui::Modifiers) { + h.input_mut().events.push(egui::Event::ModifiersChanged(modifiers)); + h.input_mut().events.push(egui::Event::PointerMoved(at)); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: true, modifiers }); + h.input_mut().events.push(egui::Event::PointerButton { pos: at, button: egui::PointerButton::Primary, pressed: false, modifiers }); + h.run_steps(3); + h.input_mut().events.push(egui::Event::ModifiersChanged(Default::default())); + h.step(); +} + +#[test] +fn viewer_points_selection_and_effect_controls() { + let (a, plate, uid) = app(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).build_eframe(|_| a); + h.state_mut().show_panel(PanelKind::Composition); + h.run_steps(4); + let all = ids(&h); + assert!(all.iter().any(|i| i == "viewer.cameraTracker"), "track point overlay registered"); + let mut pts: Vec<&String> = all.iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + pts.sort(); + pts.dedup(); + let n = pts.len(); + assert_eq!(n, 24, "every visible point is drawn"); + // Click a point: it is selected; Shift-click another adds it. + let r = rect(&h, "viewer.cameraTracker.point.7"); + click(&mut h, r.center(), Default::default()); + assert_eq!(h.state().session.state.camera_points, vec![7]); + let r = rect(&h, "viewer.cameraTracker.point.9"); + click(&mut h, r.center(), egui::Modifiers::SHIFT); + assert_eq!(h.state().session.state.camera_points, vec![7, 9]); + // Marquee over the first row selects its six points. + let (a0, a5) = (rect(&h, "viewer.cameraTracker.point.0"), rect(&h, "viewer.cameraTracker.point.5")); + let from = a0.left_top() - egui::vec2(6.0, 6.0); + let to = a5.right_bottom() + egui::vec2(6.0, 6.0); + h.input_mut().events.push(egui::Event::PointerMoved(from)); + h.input_mut().events.push(egui::Event::PointerButton { pos: from, button: egui::PointerButton::Primary, pressed: true, modifiers: Default::default() }); + h.step(); + for i in 1..=8 { + h.input_mut().events.push(egui::Event::PointerMoved(from + (to - from) * (i as f32 / 8.0))); + h.step(); + } + h.input_mut().events.push(egui::Event::PointerButton { pos: to, button: egui::PointerButton::Primary, pressed: false, modifiers: Default::default() }); + h.run_steps(3); + let mut sel = h.state().session.state.camera_points.clone(); + sel.sort(); + assert_eq!(sel, vec![0, 1, 2, 3, 4, 5]); + // The right-click menu's Create Solid and Camera (via its command). + let ctx = h.ctx.clone(); + effectcraft_ui_egui::menus::invoke(h.state_mut(), &ctx, "camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + let comp = h.state().session.active_comp().unwrap(); + assert!(comp.layers.iter().any(|l| l.name == "3D Tracker Camera" && l.is_camera())); + assert!(comp.layers.iter().any(|l| l.name.starts_with("Track Solid"))); + + // Effect Controls: Analyze and Create Camera. + h.state_mut().session.execute("layer.select", json!({"layers": [plate]})).unwrap(); + h.state_mut().show_panel(PanelKind::EffectControls); + h.run_steps(4); + let all = ids(&h); + for want in [format!("effectControls.cameraTracker.{uid}.analyze"), format!("effectControls.cameraTracker.{uid}.createCamera")] { + assert!(all.contains(&want), "missing {want}"); + } + // Tracker panel: Track Camera is available for the selected layer. + h.state_mut().show_panel(PanelKind::Tracker); + h.run_steps(3); + assert!(ids(&h).iter().any(|i| i == "tracker.trackCamera")); + assert!(h.state().session.is_enabled("track.camera")); +} + +/// Step the UI until background frame renders have landed. +fn settle(h: &mut Harness<'_, EffectcraftApp>) { + for _ in 0..600 { + h.step(); + if h.state().frames.inflight() == 0 && h.state().frames.last_ms.lock().map(|v| *v > 0.0).unwrap_or(false) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + for _ in 0..4 { + h.step(); + } +} + +#[test] +#[ignore = "analyses the demo's 3D Showcase and renders with wgpu; run with --ignored (set EC_SNAPSHOT_DIR to keep PNGs)"] +fn snapshot_showcase_track() { + let dir = std::env::var("EC_SNAPSHOT_DIR").ok(); + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("comp.new", json!({"name": "Camera Track", "width": 1920, "height": 1080, "duration": 4, "frameRate": 29.97})).unwrap(); + s.execute("layer.addItem", json!({"item": "3D Showcase"})).unwrap(); + s.execute("layer.select", json!({"layers": ["#1"]})).unwrap(); + s.execute("track.camera", json!({"layer": "#1", "wait": true})).unwrap(); + s.execute("time.set", json!({"time": 2.0})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 1000.0)).with_pixels_per_point(1.0).wgpu().build_eframe(|_| EffectcraftApp::new(s)); + h.state_mut().show_panel(PanelKind::Composition); + settle(&mut h); + let pts: Vec = ids(&h).into_iter().filter(|i| i.starts_with("viewer.cameraTracker.point.")).collect(); + assert!(pts.len() > 10, "{}", pts.len()); + // Hover between points to show a target, then select three and show the selection's target. + let r = rect(&h, &pts[0]); + h.input_mut().events.push(egui::Event::PointerMoved(r.center() + egui::vec2(12.0, 12.0))); + h.run_steps(2); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_hover.png")).unwrap(); + } + let sel: Vec = pts.iter().take(5).filter_map(|i| i.rsplit('.').next()?.parse().ok()).collect(); + h.state_mut().session.execute("camera.selectPoints", json!({"points": sel})).unwrap(); + h.state_mut().session.execute("camera.createFromSolve", json!({"kind": "solid"})).unwrap(); + h.state_mut().session.execute("layer.select", json!({"layers": ["#3"]})).unwrap(); + settle(&mut h); + if let Some(d) = &dir { + h.render().expect("render").save(format!("{d}/camera_tracker_solid.png")).unwrap(); + } +} diff --git a/docs/agents.md b/docs/agents.md index 510bc356d..bda72461d 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -146,6 +146,34 @@ effectcraft-cli exec warp.analyze '{"layer":"#1","wait":true}' shaky.ecproj --sa effectcraft-cli exec warp.status '{"layer":"#1"}' shaky.ecproj --json ``` +### 3D Camera Tracker + +Animation ▸ Track Camera and the Tracker panel's button (`track.camera {layer?, shotType?: +fixed|variable|specify, aov?, solveMethod?: auto|typical|flat|tripod, detailed?, wait?}`) apply +Effect ▸ Perspective ▸ 3D Camera Tracker (or reuse the layer's) and analyse it in the background: +"Analyzing in background (step 1 of 2)" tracks features, "Solving camera" solves. +`camera.analyze {layer?, effect?, wait?}` re-runs it, `camera.cancel` stops it without writing +anything. `camera.solveStatus` reports `progress`/`banner`, `analyzed`, `solved`, `methodUsed`, +`averageError` (pixels), `focalLength`, `horizontalAngleOfView`, the current frame's `camera` +(position, orientation, zoom) and `groundPlane`. `camera.points {time?}` lists the solved points +visible now with their `id`, `comp` position, `depth`, `world` position and `error`. + +Select points with `camera.selectPoints {points, add?, toggle?}` (the viewer's click / Shift-click / +marquee), then the right-click menu's commands: `camera.setGroundPlane {points?}` (Set Ground Plane +and Origin), `camera.createFromSolve {kind: text|solid|null|shadowCatcher, points? | target: +{center, normal, size?}, multiple?}` (Create Text / Solid / Null / Shadow Catcher, Camera and Light, +Create Multiple …), `camera.deletePoints {points?, wait?}` (re-solves; with Auto-delete Points +Across Time the same feature's other tracks go too) and `camera.create` (the Create Camera button). +Each is one undo step; the first create adds the one-node "3D Tracker Camera" keyed on every frame. +Changing the layer's frames clears the analysis; changing `effects/#1/shotType`, +`effects/#1/horizontalAngleOfView`, `effects/#1/advanced/solveMethod` re-solves the stored tracks. + +```sh +effectcraft-cli run shot.ecproj track.camera '{"layer":"#1","wait":true}' \ + camera.points '{}' --json +effectcraft-cli run shot.ecproj camera.createFromSolve '{"kind":"solid","points":[12,40,77]}' --save +``` + ### Text: styles and editing Source Text holds character style runs and per-paragraph settings. `layer.setText` changes the diff --git a/docs/architecture.md b/docs/architecture.md index 033ed6cb5..f9f0586fa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -176,6 +176,22 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**3D Camera Tracker** (`effects::camera_tracker`, `engine::camera_track`, `effectcraft-track`'s +`camtrack`): `camera.analyze` / `track.camera` render the layer's input to the effect on a +background thread and run structure from motion in two steps. Step 1 follows Shi–Tomasi features +through the clip with pyramidal Lucas–Kanade (forward–backward checked, re-detected where the frame +has none) into long 2D tracks. Step 2 solves the camera: parallax keyframes; a two-view start from +the essential matrix (normalised 8-point in RANSAC) or a planar homography decomposition; incremental +resection and triangulation; a sparse Levenberg–Marquardt bundle adjustment (Schur complement over +the points, Huber loss) with the focal length fixed, shared, or per frame; a log-spaced and +golden-section focal search for Fixed Angle of View / Variable Zoom; a rotation-only model for tripod +pans, chosen by Auto Detect when it explains the tracks as well. Tracks and solve are stored as JSON +in hidden effect parameters with keys: changing the layer's frames clears both, changing Shot Type, +Angle of View, Solve Method or deleting points only re-solves. The solve's canonical frame maps to +comp space so the first frame's camera is the default comp camera (or so a chosen ground plane is +the X-Z plane at the origin); `camera.createFromSolve` keys a one-node "3D Tracker Camera" on every +frame and places text, solids, nulls or a shadow catcher and light on the target plane. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, diff --git a/docs/parity.md b/docs/parity.md index d259fec24..25e5a0801 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -15,7 +15,7 @@ rendered, does not count as done. | P1 (professional daily use) | ≈ 38% | | P2 (long tail) | ≈ 17% | | Features done / partial / missing | 18 / 60 / 14 of 92 | -| **Effects** | **257 of 298** After Effects 2026 effects by name (86%); ≈ 70% allowing for simplified implementations | +| **Effects** | **258 of 298** After Effects 2026 effects by name (87%); ≈ 70% allowing for simplified implementations | | Remaining work | ≈ **167 agent-hours** (139 h of features + ≈ 20% for 1:1 polish against After Effects) | | Wall-clock estimate | ≈ **42–50 hours** with five agents working in parallel and one integrating; ≈ 31–35 h for P0 + P1 only | @@ -73,7 +73,7 @@ library groundwork for mask tracking and Warp Stabilizer. | Remaining (audit-scale agent-hours) | ≈ 167 | ≈ 100 | | Wall clock, five parallel agents | 42–50 h | **≈ 8–10 h at today's measured pace; ≈ 25 h by the audit's conservative figures** | -What is left is concentrated in large systems: Roto Brush and Refine Edge, the 3D camera tracker, +What is left is concentrated in large systems: Roto Brush and Refine Edge, Advanced 3D (models, PBR, image-based lighting), the Warp Stabilizer and mask-tracking UI on top of the new tracking library, a JavaScript scripting object model, SVG/PSD import, WebM, a disk cache, the missing effect categories (3D Channel, Immersive Video, OCIO), native macOS menus, and @@ -93,13 +93,13 @@ wasm threads. | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 85% | 1.5 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no text on 3D bevels; per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | -| 3D | 58% | 20.5 | Advanced 3D (models, PBR, image-based light), 3D camera tracker, multi-view layouts | +| 3D | 62% | 18.5 | Advanced 3D (models, PBR, image-based light), multi-view layouts (the 3D Camera Tracker landed in M12.6) | | Effects | 58% | 13.6 | Effect Controls widgets (angle dial, point crosshair, eyedropper, curves), GPU versions of more effects, 41 missing effects | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | | Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | -| Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking, Subspace Warp's mesh warp and rolling-shutter repair (point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | +| Tracking | ≈ 85% | 2.5 | face tracking, Subspace Warp's mesh warp, rolling-shutter repair, lens distortion in the camera solve (point tracker, mask tracking, Warp Stabilizer and the 3D Camera Tracker landed in M6.x / M12.5 / M12.6) | | Paint | 0% | 4.0 | Brush, Clone Stamp, Eraser | ## Effects still missing @@ -107,7 +107,7 @@ wasm threads. Immersive Video (all 12), 3D Channel (all 8, including Cryptomatte), Color Stabilizer and the five OCIO transforms, CC Flo Motion, Liquify, Rolling Shutter Repair, Compressor, Distortion and Gate (audio), Camera-Shake Deblur, CC Radial Blur, CC Hair, Particle Playground, -Keylight (our Screen Key stands in), Color Profile Converter and the 3D Camera Tracker. Boris FX +Keylight (our Screen Key stands in) and Color Profile Converter. Boris FX Mocha and Cineware are third-party and not counted. ## Highest-value gaps, in order From c33b80614ace5e598c48e998f59b49aaf96c26ed Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:11:03 -0400 Subject: [PATCH 5/9] =?UTF-8?q?PRJ-8:=20proxies=20=E2=80=94=20Set=20Proxy?= =?UTF-8?q?=20/=20None=20/=20Use=20Proxy=20/=20Create=20Proxy=20/=20Interp?= =?UTF-8?q?ret=20Proxy,=20Proxy=20Use;=20PRJ-3=20interpret=20fields,=20PAR?= =?UTF-8?q?,=20alpha=20guess?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Items (footage and comps) carry a proxy rendered in their place at the source's size when its switch and Render Settings > Proxy Use allow it; Create Proxy queues a half-resolution render whose post-render action attaches it. Interpret Footage gains Separate Fields (upper/lower first, fields become frames at twice the rate), Guess alpha with matte colour, explicit matte colour, Invert Alpha, Interpret As Linear Light; non-square footage, solids and precomps are stretched by their pixel aspect in the comp. The four proxy stubs are gone. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/file_more.rs | 207 +++++++++++++--- crates/engine/src/commands/layer.rs | 2 +- crates/engine/src/commands/mod.rs | 9 +- crates/engine/src/commands/proxy.rs | 223 +++++++++++++++++ crates/engine/src/commands/render_queue.rs | 8 +- crates/engine/src/commands/stubs.rs | 6 - crates/engine/src/lib.rs | 5 +- crates/engine/src/render_queue.rs | 5 + crates/engine/src/tests_menu_cmds.rs | 2 +- crates/engine/src/tests_proxy.rs | 275 +++++++++++++++++++++ crates/engine/src/tests_stubs.rs | 2 +- crates/export/src/lib.rs | 1 + crates/host/src/lib.rs | 53 ++++ crates/project/src/render_queue.rs | 51 ++++ crates/render/src/cache.rs | 2 + crates/render/src/eval.rs | 25 +- crates/render/src/lib.rs | 130 ++++++++-- crates/ui-egui/src/lib.rs | 1 + 18 files changed, 938 insertions(+), 69 deletions(-) create mode 100644 crates/engine/src/commands/proxy.rs create mode 100644 crates/engine/src/tests_proxy.rs diff --git a/crates/engine/src/commands/file_more.rs b/crates/engine/src/commands/file_more.rs index f5f4d3709..45e6865ae 100644 --- a/crates/engine/src/commands/file_more.rs +++ b/crates/engine/src/commands/file_more.rs @@ -400,56 +400,179 @@ fn run_script(s: &mut Session, p: &Value) -> Result { // ---------------------------------------------------------------- footage +/// Interpret Footage settings parsed from command parameters (every field optional). +#[derive(Clone, Debug, Default)] +pub(crate) struct Interpretation { + rate: Option, + /// Back to the file's own rate (Use frame rate from file). + native: bool, + alpha: Option, + /// Alpha ▸ Guess: decide Straight / Premultiplied (and the matte colour) from a frame. + guess: bool, + matte: Option<[f32; 3]>, + invert_alpha: Option, + loops: Option, + par: Option, + fields: Option, + profile: Option>, + linear: Option, +} + +impl Interpretation { + pub(crate) fn parse(p: &Value, cmd: &str) -> Result { + let mut it = Interpretation { rate: f_p(p, "frameRate").map(FrameRate::from_f64), ..Default::default() }; + if str_p(p, "frameRate") == Some("file") { + it.native = true; + } + match str_p(p, "alpha") { + Some("straight") => it.alpha = Some(AlphaMode::Straight), + Some("premultiplied") => it.alpha = Some(AlphaMode::Premultiplied), + Some("ignore") => it.alpha = Some(AlphaMode::Ignore), + Some("guess") => it.guess = true, + Some(a) => return Err(bad(cmd, format!("alpha: straight|premultiplied|ignore|guess, not `{a}`"))), + None => {} + } + it.guess |= super::b_p(p, "guessAlpha").unwrap_or(false); + it.matte = match p.get("matteColor").or(p.get("premulColor")) { + None => None, + Some(v) => Some( + effectcraft_keyframe::Value::Color([0.0, 0.0, 0.0, 1.0]) + .coerce_json(v) + .map(|c| { + let c = c.as_color(); + [c[0], c[1], c[2]] + }) + .ok_or_else(|| bad(cmd, "matteColor: #hex or [r, g, b] (0..1)"))?, + ), + }; + it.invert_alpha = super::b_p(p, "invertAlpha"); + it.loops = p.get("loop").and_then(Value::as_u64).map(|v| v.clamp(1, 9999) as u32); + it.par = f_p(p, "pixelAspect").filter(|x| *x > 0.0); + it.fields = match str_p(p, "fields") { + Some(f) => Some(effectcraft_project::FieldOrder::parse(f).ok_or_else(|| bad(cmd, "fields: off|upper|lower"))?), + None => None, + }; + // Color ▸ Assign Profile: a colour space id, or "auto"/"none" for the file's own (sRGB). + it.profile = match str_p(p, "colorProfile") { + Some("auto" | "none" | "") => Some(None), + Some(c) => match effectcraft_project::ColorSpace::parse(c) { + Some(cs) => Some(Some(cs)), + None => return Err(bad(cmd, format!("colorProfile: srgb|rec709|rec2020|p3|auto, not `{c}`"))), + }, + None => None, + }; + it.linear = super::b_p(p, "linearLight"); + Ok(it) + } + + pub(crate) fn apply(&self, f: &mut Footage, guessed: Option<(AlphaMode, [f32; 3])>) { + if self.native { + if let Some(n) = f.native_rate.take() { + f.frame_rate = n; + } + } else if let Some(r) = self.rate { + if f.native_rate.is_none() { + f.native_rate = Some(f.frame_rate); + } + f.frame_rate = r; + } + if let Some(a) = self.alpha { + f.alpha = a; + } + if let Some((a, m)) = guessed { + f.alpha = a; + f.premul_color = m; + } + if let Some(m) = self.matte { + f.premul_color = m; + } + if let Some(v) = self.invert_alpha { + f.invert_alpha = v; + } + if let Some(l) = self.loops { + f.loop_count = l; + } + if let Some(x) = self.par { + f.pixel_aspect = x; + } + if let Some(x) = self.fields { + f.fields = x; + } + if let Some(c) = self.profile { + f.color_profile = c; + } + if let Some(v) = self.linear { + f.linear_light = v; + } + } +} + +/// Interpret Footage ▸ Alpha ▸ Guess: read the first frame as straight colour and decide +/// whether the file is premultiplied (every partly transparent pixel's colour within its alpha +/// of a black or white matte) or straight. `None` when the frame has no partial alpha. +pub(crate) fn guess_alpha(s: &Session, item: ItemId, f: &Footage) -> Option<(AlphaMode, [f32; 3])> { + let probe = Footage { alpha: AlphaMode::Straight, invert_alpha: false, ..f.clone() }; + let img = s.footage.frame(item, &probe, Tick::ZERO)?; + let (mut partial, mut black, mut white) = (0usize, true, true); + let eps = 1.5 / 255.0; + for px in img.data.iter() { + let a = px[3]; + if a <= eps || a >= 1.0 - eps { + continue; + } + partial += 1; + for c in &px[..3] { + // The file's colour (the decoder premultiplied it as straight). + let file = c / a; + black &= file <= a + eps; + white &= file >= 1.0 - a - eps; + } + } + if partial == 0 { + return None; + } + Some(if black { + (AlphaMode::Premultiplied, [0.0; 3]) + } else if white { + (AlphaMode::Premultiplied, [1.0; 3]) + } else { + (AlphaMode::Straight, f.premul_color) + }) +} + fn interpret(s: &mut Session, p: &Value) -> Result { let items = items_p(s, p); if items.is_empty() { return Err(bad("file.interpretFootage", "select footage")); } - let rate = f_p(p, "frameRate").map(FrameRate::from_f64); - let alpha = match str_p(p, "alpha") { - Some("straight") => Some(AlphaMode::Straight), - Some("premultiplied") => Some(AlphaMode::Premultiplied), - Some("ignore") => Some(AlphaMode::Ignore), - Some(a) => return Err(bad("file.interpretFootage", format!("alpha: straight|premultiplied|ignore, not `{a}`"))), - None => None, - }; - let loops = p.get("loop").and_then(Value::as_u64).map(|v| v.max(1) as u32); - let par = f_p(p, "pixelAspect"); - // Color ▸ Assign Profile: a colour space id, or "auto"/"none" for the file's own (sRGB). - let profile = match str_p(p, "colorProfile") { - Some("auto" | "none" | "") => Some(None), - Some(c) => match effectcraft_project::ColorSpace::parse(c) { - Some(cs) => Some(Some(cs)), - None => return Err(bad("file.interpretFootage", format!("colorProfile: srgb|rec709|rec2020|p3|auto, not `{c}`"))), - }, - None => None, - }; + let it = Interpretation::parse(p, "file.interpretFootage")?; + let guesses: Vec> = items + .iter() + .map(|i| match (it.guess, s.project.item(*i).map(|x| &x.kind)) { + (true, Some(ItemKind::Footage(f))) => guess_alpha(s, *i, f), + _ => None, + }) + .collect(); s.edit("Interpret Footage", None, |proj, _| { - for i in &items { + for (i, g) in items.iter().zip(&guesses) { if let Some(ItemKind::Footage(f)) = proj.item_mut(*i).map(|x| &mut x.kind) { - if let Some(r) = rate { - if f.native_rate.is_none() { - f.native_rate = Some(f.frame_rate); - } - f.frame_rate = r; - } - if let Some(a) = alpha { - f.alpha = a; - } - if let Some(l) = loops { - f.loop_count = l; - } - if let Some(x) = par { - f.pixel_aspect = x; - } - if let Some(c) = profile { - f.color_profile = c; - } + it.apply(f, *g); } } Ok(()) })?; - Ok(Value::Null) + let out: Vec = items + .iter() + .filter_map(|i| match s.project.item(*i).map(|x| &x.kind) { + Some(ItemKind::Footage(f)) => Some(json!({ + "item": i.0, "alpha": format!("{:?}", f.alpha), "matteColor": f.premul_color, "invertAlpha": f.invert_alpha, + "fields": f.fields.label(), "pixelAspect": f.pixel_aspect, "loop": f.loop_count, "frameRate": f.frame_rate.as_f64(), + "linearLight": f.linear_light, + })), + _ => None, + }) + .collect(); + Ok(json!({"items": out})) } fn remember_interpretation(s: &mut Session, p: &Value) -> Result { @@ -470,6 +593,10 @@ fn apply_interpretation(s: &mut Session, p: &Value) -> Result { f.premul_color = src.premul_color; f.pixel_aspect = src.pixel_aspect; f.loop_count = src.loop_count; + f.fields = src.fields; + f.invert_alpha = src.invert_alpha; + f.linear_light = src.linear_light; + f.color_profile = src.color_profile; if src.native_rate.is_some() { if f.native_rate.is_none() { f.native_rate = Some(f.frame_rate); @@ -648,7 +775,7 @@ pub fn specs() -> Vec { "Main...", ["File", "Interpret Footage"], Some("Cmd+Alt+G"), - "{items?, frameRate?, alpha?: straight|premultiplied|ignore, loop?, pixelAspect?, colorProfile?: srgb|rec709|rec2020|p3|auto}", + "{items?, frameRate?: fps|\"file\", alpha?: straight|premultiplied|ignore|guess, guessAlpha?, matteColor?, invertAlpha?, loop?, pixelAspect?, fields?: off|upper|lower, colorProfile?: srgb|rec709|rec2020|p3|auto, linearLight?}", has_footage_selection, interpret ), diff --git a/crates/engine/src/commands/layer.rs b/crates/engine/src/commands/layer.rs index e57b1c0a6..6f326b023 100644 --- a/crates/engine/src/commands/layer.rs +++ b/crates/engine/src/commands/layer.rs @@ -53,7 +53,7 @@ fn new_solid_like(s: &mut Session, p: &Value, adjustment: bool) -> Result let name = str_p(p, "name").map(str::to_string).unwrap_or(default_name); let id = s.edit(if adjustment { "New Adjustment Layer" } else { "New Solid" }, None, |proj, st| { let folder = solids_folder(proj); - let sid = proj.add_item(&name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: 1.0 })); + let sid = proj.add_item(&name, Label::Red, Some(folder), ItemKind::Solid(Solid { color, width: w, height: h, pixel_aspect: comp.pixel_aspect })); let mut l = build::layer(proj, &comp, &name, LayerSource::Solid { item: sid }, (w, h), None); if adjustment { l.switches.adjustment = true; diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index e1b3f15f7..7d44f5e52 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -11,7 +11,7 @@ mod effect; pub mod essential; pub mod expr_tools; mod file; -mod file_more; +pub(crate) mod file_more; mod frontend; mod help; mod key_transform; @@ -29,6 +29,7 @@ pub mod paint; mod paths; mod project_items; mod prop; +mod proxy; pub mod puppet; mod query; mod render_queue; @@ -140,11 +141,17 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(settings::specs()); v.extend(essential::specs()); v.extend(expr_tools::specs()); + v.extend(proxy::specs()); v.extend(stubs::specs()); v }) } +/// Set `footage` as the proxy of `item` (File ▸ Create Proxy's post-render action). +pub(crate) fn proxy_set(s: &mut Session, item: ItemId, footage: effectcraft_project::Footage) -> Result<()> { + proxy::set_proxy_footage(s, &[item], footage) +} + /// (id, name) of the text animation presets. pub fn text_preset_list() -> Vec<(String, String)> { text_anim::preset_names() diff --git a/crates/engine/src/commands/proxy.rs b/crates/engine/src/commands/proxy.rs new file mode 100644 index 000000000..607dd5ddc --- /dev/null +++ b/crates/engine/src/commands/proxy.rs @@ -0,0 +1,223 @@ +//! Proxies (File ▸ Create Proxy / Set Proxy / Interpret Footage ▸ Proxy): a stand-in file for +//! a footage item or a composition. The Project panel's proxy switch (`file.useProxy`) turns +//! it on or off per item; renders follow Render Settings ▸ Proxy Use (the viewer uses each +//! item's switch). + +use effectcraft_project::render_queue::{Channels, OutputFormat, OutputModule, PostRenderAction, ProResProfile, RenderQuality, RenderQueueItem, TimeSpan}; +use effectcraft_project::{Footage, ItemId, ItemKind, Proxy}; +use effectcraft_time::Tick; +use serde_json::{Value, json}; + +use super::{CommandSpec, b_p, bad, has_project_selection, str_p}; +use crate::{EngineError, Result, Session, cmd}; + +/// Footage items and comps a proxy command acts on: `items`/`item`, else the Project panel +/// selection. +fn targets(s: &Session, p: &Value) -> Vec { + let ids: Vec = match p.get("items").or(p.get("item")) { + Some(Value::Array(a)) => a.iter().filter_map(Value::as_u64).map(ItemId).collect(), + Some(Value::Number(n)) => n.as_u64().map(ItemId).into_iter().collect(), + Some(Value::String(name)) => s.project.find_by_name(name).map(|i| i.id).into_iter().collect(), + _ => s.state.project_selection.clone(), + }; + ids.into_iter().filter(|i| matches!(s.project.item(*i).map(|x| &x.kind), Some(ItemKind::Footage(_) | ItemKind::Comp(_)))).collect() +} + +fn has_targets(s: &Session) -> std::result::Result<(), String> { + has_project_selection(s)?; + if targets(s, &Value::Null).is_empty() { Err("select footage or a composition in the Project panel".into()) } else { Ok(()) } +} + +fn has_proxy(s: &Session) -> std::result::Result<(), String> { + has_targets(s)?; + if targets(s, &Value::Null).iter().any(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())) { + Ok(()) + } else { + Err("the selected items have no proxy".into()) + } +} + +/// Probe a proxy file through the importer. +fn probe(s: &Session, path: &str, cmd: &str) -> Result { + let importer = s.importer.clone().ok_or_else(|| EngineError::Other("media import is not available in this build".into()))?; + let f = importer.probe(path).map_err(|e| bad(cmd, format!("{path}: {e}")))?; + if !f.has_video { + return Err(bad(cmd, format!("{path} has no video to use as a proxy"))); + } + Ok(f) +} + +/// Attach `footage` as the proxy of `items` (Use Proxy on). One undo step. +pub(crate) fn set_proxy_footage(s: &mut Session, items: &[ItemId], footage: Footage) -> Result<()> { + s.edit("Set Proxy", None, |proj, _| { + for i in items { + if let Some(it) = proj.item_mut(*i) { + it.proxy = Some(Box::new(Proxy { footage: footage.clone(), enabled: true })); + } + } + Ok(()) + }) +} + +fn set_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.setProxy"; + let items = targets(s, p); + if items.is_empty() { + return Err(bad(C, "select footage or a composition")); + } + let path = str_p(p, "path").ok_or_else(|| bad(C, "missing `path`"))?.to_string(); + let f = probe(s, &path, C)?; + let (w, h) = (f.width, f.height); + set_proxy_footage(s, &items, f)?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>(), "path": path, "width": w, "height": h})) +} + +fn set_proxy_none(s: &mut Session, p: &Value) -> Result { + let items = targets(s, p); + s.edit("Set Proxy None", None, |proj, _| { + for i in &items { + if let Some(it) = proj.item_mut(*i) { + it.proxy = None; + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>()})) +} + +/// The Project panel's proxy switch: use the proxy (filled box) or the full-resolution source. +fn use_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.useProxy"; + let items: Vec = targets(s, p).into_iter().filter(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())).collect(); + if items.is_empty() { + return Err(bad(C, "the selected items have no proxy")); + } + let first = s.project.item(items[0]).and_then(|x| x.proxy.as_ref()).is_some_and(|px| px.enabled); + let on = b_p(p, "on").unwrap_or(!first); + s.edit(if on { "Use Proxy" } else { "Use Full Resolution" }, None, |proj, _| { + for i in &items { + if let Some(px) = proj.item_mut(*i).and_then(|x| x.proxy.as_mut()) { + px.enabled = on; + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>(), "on": on})) +} + +/// File ▸ Interpret Footage ▸ Proxy: Interpret Footage settings for the proxy file. +fn interpret_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.interpretProxy"; + let items: Vec = targets(s, p).into_iter().filter(|i| s.project.item(*i).is_some_and(|x| x.proxy.is_some())).collect(); + if items.is_empty() { + return Err(bad(C, "the selected items have no proxy")); + } + let it = super::file_more::Interpretation::parse(p, C)?; + let guesses: Vec<_> = items + .iter() + .map(|i| { + let px = s.project.item(*i).and_then(|x| x.proxy.as_ref()); + match (p.get("alpha").and_then(Value::as_str) == Some("guess") || b_p(p, "guessAlpha") == Some(true), px) { + (true, Some(px)) => super::file_more::guess_alpha(s, *i, &px.footage), + _ => None, + } + }) + .collect(); + s.edit("Interpret Proxy", None, |proj, _| { + for (i, g) in items.iter().zip(&guesses) { + if let Some(px) = proj.item_mut(*i).and_then(|x| x.proxy.as_mut()) { + it.apply(&mut px.footage, *g); + } + } + Ok(()) + })?; + Ok(json!({"items": items.iter().map(|i| i.0).collect::>()})) +} + +/// File ▸ Create Proxy ▸ Still / Movie: queue a half-resolution render of the composition whose +/// post-render action sets the result as its proxy. +fn create_proxy(s: &mut Session, p: &Value) -> Result { + const C: &str = "file.createProxy"; + let cid = match p.get("comp").or(p.get("item")) { + Some(_) => { + let q = json!({"comp": p.get("comp").or(p.get("item")).cloned()}); + super::comp_id(s, &q)? + } + None => s + .state + .project_selection + .iter() + .copied() + .find(|i| s.project.comp(*i).is_some()) + .or(s.active_comp_id()) + .ok_or_else(|| bad(C, "select a composition"))?, + }; + let comp = s.project.comp(cid).ok_or(EngineError::NoComp)?.clone(); + let still = match str_p(p, "kind").unwrap_or("movie") { + "still" => true, + "movie" => false, + k => return Err(bad(C, format!("kind: still|movie, not `{k}`"))), + }; + let mut it = RenderQueueItem::new(0, cid); + it.settings.quality = RenderQuality::Draft; + it.settings.resolution = p.get("resolution").and_then(Value::as_f64).unwrap_or(0.5).clamp(0.05, 1.0); + if still { + // One frame at the current time (the comp's poster frame when not the active comp). + let t = if s.active_comp_id() == Some(cid) { s.time() } else { comp.poster_time }; + it.settings.time_span = TimeSpan::Custom { start: t, end: t + comp.frame_duration() }; + it.output = OutputModule::for_format(OutputFormat::PngSequence); + it.output.output = "[compName]_proxy_[#####].png".into(); + } else { + it.settings.time_span = TimeSpan::LengthOfComp; + it.output = OutputModule::for_format(OutputFormat::ProRes); + it.output.prores_profile = ProResProfile::P4444; + it.output.output = "[compName]_proxy.[fileExtension]".into(); + } + it.output.channels = Channels::Rgba; + if let Some(o) = str_p(p, "path").or(str_p(p, "output")) { + it.output.output = o.to_string(); + } + it.post_render = PostRenderAction::SetProxy; + let id = s.edit("Create Proxy", None, |proj, _| { + it.id = proj.render_queue.iter().map(|i| i.id).max().unwrap_or(0) + 1; + let id = it.id; + proj.render_queue.push(it); + Ok(id) + })?; + let path = s.project.render_queue.last().and_then(|i| s.resolve_output(i)); + Ok(json!({"item": id, "comp": cid.0, "kind": if still { "still" } else { "movie" }, "output": path, "time": Tick::ZERO.seconds()})) +} + +fn has_comp_selection(s: &Session) -> std::result::Result<(), String> { + if s.state.project_selection.iter().any(|i| s.project.comp(*i).is_some()) || s.active_comp_id().is_some() { + Ok(()) + } else { + Err("select a composition".into()) + } +} + +pub fn specs() -> Vec { + vec![ + cmd!( + "file.createProxy", + "Create Proxy", + [], + None, + "{kind: still|movie, comp?|item?, path?|output? (template), resolution? (default 0.5)}", + has_comp_selection, + create_proxy + ), + cmd!("file.setProxy", "File...", ["File", "Set Proxy"], None, "{path, items?|item?}", has_targets, set_proxy), + cmd!("file.setProxyNone", "None", ["File", "Set Proxy"], None, "{items?|item?}", has_proxy, set_proxy_none), + cmd!("file.useProxy", "Use Proxy", [], None, "{items?|item?, on?}", has_proxy, use_proxy), + cmd!( + "file.interpretProxy", + "Proxy...", + ["File", "Interpret Footage"], + None, + "{items?|item?, alpha?, matteColor?, invertAlpha?, frameRate?, loop?, pixelAspect?, fields?, colorProfile?, linearLight?}", + has_proxy, + interpret_proxy + ), + ] +} diff --git a/crates/engine/src/commands/render_queue.rs b/crates/engine/src/commands/render_queue.rs index c905ce6c2..acdf49127 100644 --- a/crates/engine/src/commands/render_queue.rs +++ b/crates/engine/src/commands/render_queue.rs @@ -104,6 +104,10 @@ fn apply_settings(rs: &mut RenderSettings, p: &Value, cmd: &str) -> Result { return Err(bad(cmd, "end must be after start")); } + if let Some(u) = str_p(p, "proxyUse") { + rs.proxy_use = effectcraft_project::render_queue::ProxyUse::parse(u).ok_or_else(|| bad(cmd, "proxyUse: current|all|comp|none"))?; + any = true; + } match p.get("frameRate") { Some(Value::Null) => { rs.frame_rate = None; @@ -509,7 +513,7 @@ pub fn specs() -> Vec { "Add to Render Queue", ["Composition"], Some("Cmd+M"), - "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", + "{comp?: id|name, format?: h264|prores|png|jpeg|tiff|exr|gif, output?: path|template, quality?: best|draft|1-100 (jpeg), resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none, channels?: rgb|rgba, bitrate?: kbps, proresProfile?: proxy|lt|standard|hq|4444|4444xq, audio?: auto|on|off, sampleRate?, loop?: bool}", can_add, add ), @@ -520,7 +524,7 @@ pub fn specs() -> Vec { "Render Settings...", [], None, - "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool}", + "{item?|index?, quality?: best|draft, resolution?: full|half|third|quarter|scale, timeSpan?: workArea|comp|custom, start?: s, end?: s, frameRate?: fps|null, motionBlur?: bool, skipExisting?: bool, proxyUse?: current|all|comp|none}", has_items, set_render_settings ), diff --git a/crates/engine/src/commands/stubs.rs b/crates/engine/src/commands/stubs.rs index 8b244bf68..b62f36546 100644 --- a/crates/engine/src/commands/stubs.rs +++ b/crates/engine/src/commands/stubs.rs @@ -20,12 +20,6 @@ pub fn specs() -> Vec { // macOS application menu items handled by the OS. stub!("app.hideOthers", "Hide Others", [], "{}"), stub!("app.showAll", "Show All", [], "{}"), - // Render queue / export (export milestone). - stub!("file.createProxy", "Create Proxy", [], "{kind: still|movie}"), - // Proxies (no proxy model yet). - stub!("file.setProxy", "File...", ["File", "Set Proxy"], "{path}"), - stub!("file.setProxyNone", "None", ["File", "Set Proxy"], "{}"), - stub!("file.interpretProxy", "Proxy...", ["File", "Interpret Footage"], "{}"), stub!("keys.selectLabelGroup", "Select Keyframe Label Group", [], "{scope}"), // Layer ▸ New (3D / content-aware fill). stub!("layer.newContentAwareFill", "Content-Aware Fill Layer...", ["Layer", "New"], "{}"), diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d0a4ba1a8..b2c4d342e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -340,7 +340,8 @@ impl Session { let spec = commands::find(id).ok_or_else(|| EngineError::UnknownCommand(id.to_string()))?; if let Err(why) = (spec.enabled)(self) { // Explicit targets (agents, scripts) don't need a UI selection. - let explicit = ["layer", "layers", "prop", "keys"].iter().any(|k| params.get(k).is_some()) && commands::has_comp(self).is_ok(); + let explicit = (["layer", "layers", "prop", "keys"].iter().any(|k| params.get(k).is_some()) && commands::has_comp(self).is_ok()) + || ["item", "items"].iter().any(|k| params.get(k).is_some()); if !explicit { return Err(EngineError::Disabled(id.to_string(), why)); } @@ -601,6 +602,8 @@ mod tests_model3d; #[cfg(test)] mod tests_project_items; #[cfg(test)] +mod tests_proxy; +#[cfg(test)] mod tests_settings; #[cfg(test)] mod tests_stubs; diff --git a/crates/engine/src/render_queue.rs b/crates/engine/src/render_queue.rs index bc490f472..efeb670fd 100644 --- a/crates/engine/src/render_queue.rs +++ b/crates/engine/src/render_queue.rs @@ -388,6 +388,11 @@ impl Session { } let importer = self.importer.clone().ok_or("media import is not available in this build")?; let footage = importer.probe(path)?; + if item.post_render == PostRenderAction::SetProxy { + crate::commands::proxy_set(self, item.comp, footage).map_err(|e| e.to_string())?; + self.toast(format!("Proxy set from {path}")); + return Ok(None); + } let name = std::path::Path::new(path).file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_else(|| path.to_string()); let replace = item.post_render == PostRenderAction::ImportAndReplace; let comp = item.comp; diff --git a/crates/engine/src/tests_menu_cmds.rs b/crates/engine/src/tests_menu_cmds.rs index 040c494ec..13087ca3d 100644 --- a/crates/engine/src/tests_menu_cmds.rs +++ b/crates/engine/src/tests_menu_cmds.rs @@ -487,7 +487,7 @@ fn frontend_commands_emit_events_and_stubs_are_disabled() { let ev = frontend_events(&mut s); assert_eq!(ev[0].0, "view.zoomIn"); assert_eq!(ev[1], ("window.panel".to_string(), json!({"panel": "align"}))); - for id in ["file.createProxy", "camera.stereoRig", "view.displayColorManagement", "view.3d.default", "layer.create", "track.motion"] { + for id in ["keys.audioToKeyframes", "camera.stereoRig", "view.displayColorManagement", "view.3d.default", "layer.create", "track.motion"] { assert!(!s.is_enabled(id), "{id}"); assert!(matches!(s.execute(id, json!({})), Err(EngineError::Disabled(..))), "{id}"); } diff --git a/crates/engine/src/tests_proxy.rs b/crates/engine/src/tests_proxy.rs new file mode 100644 index 000000000..83ae59be2 --- /dev/null +++ b/crates/engine/src/tests_proxy.rs @@ -0,0 +1,275 @@ +//! Proxies (Set Proxy, Use Proxy, Proxy Use render settings, comp proxies, Create Proxy) and +//! Interpret Footage (fields, pixel aspect, alpha guess / invert, linear light), rendered through +//! a synthetic footage source. + +use std::sync::Arc; + +use effectcraft_project::render_queue::ProxyUse; +use effectcraft_project::{AlphaMode, Footage, FootageKind, ItemId, ItemKind}; +use effectcraft_raster::Image; +use effectcraft_time::{FrameRate, Tick}; +use serde_json::json; + +use crate::Session; +use crate::render::{FootageSource, RenderOpts}; + +/// Files by name: `red16.png` (16×16 red), `blue8.png` (8×8 blue), `lines.mov` (interlaced: +/// even rows red, odd rows blue; 4×4, 10 fps), `matte_black.png` / `matte_white.png` (a +/// semi-transparent green premultiplied against black / white, read as straight), `half.png` +/// (white at 25 % alpha), `wide.png` (10×10 white). +struct Synthetic; + +fn file_px(path: &str, x: u32, y: u32) -> [f32; 4] { + let name = std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or(""); + match name { + "red16.png" => [1.0, 0.0, 0.0, 1.0], + "blue8.png" => [0.0, 0.0, 1.0, 1.0], + "lines.mov" => { + if y.is_multiple_of(2) { + [1.0, 0.0, 0.0, 1.0] + } else { + [0.0, 0.0, 1.0, 1.0] + } + } + // Straight interpretation of a file whose colour is green·a + matte·(1 − a), a = 0.5. + "matte_black.png" => [0.0, 0.5, 0.0, 0.5], + "matte_white.png" => [0.5, 1.0, 0.5, 0.5], + "half.png" => [1.0, 1.0, 1.0, 0.25], + _ => { + let _ = x; + [1.0, 1.0, 1.0, 1.0] + } + } +} + +fn size_of(path: &str) -> (u32, u32) { + match std::path::Path::new(path).file_name().and_then(|n| n.to_str()).unwrap_or("") { + "red16.png" => (16, 16), + "blue8.png" => (8, 8), + "lines.mov" | "matte_black.png" | "matte_white.png" | "half.png" => (4, 4), + _ => (10, 10), + } +} + +impl FootageSource for Synthetic { + fn frame(&self, _: ItemId, f: &Footage, _t: Tick) -> Option> { + let (w, h) = size_of(&f.path); + let mut img = Image::new(w, h); + for y in 0..h { + for x in 0..w { + // Decoders hand out premultiplied pixels (straight file colour × alpha). + let p = file_px(&f.path, x, y); + img.set(x, y, [p[0] * p[3], p[1] * p[3], p[2] * p[3], p[3]]); + } + } + Some(Arc::new(img)) + } +} + +struct Probe; +impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + let (w, h) = size_of(path); + let video = path.ends_with(".mov"); + Ok(Footage { + path: path.into(), + kind: if video { FootageKind::Video } else { FootageKind::Still }, + width: w, + height: h, + has_video: true, + frame_rate: FrameRate::from_f64(10.0), + duration: Tick::from_seconds_f64(if video { 2.0 } else { 0.0 }), + ..Default::default() + }) + } +} + +fn session() -> Session { + Session { footage: Arc::new(Synthetic), importer: Some(Arc::new(Probe)), ..Default::default() } +} + +fn import(s: &mut Session, path: &str) -> ItemId { + ItemId(s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap()) +} + +fn px(s: &Session, comp: ItemId, t: f64, opts: RenderOpts, x: i64, y: i64) -> [f32; 4] { + s.render(comp, Tick::from_seconds_f64(t), opts).get(x, y) +} + +fn red(p: [f32; 4]) -> bool { + p[0] > 0.99 && p[2] < 0.01 && p[3] > 0.99 +} +fn blue(p: [f32; 4]) -> bool { + p[2] > 0.99 && p[0] < 0.01 && p[3] > 0.99 +} + +#[test] +fn footage_proxy_follows_use_proxy_and_proxy_use() { + let mut s = session(); + let full = import(&mut s, "/m/red16.png"); + s.execute("comp.new", json!({"name": "C", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": full.0})).unwrap(); + let o = RenderOpts::default(); + assert!(red(px(&s, c, 0.0, o, 8, 8))); + s.execute("file.setProxy", json!({"item": full.0, "path": "/m/blue8.png"})).unwrap(); + // The 8×8 proxy fills the 16×16 footage frame. + for (x, y) in [(1, 1), (8, 8), (14, 14)] { + assert!(blue(px(&s, c, 0.0, o, x, y)), "proxy at {x},{y}"); + } + assert!(s.project.item(full).unwrap().proxy.as_ref().unwrap().enabled); + // Render Settings ▸ Proxy Use. + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseNone, ..o }, 8, 8))); + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseCompOnly, ..o }, 8, 8))); + // Use Proxy off: the viewer shows the source, Use All Proxies still uses it. + s.execute("file.useProxy", json!({"item": full.0})).unwrap(); + assert!(!s.project.item(full).unwrap().proxy.as_ref().unwrap().enabled); + assert!(red(px(&s, c, 0.0, o, 8, 8))); + assert!(blue(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseAll, ..o }, 8, 8))); + // Undo / Set Proxy ▸ None. + s.execute("edit.undo", json!({})).unwrap(); + assert!(blue(px(&s, c, 0.0, o, 8, 8))); + s.execute("file.setProxyNone", json!({"item": full.0})).unwrap(); + assert!(s.project.item(full).unwrap().proxy.is_none()); + assert!(red(px(&s, c, 0.0, RenderOpts { proxy: ProxyUse::UseAll, ..o }, 8, 8))); + // Render queue items carry Proxy Use (Best Settings: none). + let r = s.execute("renderQueue.add", json!({"proxyUse": "all"})).unwrap(); + assert_eq!(s.project.render_queue[0].settings.proxy_use, ProxyUse::UseAll, "{r}"); + assert_eq!(effectcraft_project::render_queue::RenderSettings::default().proxy_use, ProxyUse::UseNone); +} + +#[test] +fn comp_proxy_and_create_proxy() { + let mut s = session(); + s.execute("comp.new", json!({"name": "Inner", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let inner = s.active_comp_id().unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + s.execute("comp.new", json!({"name": "Outer", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let outer = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": inner.0})).unwrap(); + let o = RenderOpts::default(); + assert!(red(px(&s, outer, 0.0, o, 8, 8))); + s.execute("file.setProxy", json!({"item": inner.0, "path": "/m/blue8.png"})).unwrap(); + assert!(blue(px(&s, outer, 0.0, o, 14, 1))); + assert!(blue(px(&s, outer, 0.0, RenderOpts { proxy: ProxyUse::UseCompOnly, ..o }, 8, 8))); + assert!(red(px(&s, outer, 0.0, RenderOpts { proxy: ProxyUse::UseNone, ..o }, 8, 8))); + s.execute("file.setProxyNone", json!({"item": inner.0})).unwrap(); + // Create Proxy queues a half-resolution render whose post-render action sets the proxy. + s.state.project_selection = vec![inner]; + let r = s.execute("file.createProxy", json!({"kind": "still"})).unwrap(); + let it = s.project.render_queue.last().unwrap().clone(); + assert_eq!(it.comp, inner); + assert_eq!(it.post_render, effectcraft_project::render_queue::PostRenderAction::SetProxy); + assert_eq!(it.settings.resolution, 0.5); + assert!(r["output"].as_str().unwrap().contains("Inner_proxy_"), "{r}"); + let m = s.execute("file.createProxy", json!({"kind": "movie", "comp": inner.0})).unwrap(); + assert!(m["output"].as_str().unwrap().ends_with("Inner_proxy.mov"), "{m}"); + // The post-render action (what a finished render runs) attaches the file. + let id = it.id; + s.post_render_action(id, "/m/blue8.png").unwrap(); + assert_eq!(s.project.item(inner).unwrap().proxy.as_ref().unwrap().footage.path, "/m/blue8.png"); + assert!(blue(px(&s, outer, 0.0, o, 8, 8))); +} + +#[test] +fn interpret_proxy_settings_apply_to_the_proxy() { + let mut s = session(); + let full = import(&mut s, "/m/red16.png"); + s.execute("file.setProxy", json!({"item": full.0, "path": "/m/half.png"})).unwrap(); + s.execute("file.interpretProxy", json!({"item": full.0, "alpha": "ignore", "invertAlpha": true})).unwrap(); + let px = s.project.item(full).unwrap().proxy.clone().unwrap(); + assert_eq!(px.footage.alpha, AlphaMode::Ignore); + assert!(px.footage.invert_alpha); + let f = match &s.project.item(full).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + _ => unreachable!(), + }; + assert_eq!(f.alpha, AlphaMode::Straight, "the main footage keeps its interpretation"); +} + +/// Separate Fields: each field becomes a frame at twice the frame rate, the other field's lines +/// interpolated; the dominant field comes first. +#[test] +fn separate_fields_make_frames_from_fields() { + let mut s = session(); + let mov = import(&mut s, "/m/lines.mov"); + s.execute("comp.new", json!({"name": "C", "width": 4, "height": 4, "frameRate": 20, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": mov.0})).unwrap(); + let o = RenderOpts::default(); + // Progressive: alternating rows. + assert!(red(px(&s, c, 0.0, o, 1, 0)) && blue(px(&s, c, 0.0, o, 1, 1))); + s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "upper"})).unwrap(); + // Field 1 (upper = even rows, red) then field 2 (odd rows, blue), each 1/20 s. + for y in 0..3 { + assert!(red(px(&s, c, 0.0, o, 1, y)), "upper field at row {y}"); + assert!(blue(px(&s, c, 0.05, o, 1, y + 1)), "lower field at row {}", y + 1); + } + s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "lower"})).unwrap(); + assert!(blue(px(&s, c, 0.0, o, 1, 2))); + assert!(red(px(&s, c, 0.05, o, 1, 1))); + let r = s.execute("file.interpretFootage", json!({"item": mov.0, "fields": "off"})).unwrap(); + assert_eq!(r["items"][0]["fields"], "Off"); +} + +/// Non-square pixels: a 10×10 footage with pixel aspect 2 covers 20×10 comp pixels. +#[test] +fn pixel_aspect_stretches_footage_in_the_comp() { + let mut s = session(); + let w = import(&mut s, "/m/wide.png"); + s.execute("comp.new", json!({"name": "C", "width": 40, "height": 20, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": w.0})).unwrap(); + let o = RenderOpts::default(); + let covered = |s: &Session| s.render(c, Tick::ZERO, o).data.iter().filter(|p| p[3] > 0.5).count(); + assert_eq!(covered(&s), 100); + s.execute("file.interpretFootage", json!({"item": w.0, "pixelAspect": 2.0})).unwrap(); + assert_eq!(covered(&s), 200); + // Centered on the comp: x 10..30 (edge pixels are filtered). + assert!(px(&s, c, 0.0, o, 11, 10)[3] > 0.99 && px(&s, c, 0.0, o, 28, 10)[3] > 0.99); + assert!(px(&s, c, 0.0, o, 8, 10)[3] < 0.01 && px(&s, c, 0.0, o, 31, 10)[3] < 0.01); + // A comp with the same pixel aspect shows the footage 1:1. + s.execute("comp.settings", json!({"pixelAspect": 2.0})).unwrap(); + assert_eq!(covered(&s), 100); +} + +/// Alpha ▸ Guess: premultiplied files are recognised with their matte colour; Invert Alpha +/// and Interpret As Linear Light change the decoded pixels. +#[test] +fn guess_alpha_invert_and_linear_light() { + let mut s = session(); + let b = import(&mut s, "/m/matte_black.png"); + let w = import(&mut s, "/m/matte_white.png"); + let h = import(&mut s, "/m/half.png"); + let foot = |s: &Session, i: ItemId| match &s.project.item(i).unwrap().kind { + ItemKind::Footage(f) => f.clone(), + _ => unreachable!(), + }; + s.execute("file.interpretFootage", json!({"items": [b.0, w.0, h.0], "alpha": "guess"})).unwrap(); + assert_eq!(foot(&s, b).alpha, AlphaMode::Premultiplied); + assert_eq!(foot(&s, b).premul_color, [0.0; 3]); + assert_eq!(foot(&s, w).alpha, AlphaMode::Premultiplied); + assert_eq!(foot(&s, w).premul_color, [1.0; 3]); + // White at 25 %: not explained by a black matte (1 > 0.25) but it is by white… either way + // the guess is consistent with the pixels. + assert!(matches!(foot(&s, h).alpha, AlphaMode::Straight | AlphaMode::Premultiplied)); + // An explicit matte colour. + s.execute("file.interpretFootage", json!({"item": b.0, "alpha": "premultiplied", "matteColor": "#808080"})).unwrap(); + assert!((foot(&s, b).premul_color[0] - 128.0 / 255.0).abs() < 1e-3); + // Invert Alpha: 25 % → 75 %. + s.execute("file.interpretFootage", json!({"item": h.0, "alpha": "straight", "invertAlpha": true})).unwrap(); + s.execute("comp.new", json!({"name": "C", "width": 4, "height": 4, "frameRate": 10, "duration": 1})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": h.0})).unwrap(); + let p = px(&s, c, 0.0, RenderOpts::default(), 1, 1); + assert!((p[3] - 0.75).abs() < 0.01 && p[0] > 0.7, "{p:?}"); + // Interpret As Linear Light keeps alpha and re-encodes the colour. + s.execute("file.interpretFootage", json!({"item": h.0, "invertAlpha": false, "linearLight": true})).unwrap(); + let p = px(&s, c, 0.0, RenderOpts::default(), 1, 1); + assert!((p[3] - 0.25).abs() < 0.01, "{p:?}"); + // Remember / Apply Interpretation carries the new settings. + s.state.project_selection = vec![h]; + s.execute("file.rememberInterpretation", json!({"item": h.0})).unwrap(); + s.execute("file.applyInterpretation", json!({"items": [w.0]})).unwrap(); + assert!(foot(&s, w).linear_light); +} diff --git a/crates/engine/src/tests_stubs.rs b/crates/engine/src/tests_stubs.rs index 5cb702469..924fb5793 100644 --- a/crates/engine/src/tests_stubs.rs +++ b/crates/engine/src/tests_stubs.rs @@ -332,7 +332,7 @@ fn implemented_stubs_are_enabled_and_workspaces_are_frontend() { assert!(s.drain_events().iter().any(|e| matches!(e, Event::Frontend { command, .. } if command == id))); } // What remains a stub is still disabled. - for id in ["view.displayColorManagement", "file.createProxy", "track.camera"] { + for id in ["view.displayColorManagement", "keys.audioToKeyframes", "track.camera"] { assert!(!s.is_enabled(id), "{id}"); } } diff --git a/crates/export/src/lib.rs b/crates/export/src/lib.rs index dc4c35c83..86568871e 100644 --- a/crates/export/src/lib.rs +++ b/crates/export/src/lib.rs @@ -176,6 +176,7 @@ pub(crate) fn render_frame(job: &Job, comp: &Comp, i: u64) -> Image { view: None, backend: effectcraft_render::Backend::Auto, roi: None, + proxy: job.settings.proxy_use, }; let mut r = Renderer::new(job.project, job.footage, opts); r.expr = job.expr; diff --git a/crates/host/src/lib.rs b/crates/host/src/lib.rs index dccf1aacf..b90a42f0b 100644 --- a/crates/host/src/lib.rs +++ b/crates/host/src/lib.rs @@ -297,6 +297,59 @@ mod tests { assert!(px[1] > 0.99 && px[0] < 0.01, "the box is filled with the sampled green: {px:?}"); } + /// Proxies through the real media and export layers: Render Settings ▸ Proxy Use decides + /// whether an export uses the proxy, and Create Proxy renders and attaches one. + #[test] + fn proxies_in_exports_and_create_proxy() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/test-out/host-proxy"); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let p = |n: &str| dir.join(n).to_string_lossy().to_string(); + let mut s = super::session(); + // Source and proxy stills, written by the app itself. + s.execute("comp.new", json!({"name": "Red", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let red = s.active_comp_id().unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000"})).unwrap(); + s.execute("comp.saveFrameAs", json!({"path": p("red.png")})).unwrap(); + s.execute("comp.new", json!({"name": "Blue", "width": 8, "height": 8, "frameRate": 10, "duration": 1})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#0000ff"})).unwrap(); + s.execute("comp.saveFrameAs", json!({"path": p("blue.png")})).unwrap(); + let foot = s.execute("file.import", json!({"paths": [p("red.png")]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("file.setProxy", json!({"item": foot, "path": p("blue.png")})).unwrap(); + s.execute("comp.new", json!({"name": "Out", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + let out = s.active_comp_id().unwrap(); + s.execute("layer.addItem", json!({"item": foot})).unwrap(); + // Export one frame with each Proxy Use. + for (name, using) in [("all", "use_all_[#].png"), ("none", "use_none_[#].png")] { + s.execute( + "renderQueue.add", + json!({"comp": out.0, "format": "png", "output": p(using), "timeSpan": "custom", "start": 0.0, "end": 0.1, "proxyUse": name}), + ) + .unwrap(); + } + s.execute("renderQueue.render", json!({"wait": true})).unwrap(); + s.poll_render(); + let centre = |s: &mut effectcraft_engine::Session, path: &str| { + let id = s.execute("file.import", json!({"paths": [path]})).unwrap()["items"][0].as_u64().unwrap(); + s.execute("comp.new", json!({"name": "Probe", "width": 16, "height": 16, "frameRate": 10, "duration": 1})).unwrap(); + s.execute("layer.addItem", json!({"item": id})).unwrap(); + let c = s.active_comp_id().unwrap(); + s.render(c, Default::default(), Default::default()).get(8, 8) + }; + let all = centre(&mut s, &p("use_all_0.png")); + assert!(all[2] > 0.99 && all[0] < 0.01, "Use All Proxies renders the blue proxy: {all:?}"); + let none = centre(&mut s, &p("use_none_0.png")); + assert!(none[0] > 0.99 && none[2] < 0.01, "Use No Proxies renders the red source: {none:?}"); + // Create Proxy ▸ Still: rendered at half size and attached to the comp. + s.execute("file.createProxy", json!({"kind": "still", "comp": red.0, "path": p("red_proxy_[#####].png")})).unwrap(); + s.execute("renderQueue.render", json!({"wait": true})).unwrap(); + s.poll_render(); + let px = s.project.item(red).unwrap().proxy.clone().expect("Create Proxy attached the rendered still"); + assert!(px.enabled); + assert_eq!((px.footage.width, px.footage.height), (8, 8)); + assert!(std::path::Path::new(&px.footage.path).exists(), "{}", px.footage.path); + } + #[test] fn expression_syntax_errors_disable_the_expression() { let mut s = super::session(); diff --git a/crates/project/src/render_queue.rs b/crates/project/src/render_queue.rs index c41462b5a..31537a5b4 100644 --- a/crates/project/src/render_queue.rs +++ b/crates/project/src/render_queue.rs @@ -39,6 +39,51 @@ pub enum TimeSpan { Custom { start: Tick, end: Tick }, } +/// Render Settings ▸ Proxy Use: which proxies a render uses. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ProxyUse { + /// Each item's Use Proxy switch (what the viewer shows). + #[default] + CurrentSettings, + /// Every item that has a proxy uses it. + UseAll, + /// Only compositions' proxies. + UseCompOnly, + /// No proxies (full-resolution sources). + UseNone, +} + +impl ProxyUse { + pub const ALL: [ProxyUse; 4] = [ProxyUse::CurrentSettings, ProxyUse::UseAll, ProxyUse::UseCompOnly, ProxyUse::UseNone]; + pub fn label(self) -> &'static str { + match self { + ProxyUse::CurrentSettings => "Current Settings", + ProxyUse::UseAll => "Use All Proxies", + ProxyUse::UseCompOnly => "Use Comp Proxies Only", + ProxyUse::UseNone => "Use No Proxies", + } + } + pub fn parse(s: &str) -> Option { + match s.to_ascii_lowercase().replace([' ', '_', '-'], "").as_str() { + "current" | "currentsettings" => Some(ProxyUse::CurrentSettings), + "all" | "useall" | "useallproxies" => Some(ProxyUse::UseAll), + "comp" | "componly" | "usecompproxiesonly" | "usecomponly" => Some(ProxyUse::UseCompOnly), + "none" | "usenone" | "usenoproxies" => Some(ProxyUse::UseNone), + _ => None, + } + } + /// Whether an item's proxy is used: `comp` = the item is a composition, `enabled` = its Use + /// Proxy switch. + pub fn uses(self, comp: bool, enabled: bool) -> bool { + match self { + ProxyUse::CurrentSettings => enabled, + ProxyUse::UseAll => true, + ProxyUse::UseCompOnly => comp, + ProxyUse::UseNone => false, + } + } +} + #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(default)] pub struct RenderSettings { @@ -52,6 +97,8 @@ pub struct RenderSettings { pub motion_blur: bool, /// Image sequences: skip frames whose file already exists. pub skip_existing: bool, + /// Proxy Use (Best Settings: Use No Proxies). + pub proxy_use: ProxyUse, } impl Default for RenderSettings { @@ -63,6 +110,7 @@ impl Default for RenderSettings { frame_rate: None, motion_blur: true, skip_existing: false, + proxy_use: ProxyUse::UseNone, } } } @@ -419,6 +467,8 @@ pub enum PostRenderAction { Import, /// Import it and replace every use of the rendered composition (Composition ▸ Pre-render). ImportAndReplace, + /// Set the rendered file as the composition's proxy (File ▸ Create Proxy). + SetProxy, } impl PostRenderAction { @@ -430,6 +480,7 @@ impl PostRenderAction { PostRenderAction::None => "None", PostRenderAction::Import => "Import", PostRenderAction::ImportAndReplace => "Import & Replace Usage", + PostRenderAction::SetProxy => "Set Proxy", } } } diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 8cd27d598..ca850a866 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -286,6 +286,8 @@ fn key_with(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, footage: bool let it = ctx.project.item(*item)?; let ItemKind::Footage(f) = &it.kind else { return None }; hash_debug(&mut h, f); + // A proxy (and its Use Proxy switch) changes the pixels. + hash_debug(&mut h, &it.proxy); item.hash(&mut h); ctx.source_time(layer).0.hash(&mut h); // Frame blending mixes neighbouring source frames. diff --git a/crates/render/src/eval.rs b/crates/render/src/eval.rs index aff4d93c5..d8d3f6417 100644 --- a/crates/render/src/eval.rs +++ b/crates/render/src/eval.rs @@ -120,6 +120,28 @@ impl<'a> EvalCtx<'a> { /// Local (parent-space) transform of a layer. pub fn local_matrix(&self, layer: &Layer) -> Mat4 { + self.local_matrix_par(layer, 1.0) + } + + /// Source pixel aspect relative to the comp's: non-square footage (or solids, or precomps + /// with another pixel aspect) is stretched horizontally by this factor in the comp. + pub fn par_ratio(&self, layer: &Layer) -> f64 { + let par = match &layer.source { + LayerSource::Footage { item } | LayerSource::Solid { item } | LayerSource::Comp { item } => match self.project.item(*item).map(|i| &i.kind) { + Some(effectcraft_project::ItemKind::Footage(f)) => f.pixel_aspect, + Some(effectcraft_project::ItemKind::Solid(s)) => s.pixel_aspect, + Some(effectcraft_project::ItemKind::Comp(c)) => c.pixel_aspect, + _ => 1.0, + }, + _ => 1.0, + }; + let r = par / if self.comp.pixel_aspect > 0.0 { self.comp.pixel_aspect } else { 1.0 }; + if r.is_finite() && r > 0.0 && (r - 1.0).abs() > 1e-9 { r } else { 1.0 } + } + + /// [`Self::local_matrix`] with the source's pixel aspect folded into the scale (`par`; 1 for + /// a parent's matrix, which children don't inherit). + fn local_matrix_par(&self, layer: &Layer, par: f64) -> Mat4 { let Some(tr) = layer.transform() else { return Mat4::IDENTITY }; let three = layer.is_3d(); let pos = self.position(layer, tr); @@ -130,6 +152,7 @@ impl<'a> EvalCtx<'a> { } let anchor = self.v3(layer, tr, "anchor", [0.0; 3]); let mut scale = self.v3(layer, tr, "scale", [100.0; 3]); + scale[0] *= par; if !three { scale[2] = 100.0; let a = Vec3::from([anchor[0], anchor[1], 0.0]); @@ -154,7 +177,7 @@ impl<'a> EvalCtx<'a> { /// Layer space → comp (world) space, including parents. pub fn world_matrix(&self, layer: &Layer) -> Mat4 { - let mut m = self.local_matrix(layer); + let mut m = self.local_matrix_par(layer, self.par_ratio(layer)); let mut cur = layer.parent; let mut guard = 0; while let Some(pid) = cur { diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 6699d68d9..01f5f96af 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -247,11 +247,22 @@ pub struct RenderOpts { /// Region of interest `[x, y, w, h]` in comp pixels (viewer only): the top-level frame covers /// just this rectangle, and only its pixels are composited. pub roi: Option<[f64; 4]>, + /// Which proxies stand in for footage and compositions (Render Settings ▸ Proxy Use). + pub proxy: effectcraft_project::render_queue::ProxyUse, } impl Default for RenderOpts { fn default() -> Self { - RenderOpts { scale: 1.0, motion_blur: true, guides: false, draft: false, view: None, roi: None, backend: Backend::Cpu } + RenderOpts { + scale: 1.0, + motion_blur: true, + guides: false, + draft: false, + view: None, + roi: None, + backend: Backend::Cpu, + proxy: effectcraft_project::render_queue::ProxyUse::CurrentSettings, + } } } @@ -675,6 +686,13 @@ impl<'a> Renderer<'a> { if self.project.comp_contains(*item, ctx.comp_id) { return None; } + // A composition proxy (a rendered still or movie) stands in for the comp. + if let Some(pf) = self.proxy_for(*item) + && let Some(nc) = self.project.comp(*item) + { + let img = self.footage.frame(*item, pf, ctx.source_time(layer))?; + return Some(self.footage_buf(&img, pf, nc.width, nc.height)); + } // Essential Properties overrides render the nested comp with this instance's // values. let ov = essential_overrides(ctx, layer); @@ -706,20 +724,10 @@ impl<'a> Renderer<'a> { return None; } let lt = ctx.source_time(layer); - let img = self.footage_frame(ctx, layer, *item, f, lt)?; - let k = if img.width > 0 { f.width.max(1) as f64 / img.width as f64 } else { 1.0 }; - // Keep native pixels when downsampling is small; resample otherwise. - let mut buf = if s < 0.75 { - let w = ((f.width as f64 * s).round() as u32).max(1); - let h = ((f.height as f64 * s).round() as u32).max(1); - Buf { img: effectcraft_raster::resample(&img, w, h), offset: [0.0; 2], scale: s } - } else { - Buf { img: (*img).clone(), offset: [0.0; 2], scale: 1.0 / k } - }; - if let Some(c) = self.pipe.media_in(f.color_profile) { - color::convert(&mut buf.img, &c); - } - Some(buf) + // The proxy is decoded instead, at its own size, and fills the footage's frame. + let pf = self.proxy_for(*item).unwrap_or(f); + let img = self.footage_frame(ctx, layer, *item, pf, lt)?; + Some(self.footage_buf(&img, pf, f.width, f.height)) } LayerSource::Text => Some(self.authored(text::render(ctx, layer, self.raster_scale(ctx, layer)))), LayerSource::Shape => layer.props.sub("contents").map(|c| self.authored(shapes::render(ctx, layer, c, self.raster_scale(ctx, layer)))), @@ -727,6 +735,32 @@ impl<'a> Renderer<'a> { } } + /// The proxy footage used for `item` under this render's Proxy Use, if any. + pub fn proxy_for(&self, item: ItemId) -> Option<&'a Footage> { + let it = self.project.item(item)?; + let px = it.proxy.as_deref()?; + (self.opts.proxy.uses(matches!(it.kind, ItemKind::Comp(_)), px.enabled) && px.footage.has_video).then_some(&px.footage) + } + + /// A decoded frame of `f` as a layer buffer covering `w`×`h` layer pixels (the footage's + /// nominal size; proxies are scaled up to it), converted from its colour profile. + fn footage_buf(&self, img: &Image, f: &Footage, w: u32, h: u32) -> Buf { + let s = self.opts.scale; + let k = if img.width > 0 { w.max(1) as f64 / img.width as f64 } else { 1.0 }; + // Keep native pixels when downsampling is small; resample otherwise. + let mut buf = if s < 0.75 && k <= 1.0 + 1e-9 { + let rw = ((w as f64 * s).round() as u32).max(1); + let rh = ((h as f64 * s).round() as u32).max(1); + Buf { img: effectcraft_raster::resample(img, rw, rh), offset: [0.0; 2], scale: s } + } else { + Buf { img: img.clone(), offset: [0.0; 2], scale: 1.0 / k } + }; + if let Some(c) = self.pipe.media_in(f.color_profile) { + color::convert(&mut buf.img, &c); + } + buf + } + /// Authored colours into the working space (linear working spaces). fn authored(&self, mut buf: Buf) -> Buf { if let Some(c) = self.pipe.authored_in() { @@ -739,6 +773,21 @@ impl<'a> Renderer<'a> { /// when the layer's Frame Blending switch (and the comp's Enable Frame Blending) is on and /// `t` falls between frames (footage rate ≠ comp rate, time stretch, time remapping). fn footage_frame(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { + // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. + if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { + let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); + let i = field_rate.frame_at(t).max(0); + let img = self.footage.frame(item, f, f.frame_rate.tick_of(i / 2))?; + let dominant_upper = f.fields == effectcraft_project::FieldOrder::UpperFirst; + // Upper field = even lines (0, 2, …). + let parity = if (i % 2 == 0) == dominant_upper { 0 } else { 1 }; + return Some(Arc::new(interpret_pixels(&field_frame(&img, parity), f))); + } + let img = self.footage_frame_raw(ctx, layer, item, f, t)?; + if f.invert_alpha || f.linear_light { Some(Arc::new(interpret_pixels(&img, f))) } else { Some(img) } + } + + fn footage_frame_raw(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick) -> Option> { let mode = frame_blend_mode(ctx, layer); if mode != FrameBlend::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let (i, w) = frame_position(t, f.frame_rate); @@ -1307,6 +1356,57 @@ impl<'a> Renderer<'a> { } /// The layer's frame blending mode, if the comp's Enable Frame Blending switch is on. +/// One field of an interlaced frame as a full frame: the field's lines (`parity` 0 = even / +/// upper, 1 = odd / lower) kept, the other lines interpolated from their neighbours. +pub fn field_frame(img: &Image, parity: usize) -> Image { + let (w, h) = (img.width as usize, img.height as usize); + let mut out = img.clone(); + if h < 2 { + return out; + } + for y in 0..h { + if y % 2 == parity { + continue; + } + let above = y.checked_sub(1); + let below = (y + 1 < h).then_some(y + 1); + for x in 0..w { + let px = match (above, below) { + (Some(a), Some(b)) => { + let (p, q) = (img.data[a * w + x], img.data[b * w + x]); + [(p[0] + q[0]) * 0.5, (p[1] + q[1]) * 0.5, (p[2] + q[2]) * 0.5, (p[3] + q[3]) * 0.5] + } + (Some(a), None) => img.data[a * w + x], + (None, Some(b)) => img.data[b * w + x], + (None, None) => img.data[y * w + x], + }; + out.data[y * w + x] = px; + } + } + out +} + +/// Interpret Footage ▸ Invert Alpha and Interpret As Linear Light on decoded (premultiplied) +/// pixels. +pub fn interpret_pixels(img: &Image, f: &Footage) -> Image { + if !f.invert_alpha && !f.linear_light { + return img.clone(); + } + let space = f.color_profile.unwrap_or(effectcraft_color::ColorSpace::Srgb); + let mut out = img.clone(); + for p in out.data.iter_mut() { + let a = p[3]; + let mut c = if a > 1e-6 { [p[0] / a, p[1] / a, p[2] / a] } else { [0.0; 3] }; + if f.linear_light { + // The file's values are linear light: encode them like the rest of the footage. + c = c.map(|v| space.encode(v)); + } + let a = if f.invert_alpha { 1.0 - a } else { a }; + *p = [c[0] * a, c[1] * a, c[2] * a, a]; + } + out +} + fn frame_blend_mode(ctx: &EvalCtx, layer: &Layer) -> FrameBlend { if ctx.comp.enable_frame_blending { layer.switches.frame_blend } else { FrameBlend::Off } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 071c19487..cc12a3416 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -419,6 +419,7 @@ impl EffectcraftApp { view: self.session.view_camera(comp), roi, backend: effectcraft_engine::render::Backend::Auto, + proxy: Default::default(), }; if urgent { self.frames.request_urgent(&self.render_source(), key, comp, t, opts); From 2cba455175df975f72b9c45d8173265498166805 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:31:50 -0400 Subject: [PATCH 6/9] CMP-7: Essential Graphics panel, expression error bar and Expression Language menu, proxy indicator, Interpret Footage dialog Window > Essential Graphics: Primary comp, Solo Supported Properties (timeline filter), template name, typed controls editing the source properties, drag a property name from the timeline onto the list, rename/reorder/remove, Add Formatting (group, comment), Media Replacement, Export Template, and the selected instance's Essential Properties with Revert / Push. The comp viewer shows an expression error bar (click to reveal); the expression editor's f button opens the categorised Expression Language menu. Project panel proxy switch; Interpret Footage dialog gains Guess, matte colour, Invert Alpha, Separate Fields, Linear Light, Proxy. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/essential.rs | 17 +- crates/engine/src/commands/expr_tools.rs | 14 +- crates/ui-egui/src/dock.rs | 5 +- crates/ui-egui/src/menus.rs | 5 +- crates/ui-egui/src/panels/essential.rs | 427 ++++++++++++++++++++++ crates/ui-egui/src/panels/expr_bar.rs | 95 +++++ crates/ui-egui/src/panels/forms.rs | 38 +- crates/ui-egui/src/panels/mod.rs | 5 + crates/ui-egui/src/panels/project.rs | 18 + crates/ui-egui/src/panels/render_queue.rs | 4 + crates/ui-egui/src/panels/timeline.rs | 64 +++- crates/ui-egui/src/panels/viewer.rs | 8 +- 12 files changed, 675 insertions(+), 25 deletions(-) create mode 100644 crates/ui-egui/src/panels/essential.rs create mode 100644 crates/ui-egui/src/panels/expr_bar.rs diff --git a/crates/engine/src/commands/essential.rs b/crates/engine/src/commands/essential.rs index 65e93d4e7..a24fcdc0f 100644 --- a/crates/engine/src/commands/essential.rs +++ b/crates/engine/src/commands/essential.rs @@ -65,7 +65,8 @@ fn set_primary(s: &mut Session, p: &Value) -> Result { fn set_name(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let name = str_p(p, "name").ok_or_else(|| bad("essential.setName", "missing `name`"))?.to_string(); - s.edit("Essential Graphics Name", None, |proj, _| { + s.edit("Essential Graphics Name", None, |proj, st| { + st.essential_primary.get_or_insert(cid); eg_of(proj, cid)?.name = name.clone(); Ok(()) })?; @@ -99,7 +100,10 @@ fn add_property(s: &mut Session, p: &Value) -> Result { } added.push((*cid, *lid, *uid, rename.clone().unwrap_or_else(|| pr.name.clone()))); } - let ids = s.edit("Add Property to Essential Graphics", None, |proj, _| { + let ids = s.edit("Add Property to Essential Graphics", None, |proj, st| { + if let Some((cid, ..)) = added.first() { + st.essential_primary.get_or_insert(*cid); + } let mut ids = vec![]; for (k, (cid, lid, uid, name)) in added.iter().enumerate() { let id = proj.alloc(); @@ -126,7 +130,8 @@ fn add_media(s: &mut Session, p: &Value) -> Result { } let name = str_p(p, "name").map(str::to_string).unwrap_or_else(|| format!("{} (Media)", l.name)); let group = p.get("group").and_then(Value::as_u64); - let id = s.edit("Add Media Replacement", None, |proj, _| { + let id = s.edit("Add Media Replacement", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); let eg = eg_of(proj, cid)?; if eg.flat().iter().any(|c| matches!(c.kind, EgKind::Media { layer } if layer == lid)) { @@ -142,7 +147,8 @@ fn add_group(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let name = str_p(p, "name").unwrap_or("Group").to_string(); let index = p.get("index").and_then(Value::as_u64).map(|i| i as usize); - let id = s.edit("Add Essential Graphics Group", None, |proj, _| { + let id = s.edit("Add Essential Graphics Group", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); insert(eg_of(proj, cid)?, EgControl { id, name, kind: EgKind::Group { children: vec![] } }, None, index, "essential.addGroup")?; Ok(id) @@ -154,7 +160,8 @@ fn add_comment(s: &mut Session, p: &Value) -> Result { let cid = eg_comp(s, p)?; let text = str_p(p, "text").unwrap_or("Comment").to_string(); let group = p.get("group").and_then(Value::as_u64); - let id = s.edit("Add Essential Graphics Comment", None, |proj, _| { + let id = s.edit("Add Essential Graphics Comment", None, |proj, st| { + st.essential_primary.get_or_insert(cid); let id = proj.alloc(); insert(eg_of(proj, cid)?, EgControl { id, name: "Comment".into(), kind: EgKind::Comment { text } }, group, None, "essential.addComment")?; Ok(id) diff --git a/crates/engine/src/commands/expr_tools.rs b/crates/engine/src/commands/expr_tools.rs index 868be63a7..5648fec1e 100644 --- a/crates/engine/src/commands/expr_tools.rs +++ b/crates/engine/src/commands/expr_tools.rs @@ -212,7 +212,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Sub-objects", + "Layer > Sub-objects", vec![ ("source", "source"), ("effect(name)", "effect(\"name\")"), @@ -222,7 +222,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ General", + "Layer > General", vec![ ("width", "width"), ("height", "height"), @@ -238,7 +238,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Properties", + "Layer > Properties", vec![ ("anchorPoint", "anchorPoint"), ("position", "position"), @@ -250,7 +250,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Space Transforms", + "Layer > Space Transforms", vec![ ("toComp(point, t = time)", "toComp(point, t = time)"), ("fromComp(point, t = time)", "fromComp(point, t = time)"), @@ -263,7 +263,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Layer ▸ Other", + "Layer > Other", vec![ ("sourceRectAtTime(t = time, includeExtents = false)", "sourceRectAtTime(t = time, includeExtents = false)"), ("sampleImage(point, radius = [.5, .5], postEffect = true, t = time)", "sampleImage(point, radius = [.5, .5], postEffect = true, t = time)"), @@ -307,7 +307,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ], ), ( - "Text ▸ Style", + "Text > Style", vec![ ("text.sourceText.style", "text.sourceText.style"), ("getStyleAt(charIndex, t = time)", "getStyleAt(charIndex, t = time)"), @@ -317,7 +317,7 @@ pub fn language_menu() -> Vec<(&'static str, Vec<(&'static str, &'static str)>)> ("setText(value)", "setText(value)"), ], ), - ("Text ▸ Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), + ("Text > Expression Selector", vec![("textIndex", "textIndex"), ("textTotal", "textTotal"), ("selectorValue", "selectorValue")]), ("Key", vec![("value", "value"), ("time", "time"), ("index", "index")]), ( "Marker Key", diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index 99bc7c5e4..3644d7ac0 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -34,10 +34,11 @@ pub enum PanelKind { History, Markers, MaskInterpolation, + EssentialGraphics, } impl PanelKind { - pub const ALL: [PanelKind; 24] = [ + pub const ALL: [PanelKind; 25] = [ PanelKind::Project, PanelKind::EffectControls, PanelKind::Composition, @@ -62,6 +63,7 @@ impl PanelKind { PanelKind::History, PanelKind::Markers, PanelKind::MaskInterpolation, + PanelKind::EssentialGraphics, ]; pub fn title(self) -> &'static str { match self { @@ -89,6 +91,7 @@ impl PanelKind { PanelKind::History => "History", PanelKind::Markers => "Markers", PanelKind::MaskInterpolation => "Mask Interpolation", + PanelKind::EssentialGraphics => "Essential Graphics", } } pub fn id(self) -> String { diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index 22b2bd401..f97851f88 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -700,6 +700,9 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option ("path", Ask::Open(&["jsonl", "json", "txt"])), "file.replaceFootage" => ("path", Ask::Import), "file.collectFiles" => ("folder", Ask::Save("Collected Files")), + "essential.exportTemplate" => ("path", Ask::Save("Template.ectemplate")), + "essential.importTemplate" => ("path", Ask::Open(&["ectemplate"])), + "file.setProxy" => ("path", Ask::Open(&["mp4", "mov", "m4v", "mkv", "webm", "png", "jpg", "jpeg", "gif", "webp", "tif", "tiff", "bmp", "exr"])), _ => return None, }; if params.get(key).is_some() @@ -716,7 +719,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option None, diff --git a/crates/ui-egui/src/panels/essential.rs b/crates/ui-egui/src/panels/essential.rs new file mode 100644 index 000000000..f4c565c52 --- /dev/null +++ b/crates/ui-egui/src/panels/essential.rs @@ -0,0 +1,427 @@ +//! Essential Graphics panel (Window ▸ Essential Graphics). +//! +//! - **Primary** composition, **Solo Supported Properties** (the timeline lists only what can +//! be exposed) and the template name. +//! - The controls: drag a property name from the timeline onto the list (or Animation ▸ Add +//! Property to Essential Graphics); each shows as a text / colour / slider / checkbox / point / +//! angle / dropdown control editing the source property; rename by double-clicking the name; +//! reorder with ▲▼, remove with ✕; groups and comments from Add Formatting; Media +//! Replacement for the selected footage layer. +//! - **Export Template…** writes an open `.ectemplate`. +//! - With a precomp layer that has Essential Properties selected, its instance values: edit them +//! (overrides), Revert or Push to Comp. +//! +//! Every control runs an engine `essential.*` / `prop.set` command and registers an automation +//! id (`essential.primary`, `essential.solo`, `essential.name`, `essential.control..value`, +//! `essential.instance..value`, `essential.export`…). + +use effectcraft_engine::keyframe::Value as KV; +use effectcraft_engine::project::essential::{self, ControlType, EgControl, EgKind}; +use effectcraft_engine::project::{ItemId, ItemKind, LayerSource, ParamUi, Property}; +use egui::{Rect, RichText}; +use serde_json::{Value, json}; + +use crate::EffectcraftApp; +use crate::panels::DragPayload; + +type Actions = Vec<(String, Value)>; + +/// The comp the panel edits: the Primary comp, else the active one. +fn primary(app: &EffectcraftApp) -> Option { + let s = &app.session; + s.state.essential_primary.filter(|c| s.project.comp(*c).is_some()).or(s.active_comp_id()) +} + +pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { + let ctx = ui.ctx().clone(); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink(8.0)).layout(egui::Layout::top_down(egui::Align::Min))); + let mut out = None; + egui::ScrollArea::vertical().id_salt("eg-panel").auto_shrink([false, false]).show(&mut child, |ui| out = Some(body(app, ui))); + let Some((actions, invoke)) = out else { return }; + for (id, p) in actions { + if let Err(e) = app.session.execute(&id, p) { + app.ui.status = e.to_string(); + } + } + for (id, p) in invoke { + if let Err(e) = crate::menus::invoke(app, &ctx, &id, p) { + app.ui.status = e; + } + } +} + +fn body(app: &mut EffectcraftApp, ui: &mut egui::Ui) -> (Actions, Actions) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut actions: Actions = vec![]; + let mut invoke: Vec<(String, Value)> = vec![]; + let Some(cid) = primary(app) else { + ui.label(RichText::new("Open a composition to build Essential Graphics").color(t.text_faint)); + return (actions, invoke); + }; + let project = app.session.project.clone(); + let comp_name = project.item(cid).map(|i| i.name.clone()).unwrap_or_default(); + + // ---- Primary + Solo. + ui.horizontal(|ui| { + ui.label(RichText::new("Primary:").color(t.text_dim)); + let r = egui::ComboBox::from_id_salt("eg-primary").selected_text(&comp_name).width(150.0).show_ui(ui, |ui| { + for (id, _) in project.comps() { + let name = project.item(*id).map(|i| i.name.clone()).unwrap_or_default(); + if ui.selectable_label(*id == cid, name).clicked() { + actions.push(("essential.setPrimary".into(), json!({"comp": id.0}))); + } + } + }); + app.auto.add("essential.primary", r.response.rect, &comp_name); + }); + let solo = app.session.state.essential_solo; + let r = ui.add(egui::Button::new("Solo Supported Properties").selected(solo)); + app.auto.add("essential.solo", r.rect, "Solo Supported Properties"); + if r.clicked() { + actions.push(("essential.soloSupported".into(), json!({"on": !solo}))); + } + // ---- Name. + let eg = project.comp(cid).and_then(|c| c.essential.clone()).unwrap_or_default(); + let shown_name = if eg.name.is_empty() { comp_name.clone() } else { eg.name.clone() }; + let name_id = egui::Id::new("eg-name-buf"); + let mut name: String = ctx.data(|d| d.get_temp(name_id)).unwrap_or_else(|| shown_name.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut name).hint_text("Template name").desired_width(f32::INFINITY)); + app.auto.add("essential.name", r.rect, &shown_name); + if r.has_focus() { + ctx.data_mut(|d| d.insert_temp(name_id, name.clone())); + } else { + if r.lost_focus() && name.trim() != shown_name { + actions.push(("essential.setName".into(), json!({"comp": cid.0, "name": name.trim()}))); + } + ctx.data_mut(|d| d.remove::(name_id)); + } + ui.separator(); + + // ---- Controls (drop target for timeline properties). + let list = ui.scope(|ui| { + ui.set_min_height(60.0); + if eg.controls.is_empty() { + ui.label(RichText::new("Drag properties here from the timeline,\nor use Animation > Add Property to Essential Graphics.").color(t.text_faint)); + } + controls(app, ui, cid, &eg.controls, None, &mut actions); + }); + let drop_rect = list.response.rect.expand2(egui::vec2(0.0, 4.0)); + app.auto.add("essential.dropZone", drop_rect, "Drag properties here"); + if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && let DragPayload::Property { layer, prop } = *payload + && ui.rect_contains_pointer(drop_rect) + { + ui.painter().rect_stroke(drop_rect, 2.0, egui::Stroke::new(2.0, t.accent), egui::StrokeKind::Inside); + if ctx.input(|i| i.pointer.any_released()) { + let comp = app.session.active_comp_id().map(|c| c.0); + actions.push(("essential.addProperty".into(), json!({"comp": comp, "layer": layer, "prop": prop}))); + egui::DragAndDrop::clear_payload(&ctx); + } + } + ui.separator(); + + // ---- Bottom buttons. + ui.horizontal_wrapped(|ui| { + let r = ui.menu_button("Add Formatting", |ui| { + if ui.button("Add Group").clicked() { + actions.push(("essential.addGroup".into(), json!({"comp": cid.0}))); + ui.close(); + } + if ui.button("Add Comment").clicked() { + actions.push(("essential.addComment".into(), json!({"comp": cid.0}))); + ui.close(); + } + }); + app.auto.add("essential.addFormatting", r.response.rect, "Add Formatting"); + let footage_layer = app.session.active_comp().and_then(|c| { + app.session.state.selected_layers.iter().filter_map(|l| c.layer(*l)).find(|l| matches!(l.source, LayerSource::Footage { .. })).map(|l| l.id.0) + }); + let r = ui.add_enabled(footage_layer.is_some(), egui::Button::new("Media Replacement")); + app.auto.add("essential.addMedia", r.rect, "Add Media Replacement"); + if r.clicked() + && let Some(l) = footage_layer + { + actions.push(("essential.addMedia".into(), json!({"layer": l}))); + } + let r = ui.add_enabled(!eg.controls.is_empty(), egui::Button::new("Export Template…")); + app.auto.add("essential.export", r.rect, "Export Template"); + if r.clicked() { + invoke.push(("essential.exportTemplate".into(), json!({"comp": cid.0}))); + } + }); + + // ---- The selected instance's Essential Properties. + instance(app, ui, &mut actions); + (actions, invoke) +} + +/// One list of controls (top level or a group's children). +fn controls(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, list: &[EgControl], group: Option, actions: &mut Actions) { + let t = app.tokens; + let n = list.len(); + for (i, c) in list.iter().enumerate() { + let row = ui.horizontal(|ui| { + // Reorder / remove. + for (label, auto, enabled, to) in [("▲", "up", i > 0, i.saturating_sub(1)), ("▼", "down", i + 1 < n, i + 1)] { + let r = ui.add_enabled(enabled, egui::Button::new(RichText::new(label).size(9.0)).small().frame(false)); + app.auto.add(&format!("essential.control.{}.{auto}", c.id), r.rect, label); + if r.clicked() { + actions.push(("essential.move".into(), json!({"comp": cid.0, "control": c.id, "group": group, "index": to}))); + } + } + match &c.kind { + EgKind::Group { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name).strong(), actions); + } + EgKind::Comment { text } => { + let shown = EgControl { name: text.clone(), ..c.clone() }; + name_label(app, ui, cid, &shown, RichText::new(text).italics().color(t.text_dim), actions); + } + EgKind::Property { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let src = essential::source_prop(&app.session.project, cid, c).map(|(l, p)| (l.id.0, l.layer_time(app.session.time()), p.clone())); + match src { + Some((layer, lt, p)) => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &format!("essential.control.{}.value", c.id), &p, &v) { + actions.push(( + "prop.set".into(), + json!({"comp": cid.0, "layer": layer, "prop": p.uid, "value": nv, "merge": format!("eg-{}", c.id)}), + )); + } + } + None => { + ui.label(RichText::new("(missing)").color(t.danger)); + } + } + } + EgKind::Media { .. } => { + name_label(app, ui, cid, c, RichText::new(&c.name), actions); + let item = essential::source_media(&app.session.project, cid, c); + let name = item.and_then(|i| app.session.project.item(i)).map(|i| i.name.clone()).unwrap_or_default(); + ui.label(RichText::new(format!("▣ {name}")).color(t.text_dim)); + } + } + let r = ui.add(egui::Button::new(RichText::new("×").size(10.0)).small().frame(false)).on_hover_text("Remove"); + app.auto.add(&format!("essential.control.{}.remove", c.id), r.rect, "Remove"); + if r.clicked() { + actions.push(("essential.remove".into(), json!({"comp": cid.0, "control": c.id}))); + } + }); + app.auto.add(&format!("essential.control.{}.row", c.id), row.response.rect, &c.name); + if let EgKind::Group { children } = &c.kind { + ui.indent(("eg-group", c.id), |ui| controls(app, ui, cid, children, Some(c.id), actions)); + } + } +} + +/// A control's name; double-click to rename (comments: edit the text). +fn name_label(app: &mut EffectcraftApp, ui: &mut egui::Ui, cid: ItemId, c: &EgControl, text: RichText, actions: &mut Actions) { + let edit_id = egui::Id::new(("eg-rename", c.id)); + let editing: Option = ui.ctx().data(|d| d.get_temp(edit_id)); + match editing { + Some(mut buf) => { + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(120.0)); + app.auto.add(&format!("essential.control.{}.rename", c.id), r.rect, &c.name); + r.request_focus(); + if r.lost_focus() || ui.input(|i| i.key_pressed(egui::Key::Enter)) { + if !buf.trim().is_empty() && buf.trim() != c.name { + actions.push(("essential.rename".into(), json!({"comp": cid.0, "control": c.id, "name": buf.trim()}))); + } + ui.ctx().data_mut(|d| d.remove::(edit_id)); + } else { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, buf)); + } + } + None => { + let r = ui.add(egui::Label::new(text).sense(egui::Sense::click()).truncate()); + app.auto.add(&format!("essential.control.{}.name", c.id), r.rect, &c.name); + if r.double_clicked() { + ui.ctx().data_mut(|d| d.insert_temp(edit_id, c.name.clone())); + } + } + } +} + +/// The editor for a control's value; returns the new value (JSON for `prop.set`). +fn value_widget(app: &mut EffectcraftApp, ui: &mut egui::Ui, auto: &str, p: &Property, v: &KV) -> Option { + let ty = essential::control_type(p)?; + let mut out = None; + let rect; + match ty { + ControlType::Text => { + let cur = match v { + KV::Text(d) => d.text.clone(), + KV::Str(s) => s.clone(), + _ => String::new(), + }; + let id = egui::Id::new(("eg-text", auto)); + let mut buf: String = ui.ctx().data(|d| d.get_temp(id)).unwrap_or_else(|| cur.clone()); + let r = ui.add(egui::TextEdit::singleline(&mut buf).desired_width(140.0)); + if r.has_focus() { + ui.ctx().data_mut(|d| d.insert_temp(id, buf)); + } else { + if r.lost_focus() && buf != cur { + out = Some(json!(buf)); + } + ui.ctx().data_mut(|d| d.remove::(id)); + } + rect = r.rect; + } + ControlType::Color => { + let c = v.as_color(); + let mut rgba = egui::Rgba::from_rgba_unmultiplied(c[0], c[1], c[2], c[3]); + let r = egui::widgets::color_picker::color_edit_button_rgba(ui, &mut rgba, egui::widgets::color_picker::Alpha::Opaque); + if r.changed() { + let u = rgba.to_rgba_unmultiplied(); + out = Some(json!([u[0], u[1], u[2], u[3]])); + } + rect = r.rect; + } + ControlType::Checkbox => { + let mut on = v.as_bool(); + let r = ui.checkbox(&mut on, ""); + if r.changed() { + out = Some(json!(on)); + } + rect = r.rect; + } + ControlType::Dropdown => { + let opts = match &p.ui { + ParamUi::Popup { options } => options.clone(), + _ => vec![], + }; + let cur = v.as_enum() as usize; + let r = egui::ComboBox::from_id_salt(("eg-dd", auto)).selected_text(opts.get(cur).cloned().unwrap_or_default()).show_ui(ui, |ui| { + for (i, o) in opts.iter().enumerate() { + if ui.selectable_label(i == cur, o).clicked() { + out = Some(json!(i)); + } + } + }); + rect = r.response.rect; + } + ControlType::Point => { + let mut a = v.as_vec2(); + let r1 = ui.add(egui::DragValue::new(&mut a[0]).speed(1.0).max_decimals(1)); + let r2 = ui.add(egui::DragValue::new(&mut a[1]).speed(1.0).max_decimals(1)); + if r1.changed() || r2.changed() { + out = Some(json!(a)); + } + rect = r1.rect.union(r2.rect); + } + ControlType::Angle => { + let mut a = v.as_f64(); + let r = ui.add(egui::DragValue::new(&mut a).speed(1.0).suffix("°").max_decimals(1)); + if r.changed() { + out = Some(json!(a)); + } + rect = r.rect; + } + ControlType::Slider | ControlType::Media => { + let mut x = v.as_f64(); + let r = match p.ui { + ParamUi::Slider { slider_min, slider_max, .. } => ui.add(egui::Slider::new(&mut x, slider_min..=slider_max).max_decimals(2)), + _ => ui.add(egui::DragValue::new(&mut x).speed(0.5).max_decimals(2)), + }; + if r.changed() { + out = Some(json!(x)); + } + rect = r.rect; + } + } + app.auto.add(auto, rect, &p.name); + out +} + +/// Essential Properties of the selected precomp layer: per-instance values, overrides marked ●, +/// Revert and Push to Comp. +fn instance(app: &mut EffectcraftApp, ui: &mut egui::Ui, actions: &mut Actions) { + let t = app.tokens; + let Some(comp) = app.session.active_comp().cloned() else { return }; + let Some(layer) = app.session.state.selected_layers.iter().filter_map(|l| comp.layer(*l)).find(|l| essential::group(l).is_some()).cloned() else { return }; + let LayerSource::Comp { item: src } = layer.source else { return }; + let Some(eg) = app.session.project.comp(src).and_then(|c| c.essential.clone()) else { return }; + let over = essential::overridden(&layer); + let lt = layer.layer_time(app.session.time()); + ui.separator(); + ui.label(RichText::new(format!("Essential Properties — {}", layer.name)).strong()); + let group = essential::group(&layer).cloned().unwrap_or_else(|| effectcraft_engine::project::PropGroup::new(0, "", "")); + for c in eg.flat() { + let m = essential::match_id(c.id); + let mut pr = None; + group.walk("", &mut |_, x| { + if x.match_id == m { + pr = Some(x.clone()); + } + }); + let Some(p) = pr else { continue }; + let is_over = over.contains(&p.uid); + ui.horizontal(|ui| { + ui.label(RichText::new(if is_over { "●" } else { "○" }).color(if is_over { t.accent } else { t.text_faint })).on_hover_text(if is_over { + "Overridden in this instance" + } else { + "Follows the source composition" + }); + ui.label(&c.name); + let auto = format!("essential.instance.{}.value", c.id); + match c.kind { + EgKind::Media { .. } => { + let cur = ItemId(p.value.as_f64() as u64); + let cur_name = app.session.project.item(cur).map(|i| i.name.clone()).unwrap_or_default(); + let items: Vec<(ItemId, String)> = app + .session + .project + .items + .values() + .filter(|i| matches!(&i.kind, ItemKind::Footage(f) if f.has_video)) + .map(|i| (i.id, i.name.clone())) + .collect(); + let r = egui::ComboBox::from_id_salt(("eg-media", layer.id.0, c.id)).selected_text(cur_name).show_ui(ui, |ui| { + for (id, name) in &items { + if ui.selectable_label(*id == cur, name).clicked() { + actions.push(("essential.set".into(), json!({"layer": layer.id.0, "control": c.id, "item": id.0}))); + } + } + }); + app.auto.add(&auto, r.response.rect, &c.name); + } + _ => { + let v = p.value_at(lt); + if let Some(nv) = value_widget(app, ui, &auto, &p, &v) { + actions.push(( + "essential.set".into(), + json!({"layer": layer.id.0, "control": c.id, "value": nv, "merge": format!("egi-{}-{}", layer.id.0, c.id)}), + )); + } + } + } + if is_over { + let r = ui.small_button("Revert"); + app.auto.add(&format!("essential.instance.{}.revert", c.id), r.rect, "Revert"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + let r = ui.small_button("Push"); + app.auto.add(&format!("essential.instance.{}.push", c.id), r.rect, "Push to Comp"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0, "control": c.id}))); + } + } + }); + } + ui.horizontal(|ui| { + let any = !over.is_empty(); + let r = ui.add_enabled(any, egui::Button::new("Push All to Comp")); + app.auto.add("essential.instance.pushAll", r.rect, "Push Override Values to Source"); + if r.clicked() { + actions.push(("essential.pushToComp".into(), json!({"layer": layer.id.0}))); + } + let r = ui.add_enabled(any, egui::Button::new("Revert All")); + app.auto.add("essential.instance.revertAll", r.rect, "Revert All"); + if r.clicked() { + actions.push(("essential.revert".into(), json!({"layer": layer.id.0}))); + } + }); +} diff --git a/crates/ui-egui/src/panels/expr_bar.rs b/crates/ui-egui/src/panels/expr_bar.rs new file mode 100644 index 000000000..09d4a608b --- /dev/null +++ b/crates/ui-egui/src/panels/expr_bar.rs @@ -0,0 +1,95 @@ +//! The expression error bar: a strip at the bottom of the Composition viewer listing failing +//! expressions of the comp at the current time (`expr.errors`), with ◀ ▶ to step through them. +//! Clicking the message selects the layer and reveals the property in the timeline. +//! Automation ids: `viewer.exprErrors`, `viewer.exprErrors.prev`, `viewer.exprErrors.next`. + +use effectcraft_engine::commands::expr_tools::{self, ExprError}; +use effectcraft_engine::project::ItemId; +use egui::{Align2, Color32, Rect, Sense, pos2, vec2}; +use serde_json::json; + +use crate::EffectcraftApp; +use crate::theme::Tokens; + +/// Height of the bar. +pub const HEIGHT: f32 = 22.0; + +#[derive(Clone, Default)] +struct Cache { + key: (u64, u64, i64), + errors: Vec, + /// Shown error (index). + current: usize, +} + +fn cache_id() -> egui::Id { + egui::Id::new("expr-error-bar") +} + +/// The comp's failing expressions (recomputed when the project or the time changes; kept while +/// playing). +fn errors(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> Cache { + let t = app.session.time_of(cid); + let key = (app.session.revision, cid.0, t.0); + let mut c: Cache = ctx.data(|d| d.get_temp(cache_id())).unwrap_or_default(); + let stale = c.key != key && !(app.playback.playing && c.key.1 == cid.0); + if stale { + c.errors = expr_tools::errors(&app.session, cid, t); + c.key = key; + c.current = c.current.min(c.errors.len().saturating_sub(1)); + ctx.data_mut(|d| d.insert_temp(cache_id(), c.clone())); + } + c +} + +/// Height the bar needs for `cid` (0 when every expression evaluates). +pub fn height(app: &EffectcraftApp, ctx: &egui::Context, cid: ItemId) -> f32 { + if app.session.expr.is_none() || errors(app, ctx, cid).errors.is_empty() { 0.0 } else { HEIGHT } +} + +pub fn draw(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, cid: ItemId) { + let t = app.tokens; + let ctx = ui.ctx().clone(); + let mut c = errors(app, &ctx, cid); + if c.errors.is_empty() { + return; + } + let p = ui.painter().with_clip_rect(rect); + p.rect_filled(rect, 0.0, Color32::from_rgb(0x4a, 0x3a, 0x12)); + let warn = Color32::from_rgb(0xf5, 0xb0, 0x30); + p.text(pos2(rect.min.x + 10.0, rect.center().y), Align2::LEFT_CENTER, "⚠", Tokens::ui(13.0), warn); + let n = c.errors.len(); + let i = c.current.min(n - 1); + // ◀ i/n ▶ on the right. + let next = Rect::from_center_size(pos2(rect.max.x - 14.0, rect.center().y), vec2(16.0, 16.0)); + let count = Rect::from_center_size(pos2(rect.max.x - 44.0, rect.center().y), vec2(40.0, 16.0)); + let prev = Rect::from_center_size(pos2(rect.max.x - 74.0, rect.center().y), vec2(16.0, 16.0)); + for (r, label, auto, d) in [(prev, "◀", "viewer.exprErrors.prev", n - 1), (next, "▶", "viewer.exprErrors.next", 1)] { + let resp = ui.interact(r, egui::Id::new(("expr-bar", auto)), Sense::click()); + p.text(r.center(), Align2::CENTER_CENTER, label, Tokens::ui(11.0), if resp.hovered() { Color32::WHITE } else { t.text }); + app.auto.add(auto, r, label); + if resp.clicked() { + c.current = (i + d) % n; + ctx.data_mut(|dd| dd.insert_temp(cache_id(), c.clone())); + } + } + p.text(count.center(), Align2::CENTER_CENTER, format!("{}/{n}", i + 1), Tokens::ui(11.0), t.text); + let comp_name = app.session.project.item(cid).map(|it| it.name.clone()).unwrap_or_default(); + let e = c.errors[i].clone(); + let msg_rect = Rect::from_min_max(pos2(rect.min.x + 26.0, rect.min.y), pos2(prev.min.x - 6.0, rect.max.y)); + let resp = ui.interact(msg_rect, egui::Id::new("expr-bar-msg"), Sense::click()).on_hover_text("Click to reveal the expression in the timeline"); + p.with_clip_rect(msg_rect).text( + pos2(msg_rect.min.x, rect.center().y), + Align2::LEFT_CENTER, + e.describe(&comp_name), + Tokens::ui(11.5), + if resp.hovered() { Color32::WHITE } else { t.text }, + ); + app.auto.add("viewer.exprErrors", msg_rect, &e.describe(&comp_name)); + if resp.clicked() { + let _ = app.session.execute("layer.select", json!({"comp": cid.0, "layers": [e.layer]})); + let _ = app.session.execute("prop.select", json!({"comp": cid.0, "layer": e.layer, "prop": e.prop, "selectKeys": false})); + let _ = crate::menus::frontend(app, &ctx, "timeline.revealProps", json!({"props": [{"layer": e.layer, "prop": e.prop}]})); + app.ui.timeline.expr_closed.remove(&e.prop); + } +} diff --git a/crates/ui-egui/src/panels/forms.rs b/crates/ui-egui/src/panels/forms.rs index 4b731cca6..af117e8eb 100644 --- a/crates/ui-egui/src/panels/forms.rs +++ b/crates/ui-egui/src/panels/forms.rs @@ -218,27 +218,48 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ), ], ), - "file.interpretFootage" if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile"]) => { - let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match &it.kind { - effectcraft_engine::project::ItemKind::Footage(f) => Some(f.clone()), + "file.interpretFootage" | "file.interpretProxy" + if !has(p, &["frameRate", "alpha", "loop", "pixelAspect", "colorProfile", "fields", "invertAlpha", "matteColor", "linearLight"]) => + { + let proxy = id == "file.interpretProxy"; + let f = s.state.project_selection.first().and_then(|i| s.project.item(*i)).and_then(|it| match (&it.kind, &it.proxy) { + (_, Some(px)) if proxy => Some(px.footage.clone()), + (effectcraft_engine::project::ItemKind::Footage(f), _) if !proxy => Some(f.clone()), _ => None, }); let Some(f) = f else { return false }; + let hex = |c: [f32; 3]| format!("#{:02x}{:02x}{:02x}", (c[0] * 255.0).round() as u8, (c[1] * 255.0).round() as u8, (c[2] * 255.0).round() as u8); ( - "Interpret Footage".into(), + if proxy { "Interpret Footage: Proxy".into() } else { "Interpret Footage".into() }, vec![ - Field::num("frameRate", "Frame rate", f.frame_rate.as_f64()), + // Main Options ▸ Alpha. Field::choice( "alpha", "Alpha", - &[("Straight", json!("straight")), ("Premultiplied", json!("premultiplied")), ("Ignore", json!("ignore"))], + &[ + ("Interpret Straight - Unmatted", json!("straight")), + ("Interpret Premultiplied - Matted With Color", json!("premultiplied")), + ("Ignore", json!("ignore")), + ("Guess", json!("guess")), + ], f.alpha as usize, ), + Field::text("matteColor", "Matte color (premultiplied)", &hex(f.premul_color)), + Field::bool("invertAlpha", "Invert Alpha", f.invert_alpha), + // Main Options ▸ Frame Rate, Fields and Pulldown, Other Options. + Field::num("frameRate", "Assume this frame rate", f.frame_rate.as_f64()), + Field::choice( + "fields", + "Separate Fields", + &[("Off", json!("off")), ("Upper Field First", json!("upper")), ("Lower Field First", json!("lower"))], + f.fields as usize, + ), + Field::num("pixelAspect", "Pixel Aspect Ratio", f.pixel_aspect), Field::num("loop", "Loop (times)", f.loop_count as f64), - Field::num("pixelAspect", "Pixel aspect ratio", f.pixel_aspect), + // Color. Field::choice( "colorProfile", - "Color profile", + "Assign Profile", &[ ("Embedded / sRGB", json!("auto")), ("sRGB IEC61966-2.1", json!("srgb")), @@ -248,6 +269,7 @@ pub fn open_form(app: &mut EffectcraftApp, id: &str, p: &Value) -> bool { ], f.color_profile.map_or(0, |c| 1 + effectcraft_engine::project::ColorSpace::ALL.iter().position(|x| *x == c).unwrap_or(0)), ), + Field::bool("linearLight", "Interpret As Linear Light", f.linear_light), ], ) } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 3f7375022..df60beb3a 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -6,6 +6,8 @@ pub mod dialogs; pub mod dialogs_3d; pub mod effect_controls; pub mod effects_presets; +pub mod essential; +pub mod expr_bar; pub mod forms; pub mod fx_widgets; pub mod graph; @@ -47,6 +49,8 @@ pub enum DragPayload { Item(u64), /// An effect id from Effects & Presets. Effect(String), + /// A property dragged from the timeline (to the Essential Graphics panel). + Property { layer: u64, prop: u64 }, } /// The current time formatted per project settings (timecode with `;` for drop-frame, or frames). @@ -90,6 +94,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rec PanelKind::Layer => layer_panel::show(app, ui, rect), PanelKind::Paint => paint_panels::paint(app, ui, rect), PanelKind::Brushes => paint_panels::brushes(app, ui, rect), + PanelKind::EssentialGraphics => essential::show(app, ui, rect), other => misc::placeholder(app, ui, rect, other), } } diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index d13f37687..6482b555e 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -322,6 +322,24 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { app.auto.add(&format!("project.item.{}.twirl", id.0), tw, &it.name); } icons::paint(&lp, Rect::from_center_size(pos2(x0 + 18.0, r.center().y), vec2(14.0, 14.0)), item_icon(&it), t.text_dim); + // Proxy indicator: filled = the proxy is used, hollow = set but off. Click to switch. + if let Some(px) = it.proxy.as_ref() { + let pr = Rect::from_center_size(pos2(x0 + 4.0, r.center().y), vec2(9.0, 9.0)); + if px.enabled { + lp.rect_filled(pr, 1.0, t.text); + } else { + lp.rect_stroke(pr, 1.0, Stroke::new(1.0, t.text), egui::StrokeKind::Inside); + } + let presp = ui.interact(pr.expand(2.0).intersect(list), egui::Id::new(("pproxy", id.0)), Sense::click()).on_hover_text(format!( + "Proxy: {} ({})", + px.footage.path, + if px.enabled { "in use" } else { "off" } + )); + app.auto.add(&format!("project.item.{}.proxy", id.0), pr, if px.enabled { "Proxy in use" } else { "Proxy off" }); + if presp.clicked() { + actions.push(("file.useProxy".into(), json!({"item": id.0}))); + } + } let name_clip = Rect::from_min_max(pos2(x0 + 30.0, r.min.y), pos2(r.min.x + name_w + 4.0, r.max.y)).intersect(list); let name_rect = Rect::from_min_max(pos2(x0 + 28.0, r.min.y + 2.0), pos2(r.min.x + name_w + 2.0, r.max.y - 2.0)); // Inline edits (rename / comment). diff --git a/crates/ui-egui/src/panels/render_queue.rs b/crates/ui-egui/src/panels/render_queue.rs index 154dd4c2d..b29bcf185 100644 --- a/crates/ui-egui/src/panels/render_queue.rs +++ b/crates/ui-egui/src/panels/render_queue.rs @@ -77,6 +77,10 @@ fn settings_menu(it: &RenderQueueItem) -> Vec<(String, Value)> { v.push(("-".into(), Value::Null)); v.push((mark(s.motion_blur, "Motion Blur: On for Checked Layers"), json!({"motionBlur": !s.motion_blur}))); v.push((mark(s.skip_existing, "Skip Existing Files"), json!({"skipExisting": !s.skip_existing}))); + v.push(("-".into(), Value::Null)); + for (u, k) in effectcraft_engine::project::render_queue::ProxyUse::ALL.iter().zip(["current", "all", "comp", "none"]) { + v.push((mark(s.proxy_use == *u, &format!("Proxy Use: {}", u.label())), json!({"proxyUse": k}))); + } v } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index cd3bf6f92..b05e523a6 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -234,6 +234,19 @@ fn build_rows(app: &EffectcraftApp, comp: &Comp) -> Vec { continue; } rows.push(Row { layer: l.id, depth: 0, kind: RowKind::Layer }); + // Essential Graphics ▸ Solo Supported Properties: every property it can expose. + if app.session.state.essential_solo { + let mut v = vec![]; + for (i, c) in l.props.children.iter().enumerate() { + if c.match_id() == effectcraft_engine::project::essential::GROUP { + continue; + } + let g = PropGroup { children: vec![l.props.children[i].clone()], ..PropGroup::new(0, "", "") }; + collect_props(&g, &mut v, &|p| prop_visible(p, l) && effectcraft_engine::project::essential::control_type(p).is_some()); + } + rows.extend(v.into_iter().map(|uid| Row { layer: l.id, depth: 1, kind: RowKind::Prop { uid } })); + continue; + } if !tl.open_layers.contains(&l.id.0) { continue; } @@ -1143,6 +1156,40 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { lp.text(wr.center(), Align2::CENTER_CENTER, "⚠", Tokens::ui(12.0), Color32::from_rgb(0xf0, 0xa0, 0x30)); let _ = ui.interact(wr, egui::Id::new(("expr-err", uid)), Sense::hover()).on_hover_text(e); } + // ⓕ: the Expression Language menu inserts at the cursor. + let lang_r = Rect::from_center_size(pos2(cw.switches + 70.0, cy), vec2(16.0, 16.0)); + let lang = ui.interact(lang_r, egui::Id::new(("expr-lang", uid)), Sense::click()).on_hover_text("Expression Language Menu"); + lp.circle_stroke(lang_r.center(), 6.5, Stroke::new(1.0, if lang.hovered() { t.text } else { t.text_dim })); + lp.text(lang_r.center(), Align2::CENTER_CENTER, "f", Tokens::semibold(10.0), if lang.hovered() { t.text } else { t.text_dim }); + app.auto.add(&format!("timeline.prop.{uid}.exprLanguage"), lang_r, "Expression Language Menu"); + let mut picked: Option<&'static str> = None; + egui::Popup::menu(&lang).show(|ui| { + for (cat, items) in effectcraft_engine::commands::expr_tools::language_menu() { + ui.menu_button(cat, |ui| { + for (label, text) in items { + if ui.button(label).clicked() { + picked = Some(text); + ui.close(); + } + } + }); + } + }); + if let Some(text) = picked { + let edit_id = egui::Id::new(("expr-edit", uid)); + let buf_id = egui::Id::new(("expr-buf", uid)); + let cur: String = ctx.data(|d| d.get_temp(buf_id)).unwrap_or_else(|| ex.text.clone()); + let at = egui::text_edit::TextEditState::load(&ctx, edit_id) + .and_then(|st| st.cursor.char_range()) + .map(|r| r.primary.index.0) + .unwrap_or(cur.chars().count()) + .min(cur.chars().count()); + let byte = cur.char_indices().nth(at).map(|(b, _)| b).unwrap_or(cur.len()); + let mut next = cur.clone(); + next.insert_str(byte, text); + actions.push(("prop.setExpression".into(), json!({"layer": layer.id.0, "prop": uid, "expression": next}))); + ctx.data_mut(|d| d.remove::(buf_id)); + } // The editor in the time-graph area. ui.set_clip_rect(right_clip); gp.rect_filled(Rect::from_min_max(pos2(graph_x0, r.min.y), r.max), 0.0, Color32::from_rgb(0x1a, 0x1a, 0x1a)); @@ -1253,8 +1300,19 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { Tokens::ui(12.0), if prop.has_expression() { Color32::from_rgb(0xe8, 0x7c, 0x5c) } else { t.text }, ); - let name_resp = - ui.interact(Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), egui::Id::new(("pname", uid)), Sense::click()); + let name_resp = ui.interact( + Rect::from_min_max(pos2(name_x, r.min.y), pos2(cw.switches - 2.0, r.max.y)), + egui::Id::new(("pname", uid)), + Sense::click_and_drag(), + ); + if name_resp.drag_started() { + // Drop on the Essential Graphics panel to expose the property. + egui::DragAndDrop::set_payload(&ctx, crate::panels::DragPayload::Property { layer: layer.id.0, prop: *uid }); + } + if name_resp.dragged() { + egui::Tooltip::always_open(ctx.clone(), ui.layer_id(), egui::Id::new("prop-drag-tip"), egui::PopupAnchor::Pointer) + .show(|ui| ui.label(&prop.name)); + } if name_resp.clicked() { actions.push(("prop.select".into(), json!({"layer": layer.id.0, "prop": uid, "add": ui.input(|i| i.modifiers.shift)}))); } @@ -1494,6 +1552,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Drop targets: footage/comps from the Project panel, effects from Effects & Presets. if let Some(payload) = egui::DragAndDrop::payload::(&ctx) + && !matches!(*payload, crate::panels::DragPayload::Property { .. }) && ui.rect_contains_pointer(rows_rect) { p.rect_stroke(rows_rect, 0.0, Stroke::new(2.0, t.accent), StrokeKind::Inside); @@ -1507,6 +1566,7 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { actions.push(("effect.apply".into(), json!({"effect": e, "layers": [l.0]}))); } } + crate::panels::DragPayload::Property { .. } => {} } egui::DragAndDrop::clear_payload(&ctx); } diff --git a/crates/ui-egui/src/panels/viewer.rs b/crates/ui-egui/src/panels/viewer.rs index 76ef92a9e..227e916ad 100644 --- a/crates/ui-egui/src/panels/viewer.rs +++ b/crates/ui-egui/src/panels/viewer.rs @@ -347,7 +347,10 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { // Bottom control bar. let bar_h = 30.0; let bar = Rect::from_min_max(pos2(rect.min.x, rect.max.y - bar_h), rect.max); - let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, bar.min.y)); + // The expression error bar sits above the control bar while expressions fail. + let err_h = super::expr_bar::height(app, &ctx, cid); + let err_bar = Rect::from_min_max(pos2(rect.min.x, bar.min.y - err_h), pos2(rect.max.x, bar.min.y)); + let full = Rect::from_min_max(pos2(rect.min.x, nav.max.y), pos2(rect.max.x, err_bar.min.y)); let pasteboard = app.ui.viewer.pasteboard.map(|[r, g, b]| Color32::from_rgb(r, g, b)).unwrap_or(t.pasteboard); // View ▸ Switch View Layout: extra views (Top / Front / Right) beside the main view, which // keeps the overlays and the interaction. @@ -1158,6 +1161,9 @@ pub fn show(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect) { vt::draw_snap(&ctx, &painter, &map, &ectx); vt::rulers(app, ui, &map, outer, area); vt::bottom_bar(app, ui, bar, zoom, fit, time, &comp); + if err_h > 0.0 { + super::expr_bar::draw(app, ui, err_bar, cid); + } } /// Parent-space position change per comp pixel of drag (x and y) for a layer: 2D layers map From b927964d983432bb8f817fb921e95bc81170602d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:32:13 -0400 Subject: [PATCH 7/9] CMP-7: document Essential Graphics, templates, responsive time, proxies and expression data Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 24 +++++++++++++++++++++++- docs/parity.md | 6 +++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 95bad818f..a030e26c1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -205,11 +205,33 @@ auto-scale; cached per analysis and settings) and warps each frame; Synthesize E borders from neighbouring frames read with `EffectHost::self_at`. The effect lives in the effects crate, hence the `effects → track` edge. +**Essential Graphics** (`project::essential`, `engine::commands::essential`): a comp's +`essential` field lists its exposed controls (properties, Media Replacement, groups, comments). +Every precomp layer of such a comp gets an Essential Properties group (`GroupKind::Essential`, +children `eg`), kept in step by `Session::edit`; editing a child overrides it for +that instance. The renderer draws an instance with overrides from a copy of the project with the +overridden source properties set (`essential::with_overrides`). Templates are `.ectemplate` +ZIPs: `manifest.json` (controls), `project.ecproj` (the comp and its dependencies), `media/…` and +`poster.png`; importing offsets every id past the project's (`essential::offset_ids`). +**Responsive Design — Time**: a time-stretched precomp maps time piecewise so its protected +marker regions play at 100 % (`eval::responsive_source_time`). + +**Proxies and interpretation**: an item's `proxy` (footage or comp) is decoded in its place at +the source's nominal size when `RenderOpts::proxy` (Render Settings ▸ Proxy Use) allows it. +Interpret Footage's Separate Fields turns each field into a frame at twice the rate (the other +lines interpolated); pixel aspect stretches footage, solids and precomps in the comp +(`EvalCtx::par_ratio`, not inherited by children); Invert Alpha and Interpret As Linear Light +apply after decoding. + ## 5. Expressions Expressions are JavaScript, run by boa, with After Effects' object model (`thisComp`, `thisLayer`, `time`, `value`, `wiggle`, `loopOut`, vector maths on arrays, and so on). A syntax error keeps the -text, disables the expression and shows a warning, like After Effects. +text, disables the expression and shows a warning, like After Effects. `sampleImage` renders the +sampled layer through the evaluating renderer's footage source (`EvalCtx::footage`), cached per +thread and frame; `footage(name)` reads data footage (JSON, CSV, TSV imported with File ▸ Import, +text kept in the project) through `sourceData`, `sourceText` and `dataValue`. `expr.errors` +lists failing expressions for the viewer's error bar. ## 6. Commands, the UI seam and automation diff --git a/docs/parity.md b/docs/parity.md index 419b407cc..37b2d670a 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -89,14 +89,14 @@ and OCIO, landed in M9.11.) | Import | 78% | 4.5 | PSD, SVG as shapes, Lottie | | Automation | 76% | 3.3 | a JavaScript scripting object model | | Shapes | 68% | 3.9 | taper and wave strokes (pen tool for shape paths and vertex editing landed in M6.5) | -| Compositions | 67% | 7.4 | marker dialog, flowchart, Essential Graphics | +| Compositions | ≈ 75% | 5.0 | marker dialog, flowchart (Essential Graphics with master properties, `.ectemplate` templates and Responsive Design — Time landed: CMP-7) | | Animation | 65% | 10.4 | puppet, Wiggler/Smoother/Motion Sketch (motion-path handles and the graph editor transform box landed in M5.8) | | Text | ≈ 86% | 1.3 | vertical type is basic (upright characters, no tate-chu-yoko), no OpenType feature panel, no extruded strokes (extruded, bevelled text landed in M7.6); per-character styles, paragraph settings, on-canvas editing and the `sourceText` style API landed (M9.9–M9.10) | | Web | 60% | 2.5 | threads, browser storage, audio | | 3D | 72% | 12.0 | 3D camera tracker, multi-view layouts, Advanced 3D motion blur and blend modes, cameras/lights from models; Advanced 3D (glTF/OBJ models, primitives, extruded text and shapes, PBR, image-based light, shadow maps, GPU rasteriser) landed in M7.4–M7.6 | | Effects | ≈ 75% | 5.0 | GPU versions of more effects and the 3D Camera Tracker (every other After Effects effect exists since M9.11 and M12.5) | | Interface | 55% | 10.4 | drag-to-dock and floating panels, preferences, native macOS menus (viewer rulers/snapping/channels/snapshots landed in M0.13) | -| Project | 53% | 8.1 | 8/16/32-bit pipeline and colour management, auto-save, proxies, folder moves | +| Project | ≈ 60% | 6.5 | 8/16/32-bit pipeline and colour management, auto-save, folder moves (proxies and Interpret Footage fields / pixel aspect / alpha guess landed: PRJ-8, PRJ-3) | | Masks & roto | 52% | 10.0 | Roto Brush, variable-width mask feather points (mask tracking and Mask Interpolation landed in M6.6) | | Preview | 55% | 6.1 | GPU 3D and adjustment layers, disk cache (region of interest, snapshots, exposure and Fast Previews landed in M0.13) | | Tracking | ≈ 70% | 4.0 | 3D camera tracker, face tracking and Subspace Warp's mesh warp (Rolling Shutter Repair landed in M9.11; point tracker, mask tracking and Warp Stabilizer landed in M6.x / M12.5) | @@ -136,7 +136,7 @@ Mocha and Cineware are third-party and not counted. 9. A real 8/16/32-bit pipeline with linear blending and colour management. 10. Drag-to-dock and floating panels, saved workspaces, native macOS menus. 11. Auto-save, crash recovery, recent projects (in progress). -12. Expression gaps: `sampleImage`, `footage()` (the `sourceText` style API landed in M9.9). +12. ~~Expression gaps: `sampleImage`, `footage()`~~ (landed with data footage, the error bar and the Expression Language menu; the `sourceText` style API landed in M9.9). 13. Lottie (in progress), WebM, SVG and PSD import. 14. Puppet and paint tools (in progress). 15. Preferences and a shortcut editor that can rebind (in progress); real Wiggler, Smoother and Motion Sketch; the marker dialog. From 5df98d0982ac963b508631cc0b6a1f56c00a4403 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:41:28 -0400 Subject: [PATCH 8/9] =?UTF-8?q?M0.11:=20parity=20audit=20at=20fa26ad9=20(?= =?UTF-8?q?=E2=89=8893%=20weighted;=20298/298=20effects;=20=E2=89=8811?= =?UTF-8?q?=E2=80=9314=20h=20wall=20clock=20left)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 13 ++++++------- docs/parity.md | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 2bf7d8ba7..724c9d701 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,13 +25,12 @@ once. ## How far from full parity -Measured feature by feature in [docs/parity.md](docs/parity.md) (end of 2 October 2026): +Measured feature by feature in [docs/parity.md](docs/parity.md) (late 2 October 2026): -- **≈ 79% of After Effects' features, weighted by importance** (87% of the essentials, 67% of - professional daily-use features, 21% of the long tail); 298 of 298 effects, 16 of them on the GPU. -- **≈ 100 agent-hours of work remain by the audit's estimate.** With five Claude Opus 5.5 agents in - parallel that is **≈ 8–10 hours of wall-clock time at the pace measured so far, up to ≈ 25 hours** - by the audit's conservative per-feature figures. -- Hardest remaining pieces: Advanced 3D, a scripting object model (Roto Brush & Refine Edge landed in M6.7, the 3D Camera Tracker in M12.6). +- **≈ 93% of After Effects' features, weighted by importance** (96% of the essentials, 90% of + professional daily-use features, 65% of the long tail); all 298 After Effects effects. +- **≈ 50–55 agent-hours of work remain**: about **11–14 hours of wall-clock time** with five Claude + Opus 5.5 agents in parallel, ≈ 6–8 hours for the essentials and daily-use features. +- Hardest remaining pieces: the web app's depth, GPU 3D, Content-Aware Fill, a plug-in API. Come tell us what matters most to you on [Discord](https://discord.gg/artcraft). diff --git a/docs/parity.md b/docs/parity.md index 0f3103ac4..32cb65b79 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -1,9 +1,36 @@ # Parity with After Effects How close EffectCraft is to After Effects 2026, feature by feature, and how much work is left. -Audited against the code at commit `5273f1b` (2 October 2026). Every feature in our catalog was -graded from the code itself: a disabled menu entry, or a setting that is stored but never -rendered, does not count as done. + +## Current status (audit at commit `fa26ad9`, 2 October 2026, late) + +| Measure | Value | +|---|---| +| **Feature parity, weighted by tier** (P0 ×3, P1 ×2, P2 ×1) | **≈ 93%** | +| Unweighted | ≈ 91% | +| P0 / P1 / P2 | ≈ 96% / 90% / 65% | +| Features done / partial / missing | 41 / 50 / 1 of 92 (missing: a plug-in API) | +| **Effects** | **298 of 298** After Effects 2026 effects exist (some still simplified) | +| Disabled menu entries left | 26 commands (31 menu items) | +| Remaining work | ≈ 50–55 agent-hours at the pace measured so far (≈ 150 by the original conservative audit scale) | +| **Wall-clock estimate** | **≈ 11–14 hours** with five agents in parallel; ≈ 6–8 hours for 100% of P0 + P1 | + +By area: Layers 98%, Output 95%, Compositions 95%, Automation 96%, Paint 95%, Text 95%, Import 96%, +Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio 90%, Project 90%, +Tracking 88%, Effects 85%, Web 65%. + +What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; real +`sampleImage` and `footage()` in expressions; camera iris/bokeh and focus-link commands; GPU +rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; +approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur +and blend modes; a few viewer and colour-management menu items; OpenType features and variable +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; proxies; ScriptUI; +Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and +the "better than After Effects" items (a plug-in API, branching history, GPU particles). + +The sections below are the original audit (morning of 2 October, ≈ 64%) and its updates, kept for +history. ## Summary From cea56a12d835440dea712e7deeb97b16b9933d45 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 2 Oct 2026 23:44:06 -0400 Subject: [PATCH 9/9] =?UTF-8?q?M0.11:=20parity=20summary=20=E2=80=94=20sam?= =?UTF-8?q?pleImage/footage()=20and=20proxies=20landed=20(CMP-7,=20ANM-5,?= =?UTF-8?q?=20PRJ-8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- docs/parity.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/parity.md b/docs/parity.md index 9704e935e..3517c4845 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -20,12 +20,11 @@ Animation 94%, Masks 94%, Preview 94%, Interface 93%, Shapes 93%, 3D 91%, Audio Tracking 88%, Effects 85%, Web 65%. What is left, in priority order: the web app's depth (threads, storage, audio, non-blocking -renders); stroke taper/wave and multi-segment dashes; variable mask feather points; real -`sampleImage` and `footage()` in expressions; camera iris/bokeh and focus-link commands; GPU +renders); stroke taper/wave and multi-segment dashes; variable mask feather points; camera iris/bokeh and focus-link commands; GPU rendering of 3D runs and adjustment layers; Render Queue field render, crop/resize and templates; approximated effects (Subspace Warp, Key Cleaner) and Liquify's viewer brush; Advanced 3D motion blur and blend modes; a few viewer and colour-management menu items; OpenType features and variable -font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; proxies; ScriptUI; +font axes; AI/EPS/PDF import; more codecs; 59 preferences not yet wired; ScriptUI; Lumetri Scopes, Footage, Media Browser and Metadata panels; Content-Aware Fill; face tracking; and the "better than After Effects" items (a plug-in API, branching history, GPU particles).