From 5cf0e24f430176fdac37dbd2a284faa7215f1dd7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:12:43 +0200 Subject: [PATCH 01/51] Charts: one combo plot-kind rule, no fixed_scale helper, ticks off the step grid (#1146) Co-Authored-By: Claude Sonnet 5.5 --- ROADMAP.md | 3 ++- crates/chart/src/axis.rs | 11 ++++------- crates/chart/src/render.rs | 20 +++++++++++--------- crates/chart/src/resolve.rs | 10 ++++++++-- crates/chart/src/tests.rs | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 57 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 070552d07..2e9336683 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~25 points of "ready for real work" and ~160–240 h away -> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-12 · **Change:** minor (rows auto-fit oversized fonts on open and whole-row/column formatting) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-10-11 · **Last updated:** 2026-10-13 · **Change:** trivial (chart combo plot-kind rule, scale ticks) · **Target:** Microsoft Excel (Microsoft 365) GridCraft aims at full Microsoft Excel parity, and at being better in places: faster, open (XLSX is the native format), fully scriptable by agents (MCP, control channel, CLI), and on the web. @@ -132,6 +132,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-10-13 | Charts: one combo plot-kind rule for resolving and drawing; ticks of a scale off the step grid keep their own values (#1146) | | 2026-10-12 | Rows auto-fit oversized fonts: opening a file and whole-row/column formatting grow non-custom rows to their tallest font (#833) | | 2026-10-12 | Number formats: parsing a format code is linear in its length (long runs of `[`, `,` or `/` no longer take quadratic time), and an unclosed `[` is a literal when splitting sections as well as when tokenizing (#810) | | 2026-10-12 | Number formats: a fraction denominator with 65,536 zeros no longer crashes; format codes are capped at 255 characters like Excel (longer codes display as General, `TEXT` gives `#VALUE!`) (#807) | diff --git a/crates/chart/src/axis.rs b/crates/chart/src/axis.rs index 8686bf7d9..46fc43f99 100644 --- a/crates/chart/src/axis.rs +++ b/crates/chart/src/axis.rs @@ -25,9 +25,11 @@ impl Scale { (0..=n) .map(|i| { let v = self.min + i as f64 * self.step; - // Snap away accumulated float error. + // Snap away accumulated float error, but only onto a multiple of the step that is + // really that close: a `min` off the step grid must keep its own ticks. let snapped = (v / self.step).round() * self.step; - if snapped.abs() < self.step * 1e-9 { 0.0 } else { snapped } + let v = if (snapped - v).abs() < self.step * 1e-9 { snapped } else { v }; + if v.abs() < self.step * 1e-9 { 0.0 } else { v } }) .collect() } @@ -107,11 +109,6 @@ pub fn nice_scale(lo: f64, hi: f64, force_zero: bool) -> Scale { Scale { min, max, step } } -/// Exact fixed scale (100% stacked axes). -pub(crate) fn fixed_scale(min: f64, max: f64, step: f64) -> Scale { - Scale { min, max, step } -} - /// A number format prepared once and reused for many labels. pub(crate) struct Fmt { general: bool, diff --git a/crates/chart/src/render.rs b/crates/chart/src/render.rs index 037b9099e..82769bcdc 100644 --- a/crates/chart/src/render.rs +++ b/crates/chart/src/render.rs @@ -5,7 +5,7 @@ use std::f32::consts::{PI, TAU}; use gridcraft_model::{Chart, ChartKind, LegendPos, LineDash, Theme, TrendlineKind}; -use crate::axis::{Fmt, Scale, clean, fixed_scale, nice_scale}; +use crate::axis::{Fmt, Scale, clean, nice_scale}; use crate::resolve::{MAX_POINTS, MAX_SERIES, MAX_TRENDLINES, series_color}; use crate::trend::{auto_name, fit_trendline, moving_average}; use crate::{ChartData, HAlign, Measure, Prim, Rgba, SeriesData, VAlign}; @@ -120,9 +120,10 @@ pub enum Swatch { Marker, } -/// The kind a series is drawn as (per-series kinds only apply to combo charts). -fn plot_kind(chart: &Chart, s: &SeriesData) -> ChartKind { - if chart.kind == ChartKind::Combo { if s.kind == ChartKind::Combo { ChartKind::Line } else { s.kind } } else { chart.kind } +/// The kind series `i` is drawn as (per-series kinds only apply to combo charts); the same rule +/// as `series_plot_kind`, applied to the already resolved series kind. +fn plot_kind(chart: &Chart, s: &SeriesData, i: usize) -> ChartKind { + crate::resolve::plot_kind(chart.kind, Some(s.kind), i) } /// Colour of point `j` for charts that colour by point (pie, doughnut, treemap, sunburst). @@ -163,8 +164,9 @@ pub fn legend_entries(chart: &Chart, data: &ChartData) -> Vec<(String, Rgba, Swa let series = data.series.iter().take(MAX_SERIES); let mut out: Vec<(String, Rgba, Swatch)> = series .clone() - .map(|s| { - let sw = match plot_kind(chart, s) { + .enumerate() + .map(|(i, s)| { + let sw = match plot_kind(chart, s, i) { ChartKind::Line | ChartKind::LineStacked | ChartKind::Radar | ChartKind::ScatterLines => Swatch::Line, ChartKind::LineMarkers => Swatch::LineMarker, ChartKind::Scatter | ChartKind::Bubble => Swatch::Marker, @@ -174,7 +176,7 @@ pub fn legend_entries(chart: &Chart, data: &ChartData) -> Vec<(String, Rgba, Swa }) .collect(); // Trendlines follow the series, named "Linear (Sales)" unless given a name. - for s in series.filter(|s| supports_trendline(plot_kind(chart, s))) { + for (_, s) in series.enumerate().filter(|(i, s)| supports_trendline(plot_kind(chart, s, *i))) { for t in s.trendlines.iter().take(MAX_TRENDLINES) { out.push((t.spec.name.clone().unwrap_or_else(|| auto_name(t.spec.kind, &s.name)), t.color, Swatch::Line)); } @@ -833,7 +835,7 @@ fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizonta if ncat == 0 { return; } - let kinds: Vec = data.series.iter().map(|s| if horizontal { chart.kind } else { plot_kind(chart, s) }).collect(); + let kinds: Vec = data.series.iter().enumerate().map(|(i, s)| if horizontal { chart.kind } else { plot_kind(chart, s, i) }).collect(); let fams: Vec = kinds.iter().map(|k| family(*k)).collect(); let any_primary = data.series.iter().any(|s| !s.secondary); let sec: Vec = data.series.iter().map(|s| s.secondary && any_primary && !horizontal).collect(); @@ -915,7 +917,7 @@ fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizonta })); let (lo, hi) = range.unwrap_or((0.0, 0.0)); let (scale, fmt) = if all100 { - (fixed_scale(if lo < -1e-9 { -1.0 } else { 0.0 }, if hi > 1e-9 || lo >= -1e-9 { 1.0 } else { 0.0 }, 0.2), Fmt::new("0%")) + (Scale { min: if lo < -1e-9 { -1.0 } else { 0.0 }, max: if hi > 1e-9 || lo >= -1e-9 { 1.0 } else { 0.0 }, step: 0.2 }, Fmt::new("0%")) } else { let code = members.first().and_then(|i| data.series.get(*i)).map(|s| s.number_format.as_str()).unwrap_or("General"); (nice_scale(lo, hi, force_zero), Fmt::new(code)) diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index c007e3e90..2faeeb356 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -170,7 +170,7 @@ fn series_name(wb: &Workbook, sheet: usize, name: &str) -> String { t.to_string() } -fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { +pub(crate) fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { match (chart_kind, series_kind) { (ChartKind::Combo, Some(k)) if k != ChartKind::Combo => k, (ChartKind::Combo, _) => { @@ -185,9 +185,15 @@ fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usiz } } +/// The kind series `i` is drawn as: its own kind in a combo chart (first series a column, the +/// rest lines when it has none), else the chart's. The one rule shared by resolving and drawing. +pub(crate) fn plot_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { + if chart_kind == ChartKind::Combo { effective_kind(chart_kind, series_kind, i) } else { chart_kind } +} + /// The kind series `i` of `chart` is drawn as: its own kind in a combo chart, else the chart's. pub fn series_plot_kind(chart: &Chart, i: usize) -> ChartKind { - if chart.kind == ChartKind::Combo { effective_kind(chart.kind, chart.series.get(i).and_then(|s| s.kind), i) } else { chart.kind } + plot_kind(chart.kind, chart.series.get(i).and_then(|s| s.kind), i) } /// Evaluates a chart's series formulas: names, values, categories (default "1", "2", …), X diff --git a/crates/chart/src/tests.rs b/crates/chart/src/tests.rs index 01f1aac49..4dcfcde84 100644 --- a/crates/chart/src/tests.rs +++ b/crates/chart/src/tests.rs @@ -1057,3 +1057,35 @@ fn area_charts_draw_source_value_labels_only_when_enabled() { assert!(first_label > last_poly, "{kind:?}: label drawn under a later area"); } } + +#[test] +fn drawing_and_resolving_agree_on_the_combo_plot_kind() { + // Resolved series carry `effective_kind`; drawing re-derives the kind from it. Both paths + // must give the same kind for every chart kind, series kind (or none) and series index. + let series_kinds = std::iter::once(None).chain(ALL_KINDS.iter().map(|k| Some(*k))); + for ck in ALL_KINDS { + for sk in series_kinds.clone() { + for i in 0..3 { + let mut c = chart(ck); + c.series = (0..3).map(|j| Series { kind: if j == i { sk } else { None }, ..Default::default() }).collect(); + let from_chart = series_plot_kind(&c, i); + let resolved = crate::resolve::effective_kind(ck, sk, i); + assert_eq!(crate::resolve::plot_kind(ck, Some(resolved), i), from_chart, "{ck:?} {sk:?} {i}"); + if ck != ChartKind::Combo { + assert_eq!(from_chart, ck); + } else { + assert_ne!(from_chart, ChartKind::Combo); + } + } + } + } +} + +#[test] +fn ticks_of_a_scale_off_the_step_grid_start_at_its_min() { + let s = Scale { min: 3.0, max: 7.0, step: 4.0 }; + assert_eq!(s.ticks(), vec![3.0, 7.0]); + // Aligned scales still snap float error away. + let s = Scale { min: 0.0, max: 1.0, step: 0.1 }; + assert!((s.ticks()[3] - 0.3).abs() < 1e-12 && s.ticks()[0] == 0.0); +} From 79cfd9b151c7f36793d230fac10857b02cfbe33d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:13:20 +0200 Subject: [PATCH 02/51] Charts: label fitting and legend measuring avoid hundreds of text layouts per render (#1149) Binary search in Ctx::fit, early exit of the side-legend column measure, no per-point Vecs in value_range/squarify, treemap carries the value. Co-Authored-By: Claude Sonnet 5.5 --- ROADMAP.md | 1 + crates/chart/src/render.rs | 90 ++++++--- crates/chart/src/render_perf_tests.rs | 254 ++++++++++++++++++++++++++ 3 files changed, 317 insertions(+), 28 deletions(-) create mode 100644 crates/chart/src/render_perf_tests.rs diff --git a/ROADMAP.md b/ROADMAP.md index 070552d07..ae8286ad3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -132,6 +132,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-10-13 | Charts: label shortening needs about nine text measurements instead of up to 400, side legends stop measuring once the column is at its cap and treemap layout avoids per-item allocation (#1149) | | 2026-10-12 | Rows auto-fit oversized fonts: opening a file and whole-row/column formatting grow non-custom rows to their tallest font (#833) | | 2026-10-12 | Number formats: parsing a format code is linear in its length (long runs of `[`, `,` or `/` no longer take quadratic time), and an unclosed `[` is a literal when splitting sections as well as when tokenizing (#810) | | 2026-10-12 | Number formats: a fraction denominator with 65,536 zeros no longer crashes; format codes are capped at 255 characters like Excel (longer codes display as General, `TEXT` gives `#VALUE!`) (#807) | diff --git a/crates/chart/src/render.rs b/crates/chart/src/render.rs index 037b9099e..396423d2b 100644 --- a/crates/chart/src/render.rs +++ b/crates/chart/src/render.rs @@ -62,25 +62,47 @@ struct Ctx<'a> { m: &'a dyn Measure, } +#[cfg(test)] +#[path = "render_perf_tests.rs"] +mod perf_tests; + impl Ctx<'_> { fn tw(&self, s: &str, size: f32) -> f32 { let w = self.m.text_width(s, size, false); if w.is_finite() { w.max(0.0) } else { 0.0 } } + /// `s` shortened with an ellipsis to fit `maxw`; empty when not even one character fits. + /// + /// Binary search over the number of kept characters (at most 400 are tried), about nine + /// `Measure` calls instead of one per character. It assumes the width of `prefix + "…"` does not + /// shrink as the prefix grows, which holds for per-character advance sums (`ApproxMeasure`, the + /// PDF metrics) and for the UI's text layout up to kerning of a few hundredths of a point; with + /// such a non-monotone measure the result may differ from the old linear scan by a character. fn fit(&self, s: &str, size: f32, maxw: f32) -> String { if self.tw(s, size) <= maxw { return s.to_string(); } - let chars: Vec = s.chars().collect(); - let mut n = chars.len().min(400); - while n > 0 { - n -= 1; - let t: String = chars.iter().take(n).collect::() + "…"; - if self.tw(&t, size) <= maxw { - return if n == 0 { String::new() } else { t }; + // Byte offset where each of the first 400 characters starts; a prefix of `k` characters + // (k < number of characters, so the offset exists) ends at `starts[k]`. + let starts: Vec = s.char_indices().map(|(i, _)| i).take(400).collect(); + let top = starts.len().saturating_sub(1); + let fits = |k: usize| -> bool { + let end = starts.get(k).copied().unwrap_or(s.len()); + let mut t = String::with_capacity(end + 3); + t.push_str(s.get(..end).unwrap_or("")); + t.push('…'); + self.tw(&t, size) <= maxw + }; + let (mut lo, mut hi) = (0usize, top); + while lo < hi { + let mid = lo + (hi - lo).div_ceil(2); + if fits(mid) { + lo = mid; + } else { + hi = mid - 1; } } - String::new() + if lo == 0 { String::new() } else { format!("{}…", s.get(..starts.get(lo).copied().unwrap_or(s.len())).unwrap_or("")) } } #[allow(clippy::too_many_arguments)] fn text(&mut self, x: f32, y: f32, s: &str, size: f32, color: Rgba, align: HAlign, valign: VAlign) { @@ -369,7 +391,16 @@ fn draw_legend(ctx: &mut Ctx, entries: &[(String, Rgba, Swatch)], pos: LegendPos if pos == LegendPos::Bottom { Bx { h: area.h - used, ..area } } else { Bx { y: area.y + used, h: area.h - used, ..area } } } LegendPos::Left | LegendPos::Right => { - let colw = entries.iter().map(|e| item_w(ctx, e)).fold(0.0f32, f32::max).min(area.w * 0.35); + // Widest entry, capped at 35% of the area: once the cap is reached the rest cannot change it. + let cap = area.w * 0.35; + let mut colw = 0.0f32; + for e in entries { + if colw >= cap { + break; + } + colw = colw.max(item_w(ctx, e)); + } + let colw = colw.min(cap); let max_rows = ((area.h / row_h).floor() as usize).max(1); let n = entries.len().min(max_rows); let lh = n as f32 * row_h; @@ -911,7 +942,8 @@ fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizonta let all100 = members.iter().all(|i| fams.get(*i) == Some(&Fam::Col100)); let range = value_range(members.iter().flat_map(|i| { let line = fams.get(*i).is_some_and(|f| is_line(*f)); - spans.get(*i).into_iter().flatten().flatten().flat_map(move |(b, t)| if line { vec![*t] } else { vec![*b, *t] }) + // Lines contribute only their top value, bars both ends (no per-point `Vec`). + spans.get(*i).into_iter().flatten().flatten().flat_map(move |(b, t)| [*b, *t].into_iter().skip(usize::from(line))) })); let (lo, hi) = range.unwrap_or((0.0, 0.0)); let (scale, fmt) = if all100 { @@ -1399,19 +1431,19 @@ fn pie(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, inner: f32, nam // ---------------------------------------------------------------- treemap -fn worst(row: &[f64], side: f64) -> f64 { - let s: f64 = row.iter().sum(); +/// Worst aspect ratio of a row with the given area sum, smallest and largest item, laid along `side`. +fn worst(s: f64, mn: f64, mx: f64, side: f64) -> f64 { if s <= 0.0 || side <= 0.0 { return f64::INFINITY; } - let (mn, mx) = row.iter().fold((f64::INFINITY, 0.0f64), |(a, b), v| (a.min(*v), b.max(*v))); let s2 = s * s; let w2 = side * side; (w2 * mx / s2).max(s2 / (w2 * mn)) } -/// Squarified treemap layout of (index, area) items, sorted descending, into `r`. -fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { +/// Squarified treemap layout of (index, value) items, sorted descending, into `r`; each block +/// comes back with its index and value. +fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, f64, Bx)> { let total: f64 = items.iter().map(|i| i.1).sum(); let area = (r.w as f64) * (r.h as f64); if !(total > 0.0 && area > 0.0) { @@ -1420,9 +1452,10 @@ fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { let scale = area / total; let mut out = Vec::with_capacity(items.len()); let mut rem = r; - let mut row: Vec<(usize, f64)> = Vec::new(); + let mut row: Vec<(usize, f64, f64)> = Vec::new(); + let (mut sum, mut mn, mut mx) = (0.0f64, f64::INFINITY, 0.0f64); let mut i = 0; - let layout = |row: &[(usize, f64)], rem: &mut Bx, out: &mut Vec<(usize, Bx)>| { + let layout = |row: &[(usize, f64, f64)], rem: &mut Bx, out: &mut Vec<(usize, f64, Bx)>| { let s: f64 = row.iter().map(|x| x.1).sum(); if s <= 0.0 { return; @@ -1430,9 +1463,9 @@ fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { if rem.w >= rem.h { let cw = (s / rem.h.max(1e-6) as f64) as f32; let mut y = rem.y; - for (j, a) in row { + for (j, a, v) in row { let h = (*a / cw.max(1e-6) as f64) as f32; - out.push((*j, Bx { x: rem.x, y, w: cw, h })); + out.push((*j, *v, Bx { x: rem.x, y, w: cw, h })); y += h; } rem.x += cw; @@ -1440,9 +1473,9 @@ fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { } else { let rh = (s / rem.w.max(1e-6) as f64) as f32; let mut x = rem.x; - for (j, a) in row { + for (j, a, v) in row { let w = (*a / rh.max(1e-6) as f64) as f32; - out.push((*j, Bx { x, y: rem.y, w, h: rh })); + out.push((*j, *v, Bx { x, y: rem.y, w, h: rh })); x += w; } rem.y += rh; @@ -1452,15 +1485,17 @@ fn squarify(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { while let Some(&(j, v)) = items.get(i) { let a = v * scale; let side = rem.w.min(rem.h) as f64; - let cur: Vec = row.iter().map(|x| x.1).collect(); - let mut with = cur.clone(); - with.push(a); - if row.is_empty() || worst(&with, side) <= worst(&cur, side) { - row.push((j, a)); + // Sum, smallest and largest item of the open row, kept incrementally (same fold order as + // summing the row afresh, so the numbers are bit-identical). + let with = (sum + a, mn.min(a), mx.max(a)); + if row.is_empty() || worst(with.0, with.1, with.2, side) <= worst(sum, mn, mx, side) { + row.push((j, a, v)); + (sum, mn, mx) = with; i += 1; } else { layout(&row, &mut rem, &mut out); row.clear(); + (sum, mn, mx) = (0.0, f64::INFINITY, 0.0); } } if !row.is_empty() { @@ -1475,14 +1510,13 @@ fn treemap(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx) { s.values.iter().take(MAX_MARKERS).enumerate().filter_map(|(j, v)| v.filter(|v| *v > 0.0).map(|v| (j, v))).collect(); items.sort_by(|a, b| b.1.total_cmp(&a.1)); let f = Fmt::new(&s.number_format); - for (j, b) in squarify(&items, area) { + for (j, v, b) in squarify(&items, area) { let fill = point_color(data, j); ctx.rect(b.x, b.y, b.w, b.h, fill, Some((WHITE, 1.5))); if b.w > 24.0 && b.h > LABEL_SIZE * 1.6 { let name = ctx.fit(&cat_label(data, j), LABEL_SIZE, b.w - 8.0); ctx.text(b.x + 4.0, b.y + 4.0, &name, LABEL_SIZE, on(fill), HAlign::Left, VAlign::Top); if chart.data_labels && b.h > LABEL_SIZE * 3.2 { - let v = items.iter().find(|x| x.0 == j).map(|x| x.1).unwrap_or(0.0); let t = ctx.fit(&f.format(v, None), LABEL_SIZE, b.w - 8.0); ctx.text(b.x + 4.0, b.y + 4.0 + LABEL_SIZE * 1.4, &t, LABEL_SIZE, on(fill), HAlign::Left, VAlign::Top); } diff --git a/crates/chart/src/render_perf_tests.rs b/crates/chart/src/render_perf_tests.rs new file mode 100644 index 000000000..7bef70722 --- /dev/null +++ b/crates/chart/src/render_perf_tests.rs @@ -0,0 +1,254 @@ +//! Equality and counter tests for the label-fitting, legend-measuring and treemap rewrites. +//! The old implementations live here as references. + +use std::cell::Cell; + +use super::*; +use crate::{ApproxMeasure, Measure}; + +/// The pre-rewrite `Ctx::fit`: one `Measure` call per candidate length, longest first. +fn fit_reference(ctx: &Ctx, s: &str, size: f32, maxw: f32) -> String { + if ctx.tw(s, size) <= maxw { + return s.to_string(); + } + let chars: Vec = s.chars().collect(); + let mut n = chars.len().min(400); + while n > 0 { + n -= 1; + let t: String = chars.iter().take(n).collect::() + "…"; + if ctx.tw(&t, size) <= maxw { + return if n == 0 { String::new() } else { t }; + } + } + String::new() +} + +/// Counts calls; advances differ per character (wide CJK/emoji, zero-width combining marks) and +/// "AV"-style pairs are kerned tighter, but a character never makes the text narrower. +struct Varied { + calls: Cell, +} + +impl Measure for Varied { + fn text_width(&self, text: &str, size: f32, bold: bool) -> f32 { + self.calls.set(self.calls.get() + 1); + let mut w = 0.0f32; + let mut prev = ' '; + for c in text.chars() { + let adv = match c as u32 { + 0x300..=0x36f => 0.0, + 0x1f300..=0x1faff | 0x4e00..=0x9fff => 1.0, + u => 0.3 + (u % 7) as f32 * 0.07, + }; + let kern = if prev == 'A' && c == 'V' { 0.1 } else { 0.0 }; + w += (adv - kern) * size; + prev = c; + } + if bold { w * 1.1 } else { w } + } +} + +struct Counting<'a>(&'a dyn Measure, Cell); + +impl Measure for Counting<'_> { + fn text_width(&self, text: &str, size: f32, bold: bool) -> f32 { + self.1.set(self.1.get() + 1); + self.0.text_width(text, size, bold) + } +} + +fn corpus() -> Vec { + let mut v: Vec = + ["", "a", "ab", "Hello, world", "AVAVAV tight", "日本語のラベルです", "👨‍👩‍👧 family 🎉 party", "e\u{301}e\u{301}e\u{301} combining", "…", "……"] + .iter() + .map(|s| s.to_string()) + .collect(); + v.push("x".repeat(399)); + v.push("x".repeat(400)); + v.push("x".repeat(401)); + v.push("é🎉".repeat(500)); + v.push("long label ".repeat(100)); + v +} + +fn run_fit(m: &dyn Measure) { + let ctx = Ctx { out: vec![], m }; + for s in corpus() { + for size in [9.0f32, 11.0, 40.0] { + for maxw in [-5.0, 0.0, 0.4, 6.0, 25.0, 61.5, 120.0, 800.0, 1e6, f32::INFINITY, f32::NAN] { + assert_eq!(ctx.fit(&s, size, maxw), fit_reference(&ctx, &s, size, maxw), "{s:?} size {size} maxw {maxw}"); + } + } + } +} + +#[test] +fn fit_matches_linear_reference() { + run_fit(&ApproxMeasure); + run_fit(&Varied { calls: Cell::new(0) }); +} + +#[test] +fn fit_measures_logarithmically() { + let long = "x".repeat(1000); + let c = Counting(&ApproxMeasure, Cell::new(0)); + let ctx = Ctx { out: vec![], m: &c }; + let r = ctx.fit(&long, 11.0, 60.0); + assert!(r.ends_with('…') && r.chars().count() > 3); + // 1 whole-string check + ceil(log2(399)) + 1 probes; the old scan needed ~390 here. + assert!(c.1.get() <= 12, "{} Measure calls", c.1.get()); + let c2 = Counting(&ApproxMeasure, Cell::new(0)); + let ctx2 = Ctx { out: vec![], m: &c2 }; + assert_eq!(ctx2.fit("short", 11.0, 500.0), "short"); + assert_eq!(c2.1.get(), 1); +} + +fn legend_entries(n: usize, label: &dyn Fn(usize) -> String) -> Vec<(String, Rgba, Swatch)> { + (0..n).map(|i| (label(i), [10, 20, 30, 255], if i % 3 == 0 { Swatch::Line } else { Swatch::Box })).collect() +} + +/// Colour column width as the old code computed it, from the returned remaining area. +fn old_colw(m: &dyn Measure, entries: &[(String, Rgba, Swatch)], area: Bx) -> f32 { + let ctx = Ctx { out: vec![], m }; + let size = LABEL_SIZE; + let s = size * 0.8; + entries.iter().map(|e| swatch_w(e.2, s) + 4.0 + ctx.tw(&e.0, size)).fold(0.0f32, f32::max).min(area.w * 0.35) +} + +#[test] +fn side_legend_matches_old_column_width_and_stops_measuring_at_the_cap() { + let area = Bx { x: 0.0, y: 0.0, w: 400.0, h: 300.0 }; + let labels: [&dyn Fn(usize) -> String; 3] = + [&|i| format!("entry {i}"), &|i| if i == 7 { "w".repeat(80) } else { format!("e{i}") }, &|i| "very long legend label ".repeat(1 + i % 3)]; + for pos in [LegendPos::Left, LegendPos::Right] { + for (k, label) in labels.iter().enumerate() { + for n in [0usize, 1, 5, 40] { + let entries = legend_entries(n, label); + let mut ctx = Ctx { out: vec![], m: &ApproxMeasure }; + let rest = draw_legend(&mut ctx, &entries, pos, area); + let colw = old_colw(&ApproxMeasure, &entries, area); + assert!((rest.w - (area.w - (colw + 8.0))).abs() < 1e-4, "{pos:?} labels {k} n {n}"); + } + } + } + // 20,000 entries whose first one already fills the 35% cap: the old code measured all of them + // plus the drawn rows, now only the first entry and the drawn rows are measured. + let entries = legend_entries(20_000, &|i| if i == 0 { "w".repeat(200) } else { format!("e{i}") }); + let c = Counting(&ApproxMeasure, Cell::new(0)); + let mut ctx = Ctx { out: vec![], m: &c }; + let _ = draw_legend(&mut ctx, &entries, LegendPos::Right, area); + let drawn = ctx.out.iter().filter(|p| matches!(p, Prim::Text { .. })).count(); + assert!(c.1.get() <= drawn * 10 + 5, "{} Measure calls for {drawn} drawn entries", c.1.get()); + assert!(c.1.get() < 500); +} + +/// The pre-rewrite `worst`/`squarify` (a `Vec` per item, row recomputed from scratch). +fn worst_reference(row: &[f64], side: f64) -> f64 { + let s: f64 = row.iter().sum(); + if s <= 0.0 || side <= 0.0 { + return f64::INFINITY; + } + let (mn, mx) = row.iter().fold((f64::INFINITY, 0.0f64), |(a, b), v| (a.min(*v), b.max(*v))); + let s2 = s * s; + let w2 = side * side; + (w2 * mx / s2).max(s2 / (w2 * mn)) +} + +fn squarify_reference(items: &[(usize, f64)], r: Bx) -> Vec<(usize, Bx)> { + let total: f64 = items.iter().map(|i| i.1).sum(); + let area = (r.w as f64) * (r.h as f64); + if !(total > 0.0 && area > 0.0) { + return vec![]; + } + let scale = area / total; + let mut out = Vec::with_capacity(items.len()); + let mut rem = r; + let mut row: Vec<(usize, f64)> = Vec::new(); + let mut i = 0; + let layout = |row: &[(usize, f64)], rem: &mut Bx, out: &mut Vec<(usize, Bx)>| { + let s: f64 = row.iter().map(|x| x.1).sum(); + if s <= 0.0 { + return; + } + if rem.w >= rem.h { + let cw = (s / rem.h.max(1e-6) as f64) as f32; + let mut y = rem.y; + for (j, a) in row { + let h = (*a / cw.max(1e-6) as f64) as f32; + out.push((*j, Bx { x: rem.x, y, w: cw, h })); + y += h; + } + rem.x += cw; + rem.w = (rem.w - cw).max(0.0); + } else { + let rh = (s / rem.w.max(1e-6) as f64) as f32; + let mut x = rem.x; + for (j, a) in row { + let w = (*a / rh.max(1e-6) as f64) as f32; + out.push((*j, Bx { x, y: rem.y, w, h: rh })); + x += w; + } + rem.y += rh; + rem.h = (rem.h - rh).max(0.0); + } + }; + while let Some(&(j, v)) = items.get(i) { + let a = v * scale; + let side = rem.w.min(rem.h) as f64; + let cur: Vec = row.iter().map(|x| x.1).collect(); + let mut with = cur.clone(); + with.push(a); + if row.is_empty() || worst_reference(&with, side) <= worst_reference(&cur, side) { + row.push((j, a)); + i += 1; + } else { + layout(&row, &mut rem, &mut out); + row.clear(); + } + } + if !row.is_empty() { + layout(&row, &mut rem, &mut out); + } + out +} + +#[test] +fn squarify_matches_reference() { + let mut seed = 0x2545_f491_4f6c_dd1du64; + let mut rnd = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 11) as f64 / (1u64 << 53) as f64 + }; + let mut cases: Vec> = vec![ + vec![], + vec![5.0], + vec![6.0, 6.0, 4.0, 3.0, 2.0, 2.0, 1.0], + vec![3.0, 0.0, 0.0, 1.0], + vec![0.0, 0.0], + vec![4.0, -1.0, 2.0], + vec![4.0, f64::NAN, 2.0, 1.0], + vec![f64::INFINITY, 1.0], + vec![1e-300, 1e-310, 0.0], + ]; + for n in [2usize, 9, 40, 300] { + let mut v: Vec = (0..n).map(|_| rnd() * 100.0).collect(); + v.sort_by(|a, b| b.total_cmp(a)); + cases.push(v); + } + for r in [Bx { x: 0.0, y: 0.0, w: 300.0, h: 200.0 }, Bx { x: 5.0, y: 7.0, w: 90.0, h: 400.0 }, Bx { x: 0.0, y: 0.0, w: 0.0, h: 50.0 }] { + for vals in &cases { + let items: Vec<(usize, f64)> = vals.iter().copied().enumerate().collect(); + let new: Vec<(usize, Bx)> = squarify(&items, r) + .into_iter() + .map(|(j, v, b)| { + assert!(vals.get(j).is_some_and(|x| x.to_bits() == v.to_bits()), "value carried through"); + (j, b) + }) + .collect(); + let old = squarify_reference(&items, r); + assert_eq!(format!("{new:?}"), format!("{old:?}"), "{vals:?} in {r:?}"); + } + } +} From 4a1e6de3034f0e953c679c1b2fa906830ee8b5fe Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:21:24 +0200 Subject: [PATCH 03/51] Charts: work and memory are bounded by series x points; ranges are clipped before evaluation (#1147) A chart keeps at most 2,000,000 series x points (every series and the categories keep their first points), 16 trendlines per chart; columns and bars narrower than 0.25 px are merged per pixel column; series references are cut after the rows that can matter before calc evaluates them; the moving average is linear; fit_trendline allocates no Vec per point; render borrows clean data instead of cloning it. Co-Authored-By: Claude Sonnet 5.5 --- ROADMAP.md | 1 + crates/chart/src/render.rs | 105 +++++++++++++++- crates/chart/src/resolve.rs | 138 ++++++++++++++++++--- crates/chart/src/tests.rs | 2 + crates/chart/src/tests/budget.rs | 172 ++++++++++++++++++++++++++ crates/chart/src/trend.rs | 205 ++++++++++++++++++++++++++++++- 6 files changed, 602 insertions(+), 21 deletions(-) create mode 100644 crates/chart/src/tests/budget.rs diff --git a/ROADMAP.md b/ROADMAP.md index 070552d07..06b50e9cf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -132,6 +132,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-10-13 | Charts: work and memory are bounded by series x points (2,000,000 per chart: every series keeps its first points; 16 trendlines per chart), hairline columns and bars are merged per pixel column, series ranges are clipped before they are evaluated, the moving average is linear and rendering clean data no longer copies it (#1147) | | 2026-10-12 | Rows auto-fit oversized fonts: opening a file and whole-row/column formatting grow non-custom rows to their tallest font (#833) | | 2026-10-12 | Number formats: parsing a format code is linear in its length (long runs of `[`, `,` or `/` no longer take quadratic time), and an unclosed `[` is a literal when splitting sections as well as when tokenizing (#810) | | 2026-10-12 | Number formats: a fraction denominator with 65,536 zeros no longer crashes; format codes are capped at 255 characters like Excel (longer codes display as General, `TEXT` gives `#VALUE!`) (#807) | diff --git a/crates/chart/src/render.rs b/crates/chart/src/render.rs index 037b9099e..8df7c2800 100644 --- a/crates/chart/src/render.rs +++ b/crates/chart/src/render.rs @@ -1,12 +1,13 @@ //! Chart layout: title, legend, axes and every chart kind, drawn into [`Prim`]s. +use std::borrow::Cow; use std::collections::HashSet; use std::f32::consts::{PI, TAU}; use gridcraft_model::{Chart, ChartKind, LegendPos, LineDash, Theme, TrendlineKind}; use crate::axis::{Fmt, Scale, clean, fixed_scale, nice_scale}; -use crate::resolve::{MAX_POINTS, MAX_SERIES, MAX_TRENDLINES, series_color}; +use crate::resolve::{MAX_POINTS, MAX_SERIES, MAX_TRENDLINES, points_per_series, series_color}; use crate::trend::{auto_name, fit_trendline, moving_average}; use crate::{ChartData, HAlign, Measure, Prim, Rgba, SeriesData, VAlign}; @@ -30,6 +31,8 @@ const MAX_LINE_PTS: usize = 2000; const MAX_MARKERS: usize = 5000; /// Data labels are skipped when a series has more points than this. const MAX_LABELS: usize = 200; +/// Columns and bars narrower than this (in pixels) are merged per pixel column into one rectangle. +const MIN_BAR_PX: f32 = 0.25; #[derive(Clone, Copy, Debug)] struct Bx { @@ -188,7 +191,7 @@ fn sanitize_vals(v: &[Option]) -> Vec> { v.iter().take(MAX_POINTS).map(|x| x.filter(|n| n.is_finite()).map(clean)).collect() } -fn sanitize(d: &ChartData) -> ChartData { +pub(crate) fn sanitize(d: &ChartData) -> ChartData { ChartData { palette: d.palette.iter().take(MAX_POINTS).copied().collect(), categories: d.categories.iter().take(MAX_POINTS).cloned().collect(), @@ -212,6 +215,54 @@ fn sanitize(d: &ChartData) -> ChartData { } } +/// Whether `v` survives `sanitize_vals` unchanged. +fn is_clean(v: &Option) -> bool { + v.is_none_or(|n| n.is_finite() && clean(n) == n) +} + +/// Whether [`sanitize`] and [`apply_budget`] would change `d`. +fn needs_sanitizing(d: &ChartData) -> bool { + let cap = points_per_series(d.series.len()); + let long = |n: usize| n > cap; + let mut trendlines = 0usize; + d.palette.len() > MAX_POINTS + || long(d.categories.len()) + || d.series.len() > MAX_SERIES + || d.series.iter().any(|s| { + trendlines += s.trendlines.len(); + long(s.values.len()) + || !s.values.iter().all(is_clean) + || s.x.as_ref().is_some_and(|x| long(x.len()) || !x.iter().all(is_clean)) + || s.sizes.as_ref().is_some_and(|x| long(x.len()) || !x.iter().all(is_clean)) + || s.trendlines.len() > MAX_TRENDLINES + }) + || trendlines > MAX_TRENDLINES +} + +/// Enforces the chart's budget of points (`MAX_TOTAL_POINTS` = series x points: every series and +/// the category list keep their first `MAX_TOTAL_POINTS / series` points) and of trendlines +/// (`MAX_TRENDLINES` per chart, in series order). +fn apply_budget(mut d: ChartData) -> ChartData { + let cap = points_per_series(d.series.len()); + d.categories.truncate(cap); + let mut trendlines = MAX_TRENDLINES; + for s in &mut d.series { + s.values.truncate(cap); + for v in [&mut s.x, &mut s.sizes].into_iter().flatten() { + v.truncate(cap); + } + s.trendlines.truncate(trendlines); + trendlines = trendlines.saturating_sub(s.trendlines.len()); + } + d +} + +/// The data `render` draws: `d` itself when it is already clean and within budget (the usual +/// case: no copy), else a cleaned, truncated copy. +pub(crate) fn sanitized(d: &ChartData) -> Cow<'_, ChartData> { + if needs_sanitizing(d) { Cow::Owned(apply_budget(sanitize(d))) } else { Cow::Borrowed(d) } +} + fn chart_title(chart: &Chart, data: &ChartData) -> Option { match &chart.title { Some(t) => Some(t.trim().to_string()).filter(|t| !t.is_empty()), @@ -225,7 +276,7 @@ pub fn render(chart: &Chart, data: &ChartData, w: f32, h: f32, m: &dyn Measure) if !(w.is_finite() && h.is_finite()) || w < 1.0 || h < 1.0 || w > 1e7 || h > 1e7 { return vec![]; } - let data = sanitize(data); + let data = sanitized(data); let mut ctx = Ctx { out: Vec::new(), m }; ctx.out.push(Prim::Rect { x: 0.0, y: 0.0, w, h, fill: Some(WHITE), stroke: Some((GRID, 1.0)), radius: 0.0 }); let pad = (w.min(h) * 0.04).clamp(2.0, 10.0); @@ -828,6 +879,36 @@ fn is_line(f: Fam) -> bool { matches!(f, Fam::Line | Fam::LineStack) } +/// Bars that fall into one pixel column, merged: extent along the category axis (`a0..=a1`, the +/// bars' start positions) and along the value axis (`lo..hi`). +struct BarRun { + bucket: f32, + a0: f32, + a1: f32, + lo: f32, + hi: f32, +} + +impl BarRun { + fn new(pos: f32, lo: f32, hi: f32) -> Self { + BarRun { bucket: pos.floor(), a0: pos, a1: pos, lo, hi } + } + fn add(&mut self, pos: f32, lo: f32, hi: f32) { + self.a0 = self.a0.min(pos); + self.a1 = self.a1.max(pos); + self.lo = self.lo.min(lo); + self.hi = self.hi.max(hi); + } + fn draw(&self, ctx: &mut Ctx, bw: f32, horizontal: bool, color: Rgba) { + let along = self.a1 - self.a0 + bw; + if horizontal { + ctx.rect(self.lo, self.a0, self.hi - self.lo, along, color, None); + } else { + ctx.rect(self.a0, self.lo, along, self.hi - self.lo, color, None); + } + } +} + fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizontal: bool) { let ncat = data.categories.len().max(data.series.iter().map(|s| s.values.len()).max().unwrap_or(0)).min(MAX_CATS); if ncat == 0 { @@ -1026,12 +1107,27 @@ fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizonta } 1 => { let k = slot_of.get(i).copied().unwrap_or(0) as f32; + // Hairline bars are merged per pixel column (their envelope is what shows). + let cull = bw < MIN_BAR_PX && !labels_on; + let mut run: Option = None; for (j, sp) in row.iter().enumerate() { let Some((b, t)) = sp else { continue }; let (pb, pt) = (vpos(*b, &sc), vpos(*t, &sc)); let c = center(j); let lo = pb.min(pt); let len = (pb - pt).abs(); + if cull { + let pos = if horizontal { c + nslots * bw / 2.0 - (k + 1.0) * bw } else { c - nslots * bw / 2.0 + k * bw }; + match &mut run { + Some(m) if m.bucket == pos.floor() => m.add(pos, lo, lo + len), + _ => { + if let Some(m) = run.replace(BarRun::new(pos, lo, lo + len)) { + m.draw(ctx, bw, horizontal, s.color); + } + } + } + continue; + } if horizontal { let y = c + nslots * bw / 2.0 - (k + 1.0) * bw; ctx.rect(lo, y, len, bw, s.color, None); @@ -1064,6 +1160,9 @@ fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx, horizonta } } } + if let Some(m) = run { + m.draw(ctx, bw, horizontal, s.color); + } } _ => { let pts: Vec> = row.iter().enumerate().map(|(j, sp)| sp.map(|(_, t)| [center(j), vpos(t, &sc)])).collect(); diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index c007e3e90..c0dbaa173 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -11,8 +11,17 @@ use crate::{ChartData, Rgba, SeriesData, TrendlineData}; pub(crate) const MAX_POINTS: usize = 100_000; /// Cap on series per chart. pub(crate) const MAX_SERIES: usize = 255; -/// Cap on trendlines drawn per series. +/// Cap on trendlines per chart (not per series: each one is a fit over a series' points). pub(crate) const MAX_TRENDLINES: usize = 16; +/// Budget for series x points in one chart. When a chart would hold more, every series (and the +/// category list) keeps its first `MAX_TOTAL_POINTS / series` points; no series is dropped. +pub(crate) const MAX_TOTAL_POINTS: usize = 2_000_000; + +/// Points kept per series (and categories) in a chart of `series` series: `MAX_POINTS`, or less +/// when `series x MAX_POINTS` would pass `MAX_TOTAL_POINTS`. +pub(crate) fn points_per_series(series: usize) -> usize { + (MAX_TOTAL_POINTS / series.clamp(1, MAX_SERIES)).clamp(1, MAX_POINTS) +} /// Tints used for series beyond the six theme accents: lighter, darker, lighter still… const CYCLE_TINTS: [f64; 6] = [0.0, 0.4, -0.25, 0.6, -0.5, 0.2]; @@ -77,10 +86,34 @@ fn union_parts(f: &str) -> Option> { /// Evaluates a series formula to a flat list of values (row-major), capped. A union of /// references (`(A1:A3,A5)`, as PivotCharts use) is read part by part. fn eval_list(wb: &Workbook, sheet: usize, formula: &str) -> Vec { - eval_list_in(wb, sheet, formula, 0) + eval_list_in(wb, sheet, formula, MAX_POINTS, 0) } -fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, depth: u32) -> Vec { +/// calc reads a reference of up to this many cells at its full size and trims a larger one to the +/// sheet's used range (`FULL_SIZE_CELLS` in `gridcraft_calc::eval`). A clipped reference must stay +/// above it, or the clip would turn a trimmed read into a full-size one with trailing blanks. +const CALC_FULL_SIZE_CELLS: u64 = 16_384; + +/// `formula` as a plain reference (`Sheet!A1:A1000000`) cut after the rows that can still hold +/// one of the first `limit` points, so that calc never reads the rest; `None` when `formula` is +/// no reference or already small enough. Values are read row by row, and calc trims only the +/// bottom and right of a reference, so keeping `limit` rows keeps the first `limit` values. +fn clipped_reference(wb: &Workbook, sheet: usize, formula: &str, limit: usize) -> Option { + let (_, r) = parse_ref(wb, sheet, formula)?; + let (h, w) = (u64::from(r.height()), u64::from(r.width())); + if h.saturating_mul(w) <= CALC_FULL_SIZE_CELLS { + return None; + } + let keep = (limit as u64).max(CALC_FULL_SIZE_CELLS / w + 1); + if h <= keep { + return None; + } + let end = CellRef::new(r.start.row.saturating_add(u32::try_from(keep).ok()?).saturating_sub(1), r.end.col); + let prefix = formula.rfind('!').and_then(|i| formula.get(..=i)).unwrap_or(""); + Some(format!("{prefix}{}", RangeRef::new(r.start, end))) +} + +fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, limit: usize, depth: u32) -> Vec { let f = formula.trim(); if f.is_empty() { return vec![]; @@ -90,16 +123,17 @@ fn eval_list_in(wb: &Workbook, sheet: usize, formula: &str, depth: u32) -> Vec= MAX_POINTS { + if out.len() >= limit { break; } - let rest = MAX_POINTS - out.len(); - out.extend(eval_list_in(wb, sheet, p, depth + 1).into_iter().take(rest)); + let rest = limit - out.len(); + out.extend(eval_list_in(wb, sheet, p, rest, depth + 1).into_iter().take(rest)); } return out; } - match evaluate(wb, sheet, CellRef::default(), f) { - Value::Array(a) => a.iter().take(MAX_POINTS).cloned().collect(), + let clipped = clipped_reference(wb, sheet, f, limit); + match evaluate(wb, sheet, CellRef::default(), clipped.as_deref().unwrap_or(f)) { + Value::Array(a) => a.iter().take(limit).cloned().collect(), v => vec![v], } } @@ -111,8 +145,8 @@ fn to_num(v: &Value) -> Option { } } -fn nums(wb: &Workbook, sheet: usize, formula: &str) -> Vec> { - eval_list(wb, sheet, formula).iter().map(to_num).collect() +fn nums(wb: &Workbook, sheet: usize, formula: &str, limit: usize) -> Vec> { + eval_list_in(wb, sheet, formula, limit, 0).iter().map(to_num).collect() } /// Number format code of the first cell of a reference ("General" when not a reference). @@ -197,35 +231,41 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData { let numeric_x = matches!(chart.kind, ChartKind::Scatter | ChartKind::ScatterLines | ChartKind::Bubble); let mut series = Vec::new(); let mut categories: Option> = None; + let limit = points_per_series(chart.series.len()); + let mut trend_left = MAX_TRENDLINES; for (i, s) in chart.series.iter().take(MAX_SERIES).enumerate() { - let values = nums(wb, sheet, &s.values); + let values = nums(wb, sheet, &s.values, limit); let name = s.name.as_deref().map(|n| series_name(wb, sheet, n)).filter(|n| !n.is_empty()).unwrap_or_else(|| format!("Series{}", i + 1)); let color = s.color.as_ref().and_then(|c| c.resolve(&wb.theme)).map(|[r, g, b]| [r, g, b, 0xFF]).unwrap_or_else(|| series_color(&wb.theme, i)); let kind = effective_kind(chart.kind, s.kind, i); let x = match (&s.categories, numeric_x || matches!(kind, ChartKind::Scatter | ChartKind::ScatterLines | ChartKind::Bubble)) { // X values with text in them (labels) are ignored: the points are numbered 1..n. - (Some(f), true) => Some(eval_list(wb, sheet, f)).filter(|xs| !xs.iter().any(Value::is_text)).map(|xs| xs.iter().map(to_num).collect()), + (Some(f), true) => { + Some(eval_list_in(wb, sheet, f, limit, 0)).filter(|xs| !xs.iter().any(Value::is_text)).map(|xs| xs.iter().map(to_num).collect()) + } _ => None, }; if categories.is_none() && let Some(f) = &s.categories { - let c = category_texts(wb, sheet, f); + let mut c = category_texts(wb, sheet, f); + c.truncate(limit); if !c.is_empty() { categories = Some(c); } } - let sizes = s.bubble_sizes.as_deref().map(|f| nums(wb, sheet, f)); + let sizes = s.bubble_sizes.as_deref().map(|f| nums(wb, sheet, f, limit)); let trendlines = s .trendlines .iter() - .take(MAX_TRENDLINES) + .take(trend_left) .map(|t| TrendlineData { spec: t.clone(), color: t.color.as_ref().and_then(|c| c.resolve(&wb.theme)).map(|[r, g, b]| [r, g, b, 0xFF]).unwrap_or(color), }) - .collect(); + .collect::>(); + trend_left = trend_left.saturating_sub(trendlines.len()); series.push(SeriesData { name, values, @@ -244,3 +284,69 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData { let palette = default_palette(&wb.theme, n.min(PALETTE_CYCLE)); ChartData { categories, series, palette } } + +#[cfg(test)] +mod tests { + use gridcraft_model::Workbook; + + use super::*; + + fn book(rows: u32, cols: u32) -> Workbook { + let mut wb = Workbook::new(); + if let Some(s) = wb.sheet_mut(0) { + s.name = "My Sheet".into(); + for r in 0..rows { + for c in 0..cols { + s.set_value(CellRef::new(r, c), Value::Number(f64::from(r * 10 + c))); + } + } + } + wb + } + + #[test] + fn a_long_reference_is_clipped_before_it_is_evaluated() { + let wb = book(1, 1); + let rows = |f: &str, limit| { + let g = clipped_reference(&wb, 0, f, limit)?; + let (_, r) = parse_ref(&wb, 0, &g)?; + Some((g, u64::from(r.height()))) + }; + // 1,000,000 rows: at most `MAX_POINTS` of them are asked for. + assert_eq!(rows("'My Sheet'!$A$1:$A$1000000", MAX_POINTS), Some(("'My Sheet'!A1:A100000".to_string(), 100_000))); + assert_eq!(rows("A5:A1048576", MAX_POINTS).map(|r| r.1), Some(100_000)); + assert_eq!(rows("A:A", 50_000).map(|r| r.1), Some(50_000)); + // Never below what keeps calc trimming the clipped reference like the whole one. + assert_eq!(rows("A1:A1000000", 100).map(|r| r.1), Some(16_385)); + // Wide references keep whole rows. + assert_eq!(rows("A1:C1000000", MAX_POINTS).map(|r| r.1), Some(100_000)); + // Small references, single cells and anything that isn't a plain reference are left alone. + assert_eq!(rows("A1:A9000", MAX_POINTS), None); + assert_eq!(rows("A1:A20000", MAX_POINTS), None); + assert_eq!(rows("B2", MAX_POINTS), None); + assert_eq!(rows("SUM(A1:A1000000)", MAX_POINTS), None); + assert_eq!(rows("{1,2,3}", MAX_POINTS), None); + } + + #[test] + fn clipping_does_not_change_the_points() { + for (rows, cols) in [(10, 1), (30_000, 1), (30_000, 2), (3, 4)] { + let wb = book(rows, cols); + for f in ["'My Sheet'!A1:A1000000", "'My Sheet'!$A$3:$B$1048576", "A:A", "A1:C1000000", "(A1:A1000000,B1:B1000000)", "A2:A20000"] { + for limit in [100, 7_843, 25_000, MAX_POINTS] { + let clipped = eval_list_in(&wb, 0, f, limit, 0); + let whole: Vec = match evaluate(&wb, 0, CellRef::default(), f) { + Value::Array(a) => a.iter().take(limit).cloned().collect(), + v => vec![v], + }; + if f.starts_with('(') { + // A union is read part by part; each part keeps its own order. + assert!(clipped.len() <= limit); + continue; + } + assert_eq!(clipped, whole, "{f} limit {limit} on {rows}x{cols}"); + } + } + } + } +} diff --git a/crates/chart/src/tests.rs b/crates/chart/src/tests.rs index 01f1aac49..587ddd958 100644 --- a/crates/chart/src/tests.rs +++ b/crates/chart/src/tests.rs @@ -5,6 +5,8 @@ use gridcraft_model::{Anchor, Chart, ChartKind, Color, LegendPos, Series, Sparkl use crate::*; +mod budget; + const ALL_KINDS: [ChartKind; 25] = [ ChartKind::ColumnClustered, ChartKind::ColumnStacked, diff --git a/crates/chart/src/tests/budget.rs b/crates/chart/src/tests/budget.rs new file mode 100644 index 000000000..e151f635d --- /dev/null +++ b/crates/chart/src/tests/budget.rs @@ -0,0 +1,172 @@ +//! Work and memory are bounded by the product of the chart limits (series x points, trendlines +//! per chart, hairline bars), and clean data is rendered without a copy. + +use std::borrow::Cow; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_model::{ChartKind, Series, Trendline, TrendlineKind, Workbook}; + +use super::{assert_sane, chart}; +use crate::render::{sanitize, sanitized}; +use crate::resolve::{MAX_POINTS, MAX_TOTAL_POINTS, MAX_TRENDLINES, points_per_series}; +use crate::*; + +fn data_of(series: usize, points: usize) -> ChartData { + ChartData { + palette: vec![], + categories: (0..points).map(|i| i.to_string()).collect(), + series: (0..series) + .map(|s| SeriesData { + name: format!("S{s}"), + values: (0..points).map(|i| Some(((i * 7 + s * 13) % 100 + 1) as f64)).collect(), + color: series_color(&Default::default(), s), + ..SeriesData::default() + }) + .collect(), + } +} + +fn series_rects(prims: &[Prim], color: Rgba) -> Vec<(f32, f32, f32, f32)> { + prims + .iter() + .filter_map(|p| match p { + Prim::Rect { x, y, w, h, fill: Some(f), .. } if *f == color => Some((*x, *y, *w, *h)), + _ => None, + }) + .collect() +} + +#[test] +fn the_point_budget_keeps_every_series_with_fewer_points() { + assert_eq!(points_per_series(1), MAX_POINTS); + assert_eq!(points_per_series(20), MAX_POINTS); + assert_eq!(points_per_series(21), MAX_TOTAL_POINTS / 21); + assert_eq!(points_per_series(255), MAX_TOTAL_POINTS / 255); + assert_eq!(points_per_series(0), MAX_POINTS); + + // 25 series x 100,000 points is 2.5 million: over budget. + let d = data_of(25, MAX_POINTS); + let kept = sanitized(&d); + assert_eq!(kept.series.len(), 25, "no series is dropped"); + assert!(kept.series.iter().all(|s| s.values.len() == MAX_TOTAL_POINTS / 25)); + assert_eq!(kept.categories.len(), MAX_TOTAL_POINTS / 25); + assert!(kept.series.iter().map(|s| s.values.len()).sum::() <= MAX_TOTAL_POINTS); + // The first points are the ones kept. + assert_eq!(kept.series[3].values[..5], d.series[3].values[..5]); + + // Drawing it stays bounded: hairline columns are merged per pixel column. + let mut c = chart(ChartKind::ColumnClustered); + c.data_labels = false; + c.legend = LegendPos::None; + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + assert_sane(&prims, 400.0, 300.0); + assert!(prims.len() < 25 * 450 + 200, "{} primitives", prims.len()); +} + +#[test] +fn trendlines_are_limited_per_chart() { + let line = |kind| TrendlineData { spec: Trendline { kind, ..Trendline::default() }, color: [1, 2, 3, 255] }; + let mut d = data_of(3, 30); + for s in &mut d.series { + s.trendlines = vec![line(TrendlineKind::Linear); 10]; + } + let kept = sanitized(&d); + let counts: Vec = kept.series.iter().map(|s| s.trendlines.len()).collect(); + assert_eq!(counts, vec![10, MAX_TRENDLINES - 10, 0], "series order decides which trendlines stay"); + let entries = legend_entries(&chart(ChartKind::Line), &kept); + assert_eq!(entries.len(), 3 + MAX_TRENDLINES); +} + +#[test] +fn hairline_columns_are_merged_and_wide_ones_are_not() { + let mut c = chart(ChartKind::ColumnClustered); + c.data_labels = false; + c.legend = LegendPos::None; + + // Wide bars (>= 0.25 px): one rectangle per series and category, as before. + let d = data_of(3, 40); + let prims = render(&c, &d, 640.0, 400.0, &ApproxMeasure); + for s in &d.series { + assert_eq!(series_rects(&prims, s.color).len(), 40); + } + + // 4 series x 20,000 categories in 400 px: the bars are hairlines. + let d = data_of(4, 20_000); + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + assert_sane(&prims, 400.0, 300.0); + for s in &d.series { + let n = series_rects(&prims, s.color).len(); + assert!((1..=420).contains(&n), "{n} rectangles for one series"); + } + + // The merged rectangles keep the envelope: alternating 1 and 10 draws the full-height bar. + let mut d = data_of(1, 20_000); + for (i, v) in d.series[0].values.iter_mut().enumerate() { + *v = Some(if i % 2 == 0 { 1.0 } else { 10.0 }); + } + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + let rects = series_rects(&prims, d.series[0].color); + let tallest = rects.iter().map(|r| r.3).fold(0.0f32, f32::max); + assert!(rects.len() > 100 && tallest > 100.0); + assert!(rects.iter().all(|r| (r.3 - tallest).abs() < 0.01), "every pixel column shows the tall bar"); + // And together they cover the plot's width without a gap wider than a pixel. + let mut xs: Vec<(f32, f32)> = rects.iter().map(|r| (r.0, r.0 + r.2)).collect(); + xs.sort_by(|a, b| a.0.total_cmp(&b.0)); + assert!(xs.windows(2).all(|w| w[1].0 - w[0].1 < 1.0)); + + // Horizontal bars too. + let mut c = chart(ChartKind::BarClustered); + c.data_labels = false; + c.legend = LegendPos::None; + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + let n = series_rects(&prims, d.series[0].color).len(); + assert!((1..=320).contains(&n), "{n} bars"); +} + +#[test] +fn clean_data_is_rendered_without_a_copy() { + let d = data_of(3, 50); + assert!(matches!(sanitized(&d), Cow::Borrowed(_))); + + // Dirty data takes the copying path and comes out as `sanitize` has always produced it. + let mut dirty = data_of(3, 50); + dirty.series[0].values[4] = Some(f64::NAN); + dirty.series[1].values[5] = Some(f64::INFINITY); + dirty.series[2].values[6] = Some(1e301); + dirty.series[2].x = Some(vec![Some(f64::NEG_INFINITY), Some(2.0)]); + dirty.series[1].sizes = Some(vec![Some(-1e305)]); + let got = sanitized(&dirty); + assert!(matches!(got, Cow::Owned(_))); + assert_eq!(*got, sanitize(&dirty)); + assert_eq!(got.series[0].values[4], None); + assert_eq!(got.series[2].values[6], Some(1e300)); + + // Over the series cap (within the point budget) the same. + let many = data_of(300, 3); + let got = sanitized(&many); + assert_eq!(got.series.len(), 255); + assert_eq!(*got, sanitize(&many)); +} + +#[test] +fn resolve_keeps_every_series_within_the_point_budget() { + // 40 series x 60,000 points is 2.4 million: over the 2 million budget. + let mut wb = Workbook::new(); + if let Some(s) = wb.sheet_mut(0) { + for r in 0..60_000u32 { + s.set_value(CellRef::new(r, 0), Value::Number(f64::from(r))); + } + } + let mut c = chart(ChartKind::ColumnClustered); + c.series = + (0..40).map(|_| Series { values: "Sheet1!$A$1:$A$1000000".into(), categories: Some("A1:A1000000".into()), ..Series::default() }).collect(); + let d = resolve(&wb, 0, &c); + assert_eq!(d.series.len(), 40); + assert!(d.series.iter().all(|s| s.values.len() == MAX_TOTAL_POINTS / 40)); + assert_eq!(d.categories.len(), MAX_TOTAL_POINTS / 40); + assert_eq!(d.series[7].values.first(), Some(&Some(0.0))); + // Trendlines: 16 per chart, in series order. + c.series = (0..4).map(|_| Series { values: "A1:A5".into(), trendlines: vec![Default::default(); 9], ..Series::default() }).collect(); + let counts: Vec = resolve(&wb, 0, &c).series.iter().map(|s| s.trendlines.len()).collect(); + assert_eq!(counts, vec![9, 7, 0, 0]); +} diff --git a/crates/chart/src/trend.rs b/crates/chart/src/trend.rs index d987ab58d..6b22ed7ad 100644 --- a/crates/chart/src/trend.rs +++ b/crates/chart/src/trend.rs @@ -72,7 +72,12 @@ pub(crate) fn fit_trendline(t: &Trendline, pts: &[(f64, f64)]) -> Option = (first..=d).map(|k| u.powi(k as i32)).collect(); + // `d` is at most 6, so the powers `first..=d` always fit the array (no allocation per point). + let mut buf = [0.0f64; 7]; + for (slot, k) in buf.iter_mut().zip(first..=d) { + *slot = u.powi(k as i32); + } + let pows = buf.get(..m)?; for (i, pi) in pows.iter().enumerate() { if let Some(v) = aty.get_mut(i) { *v += pi * (y - b0); @@ -258,10 +263,41 @@ fn trim_zeros(s: &str) -> String { /// Moving average of `period` consecutive points in plotted order: point `i` (from /// `period - 1`) is `(xᵢ, mean(yᵢ₋ₚ₊₁..=yᵢ))`. Empty when there are fewer points than `period`. +/// +/// Linear in the number of points: the points are cut into blocks of `period`, and a window is +/// the tail of one block plus the head of the next, taken from running sums that only ever add +/// (a running sum that also subtracts would lose small values after a huge one passed through). pub(crate) fn moving_average(pts: &[(f64, f64)], period: u32) -> Vec<(f64, f64)> { let p = period.clamp(2, 255) as usize; let pts: Vec<(f64, f64)> = pts.iter().copied().filter(|(x, y)| x.is_finite() && y.is_finite()).collect(); - pts.windows(p).filter_map(|w| w.last().map(|(x, _)| (*x, w.iter().map(|q| q.1).sum::() / p as f64))).collect() + let n = pts.len(); + if n < p { + return vec![]; + } + // `head[i]`: sum of the block's points up to and including `i`; `tail[i]`: from `i` to the block's end. + let mut head = vec![0.0f64; n]; + let mut tail = vec![0.0f64; n]; + for (b, block) in pts.chunks(p).enumerate() { + let (mut h, mut t) = (0.0f64, 0.0f64); + for k in 0..block.len() { + h += block.get(k).map_or(0.0, |q| q.1); + if let Some(v) = head.get_mut(b * p + k) { + *v = h; + } + let r = block.len() - 1 - k; + t += block.get(r).map_or(0.0, |q| q.1); + if let Some(v) = tail.get_mut(b * p + r) { + *v = t; + } + } + } + (p - 1..n) + .filter_map(|i| { + let s = i + 1 - p; + let sum = tail.get(s)? + if s.is_multiple_of(p) { 0.0 } else { *head.get(i)? }; + Some((pts.get(i)?.0, sum / p as f64)) + }) + .collect() } /// Excel's automatic trendline name for the legend, e.g. `Linear (Sales)`. @@ -276,3 +312,168 @@ pub(crate) fn auto_name(kind: TrendlineKind, series: &str) -> String { }; format!("{k} ({series})") } + +#[cfg(test)] +mod tests { + use super::*; + + /// xorshift64 for repeatable pseudo-random data. + struct Rng(u64); + + impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + fn unit(&mut self) -> f64 { + (self.next() % 1_000_000) as f64 / 1_000_000.0 - 0.5 + } + } + + /// The window-by-window moving average this module used to compute. + fn moving_average_naive(pts: &[(f64, f64)], period: u32) -> Vec<(f64, f64)> { + let p = period.clamp(2, 255) as usize; + let pts: Vec<(f64, f64)> = pts.iter().copied().filter(|(x, y)| x.is_finite() && y.is_finite()).collect(); + pts.windows(p).filter_map(|w| w.last().map(|(x, _)| (*x, w.iter().map(|q| q.1).sum::() / p as f64))).collect() + } + + #[test] + fn linear_moving_average_matches_the_window_sums() { + let mut r = Rng(0x9E37_79B9_7F4A_7C15); + for case in 0..60 { + let n = (r.next() % 700) as usize; + let pts: Vec<(f64, f64)> = (0..n) + .map(|i| { + let y = match case % 5 { + 0 => r.unit() * 100.0, + 1 => 1e9 + r.unit(), + // A huge value passing through must not leave rounding residue behind. + 2 => { + if r.next().is_multiple_of(50) { + 1e300 * r.unit().signum() + } else { + r.unit() + } + } + 3 => { + if i % 2 == 0 { + 1e15 + } else { + -1e15 + r.unit() + } + } + _ => { + if r.next().is_multiple_of(9) { + f64::NAN + } else { + r.unit() * 1e-3 + } + } + }; + (i as f64, y) + }) + .collect(); + for period in [0, 2, 3, 7, 50, 255, 400] { + let fast = moving_average(&pts, period); + let slow = moving_average_naive(&pts, period); + assert_eq!(fast.len(), slow.len(), "case {case} period {period}"); + let p = period.clamp(2, 255) as usize; + let finite: Vec<(f64, f64)> = pts.iter().copied().filter(|(x, y)| x.is_finite() && y.is_finite()).collect(); + for (i, ((fx, fy), (sx, sy))) in fast.iter().zip(&slow).enumerate() { + assert_eq!(fx, sx); + let mass: f64 = finite.iter().skip(i).take(p).map(|q| q.1.abs()).sum::() / p as f64; + assert!((fy - sy).abs() <= 1e-12 * mass, "case {case} period {period} point {i}: {fy} vs {sy}"); + } + } + } + } + + /// The trendline fit as it was written with a `Vec` of powers per data point. + fn fit_reference(t: &Trendline, pts: &[(f64, f64)]) -> Option { + let kind = t.kind; + let d = degree(kind)?; + let data: Vec<(f64, f64)> = pts + .iter() + .filter(|(x, y)| x.is_finite() && y.is_finite() && (!log_x(kind) || *x > 0.0) && (!log_y(kind) || *y > 0.0)) + .map(|&(x, y)| (if log_x(kind) { x.ln() } else { x }, if log_y(kind) { y.ln() } else { y })) + .collect(); + if data.len() < 2 || data.len() <= d { + return None; + } + let fixed = match kind { + TrendlineKind::Linear | TrendlineKind::Polynomial(_) => t.intercept, + TrendlineKind::Exponential => t.intercept.filter(|b| *b > 0.0).map(f64::ln), + _ => None, + } + .filter(|b| b.is_finite()); + let n = data.len() as f64; + let center = if fixed.is_some() { 0.0 } else { data.iter().map(|p| p.0).sum::() / n }; + let scale = data.iter().map(|p| (p.0 - center).abs()).fold(0.0f64, f64::max); + if !(scale.is_finite() && scale > 0.0) { + return None; + } + let first = usize::from(fixed.is_some()); + let b0 = fixed.unwrap_or(0.0); + let m = d + 1 - first; + let mut ata = vec![vec![0.0f64; m]; m]; + let mut aty = vec![0.0f64; m]; + for &(x, y) in &data { + let u = (x - center) / scale; + let pows: Vec = (first..=d).map(|k| u.powi(k as i32)).collect(); + for (i, pi) in pows.iter().enumerate() { + aty[i] += pi * (y - b0); + for (j, pj) in pows.iter().enumerate() { + ata[i][j] += pi * pj; + } + } + } + let sol = solve(ata, aty)?; + let mut a = vec![b0; d + 1]; + for (k, v) in sol.into_iter().enumerate() { + a[k + first] = v; + } + let coef = expand(&a, center, scale)?; + let fit = TrendFit { kind, coef, center, scale, a, r2: None }; + let mean = data.iter().map(|p| p.1).sum::() / n; + let (mut ss_res, mut ss_tot) = (0.0f64, 0.0f64); + for &(x, y) in &data { + ss_res += (y - fit.poly(x)).powi(2); + ss_tot += if fixed.is_some() { (y - b0).powi(2) } else { (y - mean).powi(2) }; + } + let r2 = (ss_tot > 0.0 && ss_tot.is_finite() && ss_res.is_finite()).then(|| 1.0 - ss_res / ss_tot); + Some(TrendFit { r2, ..fit }) + } + + #[test] + fn fit_without_a_vec_per_point_is_bit_identical() { + let mut r = Rng(0x1234_5678_9ABC_DEF1); + let kinds = [ + TrendlineKind::Linear, + TrendlineKind::Exponential, + TrendlineKind::Logarithmic, + TrendlineKind::Power, + TrendlineKind::Polynomial(2), + TrendlineKind::Polynomial(3), + TrendlineKind::Polynomial(6), + TrendlineKind::MovingAverage(3), + ]; + for case in 0..40 { + let n = 2 + (r.next() % 60) as usize; + let pts: Vec<(f64, f64)> = (0..n) + .map(|i| { + let x = i as f64 * (1.0 + r.unit()) + 1.0; + let y = if case % 4 == 0 { f64::NAN } else { (r.unit() + 0.6) * 10f64.powi((case % 7) - 2) }; + (x, if case % 4 == 0 && i % 3 != 0 { r.unit() + 1.0 } else { y }) + }) + .collect(); + for kind in kinds { + for intercept in [None, Some(0.5), Some(-1.0)] { + let t = Trendline { kind, intercept, ..Trendline::default() }; + assert_eq!(fit_trendline(&t, &pts), fit_reference(&t, &pts), "case {case} {kind:?} {intercept:?}"); + } + } + } + } +} From 890870cd57167fda190716689b64897fcb16596e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:11:08 +0100 Subject: [PATCH 04/51] perf: select instead of sort in MEDIAN, PERCENTILE, QUARTILE, LARGE, SMALL Read the needed order statistics with select_nth_unstable_by(total_cmp) instead of a full sort. Results are bit-identical; a test compares them against the previous full-sort code on randomised data. Fixes #2679 Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat.rs | 149 +++++++++++++++++++++++++++++++---- 1 file changed, 132 insertions(+), 17 deletions(-) diff --git a/crates/functions/src/stat.rs b/crates/functions/src/stat.rs index 344ba9ff8..4ef2625cc 100644 --- a/crates/functions/src/stat.rs +++ b/crates/functions/src/stat.rs @@ -51,10 +51,11 @@ fn median(v: Vec) -> R { if v.is_empty() { return Err(CellError::Num); } - let s = sorted(v); - let n = s.len(); - let hi = s.get(n / 2).copied().unwrap_or(0.0); - if n % 2 == 1 { Ok(hi) } else { Ok(s.get(n / 2 - 1).copied().unwrap_or(0.0).midpoint(hi)) } + let mut v = v; + let n = v.len(); + let (lower, hi, _) = v.select_nth_unstable_by(n / 2, f64::total_cmp); + let hi = *hi; + if n % 2 == 1 { Ok(hi) } else { Ok(lower.iter().copied().max_by(f64::total_cmp).unwrap_or(0.0).midpoint(hi)) } } /// Values occurring at least twice, ordered by first occurrence, with their (equal, maximal) count. @@ -125,29 +126,38 @@ fn kth(v: Vec, k: f64, largest: bool) -> R { if k.is_nan() || k < 1.0 || k > v.len() as f64 { return Err(CellError::Num); } - let s = sorted(v); + let mut v = v; let k = k as usize; - let idx = if largest { s.len() - k } else { k - 1 }; - s.get(idx).copied().ok_or(CellError::Num) + let idx = if largest { v.len() - k } else { k - 1 }; + // Selection instead of a full sort; same total order, so the same bit pattern as `sorted(v)[idx]`. + Ok(*v.select_nth_unstable_by(idx, f64::total_cmp).1) } -fn interp(s: &[f64], pos: f64) -> R { +/// Value at fractional rank `pos` of the sorted data, linearly interpolated, found by selection. +fn interp(mut v: Vec, pos: f64) -> R { let i = pos.floor(); let frac = pos - i; let i = i as usize; - let lo = *s.get(i).ok_or(CellError::Num)?; - match s.get(i + 1) { - Some(hi) if frac > 0.0 => Ok(lo + frac * (hi - lo)), - _ => Ok(lo), + if i >= v.len() { + return Err(CellError::Num); } + let (_, lo, upper) = v.select_nth_unstable_by(i, f64::total_cmp); + let lo = *lo; + // The next order statistic is the minimum of the upper partition. + if frac > 0.0 + && let Some(hi) = upper.iter().copied().min_by(f64::total_cmp) + { + return Ok(lo + frac * (hi - lo)); + } + Ok(lo) } fn percentile_inc(v: Vec, k: f64) -> R { if v.is_empty() || !(0.0..=1.0).contains(&k) { return Err(CellError::Num); } - let s = sorted(v); - interp(&s, k * (s.len() - 1) as f64) + let pos = k * (v.len() - 1) as f64; + interp(v, pos) } fn percentile_exc(v: Vec, k: f64) -> R { @@ -159,8 +169,7 @@ fn percentile_exc(v: Vec, k: f64) -> R { if pos < 1.0 || pos > n { return Err(CellError::Num); } - let s = sorted(v); - interp(&s, pos - 1.0) + interp(v, pos - 1.0) } fn quartile_inc(v: Vec, q: f64) -> R { @@ -2274,7 +2283,9 @@ pub(crate) fn specs() -> Vec { #[cfg(test)] mod tests { - use super::{MAX_REGRESSORS, aggregate_values, aggregate_values_k, check_regression_size}; + use super::{ + MAX_REGRESSORS, R, aggregate_values, aggregate_values_k, check_regression_size, kth, median, percentile_exc, percentile_inc, sorted, + }; use crate::util::testutil::*; use gridcraft_core::{CellError, Value}; @@ -2463,6 +2474,110 @@ mod tests { close(ev("RANK.AVG", vec![n(3.5), r2, n(1.0)]), 3.5); } + // Reference implementations: the previous full-sort versions, as the oracle for the selection ones. + fn old_interp(s: &[f64], pos: f64) -> R { + let i = pos.floor(); + let frac = pos - i; + let i = i as usize; + let lo = *s.get(i).ok_or(CellError::Num)?; + match s.get(i + 1) { + Some(hi) if frac > 0.0 => Ok(lo + frac * (hi - lo)), + _ => Ok(lo), + } + } + + fn old_median(v: Vec) -> R { + if v.is_empty() { + return Err(CellError::Num); + } + let s = sorted(v); + let n = s.len(); + let hi = s.get(n / 2).copied().unwrap_or(0.0); + if n % 2 == 1 { Ok(hi) } else { Ok(s.get(n / 2 - 1).copied().unwrap_or(0.0).midpoint(hi)) } + } + + fn old_kth(v: Vec, k: f64, largest: bool) -> R { + let k = k.ceil(); + if k.is_nan() || k < 1.0 || k > v.len() as f64 { + return Err(CellError::Num); + } + let s = sorted(v); + let k = k as usize; + let idx = if largest { s.len() - k } else { k - 1 }; + s.get(idx).copied().ok_or(CellError::Num) + } + + fn old_percentile_inc(v: Vec, k: f64) -> R { + if v.is_empty() || !(0.0..=1.0).contains(&k) { + return Err(CellError::Num); + } + let s = sorted(v); + old_interp(&s, k * (s.len() - 1) as f64) + } + + fn old_percentile_exc(v: Vec, k: f64) -> R { + let n = v.len() as f64; + if v.is_empty() || !(k > 0.0 && k < 1.0) { + return Err(CellError::Num); + } + let pos = k * (n + 1.0); + if pos < 1.0 || pos > n { + return Err(CellError::Num); + } + let s = sorted(v); + old_interp(&s, pos - 1.0) + } + + /// Same Ok bit pattern or same error. + fn same(a: R, b: R) -> bool { + match (a, b) { + (Ok(x), Ok(y)) => x.to_bits() == y.to_bits(), + (Err(x), Err(y)) => x == y, + _ => false, + } + } + + #[test] + fn selection_matches_full_sort_bit_for_bit() { + let mut state = 0x9E37_79B9_7F4A_7C15_u64; + let mut next = move || { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state + }; + let specials = [0.0, -0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, -f64::NAN, f64::MAX, f64::MIN_POSITIVE, 1.0, -1.0]; + for len in (0..=40).chain([63, 64, 65, 100, 257, 1000, 4096]) { + for round in 0..12 { + let v: Vec = (0..len) + .map(|_| match (round % 4, next() % 8) { + // Few distinct values: many ties. + (0, r) => (r % 3) as f64, + // Specials, including both zeros and NaNs. + (1, r) => specials[(next() as usize + r as usize) % specials.len()], + // Wide random range. + (2, _) => (next() as i64 as f64) / 1e3, + // Integers with some specials mixed in. + (_, 0) => specials[next() as usize % specials.len()], + _ => (next() % 50) as f64, + }) + .collect(); + assert!(same(median(v.clone()), old_median(v.clone())), "median len={len}"); + let ks = [-1.0, 0.0, 0.2, 1.0, 1.5, 2.0, len as f64 / 2.0, len as f64 - 0.5, len as f64, len as f64 + 1.0, f64::NAN]; + for &k in &ks { + for largest in [true, false] { + assert!(same(kth(v.clone(), k, largest), old_kth(v.clone(), k, largest)), "kth len={len} k={k}"); + } + } + let ps = [-0.1, 0.0, 0.01, 0.1, 0.25, 0.3, 0.5, 0.75, 0.9, 0.99, 1.0, 1.1, f64::NAN, (next() % 1001) as f64 / 1000.0]; + for &p in &ps { + assert!(same(percentile_inc(v.clone(), p), old_percentile_inc(v.clone(), p)), "pinc len={len} p={p}"); + assert!(same(percentile_exc(v.clone(), p), old_percentile_exc(v.clone(), p)), "pexc len={len} p={p}"); + } + } + } + } + #[test] fn percentiles() { close(ev("PERCENTILE.INC", vec![c(&[1.0, 3.0, 2.0, 4.0]), n(0.3)]), 1.9); From 96bfba93f880a6c00c15d5a9afcc0eace1575be3 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:17:29 +0100 Subject: [PATCH 05/51] perf(calc): remember SUM/COUNT/AVERAGE of constant-only blocks per pass (#2681) A block that holds only constants can't change during a pass, so its sum and count are worked out once and reused by every formula that reads the same block. A block that is the same rows from the top but longer (a running total) continues the shorter block's sum, which is the same additions in the same order as one walk from the start. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/eval.rs | 14 +++++ crates/calc/src/recalc.rs | 49 +++++++++++++++++ crates/calc/src/tests.rs | 113 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 176 insertions(+) diff --git a/crates/calc/src/eval.rs b/crates/calc/src/eval.rs index 4bdb3a30d..54777f98b 100644 --- a/crates/calc/src/eval.rs +++ b/crates/calc/src/eval.rs @@ -103,6 +103,12 @@ pub trait Host { fn visit_range(&mut self, _sheet: usize, _range: RangeRef, _f: &mut dyn FnMut(&Value) -> bool) -> bool { false } + /// `(sum, count)` of the numbers in a block that holds nothing but constants (no formula, + /// error or spilled value), added row-major from `0.0` exactly as `visit_range` would hand + /// them over. `None`: the block isn't like that, or this host can't tell. + fn range_sum(&mut self, _sheet: usize, _range: RangeRef) -> Option<(f64, u64)> { + None + } /// While a formula is calculated on demand inside another one: `stack_position()` where the /// outermost one started, which the stack budget is counted from. fn stack_top(&self) -> Option { @@ -984,6 +990,14 @@ impl<'h> Evaluator<'h> { /// Adds one area to `acc`; `Err` is the result the function ends with. fn aggregate_area(&mut self, kind: Agg, a: &Area, acc: &mut Acc) -> Result<(), Ev> { if !a.range.is_single() { + // Adding the block's remembered sum to anything but +0.0 would add in another order. + if acc.sum.to_bits() == 0 + && let Some((sum, count)) = self.host.range_sum(a.sheet, a.range) + { + acc.sum = sum; + acc.count = acc.count.saturating_add(count); + return Ok(()); + } let mut error = None; let walked = self.host.visit_range(a.sheet, a.range, &mut |v| match acc.add_cell(kind, v) { Ok(()) => true, diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 17a0e6649..753fd1971 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -436,6 +436,10 @@ struct PassHost<'a> { stack_top: usize, /// Whether an evaluation ran past the stack budget since the innermost formula started. out_of_stack: bool, + /// Sum and count of blocks that hold only constants, for `range_sum`. Keyed by sheet, first + /// cell and last column; each value maps a last row to the block's result (`None`: the block + /// has a formula, an error value or a spilled value, so every longer one has too). + sums: HashMap<(usize, CellRef, u32), std::collections::BTreeMap>>, } impl<'a> PassHost<'a> { @@ -468,6 +472,7 @@ impl<'a> PassHost<'a> { taint: false, stack_top: 0, out_of_stack: false, + sums: HashMap::default(), } } } @@ -909,6 +914,50 @@ impl Host for PassHost<'_> { } true } + fn range_sum(&mut self, sheet: usize, range: RangeRef) -> Option<(f64, u64)> { + // Constants can't change during a pass, so a block's result stays valid for it. Spills + // found in this pass (and iteration, which tracks what a read depends on) go the long way. + if self.iterate || self.spills.keys().any(|a| a.0 == sheet) { + return None; + } + let sh = self.wb.sheet(sheet)?; + let (key, last) = ((sheet, range.start, range.end.col), range.end.row); + let known = self.sums.entry(key).or_default(); + if let Some(r) = known.get(&last) { + return *r; + } + // Start from the longest shorter block of the same rows from the top: summing row-major + // from there on is the same additions in the same order as one walk from the start. + let (from, mut sum, mut count) = match known.range(..last).next_back() { + Some((_, None)) => { + known.insert(last, None); + return None; + } + Some((&row, &Some((sum, count)))) => (row + 1, sum, count), + None => (range.start.row, 0.0, 0), + }; + let rest = RangeRef::new(CellRef::new(from, range.start.col), range.end); + let mut plain = sh.spill.range(rest.start..=rest.end).all(|(c, _)| !rest.contains(*c)); + if plain { + for (_, cell) in sh.cells.iter_range(rest) { + match &cell.value { + _ if cell.formula.is_some() => plain = false, + Value::Error(_) => plain = false, + Value::Number(n) => { + sum += n; + count = u64::saturating_add(count, 1); + } + _ => {} + } + if !plain { + break; + } + } + } + let result = plain.then_some((sum, count)); + known.insert(last, result); + result + } fn used_range(&mut self, sheet: usize) -> Option { let wb = self.wb; *self.used.entry(sheet).or_insert_with(|| wb.sheet(sheet).and_then(|s| s.used_range())) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..a2db89be0 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,116 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +/// `SUM`, `COUNT` and `AVERAGE` over blocks of constants share one pass-wide answer per block, and a +/// running total extends the shorter block's. Every result must equal the left-to-right sum of the +/// cells, bit for bit, with text, blanks, errors, formulas and a nonzero start in the mix. +#[test] +fn aggregates_over_ranges_match_a_plain_fold_bit_for_bit() { + struct Rng(u64); + impl Rng { + fn next(&mut self, n: u64) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 % n + } + } + let nums = [0.1, 0.2, 0.3, 1e16, -1e16, 1.0, 3.7, -0.0, 12345.678, 1e-9, 7.0]; + let rows = 40u32; + let mut rng = Rng(0x9E37_79B9_7F4A_7C15); + let mut wb = Workbook::new(); + let fresh = |rng: &mut Rng, plain: bool| match rng.next(if plain { 8 } else { 14 }) { + 0 => Cell::value(Value::text("t")), + 1 => Cell::default(), + 2 => Cell::value(Value::Bool(true)), + 3 if !plain => Cell::value(Value::Error(CellError::NA)), + 4 if !plain => Cell::formula(Formula::new("=A1*2")), + 5 if !plain => Cell::formula(Formula::new("=\"x\"&B2")), + _ => Cell::value(Value::Number(nums[rng.next(nums.len() as u64) as usize])), + }; + // Columns A:C mostly numbers; E holds a formula column and some errors only in a few rows. + for r in 0..rows { + for col in 0..3 { + let plain = !(r % 13 == 7 && col == 1); + wb.sheet_mut(0).unwrap().cells.set(CellRef::new(r, col), fresh(&mut rng, plain)); + } + } + // Formulas in column H and up: (function, optional leading number, blocks). + let mut oracle: Vec<(CellRef, &str, Option, Vec<(u32, u32, u32, u32)>)> = vec![]; + let funcs = ["SUM", "COUNT", "AVERAGE"]; + let mut at = 0; + let mut add = |wb: &mut Workbook, f: &'static str, lead: Option, blocks: Vec<(u32, u32, u32, u32)>| { + let args: Vec = blocks + .iter() + .map(|&(r0, c0, r1, c1)| format!("{}{}:{}{}", (b'A' + c0 as u8) as char, r0 + 1, (b'A' + c1 as u8) as char, r1 + 1)) + .collect(); + let lead_s = lead.map(|l| format!("{l:e},")).unwrap_or_default(); + let cell = CellRef::new(at % rows, 7 + at / rows); + wb.sheet_mut(0).unwrap().cells.set(cell, Cell::formula(Formula::new(&format!("={f}({lead_s}{})", args.join(","))))); + oracle.push((cell, f, lead, blocks)); + at += 1; + }; + // Running totals over a clean column and over a column with an error further down. + for r in 0..rows { + add(&mut wb, "SUM", None, vec![(0, 0, r, 0)]); + add(&mut wb, "SUM", None, vec![(0, 1, r, 1)]); + add(&mut wb, "COUNT", None, vec![(0, 0, r, 2)]); + } + for _ in 0..120 { + let (r0, r1) = (rng.next(rows as u64) as u32, rng.next(rows as u64) as u32); + let (c0, c1) = (rng.next(3) as u32, rng.next(3) as u32); + let n = 1 + rng.next(2) as usize; + let blocks = (0..n).map(|_| (r0.min(r1), c0.min(c1), r0.max(r1), c0.max(c1))).collect(); + let lead = [None, Some(1e16), Some(0.5), Some(2.0)][rng.next(4) as usize]; + let f = funcs[rng.next(3) as usize]; + add(&mut wb, f, lead, blocks); + } + let check = |wb: &Workbook, when: &str| { + let sh = wb.sheet(0).unwrap(); + for (cell, f, lead, blocks) in &oracle { + let (mut sum, mut count, mut error) = (lead.unwrap_or(0.0), u64::from(lead.is_some()), None); + for &(r0, c0, r1, c1) in blocks { + for r in r0..=r1 { + for col in c0..=c1 { + match sh.value(CellRef::new(r, col)) { + Value::Number(n) => { + sum += n; + count += 1; + } + Value::Error(e) if *f != "COUNT" => error = error.or(Some(e)), + _ => {} + } + } + } + } + let want = match (error, *f) { + (Some(e), _) => Value::Error(e), + (_, "SUM") => Value::number(sum), + (_, "COUNT") => Value::number(count as f64), + (_, _) if count == 0 => Value::Error(CellError::Div0), + _ => Value::number(sum / count as f64), + }; + let got = sh.value(*cell); + let same = match (&got, &want) { + (Value::Number(a), Value::Number(b)) => a.to_bits() == b.to_bits(), + (a, b) => a == b, + }; + assert!(same, "{when}: {:?} = {:?}, plain fold {:?}", cell, got, want); + } + }; + let mut calc = Calc::new(); + calc.rebuild(&wb); + calc.recalc_all(&mut wb); + check(&wb, "full recalc"); + for round in 0..30 { + let mut changed = vec![]; + for _ in 0..1 + rng.next(4) { + let at = CellRef::new(rng.next(rows as u64) as u32, rng.next(3) as u32); + wb.sheet_mut(0).unwrap().cells.set(at, fresh(&mut rng, round % 2 == 0)); + changed.push((0, at)); + } + calc.cells_changed(&mut wb, &changed); + check(&wb, &format!("edit round {round}")); + } +} From c567511cf1e5335e4078e5bdd2036e14e72293c6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:21:15 +0100 Subject: [PATCH 06/51] calc: rebuild the dependency graph from shared syntax trees Calc::rebuild and Calc::cells_changed called Formula::expr(), which deep-copies each formula's syntax tree only to read its references. Use expr_arc() (already used when formulas are calculated), which hands out the cached Arc without copying. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 17a0e6649..6cd1551fe 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -1010,7 +1010,7 @@ impl Calc { for (si, sheet) in wb.sheets.iter().enumerate() { for (c, cell) in sheet.cells.iter() { if let Some(f) = &cell.formula - && let Some(e) = f.expr() + && let Some(e) = f.expr_arc() { let (areas, dynamic) = precedents(wb, si, &e); self.graph.insert((si, c), Node { areas, dynamic }); @@ -1088,7 +1088,7 @@ impl Calc { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); - match formula.and_then(|f| f.expr()) { + match formula.and_then(|f| f.expr_arc()) { Some(e) => { let (areas, dynamic) = precedents(wb, k.0, &e); self.graph.insert(k, Node { areas, dynamic }); From 6c37a63704253075feadd4ffd1e253a19b64b779 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:22:22 +0100 Subject: [PATCH 07/51] perf(calc): index the pass's spill areas so reads don't scan every spill (#2683) A cell read and a range read looked at every spill computed so far in the pass, and cells_changed looked at every spill range of the sheet for each changed cell. A spatial index (16 x 16 cell blocks) now finds the areas covering a cell or meeting a range. Where areas overlap, the one that counted before (first/last in the pass's spill map order) still counts. Fixes #2683 Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/lib.rs | 1 + crates/calc/src/recalc.rs | 152 +++++++++++++++++++--------- crates/calc/src/spill_index.rs | 175 +++++++++++++++++++++++++++++++++ crates/calc/src/tests.rs | 29 ++++++ 4 files changed, 310 insertions(+), 47 deletions(-) create mode 100644 crates/calc/src/spill_index.rs diff --git a/crates/calc/src/lib.rs b/crates/calc/src/lib.rs index 27c9e1961..c71d8b7b1 100644 --- a/crates/calc/src/lib.rs +++ b/crates/calc/src/lib.rs @@ -5,6 +5,7 @@ pub mod eval; pub mod recalc; +mod spill_index; mod trimrange; pub use eval::{Area, SPECIAL_FUNCTIONS, is_known_function}; diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 17a0e6649..d86e73fa5 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -45,6 +45,7 @@ use gridcraft_formula::Expr; use gridcraft_model::{CalcMode, Workbook}; use crate::eval::{Area, Evaluator, Host, precedents, stack_position}; +use crate::spill_index::{Cover, SpillIndex}; pub type Key = (usize, CellRef); @@ -393,6 +394,8 @@ struct PassHost<'a> { in_progress: HashSet, /// New spills found during the pass: anchor → array. spills: HashMap>, + /// Where the areas in `spills` are, so a read finds the ones it meets without scanning them all. + spill_index: SpillIndex, /// Used range per sheet: the workbook doesn't change during a pass, and working it out walks /// every row, so once per sheet rather than once per range read. used: HashMap>, @@ -447,6 +450,7 @@ impl<'a> PassHost<'a> { pending: exprs.keys().copied().collect(), in_progress: HashSet::default(), spills: HashMap::default(), + spill_index: SpillIndex::default(), used: HashMap::default(), rng, now, @@ -472,6 +476,33 @@ impl<'a> PassHost<'a> { } } +/// Most spill areas a range read meets before they are checked for overlapping one another by +/// walking all of the pass's spills instead of comparing pairs. +/// Changed cells (in one `cells_changed`) from which the sheet's spill areas are indexed. +const INDEX_SPILLS_FROM: usize = 16; + +const MAX_UNORDERED_HITS: usize = 16; + +/// Whether an array at `anchor` covers `c`. +fn spill_covers(anchor: CellRef, arr: &Array, c: CellRef) -> bool { + c.row >= anchor.row && c.col >= anchor.col && ((c.row - anchor.row) as usize) < arr.rows && ((c.col - anchor.col) as usize) < arr.cols +} + +/// Whether two of the spill areas anchored at `hits` share a cell inside `range`. +fn spill_hits_overlap(index: &SpillIndex, sheet: usize, hits: &[CellRef], range: RangeRef) -> bool { + let rects: Vec = hits.iter().filter_map(|a| index.area(sheet, *a)).filter(|(_, empty)| !empty).map(|(r, _)| r).collect(); + rects.iter().enumerate().any(|(i, a)| { + rects.iter().skip(i + 1).any(|b| { + a.intersects(b) + && RangeRef::new( + CellRef::new(a.start.row.max(b.start.row), a.start.col.max(b.start.col)), + CellRef::new(a.end.row.min(b.end.row), a.end.col.min(b.end.col)), + ) + .intersects(&range) + }) + }) +} + /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, /// overlaps merged cells, holds other cells, or another formula's spill (unless that formula is /// recalculated in this pass, and so may spill elsewhere now). @@ -659,6 +690,7 @@ impl PassHost<'_> { // value -- a FILTER matching one row, a SORT or UNIQUE over one item -- so `C1#` // is the 1 x 1 range and `ROWS(C1#)` is 1, as in Excel (#1491). A formula that // returns a scalar never reaches this arm, so it does not become an anchor. + self.spill_index.insert(k, a.rows, a.cols); self.spills.insert(k, a.clone()); a.data.first().cloned().unwrap_or_default() } @@ -669,6 +701,7 @@ impl PassHost<'_> { let v = match self.spills.get(&k) { Some(a) if spill_blocked(self, k, a) => { self.spills.remove(&k); + self.spill_index.remove(k); Value::Error(CellError::Spill) } _ => v, @@ -715,7 +748,11 @@ impl PassHost<'_> { while passes < max_iterations.min(max_passes) && !self.converged(max_change) { self.prev = again.iter().filter_map(|k| self.results.remove(k).map(|v| (*k, v))).collect(); self.pending = set.clone(); - self.spills.retain(|a, _| !set.contains(a)); + let gone: Vec = self.spills.keys().filter(|a| set.contains(a)).copied().collect(); + for a in gone { + self.spills.remove(&a); + self.spill_index.remove(a); + } for k in &again { self.calculate(*k); } @@ -799,17 +836,19 @@ impl Host for PassHost<'_> { if self.pending.contains(&k) || self.results.contains_key(&k) { return self.compute(k); } - // A cell covered by a spill computed in this pass. - for (anchor, arr) in &self.spills { - if anchor.0 == sheet && c.row >= anchor.1.row && c.col >= anchor.1.col { - let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize); - if dr < arr.rows && dc < arr.cols { - if self.iterate && self.tainted.contains(anchor) { - self.taint = true; - } - return arr.get(dr, dc).cloned().unwrap_or_default(); - } + // A cell covered by a spill computed in this pass. Where several overlap, the first in + // `spills`' iteration order counts, as it always has. + let covering = match self.spill_index.covering(sheet, c) { + Cover::None => None, + Cover::One(a) => self.spills.get_key_value(&(sheet, a)), + Cover::Many => self.spills.iter().find(|(a, arr)| a.0 == sheet && spill_covers(a.1, arr, c)), + }; + if let Some((anchor, arr)) = covering { + let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize); + if self.iterate && self.tainted.contains(anchor) { + self.taint = true; } + return arr.get(dr, dc).cloned().unwrap_or_default(); } self.wb.sheet(sheet).map(|s| s.value(c)).unwrap_or_default() } @@ -845,41 +884,44 @@ impl Host for PassHost<'_> { } } if !self.spills.is_empty() { - let new: Vec<(usize, Value)> = self - .spills - .iter() - .filter(|(a, _)| a.0 == sheet) - .flat_map(|(a, arr)| { - let mut v = Vec::new(); - for r in 0..arr.rows { - for cc in 0..arr.cols { - let c = CellRef::new(a.1.row.saturating_add(r as u32), a.1.col.saturating_add(cc as u32)); - if c != a.1 && range.contains(c) { - v.push(( - (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize, - arr.get(r, cc).cloned().unwrap_or_default(), - )); - } - } - } - v - }) - .collect(); + let mut hits = Vec::new(); + self.spill_index.meeting(sheet, range, &mut hits); // Iterating: an array spilled by a formula that read a previous value. - if self.iterate - && self.spills.iter().any(|(a, arr)| { - let end = CellRef::new( - a.1.row.saturating_add((arr.rows as u32).saturating_sub(1)), - a.1.col.saturating_add((arr.cols as u32).saturating_sub(1)), - ); - a.0 == sheet && self.tainted.contains(a) && RangeRef::new(a.1, end).intersects(&range) - }) - { + if self.iterate && hits.iter().any(|a| self.tainted.contains(&(sheet, *a))) { self.taint = true; } - for (i, v) in new { - if let Some(slot) = out.get_mut(i) { - *slot = v; + // Where areas overlap inside the range, the last in `spills`' iteration order wins, + // as it always has; otherwise the order doesn't matter. + let ordered = hits.len() > MAX_UNORDERED_HITS || spill_hits_overlap(&self.spill_index, sheet, &hits, range); + let mut put = |anchor: CellRef, arr: &Array| { + if arr.rows == 0 || arr.cols == 0 { + return; + } + let (r0, c0) = (range.start.row.max(anchor.row), range.start.col.max(anchor.col)); + let r1 = range.end.row.min(anchor.row.saturating_add((arr.rows as u32).saturating_sub(1))); + let c1 = range.end.col.min(anchor.col.saturating_add((arr.cols as u32).saturating_sub(1))); + for r in r0..=r1 { + for cc in c0..=c1 { + if (r, cc) != (anchor.row, anchor.col) + && let Some(slot) = out.get_mut((r - range.start.row) as usize * w + (cc - range.start.col) as usize) + { + *slot = arr.get((r - anchor.row) as usize, (cc - anchor.col) as usize).cloned().unwrap_or_default(); + } + } + } + }; + if ordered { + let wanted: HashSet = hits.iter().map(|a| (sheet, *a)).collect(); + for (a, arr) in &self.spills { + if wanted.contains(a) { + put(a.1, arr); + } + } + } else { + for a in &hits { + if let Some(arr) = self.spills.get(&(sheet, *a)) { + put(*a, arr); + } } } } @@ -898,7 +940,7 @@ impl Host for PassHost<'_> { } // Spilled values sit between the stored cells (and the formulas just calculated may have // produced some): leave such blocks to `range_values`. - if self.spills.keys().any(|a| a.0 == sheet) || sh.spill.range(range.start..=range.end).any(|(c, _)| range.contains(*c)) { + if self.spill_index.has_sheet(sheet) || sh.spill.range(range.start..=range.end).any(|(c, _)| range.contains(*c)) { return false; } for (c, cell) in sh.cells.iter_range(range) { @@ -1124,11 +1166,27 @@ impl Calc { let mut emptied: Vec = Vec::new(); // Typing into (or clearing) a cell a dynamic array spills over (or would spill over) // re-evaluates the anchor, which may now be blocked or unblocked. + // With many changed cells (a paste) the sheet's spill areas are indexed once instead of + // being walked for each one. + let mut areas = SpillIndex::default(); + let mut indexed: HashSet = HashSet::default(); + let mut at = Vec::new(); for &k in changed { if let Some(sh) = wb.sheet(k.0) { - for (anchor, r) in &sh.spill_ranges { - if *anchor != k.1 && r.contains(k.1) { - seeds.push((k.0, *anchor)); + if changed.len() > INDEX_SPILLS_FROM && !sh.spill_ranges.is_empty() { + if indexed.insert(k.0) { + for (anchor, r) in &sh.spill_ranges { + areas.insert_rect((k.0, *anchor), *r, false); + } + } + // Anchor order, as walking `spill_ranges` gives. + areas.meeting(k.0, RangeRef::new(k.1, k.1), &mut at); + seeds.extend(at.iter().filter(|a| **a != k.1).map(|a| (k.0, *a))); + } else { + for (anchor, r) in &sh.spill_ranges { + if *anchor != k.1 && r.contains(k.1) { + seeds.push((k.0, *anchor)); + } } } for (c, cell) in sh.cells.iter_range(RangeRef::new(CellRef::new(k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)), k.1)) { diff --git a/crates/calc/src/spill_index.rs b/crates/calc/src/spill_index.rs new file mode 100644 index 000000000..9a50996a3 --- /dev/null +++ b/crates/calc/src/spill_index.rs @@ -0,0 +1,175 @@ +//! A spatial index over spill areas: which areas cover a cell, and which meet a range, without +//! looking at every area. Areas are registered in 16 x 16 cell blocks, so a lookup touches one +//! block (a cell) or the blocks a range covers (a range) rather than every spill on the sheet. + +use crate::recalc::{FxHasher, Key}; +use gridcraft_core::{CellRef, RangeRef}; +use std::hash::BuildHasherDefault; + +type HashMap = std::collections::HashMap>; + +/// Block edge in cells (rows and columns). +const BLOCK: u32 = 16; + +/// One indexed area: its rectangle, and whether it covers no cells (an array with no rows or no +/// columns still meets the ranges its anchor is in, as a 1 x 1 area does for taint tracking). +#[derive(Clone, Copy)] +struct Entry { + rect: RangeRef, + empty: bool, +} + +#[derive(Default)] +struct SheetAreas { + entries: HashMap, + blocks: HashMap<(u32, u32), Vec>, +} + +/// What covers a cell. +pub(crate) enum Cover { + None, + One(CellRef), + /// Several areas overlap here: the caller decides which one counts. + Many, +} + +#[derive(Default)] +pub(crate) struct SpillIndex { + sheets: HashMap, +} + +fn block_span(rect: &RangeRef) -> (u32, u32, u32, u32) { + (rect.start.row / BLOCK, rect.end.row / BLOCK, rect.start.col / BLOCK, rect.end.col / BLOCK) +} + +impl SpillIndex { + /// Whether any area is indexed on `sheet`. + pub(crate) fn has_sheet(&self, sheet: usize) -> bool { + self.sheets.get(&sheet).is_some_and(|s| !s.entries.is_empty()) + } + + /// Indexes the area an array of `rows` x `cols` anchored at `anchor` covers (replacing the + /// one already indexed for that anchor). + pub(crate) fn insert(&mut self, (sheet, anchor): Key, rows: usize, cols: usize) { + let end = + CellRef::new(anchor.row.saturating_add((rows as u32).saturating_sub(1)), anchor.col.saturating_add((cols as u32).saturating_sub(1))); + self.insert_rect((sheet, anchor), RangeRef::new(anchor, end), rows == 0 || cols == 0); + } + + /// Indexes an area given as a range. + pub(crate) fn insert_rect(&mut self, (sheet, anchor): Key, rect: RangeRef, empty: bool) { + self.remove((sheet, anchor)); + let areas = self.sheets.entry(sheet).or_default(); + let (r0, r1, c0, c1) = block_span(&rect); + for br in r0..=r1 { + for bc in c0..=c1 { + areas.blocks.entry((br, bc)).or_default().push(anchor); + } + } + areas.entries.insert(anchor, Entry { rect, empty }); + } + + pub(crate) fn remove(&mut self, (sheet, anchor): Key) { + let Some(areas) = self.sheets.get_mut(&sheet) else { return }; + let Some(old) = areas.entries.remove(&anchor) else { return }; + let (r0, r1, c0, c1) = block_span(&old.rect); + for br in r0..=r1 { + for bc in c0..=c1 { + if let Some(list) = areas.blocks.get_mut(&(br, bc)) { + list.retain(|a| *a != anchor); + if list.is_empty() { + areas.blocks.remove(&(br, bc)); + } + } + } + } + } + + /// The areas that cover `c`. + pub(crate) fn covering(&self, sheet: usize, c: CellRef) -> Cover { + let Some(areas) = self.sheets.get(&sheet) else { return Cover::None }; + let Some(list) = areas.blocks.get(&(c.row / BLOCK, c.col / BLOCK)) else { return Cover::None }; + let mut found = Cover::None; + for a in list { + if let Some(e) = areas.entries.get(a) + && !e.empty + && e.rect.contains(c) + { + found = match found { + Cover::None => Cover::One(*a), + _ => return Cover::Many, + }; + } + } + found + } + + /// The anchors of the areas whose rectangle meets `range` (an empty area counts through its + /// anchor cell), sorted by anchor. + pub(crate) fn meeting(&self, sheet: usize, range: RangeRef, out: &mut Vec) { + out.clear(); + let Some(areas) = self.sheets.get(&sheet) else { return }; + let (r0, r1, c0, c1) = block_span(&range); + let touched = u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1); + if touched > 4 * areas.entries.len() as u64 + 16 { + // A range wider than the areas are many: look at each area once. + out.extend(areas.entries.iter().filter(|(_, e)| e.rect.intersects(&range)).map(|(a, _)| *a)); + } else { + for br in r0..=r1 { + for bc in c0..=c1 { + if let Some(list) = areas.blocks.get(&(br, bc)) { + out.extend(list.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range)))); + } + } + } + } + out.sort_unstable(); + out.dedup(); + } + + /// The rectangle indexed for `anchor` and whether it covers no cells. + pub(crate) fn area(&self, sheet: usize, anchor: CellRef) -> Option<(RangeRef, bool)> { + self.sheets.get(&sheet)?.entries.get(&anchor).map(|e| (e.rect, e.empty)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(r: u32, c: u32) -> CellRef { + CellRef::new(r, c) + } + + #[test] + fn finds_the_areas_covering_a_cell_and_meeting_a_range() { + let mut ix = SpillIndex::default(); + ix.insert((0, at(0, 0)), 40, 3); + ix.insert((0, at(30, 1)), 1, 40); + ix.insert((1, at(0, 0)), 5, 5); + assert!(matches!(ix.covering(0, at(39, 2)), Cover::One(a) if a == at(0, 0))); + assert!(matches!(ix.covering(0, at(30, 2)), Cover::Many)); + assert!(matches!(ix.covering(0, at(40, 0)), Cover::None)); + assert!(matches!(ix.covering(2, at(0, 0)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(30, 30), at(50, 60)), &mut hits); + assert_eq!(hits, vec![at(30, 1)]); + ix.meeting(0, RangeRef::new(at(0, 0), at(1_000_000, 16_000)), &mut hits); + assert_eq!(hits, vec![at(0, 0), at(30, 1)]); + ix.remove((0, at(30, 1))); + assert!(matches!(ix.covering(0, at(30, 2)), Cover::One(_))); + assert!(ix.has_sheet(1) && !ix.has_sheet(2)); + ix.insert((0, at(0, 0)), 1, 1); + assert!(matches!(ix.covering(0, at(39, 2)), Cover::None)); + } + + #[test] + fn an_area_without_cells_covers_none_but_is_met_at_its_anchor() { + let mut ix = SpillIndex::default(); + ix.insert((0, at(3, 3)), 0, 4); + assert!(matches!(ix.covering(0, at(3, 3)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(3, 3), at(3, 3)), &mut hits); + assert_eq!(hits, vec![at(3, 3)]); + } +} diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..a90a997ec 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,32 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +#[test] +fn many_spills_are_read_and_blocked_by_a_paste() { + let mut t = T::new(); + // 40 arrays of 3 rows side by side in rows 1-3, and a reader of a range over two of them. + for i in 0..40u32 { + let anchor = CellRef::new(0, 2 * i); + t.wb.sheet_mut(0).unwrap().cells.set(anchor, Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i)))); + } + t.set("A10", "=SUM(C1:F3)"); + t.set("A12", "=COUNT(C1:F3)"); + t.set("A11", "=C2*1"); + t.calc.recalc_all(&mut t.wb); + assert_eq!(t.num("A10"), (10 + 11 + 12 + 20 + 21 + 22) as f64); + assert_eq!(t.num("A11"), 11.0); + assert_eq!(t.num("A12"), 6.0); + // Pasting 20 cells (more than one spill's width worth, so the changed set is indexed) over + // the second row of every other array blocks those arrays; the rest are unchanged. + let changed: Vec<(usize, CellRef)> = (0..20u32).map(|i| (0, CellRef::new(1, 4 * i))).collect(); + for k in &changed { + t.wb.sheet_mut(0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0))); + } + t.calc.cells_changed(&mut t.wb, &changed); + assert_eq!(t.get("E1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("C1"), Value::Number(10.0)); + assert_eq!(t.get("G2"), Value::Number(31.0)); + assert_eq!(t.get("A10"), Value::Error(CellError::Spill)); + assert_eq!(t.num("A12"), 4.0); +} From 72236e4aab3cdf94123023d00307e08a23a9c0aa Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:26:15 +0100 Subject: [PATCH 08/51] perf(calc): find the formulas a cleared cell may unblock from a set of #SPILL! cells (#2684) Clearing a cell walked every formula in the graph, and each changed cell scanned a 64 x 64 window of stored cells, to find formulas showing #SPILL!. Calc now keeps the set of cells whose value is #SPILL! (filled in rebuild, kept up to date by cells_changed and the write-back of a run) and takes those candidates from it. The same set replaces the walk over the graph in run for areas a formula stopped spilling over. Each entry is checked against the workbook before it is used. Fixes #2684 Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 80 ++++++++++++++++++++++++++++----------- crates/calc/src/tests.rs | 28 ++++++++++++++ 2 files changed, 85 insertions(+), 23 deletions(-) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 17a0e6649..4a90a6775 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -377,11 +377,23 @@ pub struct Calc { /// Cells whose formulas have errors from cycles. pub circular: Vec, pub last_recalc_cells: usize, + /// The cells whose value is `#SPILL!`, kept so that finding the formulas an emptied cell or a + /// freed spill area may unblock doesn't walk every formula or window of cells. Filled in + /// `rebuild`, kept up to date by `cells_changed` and by the write-back of a run; an entry is + /// always checked against the workbook before it is used. + spill_errors: BTreeSet, } impl Default for Calc { fn default() -> Self { - Calc { graph: Graph::default(), rng: 0x9E37_79B9_7F4A_7C15, fixed_now: None, circular: vec![], last_recalc_cells: 0 } + Calc { + graph: Graph::default(), + rng: 0x9E37_79B9_7F4A_7C15, + fixed_now: None, + circular: vec![], + last_recalc_cells: 0, + spill_errors: BTreeSet::new(), + } } } @@ -1007,8 +1019,12 @@ impl Calc { pub fn rebuild(&mut self, wb: &Workbook) { self.graph = Graph::default(); let mut circ = Vec::new(); + self.spill_errors.clear(); for (si, sheet) in wb.sheets.iter().enumerate() { for (c, cell) in sheet.cells.iter() { + if cell.value == Value::Error(CellError::Spill) { + self.spill_errors.insert((si, c)); + } if let Some(f) = &cell.formula && let Some(e) = f.expr() { @@ -1066,6 +1082,25 @@ impl Calc { out } + /// The cells of `sheet` in `rows` x `cols` (inclusive, `cols` a lower and an upper column) that + /// show `#SPILL!`, in row-major order. + fn spill_errors_in(&self, wb: &Workbook, sheet: usize, rows: (u32, u32), cols: (u32, u32)) -> Vec { + let Some(sh) = wb.sheet(sheet) else { return Vec::new() }; + self.spill_errors + .range((sheet, CellRef::new(rows.0, 0))..=(sheet, CellRef::new(rows.1, u32::MAX))) + .filter(|(_, c)| c.col >= cols.0 && c.col <= cols.1) + .filter(|(_, c)| sh.cells.get(*c).is_some_and(|x| x.value == Value::Error(CellError::Spill))) + .map(|(_, c)| *c) + .collect() + } + + /// The formulas showing `#SPILL!` on `sheet` at or above and to the left of `end`: what a + /// cell emptied or a spill area freed there may unblock. + fn unblock_candidates(&self, wb: &Workbook, sheet: usize, end: CellRef) -> Vec { + let cells = self.spill_errors_in(wb, sheet, (0, end.row), (0, end.col)); + cells.into_iter().filter(|c| self.graph.nodes.contains_key(&(sheet, *c))).map(|c| (sheet, c)).collect() + } + /// Recalculates every formula (F9 / Ctrl+Alt+F9, and after loading). pub fn recalc_all(&mut self, wb: &mut Workbook) { self.rebuild(wb); @@ -1087,6 +1122,11 @@ impl Calc { pub fn cells_changed(&mut self, wb: &mut Workbook, changed: &[Key]) { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { + if wb.sheet(k.0).and_then(|s| s.cell(k.1)).is_some_and(|c| c.value == Value::Error(CellError::Spill)) { + self.spill_errors.insert(k); + } else { + self.spill_errors.remove(&k); + } let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); match formula.and_then(|f| f.expr()) { Some(e) => { @@ -1109,6 +1149,7 @@ impl Calc { && cell.formula.is_some() { cell.value = Value::Error(CellError::Name); + self.spill_errors.remove(&k); } } } @@ -1131,10 +1172,9 @@ impl Calc { seeds.push((k.0, *anchor)); } } - for (c, cell) in sh.cells.iter_range(RangeRef::new(CellRef::new(k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)), k.1)) { - if c != k.1 && cell.value == Value::Error(CellError::Spill) { - seeds.push((k.0, c)); - } + if !self.spill_errors.is_empty() { + let (r0, c0) = (k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)); + seeds.extend(self.spill_errors_in(wb, k.0, (r0, k.1.row), (c0, k.1.col)).into_iter().filter(|c| *c != k.1).map(|c| (k.0, c))); } if !sh.cells.has(k.1) { match emptied.iter_mut().find(|x| x.0 == k.0) { @@ -1146,11 +1186,7 @@ impl Calc { } // An emptied cell can unblock an anchor any distance above or to the left of it. for (si, end) in emptied { - for &(s, c) in self.graph.nodes.keys() { - if s == si && c.row <= end.row && c.col <= end.col && wb.sheet(s).is_some_and(|sh| sh.value(c) == Value::Error(CellError::Spill)) { - seeds.push((s, c)); - } - } + seeds.extend(self.unblock_candidates(wb, si, end)); } // Spill areas of changed anchors also change. for &k in changed { @@ -1254,10 +1290,15 @@ impl Calc { let mut freed: Vec<(usize, RangeRef)> = Vec::new(); for (k, v) in results { let Some(sheet) = wb.sheet_mut(k.0) else { continue }; - if let Some(cell) = sheet.cells.get_mut(k.1) - && cell.value != v - { - cell.value = v; + if let Some(cell) = sheet.cells.get_mut(k.1) { + if v == Value::Error(CellError::Spill) { + self.spill_errors.insert(k); + } else { + self.spill_errors.remove(&k); + } + if cell.value != v { + cell.value = v; + } } } for k in &dirty { @@ -1293,6 +1334,7 @@ impl Calc { if blocked { if let Some(cell) = sheet.cells.get_mut(k.1) { cell.value = Value::Error(CellError::Spill); + self.spill_errors.insert(*k); } // Not blocked while the pass ran (it would have been #SPILL! there), so // blocked by another array of this pass: the formulas that read this one, @@ -1338,15 +1380,7 @@ impl Calc { } // An area a formula no longer spills over may unblock another formula's array. for (si, o) in freed { - for &(s, c) in self.graph.nodes.keys() { - if s == si - && c.row <= o.end.row - && c.col <= o.end.col - && wb.sheet(s).is_some_and(|sh| sh.value(c) == Value::Error(CellError::Spill)) - { - spill_changes.push((s, c)); - } - } + spill_changes.extend(self.unblock_candidates(wb, si, o.end)); } if full { // Everything was already evaluated once; spill dependents need another pass. diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..fa8aab3ca 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,31 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +#[test] +fn clearing_a_cell_unblocks_the_arrays_above_and_left_of_it() { + let mut t = T::new(); + for r in 1..=50 { + t.set(&format!("D{r}"), &format!("={r}*2")); + } + t.set("B2", "7"); + t.set("B1", "=SEQUENCE(3)"); + t.set("C1", "=SEQUENCE(1,3)"); + t.set("A20", "=B1+1"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("A20"), Value::Error(CellError::Spill)); + // Emptied: the array above it spills. + t.wb.sheet_mut(0).unwrap().cells.remove(c("B2")); + t.calc.cells_changed(&mut t.wb, &[(0, c("B2"))]); + assert_eq!(t.num("B3"), 3.0); + assert_eq!(t.num("A20"), 2.0); + // Typing over the area blocks it again; far from it, nothing is disturbed. + t.set("B3", "9"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + t.set("D50", "1"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("D49"), 98.0); + // Clearing it again frees the array. + t.set("B3", ""); + assert_eq!(t.num("B3"), 3.0); +} From 5f3fcc8140670454146044ddce955cac7f18539c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:45:28 +0100 Subject: [PATCH 09/51] perf(calc): look up #SPILL! cells by row and column range, not by row band The spill-error set was ranged by rows only and filtered by column afterwards, so an edit in column A walked every #SPILL! cell to its right in the same rows. Each row now keeps its own column set and a window lookup takes one column range per row that has an entry. Results are unchanged. Also covers an array spilling right over the cleared cell. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 51 ++++++++++++++++++++++++++++++++------- crates/calc/src/tests.rs | 48 ++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index d139486c1..ce01c2984 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -10,7 +10,7 @@ //! and the formulas that read one are evaluated again until the loop settles //! (`PassHost::iterate_loops`). -use std::collections::{BTreeSet, VecDeque}; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::hash::{BuildHasherDefault, Hasher}; use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; @@ -42,7 +42,45 @@ pub struct Calc { /// freed spill area may unblock doesn't walk every formula or window of cells. Filled in /// `rebuild`, kept up to date by `cells_changed` and by the write-back of a run; an entry is /// always checked against the workbook before it is used. - spill_errors: BTreeSet, + spill_errors: SpillErrors, +} + +/// A set of cells by row, each row's columns in their own set: a window of rows and columns is +/// found by one column range per row that has an entry, so the cost follows the cells in the +/// window, not those to the side of it in the same rows. +#[derive(Clone, Debug, Default)] +struct SpillErrors { + rows: BTreeMap<(usize, u32), BTreeSet>, +} + +impl SpillErrors { + fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + fn clear(&mut self) { + self.rows.clear(); + } + + fn insert(&mut self, (sheet, c): Key) { + self.rows.entry((sheet, c.row)).or_default().insert(c.col); + } + + fn remove(&mut self, (sheet, c): &Key) { + if let Some(cols) = self.rows.get_mut(&(*sheet, c.row)) { + cols.remove(&c.col); + if cols.is_empty() { + self.rows.remove(&(*sheet, c.row)); + } + } + } + + /// The cells of `sheet` in `rows` x `cols` (both inclusive), in row-major order. + fn range(&self, sheet: usize, rows: (u32, u32), cols: (u32, u32)) -> impl Iterator + '_ { + self.rows + .range((sheet, rows.0)..=(sheet, rows.1)) + .flat_map(move |(&(_, row), set)| set.range(cols.0..=cols.1).map(move |&col| CellRef::new(row, col))) + } } impl Default for Calc { @@ -53,7 +91,7 @@ impl Default for Calc { fixed_now: None, circular: vec![], last_recalc_cells: 0, - spill_errors: BTreeSet::new(), + spill_errors: SpillErrors::default(), } } } @@ -205,12 +243,7 @@ impl Calc { /// show `#SPILL!`, in row-major order. fn spill_errors_in(&self, wb: &Workbook, sheet: usize, rows: (u32, u32), cols: (u32, u32)) -> Vec { let Some(sh) = wb.sheet(sheet) else { return Vec::new() }; - self.spill_errors - .range((sheet, CellRef::new(rows.0, 0))..=(sheet, CellRef::new(rows.1, u32::MAX))) - .filter(|(_, c)| c.col >= cols.0 && c.col <= cols.1) - .filter(|(_, c)| sh.cells.get(*c).is_some_and(|x| x.value == Value::Error(CellError::Spill))) - .map(|(_, c)| *c) - .collect() + self.spill_errors.range(sheet, rows, cols).filter(|c| sh.cells.get(*c).is_some_and(|x| x.value == Value::Error(CellError::Spill))).collect() } /// The formulas showing `#SPILL!` on `sheet` at or above and to the left of `end`: what a diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index fa8aab3ca..9de680e2f 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2302,3 +2302,51 @@ fn clearing_a_cell_unblocks_the_arrays_above_and_left_of_it() { t.set("B3", ""); assert_eq!(t.num("B3"), 3.0); } + +/// A sheet with `#SPILL!` cells in rows 0..64, columns 100..16384 (as blocked arrays leave), none +/// near column A. +fn spill_error_field() -> T { + let mut t = T::new(); + let sh = t.wb.sheet_mut(0).unwrap(); + for r in 0..64 { + for col in 100..16_384 { + sh.cells.set(CellRef::new(r, col), Cell::value(Value::Error(CellError::Spill))); + } + } + t.calc.recalc_all(&mut t.wb); + t +} + +#[test] +fn an_edit_far_left_of_many_spill_errors_does_not_disturb_them() { + let mut t = spill_error_field(); + t.set("A65", "5"); + assert_eq!(t.num("A65"), 5.0); + assert_eq!(t.get("CW1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("XFD64"), Value::Error(CellError::Spill)); + // A blocked formula inside the window is still found when a cell to its right is cleared. + t.set("DA66", "7"); + t.set("CZ66", "=SEQUENCE(1,3)"); + assert_eq!(t.get("CZ66"), Value::Error(CellError::Spill)); + t.wb.sheet_mut(0).unwrap().cells.remove(c("DA66")); + t.calc.cells_changed(&mut t.wb, &[(0, c("DA66"))]); + assert_eq!(t.num("DA66"), 2.0); +} + +/// Probe: single-cell edits in column A below a block of `#SPILL!` cells to its right. +/// `cargo test -p gridcraft-calc --release -- --ignored --nocapture spill_error_field_probe` +#[test] +#[ignore] +#[allow(clippy::disallowed_methods)] +fn spill_error_field_probe() { + let mut t = spill_error_field(); + let mut best = std::time::Duration::MAX; + for i in 0..9 { + let start = std::time::Instant::now(); + for _ in 0..100 { + t.set("A65", &format!("{i}")); + } + best = best.min(start.elapsed()); + } + eprintln!("100 edits of A65 beside 1,040,000 spill errors: best of 9 = {best:?}"); +} From 4523b817ce53be54a9ce9d5b5b49761c3907a75d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 29 Oct 2026 00:51:12 +0100 Subject: [PATCH 10/51] Read already-sorted input directly instead of selecting A full sort recognises ascending and descending runs in O(n), which a selection cannot beat, so sorted, reverse-sorted and tie-block input was up to 1.8x slower than before. Check for a monotone run first (one pass that stops at the first out-of-order pair) and index into it. Extend the bit-for-bit oracle test with ascending, descending and nearly ascending data. Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat/descriptive.rs | 32 +++++++++++++++++++++++- crates/functions/src/stat/tests.rs | 22 +++++++++++++--- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/crates/functions/src/stat/descriptive.rs b/crates/functions/src/stat/descriptive.rs index 49d30d42c..08938d039 100644 --- a/crates/functions/src/stat/descriptive.rs +++ b/crates/functions/src/stat/descriptive.rs @@ -38,12 +38,31 @@ pub(super) fn sorted(mut v: Vec) -> Vec { v } +/// Sorts `v` if it is already monotone in `total_cmp` order (a descending slice is reversed) and +/// says whether it did. One cheap pass that stops at the first out-of-order pair, so unordered +/// data costs next to nothing. Needed because the std sort recognises ascending and descending +/// runs in O(n), which a selection cannot beat. +fn sorted_if_monotone(v: &mut [f64]) -> bool { + if v.is_sorted_by(|a, b| a.total_cmp(b).is_le()) { + return true; + } + if v.is_sorted_by(|a, b| a.total_cmp(b).is_ge()) { + v.reverse(); + return true; + } + false +} + pub(super) fn median(v: Vec) -> R { if v.is_empty() { return Err(CellError::Num); } let mut v = v; let n = v.len(); + if sorted_if_monotone(&mut v) { + let hi = v.get(n / 2).copied().unwrap_or(0.0); + return if n % 2 == 1 { Ok(hi) } else { Ok(v.get(n / 2 - 1).copied().unwrap_or(0.0).midpoint(hi)) }; + } let (lower, hi, _) = v.select_nth_unstable_by(n / 2, f64::total_cmp); let hi = *hi; if n % 2 == 1 { Ok(hi) } else { Ok(lower.iter().copied().max_by(f64::total_cmp).unwrap_or(0.0).midpoint(hi)) } @@ -120,7 +139,11 @@ pub(super) fn kth(v: Vec, k: f64, largest: bool) -> R { let mut v = v; let k = k as usize; let idx = if largest { v.len() - k } else { k - 1 }; - // Selection instead of a full sort; same total order, so the same bit pattern as `sorted(v)[idx]`. + // Selection instead of a full sort (monotone input is read directly); same total order, so the + // same bit pattern as `sorted(v)[idx]`. + if sorted_if_monotone(&mut v) { + return v.get(idx).copied().ok_or(CellError::Num); + } Ok(*v.select_nth_unstable_by(idx, f64::total_cmp).1) } @@ -132,6 +155,13 @@ fn interp(mut v: Vec, pos: f64) -> R { if i >= v.len() { return Err(CellError::Num); } + if sorted_if_monotone(&mut v) { + let lo = v.get(i).copied().ok_or(CellError::Num)?; + return match v.get(i + 1) { + Some(hi) if frac > 0.0 => Ok(lo + frac * (hi - lo)), + _ => Ok(lo), + }; + } let (_, lo, upper) = v.select_nth_unstable_by(i, f64::total_cmp); let lo = *lo; // The next order statistic is the minimum of the upper partition. diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index 60da9f59e..f11629369 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -263,8 +263,8 @@ fn selection_matches_full_sort_bit_for_bit() { }; let specials = [0.0, -0.0, f64::INFINITY, f64::NEG_INFINITY, f64::NAN, -f64::NAN, f64::MAX, f64::MIN_POSITIVE, 1.0, -1.0]; for len in (0..=40).chain([63, 64, 65, 100, 257, 1000, 4096]) { - for round in 0..12 { - let v: Vec = (0..len) + for round in 0..24 { + let mut v: Vec = (0..len) .map(|_| match (round % 4, next() % 8) { // Few distinct values: many ties. (0, r) => (r % 3) as f64, @@ -277,7 +277,23 @@ fn selection_matches_full_sort_bit_for_bit() { _ => (next() % 50) as f64, }) .collect(); - assert!(same(median(v.clone()), old_median(v.clone())), "median len={len}"); + // Rounds 12..24 repeat the data as already ascending, already descending and nearly + // ascending (ends swapped): the monotone shortcut and its fallback to selection. + match round / 4 { + 3 => v.sort_by(f64::total_cmp), + 4 => { + v.sort_by(f64::total_cmp); + v.reverse(); + } + 5 => { + v.sort_by(f64::total_cmp); + if let Some(last) = v.len().checked_sub(1) { + v.swap(0, last); + } + } + _ => {} + } + assert!(same(median(v.clone()), old_median(v.clone())), "median len={len} round={round}"); let ks = [-1.0, 0.0, 0.2, 1.0, 1.5, 2.0, len as f64 / 2.0, len as f64 - 0.5, len as f64, len as f64 + 1.0, f64::NAN]; for &k in &ks { for largest in [true, false] { From b6bbdcd5cf23626171768d0cf061e4b0319a67c1 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 29 Oct 2026 00:51:45 +0100 Subject: [PATCH 11/51] calc: build precedents from a borrowed syntax tree, no Arc for text-only formulas Formula::with_expr runs a closure on the cached tree, or on a freshly parsed one that is dropped afterwards, so rebuild and cells_changed need neither a tree copy nor an Arc allocation / reference count. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 11 +++++------ crates/calc/src/tests.rs | 39 +++++++++++++++++++++++++++++++++++++++ crates/model/src/cell.rs | 10 ++++++++++ 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 34c712ddc..9886ca9bd 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -128,9 +128,8 @@ impl Calc { for (si, sheet) in wb.sheets.iter().enumerate() { for (c, cell) in sheet.cells.iter() { if let Some(f) = &cell.formula - && let Some(e) = f.expr_arc() + && let Some((areas, dynamic)) = f.with_expr(|e| precedents(wb, si, e)) { - let (areas, dynamic) = precedents(wb, si, &e); self.graph.insert((si, c), Node { areas, dynamic }); if cell.value == Value::Error(CellError::Circ) { circ.push((si, c)); @@ -205,10 +204,10 @@ impl Calc { pub fn cells_changed(&mut self, wb: &mut Workbook, changed: &[Key]) { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { - let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); - match formula.and_then(|f| f.expr_arc()) { - Some(e) => { - let (areas, dynamic) = precedents(wb, k.0, &e); + let precedent = + wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.as_ref()).and_then(|f| f.with_expr(|e| precedents(wb, k.0, e))); + match precedent { + Some((areas, dynamic)) => { self.graph.insert(k, Node { areas, dynamic }); } None => { diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..0d4c973d4 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,42 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +/// `rebuild` and `cells_changed` read formulas through `Formula::with_expr`, which borrows the +/// cached tree or parses on the fly. The dependency graph must come out the same for formulas that +/// carry their tree and for formulas that only carry their text (as after loading a file). +#[test] +fn graph_built_from_text_only_formulas_matches_cached() { + fn run(cached: bool) -> Vec { + let mut t = T::new(); + t.set("A1", "2"); + for (at, f) in [("B1", "=A1*10"), ("C1", "=B1+A1"), ("D1", "=SUM(A1:C1)"), ("E1", "=NOPE(")] { + // E1 does not parse, so it has no tree either way and only covers the `None` branch. + let mut formula = Formula::new(f); + if !cached { + formula.expr = None; + } + t.wb.sheet_mut(0).unwrap().cells.set(c(at), Cell::formula(formula)); + } + t.calc.rebuild(&t.wb); + t.calc.recalc_all(&mut t.wb); + let mut out: Vec = ["B1", "C1", "D1", "E1"].iter().map(|a| t.get(a)).collect(); + // Editing the input must reach the dependents through the rebuilt graph. + t.set("A1", "5"); + out.extend(["B1", "C1", "D1"].iter().map(|a| t.get(a))); + // A text-only formula that is rewritten through `cells_changed` joins the graph too. + let mut formula = Formula::new("=A1+100"); + if !cached { + formula.expr = None; + } + t.wb.sheet_mut(0).unwrap().cells.set(c("F1"), Cell::formula(formula)); + t.calc.cells_changed(&mut t.wb, &[(0, c("F1"))]); + t.set("A1", "7"); + out.push(t.get("F1")); + out + } + let cached = run(true); + assert_eq!(cached, run(false)); + assert_eq!(cached[4], Value::Number(50.0)); + assert_eq!(cached[7], Value::Number(107.0)); +} diff --git a/crates/model/src/cell.rs b/crates/model/src/cell.rs index d4466365f..7d3a7715e 100644 --- a/crates/model/src/cell.rs +++ b/crates/model/src/cell.rs @@ -46,6 +46,16 @@ impl Formula { pub fn expr_arc(&self) -> Option> { self.expr.clone().or_else(|| gridcraft_formula::parse(&self.text).ok().map(Arc::new)) } + /// Runs `f` on the parsed expression, borrowing the cached tree or, after deserialization, + /// a freshly parsed one that is dropped afterwards. Unlike [`expr`](Self::expr) it never copies + /// the tree, and unlike [`expr_arc`](Self::expr_arc) it needs no reference count or `Arc` + /// allocation. `None` when the text does not parse. + pub fn with_expr(&self, f: impl FnOnce(&Expr) -> R) -> Option { + match &self.expr { + Some(e) => Some(f(e)), + None => gridcraft_formula::parse(&self.text).ok().map(|e| f(&e)), + } + } } /// A static picture stored as cell content, independent of floating drawing anchors. From 8b330ac9e213c05906b4dee0468444e558f6e4a0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 29 Oct 2026 00:52:59 +0100 Subject: [PATCH 12/51] Tighten selection comment and test index arithmetic Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat/descriptive.rs | 7 ++++--- crates/functions/src/stat/tests.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/functions/src/stat/descriptive.rs b/crates/functions/src/stat/descriptive.rs index 08938d039..3c8f78f69 100644 --- a/crates/functions/src/stat/descriptive.rs +++ b/crates/functions/src/stat/descriptive.rs @@ -39,9 +39,10 @@ pub(super) fn sorted(mut v: Vec) -> Vec { } /// Sorts `v` if it is already monotone in `total_cmp` order (a descending slice is reversed) and -/// says whether it did. One cheap pass that stops at the first out-of-order pair, so unordered -/// data costs next to nothing. Needed because the std sort recognises ascending and descending -/// runs in O(n), which a selection cannot beat. +/// says whether it did. At most two passes, each stopping at the first out-of-order pair, so +/// unordered data costs next to nothing. Needed because the std sort recognises ascending and +/// descending runs in O(n) with a small constant; selection is O(n) too but measured 1.2-1.8x +/// slower on such input. fn sorted_if_monotone(v: &mut [f64]) -> bool { if v.is_sorted_by(|a, b| a.total_cmp(b).is_le()) { return true; diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index f11629369..023b36815 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -269,7 +269,7 @@ fn selection_matches_full_sort_bit_for_bit() { // Few distinct values: many ties. (0, r) => (r % 3) as f64, // Specials, including both zeros and NaNs. - (1, r) => specials[(next() as usize + r as usize) % specials.len()], + (1, r) => specials[(next() as usize).wrapping_add(r as usize) % specials.len()], // Wide random range. (2, _) => (next() as i64 as f64) / 1e3, // Integers with some specials mixed in. From 95c3bb9b89e229759077aa025032396e951e30a8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 29 Oct 2026 23:05:54 +0100 Subject: [PATCH 13/51] Fix clippy complexity after merging upstream/main --- crates/chart/src/render/cartesian.rs | 84 +++++++++++++++++++--------- 1 file changed, 58 insertions(+), 26 deletions(-) diff --git a/crates/chart/src/render/cartesian.rs b/crates/chart/src/render/cartesian.rs index f30283b58..78774b078 100644 --- a/crates/chart/src/render/cartesian.rs +++ b/crates/chart/src/render/cartesian.rs @@ -3,8 +3,8 @@ use super::legend::{cat_label, plot_kind}; use super::lines::{decimate, draw_markers, draw_runs}; use super::trend::{draw_trend_labels, draw_trendlines, max_backward, max_forward, max_forward_slots, supports_trendline}; use super::{ - Chart, ChartData, ChartKind, Ctx, Fmt, HAlign, LABEL_SIZE, MAX_CATS, MAX_LABELS, MAX_LINE_PTS, MAX_MARKERS, Prim, Scale, SeriesData, TEXT, - VAlign, WHITE, clean, fixed_scale, nice_scale, + Chart, ChartData, ChartKind, Ctx, Fmt, HAlign, LABEL_SIZE, MAX_CATS, MAX_LABELS, MAX_LINE_PTS, MAX_MARKERS, MIN_BAR_PX, Prim, Rgba, Scale, + SeriesData, TEXT, VAlign, WHITE, clean, fixed_scale, nice_scale, }; // ---------------------------------------------------------------- cartesian (column/bar/line/area/combo) @@ -81,6 +81,58 @@ impl BarRun { } } +/// Whether bars of width `bw` are merged per pixel column (never when they carry data labels). +fn merge_hairlines(bw: f32, labels_on: bool) -> bool { + bw < MIN_BAR_PX && !labels_on +} + +/// Draws the bars of one series: a rectangle each, or, for hairline bars, one merged [`BarRun`] +/// per pixel column. +struct BarSink { + run: Option, + bw: f32, + horizontal: bool, + color: Rgba, + nslots: f32, + k: f32, +} + +impl BarSink { + fn new(bw: f32, horizontal: bool, color: Rgba, nslots: f32, k: f32) -> Self { + BarSink { run: None, bw, horizontal, color, nslots, k } + } + /// Start of this series' slot along the category axis, for the category centered at `c`. + fn pos(&self, c: f32) -> f32 { + if self.horizontal { c + self.nslots * self.bw / 2.0 - (self.k + 1.0) * self.bw } else { c - self.nslots * self.bw / 2.0 + self.k * self.bw } + } + /// Draws the bar at category center `c` spanning `lo..lo + len` on the value axis, or, when + /// `merge` is set, adds it to the current run. + fn bar(&mut self, ctx: &mut Ctx, merge: bool, c: f32, lo: f32, len: f32) { + let pos = self.pos(c); + if !merge { + if self.horizontal { + ctx.rect(lo, pos, len, self.bw, self.color, None); + } else { + ctx.rect(pos, lo, self.bw, len, self.color, None); + } + return; + } + match &mut self.run { + Some(m) if m.bucket == pos.floor() => m.add(pos, lo, lo + len), + run => { + if let Some(m) = run.replace(BarRun::new(pos, lo, lo + len)) { + m.draw(ctx, self.bw, self.horizontal, self.color); + } + } + } + } + fn finish(self, ctx: &mut Ctx) { + if let Some(m) = self.run { + m.draw(ctx, self.bw, self.horizontal, self.color); + } + } +} + fn series_spans(data: &ChartData, fams: &[Fam], sec: &[bool], ncat: usize) -> Vec>> { // Totals for 100% stacking, per axis and family (100% columns and 100% areas stack apart). let mut totals: Vec<((bool, Fam), Vec)> = Vec::new(); @@ -307,33 +359,15 @@ pub(super) fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx 1 => { let k = slot_of.get(i).copied().unwrap_or(0) as f32; // Hairline bars are merged per pixel column (their envelope is what shows). - let cull = bw < MIN_BAR_PX && !labels_on; - let mut run: Option = None; + let cull = merge_hairlines(bw, labels_on); + let mut sink = BarSink::new(bw, horizontal, s.color, nslots, k); for (j, sp) in row.iter().enumerate() { let Some((b, t)) = sp else { continue }; let (pb, pt) = (vpos(*b, &sc), vpos(*t, &sc)); let c = center(j); let lo = pb.min(pt); let len = (pb - pt).abs(); - if cull { - let pos = if horizontal { c + nslots * bw / 2.0 - (k + 1.0) * bw } else { c - nslots * bw / 2.0 + k * bw }; - match &mut run { - Some(m) if m.bucket == pos.floor() => m.add(pos, lo, lo + len), - _ => { - if let Some(m) = run.replace(BarRun::new(pos, lo, lo + len)) { - m.draw(ctx, bw, horizontal, s.color); - } - } - } - continue; - } - if horizontal { - let y = c + nslots * bw / 2.0 - (k + 1.0) * bw; - ctx.rect(lo, y, len, bw, s.color, None); - } else { - let x = c - nslots * bw / 2.0 + k * bw; - ctx.rect(x, lo, bw, len, s.color, None); - } + sink.bar(ctx, cull, c, lo, len); if labels_on && let Some(v) = s.values.get(j).copied().flatten() { let t = fmt.format(v, None); let stacked = fam != Fam::Col; @@ -359,9 +393,7 @@ pub(super) fn cartesian(ctx: &mut Ctx, chart: &Chart, data: &ChartData, area: Bx } } } - if let Some(m) = run { - m.draw(ctx, bw, horizontal, s.color); - } + sink.finish(ctx); } _ => { let pts: Vec> = row.iter().enumerate().map(|(j, sp)| sp.map(|(_, t)| [center(j), vpos(t, &sc)])).collect(); From 6c912d2eb130130f93d9281baca435a37e5d30e7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 29 Oct 2026 23:48:11 +0100 Subject: [PATCH 14/51] perf(calc): keep tall and wide spill areas out of the block index A full-height column spill took 65,536 block registrations, on every pass that produced it. Areas over 1024 blocks are now tested directly by each lookup instead. Adds a randomized comparison against a scan of all areas. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/spill_index.rs | 121 +++++++++++++++++++++++++++++---- crates/calc/src/tests.rs | 21 ++++++ 2 files changed, 127 insertions(+), 15 deletions(-) diff --git a/crates/calc/src/spill_index.rs b/crates/calc/src/spill_index.rs index 7cf2d44fe..1bbf0f9c0 100644 --- a/crates/calc/src/spill_index.rs +++ b/crates/calc/src/spill_index.rs @@ -1,6 +1,9 @@ //! A spatial index over spill areas: which areas cover a cell, and which meet a range, without //! looking at every area. Areas are registered in 16 x 16 cell blocks, so a lookup touches one //! block (a cell) or the blocks a range covers (a range) rather than every spill on the sheet. +//! An area that would take more than `BIG_AREA_BLOCKS` blocks (a full-height column spill takes +//! 65,536) is not registered in blocks but kept in a short list that every lookup tests directly, +//! so registering costs what the area's size warrants only for small areas. use crate::recalc::{FxHasher, Key}; use gridcraft_core::{CellRef, RangeRef}; @@ -11,6 +14,9 @@ type HashMap = std::collections::HashMap, blocks: HashMap<(u32, u32), Vec>, + /// Anchors of the areas too big to register in blocks. + big: Vec, } /// What covers a cell. @@ -42,16 +50,24 @@ fn block_span(rect: &RangeRef) -> (u32, u32, u32, u32) { (rect.start.row / BLOCK, rect.end.row / BLOCK, rect.start.col / BLOCK, rect.end.col / BLOCK) } +fn block_count(rect: &RangeRef) -> u64 { + let (r0, r1, c0, c1) = block_span(rect); + u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1) +} + /// Whether registering `areas` in blocks costs less than checking each of them against each of /// `lookups` cells would: a few block entries cost about one containment check each. A tall or -/// wide area (a full-height column spill takes 65,536 blocks) is cheaper to test directly. +/// wide area is not registered at all (see `BIG_AREA_BLOCKS`) and so costs nothing here. pub(crate) fn indexing_pays<'a>(areas: impl ExactSizeIterator, lookups: usize) -> bool { const BLOCK_COST: u64 = 16; let budget = (lookups as u64).saturating_mul(areas.len() as u64) / BLOCK_COST; let mut blocks = 0u64; for r in areas { - let (r0, r1, c0, c1) = block_span(r); - blocks = blocks.saturating_add(u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1)); + let n = block_count(r); + if n > BIG_AREA_BLOCKS { + continue; + } + blocks = blocks.saturating_add(n); if blocks > budget { return false; } @@ -77,10 +93,14 @@ impl SpillIndex { pub(crate) fn insert_rect(&mut self, (sheet, anchor): Key, rect: RangeRef, empty: bool) { self.remove((sheet, anchor)); let areas = self.sheets.entry(sheet).or_default(); - let (r0, r1, c0, c1) = block_span(&rect); - for br in r0..=r1 { - for bc in c0..=c1 { - areas.blocks.entry((br, bc)).or_default().push(anchor); + if block_count(&rect) > BIG_AREA_BLOCKS { + areas.big.push(anchor); + } else { + let (r0, r1, c0, c1) = block_span(&rect); + for br in r0..=r1 { + for bc in c0..=c1 { + areas.blocks.entry((br, bc)).or_default().push(anchor); + } } } areas.entries.insert(anchor, Entry { rect, empty }); @@ -89,6 +109,10 @@ impl SpillIndex { pub(crate) fn remove(&mut self, (sheet, anchor): Key) { let Some(areas) = self.sheets.get_mut(&sheet) else { return }; let Some(old) = areas.entries.remove(&anchor) else { return }; + if block_count(&old.rect) > BIG_AREA_BLOCKS { + areas.big.retain(|a| *a != anchor); + return; + } let (r0, r1, c0, c1) = block_span(&old.rect); for br in r0..=r1 { for bc in c0..=c1 { @@ -105,9 +129,9 @@ impl SpillIndex { /// The areas that cover `c`. pub(crate) fn covering(&self, sheet: usize, c: CellRef) -> Cover { let Some(areas) = self.sheets.get(&sheet) else { return Cover::None }; - let Some(list) = areas.blocks.get(&(c.row / BLOCK, c.col / BLOCK)) else { return Cover::None }; + let in_block = areas.blocks.get(&(c.row / BLOCK, c.col / BLOCK)).map_or(&[][..], |l| l.as_slice()); let mut found = Cover::None; - for a in list { + for a in in_block.iter().chain(&areas.big) { if let Some(e) = areas.entries.get(a) && !e.empty && e.rect.contains(c) @@ -139,6 +163,7 @@ impl SpillIndex { } } } + out.extend(areas.big.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range)))); } out.sort_unstable(); out.dedup(); @@ -181,15 +206,81 @@ mod tests { } #[test] - fn tall_or_wide_areas_are_not_worth_indexing_for_a_small_paste() { + fn tall_or_wide_areas_are_not_registered_in_blocks() { let column = RangeRef::new(at(0, 0), at(1_048_575, 0)); - assert!(!indexing_pays([column].iter(), 17)); - assert!(!indexing_pays([column].iter(), 1_000)); - // Many small areas pay back for a paste of a hundred cells. + assert!(block_count(&column) > BIG_AREA_BLOCKS); + let mut ix = SpillIndex::default(); + ix.insert_rect((0, at(0, 0)), column, false); + assert!(ix.sheets[&0].blocks.is_empty()); + assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::One(a) if a == at(0, 0))); + assert!(matches!(ix.covering(0, at(900_000, 1)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(5, 0), at(6, 3)), &mut hits); + assert_eq!(hits, vec![at(0, 0)]); + ix.remove((0, at(0, 0))); + assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::None) && ix.sheets[&0].big.is_empty()); + // Registering a big area costs nothing (lookups test it directly), so a small paste still indexes next to many small areas. + assert!(indexing_pays([column].iter(), 17)); + // Many small areas pay back for a paste of a hundred cells, but not for a handful. let small: Vec = (0..1_000).map(|i| RangeRef::new(at(i * 3, 0), at(i * 3 + 2, 0))).collect(); assert!(indexing_pays(small.iter(), 100)); - // A big area among many areas is paid for by a big paste. - assert!(indexing_pays([column].iter(), 2_000_000)); + assert!(!indexing_pays(small.iter(), 17)); + } + + /// The index against the old way: test every area for every lookup. Mixed shapes: small, + /// thin and tall, wide, big (kept out of the blocks), empty, on two sheets, replaced and removed. + #[test] + fn matches_a_scan_of_every_area_on_mixed_shapes() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = |n: u32| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 11) as u32 % n + }; + let mut ix = SpillIndex::default(); + let mut model: std::collections::BTreeMap<(usize, CellRef), (RangeRef, bool)> = Default::default(); + for step in 0..3_000 { + let sheet = next(2) as usize; + let anchor = at(next(400) * 5, next(40) * 3); + if next(4) == 0 { + ix.remove((sheet, anchor)); + model.remove(&(sheet, anchor)); + } else { + let (rows, cols) = match next(8) { + 0 => (0, 1 + next(5) as usize), + 1 => (1 + next(5) as usize, 0), + 2 => (5_000 + next(100_000) as usize, 1), + 3 => (1, 1 + next(40_000) as usize), + 4 => (200 + next(300) as usize, 40 + next(40) as usize), + _ => (1 + next(60) as usize, 1 + next(40) as usize), + }; + ix.insert((sheet, anchor), rows, cols); + let end = at(anchor.row + (rows as u32).saturating_sub(1), anchor.col + (cols as u32).saturating_sub(1)); + model.insert((sheet, anchor), (RangeRef::new(anchor, end), rows == 0 || cols == 0)); + } + if step % 7 != 0 { + continue; + } + let sheet = next(3) as usize; + for _ in 0..20 { + let cell = at(next(2_100), next(130)); + let want: Vec = + model.iter().filter(|((s, _), (r, e))| *s == sheet && !e && r.contains(cell)).map(|((_, a), _)| *a).collect(); + match ix.covering(sheet, cell) { + Cover::None => assert!(want.is_empty(), "{cell:?}"), + Cover::One(a) => assert_eq!(want, vec![a], "{cell:?}"), + Cover::Many => assert!(want.len() > 1, "{cell:?}"), + } + let (r0, c0) = (next(2_100), next(130)); + let (tall, wide) = (if next(5) == 0 { 3_000 } else { 40 }, if next(5) == 0 { 130 } else { 20 }); + let range = RangeRef::new(at(r0, c0), at(r0 + next(tall), c0 + next(wide))); + let want: Vec = model.iter().filter(|((s, _), (r, _))| *s == sheet && r.intersects(&range)).map(|((_, a), _)| *a).collect(); + let mut got = Vec::new(); + ix.meeting(sheet, range, &mut got); + assert_eq!(got, want, "{range:?}"); + } + } } #[test] diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 9dd1fd925..ac254239a 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2350,3 +2350,24 @@ fn many_spills_are_read_and_blocked_by_a_paste() { assert_eq!(t.get("A10"), Value::Error(CellError::Spill)); assert_eq!(t.num("A12"), 4.0); } + +#[test] +fn a_paste_over_small_and_tall_spills_blocks_those_it_touches() { + let mut t = T::new(); + for i in 0..40u32 { + t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 2 * i), Cell::formula(Formula::new("=SEQUENCE(3)"))); + } + t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 200), Cell::formula(Formula::new("=SEQUENCE(1048576)"))); + t.calc.recalc_all(&mut t.wb); + // 20 cells: the middle row of the small arrays in columns 0, 8, 16 ... 72 (columns 80 and up hit + // nothing), and one deep inside the tall one. + let mut cells: Vec<(u32, u32)> = (0..19u32).map(|i| (1, 8 * i)).collect(); + cells.push((700_000, 200)); + paste_into(&mut t, &cells); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("GS1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("C1"), 1.0); + assert_eq!(t.num("C3"), 3.0); + assert_eq!(t.get("I1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("E3"), 3.0); +} From bdbd29e623c7ddf4df27331a7f7834295b5f770c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Fri, 30 Oct 2026 00:04:25 +0100 Subject: [PATCH 15/51] perf(calc): size the spill-index decision by the sheet's own changed cells A batch spread over several sheets used its total length for every sheet, so a one-cell sheet with many spills built the block index as if it had the whole batch's lookups. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 44 +++++++++++++++++++++++++++++---------- crates/calc/src/tests.rs | 22 ++++++++++++++++++++ 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index aae212dd3..7c3a6f361 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -22,6 +22,13 @@ use crate::spill_index::{SpillIndex, indexing_pays}; /// Changed cells (in one `cells_changed`) from which a sheet's spill areas may be indexed. const INDEX_SPILLS_FROM: usize = 16; +/// Whether a sheet's spill areas are indexed for a batch of `batch` changed cells of which +/// `on_sheet` are on that sheet: the batch must be a paste, and registering the areas must cost +/// less than walking them once for each of the sheet's own changed cells. +fn index_spills<'a>(batch: usize, on_sheet: usize, areas: impl ExactSizeIterator) -> bool { + batch > INDEX_SPILLS_FROM && areas.len() > 0 && indexing_pays(areas, on_sheet) +} + mod graph; mod pass_host; @@ -250,20 +257,22 @@ impl Calc { // walked for each one, when registering them costs less than walking them would. let mut areas = SpillIndex::default(); let mut strategy: HashMap = HashMap::default(); + let mut per_sheet: HashMap = HashMap::default(); + for k in changed { + *per_sheet.entry(k.0).or_insert(0) += 1; + } let mut at = Vec::new(); for &k in changed { if let Some(sh) = wb.sheet(k.0) { - let indexed = changed.len() > INDEX_SPILLS_FROM - && !sh.spill_ranges.is_empty() - && *strategy.entry(k.0).or_insert_with(|| { - let indexed = indexing_pays(sh.spill_ranges.values(), changed.len()); - if indexed { - for (anchor, r) in &sh.spill_ranges { - areas.insert_rect((k.0, *anchor), *r, false); - } + let indexed = *strategy.entry(k.0).or_insert_with(|| { + let indexed = index_spills(changed.len(), per_sheet[&k.0], sh.spill_ranges.values()); + if indexed { + for (anchor, r) in &sh.spill_ranges { + areas.insert_rect((k.0, *anchor), *r, false); } - indexed - }); + } + indexed + }); if indexed { // Anchor order, as walking `spill_ranges` gives. areas.meeting(k.0, RangeRef::new(k.1, k.1), &mut at); @@ -556,7 +565,7 @@ pub(crate) mod tests { use gridcraft_core::{CellRef, RangeRef, Value}; use gridcraft_model::{Cell, Formula, Workbook}; - use super::{Calc, HashMap, Host, Intervals, Key, PassHost}; + use super::{Calc, HashMap, Host, Intervals, Key, PassHost, index_spills}; /// xorshift64 for repeatable pseudo-random cases. fn rng(state: &mut u64) -> u64 { @@ -573,6 +582,19 @@ pub(crate) mod tests { v } + /// One changed cell on a sheet with many small spills does not index them just because the + /// batch (mostly on another sheet) is large; the sheet's own share of the batch decides. + #[test] + fn the_spill_index_is_chosen_from_the_sheets_own_changed_cells() { + let small: Vec = (0..1_000).map(|i| RangeRef::new(CellRef::new(i * 3, 0), CellRef::new(i * 3 + 2, 0))).collect(); + // Same areas, same batch of 5000: indexed when 5000 of them are on the sheet, walked when 1 is. + assert!(index_spills(5_000, 5_000, small.iter())); + assert!(!index_spills(5_000, 1, small.iter())); + // Not a paste, or nothing to index: never. + assert!(!index_spills(16, 16, small.iter())); + assert!(!index_spills(5_000, 5_000, [].iter())); + } + #[test] fn intervals_match_a_linear_scan() { let mut iv: Intervals = Intervals::default(); diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index ac254239a..a3e7f810d 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2371,3 +2371,25 @@ fn a_paste_over_small_and_tall_spills_blocks_those_it_touches() { assert_eq!(t.get("I1"), Value::Error(CellError::Spill)); assert_eq!(t.num("E3"), 3.0); } + +#[test] +fn a_big_paste_on_one_sheet_and_one_cell_over_many_spills_on_another() { + let mut t = T::new(); + t.wb.sheets.push(std::sync::Arc::new(gridcraft_model::Sheet::new("Data"))); + for i in 0..1_500u32 { + t.wb.sheet_mut(1).unwrap().cells.set(CellRef::new(4 * i, 0), Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i)))); + } + t.calc.recalc_all(&mut t.wb); + assert_eq!(t.wb.sheet(1).unwrap().value(CellRef::new(4 * 700 + 2, 0)), Value::Number(7002.0)); + // 5000 cells pasted on the first sheet, one over the middle of the 701st array on the second. + let mut changed: Vec<(usize, CellRef)> = (0..5_000u32).map(|i| (0, CellRef::new(i / 50, i % 50))).collect(); + changed.push((1, CellRef::new(4 * 700 + 1, 0))); + for k in &changed { + t.wb.sheet_mut(k.0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0))); + } + t.calc.cells_changed(&mut t.wb, &changed); + let data = t.wb.sheet(1).unwrap(); + assert_eq!(data.value(CellRef::new(4 * 700, 0)), Value::Error(CellError::Spill)); + assert_eq!(data.value(CellRef::new(4 * 699 + 2, 0)), Value::Number(6992.0)); + assert_eq!(data.value(CellRef::new(4 * 701 + 2, 0)), Value::Number(7012.0)); +} From 494a7dbb5c0fe57d819b83f1981d62e15d4b14d4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Fri, 30 Oct 2026 00:54:56 +0100 Subject: [PATCH 16/51] Keep clustered hairline bars of each series in their own strip of the pixel column Co-Authored-By: Claude Sonnet 5.5 --- crates/chart/src/render/cartesian.rs | 32 ++++++++++++++++++++------ crates/chart/src/tests/budget.rs | 34 ++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/crates/chart/src/render/cartesian.rs b/crates/chart/src/render/cartesian.rs index 78774b078..1d643c21f 100644 --- a/crates/chart/src/render/cartesian.rs +++ b/crates/chart/src/render/cartesian.rs @@ -59,24 +59,32 @@ struct BarRun { a1: f32, lo: f32, hi: f32, + /// More than one bar was merged. + merged: bool, } impl BarRun { fn new(pos: f32, lo: f32, hi: f32) -> Self { - BarRun { bucket: pos.floor(), a0: pos, a1: pos, lo, hi } + BarRun { bucket: pos.floor(), a0: pos, a1: pos, lo, hi, merged: false } } fn add(&mut self, pos: f32, lo: f32, hi: f32) { + self.merged = true; self.a0 = self.a0.min(pos); self.a1 = self.a1.max(pos); self.lo = self.lo.min(lo); self.hi = self.hi.max(hi); } - fn draw(&self, ctx: &mut Ctx, bw: f32, horizontal: bool, color: Rgba) { - let along = self.a1 - self.a0 + bw; + /// `band` is the part of the pixel column reserved for this series (`start`, `len`): in a + /// clustered chart a merged run stays inside it, so the other series keep a visible strip. + fn draw(&self, ctx: &mut Ctx, bw: f32, horizontal: bool, color: Rgba, band: Option<(f32, f32)>) { + let (a0, along) = match band { + Some((start, len)) if self.merged => (self.bucket + start, len), + _ => (self.a0, self.a1 - self.a0 + bw), + }; if horizontal { - ctx.rect(self.lo, self.a0, self.hi - self.lo, along, color, None); + ctx.rect(self.lo, a0, self.hi - self.lo, along, color, None); } else { - ctx.rect(self.a0, self.lo, along, self.hi - self.lo, color, None); + ctx.rect(a0, self.lo, along, self.hi - self.lo, color, None); } } } @@ -105,10 +113,19 @@ impl BarSink { fn pos(&self, c: f32) -> f32 { if self.horizontal { c + self.nslots * self.bw / 2.0 - (self.k + 1.0) * self.bw } else { c - self.nslots * self.bw / 2.0 + self.k * self.bw } } + /// This series' strip of a pixel column (offset, length) when the columns hold several + /// clustered series, in the direction the slots run. + fn band(&self) -> Option<(f32, f32)> { + (self.nslots > 1.0).then(|| { + let idx = if self.horizontal { self.nslots - 1.0 - self.k } else { self.k }; + (idx / self.nslots, 1.0 / self.nslots) + }) + } /// Draws the bar at category center `c` spanning `lo..lo + len` on the value axis, or, when /// `merge` is set, adds it to the current run. fn bar(&mut self, ctx: &mut Ctx, merge: bool, c: f32, lo: f32, len: f32) { let pos = self.pos(c); + let band = self.band(); if !merge { if self.horizontal { ctx.rect(lo, pos, len, self.bw, self.color, None); @@ -121,14 +138,15 @@ impl BarSink { Some(m) if m.bucket == pos.floor() => m.add(pos, lo, lo + len), run => { if let Some(m) = run.replace(BarRun::new(pos, lo, lo + len)) { - m.draw(ctx, self.bw, self.horizontal, self.color); + m.draw(ctx, self.bw, self.horizontal, self.color, band); } } } } fn finish(self, ctx: &mut Ctx) { + let band = self.band(); if let Some(m) = self.run { - m.draw(ctx, self.bw, self.horizontal, self.color); + m.draw(ctx, self.bw, self.horizontal, self.color, band); } } } diff --git a/crates/chart/src/tests/budget.rs b/crates/chart/src/tests/budget.rs index 1f2e35ab3..54d33aa7d 100644 --- a/crates/chart/src/tests/budget.rs +++ b/crates/chart/src/tests/budget.rs @@ -124,6 +124,40 @@ fn hairline_columns_are_merged_and_wide_ones_are_not() { assert!((1..=320).contains(&n), "{n} bars"); } +#[test] +fn clustered_hairline_series_keep_a_visible_strip() { + // Four series of similar height, 20,000 categories in 400 px: every pixel column holds a + // merged bar of each series, and they must not cover each other. + for kind in [ChartKind::ColumnClustered, ChartKind::BarClustered] { + let horizontal = kind == ChartKind::BarClustered; + let mut c = chart(kind); + c.data_labels = false; + c.legend = LegendPos::None; + let mut d = data_of(4, 20_000); + for (s, series) in d.series.iter_mut().enumerate() { + for (i, v) in series.values.iter_mut().enumerate() { + *v = Some(50.0 + ((i + s) % 3) as f64); + } + } + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + let along = |r: &(f32, f32, f32, f32)| if horizontal { (r.1, r.1 + r.3) } else { (r.0, r.0 + r.2) }; + let mut strips: Vec<(f32, f32, usize)> = Vec::new(); + for (s, series) in d.series.iter().enumerate() { + strips.extend(series_rects(&prims, series.color).iter().map(|r| (along(r).0, along(r).1, s))); + } + strips.sort_by(|a, b| a.0.total_cmp(&b.0)); + assert!(strips.len() > 400); + // Rectangles of different series never overlap (more than rounding). + for w in strips.windows(2) { + assert!(w[0].2 == w[1].2 || w[1].0 >= w[0].1 - 1e-3, "series {} and {} overlap", w[0].2, w[1].2); + } + // And all four series show up in the same pixel column. + let col = strips[strips.len() / 2].0.floor(); + let seen: std::collections::BTreeSet = strips.iter().filter(|s| s.0.floor() == col).map(|s| s.2).collect(); + assert_eq!(seen.len(), 4, "series visible in pixel column {col}"); + } +} + #[test] fn clean_data_is_rendered_without_a_copy() { let d = data_of(3, 50); From 96672e8ad5900dd20bd2034a99d4abebaccda297 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 2 Nov 2026 19:33:53 +0100 Subject: [PATCH 17/51] Drop helpers the merged range reads no longer use Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc/pass_host.rs | 7 ------- crates/calc/src/spill_index.rs | 1 + 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 9d00f9784..9a733501b 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -185,13 +185,6 @@ pub(super) fn spills_as_before(sheet: &gridcraft_model::Sheet, anchor: CellRef, .all(|(i, c)| c == anchor || (!sheet.cells.has(c) && sheet.spill.get(&c) == Some(arr.data.get(i).unwrap_or(&Value::Empty)))) } -/// The area an array anchored at `anchor` covers. -fn array_area(anchor: CellRef, a: &Array) -> RangeRef { - let end = - CellRef::new(anchor.row.saturating_add((a.rows as u32).saturating_sub(1)), anchor.col.saturating_add((a.cols as u32).saturating_sub(1))); - RangeRef::new(anchor, end) -} - /// The spilled values `sheet` stores inside `range`, passed to `f` until it returns false. Values /// outside the range are not walked: with fewer rows in the range than spilled values, each row /// is looked up on its own; otherwise one ordered scan visits each spilled value at most once. diff --git a/crates/calc/src/spill_index.rs b/crates/calc/src/spill_index.rs index 1bbf0f9c0..d642bdee7 100644 --- a/crates/calc/src/spill_index.rs +++ b/crates/calc/src/spill_index.rs @@ -77,6 +77,7 @@ pub(crate) fn indexing_pays<'a>(areas: impl ExactSizeIterator bool { self.sheets.get(&sheet).is_some_and(|s| !s.entries.is_empty()) } From 00f565bc11d69623f5260a13e143c20c65d0c85f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 2 Nov 2026 19:44:16 +0100 Subject: [PATCH 18/51] Factor out types clippy flags as too complex Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc/pass_host.rs | 5 ++++- crates/calc/src/tests.rs | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 6151b74d3..9964b3363 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -13,6 +13,9 @@ const MAX_NESTED: usize = 64; /// evaluated again), however high Maximum Iterations is. const MAX_ITERATION_WORK: usize = 10_000_000; +/// A block's results by last row: its sum and count, or `None` when it can't be summed from constants. +type BlockSums = std::collections::BTreeMap>; + pub(super) struct PassHost<'a> { wb: &'a Workbook, exprs: &'a HashMap>, @@ -71,7 +74,7 @@ pub(super) struct PassHost<'a> { /// Sum and count of blocks that hold only constants, for `range_sum`. Keyed by sheet, first /// cell and last column; each value maps a last row to the block's result (`None`: the block /// has a formula, a Python cell, an error value or a spilled value, so every longer one has too). - sums: HashMap<(usize, CellRef, u32), std::collections::BTreeMap>>, + sums: HashMap<(usize, CellRef, u32), BlockSums>, /// The kept results of the Python cells in the pass: a Python cell isn't evaluated, its /// result is laid out (an array spills from the cell, as a formula's does). Each one is in /// `pending` until it is laid out, so a formula that reads it first gets its new value. diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index a55231794..0c9ec4aa1 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2348,7 +2348,8 @@ fn aggregates_over_ranges_match_a_plain_fold_bit_for_bit() { } } // Formulas in column H and up: (function, optional leading number, blocks). - let mut oracle: Vec<(CellRef, &str, Option, Vec<(u32, u32, u32, u32)>)> = vec![]; + type Oracle<'a> = (CellRef, &'a str, Option, Vec<(u32, u32, u32, u32)>); + let mut oracle: Vec = vec![]; let funcs = ["SUM", "COUNT", "AVERAGE"]; let mut at = 0; let mut add = |wb: &mut Workbook, f: &'static str, lead: Option, blocks: Vec<(u32, u32, u32, u32)>| { From e85b24df5724d224ae69b2d9b7bc5cf7554f4d42 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 2 Nov 2026 20:31:52 +0100 Subject: [PATCH 19/51] calc: register a Python result's spill in the spill index python_result wrote into spills but not into spill_index, so indexed readers (cell_value, range_values, visit_range) missed a Python array laid out in the same pass. Insert on spill, remove when blocked, as the formula path does. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/recalc.rs | 33 +++++++++++++++++++++++++++++ crates/calc/src/recalc/pass_host.rs | 2 ++ 2 files changed, 35 insertions(+) diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 77d5b68dd..286569822 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -663,6 +663,39 @@ pub(crate) mod tests { assert!(!host.visit_range(0, inside, &mut |_| true), "an overlapping read leaves the block to range_values"); } + /// A Python array laid out in this pass is seen by the indexed readers, whether the sheet + /// has no spill from before or an older, smaller one. + #[test] + fn a_python_spill_found_in_the_pass_is_read_through_the_index() { + for stale in [false, true] { + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let anchor = CellRef::new(0, 0); + let py = gridcraft_model::PyCode::new("grid"); + sh.set_cell(anchor, Cell { value: Value::Number(0.0), py: Some(std::sync::Arc::new(py)), ..Cell::default() }); + if stale { + // The last calculation spilled a 2 x 2 block of 100s. + sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(1, 1))); + for c in RangeRef::new(anchor, CellRef::new(1, 1)).iter().skip(1) { + sh.spill.insert(c, Value::Number(100.0)); + } + } + let data: Vec = (1..=9).map(|i| Value::Number(f64::from(i))).collect(); + let array = gridcraft_core::Array::new(3, 3, data).unwrap(); + let mut python = HashMap::default(); + python.insert((0, anchor), Value::Array(std::sync::Arc::new(array))); + let exprs = HashMap::default(); + let mut rng = 1u64; + let mut host = PassHost::new(&wb, &exprs, &mut rng, 0.0).with_python(python); + assert_eq!(host.cell_value(0, anchor), Value::Number(1.0)); + let all = RangeRef::parse("A1:C3").unwrap(); + let sum: f64 = host.range_values(0, all).iter().map(|v| if let Value::Number(n) = v { *n } else { 0.0 }).sum(); + assert_eq!(sum, 45.0, "stale={stale}: a SUM over the new 3 x 3 spill reads all of it"); + assert_eq!(host.cell_value(0, CellRef::new(2, 2)), Value::Number(9.0), "stale={stale}"); + assert!(!host.visit_range(0, all, &mut |_| true), "stale={stale}: the new spill leaves the block to range_values"); + } + } + #[test] fn intervals_match_a_linear_scan() { let mut iv: Intervals = Intervals::default(); diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 9a733501b..1ef24bc42 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -400,6 +400,7 @@ impl PassHost<'_> { && a.cols >= 1 && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS); if sane { + self.spill_index.insert(k, a.rows, a.cols); self.spills.insert(k, a.clone()); a.data.first().cloned().unwrap_or_default() } else { @@ -411,6 +412,7 @@ impl PassHost<'_> { let v = match self.spills.get(&k) { Some(a) if spill_blocked(self, k, a) => { self.spills.remove(&k); + self.spill_index.remove(k); Value::Error(CellError::Spill) } _ => v, From 472132b1585bef6bec6a296128f2ac72166034ca Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 19:44:55 +0100 Subject: [PATCH 20/51] math: refuse macro definitions and cap matrix cells RaTeX counts macro expansions (1,000) but not how large they grow, so a definition whose argument doubles on every call keeps the expander busy without bound. User macros have no use in an equation, so check_caps now refuses the definers (def, gdef, edef, xdef, let, futurelet, global, long, newcommand, renewcommand, providecommand) before RaTeX sees the source. The scan reads control words, verbatim text and comments the way RaTeX's lexer does. A matrix is laid out and written as rows times the widest row, so one long row in a tall matrix multiplied the output. check_caps now counts the padded cells of every environment from the source and refuses more than 2,500 in total; to_omml checks the parsed grid as well. Co-Authored-By: Claude Sonnet 5.5 --- crates/math/src/lib.rs | 232 ++++++++++++++++++++++++++++++++++++++-- crates/math/src/omml.rs | 3 + docs/decisions.md | 5 +- 3 files changed, 232 insertions(+), 8 deletions(-) diff --git a/crates/math/src/lib.rs b/crates/math/src/lib.rs index 5e1e99a41..e1f783dca 100644 --- a/crates/math/src/lib.rs +++ b/crates/math/src/lib.rs @@ -3,7 +3,7 @@ //! without calling RaTeX again. //! //! RaTeX is young (0.1.x), so it only ever runs inside this crate's entry points ([`layout`], -//! [`to_omml`], [`check`]). Each one caps its input (4 KB, nesting 32) and runs under +//! [`to_omml`], [`check`]). Each one caps its input (4 KB, nesting 32, no macro definitions, at most [`MAX_CELLS`] matrix cells) and runs under //! `catch_unwind`, so a stray panic becomes a [`MathError`]. See the RaTeX gate in //! `docs/decisions.md` and `tests/gate.rs`. //! @@ -33,6 +33,9 @@ pub const MAX_DEPTH: usize = 32; pub const MAX_EM: f32 = 10_000.0; /// Most items one equation may produce. pub const MAX_ITEMS: usize = 50_000; +/// Most matrix cells one equation may have, counting the empty cells that pad a short row to the +/// width of the widest one. A ragged matrix costs rows times the widest row, not what is typed. +pub const MAX_CELLS: usize = 2_500; /// The error for a source over [`MAX_SOURCE_BYTES`]. It is one shared value so that a cache can /// answer for such a source by its length alone, without reading it. @@ -223,20 +226,61 @@ fn parse(latex: &str) -> Result, MathError> { }) } -/// The size and nesting caps, checked before RaTeX sees the source. +/// Control words that define or rebind macros. User macros have no use in an equation, and +/// RaTeX caps only how many expansions run, not how large they grow, so a self-referencing +/// definition whose argument doubles on every call keeps the expander busy without bound. They are +/// refused before RaTeX sees the source. +const MACRO_DEFINERS: &[&str] = + &["def", "gdef", "edef", "xdef", "let", "futurelet", "global", "long", "newcommand", "renewcommand", "providecommand"]; + +/// One open `\begin…\end` environment, counted from the source text. +struct Grid { + /// Rows ended so far (by `\\` or `\cr`). + rows: usize, + /// Cells in the current row. + cells: usize, + /// Cells in the widest row so far. + widest: usize, +} + +impl Grid { + fn new() -> Grid { + Grid { rows: 0, cells: 1, widest: 0 } + } + fn end_row(&mut self) { + self.widest = self.widest.max(self.cells); + self.rows += 1; + self.cells = 1; + } + /// The padded grid RaTeX lays out and `to_omml` writes: every row as wide as the widest. + fn padded_cells(&self) -> usize { + (self.rows + 1).saturating_mul(self.widest.max(self.cells)) + } +} + +/// The size, nesting, macro and matrix caps, checked before RaTeX sees the source. fn check_caps(latex: &str) -> Result<(), MathError> { if latex.len() > MAX_SOURCE_BYTES { return Err(SOURCE_TOO_LONG.clone()); } + // `%` starts a comment, except inside the address of `\href` and `\url`, where RaTeX reads it + // as a character. Which one it is decides what the rest of the line means, so the two are not + // allowed together. + if latex.contains('%') && (latex.contains("\\href") || latex.contains("\\url")) { + return Err(MathError::new("A % can't be combined with a link in an equation.", latex.find('%').map(|i| i..i + 1))); + } let mut depth = 0usize; + let mut grids: Vec = Vec::new(); + let mut cells_total = 0usize; let mut chars = latex.char_indices().peekable(); while let Some((i, c)) = chars.next() { let open = match c { '\\' => { - // A control word, or an escaped character (`\{` is not a group). + // A control word (letters and `@`, as RaTeX's lexer reads it), or an escaped + // character (`\{` is not a group). let mut word = String::new(); while let Some(&(_, n)) = chars.peek() { - if n.is_ascii_alphabetic() { + if n.is_ascii_alphabetic() || n == '@' { word.push(n); chars.next(); } else { @@ -244,15 +288,69 @@ fn check_caps(latex: &str) -> Result<(), MathError> { } } if word.is_empty() { - chars.next(); + if let Some((_, n)) = chars.next() + && n == '\\' + && let Some(g) = grids.last_mut() + { + g.end_row(); + } continue; } + if MACRO_DEFINERS.contains(&word.as_str()) { + return Err(MathError::new("Macro definitions are not supported in equations.", Some(i..i + 1 + word.len()))); + } match word.as_str() { - "left" | "begin" => Some(true), - "right" | "end" => Some(false), + "verb" => { + // Verbatim text runs to the next copy of its delimiter, whatever it + // holds: no group, comment, row or definition starts inside it. + if chars.peek().is_some_and(|&(_, n)| n == '*') { + chars.next(); + } + if let Some((_, delim)) = chars.next() { + for (_, n) in chars.by_ref() { + if n == delim { + break; + } + } + } + None + } + "cr" => { + if let Some(g) = grids.last_mut() { + g.end_row(); + } + None + } + "begin" => { + if let Some(n) = alignat_columns(latex.get(i + 6..).unwrap_or_default()) + && n > MAX_ALIGNAT_PAIRS + { + return Err(too_many_cells(i..i + 6)); + } + grids.push(Grid::new()); + Some(true) + } + "end" => { + if let Some(g) = grids.pop() { + cells_total = cells_total.saturating_add(g.padded_cells()); + if cells_total > MAX_CELLS { + return Err(too_many_cells(i..i + 1 + word.len())); + } + } + Some(false) + } + "left" => Some(true), + "right" => Some(false), _ => None, } } + '&' | '@' => { + // `@` makes an arrow cell and an empty cell beside it in a `CD` diagram. + if let Some(g) = grids.last_mut() { + g.cells += if c == '@' { 2 } else { 1 }; + } + None + } '%' => { // A comment runs to the end of the line. for (_, n) in chars.by_ref() { @@ -277,9 +375,39 @@ fn check_caps(latex: &str) -> Result<(), MathError> { None => {} } } + // An environment left open is a syntax error for the parser to report, but it still counts. + for g in grids { + cells_total = cells_total.saturating_add(g.padded_cells()); + } + if cells_total > MAX_CELLS { + return Err(too_many_cells(latex.len()..latex.len())); + } Ok(()) } +/// Most column pairs `\begin{alignedat}{N}` may ask for. RaTeX allocates 2N column specs before +/// it lays anything out. +const MAX_ALIGNAT_PAIRS: usize = 50; + +/// N of `{alignat}{N}`, `{alignat*}{N}` or `{alignedat}{N}` at the start of `rest` (just after +/// `\begin`); `usize::MAX` when the count is present but too long to read. +fn alignat_columns(rest: &str) -> Option { + let rest = rest.trim_start().strip_prefix('{')?; + let (name, after) = rest.split_once('}')?; + if !matches!(name, "alignat" | "alignat*" | "alignedat") { + return None; + } + let digits = after.trim_start().strip_prefix('{')?.split('}').next()?.trim(); + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + Some(digits.parse().unwrap_or(usize::MAX)) +} + +fn too_many_cells(span: Range) -> MathError { + MathError::new(format!("An equation can have at most {MAX_CELLS} matrix cells (rows times the widest row)."), Some(span)) +} + fn color(c: &Color) -> Option<[u8; 3]> { let q = |v: f32| (v.clamp(0.0, 1.0) * 255.0).round() as u8; let rgb = [q(c.r), q(c.g), q(c.b)]; @@ -509,6 +637,96 @@ mod tests { } } + /// Runs `f` on its own thread and fails if it has not returned within `secs` seconds. + fn within(secs: u64, f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .stack_size(2 << 20) + .spawn(move || { + let _ = tx.send(f()); + }) + .unwrap(); + match rx.recv_timeout(std::time::Duration::from_secs(secs)) { + Ok(v) => v, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => panic!("did not return within {secs} s"), + Err(e) => panic!("the worker thread died: {e}"), + } + } + + #[test] + fn macro_definitions_are_refused_before_the_expander_runs() { + let defs = [ + "\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\gdef\\a#1{\\a{#1#1#1#1}}\\a{x}", + "\\edef\\a#1{\\a{#1#1}}\\a{x}", + "\\xdef\\a#1{{#1}{#1}}\\a{x}", + "\\newcommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\renewcommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\providecommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\let\\a\\frac \\a{1}{2}", + "\\futurelet\\a\\frac \\a{1}{2}", + "\\global\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\long\\def\\a#1{\\a{#1#1}}\\a{x}", + "{\\def\\a#1{\\a{#1#1}}}\\a{x}", + ]; + for src in defs { + // `check_caps` alone, so this cannot hang even if the refusal regresses. + let e = check_caps(src).unwrap_err(); + assert!(e.message.contains("Macro definitions"), "{src}: {e}"); + assert_eq!(e.span.as_ref().map(|r| &src[r.clone()]).map(|w| w.starts_with('\\')), Some(true)); + let s = src.to_string(); + let (c, l, o) = within(10, move || (check(&s).is_err(), layout(&s).is_err(), to_omml(&s, 11.0, None).is_err())); + assert!(c && l && o, "{src}"); + } + // Names that merely start with a definer, escaped backslashes and comments are not macros. + for ok in ["\\default", "\\letter", "\\\\def", "x % \\def\\a\n+1", "\\longrightarrow"] { + assert!(check_caps(ok).is_ok(), "{ok}"); + } + assert!(layout("\\longrightarrow x").is_ok()); + // Text the lexer reads differently from a plain scan must not hide a definition: a + // verbatim `%` is no comment, a `%` in a link address is no comment, `@` joins a word. + for src in [ + "\\verb|%|\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\verb\\x\\\\def\\a#1{\\a{#1#1}}\\a{x}", + "%\\verb|\n\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\href{a%b}{x}\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\url{a%}\\def\\a#1{\\a{#1#1}}\\a{x}", + ] { + assert!(check_caps(src).is_err(), "{src}"); + } + assert!(check_caps("\\verb|\\def| x").is_ok(), "verbatim text is not a definition"); + assert!(check_caps("\\def@x").is_ok(), "`\\def@x` is one unknown word to the lexer"); + } + + #[test] + fn ragged_matrices_are_capped_by_their_padded_size() { + let ragged = format!("\\begin{{matrix}}{}{}\\end{{matrix}}", "a\\\\".repeat(650), "a&".repeat(999) + "a"); + assert!(ragged.len() < MAX_SOURCE_BYTES); + let e = check_caps(&ragged).unwrap_err(); + assert!(e.message.contains("matrix cells"), "{e}"); + let r = ragged.clone(); + let (l, o) = within(10, move || (layout(&r).unwrap_err(), to_omml(&r, 11.0, None).unwrap_err())); + assert!(l.message.contains("matrix cells") && o.message.contains("matrix cells")); + // Without the environment words there is no grid to count; the parser rejects it. + // Ordinary and wide-but-regular matrices still work, and so does a trailing `\\`. + let wide = format!("\\begin{{pmatrix}}{}\\end{{pmatrix}}", vec!["a&".repeat(39) + "a"; 40].join("\\\\")); + assert!(wide.len() < MAX_SOURCE_BYTES); + assert!(layout(&wide).is_ok()); + assert!(to_omml(&wide, 11.0, None).is_ok()); + assert!(layout("\\begin{pmatrix} a & b \\\\ c & d \\\\ \\end{pmatrix}").is_ok()); + // Diagram arrows and `alignedat` column counts size a grid without any `&`. + let cd = format!("\\begin{{CD}}{}{}\\end{{CD}}", "A\\\\".repeat(900), "@>>>".repeat(300)); + assert!(cd.len() < MAX_SOURCE_BYTES); + assert!(check_caps(&cd).is_err_and(|e| e.message.contains("matrix cells"))); + let r = "\\begin{alignedat}{99999999}a\\end{alignedat}"; + assert!(check_caps(r).is_err_and(|e| e.message.contains("matrix cells"))); + assert!(check_caps("\\begin{alignedat}{2}a&b&c&d\\end{alignedat}").is_ok()); + assert!(layout("\\begin{CD}A @>f>> B \\\\ @VVV @VVV \\\\ C @>>> D\\end{CD}").is_ok()); + assert!(check_caps("\\operatorname{curl} 50\\%").is_ok()); + let two = format!("{0}{0}", "\\begin{matrix}a&b\\\\c&d\\end{matrix}"); + assert!(check_caps(&two).is_ok()); + } + #[test] fn an_injected_panic_is_an_error_not_a_crash() { INJECT_PANIC.with(|p| p.set(true)); diff --git a/crates/math/src/omml.rs b/crates/math/src/omml.rs index dc0141aed..7d5c66835 100644 --- a/crates/math/src/omml.rs +++ b/crates/math/src/omml.rs @@ -406,6 +406,9 @@ impl Writer { } ParseNode::Array { body, is_cd: None | Some(false), .. } => { let cols = body.iter().map(Vec::len).max().unwrap_or(0).max(1); + if body.len().saturating_mul(cols) > crate::MAX_CELLS { + return Err(MathError::new(format!("An equation can have at most {} matrix cells.", crate::MAX_CELLS), None)); + } self.out.push_str(&format!( "" )); diff --git a/docs/decisions.md b/docs/decisions.md index 09bc9b5cc..7ccc445c8 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -95,7 +95,10 @@ LaTeX subset: access. Indexing is not counted; the fuzz run covers it. - **Fuzzing.** 70,000 random token strings and 50,000 mutations of valid equations: no panic, no input slower than 5 ms. Self-referencing and exponential macros stop at the parser's - 1,000-expansion limit. Nesting deeper than 32 is refused ("Recursion limit exceeded"), and + 1,000-expansion limit, which counts expansions and not their size: a definition whose + argument doubles on every call never reaches it. The wrapper therefore refuses macro + definitions (`\def`, `\newcommand`, `\let` and the like) before RaTeX sees the source, and + caps the padded size of matrices at 2,500 cells. Nesting deeper than 32 is refused ("Recursion limit exceeded"), and the deepest accepted input needs under 256 KB of stack in a release build, well inside the smallest main-thread stack we run on (1 MB on Windows). Absurd sizes (`\rule{1e308em}{1em}`) lay out with huge dimensions, so the wrapper caps them. From 1ff10df55756d3823bcf91e3d3fc31462b6276b4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 20:56:00 +0100 Subject: [PATCH 21/51] fix(autosum): stop the guessed range at a blank cell (#3291) The single-cell AutoSum guess stepped over one blank cell when a number sat above it, so a second block of numbers below a gap was summed together with the first block. It now stops at the first non-number, as Excel does. Co-Authored-By: Claude Sonnet 5.5 --- crates/engine/src/cmd/formulas.rs | 5 +---- crates/engine/src/tests.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/formulas.rs b/crates/engine/src/cmd/formulas.rs index d72f814a1..b38571d64 100644 --- a/crates/engine/src/cmd/formulas.rs +++ b/crates/engine/src/cmd/formulas.rs @@ -179,10 +179,7 @@ fn guessed_range(sh: &gridcraft_model::sheet::Sheet, at: CellRef) -> Option 0 && (numeric(CellRef::new(at.row - 1, at.col)) || at.col == 0 || !numeric(CellRef::new(at.row, at.col - 1))) { let mut top = at.row - 1; - while top > 0 - && (numeric(CellRef::new(top - 1, at.col)) - || sh.value(CellRef::new(top - 1, at.col)).is_empty() && top > 1 && numeric(CellRef::new(top - 2, at.col))) - { + while top > 0 && numeric(CellRef::new(top - 1, at.col)) { top -= 1; } if numeric(CellRef::new(at.row - 1, at.col)) { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index fe42b6977..0fccb81e1 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -2067,6 +2067,19 @@ fn formula_at(s: &Session, a: &str) -> Option { s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse(a).unwrap()).and_then(|c| c.formula.as_ref().map(|f| f.text.clone())) } +#[test] +fn autosum_of_a_single_cell_stops_at_a_blank_cell() { + // #3291: a blank row between two blocks of numbers ended the guess one block too late, so the + // second total summed the first block (and its total) as well. + let mut s = s(); + s.execute("range.setValues", json!({"range": "B4", "values": [[1], [2], [3]]})).unwrap(); + s.execute("cell.set", json!({"cell": "B7", "input": "=SUM(B4:B6)"})).unwrap(); + s.execute("range.setValues", json!({"range": "B9", "values": [[10], [20]]})).unwrap(); + s.execute("selection.set", json!({"cell": "B11"})).unwrap(); + s.execute("formulas.autoSum", json!({})).unwrap(); + assert_eq!(v(&s, "B11"), Value::Number(30.0)); +} + #[test] fn autosum_over_a_row_totals_it_to_the_right() { // As in Excel: a row selected and AutoSum puts one total to its right, not a one-cell total From 22c06b7a4a394334e0d35f8880da2e9ce7d830e6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 23:49:52 +0100 Subject: [PATCH 22/51] BINOM.DIST, POISSON.DIST: Loader mass, recurrence sums, exact prefactors (#3324) The mass came from ln_gamma differences (relative error 1.1e-13 at n = 100, 1.5e-11 at n = 10,000) and the cumulative forms summed up to 10,000 of them. The mass now comes from Loader's saddle-point formula, the cumulative sum walks away from the mode by the ratio of neighbouring terms and stops when terms stop mattering, and the incomplete beta / gamma branches take the same mass as their prefactor. Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/special.rs | 36 +++- crates/functions/src/stat.rs | 5 +- crates/functions/src/stat/distributions.rs | 191 +++++++++++++++-- crates/functions/src/stat/tests.rs | 230 +++++++++++++++++++++ 4 files changed, 436 insertions(+), 26 deletions(-) diff --git a/crates/functions/src/special.rs b/crates/functions/src/special.rs index 1541cd8a1..b173ff765 100644 --- a/crates/functions/src/special.rs +++ b/crates/functions/src/special.rs @@ -88,11 +88,17 @@ pub(crate) fn reg_gamma_p(a: f64, x: f64) -> f64 { if x.is_infinite() { return 1.0; } - if x < a + 1.0 { gamma_series(a, x) } else { 1.0 - gamma_cf(a, x) } + if x < a + 1.0 { gamma_series(a, x, front_of(a, x)) } else { 1.0 - gamma_cf(a, x, front_of(a, x)) } } /// Regularised upper incomplete gamma Q(a, x) = 1 − P(a, x). pub(crate) fn reg_gamma_q(a: f64, x: f64) -> f64 { + let front = if a > 0.0 && x > 0.0 && x.is_finite() { (-x + a * x.ln() - ln_gamma(a)).exp() } else { 0.0 }; + reg_gamma_q_front(a, x, front) +} + +/// Q(a, x) given the prefactor `e^-x x^a / Γ(a)` (the Poisson mass at a - 1, times x). +pub(crate) fn reg_gamma_q_front(a: f64, x: f64, front: f64) -> f64 { if a.is_nan() || a <= 0.0 || x.is_nan() || x < 0.0 || !a.is_finite() { return f64::NAN; } @@ -102,10 +108,15 @@ pub(crate) fn reg_gamma_q(a: f64, x: f64) -> f64 { if x.is_infinite() { return 0.0; } - if x < a + 1.0 { 1.0 - gamma_series(a, x) } else { gamma_cf(a, x) } + if x < a + 1.0 { 1.0 - gamma_series(a, x, front) } else { gamma_cf(a, x, front) } } -fn gamma_series(a: f64, x: f64) -> f64 { +/// `e^-x x^a / Γ(a)`. +fn front_of(a: f64, x: f64) -> f64 { + (-x + a * x.ln() - ln_gamma(a)).exp() +} + +fn gamma_series(a: f64, x: f64, front: f64) -> f64 { let mut ap = a; let mut sum = 1.0 / a; let mut del = sum; @@ -117,10 +128,10 @@ fn gamma_series(a: f64, x: f64) -> f64 { break; } } - sum * (-x + a * x.ln() - ln_gamma(a)).exp() + sum * front } -fn gamma_cf(a: f64, x: f64) -> f64 { +fn gamma_cf(a: f64, x: f64, front: f64) -> f64 { let tiny = 1e-300; let mut b = x + 1.0 - a; let mut c = 1.0 / tiny; @@ -144,7 +155,7 @@ fn gamma_cf(a: f64, x: f64) -> f64 { break; } } - (-x + a * x.ln() - ln_gamma(a)).exp() * h + front * h } /// Error function. @@ -187,6 +198,17 @@ pub(crate) fn erfc(x: f64) -> f64 { /// Regularised incomplete beta I_x(a, b). pub(crate) fn reg_inc_beta(x: f64, a: f64, b: f64) -> f64 { + if x > 0.0 && x < 1.0 && a > 0.0 && b > 0.0 { + let ln_bt = -ln_beta(a, b) + a * x.ln() + b * (1.0 - x).ln(); + return reg_inc_beta_front(x, a, b, ln_bt.exp()); + } + reg_inc_beta_front(x, a, b, 0.0) +} + +/// I_x(a, b) given the prefactor `bt = x^a (1-x)^b / B(a, b)` (symmetric in the two sides). A +/// caller that knows `bt` more accurately than the log-gamma route gives (the binomial mass, for +/// instance) passes it here. +pub(crate) fn reg_inc_beta_front(x: f64, a: f64, b: f64, bt: f64) -> f64 { if a.is_nan() || a <= 0.0 || b.is_nan() || b <= 0.0 || !(0.0..=1.0).contains(&x) { return f64::NAN; } @@ -196,8 +218,6 @@ pub(crate) fn reg_inc_beta(x: f64, a: f64, b: f64) -> f64 { if x == 1.0 { return 1.0; } - let ln_bt = -ln_beta(a, b) + a * x.ln() + b * (1.0 - x).ln(); - let bt = ln_bt.exp(); if x < (a + 1.0) / (a + b + 2.0) { bt * beta_cf(a, b, x) / a } else { 1.0 - bt * beta_cf(b, a, 1.0 - x) / b } } diff --git a/crates/functions/src/stat.rs b/crates/functions/src/stat.rs index 270bd2373..790915c95 100644 --- a/crates/functions/src/stat.rs +++ b/crates/functions/src/stat.rs @@ -3,7 +3,10 @@ use gridcraft_core::{CellError, Value}; -use crate::special::{erf, gamma, invert, ln_beta, ln_gamma, norm_cdf, norm_inv, norm_pdf, reg_gamma_p, reg_gamma_q, reg_inc_beta}; +use crate::special::{ + erf, gamma, invert, ln_beta, ln_gamma, norm_cdf, norm_inv, norm_pdf, reg_gamma_p, reg_gamma_q, reg_gamma_q_front, reg_inc_beta, + reg_inc_beta_front, +}; use crate::util::{A, R, S, arg, array_numbers, array_val, as_array, boolean, has, num, num_val, numbers, numbers_a, opt_num}; use crate::{Arg, Ctx, FnSpec, VAR}; diff --git a/crates/functions/src/stat/distributions.rs b/crates/functions/src/stat/distributions.rs index a3964ad80..7fa486e93 100644 --- a/crates/functions/src/stat/distributions.rs +++ b/crates/functions/src/stat/distributions.rs @@ -1,7 +1,7 @@ use super::descriptive::{arg_nums, arg_pairs, mean, var_s}; use super::{ Arg, C, CellError, R, Value, arg, as_array, boolean, erf, gamma, has, invert, ln_beta, ln_gamma, norm_cdf, norm_inv, norm_pdf, num, num_val, - reg_gamma_p, reg_gamma_q, reg_inc_beta, + reg_gamma_p, reg_gamma_q, reg_gamma_q_front, reg_inc_beta, reg_inc_beta_front, }; // --------------------------------------------------------------------------------------------- @@ -93,6 +93,95 @@ fn f_pdf(x: f64, d1: f64, d2: f64) -> f64 { const MAX_BINOM_TRIALS: f64 = 2_147_483_646.0; +/// Stirling-series error `ln n! - ((n + 1/2) ln n - n + ln(2π)/2)` for integer `n >= 1`: exact +/// table (computed with 60-digit Decimal arithmetic) up to 30, then the series, whose first +/// omitted term is below 1e-19 there. +fn stirlerr(n: f64) -> f64 { + const TABLE: [f64; 31] = [ + 0.0, + 0.08106146679532726, + 0.0413406959554093, + 0.02767792568499834, + 0.020790672103765093, + 0.016644691189821193, + 0.013876128823070748, + 0.01189670994589177, + 0.010411265261972096, + 0.009255462182712733, + 0.00833056343336287, + 0.007573675487951841, + 0.00694284010720953, + 0.006408994188004207, + 0.0059513701127588475, + 0.005554733551962801, + 0.0052076559196096404, + 0.004901395948434738, + 0.004629153749334028, + 0.004385560249232324, + 0.004166319691996922, + 0.00396795421864086, + 0.0037876180684444346, + 0.0036229602246830948, + 0.003472021382978767, + 0.003333155636728093, + 0.003204970228055038, + 0.0030862786826087773, + 0.002976063983550409, + 0.0028734493623524663, + 0.0027776749297526936, + ]; + if n <= 30.0 { + return TABLE.get(n as usize).copied().unwrap_or(0.0); + } + let r = 1.0 / n; + let r2 = r * r; + r * (1.0 / 12.0 - r2 * (1.0 / 360.0 - r2 * (1.0 / 1260.0 - r2 * (1.0 / 1680.0 - r2 * (1.0 / 1188.0))))) +} + +/// `x ln(x/np) + np - x` without the cancellation of the direct formula when `x` is near `np` +/// (Loader, 2000). +fn bd0(x: f64, np: f64) -> f64 { + if (x - np).abs() < 0.1 * (x + np) { + let v = (x - np) / (x + np); + let mut s = (x - np) * v; + if s.abs() < f64::MIN_POSITIVE { + return s; + } + let mut ej = 2.0 * x * v; + let v2 = v * v; + for j in 1..1000 { + ej *= v2; + let s1 = s + ej / f64::from(2 * j + 1); + if s1 == s { + return s1; + } + s = s1; + } + } + x * (x / np).ln() + np - x +} + +/// `(1 - p)^e` with the rounding error of `1 - p` (it is not exact for `p < 1/2`) folded back in, +/// which `e` would otherwise multiply. +fn pow_one_minus(p: f64, e: f64) -> f64 { + let hi = 1.0 - p; + let lo = (1.0 - hi) - p; + hi.powf(e) * (e * (lo / hi).ln_1p()).exp() +} + +/// `C(n, k)` for a small `k` as a product of `k` ratios. +fn choose_small(n: f64, k: f64) -> f64 { + let mut c = 1.0; + let mut j = 0.0; + while j < k { + c *= (n - j) / (j + 1.0); + j += 1.0; + } + c +} + +/// Binomial mass at integer `k`, `0 <= k <= n`, by Loader's saddle-point formula: no log-gamma +/// of large numbers, so the relative error stays near 1e-15 for any `n`. fn binom_pmf(k: f64, n: f64, p: f64) -> f64 { if k < 0.0 || k > n { return 0.0; @@ -103,7 +192,60 @@ fn binom_pmf(k: f64, n: f64, p: f64) -> f64 { if p == 1.0 { return if k == n { 1.0 } else { 0.0 }; } - (ln_comb(n, k) + k * p.ln() + (n - k) * (1.0 - p).ln()).exp() + if k == 0.0 { + return if n == 0.0 { 1.0 } else { pow_one_minus(p, n) }; + } + if k == n { + return p.powf(n); + } + // A few successes or a few failures: the product form has no large cancelling terms. + if k < 16.0 { + return choose_small(n, k) * p.powi(k as i32) * pow_one_minus(p, n - k); + } + if n - k < 16.0 { + return choose_small(n, n - k) * p.powf(k) * pow_one_minus(p, n - k); + } + let q = 1.0 - p; + let lc = stirlerr(n) - stirlerr(k) - stirlerr(n - k) - bd0(k, n * p) - bd0(n - k, n * q); + let lf = std::f64::consts::TAU.ln() + k.ln() + (-k / n).ln_1p(); + (lc - 0.5 * lf).exp() +} + +/// Sum of the binomial masses 0..=k for `n <= 10_000`: starts at the term nearest the mode +/// (accurate by the formula above) and walks away from it by the ratio of neighbouring masses, +/// one multiply per term, stopping once the terms no longer matter. Above the mode it sums the +/// upper tail instead and subtracts it from 1. +fn binom_cdf_sum(k: f64, n: f64, p: f64) -> f64 { + let q = 1.0 - p; + if p == 0.0 { + return 1.0; + } + if p == 1.0 { + return 0.0; + } + if k <= n * p { + // mass(i - 1) = mass(i) * i / (n - i + 1) * q / p + let mut t = binom_pmf(k, n, p); + let mut s = t; + let mut i = k; + while i >= 1.0 && t > s * 1e-18 { + t *= i / (n - i + 1.0) * (q / p); + s += t; + i -= 1.0; + } + s.min(1.0) + } else { + // mass(i + 1) = mass(i) * (n - i) / (i + 1) * p / q + let mut t = binom_pmf(k + 1.0, n, p); + let mut s = t; + let mut i = k + 1.0; + while i < n && t > s * 1e-18 { + t *= (n - i) / (i + 1.0) * (p / q); + s += t; + i += 1.0; + } + (1.0 - s).max(0.0) + } } fn binom_cdf(k: f64, n: f64, p: f64) -> f64 { @@ -114,13 +256,7 @@ fn binom_cdf(k: f64, n: f64, p: f64) -> f64 { return 1.0; } if n <= 10_000.0 { - let mut s = 0.0; - let mut i = 0.0; - while i <= k { - s += binom_pmf(i, n, p); - i += 1.0; - } - return s.min(1.0); + return binom_cdf_sum(k, n, p); } if p == 0.0 { return 1.0; @@ -128,14 +264,19 @@ fn binom_cdf(k: f64, n: f64, p: f64) -> f64 { if p == 1.0 { return 0.0; } - reg_inc_beta(1.0 - p, n - k, k + 1.0) + // I_(1-p)(n-k, k+1) has prefactor (n-k) p mass(k). + reg_inc_beta_front(1.0 - p, n - k, k + 1.0, (n - k) * p * binom_pmf(k, n, p)) } +/// Poisson mass at integer `x >= 0` by Loader's formula. fn poisson_pmf(x: f64, m: f64) -> f64 { if m == 0.0 { return if x == 0.0 { 1.0 } else { 0.0 }; } - (-m + x * m.ln() - ln_gamma(x + 1.0)).exp() + if x == 0.0 { + return (-m).exp(); + } + (-stirlerr(x) - bd0(x, m)).exp() / (std::f64::consts::TAU * x).sqrt() } fn poisson_cdf(x: f64, m: f64) -> f64 { @@ -143,15 +284,31 @@ fn poisson_cdf(x: f64, m: f64) -> f64 { return 1.0; } if x <= 1000.0 { - let mut s = 0.0; - let mut i = 0.0; - while i <= x { - s += poisson_pmf(i, m); + if x <= m { + // mass(i - 1) = mass(i) * i / m, downwards from x while the terms matter + let mut t = poisson_pmf(x, m); + let mut s = t; + let mut i = x; + while i >= 1.0 && t > s * 1e-18 { + t *= i / m; + s += t; + i -= 1.0; + } + return s.min(1.0); + } + // Above the mean: 1 minus the upper tail, mass(i + 1) = mass(i) * m / (i + 1). + let mut t = poisson_pmf(x + 1.0, m); + let mut s = t; + let mut i = x + 1.0; + while t > s * 1e-18 { + t *= m / (i + 1.0); + s += t; i += 1.0; } - return s.min(1.0); + return (1.0 - s).max(0.0); } - reg_gamma_q(x + 1.0, m) + // Q(x+1, m) has prefactor m mass(x). + reg_gamma_q_front(x + 1.0, m, m * poisson_pmf(x, m)) } pub(super) fn norm_dist(a: &[Arg], _: C) -> R { diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index ba59116cd..f8a8c1890 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -617,3 +617,233 @@ fn aggregate_helper() { close(aggregate_values_k(9, &vals, &nv(2.0)), 9.0); is_err(aggregate_values_k(14, &vals, &nv(10.0)), CellError::Num); } + +// B1: BINOM.DIST and POISSON.DIST went through ln_gamma differences (relative error 1.1e-13 at +// n = 100, 1.5e-11 at n = 10,000). Reference values: 70-digit Decimal arithmetic on the exact +// double `p` (mass by the neighbour-ratio recurrence from q^n, cdf as the running sum), rounded +// once to f64. Columns: k, n, p, mass, cdf. +#[rustfmt::skip] +const BINOM_REF: &[(f64, f64, f64, f64, f64)] = &[ + (0.0, 10.0, 0.5, 0.0009765625, 0.0009765625), + (1.0, 10.0, 0.5, 0.009765625, 0.0107421875), + (3.0, 10.0, 0.5, 0.1171875, 0.171875), + (5.0, 10.0, 0.5, 0.24609375, 0.623046875), + (7.0, 10.0, 0.5, 0.1171875, 0.9453125), + (9.0, 10.0, 0.5, 0.009765625, 0.9990234375), + (0.0, 10.0, 0.3, 0.028247524900000005, 0.028247524900000005), + (1.0, 10.0, 0.3, 0.12106082100000001, 0.14930834590000003), + (2.0, 10.0, 0.3, 0.23347444050000002, 0.3827827864), + (3.0, 10.0, 0.3, 0.266827932, 0.6496107184000001), + (4.0, 10.0, 0.3, 0.200120949, 0.8497316674000001), + (5.0, 10.0, 0.3, 0.10291934519999998, 0.9526510126), + (7.0, 10.0, 0.3, 0.009001691999999999, 0.9984096136), + (9.0, 10.0, 0.3, 0.00013778099999999996, 0.9999940951), + (0.0, 10.0, 0.999, 1.0000000000000088e-30, 1.0000000000000088e-30), + (1.0, 10.0, 0.999, 9.99000000000008e-27, 9.99100000000008e-27), + (5.0, 10.0, 0.999, 2.507425174812609e-13, 2.50951798425701e-13), + (9.0, 10.0, 0.999, 0.00991035916125875, 0.009955119790251798), + (0.0, 100.0, 0.5, 7.888609052210118e-31, 7.888609052210118e-31), + (1.0, 100.0, 0.5, 7.888609052210118e-29, 7.967495142732219e-29), + (35.0, 100.0, 0.5, 0.0008638556657416528, 0.0017588208614850792), + (45.0, 100.0, 0.5, 0.048474296626430755, 0.18410080866334813), + (50.0, 100.0, 0.5, 0.07958923738717877, 0.5397946186935894), + (55.0, 100.0, 0.5, 0.048474296626430755, 0.8643734879630827), + (65.0, 100.0, 0.5, 0.0008638556657416528, 0.9991050348042566), + (99.0, 100.0, 0.5, 7.888609052210118e-29, 1.0), + (0.0, 100.0, 0.01, 0.3660323412732295, 0.3660323412732295), + (1.0, 100.0, 0.01, 0.3697296376497268, 0.7357619789229563), + (2.0, 100.0, 0.01, 0.1848648188248634, 0.9206267977478196), + (4.0, 100.0, 0.01, 0.014941714856895142, 0.9965676784122455), + (50.0, 100.0, 0.01, 6.103987557173001e-72, 1.0), + (99.0, 100.0, 0.01, 9.900000000000021e-197, 1.0), + (0.0, 100.0, 0.7, 5.15377520732019e-53, 5.15377520732019e-53), + (1.0, 100.0, 0.7, 1.2025475483747105e-50, 1.207701323582031e-50), + (50.0, 100.0, 0.7, 1.302622713144541e-05, 2.2060913327166086e-05), + (56.0, 100.0, 0.7, 0.001028871216633018, 0.0021146172813184595), + (65.0, 100.0, 0.7, 0.04677968235272982, 0.16285828837178737), + (70.0, 100.0, 0.7, 0.08678386475342809, 0.5376602639846381), + (75.0, 100.0, 0.7, 0.04955992276217012, 0.8864298182958186), + (84.0, 100.0, 0.7, 0.0005638655029474704, 0.9995950004580563), + (99.0, 100.0, 0.7, 1.3862042184106021e-14, 0.9999999999999997), + (0.0, 1000.0, 0.5, 9.332636185032189e-302, 9.332636185032189e-302), + (1.0, 1000.0, 0.5, 9.332636185032189e-299, 9.341968821217221e-299), + (453.0, 1000.0, 0.5, 0.00030353498700958767, 0.00162629990658169), + (484.0, 1000.0, 0.5, 0.015123665829158384, 0.1634683412157977), + (500.0, 1000.0, 0.5, 0.0252250181783608, 0.5126125090891804), + (516.0, 1000.0, 0.5, 0.015123665829158384, 0.8516553246133607), + (547.0, 1000.0, 0.5, 0.00030353498700958767, 0.9986772350804279), + (999.0, 1000.0, 0.5, 9.332636185032189e-299, 1.0), + (0.0, 1000.0, 0.3, 1.2532566399657381e-155, 1.2532566399657381e-155), + (1.0, 1000.0, 0.3, 5.371099885567449e-153, 5.383632451967106e-153), + (257.0, 1000.0, 0.3, 0.00030959977077928017, 0.0014835731103312242), + (286.0, 1000.0, 0.3, 0.0174263984921581, 0.1759288949498673), + (300.0, 1000.0, 0.3, 0.027521003821268385, 0.5155935198141203), + (314.0, 1000.0, 0.3, 0.017109129819617666, 0.8414952343056957), + (343.0, 1000.0, 0.3, 0.00036310112605077133, 0.9985016810663147), + (500.0, 1000.0, 0.3, 3.4791609391626546e-40, 1.0), + (999.0, 1000.0, 0.3, 0.0, 1.0), + (0.0, 1000.0, 0.001, 0.36769542477096406, 0.36769542477096406), + (1.0, 1000.0, 0.001, 0.36806348825922325, 0.7357589130301873), + (2.0, 1000.0, 0.001, 0.18403174412961162, 0.9197906571597989), + (4.0, 1000.0, 0.001, 0.01528995542083862, 0.9963631219704782), + (500.0, 1000.0, 0.001, 0.0, 1.0), + (999.0, 1000.0, 0.001, 0.0, 1.0), + (0.0, 10000.0, 0.5, 0.0, 0.0), + (1.0, 10000.0, 0.5, 0.0, 0.0), + (4850.0, 10000.0, 0.5, 8.861480551702583e-05, 0.0013939837588563594), + (4950.0, 10000.0, 0.5, 0.004839495142316416, 0.161087099897656), + (5000.0, 10000.0, 0.5, 0.007978646139382154, 0.5039893230696911), + (5050.0, 10000.0, 0.5, 0.004839495142316416, 0.8437523952446604), + (5150.0, 10000.0, 0.5, 8.861480551702583e-05, 0.9986946310466607), + (9999.0, 10000.0, 0.5, 0.0, 1.0), + (0.0, 10000.0, 0.3, 0.0, 0.0), + (1.0, 10000.0, 0.3, 0.0, 0.0), + (2863.0, 10000.0, 0.3, 9.71726742023389e-05, 0.0013927715709303387), + (2954.0, 10000.0, 0.3, 0.0052756073908582085, 0.16038574844431697), + (3000.0, 10000.0, 0.3, 0.008705361365067056, 0.504932983819298), + (3046.0, 10000.0, 0.3, 0.005244996814931252, 0.8448693061622872), + (3137.0, 10000.0, 0.3, 0.00010232641238302388, 0.9986013755801396), + (5000.0, 10000.0, 0.3, 0.0, 1.0), + (9999.0, 10000.0, 0.3, 0.0, 1.0), + (0.0, 10000.0, 0.0001, 0.3678610464329299, 0.3678610464329299), + (1.0, 10000.0, 0.0001, 0.3678978362165516, 0.7357588826494815), + (2.0, 10000.0, 0.0001, 0.1839489181082758, 0.9197078007577573), + (4.0, 10000.0, 0.0001, 0.015324477632794846, 0.996342452182816), + (5000.0, 10000.0, 0.0001, 0.0, 1.0), + (9999.0, 10000.0, 0.0001, 0.0, 1.0), + (0.0, 10000.0, 0.999, 0.0, 0.0), + (1.0, 10000.0, 0.999, 0.0, 0.0), + (5000.0, 10000.0, 0.999, 0.0, 0.0), + (9981.0, 10000.0, 0.999, 0.0037205945263045544, 0.007158161342006954), + (9987.0, 10000.0, 0.999, 0.07292252318084555, 0.20834867055478343), + (9990.0, 10000.0, 0.999, 0.12517263665023892, 0.5421328760209826), + (9993.0, 10000.0, 0.999, 0.09007019224850273, 0.8699847268232276), + (9999.0, 10000.0, 0.999, 0.00045218564541589875, 0.9999548266540229), + (0.0, 100000.0, 0.5, 0.0, 0.0), + (1.0, 100000.0, 0.5, 0.0, 0.0), + (49526.0, 100000.0, 0.5, 2.821098952769392e-05, 0.00137358872189177), + (49842.0, 100000.0, 0.5, 0.0015314620321580088, 0.15959653953914368), + (50000.0, 100000.0, 0.5, 0.00252312621419674, 0.5012615631070984), + (50158.0, 100000.0, 0.5, 0.0015314620321580088, 0.8419349224930143), + (50474.0, 100000.0, 0.5, 2.821098952769392e-05, 0.9986546222676359), + (99999.0, 100000.0, 0.5, 0.0, 1.0), + (0.0, 100000.0, 0.3, 0.0, 0.0), + (1.0, 100000.0, 0.3, 0.0, 0.0), + (29565.0, 100000.0, 0.3, 3.016598258904529e-05, 0.0013408650955754554), + (29855.0, 100000.0, 0.3, 0.0016703029966719045, 0.15934769308464344), + (30000.0, 100000.0, 0.3, 0.002752954648397428, 0.5015600059172672), + (30145.0, 100000.0, 0.3, 0.0016672321665529399, 0.8423208126454166), + (30435.0, 100000.0, 0.3, 3.067097885268188e-05, 0.9986570625478413), + (50000.0, 100000.0, 0.3, 0.0, 1.0), + (99999.0, 100000.0, 0.3, 0.0, 1.0), + (0.0, 100000.0, 0.99, 0.0, 0.0), + (1.0, 100000.0, 0.99, 0.0, 0.0), + (50000.0, 100000.0, 0.99, 0.0, 0.0), + (98906.0, 100000.0, 0.99, 0.00015962501729101418, 0.001680685850467731), + (98969.0, 100000.0, 0.99, 0.007723834924565287, 0.1660975153530963), + (99000.0, 100000.0, 0.99, 0.012678161323544589, 0.5042686879721229), + (99031.0, 100000.0, 0.99, 0.007885853372194001, 0.8416321140066325), + (99094.0, 100000.0, 0.99, 0.00013276363518518057, 0.9988439452168577), + (99999.0, 100000.0, 0.99, 0.0, 1.0), +]; + +// Columns: x, mean, mass, cdf. The last block has x > 1000, which takes the incomplete-gamma branch. +#[rustfmt::skip] +const POISSON_REF: &[(f64, f64, f64, f64)] = &[ + (0.0, 0.5, 0.6065306597126334, 0.6065306597126334), + (1.0, 0.5, 0.3032653298563167, 0.9097959895689501), + (3.0, 0.5, 0.012636055410679864, 0.9982483774437092), + (0.0, 5.0, 0.006737946999085467, 0.006737946999085467), + (2.0, 5.0, 0.08422433748856833, 0.12465201948308115), + (5.0, 5.0, 0.1754673697678507, 0.6159606548330632), + (10.0, 5.0, 0.018132788707821874, 0.986304731401617), + (25.0, 5.0, 1.294589320478486e-10, 0.9999999999695003), + (70.0, 100.0, 0.00031056089812001654, 0.0009714440282773882), + (100.0, 100.0, 0.039860996809147134, 0.5265621985299984), + (130.0, 100.0, 0.000575252683815463, 0.9982931596294985), + (900.0, 1000.0, 7.516954352125952e-05, 0.0006977673277963068), + (1000.0, 1000.0, 0.012614611348721499, 0.508409367168506), + (1100.0, 1000.0, 9.498944242299507e-05, 0.9991323590365564), + (1050.0, 1000.0, 0.003599490404803648, 0.9439711616363267), + (1200.0, 1000.0, 7.992642848843571e-11, 0.9999999996115061), + (1500.0, 2000.0, 1.878037542328086e-32, 7.468034373687017e-32), + (2000.0, 2000.0, 0.00892024889598624, 0.5059467004517837), + (2100.0, 2000.0, 0.0007442234118916615, 0.9872092799080792), + (2300.0, 2000.0, 4.011826753210622e-12, 0.999999999973895), + (4800.0, 5000.0, 9.987733294132511e-05, 0.002269403878343364), + (5000.0, 5000.0, 0.005641801804664022, 0.5037611677708467), +]; + +fn rel_err(got: Value, want: f64) -> f64 { + let g = got.as_f64().unwrap_or(f64::NAN); + if want == 0.0 { g.abs() } else { ((g - want) / want).abs() } +} + +/// Error bound the grid below must meet: 1e-13, plus 2e-16 per unit of |ln value| for far-tail +/// results (an exp of a sum of large logs cannot do better in double precision). Measured: at +/// most 9e-14 on the grid (the n = 100,000 cumulative cases), 2e-14 for n <= 10,000 away from the +/// far tails, 2e-15 on the Excel-sized central cases. The log-gamma code was off by 1.5e-11. +fn allowed(want: f64) -> f64 { + 1e-13 + 2e-16 * want.abs().ln().abs() +} + +#[test] +fn binom_and_poisson_match_exact_decimal_references() { + // Worst error relative to the allowance (above 1 is a failure) and plain worst error. + let (mut worst, mut worst_plain) = (0.0f64, 0.0f64); + let mut check = |what: &str, got: Value, want: f64| { + let e = rel_err(got, want); + let ratio = e / allowed(want); + if ratio > 1.0 { + println!("{what}: error {e:e}, allowed {:e}", allowed(want)); + } + worst = worst.max(ratio); + if want > 1e-3 { + worst_plain = worst_plain.max(e); + } + }; + for &(k, nn, p, pmf, cdf) in BINOM_REF { + check(&format!("BINOM.DIST({k},{nn},{p},FALSE)"), ev("BINOM.DIST", vec![n(k), n(nn), n(p), b(false)]), pmf); + check(&format!("BINOM.DIST({k},{nn},{p},TRUE)"), ev("BINOM.DIST", vec![n(k), n(nn), n(p), b(true)]), cdf); + } + for &(x, m, pmf, cdf) in POISSON_REF { + check(&format!("POISSON.DIST({x},{m},FALSE)"), ev("POISSON.DIST", vec![n(x), n(m), b(false)]), pmf); + check(&format!("POISSON.DIST({x},{m},TRUE)"), ev("POISSON.DIST", vec![n(x), n(m), b(true)]), cdf); + } + println!("worst error / allowance {worst:.3}; worst relative error where the result is above 1e-3: {worst_plain:e}"); + assert!(worst <= 1.0); + assert!(worst_plain < 1e-13); +} + +#[test] +fn binom_and_poisson_edge_cases() { + for cum in [false, true] { + let pm = |k: f64, nn: f64, p: f64| ev("BINOM.DIST", vec![n(k), n(nn), n(p), b(cum)]); + // p = 0 and p = 1 + close(pm(0.0, 50.0, 0.0), 1.0); + close(pm(3.0, 50.0, 1.0), 0.0); + close(pm(50.0, 50.0, 1.0), 1.0); + // n = 0 + close(pm(0.0, 0.0, 0.3), 1.0); + // k = n is certain when cumulative and p^n otherwise + close(pm(20.0, 20.0, 0.5), if cum { 1.0 } else { 0.5f64.powi(20) }); + close_tol(pm(10_000.0, 10_000.0, 0.999), if cum { 1.0 } else { 0.999f64.powi(10_000) }, 1e-12); + // k = 0 is q^n + close_tol(pm(0.0, 1000.0, 0.001), 0.999f64.powi(1000), 1e-12); + close_tol(pm(0.0, 100_000.0, 0.00001), (1.0f64 - 1e-5).powi(100_000), 1e-9); + } + close(ev("POISSON.DIST", vec![n(0.0), n(0.0), b(false)]), 1.0); + close(ev("POISSON.DIST", vec![n(4.0), n(0.0), b(false)]), 0.0); + close(ev("POISSON.DIST", vec![n(4.0), n(0.0), b(true)]), 1.0); + close_tol(ev("POISSON.DIST", vec![n(0.0), n(3.0), b(true)]), (-3.0f64).exp(), 1e-15); + // Fractions truncate as before. + close(ev("BINOM.DIST", vec![n(6.9), n(10.9), n(0.5), b(false)]), 0.205078125); + // The 10,000 trial switch of the method no longer shows: neighbours agree to 1e-14. + let lo = ev("BINOM.DIST", vec![n(5000.0), n(10_000.0), n(0.5), b(true)]).as_f64().unwrap(); + let hi = ev("BINOM.DIST", vec![n(5000.0), n(10_001.0), n(0.5), b(true)]).as_f64().unwrap(); + let pmf = ev("BINOM.DIST", vec![n(5000.0), n(10_000.0), n(0.5), b(false)]).as_f64().unwrap(); + // cdf(5000; 10,001) = cdf(5000; 10,000) - p * mass(5000; 10,000) by the recurrence on n + assert!(((hi - (lo - 0.5 * pmf)) / hi).abs() < 1e-13, "{lo} {hi}"); + // Inverse stays consistent: the smallest k with cdf >= alpha. + close(ev("BINOM.INV", vec![n(10_000.0), n(0.5), n(0.5)]), 5000.0); +} From 7e8b94fc39f345d8061402c2bdbda5b998018407 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 3 Nov 2026 23:54:53 +0100 Subject: [PATCH 23/51] BINOM.DIST: keep tiny masses and huge n out of the product form Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat/distributions.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/crates/functions/src/stat/distributions.rs b/crates/functions/src/stat/distributions.rs index 7fa486e93..d2e4e3421 100644 --- a/crates/functions/src/stat/distributions.rs +++ b/crates/functions/src/stat/distributions.rs @@ -166,7 +166,9 @@ fn bd0(x: f64, np: f64) -> f64 { fn pow_one_minus(p: f64, e: f64) -> f64 { let hi = 1.0 - p; let lo = (1.0 - hi) - p; - hi.powf(e) * (e * (lo / hi).ln_1p()).exp() + let r = hi.powf(e) * (e * (lo / hi).ln_1p()).exp(); + // An exponent so large that the two factors under- and overflow: the plain form is enough. + if r.is_nan() { (e * (-p).ln_1p()).exp() } else { r } } /// `C(n, k)` for a small `k` as a product of `k` ratios. @@ -199,11 +201,15 @@ fn binom_pmf(k: f64, n: f64, p: f64) -> f64 { return p.powf(n); } // A few successes or a few failures: the product form has no large cancelling terms. - if k < 16.0 { - return choose_small(n, k) * p.powi(k as i32) * pow_one_minus(p, n - k); - } - if n - k < 16.0 { - return choose_small(n, n - k) * p.powf(k) * pow_one_minus(p, n - k); + // (Not for astronomically large n or a power of p near underflow: the saddle-point form + // below handles those through logs.) + if n < 1e30 && p.powf(k) > 1e-280 { + if k < 16.0 { + return choose_small(n, k) * p.powi(k as i32) * pow_one_minus(p, n - k); + } + if n - k < 16.0 { + return choose_small(n, n - k) * p.powf(k) * pow_one_minus(p, n - k); + } } let q = 1.0 - p; let lc = stirlerr(n) - stirlerr(k) - stirlerr(n - k) - bd0(k, n * p) - bd0(n - k, n * q); From d8ba397bb721ff16124b207ff3cedbb8bb5cc7d2 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 00:04:32 +0100 Subject: [PATCH 24/51] math: refuse an alignedat count hidden by a comment or spaces (review point: cap bypass) CodeRabbit found that \begin{alignedat}%{99999999} and {9 9} made the count parser return None, which skipped the 50-pair cap. The lookahead now drops comments and spaces as RaTeX's lexer does, and a matched alignat name with a count that is not a plain digit string is refused. Co-Authored-By: Claude Sonnet 5.5 --- crates/math/src/lib.rs | 45 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/crates/math/src/lib.rs b/crates/math/src/lib.rs index e1f783dca..62033f2b8 100644 --- a/crates/math/src/lib.rs +++ b/crates/math/src/lib.rs @@ -390,16 +390,39 @@ fn check_caps(latex: &str) -> Result<(), MathError> { const MAX_ALIGNAT_PAIRS: usize = 50; /// N of `{alignat}{N}`, `{alignat*}{N}` or `{alignedat}{N}` at the start of `rest` (just after -/// `\begin`); `usize::MAX` when the count is present but too long to read. +/// `\begin`). `usize::MAX` when the count is present but is not a plain digit string RaTeX reads +/// the same way (too long, or hidden by a comment, inner spaces or a macro): refused, not skipped. +/// +/// RaTeX reads the environment name and the count through its lexer, which drops comments and +/// spaces, so they are dropped here too before the two groups are compared. fn alignat_columns(rest: &str) -> Option { - let rest = rest.trim_start().strip_prefix('{')?; - let (name, after) = rest.split_once('}')?; + // Enough for the longest name and any count that fits `usize`; a longer count is refused. + const WINDOW: usize = 64; + let mut kept = String::new(); + let mut chars = rest.chars(); + while kept.len() < WINDOW { + match chars.next() { + None => break, + Some('%') => { + for n in chars.by_ref() { + if n == '\n' { + break; + } + } + } + Some(c) if c.is_whitespace() => {} + Some(c) => kept.push(c), + } + } + let (name, after) = kept.strip_prefix('{')?.split_once('}')?; if !matches!(name, "alignat" | "alignat*" | "alignedat") { return None; } - let digits = after.trim_start().strip_prefix('{')?.split('}').next()?.trim(); + let Some(digits) = after.strip_prefix('{').and_then(|a| a.split_once('}')).map(|(d, _)| d) else { + return Some(usize::MAX); + }; if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { - return None; + return Some(usize::MAX); } Some(digits.parse().unwrap_or(usize::MAX)) } @@ -721,6 +744,18 @@ mod tests { let r = "\\begin{alignedat}{99999999}a\\end{alignedat}"; assert!(check_caps(r).is_err_and(|e| e.message.contains("matrix cells"))); assert!(check_caps("\\begin{alignedat}{2}a&b&c&d\\end{alignedat}").is_ok()); + // A comment, inner spaces or a macro in the count do not hide it from the cap. + for hidden in [ + "\\begin{alignedat}%\n{99999999}a\\end{alignedat}", + "\\begin %c\n {alignedat}{99999999}a\\end{alignedat}", + "\\begin{alignedat}{9 9}a\\end{alignedat}", + "\\begin{alignedat}{9%x\n9}a\\end{alignedat}", + "\\begin{align%x\nedat}{99}a\\end{alignedat}", + "\\begin{alignedat}{9\\relax9}a\\end{alignedat}", + ] { + assert!(check_caps(hidden).is_err_and(|e| e.message.contains("matrix cells")), "{hidden}"); + } + assert!(check_caps("\\begin{alignedat} {2} % two pairs\na&b&c&d\\end{alignedat}").is_ok()); assert!(layout("\\begin{CD}A @>f>> B \\\\ @VVV @VVV \\\\ C @>>> D\\end{CD}").is_ok()); assert!(check_caps("\\operatorname{curl} 50\\%").is_ok()); let two = format!("{0}{0}", "\\begin{matrix}a&b\\\\c&d\\end{matrix}"); From 40e75c79127ffa545040d2d97e6cc7d596bafe9b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 00:05:58 +0100 Subject: [PATCH 25/51] binom/poisson reference test: fail on errors, NaN and non-zero results for zero references (review point) CodeRabbit pointed out that an error or NaN result passed the accuracy test and that a zero reference allowed any finite value (the bound was infinite). Errors now count as infinite error and a zero reference allows 1e-300. Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat/tests.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index f8a8c1890..67e626251 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -775,7 +775,8 @@ const POISSON_REF: &[(f64, f64, f64, f64)] = &[ ]; fn rel_err(got: Value, want: f64) -> f64 { - let g = got.as_f64().unwrap_or(f64::NAN); + // An error or a non-finite result is the worst possible miss, not a skipped one. + let Some(g) = got.as_f64().filter(|g| g.is_finite()) else { return f64::INFINITY }; if want == 0.0 { g.abs() } else { ((g - want) / want).abs() } } @@ -784,6 +785,10 @@ fn rel_err(got: Value, want: f64) -> f64 { /// most 9e-14 on the grid (the n = 100,000 cumulative cases), 2e-14 for n <= 10,000 away from the /// far tails, 2e-15 on the Excel-sized central cases. The log-gamma code was off by 1.5e-11. fn allowed(want: f64) -> f64 { + // A reference that underflows to 0 must come back as (almost) 0, not as any finite value. + if want == 0.0 { + return 1e-300; + } 1e-13 + 2e-16 * want.abs().ln().abs() } From 86c5c8dd813e05b2f9912bd3dcc4a1e9e042e5a5 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 01:31:04 +0100 Subject: [PATCH 26/51] Math: count \newline as a row break in the matrix cell cap (#3327) RaTeX reads \newline as \\, but the source scan only counted \\ and \cr, so a matrix built with \newline skipped the 2,500 cell cap. Co-Authored-By: Claude Sonnet 5.5 --- crates/math/src/lib.rs | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/math/src/lib.rs b/crates/math/src/lib.rs index 62033f2b8..3a5b310cf 100644 --- a/crates/math/src/lib.rs +++ b/crates/math/src/lib.rs @@ -235,7 +235,7 @@ const MACRO_DEFINERS: &[&str] = /// One open `\begin…\end` environment, counted from the source text. struct Grid { - /// Rows ended so far (by `\\` or `\cr`). + /// Rows ended so far (by `\\`, `\cr` or `\newline`). rows: usize, /// Cells in the current row. cells: usize, @@ -315,7 +315,8 @@ fn check_caps(latex: &str) -> Result<(), MathError> { } None } - "cr" => { + // RaTeX reads `\newline` as `\\`, so it ends a row like `\cr` does. + "cr" | "newline" => { if let Some(g) = grids.last_mut() { g.end_row(); } @@ -783,4 +784,20 @@ mod tests { let l = layout("").unwrap(); assert!(l.items.is_empty()); } + + #[test] + fn newline_rows_count_toward_the_matrix_cap() { + // RaTeX reads `\newline` as `\\`, so it ends a row like `\cr` does. + for brk in ["\\newline ", "\\cr ", "\\\\"] { + let src = format!("\\begin{{matrix}}{}{}\\end{{matrix}}", format!("a{brk}").repeat(250), "a&".repeat(700) + "a"); + assert!(src.len() < MAX_SOURCE_BYTES); + let e = check_caps(&src).unwrap_err(); + assert!(e.message.contains("matrix cells"), "{brk}: {e}"); + let s = src.clone(); + let (l, o) = within(10, move || (layout(&s).unwrap_err(), to_omml(&s, 11.0, None).unwrap_err())); + assert!(l.message.contains("matrix cells") && o.message.contains("matrix cells"), "{brk}"); + } + assert!(check_caps("\\begin{matrix}a&b\\newline c&d\\end{matrix}").is_ok()); + assert!(layout("\\begin{matrix}a&b\\newline c&d\\end{matrix}").is_ok()); + } } From b3a709f314ef7203959f23a121836868342451aa Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 02:12:09 +0100 Subject: [PATCH 27/51] contributors: set names for vangel76 Co-Authored-By: Claude Sonnet 5.5 --- contributors/people.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contributors/people.toml b/contributors/people.toml index 00f6ce76a..1b8bcf996 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -33,7 +33,8 @@ sync_github_name = false [people.vangel76] real_name = "Valentin Klein" -sync_github_name = true +display_name = "vangel76" +sync_github_name = false [people.MacJedi42] sync_github_name = true From 7e828a982c608c5c1e418065af37bc09729ea53c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 01:02:21 +0100 Subject: [PATCH 28/51] Tabs: Ctrl-click on the sheet arrows jumps to the first or last visible sheet (#3350) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/tabs.rs | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/src/tabs.rs b/crates/ui-egui/src/tabs.rs index ea11615d0..917dc3b7f 100644 --- a/crates/ui-egui/src/tabs.rs +++ b/crates/ui-egui/src/tabs.rs @@ -22,6 +22,12 @@ fn navigator_items(sheets: &[Arc]) -> Vec<(String, String)> { sheets.iter().enumerate().filter(|(_, s)| s.visibility == Visibility::Visible).map(|(i, s)| (i.to_string(), s.name.clone())).collect() } +/// Workbook index of the first (or, with `last`, the last) visible sheet: where Ctrl-clicking a tab arrow lands. +fn edge_sheet(sheets: &[Arc], last: bool) -> Option { + let mut visible = sheets.iter().enumerate().filter(|(_, s)| s.visibility == Visibility::Visible).map(|(i, _)| i); + if last { visible.next_back() } else { visible.next() } +} + /// Switch to sheet `i` from the tab strip or the navigator. In point mode the formula being typed keeps going on the /// other sheet; otherwise the edit is committed first. fn activate_sheet(app: &mut SheetApp, i: usize) { @@ -117,10 +123,22 @@ pub fn sheet_tabs(app: &mut SheetApp, ui: &mut Ui) { ui.add_space(8.0); if view.show_sheet_tabs { if crate::widgets::icon_button(ui, Icon::Left, t.text_dim, app.ui.language.tr("Previous sheet"), vec2(24.0, 24.0)).clicked() { - app.run_or_alert("sheet.previous", json!({})); + // Ctrl-click jumps to the first visible sheet, like Excel's tab scroll buttons. + let edge = + if ui.input(|i| i.modifiers.command) { app.session.active().and_then(|d| edge_sheet(&d.wb.sheets, false)) } else { None }; + match edge { + Some(i) => activate_sheet(app, i), + None => app.run_or_alert("sheet.previous", json!({})), + } } if crate::widgets::icon_button(ui, Icon::Right, t.text_dim, app.ui.language.tr("Next sheet"), vec2(24.0, 24.0)).clicked() { - app.run_or_alert("sheet.next", json!({})); + // Ctrl-click jumps to the last visible sheet. + let edge = + if ui.input(|i| i.modifiers.command) { app.session.active().and_then(|d| edge_sheet(&d.wb.sheets, true)) } else { None }; + match edge { + Some(i) => activate_sheet(app, i), + None => app.run_or_alert("sheet.next", json!({})), + } } let all = crate::widgets::icon_button(ui, Icon::SheetList, t.text_dim, app.ui.language.tr("All sheets"), vec2(24.0, 24.0)); all.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, app.ui.language.tr("All sheets"))); @@ -522,7 +540,8 @@ fn status_bar_menu(app: &mut SheetApp, ui: &mut Ui) { #[cfg(test)] mod tests { use super::{ - MIN_TRACK, NEW_SHEET_WIDTH, SPLIT_WIDTH, activate_sheet, dragged_share, drop_copies_sheet, navigator_items, rename_needs_focus, tab_drop, + MIN_TRACK, NEW_SHEET_WIDTH, SPLIT_WIDTH, activate_sheet, dragged_share, drop_copies_sheet, edge_sheet, navigator_items, rename_needs_focus, + tab_drop, }; use crate::SheetApp; use egui::Modifiers; @@ -614,6 +633,16 @@ mod tests { assert_eq!(navigator_items(&sheets), vec![("0".to_string(), "Data".to_string()), ("3".to_string(), "Summary".to_string())]); } + #[test] + fn ctrl_click_on_the_tab_arrows_lands_on_the_first_and_last_visible_sheet() { + let mut hidden = Sheet::new("Hidden"); + hidden.visibility = Visibility::Hidden; + let sheets = [hidden.clone(), Sheet::new("A"), Sheet::new("B"), Sheet::new("C"), hidden].map(Arc::new); + assert_eq!(edge_sheet(&sheets, false), Some(1)); + assert_eq!(edge_sheet(&sheets, true), Some(3)); + assert_eq!(edge_sheet(&[], true), None); + } + #[test] fn a_rejected_commit_keeps_the_sheet_and_the_edit() { let mut session = gridcraft_engine::Session::new(); From 23c909e4775ac8d28e722ada888980f4faefe42f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 01:53:24 +0100 Subject: [PATCH 29/51] chart perf tests: import LegendPos (legend module now imports it itself) Co-Authored-By: Claude Sonnet 5.5 --- crates/chart/src/render_perf_tests.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/chart/src/render_perf_tests.rs b/crates/chart/src/render_perf_tests.rs index 461c2f159..ce15e7861 100644 --- a/crates/chart/src/render_perf_tests.rs +++ b/crates/chart/src/render_perf_tests.rs @@ -7,6 +7,7 @@ use super::kinds::squarify; use super::legend::swatch_w; use super::*; use crate::{ApproxMeasure, Measure}; +use gridcraft_model::LegendPos; /// The pre-rewrite `Ctx::fit`: one `Measure` call per candidate length, longest first. fn fit_reference(ctx: &Ctx, s: &str, size: f32, maxw: f32) -> String { From 7d3ede854f3d1c1d8f84b386b464a771c7dac61d Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 00:27:07 +0600 Subject: [PATCH 30/51] Sort: the closed Sort by box names the column, as its list does (#3446) The Sort dialog's closed "Sort by" box showed only the stored column letter ("B") while its open list showed "B: Revenue". It now looks the letter up in the dialog's column labels and shows the same label, falling back to the bare letter for a column outside the data block. --- crates/ui-egui/src/dialogs/data.rs | 5 ++++- crates/ui-egui/tests/it/sort_dialog_columns.rs | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/dialogs/data.rs b/crates/ui-egui/src/dialogs/data.rs index e3eabda79..e57a90c4e 100644 --- a/crates/ui-egui/src/dialogs/data.rs +++ b/crates/ui-egui/src/dialogs/data.rs @@ -218,8 +218,11 @@ pub(super) fn sort(app: &mut SheetApp, ui: &mut egui::Ui, d: &mut Dialog, confir if columns.is_empty() { ui.add(egui::TextEdit::singleline(&mut col).desired_width(50.0)).labelled_by(label.id); } else { + // The closed box shows the same "B: Revenue" label as the list, not the bare letter (#3446). + let shown = + columns.iter().find(|(letter, _)| letter.eq_ignore_ascii_case(&col)).map_or_else(|| col.clone(), |(_, label)| label.clone()); craft_ui::ComboBox::from_id_salt(("sort_col", i)) - .selected_text(col.clone()) + .selected_text(shown) .show_ui(ui, |ui| { for (letter, label) in &columns { ui.selectable_value(&mut col, letter.clone(), label.as_str()); diff --git a/crates/ui-egui/tests/it/sort_dialog_columns.rs b/crates/ui-egui/tests/it/sort_dialog_columns.rs index b6cbb54f6..098f9c3e6 100644 --- a/crates/ui-egui/tests/it/sort_dialog_columns.rs +++ b/crates/ui-egui/tests/it/sort_dialog_columns.rs @@ -34,6 +34,18 @@ fn sort_dialog_seeds_the_active_column() { assert_eq!(col(&h), "B", "Sort by starts on the column the cursor is in, not A"); } +/// #3446: the closed "Sort by" box shows the column's name next to its letter, +/// as the open list does ("B: Revenue", not just "B"). +#[test] +fn closed_column_box_shows_the_column_name() { + use egui::accesskit::Role; + let mut h = harness(sales()); + h.state_mut().open_dialog("sort", json!({})); + h.run_steps(2); + let shown: Vec = h.get_all_by_role(Role::ComboBox).filter_map(|n| n.value()).collect(); + assert!(shown.iter().any(|v| v == "B: Revenue"), "the Sort by box shows the column name: {shown:?}"); +} + #[test] fn add_level_picks_a_column_no_level_uses_yet() { let mut h = harness(sales()); From 1688b43ae225fc4f89879d57ebacba66c7908a5a Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 00:27:07 +0600 Subject: [PATCH 31/51] View: a sheet keeps its own scroll position when sheets move (#3485) Scroll positions were stored by (window, sheet index), and nothing re-keyed them when sheets were inserted, deleted or moved. A new PivotTable sheet, which goes in front of its source, opened at the source's position, and the source took over whatever was stored at its new index. Positions are now keyed by (window, sheet uid), so they follow the sheet through inserts, deletes, moves and undo. A closed window's positions are dropped. Saving the active cell still happens when no window is active. Co-Authored-By: Olivier <5681030+tutis@users.noreply.github.com> --- ROADMAP.md | 2 + crates/ui-egui/src/app/commands.rs | 1 + crates/ui-egui/src/app/view.rs | 16 +++-- crates/ui-egui/src/lib.rs | 5 +- crates/ui-egui/tests/it/scroll_position.rs | 83 ++++++++++++++++++++++ 5 files changed, 101 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 59b3bf2d8..7a794e06e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -135,6 +135,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-11-06 | A sheet keeps its own scroll position when sheets are inserted, deleted or moved before it: a new PivotTable or inserted sheet opens at the top instead of where its neighbour was scrolled; a closed window's positions are let go (#3485) | | 2026-11-02 | Form controls: Insert › Controls places buttons, sliders (scroll bars) and spin buttons on the sheet. A slider or spin button holds a whole number from 0 to 30000 in a linked cell (a drag or a held arrow is one undo step, and editing the cell moves the control); a button runs a macro or one command through the macro policy, asking first in a workbook opened from a file; Assign Action and Format Control edit them, and XLSX files read and write them (control properties, `` and VML), alongside the form controls GridCraft keeps unchanged | | 2026-11-02 | Dialogs take ranges by selection on the grid: reference boxes (RefEdit) with a collapse button that shrinks the dialog while the person selects, in Goal Seek, Solver (including the Model Range of Load/Save) and Data Analysis; while collapsed, the arrow keys move the selection and Shift+arrow extends it, and F4 cycles a reference's anchors | | 2026-11-02 | Python mode in the formula bar: Ctrl+Alt+Shift+P, `=PY` then Tab, or editing a Python cell edits its code in the formula bar, with Enter for a new line, Tab to indent and Ctrl+Enter to commit through `python.set`. The code still doesn't run | @@ -259,6 +260,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | Change | Summary | |---|---|---| +| 2026-11-06 | trivial | Progress log: scroll positions follow their sheet through sheet inserts, deletes and moves (#3485) | | 2026-11-02 | minor | Form controls; catalog 277 / 292 → 281 / 296 (Developer › Controls); progress log row | | 2026-11-02 | minor | Progress log: RefEdit reference boxes in dialogs | | 2026-11-02 | minor | Progress log: Python mode in the formula bar | diff --git a/crates/ui-egui/src/app/commands.rs b/crates/ui-egui/src/app/commands.rs index 4aba80024..6dff45cb2 100644 --- a/crates/ui-egui/src/app/commands.rs +++ b/crates/ui-egui/src/app/commands.rs @@ -302,6 +302,7 @@ impl SheetApp { // The requests belong to the window the command left active. self.follow_active_window(); self.prune_selected_object(); + self.prune_views(); for req in self.session.take_ui_requests() { match req { UiRequest::Dialog(name, params) => self.open_dialog(&name, params), diff --git a/crates/ui-egui/src/app/view.rs b/crates/ui-egui/src/app/view.rs index aea36abc2..482372448 100644 --- a/crates/ui-egui/src/app/view.rs +++ b/crates/ui-egui/src/app/view.rs @@ -7,8 +7,8 @@ impl SheetApp { } /// The active window and sheet: each window scrolls on its own. - fn scroll_key(&self) -> Option<(u64, usize)> { - Some((self.session.active_window()?, self.session.active()?.wb.active_sheet())) + fn scroll_key(&self) -> Option<(u64, u64)> { + Some((self.session.active_window()?, self.session.active()?.wb.active()?.uid.get())) } pub fn view(&self) -> SheetView { @@ -41,8 +41,10 @@ impl SheetApp { let window = self.session.active_window(); let Some(d) = self.session.active_mut() else { return }; let mut tops = Vec::new(); - for (&(w, si), v) in &self.views { - let Some(sh) = d.wb.sheets.get(si).filter(|sh| Some(w) == window && sh.view == gridcraft_engine::model::SheetViewMode::Normal) else { + for (si, sh) in d.wb.sheets.iter().enumerate() { + let Some(v) = + window.and_then(|w| self.views.get(&(w, sh.uid.get()))).filter(|_| sh.view == gridcraft_engine::model::SheetViewMode::Normal) + else { continue; }; let (fr, fc) = sh.freeze.unwrap_or((0, 0)); @@ -53,4 +55,10 @@ impl SheetApp { } d.store_view(&tops); } + + /// Drops the scroll positions of windows that have closed: window ids are never reused. + pub(super) fn prune_views(&mut self) { + let session = &self.session; + self.views.retain(|&(w, _), _| session.window(w).is_some()); + } } diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index e773a207d..dffe3ed89 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -222,8 +222,9 @@ pub struct SheetApp { pub ui: UiState, /// The window chrome to draw (see [`Chrome`]). pub chrome: Chrome, - /// Scroll position per (window id, sheet index). - pub views: HashMap<(u64, usize), SheetView>, + /// Scroll position per (window id, sheet uid): each sheet keeps its own when sheets are + /// inserted, deleted or moved. + pub views: HashMap<(u64, u64), SheetView>, pub editor: Option, pub text_box_editor: Option, pub dialog: Option, diff --git a/crates/ui-egui/tests/it/scroll_position.rs b/crates/ui-egui/tests/it/scroll_position.rs index 520bb1691..ffcb56dd6 100644 --- a/crates/ui-egui/tests/it/scroll_position.rs +++ b/crates/ui-egui/tests/it/scroll_position.rs @@ -1,4 +1,6 @@ //! The saved scroll position (`sheetView topLeftCell`) is restored on open and saved back (#378). +//! Each window keeps each sheet's own scroll position while sheets are inserted, deleted or +//! moved (#3485). use gridcraft_engine::core::{CellRef, Value}; use gridcraft_engine::model::Workbook; @@ -58,3 +60,84 @@ fn saved_scroll_position_is_restored_and_saved_back() { assert!(!d.is_dirty(), "scrolling and saving view state does not dirty the workbook"); } } + +#[test] +fn a_pivottable_sheet_opens_at_the_top_and_its_source_sheet_keeps_its_place() { + // A new PivotTable sheet goes in front of its source sheet. Scroll positions were kept by + // sheet index, so the new sheet took over the source sheet's position and opened at row 51. + let mut h = harness(None); + h.state_mut().run("range.setValues", json!({"range": "C1:D3", "values": [["Region", "Sales"], ["North", 10], ["South", 20]]})).unwrap(); + { + let app = h.state_mut(); + let sh = app.session.active().unwrap().wb.active().unwrap().clone(); + app.view_mut().unwrap().scroll.y = sh.row_top(50) as f32; + } + h.run_steps(2); + assert_eq!(first_scrolled_row(&h), 50, "setup: Sheet1 is scrolled to row 51"); + h.state_mut().run("insert.pivotTable", json!({"source": "Sheet1!$C$1:$D$3", "rows": ["Region"]})).unwrap(); + h.run_steps(4); + let wb = &h.state().session.active().unwrap().wb; + assert!(!wb.active().unwrap().pivots.is_empty(), "setup: the PivotTable's new sheet is active"); + assert_eq!(first_scrolled_row(&h), 0, "the new sheet opens at row 1"); + h.state_mut().run("sheet.next", json!({})).unwrap(); + h.run_steps(4); + assert_eq!(h.state().session.active().unwrap().wb.active().unwrap().name, "Sheet1"); + assert_eq!(first_scrolled_row(&h), 50, "Sheet1 is still scrolled to row 51"); +} + +/// Scrolls the active sheet so `row` (0-based) is the first one in view. +fn scroll_to_row(h: &mut Harness, row: u32) { + let app = h.state_mut(); + let sh = app.session.active().unwrap().wb.active().unwrap().clone(); + app.view_mut().unwrap().scroll.y = sh.row_top(row) as f32; + h.run_steps(2); +} + +fn activate(h: &mut Harness, sheet: &str) { + h.state_mut().run("sheet.activate", json!({"sheet": sheet})).unwrap(); + h.run_steps(4); + assert_eq!(h.state().session.active().unwrap().wb.active().unwrap().name, sheet); +} + +#[test] +fn a_sheet_keeps_its_scroll_position_when_sheets_are_inserted_deleted_or_moved() { + let mut h = harness(None); + scroll_to_row(&mut h, 50); + h.state_mut().run("home.insertSheet", json!({"name": "Two", "before": 0})).unwrap(); + h.run_steps(4); + assert_eq!(first_scrolled_row(&h), 0, "the inserted sheet opens at row 1"); + scroll_to_row(&mut h, 10); + activate(&mut h, "Sheet1"); + assert_eq!(first_scrolled_row(&h), 50, "Sheet1 moved one place right and is still at row 51"); + h.state_mut().run("home.deleteSheet", json!({"sheet": "Two"})).unwrap(); + h.run_steps(4); + assert_eq!(first_scrolled_row(&h), 50, "Sheet1 moved back and is still at row 51"); + h.state_mut().run("edit.undo", json!({})).unwrap(); + h.run_steps(4); + activate(&mut h, "Two"); + assert_eq!(first_scrolled_row(&h), 10, "the sheet undo brings back is where it was left"); + h.state_mut().run("sheet.move", json!({"sheet": 0, "to": 1})).unwrap(); + h.run_steps(4); + activate(&mut h, "Two"); + assert_eq!(first_scrolled_row(&h), 10, "a moved sheet keeps its place"); + activate(&mut h, "Sheet1"); + assert_eq!(first_scrolled_row(&h), 50, "and so does the sheet it moved past"); +} + +#[test] +fn each_window_scrolls_on_its_own_and_a_closed_windows_positions_are_dropped() { + let mut h = harness(None); + scroll_to_row(&mut h, 50); + let first = h.state().session.active_window().unwrap(); + h.state_mut().run("view.newWindow", json!({})).unwrap(); + h.run_steps(4); + let second = h.state().session.active_window().unwrap(); + assert_ne!(first, second); + scroll_to_row(&mut h, 10); + assert!(h.state().views.keys().any(|(w, _)| *w == second), "setup: the second window has its own position"); + h.state_mut().run("window.close", json!({"window": second})).unwrap(); + h.run_steps(4); + assert_eq!(h.state().session.active_window(), Some(first)); + assert_eq!(first_scrolled_row(&h), 50, "the first window kept its own position"); + assert!(h.state().views.keys().all(|(w, _)| *w == first), "nothing is kept for the closed window: {:?}", h.state().views.keys()); +} From 20473a4fae31023038a160b5651aa50657349917 Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 00:27:08 +0600 Subject: [PATCH 32/51] Selection: Ctrl+Shift+Arrow searches the active cell's line (#3484) Ctrl+Shift+Arrow searched from the selection's moving corner, so after Ctrl+Shift+Right from A2 the next Ctrl+Shift+Down searched the far column and ran to the last row. As in Excel, it now searches the active cell's column (Up/Down) or row (Left/Right) from the edge being moved, and moves only that edge. A diagonal move from command params keeps the old jump. Co-Authored-By: Olivier <5681030+tutis@users.noreply.github.com> --- crates/engine/src/cmd/edit/selection.rs | 9 ++- .../tests/it/ctrl_shift_arrow_active_line.rs | 56 +++++++++++++++++++ crates/engine/tests/it/main.rs | 1 + 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 crates/engine/tests/it/ctrl_shift_arrow_active_line.rs diff --git a/crates/engine/src/cmd/edit/selection.rs b/crates/engine/src/cmd/edit/selection.rs index 8115f6390..f171c65a8 100644 --- a/crates/engine/src/cmd/edit/selection.rs +++ b/crates/engine/src/cmd/edit/selection.rs @@ -66,7 +66,14 @@ pub(super) fn selection_move(s: &mut Session, p: &Json) -> Result { // Extension moves the far corner, not the active cell. let from = if extend { sel.moving_end() } else { sel.active }; // Skip hidden rows/columns and step over merged areas. - let mut to = if jmp { + let mut to = if jmp && extend && (dr == 0) != (dc == 0) { + // Excel: Ctrl+Shift+arrow searches the active cell's column (or row) from the moving + // edge, so a header row selected with Ctrl+Shift+Right extends down to the end of the + // active column's data. + let probe = if dr != 0 { CellRef::new(from.row, sel.active.col) } else { CellRef::new(sel.active.row, from.col) }; + let edge = jump(sheet, probe, dr.signum(), dc.signum()); + if dr != 0 { CellRef::new(edge.row, from.col) } else { CellRef::new(from.row, edge.col) } + } else if jmp { jump(sheet, from, dr.signum(), dc.signum()) } else { let mut t = from.offset_clamped(dr, dc); diff --git a/crates/engine/tests/it/ctrl_shift_arrow_active_line.rs b/crates/engine/tests/it/ctrl_shift_arrow_active_line.rs new file mode 100644 index 000000000..d296d50bb --- /dev/null +++ b/crates/engine/tests/it/ctrl_shift_arrow_active_line.rs @@ -0,0 +1,56 @@ +//! Ctrl/Cmd+Shift+arrow searches the active cell's column (or row) from the moving edge (#3484). + +use gridcraft_engine::Session; +use serde_json::json; + +fn table() -> Session { + // Headers in A2:K2, a key column in A3:A13, nothing else filled. + let mut s = Session::new(); + s.new_workbook(); + let header: Vec = (1..=11).collect(); + s.execute("range.setValues", json!({"range": "A2:K2", "values": [header]})).unwrap(); + let keys: Vec> = (1..=11).map(|i| vec![i]).collect(); + s.execute("range.setValues", json!({"range": "A3:A13", "values": keys})).unwrap(); + s +} + +fn ctrl_shift(s: &mut Session, dr: i64, dc: i64) -> String { + s.execute("selection.move", json!({"dr": dr, "dc": dc, "extend": true, "jump": true})).unwrap(); + s.doc().unwrap().selection.a1() +} + +#[test] +fn ctrl_shift_down_after_ctrl_shift_right_follows_the_active_column() { + let mut s = table(); + s.execute("selection.set", json!({"cell": "A2"})).unwrap(); + assert_eq!(ctrl_shift(&mut s, 0, 1), "A2:K2"); + assert_eq!(ctrl_shift(&mut s, 1, 0), "A2:K13", "column A's data ends on row 13; column K's ends on row 2"); + // From the end of column A's data the next press runs to the sheet's last row. + assert_eq!(ctrl_shift(&mut s, 1, 0), "A2:K1048576"); +} + +#[test] +fn ctrl_shift_right_after_ctrl_shift_down_follows_the_active_row() { + let mut s = table(); + s.execute("selection.set", json!({"cell": "A2"})).unwrap(); + assert_eq!(ctrl_shift(&mut s, 1, 0), "A2:A13"); + assert_eq!(ctrl_shift(&mut s, 0, 1), "A2:K13", "row 2's data ends in column K; row 13's ends in column A"); +} + +#[test] +fn ctrl_shift_down_after_ctrl_shift_left_follows_the_active_column() { + // The active cell is on the right: column K, empty below the header, decides, not column A. + let mut s = table(); + s.execute("selection.set", json!({"cell": "K2"})).unwrap(); + assert_eq!(ctrl_shift(&mut s, 0, -1), "A2:K2"); + assert_eq!(ctrl_shift(&mut s, 1, 0), "A2:K1048576"); +} + +#[test] +fn ctrl_shift_right_after_ctrl_shift_up_follows_the_active_row() { + // The active cell is at the bottom: row 13, empty right of the key, decides, not row 2. + let mut s = table(); + s.execute("selection.set", json!({"cell": "A13"})).unwrap(); + assert_eq!(ctrl_shift(&mut s, -1, 0), "A2:A13"); + assert_eq!(ctrl_shift(&mut s, 0, 1), "2:13", "row 13's data ends in column A, so the rows extend to the last column"); +} diff --git a/crates/engine/tests/it/main.rs b/crates/engine/tests/it/main.rs index a9cd7cbfe..0ba7c0db8 100644 --- a/crates/engine/tests/it/main.rs +++ b/crates/engine/tests/it/main.rs @@ -7,6 +7,7 @@ mod chart_workbook_names; mod command_sequence_fuzz; mod cross_sheet_cut; mod csv_export_budget; +mod ctrl_shift_arrow_active_line; mod custom_filter_blanks; mod custom_view_freeze; mod dynamicmetadata; From 89ec06f0415f53d06437889e778face34f53644a Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Thu, 5 Nov 2026 21:58:59 +0800 Subject: [PATCH 33/51] gridcraft-web: window.gridcraft, the page command API from craft-web-api The web shell queues the page's calls as control requests (the channel the desktop control server uses) and answers them on the next frame. A call the page gave up on is released, so the app skips it instead of running it late. The API is installed in every mode; opening, saving, the file name and the unsaved flag stay with the host messages. --- Cargo.lock | 14 ++- apps/gridcraft-web/Cargo.toml | 4 +- apps/gridcraft-web/src/api.rs | 151 +++++++++++++++++++++++++++++++++ apps/gridcraft-web/src/main.rs | 5 ++ apps/gridcraft-web/src/web.rs | 1 + packaging/web/README.md | 12 +++ 6 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 apps/gridcraft-web/src/api.rs diff --git a/Cargo.lock b/Cargo.lock index dfdde30e7..a39620f96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -995,10 +995,21 @@ dependencies = [ "zip 9.1.0", ] +[[package]] +name = "craft-web-api" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +dependencies = [ + "js-sys", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "craft-web-host" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=2e2312ddaf21bc3d85cd3395d0955c7d239306f6#2e2312ddaf21bc3d85cd3395d0955c7d239306f6" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" dependencies = [ "wasm-bindgen", "web-sys", @@ -2195,6 +2206,7 @@ dependencies = [ name = "gridcraft-web" version = "0.16.0" dependencies = [ + "craft-web-api", "craft-web-host", "eframe", "egui", diff --git a/apps/gridcraft-web/Cargo.toml b/apps/gridcraft-web/Cargo.toml index e5c599bd8..09c6d7e2a 100644 --- a/apps/gridcraft-web/Cargo.toml +++ b/apps/gridcraft-web/Cargo.toml @@ -17,7 +17,9 @@ path = "src/main.rs" gridcraft-engine = { workspace = true } gridcraft-ui-egui = { workspace = true } # The host messages (storytold/craft-libs): one contract and one bridge for the web builds. -craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "2e2312ddaf21bc3d85cd3395d0955c7d239306f6", version = "0.1.0" } +# The page command API (storytold/craft-libs#413): window.gridcraft over the control channel. +craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } +craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } egui = { workspace = true } log = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft-web/src/api.rs b/apps/gridcraft-web/src/api.rs new file mode 100644 index 000000000..74f7206f6 --- /dev/null +++ b/apps/gridcraft-web/src/api.rs @@ -0,0 +1,151 @@ +//! `window.gridcraft`: the page command API (`craft-web-api`, storytold/craft-libs#413) over the +//! app's own control channel, the one the desktop control server drives (`docs/control-protocol.md`). +//! +//! The crate owns the JavaScript side (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, +//! `screenshot`, as promises with a timeout). This module is GridCraft's part: a [`Request`] that +//! queues a [`ControlRequest`] for the app, which answers it on its next frame, and waits for the +//! reply. A call the page gave up on is released ([`Pending`]), so the app drops it unrun instead of +//! applying a command the page was told had failed. + +use std::sync::Arc; +use std::sync::atomic::AtomicU8; +use std::sync::mpsc::{Receiver, TryRecvError}; + +use gridcraft_ui_egui::ControlRequest; +use serde_json::{Value, json}; + +/// How often a waiting call looks for the app's reply, in milliseconds (about a frame). +#[cfg(target_arch = "wasm32")] +const POLL_MS: i32 = 16; + +/// A queued control request whose reply has not arrived. Dropping it before the reply (the page's +/// timeout fired, or the call was abandoned) releases the request: the app skips it unrun. +struct Pending { + reply: Receiver, + state: Arc, + answered: bool, +} + +impl Pending { + /// `Some` once the app has answered or gone away; `None` while the request is still waiting. + fn poll(&mut self) -> Option { + match self.reply.try_recv() { + Ok(reply) => { + self.answered = true; + Some(reply) + } + Err(TryRecvError::Empty) => None, + Err(TryRecvError::Disconnected) => { + self.answered = true; + Some(json!({"ok": false, "error": "the app dropped the request"})) + } + } + } +} + +impl Drop for Pending { + fn drop(&mut self) { + if !self.answered { + ControlRequest::abandon(&self.state); + } + } +} + +#[cfg(target_arch = "wasm32")] +pub use web::install; + +#[cfg(target_arch = "wasm32")] +mod web { + use std::rc::Rc; + use std::sync::mpsc::channel; + + use craft_web_api::methods::Methods; + use craft_web_api::page::{PageApi, Request}; + use gridcraft_ui_egui::{ControlRequest, SheetApp}; + use serde_json::json; + use wasm_bindgen_futures::JsFuture; + + use super::{POLL_MS, Pending}; + + /// The app id in `window.`. + const APP: &str = "gridcraft"; + + /// Give the app its control queue and put `window.gridcraft` on the page. An existing + /// `window.gridcraft` keeps the functions it already has. + pub fn install(app: &mut SheetApp, ctx: egui::Context) { + let (queue, requests) = channel::(); + app.control_rx = Some(requests); + let request: Request = Rc::new(move |method, params| { + let (control, reply) = ControlRequest::new(method, params); + let mut pending = Pending { reply, state: control.state.clone(), answered: false }; + if queue.send(control).is_err() { + pending.answered = true; + return Box::pin(async { json!({"ok": false, "error": "the app is not running"}) }); + } + ctx.request_repaint(); + Box::pin(async move { + loop { + if let Some(reply) = pending.poll() { + return reply; + } + sleep(POLL_MS).await; + } + }) + }); + if let Err(e) = Rc::new(PageApi::new(APP, Methods::default(), request)).install() { + log::warn!("couldn't install window.{APP}: {e:?}"); + } + } + + async fn sleep(ms: i32) { + let promise = js_sys::Promise::new(&mut |resolve, _| { + if let Some(window) = web_sys::window() { + let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms); + } + }); + let _ = JsFuture::from(promise).await; + } +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::Ordering; + + use super::*; + + /// `ControlRequest`'s `state` values (`gridcraft_ui_egui::control`): the app has started it, or its + /// sender gave up. + const TAKEN: u8 = 1; + const ABANDONED: u8 = 2; + + fn pending() -> (Pending, ControlRequest) { + let (control, reply) = ControlRequest::new("document.inspect", json!({})); + (Pending { reply, state: control.state.clone(), answered: false }, control) + } + + #[test] + fn a_call_the_page_gave_up_on_is_released_so_the_app_skips_it() { + let (waiting, control) = pending(); + drop(waiting); + assert_eq!(control.state.load(Ordering::SeqCst), ABANDONED); + assert!(!control.take(), "the app must not run a request the page was told had failed"); + } + + #[test] + fn an_answered_call_is_not_released() { + let (mut waiting, control) = pending(); + assert!(control.take()); + control.reply.send(json!({"ok": true, "result": 1})).unwrap_or_default(); + assert_eq!(waiting.poll(), Some(json!({"ok": true, "result": 1}))); + drop(waiting); + assert_eq!(control.state.load(Ordering::SeqCst), TAKEN); + } + + #[test] + fn a_request_the_app_never_runs_waits_until_it_is_dropped() { + let (mut waiting, control) = pending(); + assert_eq!(waiting.poll(), None); + drop(control); + assert_eq!(waiting.poll().and_then(|r| r["ok"].as_bool()), Some(false), "the app went away"); + } +} diff --git a/apps/gridcraft-web/src/main.rs b/apps/gridcraft-web/src/main.rs index 2c701feb4..823a68e9b 100644 --- a/apps/gridcraft-web/src/main.rs +++ b/apps/gridcraft-web/src/main.rs @@ -13,11 +13,16 @@ //! workbook to open and stores what GridCraft saves (`host`). A page can host GridCraft in its //! own document (no iframe) with `data-host="page"` on the canvas, talking over DOM `CustomEvent`s //! on the canvas. `&author=` names the user for notes and comments in either host mode. +//! `window.gridcraft` (`craft-web-api`) is the page command API in every mode: `request`, +//! `execute`, `commands`, `docInspect`, `uiInspect` and `screenshot` over the control methods +//! (`api`). //! `?chrome=none|notitlebar|nofiletab` leaves the title bar and/or the File tab undrawn for a host //! that embeds the sheet in its own window (see `Chrome`). Preferences (theme, interface size, //! editing options…) are kept in `localStorage` (`prefs`). #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(any(target_arch = "wasm32", test))] +mod api; #[cfg(target_arch = "wasm32")] mod host; #[cfg(any(target_arch = "wasm32", test))] diff --git a/apps/gridcraft-web/src/web.rs b/apps/gridcraft-web/src/web.rs index 268560520..1108c4c11 100644 --- a/apps/gridcraft-web/src/web.rs +++ b/apps/gridcraft-web/src/web.rs @@ -99,6 +99,7 @@ pub fn start() { } bridge.post_ready(); } + crate::api::install(&mut app, cc.egui_ctx.clone()); Ok(Box::new(WebShell { app, inbox, bridge: app_bridge, prefs })) }), ) diff --git a/packaging/web/README.md b/packaging/web/README.md index 1f346cde0..833a64e9f 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -157,6 +157,18 @@ The messages and both transports are the shared contract of [`craft-web-host`](h - **Files from the host are untrusted input.** An opened file goes through the same reading as one picked with File ▸ Open or dropped on the window, with the same format limits; before it is copied into the app's memory, its size is checked against the host open limit (512 MiB) and a larger file is refused, and a name is cut to 255 characters. - **The event and message names are the interface.** `gridcraft:ready`, `gridcraft:open`, `gridcraft:save`, `gridcraft:dirty` and `gridcraft:failed`, and the fields they carry (the same as the `?host=parent` messages), stay stable; changing one is a breaking change. +### Driving it from the page (`window.gridcraft`) + +Every GridCraft web build puts `window.gridcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app. `gridcraft.request(method, params)` calls any method of the control channel the desktop app serves (`docs/control-protocol.md`), and `execute(command, params)`, `commands()`, `docInspect(params)`, `uiInspect(params)` and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app drops it unrun. + +```js +const book = await gridcraft.docInspect(); // sheets, used ranges, tables, charts, names, selection +await gridcraft.execute("cell.set", { cell: "A1", input: "Total" }); +const { data } = await gridcraft.screenshot(); // the canvas as PNG (base64) +``` + +The API is there in every mode, with or without a host. Opening a workbook, handing a saved or exported file to the page, the file name and the unsaved flag stay with the host messages above. + ## Renderer selection GridCraft's web build is the same egui app as the desktop one, compiled to wasm From fb17d0d1d5295f9c168185082d7998a591d41cc3 Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:28:03 +0800 Subject: [PATCH 34/51] gridcraft-web: say a call the app already took still finishes after a timeout --- apps/gridcraft-web/src/api.rs | 4 ++-- packaging/web/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/gridcraft-web/src/api.rs b/apps/gridcraft-web/src/api.rs index 74f7206f6..34aca8c18 100644 --- a/apps/gridcraft-web/src/api.rs +++ b/apps/gridcraft-web/src/api.rs @@ -4,7 +4,7 @@ //! The crate owns the JavaScript side (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, //! `screenshot`, as promises with a timeout). This module is GridCraft's part: a [`Request`] that //! queues a [`ControlRequest`] for the app, which answers it on its next frame, and waits for the -//! reply. A call the page gave up on is released ([`Pending`]), so the app drops it unrun instead of +//! reply. A call the page gave up on is released ([`Pending`]), so the app skips it if it has not started it, instead of //! applying a command the page was told had failed. use std::sync::Arc; @@ -19,7 +19,7 @@ use serde_json::{Value, json}; const POLL_MS: i32 = 16; /// A queued control request whose reply has not arrived. Dropping it before the reply (the page's -/// timeout fired, or the call was abandoned) releases the request: the app skips it unrun. +/// timeout fired, or the call was abandoned) releases the request: the app skips it if it has not started it (one it has taken still finishes). struct Pending { reply: Receiver, state: Arc, diff --git a/packaging/web/README.md b/packaging/web/README.md index 833a64e9f..b048a5b52 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -159,7 +159,7 @@ The messages and both transports are the shared contract of [`craft-web-host`](h ### Driving it from the page (`window.gridcraft`) -Every GridCraft web build puts `window.gridcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app. `gridcraft.request(method, params)` calls any method of the control channel the desktop app serves (`docs/control-protocol.md`), and `execute(command, params)`, `commands()`, `docInspect(params)`, `uiInspect(params)` and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app drops it unrun. +Every GridCraft web build puts `window.gridcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app. `gridcraft.request(method, params)` calls any method of the control channel the desktop app serves (`docs/control-protocol.md`), and `execute(command, params)`, `commands()`, `docInspect(params)`, `uiInspect(params)` and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app skips it if it has not started it; a call the app has already taken still finishes, so a retry after a timeout can apply a command twice. ```js const book = await gridcraft.docInspect(); // sheets, used ranges, tables, charts, names, selection From bc1625248e325179f8da7d14f06b99b04c99a8d7 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:25:23 +0100 Subject: [PATCH 35/51] Chart: cap category levels in the clean-data check, recompute overflowing moving-average windows, keep disjoint stacked hairline spans apart Co-Authored-By: Claude Sonnet 5.5 --- crates/chart/src/render.rs | 2 ++ crates/chart/src/render/cartesian.rs | 6 +++++- crates/chart/src/tests/budget.rs | 27 +++++++++++++++++++++++++++ crates/chart/src/trend.rs | 15 ++++++++++++++- 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/crates/chart/src/render.rs b/crates/chart/src/render.rs index 3fa9f8431..3abf8bf14 100644 --- a/crates/chart/src/render.rs +++ b/crates/chart/src/render.rs @@ -141,6 +141,8 @@ fn needs_sanitizing(d: &ChartData) -> bool { let mut trendlines = 0usize; d.palette.len() > MAX_POINTS || long(d.categories.len()) + || d.category_levels.len() > 3 + || d.category_levels.iter().any(|l| l.len() > MAX_POINTS) || d.series.len() > MAX_SERIES || d.series.iter().any(|s| { trendlines += s.trendlines.len(); diff --git a/crates/chart/src/render/cartesian.rs b/crates/chart/src/render/cartesian.rs index 6689bf202..44c7a3dc9 100644 --- a/crates/chart/src/render/cartesian.rs +++ b/crates/chart/src/render/cartesian.rs @@ -67,6 +67,10 @@ impl BarRun { fn new(pos: f32, lo: f32, hi: f32) -> Self { BarRun { bucket: pos.floor(), a0: pos, a1: pos, lo, hi, merged: false } } + /// Whether `lo..hi` touches the run's value span; a disjoint bar would be joined by a filled gap. + fn touches(&self, lo: f32, hi: f32) -> bool { + lo <= self.hi && hi >= self.lo + } fn add(&mut self, pos: f32, lo: f32, hi: f32) { self.merged = true; self.a0 = self.a0.min(pos); @@ -135,7 +139,7 @@ impl BarSink { return; } match &mut self.run { - Some(m) if m.bucket == pos.floor() => m.add(pos, lo, lo + len), + Some(m) if m.bucket == pos.floor() && m.touches(lo, lo + len) => m.add(pos, lo, lo + len), run => { if let Some(m) = run.replace(BarRun::new(pos, lo, lo + len)) { m.draw(ctx, self.bw, self.horizontal, self.color, band); diff --git a/crates/chart/src/tests/budget.rs b/crates/chart/src/tests/budget.rs index 54d33aa7d..6c5a10879 100644 --- a/crates/chart/src/tests/budget.rs +++ b/crates/chart/src/tests/budget.rs @@ -158,6 +158,26 @@ fn clustered_hairline_series_keep_a_visible_strip() { } } +#[test] +fn stacked_hairline_bars_do_not_fill_the_gap_between_disjoint_spans() { + // Series 1 alternates +10 (stacked on the 100) and -10 (below zero): its two spans in one + // pixel column are 100..110 and -10..0, and the 100 units between them must stay empty. + let mut c = chart(ChartKind::ColumnStacked); + c.data_labels = false; + c.legend = LegendPos::None; + let mut d = data_of(2, 20_000); + for v in d.series[0].values.iter_mut() { + *v = Some(100.0); + } + for (i, v) in d.series[1].values.iter_mut().enumerate() { + *v = Some(if i % 2 == 0 { 10.0 } else { -10.0 }); + } + let prims = render(&c, &d, 400.0, 300.0, &ApproxMeasure); + let rects = series_rects(&prims, d.series[1].color); + let tallest = rects.iter().map(|r| r.3).fold(0.0f32, f32::max); + assert!(!rects.is_empty() && tallest < 60.0, "tallest merged bar is {tallest} px"); +} + #[test] fn clean_data_is_rendered_without_a_copy() { let d = data_of(3, 50); @@ -181,6 +201,13 @@ fn clean_data_is_rendered_without_a_copy() { let got = sanitized(&many); assert_eq!(got.series.len(), 255); assert_eq!(*got, sanitize(&many)); + + // Too many category levels take the copying path too. + let mut levels = data_of(3, 50); + levels.category_levels = vec![vec!["a".into()]; 5]; + let got = sanitized(&levels); + assert!(matches!(got, Cow::Owned(_))); + assert_eq!(got.category_levels.len(), 3); } #[test] diff --git a/crates/chart/src/trend.rs b/crates/chart/src/trend.rs index 6b22ed7ad..818c75796 100644 --- a/crates/chart/src/trend.rs +++ b/crates/chart/src/trend.rs @@ -294,7 +294,11 @@ pub(crate) fn moving_average(pts: &[(f64, f64)], period: u32) -> Vec<(f64, f64)> (p - 1..n) .filter_map(|i| { let s = i + 1 - p; - let sum = tail.get(s)? + if s.is_multiple_of(p) { 0.0 } else { *head.get(i)? }; + let mut sum = tail.get(s)? + if s.is_multiple_of(p) { 0.0 } else { *head.get(i)? }; + if !sum.is_finite() { + // A block sum overflowed although the window itself may not: sum it directly. + sum = pts.get(s..=i)?.iter().map(|q| q.1).sum(); + } Some((pts.get(i)?.0, sum / p as f64)) }) .collect() @@ -390,6 +394,15 @@ mod tests { } } + #[test] + fn moving_average_recomputes_a_window_whose_block_sums_overflow() { + let pts: Vec<(f64, f64)> = [0.0, -1e308, 1e308, 1e308, -1e308].iter().enumerate().map(|(i, y)| (i as f64, *y)).collect(); + let fast = moving_average(&pts, 4); + let slow = moving_average_naive(&pts, 4); + assert_eq!(fast, slow); + assert!(fast.iter().all(|q| q.1.is_finite())); + } + /// The trendline fit as it was written with a `Vec` of powers per data point. fn fit_reference(t: &Trendline, pts: &[(f64, f64)]) -> Option { let kind = t.kind; From 76002fe0bd1eadad4c59fe40778fbde126297a67 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:58:19 +0100 Subject: [PATCH 36/51] Reuse remembered range sums for every COUNT block COUNT never reads the running sum, so the zero-sum gate that keeps SUM and AVERAGE additions in order does not apply to it. Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/eval/call.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/calc/src/eval/call.rs b/crates/calc/src/eval/call.rs index 46f0d02c9..8ac58c8d1 100644 --- a/crates/calc/src/eval/call.rs +++ b/crates/calc/src/eval/call.rs @@ -384,7 +384,8 @@ impl<'h> Evaluator<'h> { fn aggregate_area(&mut self, kind: Agg, a: &Area, acc: &mut Acc) -> Result<(), Ev> { if !a.range.is_single() { // Adding the block's remembered sum to anything but +0.0 would add in another order. - if acc.sum.to_bits() == 0 + // COUNT never reads the sum, so it may reuse every block. + if (kind == Agg::Count || acc.sum.to_bits() == 0) && let Some((sum, count)) = self.host.range_sum(a.sheet, a.range) { acc.sum = sum; From 225496e5445eda6f0bd2cd7edf4cefc688b62f02 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:19:20 +0100 Subject: [PATCH 37/51] Math: refuse computed environment names in the alignat pair cap A name such as align\@char{97}t reads as alignat to the parser but split at the first } looked like another environment, so the pair limit was skipped. Names that contain a backslash or brace, or do not close in the window, are now refused like an unreadable count. Co-Authored-By: Claude Sonnet 5.5 --- crates/math/src/lib.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/crates/math/src/lib.rs b/crates/math/src/lib.rs index 62033f2b8..fb01b2374 100644 --- a/crates/math/src/lib.rs +++ b/crates/math/src/lib.rs @@ -392,6 +392,7 @@ const MAX_ALIGNAT_PAIRS: usize = 50; /// N of `{alignat}{N}`, `{alignat*}{N}` or `{alignedat}{N}` at the start of `rest` (just after /// `\begin`). `usize::MAX` when the count is present but is not a plain digit string RaTeX reads /// the same way (too long, or hidden by a comment, inner spaces or a macro): refused, not skipped. +/// An environment name that is computed (contains `\` or `{`) is refused the same way. /// /// RaTeX reads the environment name and the count through its lexer, which drops comments and /// spaces, so they are dropped here too before the two groups are compared. @@ -414,7 +415,16 @@ fn alignat_columns(rest: &str) -> Option { Some(c) => kept.push(c), } } - let (name, after) = kept.strip_prefix('{')?.split_once('}')?; + let group = kept.strip_prefix('{')?; + // A name that does not close inside the window cannot be read, so it is refused. + let Some((name, after)) = group.split_once('}') else { + return Some(usize::MAX); + }; + // A computed name (`align\@char{97}t` reads as `alignat`) cannot be resolved here: refused + // too. Every real environment name is letters, optionally with a trailing `*`. + if name.contains(['\\', '{']) { + return Some(usize::MAX); + } if !matches!(name, "alignat" | "alignat*" | "alignedat") { return None; } @@ -755,6 +765,14 @@ mod tests { ] { assert!(check_caps(hidden).is_err_and(|e| e.message.contains("matrix cells")), "{hidden}"); } + // A computed environment name reads as `alignat` to the parser, so it is capped too. + for name in ["align\\@char{97}t", "alignedat\\relax", "al\\@char{105}gnat*"] { + let over = format!("\\begin{{{name}}}{{51}}a\\end{{alignat}}"); + assert!(check_caps(&over).is_err_and(|e| e.message.contains("matrix cells")), "{over}"); + } + let long = format!("\\begin{{{}}}{{51}}a\\end{{alignat}}", "a".repeat(80)); + assert!(check_caps(&long).is_err_and(|e| e.message.contains("matrix cells"))); + assert!(check_caps("\\begin{alignat}{50}a\\end{alignat}").is_ok()); assert!(check_caps("\\begin{alignedat} {2} % two pairs\na&b&c&d\\end{alignedat}").is_ok()); assert!(layout("\\begin{CD}A @>f>> B \\\\ @VVV @VVV \\\\ C @>>> D\\end{CD}").is_ok()); assert!(check_caps("\\operatorname{curl} 50\\%").is_ok()); From 12f69f99bf40306008c243a89b44c2f122d30f06 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:20:53 +0100 Subject: [PATCH 38/51] BINOM.DIST: keep small-k masses whose complement power underflows The small-k product also needs (1 - p)^(n - k) to stay a normal number; otherwise it returned 0 for BINOM.DIST(15,1100,0.5,FALSE) (exact 2.1367646606291324e-298) and the cumulative form built on it. Those cases now use the logarithmic formula. Reference grid gets k = 10..25 at n = 1100. Co-Authored-By: Claude Sonnet 5.5 --- crates/functions/src/stat/distributions.rs | 9 ++++++--- crates/functions/src/stat/tests.rs | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/functions/src/stat/distributions.rs b/crates/functions/src/stat/distributions.rs index d2e4e3421..0853675a6 100644 --- a/crates/functions/src/stat/distributions.rs +++ b/crates/functions/src/stat/distributions.rs @@ -201,11 +201,14 @@ fn binom_pmf(k: f64, n: f64, p: f64) -> f64 { return p.powf(n); } // A few successes or a few failures: the product form has no large cancelling terms. - // (Not for astronomically large n or a power of p near underflow: the saddle-point form - // below handles those through logs.) + // (Not for astronomically large n, or when either power underflows: the + // saddle-point form below handles those through logs.) if n < 1e30 && p.powf(k) > 1e-280 { if k < 16.0 { - return choose_small(n, k) * p.powi(k as i32) * pow_one_minus(p, n - k); + let rest = pow_one_minus(p, n - k); + if rest >= f64::MIN_POSITIVE { + return choose_small(n, k) * p.powi(k as i32) * rest; + } } if n - k < 16.0 { return choose_small(n, n - k) * p.powf(k) * pow_one_minus(p, n - k); diff --git a/crates/functions/src/stat/tests.rs b/crates/functions/src/stat/tests.rs index 0eb0de0d1..94b35e1f2 100644 --- a/crates/functions/src/stat/tests.rs +++ b/crates/functions/src/stat/tests.rs @@ -745,6 +745,13 @@ const BINOM_REF: &[(f64, f64, f64, f64, f64)] = &[ (99031.0, 100000.0, 0.99, 0.007885853372194001, 0.8416321140066325), (99094.0, 100000.0, 0.99, 0.00013276363518518057, 0.9988439452168577), (99999.0, 100000.0, 0.99, 0.0, 1.0), + // Small k where (1 - p)^(n - k) underflows although the mass is a normal number. + (10.0, 1100.0, 0.5, 5.050689010687844e-308, 5.097367491037505e-308), + (14.0, 1100.0, 0.5, 2.9513324041838846e-300, 2.989803284040754e-300), + (15.0, 1100.0, 0.5, 2.1367646606291324e-298, 2.16666269346954e-298), + (16.0, 1100.0, 0.5, 1.4489935354891303e-296, 1.4706601624238258e-296), + (20.0, 1100.0, 0.5, 1.711087104904223e-289, 1.7433098849775326e-289), + (25.0, 1100.0, 0.5, 3.906996229178926e-281, 3.999839082741416e-281), ]; // Columns: x, mean, mass, cdf. The last block has x > 1000, which takes the incomplete-gamma branch. From 87cb0f11d2a65e929c0d1a2af2e81f58720365ae Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:22:54 +0100 Subject: [PATCH 39/51] Tab strip: reveal the edge tab after a repeated Ctrl-click on an arrow When the edge sheet was already active nothing changed, so the strip was not scrolled back to it. A Ctrl-click on the arrow now asks for the reveal itself. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/tabs.rs | 27 ++++++++++++++-- crates/ui-egui/tests/it/sheet_tabs.rs | 44 +++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/src/tabs.rs b/crates/ui-egui/src/tabs.rs index 917dc3b7f..462dc2e4c 100644 --- a/crates/ui-egui/src/tabs.rs +++ b/crates/ui-egui/src/tabs.rs @@ -61,6 +61,18 @@ fn tab_drop(i: usize, steps: i64, copy: bool) -> Option { Some(json!({"sheet": i, "to": to})) } +/// Where a request to scroll the active tab into view is kept until the tabs are drawn. +fn reveal_request_id(scope: egui::Id) -> egui::Id { + scope.with("worksheet_tabs_reveal_request") +} + +/// Asks the tab strip to bring the active tab into view in this frame, even when the active sheet +/// did not change (a Ctrl-click on an arrow whose edge sheet is already active). +fn request_reveal(ui: &Ui) { + let id = reveal_request_id(ui.scope_id()); + ui.ctx().data_mut(|data| data.insert_temp(id, true)); +} + /// The New sheet (+) button and the gap before it. const NEW_SHEET_WIDTH: f32 = 32.0; /// The handle between the tabs and the horizontal scrollbar. @@ -127,7 +139,10 @@ pub fn sheet_tabs(app: &mut SheetApp, ui: &mut Ui) { let edge = if ui.input(|i| i.modifiers.command) { app.session.active().and_then(|d| edge_sheet(&d.wb.sheets, false)) } else { None }; match edge { - Some(i) => activate_sheet(app, i), + Some(i) => { + activate_sheet(app, i); + request_reveal(ui); + } None => app.run_or_alert("sheet.previous", json!({})), } } @@ -136,7 +151,10 @@ pub fn sheet_tabs(app: &mut SheetApp, ui: &mut Ui) { let edge = if ui.input(|i| i.modifiers.command) { app.session.active().and_then(|d| edge_sheet(&d.wb.sheets, true)) } else { None }; match edge { - Some(i) => activate_sheet(app, i), + Some(i) => { + activate_sheet(app, i); + request_reveal(ui); + } None => app.run_or_alert("sheet.next", json!({})), } } @@ -181,10 +199,13 @@ pub fn sheet_tabs(app: &mut SheetApp, ui: &mut Ui) { // allow browsing other tabs without snapping back on every frame. let reveal_key = egui::Id::unique((d.uid, d.wb.active_sheet(), &sheets, tabs_width.to_bits())); let reveal_id = ui.scope_id().with("worksheet_tabs_reveal"); + let request_id = reveal_request_id(ui.scope_id()); let reveal_active = ui.ctx().data_mut(|data| { let changed = data.get_temp::(reveal_id) != Some(reveal_key); data.insert_temp(reveal_id, reveal_key); - changed + // A Ctrl-click on an arrow whose edge sheet is already active changes nothing above, + // yet asks to bring that tab into view. + changed | data.remove_temp::(request_id).unwrap_or(false) }); if view.show_sheet_tabs { ui.allocate_ui_with_layout(vec2(tabs_width, 30.0), egui::Layout::left_to_right(egui::Align::Center), |ui| { diff --git a/crates/ui-egui/tests/it/sheet_tabs.rs b/crates/ui-egui/tests/it/sheet_tabs.rs index 460162e67..725e761c6 100644 --- a/crates/ui-egui/tests/it/sheet_tabs.rs +++ b/crates/ui-egui/tests/it/sheet_tabs.rs @@ -58,6 +58,20 @@ fn click(h: &mut Harness, at: egui::Pos2) { h.run_steps(20); } +fn ctrl_click(h: &mut Harness, at: egui::Pos2) { + let m = Modifiers::COMMAND; + h.input_mut().events.extend([ + Event::ModifiersChanged(m), + Event::PointerMoved(at), + Event::PointerButton { pos: at, button: PointerButton::Primary, pressed: true, modifiers: m }, + ]); + h.step(); + h.input_mut().events.push(Event::PointerButton { pos: at, button: PointerButton::Primary, pressed: false, modifiers: m }); + h.run_steps(20); + h.input_mut().events.push(Event::ModifiersChanged(Modifiers::NONE)); + h.run_steps(2); +} + #[test] fn opening_and_activating_late_sheets_reveals_their_tabs() { for width in [500.0, 1000.0] { @@ -85,6 +99,36 @@ fn navigation_arrows_reveal_the_selected_tab() { assert_visible(&h, "Tab29"); } +/// A Ctrl/Cmd-click on the Next arrow jumps to the last sheet; repeated after the strip was scrolled +/// away from it, the already active edge tab is brought back into view. +#[test] +fn repeated_ctrl_click_on_the_arrow_reveals_the_active_edge_tab() { + let mut h = harness(700.0, 0); + let (_, clip) = tab(&h, "Tab00").unwrap(); + let next = pos2(clip.left() - 44.0, clip.center().y); + ctrl_click(&mut h, next); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 29); + assert_visible(&h, "Tab29"); + // Browse the strip back to the start without changing the selection. + h.input_mut().events.extend([Event::PointerMoved(clip.center())]); + for _ in 0..3 { + h.input_mut().events.push(Event::MouseWheel { + unit: egui::MouseWheelUnit::Point, + source: egui::MouseWheelSource::Unknown, + phase: egui::TouchPhase::Move, + delta: vec2(0.0, 600.0), + modifiers: Modifiers::NONE, + }); + h.run_steps(10); + } + h.run_steps(40); + assert!(!tab(&h, "Tab29").is_some_and(|(r, c)| c.contains_rect(r)), "the strip was scrolled away from the active tab"); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 29); + ctrl_click(&mut h, next); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 29); + assert_visible(&h, "Tab29"); +} + #[test] /// Horizontal wheel/trackpad browsing leaves the workbook unchanged, does not snap back on idle frames, and activating a sheet still reveals its tab. fn manual_wheel_and_trackpad_browsing_does_not_change_selection_or_snap_back() { From 05cffdceb1f8356c64ade34858edbcfde32ea608 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:27:07 +0100 Subject: [PATCH 40/51] Tab strip: test the Previous arrow reveal for the already active first tab Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/sheet_tabs.rs | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/ui-egui/tests/it/sheet_tabs.rs b/crates/ui-egui/tests/it/sheet_tabs.rs index 725e761c6..6703531a0 100644 --- a/crates/ui-egui/tests/it/sheet_tabs.rs +++ b/crates/ui-egui/tests/it/sheet_tabs.rs @@ -129,6 +129,35 @@ fn repeated_ctrl_click_on_the_arrow_reveals_the_active_edge_tab() { assert_visible(&h, "Tab29"); } +/// The same for the Previous arrow and the first sheet. +#[test] +fn repeated_ctrl_click_on_the_previous_arrow_reveals_the_first_tab() { + let mut h = harness(700.0, 29); + let (_, clip) = tab(&h, "Tab29").unwrap(); + let prev = pos2(clip.left() - 68.0, clip.center().y); + ctrl_click(&mut h, prev); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 0); + assert_visible(&h, "Tab00"); + // Browse the strip to its end without changing the selection. + h.input_mut().events.extend([Event::PointerMoved(clip.center())]); + for _ in 0..3 { + h.input_mut().events.push(Event::MouseWheel { + unit: egui::MouseWheelUnit::Point, + source: egui::MouseWheelSource::Unknown, + phase: egui::TouchPhase::Move, + delta: vec2(0.0, -600.0), + modifiers: Modifiers::NONE, + }); + h.run_steps(10); + } + h.run_steps(40); + assert!(!tab(&h, "Tab00").is_some_and(|(r, c)| c.contains_rect(r)), "the strip was scrolled away from the active tab"); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 0); + ctrl_click(&mut h, prev); + assert_eq!(h.state().session.doc().unwrap().wb.active_sheet(), 0); + assert_visible(&h, "Tab00"); +} + #[test] /// Horizontal wheel/trackpad browsing leaves the workbook unchanged, does not snap back on idle frames, and activating a sheet still reveals its tab. fn manual_wheel_and_trackpad_browsing_does_not_change_selection_or_snap_back() { From 6b19e445f44f59643fd673bbfeb5e635fbf5fdfe Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Sat, 7 Nov 2026 17:27:25 +0100 Subject: [PATCH 41/51] fix(chart): pass series index to data-table keys (#1150) --- crates/chart/src/render/table.rs | 2 +- crates/chart/src/tests/data_table.rs | 29 ++++++++++++++++++++++++++++ docs/chart-parity.md | 3 ++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/chart/src/render/table.rs b/crates/chart/src/render/table.rs index f0839dcfe..4f641afd2 100644 --- a/crates/chart/src/render/table.rs +++ b/crates/chart/src/render/table.rs @@ -70,7 +70,7 @@ pub(super) fn draw(ctx: &mut Ctx, chart: &Chart, data: &ChartData, plot: Bx, lay let y = top + (layout.headers as f32 + i as f32 + 0.5) * layout.row_height; let name_x = if settings.show_keys { x0 + 25.0 } else { x0 + 4.0 }; if settings.show_keys { - let swatch = match plot_kind(chart, series) { + let swatch = match plot_kind(chart, series, i) { ChartKind::Line | ChartKind::LineStacked => Swatch::Line, ChartKind::LineMarkers => Swatch::LineMarker, _ => Swatch::Box, diff --git a/crates/chart/src/tests/data_table.rs b/crates/chart/src/tests/data_table.rs index 13e2baf6c..ddeee07bc 100644 --- a/crates/chart/src/tests/data_table.rs +++ b/crates/chart/src/tests/data_table.rs @@ -1,6 +1,35 @@ use super::*; use gridcraft_model::ChartDataTable; +#[test] +fn combo_table_keys_use_each_series_index() { + let mut c = chart(ChartKind::Combo); + c.legend = LegendPos::None; + c.data_labels = false; + c.data_table = Some(ChartDataTable::default()); + let mut d = sample(c.kind); + // The public renderer also accepts Combo series: the first defaults to columns, later ones to lines. + for s in &mut d.series { + s.kind = ChartKind::Combo; + } + let p = render(&c, &d, 900.0, 500.0, &ApproxMeasure); + assert_sane(&p, 900.0, 500.0); + let header_y = p + .iter() + .find_map(|p| match p { + Prim::Text { text, y, .. } if text == "Q1" => Some(*y), + _ => None, + }) + .unwrap(); + for (i, s) in d.series.iter().enumerate() { + let boxes = p.iter().filter(|p| matches!(p, Prim::Rect { y, fill: Some(f), .. } if *f == s.color && *y > header_y)).count(); + let lines = + p.iter().filter(|p| matches!(p, Prim::Line { pts, color, .. } if *color == s.color && pts.iter().all(|p| p[1] > header_y))).count(); + assert_eq!(boxes, usize::from(i == 0), "series {i}: column key"); + assert_eq!(lines, usize::from(i > 0), "series {i}: line key"); + } +} + #[test] fn tables_draw_source_values_for_every_supported_chart_kind() { for kind in ALL_KINDS { diff --git a/docs/chart-parity.md b/docs/chart-parity.md index 311bd2635..39851f2bc 100644 --- a/docs/chart-parity.md +++ b/docs/chart-parity.md @@ -1,6 +1,6 @@ # Chart parity -> **Last reviewed:** 2026-10-18 · **Last updated:** 2026-11-02 · **Change:** minor (doughnut charts: every series is a concentric ring, the first series innermost; hole size and first slice angle kept; rotated slices cover their own sweep; legend names categories; earlier: charts resize from eight handles, chart data tables, legend keys and borders, native XLSX round-trip and print/PDF; earlier: selected chart source outlines stay beneath floating objects) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 +> **Last reviewed:** 2026-10-18 · **Last updated:** 2026-11-07 · **Change:** minor (chart-data-table keys pass the series index to the shared combo plot-kind rule, restoring compilation after #1150; earlier: doughnut charts: every series is a concentric ring, the first series innermost; hole size and first slice angle kept; rotated slices cover their own sweep; legend names categories; earlier: charts resize from eight handles, chart data tables, legend keys and borders, native XLSX round-trip and print/PDF; earlier: selected chart source outlines stay beneath floating objects) · **Target:** Microsoft Excel (Microsoft 365), Excel for Mac 16.113.4 Excel's chart types and chart elements against GridCraft. Part of [target-app-parity.md](target-app-parity.md). **Breadth ~50%, ready ~30%, 40–60 h** (estimated). @@ -107,6 +107,7 @@ are (#1686). Such series get no `numCache`/`strCache` on save (#1690). | Date | Change | Summary | |---|---|---| +| 2026-11-07 | minor | Chart-data-table keys use the shared plot-kind helper with their series index after #1150. A focused regression checks the first combo series uses a column key and later series use line keys. No readiness score change. | | 2026-11-02 | minor | Charts move and resize from eight handles, each moving only its own edges (#190); resizing leaves the work list (1–2 h, within the 40–60 h rounding) | | 2026-11-01 | minor | Chart data tables for column, bar, line, area and compatible combo charts: command/UI controls, formatted source values, blanks, grouped category headers, legend keys, independent borders, shared print/PDF rendering and native c:dTable import/export. Dense tables show a space notice; no readiness score change. | | 2026-10-29 | minor | Selected charts highlight their source cells beneath floating objects, so the blue outline no longer crosses the chart; uncovered source cells remain highlighted (#2861) | From 6a7cf9cf667956b486a719977106ff1f9152a7c4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:31:27 +0100 Subject: [PATCH 42/51] Chart: bound the category axis by the point cap, not the raw cell count Read only the cells the category levels use and cut the category dimension at the per-series cap, so a multi-level axis keeps all its levels. Co-Authored-By: Claude Sonnet 5.5 --- crates/chart/src/resolve.rs | 36 +++++++++++++++++++++---- crates/chart/src/tests/budget.rs | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 5 deletions(-) diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index 1ee635a14..bd6849435 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -189,8 +189,29 @@ fn value_text(v: &Value) -> String { /// taller than a row of values) groups its innermost labels, year beside month. Each outer level /// has one entry per category, non-empty where a group starts, as the cells are. fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limit: usize) -> (Vec, Vec>) { - let vals = eval_list_in(wb, sheet, formula, limit, 0); let cells: Option<(usize, RangeRef)> = parse_ref(wb, sheet, formula); + let value_range = parse_ref(wb, sheet, values).map(|(_, r)| r); + // Bound the category dimension by `limit` and read only the cells the levels use (the + // innermost label line and up to three outer ones), so that every kept category keeps its + // cells in every level. Counting raw cells would cut a two-level axis in the middle. + let levels = cells.and_then(|(_, r)| category_levels(&r, value_range.as_ref())); + let (cells, read) = match (cells, levels) { + (Some((sh, r)), Some(lv)) => { + let cut = u32::try_from(limit.saturating_sub(1)).unwrap_or(u32::MAX); + let outer = lv.outer; + let end = if lv.by_columns { CellRef::new(r.end.row.min(r.start.row.saturating_add(cut)), r.end.col) } else { CellRef::new(r.end.row, r.end.col.min(r.start.col.saturating_add(cut))) }; + let start = if lv.by_columns { CellRef::new(r.start.row, r.start.col.max(end.col.saturating_sub(outer))) } else { CellRef::new(r.start.row.max(end.row.saturating_sub(outer)), r.start.col) }; + let r = RangeRef::new(start, end); + let prefix = formula.rfind('!').and_then(|i| formula.get(..=i)).unwrap_or(""); + let cap = (r.width() as usize).saturating_mul(r.height() as usize); + (Some((sh, r)), Some((format!("{prefix}{r}"), cap))) + } + (c, _) => (c, None), + }; + let vals = match &read { + Some((f, cap)) => eval_list_in(wb, sheet, f, *cap, 0), + None => eval_list_in(wb, sheet, formula, limit, 0), + }; let src = cells.and_then(|(sh, r)| wb.sheet(sh).map(|s| (s, r))); let texts: Vec = vals .iter() @@ -210,8 +231,7 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi }) .collect(); let Some((_, r)) = src else { return (texts, vec![]) }; - let value_range = parse_ref(wb, sheet, values).map(|(_, r)| r); - let Some(levels) = category_levels(&r, value_range.as_ref()) else { return (texts, vec![]) }; + let Some(levels) = levels else { return (texts, vec![]) }; let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); if texts.len() != w.saturating_mul(h) { return (texts, vec![]); // truncated or not a plain block: a flat list @@ -252,7 +272,7 @@ fn series_name(wb: &Workbook, sheet: usize, name: &str) -> String { t.to_string() } -fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { +pub(crate) fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { match (chart_kind, series_kind) { (ChartKind::Combo, Some(k)) if k != ChartKind::Combo => k, (ChartKind::Combo, _) => { @@ -267,9 +287,15 @@ fn effective_kind(chart_kind: ChartKind, series_kind: Option, i: usiz } } +/// The kind series `i` is drawn as: its own kind in a combo chart (first series a column, the +/// rest lines when it has none), else the chart's. The one rule shared by resolving and drawing. +pub(crate) fn plot_kind(chart_kind: ChartKind, series_kind: Option, i: usize) -> ChartKind { + if chart_kind == ChartKind::Combo { effective_kind(chart_kind, series_kind, i) } else { chart_kind } +} + /// The kind series `i` of `chart` is drawn as: its own kind in a combo chart, else the chart's. pub fn series_plot_kind(chart: &Chart, i: usize) -> ChartKind { - if chart.kind == ChartKind::Combo { effective_kind(chart.kind, chart.series.get(i).and_then(|s| s.kind), i) } else { chart.kind } + plot_kind(chart.kind, chart.series.get(i).and_then(|s| s.kind), i) } /// Evaluates a chart's series formulas: names, values, categories (default "1", "2", …), X diff --git a/crates/chart/src/tests/budget.rs b/crates/chart/src/tests/budget.rs index 6c5a10879..fad349ba6 100644 --- a/crates/chart/src/tests/budget.rs +++ b/crates/chart/src/tests/budget.rs @@ -232,3 +232,48 @@ fn resolve_keeps_every_series_within_the_point_budget() { let counts: Vec = resolve(&wb, 0, &c).series.iter().map(|s| s.trendlines.len()).collect(); assert_eq!(counts, vec![9, 7, 0, 0]); } + +/// 100 series lower the per-series cap to 20,000, below the number of category cells of a +/// two-level axis (#1155). Each kept category keeps its inner label and its outer level. +fn two_level_chart(n: u32, vertical: bool) -> ChartData { + let at = |i: u32, level: u32| if vertical { CellRef::new(i, level) } else { CellRef::new(level, i) }; + let mut wb = Workbook::new(); + if let Some(s) = wb.sheet_mut(0) { + for i in 0..n { + s.set_value(at(i, 0), Value::Text(format!("Y{}", i / 3).into())); + s.set_value(at(i, 1), Value::Text(format!("C{i}").into())); + s.set_value(at(i, 2), Value::Number(f64::from(i))); + } + } + let cats = format!("{}:{}", at(0, 0), at(n - 1, 1)); + let values = format!("{}:{}", at(0, 2), at(n - 1, 2)); + let mut c = chart(ChartKind::ColumnClustered); + c.series = (0..100).map(|_| Series { values: values.clone(), categories: Some(cats.clone()), ..Series::default() }).collect(); + resolve(&wb, 0, &c) +} + +#[test] +fn multi_level_categories_keep_their_levels_under_the_point_cap() { + for vertical in [true, false] { + let d = two_level_chart(15_000, vertical); + assert_eq!(d.series[0].values.len(), 15_000, "vertical {vertical}"); + assert_eq!(d.categories.len(), 15_000, "vertical {vertical}"); + assert_eq!(d.categories[0], "C0"); + assert_eq!(d.categories[14_999], "C14999"); + assert_eq!(d.category_levels.len(), 1, "one outer level, vertical {vertical}"); + assert_eq!(d.category_levels[0].len(), 15_000); + assert_eq!(d.category_levels[0][3], "Y1"); + } +} + +#[test] +fn multi_level_categories_longer_than_the_cap_are_cut_together_with_their_levels() { + // 25,000 categories against a cap of 20,000. + let d = two_level_chart(25_000, true); + let cap = points_per_series(100); + assert_eq!(d.categories.len(), cap); + assert_eq!(d.category_levels.len(), 1); + assert_eq!(d.category_levels[0].len(), cap); + assert_eq!(d.categories[cap - 1], format!("C{}", cap - 1)); + assert_eq!(d.category_levels[0][cap - 3], format!("Y{}", (cap - 3) / 3)); +} From 9ba79c3e3850fc14e4ea12e1c2c76d3ef2952206 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:31:41 +0100 Subject: [PATCH 43/51] fmt --- crates/chart/src/resolve.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index bd6849435..11213a5e7 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -199,8 +199,16 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi (Some((sh, r)), Some(lv)) => { let cut = u32::try_from(limit.saturating_sub(1)).unwrap_or(u32::MAX); let outer = lv.outer; - let end = if lv.by_columns { CellRef::new(r.end.row.min(r.start.row.saturating_add(cut)), r.end.col) } else { CellRef::new(r.end.row, r.end.col.min(r.start.col.saturating_add(cut))) }; - let start = if lv.by_columns { CellRef::new(r.start.row, r.start.col.max(end.col.saturating_sub(outer))) } else { CellRef::new(r.start.row.max(end.row.saturating_sub(outer)), r.start.col) }; + let end = if lv.by_columns { + CellRef::new(r.end.row.min(r.start.row.saturating_add(cut)), r.end.col) + } else { + CellRef::new(r.end.row, r.end.col.min(r.start.col.saturating_add(cut))) + }; + let start = if lv.by_columns { + CellRef::new(r.start.row, r.start.col.max(end.col.saturating_sub(outer))) + } else { + CellRef::new(r.start.row.max(end.row.saturating_sub(outer)), r.start.col) + }; let r = RangeRef::new(start, end); let prefix = formula.rfind('!').and_then(|i| formula.get(..=i)).unwrap_or(""); let cap = (r.width() as usize).saturating_mul(r.height() as usize); From 9354fdb0c4d155b1124facda01142564b24d7887 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:37:20 +0100 Subject: [PATCH 44/51] Chart: fit labels with a line break by a linear scan The width of a prefix plus an ellipsis is not monotone when the text has a line break, so the binary search could drop a label that fits. Also pass the series index to plot_kind in the data table (upstream/main does not compile without it). Co-Authored-By: Claude Sonnet 5.5 --- crates/chart/src/render.rs | 6 ++++ crates/chart/src/render/table.rs | 2 +- crates/chart/src/render_perf_tests.rs | 41 +++++++++++++++++++++++++++ crates/ui-egui/src/chartview.rs | 27 ++++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) diff --git a/crates/chart/src/render.rs b/crates/chart/src/render.rs index 1175ffc19..65b6d4f1a 100644 --- a/crates/chart/src/render.rs +++ b/crates/chart/src/render.rs @@ -71,6 +71,8 @@ impl Ctx<'_> { /// shrink as the prefix grows, which holds for per-character advance sums (`ApproxMeasure`, the /// PDF metrics) and for the UI's text layout up to kerning of a few hundredths of a point; with /// such a non-monotone measure the result may differ from the old linear scan by a character. + /// A label with a line break is not monotone (the ellipsis moves to a new line and the widest + /// line can get shorter), so it takes the linear scan, longest prefix first. fn fit(&self, s: &str, size: f32, maxw: f32) -> String { if self.tw(s, size) <= maxw { return s.to_string(); @@ -86,6 +88,10 @@ impl Ctx<'_> { t.push('…'); self.tw(&t, size) <= maxw }; + if s.contains('\n') { + let k = (1..=top).rev().find(|&k| fits(k)); + return k.map_or_else(String::new, |k| format!("{}…", s.get(..starts.get(k).copied().unwrap_or(s.len())).unwrap_or(""))); + } let (mut lo, mut hi) = (0usize, top); while lo < hi { let mid = lo + (hi - lo).div_ceil(2); diff --git a/crates/chart/src/render/table.rs b/crates/chart/src/render/table.rs index f0839dcfe..4f641afd2 100644 --- a/crates/chart/src/render/table.rs +++ b/crates/chart/src/render/table.rs @@ -70,7 +70,7 @@ pub(super) fn draw(ctx: &mut Ctx, chart: &Chart, data: &ChartData, plot: Bx, lay let y = top + (layout.headers as f32 + i as f32 + 0.5) * layout.row_height; let name_x = if settings.show_keys { x0 + 25.0 } else { x0 + 4.0 }; if settings.show_keys { - let swatch = match plot_kind(chart, series) { + let swatch = match plot_kind(chart, series, i) { ChartKind::Line | ChartKind::LineStacked => Swatch::Line, ChartKind::LineMarkers => Swatch::LineMarker, _ => Swatch::Box, diff --git a/crates/chart/src/render_perf_tests.rs b/crates/chart/src/render_perf_tests.rs index ce15e7861..9d2d7177e 100644 --- a/crates/chart/src/render_perf_tests.rs +++ b/crates/chart/src/render_perf_tests.rs @@ -60,6 +60,30 @@ impl Measure for Counting<'_> { } } +/// Like a text layout that breaks at `\n`: the width is that of the widest line. Adding a +/// newline can make the text narrower, so the width of `prefix + "…"` is not monotone. +struct WidestLine; + +impl Measure for WidestLine { + fn text_width(&self, text: &str, size: f32, _bold: bool) -> f32 { + text.split('\n') + .map(|line| { + line.chars() + .map(|c| { + size * if c == 'i' { + 0.25 + } else if c == 'M' { + 0.9 + } else { + 0.5 + } + }) + .sum::() + }) + .fold(0.0, f32::max) + } +} + fn corpus() -> Vec { let mut v: Vec = ["", "a", "ab", "Hello, world", "AVAVAV tight", "日本語のラベルです", "👨‍👩‍👧 family 🎉 party", "e\u{301}e\u{301}e\u{301} combining", "…", "……"] @@ -71,6 +95,9 @@ fn corpus() -> Vec { v.push("x".repeat(401)); v.push("é🎉".repeat(500)); v.push("long label ".repeat(100)); + for s in ["iiii\nMM", "iiii\n", "\niiii", "ab\ncd\nef", "iiiiiiii\niiiiMMMM\nMM", "line one\nline two is longer\nx"] { + v.push(s.to_string()); + } v } @@ -89,6 +116,20 @@ fn run_fit(m: &dyn Measure) { fn fit_matches_linear_reference() { run_fit(&ApproxMeasure); run_fit(&Varied { calls: Cell::new(0) }); + run_fit(&WidestLine); +} + +#[test] +fn a_multiline_label_keeps_its_fitting_prefix() { + // The whole label is 18 wide and "iii…" is 12.5, both too wide for 10.5, but "iiii\n…" is + // 10 wide and fits. A binary search that stops at the failing midpoint would return nothing. + let ctx = Ctx { out: vec![], m: &WidestLine }; + let s = "iiii\nMM"; + let size = 10.0; + let maxw = 10.5; + assert!(ctx.tw(s, size) > maxw); + assert!(ctx.tw("iii…", size) > maxw && ctx.tw("iiii\n…", size) <= maxw); + assert_eq!(ctx.fit(s, size, maxw), "iiii\n…"); } #[test] diff --git a/crates/ui-egui/src/chartview.rs b/crates/ui-egui/src/chartview.rs index f1a366d9e..dbfba039c 100644 --- a/crates/ui-egui/src/chartview.rs +++ b/crates/ui-egui/src/chartview.rs @@ -581,3 +581,30 @@ pub fn hit(app: &SheetApp, geo: &Geo, sh: &Sheet, p: Pos2) -> Option<(ObjRef, Re pub(crate) fn ink_color(line: Color32, width_pt: f32) -> Color32 { if width_pt > 8.0 { line.gamma_multiply(0.45) } else { line } } + +#[cfg(test)] +mod tests { + use gridcraft_chart::Measure; + + use super::*; + + /// The production measure is not monotone for labels with a line break: egui lays the text out + /// line by line, so ending a prefix with `\n…` can be narrower than the shorter `iii…`. The + /// chart's label fitting must not assume otherwise (#1152). + #[test] + fn egui_measure_shrinks_when_a_prefix_ends_in_a_line_break() { + let ctx = egui::Context::default(); + ctx.set_fonts(theme::pinned_font_definitions()); + let mut checked = false; + // Fonts are ready from the second pass on. + for _ in 0..2 { + let mut out = ctx.run_ui(Default::default(), |ui| { + let m = EguiMeasure { painter: ui.painter(), z: 1.0 }; + let (short, long) = (m.text_width("iii…", 11.0, false), m.text_width("iiii\n…", 11.0, false)); + checked = long < short; + }); + out.textures_delta.clear(); + } + assert!(checked); + } +} From 640fbf22f64e8726188ff604480a2c6f3e6aad42 Mon Sep 17 00:00:00 2001 From: richard1912 <59070720+richard1912@users.noreply.github.com> Date: Sun, 8 Nov 2026 12:30:06 +1100 Subject: [PATCH 45/51] fix(edit): paste, Ctrl+Enter, Fill and Copy skip rows a filter hides (#1101) (#1175) * Paste, Ctrl+Enter and Fill skip rows hidden by a filter (#1101) Repeating one cell or row down a filtered selection (paste of a single cell or row, Ctrl+Enter, Fill Down/Up/Right/Left) now leaves the rows an applied filter hides alone, as Excel does. A taller pasted block still goes straight down, and rows hidden by hand are still written, also as in Excel. Each case was measured in Excel for Microsoft 365. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01A2bJ3LxrZ6YnmppgrfwHwr * Copying a filtered range takes only the rows showing (#1101) A copied range with rows hidden by an applied filter is split into its runs of showing rows and packed like a multi-area copy (formulas as values), so pasting it inside GridCraft no longer brings the hidden rows back. One run is copied as a plain range, keeping formulas. Cut still moves every row. All as measured in Excel for Microsoft 365. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01A2bJ3LxrZ6YnmppgrfwHwr * Find the rows a filter hides without rescanning every table per row filtered_rows asks the sheet for its filter row spans once instead of checking every table for each hidden row. * Paste over filtered rows leaves their validation rules alone When one row is tiled over filtered rows, paste_validations no longer clears or copies rules on the rows the filter hides. * Filtered copy packs links and validation with the rows showing slide_together now moves hyperlinks and validation rules with their rows, and a filtered whole-row copy records the packed rows as its selection so Insert Copied Cells inserts only the rows copied. * Group validation paste options and format the new tests * Pack a filtered copy in one pass over notes, merges, links and spills slide_together finds each entry's area by search instead of scanning every map once per area, so a filter leaving thousands of separate runs no longer costs runs times map size on every copy. --------- Co-authored-by: Richard Co-authored-by: Claude Opus 5.5 Co-authored-by: skylar --- crates/engine/src/cmd/edit.rs | 7 +- crates/engine/src/cmd/edit/cell.rs | 4 + crates/engine/src/cmd/edit/clipboard.rs | 142 +++++++++-- crates/engine/src/cmd/edit/fill.rs | 3 +- crates/engine/src/cmd/edit/paste_special.rs | 48 +++- crates/engine/src/fill.rs | 15 +- crates/engine/src/tests.rs | 266 ++++++++++++++++++++ crates/model/src/sheet.rs | 14 +- 8 files changed, 469 insertions(+), 30 deletions(-) diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 83133c01c..8c7072f52 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -239,7 +239,12 @@ fn redo(s: &mut Session, p: &Json) -> Result { /// The rows of `range` an applied filter hides (rows hidden by hand are not among them). fn filtered_rows(sh: &gridcraft_model::Sheet, range: RangeRef) -> std::collections::BTreeSet { - sh.rows.range(range.start.row..=range.end.row).filter(|(r, info)| info.hidden && sh.is_row_filtered(**r)).map(|(r, _)| *r).collect() + let spans = sh.filter_row_spans(); + if spans.is_empty() { + return Default::default(); + } + let filtered = |row: u32| spans.iter().any(|&(first, last)| row >= first && row <= last); + sh.rows.range(range.start.row..=range.end.row).filter(|(r, info)| info.hidden && filtered(**r)).map(|(r, _)| *r).collect() } /// The Clear commands (Delete is Clear Contents). As in Excel, the rows an applied filter hides diff --git a/crates/engine/src/cmd/edit/cell.rs b/crates/engine/src/cmd/edit/cell.rs index 610135682..c3fe44385 100644 --- a/crates/engine/src/cmd/edit/cell.rs +++ b/crates/engine/src/cmd/edit/cell.rs @@ -328,7 +328,11 @@ pub(super) fn range_fill(s: &mut Session, p: &Json) -> Result { if r.count() > 2_000_000 { return Err(bad("range.fill", "range too large")); } + let skip = cx.wb.sheet(sheet).map(|sh| super::filtered_rows(sh, *r)).unwrap_or_default(); for at in r.iter() { + if skip.contains(&at.row) { + continue; + } let style = cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(); let base = match by_style.get(&style) { Some(c) => c.clone(), diff --git a/crates/engine/src/cmd/edit/clipboard.rs b/crates/engine/src/cmd/edit/clipboard.rs index da34c017a..9feb380a0 100644 --- a/crates/engine/src/cmd/edit/clipboard.rs +++ b/crates/engine/src/cmd/edit/clipboard.rs @@ -1,8 +1,8 @@ use super::cell::input_to_cell; use super::paste_special::paste_special; use super::{ - Cell, Clipboard, EngineError, Json, RangeRef, Result, Session, Value, bad, bool_param, cell_param_on, edit, json, ok, str_param, target_ranges, - target_sheet, + Cell, CellRef, Clipboard, EngineError, Json, RangeRef, Result, Session, Value, bad, bool_param, cell_param_on, edit, json, ok, str_param, + target_ranges, target_sheet, }; // ---------------------------------------------------------------- clipboard @@ -15,7 +15,7 @@ fn copy_impl(s: &mut Session, p: &Json, cut: bool) -> Result { return Err(multiple()); } // The selection as made (untrimmed), for Insert Copied Cells. - let selected = ranges.first().copied().unwrap_or_default(); + let mut selected = ranges.first().copied().unwrap_or_default(); let d = s.doc()?; let Some(sh) = d.wb.sheet(sheet) else { return Err(EngineError::NoDocument) }; let trim = |range: RangeRef| { @@ -31,7 +31,27 @@ fn copy_impl(s: &mut Session, p: &Json, cut: bool) -> Result { let (wb, block) = slide_together(&d.wb, sheet, &areas).ok_or_else(multiple)?; (std::sync::Arc::new(wb), block, areas) } else { - (d.wb.clone(), trim(ranges.first().copied().unwrap_or_default()), Vec::new()) + let range = trim(ranges.first().copied().unwrap_or_default()); + // A copied range with rows a filter hides takes only the rows showing: they are copied + // like a multi-area selection stacked top to bottom (formulas as values), as in Excel. + // A cut still takes every row. + let hidden = if cut { Default::default() } else { super::filtered_rows(sh, range) }; + let visible = if hidden.is_empty() { Vec::new() } else { visible_row_runs(range, &hidden) }; + let any_visible = !visible.is_empty(); + let packed = match visible.as_slice() { + [] => (d.wb.clone(), range, Vec::new()), + [only] => (d.wb.clone(), *only, Vec::new()), + _ => { + let (wb, block) = slide_together(&d.wb, sheet, &visible).ok_or_else(multiple)?; + (std::sync::Arc::new(wb), block, visible) + } + }; + // Whole rows copied with some hidden by a filter come as the block of rows showing, so + // Insert Copied Cells makes room for those rows, not for the hidden ones too. + if !hidden.is_empty() && any_visible && selected.is_full_rows() { + selected = RangeRef::rows(packed.1.start.row, packed.1.end.row); + } + packed }; let text = crate::display::range_text(&wb, sheet, range); // The HTML rendering is for the host clipboard (UI); it can reach 4 MB, so agents and @@ -46,6 +66,56 @@ fn copy_impl(s: &mut Session, p: &Json, cut: bool) -> Result { Ok(result) } +/// The runs of consecutive rows of `range` not in `hidden`, as ranges of `range`'s columns. +fn visible_row_runs(range: RangeRef, hidden: &std::collections::BTreeSet) -> Vec { + let mut runs = Vec::new(); + let mut start = None; + for row in range.start.row..=range.end.row { + match (hidden.contains(&row), start) { + (false, None) => start = Some(row), + (true, Some(first)) => { + runs.push(RangeRef::new(CellRef::new(first, range.start.col), CellRef::new(row - 1, range.end.col))); + start = None; + } + _ => {} + } + } + if let Some(first) = start { + runs.push(RangeRef::new(CellRef::new(first, range.start.col), CellRef::new(range.end.row, range.end.col))); + } + runs +} + +/// The validation rules of `src` cut into the areas of a packed block: one rule per area a rule +/// touches, over that area's part moved by the area's shift. `sorted` areas run in order. +fn pack_validations(src: &gridcraft_model::Sheet, sorted: &[RangeRef], shifts: &[(i64, i64)], across: bool) -> Vec { + let key = |c: CellRef| if across { c.col } else { c.row }; + let mut packed_rules = Vec::new(); + for dv in &src.validations { + let Some(from) = dv.ranges.first().map(|r| r.start) else { continue }; + // The areas each of the rule's ranges touches lie side by side in the sorted list. + let end_key = |a: &RangeRef| if across { a.end.col } else { a.end.row }; + let mut by_area: std::collections::BTreeMap> = Default::default(); + for r in &dv.ranges { + let first = sorted.partition_point(|a| end_key(a) < key(r.start)); + for (i, a) in sorted.iter().enumerate().skip(first).take_while(|(_, a)| key(a.start) <= end_key(r)) { + let (dr, dc) = shifts[i]; + let moved = r.intersection(a).and_then(|x| Some(RangeRef::new(x.start.offset(dr, dc)?, x.end.offset(dr, dc)?))); + by_area.entry(i).or_default().extend(moved); + } + } + for ranges in by_area.into_values() { + let Some(to) = ranges.first().map(|r| r.start) else { continue }; + let mut rule = dv.clone(); + rule.f1 = super::super::data::move_rule_formula(&dv.f1, from, to); + rule.f2 = dv.f2.as_ref().map(|f| super::super::data::move_rule_formula(f, from, to)); + rule.ranges = ranges; + packed_rules.push(rule); + } + } + packed_rules +} + /// Excel copies a multi-area selection only when the areas line up: all cover the same rows (they /// slide together side by side, left to right) or all cover the same columns (stacked, top to /// bottom), without overlapping. Returns a copy of the workbook whose sheet holds the areas packed @@ -80,17 +150,35 @@ fn slide_together(wb: &gridcraft_model::Workbook, sheet: usize, areas: &[RangeRe dst.spill.retain(|c, _| !block.contains(*c)); dst.spill_ranges.retain(|c, _| !block.contains(*c)); dst.comments.retain(|c, _| !block.contains(*c)); + dst.hyperlinks.retain(|c, _| !block.contains(*c)); + super::super::data::remove_validation_cells(&mut dst.validations, &[block]); dst.merges.retain(|m| !m.intersects(&block)); if across { dst.cols.retain(|c, _| !(block.start.col..=block.end.col).contains(c)); } else { dst.rows.retain(|r, _| !(block.start.row..=block.end.row).contains(r)); } + // Where each area lands: the shift of its cells, and its first row or column in the block. + let key = |c: CellRef| if across { c.col } else { c.row }; let mut next = if across { anchor.col } else { anchor.row }; + let mut shifts: Vec<(i64, i64)> = Vec::with_capacity(sorted.len()); for a in &sorted { - let (dr, dc) = if across { (0, i64::from(next) - i64::from(a.start.col)) } else { (i64::from(next) - i64::from(a.start.row), 0) }; - let cells = src.cells.iter_range(*a).map(|(c, _)| c).chain(src.spill.keys().copied().filter(|c| a.contains(*c))); - for c in cells { + let to = i64::from(next); + shifts.push(if across { (0, to - i64::from(a.start.col)) } else { (to - i64::from(a.start.row), 0) }); + next = next.saturating_add(if across { a.width() } else { a.height() }); + } + // The areas never overlap and run in order, so the area holding a cell is found by search: + // each map below is read once, whatever the number of areas. + let area_of = |c: CellRef| -> Option { + let i = sorted.partition_point(|a| key(a.start) <= key(c)).checked_sub(1)?; + sorted.get(i)?.contains(c).then_some(i) + }; + let moved = |c: CellRef| -> Option { + let (dr, dc) = shifts[area_of(c)?]; + c.offset(dr, dc) + }; + for (a, &(dr, dc)) in sorted.iter().zip(&shifts) { + for (c, _) in src.cells.iter_range(*a) { let Some(to) = c.offset(dr, dc) else { continue }; let style = src.cell(c).map(|x| x.style).unwrap_or_default(); dst.set_cell(to, Cell { value: src.value(c), formula: None, style, py: None }); @@ -98,17 +186,38 @@ fn slide_together(wb: &gridcraft_model::Workbook, sheet: usize, areas: &[RangeRe dst.cell_pictures.insert(to, picture.clone()); } } - for (c, comment) in src.comments.iter().filter(|(c, _)| a.contains(**c)) { - if let Some(to) = c.offset(dr, dc) { - dst.comments.insert(to, comment.clone()); - } + } + for c in src.spill.keys().copied() { + let Some(to) = moved(c) else { continue }; + let style = src.cell(c).map(|x| x.style).unwrap_or_default(); + dst.set_cell(to, Cell { value: src.value(c), formula: None, style, py: None }); + if let Some(picture) = src.cell_pictures.get(&c) { + dst.cell_pictures.insert(to, picture.clone()); } - for m in src.merges.iter().filter(|m| a.contains_range(m)) { - if let (Some(s), Some(e)) = (m.start.offset(dr, dc), m.end.offset(dr, dc)) { - dst.merges.push(RangeRef::new(s, e)); - } + } + // Links and validation rules follow their cells too, so a row the filter hides leaves its own + // behind. + for (c, link) in &src.hyperlinks { + if let Some(to) = moved(*c) { + dst.hyperlinks.insert(to, link.clone()); } - // Column widths (side by side) or row heights and hidden rows (stacked) follow their cells. + } + let packed_rules = pack_validations(src, &sorted, &shifts, across); + for (c, comment) in &src.comments { + if let Some(to) = moved(*c) { + dst.comments.insert(to, comment.clone()); + } + } + for m in &src.merges { + let Some(i) = area_of(m.start).filter(|&i| sorted[i].contains_range(m)) else { continue }; + let (dr, dc) = shifts[i]; + if let (Some(s), Some(e)) = (m.start.offset(dr, dc), m.end.offset(dr, dc)) { + dst.merges.push(RangeRef::new(s, e)); + } + } + // Column widths (side by side) or row heights and hidden rows (stacked) follow their cells. + let mut next = if across { anchor.col } else { anchor.row }; + for a in &sorted { if across { for (c, info) in src.cols.range(a.start.col..=a.end.col) { dst.cols.insert(next.saturating_add(c.saturating_sub(a.start.col)), *info); @@ -120,6 +229,7 @@ fn slide_together(wb: &gridcraft_model::Workbook, sheet: usize, areas: &[RangeRe } next = next.saturating_add(if across { a.width() } else { a.height() }); } + dst.validations.extend(packed_rules); Some((out, block)) } diff --git a/crates/engine/src/cmd/edit/fill.rs b/crates/engine/src/cmd/edit/fill.rs index 8cd1e3d88..489eca2a4 100644 --- a/crates/engine/src/cmd/edit/fill.rs +++ b/crates/engine/src/cmd/edit/fill.rs @@ -31,8 +31,9 @@ pub(super) fn fill_dir(s: &mut Session, p: &Json, dr: i64, dc: i64) -> Result Result { && !sel.is_full_rows() && sel.count() <= 4_000_000; let (tiles_r, tiles_c) = if tile { (sel.height() / h, sel.width() / w) } else { (1, 1) }; + // A single cell or row repeated down a filtered selection skips the rows the filter hides; a + // taller block is pasted straight down, hidden rows included, as in Excel. + let skip_rows = match s.doc()?.wb.sheet(dest_sheet) { + Some(sh) if tile && h == 1 => super::filtered_rows(sh, sel), + _ => Default::default(), + }; // Excel refuses a paste that would run off the sheet (whole columns pasted below row 1, whole // rows right of column A, a whole column transposed) instead of pasting part of it. if at.offset(i64::from(tiles_r * h) - 1, i64::from(tiles_c * w) - 1).is_none() { @@ -134,6 +140,9 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { let sc = CellRef::new(src.start.row + r, src.start.col + c); let (dr, dc) = if transpose { (c, r) } else { (r, c) }; let Some(dest) = at.offset((tr * h + dr) as i64, (tc * w + dc) as i64) else { continue }; + if skip_rows.contains(&dest.row) { + continue; + } let src_cell = if cut_move { moved_formulas.get(&sc).cloned() } else { src_sheet.cell(sc).cloned() }; let src_val = src_sheet.value(sc); let src_picture = if cut_move { moved_pictures.get(&sc) } else { src_sheet.cell_pictures.get(&sc) }.cloned(); @@ -326,7 +335,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { } let pasted = RangeRef::new(at, at.offset_clamped(i64::from(tiles_r * h) - 1, i64::from(tiles_c * w) - 1)); if !link && matches!(what, What::All | What::AllExceptBorders | What::Validation) { - paste_validations(cx, src_sheet, src, dest_sheet, pasted, transpose, skip_blanks)?; + paste_validations(cx, src_sheet, src, dest_sheet, pasted, PasteMode { transpose, skip_blanks, skip_rows: &skip_rows })?; } if what == What::ColumnWidths { // Each pasted column takes the width of the column copied to it (transposed, the @@ -394,6 +403,31 @@ const MAX_PASTE_LINK_CELLS: u64 = 1_048_576; /// selection makes one range per tile). const MAX_PASTE_VALIDATION_RANGES: usize = 100_000; +/// How a validation paste lays the rules out: transposed or not, keeping the destination's rule under +/// blank source cells, and leaving the rows a filter hides as they are. +#[derive(Clone, Copy)] +struct PasteMode<'a> { + transpose: bool, + skip_blanks: bool, + skip_rows: &'a std::collections::BTreeSet, +} + +/// `range` less the rows in `skip`: the runs of rows between them, as ranges of its columns. +fn without_rows(range: RangeRef, skip: &std::collections::BTreeSet) -> Vec { + let mut out = Vec::new(); + let mut next = range.start.row; + for &row in skip.range(range.start.row..=range.end.row) { + if row > next { + out.push(RangeRef::new(CellRef::new(next, range.start.col), CellRef::new(row - 1, range.end.col))); + } + next = row + 1; + } + if next <= range.end.row { + out.push(RangeRef::new(CellRef::new(next, range.start.col), CellRef::new(range.end.row, range.end.col))); + } + out +} + /// Pasting validation works on ranges, as Excel stores it: the pasted rectangle is cut out of the /// destination's rules (cells whose source had no rule end up with none), then each source rule /// that covers part of the copied block `src` comes along as one rule over the translated parts. @@ -404,9 +438,9 @@ fn paste_validations( src: RangeRef, dest_sheet: usize, pasted: RangeRef, - transpose: bool, - skip_blanks: bool, + mode: PasteMode<'_>, ) -> Result<()> { + let PasteMode { transpose, skip_blanks, skip_rows } = mode; let (h, w) = if transpose { (src.width(), src.height()) } else { (src.height(), src.width()) }; let (tiles_r, tiles_c) = (pasted.height() / h.max(1), pasted.width() / w.max(1)); let mut rules = Vec::new(); @@ -426,7 +460,9 @@ fn paste_validations( let (r, c) = if transpose { (c, r) } else { (r, c) }; for tr in 0..tiles_r { for tc in 0..tiles_c { - if let Some(d) = pasted.start.offset(i64::from(tr * h + r), i64::from(tc * w + c)) { + if let Some(d) = pasted.start.offset(i64::from(tr * h + r), i64::from(tc * w + c)) + && !skip_rows.contains(&d.row) + { cells.push((d.row, d.col)); } } @@ -490,6 +526,8 @@ fn paste_validations( } } } + // The rows a filter hides keep what they have. + let ranges: Vec = ranges.into_iter().flat_map(|r| without_rows(r, skip_rows)).collect(); let Some(anchor) = ranges.first().map(|r| r.start) else { continue }; let mut rule = dv.clone(); // Relative references move with the rule, as pasted formulas do: they are written for the @@ -507,7 +545,7 @@ fn paste_validations( cleared.extend(rules.iter().flat_map(|r| r.ranges.iter().copied())); super::super::data::remove_validation_cells(&mut sh.validations, &cleared); } else { - super::super::data::remove_validation_cells(&mut sh.validations, &[pasted]); + super::super::data::remove_validation_cells(&mut sh.validations, &without_rows(pasted, skip_rows)); } sh.validations.extend(rules); Ok(()) diff --git a/crates/engine/src/fill.rs b/crates/engine/src/fill.rs index efa09310a..25e137878 100644 --- a/crates/engine/src/fill.rs +++ b/crates/engine/src/fill.rs @@ -87,6 +87,18 @@ pub fn holds_source_in_corner(src: RangeRef, target: RangeRef) -> bool { /// Fills `target` (which contains `src` at one edge) from the pattern in `src`. pub fn fill(cx: &mut Ctx, sheet: usize, src: RangeRef, target: RangeRef, mode: FillMode) -> Result<()> { + fill_except(cx, sheet, src, target, mode, &std::collections::BTreeSet::new()) +} + +/// [`fill`], leaving the rows in `skip_rows` untouched (Fill Down over a filtered range). +pub fn fill_except( + cx: &mut Ctx, + sheet: usize, + src: RangeRef, + target: RangeRef, + mode: FillMode, + skip_rows: &std::collections::BTreeSet, +) -> Result<()> { let custom = cx.wb.custom_lists.clone(); let Some(sh) = cx.wb.sheet(sheet) else { return Ok(()) }; // Direction: down/up if target extends rows, else right/left. @@ -119,7 +131,7 @@ pub fn fill(cx: &mut Ctx, sheet: usize, src: RangeRef, target: RangeRef, mode: F (c, sh.cell(c).cloned()) }) .collect(); - let targets: Vec = if vertical { + let mut targets: Vec = if vertical { let rows: Vec = if forward { (src.end.row + 1..=target.end.row).collect() } else { (target.start.row..src.start.row).rev().collect() }; rows.into_iter().map(|r| CellRef::new(r, lane)).collect() @@ -128,6 +140,7 @@ pub fn fill(cx: &mut Ctx, sheet: usize, src: RangeRef, target: RangeRef, mode: F if forward { (src.end.col + 1..=target.end.col).collect() } else { (target.start.col..src.start.col).rev().collect() }; cols.into_iter().map(|c| CellRef::new(lane, c)).collect() }; + targets.retain(|t| !skip_rows.contains(&t.row)); let fmt = |c: &(CellRef, Option)| cx.wb.styles.get(c.1.as_ref().map(|c| c.style).unwrap_or_default()).num_fmt.as_str(); let pattern = if mode.extends_series() || mode == FillMode::Alternate { let first_fmt = src_cells.first().map(fmt).unwrap_or_default(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 445ccf545..f97e1bc10 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -4997,6 +4997,246 @@ fn precision_as_displayed_rounds_numbers_written_to_a_new_sheet() { assert!(rounded > 0, "the histogram's formatted cells should be rounded"); } +/// #1101: repeating one cell or row down a filtered selection (paste, Ctrl+Enter, Fill Down) skips +/// the rows the filter hides, as in Excel. A taller pasted block goes straight down, and rows +/// hidden by hand are written, also as in Excel (each case measured in Excel for Microsoft 365). +#[test] +fn fills_and_single_row_pastes_skip_filtered_rows() { + let fresh = s; + let filtered = || { + let mut s = fresh(); + let mut rows = vec![json!(["Val", "Tag"])]; + for i in 2..=11 { + rows.push(json!([format!("v{i}"), if [3, 5, 7, 9].contains(&i) { "hide" } else { "keep" }])); + } + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + s.execute("range.setValues", json!({"range": "E1", "values": [["X", "W"], ["Y", ""]]})).unwrap(); + s.execute("selection.set", json!({"range": "A1:B1"})).unwrap(); + s.execute("data.filter", json!({})).unwrap(); + assert_eq!(s.execute("data.filterBy", json!({"column": "B", "values": ["keep"]})).unwrap()["hiddenRows"], 4); + s + }; + let col = |s: &Session, c: &str| -> Vec { (2..=11).map(|r| v(s, &format!("{c}{r}"))).collect() }; + let skipped = + |x: &str| -> Vec { (2..=11).map(|r| Value::from(if [3, 5, 7, 9].contains(&r) { format!("v{r}") } else { x.to_string() })).collect() }; + + // One cell pasted over the filtered rows. + let mut s = filtered(); + s.execute("edit.copy", json!({"range": "E1"})).unwrap(); + s.execute("selection.set", json!({"range": "A2:A11"})).unwrap(); + s.execute("edit.paste", json!({})).unwrap(); + assert_eq!(col(&s, "A"), skipped("X")); + + // One row (two cells) pasted over the filtered rows. + let mut s = filtered(); + s.execute("edit.copy", json!({"range": "E1:F1"})).unwrap(); + s.execute("selection.set", json!({"range": "A2:B11"})).unwrap(); + s.execute("edit.paste", json!({})).unwrap(); + assert_eq!(col(&s, "A"), skipped("X")); + assert_eq!(v(&s, "B3"), Value::from("hide")); + assert_eq!(v(&s, "B4"), Value::from("W")); + + // A two-row block is pasted straight down, hidden rows included. + let mut s = filtered(); + s.execute("edit.copy", json!({"range": "E1:E2"})).unwrap(); + s.execute("selection.set", json!({"range": "A2:A11"})).unwrap(); + s.execute("edit.paste", json!({})).unwrap(); + assert_eq!(col(&s, "A"), ["X", "Y", "X", "Y", "X", "Y", "X", "Y", "X", "Y"].map(Value::from)); + + // Ctrl+Enter. + let mut s = filtered(); + s.execute("selection.set", json!({"range": "A2:A11"})).unwrap(); + s.execute("range.fill", json!({"input": "Z"})).unwrap(); + assert_eq!(col(&s, "A"), skipped("Z")); + + // Fill Down, and a formula filled down still adjusts to the row it lands in. + let mut s = filtered(); + s.execute("edit.fillDown", json!({"range": "A2:A11"})).unwrap(); + assert_eq!(col(&s, "A"), skipped("v2")); + s.execute("cell.set", json!({"cell": "C2", "input": "=ROW()"})).unwrap(); + s.execute("edit.fillDown", json!({"range": "C2:C11"})).unwrap(); + assert_eq!(v(&s, "C4"), Value::Number(4.0)); + assert!(v(&s, "C3").is_empty()); + + // Fill Up and Fill Right skip them too. + let mut s = filtered(); + s.execute("edit.fillUp", json!({"range": "A2:A11"})).unwrap(); + assert_eq!(col(&s, "A"), skipped("v11")); + let mut s = filtered(); + s.execute("range.setValues", json!({"range": "C2", "values": [["c2"], ["c3"], ["c4"]]})).unwrap(); + s.execute("edit.fillRight", json!({"range": "A2:C4"})).unwrap(); + assert_eq!([v(&s, "C2"), v(&s, "C3"), v(&s, "C4")], [Value::from("v2"), Value::from("c3"), Value::from("v4")]); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(v(&s, "C4"), Value::from("c4")); + + // Copying the filtered range takes only the rows showing, formulas as their values; a cut + // still moves every row. + let mut s = filtered(); + s.execute("cell.set", json!({"cell": "C2", "input": "=ROW()"})).unwrap(); + s.execute("edit.fillDown", json!({"range": "C2:C11"})).unwrap(); + let r = s.execute("edit.copy", json!({"range": "A1:C11"})).unwrap(); + assert_eq!( + r["text"], + "Val Tag +v2 keep 2 +v4 keep 4 +v6 keep 6 +v8 keep 8 +v10 keep 10 +v11 keep 11 +" + ); + s.execute("edit.paste", json!({"at": "H20"})).unwrap(); + let pasted: Vec = (20..=27).map(|r| v(&s, &format!("H{r}"))).collect(); + assert_eq!(pasted, ["Val", "v2", "v4", "v6", "v8", "v10", "v11"].map(Value::from).into_iter().chain([Value::Empty]).collect::>()); + assert_eq!(v(&s, "J22"), Value::Number(4.0)); + assert!(s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse("J22").unwrap()).unwrap().formula.is_none()); + let mut s = filtered(); + s.execute("edit.cut", json!({"range": "A1:B11"})).unwrap(); + s.execute("edit.paste", json!({"at": "H20"})).unwrap(); + assert_eq!(v(&s, "H22"), Value::from("v3")); + + // Rows hidden by hand (no filter) are written, as in Excel. + let mut s = fresh(); + s.execute("range.setValues", json!({"range": "A1", "values": [["a"], ["b"], ["c"]]})).unwrap(); + s.execute("home.hideRows", json!({"rows": "2:2"})).unwrap(); + s.execute("selection.set", json!({"range": "A1:A3"})).unwrap(); + s.execute("range.fill", json!({"input": "Z"})).unwrap(); + assert_eq!(v(&s, "A2"), Value::from("Z")); +} + +/// A table of ten rows (A1:B11) with a link on every value cell, four of its rows hidden by a +/// filter (3, 5, 7, 9), and a spare corner at E1 to copy from. +fn filtered_rows_with_links() -> Session { + let mut s = s(); + let mut rows = vec![json!(["Val", "Tag"])]; + for i in 2..=11 { + rows.push(json!([format!("v{i}"), if [3, 5, 7, 9].contains(&i) { "hide" } else { "keep" }])); + } + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + s.execute("range.setValues", json!({"range": "E1", "values": [["X"]]})).unwrap(); + let sh = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb).sheet_mut(0).unwrap(); + for r in 2..=11u32 { + sh.hyperlinks + .insert(CellRef::new(r - 1, 0), gridcraft_model::Hyperlink { target: format!("https://example.com/{r}"), tooltip: None, external: true }); + } + s.execute("selection.set", json!({"range": "A1:B1"})).unwrap(); + s.execute("data.filter", json!({})).unwrap(); + s.execute("data.filterBy", json!({"column": "B", "values": ["keep"]})).unwrap(); + s +} + +fn cell_link_target(s: &Session, cell: &str) -> Option { + s.doc().unwrap().wb.active().unwrap().hyperlinks.get(&CellRef::parse(cell).unwrap()).map(|h| h.target.clone()) +} + +fn cell_validation(s: &Session, cell: &str) -> Option { + let c = CellRef::parse(cell).unwrap(); + s.doc().unwrap().wb.active().unwrap().validations.iter().find(|d| d.ranges.iter().any(|r| r.contains(c))).map(|d| d.f1.clone()) +} + +/// Copying a filtered range packs the rows showing into one block; their links and validation +/// rules travel with them, and those of the hidden rows are left behind. +#[test] +fn filtered_copy_pastes_links_and_validation_of_the_rows_showing_only() { + let mut s = filtered_rows_with_links(); + for (cell, source) in [("A3", "\"hidden\""), ("A4", "\"showing\""), ("A9", "\"hidden too\"")] { + s.execute("data.validation", json!({"range": cell, "type": "list", "formula1": source})).unwrap(); + } + s.execute("edit.copy", json!({"range": "A1:B11"})).unwrap(); + s.execute("edit.paste", json!({"at": "H20"})).unwrap(); + let links: Vec> = (20..=26).map(|r| cell_link_target(&s, &format!("H{r}"))).collect(); + let expected: Vec> = + [None, Some(2u32), Some(4), Some(6), Some(8), Some(10), Some(11)].iter().map(|n| n.map(|n| format!("https://example.com/{n}"))).collect(); + assert_eq!(links, expected); + let rules: Vec> = (20..=26).map(|r| cell_validation(&s, &format!("H{r}"))).collect(); + assert_eq!(rules, [None, None, Some("\"showing\"".to_string()), None, None, None, None]); + // The source keeps its own links and rules where they were. + assert_eq!(cell_link_target(&s, "A3").as_deref(), Some("https://example.com/3")); + assert_eq!(cell_validation(&s, "A3").as_deref(), Some("\"hidden\"")); +} + +/// One cell pasted over a filtered column leaves the rules of the rows the filter hides alone, and +/// does not give them the copied rule, with or without Skip Blanks. +#[test] +fn single_row_paste_over_filtered_rows_leaves_their_validation_alone() { + for skip_blanks in [false, true] { + let mut s = filtered_rows_with_links(); + s.execute("data.validation", json!({"range": "E1", "type": "list", "formula1": "\"copied\""})).unwrap(); + s.execute("data.validation", json!({"range": "A2:A5", "type": "list", "formula1": "\"old\""})).unwrap(); + s.execute("edit.copy", json!({"range": "E1"})).unwrap(); + s.execute("selection.set", json!({"range": "A2:A11"})).unwrap(); + s.execute("edit.pasteSpecial", json!({"what": "all", "skipBlanks": skip_blanks})).unwrap(); + let rules: Vec> = (2..=11).map(|r| cell_validation(&s, &format!("A{r}"))).collect(); + let showing = Some("\"copied\"".to_string()); + let old = Some("\"old\"".to_string()); + assert_eq!( + rules, + [showing.clone(), old.clone(), showing.clone(), old, showing.clone(), None, showing.clone(), None, showing.clone(), showing], + "skip blanks: {skip_blanks}" + ); + } +} + +/// Insert Copied Cells after a filtered copy of whole rows makes room for the rows copied, not for +/// the rows that were selected. +#[test] +fn insert_copied_cells_after_a_filtered_row_copy_inserts_only_the_rows_showing() { + let mut s = filtered_rows_with_links(); + s.execute("cell.set", json!({"cell": "A30", "input": "below"})).unwrap(); + s.execute("edit.copy", json!({"range": "1:11"})).unwrap(); + s.execute("selection.set", json!({"range": "A20"})).unwrap(); + s.execute("home.insertCopiedCells", json!({})).unwrap(); + // Seven rows were copied (the header and six rows showing). + assert_eq!(v(&s, "A37"), Value::from("below")); + assert_eq!(v(&s, "A20"), Value::from("Val")); + assert_eq!(v(&s, "A26"), Value::from("v11")); + assert_eq!(v(&s, "A27"), Value::Empty); +} + +/// Copying a filtered range with many separate runs of rows showing moves each row's note, link and +/// merge with it, and leaves the hidden rows' behind. +#[test] +fn filtered_copy_with_many_runs_moves_notes_and_merges_with_their_rows() { + let mut s = s(); + let mut rows = vec![json!(["Val", "Tag"])]; + for i in 2..=2001 { + rows.push(json!([format!("v{i}"), if i % 2 == 1 { "hide" } else { "keep" }])); + } + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let sh = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb).sheet_mut(0).unwrap(); + for r in 2..=2001u32 { + let note = gridcraft_model::Comment { + author: "a".into(), + text: format!("note {r}"), + replies: vec![], + threaded: false, + resolved: false, + visible: false, + dates: vec![], + note_box: None, + }; + sh.comments.insert(CellRef::new(r - 1, 0), note); + } + sh.merges.push(gridcraft_core::RangeRef::new(CellRef::new(3, 1), CellRef::new(3, 2))); + sh.merges.push(gridcraft_core::RangeRef::new(CellRef::new(2, 1), CellRef::new(2, 2))); + s.execute("selection.set", json!({"range": "A1:B1"})).unwrap(); + s.execute("data.filter", json!({})).unwrap(); + s.execute("data.filterBy", json!({"column": "B", "values": ["keep"]})).unwrap(); + s.execute("edit.copy", json!({"range": "A1:C2001"})).unwrap(); + s.execute("edit.paste", json!({"at": "H1"})).unwrap(); + let sh = s.doc().unwrap().wb.active().unwrap(); + let note = |row: u32| sh.comments.get(&CellRef::new(row - 1, 7)).map(|c| c.text.clone()); + assert_eq!(note(1), None); + assert_eq!(note(2).as_deref(), Some("note 2")); + assert_eq!(note(3).as_deref(), Some("note 4")); + assert_eq!(note(1001).as_deref(), Some("note 2000")); + assert_eq!(sh.comments.keys().filter(|c| c.col == 7).count(), 1000); + // Only the merge on a row showing (row 4) came along, packed to the row it landed in (row 3). + assert!(sh.merges.contains(&gridcraft_core::RangeRef::new(CellRef::new(2, 8), CellRef::new(2, 9)))); + assert_eq!(sh.merges.iter().filter(|m| m.start.col >= 7).count(), 1); +} + #[test] fn editing_quote_prefixed_text_keeps_it_text() { let mut s = s(); @@ -5257,6 +5497,32 @@ fn paste_validation_skip_blanks_large_populated_paste() { assert!(validation_areas(&s).is_empty()); } +/// A whole-column copy of a filtered range takes only the rows showing, also when they form one +/// block (the hidden rows are all at the end) and the selection covers the whole column. +#[test] +fn whole_column_copy_of_a_filtered_range_leaves_the_hidden_rows_out() { + for hidden_tail in [false, true] { + let mut s = s(); + let mut rows = vec![json!(["Val", "Tag"])]; + for i in 2..=11 { + let hide = if hidden_tail { i >= 8 } else { [3, 5, 7, 9].contains(&i) }; + rows.push(json!([format!("v{i}"), if hide { "hide" } else { "keep" }])); + } + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + s.execute("selection.set", json!({"range": "A1:B1"})).unwrap(); + s.execute("data.filter", json!({})).unwrap(); + s.execute("data.filterBy", json!({"column": "B", "values": ["keep"]})).unwrap(); + s.execute("cell.set", json!({"cell": "H30", "input": "old"})).unwrap(); + s.execute("edit.copy", json!({"range": "A:B"})).unwrap(); + s.execute("edit.paste", json!({"at": "H1"})).unwrap(); + let col: Vec = (1..=11).map(|r| v(&s, &format!("H{r}"))).collect(); + let want: Vec<&str> = + if hidden_tail { vec!["Val", "v2", "v3", "v4", "v5", "v6", "v7"] } else { vec!["Val", "v2", "v4", "v6", "v8", "v10", "v11"] }; + let want: Vec = want.into_iter().map(Value::from).chain(std::iter::repeat(Value::Empty)).take(11).collect(); + assert_eq!(col, want, "hidden rows stay out of a whole-column copy (hidden tail: {hidden_tail})"); + } +} + /// Open a workbook by path, and `CELL("filename")` names the file it came from, as Excel does; /// Save As makes the new file the one it names. The sheet-name formula of #1489 depends on it. #[test] diff --git a/crates/model/src/sheet.rs b/crates/model/src/sheet.rs index 67ca1f13e..440647d42 100644 --- a/crates/model/src/sheet.rs +++ b/crates/model/src/sheet.rs @@ -491,12 +491,14 @@ impl Sheet { /// sheet AutoFilter while it has criteria, or a hidden data row of a filtered table. /// Other hidden rows were hidden by hand. pub fn is_row_filtered(&self, row: u32) -> bool { - if !self.is_row_hidden(row) { - return false; - } - let in_sheet_filter = - self.autofilter.as_ref().is_some_and(|af| !af.criteria.is_empty() && row > af.range.start.row && row <= af.range.end.row); - in_sheet_filter || self.tables.iter().any(|t| !t.filter.is_empty() && t.data_range().is_some_and(|r| row >= r.start.row && row <= r.end.row)) + self.is_row_hidden(row) && self.filter_row_spans().iter().any(|&(first, last)| row >= first && row <= last) + } + /// The row spans (first, last) an applied filter can hide: the data rows of the sheet + /// AutoFilter while it has criteria, and of each filtered table. + pub fn filter_row_spans(&self) -> Vec<(u32, u32)> { + let sheet = self.autofilter.as_ref().filter(|af| !af.criteria.is_empty()).map(|af| (af.range.start.row.saturating_add(1), af.range.end.row)); + let tables = self.tables.iter().filter(|t| !t.filter.is_empty()).filter_map(|t| t.data_range()).map(|r| (r.start.row, r.end.row)); + sheet.into_iter().chain(tables).collect() } pub fn is_col_hidden(&self, col: u32) -> bool { self.cols.get(&col).is_some_and(|r| r.hidden) From 4bba98fd500665fd3c24511302c255d3eb2b424e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 02:57:51 +0100 Subject: [PATCH 46/51] perf(model): cache formula trees when reading a workbook from JSON A Formula read from JSON had no cached tree (`expr` is not saved), so every graph rebuild and every recalculation parsed each formula again. Reading one now parses its text once and keeps the tree, as Formula::new does. The saved text is kept as written. Fixes #3577 Co-Authored-By: Claude Opus 5.5 --- crates/model/src/cell.rs | 82 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/crates/model/src/cell.rs b/crates/model/src/cell.rs index 0ca1dbe74..33186bd26 100644 --- a/crates/model/src/cell.rs +++ b/crates/model/src/cell.rs @@ -11,6 +11,7 @@ use crate::style::StyleId; /// A parsed formula. `text` is the canonical formula text without the leading `=`. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(from = "SavedFormula")] pub struct Formula { pub text: String, #[serde(skip)] @@ -39,15 +40,15 @@ impl Formula { pub fn from_expr(e: Expr) -> Formula { Formula { text: gridcraft_formula::print(&e), expr: Some(Arc::new(e)), array: None, dynamic: false } } - /// The parsed expression, re-parsing after deserialization. + /// The parsed expression, re-parsing when no tree is cached. pub fn expr(&self) -> Option { self.expr.as_deref().cloned().or_else(|| gridcraft_formula::parse(&self.text).ok()) } - /// The parsed expression without copying it (re-parses after deserialization). + /// The parsed expression without copying it (re-parses when no tree is cached). pub fn expr_arc(&self) -> Option> { self.expr.clone().or_else(|| gridcraft_formula::parse(&self.text).ok().map(Arc::new)) } - /// Runs `f` on the parsed expression, borrowing the cached tree or, after deserialization, + /// Runs `f` on the parsed expression, borrowing the cached tree or, when none is cached, /// a freshly parsed one that is dropped afterwards. Unlike [`expr`](Self::expr) it never copies /// the tree, and unlike [`expr_arc`](Self::expr_arc) it needs no reference count or `Arc` /// allocation. `None` when the text does not parse. @@ -59,6 +60,25 @@ impl Formula { } } +/// A formula as a saved workbook holds it: the tree is not saved. Reading one parses its text +/// once and caches the tree, so a workbook opened from JSON does not parse every formula again on +/// each recalculation. The text is kept as saved, and text that doesn't parse caches nothing, as +/// [`Formula::new`] does. +#[derive(Deserialize)] +struct SavedFormula { + text: String, + array: Option, + #[serde(default)] + dynamic: bool, +} + +impl From for Formula { + fn from(f: SavedFormula) -> Formula { + let expr = gridcraft_formula::parse(&f.text).ok().map(Arc::new); + Formula { text: f.text, expr, array: f.array, dynamic: f.dynamic } + } +} + /// A static picture stored as cell content, independent of floating drawing anchors. #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] pub struct CellPicture { @@ -131,3 +151,59 @@ impl Cell { } } } + +#[cfg(test)] +mod tests { + use super::*; + use gridcraft_core::CellRef; + + #[test] + fn formulas_read_from_json_keep_their_text_and_cache_the_tree() -> Result<(), serde_json::Error> { + let mut array = Formula::new("=A1:B2*2"); + array.array = Some(RangeRef::new(CellRef::new(0, 2), CellRef::new(1, 3))); + array.dynamic = true; + for f in [Formula::new("=SUM(A1:A3)+'My Sheet'!B2"), Formula::new("=LET(x,A1*2,x+1)"), array] { + let back: Formula = serde_json::from_str(&serde_json::to_string(&f)?)?; + assert!(back.expr.is_some(), "{}: the tree is cached after reading", f.text); + assert_eq!(back, f); + } + // Saved text is kept as written, even when the canonical print differs, and its tree is + // the one the text parses to. + let back: Formula = serde_json::from_str(r#"{"text": "sum( a1 , 2 )", "array": null}"#)?; + assert_eq!(back.text, "sum( a1 , 2 )"); + assert!(!back.dynamic); + assert_eq!(back.expr.as_deref(), gridcraft_formula::parse("sum( a1 , 2 )").ok().as_ref()); + assert!(back.expr.is_some()); + // Text that doesn't parse caches nothing, as `Formula::new` does. + let bad = Formula::new("=SUM("); + let back: Formula = serde_json::from_str(&serde_json::to_string(&bad)?)?; + assert_eq!(back, bad); + assert!(back.expr.is_none()); + Ok(()) + } + + #[test] + fn a_workbook_read_from_json_has_every_formula_tree() -> Result<(), Box> { + let mut wb = crate::Workbook::new(); + let mut array = Formula::new("=A1:A3*2"); + array.array = Some(RangeRef::new(CellRef::new(0, 3), CellRef::new(2, 3))); + let sheet = wb.sheet_mut(0).ok_or("no sheet")?; + sheet.cells.set(CellRef::new(0, 0), Cell::value(Value::Number(1.0))); + sheet.cells.set(CellRef::new(0, 1), Cell::formula(Formula::new("=SUM(A1:A3)"))); + sheet.cells.set(CellRef::new(1, 1), Cell::formula(Formula::new("=IF(A1>0,\"yes\",\"no\")"))); + sheet.cells.set(CellRef::new(2, 1), Cell::formula(Formula::new("=SUM("))); + sheet.cells.set(CellRef::new(0, 3), Cell::formula(array)); + let json = serde_json::to_string(&wb)?; + let back: crate::Workbook = serde_json::from_str(&json)?; + let (before, after) = (wb.sheet(0).ok_or("no sheet")?, back.sheet(0).ok_or("no sheet")?); + let mut formulas = 0; + for (c, cell) in before.cells.iter() { + let (Some(f), Some(read)) = (&cell.formula, after.cell(c).and_then(|cell| cell.formula.as_ref())) else { continue }; + assert_eq!(read, f, "{c:?}: text, tree and flags as saved"); + assert_eq!(read.expr.is_some(), gridcraft_formula::parse(&f.text).is_ok(), "{c:?}: every formula that parses has its tree"); + formulas += 1; + } + assert_eq!(formulas, 4); + Ok(()) + } +} From 7cb0088361e1118b1df96ea008a7065bcaf4b9f8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 03:04:24 +0100 Subject: [PATCH 47/51] perf(formula): adjust_if_changed skips formulas a structural edit leaves alone adjust had no way to report that an edit does not affect a formula, so callers copied, rebuilt and printed every formula in the workbook. adjust_if_changed borrows the formula and returns None when nothing moves. Its check reuses adjust_ref on each reference into the edited sheet, on a copy without the sheet name, so it cannot disagree with adjust. adjust_ref and shift_ref take the reference by value and no longer clone sheet names. An integration test keeps the previous implementation as an oracle over every edit kind and a randomized property test. Fixes #3576 Co-Authored-By: Claude Opus 5.5 --- crates/formula/src/adjust.rs | 73 ++-- crates/formula/tests/it/adjust_oracle.rs | 494 +++++++++++++++++++++++ crates/formula/tests/it/main.rs | 1 + 3 files changed, 545 insertions(+), 23 deletions(-) create mode 100644 crates/formula/tests/it/adjust_oracle.rs diff --git a/crates/formula/src/adjust.rs b/crates/formula/src/adjust.rs index 319474b4e..ba6616d64 100644 --- a/crates/formula/src/adjust.rs +++ b/crates/formula/src/adjust.rs @@ -8,7 +8,7 @@ use crate::ast::{Anchor, Expr, RefKind, Reference, SheetSel, clamp_anchor}; /// Copy/fill: relative parts move by `(dr, dc)`. References pushed off the sheet become `#REF!`. pub fn shift_relative(e: Expr, dr: i64, dc: i64) -> Expr { e.map(&mut |x| match x { - Expr::Ref(r) => match shift_ref(&r, dr, dc) { + Expr::Ref(r) => match shift_ref(r, dr, dc) { Some(r) => Expr::Ref(r), None => Expr::Error(CellError::Ref), }, @@ -60,14 +60,14 @@ fn shift_line(v: u32, abs: bool, d: i64, max: u32) -> Option { if (0..max as i64).contains(&n) { Some(n as u32) } else { None } } -fn shift_ref(r: &Reference, dr: i64, dc: i64) -> Option { +fn shift_ref(r: Reference, dr: i64, dc: i64) -> Option { let kind = match &r.kind { RefKind::Cell(a) => RefKind::Cell(shift_anchor(a, dr, dc)?), RefKind::Range(a, b) => RefKind::Range(shift_anchor(a, dr, dc)?, shift_anchor(b, dr, dc)?), RefKind::Rows(r0, a0, r1, a1) => RefKind::Rows(shift_line(*r0, *a0, dr, MAX_ROWS)?, *a0, shift_line(*r1, *a1, dr, MAX_ROWS)?, *a1), RefKind::Cols(c0, a0, c1, a1) => RefKind::Cols(shift_line(*c0, *a0, dc, MAX_COLS)?, *a0, shift_line(*c1, *a1, dc, MAX_COLS)?, *a1), }; - Some(Reference { sheet: r.sheet.clone(), kind }) + Some(Reference { sheet: r.sheet, kind }) } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -95,27 +95,54 @@ pub enum Edit { MoveToSheet { from: RangeRef, to_sheet: String, to_row: u32, to_col: u32 }, } +/// Whether `r`, in a formula living on sheet `host`, points into sheet `target`. +fn on_target(r: &Reference, host: &str, target: &str) -> bool { + match &r.sheet { + SheetSel::Current => host.eq_ignore_ascii_case(target), + SheetSel::Named(n) => n.eq_ignore_ascii_case(target), + SheetSel::Span(..) => false, + } +} + +/// Whether [`adjust`] would change `r`. It runs the same [`adjust_ref`] on a copy without the +/// sheet name, so no name is copied: the adjustment reads only the position, and the sheet a cut +/// to another sheet assigns is compared afterwards. +fn ref_changes(r: &Reference, edit: &Edit) -> bool { + let probe = Reference { sheet: SheetSel::Current, kind: r.kind.clone() }; + match adjust_ref(probe, edit) { + None => true, + Some(p) => p.kind != r.kind || (p.sheet != SheetSel::Current && p.sheet != r.sheet), + } +} + +/// Whether [`adjust`] would change the formula `e` at all, without copying or rebuilding it. +fn changes(e: &Expr, host: &str, target: &str, edit: &Edit) -> bool { + let mut changed = false; + e.walk(&mut |x| { + if !changed && let Expr::Ref(r) = x { + changed = on_target(r, host, target) && ref_changes(r, edit); + } + }); + changed +} + /// Adjusts a formula living on sheet `host` after `edit` on sheet `target`. pub fn adjust(e: Expr, host: &str, target: &str, edit: &Edit) -> Expr { e.map(&mut |x| match x { - Expr::Ref(r) => { - let on_target = match &r.sheet { - SheetSel::Current => host.eq_ignore_ascii_case(target), - SheetSel::Named(n) => n.eq_ignore_ascii_case(target), - SheetSel::Span(..) => false, - }; - if !on_target { - return Expr::Ref(r); - } - match adjust_ref(&r, edit) { - Some(r) => Expr::Ref(r), - None => Expr::Error(CellError::Ref), - } - } + Expr::Ref(r) if on_target(&r, host, target) => match adjust_ref(r, edit) { + Some(r) => Expr::Ref(r), + None => Expr::Error(CellError::Ref), + }, other => other, }) } +/// [`adjust`] for a borrowed formula: `None` when the edit leaves it exactly as it is, so a +/// caller can skip unaffected formulas without copying, rebuilding or printing them. +pub fn adjust_if_changed(e: &Expr, host: &str, target: &str, edit: &Edit) -> Option { + changes(e, host, target, edit).then(|| adjust(e.clone(), host, target, edit)) +} + /// New position of an index after an insert (`None` never) or delete (`None` = deleted). fn map_index(v: u32, at: u32, count: u32, insert: bool, max: u32) -> Option { if insert { @@ -168,7 +195,7 @@ fn map_span(lo: u32, hi: u32, at: u32, count: u32, insert: bool, max: u32) -> Op } } -fn adjust_ref(r: &Reference, edit: &Edit) -> Option { +fn adjust_ref(r: Reference, edit: &Edit) -> Option { match edit { Edit::Insert { axis, at, count } | Edit::Delete { axis, at, count } => { let insert = matches!(edit, Edit::Insert { .. }); @@ -193,7 +220,7 @@ fn adjust_ref(r: &Reference, edit: &Edit) -> Option { } (k, _) => k.clone(), }; - Some(Reference { sheet: r.sheet.clone(), kind }) + Some(Reference { sheet: r.sheet, kind }) } Edit::InsertCells { axis, range } | Edit::DeleteCells { axis, range } => { // Like inserting/deleting lines, for references that lie within the shifted @@ -204,7 +231,7 @@ fn adjust_ref(r: &Reference, edit: &Edit) -> Option { Axis::Cols => (range.start.row <= span.start.row && span.end.row <= range.end.row, range.start.col, range.width()), }; if !inside { - return Some(r.clone()); + return Some(r); } let axis = *axis; let lines = if matches!(edit, Edit::InsertCells { .. }) { Edit::Insert { axis, at, count } } else { Edit::Delete { axis, at, count } }; @@ -215,7 +242,7 @@ fn adjust_ref(r: &Reference, edit: &Edit) -> Option { let dc = *to_col as i64 - from.start.col as i64; let range = r.range(); if !from.contains_range(&range) { - return Some(r.clone()); + return Some(r); } let mv = |a: &Anchor| -> Option { let (row, col) = clamp_anchor(a.row as i64 + dr, a.col as i64 + dc)?; @@ -233,11 +260,11 @@ fn adjust_ref(r: &Reference, edit: &Edit) -> Option { RefKind::Cols(shift_line(*a, false, dc, MAX_COLS)?, *aa, shift_line(*b, false, dc, MAX_COLS)?, *ba) } }; - Some(Reference { sheet: r.sheet.clone(), kind }) + Some(Reference { sheet: r.sheet, kind }) } Edit::MoveToSheet { from, to_sheet, to_row, to_col } => { if !from.contains_range(&r.range()) { - return Some(r.clone()); + return Some(r); } let mut moved = adjust_ref(r, &Edit::Move { from: *from, to_row: *to_row, to_col: *to_col })?; moved.sheet = SheetSel::Named(to_sheet.clone()); diff --git a/crates/formula/tests/it/adjust_oracle.rs b/crates/formula/tests/it/adjust_oracle.rs new file mode 100644 index 000000000..42c5314dc --- /dev/null +++ b/crates/formula/tests/it/adjust_oracle.rs @@ -0,0 +1,494 @@ +//! `adjust`, `adjust_if_changed` and `shift_relative` against a verbatim copy of the previous +//! implementation, which cloned and rebuilt every formula. The rewrite must give the same tree +//! (and so the same text) for every formula and edit, and `adjust_if_changed` must say +//! "unchanged" exactly when the old code returned the formula as it was. + +use gridcraft_core::{CellRef, MAX_COLS, MAX_ROWS, RangeRef}; +use gridcraft_formula::adjust::{Axis, Edit, adjust, adjust_if_changed, shift_relative}; +use gridcraft_formula::{Expr, parse, print}; + +/// The implementation before the borrowed check, kept as the test oracle. +mod old { + use gridcraft_core::{CellError, MAX_COLS, MAX_ROWS}; + use gridcraft_formula::adjust::{Axis, Edit}; + use gridcraft_formula::{Anchor, Expr, RefKind, Reference, SheetSel}; + + fn clamp_anchor(row: i64, col: i64) -> Option<(u32, u32)> { + if (0..MAX_ROWS as i64).contains(&row) && (0..MAX_COLS as i64).contains(&col) { Some((row as u32, col as u32)) } else { None } + } + + pub fn shift_relative(e: Expr, dr: i64, dc: i64) -> Expr { + e.map(&mut |x| match x { + Expr::Ref(r) => match shift_ref(&r, dr, dc) { + Some(r) => Expr::Ref(r), + None => Expr::Error(CellError::Ref), + }, + other => other, + }) + } + + fn shift_anchor(a: &Anchor, dr: i64, dc: i64) -> Option { + let row = if a.row_abs { a.row as i64 } else { a.row as i64 + dr }; + let col = if a.col_abs { a.col as i64 } else { a.col as i64 + dc }; + let (row, col) = clamp_anchor(row, col)?; + Some(Anchor { row, col, ..*a }) + } + + fn shift_line(v: u32, abs: bool, d: i64, max: u32) -> Option { + if abs { + return Some(v); + } + let n = v as i64 + d; + if (0..max as i64).contains(&n) { Some(n as u32) } else { None } + } + + fn shift_ref(r: &Reference, dr: i64, dc: i64) -> Option { + let kind = match &r.kind { + RefKind::Cell(a) => RefKind::Cell(shift_anchor(a, dr, dc)?), + RefKind::Range(a, b) => RefKind::Range(shift_anchor(a, dr, dc)?, shift_anchor(b, dr, dc)?), + RefKind::Rows(r0, a0, r1, a1) => RefKind::Rows(shift_line(*r0, *a0, dr, MAX_ROWS)?, *a0, shift_line(*r1, *a1, dr, MAX_ROWS)?, *a1), + RefKind::Cols(c0, a0, c1, a1) => RefKind::Cols(shift_line(*c0, *a0, dc, MAX_COLS)?, *a0, shift_line(*c1, *a1, dc, MAX_COLS)?, *a1), + }; + Some(Reference { sheet: r.sheet.clone(), kind }) + } + + pub fn adjust(e: Expr, host: &str, target: &str, edit: &Edit) -> Expr { + e.map(&mut |x| match x { + Expr::Ref(r) => { + let on_target = match &r.sheet { + SheetSel::Current => host.eq_ignore_ascii_case(target), + SheetSel::Named(n) => n.eq_ignore_ascii_case(target), + SheetSel::Span(..) => false, + }; + if !on_target { + return Expr::Ref(r); + } + match adjust_ref(&r, edit) { + Some(r) => Expr::Ref(r), + None => Expr::Error(CellError::Ref), + } + } + other => other, + }) + } + + fn map_index(v: u32, at: u32, count: u32, insert: bool, max: u32) -> Option { + if insert { + if v >= at { + let n = v as u64 + count as u64; + if n >= max as u64 { None } else { Some(n as u32) } + } else { + Some(v) + } + } else if v < at { + Some(v) + } else if v >= at.saturating_add(count) { + Some(v - count) + } else { + None + } + } + + fn map_span(lo: u32, hi: u32, at: u32, count: u32, insert: bool, max: u32) -> Option<(u32, u32)> { + if insert { + let lo2 = if lo >= at { lo as u64 + count as u64 } else { lo as u64 }; + let hi2 = if hi >= at { hi as u64 + count as u64 } else { hi as u64 }; + let hi2 = hi2.min(max as u64 - 1); + if lo2 > hi2 { + return None; + } + Some((lo2 as u32, hi2 as u32)) + } else { + let end = at.saturating_add(count); + if lo >= at && hi < end { + return None; + } + let lo2 = if lo < at { + lo + } else if lo >= end { + lo - count + } else { + at + }; + let hi2 = if hi < at { + hi + } else if hi >= end { + hi - count + } else { + at.saturating_sub(1) + }; + Some((lo2, hi2.max(lo2))) + } + } + + fn adjust_ref(r: &Reference, edit: &Edit) -> Option { + match edit { + Edit::Insert { axis, at, count } | Edit::Delete { axis, at, count } => { + let insert = matches!(edit, Edit::Insert { .. }); + let kind = match (&r.kind, axis) { + (RefKind::Cell(a), Axis::Rows) => RefKind::Cell(Anchor { row: map_index(a.row, *at, *count, insert, MAX_ROWS)?, ..*a }), + (RefKind::Cell(a), Axis::Cols) => RefKind::Cell(Anchor { col: map_index(a.col, *at, *count, insert, MAX_COLS)?, ..*a }), + (RefKind::Range(a, b), Axis::Rows) => { + let (lo, hi) = map_span(a.row.min(b.row), a.row.max(b.row), *at, *count, insert, MAX_ROWS)?; + RefKind::Range(Anchor { row: lo, ..*a }, Anchor { row: hi, ..*b }) + } + (RefKind::Range(a, b), Axis::Cols) => { + let (lo, hi) = map_span(a.col.min(b.col), a.col.max(b.col), *at, *count, insert, MAX_COLS)?; + RefKind::Range(Anchor { col: lo, ..*a }, Anchor { col: hi, ..*b }) + } + (RefKind::Rows(r0, a0, r1, a1), Axis::Rows) => { + let (lo, hi) = map_span(*r0.min(r1), *r0.max(r1), *at, *count, insert, MAX_ROWS)?; + RefKind::Rows(lo, *a0, hi, *a1) + } + (RefKind::Cols(c0, a0, c1, a1), Axis::Cols) => { + let (lo, hi) = map_span(*c0.min(c1), *c0.max(c1), *at, *count, insert, MAX_COLS)?; + RefKind::Cols(lo, *a0, hi, *a1) + } + (k, _) => k.clone(), + }; + Some(Reference { sheet: r.sheet.clone(), kind }) + } + Edit::InsertCells { axis, range } | Edit::DeleteCells { axis, range } => { + let span = r.range(); + let (inside, at, count) = match axis { + Axis::Rows => (range.start.col <= span.start.col && span.end.col <= range.end.col, range.start.row, range.height()), + Axis::Cols => (range.start.row <= span.start.row && span.end.row <= range.end.row, range.start.col, range.width()), + }; + if !inside { + return Some(r.clone()); + } + let axis = *axis; + let lines = + if matches!(edit, Edit::InsertCells { .. }) { Edit::Insert { axis, at, count } } else { Edit::Delete { axis, at, count } }; + adjust_ref(r, &lines) + } + Edit::Move { from, to_row, to_col } => { + let dr = *to_row as i64 - from.start.row as i64; + let dc = *to_col as i64 - from.start.col as i64; + let range = r.range(); + if !from.contains_range(&range) { + return Some(r.clone()); + } + let mv = |a: &Anchor| -> Option { + let (row, col) = clamp_anchor(a.row as i64 + dr, a.col as i64 + dc)?; + Some(Anchor { row, col, ..*a }) + }; + let kind = match &r.kind { + RefKind::Cell(a) => RefKind::Cell(mv(a)?), + RefKind::Range(a, b) => RefKind::Range(mv(a)?, mv(b)?), + RefKind::Rows(a, aa, b, ba) => { + let dr = *to_row as i64 - from.start.row as i64; + RefKind::Rows(shift_line(*a, false, dr, MAX_ROWS)?, *aa, shift_line(*b, false, dr, MAX_ROWS)?, *ba) + } + RefKind::Cols(a, aa, b, ba) => { + let dc = *to_col as i64 - from.start.col as i64; + RefKind::Cols(shift_line(*a, false, dc, MAX_COLS)?, *aa, shift_line(*b, false, dc, MAX_COLS)?, *ba) + } + }; + Some(Reference { sheet: r.sheet.clone(), kind }) + } + Edit::MoveToSheet { from, to_sheet, to_row, to_col } => { + if !from.contains_range(&r.range()) { + return Some(r.clone()); + } + let mut moved = adjust_ref(r, &Edit::Move { from: *from, to_row: *to_row, to_col: *to_col })?; + moved.sheet = SheetSel::Named(to_sheet.clone()); + Some(moved) + } + } + } +} + +/// Formulas with every reference shape the adjustment distinguishes. +const FORMULAS: &[&str] = &[ + "A1", + "B2+C3*D4", + "$B$2+B$2+$B2", + "SUM(A1:A10)+SUM(B2:C3)+SUM($B$2:$C$3)", + "SUM(C3:B2)", + "SUM(A:A)+SUM($B:$C)+SUM(2:2)+SUM($3:$5)", + "SUM(A1:.A7)+SUM(B:.B)+ROWS(2.:.6)+SUM(A1.:B1)", + "S!B2+s!C3+'S'!A1:B9", + "Other!B2+Other!A:A+Other!2:3", + "'My Sheet'!B2+'my sheet'!C3:D4", + "Dest!B2+S!B2", + "SUM(S:Other!B2)+SUM(Other:S!A1:C3)", + "XFD1048576+XFD1+A1048576+SUM(XFC1048575:XFD1048576)", + "SUM(1048576:1048576)+SUM(XFD:XFD)", + "VLOOKUP($B2,Lookup!$A$1:$D$500,3,FALSE)", + "IF(AND(C2>0,D2<100),C2*1.07,IF(E2=\"x\",\"yes\",\"no\"))", + "Table1[@Qty]*Table1[@Price]+SUM(Table1[[#Data],[Qty]])", + "SUMPRODUCT(A2:C2,{1,2,3})", + "LET(x,A2*2,y,B2+x,x*y)", + "LAMBDA(x,x+B2)(C3)", + "IFERROR(INDEX(Lookup!$A:$A,MATCH(F2,Lookup!$B:$B,0)),\"\")", + "-B2%+(C3)^2&\"a\"", + "A1#+S!B2#", + "S!MyName+MyName+#REF!+S!#REF!", + "COUNTIFS(Data!$A:$A,A2,Data!$C:$C,\">=\"&$G$2)", + "1+2", + "\"text\"", + "B2:C3 B3:D4", + "(B2:C3,E5)", + "INDIRECT(\"B2\")+OFFSET(B2,1,1)", + // From the parser edge cases (prefixes and LET parameter names). + "LET(_xlpm.TRUE,1,_xlpm.TRUE+1)", + "LET(_xlpm.A1,1,_xlpm.A1)", + "LAMBDA(_xlpm.x,_xlpm.x+1)(2)", + "_xlfn.LET(_xlpm.R1C1,1,_xlpm.R1C1*2)", + "LET(_xlpm.FALSE,2,_xlpm.FALSE)", + "_xlpm.SUM(1)", + "LET(_xlpm.x_1,1,_xlpm.x_1)", + "_xlfn.ANCHORARRAY(A1)+_xlfn.SINGLE(B1:B3)", + "SUM(_xlfn._TRO_ALL(A1:E10))", +]; + +fn col(c: u32) -> String { + gridcraft_core::addr::col_to_letters(c) +} + +/// The realistic mix the structural-edit benchmark uses, row-dependent. +fn mix(i: usize, r: u32) -> String { + let r1 = r + 1; + match i % 12 { + 0 => format!("SUM(A{r1}:A{})", r1 + 9), + 1 => format!("VLOOKUP($B{r1},Lookup!$A$1:$D$500,3,FALSE)"), + 2 => format!("IF(AND(C{r1}>0,D{r1}<100),C{r1}*1.07,IF(E{r1}=\"x\",\"yes\",\"no\"))"), + 3 => format!("'My Sheet'!B{r1}+Data!C{r1}*2-1"), + 4 => "Table1[@Qty]*Table1[@Price]".to_string(), + 5 => format!("SUMPRODUCT(A{r1}:C{r1},{{1,2,3}})+SUM(Table1[[#Data],[Qty]])"), + 6 => format!("\"Lorem ipsum\"&A{r1}&\" units\""), + 7 => format!("IFERROR(INDEX(Lookup!$A:$A,MATCH(F{r1},Lookup!$B:$B,0)),\"\")"), + 8 => (0..20).map(|k| format!("{}{r1}", col(k))).collect::>().join("+"), + 9 => format!("LET(x,A{r1}*2,y,B{r1}+x,x*y)"), + 10 => format!("ROUND(AVERAGE(B{r1}:F{r1})*(1+$H$1)^2,2)"), + _ => format!("COUNTIFS(Data!$A:$A,A{r1},Data!$C:$C,\">=\"&$G$2)"), + } +} + +fn range(r0: u32, c0: u32, r1: u32, c1: u32) -> RangeRef { + RangeRef::new(CellRef::new(r0, c0), CellRef::new(r1, c1)) +} + +/// Every edit kind, at the sheet's start, middle and edges, with counts that run past the end. +fn edits() -> Vec { + let mut v = Vec::new(); + for axis in [Axis::Rows, Axis::Cols] { + let max = if axis == Axis::Rows { MAX_ROWS } else { MAX_COLS }; + for at in [0, 1, 2, 3, 9, max - 2, max - 1] { + for count in [0, 1, 2, 5, max - 1, max, u32::MAX] { + v.push(Edit::Insert { axis, at, count }); + v.push(Edit::Delete { axis, at, count }); + } + } + for r in [ + range(1, 1, 2, 2), + range(0, 0, 0, 0), + range(1, 0, 1, MAX_COLS - 1), + range(0, 1, MAX_ROWS - 1, 1), + range(MAX_ROWS - 2, MAX_COLS - 2, MAX_ROWS - 1, MAX_COLS - 1), + ] { + v.push(Edit::InsertCells { axis, range: r }); + v.push(Edit::DeleteCells { axis, range: r }); + } + } + let blocks = [ + range(1, 1, 2, 2), + range(0, 0, 0, 0), + range(0, 0, MAX_ROWS - 1, MAX_COLS - 1), + range(0, 0, MAX_ROWS - 1, 2), + range(1, 0, 4, MAX_COLS - 1), + range(0, 0, 20, 10), + ]; + let targets = [(9, 3), (0, 0), (1, 1), (MAX_ROWS - 1, MAX_COLS - 1), (MAX_ROWS - 2, 0)]; + for from in blocks { + for (to_row, to_col) in targets { + v.push(Edit::Move { from, to_row, to_col }); + for to_sheet in ["Dest", "S", "s", "Other"] { + v.push(Edit::MoveToSheet { from, to_sheet: to_sheet.to_string(), to_row, to_col }); + } + } + } + v +} + +/// Compares the new and the old adjustment of `src` for every host and edit; returns the +/// number of (formula, host, edit) cases checked, or 0 when `src` does not parse. +fn check(src: &str, edits: &[Edit]) -> usize { + let Ok(e) = parse(src) else { return 0 }; + let mut n = 0; + for host in ["S", "s", "Other", "My Sheet"] { + for edit in edits { + let want = old::adjust(e.clone(), host, "S", edit); + let got = adjust(e.clone(), host, "S", edit); + assert!(got == want, "{src} on {host} after {edit:?}: {} vs {}", print(&got), print(&want)); + assert_eq!(print(&got), print(&want), "{src} on {host} after {edit:?}"); + match adjust_if_changed(&e, host, "S", edit) { + None => assert!(want == e, "{src} on {host} after {edit:?}: reported unchanged, old gives {}", print(&want)), + Some(x) => { + assert!(want != e, "{src} on {host} after {edit:?}: reported changed but the old code keeps it"); + assert!(x == want, "{src} on {host} after {edit:?}: {} vs {}", print(&x), print(&want)); + } + } + n += 1; + } + } + for (dr, dc) in [(0, 0), (1, 0), (0, 1), (-1, -1), (5, 3), (-3, 2), (MAX_ROWS as i64, 0), (0, -(MAX_COLS as i64)), (1_048_575, 16_383)] { + let want = old::shift_relative(e.clone(), dr, dc); + let got = shift_relative(e.clone(), dr, dc); + assert!(got == want, "{src} shifted by ({dr}, {dc}): {} vs {}", print(&got), print(&want)); + } + n +} + +#[test] +fn adjust_matches_the_previous_implementation_for_every_edit_kind() { + let edits = edits(); + let mut cases = 0; + for src in FORMULAS { + let n = check(src, &edits); + assert!(n > 0, "{src} does not parse"); + cases += n; + } + for i in 0..48 { + // Rows 1-2 and 1,048,565-1,048,566 of the mix (its longest range reaches 9 rows further + // down, to the last row): near both edges of the sheet. + let row = if i < 24 { (i / 12) as u32 } else { MAX_ROWS - 12 + (i / 12 - 2) as u32 }; + let src = mix(i, row); + let n = check(&src, &edits); + assert!(n > 0, "{src} does not parse"); + cases += n; + } + assert!(cases > 100_000, "{cases}"); +} + +/// A small deterministic generator: a failing case is reproduced by rerunning the test, and +/// its message names the case's index in the stream. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + self.0 >> 33 + } + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } + fn line(&mut self, max: u32) -> u32 { + match self.below(3) { + 0 => self.below(12) as u32, + 1 => max - 1 - self.below(12) as u32, + _ => self.below(max as u64) as u32, + } + } +} + +fn random_ref(rng: &mut Rng) -> String { + let sheet = ["", "", "S!", "s!", "Other!", "'My Sheet'!", "S:Other!"][rng.below(7) as usize]; + let dollar = |rng: &mut Rng| if rng.below(3) == 0 { "$" } else { "" }; + let cell = |rng: &mut Rng| format!("{}{}{}{}", dollar(rng), col(rng.line(MAX_COLS)), dollar(rng), rng.line(MAX_ROWS) + 1); + let body = match rng.below(4) { + 0 => cell(rng), + 1 => format!("{}:{}", cell(rng), cell(rng)), + 2 => { + let (a, b) = (rng.line(MAX_ROWS) + 1, rng.line(MAX_ROWS) + 1); + format!("{}{a}:{}{b}", dollar(rng), dollar(rng)) + } + _ => { + let (a, b) = (rng.line(MAX_COLS), rng.line(MAX_COLS)); + format!("{}{}:{}{}", dollar(rng), col(a), dollar(rng), col(b)) + } + }; + format!("{sheet}{body}") +} + +fn random_edit(rng: &mut Rng) -> Edit { + let axis = if rng.below(2) == 0 { Axis::Rows } else { Axis::Cols }; + let max = if axis == Axis::Rows { MAX_ROWS } else { MAX_COLS }; + let block = |rng: &mut Rng| { + let (r0, r1) = (rng.line(MAX_ROWS), rng.line(MAX_ROWS)); + let (c0, c1) = (rng.line(MAX_COLS), rng.line(MAX_COLS)); + range(r0.min(r1), c0.min(c1), r0.max(r1), c0.max(c1)) + }; + let count = if rng.below(8) == 0 { rng.below(u32::MAX as u64) as u32 } else { rng.below(20) as u32 }; + match rng.below(7) { + 0 => Edit::Insert { axis, at: rng.line(max), count }, + 1 => Edit::Delete { axis, at: rng.line(max), count }, + 2 => Edit::InsertCells { axis, range: block(rng) }, + 3 => Edit::DeleteCells { axis, range: block(rng) }, + 4 => Edit::Move { from: block(rng), to_row: rng.line(MAX_ROWS), to_col: rng.line(MAX_COLS) }, + _ => Edit::MoveToSheet { + from: block(rng), + to_sheet: ["Dest", "S", "other"][rng.below(3) as usize].to_string(), + to_row: rng.line(MAX_ROWS), + to_col: rng.line(MAX_COLS), + }, + } +} + +/// Random references near the sheet's edges and random edits: "unchanged" is reported exactly +/// when the previous implementation leaves the formula as it was, and a reported change is the +/// old result. +#[test] +fn adjust_if_changed_reports_unchanged_exactly_when_nothing_moves() { + let mut rng = Rng(0x5eed); + let (mut unchanged, mut changed) = (0, 0); + for case in 0..20_000 { + let refs: Vec = (0..1 + rng.below(4)).map(|_| random_ref(&mut rng)).collect(); + let src = format!("SUM({})", refs.join(",")); + let Ok(e) = parse(&src) else { panic!("case {case}: {src} does not parse") }; + let edit = random_edit(&mut rng); + let host = ["S", "Other"][rng.below(2) as usize]; + let want = old::adjust(e.clone(), host, "S", &edit); + assert!(adjust(e.clone(), host, "S", &edit) == want, "case {case}: {src} on {host} after {edit:?}"); + match adjust_if_changed(&e, host, "S", &edit) { + None => { + assert!(want == e, "case {case}: {src} on {host} after {edit:?}: reported unchanged, old gives {}", print(&want)); + unchanged += 1; + } + Some(x) => { + assert!(want != e && x == want, "case {case}: {src} on {host} after {edit:?}"); + changed += 1; + } + } + } + // Both outcomes are exercised in quantity. + assert!(unchanged > 2_000 && changed > 2_000, "unchanged {unchanged}, changed {changed}"); +} + +/// The whole formula corpus (one formula per line, with or without `=`) when +/// `GRIDCRAFT_ADJUST_CORPUS` names a file: `cargo test -p gridcraft-formula --test it -- --ignored adjust_corpus`. +#[test] +#[ignore = "needs GRIDCRAFT_ADJUST_CORPUS"] +fn adjust_corpus_matches_the_previous_implementation() { + let Ok(path) = std::env::var("GRIDCRAFT_ADJUST_CORPUS") else { return }; + let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{path}: {e}")); + let edits = edits(); + let (mut parsed, mut skipped, mut cases) = (0, 0, 0); + for line in text.lines().filter(|l| !l.trim().is_empty()) { + match check(line, &edits) { + 0 => skipped += 1, + n => { + parsed += 1; + cases += n; + } + } + } + eprintln!("corpus: {parsed} formulas parsed, {skipped} skipped (do not parse), {cases} cases"); +} + +#[test] +fn unaffected_formula_is_returned_as_is() { + let e: Expr = parse("Other!B2+SUM(A1:A3)").unwrap(); + let ins = Edit::Insert { axis: Axis::Rows, at: 5, count: 1 }; + // Row 6 inserted: A1:A3 lies above it and Other!B2 is not on S, so nothing moves, whichever + // sheet the formula lives on (on Other, A1:A3 is on Other too). + assert!(adjust_if_changed(&e, "S", "S", &ins).is_none()); + assert!(adjust_if_changed(&e, "Other", "S", &ins).is_none()); + // Row 1 inserted: A1:A3 on S moves, Other!B2 does not. + let top = Edit::Insert { axis: Axis::Rows, at: 0, count: 1 }; + assert_eq!(adjust_if_changed(&e, "S", "S", &top).map(|x| print(&x)).as_deref(), Some("Other!B2+SUM(A2:A4)")); + // A cut that moves a block onto the same place still qualifies its references with the sheet. + let same_place = Edit::MoveToSheet { from: range(0, 0, 2, 0), to_sheet: "S".into(), to_row: 0, to_col: 0 }; + assert_eq!(adjust_if_changed(&e, "S", "S", &same_place).map(|x| print(&x)).as_deref(), Some("Other!B2+SUM(S!A1:A3)")); +} diff --git a/crates/formula/tests/it/main.rs b/crates/formula/tests/it/main.rs index 01bff98c1..0f0292479 100644 --- a/crates/formula/tests/it/main.rs +++ b/crates/formula/tests/it/main.rs @@ -1,4 +1,5 @@ //! The crate's integration tests, one module per file, built as one test binary. +mod adjust_oracle; mod printer_roundtrip; mod spanish_input; From 2ee261c0e8ff389be25ac494d121fc7bde4a622c Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 03:51:57 +0100 Subject: [PATCH 48/51] perf(model): move whole rows when inserting or deleting rows and columns (#3575) CellStore::shift_rows, shift_cols, shift_cols_in_rows and take_range removed and re-set every affected cell one by one. They now take whole row maps (or the split-off part of one) out of the bands and put them back at their new position, so a row shift costs a few map operations per row instead of a band lookup, a remove and a set per cell. take_range does this when the block reaches the last column. Behaviour is unchanged, including blank cells left behind by get_mut, cells off the grid (they are still dropped, and still clear the cell they land on), cells shifted off the sheet, and copy-on-write: bands without moved cells stay shared with undo snapshots. A differential test runs 12,000 random operations against the old per-cell code and compares contents, length, iteration order and band sharing. shift_rows_in_cols, and take_range on a block with columns to its right, keep moving cell by cell: the row-map version measured no faster there. Co-Authored-By: Claude Opus 5.5 --- crates/model/src/store.rs | 402 ++++++++++++++++++++++++++++++++++---- 1 file changed, 367 insertions(+), 35 deletions(-) diff --git a/crates/model/src/store.rs b/crates/model/src/store.rs index be52bbe35..e2da31e43 100644 --- a/crates/model/src/store.rs +++ b/crates/model/src/store.rs @@ -4,6 +4,8 @@ //! copies only the band pointers; editing a cell copies just its band. use std::collections::BTreeMap; +use std::collections::btree_map::Entry; +use std::ops::RangeInclusive; use std::sync::Arc; use gridcraft_core::{CellRef, RangeRef}; @@ -266,64 +268,187 @@ impl CellStore { } bytes } - /// Removes every cell in `r` and returns them. + /// Removes every cell in `r` and returns them, row-major. pub fn take_range(&mut self, r: RangeRef) -> Vec<(CellRef, Cell)> { - let keys: Vec = self.iter_range(r).map(|(c, _)| c).collect(); - keys.into_iter().filter_map(|c| self.remove(c).map(|cell| (c, cell))).collect() + if r.end.col < gridcraft_core::MAX_COLS - 1 { + // A block with columns to its right: cutting it out of each row map measured slower + // than removing its cells one by one (#3575). + let keys: Vec = self.iter_range(r).map(|(c, _)| c).collect(); + return keys.into_iter().filter_map(|c| self.remove(c).map(|cell| (c, cell))).collect(); + } + let rows = self.take_rows(r.start.row..=r.end.row, r.start.col..=r.end.col); + rows.into_iter().flat_map(|(row, cols)| cols.into_iter().map(move |(col, cell)| (CellRef::new(row, col), cell))).collect() } /// Rows strictly greater than or equal to `from` moved by `delta` (positive = down). Rows that /// would land before `from`... callers delete the vacated block first. pub fn shift_rows(&mut self, from: u32, delta: i64) { - let moved: Vec<(CellRef, Cell)> = { - let keys: Vec = self.iter().filter(|(c, _)| c.row >= from).map(|(c, _)| c).collect(); - keys.into_iter().filter_map(|c| self.remove(c).map(|cell| (c, cell))).collect() - }; - for (c, cell) in moved { - let nr = c.row as i64 + delta; - if (0..gridcraft_core::MAX_ROWS as i64).contains(&nr) { - self.set(CellRef::new(nr as u32, c.col), cell); - } - } + let moved = self.take_rows(from..=u32::MAX, 0..=u32::MAX); + self.put_rows(moved.into_iter().filter_map(|(row, cols)| Some((shifted(row, delta, gridcraft_core::MAX_ROWS)?, cols)))); } pub fn shift_cols(&mut self, from: u32, delta: i64) { - let keys: Vec = self.iter().filter(|(c, _)| c.col >= from).map(|(c, _)| c).collect(); - let moved: Vec<(CellRef, Cell)> = keys.into_iter().filter_map(|c| self.remove(c).map(|cell| (c, cell))).collect(); - for (c, cell) in moved { - let nc = c.col as i64 + delta; - if (0..gridcraft_core::MAX_COLS as i64).contains(&nc) { - self.set(CellRef::new(c.row, nc as u32), cell); - } - } + let moved = self.take_rows(0..=u32::MAX, from..=u32::MAX); + self.put_rows(moved.into_iter().map(|(row, cols)| (row, shift_keys(cols, delta)))); } /// Shifts cells within rows `r0..=r1` and columns `>= from` horizontally (Insert/Delete Cells /// with shift right/left). pub fn shift_cols_in_rows(&mut self, r0: u32, r1: u32, from: u32, delta: i64) { - let range = RangeRef::new(CellRef::new(r0, from), CellRef::new(r1, gridcraft_core::MAX_COLS - 1)); - let moved = self.take_range(range); - for (c, cell) in moved { - let nc = c.col as i64 + delta; - if (0..gridcraft_core::MAX_COLS as i64).contains(&nc) { - self.set(CellRef::new(c.row, nc as u32), cell); - } - } + let r = RangeRef::new(CellRef::new(r0, from), CellRef::new(r1, gridcraft_core::MAX_COLS - 1)); + let moved = self.take_rows(r.start.row..=r.end.row, r.start.col..=r.end.col); + self.put_rows(moved.into_iter().map(|(row, cols)| (row, shift_keys(cols, delta)))); } /// Shifts cells within columns `c0..=c1` and rows `>= from` vertically. pub fn shift_rows_in_cols(&mut self, c0: u32, c1: u32, from: u32, delta: i64) { + // Cell by cell: the moved cells sit between columns that stay, so moving row maps + // measured no faster here (#3575). let range = RangeRef::new(CellRef::new(from, c0), CellRef::new(gridcraft_core::MAX_ROWS - 1, c1)); - let moved = self.take_range(range); - for (c, cell) in moved { - let nr = c.row as i64 + delta; - if (0..gridcraft_core::MAX_ROWS as i64).contains(&nr) { - self.set(CellRef::new(nr as u32, c.col), cell); + for (c, cell) in self.take_range(range) { + if let Some(nr) = shifted(c.row, delta, gridcraft_core::MAX_ROWS) { + self.set(CellRef::new(nr, c.col), cell); + } + } + } + /// Removes the cells in `rows` x `cols` and returns them grouped by row, in row order. Moves + /// whole column maps (or the split-off part of one) instead of removing cell by cell, and only + /// copies bands that hold a cell in the block, so snapshots keep sharing the others. + fn take_rows(&mut self, rows: RangeInclusive, cols: RangeInclusive) -> Vec<(u32, BTreeMap)> { + let (r0, r1, c0, c1) = (*rows.start(), *rows.end(), *cols.start(), *cols.end()); + let mut out = Vec::new(); + if r0 > r1 || c0 > c1 { + return out; + } + let whole_rows = c0 == 0 && c1 == u32::MAX; + let keys: Vec = self + .bands + .range(r0 / BAND..=r1 / BAND) + .filter(|(_, band)| band.rows.range(r0..=r1).any(|(_, row)| row.range(c0..=c1).next().is_some())) + .map(|(k, _)| *k) + .collect(); + for key in keys { + let Some(band) = self.bands.get_mut(&key) else { continue }; + let inside = |row: Option<&u32>| row.is_some_and(|r| (r0..=r1).contains(r)); + if whole_rows && inside(band.rows.keys().next()) && inside(band.rows.keys().next_back()) { + // The whole band moves: no splitting. + let Some(band) = self.bands.remove(&key) else { continue }; + for (row, taken) in Arc::unwrap_or_clone(band).rows { + self.len -= taken.len(); + out.push((row, taken)); + } + continue; + } + let band = Arc::make_mut(band); + let mut block = band.rows.split_off(&r0); + let mut tail = match r1.checked_add(1) { + Some(next) => block.split_off(&next), + None => BTreeMap::new(), + }; + for (row, mut kept) in block { + let taken = if whole_rows { + std::mem::take(&mut kept) + } else if c1.checked_add(1).is_none_or(|next| kept.range(next..).next().is_none()) { + // Nothing right of the block: one split. + kept.split_off(&c0) + } else { + let mut taken = BTreeMap::new(); + for (col, cell) in kept.extract_if(c0..=c1, |_, _| true) { + taken.insert(col, cell); + } + taken + }; + if !kept.is_empty() { + band.rows.insert(row, kept); + } + if !taken.is_empty() { + self.len -= taken.len(); + out.push((row, taken)); + } + } + band.rows.append(&mut tail); + if band.rows.is_empty() { + self.bands.remove(&key); + } + } + out + } + /// Stores rows of cells (each row at most once) the way `set` stores each cell: a cell + /// replaces what is there, and a blank cell clears its destination instead of being stored. + /// A row with nothing stored yet takes the whole map. Rows in order share one band lookup. + fn put_rows(&mut self, rows: impl Iterator)>) { + let mut rows = rows.peekable(); + let mut group = Vec::new(); + while let Some((first, cols)) = rows.next() { + let key = first / BAND; + group.clear(); + group.push((first, cols)); + while let Some(next) = rows.next_if(|(row, _)| row / BAND == key) { + group.push(next); + } + // Leave the band shared when only blank cells land on empty positions. + let existing = self.bands.get(&key); + let writes = group.iter().any(|(row, cols)| { + let dest = existing.and_then(|band| band.rows.get(row)); + cols.iter().any(|(col, cell)| !cell.is_blank() || dest.is_some_and(|d| d.contains_key(col))) + }); + if !writes { + continue; + } + let band = Arc::make_mut(self.bands.entry(key).or_default()); + for (row, mut cols) in group.drain(..) { + match band.rows.entry(row) { + Entry::Vacant(slot) => { + if cols.values().any(Cell::is_blank) { + cols.retain(|_, cell| !cell.is_blank()); + } + if !cols.is_empty() { + self.len += cols.len(); + slot.insert(cols); + } + } + Entry::Occupied(mut slot) => { + let dest = slot.get_mut(); + for (col, cell) in cols { + if cell.is_blank() { + if dest.remove(&col).is_some() { + self.len -= 1; + } + } else if dest.insert(col, cell).is_none() { + self.len += 1; + } + } + if dest.is_empty() { + slot.remove(); + } + } + } + } + if band.rows.is_empty() { + self.bands.remove(&key); } } } } +/// `at + delta` when it lies in `0..limit`. +fn shifted(at: u32, delta: i64, limit: u32) -> Option { + let to = (at as i64).checked_add(delta)?; + if (0..limit as i64).contains(&to) { Some(to as u32) } else { None } +} + +/// The cells of one row with their columns moved by `delta`; cells that would leave the sheet are +/// dropped. +fn shift_keys(cols: BTreeMap, delta: i64) -> BTreeMap { + let mut out = BTreeMap::new(); + for (col, cell) in cols { + if let Some(c) = shifted(col, delta, gridcraft_core::MAX_COLS) { + out.insert(c, cell); + } + } + out +} + #[cfg(test)] mod tests { use super::*; - use gridcraft_core::Value; + use gridcraft_core::{MAX_COLS, MAX_ROWS, Value}; fn c(r: u32, col: u32) -> CellRef { CellRef::new(r, col) @@ -396,4 +521,211 @@ mod tests { s.shift_cols(0, 1); assert!(s.get(c(7, 1)).is_some()); } + + /// The per-cell algorithms the shifts used before they moved whole rows (#3575), kept as the + /// reference the structural versions must match cell for cell. + mod per_cell { + use super::*; + + fn reset(s: &mut CellStore, moved: Vec<(CellRef, Cell)>, to: impl Fn(CellRef) -> Option) { + for (at, cell) in moved { + if let Some(at) = to(at) { + s.set(at, cell); + } + } + } + fn row_by(delta: i64) -> impl Fn(CellRef) -> Option { + move |at| { + let r = at.row as i64 + delta; + (0..MAX_ROWS as i64).contains(&r).then(|| CellRef::new(r as u32, at.col)) + } + } + fn col_by(delta: i64) -> impl Fn(CellRef) -> Option { + move |at| { + let c = at.col as i64 + delta; + (0..MAX_COLS as i64).contains(&c).then(|| CellRef::new(at.row, c as u32)) + } + } + pub fn take_range(s: &mut CellStore, r: RangeRef) -> Vec<(CellRef, Cell)> { + let keys: Vec = s.iter_range(r).map(|(c, _)| c).collect(); + keys.into_iter().filter_map(|c| s.remove(c).map(|cell| (c, cell))).collect() + } + pub fn shift_rows(s: &mut CellStore, from: u32, delta: i64) { + let keys: Vec = s.iter().filter(|(c, _)| c.row >= from).map(|(c, _)| c).collect(); + let moved = keys.into_iter().filter_map(|c| s.remove(c).map(|cell| (c, cell))).collect(); + reset(s, moved, row_by(delta)); + } + pub fn shift_cols(s: &mut CellStore, from: u32, delta: i64) { + let keys: Vec = s.iter().filter(|(c, _)| c.col >= from).map(|(c, _)| c).collect(); + let moved = keys.into_iter().filter_map(|c| s.remove(c).map(|cell| (c, cell))).collect(); + reset(s, moved, col_by(delta)); + } + pub fn shift_cols_in_rows(s: &mut CellStore, r0: u32, r1: u32, from: u32, delta: i64) { + let moved = take_range(s, RangeRef::new(CellRef::new(r0, from), CellRef::new(r1, MAX_COLS - 1))); + reset(s, moved, col_by(delta)); + } + pub fn shift_rows_in_cols(s: &mut CellStore, c0: u32, c1: u32, from: u32, delta: i64) { + let moved = take_range(s, RangeRef::new(CellRef::new(from, c0), CellRef::new(MAX_ROWS - 1, c1))); + reset(s, moved, row_by(delta)); + } + } + + struct Lcg(u64); + impl Lcg { + fn below(&mut self, n: u64) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (self.0 >> 33) % n.max(1) + } + fn pick(&mut self, xs: &[T]) -> T { + xs[self.below(xs.len() as u64) as usize] + } + } + + /// A position near a band edge, the sheet edge, or anywhere below `max`. + fn coord(rng: &mut Lcg, max: u32, edges: &[u32]) -> u32 { + match rng.below(3) { + 0 => rng.pick(edges).min(max - 1), + 1 => max - 1 - rng.below(70.min(max as u64)) as u32, + _ => rng.below(max as u64) as u32, + } + } + + /// Random store with values, styled-only cells, a few cells off the grid, and blank cells left + /// behind by `get_mut` (which `set` never stores, but the old shifts dropped and let clear + /// their destination). + fn random_store(rng: &mut Lcg, max_r: u32, max_c: u32) -> CellStore { + let row_edges = [0, 1, 62, 63, 64, 65, 127, 128, 129, 191, 192, MAX_ROWS - 64, MAX_ROWS - 1]; + let col_edges = [0, 1, 2, 63, 64, 65, MAX_COLS - 2, MAX_COLS - 1]; + let mut s = CellStore::new(); + for _ in 0..rng.below(300) { + let mut at = CellRef::new(coord(rng, max_r, &row_edges), coord(rng, max_c, &col_edges)); + // Now and then a cell outside the grid (`set` does not check): the shifts must treat + // it as the old code did. + match rng.below(60) { + 0 => at.row = MAX_ROWS + rng.below(3) as u32, + 1 => at.col = MAX_COLS + rng.below(3) as u32, + _ => {} + } + let cell = match rng.below(5) { + 0 => Cell { style: crate::style::StyleId(1 + rng.below(3) as u32), ..Cell::default() }, + 1 => Cell::value(Value::from("t")), + _ => Cell::value(Value::Number(rng.below(1000) as f64)), + }; + s.set(at, cell); + if rng.below(8) == 0 + && let Some(cell) = s.get_mut(at) + { + *cell = Cell::default(); + } + } + s + } + + fn assert_same(new: &CellStore, old: &CellStore, base: &CellStore, base_copy: &CellStore, what: &str) { + assert_eq!(new, old, "{what}"); + assert_eq!(new.len(), new.iter().count(), "{what}: len"); + assert!(new.iter().map(|(c, x)| (c, x.clone())).eq(old.iter().map(|(c, x)| (c, x.clone()))), "{what}: iteration"); + assert!(new.bands.values().all(|b| !b.rows.is_empty() && b.rows.values().all(|r| !r.is_empty())), "{what}: empty band or row"); + // Copy-on-write: every band the old code left shared with the snapshot stays shared. + for (k, b) in &base.bands { + if old.bands.get(k).is_some_and(|o| Arc::ptr_eq(o, b)) { + assert!(new.bands.get(k).is_some_and(|n| Arc::ptr_eq(n, b)), "{what}: band {k} copied"); + } + } + assert_eq!(base, base_copy, "{what}: snapshot changed"); + } + + #[test] + fn structural_shifts_match_per_cell_shifts() { + let mut rng = Lcg(3575); + let deltas = [0i64, 1, -1, 2, -2, 63, -63, 64, -64, 65, -65, 128, -128, 1000, -1000, MAX_ROWS as i64, -(MAX_ROWS as i64)]; + let row_edges = [0, 1, 63, 64, 65, 127, 128, MAX_ROWS - 1, MAX_ROWS]; + let col_edges = [0, 1, 63, 64, 65, MAX_COLS - 1, MAX_COLS]; + for trial in 0..3000 { + let max_r = [200, 5000, MAX_ROWS][trial % 3]; + let max_c = [30, MAX_COLS][trial / 3 % 2]; + let mut base = random_store(&mut rng, max_r, max_c); + for step in 0..4 { + let snapshot = base.clone(); + let (mut new, mut old) = (base.clone(), base.clone()); + let delta = if rng.below(2) == 0 { rng.pick(&deltas) } else { rng.below(301) as i64 - 150 }; + let row = if rng.below(2) == 0 { rng.pick(&row_edges) } else { rng.below(max_r as u64 + 1) as u32 }; + let col = if rng.below(2) == 0 { rng.pick(&col_edges) } else { rng.below(max_c as u64 + 1) as u32 }; + let (row2, col2) = (rng.below(max_r as u64 + 1) as u32, rng.below(max_c as u64 + 1) as u32); + let op = rng.below(5); + match op { + 0 => { + new.shift_rows(row, delta); + per_cell::shift_rows(&mut old, row, delta); + } + 1 => { + new.shift_cols(col, delta); + per_cell::shift_cols(&mut old, col, delta); + } + 2 => { + new.shift_cols_in_rows(row, row2, col, delta); + per_cell::shift_cols_in_rows(&mut old, row, row2, col, delta); + } + 3 => { + new.shift_rows_in_cols(col, col2, row, delta); + per_cell::shift_rows_in_cols(&mut old, col, col2, row, delta); + } + _ => { + let r = RangeRef::new(CellRef::new(row, col), CellRef::new(row2, col2)); + assert_eq!(new.take_range(r), per_cell::take_range(&mut old, r), "trial {trial} step {step}: taken cells"); + } + } + let what = format!("trial {trial} step {step} op {op} row {row}..{row2} col {col}..{col2} delta {delta}"); + assert_same(&new, &old, &base, &snapshot, &what); + base = new; + } + } + } + + #[test] + fn off_grid_cells_right_of_a_block_stay_put() { + let mut s = CellStore::new(); + for col in [0, 5, MAX_COLS - 1, MAX_COLS + 1] { + s.set(c(3, col), Cell::value((col as f64).into())); + } + let mut old = s.clone(); + s.shift_cols_in_rows(3, 3, 0, 1); + per_cell::shift_cols_in_rows(&mut old, 3, 3, 0, 1); + assert_eq!(s, old); + assert_eq!(s.get(c(3, MAX_COLS + 1)).map(|x| x.value.clone()), Some(Value::Number((MAX_COLS + 1) as f64))); + let r = RangeRef::rows(3, 3); + assert_eq!(s.take_range(r), per_cell::take_range(&mut old, r)); + assert_eq!(s, old); + } + + #[test] + fn moved_blank_cell_clears_its_destination() { + let mut s = CellStore::new(); + s.set(c(3, 0), Cell::value(1.0.into())); + s.set(c(5, 0), Cell::value(2.0.into())); + s.set(c(5, 1), Cell::value(3.0.into())); + if let Some(cell) = s.get_mut(c(5, 0)) { + *cell = Cell::default(); + } + s.shift_rows(5, -2); + assert!(s.get(c(3, 0)).is_none()); + assert_eq!(s.get(c(3, 1)).map(|x| x.value.clone()), Some(Value::Number(3.0))); + assert_eq!(s.len(), 1); + } + + #[test] + fn shifting_keeps_untouched_bands_shared() { + let mut s = CellStore::new(); + for r in 0..256 { + s.set(c(r, 0), Cell::value((r as f64).into())); + } + s.set(c(200, 5), Cell::value("x".into())); + let snap = s.clone(); + s.shift_rows(130, 1); + assert!(Arc::ptr_eq(&s.bands[&0], &snap.bands[&0])); + assert!(Arc::ptr_eq(&s.bands[&1], &snap.bands[&1])); + s.shift_cols(5, 1); + assert!(Arc::ptr_eq(&s.bands[&0], &snap.bands[&0])); + assert_eq!(s.get(c(201, 6)).map(|x| x.value.clone()), Some(Value::from("x"))); + } } From 9ab297ced1115c2088bb0065588a4500563476f2 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 04:06:21 +0100 Subject: [PATCH 49/51] perf(engine): rewrite only the formulas a structural edit changes rewrite_all_formulas borrows each tree with Formula::with_expr and calls adjust_if_changed, so a formula the edit leaves alone is neither copied nor printed, and a changed one is printed once. A legacy array whose range moves while its formula stays the same still gets the new range. Sheet rename and delete compare the adjusted tree with the old one instead of printed text, and replace only formulas that changed. Delete no longer rebuilds every formula in the workbook, so unrelated formulas keep their array range and dynamic-array mark. A stored non-canonical formula text is no longer re-printed by an edit that does not change the formula; a test pins this. Part of #3576 Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/cmd/edit/paste_special.rs | 18 ++-- crates/engine/src/cmd/sheet/tabs.rs | 25 +++--- crates/engine/tests/it/main.rs | 1 + .../tests/it/structural_edit_rewrite.rs | 84 +++++++++++++++++++ 4 files changed, 106 insertions(+), 22 deletions(-) create mode 100644 crates/engine/tests/it/structural_edit_rewrite.rs diff --git a/crates/engine/src/cmd/edit/paste_special.rs b/crates/engine/src/cmd/edit/paste_special.rs index df2b0de61..c90dd9ec8 100644 --- a/crates/engine/src/cmd/edit/paste_special.rs +++ b/crates/engine/src/cmd/edit/paste_special.rs @@ -567,19 +567,19 @@ pub(crate) fn rewrite_all_formulas(wb: &mut gridcraft_model::Workbook, target: & let on_target = moves_arrays && host.eq_ignore_ascii_case(target); let Some(sheet) = wb.sheet_mut(si) else { continue }; for (c, f) in keys { - let Some(expr) = f.expr() else { continue }; - let new = gridcraft_formula::adjust::adjust(expr, &host, target, e); - let text = gridcraft_formula::print(&new); + // Borrows the tree and copies and prints only the formulas the edit changes. A formula + // the edit leaves alone keeps its stored text, even if that text is not canonical. + let Some(new) = f.with_expr(|x| gridcraft_formula::adjust::adjust_if_changed(x, &host, target, e)) else { continue }; let array = match f.array { Some(a) if on_target => crate::cmd::sheet::map_range(a, e), a => a, }; - if (text != f.text || array != f.array) - && let Some(cell) = sheet.cells.get_mut(c) - { - let mut nf = Formula::from_expr(new); - nf.array = array; - nf.dynamic = f.dynamic; + let nf = match new { + Some(new) => Formula { array, dynamic: f.dynamic, ..Formula::from_expr(new) }, + None if array != f.array => Formula { array, ..(*f).clone() }, + None => continue, + }; + if let Some(cell) = sheet.cells.get_mut(c) { cell.formula = Some(std::sync::Arc::new(nf)); } } diff --git a/crates/engine/src/cmd/sheet/tabs.rs b/crates/engine/src/cmd/sheet/tabs.rs index d21851553..2c1657ef9 100644 --- a/crates/engine/src/cmd/sheet/tabs.rs +++ b/crates/engine/src/cmd/sheet/tabs.rs @@ -55,11 +55,12 @@ pub(super) fn delete_sheet(s: &mut Session, p: &Json) -> Result { cx.wb.sheets.get(si).map(|s| s.cells.iter().filter_map(|(c, x)| x.formula.clone().map(|f| (c, f))).collect()).unwrap_or_default(); let Some(sh) = cx.wb.sheet_mut(si) else { continue }; for (c, f) in keys { - if let Some(e) = f.expr() { - let ne = gridcraft_formula::adjust::delete_sheet(e, &name, &order); - if let Some(cell) = sh.cells.get_mut(c) { - cell.formula = Some(Arc::new(gridcraft_model::Formula::from_expr(ne))); - } + // Only formulas whose tree changes are rewritten; the others keep their text. + let changed = f.with_expr(|e| Some(gridcraft_formula::adjust::delete_sheet(e.clone(), &name, &order)).filter(|ne| ne != e)); + if let Some(Some(ne)) = changed + && let Some(cell) = sh.cells.get_mut(c) + { + cell.formula = Some(Arc::new(gridcraft_model::Formula::from_expr(ne))); } } } @@ -109,14 +110,12 @@ pub(super) fn rename_sheet(s: &mut Session, p: &Json) -> Result { cx.wb.sheets.get(si).map(|s| s.cells.iter().filter_map(|(c, x)| x.formula.clone().map(|f| (c, f))).collect()).unwrap_or_default(); let Some(sh) = cx.wb.sheet_mut(si) else { continue }; for (c, f) in keys { - if let Some(e) = f.expr() { - let ne = gridcraft_formula::adjust::rename_sheet(e, &old, &name); - let text = gridcraft_formula::print(&ne); - if text != f.text - && let Some(cell) = sh.cells.get_mut(c) - { - cell.formula = Some(Arc::new(gridcraft_model::Formula::from_expr(ne))); - } + // Compares trees, so unchanged formulas are not printed; they keep their text. + let changed = f.with_expr(|e| Some(gridcraft_formula::adjust::rename_sheet(e.clone(), &old, &name)).filter(|ne| ne != e)); + if let Some(Some(ne)) = changed + && let Some(cell) = sh.cells.get_mut(c) + { + cell.formula = Some(Arc::new(gridcraft_model::Formula::from_expr(ne))); } } for ch in sh.charts.iter_mut() { diff --git a/crates/engine/tests/it/main.rs b/crates/engine/tests/it/main.rs index 0ba7c0db8..ca960d759 100644 --- a/crates/engine/tests/it/main.rs +++ b/crates/engine/tests/it/main.rs @@ -29,6 +29,7 @@ mod sheet_copy_spill; mod sheet_name_quoting; mod sparse_clear_invalidation; mod stored_list_shape; +mod structural_edit_rewrite; mod table_resize_overlap; mod table_total_row_last_row; mod trace_error; diff --git a/crates/engine/tests/it/structural_edit_rewrite.rs b/crates/engine/tests/it/structural_edit_rewrite.rs new file mode 100644 index 000000000..bb87ba3c4 --- /dev/null +++ b/crates/engine/tests/it/structural_edit_rewrite.rs @@ -0,0 +1,84 @@ +//! Structural edits (insert and delete of lines, sheet rename and delete) rewrite only the formulas +//! whose tree they change. A formula they leave alone keeps its stored text and its flags. + +use std::sync::Arc; + +use gridcraft_core::{CellRef, RangeRef}; +use gridcraft_engine::Session; +use gridcraft_model::{Cell, Formula}; +use serde_json::{Value as Json, json}; + +fn run(s: &mut Session, command: &str, params: Json) { + s.execute(command, params).unwrap(); +} + +fn workbook() -> Session { + let mut s = Session::new(); + s.new_workbook(); + run(&mut s, "home.insertSheet", json!({"name": "Other"})); + run(&mut s, "home.insertSheet", json!({"name": "Spare"})); + run(&mut s, "sheet.activate", json!({"sheet": "Sheet1"})); + s +} + +fn formula(s: &Session, sheet: &str, cell: &str) -> Arc { + let wb = &s.doc().unwrap().wb; + wb.sheet(wb.sheet_index(sheet).unwrap()).unwrap().cell(CellRef::parse(cell).unwrap()).unwrap().formula.clone().unwrap() +} + +/// Stores a formula with this exact text and no cached tree, as a JSON workbook loads it. +fn put_raw(s: &mut Session, cell: &str, text: &str) { + let wb = Arc::make_mut(&mut s.doc_mut().unwrap().wb); + let i = wb.sheet_index("Sheet1").unwrap(); + let f = Formula { text: text.to_string(), expr: None, array: None, dynamic: false }; + wb.sheet_mut(i).unwrap().cells.set(CellRef::parse(cell).unwrap(), Cell::formula(f)); +} + +#[test] +fn an_edit_that_leaves_a_formula_alone_keeps_its_stored_text() { + // The rewrite compares trees, not printed text, so a stored non-canonical text is not + // re-printed by an edit that does not change the formula. An edit that changes it prints it. + let mut s = workbook(); + put_raw(&mut s, "C1", "sum(a1, Other!b2)"); + run(&mut s, "home.insertRows", json!({"rows": "5:5"})); + run(&mut s, "home.insertColumns", json!({"cols": "F:F"})); + run(&mut s, "sheet.rename", json!({"sheet": "Spare", "name": "Spare2"})); + run(&mut s, "home.deleteSheet", json!({"sheet": "Spare2"})); + assert_eq!(formula(&s, "Sheet1", "C1").text, "sum(a1, Other!b2)"); + + run(&mut s, "home.insertRows", json!({"rows": "1:1"})); + assert_eq!(formula(&s, "Sheet1", "C2").text, "SUM(A2,Other!B2)"); +} + +#[test] +fn renaming_a_referenced_sheet_rewrites_the_formula() { + let mut s = workbook(); + put_raw(&mut s, "C1", "sum(a1, Other!b2)"); + run(&mut s, "sheet.rename", json!({"sheet": "Other", "name": "Data"})); + assert_eq!(formula(&s, "Sheet1", "C1").text, "SUM(A1,Data!B2)"); + run(&mut s, "home.deleteSheet", json!({"sheet": "Data"})); + assert_eq!(formula(&s, "Sheet1", "C1").text, "SUM(A1,#REF!)"); +} + +#[test] +fn inserting_rows_above_a_legacy_array_moves_its_range_when_the_formula_is_unchanged() { + let mut s = workbook(); + run(&mut s, "range.setValues", json!({"range": "B1", "values": [[1], [2]]})); + run(&mut s, "selection.set", json!({"range": "C5:C6"})); + run(&mut s, "cell.set", json!({"cell": "C5", "input": "=B1:B2*2", "array": true})); + assert_eq!(formula(&s, "Sheet1", "C5").array, Some(RangeRef::parse("C5:C6").unwrap())); + run(&mut s, "home.insertRows", json!({"rows": "3:3"})); + let f = formula(&s, "Sheet1", "C6"); + assert_eq!(f.text, "B1:B2*2"); + assert_eq!(f.array, Some(RangeRef::parse("C6:C7").unwrap())); +} + +#[test] +fn deleting_an_unrelated_sheet_keeps_legacy_arrays() { + let mut s = workbook(); + run(&mut s, "range.setValues", json!({"range": "B1", "values": [[1], [2]]})); + run(&mut s, "selection.set", json!({"range": "C1:C2"})); + run(&mut s, "cell.set", json!({"cell": "C1", "input": "=B1:B2*2", "array": true})); + run(&mut s, "home.deleteSheet", json!({"sheet": "Spare"})); + assert_eq!(formula(&s, "Sheet1", "C1").array, Some(RangeRef::parse("C1:C2").unwrap())); +} From 99bbd237fcfdd851dd56f26cff0efc6758151c89 Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sun, 8 Nov 2026 11:53:25 -0800 Subject: [PATCH 50/51] Updates: the updater reports installing, ready, installed and failed, and installs at exit craft-update, craft-window, craft-print and craft-ui move from craft-libs c5a1cbfb to c7c84230, a branch with storytold/craft-libs#687 and #696 merged into craft-libs main. - The check result window shows whenever the updater's status is not idle, so the first start after an update reports that it installed or why it failed. Restart to Update asks the app to quit once nothing is unsaved. - An update's close request quits the app (Quit asks about each unsaved workbook), so every workbook window closes. - on_exit starts the install helper of a ready update before the macOS process exit. --- Cargo.lock | 17 ++++----- Cargo.toml | 6 +-- apps/gridcraft/Cargo.toml | 2 +- apps/gridcraft/src/main.rs | 3 ++ crates/ui-egui/Cargo.toml | 2 +- crates/ui-egui/src/app/prefs.rs | 8 ++++ crates/ui-egui/src/updates_ui.rs | 63 +++++++++++++++++++++++--------- 7 files changed, 69 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a39620f96..5d78dbed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,12 +910,12 @@ dependencies = [ [[package]] name = "craft-fonts-build" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" [[package]] name = "craft-i18n" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" dependencies = [ "fixed_decimal", "icu_decimal", @@ -938,7 +938,7 @@ dependencies = [ [[package]] name = "craft-print" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" dependencies = [ "craft-winprint-gdi", "log", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "craft-ui" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" dependencies = [ "craft-fonts-build", "craft-i18n", @@ -972,7 +972,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" dependencies = [ "chrono", "dunce", @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "craft-window" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" dependencies = [ "egui", "objc2 0.6.5", @@ -1029,7 +1029,7 @@ dependencies = [ [[package]] name = "craft-winprint-gdi" version = "0.1.0" -source = "git+https://github.com/storytold/winprint.rs?rev=417ac3c4d14d3da5485a99c7d5ce8f05664ec312#417ac3c4d14d3da5485a99c7d5ce8f05664ec312" +source = "git+https://github.com/storytold/winprint.rs?rev=64e7afd4d1483512385012c340ecaaaf0f3beb7f#64e7afd4d1483512385012c340ecaaaf0f3beb7f" dependencies = [ "windows-sys 0.61.2", ] @@ -6485,7 +6485,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=5f80b9baf6b58937c401656634e42d33f00ddf49#5f80b9baf6b58937c401656634e42d33f00ddf49" +source = "git+https://github.com/storytold/winprint.rs?rev=64e7afd4d1483512385012c340ecaaaf0f3beb7f#64e7afd4d1483512385012c340ecaaaf0f3beb7f" dependencies = [ "flate2", "fmt-derive", @@ -6935,7 +6935,6 @@ dependencies = [ "nix", "typed-path", "windows-sys 0.61.2", - "zopfli", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 594b29273..bc69f4847 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,9 +26,9 @@ gridcraft-mcp = { path = "crates/mcp" } gridcraft-pdf = { path = "crates/pdf" } gridcraft-math = { path = "crates/math" } # Every craft-libs crate (craft-update and craft-window here, craft-ui in ui-egui, craft-print in -# the app) comes from one storytold/craft-libs revision. +# the app) comes from one craft-libs revision. # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7", version = "=0.1.0", features = ["egui"] } +craft-update = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae", version = "=0.1.0", features = ["egui"] } # foundation thiserror = "2" @@ -71,7 +71,7 @@ skrifa = { version = "0.44", default-features = false, features = ["std"] } epaint_default_fonts = "0.37" # One native window per workbook: the window list, Window menu titles and the immediate-viewport # driver. Same craft-libs revision as craft-update above. -craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7", default-features = false, features = ["egui"] } +craft-window = { version = "0.1.0", git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae", default-features = false, features = ["egui"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/apps/gridcraft/Cargo.toml b/apps/gridcraft/Cargo.toml index a7472821f..e706279ce 100644 --- a/apps/gridcraft/Cargo.toml +++ b/apps/gridcraft/Cargo.toml @@ -34,7 +34,7 @@ winit = { version = "0.30", default-features = false, features = ["x11", "waylan [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Same craft-libs revision as the workspace's craft-update and craft-window (see the root # Cargo.toml). -craft-print = { git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7" } +craft-print = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae" } craft-update = { workspace = true } rfd = { workspace = true } arboard = { workspace = true, features = ["image-data"] } diff --git a/apps/gridcraft/src/main.rs b/apps/gridcraft/src/main.rs index 2aa98b6eb..cc67e241a 100644 --- a/apps/gridcraft/src/main.rs +++ b/apps/gridcraft/src/main.rs @@ -64,6 +64,9 @@ impl eframe::App for App { fn on_exit(&mut self) { print::finish_pending_jobs(); save_prefs(&self.0); + // A ready update installs once the process has exited. The macOS exit below skips + // dropping the app, so the helper is started here. + self.0.start_pending_update(); // macOS (#117): end the process here, before eframe tears the window down. AccessKit's // macOS adapter subclasses the window's content view and restores its class on teardown; // AppKit's Touch Bar support observes that view (KVO), so in the next display cycle diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index fd18f7041..7dccfd584 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -11,7 +11,7 @@ workspace = true [dependencies] # The shared capture slot (`craft_ui::capture`) comes from the one craft-libs revision every # craft-libs dependency uses (see the root Cargo.toml). -craft-ui = { git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7" } +craft-ui = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae" } gridcraft-engine = { workspace = true } gridcraft-chart = { workspace = true } # `monochrome_emoji_fonts` restores the Noto Emoji and emoji-icon faces that 0.37 moved behind diff --git a/crates/ui-egui/src/app/prefs.rs b/crates/ui-egui/src/app/prefs.rs index 216e4a568..151374e6e 100644 --- a/crates/ui-egui/src/app/prefs.rs +++ b/crates/ui-egui/src/app/prefs.rs @@ -64,4 +64,12 @@ impl SheetApp { self.message = Some(("GridCraft".into(), "The web version is always the latest release.".into())); } } + + /// Starts the install helper of an update that is ready, so it installs once the process has + /// exited (a no-op on the web, and when no update is ready). Call it in `on_exit` when the + /// process ends there, since the app is then never dropped. + pub fn start_pending_update(&mut self) { + #[cfg(not(target_arch = "wasm32"))] + updates_ui::start_pending_update(self); + } } diff --git a/crates/ui-egui/src/updates_ui.rs b/crates/ui-egui/src/updates_ui.rs index 2553fbbc7..cded96c85 100644 --- a/crates/ui-egui/src/updates_ui.rs +++ b/crates/ui-egui/src/updates_ui.rs @@ -15,13 +15,11 @@ pub(crate) struct Runtime { startup: bool, /// Help ▸ Check for Updates… was chosen; the next `tick` starts it. manual: bool, - /// The result dialog of a requested check is showing. - result_open: bool, } impl Default for Runtime { fn default() -> Self { - Self { updater: Updater::new(&crate::UPDATE_APP, VERSION), startup: false, manual: false, result_open: false } + Self { updater: Updater::new(&crate::UPDATE_APP, VERSION), startup: false, manual: false } } } @@ -38,19 +36,26 @@ pub(crate) fn tick(app: &mut SheetApp, ctx: &egui::Context) { let rt = &mut app.updates_rt; if std::mem::take(&mut rt.manual) { rt.updater.check_now(ctx); - rt.result_open = true; } else if rt.startup { rt.updater.check_at_start(ctx); } rt.updater.tick(ctx); - // A requested check that found a release leaves the status idle and the prompt opens. - if matches!(rt.updater.status(), Status::Idle) { - rt.result_open = false; - } +} + +/// Starts the helper of a ready update; it installs once the process has exited. +pub(crate) fn start_pending_update(app: &mut SheetApp) { + app.updates_rt.updater.start_pending_update(); +} + +/// Whether the check result window shows: for every status but idle (a requested check, and the +/// first start after an update, which reports that it installed or why it failed), and never +/// over the startup prompt. +fn result_window_shows(status: &Status, prompt_open: bool) -> bool { + !prompt_open && !matches!(status, Status::Idle) } pub(crate) fn show(app: &mut SheetApp, ctx: &egui::Context) { - // The restart helper is scheduled before the app closes, so no update starts over unsaved work. + // The close for a ready update waits until no workbook holds unsaved work. let clean = !app.session.documents().iter().any(|d| d.is_dirty()); match app.updates_rt.updater.show_if(ctx, clean) { Action::OpenUrl(url) => { @@ -58,20 +63,42 @@ pub(crate) fn show(app: &mut SheetApp, ctx: &egui::Context) { open(&url); } } - // The guarded close path asks about unsaved work; the restart helper relaunches once the - // process has exited. - Action::Close => ctx.send_viewport_cmd(egui::ViewportCommand::Close), + // Quit closes every workbook window. The install helper starts once the app exits. + Action::Close => app.request_quit(ctx), Action::None => {} } - if app.updates_rt.result_open { - let rt = &mut app.updates_rt; + let rt = &mut app.updates_rt; + if result_window_shows(rt.updater.status(), rt.updater.prompt_open()) { match craft_update::CheckResultDialog::new(&crate::UPDATE_APP, VERSION).show(ctx, rt.updater.status()) { - CheckResultResponse::Close => { - rt.updater.dismiss_status(); - rt.result_open = false; - } + CheckResultResponse::Close => rt.updater.dismiss_status(), CheckResultResponse::CheckAgain => rt.updater.check_now(ctx), + // The next `show_if` asks the window to close once nothing is unsaved. + CheckResultResponse::RestartToUpdate => rt.updater.restart_to_update(), CheckResultResponse::None => {} } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn result_window_shows_every_status_but_idle() { + let shown = [ + Status::Checking, + Status::UpToDate, + Status::Failed("no network".into()), + Status::Installing, + Status::Ready, + Status::Managed, + Status::Updated("0.2.0".into()), + Status::InstallFailed("disk full".into()), + ]; + for status in &shown { + assert!(result_window_shows(status, false), "{status:?}"); + assert!(!result_window_shows(status, true), "{status:?} over the prompt"); + } + assert!(!result_window_shows(&Status::Idle, false)); + } +} From 3004a05e463c3aac87b8901152d81a99b63672ac Mon Sep 17 00:00:00 2001 From: dotCooCoo Date: Sun, 8 Nov 2026 19:45:34 -0800 Subject: [PATCH 51/51] Updates: take craft-update from craft-libs main at 1b34eba41 --- Cargo.lock | 14 +++++++------- Cargo.toml | 4 ++-- apps/gridcraft/Cargo.toml | 2 +- crates/ui-egui/Cargo.toml | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d78dbed4..cb783e546 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,12 +910,12 @@ dependencies = [ [[package]] name = "craft-fonts-build" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" [[package]] name = "craft-i18n" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" dependencies = [ "fixed_decimal", "icu_decimal", @@ -938,7 +938,7 @@ dependencies = [ [[package]] name = "craft-print" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" dependencies = [ "craft-winprint-gdi", "log", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "craft-ui" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" dependencies = [ "craft-fonts-build", "craft-i18n", @@ -972,7 +972,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" dependencies = [ "chrono", "dunce", @@ -1018,7 +1018,7 @@ dependencies = [ [[package]] name = "craft-window" version = "0.1.0" -source = "git+https://github.com/dotCooCoo/craft-libs?rev=c7c84230631cbc01ee928f0a0e0d838b1ed612ae#c7c84230631cbc01ee928f0a0e0d838b1ed612ae" +source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" dependencies = [ "egui", "objc2 0.6.5", @@ -6485,7 +6485,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=64e7afd4d1483512385012c340ecaaaf0f3beb7f#64e7afd4d1483512385012c340ecaaaf0f3beb7f" +source = "git+https://github.com/storytold/winprint.rs?rev=51934db382f225454cf152a8657eb03789be63c4#51934db382f225454cf152a8657eb03789be63c4" dependencies = [ "flate2", "fmt-derive", diff --git a/Cargo.toml b/Cargo.toml index bc69f4847..1941c9d1e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ gridcraft-math = { path = "crates/math" } # Every craft-libs crate (craft-update and craft-window here, craft-ui in ui-egui, craft-print in # the app) comes from one craft-libs revision. # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae", version = "=0.1.0", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", version = "=0.1.0", features = ["egui"] } # foundation thiserror = "2" @@ -71,7 +71,7 @@ skrifa = { version = "0.44", default-features = false, features = ["std"] } epaint_default_fonts = "0.37" # One native window per workbook: the window list, Window menu titles and the immediate-viewport # driver. Same craft-libs revision as craft-update above. -craft-window = { version = "0.1.0", git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae", default-features = false, features = ["egui"] } +craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d", default-features = false, features = ["egui"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/apps/gridcraft/Cargo.toml b/apps/gridcraft/Cargo.toml index e706279ce..75acee3ff 100644 --- a/apps/gridcraft/Cargo.toml +++ b/apps/gridcraft/Cargo.toml @@ -34,7 +34,7 @@ winit = { version = "0.30", default-features = false, features = ["x11", "waylan [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Same craft-libs revision as the workspace's craft-update and craft-window (see the root # Cargo.toml). -craft-print = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae" } +craft-print = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d" } craft-update = { workspace = true } rfd = { workspace = true } arboard = { workspace = true, features = ["image-data"] } diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 7dccfd584..b4c049afa 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -11,7 +11,7 @@ workspace = true [dependencies] # The shared capture slot (`craft_ui::capture`) comes from the one craft-libs revision every # craft-libs dependency uses (see the root Cargo.toml). -craft-ui = { git = "https://github.com/dotCooCoo/craft-libs", rev = "c7c84230631cbc01ee928f0a0e0d838b1ed612ae" } +craft-ui = { git = "https://github.com/storytold/craft-libs", rev = "1b34eba41e62138fc2438d2734523b302840b45d" } gridcraft-engine = { workspace = true } gridcraft-chart = { workspace = true } # `monochrome_emoji_fonts` restores the Noto Emoji and emoji-icon faces that 0.37 moved behind