diff --git a/crates/calc/src/eval/array.rs b/crates/calc/src/eval/array.rs index 67662c83c..39a6e0f53 100644 --- a/crates/calc/src/eval/array.rs +++ b/crates/calc/src/eval/array.rs @@ -86,6 +86,30 @@ pub fn zip2(a: &Value, b: &Value, f: impl Fn(&Value, &Value) -> Value) -> Value } } +/// [`zip2`] that stops at the first cell `f` refuses (`None`), before the rest is built: the +/// result grows as cells are made rather than being reserved for the whole broadcast up front. +fn try_zip2(a: &Value, b: &Value, mut f: impl FnMut(&Value, &Value) -> Option) -> Option { + Some(match (a, b) { + (Value::Array(x), Value::Array(y)) => { + let rows = broadcast_dim(x.rows, y.rows); + let cols = broadcast_dim(x.cols, y.cols); + if broadcast_len(rows, cols).is_none() { + return Some(Value::Error(CellError::Num)); + } + let mut data = Vec::new(); + for r in 0..rows { + for c in 0..cols { + data.push(f(&x.get_broadcast(r, c), &y.get_broadcast(r, c))?); + } + } + Value::from(Array { rows, cols, data }) + } + (Value::Array(x), s) => Value::from(Array { rows: x.rows, cols: x.cols, data: x.data.iter().map(|v| f(v, s)).collect::>()? }), + (s, Value::Array(y)) => Value::from(Array { rows: y.rows, cols: y.cols, data: y.data.iter().map(|v| f(s, v)).collect::>()? }), + (x, y) => f(x, y)?, + }) +} + /// Cell count of a `rows`×`cols` broadcast result, or `None` when it would exceed [`MAX_CELLS`]. /// Each operand passes its own size cap, but their broadcast product can be far larger /// (`SEQUENCE(1048576)*SEQUENCE(1,16384)` is 17 billion cells), so it is checked before allocating. @@ -135,10 +159,36 @@ pub(super) fn single(v: Value) -> Option { } } +/// Text bytes that one `&` over arrays may produce in all. Each piece is capped on its own, but a +/// compact formula can broadcast a long text across millions of cells +/// (`SEQUENCE(4000,4000)&REPT("x",30000)` is 480 GB): past this the whole result is `#NUM!`, the +/// error for a result too large to hold, before the rest is built. +pub(super) const MAX_CONCAT_ARRAY_BYTES: usize = 512 * 1024 * 1024; + pub fn binary_values(op: BinOp, x: &Value, y: &Value, sys: gridcraft_core::DateSystem) -> Value { + if op == BinOp::Concat && (matches!(x, Value::Array(_)) || matches!(y, Value::Array(_))) { + return concat_arrays(x, y, sys, MAX_CONCAT_ARRAY_BYTES); + } zip2(x, y, |a, b| binary_scalar(op, a, b, sys)) } +/// `x & y` with at least one array operand, giving `#NUM!` once the texts built so far pass `budget` +/// bytes. +fn concat_arrays(x: &Value, y: &Value, sys: gridcraft_core::DateSystem, budget: usize) -> Value { + let mut used = 0usize; + try_zip2(x, y, |a, b| { + let v = binary_scalar(BinOp::Concat, a, b, sys); + if let Value::Text(t) = &v { + used = used.saturating_add(t.len()); + if used > budget { + return None; + } + } + Some(v) + }) + .unwrap_or(Value::Error(CellError::Num)) +} + fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSystem) -> Value { if let Value::Error(e) = a { return Value::Error(*e); @@ -187,7 +237,35 @@ fn binary_scalar(op: BinOp, a: &Value, b: &Value, sys: gridcraft_core::DateSyste #[cfg(test)] mod tests { - use super::{MAX_CELLS, broadcast_len}; + use super::{Array, BinOp, CellError, MAX_CELLS, Value, broadcast_len, concat_arrays, try_zip2}; + + /// The broadcast stops at the first refused cell: nothing after it is visited. + #[test] + fn a_refused_cell_stops_the_broadcast() { + let col = Value::from(Array::column((0..4_000).map(|i| Value::number(i as f64)).collect())); + let row = Value::from(Array { rows: 1, cols: 4_000, data: (0..4_000).map(|i| Value::number(i as f64)).collect() }); + let mut calls = 0; + let r = try_zip2(&col, &row, |a, _| { + calls += 1; + (calls <= 10).then(|| a.clone()) + }); + assert_eq!((r, calls), (None, 11)); + } + + #[test] + fn concatenating_arrays_stops_at_the_byte_budget() { + let sys = gridcraft_core::DateSystem::default(); + let col = Value::from(Array::column((0..100).map(|i| Value::number(i as f64)).collect())); + let long = Value::text("x".repeat(100)); + // 100 cells of 101 or 102 bytes each: about 10 KB in all. + let r = concat_arrays(&col, &long, sys, 20_000); + assert!(matches!(&r, Value::Array(a) if a.data.len() == 100), "{r:?}"); + assert_eq!(concat_arrays(&col, &long, sys, 5_000), Value::Error(CellError::Num)); + assert_eq!(concat_arrays(&long, &col, sys, 5_000), Value::Error(CellError::Num)); + // Through the operator, a small result is unaffected. + let r = super::binary_values(BinOp::Concat, &col, &Value::text("a"), sys); + assert!(matches!(&r, Value::Array(a) if a.data.first() == Some(&Value::text("0a"))), "{r:?}"); + } #[test] fn broadcast_len_caps_at_max_cells() {