From 691883f99c65772ed3d25a6247f3e71b28c0f184 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 20:56:16 +0200 Subject: [PATCH 01/51] Functions: VLOOKUP/HLOOKUP/LOOKUP/XLOOKUP keep an empty target blank (#374) A looked-up empty cell was turned into the number 0 inside the function, so ISBLANK saw FALSE and & / CONCATENATE inserted "0". The value now stays blank, like INDEX; a bare lookup still displays 0 as a formula's final result. Co-Authored-By: Claude Opus 5.5 --- crates/calc/src/tests.rs | 23 +++++++++++++++++++++++ crates/functions/src/lookup.rs | 22 ++++++++++------------ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 513340983..ac58b3d31 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -968,3 +968,26 @@ fn trim_references() { t.set("K20", "=SUM(Q1:.Q9)"); assert_eq!(t.get("K20"), Value::Error(CellError::Calc)); } + +#[test] +fn lookup_of_an_empty_cell_stays_blank() { + // #374: B1 (the value for key "k") is empty. + let mut t = T::new(); + t.set("A1", "k"); + t.set("A2", "j"); + t.set("B2", "v"); + for (at, f, want) in [ + ("D1", "=ISBLANK(VLOOKUP(\"k\",A1:B2,2,0))", Value::Bool(true)), + ("D2", "=\"x\"&VLOOKUP(\"k\",A1:B2,2,0)&\"y\"", Value::from("xy")), + ("D3", "=CONCATENATE(\"x\",VLOOKUP(\"k\",A1:B2,2,0),\"y\")", Value::from("xy")), + ("D4", "=ISBLANK(XLOOKUP(\"k\",A1:A2,B1:B2))", Value::Bool(true)), + ("D5", "=\"x\"&XLOOKUP(\"k\",A1:A2,B1:B2)&\"y\"", Value::from("xy")), + ("D6", "=ISBLANK(LOOKUP(\"k\",A1:A1,B1:B1))", Value::Bool(true)), + ("D7", "=ISBLANK(HLOOKUP(\"j\",A2:B3,2,0))", Value::Bool(true)), + // A bare lookup still shows 0, like a reference to an empty cell. + ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} diff --git a/crates/functions/src/lookup.rs b/crates/functions/src/lookup.rs index 56739c933..a04ce9960 100644 --- a/crates/functions/src/lookup.rs +++ b/crates/functions/src/lookup.rs @@ -119,11 +119,6 @@ fn row(a: &Array, r: usize) -> Vec { (0..a.cols).map(|c| a.get(r, c).cloned().unwrap_or(Value::Empty)).collect() } -/// A looked-up cell value: blanks read as 0 like a reference to an empty cell. -fn cell_result(v: Value) -> Value { - if matches!(v, Value::Empty) { Value::Number(0.0) } else { v } -} - // --------------------------------------------------------------------------------------------- // VLOOKUP / HLOOKUP / LOOKUP / MATCH @@ -146,7 +141,9 @@ fn vh_lookup(args: &[Arg], vertical: bool) -> R { let pos = if approx { bsearch_last(&x, &keys, false) } else { find_exact(&x, &keys, true, false) }; let pos = pos.ok_or(CellError::NA)?; let v = if vertical { table.get(pos, idx) } else { table.get(idx, pos) }; - Ok(cell_result(v.cloned().unwrap_or(Value::Empty))) + // An empty target stays blank, as in Excel: ISBLANK sees it as blank and `&` / + // CONCATENATE treat it as ""; only a formula's final result displays 0 (#374). + Ok(v.cloned().unwrap_or(Value::Empty)) } fn vlookup(a: &[Arg], _c: &mut dyn Ctx) -> R { @@ -179,18 +176,18 @@ fn lookup(args: &[Arg], _c: &mut dyn Ctx) -> R { } else { None }; - return v.map(cell_result).ok_or(CellError::NA); + return v.ok_or(CellError::NA); } // Array form: search the first row of a wide array, else the first column; return the // matching entry from the last row / column. if la.cols > la.rows { let keys = row(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty))) + Ok(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty)) } else { let keys = column(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty))) + Ok(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty)) } } @@ -311,13 +308,13 @@ fn xlookup(args: &[Arg], _c: &mut dyn Ctx) -> R { if vertical { let r = row(&ra, p); if r.len() == 1 { - return Ok(cell_result(r.into_iter().next().unwrap_or(Value::Empty))); + return Ok(r.into_iter().next().unwrap_or(Value::Empty)); } array_val(1, r.len(), r) } else { let c = column(&ra, p); if c.len() == 1 { - return Ok(cell_result(c.into_iter().next().unwrap_or(Value::Empty))); + return Ok(c.into_iter().next().unwrap_or(Value::Empty)); } array_val(c.len(), 1, c) } @@ -581,7 +578,8 @@ mod tests { is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), b(false)]), CellError::NA); is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), empty()]), CellError::NA); close(ev("VLOOKUP", vec![n(3.0), rf(table()), n(3.0), b(false)]), 30.0); - close(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), 0.0); + // An empty target stays blank (the cell displays 0 only as a final result) (#374). + assert_eq!(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), Value::Empty); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(4.0)]), CellError::Ref); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(0.0)]), CellError::Value); close(ev("VLOOKUP", vec![t("b*"), rf(names()), n(2.0), b(false)]), 2.0); From d2b9affbc519c54d438e3b6b74f930a0c3de6c8f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 20:59:09 +0200 Subject: [PATCH 02/51] Functions: test that arithmetic reads a blank lookup result as 0 (#374) Co-Authored-By: Claude Opus 5.5 --- crates/calc/src/tests.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index ac58b3d31..fe55dc3e1 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -986,6 +986,8 @@ fn lookup_of_an_empty_cell_stays_blank() { ("D7", "=ISBLANK(HLOOKUP(\"j\",A2:B3,2,0))", Value::Bool(true)), // A bare lookup still shows 0, like a reference to an empty cell. ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), + // Arithmetic still reads the blank as 0. + ("D9", "=VLOOKUP(\"k\",A1:B2,2,0)+1", Value::Number(1.0)), ] { t.set(at, f); assert_eq!(t.get(at), want, "{f}"); From 423810750d8d43f732163d57267905a0110afe83 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 12 Oct 2026 19:18:01 +0200 Subject: [PATCH 03/51] Functions: don't assert ISBLANK(VLOOKUP/HLOOKUP/LOOKUP), unverified against Excel (#374) Co-Authored-By: Claude Sonnet 5.5 --- crates/calc/src/tests.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index fe55dc3e1..cfce701d1 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -977,13 +977,10 @@ fn lookup_of_an_empty_cell_stays_blank() { t.set("A2", "j"); t.set("B2", "v"); for (at, f, want) in [ - ("D1", "=ISBLANK(VLOOKUP(\"k\",A1:B2,2,0))", Value::Bool(true)), ("D2", "=\"x\"&VLOOKUP(\"k\",A1:B2,2,0)&\"y\"", Value::from("xy")), ("D3", "=CONCATENATE(\"x\",VLOOKUP(\"k\",A1:B2,2,0),\"y\")", Value::from("xy")), ("D4", "=ISBLANK(XLOOKUP(\"k\",A1:A2,B1:B2))", Value::Bool(true)), ("D5", "=\"x\"&XLOOKUP(\"k\",A1:A2,B1:B2)&\"y\"", Value::from("xy")), - ("D6", "=ISBLANK(LOOKUP(\"k\",A1:A1,B1:B1))", Value::Bool(true)), - ("D7", "=ISBLANK(HLOOKUP(\"j\",A2:B3,2,0))", Value::Bool(true)), // A bare lookup still shows 0, like a reference to an empty cell. ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), // Arithmetic still reads the blank as 0. From 9977d850a4db34ebe2028ebe0320e027e43eb3af Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 17 Oct 2026 15:23:42 +0200 Subject: [PATCH 04/51] test(calc): Excel-verified blank results for VLOOKUP, HLOOKUP and LOOKUP (#374) Excel 365 for Windows (2609, build 16.0.20430.20032) keeps a lookup of an empty cell blank inside ISBLANK and concatenation for VLOOKUP, HLOOKUP and LOOKUP. The cases were checked in Excel by scottcampbelldata in vangel76/gridcraft#1. Co-Authored-By: scottcampbelldata <251750033+scottcampbelldata@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- crates/calc/src/tests.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 4827b1bcc..357249f56 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -1193,6 +1193,24 @@ fn lookup_of_an_empty_cell_stays_blank() { } } +#[test] +fn blank_lookup_values_match_excel_windows_observation() { + // Excel 365 Windows 2609, build 16.0.20430.20032, black-box check 2026-10-13: + // A1="k", B1 and A2 genuinely empty. + // All three value lookups preserve blankness inside ISBLANK/concatenation. + let mut t = T::new(); + t.set("A1", "k"); + for (at, f, want) in [ + ("D1", "=ISBLANK(VLOOKUP(\"k\",A1:B1,2,FALSE))", Value::Bool(true)), + ("D2", "=ISBLANK(HLOOKUP(\"k\",A1:A2,2,FALSE))", Value::Bool(true)), + ("D3", "=ISBLANK(LOOKUP(\"k\",A1:A1,B1:B1))", Value::Bool(true)), + ("D4", "=\"x\"&HLOOKUP(\"k\",A1:A2,2,FALSE)&\"y\"", Value::from("xy")), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} + #[test] fn cell_width_is_the_column_width_dialog_value_rounded() { // #735: Excel 365 values. Widths in px as `home.columnWidth {"chars"}` sets them From 4ab3edaf3b5a3ee540a76301039a80a79e063bb0 Mon Sep 17 00:00:00 2001 From: Higgs32584 Date: Wed, 21 Oct 2026 20:06:33 -0400 Subject: [PATCH 05/51] xlsx: escape shape font typeface attribute (#2170) Co-Authored-By: Claude Opus 5.5 --- crates/xlsx/src/drawing.rs | 2 +- crates/xlsx/src/tests/shape_text_rtl.rs | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/crates/xlsx/src/drawing.rs b/crates/xlsx/src/drawing.rs index a6892f0b8..6fd2c22f3 100644 --- a/crates/xlsx/src/drawing.rs +++ b/crates/xlsx/src/drawing.rs @@ -471,7 +471,7 @@ fn run_props(f: &ShapeFormat, theme: &Theme) -> (String, String) { let _ = write!(body, "{c}"); } if let Some(font) = &f.font { - let _ = write!(body, "", esc(font)); + let _ = write!(body, "", esc_attr(font)); } (attrs, body) } diff --git a/crates/xlsx/src/tests/shape_text_rtl.rs b/crates/xlsx/src/tests/shape_text_rtl.rs index 36bdb861d..2ece04d31 100644 --- a/crates/xlsx/src/tests/shape_text_rtl.rs +++ b/crates/xlsx/src/tests/shape_text_rtl.rs @@ -84,3 +84,19 @@ fn shape_text_without_rtl_writes_unchanged_ppr() { let xml = written_drawing(&text_box("plain", ShapeFormat::default())); assert!(!xml.contains(" Date: Thu, 22 Oct 2026 22:19:05 -0500 Subject: [PATCH 06/51] MCP: the stdio server keeps at most 16 MB of a message (#2164) `serve` read a whole line before checking it against `MAX_LINE`, so a client sending a newline-free payload grew the buffer without limit. The line is now read through `take`, as the control server does; the rest of an over-long line is read and dropped, the client gets the "message too large" error, and the next message is served (stdio has one client, so the stream stays open rather than closing as the control server's connections do). Co-Authored-By: Claude Opus 5.5 --- crates/mcp/src/server.rs | 56 ++++++++++++++++++++++++++++------ crates/mcp/src/tests.rs | 65 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 9 deletions(-) diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 75b82d8d8..0d262dbe7 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -1,6 +1,6 @@ //! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts methods. -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; use serde_json::{Value, json}; @@ -45,6 +45,45 @@ fn response(id: Value, result: Value) -> Value { json!({"jsonrpc": "2.0", "id": id, "result": result}) } +/// Reads the next newline-delimited message into `buf`, keeping at most `MAX_LINE` bytes of it: +/// a client sending no newline can't make the server allocate more (#2164). `None` at the end of +/// `input`; `Some(false)` for a line over the limit, whose rest is read and dropped so the next +/// message is read whole. +pub(crate) fn read_message(input: &mut impl BufRead, buf: &mut Vec) -> std::io::Result> { + buf.clear(); + // One byte past the limit tells an over-long line from one of exactly `MAX_LINE` bytes. + let limit = u64::try_from(MAX_LINE).unwrap_or(u64::MAX).saturating_add(1); + if input.take(limit).read_until(b'\n', buf)? == 0 { + return Ok(None); + } + if buf.len() > MAX_LINE && buf.last() != Some(&b'\n') { + skip_line(input)?; + buf.clear(); + return Ok(Some(false)); + } + Ok(Some(true)) +} + +/// Reads up to and including the next newline (or to the end of `input`) without keeping it. +fn skip_line(input: &mut impl BufRead) -> std::io::Result<()> { + loop { + let chunk = input.fill_buf()?; + if chunk.is_empty() { + return Ok(()); + } + match chunk.iter().position(|&b| b == b'\n') { + Some(i) => { + input.consume(i + 1); + return Ok(()); + } + None => { + let n = chunk.len(); + input.consume(n); + } + } + } +} + fn error(id: Value, code: i64, message: impl Into) -> Value { json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}}) } @@ -68,14 +107,13 @@ impl Server { pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> { let mut buf = Vec::new(); loop { - buf.clear(); - let n = input.read_until(b'\n', &mut buf)?; - if n == 0 { - return Ok(()); - } - let reply = match std::str::from_utf8(&buf) { - Ok(line) => self.handle_line(line), - Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + let reply = match read_message(&mut input, &mut buf)? { + None => return Ok(()), + Some(false) => Some(error(Value::Null, INVALID_REQUEST, "message too large (16 MB max)").to_string()), + Some(true) => match std::str::from_utf8(&buf) { + Ok(line) => self.handle_line(line), + Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + }, }; if let Some(reply) = reply { output.write_all(reply.as_bytes())?; diff --git a/crates/mcp/src/tests.rs b/crates/mcp/src/tests.rs index 71631b5e8..5a5297246 100644 --- a/crates/mcp/src/tests.rs +++ b/crates/mcp/src/tests.rs @@ -359,6 +359,71 @@ fn serve_loop_over_buffers() { assert!(lines[2]["result"]["content"][0]["text"].as_str().unwrap().contains("42")); } +/// Streams `len` bytes of `a` from one reused buffer, then `tail`: a huge line costs no memory to +/// produce, so only the server can make it expensive. +struct HugeLine { + left: u64, + tail: &'static [u8], + chunk: Vec, +} + +impl std::io::Read for HugeLine { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let buf = std::io::BufRead::fill_buf(self)?; + let n = buf.len().min(out.len()); + out[..n].copy_from_slice(&buf[..n]); + std::io::BufRead::consume(self, n); + Ok(n) + } +} + +impl std::io::BufRead for HugeLine { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.left > 0 { + let n = self.chunk.len().min(usize::try_from(self.left).unwrap_or(usize::MAX)); + return Ok(&self.chunk[..n]); + } + Ok(self.tail) + } + fn consume(&mut self, n: usize) { + if self.left > 0 { + self.left -= n as u64; + } else { + self.tail = &self.tail[n..]; + } + } +} + +#[test] +fn a_line_without_newline_keeps_at_most_the_limit_in_memory() { + // #2164: the whole line was buffered before its size was checked (4 GiB here; any size a + // client cares to send). + let mut input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut buf = Vec::new(); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(false)); + assert!(buf.capacity() <= 2 * (16 << 20) + 1, "kept {} bytes", buf.capacity()); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(true)); + assert!(buf.starts_with(br#"{"jsonrpc""#)); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), None); +} + +#[test] +fn serve_loop_bounds_a_line_without_newline() { + // #2164: a 4 GiB line was read whole before its size was checked. Now the server keeps at + // most 16 MB of it, answers that it is too large, and reads the next message normally. + let input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut out = Vec::new(); + server().serve(input, &mut out).unwrap(); + let lines: Vec = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0]["error"]["code"], -32600); + assert!(lines[0]["error"]["message"].as_str().unwrap().contains("too large")); + assert_eq!(lines[1]["id"], 7); + assert!(lines[1].get("result").is_some(), "{}", lines[1]); +} + #[test] fn cell_set_without_input_is_error() { let mut s = server(); From 8c1c7cfd27a22b366ce2ddb94120165b5978147e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 00:47:47 +0100 Subject: [PATCH 07/51] Manage Rules: scroll the rule list and lay out only the visible rules (#3321) The dialog laid out one row per rule with no height limit and ran home.manageRules (copying and serialising every rule, and journaling a step) on every frame. A sheet with thousands of rules made the window as tall as the whole list and the application window go blank. Co-Authored-By: Claude Sonnet 5.5 --- .../ui-egui/src/dialogs/cond_format_rules.rs | 54 ++++++++++------ crates/ui-egui/tests/it/cf_rules_manager.rs | 64 +++++++++++++++++++ crates/ui-egui/tests/it/main.rs | 1 + 3 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 crates/ui-egui/tests/it/cf_rules_manager.rs diff --git a/crates/ui-egui/src/dialogs/cond_format_rules.rs b/crates/ui-egui/src/dialogs/cond_format_rules.rs index 4d8a22375..23af88aae 100644 --- a/crates/ui-egui/src/dialogs/cond_format_rules.rs +++ b/crates/ui-egui/src/dialogs/cond_format_rules.rs @@ -1,28 +1,40 @@ use super::{Json, SheetApp, json}; +/// The list scrolls inside the dialog: a sheet read from a file can carry thousands of rules, and +/// a list laid out in full made the window as tall as all of them (issue #3321). +const LIST_HEIGHT: f32 = 360.0; + pub(super) fn manage_rules(app: &mut SheetApp, ui: &mut egui::Ui) { - let list = app.session.run("home.manageRules", json!({})).unwrap_or(Json::Null); - let rules = list["rules"].as_array().cloned().unwrap_or_default(); - if rules.is_empty() { + // Read the rules in place: running `home.manageRules` for them each frame copied and + // serialised every rule and journaled a step 60 times a second. + let count = app.session.active().and_then(|d| d.wb.active()).map_or(0, |sh| sh.cond_formats.len()); + if count == 0 { ui.label("There are no conditional formatting rules on this sheet."); + return; } - for (i, r) in rules.iter().enumerate() { - ui.horizontal(|ui| { - let kind = r["rule"].as_object().and_then(|o| o.keys().next().cloned()).unwrap_or_default(); - ui.label(format!( - "{}. {kind} applies to {}", - i + 1, - r["ranges"].as_array().map(|a| a.iter().filter_map(|x| x.as_str()).collect::>().join(",")).unwrap_or_default() - )); - if ui.small_button("▲").clicked() { - app.run_or_alert("home.manageRules", json!({"moveUp": i})); - } - if ui.small_button("▼").clicked() { - app.run_or_alert("home.manageRules", json!({"moveDown": i})); - } - if ui.small_button("Delete").clicked() { - app.run_or_alert("home.manageRules", json!({"delete": i})); - } - }); + let mut action: Option = None; + let row_height = ui.spacing().interact_size.y; + egui::ScrollArea::vertical().max_height(LIST_HEIGHT).auto_shrink([false, true]).show_rows(ui, row_height, count, |ui, rows| { + let Some(sh) = app.session.active().and_then(|d| d.wb.active()) else { return }; + for i in rows { + let Some(cf) = sh.cond_formats.get(i) else { break }; + let kind = serde_json::to_value(&cf.rule).ok().and_then(|r| r.as_object().and_then(|o| o.keys().next().cloned())).unwrap_or_default(); + let ranges = cf.ranges.iter().map(|r| r.a1()).collect::>().join(","); + ui.horizontal(|ui| { + ui.label(format!("{}. {kind} applies to {ranges}", i + 1)); + if ui.small_button("▲").clicked() { + action = Some(json!({"moveUp": i})); + } + if ui.small_button("▼").clicked() { + action = Some(json!({"moveDown": i})); + } + if ui.small_button("Delete").clicked() { + action = Some(json!({"delete": i})); + } + }); + } + }); + if let Some(params) = action { + app.run_or_alert("home.manageRules", params); } } diff --git a/crates/ui-egui/tests/it/cf_rules_manager.rs b/crates/ui-egui/tests/it/cf_rules_manager.rs new file mode 100644 index 000000000..540b19403 --- /dev/null +++ b/crates/ui-egui/tests/it/cf_rules_manager.rs @@ -0,0 +1,64 @@ +//! Manage Rules on a sheet with thousands of conditional formats (issue #3321): the list scrolls +//! inside the dialog, and only the visible rules are laid out, instead of one row per rule +//! making the window as tall as the whole list and the application window go blank. + +use egui::vec2; +use egui_kittest::kittest::Queryable; +use gridcraft_engine::Session; +use gridcraft_ui_egui::SheetApp; +use serde_json::json; + +type Harness = egui_kittest::Harness<'static, SheetApp>; + +const RULES: usize = 3000; +const SCREEN_H: f32 = 800.0; + +fn harness_with_rules(n: usize) -> Harness { + let mut session = Session::new(); + session.new_workbook(); + for i in 0..n { + let range = format!("A{}", i + 1); + session + .execute( + "home.conditionalFormat", + json!({"range": range, "rule": {"type": "cellIs", "operator": "greaterThan", "value": "5", "preset": "yellowFill"}}), + ) + .unwrap(); + } + let mut h = crate::common::builder().with_size(vec2(1200.0, SCREEN_H)).build_ui_state( + |ui, app: &mut SheetApp| { + app.logic(ui.ctx()); + app.ui(ui); + }, + crate::common::app(session), + ); + h.run_steps(4); + h.state_mut().open_dialog("manageRules", json!({})); + h.run_steps(6); + h +} + +#[test] +fn many_rules_scroll_inside_a_dialog_that_fits_the_window() { + let mut h = harness_with_rules(RULES); + let deletes: Vec<_> = h.query_all_by_label("Delete").collect(); + assert!(!deletes.is_empty(), "the first rules are listed"); + // One more Delete sits in the ribbon. + assert!(deletes.len() < 100, "only the visible rules are laid out, got {}", deletes.len()); + let lowest = deletes.iter().map(|d| d.rect().bottom()).fold(0.0, f32::max); + assert!(lowest <= SCREEN_H, "the list stays inside the window, lowest button at {lowest}"); + drop(deletes); + // Deleting through the dialog removes the first rule and the dialog stays usable. + // (The ribbon has a Delete button of its own, outside the dialog's window.) + let window = h.get_by_label("Conditional Formatting Rules Manager").rect(); + h.query_all_by_label("Delete").find(|d| window.contains_rect(d.rect())).unwrap().click(); + h.run_steps(4); + let left = h.state().session.active().unwrap().wb.active().unwrap().cond_formats.len(); + assert_eq!(left, RULES - 1); +} + +#[test] +fn an_empty_sheet_says_so() { + let h = harness_with_rules(0); + h.get_by_label("There are no conditional formatting rules on this sheet."); +} diff --git a/crates/ui-egui/tests/it/main.rs b/crates/ui-egui/tests/it/main.rs index efc4fdc23..9cb395e5d 100644 --- a/crates/ui-egui/tests/it/main.rs +++ b/crates/ui-egui/tests/it/main.rs @@ -15,6 +15,7 @@ mod borders; mod cell_editor_caret; mod cell_pictures; mod cf_rule_pick; +mod cf_rules_manager; mod chart_data_table; mod chart_resize; mod chart_source_layering; From 51e242e1fa1ad79764115cb7f3427d1e539930ec Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 5 Nov 2026 00:52:15 +0100 Subject: [PATCH 08/51] Manage Rules test: check the dialog rect and which rule was deleted Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/cf_rules_manager.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/cf_rules_manager.rs b/crates/ui-egui/tests/it/cf_rules_manager.rs index 540b19403..d6f19a715 100644 --- a/crates/ui-egui/tests/it/cf_rules_manager.rs +++ b/crates/ui-egui/tests/it/cf_rules_manager.rs @@ -48,13 +48,17 @@ fn many_rules_scroll_inside_a_dialog_that_fits_the_window() { let lowest = deletes.iter().map(|d| d.rect().bottom()).fold(0.0, f32::max); assert!(lowest <= SCREEN_H, "the list stays inside the window, lowest button at {lowest}"); drop(deletes); + let window = h.get_by_label("Conditional Formatting Rules Manager").rect(); + assert!(window.bottom() <= SCREEN_H, "the dialog fits the window, bottom at {}", window.bottom()); // Deleting through the dialog removes the first rule and the dialog stays usable. // (The ribbon has a Delete button of its own, outside the dialog's window.) - let window = h.get_by_label("Conditional Formatting Rules Manager").rect(); h.query_all_by_label("Delete").find(|d| window.contains_rect(d.rect())).unwrap().click(); h.run_steps(4); let left = h.state().session.active().unwrap().wb.active().unwrap().cond_formats.len(); assert_eq!(left, RULES - 1); + let first = h.state().session.active().unwrap().wb.active().unwrap().cond_formats[0].ranges[0].a1(); + // New rules go first, so the one deleted was A3000. + assert_eq!(first, format!("A{}", RULES - 1), "the first rule was the one removed"); } #[test] From d7c0158f0a6de45660750d1387969b36369025ac Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 17:08:44 +0100 Subject: [PATCH 09/51] Manage Rules: let the rule list scroll sideways A long range list pushed the move and Delete buttons out of the vertical-only scroll area, where they could not be clicked. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/dialogs/cond_format_rules.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/src/dialogs/cond_format_rules.rs b/crates/ui-egui/src/dialogs/cond_format_rules.rs index 23af88aae..79df9897f 100644 --- a/crates/ui-egui/src/dialogs/cond_format_rules.rs +++ b/crates/ui-egui/src/dialogs/cond_format_rules.rs @@ -14,7 +14,7 @@ pub(super) fn manage_rules(app: &mut SheetApp, ui: &mut egui::Ui) { } let mut action: Option = None; let row_height = ui.spacing().interact_size.y; - egui::ScrollArea::vertical().max_height(LIST_HEIGHT).auto_shrink([false, true]).show_rows(ui, row_height, count, |ui, rows| { + egui::ScrollArea::both().max_height(LIST_HEIGHT).auto_shrink([false, true]).show_rows(ui, row_height, count, |ui, rows| { let Some(sh) = app.session.active().and_then(|d| d.wb.active()) else { return }; for i in rows { let Some(cf) = sh.cond_formats.get(i) else { break }; From 684f051f0da03c797ee39f89bda3708bf81f026b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 12:51:27 +0100 Subject: [PATCH 10/51] Manage Rules: test reaching rules by scrolling down and sideways Scroll the 3,000-rule list to a row that was off screen and move and delete that rule through its own buttons; give one rule a label of 200 ranges whose buttons start outside the dialog, scroll sideways, then move and delete it. Each test checks the intended rule changed. Co-Authored-By: Claude Opus 5.5 --- crates/ui-egui/tests/it/cf_rules_manager.rs | 117 +++++++++++++++++++- 1 file changed, 114 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/tests/it/cf_rules_manager.rs b/crates/ui-egui/tests/it/cf_rules_manager.rs index d6f19a715..013a0b28f 100644 --- a/crates/ui-egui/tests/it/cf_rules_manager.rs +++ b/crates/ui-egui/tests/it/cf_rules_manager.rs @@ -2,7 +2,7 @@ //! inside the dialog, and only the visible rules are laid out, instead of one row per rule //! making the window as tall as the whole list and the application window go blank. -use egui::vec2; +use egui::{Event, Modifiers, MouseWheelSource, MouseWheelUnit, PointerButton, Pos2, Rect, TouchPhase, Vec2, pos2, vec2}; use egui_kittest::kittest::Queryable; use gridcraft_engine::Session; use gridcraft_ui_egui::SheetApp; @@ -14,10 +14,15 @@ const RULES: usize = 3000; const SCREEN_H: f32 = 800.0; fn harness_with_rules(n: usize) -> Harness { + let ranges: Vec = (0..n).map(|i| format!("A{}", i + 1)).collect(); + harness_with_ranges(&ranges) +} + +/// One rule per entry of `ranges`, added in order (so the last one is listed first). +fn harness_with_ranges(ranges: &[String]) -> Harness { let mut session = Session::new(); session.new_workbook(); - for i in 0..n { - let range = format!("A{}", i + 1); + for range in ranges { session .execute( "home.conditionalFormat", @@ -38,6 +43,67 @@ fn harness_with_rules(n: usize) -> Harness { h } +fn dialog(h: &Harness) -> Rect { + h.get_by_label("Conditional Formatting Rules Manager").rect() +} + +/// The rows of the list shown inside the dialog: (rule number, row rectangle). +fn visible_rows(h: &Harness) -> Vec<(usize, Rect)> { + let window = dialog(h); + let mut rows: Vec<(usize, Rect)> = h + .query_all_by_label_contains("applies to") + .filter(|n| window.contains(n.rect().left_center())) + .filter_map(|n| Some((n.value()?.split('.').next()?.parse().ok()?, n.rect()))) + .collect(); + rows.sort_by(|a, b| a.1.top().total_cmp(&b.1.top())); + rows +} + +/// The row of rule `n`, wherever it is laid out (its label may reach past the dialog's edges). +fn row_of(h: &Harness, n: usize) -> Rect { + let prefix = format!("{n}. "); + h.query_all_by_label_contains("applies to") + .find(|r| r.value().is_some_and(|l| l.starts_with(&prefix))) + .expect("the rule's row is laid out") + .rect() +} + +/// The `label` button on the row at `row`, if it is inside the dialog. +fn row_button(h: &Harness, row: Rect, label: &str) -> Option { + let window = dialog(h); + h.query_all_by_label(label).map(|n| n.rect()).find(|r| window.contains_rect(*r) && (r.center().y - row.center().y).abs() < row.height() / 2.0) +} + +fn wheel(h: &mut Harness, at: Pos2, delta: Vec2) { + h.input_mut().events.push(Event::PointerMoved(at)); + h.run_steps(2); + h.input_mut().events.push(Event::MouseWheel { + unit: MouseWheelUnit::Point, + delta, + phase: TouchPhase::Move, + source: MouseWheelSource::Unknown, + modifiers: Modifiers::NONE, + }); + h.run_steps(30); +} + +fn click(h: &mut Harness, r: Rect) { + let pos = r.center(); + for pressed in [true, false] { + h.input_mut() + .events + .extend([Event::PointerMoved(pos), Event::PointerButton { pos, button: PointerButton::Primary, pressed, modifiers: Modifiers::NONE }]); + h.step(); + } + h.run_steps(4); +} + +/// The ranges of every rule on the sheet, in list order. +fn rule_ranges(h: &Harness) -> Vec { + let sh = h.state().session.active().unwrap().wb.active().unwrap(); + sh.cond_formats.iter().map(|cf| cf.ranges.iter().map(|r| r.a1()).collect::>().join(",")).collect() +} + #[test] fn many_rules_scroll_inside_a_dialog_that_fits_the_window() { let mut h = harness_with_rules(RULES); @@ -66,3 +132,48 @@ fn an_empty_sheet_says_so() { let h = harness_with_rules(0); h.get_by_label("There are no conditional formatting rules on this sheet."); } + +#[test] +fn a_rule_scrolled_into_view_is_moved_and_deleted() { + let mut h = harness_with_rules(RULES); + let before = visible_rows(&h); + let last_shown = before.iter().map(|r| r.0).max().expect("the first rules are listed"); + // Scroll the list down, then act on a row in the middle of what is shown now. + let at = pos2(dialog(&h).left() + 40.0, before[before.len() / 2].1.center().y); + wheel(&mut h, at, vec2(0.0, -2000.0)); + let rows = visible_rows(&h); + let (n, row) = rows[rows.len() / 2]; + assert!(n > last_shown, "rule {n} was not on screen before scrolling (rules 1 to {last_shown} were)"); + // Rule n is the n-th listed; new rules go first, so it applies to A{RULES + 1 - n}. + let target = format!("A{}", RULES + 1 - n); + assert_eq!(rule_ranges(&h)[n - 1], target); + let down = row_button(&h, row, "▼").expect("the row's move-down button is in the dialog"); + click(&mut h, down); + let ranges = rule_ranges(&h); + assert_eq!(ranges[n], target, "rule {n} moved down one place"); + assert_eq!(ranges[n - 1], format!("A{}", RULES - n), "the rule below it moved up"); + // The moved rule is now rule n + 1, on the next row. + let delete = row_button(&h, row_of(&h, n + 1), "Delete").expect("the row's Delete button is in the dialog"); + click(&mut h, delete); + let ranges = rule_ranges(&h); + assert_eq!(ranges.len(), RULES - 1); + assert!(!ranges.contains(&target), "the rule scrolled to was deleted"); +} + +#[test] +fn a_rule_with_a_long_label_is_reached_by_scrolling_sideways() { + let long: Vec = (0..200).map(|i| format!("C{}", 2 * i + 1)).collect(); + let long = long.join(","); + let mut h = harness_with_ranges(&["A1".into(), long.clone(), "B1".into()]); + assert_eq!(rule_ranges(&h), ["B1", long.as_str(), "A1"]); + let row = row_of(&h, 2); + assert!(row_button(&h, row, "▼").is_none(), "the long rule's buttons start out of view"); + let window = dialog(&h); + wheel(&mut h, pos2(window.left() + 40.0, row.center().y), vec2(-20000.0, 0.0)); + let down = row_button(&h, row_of(&h, 2), "▼").expect("scrolling sideways brings the move-down button into the dialog"); + click(&mut h, down); + assert_eq!(rule_ranges(&h), ["B1", "A1", long.as_str()], "the long rule moved down"); + let delete = row_button(&h, row_of(&h, 3), "Delete").expect("the long rule's Delete button is in the dialog on its new row"); + click(&mut h, delete); + assert_eq!(rule_ranges(&h), ["B1", "A1"], "the long rule was deleted"); +} From 63150810b35539a05492c1269bb438ff4a2f7766 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 13:21:44 +0100 Subject: [PATCH 11/51] fix(functions): a blank lookup given as an argument is not an omitted one FILTER(1,FALSE,VLOOKUP(...)) with a blank lookup result returned #CALC! because util::has read the blank value as a left-out argument. Arguments written in the formula that compute a blank are now marked as given. Co-Authored-By: Claude Opus 5.5 --- crates/calc/src/eval/call.rs | 10 ++++++---- crates/calc/src/eval/reference_fns.rs | 7 ++++--- crates/calc/src/tests.rs | 3 +++ crates/functions/src/info.rs | 2 +- crates/functions/src/lib.rs | 7 +++++-- crates/functions/src/lift.rs | 7 +------ crates/functions/src/util.rs | 2 +- 7 files changed, 21 insertions(+), 17 deletions(-) diff --git a/crates/calc/src/eval/call.rs b/crates/calc/src/eval/call.rs index 8ac58c8d1..848762ab9 100644 --- a/crates/calc/src/eval/call.rs +++ b/crates/calc/src/eval/call.rs @@ -203,9 +203,9 @@ impl<'h> Evaluator<'h> { let index = self.value(&args[0]); if let Value::Array(_) = index { let Some(spec) = gridcraft_functions::lookup("CHOOSE") else { return err(CellError::Name) }; - let mut evaluated = vec![Arg { value: index, from_ref: false }]; + let mut evaluated = vec![Arg::val(index)]; for argument in &args[1..] { - evaluated.push(Arg { value: self.value(argument), from_ref: false }); + evaluated.push(Arg::val(self.value(argument))); } let mut context = FnCtx { ev: self, criteria: None, lambda: None }; return gridcraft_functions::call(spec, &evaluated, &mut context).pipe(Ev::V); @@ -351,7 +351,9 @@ impl<'h> Evaluator<'h> { } ev => self.deref(ev), }; - evaluated.push(Arg { value, from_ref }); + // Only an argument left out (`f(a,,b)`) is omitted; one that computes a blank is given. + let blank_value = !from_ref && value == Value::Empty && lambda_at != Some(i) && !matches!(a, Expr::Missing); + evaluated.push(Arg { value, from_ref, blank_value }); } let mut ctx = FnCtx { ev: self, criteria, lambda }; gridcraft_functions::call(spec, &evaluated, &mut ctx).pipe(Ev::V) @@ -414,7 +416,7 @@ impl<'h> Evaluator<'h> { let (Some(sum), Some(count)) = (gridcraft_functions::lookup("SUM"), gridcraft_functions::lookup("COUNT")) else { return Err(err(CellError::Name)); }; - let args = [Arg { value, from_ref }]; + let args = [Arg { value, from_ref, blank_value: false }]; if kind != Agg::Count { match gridcraft_functions::call(sum, &args, &mut FnCtx { ev: self, criteria: None, lambda: None }) { Value::Number(n) => acc.sum += n, diff --git a/crates/calc/src/eval/reference_fns.rs b/crates/calc/src/eval/reference_fns.rs index 4f5154c2d..6bef99732 100644 --- a/crates/calc/src/eval/reference_fns.rs +++ b/crates/calc/src/eval/reference_fns.rs @@ -203,7 +203,7 @@ impl<'h> Evaluator<'h> { let horizontal = l.range.height() == 1 && l.range.width() > 1; let same_len = if vertical { r.range.height() == l.range.height() } else { r.range.width() == l.range.width() }; if !matches!(key, Value::Array(_)) && (vertical || horizontal) && same_len { - let mut xargs = vec![Arg { value: key, from_ref: false }, Arg { value: self.area_value(&l), from_ref: true }]; + let mut xargs = vec![Arg::val(key), Arg::reference(self.area_value(&l))]; // An empty match_mode before a given search_mode is the default (0), not the end. let given = |i: usize| args.get(i).is_some_and(|e| !matches!(e, Expr::Missing)); let last = if given(5) { @@ -215,10 +215,11 @@ impl<'h> Evaluator<'h> { }; for i in 4..=last { match args.get(i) { - Some(Expr::Missing) | None => xargs.push(Arg { value: Value::Number(0.0), from_ref: false }), + Some(Expr::Missing) | None => xargs.push(Arg::val(Value::Number(0.0))), Some(e) => { let value = self.value(e); - xargs.push(Arg { value, from_ref: false }); + let blank_value = value == Value::Empty; + xargs.push(Arg { value, from_ref: false, blank_value }); } } } diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index fffbc59a1..3c1330dbe 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -1842,6 +1842,9 @@ fn lookup_of_an_empty_cell_stays_blank() { ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), // Arithmetic still reads the blank as 0. ("D9", "=VLOOKUP(\"k\",A1:B2,2,0)+1", Value::Number(1.0)), + // A blank lookup given as an argument is still given, not omitted. + ("D10", "=FILTER(1,FALSE,VLOOKUP(\"k\",A1:B2,2,FALSE))", Value::Number(0.0)), + ("D11", "=XLOOKUP(\"z\",A1:A2,B1:B2,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), ] { t.set(at, f); assert_eq!(t.get(at), want, "{f}"); diff --git a/crates/functions/src/info.rs b/crates/functions/src/info.rs index 309f0323e..82eda5828 100644 --- a/crates/functions/src/info.rs +++ b/crates/functions/src/info.rs @@ -91,7 +91,7 @@ fn na(_a: &[Arg], _c: &mut dyn Ctx) -> R { fn n_fn(a: &[Arg], _c: &mut dyn Ctx) -> R { // N(range) reads the range's first cell, but a computed array (N(A1:A3>0), N({...})) is // converted element by element, so N can stand in for `--` inside SUMPRODUCT. - if let Some(Arg { value: Value::Array(arr), from_ref: false }) = a.first() { + if let Some(Arg { value: Value::Array(arr), from_ref: false, .. }) = a.first() { let data = arr.data.iter().map(|v| n_of(v.clone()).unwrap_or_else(Value::Error)).collect(); return Array::new(arr.rows, arr.cols, data).map(Value::from).ok_or(CellError::Calc); } diff --git a/crates/functions/src/lib.rs b/crates/functions/src/lib.rs index ccc61b026..6cb035b28 100644 --- a/crates/functions/src/lib.rs +++ b/crates/functions/src/lib.rs @@ -87,16 +87,19 @@ pub use stat::{aggregate_values, aggregate_values_k}; pub struct Arg { pub value: Value, pub from_ref: bool, + /// A written argument whose value came out blank (a lookup of an empty cell): given, not + /// omitted, although its value is `Value::Empty`. + pub blank_value: bool, } impl Arg { /// A literal (non-reference) argument. pub fn val(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: false } + Arg { value: value.into(), from_ref: false, blank_value: false } } /// An argument that came from a cell or range reference. pub fn reference(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: true } + Arg { value: value.into(), from_ref: true, blank_value: false } } } diff --git a/crates/functions/src/lift.rs b/crates/functions/src/lift.rs index ea7f07fd9..6d2e3fbc4 100644 --- a/crates/functions/src/lift.rs +++ b/crates/functions/src/lift.rs @@ -50,12 +50,7 @@ fn reject_multi_cell_refs(spec: &FnSpec, args: &[Arg]) -> Option> { if !args.iter().enumerate().any(|(i, a)| multi(i, a)) { return None; } - Some( - args.iter() - .enumerate() - .map(|(i, a)| if multi(i, a) { Arg { value: Value::Error(CellError::Value), from_ref: false } } else { a.clone() }) - .collect(), - ) + Some(args.iter().enumerate().map(|(i, a)| if multi(i, a) { Arg::val(Value::Error(CellError::Value)) } else { a.clone() }).collect()) } /// Lookups whose every call searches a lookup vector from its start, so a call lifted over `n` diff --git a/crates/functions/src/util.rs b/crates/functions/src/util.rs index 14c627937..4e7853d2c 100644 --- a/crates/functions/src/util.rs +++ b/crates/functions/src/util.rs @@ -51,7 +51,7 @@ pub(crate) fn arg(args: &[Arg], i: usize) -> R<&Arg> { /// Whether argument `i` was supplied (present and not an empty literal like `F(1,)`). pub(crate) fn has(args: &[Arg], i: usize) -> bool { - matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref) + matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref || a.blank_value) } /// Number coercion for a scalar parameter: empty = 0, TRUE = 1, numeric text parses. From 642fa97fa5360ff54a4cff82d5d415dbadcc2089 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 8 Nov 2026 14:19:05 +0100 Subject: [PATCH 12/51] fix(functions): a blank element of a lifted argument is given, not omitted CEILING.MATH(6.7,VLOOKUP({"k","k"},A1:B1,2,FALSE)) with B1 empty now gives 0 per element, as the scalar call does, instead of treating the significance as left out. Co-Authored-By: Claude Opus 5.5 --- crates/calc/src/tests.rs | 5 +++++ crates/functions/src/lift.rs | 2 ++ 2 files changed, 7 insertions(+) diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 3c1330dbe..d29dc9653 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -1845,6 +1845,11 @@ fn lookup_of_an_empty_cell_stays_blank() { // A blank lookup given as an argument is still given, not omitted. ("D10", "=FILTER(1,FALSE,VLOOKUP(\"k\",A1:B2,2,FALSE))", Value::Number(0.0)), ("D11", "=XLOOKUP(\"z\",A1:A2,B1:B2,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + // The same holds per element when the call is lifted over an array of blank lookups. + ("D12", "=CEILING.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D13", "=SUM(CEILING.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), + ("D14", "=FLOOR.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D15", "=SUM(FLOOR.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), ] { t.set(at, f); assert_eq!(t.get(at), want, "{f}"); diff --git a/crates/functions/src/lift.rs b/crates/functions/src/lift.rs index 6d2e3fbc4..d474bf12d 100644 --- a/crates/functions/src/lift.rs +++ b/crates/functions/src/lift.rs @@ -115,6 +115,8 @@ pub(crate) fn call_lifted(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx) -> Val { let v = arr.get_broadcast(r, c); slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v }; + // A blank element of a written array argument is given, as a scalar blank is. + slot.blank_value = !slot.from_ref && matches!(slot.value, Value::Empty); } } let out = (spec.imp)(&cell_args, ctx); From 9c8b4a37eafd10c9193e339d8b5a5eca8789ba1f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:03:35 -0500 Subject: [PATCH 13/51] PDF export: index merged areas when drawing a block (#3203) draw_block checked every cell against every merge meeting the block, and the fill, border and text passes searched the merge list again for each merge start, so an imported sheet with many merge ranges in a print area cost cells x merges (and merges x merges) comparisons. The merges meeting a block are now gathered once and swept down the rows: whether a cell is covered is a BTreeMap lookup among the merges active on its row, and merge starts are found through a HashMap. A merge overlapping one kept before it (only possible in a damaged file) is left out, as Excel does when it repairs the file. Hyperlink rectangles use the same index instead of scanning all sheet merges. --- crates/engine/src/cmd/print/draw.rs | 137 ++++++++++++++++++++++++++-- 1 file changed, 128 insertions(+), 9 deletions(-) diff --git a/crates/engine/src/cmd/print/draw.rs b/crates/engine/src/cmd/print/draw.rs index 7a49b8e85..595b70d86 100644 --- a/crates/engine/src/cmd/print/draw.rs +++ b/crates/engine/src/cmd/print/draw.rs @@ -228,6 +228,82 @@ fn draw_border(pg: &mut Page, b: &BorderLine, wb: &Workbook, start: Point, end: pg.stroke_line_dashed(Point::new(x0, y0), Point::new(x1, y1), lw, color, &border_dash(b.style, k)); } +/// Merged areas active at one row of a downward sweep, kept disjoint: by first column, their last +/// column and last row. Adding, dropping and looking up a merge costs log(active merges). +#[derive(Default)] +struct MergeSweep { + cols: std::collections::BTreeMap, + /// (last row, first column) of the active merges, soonest ending first. + ends: std::collections::BinaryHeap>, +} + +impl MergeSweep { + /// Moves down to `row` (rows never decrease), dropping the merges that end above it. + fn advance(&mut self, row: u32) { + while let Some(std::cmp::Reverse((end, col))) = self.ends.peek().copied() { + if end >= row { + break; + } + self.ends.pop(); + self.cols.remove(&col); + } + } + + /// Does an active merge meet columns `c0..=c1`? The intervals are disjoint, so only the last + /// one starting at or before `c1` can. + fn meets(&self, c0: u32, c1: u32) -> bool { + self.cols.range(..=c1).next_back().is_some_and(|(_, (end, _))| *end >= c0) + } + + /// Adds merge `m`, at row `row`, unless it overlaps an active one. + fn add(&mut self, m: RangeRef, row: u32) -> bool { + self.advance(row); + if self.meets(m.start.col, m.end.col) { + return false; + } + self.cols.insert(m.start.col, (m.end.col, m.end.row)); + self.ends.push(std::cmp::Reverse((m.end.row, m.start.col))); + true + } +} + +/// The merged areas that meet `block`, ordered by the row where they enter it, then by column, +/// leaving out any that overlaps one kept before it (only a damaged file has overlapping merges; +/// Excel drops them when it repairs one). One sweep down the rows instead of a scan of every +/// merge per cell. +fn block_merges(sh: &Sheet, block: RangeRef) -> Vec { + let top = |m: &RangeRef| m.start.row.max(block.start.row); + let mut merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); + merges.sort_by_key(|m| (top(m), m.start.col)); + let mut sweep = MergeSweep::default(); + merges.retain(|m| sweep.add(*m, top(m))); + merges +} + +/// Answers "is this cell inside a merge?" for cells asked in row-major order, from the merges of +/// [`block_merges`]. +struct Covered<'a> { + merges: &'a [RangeRef], + top: u32, + next: usize, + sweep: MergeSweep, +} + +impl<'a> Covered<'a> { + fn new(merges: &'a [RangeRef], block: RangeRef) -> Self { + Self { merges, top: block.start.row, next: 0, sweep: MergeSweep::default() } + } + + fn covers(&mut self, c: CellRef) -> bool { + while let Some(m) = self.merges.get(self.next).filter(|m| m.start.row.max(self.top) <= c.row) { + self.sweep.add(*m, m.start.row.max(self.top)); + self.next += 1; + } + self.sweep.advance(c.row); + self.sweep.meets(c.col, c.col) + } +} + /// Draws the cells of a block: fills, gridlines, data bars, borders and text. pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Option) { let (bx, by) = (b.xs.first().copied().unwrap_or(0.0), b.ys.first().copied().unwrap_or(0.0)); @@ -237,17 +313,18 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt } let block = b.range(); let sh = b.sh; - let merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); - let covered = |c: CellRef| merges.iter().any(|m| m.contains(c)); - // Cells worth visiting. + let merges = block_merges(sh, block); + let by_start: std::collections::HashMap = merges.iter().map(|m| (m.start, *m)).collect(); + // Cells worth visiting, in row-major order (which `Covered` needs). + let mut covered = Covered::new(&merges, block); let cells: Vec = if block.count() <= DENSE_BLOCK { - block.iter().filter(|c| !covered(*c)).collect() + block.iter().filter(|c| !covered.covers(*c)).collect() } else { let mut set: Vec = sh.cells.iter_range(block).map(|(c, _)| c).collect(); set.extend(sh.spill.range(block.start..=block.end).map(|(c, _)| *c).filter(|c| block.contains(*c))); set.sort_unstable(); set.dedup(); - set.retain(|c| !covered(*c)); + set.retain(|c| !covered.covers(*c)); set }; pg.save(); @@ -277,14 +354,14 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt .filter(|(c, l)| { l.fill.is_some() || l.cf.is_some() - || merges.iter().any(|m| m.start == *c) + || by_start.contains_key(c) || !sh.value(*c).is_empty() || l.style.borders != Borders::default() || l.style.num_fmt.as_str() == "checkbox" }) .collect(); let rect_of = |c: CellRef| -> (f32, f32, f32, f32, bool) { - match merges.iter().find(|m| m.start == c) { + match by_start.get(&c) { Some(m) => { let (x, y, w, h) = b.rect(*m); (x, y, w, h, true) @@ -322,7 +399,7 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt continue; } let bd = &l.style.borders; - let (right, bottom) = match merges.iter().find(|m| m.start == *c).filter(|_| merged) { + let (right, bottom) = match by_start.get(c).filter(|_| merged) { Some(m) => ( b.wb.styles.get(sh.style_id(CellRef::new(m.start.row, m.end.col))).borders.right, b.wb.styles.get(sh.style_id(CellRef::new(m.end.row, m.start.col))).borders.bottom, @@ -407,11 +484,12 @@ pub(super) struct PrintedBlock { pub(super) fn draw_block_links(pg: &mut Page, b: &Block, page: usize, pending: &mut Vec) { let range = b.range(); let (Some(&bx), Some(&by), Some(&right), Some(&bottom)) = (b.xs.first(), b.ys.first(), b.xs.last(), b.ys.last()) else { return }; + let by_start: std::collections::HashMap = block_merges(b.sh, range).into_iter().map(|m| (m.start, m)).collect(); for (cell, link) in b.sh.hyperlinks.range(range.start..=range.end) { if !range.contains(*cell) { continue; } - let cell_range = b.sh.merges.iter().find(|m| m.start == *cell).copied().unwrap_or(RangeRef::cell(*cell)); + let cell_range = by_start.get(cell).copied().unwrap_or(RangeRef::cell(*cell)); let (x, y, w, h) = b.rect(cell_range); let (x0, y0, x1, y1) = (x.max(bx), y.max(by), (x + w).min(right), (y + h).min(bottom)); if x1 <= x0 || y1 <= y0 { @@ -503,3 +581,44 @@ pub(super) fn make_block<'a>( } Block { wb, si, sh, rows, cols, xs, ys, k, scale: lay.scale } } + +#[cfg(test)] +mod merge_tests { + use super::{CellRef, Covered, RangeRef, Sheet, block_merges}; + + fn r(a: &str) -> RangeRef { + RangeRef::parse(a).unwrap() + } + + #[test] + fn block_merges_drop_overlaps_and_cover_like_a_scan() { + let mut sh = Sheet::new("S"); + sh.merges = vec![r("B2:C3"), r("E1:F8"), r("B2:B5"), r("A10:D10"), r("H3:H4"), r("Z1:Z2"), r("C3:D4"), r("A1:A1048576")]; + let block = r("B1:H12"); + let merges = block_merges(&sh, block); + // B2:B5 and C3:D4 overlap B2:C3, which comes first; Z1:Z2 is outside the block. + assert_eq!(merges, vec![r("E1:F8"), r("B2:C3"), r("H3:H4"), r("A10:D10")]); + let mut covered = Covered::new(&merges, block); + for c in block.iter() { + assert_eq!(covered.covers(c), merges.iter().any(|m| m.contains(c)), "{}", c.a1()); + } + // Cells asked sparsely, still in order. + let mut covered = Covered::new(&merges, block); + for (a, want) in [("C1", false), ("F5", true), ("G5", false), ("D10", true), ("H12", false)] { + assert_eq!(covered.covers(CellRef::parse(a).unwrap()), want, "{a}"); + } + } + + #[test] + fn many_stacked_merges_are_swept_once() { + // 50,000 merges over the same cells: one is kept, and the block is checked against it + // without comparing every cell with every merge. + let mut sh = Sheet::new("S"); + sh.merges = (0..50_000u32).map(|i| RangeRef::new(CellRef::new(0, 0), CellRef::new(199, 1 + i % 50))).collect(); + let block = r("A1:AZ200"); + let merges = block_merges(&sh, block); + assert_eq!(merges, vec![r("A1:B200")]); + let mut covered = Covered::new(&merges, block); + assert_eq!(block.iter().filter(|c| covered.covers(*c)).count(), 400); + } +} From 2c9e2d1eec0871fc1f292370738f63e399fc0110 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:03:53 -0500 Subject: [PATCH 14/51] Go To: index defined names when resolving and listing them (#3206) Go To resolved every visible name, and each alias inside a name was looked up with Workbook::name, a linear scan of all names. A workbook of many short aliases made the Go To list quadratic (or worse along alias chains) in the number of names. A Resolver now indexes the names by scope and folded spelling once per command and remembers, per name, sheet and depth, whether a name resolves and to how many areas, so listing names resolves each alias once. Lookups keep the old precedence and matching rules. --- crates/engine/src/cmd/edit/go_to.rs | 161 +++++++++++++++++++--------- crates/engine/src/cmd/edit/tests.rs | 22 ++++ 2 files changed, 133 insertions(+), 50 deletions(-) diff --git a/crates/engine/src/cmd/edit/go_to.rs b/crates/engine/src/cmd/edit/go_to.rs index 0cd7a3de9..3c632e3b9 100644 --- a/crates/engine/src/cmd/edit/go_to.rs +++ b/crates/engine/src/cmd/edit/go_to.rs @@ -17,11 +17,13 @@ pub(in super::super) fn go_to(s: &mut Session, p: &Json) -> Result { } let Some(reference) = str_param(p, "reference").or_else(|| str_param(p, "range")) else { return Err(bad("edit.goTo", "missing `reference`")) }; let d = s.doc()?; + let mut names = Resolver::new(&d.wb); let mut target: Option = None; let mut areas: Vec = Vec::new(); let mut named = false; for part in reference_parts(reference) { - let Some((sheet, found, by_name)) = go_to_target(&d.wb, d.wb.active_sheet(), part, 0) else { + let mut found = Vec::new(); + let Some((sheet, _, by_name)) = names.target(d.wb.active_sheet(), part, 0, Some(&mut found)) else { return Err(EngineError::Other("Reference isn't valid.".into())); }; // One selection holds the areas of one sheet. @@ -77,57 +79,114 @@ pub(super) fn reference_parts(text: &str) -> impl Iterator { }) } -/// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, its areas, -/// and whether a defined name or a table gave them. `Sheet2!Name` is the name scoped to Sheet2. -/// A name counts when what it refers to is a reference; a constant or a formula is not one. -fn go_to_target(wb: &gridcraft_model::Workbook, sheet: usize, part: &str, depth: usize) -> Option<(usize, Vec, bool)> { - if part.is_empty() || depth > GO_TO_NAME_DEPTH { - return None; - } - let (prefix, body) = split_sheet(part); - let on = match &prefix { - Some(n) => wb.sheet_index(n)?, - None => sheet, - }; - if let Some(r) = RangeRef::parse(&body.replace('$', "")) { - return Some((on, vec![r], false)); - } - let name = match prefix { - Some(_) => wb.names.iter().find(|n| n.scope == Some(on) && gridcraft_formula::same_name(&n.name, body)), - None => wb.name(body, sheet), - }; - if let Some(n) = name { - return name_areas(wb, n, sheet, depth).map(|(t, areas)| (t, areas, true)); - } - if prefix.is_none() - && let Some((si, ti)) = wb.table(body) - { - // As in Excel, a table's name stands for its data, without the header and totals rows. - let t = wb.sheet(si)?.tables.get(ti)?; - return Some((si, vec![t.data_range().unwrap_or(t.range)], true)); - } - None +/// Resolves Go To references against one workbook. Defined names are found through an index +/// built once, and whether a name refers to a reference is remembered: listing the names of a +/// workbook full of names that alias other names would otherwise scan every name for every alias +/// of every name. +struct Resolver<'a> { + wb: &'a gridcraft_model::Workbook, + /// Indexes into `wb.names` by scope and folded name, in workbook order. + by_name: std::collections::HashMap<(Option, String), Vec>, + /// [`Resolver::name_areas`] without areas, by name index, sheet and depth: the target sheet + /// and the number of areas. + memo: std::collections::HashMap<(usize, usize, usize), Option<(usize, usize)>>, } -/// The sheet and areas defined name `n` refers to, seen from sheet `sheet`, when that is a -/// reference. -fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, sheet: usize, depth: usize) -> Option<(usize, Vec)> { - let from = n.scope.unwrap_or(sheet); - let formula = n.formula.trim().trim_start_matches('='); - let mut target: Option = None; - let mut areas = Vec::new(); - for p in reference_parts(formula) { - let (s, found, _) = go_to_target(wb, from, p, depth + 1)?; - if target.is_some_and(|t| t != s) { - return None; - } - target = Some(s); - areas.extend(found); - if areas.len() > GO_TO_MAX_AREAS { - return None; +impl<'a> Resolver<'a> { + fn new(wb: &'a gridcraft_model::Workbook) -> Self { + let mut by_name: std::collections::HashMap<(Option, String), Vec> = std::collections::HashMap::new(); + for (i, n) in wb.names.iter().enumerate() { + by_name.entry((n.scope, gridcraft_formula::fold_name(&n.name))).or_default().push(i); } + Self { wb, by_name, memo: std::collections::HashMap::new() } + } + + /// The first name of scope `scope` that `same` accepts, as a scan of `wb.names` finds it. + /// Names `same` accepts fold alike, so only that bucket is checked. + fn find(&self, scope: Option, body: &str, same: impl Fn(&str, &str) -> bool) -> Option { + self.by_name + .get(&(scope, gridcraft_formula::fold_name(body)))? + .iter() + .copied() + .find(|&i| self.wb.names.get(i).is_some_and(|n| same(&n.name, body))) + } + + /// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, how many + /// areas (appended to `areas` when given), and whether a defined name or a table gave them. + /// `Sheet2!Name` is the name scoped to Sheet2. A name counts when what it refers to is a + /// reference; a constant or a formula is not one. + fn target(&mut self, sheet: usize, part: &str, depth: usize, areas: Option<&mut Vec>) -> Option<(usize, usize, bool)> { + let wb = self.wb; + if part.is_empty() || depth > GO_TO_NAME_DEPTH { + return None; + } + let (prefix, body) = split_sheet(part); + let on = match &prefix { + Some(n) => wb.sheet_index(n)?, + None => sheet, + }; + if let Some(r) = RangeRef::parse(&body.replace('$', "")) { + if let Some(a) = areas { + a.push(r); + } + return Some((on, 1, false)); + } + // As `Workbook::name`: the sheet's own name, else the workbook's. + let name = match prefix { + Some(_) => self.find(Some(on), body, gridcraft_formula::same_name), + None => self.find(Some(sheet), body, str::eq_ignore_ascii_case).or_else(|| self.find(None, body, str::eq_ignore_ascii_case)), + }; + if let Some(i) = name { + return self.name_areas(i, sheet, depth, areas).map(|(t, n)| (t, n, true)); + } + if prefix.is_none() + && let Some((si, ti)) = wb.table(body) + { + // As in Excel, a table's name stands for its data, without the header and totals rows. + let t = wb.sheet(si)?.tables.get(ti)?; + if let Some(a) = areas { + a.push(t.data_range().unwrap_or(t.range)); + } + return Some((si, 1, true)); + } + None + } + + /// The sheet and number of areas defined name `wb.names[i]` refers to, seen from sheet + /// `sheet`, when that is a reference; the areas are appended to `areas` when given. + fn name_areas(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let key = (i, sheet, depth); + if areas.is_none() + && let Some(known) = self.memo.get(&key) + { + return *known; + } + let out = self.name_areas_uncached(i, sheet, depth, areas.as_deref_mut()); + if areas.is_none() { + self.memo.insert(key, out); + } + out + } + + fn name_areas_uncached(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let n = self.wb.names.get(i)?; + let from = n.scope.unwrap_or(sheet); + let formula = n.formula.trim().trim_start_matches('='); + let mut target: Option = None; + let mut count = 0usize; + for p in reference_parts(formula) { + let (s, found, _) = self.target(from, p, depth + 1, areas.as_deref_mut())?; + if target.is_some_and(|t| t != s) { + return None; + } + target = Some(s); + count += found; + if count > GO_TO_MAX_AREAS { + return None; + } + } + target.map(|t| (t, count)) } - target.map(|t| (t, areas)) } /// The names the Go To dialog lists, seen from sheet `sheet`: the named ranges of the workbook and @@ -137,11 +196,13 @@ fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, pub fn go_to_names(wb: &gridcraft_model::Workbook, sheet: usize) -> Vec { let own: std::collections::HashSet = wb.names.iter().filter(|n| n.scope == Some(sheet)).map(|n| gridcraft_formula::fold_name(&n.name)).collect(); + let mut names = Resolver::new(wb); let mut out: Vec = wb .names .iter() - .filter(|n| !n.hidden && name_areas(wb, n, sheet, 0).is_some()) - .filter_map(|n| match n.scope { + .enumerate() + .filter(|(i, n)| !n.hidden && names.name_areas(*i, sheet, 0, None).is_some()) + .filter_map(|(_, n)| match n.scope { None if own.contains(&gridcraft_formula::fold_name(&n.name)) => None, None => Some(n.name.clone()), Some(s) if s == sheet => Some(n.name.clone()), diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index 49317caa0..dd5fad636 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -42,6 +42,28 @@ fn go_to_lists_the_named_ranges_of_every_sheet() { assert_eq!(list["names"], json!(["Areas", "Local", "Price", "Sheet1!Own", "type"]), "{list}"); } +/// Thousands of names that alias other names are listed with one lookup per alias, not a scan of +/// every name for each; aliases resolve through the chain like a single name does. +#[test] +fn go_to_lists_many_aliases_and_follows_them() { + let mut s = go_to_workbook(); + let wb = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb); + let alias = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(alias("Chain0".into(), "Areas".into())); + for i in 1..20_000 { + wb.names.push(alias(format!("Chain{i}"), format!("Chain{},price", i - 1))); + } + let list = s.execute("edit.goTo", json!({"list": true})).unwrap(); + let names: Vec<&str> = list["names"].as_array().unwrap().iter().filter_map(|n| n.as_str()).collect(); + // Chain0..Chain6 are within the name depth Go To follows; longer chains are not ranges. + for i in 0..7 { + assert!(names.contains(&format!("Chain{i}").as_str()), "Chain{i} in {names:?}"); + } + assert!(!names.contains(&"Chain7"), "{names:?}"); + let out = s.execute("edit.goTo", json!({"reference": "chain1"})).unwrap(); + assert_eq!(out["selection"], "A1,C3:C4,B2:B5", "{out}"); +} + #[test] fn go_to_a_name_scoped_to_another_sheet_goes_to_that_sheet() { let mut s = go_to_workbook(); From 1deb108a7cbdcb87bbe27949ce0d8c118dcf104a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:04:13 -0500 Subject: [PATCH 15/51] xtask: probe keytip commands without file access, and refuse text that could escape generated code (#3269) * xtask: probe keytip commands without file access, and refuse text that could escape generated code cargo xtask keytips ran every command keytips.json names, with its params, under the user's file permissions (file.exportPdf could replace any file, also during --check). It also wrote target, excel and note strings into generated Rust without escaping. Probes now run inside the engine's ScriptSandbox, and those fields may not hold quotes, backslashes, braces or control characters. * address review: make ScriptSandbox !Send so it drops on the thread that entered it --- crates/engine/src/io.rs | 6 ++-- crates/engine/src/io/tests.rs | 13 +++++++++ xtask/src/keytips.rs | 52 +++++++++++++++++++++++++++++------ 3 files changed, 59 insertions(+), 12 deletions(-) diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 690efec1d..8f2c59a97 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -888,12 +888,12 @@ thread_local! { /// While one is alive, [`read_file`] and [`write_file`] on this thread refuse every path: a /// stored script's steps run inside one (#2178), so no command reaches the file system from a /// script whatever its parameters are called. Dropping it (also on unwind) lifts the refusal. -pub(crate) struct ScriptSandbox(()); +pub struct ScriptSandbox(std::marker::PhantomData<*const ()>); impl ScriptSandbox { - pub(crate) fn enter() -> Self { + pub fn enter() -> Self { let _ = SCRIPT_SANDBOX.try_with(|c| c.set(c.get().saturating_add(1))); - ScriptSandbox(()) + ScriptSandbox(std::marker::PhantomData) } } diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index ae5ea8832..d1efc6bf8 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -351,6 +351,19 @@ fn script_sandbox_refuses_file_access_until_dropped() { let _ = std::fs::remove_dir_all(&dir); } +/// The guard's drop lowers the counter of the thread it runs on, so it must not move between threads: +/// naming the trait's method is ambiguous (a compile error) if `ScriptSandbox` is `Send`. +#[test] +fn script_sandbox_is_not_send() { + trait AmbiguousIfSend { + fn check() {} + } + impl AmbiguousIfSend<()> for T {} + struct IsSend; + impl AmbiguousIfSend for T {} + >::check(); +} + /// A minimal compound (CFB/OLE2) file — FAT in sector 0, directory in sector 1 — holding a /// root entry and the given stream names. fn cfb(streams: &[&str]) -> Vec { diff --git a/xtask/src/keytips.rs b/xtask/src/keytips.rs index b662dd101..71bd9c509 100644 --- a/xtask/src/keytips.rs +++ b/xtask/src/keytips.rs @@ -77,6 +77,8 @@ fn probe_live_commands(calls: &[(String, String)]) -> Probe { let params = if params.is_empty() { serde_json::json!({}) } else { serde_json::from_str(params).unwrap_or_default() }; let mut s = Session::new(); s.new_workbook(); + // The JSON picks the commands and their params: none of them may touch the file system. + let _sandbox = gridcraft_engine::io::ScriptSandbox::enter(); let status = match s.execute(id, params.clone()) { Ok(_) => { if s.journal.iter().any(|(jid, _)| jid == id) { @@ -181,15 +183,16 @@ fn parse_and_probe(source: &str) -> Result<(Vec, Probe), String> { } } } - entries.push(Entry { - seq, - kind, - target, - pending, - params, - excel: e.get("excel").and_then(|v| v.as_str()).unwrap_or("").to_string(), - note: e.get("note").and_then(|v| v.as_str()).unwrap_or("").to_string(), - }); + let excel = e.get("excel").and_then(|v| v.as_str()).unwrap_or("").to_string(); + let note = e.get("note").and_then(|v| v.as_str()).unwrap_or("").to_string(); + // These are written into generated Rust: a target inside string literals and format + // strings, the annotations after `//`. Refuse anything that could end either. + for (field, text) in [("target", &target), ("excel", &excel), ("note", ¬e)] { + if text.chars().any(|c| c.is_control() || matches!(c, '"' | '\\' | '{' | '}')) { + return Err(format!("{at}: `{field}` must not hold quotes, backslashes, braces or control characters: {text:?}")); + } + } + entries.push(Entry { seq, kind, target, pending, params, excel, note }); } // JSON order is authoritative and curated: the first entry for a tab is its canonical // badge (legacy aliases come later), and groups stay together. Do not sort. @@ -439,3 +442,34 @@ fn reset(h: &mut Harness) { key(h, Key::Escape); // leave keytip mode entirely } "#; + +#[cfg(all(test, feature = "engine"))] +mod tests { + use super::*; + + /// Text that would end a comment or string literal in the generated Rust is refused. + #[test] + fn annotations_and_targets_cannot_escape_generated_code() { + for entry in [ + r#"{"seq": "Q", "kind": "command", "target": "edit.copy", "note": "x\nfn evil() {}"}"#, + r#"{"seq": "Q", "kind": "command", "target": "edit.copy", "excel": "a\rb"}"#, + r#"{"seq": "Q", "kind": "dialog", "target": "x\"); evil(); (\""}"#, + r#"{"seq": "Q", "kind": "control", "target": "{0}"}"#, + ] { + let doc = format!(r#"{{"sequences": [{entry}]}}"#); + assert!(parse_and_probe(&doc).is_err(), "{entry}"); + } + assert!(parse_and_probe(r#"{"sequences": [{"seq": "Q", "kind": "control", "target": "fontSize", "note": "Font Size box"}]}"#).is_ok()); + } + + /// A probed command may not write files, whatever its params say. + #[test] + fn probes_cannot_touch_the_file_system() { + let path = std::env::temp_dir().join(format!("keytips-probe-{}.pdf", std::process::id())); + let _ = fs::remove_file(&path); + let calls = vec![("file.exportPdf".to_string(), serde_json::json!({ "path": path.to_string_lossy() }).to_string())]; + let probe = probe_live_commands(&calls); + assert_eq!(probe.values().copied().collect::>(), [2]); + assert!(!path.exists()); + } +} From 85d32c20be4da521fd8db989d68e2b78910ac6df Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:04:31 -0500 Subject: [PATCH 16/51] engine: show overlong format codes as General without hashing them per cell (#3273) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * engine: show overlong format codes as General without hashing them per cell The display cache hashed the full format code on every lookup, so one huge code shared by many cells cost its length for every cell drawn. A code with more than 4 × 255 bytes can't be within Excel's 255 characters and is now formatted as General straight away. * address review: keep the original code in the public number_format for overlong formats --- crates/engine/src/display.rs | 40 +++++++++++++++++++++++++++- crates/engine/src/io.rs | 2 +- crates/engine/src/macros/recorder.rs | 2 +- crates/engine/src/macros/run.rs | 2 +- crates/engine/src/pivot.rs | 2 +- 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index 06a0fa587..4dbdf46e8 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -8,8 +8,13 @@ use gridcraft_model::{Sheet, Workbook}; use gridcraft_numfmt::{Formatted, NumberFormat, format_value}; /// Parsed number formats are cached by code. -fn parsed(code: &str) -> NumberFormat { +pub(crate) fn parsed(code: &str) -> NumberFormat { static CACHE: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + // A code of more bytes than 4 × the character limit can't be within it: it shows as General + // without being hashed or scanned again for every cell that shares it. + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse("General"); + } let lock = || CACHE.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(f) = lock().get(code) { return f.clone(); @@ -24,7 +29,12 @@ fn parsed(code: &str) -> NumberFormat { f } +/// The parsed format of `code`; one longer than the character limit is a General format that keeps +/// the code it was made from. The engine itself formats through `parsed`, which doesn't copy it. pub fn number_format(code: &str) -> NumberFormat { + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse(code); + } parsed(code) } @@ -228,3 +238,31 @@ pub fn stats(sheet: &Sheet, ranges: &[RangeRef]) -> Stats { } s } + +#[cfg(test)] +mod tests { + use super::*; + + /// A format code far past Excel's 255 characters shows its values as General, for every cell + /// that shares it, without being hashed into the cache each time. + #[test] + fn an_overlong_code_formats_as_general() { + let wb = Workbook::new(); + let code = format!("0.00{}", "\"x\"".repeat(400_000)); + for _ in 0..1_000 { + assert_eq!(format(&Value::Number(1.5), &code, &wb).text, "1.5"); + } + assert_eq!(format(&Value::Number(1.5), "0.00", &wb).text, "1.50"); + } + + /// The public `number_format` keeps the code of an overlong format; it is General, but not the + /// same format as a real `General`. + #[test] + fn number_format_keeps_an_overlong_code() { + let code = format!("0.00{}", "\"x\"".repeat(2_000)); + let f = number_format(&code); + assert!(f.is_general()); + assert_eq!(f.code(), code); + assert_ne!(f, number_format("General")); + } +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 8f2c59a97..8711785c7 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -576,7 +576,7 @@ pub fn csv_values_bytes(wb: &Workbook) -> Result> { let Some(sh) = wb.sheet(wb.active_sheet()) else { return Ok(Vec::new()) }; gridcraft_xlsx::write_csv(sh, wb.csv_delimiter.unwrap_or(b','), |c| match sh.value(c) { gridcraft_core::Value::Number(n) if n.is_finite() => { - let shape = crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); + let shape = crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); match shape.zip(gridcraft_core::date::datetime_from_serial(wb.date_system, n)) { Some(((date, time), d)) => iso(&d, date, time), None => n.to_string(), diff --git a/crates/engine/src/macros/recorder.rs b/crates/engine/src/macros/recorder.rs index 337c41661..ef9be781f 100644 --- a/crates/engine/src/macros/recorder.rs +++ b/crates/engine/src/macros/recorder.rs @@ -440,7 +440,7 @@ pub fn infer_input(wb: &Workbook, sheet: usize, r: RangeRef) -> NamedInput { Some(match v { Value::Empty => return None, Value::Text(t) if t.is_empty() => return None, - Value::Number(_) if crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, + Value::Number(_) if crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, Value::Number(_) => ColType::Number, Value::Text(_) => ColType::Text, Value::Bool(_) => ColType::Bool, diff --git a/crates/engine/src/macros/run.rs b/crates/engine/src/macros/run.rs index 9e31e3ca9..96c6f504a 100644 --- a/crates/engine/src/macros/run.rs +++ b/crates/engine/src/macros/run.rs @@ -335,7 +335,7 @@ fn check_input(wb: &Workbook, input: &gridcraft_model::NamedInput, place: Place) gridcraft_model::ColType::Number => v.is_number(), gridcraft_model::ColType::Text => v.is_text(), gridcraft_model::ColType::Bool => matches!(v, Value::Bool(_)), - gridcraft_model::ColType::Date => v.is_number() && crate::display::number_format(&date()).is_date(), + gridcraft_model::ColType::Date => v.is_number() && crate::display::parsed(&date()).is_date(), }; if !ok { wrong.push((c, holds(&v, ty))); diff --git a/crates/engine/src/pivot.rs b/crates/engine/src/pivot.rs index bfaccace4..1b5d48155 100644 --- a/crates/engine/src/pivot.rs +++ b/crates/engine/src/pivot.rs @@ -182,7 +182,7 @@ pub fn read_source(wb: &Workbook, text: &str, default_sheet: usize) -> Result Date: Tue, 10 Nov 2026 18:04:49 -0500 Subject: [PATCH 17/51] ui-egui: embed craft-fonts files only from inside the checkout (#3281) * ui-egui: embed craft-fonts files only from inside the checkout The build hook canonicalised each manifest row's file and embedded it with include_bytes! wherever it led, so an absolute path, a .. path or a symlink in a craft-fonts checkout could put any readable file of the build machine into the binary. Manifests and font files must now resolve inside the canonical checkout directory. * address review: register the manifest path before validation and keep the Chinese manifest fallback optional --- crates/ui-egui/build.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/build.rs b/crates/ui-egui/build.rs index 0c5dbf3e8..2551abf2c 100644 --- a/crates/ui-egui/build.rs +++ b/crates/ui-egui/build.rs @@ -76,7 +76,21 @@ fn emit_craft_fonts() { /// One `CraftFont { .. }` initialiser per manifest row GridCraft may embed. fn craft_fonts(dir: &std::path::Path, wasm: bool, required: bool) -> Result { - let manifest = dir.join("fonts/manifest.txt"); + // Every file embedded comes from inside the checkout: a manifest row (or a symlink) naming a + // path outside it would put that file of the build machine into the binary. + let root = dir.canonicalize().map_err(|e| format!("{}: {e}", dir.display()))?; + let inside = |path: std::path::PathBuf| -> Result { + let real = path.canonicalize().map_err(|e| format!("{}: {e}", path.display()))?; + if real.starts_with(&root) { Ok(real) } else { Err(format!("{} is outside the craft-fonts checkout {}", path.display(), root.display())) } + }; + // Registered before validation, by the path as given (and its folder, to see the manifest appear + // or a symlink replaced): a build that skipped the fonts must rebuild once they are there. + let manifest_path = dir.join("fonts/manifest.txt"); + println!("cargo::rerun-if-changed={}", manifest_path.display()); + if let Some(parent) = manifest_path.parent() { + println!("cargo::rerun-if-changed={}", parent.display()); + } + let manifest = inside(manifest_path)?; println!("cargo::rerun-if-changed={}", manifest.display()); let text = std::fs::read_to_string(&manifest).map_err(|e| format!("{}: {e}", manifest.display()))?; let rows = craft_fonts_policy::parse_manifest(&text)?; @@ -84,7 +98,7 @@ fn craft_fonts(dir: &std::path::Path, wasm: bool, required: bool) -> Result Result = row.scripts.iter().map(|s| format!("{s:?}")).collect(); let family = &row.family; @@ -120,6 +134,10 @@ fn chinese_font_rows(dir: &std::path::Path, required: bool) -> Result match craft_fonts_policy::parse_manifest(&text).and_then(|rows| { craft_fonts_policy::check_declared_scripts(&rows)?; From fe39331ac403636d1811f7bc64cba3252d8fcf97 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:05:08 -0500 Subject: [PATCH 18/51] engine: Consolidate finds keys through hash maps and refuses an oversized table (#3285) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * engine: Consolidate finds keys through hash maps and refuses an oversized table data.consolidate searched the keys collected so far for every row and column key, so 100,000 positional rows cost about five billion comparisons. It also built every row × column combination without a size check. Keys are now indexed, and a table over 5 million cells is refused before it is built. * address review: count label cells in the consolidate output limit --- crates/engine/src/cmd/whatif/consolidate.rs | 41 ++++++++++++++++----- crates/engine/src/cmd/whatif/tests.rs | 13 +++++++ 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/crates/engine/src/cmd/whatif/consolidate.rs b/crates/engine/src/cmd/whatif/consolidate.rs index 149b72bc7..702baa312 100644 --- a/crates/engine/src/cmd/whatif/consolidate.rs +++ b/crates/engine/src/cmd/whatif/consolidate.rs @@ -61,6 +61,24 @@ enum Key { const MAX_CONSOLIDATE_CELLS: u64 = 5_000_000; +/// The cells a consolidated table writes: every row key meets every column key, plus a label for +/// each key on the sides that have labels. +fn output_cells(rows: usize, cols: usize, top: bool, left: bool) -> u64 { + let (r, c) = (rows as u64, cols as u64); + r.saturating_mul(c).saturating_add(if top { c } else { 0 }).saturating_add(if left { r } else { 0 }) +} + +#[cfg(test)] +#[test] +fn the_output_limit_counts_label_cells() { + // A million row keys and four column keys are 4 million cells, 5 million with the row labels. + assert_eq!(output_cells(1_000_000, 4, false, false), 4_000_000); + assert_eq!(output_cells(1_000_000, 4, false, true), 5_000_000); + assert!(output_cells(1_000_000, 5, false, true) > MAX_CONSOLIDATE_CELLS); + assert_eq!(output_cells(3, 2, true, true), 6 + 2 + 3); + assert_eq!(output_cells(usize::MAX, usize::MAX, true, true), u64::MAX); +} + pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.consolidate"; let func = str_param(p, "function").unwrap_or("sum").to_string(); @@ -89,20 +107,20 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } None => (d.wb.active_sheet(), d.selection.active), }; + // Keys in first-seen order, each with its index: a lookup is one probe, not a scan. let mut row_keys: Vec = Vec::new(); let mut col_keys: Vec = Vec::new(); + let mut row_index: HashMap = HashMap::new(); + let mut col_index: HashMap = HashMap::new(); let mut row_label: HashMap = HashMap::new(); let mut col_label: HashMap = HashMap::new(); let mut aggs: HashMap<(usize, usize), Agg> = HashMap::new(); let mut total: u64 = 0; - let index_of = |keys: &mut Vec, k: Key| -> usize { - match keys.iter().position(|x| *x == k) { - Some(i) => i, - None => { - keys.push(k); - keys.len() - 1 - } - } + let index_of = |keys: &mut Vec, index: &mut HashMap, k: Key| -> usize { + *index.entry(k).or_insert_with_key(|k| { + keys.push(k.clone()); + keys.len() - 1 + }) }; for t in &srcs { let (si, r) = parse_range(&d.wb, t, CMD)?; @@ -131,7 +149,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(col - c0) }; - cols.push((col, index_of(&mut col_keys, k))); + cols.push((col, index_of(&mut col_keys, &mut col_index, k))); } for row in r0..=r.end.row { let k = if left { @@ -145,7 +163,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(row - r0) }; - let ri = index_of(&mut row_keys, k); + let ri = index_of(&mut row_keys, &mut row_index, k); for (col, ci) in &cols { aggs.entry((ri, *ci)).or_default().add(&sh.value(CellRef::new(row, *col))); } @@ -157,6 +175,9 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { { return Err(EngineError::Other("The consolidated data doesn't fit below and right of the destination.".into())); } + if output_cells(row_keys.len(), col_keys.len(), top, left) > MAX_CONSOLIDATE_CELLS { + return Err(bad(CMD, "the consolidated table would be too large")); + } let mut cells: Vec<(CellRef, Value)> = Vec::new(); if top { for (ci, k) in col_keys.iter().enumerate() { diff --git a/crates/engine/src/cmd/whatif/tests.rs b/crates/engine/src/cmd/whatif/tests.rs index c1476e401..5e5da0e4c 100644 --- a/crates/engine/src/cmd/whatif/tests.rs +++ b/crates/engine/src/cmd/whatif/tests.rs @@ -297,3 +297,16 @@ fn consolidate_by_position_and_label() { assert!(s.execute("data.consolidate", json!({"sources": []})).is_err()); assert!(s.execute("data.consolidate", json!({"sources": ["A1"], "function": "median"})).is_err()); } + +/// A hundred thousand positional rows take one key lookup each; a tall source and a wide one +/// whose keys would meet in a table of 160 million cells are refused before it is built. +#[test] +fn consolidate_keys_are_indexed_and_the_output_is_bounded() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2]]})).unwrap(); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A100000"], "destination": "C1"})).unwrap(); + assert_eq!(r["rows"], 100_000); + assert_eq!(n(&s, "Sheet1!C2"), 2.0); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A10000", "A1:XFC1"], "destination": "A1"})); + assert!(r.unwrap_err().to_string().contains("too large")); +} From a3a98cf7450fe375af844e8d7fa3f1b4eabcf395 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:05:25 -0500 Subject: [PATCH 19/51] functions: REGEXREPLACE keeps only the requested occurrence's captures (#3288) * REGEXREPLACE keeps only the requested occurrence's captures * address review: do not truncate REGEXREPLACE occurrence on 32-bit targets --- crates/functions/src/lib.rs | 6 ++++++ crates/functions/src/text.rs | 22 +++++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/crates/functions/src/lib.rs b/crates/functions/src/lib.rs index ccc61b026..ab50fe62f 100644 --- a/crates/functions/src/lib.rs +++ b/crates/functions/src/lib.rs @@ -308,6 +308,12 @@ mod tests { assert_eq!(ev("REGEXREPLACE", vec![t("2026-10-22"), t("(\\d+)-(\\d+)-(\\d+)"), t("$3/$2/$1")]), Value::text("22/10/2026")); assert_eq!(ev("REGEXREPLACE", vec![t("abc"), t("x*"), t("-")]), Value::text("-a-b-c-")); assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(2.0)]), Value::text("a1b#")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-2.0)]), Value::text("a#b2")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-3.0)]), Value::text("a1b2")); + // 2^32 is no occurrence the text has (and on 32-bit targets it must not wrap to 0). + for occurrence in [4_294_967_296.0, -4_294_967_296.0] { + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(occurrence)]), Value::text("a1b2")); + } // One match whose replacement repeats it 10,000 times: refused before it is built. assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000))]), Value::Error(CellError::Value)); assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000)), n(1.0)]), Value::Error(CellError::Value)); diff --git a/crates/functions/src/text.rs b/crates/functions/src/text.rs index 24674a93c..9d61340d2 100644 --- a/crates/functions/src/text.rs +++ b/crates/functions/src/text.rs @@ -914,13 +914,25 @@ fn regexreplace(a: &[Arg], _c: &mut dyn Ctx) -> R { out.push_str(s.get(last..).unwrap_or_default()); return check_len(out); } - let caps: Vec = re.captures_iter(&s).collect(); - let k = occ.unsigned_abs() as usize; - let idx = if occ > 0 { k.checked_sub(1) } else { caps.len().checked_sub(k) }; - let Some(cap) = idx.and_then(|i| caps.get(i)) else { return text_val(s) }; + // An occurrence past what `usize` holds (32-bit targets) names no match: the text is returned as it was. + let Ok(k) = usize::try_from(occ.unsigned_abs()) else { return text_val(s) }; + // Keeps only the wanted match's captures: the nth from the front, or a window of the last `k`. + let cap = if occ > 0 { + re.captures_iter(&s).nth(k - 1) + } else { + let mut last = std::collections::VecDeque::new(); + for cap in re.captures_iter(&s) { + if last.len() == k { + last.pop_front(); + } + last.push_back(cap); + } + if last.len() == k { last.pop_front() } else { None } + }; + let Some(cap) = cap else { return text_val(s) }; let Some(m) = cap.get(0) else { return text_val(s) }; out.push_str(s.get(..m.start()).unwrap_or_default()); - expand_capped(cap, &rep, &mut out, limit).ok_or(CellError::Value)?; + expand_capped(&cap, &rep, &mut out, limit).ok_or(CellError::Value)?; out.push_str(s.get(m.end()..).unwrap_or_default()); check_len(out) } From 6a08240eb6ce7051a8ae796596446887750608a7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:05:44 -0500 Subject: [PATCH 20/51] io: refuse a FIFO before opening it; keep the replaced file's group on save (#3313) * io: refuse a FIFO before opening it; keep the replaced file's group on save * address review: create the replacement owner-only, then restore group and mode; never widen it when the group cannot be restored * address review: keep the old owner and other bits when the group cannot be restored --- crates/engine/src/io.rs | 40 +++++++++++++++++++++++++++++++---- crates/engine/src/io/tests.rs | 36 +++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 8711785c7..8b0522fb0 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -921,6 +921,12 @@ const MAX_READ_BYTES: u64 = 2 * 1024 * 1024 * 1024; pub fn read_file(path: &str) -> Result> { use std::io::Read; script_sandbox_check(path)?; + // Checked before opening: opening a FIFO nobody writes to blocks the calling (GUI) thread. + // shortcut: a FIFO swapped in between this check and the open still blocks; close that with + // O_NONBLOCK when the crate takes a libc dependency. + if !std::fs::metadata(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?.is_file() { + return Err(EngineError::Other(format!("{path}: only regular files can be opened"))); + } let f = std::fs::File::open(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; // A symlink to an endless device reports length 0 but streams forever, so metadata // alone never bounds the read: require a real file and cap the bytes actually taken. @@ -952,18 +958,37 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { // link's target; and it inherits the destination's access restrictions, so saving a // private workbook doesn't widen who can read it. let tmp = format!("{path}.sctmp"); - let prior = std::fs::metadata(path).ok().map(|m| m.permissions()); + let prior_meta = std::fs::metadata(path).ok(); + let prior = prior_meta.as_ref().map(std::fs::Metadata::permissions); let mut opts = std::fs::OpenOptions::new(); opts.write(true).create_new(true); + // Over an existing file the replacement starts owner-only, whatever the umask: its group is + // still the saving user's primary one until it is changed below, and a member of that group + // must not be able to open it (and keep the descriptor) before then. #[cfg(unix)] - if let Some(p) = &prior { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; - opts.mode(p.mode()); + if prior.is_some() { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); } let mut f = opts.open(&tmp).map_err(|e| { let _ = std::fs::remove_file(&tmp); EngineError::Other(format!("{path}: {e}")) })?; + // The replacement belongs to the saving user's primary group: restore the old file's group and + // then its mode. When the group can't be restored its access is dropped rather than handed to a + // different group; failing to read that back is a failed save. + #[cfg(unix)] + if let Some(m) = &prior_meta { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let _ = std::os::unix::fs::fchown(&f, None, Some(m.gid())); + let group_kept = f.metadata().map(|now| now.gid() == m.gid()).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + EngineError::Other(format!("{path}: {e}")) + })?; + // Best effort: a file system that can't change modes keeps the owner-only one. Without the + // group, the old owner and other bits are kept and the group, setuid and setgid bits dropped. + let _ = f.set_permissions(std::fs::Permissions::from_mode(restored_mode(m.mode(), group_kept))); + } #[cfg(not(unix))] if let Some(p) = &prior { let _ = f.set_permissions(p.clone()); @@ -980,6 +1005,13 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { }) } +/// The mode a saved file gets from the file it replaces (`old`): all of it when its group was +/// restored; without the group, the owner and other bits only. +#[cfg(unix)] +fn restored_mode(old: u32, group_kept: bool) -> u32 { + if group_kept { old & 0o7777 } else { old & 0o707 } +} + #[cfg(target_arch = "wasm32")] pub fn write_file(path: &str, _bytes: &[u8]) -> Result<()> { script_sandbox_check(path)?; diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index d1efc6bf8..c68c43f48 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -51,6 +51,14 @@ fn save_keeps_the_destination_access_restrictions() { write_file(dest.to_str().unwrap(), b"v2").unwrap(); let mode = std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777; assert_eq!(mode, 0o600); + // The mode is set outright from the old file's, not through the umask: group and other access + // it had stay as they were. + for old in [0o640, 0o664] { + std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(old)).unwrap(); + write_file(dest.to_str().unwrap(), b"v3").unwrap(); + assert_eq!(std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777, old); + } + assert!(!dir.join("private.xlsx.sctmp").exists()); // And a brand-new file is created with the restrictive, not permissive, default. let fresh = dir.join("new.xlsx"); write_file(fresh.to_str().unwrap(), b"v").unwrap(); @@ -59,6 +67,17 @@ fn save_keeps_the_destination_access_restrictions() { let _ = std::fs::remove_dir_all(&dir); } +/// Without the old group, a saved file keeps the old owner and other bits and nothing for a group. +#[cfg(not(target_arch = "wasm32"))] +#[cfg(unix)] +#[test] +fn restored_mode_drops_the_group_and_special_bits_when_the_group_is_lost() { + assert_eq!(super::restored_mode(0o100_664, true), 0o664); + assert_eq!(super::restored_mode(0o100_664, false), 0o604); + assert_eq!(super::restored_mode(0o102_444, false), 0o404, "an original read-only file stays read-only, setgid is dropped"); + assert_eq!(super::restored_mode(0o104_755, false), 0o705, "setuid is dropped"); +} + #[cfg(not(target_arch = "wasm32"))] #[test] fn open_refuses_directories_and_unknowable_lengths() { @@ -774,3 +793,20 @@ fn sylk_survives_huge_relative_offsets_and_width_floods() { let (wb, _) = open_bytes("widths.slk", &flood).unwrap(); assert_eq!(wb.sheet(0).unwrap().value(CellRef::parse("A1").unwrap()), gridcraft_core::Value::Number(1.0)); } + +/// A FIFO with no writer would block an open forever: it is refused before it is opened. +#[cfg(unix)] +#[test] +fn reading_a_fifo_is_refused_without_blocking() { + let dir = std::env::temp_dir().join(format!("gridcraft-fifo-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.xlsx"); + if !std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) { + let _ = std::fs::remove_dir_all(&dir); + return; // no mkfifo here + } + let err = read_file(fifo.to_str().unwrap()).unwrap_err().to_string(); + assert!(err.contains("only regular files"), "{err}"); + let _ = std::fs::remove_dir_all(&dir); +} From 0fed26a14fb64296ff7a97af5db9311e4a57fc36 Mon Sep 17 00:00:00 2001 From: Pranav <53838599+prxnvt@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:31:42 -0500 Subject: [PATCH 21/51] Sandbox: a crate that runs a helper program confined (Landlock and seccomp on Linux, sandbox-exec on macOS) ## Stack The Python cells work is split by layer. It merges in this order, each with a merge commit rather than a squash. Nobody can switch Python on before #3481. 1. #3475 depends on `main` 2. #3476 depends on `main` **(this pull request)** 3. #3477 depends on [#3475](https://github.com/storytold/gridcraft/pull/3475) and [#3476](https://github.com/storytold/gridcraft/pull/3476) 4. #3478 depends on [#3477](https://github.com/storytold/gridcraft/pull/3477) 5. #3479 depends on [#3478](https://github.com/storytold/gridcraft/pull/3478) 6. #3480 depends on [#3479](https://github.com/storytold/gridcraft/pull/3479) 7. #3481 depends on [#3480](https://github.com/storytold/gridcraft/pull/3480) ## Summary `crates/sandbox` runs a program that can read only the paths its policy lists, write nothing except `/dev/null`, start no process, and use no network. An application re-executes itself with a launcher flag, closes every inherited descriptor above standard error, applies the confinement, checks that it holds, and only then runs the program. If any of that fails, the program does not run. ## Changes - **`crates/sandbox`:** the policy, the Linux backend (Landlock, and seccomp that denies all sockets, with an x32 filter), the macOS backend (`/usr/bin/sandbox-exec` with a deny-default profile), the probes and a canary. - **`apps/gridcraft`, `apps/gridcraft-cli`:** the launcher flag comes first in `main`, before any window or log. - **`gridcraft-cli sandbox-check`:** says whether this system confines programs, and can run a confined self-test. - **`docs/sandbox.md`:** what is refused and how. - **`nix`:** a new dependency on Linux and macOS, for closing descriptors without `unsafe`. ## Compatibility Windows and FreeBSD have no backend here. There the crate reports that it can't confine, and nothing that needs it runs. On Linux a launch needs Landlock ABI 3, which is Linux 6.2 and later, so an older kernel is treated the same way: this excludes, for example, Debian 12's kernel (6.1) and Ubuntu 22.04's original kernel (5.15). Nothing changes for a process started without the launcher flag. ## Review fixes - **File metadata on Linux.** Landlock has no right for a file's permissions, owner, extended attributes, times or flags, and the seccomp filter allows what it doesn't name, so a confined program could change them on any file its user owns, outside the policy too. The filter now refuses the `chmod`, `chown`, `setxattr`, `removexattr` and timestamp families with their `*at` variants, by path and by descriptor, and any `ioctl` whose number says it hands the kernel data, as `FS_IOC_SETFLAGS` and `FS_IOC_FSSETXATTR` do on a descriptor opened only to read. A filter allows what it doesn't name, so a call a later kernel adds is allowed until it is named; the newest named is `file_setattr` (Linux 6.17). - **Truncation on Linux.** Landlock covers truncation only from ABI 3, so on ABI 1 or 2 `os.truncate(path, 0)` emptied any file its user can write, and `open` with `O_RDONLY | O_TRUNC` emptied one it may read. A launch now requires that right. A filter can't take its place: `openat2` hides its flags from seccomp, and refusing `O_TRUNC` would refuse opening `/dev/null` to write, as `open(os.devnull, "w")` does. - **Standard streams.** They stay open with the rights they were opened with, so a stream that is a file can be written and truncated, and one that is a socket can still send. The page and `launch_command` say so, and that a parent gives the program pipes. - **The enforcement check, again.** It now also tries to change the canary's permissions and to truncate it, asking for what it already has, so nothing changes if one works. - **Other processes on Linux.** On a kernel without Landlock's signal scope, a confined program could signal any process of the same user, and `kill(-1, 9)` would reach them all. The seccomp filter now refuses signals to any process but the program itself, whose ID `exec` keeps; changes to the limits, priority or scheduling of any other process; `pidfd_open`, `pidfd_send_signal`, `kcmp`, `get_robust_list`, `migrate_pages` and `move_pages`; every System V shared memory, message queue and semaphore call and every POSIX message queue call, which no path rule covers; and `syslog`. - **The terminal on Linux.** A program started from a shell could type into its terminal. `TIOCSTI` and `TIOCLINUX` are refused, compared in 32 bits as the kernel reads them, so high bits set on the argument don't get past the rule. - **The enforcement check.** It now also tries signal 0 on the launcher's parent, which delivers nothing, so `sandbox-check` reports a gap rather than assuming none. The launcher tries the same signal before it confines itself and refuses if that fails, and the check refuses if its parent has changed since, since a parent that refuses anyone would prove nothing. A launcher's arguments must include the current executable, which that check starts; otherwise the file rules refused the start and the check proved nothing about new processes. - **A refused launch.** The launcher read its arguments only after closing every inherited descriptor, so a refused launch had already closed them. It reads them first. - **A kernel without Landlock.** The refusal says so in that case too, which the tests that skip on such a machine look for. Only the first ABI's rights being refused, or `ENOSYS` or `EOPNOTSUPP` from creating the ruleset, says so. Any other failure, such as `EINVAL` or too many layers, is reported as what it is and doesn't skip the tests. - **Folders in the exec list on macOS.** A folder got a rule that matches only the folder, so a launcher under it failed to start where the same policy works on Linux. It now covers what is in it, in both rules. - **The sandbox page** names only what this layer has: only `gridcraft-cli sandbox-check` starts the launcher here. ## Testing All 10 steps of `cargo xtask ci` pass on macOS (arm64), and `cargo clippy` passes for this crate for both Linux architectures. The gate on ubuntu-24.04 is the hosted one on this pull request, and it executes the Linux backend through the real binary, including `a_confined_program_leaves_other_processes_alone` and `a_confined_program_changes_no_file`, in which a confined Python tries each refused call on a file inside the policy, one outside it and a descriptor, and the test then compares contents, permissions and modification time. A separate run on the fork, on Linux 7.0 with Landlock ABI 8, covered the same commit's code with parts of the fix taken out. Without the metadata rules, the launch check refuses with `change the canary's permissions: it worked`. Without those rules or the new checks, `a_confined_program_changes_no_file` fails with `permissions of ...: it worked`. With the Landlock rights limited to ABI 2, as on an older kernel, it fails with `truncate ...: it worked`. A kernel that is actually older than ABI 3, or that has no Landlock at all, has not been run. On macOS, terminal input was checked by hand on a pseudo-terminal: `TIOCSTI` is refused when confined and works when not. The commits group the change by area for reading. Only the last commit of the branch is expected to build. ## Summary by CodeRabbit * **New Features** * Added sandbox support for running commands with policy-limited file access and restrictions on network and process operations. The sandbox is available on supported Linux and macOS systems; launches are refused if confinement cannot be established or verified. * Added the `sandbox-check` command to check sandbox support and enforcement, or run a specified program in the sandbox. It reports when sandboxing is unsupported. * **Documentation** * Added guidance on sandbox behavior, platform support, access restrictions, launch checks, and `sandbox-check` usage. --- Cargo.lock | 33 ++ Cargo.toml | 1 + apps/gridcraft-cli/Cargo.toml | 1 + apps/gridcraft-cli/src/main.rs | 10 + apps/gridcraft-cli/src/sandbox_check.rs | 98 +++++ apps/gridcraft-cli/tests/it/main.rs | 1 + apps/gridcraft-cli/tests/it/sandbox.rs | 384 ++++++++++++++++++ apps/gridcraft/Cargo.toml | 1 + apps/gridcraft/src/main.rs | 6 + crates/sandbox/Cargo.toml | 22 ++ crates/sandbox/src/canary.rs | 50 +++ crates/sandbox/src/lib.rs | 263 +++++++++++++ crates/sandbox/src/linux.rs | 497 ++++++++++++++++++++++++ crates/sandbox/src/macos.rs | 92 +++++ crates/sandbox/src/policy.rs | 214 ++++++++++ crates/sandbox/src/probe.rs | 248 ++++++++++++ crates/sandbox/src/tests.rs | 160 ++++++++ docs/sandbox.md | 186 +++++++++ xtask/src/layers.rs | 2 + 19 files changed, 2269 insertions(+) create mode 100644 apps/gridcraft-cli/src/sandbox_check.rs create mode 100644 apps/gridcraft-cli/tests/it/sandbox.rs create mode 100644 crates/sandbox/Cargo.toml create mode 100644 crates/sandbox/src/canary.rs create mode 100644 crates/sandbox/src/lib.rs create mode 100644 crates/sandbox/src/linux.rs create mode 100644 crates/sandbox/src/macos.rs create mode 100644 crates/sandbox/src/policy.rs create mode 100644 crates/sandbox/src/probe.rs create mode 100644 crates/sandbox/src/tests.rs create mode 100644 docs/sandbox.md diff --git a/Cargo.lock b/Cargo.lock index 543cddfe1..d640aa5fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2034,6 +2034,7 @@ dependencies = [ "getrandom 0.4.3", "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "gridcraft-ui-egui", "image", "log", @@ -2076,6 +2077,7 @@ version = "0.16.0" dependencies = [ "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "log", "quick-xml 0.41.0", "serde_json", @@ -2176,6 +2178,17 @@ dependencies = [ name = "gridcraft-pdf" version = "0.16.0" +[[package]] +name = "gridcraft-sandbox" +version = "0.16.0" +dependencies = [ + "landlock", + "libc", + "nix", + "seccompiler", + "thiserror 2.0.21", +] + [[package]] name = "gridcraft-ui-egui" version = "0.16.0" @@ -2873,6 +2886,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "landlock" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cca98e95f35b29d469dade6724c6f96cec9236640f745a0e99b0334ec320ab1" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.21", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -4544,6 +4568,15 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + [[package]] name = "security-framework" version = "3.7.0" diff --git a/Cargo.toml b/Cargo.toml index 1941c9d1e..d3aa9b50c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ repository = "https://github.com/storytold/gridcraft" # internal gridcraft-core = { path = "crates/core" } gridcraft-numfmt = { path = "crates/numfmt" } +gridcraft-sandbox = { path = "crates/sandbox" } gridcraft-formula = { path = "crates/formula" } gridcraft-functions = { path = "crates/functions" } gridcraft-model = { path = "crates/model" } diff --git a/apps/gridcraft-cli/Cargo.toml b/apps/gridcraft-cli/Cargo.toml index 21e958005..83c45d0f2 100644 --- a/apps/gridcraft-cli/Cargo.toml +++ b/apps/gridcraft-cli/Cargo.toml @@ -14,6 +14,7 @@ name = "gridcraft-cli" path = "src/main.rs" [dependencies] +gridcraft-sandbox = { workspace = true } gridcraft-engine = { workspace = true } gridcraft-mcp = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft-cli/src/main.rs b/apps/gridcraft-cli/src/main.rs index 5170ba0c5..2c49c6033 100644 --- a/apps/gridcraft-cli/src/main.rs +++ b/apps/gridcraft-cli/src/main.rs @@ -11,6 +11,7 @@ //! gridcraft-cli functions [--json] [--search X] [--category C] //! gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] //! gridcraft-cli send [json] +//! gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] //! gridcraft-cli version | --version //! ``` #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -30,6 +31,7 @@ macro_rules! outln { } mod check; +mod sandbox_check; use gridcraft_engine::Session; use serde_json::{Value, json}; @@ -49,6 +51,8 @@ const USAGE: &str = "usage: gridcraft-cli functions [--json] [--search X] [--category C] gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] gridcraft-cli send [json] + gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] + whether this system confines programs (no program: a confined self-test) gridcraft-cli version"; /// stdout went away. A reader that stopped early (a closed pipe) ends the program quietly; any @@ -62,6 +66,11 @@ fn stdout_failed(e: std::io::Error) -> ! { } fn main() -> ExitCode { + // A confined launcher run (and its probes) comes first, before anything else starts. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + return ExitCode::from(u8::try_from(code).unwrap_or(1)); + } let args: Vec = std::env::args().skip(1).collect(); let rest = args.get(1..).unwrap_or_default(); let r = match args.first().map(String::as_str) { @@ -79,6 +88,7 @@ fn main() -> ExitCode { Some("cat") => cat(rest), Some("run") => run(rest), Some("check") => return check::main(rest), + Some("sandbox-check") => return sandbox_check::main(rest), Some("commands") => commands(rest), Some("functions") => functions(rest), Some("mcp") => mcp(rest), diff --git a/apps/gridcraft-cli/src/sandbox_check.rs b/apps/gridcraft-cli/src/sandbox_check.rs new file mode 100644 index 000000000..e40bb9ac5 --- /dev/null +++ b/apps/gridcraft-cli/src/sandbox_check.rs @@ -0,0 +1,98 @@ +//! `gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...]`: whether +//! this system confines programs, for a person or a reviewer checking a machine. +//! +//! Without a program it starts a confined copy of itself that tries what the sandbox must deny +//! (reading outside the policy, writing, the network, starting a process) and what it must still +//! allow (a thread, reading an allowed file), and prints one line per attempt. With a program it +//! runs that program confined, with the given read and executable paths and the system's shared +//! libraries, and passes its output and exit status through; its own lines then go to standard +//! error. Every path is absolute. Exit status: 0 when every attempt is +//! as required (or the program succeeded), 1 otherwise (a refused launch included), 2 when GridCraft +//! has no sandbox for this kind of system. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; + +use gridcraft_sandbox::{Canary, Policy, REFUSED_EXIT_CODE, REPORT_FLAG, Support}; + +pub fn main(args: &[String]) -> ExitCode { + match run(args) { + Ok(code) => code, + Err(e) => { + eprintln!("gridcraft-cli: sandbox-check: {e}"); + ExitCode::FAILURE + } + } +} + +fn run(args: &[String]) -> Result { + let mut policy = Policy::new().with_system_libraries(); + let mut it = args.iter(); + let mut program: Option<(PathBuf, Vec)> = None; + while let Some(a) = it.next() { + match a.as_str() { + "--read" => policy = policy.read(it.next().ok_or("--read needs a path")?), + "--exec" => policy = policy.exec(it.next().ok_or("--exec needs a path")?), + "--" => { + let p = it.next().ok_or("a program follows `--`")?; + program = Some((PathBuf::from(p), it.by_ref().map(OsString::from).collect())); + } + other => return Err(format!("unknown option `{other}`")), + } + } + // With a program, its standard output is passed through untouched, so this command's own + // lines go to standard error; the self-test's report is the output, on standard output. + let report = program.is_none(); + let say = |line: String| { + if report { + outln!("{line}"); + } else { + eprintln!("{line}"); + } + }; + match gridcraft_sandbox::support() { + Support::Available(how) => say(format!("sandbox: available ({how})")), + Support::Unsupported(why) => { + say(format!("sandbox: unsupported ({why})")); + return Ok(ExitCode::from(2)); + } + } + let canary = Canary::create().map_err(|e| e.to_string())?; + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + let (program, program_args) = match program { + Some((p, a)) => (p, a), + // A confined copy of this program, reporting each attempt; it reads its own executable as + // the allowed file. + None => { + let home = std::env::var_os("HOME").unwrap_or_default(); + (exe.clone(), vec![REPORT_FLAG.into(), canary.file().into(), exe.clone().into(), home]) + } + }; + let policy = policy.exec(program.clone()); + let mut cmd = gridcraft_sandbox::launch_command(&policy, &canary, &program, &program_args).map_err(|e| e.to_string())?; + let status = cmd.status().map_err(|e| format!("couldn't start the launcher: {e}"))?; + drop(canary); + match status.code() { + Some(0) => { + say(if report { "result: every attempt is as the sandbox requires" } else { "result: the program ran confined and succeeded" }.into()); + Ok(ExitCode::SUCCESS) + } + Some(REFUSED_EXIT_CODE) => { + // The launcher's refusal and a program's own exit share the code; the launcher's reason, + // when it refused, is the `gridcraft-sandbox: refused:` line above. + say(format!( + "result: exit code {REFUSED_EXIT_CODE}: the launcher refused to run the program (its reason is above), or the program itself exited with {REFUSED_EXIT_CODE}" + )); + Ok(ExitCode::FAILURE) + } + other => { + say(format!( + "result: {} ({})", + if report { "some attempts are not as the sandbox requires" } else { "the program failed" }, + other.map_or("no exit code".into(), |c| format!("exit code {c}")) + )); + Ok(ExitCode::FAILURE) + } + } +} diff --git a/apps/gridcraft-cli/tests/it/main.rs b/apps/gridcraft-cli/tests/it/main.rs index 5906b3967..2078aedfd 100644 --- a/apps/gridcraft-cli/tests/it/main.rs +++ b/apps/gridcraft-cli/tests/it/main.rs @@ -1,3 +1,4 @@ //! The crate's integration tests, one module per file, built as one test binary. mod cli; +mod sandbox; mod text_to_columns_quotes; diff --git a/apps/gridcraft-cli/tests/it/sandbox.rs b/apps/gridcraft-cli/tests/it/sandbox.rs new file mode 100644 index 000000000..19861f8d8 --- /dev/null +++ b/apps/gridcraft-cli/tests/it/sandbox.rs @@ -0,0 +1,384 @@ +//! The confined launcher, through the real binary: what a confined program can't do, what it +//! still can, and that a launch whose check fails runs nothing. Linux and macOS confine; other +//! systems report that they can't. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +/// Whether this machine confines programs: a supported system whose self-test passes. On a Linux +/// or macOS machine that reports no sandbox (another architecture, no `sandbox-exec`) or whose +/// kernel doesn't enforce Landlock, the confinement tests have nothing to check and pass with a +/// note. On CI (`CI` set) the machine must confine, so a failure there is never skipped. +fn confines() -> bool { + static CONFINES: std::sync::OnceLock = std::sync::OnceLock::new(); + *CONFINES.get_or_init(|| { + if !cfg!(any(target_os = "linux", target_os = "macos")) { + return false; + } + let o = cli(&["sandbox-check"]); + let out = text(&o); + let missing = out.contains("sandbox: unsupported") || out.contains("Landlock is not enforced"); + if missing && std::env::var_os("CI").is_none() { + eprintln!("this machine doesn't confine programs; skipping:\n{out}"); + return false; + } + true + }) +} + +fn cli(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).args(args).output().unwrap() +} + +fn text(o: &Output) -> String { + format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)) +} + +/// A directory of the test's own in the temporary directory, with one file. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().canonicalize().unwrap().join(format!("gridcraft-sandbox-it-{name}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("allowed.txt"), "allowed text\n").unwrap(); + dir +} + +/// `cat` as this system names it. +fn cat() -> &'static str { + if Path::new("/bin/cat").exists() { "/bin/cat" } else { "/usr/bin/cat" } +} + +#[test] +fn the_self_test_finds_every_denial() { + // A home directory of the test's own, so the home-directory attempt always runs. + let home = scratch("home"); + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).arg("sandbox-check").env("HOME", &home).output().unwrap(); + let out = text(&o); + if !cfg!(any(target_os = "linux", target_os = "macos")) { + assert_eq!(o.status.code(), Some(2), "{out}"); + assert!(out.contains("sandbox: unsupported"), "{out}"); + return; + } + if !confines() { + return; + } + assert!(o.status.success(), "{out}"); + for line in [ + "denied: read the canary", + "denied: change the canary's permissions", + "denied: truncate the canary", + "denied: write next to the canary", + "denied: write in the temporary directory", + "denied: TCP connection", + "denied: start a process", + "denied: signal another process", + "denied: UDP socket", + "denied: list the home directory", + "works: start a thread", + "works: read an allowed file", + ] { + assert!(out.contains(line), "missing `{line}` in:\n{out}"); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_confined_program_reads_only_what_it_is_given() { + if !confines() { + return; + } + let dir = scratch("read"); + let outside = scratch("outside"); + let d = dir.to_string_lossy().into_owned(); + let allowed = dir.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &allowed]); + assert!(o.status.success(), "{}", text(&o)); + // The program's output is passed through untouched; this command's own lines go to stderr. + assert_eq!(String::from_utf8_lossy(&o.stdout), "allowed text\n", "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stderr).contains("result: the program ran confined"), "{}", text(&o)); + // The same program, a file outside the policy: the read is refused and nothing is printed. + let other = outside.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &other]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(!String::from_utf8_lossy(&o.stdout).contains("allowed text"), "{}", text(&o)); + std::fs::remove_dir_all(dir).unwrap(); + std::fs::remove_dir_all(outside).unwrap(); +} + +#[test] +fn a_launch_whose_check_fails_runs_nothing() { + // A run that says it is already confined and isn't finds the canary readable and must refuse. + // On macOS that is the second stage started without `sandbox-exec`; on Linux the launcher never + // uses that flag, and the same refusal must hold. + let canary = scratch("canary"); + let echo = if Path::new("/bin/echo").exists() { "/bin/echo" } else { "/usr/bin/echo" }; + let probe = canary.join("allowed.txt").to_string_lossy().into_owned(); + // A dynamically linked `echo` needs the system's shared libraries and its loader. + // The second stage takes the process the first looked at: here, the parent of what `cli` starts. + let mut launch = vec![ + "--sandbox-confined".to_string(), + std::process::id().to_string(), + "--exec".into(), + echo.into(), + "--exec".into(), + env!("CARGO_BIN_EXE_gridcraft-cli").into(), + ]; + for lib in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"].into_iter().filter(|d| Path::new(d).exists()) { + launch.extend(["--exec".to_string(), lib.into()]); + } + launch.extend(["--probe".to_string(), probe.clone(), "--".into(), echo.into(), "the program ran".into()]); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert_eq!(o.status.code(), Some(97), "{out}"); + assert!(!out.contains("the program ran"), "{out}"); + assert!(out.contains("gridcraft-sandbox: refused"), "{out}"); + assert!(out.contains("read the canary"), "the check refused, not the arguments: {out}"); + // Without that process, the second stage has nothing to try a signal on. + let without: Vec<&str> = std::iter::once("--sandbox-confined").chain(launch[2..].iter().map(String::as_str)).collect(); + let o = cli(&without); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(text(&o).contains("needs the process"), "{}", text(&o)); + // The normal path: the launcher confines itself first, and the program runs. + if confines() { + launch.splice(0..2, ["--sandbox-launch".to_string()]); + // The launcher may be under a folder in the list, on either system. + let mut by_folder = launch.clone(); + by_folder[4] = Path::new(env!("CARGO_BIN_EXE_gridcraft-cli")).parent().unwrap().to_string_lossy().into_owned(); + let o = cli(&by_folder.iter().map(String::as_str).collect::>()); + assert!(o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("the program ran"), "{}", text(&o)); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert!(o.status.success() && out.contains("the program ran"), "{out}"); + } + // A canary inside an allowed path is refused before anything runs, everywhere. + let dir = canary.to_string_lossy().into_owned(); + let o = cli(&["--sandbox-launch", "--read", &dir, "--exec", echo, "--probe", &probe, "--", echo, "the program ran"]); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(!text(&o).contains("the program ran")); + std::fs::remove_dir_all(canary).unwrap(); +} + +/// A Python found on this system: the interpreter to run confined, and its installation, the path it +/// needs to read. `None` when there is no `python3`. +fn system_python() -> Option<(String, String)> { + // On macOS `/usr/bin/python3` is a stub that starts Xcode's Python, outside any read path given + // here; an installed Python comes first. + let candidates = if cfg!(target_os = "macos") { + ["/opt/homebrew/bin/python3", "/usr/local/bin/python3"] + } else { + ["/usr/bin/python3", "/usr/local/bin/python3"] + }; + let python = candidates.into_iter().find(|p| Path::new(p).exists())?; + let prefix = Command::new(python).args(["-c", "import sys; print(sys.base_prefix)"]).output().unwrap(); + let prefix = String::from_utf8_lossy(&prefix.stdout).trim().to_string(); + // A framework build's `bin/python3` only starts `Python.app`'s interpreter as a new process, + // which the sandbox refuses; the interpreter itself is run instead. + let framework = Path::new(&prefix).join("Resources/Python.app/Contents/MacOS/Python"); + let real = if framework.exists() { framework } else { std::fs::canonicalize(python).unwrap() }; + Some((real.to_string_lossy().into_owned(), prefix)) +} + +/// Confined, a program can't signal another process (this test, signal 0 delivering nothing), but can +/// still signal itself and its own threads. On Linux, where the filter names them, it also can't +/// change another process's limits or priority, or type into a terminal: no call here reaches a real +/// terminal, since its standard input isn't one. +#[test] +fn a_confined_program_leaves_other_processes_alone() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import os, signal, sys, threading +other = int(sys.argv[1]) +def refused(call, *args): + try: + call(*args) + except PermissionError: + return True + except OSError as e: + print("other error", call, args, e) + return False +assert refused(os.kill, other, 0), "a signal to another process" +os.kill(os.getpid(), 0) +seen = [] +signal.signal(signal.SIGUSR1, lambda *_: seen.append(1)) +signal.raise_signal(signal.SIGUSR1) +worker = threading.Thread(target=lambda: signal.pthread_kill(threading.main_thread().ident, signal.SIGUSR1)) +worker.start() +worker.join() +assert len(seen) >= 1, "a signal to itself" +if sys.platform == "linux": + import ctypes, errno, fcntl, resource, termios + libc = ctypes.CDLL(None, use_errno=True) + assert libc.shmget(0, 4096, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "System V shared memory" + assert libc.msgget(0, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V message queue" + assert libc.semget(0, 1, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V semaphore" + if hasattr(libc, "mq_open"): + assert libc.mq_open(b"/gridcraft-check", os.O_RDONLY) == -1 and ctypes.get_errno() == errno.EPERM, "a POSIX message queue" + # macOS answers these for the processes the profile lets it signal, which are only itself. + assert refused(os.kill, -1, 0), "a signal to every process" + assert refused(os.kill, 0, 0), "a signal to the process group" + assert refused(resource.prlimit, other, resource.RLIMIT_NOFILE), "another process's limits" + assert refused(os.setpriority, os.PRIO_PROCESS, other, 0), "another process's priority" + assert refused(os.setpriority, os.PRIO_USER, 0, 0), "every process's priority" + assert refused(os.sched_setaffinity, other, {0}), "another process's processors" + assert refused(os.pidfd_open, other), "a handle on another process" + resource.prlimit(0, resource.RLIMIT_NOFILE) + resource.prlimit(os.getpid(), resource.RLIMIT_NOFILE) + os.setpriority(os.PRIO_PROCESS, 0, os.getpriority(os.PRIO_PROCESS, 0)) + assert refused(fcntl.ioctl, 0, termios.TIOCSTI, b"x"), "typing into a terminal" + assert refused(fcntl.ioctl, 0, termios.TIOCSTI | (1 << 32), b"x"), "the same with high bits set" + assert refused(fcntl.ioctl, 0, 0x541C, b"x"), "TIOCLINUX" +print("left alone") +"#; + let me = std::process::id().to_string(); + let o = cli(&["sandbox-check", "--read", &prefix, "--", &python, "-I", "-c", code, &me]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("left alone"), "{}", text(&o)); +} + +/// Confined, a program changes no file, whether the policy lets it read that file or not, by path or +/// by a descriptor: not its contents, permissions, owner, extended attributes, times or flags. +/// Skipped when there is no `python3`. +#[test] +fn a_confined_program_changes_no_file() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import errno, fcntl, os, struct, sys +readable, outside = sys.argv[1:3] +def refused(what, call, *args): + try: + call(*args) + except PermissionError: + return + except OSError as e: + sys.exit(f"{what}: another error, not a refusal: {e}") + sys.exit(f"{what}: it worked") +fd = os.open(readable, os.O_RDONLY) +for target in (readable, outside, fd): + refused(f"permissions of {target}", os.chmod, target, 0) + refused(f"owner of {target}", os.chown, target, os.getuid(), os.getgid()) + refused(f"times of {target}", os.utime, target, (0, 0)) + if sys.platform == "linux": + refused(f"setting an attribute of {target}", os.setxattr, target, "user.gridcraft", b"1") + refused(f"removing an attribute of {target}", os.removexattr, target, "user.gridcraft") +for path in (readable, outside): + refused(f"truncate {path}", os.truncate, path, 0) + refused(f"open {path} with O_TRUNC", os.open, path, os.O_RDONLY | os.O_TRUNC) + refused(f"open {path} to write", os.open, path, os.O_WRONLY) +# A descriptor opened to read can't be truncated either way; the file's contents are checked after. +try: + os.ftruncate(fd, 0) +except OSError: + pass +if sys.platform == "linux": + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + # Newer calls, which the kernel running this may not have: the filter answers before it would. + for name, number in (("fchmodat2", 452), ("setxattrat", 463), ("removexattrat", 466), ("file_setattr", 469)): + ctypes.set_errno(0) + result = libc.syscall(number, -100, readable.encode(), 0, 0) + assert result == -1 and ctypes.get_errno() == errno.EPERM, f"{name}: {result}, {ctypes.get_errno()}" + # On a descriptor opened only to read, these change the file's flags and attributes. + refused("FS_IOC_SETFLAGS", fcntl.ioctl, fd, 0x40086602, struct.pack("l", 0)) + refused("FS_IOC_FSSETXATTR", fcntl.ioctl, fd, 0x401C5820, bytes(28)) + refused("FS_IOC_SET_ENCRYPTION_POLICY", fcntl.ioctl, fd, 0x800C6613, bytes(12)) + # What a program still needs: its own descriptors' flags, and a file of its own in memory. + os.set_inheritable(fd, True) + os.set_inheritable(fd, False) + os.set_blocking(fd, False) + own = os.memfd_create("gridcraft-check") + os.ftruncate(own, 16) + assert os.fstat(own).st_size == 16 +print("unchanged") +"#; + let inside = scratch("unchanged-inside"); + let elsewhere = scratch("unchanged-outside"); + let (readable, outside) = (inside.join("allowed.txt"), elsewhere.join("allowed.txt")); + let state = |file: &Path| { + let m = std::fs::metadata(file).unwrap(); + (std::fs::read(file).unwrap(), m.permissions(), m.modified().unwrap()) + }; + let before = (state(&readable), state(&outside)); + let d = inside.to_string_lossy().into_owned(); + let (r, x) = (readable.to_string_lossy().into_owned(), outside.to_string_lossy().into_owned()); + let o = cli(&["sandbox-check", "--read", &prefix, "--read", &d, "--", &python, "-I", "-c", code, &r, &x]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("unchanged"), "{}", text(&o)); + assert_eq!((state(&readable), state(&outside)), before); + std::fs::remove_dir_all(inside).unwrap(); + std::fs::remove_dir_all(elsewhere).unwrap(); +} + +/// A Python found on this system starts and imports its standard library while confined, with +/// read access to its installation only. Skipped when there is no `python3`. +#[test] +fn python_starts_confined() { + if !confines() { + return; + } + let Some((real, prefix)) = system_python() else { + return; + }; + let o = + cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import json, math, datetime, decimal; print(json.dumps([math.pi > 3]))"]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("[true]"), "{}", text(&o)); + // Confined Python can't open the network or a file outside its installation. + let o = cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import socket; socket.create_connection(('127.0.0.1', 9), 2)"]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("PermissionError") || text(&o).contains("Operation not permitted"), "{}", text(&o)); +} + +#[test] +fn a_confined_program_inherits_no_secrets() { + if !confines() { + return; + } + let env = if Path::new("/usr/bin/env").exists() { "/usr/bin/env" } else { "/bin/env" }; + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")) + .args(["sandbox-check", "--", env]) + .env("GRIDCRAFT_CONTROL_TOKEN", "secret-token") + .env("AWS_SECRET_ACCESS_KEY", "secret-key") + .env("LANG", "en_US.UTF-8") + .output() + .unwrap(); + let out = text(&o); + assert!(o.status.success(), "{out}"); + assert!(!out.contains("secret-"), "the environment reached the program:\n{out}"); + assert!(out.contains("LANG=en_US.UTF-8"), "the locale is kept:\n{out}"); + assert!(out.contains("PATH=/usr/bin:/bin"), "{out}"); +} + +/// A descriptor the parent leaves open (here fd 9, opened by a shell on a file outside any policy) +/// would survive `exec` with access the sandbox doesn't revoke, so the launcher closes it: the +/// program runs, and can't read through it. (A desktop app's system libraries leave such files open, +/// so refusing to run would block Python there.) +#[test] +fn an_inherited_descriptor_never_reaches_the_program() { + if !confines() { + return; + } + let outside = scratch("leak"); + let leaked = outside.join("allowed.txt").to_string_lossy().into_owned(); + let cli = env!("CARGO_BIN_EXE_gridcraft-cli"); + // The shell opens fd 9 (not close-on-exec) and then becomes the command, which inherits it. The + // confined shell then tries to read through it, with a built-in: starting `cat` is refused. Not + // fd 3: on macOS the system's libraries hold a socket of their own there under the sandbox. + let bash = if cfg!(target_os = "macos") { "--exec /bin/bash" } else { "" }; + let program = "read line <&9 && echo \"leaked: $line\"; echo finished"; + let script = format!("exec 9<'{leaked}'; exec '{cli}' sandbox-check {bash} -- /bin/sh -c '{program}'"); + let o = Command::new("/bin/sh").args(["-c", &script]).output().unwrap(); + let out = text(&o); + assert!(out.contains("finished"), "the program ran:\n{out}"); + assert!(!out.contains("leaked:"), "the program read through the descriptor:\n{out}"); + std::fs::remove_dir_all(outside).unwrap(); +} diff --git a/apps/gridcraft/Cargo.toml b/apps/gridcraft/Cargo.toml index 75acee3ff..9365f3e15 100644 --- a/apps/gridcraft/Cargo.toml +++ b/apps/gridcraft/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] gridcraft-engine = { workspace = true } gridcraft-ui-egui = { workspace = true } +gridcraft-sandbox = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "clipboard"] } egui = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft/src/main.rs b/apps/gridcraft/src/main.rs index cc67e241a..f05b421df 100644 --- a/apps/gridcraft/src/main.rs +++ b/apps/gridcraft/src/main.rs @@ -241,6 +241,12 @@ fn icon() -> Option { } fn main() -> eframe::Result<()> { + // A confined launcher run (`gridcraft_sandbox::launch_command`) comes before everything else: + // it opens no window, writes no log, and becomes the program it confines. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + std::process::exit(code); + } // First, so the panic hook and every start-up warning are recorded (`logging`). let logger = logging::install(); // Commands catch their panics (`Session::execute`), but the default hook only prints to diff --git a/crates/sandbox/Cargo.toml b/crates/sandbox/Cargo.toml new file mode 100644 index 000000000..ec02bfb94 --- /dev/null +++ b/crates/sandbox/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "gridcraft-sandbox" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "GridCraft's confined launcher for helper processes: Landlock and seccomp on Linux, sandbox-exec on macOS, refused elsewhere" + +[lints] +workspace = true + +[dependencies] +thiserror = { workspace = true } + +# The confinement layers' system calls sit behind these crates' safe APIs; GridCraft's own code +# has no `unsafe`. Exact pins: they are the security boundary. +[target.'cfg(target_os = "linux")'.dependencies] +landlock = "=0.4.7" +seccompiler = "=0.5.0" +libc = "0.2" + +[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies] +nix = { version = "=0.31.3", default-features = false, features = ["fs", "signal"] } diff --git a/crates/sandbox/src/canary.rs b/crates/sandbox/src/canary.rs new file mode 100644 index 000000000..d5a71096a --- /dev/null +++ b/crates/sandbox/src/canary.rs @@ -0,0 +1,50 @@ +//! The canary: a file outside every allowed path, which a confined launcher must fail to read. + +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::Result; + +static NEXT: AtomicU64 = AtomicU64::new(0); + +/// A fresh directory in the system's temporary directory holding one file. The launcher's +/// enforcement check must be denied reading the file and writing next to it. Both are removed +/// when the canary is dropped. +#[derive(Debug)] +pub struct Canary { + dir: PathBuf, + file: PathBuf, +} + +impl Canary { + pub fn create() -> Result { + let base = std::env::temp_dir().canonicalize()?; + // `create_dir` (not `create_dir_all`) fails if the name exists, so the directory found is + // new; a name another process left behind is skipped. + let mut tries = 0; + let dir = loop { + let dir = base.join(format!("gridcraft-sandbox-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); + match std::fs::create_dir(&dir) { + Ok(()) => break dir, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists && tries < 100 => tries += 1, + Err(e) => return Err(e.into()), + } + }; + let file = dir.join("canary"); + if let Err(e) = std::fs::write(&file, b"gridcraft sandbox canary\n") { + let _ = std::fs::remove_dir_all(&dir); + return Err(e.into()); + } + Ok(Canary { dir, file }) + } + + pub fn file(&self) -> &Path { + &self.file + } +} + +impl Drop for Canary { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} diff --git a/crates/sandbox/src/lib.rs b/crates/sandbox/src/lib.rs new file mode 100644 index 000000000..5c591c5d9 --- /dev/null +++ b/crates/sandbox/src/lib.rs @@ -0,0 +1,263 @@ +//! A confined launcher for helper processes that run code GridCraft doesn't trust, such as a +//! workbook's Python cells. +//! +//! The parent builds a [`Policy`] (the paths the program may read, and the programs it may +//! execute), creates a [`Canary`], and spawns the command [`launch_command`] returns. That command +//! runs the application itself again with [`LAUNCH_FLAG`]; the application hands its arguments to +//! [`handle_args`] before anything else, and the launcher: +//! +//! 1. closes every descriptor besides standard input, output and error, and refuses if one stays +//! open: the confinement doesn't revoke what is already open, and an app's system libraries leave +//! files open that a child inherits; +//! 2. confines itself: on Linux with Landlock (read-only access to the policy's paths, execute +//! access only to its programs, no writes) and seccomp (no new processes, no sockets, none of +//! the calls that escape or inspect a sandbox or that change a file where Landlock has no right +//! for it); on macOS it runs itself again under +//! `sandbox-exec` with a deny-by-default profile built from the same policy ([`CONFINED_FLAG`]); +//! 3. checks that the confinement holds: reading the canary, changing its permissions, truncating +//! it, writing next to it, a TCP connection and starting a process must each be denied; +//! 4. only then replaces itself with the program. +//! +//! When any step fails, the program never runs: the launcher writes one line, +//! `gridcraft-sandbox: refused: `, to its standard error and exits with +//! [`REFUSED_EXIT_CODE`]. Other platforms (Windows, FreeBSD, the web) have no confinement here: +//! [`support`] says so and [`launch_command`] refuses, so callers show `#BLOCKED!`. +//! +//! The crate has no `unsafe` code: the system calls sit behind the `landlock`, `seccompiler` and +//! `nix` crates, and behind `/usr/bin/sandbox-exec` on macOS. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +mod canary; +#[cfg(target_os = "linux")] +mod linux; +#[cfg(target_os = "macos")] +mod macos; +mod policy; +mod probe; + +use std::ffi::{OsStr, OsString}; +use std::path::Path; +use std::process::Command; + +pub use canary::Canary; +pub use policy::Policy; +pub use probe::{Check, Outcome}; + +/// The first argument of a launcher run: the application passes its arguments to [`handle_args`]. +pub const LAUNCH_FLAG: &str = "--sandbox-launch"; +/// The first argument of a launcher already running confined (macOS, under `sandbox-exec`): it +/// goes straight to the enforcement check, which refuses when it isn't confined after all. +pub const CONFINED_FLAG: &str = "--sandbox-confined"; +/// The first argument of the child the enforcement check tries to start; it exits at once. +pub const PROBE_CHILD_FLAG: &str = "--sandbox-probe-child"; +/// The first argument of a confined run that reports what it may and may not do +/// (`gridcraft-cli sandbox-check`). +pub const REPORT_FLAG: &str = "--sandbox-report"; +/// The exit code of a launcher that refused to run its program. +pub const REFUSED_EXIT_CODE: i32 = 97; + +/// Why a program can't be run confined. +#[derive(Debug, thiserror::Error)] +pub enum SandboxError { + #[error("this system has no supported sandbox: {0}")] + Unsupported(String), + #[error("bad sandbox policy: {0}")] + BadPolicy(String), + #[error("the sandbox is not enforced: {0}")] + NotEnforced(String), + #[error("couldn't start the sandbox: {0}")] + Io(#[from] std::io::Error), +} + +pub type Result = std::result::Result; + +/// Whether this system can confine a program. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Support { + /// The sandbox is available; the text names what provides it. + Available(String), + /// It is not, and why. + Unsupported(String), +} + +/// Whether this system can confine a program. A [`launch_command`] can still be refused when the +/// running kernel turns out not to enforce a restriction: the launcher checks again. +pub fn support() -> Support { + platform::support() +} + +/// The command that runs `program` with `args`, confined by `policy`, checked against `canary`. +/// The parent spawns it like any other command and keeps `canary` alive until the program has +/// started (its standard streams are the program's). Those stay open with the rights they were +/// opened with, which the confinement doesn't revoke: a stream that is a file can be written and +/// truncated, and one that is a socket can still send. So give the program pipes. The command runs the current executable +/// again, which must pass its arguments to [`handle_args`] first. +pub fn launch_command(policy: &Policy, canary: &Canary, program: &Path, args: &[OsString]) -> Result { + launch_command_with(&std::env::current_exe()?, policy, canary, program, args) +} + +/// [`launch_command`] with another executable as the launcher: one that also passes its arguments +/// to [`handle_args`] first. +pub fn launch_command_with(exe: &Path, policy: &Policy, canary: &Canary, program: &Path, args: &[OsString]) -> Result { + let checked = policy.checked(exe, program, canary)?; + let mut launch = vec![OsString::from(LAUNCH_FLAG)]; + launch.extend(checked.to_args(canary.file())); + launch.push(OsString::from("--")); + launch.push(checked.program().as_os_str().to_owned()); + launch.extend(args.iter().cloned()); + platform::command(&checked, exe, &launch) +} + +/// Handles the application's own sandbox runs; call it with the process's arguments (without the +/// program name) before anything else. `None`: the arguments are not one of those runs, carry on. +/// `Some(code)`: exit with `code`. A launcher run whose checks pass doesn't return at all: it +/// becomes the program. +pub fn handle_args(args: &[OsString]) -> Option { + let (first, rest) = args.split_first()?; + if first == OsStr::new(LAUNCH_FLAG) || first == OsStr::new(CONFINED_FLAG) { + let Err(e) = if first == OsStr::new(LAUNCH_FLAG) { launch(rest) } else { confined_again(rest) }; + eprintln!("gridcraft-sandbox: refused: {e}"); + return Some(REFUSED_EXIT_CODE); + } + if first == OsStr::new(PROBE_CHILD_FLAG) { + return Some(0); + } + if first == OsStr::new(REPORT_FLAG) { + return Some(probe::report(rest)); + } + None +} + +/// The launcher: check what it inherited, confine, check the confinement, become the program. +/// Returns only on failure. +fn launch(args: &[OsString]) -> Result { + // Arguments that can't be a launch are refused with nothing done. Reading them leaves nothing open. + let (checked, _) = policy::Checked::from_args(args)?; + // Before the launcher opens anything itself. On macOS this is also before the sandbox: under a + // deny-by-default profile the system's libraries hold a socket of their own, which closes at + // `exec` and which must not be taken for an inherited one. + close_inherited_descriptors()?; + // What the check tries on a process has to be allowed now, or its refusal then proves nothing. + let target = probe::signal_target().map_err(SandboxError::NotEnforced)?; + platform::confine(&checked, target, args)?; + confined(target, args) +} + +/// The launcher run again under the confinement ([`CONFINED_FLAG`]): the process the first run looked +/// at comes before its arguments. +fn confined_again(args: &[OsString]) -> Result { + let parsed = args.split_first().and_then(|(target, rest)| Some((target.to_str()?.parse().ok()?, rest))); + let (target, rest) = parsed.ok_or_else(|| SandboxError::NotEnforced(format!("{CONFINED_FLAG} needs the process the launcher looked at")))?; + confined(target, rest) +} + +/// The launcher once it is confined: check that it holds, then become the program. +fn confined(target: u32, args: &[OsString]) -> Result { + let (checked, canary) = policy::Checked::from_args(args)?; + let exe = std::env::current_exe()?; + for check in probe::enforcement(&canary, &exe, target) { + if let Outcome::Allowed(detail) = check.outcome { + return Err(SandboxError::NotEnforced(format!("{}: {detail}", check.name))); + } + } + platform::exec(checked.program(), checked.args()) +} + +#[cfg(target_os = "linux")] +use linux as platform; +#[cfg(target_os = "macos")] +use macos as platform; +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +use unsupported as platform; + +/// Platforms without confinement: every launch is refused. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +mod unsupported { + use super::{Command, OsString, Path, Result, SandboxError, Support, policy::Checked}; + + const WHY: &str = "GridCraft confines programs on Linux and macOS only"; + + pub fn support() -> Support { + Support::Unsupported(WHY.into()) + } + + pub fn command(_: &Checked, _: &Path, _: &[OsString]) -> Result { + Err(SandboxError::Unsupported(WHY.into())) + } + + pub fn confine(_: &Checked, _: u32, _: &[OsString]) -> Result<()> { + Err(SandboxError::Unsupported(WHY.into())) + } + + pub fn exec(_: &Path, _: &[OsString]) -> Result { + Err(SandboxError::Unsupported(WHY.into())) + } +} + +/// Closes every descriptor besides standard input, output and error, and refuses the launch if one +/// stays open or the list can't be read. One left open (a file outside the policy, a connected +/// socket) would survive `exec`, and the confinement doesn't revoke what is already open. Refusing +/// alone wouldn't do: in a desktop app the system's own libraries (Metal's shader cache on macOS) +/// leave files open without close-on-exec, and they reach every child. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn close_inherited_descriptors() -> Result<()> { + for fd in descriptors_above_stdio()? { + // The listing's own descriptor is on the list and is closed already; that error is fine. + let _ = nix::unistd::close(fd); + } + let left = descriptors_above_stdio()?; + // The second listing holds its own directory open, so one above stdio is expected. + if left.len() > 1 { + return Err(SandboxError::NotEnforced(format!("descriptors other than standard input, output and error stayed open ({left:?})"))); + } + Ok(()) +} + +/// The open descriptors above standard error, the listing's own included. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn descriptors_above_stdio() -> Result> { + let dir = if cfg!(target_os = "linux") { "/proc/self/fd" } else { "/dev/fd" }; + let listing = std::fs::read_dir(dir).map_err(|e| SandboxError::NotEnforced(format!("can't list open descriptors ({dir}): {e}")))?; + let mut above_stdio = Vec::new(); + for entry in listing { + let entry = entry.map_err(|e| SandboxError::NotEnforced(format!("can't list open descriptors ({dir}): {e}")))?; + match entry.file_name().to_str().and_then(|n| n.parse::().ok()) { + Some(fd) if fd > 2 => above_stdio.push(fd), + Some(_) => {} + None => return Err(SandboxError::NotEnforced(format!("unexpected entry in {dir}"))), + } + } + above_stdio.sort_unstable(); + Ok(above_stdio) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn close_inherited_descriptors() -> Result<()> { + Ok(()) +} + +/// Replaces this process with `program` (the confining platforms). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn exec_unix(program: &Path, args: &[OsString]) -> Result { + use std::os::unix::process::CommandExt as _; + let mut cmd = Command::new(program); + cmd.args(args).env_clear().env("PATH", CONFINED_PATH); + for name in KEPT_ENV { + if let Some(v) = std::env::var_os(name) { + cmd.env(name, v); + } + } + // `exec` returns only when it failed. + Err(SandboxError::Io(cmd.exec())) +} + +/// The only variables a confined program inherits, besides [`CONFINED_PATH`]: the locale and the +/// time zone. Everything else (tokens, credentials, the user's paths) stays with the parent. +pub const KEPT_ENV: [&str; 4] = ["LANG", "LC_ALL", "LC_CTYPE", "TZ"]; + +/// The `PATH` a confined program sees. +pub const CONFINED_PATH: &str = "/usr/bin:/bin"; + +#[cfg(test)] +mod tests; diff --git a/crates/sandbox/src/linux.rs b/crates/sandbox/src/linux.rs new file mode 100644 index 000000000..aaeff2647 --- /dev/null +++ b/crates/sandbox/src/linux.rs @@ -0,0 +1,497 @@ +//! Linux: the launcher confines itself with Landlock, then seccomp, then `exec`s the program, +//! which inherits both. + +use std::collections::BTreeMap; +use std::ffi::OsString; +use std::path::Path; +use std::process::Command; + +use landlock::{ + ABI, Access, AccessFs, AccessNet, CompatLevel, Compatible, CreateRulesetError, Ruleset, RulesetAttr, RulesetCreatedAttr, RulesetError, + RulesetStatus, Scope, path_beneath_rules, +}; +use seccompiler::{BpfProgram, SeccompAction, SeccompCmpArgLen, SeccompCmpOp, SeccompCondition, SeccompFilter, SeccompRule, TargetArch}; + +use crate::policy::Checked; +use crate::{Result, SandboxError, Support}; + +/// The oldest Landlock ABI a launch accepts, whose filesystem rights are required: the first that +/// covers truncating a file (Linux 6.2). Before it, `truncate` and `open` with `O_TRUNC` empty a file +/// whatever the rules say. A filter can't take its place: `openat2` hides its flags from seccomp, and +/// refusing `O_TRUNC` would refuse opening `/dev/null` to write, as `open(os.devnull, "w")` does. +const OLDEST: ABI = ABI::V3; +/// The newest Landlock ABI this crate knows. Rights it adds past [`OLDEST`] are requested on a +/// best-effort basis. +const NEWEST: ABI = ABI::V9; + +// System calls the `libc` crate doesn't name on both architectures. Numbers from 424 up are the same +// on x86_64 and aarch64. +const SYS_FCHMODAT2: libc::c_long = 452; +const SYS_SETXATTRAT: libc::c_long = 463; +const SYS_REMOVEXATTRAT: libc::c_long = 466; +const SYS_OPEN_TREE_ATTR: libc::c_long = 467; +const SYS_FILE_SETATTR: libc::c_long = 469; + +pub fn support() -> Support { + match arch() { + Some(_) => Support::Available("Landlock and seccomp (checked again at each launch)".into()), + None => Support::Unsupported(format!("seccomp filters are built for x86_64 and aarch64, not {}", std::env::consts::ARCH)), + } +} + +/// Linux runs the launcher directly: it confines itself. +pub fn command(_: &Checked, exe: &Path, launch: &[OsString]) -> Result { + arch().ok_or_else(|| SandboxError::Unsupported(format!("no seccomp filter for {}", std::env::consts::ARCH)))?; + let mut c = Command::new(exe); + c.args(launch); + Ok(c) +} + +pub fn confine(policy: &Checked, _: u32, _: &[OsString]) -> Result<()> { + landlock(policy)?; + seccomp() +} + +pub fn exec(program: &Path, args: &[OsString]) -> Result { + crate::exec_unix(program, args) +} + +fn not_enforced(e: impl std::fmt::Display) -> SandboxError { + SandboxError::NotEnforced(e.to_string()) +} + +/// A kernel without Landlock, or with it off, in the Landlock crate's own text. Only that case says +/// this, since the tests that skip on such a machine look for it: any other failure, such as too many +/// layers or a refused ruleset, comes from a machine that has Landlock, which must not be taken for +/// one that doesn't. +fn landlock_not_enforced(e: impl std::fmt::Display) -> SandboxError { + SandboxError::NotEnforced(format!("Landlock is not enforced: {e}")) +} + +/// What a failed `landlock_create_ruleset` says about the kernel: `ENOSYS` is one built without +/// Landlock and `EOPNOTSUPP` one with it off at boot. Anything else, such as `EINVAL` or too many open +/// files, comes from a kernel that has it. +fn says_no_landlock(errno: Option) -> bool { + matches!(errno, Some(libc::ENOSYS | libc::EOPNOTSUPP)) +} + +/// A kernel without the rights a launch requires ([`OLDEST`]). +fn too_old_for_truncation(e: impl std::fmt::Display) -> SandboxError { + landlock_not_enforced(format!("this kernel has no Landlock, or one older than ABI 3 (Linux 6.2), which can't refuse truncating a file: {e}")) +} + +fn ruleset_error(e: RulesetError) -> SandboxError { + let absent = + matches!(&e, RulesetError::CreateRuleset(CreateRulesetError::CreateRulesetCall { source, .. }) if says_no_landlock(source.raw_os_error())); + if absent { landlock_not_enforced(e) } else { not_enforced(e) } +} + +/// Filesystem: every right Landlock handles is denied except reading the policy's read paths, +/// executing (and reading) its exec paths, and `/dev/null`. On kernels with network rights (ABI +/// 4), TCP binding and connecting are denied too; with scopes (ABI 6), signals to and abstract +/// Unix sockets of processes outside the sandbox. +fn landlock(policy: &Checked) -> Result<()> { + let ruleset = Ruleset::default() + .set_compatibility(CompatLevel::HardRequirement) + .handle_access(AccessFs::from_all(OLDEST)) + .map_err(too_old_for_truncation)? + .set_compatibility(CompatLevel::BestEffort) + .handle_access(AccessFs::from_all(NEWEST)) + .and_then(|r| r.handle_access(AccessNet::from_all(NEWEST))) + .and_then(|r| r.scope(Scope::from_all(NEWEST))) + .map_err(not_enforced)? + .create() + .map_err(ruleset_error)?; + let read = AccessFs::ReadFile | AccessFs::ReadDir; + let ruleset = ruleset + .set_compatibility(CompatLevel::HardRequirement) + .add_rules(path_beneath_rules(&policy.read, read)) + .and_then(|r| r.add_rules(path_beneath_rules(&policy.exec, read | AccessFs::Execute))) + .and_then(|r| r.add_rules(path_beneath_rules(["/dev/null"], AccessFs::ReadFile | AccessFs::WriteFile))) + .map_err(not_enforced)?; + let status = ruleset.restrict_self().map_err(not_enforced)?; + if !status.no_new_privs { + return Err(SandboxError::NotEnforced("no_new_privs is not set".into())); + } + // The rights of `OLDEST` are a hard requirement, so "partially" means only newer rights are + // missing on this kernel; "not enforced" means nothing is. + if status.ruleset == RulesetStatus::NotEnforced { + return Err(SandboxError::NotEnforced(format!("Landlock is not enforced ({:?})", status.landlock))); + } + Ok(()) +} + +fn arch() -> Option { + match std::env::consts::ARCH { + "x86_64" => Some(TargetArch::x86_64), + "aarch64" => Some(TargetArch::aarch64), + _ => None, + } +} + +/// System calls that fail with `EPERM` in the sandbox, unconditionally. +fn denied_calls() -> Vec { + let calls = vec![ + // Network: no socket of any family (a Unix datagram socket can send without + // `connect`), and no connection or listener on one the program might hold. + libc::SYS_socket, + libc::SYS_socketpair, + libc::SYS_connect, + libc::SYS_bind, + libc::SYS_listen, + libc::SYS_accept, + libc::SYS_accept4, + // io_uring runs operations, connections included, outside the seccomp filter. + libc::SYS_io_uring_setup, + libc::SYS_io_uring_enter, + libc::SYS_io_uring_register, + // Inspecting or changing other processes. + libc::SYS_ptrace, + libc::SYS_process_vm_readv, + libc::SYS_process_vm_writev, + libc::SYS_pidfd_getfd, + libc::SYS_pidfd_open, + libc::SYS_pidfd_send_signal, + libc::SYS_kcmp, + libc::SYS_get_robust_list, + libc::SYS_migrate_pages, + libc::SYS_move_pages, + // System V shared memory, message queues and semaphores: a segment or queue this user can + // open holds what another process put there, and no path rule covers them. + libc::SYS_shmget, + libc::SYS_shmat, + libc::SYS_shmctl, + libc::SYS_shmdt, + libc::SYS_msgget, + libc::SYS_msgsnd, + libc::SYS_msgrcv, + libc::SYS_msgctl, + libc::SYS_semget, + libc::SYS_semop, + libc::SYS_semctl, + libc::SYS_semtimedop, + // What changes a file without Landlock having a right for it: permissions, owner, extended + // attributes, times and flags, by path or by descriptor, on any file this user owns. + libc::SYS_fchmod, + libc::SYS_fchmodat, + SYS_FCHMODAT2, + libc::SYS_fchown, + libc::SYS_fchownat, + libc::SYS_setxattr, + libc::SYS_lsetxattr, + libc::SYS_fsetxattr, + SYS_SETXATTRAT, + libc::SYS_removexattr, + libc::SYS_lremovexattr, + libc::SYS_fremovexattr, + SYS_REMOVEXATTRAT, + libc::SYS_utimensat, + SYS_FILE_SETATTR, + // POSIX message queues, which no path rule covers either: another process of the same user can + // open one, by name. + libc::SYS_mq_open, + libc::SYS_mq_unlink, + libc::SYS_mq_timedsend, + libc::SYS_mq_timedreceive, + libc::SYS_mq_notify, + libc::SYS_mq_getsetattr, + // Mounts, namespaces and file handles (ways around path-based rules). + libc::SYS_mount, + libc::SYS_umount2, + libc::SYS_pivot_root, + libc::SYS_chroot, + libc::SYS_unshare, + libc::SYS_setns, + libc::SYS_open_by_handle_at, + libc::SYS_name_to_handle_at, + libc::SYS_open_tree, + SYS_OPEN_TREE_ATTR, + libc::SYS_move_mount, + libc::SYS_fsopen, + libc::SYS_fsconfig, + libc::SYS_fsmount, + libc::SYS_fspick, + libc::SYS_mount_setattr, + // The kernel's own extension points and system state. + libc::SYS_bpf, + libc::SYS_perf_event_open, + libc::SYS_userfaultfd, + libc::SYS_keyctl, + libc::SYS_add_key, + libc::SYS_request_key, + libc::SYS_init_module, + libc::SYS_finit_module, + libc::SYS_delete_module, + libc::SYS_kexec_load, + libc::SYS_kexec_file_load, + libc::SYS_swapon, + libc::SYS_swapoff, + libc::SYS_reboot, + libc::SYS_syslog, + ]; + // x86_64 keeps the older process and file calls and port I/O; aarch64 has none of them. + #[cfg(target_arch = "x86_64")] + let calls = [ + calls, + vec![ + libc::SYS_fork, + libc::SYS_vfork, + libc::SYS_iopl, + libc::SYS_ioperm, + libc::SYS_chmod, + libc::SYS_chown, + libc::SYS_lchown, + libc::SYS_utime, + libc::SYS_utimes, + libc::SYS_futimesat, + ], + ] + .concat(); + calls +} + +/// `who` and `which` of `setpriority` and `ioprio_set` for one process. +const PRIO_PROCESS: u64 = 0; +const IOPRIO_WHO_PROCESS: u64 = 1; +// `libc::Ioctl` differs by C library. +#[allow(clippy::unnecessary_cast)] +const TIOCSTI: u64 = libc::TIOCSTI as u64; +#[allow(clippy::unnecessary_cast)] +const TIOCLINUX: u64 = libc::TIOCLINUX as u64; +/// The bit of an `ioctl` request, on both architectures, that says it hands the kernel data. +const IOC_WRITE: u64 = 0x4000_0000; +/// Changes a directory, though its number says it only reads, as the kernel's headers have it. +const FS_IOC_SET_ENCRYPTION_POLICY: u64 = 0x800C_6613; + +/// What the main filter refuses with `EPERM`: the calls above, and these when their arguments match. +/// - `clone` without `CLONE_THREAD`: a new process. +/// - A signal to any process but `me`, whose ID `exec` keeps. Landlock's signal scope needs a newer +/// kernel than the rest (ABI 6), and `kill(-1, 9)` must not get through on an older one. +/// - A change to the limits, priority or scheduling of any process but `me`. +/// - `TIOCSTI` and `TIOCLINUX`, which type into a terminal the program inherited, as when +/// `gridcraft-cli sandbox-check` runs a program from a shell. +/// - An `ioctl` that hands the kernel data ([`IOC_WRITE`]), as those that change a file's flags or +/// attributes do (`FS_IOC_SETFLAGS`, `FS_IOC_FSSETXATTR`) on a descriptor opened only to read. +/// Landlock's rights don't cover them for files. Those that ask, as `isatty` does, and those that +/// set a descriptor's own flags, as `FIONBIO` does, carry no such bit and still work. +/// +/// Process IDs and `ioctl` requests are compared as the kernel reads them, in 32 bits, so high bits +/// set on the 64-bit argument don't get past a rule. +fn main_rules(me: u32) -> Result>> { + let me = u64::from(me); + let word = |arg: u8, op: SeccompCmpOp, value: u64| SeccompCondition::new(arg, SeccompCmpArgLen::Dword, op, value).map_err(not_enforced); + let rule = |conditions: Vec| SeccompRule::new(conditions).map_err(not_enforced); + let mut rules: BTreeMap> = denied_calls().into_iter().map(|nr| (nr, Vec::new())).collect(); + let clone_thread = u64::try_from(libc::CLONE_THREAD).map_err(not_enforced)?; + let new_process = SeccompCondition::new(0, SeccompCmpArgLen::Qword, SeccompCmpOp::MaskedEq(clone_thread), 0).map_err(not_enforced)?; + rules.insert(libc::SYS_clone, vec![rule(vec![new_process])?]); + for nr in [libc::SYS_kill, libc::SYS_tkill, libc::SYS_tgkill, libc::SYS_rt_sigqueueinfo, libc::SYS_rt_tgsigqueueinfo] { + rules.insert(nr, vec![rule(vec![word(0, SeccompCmpOp::Ne, me)?])?]); + } + // These take 0 for the caller too. + for nr in [libc::SYS_prlimit64, libc::SYS_sched_setaffinity, libc::SYS_sched_setscheduler, libc::SYS_sched_setparam, libc::SYS_sched_setattr] { + rules.insert(nr, vec![rule(vec![word(0, SeccompCmpOp::Ne, me)?, word(0, SeccompCmpOp::Ne, 0)?])?]); + } + for (nr, one_process) in [(libc::SYS_setpriority, PRIO_PROCESS), (libc::SYS_ioprio_set, IOPRIO_WHO_PROCESS)] { + let another = rule(vec![word(1, SeccompCmpOp::Ne, me)?, word(1, SeccompCmpOp::Ne, 0)?])?; + rules.insert(nr, vec![rule(vec![word(0, SeccompCmpOp::Ne, one_process)?])?, another]); + } + rules.insert( + libc::SYS_ioctl, + vec![ + rule(vec![word(1, SeccompCmpOp::Eq, TIOCSTI)?])?, + rule(vec![word(1, SeccompCmpOp::Eq, TIOCLINUX)?])?, + rule(vec![word(1, SeccompCmpOp::MaskedEq(IOC_WRITE), IOC_WRITE)?])?, + rule(vec![word(1, SeccompCmpOp::Eq, FS_IOC_SET_ENCRYPTION_POLICY)?])?, + ], + ); + Ok(rules) +} + +/// The filters: the first answers `clone3` with `ENOSYS`, so the C library falls back to `clone`, +/// whose flags a filter can read (a thread passes `CLONE_THREAD`; a new process doesn't); the +/// second is [`main_rules`]; on x86_64 a third refuses every x32 call, which carries the same +/// architecture value but other numbers. +fn seccomp() -> Result<()> { + let arch = arch().ok_or_else(|| SandboxError::Unsupported(format!("no seccomp filter for {}", std::env::consts::ARCH)))?; + let program = |rules: BTreeMap>, errno: i32| -> Result { + let errno = u32::try_from(errno).map_err(not_enforced)?; + SeccompFilter::new(rules, SeccompAction::Allow, SeccompAction::Errno(errno), arch).and_then(BpfProgram::try_from).map_err(not_enforced) + }; + let clone3 = program(BTreeMap::from([(libc::SYS_clone3, Vec::new())]), libc::ENOSYS)?; + let main = program(main_rules(std::process::id())?, libc::EPERM)?; + // Landlock already set no_new_privs, which installing a filter without privileges needs. + if std::env::consts::ARCH == "x86_64" { + seccompiler::apply_filter(&x32_filter()).map_err(not_enforced)?; + } + seccompiler::apply_filter(&clone3).map_err(not_enforced)?; + seccompiler::apply_filter(&main).map_err(not_enforced)?; + Ok(()) +} + +/// The x32 bit of a system call number on x86_64. +const X32_SYSCALL_BIT: u32 = 0x4000_0000; + +/// A filter that refuses with `EPERM` every call whose number has the x32 bit. x32 calls report +/// the x86_64 architecture value, so the other filters' architecture check lets them through and +/// their numbers match none of the rules. A call of another architecture is left to the other +/// filters, which end the process. +fn x32_filter() -> BpfProgram { + // `struct seccomp_data`: the call number at offset 0, the architecture at offset 4. + const LD_W_ABS: u16 = 0x20; + const JEQ_K: u16 = 0x15; + const JGE_K: u16 = 0x35; + const RET_K: u16 = 0x06; + const AUDIT_ARCH_X86_64: u32 = 62 | 0x8000_0000 | 0x4000_0000; + let op = |code: u16, jt: u8, jf: u8, k: u32| seccompiler::sock_filter { code, jt, jf, k }; + let errno = u32::try_from(libc::EPERM).unwrap_or(1); + vec![ + op(LD_W_ABS, 0, 0, 4), + op(JEQ_K, 1, 0, AUDIT_ARCH_X86_64), + op(RET_K, 0, 0, libc::SECCOMP_RET_ALLOW), + op(LD_W_ABS, 0, 0, 0), + op(JGE_K, 0, 1, X32_SYSCALL_BIT), + op(RET_K, 0, 0, libc::SECCOMP_RET_ERRNO | errno), + op(RET_K, 0, 0, libc::SECCOMP_RET_ALLOW), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_filters_build_for_both_architectures() { + // Building only: installing one would confine the test process. + for arch in [TargetArch::x86_64, TargetArch::aarch64] { + let f = SeccompFilter::new(main_rules(4242).unwrap(), SeccompAction::Allow, SeccompAction::Errno(1), arch); + assert!(f.and_then(BpfProgram::try_from).is_ok(), "{arch:?}"); + } + } + + #[test] + fn the_calls_that_reach_another_process_have_rules() { + let rules = main_rules(4242).unwrap(); + for nr in [ + libc::SYS_kill, + libc::SYS_tkill, + libc::SYS_tgkill, + libc::SYS_rt_sigqueueinfo, + libc::SYS_rt_tgsigqueueinfo, + libc::SYS_prlimit64, + libc::SYS_setpriority, + libc::SYS_ioprio_set, + libc::SYS_sched_setaffinity, + libc::SYS_sched_setscheduler, + libc::SYS_sched_setparam, + libc::SYS_sched_setattr, + ] { + assert_eq!(rules.get(&nr).map(Vec::len), Some(if nr == libc::SYS_setpriority || nr == libc::SYS_ioprio_set { 2 } else { 1 }), "{nr}"); + } + for nr in [ + libc::SYS_pidfd_open, + libc::SYS_pidfd_send_signal, + libc::SYS_kcmp, + libc::SYS_get_robust_list, + libc::SYS_migrate_pages, + libc::SYS_move_pages, + libc::SYS_shmget, + libc::SYS_shmat, + libc::SYS_msgget, + libc::SYS_semget, + libc::SYS_mq_open, + libc::SYS_mq_unlink, + libc::SYS_mq_timedsend, + libc::SYS_mq_timedreceive, + libc::SYS_mq_notify, + libc::SYS_mq_getsetattr, + libc::SYS_syslog, + ] { + assert_eq!(rules.get(&nr).map(Vec::len), Some(0), "{nr} is refused outright"); + } + assert_eq!(rules.get(&libc::SYS_ioctl).map(Vec::len), Some(4)); + } + + #[test] + fn what_landlock_has_no_right_for_is_refused_outright() { + let rules = main_rules(4242).unwrap(); + let every_architecture = [ + (libc::SYS_fchmod, "fchmod"), + (libc::SYS_fchmodat, "fchmodat"), + (SYS_FCHMODAT2, "fchmodat2"), + (libc::SYS_fchown, "fchown"), + (libc::SYS_fchownat, "fchownat"), + (libc::SYS_setxattr, "setxattr"), + (libc::SYS_lsetxattr, "lsetxattr"), + (libc::SYS_fsetxattr, "fsetxattr"), + (SYS_SETXATTRAT, "setxattrat"), + (libc::SYS_removexattr, "removexattr"), + (libc::SYS_lremovexattr, "lremovexattr"), + (libc::SYS_fremovexattr, "fremovexattr"), + (SYS_REMOVEXATTRAT, "removexattrat"), + (libc::SYS_utimensat, "utimensat"), + (SYS_FILE_SETATTR, "file_setattr"), + (SYS_OPEN_TREE_ATTR, "open_tree_attr"), + ]; + for (nr, name) in every_architecture { + assert_eq!(rules.get(&nr).map(Vec::len), Some(0), "{name}"); + } + #[cfg(target_arch = "x86_64")] + for (nr, name) in [ + (libc::SYS_chmod, "chmod"), + (libc::SYS_chown, "chown"), + (libc::SYS_lchown, "lchown"), + (libc::SYS_utime, "utime"), + (libc::SYS_utimes, "utimes"), + (libc::SYS_futimesat, "futimesat"), + ] { + assert_eq!(rules.get(&nr).map(Vec::len), Some(0), "{name}"); + } + } + + #[test] + fn the_numbers_written_here_are_the_kernels() { + // `libc` names this one on x86_64, so a slip in the shared table shows there. + #[cfg(target_arch = "x86_64")] + assert_eq!(SYS_FCHMODAT2, libc::SYS_fchmodat2); + assert_eq!((SYS_SETXATTRAT, SYS_REMOVEXATTRAT, SYS_OPEN_TREE_ATTR, SYS_FILE_SETATTR), (463, 466, 467, 469)); + } + + #[test] + fn a_kernel_that_cant_refuse_truncating_a_file_is_not_accepted() { + assert!(AccessFs::from_all(OLDEST).contains(AccessFs::Truncate)); + assert!(!AccessFs::from_all(ABI::V2).contains(AccessFs::Truncate), "the oldest ABI with the right"); + // It reads as a machine that can't confine, which the tests that skip on one look for. + let e = too_old_for_truncation("the crate's text").to_string(); + assert!(e.contains("Landlock is not enforced:") && e.contains("Linux 6.2") && e.contains("the crate's text"), "{e}"); + } + + #[test] + fn a_kernel_without_landlock_says_so() { + // What the tests and a person read to tell this from a sandbox that failed its check. + assert!(landlock_not_enforced("the crate's text").to_string().contains("Landlock is not enforced: the crate's text")); + } + + #[test] + fn only_a_missing_or_disabled_landlock_says_it_is_not_enforced() { + assert!(says_no_landlock(Some(libc::ENOSYS))); + assert!(says_no_landlock(Some(libc::EOPNOTSUPP))); + for errno in [libc::EINVAL, libc::E2BIG, libc::EFAULT, libc::EMFILE, libc::ENOMEM, libc::EPERM] { + assert!(!says_no_landlock(Some(errno)), "{errno}"); + } + assert!(!says_no_landlock(None)); + // A ruleset failure of another kind, from the crate itself, on any kernel. + let e = Ruleset::default().create().err().map(ruleset_error).map(|e| e.to_string()); + assert!(e.is_some_and(|e| !e.contains("Landlock is not enforced"))); + } + + #[test] + fn the_x32_filter_refuses_numbers_with_the_x32_bit() { + let f = x32_filter(); + assert_eq!(f.len(), 7); + // Load the number, then jump past the refusal for numbers below the x32 bit. + assert_eq!((f[3].code, f[3].k), (0x20, 0)); + assert_eq!((f[4].code, f[4].jt, f[4].jf, f[4].k), (0x35, 0, 1, X32_SYSCALL_BIT)); + assert_eq!(f[5].k, libc::SECCOMP_RET_ERRNO | 1); + } +} diff --git a/crates/sandbox/src/macos.rs b/crates/sandbox/src/macos.rs new file mode 100644 index 000000000..090f28ccd --- /dev/null +++ b/crates/sandbox/src/macos.rs @@ -0,0 +1,92 @@ +//! macOS: the launcher runs itself again under `sandbox-exec` with a deny-by-default profile built +//! from the policy, checks that the profile holds, and `exec`s the program, which stays under it. + +use std::ffi::OsString; +use std::os::unix::process::CommandExt as _; +use std::path::Path; +use std::process::Command; + +use crate::policy::Checked; +use crate::{CONFINED_FLAG, Result, SandboxError, Support}; + +const SANDBOX_EXEC: &str = "/usr/bin/sandbox-exec"; + +pub fn support() -> Support { + if Path::new(SANDBOX_EXEC).exists() { + Support::Available("sandbox-exec with a deny-by-default profile (checked again at each launch)".into()) + } else { + Support::Unsupported(format!("{SANDBOX_EXEC} is missing")) + } +} + +/// The launcher starts unconfined, so it can look at what it inherited first ([`confine`]). A +/// policy the profile can't express is refused here, before anything starts. +pub fn command(policy: &Checked, exe: &Path, launch: &[OsString]) -> Result { + if !Path::new(SANDBOX_EXEC).exists() { + return Err(SandboxError::Unsupported(format!("{SANDBOX_EXEC} is missing"))); + } + profile(policy)?; + let mut c = Command::new(exe); + c.args(launch); + Ok(c) +} + +/// Runs the launcher again under `sandbox-exec`, with the profile applied before it starts; the +/// enforcement check of that run proves it. Returns only on failure. +pub fn confine(policy: &Checked, target: u32, args: &[OsString]) -> Result<()> { + let exe = std::env::current_exe()?; + let mut c = Command::new(SANDBOX_EXEC); + c.arg("-p").arg(profile(policy)?).arg(exe).arg(CONFINED_FLAG).arg(target.to_string()).args(args); + Err(SandboxError::Io(c.exec())) +} + +pub fn exec(program: &Path, args: &[OsString]) -> Result { + crate::exec_unix(program, args) +} + +/// The profile: everything is denied except what the base system profile allows a process to +/// start (`system.sb`: the system libraries, the dynamic loader's cache and a few system +/// services), reading the policy's paths, executing its programs, reading file metadata (path +/// resolution), reading system settings (`sysctl`) and writing `/dev/null`. There is no rule for +/// starting processes, writing files or the network, so they stay denied. +pub fn profile(policy: &Checked) -> Result { + let mut p = String::from("(version 1)\n(deny default)\n(import \"system.sb\")\n"); + p.push_str("(allow process-exec"); + for path in &policy.exec { + p.push_str(&path_rule(path)?); + } + p.push_str(")\n(allow file-read*"); + for path in policy.exec.iter().chain(&policy.read) { + p.push_str(&path_rule(path)?); + } + p.push_str(")\n(allow file-read-metadata)\n(allow sysctl-read)\n(allow file-write-data (literal \"/dev/null\"))\n"); + Ok(p) +} + +/// A profile rule for `path`. A literal matches only that path, so a directory, whose contents the +/// policy means, as Landlock takes it on Linux, is a subpath. +fn path_rule(path: &Path) -> Result { + let rule = if path.is_dir() { "subpath" } else { "literal" }; + Ok(format!(" ({rule} {})", quoted(path)?)) +} + +/// `path` as a profile string literal: in double quotes, with `"` and `\` escaped. Paths with +/// other control characters (the policy already refuses NUL and line breaks) or that aren't UTF-8 +/// are refused rather than quoted. +pub fn quoted(path: &Path) -> Result { + let s = path.to_str().ok_or_else(|| SandboxError::BadPolicy(format!("{} is not UTF-8", path.display())))?; + let mut out = String::with_capacity(s.len() + 2); + out.push('"'); + for c in s.chars() { + match c { + '"' | '\\' => { + out.push('\\'); + out.push(c); + } + c if c.is_control() => return Err(SandboxError::BadPolicy(format!("{} holds a control character", path.display()))), + c => out.push(c), + } + } + out.push('"'); + Ok(out) +} diff --git a/crates/sandbox/src/policy.rs b/crates/sandbox/src/policy.rs new file mode 100644 index 000000000..d39f07fcf --- /dev/null +++ b/crates/sandbox/src/policy.rs @@ -0,0 +1,214 @@ +//! What a confined program may touch, and how the launcher receives it on its command line. + +use std::ffi::{OsStr, OsString}; +use std::path::{Path, PathBuf}; + +use crate::{Canary, Result, SandboxError}; + +/// At most this many paths in each list. +pub const MAX_PATHS: usize = 64; +/// At most this many bytes in one path. +pub const MAX_PATH_BYTES: usize = 4096; + +/// The paths a confined program may read and the programs it may execute. Everything else is +/// denied: writing anywhere, network access and starting other processes have no switch. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Policy { + read: Vec, + exec: Vec, +} + +impl Policy { + pub fn new() -> Policy { + Policy::default() + } + + /// Lets the program read `path`: a file, or a directory and everything under it. + pub fn read(mut self, path: impl Into) -> Policy { + self.read.push(path.into()); + self + } + + /// Lets the program execute (and read) the file `path`. + pub fn exec(mut self, path: impl Into) -> Policy { + self.exec.push(path.into()); + self + } + + /// Adds what a dynamically linked program needs to start on this system: the shared-library + /// directories and the dynamic loader's cache on Linux (macOS's base profile covers its own). + /// Paths that don't exist here are skipped. + pub fn with_system_libraries(mut self) -> Policy { + if cfg!(target_os = "linux") { + for dir in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"] { + // The dynamic loader is executed, so its directories get execute access. + if Path::new(dir).exists() { + self.exec.push(dir.into()); + } + } + // What the C library and Python's `zoneinfo` read for the time zone. + for path in ["/etc/ld.so.cache", "/etc/localtime", "/usr/share/zoneinfo"] { + if Path::new(path).exists() { + self.read.push(path.into()); + } + } + } + // Fonts, which a plot's text is drawn with. Only the system's: fonts in the home folder stay + // out of reach. macOS's own `/System/Library/Fonts` comes with its base profile. + let fonts: &[&str] = if cfg!(target_os = "linux") { + &["/usr/share/fonts", "/usr/local/share/fonts"] + } else if cfg!(target_os = "macos") { + &["/Library/Fonts"] + } else { + &[] + }; + for dir in fonts { + if Path::new(dir).exists() { + self.read.push((*dir).into()); + } + } + self + } + + pub fn read_paths(&self) -> &[PathBuf] { + &self.read + } + + pub fn exec_paths(&self) -> &[PathBuf] { + &self.exec + } + + /// The policy as the launcher will apply it: every path absolute, existing and canonical, the + /// lists within their caps, the current executable `exe` executable (the enforcement check + /// starts it), `program` executable, and the canary outside every allowed path. + pub(crate) fn checked(&self, exe: &Path, program: &Path, canary: &Canary) -> Result { + let read = canonical_all(&self.read)?; + let mut exec = canonical_all(&self.exec)?; + let program = canonical(program)?; + for extra in [canonical(exe)?, program.clone()] { + if !exec.contains(&extra) { + exec.push(extra); + } + } + if exec.len() > MAX_PATHS { + return Err(SandboxError::BadPolicy(format!("more than {MAX_PATHS} executable paths"))); + } + let probe = canonical(canary.file())?; + if read.iter().chain(&exec).any(|p| probe.starts_with(p)) { + return Err(SandboxError::BadPolicy(format!("the canary {} is inside an allowed path", probe.display()))); + } + Ok(Checked { read, exec, program, args: Vec::new() }) + } +} + +fn canonical_all(paths: &[PathBuf]) -> Result> { + if paths.len() > MAX_PATHS { + return Err(SandboxError::BadPolicy(format!("more than {MAX_PATHS} paths"))); + } + let mut out: Vec = Vec::with_capacity(paths.len()); + for p in paths { + let c = canonical(p)?; + if !out.contains(&c) { + out.push(c); + } + } + Ok(out) +} + +/// `path` made canonical, after refusing what can't be one: an empty or relative path, one over +/// [`MAX_PATH_BYTES`], one with a NUL or a line break (which no sandbox profile can quote), and +/// one that doesn't exist. +fn canonical(path: &Path) -> Result { + let bytes = path.as_os_str().as_encoded_bytes(); + if bytes.is_empty() || !path.is_absolute() { + return Err(SandboxError::BadPolicy(format!("{} is not an absolute path", path.display()))); + } + if bytes.len() > MAX_PATH_BYTES { + return Err(SandboxError::BadPolicy(format!("a path is longer than {MAX_PATH_BYTES} bytes"))); + } + if bytes.iter().any(|b| matches!(b, 0 | b'\n' | b'\r')) { + return Err(SandboxError::BadPolicy("a path holds a NUL or a line break".into())); + } + let c = path.canonicalize().map_err(|e| SandboxError::BadPolicy(format!("{}: {e}", path.display())))?; + if c.as_os_str().as_encoded_bytes().len() > MAX_PATH_BYTES { + return Err(SandboxError::BadPolicy(format!("a path is longer than {MAX_PATH_BYTES} bytes"))); + } + Ok(c) +} + +/// A policy checked by [`Policy::checked`], with the program and its arguments: what the launcher +/// applies. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Checked { + pub read: Vec, + pub exec: Vec, + program: PathBuf, + args: Vec, +} + +impl Checked { + pub fn program(&self) -> &Path { + &self.program + } + + pub fn args(&self) -> &[OsString] { + &self.args + } + + /// The launcher's options for this policy (`--read`, `--exec`, `--probe`), without the + /// program, which follows a `--`. + pub fn to_args(&self, canary: &Path) -> Vec { + let mut out = Vec::new(); + for (flag, paths) in [("--read", &self.read), ("--exec", &self.exec)] { + for p in paths { + out.push(OsString::from(flag)); + out.push(p.as_os_str().to_owned()); + } + } + out.push(OsString::from("--probe")); + out.push(canary.as_os_str().to_owned()); + out + } + + /// Reads the launcher's arguments back: options, `--`, the program and its arguments. The + /// paths are checked again (the launcher trusts nothing), and the canary is required. + pub fn from_args(args: &[OsString]) -> Result<(Checked, PathBuf)> { + let mut read = Vec::new(); + let mut exec = Vec::new(); + let mut probe: Option = None; + let mut it = args.iter(); + loop { + let Some(flag) = it.next() else { return Err(SandboxError::BadPolicy("no program after `--`".into())) }; + if flag == OsStr::new("--") { + break; + } + let value = it.next().ok_or_else(|| SandboxError::BadPolicy(format!("{} needs a path", flag.to_string_lossy())))?; + let path = canonical(Path::new(value))?; + match flag.to_str() { + Some("--read") => read.push(path), + Some("--exec") => exec.push(path), + Some("--probe") if probe.is_none() => probe = Some(path), + _ => return Err(SandboxError::BadPolicy(format!("unknown launcher option {}", flag.to_string_lossy()))), + } + if read.len() > MAX_PATHS || exec.len() > MAX_PATHS { + return Err(SandboxError::BadPolicy(format!("more than {MAX_PATHS} paths"))); + } + } + let program = it.next().ok_or_else(|| SandboxError::BadPolicy("no program after `--`".into()))?; + let program = canonical(Path::new(program))?; + if !exec.contains(&program) { + return Err(SandboxError::BadPolicy(format!("{} is not an executable path of the policy", program.display()))); + } + let probe = probe.ok_or_else(|| SandboxError::BadPolicy("no canary (`--probe`)".into()))?; + if read.iter().chain(&exec).any(|p| probe.starts_with(p)) { + return Err(SandboxError::BadPolicy(format!("the canary {} is inside an allowed path", probe.display()))); + } + // The enforcement check starts the current executable. Outside the executable paths, the file + // rules would refuse that start, and the check would prove nothing about new processes. + let exe = canonical(&std::env::current_exe()?)?; + if !exec.iter().any(|p| exe.starts_with(p)) { + return Err(SandboxError::BadPolicy(format!("the current executable {} is not an executable path of the policy", exe.display()))); + } + Ok((Checked { read, exec, program, args: it.cloned().collect() }, probe)) + } +} diff --git a/crates/sandbox/src/probe.rs b/crates/sandbox/src/probe.rs new file mode 100644 index 000000000..bf66a0abe --- /dev/null +++ b/crates/sandbox/src/probe.rs @@ -0,0 +1,248 @@ +//! The enforcement check a confined launcher runs before it starts its program, and the fuller +//! report `gridcraft-cli sandbox-check` prints. + +use std::ffi::OsString; +use std::io::ErrorKind; +use std::net::{SocketAddr, TcpStream, UdpSocket}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use crate::PROBE_CHILD_FLAG; + +/// What one attempt found. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Outcome { + /// The attempt was refused by the system, as the sandbox requires. + Denied, + /// The attempt was not refused (it worked, or failed for another reason, which doesn't prove + /// the sandbox); the text says what happened. + Allowed(String), +} + +/// One attempt and its outcome. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Check { + pub name: &'static str, + pub outcome: Outcome, +} + +/// A loopback address with nothing to talk to: unconfined, connecting fails with "refused", which +/// is not a denial; confined, creating the socket or connecting is denied. +const TCP_TARGET: ([u8; 4], u16) = ([127, 0, 0, 1], 9); + +/// Every attempt that must be denied, against `canary` (a file outside the policy), `exe` (the +/// current executable, which the process check tries to start) and `target` (from +/// [`signal_target`]). Each must report [`Outcome::Denied`] before the launcher runs its program. +pub fn enforcement(canary: &Path, exe: &Path, target: u32) -> Vec { + vec![ + Check { name: "read the canary", outcome: denied(std::fs::read(canary).map(drop)) }, + Check { name: "change the canary's permissions", outcome: denied(chmod_probe(canary)) }, + Check { name: "truncate the canary", outcome: denied(truncate_probe(canary)) }, + Check { name: "write next to the canary", outcome: write_probe(canary.parent().unwrap_or(canary)) }, + Check { name: "write in the temporary directory", outcome: write_probe(&std::env::temp_dir()) }, + Check { name: "TCP connection", outcome: denied(tcp_probe()) }, + Check { name: "start a process", outcome: spawn_probe(exe) }, + Check { name: "signal another process", outcome: signal_probe(target) }, + ] +} + +/// Asks for the permissions the file has, so nothing changes even where this is allowed. +fn chmod_probe(file: &Path) -> std::io::Result<()> { + std::fs::set_permissions(file, std::fs::metadata(file)?.permissions()) +} + +/// Asks for the length the file has, so nothing changes even where this is allowed. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn truncate_probe(file: &Path) -> std::io::Result<()> { + let length = nix::libc::off_t::try_from(std::fs::metadata(file)?.len()).map_err(|e| std::io::Error::new(ErrorKind::InvalidInput, e))?; + nix::unistd::truncate(file, length).map_err(std::io::Error::from) +} + +/// Other systems have no confinement, so no launch gets this far. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn truncate_probe(_: &Path) -> std::io::Result<()> { + Err(std::io::Error::new(ErrorKind::Unsupported, "this system has no sandbox")) +} + +/// Signal 0 to `process`, which delivers nothing: it only asks whether a signal would be allowed. +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn signal_zero(process: u32) -> std::io::Result<()> { + let pid = i32::try_from(process).map_err(|e| std::io::Error::new(ErrorKind::InvalidInput, e))?; + nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None).map_err(std::io::Error::from) +} + +/// The process the check tries a signal on: this launcher's parent, if a signal to it is allowed +/// now, before any confinement. One that refuses already (PID 1, another user's process) would be +/// refused afterwards too, which would prove nothing, so the launch is refused instead. +#[cfg(any(target_os = "linux", target_os = "macos"))] +pub fn signal_target() -> std::result::Result { + signalable(std::os::unix::process::parent_id()) +} + +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn signalable(process: u32) -> std::result::Result { + signal_zero(process) + .map(|()| process) + .map_err(|e| format!("a signal to process {process} fails before any confinement, so its refusal would prove nothing: {e}")) +} + +/// Whether a signal to `target` is refused now. A parent that changed since [`signal_target`] looked +/// is another process, which may refuse for its own reasons (a launcher whose parent exited has PID 1 +/// for a parent). +#[cfg(any(target_os = "linux", target_os = "macos"))] +fn signal_probe(target: u32) -> Outcome { + if std::os::unix::process::parent_id() != target { + return Outcome::Allowed(format!("this launcher's parent is no longer process {target}")); + } + denied(signal_zero(target)) +} + +/// Other systems have no confinement, so no launch gets this far. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +pub fn signal_target() -> std::result::Result { + Err("this system has no sandbox".into()) +} + +/// Other systems have no confinement, so no launch gets this far. +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn signal_probe(_: u32) -> Outcome { + Outcome::Allowed("this system has no sandbox".into()) +} + +/// `Denied` when `r` failed with "permission denied" (`EACCES` or `EPERM`), else what happened. +fn denied(r: std::io::Result<()>) -> Outcome { + match r { + Ok(()) => Outcome::Allowed("it worked".into()), + Err(e) if e.kind() == ErrorKind::PermissionDenied => Outcome::Denied, + Err(e) => Outcome::Allowed(format!("it failed for another reason: {e}")), + } +} + +fn write_probe(dir: &Path) -> Outcome { + let file = dir.join(format!("gridcraft-sandbox-write-probe-{}", std::process::id())); + let r = std::fs::OpenOptions::new().write(true).create_new(true).open(&file); + if r.is_ok() { + let _ = std::fs::remove_file(&file); + } + denied(r.map(drop)) +} + +fn tcp_probe() -> std::io::Result<()> { + TcpStream::connect_timeout(&SocketAddr::from(TCP_TARGET), Duration::from_secs(2)).map(drop) +} + +fn spawn_probe(exe: &Path) -> Outcome { + let child = Command::new(exe).arg(PROBE_CHILD_FLAG).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn(); + match child { + Ok(mut c) => { + let _ = c.wait(); + Outcome::Allowed("a process started".into()) + } + // A start refused by the sandbox: no new process (seccomp, or the macOS profile). + Err(e) if e.kind() == ErrorKind::PermissionDenied => Outcome::Denied, + Err(e) => Outcome::Allowed(format!("it failed for another reason: {e}")), + } +} + +/// `--sandbox-report [ []]`, run confined: prints one line per attempt, +/// `denied: ` or `allowed: ()`, then the attempts that must work, and exits +/// with 0 when every line is as the sandbox requires, 1 otherwise. Used by +/// `gridcraft-cli sandbox-check` and the tests. +pub fn report(args: &[OsString]) -> i32 { + let (Some(canary), readable) = (args.first(), args.get(1)) else { + eprintln!("gridcraft-sandbox: --sandbox-report needs the canary's path"); + return 2; + }; + let exe = std::env::current_exe().unwrap_or_default(); + let mut ok = true; + // The launcher looked at this process's parent before it confined itself, and `exec` keeps it. + #[cfg(any(target_os = "linux", target_os = "macos"))] + let target = std::os::unix::process::parent_id(); + #[cfg(not(any(target_os = "linux", target_os = "macos")))] + let target = 0; + let mut checks = enforcement(Path::new(canary), &exe, target); + checks.push(Check { name: "UDP socket", outcome: denied(UdpSocket::bind(SocketAddr::from(([127, 0, 0, 1], 0))).map(drop)) }); + // The person's own files: not part of the launch check, since a policy may allow a directory + // under the home directory (a Python installed there), but no policy here allows the home + // directory itself. + // The confined process inherits no `HOME` (its environment is cleared), so the directory is + // an argument. + if let Some(home) = args.get(2).filter(|h| !h.is_empty()) { + checks.push(Check { name: "list the home directory", outcome: denied(std::fs::read_dir(home).map(drop)) }); + } + for c in &checks { + match &c.outcome { + Outcome::Denied => println!("denied: {}", c.name), + Outcome::Allowed(detail) => { + ok = false; + println!("allowed: {} ({detail})", c.name); + } + } + } + // What a confined program still needs. + let thread = std::thread::Builder::new().spawn(|| 2 + 2).map(|h| h.join().is_ok()); + ok &= worked("start a thread", thread.as_ref().map(|j| *j).unwrap_or(false), thread.err().map(|e| e.to_string())); + if let Some(file) = readable { + let r = std::fs::read(file); + ok &= worked("read an allowed file", r.is_ok(), r.err().map(|e| e.to_string())); + } + i32::from(!ok) +} + +fn worked(name: &str, ok: bool, err: Option) -> bool { + if ok { + println!("works: {name}"); + } else { + println!("broken: {name} ({})", err.unwrap_or_default()); + } + ok +} + +#[cfg(all(test, any(target_os = "linux", target_os = "macos")))] +mod tests { + use super::*; + + #[test] + fn a_signal_that_fails_before_any_confinement_proves_nothing_after_it() { + // This test isn't confined, and its parent has the same user. + let parent = std::os::unix::process::parent_id(); + assert_eq!(signal_target(), Ok(parent)); + assert!(matches!(signal_probe(parent), Outcome::Allowed(_)), "allowed unconfined, so no denial"); + // No such process: there is nothing to compare a refusal with, so the launch is refused. + for process in [u32::MAX, i32::MAX as u32] { + let e = signalable(process).unwrap_err(); + assert!(e.contains("before any confinement"), "{e}"); + } + } + + #[test] + fn the_checks_on_the_canary_change_nothing_where_they_are_allowed() { + let canary = crate::Canary::create().unwrap(); + let file = canary.file(); + let state = || { + let m = std::fs::metadata(file).unwrap(); + (std::fs::read(file).unwrap(), m.permissions(), m.len()) + }; + let before = state(); + // This test isn't confined, so each works, which isn't a denial. + assert_eq!(denied(chmod_probe(file)), Outcome::Allowed("it worked".into())); + assert_eq!(denied(truncate_probe(file)), Outcome::Allowed("it worked".into())); + assert_eq!(state(), before); + } + + #[test] + fn a_missing_canary_proves_nothing() { + let canary = crate::Canary::create().unwrap(); + let gone = canary.file().with_file_name("not there"); + for outcome in [denied(chmod_probe(&gone)), denied(truncate_probe(&gone))] { + assert!(matches!(&outcome, Outcome::Allowed(detail) if detail.contains("another reason")), "{outcome:?}"); + } + } + + #[test] + fn a_parent_that_changed_is_not_taken_for_the_one_looked_at() { + let other = std::os::unix::process::parent_id().wrapping_add(1); + assert!(matches!(signal_probe(other), Outcome::Allowed(detail) if detail.contains("no longer"))); + } +} diff --git a/crates/sandbox/src/tests.rs b/crates/sandbox/src/tests.rs new file mode 100644 index 000000000..af1d8ffc5 --- /dev/null +++ b/crates/sandbox/src/tests.rs @@ -0,0 +1,160 @@ +use std::ffi::OsString; +use std::path::{Path, PathBuf}; + +use crate::policy::{Checked, MAX_PATH_BYTES, MAX_PATHS}; +use crate::{Canary, PROBE_CHILD_FLAG, Policy, SandboxError, handle_args}; + +fn os(args: &[&str]) -> Vec { + args.iter().map(OsString::from).collect() +} + +fn bad_policy(r: crate::Result) -> String { + match r { + Err(SandboxError::BadPolicy(why)) => why, + other => panic!("expected a bad-policy error, got {other:?}"), + } +} + +/// A directory of its own with a file in it, outside the system's temporary directory's canary. +fn scratch() -> (Canary, PathBuf) { + let canary = Canary::create().unwrap(); + let dir = + std::env::temp_dir().canonicalize().unwrap().join(format!("gridcraft-sandbox-test-{}-{:?}", std::process::id(), std::thread::current().id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("data.txt"), "data").unwrap(); + (canary, dir) +} + +#[test] +fn a_checked_policy_is_canonical_and_executes_the_launcher_and_the_program() { + let (canary, dir) = scratch(); + let exe = std::env::current_exe().unwrap().canonicalize().unwrap(); + let file = dir.join("data.txt"); + let checked = Policy::new().read(&dir).read(&dir).exec(&file).checked(&exe, &file, &canary).unwrap(); + assert_eq!(checked.read, vec![dir.clone()], "duplicates collapse"); + assert_eq!(checked.exec, vec![file.clone(), exe.clone()], "the launcher joins the program"); + assert_eq!(checked.program(), file); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn what_a_policy_refuses() { + let (canary, dir) = scratch(); + let exe = std::env::current_exe().unwrap(); + let ok = dir.join("data.txt"); + let check = |p: Policy| bad_policy(p.checked(&exe, &ok, &canary)); + assert!(check(Policy::new().read("relative/path")).contains("absolute")); + assert!(check(Policy::new().read("")).contains("absolute")); + assert!(check(Policy::new().read(dir.join("missing"))).contains("missing")); + assert!(check(Policy::new().read(PathBuf::from(format!("/{}", "a".repeat(MAX_PATH_BYTES))))).contains("longer")); + let mut many = Policy::new(); + for _ in 0..=MAX_PATHS { + many = many.read(&dir); + } + assert!(check(many).contains("more than")); + // The canary must stay outside every allowed path, or the check would read it. + let around = canary.file().parent().unwrap().to_path_buf(); + assert!(check(Policy::new().read(around)).contains("canary")); + #[cfg(unix)] + { + use std::os::unix::ffi::OsStrExt as _; + let line_break = PathBuf::from(std::ffi::OsStr::from_bytes(b"/tmp/a\nb")); + assert!(check(Policy::new().read(line_break)).contains("line break")); + } + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn the_launcher_reads_its_arguments_back() { + let (canary, dir) = scratch(); + let exe = std::env::current_exe().unwrap(); + let program = dir.join("data.txt"); + let checked = Policy::new().read(&dir).checked(&exe, &program, &canary).unwrap(); + let mut args = checked.to_args(canary.file()); + args.push("--".into()); + args.push(program.clone().into()); + args.extend(os(&["-c", "print(1)"])); + let (back, probe) = Checked::from_args(&args).unwrap(); + assert_eq!(probe, canary.file().canonicalize().unwrap()); + assert_eq!((back.read.clone(), back.exec.clone()), (checked.read.clone(), checked.exec.clone())); + assert_eq!(back.args(), os(&["-c", "print(1)"]).as_slice()); + + let p = program.to_string_lossy().into_owned(); + let c = canary.file().to_string_lossy().into_owned(); + // No canary, a program the policy doesn't execute, an unknown option, a missing program. + assert!(bad_policy(Checked::from_args(&os(&["--exec", &p, "--", &p]))).contains("canary")); + assert!(bad_policy(Checked::from_args(&os(&["--probe", &c, "--", &p]))).contains("not an executable path")); + assert!(bad_policy(Checked::from_args(&os(&["--write", &p, "--probe", &c, "--", &p]))).contains("unknown")); + assert!(bad_policy(Checked::from_args(&os(&["--exec", &p, "--probe", &c, "--"]))).contains("no program")); + assert!(bad_policy(Checked::from_args(&os(&["--exec", &p, "--probe", &c]))).contains("no program")); + assert!(bad_policy(Checked::from_args(&os(&["--exec"]))).contains("needs a path")); + // A second canary isn't taken. + assert!(bad_policy(Checked::from_args(&os(&["--exec", &p, "--probe", &c, "--probe", &c, "--", &p]))).contains("unknown")); + // The enforcement check starts the current executable, so a hand-built list must let it. + assert!(bad_policy(Checked::from_args(&os(&["--exec", &p, "--probe", &c, "--", &p]))).contains("current executable")); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn only_the_sandbox_flags_are_handled() { + assert_eq!(handle_args(&[]), None); + assert_eq!(handle_args(&os(&["--control", "7979"])), None); + assert_eq!(handle_args(&os(&[PROBE_CHILD_FLAG])), Some(0)); +} + +/// A launch whose arguments are refused runs nothing and leaves the process's descriptors alone: +/// closing them here would pull files out from under the tests running beside this one. +#[test] +fn a_launch_refused_for_its_arguments_closes_nothing() { + use std::io::Read as _; + let (_canary, dir) = scratch(); + let mut held = std::fs::File::open(dir.join("data.txt")).unwrap(); + assert_eq!(handle_args(&os(&[crate::LAUNCH_FLAG, "--", "/bin/sh"])), Some(crate::REFUSED_EXIT_CODE)); + let mut text = String::new(); + held.read_to_string(&mut text).unwrap(); + assert_eq!(text, "data"); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[test] +fn the_canary_is_a_fresh_file_removed_on_drop() { + let a = Canary::create().unwrap(); + let b = Canary::create().unwrap(); + assert_ne!(a.file(), b.file()); + assert!(a.file().is_file()); + let dir = a.file().parent().unwrap().to_path_buf(); + drop(a); + assert!(!dir.exists()); + assert!(Path::new(b.file()).exists()); +} + +#[cfg(target_os = "macos")] +#[test] +fn a_directory_of_programs_is_a_subpath_in_both_of_its_rules() { + let (canary, dir) = scratch(); + let exe = std::env::current_exe().unwrap(); + let checked = Policy::new().exec(&dir).checked(&exe, &dir.join("data.txt"), &canary).unwrap(); + let profile = crate::macos::profile(&checked).unwrap(); + let rule = format!("(subpath \"{}\")", dir.display()); + assert_eq!(profile.matches(&rule).count(), 2, "{profile}"); + assert!(!profile.contains(&format!("(literal \"{}\")", dir.display())), "{profile}"); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[cfg(target_os = "macos")] +#[test] +fn profile_strings_are_escaped() { + assert_eq!(crate::macos::quoted(Path::new("/a/b c")).unwrap(), "\"/a/b c\""); + assert_eq!(crate::macos::quoted(Path::new("/a\"b\\c")).unwrap(), "\"/a\\\"b\\\\c\""); + assert!(crate::macos::quoted(Path::new("/a\tb")).is_err(), "a control character is refused"); + let (canary, dir) = scratch(); + let exe = std::env::current_exe().unwrap(); + let checked = Policy::new().read(&dir).checked(&exe, &dir.join("data.txt"), &canary).unwrap(); + let profile = crate::macos::profile(&checked).unwrap(); + assert!(profile.starts_with("(version 1)\n(deny default)\n")); + assert!(profile.contains(&format!("(subpath \"{}\")", dir.display()))); + for absent in ["process-fork", "network", "file-write*"] { + assert!(!profile.contains(absent), "no rule allows {absent}"); + } + std::fs::remove_dir_all(dir).unwrap(); +} diff --git a/docs/sandbox.md b/docs/sandbox.md new file mode 100644 index 000000000..78dcf1084 --- /dev/null +++ b/docs/sandbox.md @@ -0,0 +1,186 @@ +# The sandbox for helper processes + +> **Last updated:** 2026-11-07 · **Change:** minor (no file changes by any route on Linux: Landlock's truncation right is required, and seccomp refuses what Landlock has no right for; earlier: more is refused on Linux: other processes, System V and POSIX queues, the terminal; earlier: first version: the `sandbox` crate and `gridcraft-cli sandbox-check`) + +The sandbox is the layer GridCraft is to run code it doesn't trust in, such as a workbook's +Python cells: a separate process that confines itself before the code starts. Here only +`gridcraft-cli sandbox-check` starts it, and Python cells don't run ([architecture.md](architecture.md), +Python cells). The crate is `crates/sandbox`, layer 0. +This page describes what the confinement denies, how it is +checked, and where it is available. + +## Threat model + +The code is a program GridCraft doesn't trust, which may come from anyone. It runs with the person's +user account. Without a sandbox it could read and change the person's files, use the network and start +other programs. The sandbox's job is that it can do none of these: + +- **Files:** it can read only the paths its policy lists and write no file except `/dev/null`. Nor can it change + a file any other way: not its length, permissions, owner, extended attributes, times or flags, even + one it can read. The exception is a standard stream the parent made a file (see below). The home + directory, documents and other workbooks are out of reach. So `tempfile` finds no folder it can use. +- **Programs:** it can execute only the programs its policy lists, and it can't start a new + process at all, so even those only replace itself. +- **Network:** no sockets, no connections and no listeners. The exception is a standard stream the + parent made a socket (see below). +- **Other processes and the system:** no debugging or reading another process, no System V shared + memory, message queues or semaphores, no POSIX message queues, no signals to another process, no typing into a terminal it inherited, no mounts or namespaces, no kernel + modules, BPF, keyrings or performance counters. On Linux it also can't change another process's + limits, priority or scheduling. + +Data reaches the code only through its command-line arguments (which the parent chooses), its +standard input and output (which the parent holds), and the files its policy lets it read. Nothing +else is open: the launcher closes every descriptor it inherited, because the confinement doesn't +revoke what is already open. The three standard streams stay as the parent made them, with the rights +they were opened with: a stream that is a file can be written and truncated, and one that is a +socket can still send, since neither is a file or socket the program opened. A parent gives the +program pipes. `sandbox-check` passes its own streams on, so its output sent to a file is such a +stream. The +program inherits no environment: the launcher clears it and keeps only the locale and time zone +(`LANG`, `LC_ALL`, `LC_CTYPE`, `TZ`), with `PATH` set to `/usr/bin:/bin`, so tokens and credentials +in the parent's environment don't reach it. + +The sandbox doesn't limit time, memory or output size; whoever starts the program owns those limits. +The files it may read can be read, and the code can use the CPU while it runs. + +## How a launch works + +1. The parent builds a `Policy` (read paths and executable paths), creates a `Canary` (a fresh file + in the temporary directory, outside every allowed path) and spawns `launch_command`. That runs + the application's own executable again, with `--sandbox-launch` and the policy. +2. The application hands its arguments to `gridcraft_sandbox::handle_args` before anything else, + so a launcher run opens no window and writes no log. +3. The launcher closes every open descriptor besides standard input, output and error, and refuses + if one stays open. A desktop app's system libraries (Metal's shader cache on macOS) leave files + open without close-on-exec, so these reach every child. `nix` holds the system call. +4. The launcher confines itself (Linux) or runs itself again under `sandbox-exec` with the profile + (macOS, `--sandbox-confined`). On macOS the descriptors are closed first because, under a + deny-by-default profile, the system's libraries hold a close-on-exec socket of their own, which + must not be taken for an inherited one. +5. **The enforcement check.** Before the program runs, each of these must be refused by the + system with a permission error: + - reading the canary; + - changing its permissions, to those it has, and truncating it, to the length it has, so nothing + changes if the attempt works; + - creating a file next to it, and in the temporary directory; + - a TCP connection to the loopback address; + - starting a process (a copy of itself that would exit at once); + - signal 0 to the launcher's parent, which delivers nothing. Before it confines itself, the + launcher checks that the same signal is allowed, and refuses otherwise: a parent that refuses + anyone (PID 1, another user's process) would prove nothing. It refuses too if its parent has + changed since. + An attempt that works, or fails for another reason (a missing file, a refused connection), + doesn't count as a denial. +6. Only then does the launcher replace itself with the program (`exec`), which inherits the + confinement. When any step fails, the program never runs: the launcher prints + `gridcraft-sandbox: refused: ` on standard error and exits with code 97. + +## Linux + +The launcher applies two layers, then runs the check. + +- **Landlock** (ABI 3, which is Linux 6.2 and later, enabled by default on current distributions). + - **Filesystem rights:** every right Landlock handles is denied except: + - reading files and listing directories under the read paths; + - executing and reading the executable paths; + - reading and writing `/dev/null`. + - **Required and best-effort rights:** the filesystem rights up to ABI 3 are a hard requirement, + so without Landlock, or with an older one, the launch is refused. ABI 3 is the first with a right + for truncation: before it, `truncate` and `open` with `O_TRUNC` empty a file whatever the rules + say. seccomp can't take its place, since `openat2` hides its flags from a filter, and refusing + `O_TRUNC` would also refuse opening `/dev/null` to write. Newer rights are added where the kernel + has them: + - TCP bind and connect, as a second network layer (ABI 4); + - device ioctls (ABI 5); + - signals and abstract Unix sockets outside the sandbox (ABI 6). + - **`no_new_privs`** is set, so no program can gain privileges. +- **seccomp**, three filters. + - **`clone3`** fails with `ENOSYS`. The C library then uses `clone`, whose flags the filter can + read. + - **Refused with `EPERM`:** + - `clone` without `CLONE_THREAD` (a new process; threads still work), plus `fork` and + `vfork`; + - `socket` of every family (a Unix datagram socket can send without `connect`), and + `socketpair`, `connect`, `bind`, `listen`, `accept` and `accept4`; + - io_uring (it runs operations outside the filter); + - `ptrace`, `process_vm_readv`/`writev`, `pidfd_getfd`, `pidfd_open`, `pidfd_send_signal`, + `kcmp`, `get_robust_list`, `migrate_pages`, `move_pages`; + - System V shared memory, message queues and semaphores (`shmget`, `shmat`, `shmctl`, `shmdt`, + `msgget`, `msgsnd`, `msgrcv`, `msgctl`, `semget`, `semop`, `semctl`, `semtimedop`), and POSIX + message queues (`mq_open`, `mq_unlink`, `mq_timedsend`, `mq_timedreceive`, `mq_notify`, + `mq_getsetattr`), since no path rule covers them; + - signals to any process but the program itself, whose process ID `exec` keeps (`kill`, + `tkill`, `tgkill`, `rt_sigqueueinfo`, `rt_tgsigqueueinfo`), so this holds on a kernel without + Landlock's signal scope; + - changes to any process but the program itself (`prlimit64`, `setpriority`, `ioprio_set`, + `sched_setaffinity`, `sched_setscheduler`, `sched_setparam`, `sched_setattr`); + - what changes a file where Landlock has no right for it, by path or by descriptor: permissions + (`chmod`, `fchmod`, `fchmodat`, `fchmodat2`), owner (`chown`, `fchown`, `lchown`, `fchownat`), + extended attributes (`setxattr`, `lsetxattr`, `fsetxattr`, `setxattrat`, `removexattr`, + `lremovexattr`, `fremovexattr`, `removexattrat`), times (`utime`, `utimes`, `futimesat`, + `utimensat`) and flags (`file_setattr`); + - `ioctl` with `TIOCSTI` or `TIOCLINUX`, which type into a terminal the program inherited, as + when `sandbox-check` runs a program from a shell; + - any `ioctl` whose number says it hands the kernel data, such as `FS_IOC_SETFLAGS` and + `FS_IOC_FSSETXATTR`, which change a file's flags on a descriptor opened only to read, and + `FS_IOC_SET_ENCRYPTION_POLICY`, whose number says it only reads. Those that ask, such as + `isatty`'s, and those that set a descriptor's own flags, such as `FIONBIO`, still work; + - mounts, namespaces and file handles (`mount`, `umount2`, `pivot_root`, `chroot`, + `unshare`, `setns`, `open_by_handle_at`, `name_to_handle_at`, the new mount API); + - `bpf`, `perf_event_open`, `userfaultfd`, keyrings, kernel modules, `kexec`, swap, + `reboot`, `syslog`, and port I/O on x86_64. + - **x32 calls (x86_64 only)** fail with `EPERM`. They report the x86_64 architecture value with + other numbers, so the rules above wouldn't match them. + + Filters are built for x86_64 and aarch64; other architectures are refused. A filter allows what it + doesn't name, so a call a later kernel adds is allowed until it is named here; the newest named is + `file_setattr` (Linux 6.17). + + Because no socket can be made, Python's `asyncio` doesn't work in the sandbox: its event loop + needs a local socket pair. + +## macOS + +The launcher runs itself again under `sandbox-exec` with a profile built from the policy: +- `(deny default)`; +- the base system profile (`system.sb`: the system libraries, the dynamic loader's cache and a few + system services a process needs to start); +- `process-exec` for the executable paths; +- `file-read*` for those and the read paths; +- file metadata (for path resolution), `sysctl` reads, and writing `/dev/null`. + +There is no rule for `process-fork`, the network or writing any file but `/dev/null`, so those stay +denied. +A local socket pair isn't refused on macOS, so `asyncio`'s event loop starts there; it still can't +reach the network. +Paths are quoted as profile strings, and a path that can't be quoted safely is refused. + +`sandbox-exec` is deprecated by Apple; support needs `/usr/bin/sandbox-exec`, and without it +`support()` reports the system as unsupported. The check above is what proves the profile holds on a +given machine. + +A framework build of Python (Homebrew, python.org) starts its real interpreter, +`Python.app/Contents/MacOS/Python`, as a new process from `bin/python3`. Inside the sandbox that +start is refused, so callers run the interpreter itself. Homebrew's Python also loads `_ssl` and +`_sqlite3`'s libraries from `opt` and `Cellar`, so a policy for it needs those folders; Homebrew's +`etc` and `var`, which can hold a person's data, stay out of it. + +## Other systems + +Windows, FreeBSD and the web build have no confinement here: `support()` reports it and +`launch_command` refuses, so code that needs the sandbox doesn't run there (Python cells show +`#BLOCKED!`). + +## Checking a machine + +```text +gridcraft-cli sandbox-check +gridcraft-cli sandbox-check --read /usr -- /usr/bin/python3 -I -c "print(1)" +``` + +Without a program it runs a confined copy of itself that tries each denied action, plus UDP and +listing the home directory, and two that must still work (a thread, reading an allowed file). It +prints one line per attempt. With a program it runs that program confined. Exit status 0 means +everything is as required, and 1 not, including a launch the launcher refused (for example on a +Linux kernel without Landlock, or older than 6.2). 2 means GridCraft has no sandbox for this kind of system at all +(Windows, FreeBSD, an architecture other than x86_64 or aarch64, macOS without `sandbox-exec`). diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 148c80aaf..0642e234a 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -29,6 +29,8 @@ pub const TABLE: &[(&str, Class)] = &[ ("core", Class::Layer(0)), ("numfmt", Class::Layer(0)), ("pdf", Class::Layer(0)), + // The confined launcher for helper processes; the Python worker and the apps use it. + ("sandbox", Class::Layer(0)), ("formula", Class::Layer(1)), ("functions", Class::Layer(1)), // Equation layout (LaTeX to glyphs, rules and paths) and OMML, on top of RaTeX. From 55f3004cb21e4e66c1bb4363867ac2d9d66c67dc Mon Sep 17 00:00:00 2001 From: Valentin Klein Date: Wed, 11 Nov 2026 08:57:06 +0100 Subject: [PATCH 22/51] Numfmt: one built-in format table, fewer allocations rendering numbers and General (#817) (#820) * Numfmt: one built-in format table, no dead render arms, fewer allocations (#817) S1: the built-in format table lives only in gridcraft-xlsx (file formats do not depend on numfmt); numfmt's unused copy and its exports are removed, xlsx gets tests pinning its lookup rules. S3: drop the render_number arm that printed what the fallback prints; literal text is appended to the output instead of built as a String per call. O3: render_number keeps pieces, text-before and literal replacements in one buffer (Parts), fill_int writes digits left to right, Section stores the date flags (sub-second digits, calendar, AM/PM) computed at parse time, render_date writes into one buffer. O4: significant_digits formats into a stack buffer; format_general_fit re-rounds the 15 digits per width into reused strings. An equality test compares format_value, TEXT() and format_general_fit with digests recorded from the previous implementation. Co-Authored-By: Claude Sonnet 5.5 * xlsx test: builtin ids 42 and 44 are written as custom formats like 5-8 Co-Authored-By: Claude Sonnet 5.5 * numfmt: push_str for short month/day names (clippy) Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5.5 --- ROADMAP.md | 2 + crates/numfmt/src/builtin.rs | 56 --- crates/numfmt/src/decimal.rs | 199 ++++++--- crates/numfmt/src/lib.rs | 2 - crates/numfmt/src/parse.rs | 43 +- crates/numfmt/src/render.rs | 330 +++++++++------ crates/numfmt/src/tests.rs | 32 +- crates/numfmt/tests/golden/codes.txt | 152 +++++++ crates/numfmt/tests/golden/general.txt | 555 +++++++++++++++++++++++++ crates/numfmt/tests/render_equality.rs | 225 ++++++++++ crates/xlsx/src/tables.rs | 23 + 11 files changed, 1346 insertions(+), 273 deletions(-) delete mode 100644 crates/numfmt/src/builtin.rs create mode 100644 crates/numfmt/tests/golden/codes.txt create mode 100644 crates/numfmt/tests/golden/general.txt create mode 100644 crates/numfmt/tests/render_equality.rs diff --git a/ROADMAP.md b/ROADMAP.md index 36f8919a3..cb4065a2c 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -199,6 +199,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | XLSX: `[Content_Types].xml`, `_rels/.rels`, the workbook rels and `xl/workbook.xml` are the first zip entries (document properties last), so libmagic and PHP `finfo` detect the file type even with large properties (#1090) | | 2026-10-13 | Conditional formatting: a rule gathers range statistics only when it reads them (top/bottom, above average, duplicates, scales, bars, icons) and borrows them instead of copying them per cell, so the per-frame lookup of an expression or cell-value rule no longer visits its whole range (#1098, part 1) | | 2026-10-12 | Next/previous sheet navigation stops at the first/last visible worksheet instead of activating a hidden boundary sheet; the current selection is retained | +| 2026-10-12 | Number formatting: one table of built-in formats (xlsx); numbers, dates and General render with one scratch buffer instead of per-token strings; output unchanged, checked against digests recorded from the old code (#817) | | 2026-10-21 | Macro engine ([macros.md](docs/macros.md)): typed, declarative macros saved in the workbook (a JSON field; hidden names in xlsx); declared inputs checked with messages that name the cells; tokens (`Input.col(2)`, `Row`, `Output`, `Name:Total`, `{{…}}` in formulas); one undo step with full rollback; a footprint dry run with ask, fail, overwrite, shift down or shift right; a policy that keeps macros away from files and the app, with approval for macros in opened files; Repeat, If and Stop steps; a debugger; a relative recorder; `macro.*` commands beside the unchanged Automate scripts | | 2026-10-13 | Charts: work and memory are bounded by series x points (2,000,000 per chart: every series keeps its first points; 16 trendlines per chart), hairline columns and bars are merged per pixel column, series ranges are clipped before they are evaluated, the moving average is linear and rendering clean data no longer copies it (#1147) | | 2026-10-13 | Charts: one combo plot-kind rule for resolving and drawing; ticks of a scale off the step grid keep their own values (#1146) | @@ -312,6 +313,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | trivial | Progress log: shared pictures written once on XLSX export (#985) | | 2026-10-13 | minor | Progress log: conditional-format lookups no longer scan the range for rules without statistics (#1098) | | 2026-10-12 | minor | Progress log: hidden-sheet navigation at workbook boundaries | +| 2026-10-12 | minor | Numfmt cleanup: one built-in format table (gridcraft-xlsx), fewer allocations rendering numbers, dates and General, equality test against the previous output (#817); progress log row | | 2026-10-12 | minor | The control server requires an auth token and closes bad connections (#777); progress log row | | 2026-10-12 | minor | Rows auto-fit oversized fonts on open and whole-row/column formatting (#833); progress log row | | 2026-10-12 | minor | Progress log: multi-area row/column delete (#758) | diff --git a/crates/numfmt/src/builtin.rs b/crates/numfmt/src/builtin.rs deleted file mode 100644 index ed03627f7..000000000 --- a/crates/numfmt/src/builtin.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Built-in number formats (ECMA-376 §18.8.30, en-US). - -const BUILTINS: [(u32, &str); 36] = [ - (0, "General"), - (1, "0"), - (2, "0.00"), - (3, "#,##0"), - (4, "#,##0.00"), - (5, "\"$\"#,##0_);\\(\"$\"#,##0\\)"), - (6, "\"$\"#,##0_);[Red]\\(\"$\"#,##0\\)"), - (7, "\"$\"#,##0.00_);\\(\"$\"#,##0.00\\)"), - (8, "\"$\"#,##0.00_);[Red]\\(\"$\"#,##0.00\\)"), - (9, "0%"), - (10, "0.00%"), - (11, "0.00E+00"), - (12, "# ?/?"), - (13, "# ??/??"), - (14, "m/d/yyyy"), - (15, "d-mmm-yy"), - (16, "d-mmm"), - (17, "mmm-yy"), - (18, "h:mm AM/PM"), - (19, "h:mm:ss AM/PM"), - (20, "h:mm"), - (21, "h:mm:ss"), - (22, "m/d/yyyy h:mm"), - (37, "#,##0 ;(#,##0)"), - (38, "#,##0 ;[Red](#,##0)"), - (39, "#,##0.00;(#,##0.00)"), - (40, "#,##0.00;[Red](#,##0.00)"), - (41, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)"), - (42, "_(\"$\"* #,##0_);_(\"$\"* \\(#,##0\\);_(\"$\"* \"-\"_);_(@_)"), - (43, "_(* #,##0.00_);_(* \\(#,##0.00\\);_(* \"-\"??_);_(@_)"), - (44, "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"), - (45, "mm:ss"), - (46, "[h]:mm:ss"), - (47, "mmss.0"), - (48, "##0.0E+0"), - (49, "@"), -]; - -/// Built-in format code for an id (0..=49, en-US), e.g. 14 => "m/d/yyyy". Locale-specific ids -/// (23–36) and unknown ids return `None`. -pub fn builtin_format(id: u32) -> Option<&'static str> { - BUILTINS.iter().find(|(i, _)| *i == id).map(|(_, c)| *c) -} - -/// Id of a built-in format code. Matching ignores case and backslash escapes. -pub fn builtin_id(code: &str) -> Option { - if let Some((id, _)) = BUILTINS.iter().find(|(_, c)| *c == code) { - return Some(*id); - } - let norm = |s: &str| s.chars().filter(|&c| c != '\\').flat_map(char::to_lowercase).collect::(); - let n = norm(code); - BUILTINS.iter().find(|(_, c)| norm(c) == n).map(|(id, _)| *id) -} diff --git a/crates/numfmt/src/decimal.rs b/crates/numfmt/src/decimal.rs index b6c831f71..80fb23e32 100644 --- a/crates/numfmt/src/decimal.rs +++ b/crates/numfmt/src/decimal.rs @@ -4,6 +4,8 @@ //! digits and then rounded half away from zero on that decimal string. That makes 2.675 with //! "0.00" show 2.68 and 1.005 show 1.01, as in Excel, even though neither is exact in binary. +use std::fmt::Write as _; + /// Most decimals we ever compute; placeholders beyond this are padded with zeros. pub(crate) const MAX_DECIMALS: usize = 400; @@ -21,7 +23,8 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { } // 16 digits, rounded. Unless the 16th is a 5 it decides the 15th exactly (4 or less rounds // down, 6 or more up), so one format serves almost every number. - let (d16, exp) = significant_digits(a, 16); + let mut d16 = [0u8; 16]; + let exp = significant_digits(a, &mut d16); for (slot, d) in digits.iter_mut().zip(&d16) { *slot = *d; } @@ -30,11 +33,8 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { 5 if is_exact_tie(a) => (digits, exp), 5 => { // Not a tie: round the exact value (the 16-digit rounding above may be a double rounding). - let (d15, exp15) = significant_digits(a, 15); let mut out = [0u8; 15]; - for (slot, d) in out.iter_mut().zip(&d15) { - *slot = *d; - } + let exp15 = significant_digits(a, &mut out); (out, exp15) } _ => { @@ -59,12 +59,54 @@ pub(crate) fn decimal_digits(a: f64) -> ([u8; 15], i32) { } } -/// `n` significant digits of `a` (rounded half to even on the exact binary value) and the power of -/// ten of the first. -fn significant_digits(a: f64, n: usize) -> (Vec, i32) { - let s = format!("{:.*e}", n.saturating_sub(1), a); - let (mant, exp) = s.split_once('e').unwrap_or((s.as_str(), "0")); - (mant.bytes().filter(u8::is_ascii_digit).map(|b| b - b'0').collect(), exp.parse().unwrap_or(0)) +/// Writes `out.len()` significant digits of `a` (rounded half to even on the exact binary value) +/// into `out` and returns the power of ten of the first. +/// +/// Up to 32 digits are formatted in a stack buffer; more (the 800 of the tie check) use a string. +fn significant_digits(a: f64, out: &mut [u8]) -> i32 { + fn parse(s: &str, out: &mut [u8]) -> i32 { + let (mant, exp) = s.split_once('e').unwrap_or((s, "0")); + let digits = mant.bytes().filter(u8::is_ascii_digit).map(|b| b - b'0'); + for (slot, d) in out.iter_mut().zip(digits) { + *slot = d; + } + exp.parse().unwrap_or(0) + } + let prec = out.len().saturating_sub(1); + if out.len() <= 32 { + let mut buf = StackStr::default(); + if write!(buf, "{:.*e}", prec, a).is_ok() { + return parse(buf.as_str(), out); + } + } + parse(&format!("{:.*e}", prec, a), out) +} + +/// A fixed-size string buffer for short number formatting; a write that does not fit fails. +struct StackStr { + buf: [u8; 64], + len: usize, +} + +impl Default for StackStr { + fn default() -> Self { + StackStr { buf: [0; 64], len: 0 } + } +} + +impl StackStr { + fn as_str(&self) -> &str { + self.buf.get(..self.len).and_then(|b| std::str::from_utf8(b).ok()).unwrap_or("") + } +} + +impl std::fmt::Write for StackStr { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + let end = self.len.checked_add(s.len()).ok_or(std::fmt::Error)?; + self.buf.get_mut(self.len..end).ok_or(std::fmt::Error)?.copy_from_slice(s.as_bytes()); + self.len = end; + Ok(()) + } } /// True when `a`'s exact decimal expansion has a 5 as its 16th significant digit and only zeros after. @@ -72,27 +114,59 @@ fn is_exact_tie(a: f64) -> bool { let is_tie = |ds: &[u8]| ds.get(15) == Some(&5) && ds.get(16..).is_some_and(|t| t.iter().all(|&d| d == 0)); // A cheap screen first (30 digits), then the exact expansion: a double has at most 767 // significant digits and Rust prints every one of them. - is_tie(&significant_digits(a, 30).0) && is_tie(&significant_digits(a, 800).0) + let mut screen = [0u8; 30]; + significant_digits(a, &mut screen); + if !is_tie(&screen) { + return false; + } + let mut exact = vec![0u8; 800]; + significant_digits(a, &mut exact); + is_tie(&exact) } /// Rounds the number `digits × 10^exp` (see [`decimal_digits`]) to `decimals` places, half away /// from zero. Returns the integer digits ("" when the integer part is zero) and exactly /// `decimals` fraction digits. pub(crate) fn fixed_from_digits(digits: &[u8; 15], exp: i32, decimals: usize) -> (String, String) { + let (mut int, mut frac) = (String::new(), String::new()); + fixed_into(digits, exp, decimals, &mut int, &mut frac); + (int, frac) +} + +/// [`fixed_from_digits`] into caller-owned strings (cleared first), so a loop can reuse them. +/// +/// The rounded digits are `lead` (at most 16, a carry can add a leading 1) followed by `zeros` +/// zero digits (placeholders beyond the 15 known digits), which is all a `Vec` of them held. +fn fixed_into(digits: &[u8; 15], exp: i32, decimals: usize, int: &mut String, frac: &mut String) { + int.clear(); + frac.clear(); let decimals_calc = decimals.min(MAX_DECIMALS); let keep = exp as i64 + decimals_calc as i64 + 1; - let mut m: Vec = if keep <= 0 { - if keep == 0 && digits.first().copied().unwrap_or(0) >= 5 { vec![1] } else { Vec::new() } + let mut head = [0u8; 16]; + let mut head_len = 0usize; + let mut zeros = 0usize; + if keep <= 0 { + if keep == 0 && digits.first().copied().unwrap_or(0) >= 5 { + if let Some(h) = head.first_mut() { + *h = 1; + } + head_len = 1; + } } else if keep as usize >= digits.len() { - let mut v = digits.to_vec(); - v.resize(keep as usize, 0); - v + for (h, d) in head.iter_mut().zip(digits) { + *h = *d; + } + head_len = digits.len(); + zeros = keep as usize - digits.len(); } else { let keep = keep as usize; - let mut v = digits.get(..keep).map(<[u8]>::to_vec).unwrap_or_default(); + for (h, d) in head.iter_mut().zip(digits.iter().take(keep)) { + *h = *d; + } + head_len = keep; if digits.get(keep).copied().unwrap_or(0) >= 5 { let mut carry = true; - for d in v.iter_mut().rev() { + for d in head.iter_mut().take(keep).rev() { if *d == 9 { *d = 0; } else { @@ -102,26 +176,36 @@ pub(crate) fn fixed_from_digits(digits: &[u8; 15], exp: i32, decimals: usize) -> } } if carry { - v.insert(0, 1); + head.copy_within(0..keep, 1); + if let Some(h) = head.first_mut() { + *h = 1; + } + head_len += 1; } } - v - }; - let lead = m.iter().take_while(|&&d| d == 0).count(); - m.drain(..lead); - let to_s = |ds: &[u8]| ds.iter().map(|d| char::from(b'0' + d)).collect::(); - let (int, mut frac) = if m.len() <= decimals_calc { - let mut f = "0".repeat(decimals_calc - m.len()); - f.push_str(&to_s(&m)); - (String::new(), f) - } else { - let split = m.len() - decimals_calc; - (to_s(m.get(..split).unwrap_or(&[])), to_s(m.get(split..).unwrap_or(&[]))) - }; - if decimals > decimals_calc { - frac.push_str(&"0".repeat(decimals - decimals_calc)); } - (int, frac) + let total = head_len + zeros; + let mut lead = head.iter().take(head_len).take_while(|&&d| d == 0).count(); + if lead == head_len { + lead = total; // all zeros + } + let len = total - lead; + // Digit `k` of the rounded number after its leading zeros. + let digit_at = |k: usize| { + let pos = lead + k; + if pos < head_len { head.get(pos).map_or('0', |&d| char::from(b'0' + d)) } else { '0' } + }; + if len <= decimals_calc { + frac.extend(std::iter::repeat_n('0', decimals_calc - len)); + frac.extend((0..len).map(digit_at)); + } else { + let split = len - decimals_calc; + int.extend((0..split).map(digit_at)); + frac.extend((split..len).map(digit_at)); + } + if decimals > decimals_calc { + frac.extend(std::iter::repeat_n('0', decimals - decimals_calc)); + } } /// Rounds a non-negative number to `decimals` places (see [`fixed_from_digits`]). @@ -148,21 +232,25 @@ pub fn format_general_fit(n: f64, max_chars: usize) -> Option { let sign = if n < 0.0 { "-" } else { "" }; let (digits, x) = decimal_digits(n.abs()); let int_len = sign.len() + (x.max(0) as usize) + 1; - // Fixed notation for magnitudes from 1E-4 up to what fits. + // Fixed notation for magnitudes from 1E-4 up to what fits. The 15 digits are computed once; + // each width only re-rounds them, into strings that are reused. + let (mut i, mut f, mut s) = (String::new(), String::new(), String::new()); if x >= -4 && int_len <= max_chars { let avail = max_chars - int_len; - let mut decimals = if avail >= 2 { avail - 1 } else { 0 }; + // Zeros past the 400th decimal are trimmed below anyway, so a huge width needs no more. + let mut decimals = (if avail >= 2 { avail - 1 } else { 0 }).min(MAX_DECIMALS); loop { - let (i, f) = fixed_from_digits(&digits, x, decimals); - let f = f.trim_end_matches('0'); - if i.is_empty() && f.is_empty() { + fixed_into(&digits, x, decimals, &mut i, &mut f); + let ft = f.trim_end_matches('0'); + if i.is_empty() && ft.is_empty() { break; // rounded away to zero: use scientific } - let mut s = String::from(sign); + s.clear(); + s.push_str(sign); s.push_str(if i.is_empty() { "0" } else { &i }); - if !f.is_empty() { + if !ft.is_empty() { s.push('.'); - s.push_str(f); + s.push_str(ft); } if s.chars().count() <= max_chars { return Some(s); @@ -175,19 +263,20 @@ pub fn format_general_fit(n: f64, max_chars: usize) -> Option { } for md in (0..=max_chars.min(14)).rev() { let mut e = x; - let (mut mi, mut mf) = fixed_from_digits(&digits, 0, md); - if mi.len() > 1 { + fixed_into(&digits, 0, md, &mut i, &mut f); + if i.len() > 1 { e += 1; - (mi, mf) = fixed_from_digits(&digits, -1, md); + fixed_into(&digits, -1, md, &mut i, &mut f); } - let mf = mf.trim_end_matches('0'); - let mut s = String::from(sign); - s.push_str(if mi.is_empty() { "0" } else { &mi }); - if !mf.is_empty() { + let ft = f.trim_end_matches('0'); + s.clear(); + s.push_str(sign); + s.push_str(if i.is_empty() { "0" } else { &i }); + if !ft.is_empty() { s.push('.'); - s.push_str(mf); + s.push_str(ft); } - s.push_str(&format!("E{}{:02}", if e < 0 { '-' } else { '+' }, e.unsigned_abs())); + let _ = write!(s, "E{}{:02}", if e < 0 { '-' } else { '+' }, e.unsigned_abs()); if s.chars().count() <= max_chars { return Some(s); } @@ -211,6 +300,10 @@ mod tests { assert_eq!(fixed(0.0005, 3), ("".into(), "001".into())); assert_eq!(fixed(1e20, 0).0, "100000000000000000000"); assert_eq!(fixed(0.0, 2), ("".into(), "00".into())); + // Past 400 decimals only zeros are added; huge exponents keep all their digits. + assert_eq!(fixed(1.5, 450), ("1".into(), format!("5{}", "0".repeat(449)))); + assert_eq!(fixed(1e300, 1), (format!("1{}", "0".repeat(300)), "0".into())); + assert_eq!(fixed(123456789012345678.0, 0).0, "123456789012346000"); } #[test] diff --git a/crates/numfmt/src/lib.rs b/crates/numfmt/src/lib.rs index 57384cc89..b41e1bb59 100644 --- a/crates/numfmt/src/lib.rs +++ b/crates/numfmt/src/lib.rs @@ -3,7 +3,6 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] -mod builtin; mod calendar; mod decimal; mod parse; @@ -13,7 +12,6 @@ use std::sync::Arc; use gridcraft_core::{CellError, DateSystem, Value}; -pub use builtin::{builtin_format, builtin_id}; pub use decimal::format_general_fit; use parse::{DatePart, SecKind, Section, Tok}; diff --git a/crates/numfmt/src/parse.rs b/crates/numfmt/src/parse.rs index c2b121940..e9804b5c7 100644 --- a/crates/numfmt/src/parse.rs +++ b/crates/numfmt/src/parse.rs @@ -60,18 +60,28 @@ pub(crate) enum Tok { } impl Tok { + /// Appends the text this token shows when it has no special meaning in its section. + pub(crate) fn push_literal(&self, out: &mut String) { + match self { + Tok::Lit(s) | Tok::FixedDenom(s) => out.push_str(s), + Tok::Digit(c) => out.push(*c), + Tok::Point => out.push('.'), + Tok::Comma => out.push(','), + Tok::Percent => out.push('%'), + Tok::Exp { upper, plus } => { + out.push(if *upper { 'E' } else { 'e' }); + out.push(if *plus { '+' } else { '-' }); + } + Tok::Slash => out.push('/'), + Tok::Fill(_) | Tok::At | Tok::General | Tok::Date(_) => {} + } + } + /// The text this token shows when it has no special meaning in its section. pub(crate) fn literal_text(&self) -> String { - match self { - Tok::Lit(s) | Tok::FixedDenom(s) => s.clone(), - Tok::Digit(c) => c.to_string(), - Tok::Point => ".".into(), - Tok::Comma => ",".into(), - Tok::Percent => "%".into(), - Tok::Exp { upper, plus } => format!("{}{}", if *upper { 'E' } else { 'e' }, if *plus { '+' } else { '-' }), - Tok::Slash => "/".into(), - Tok::Fill(_) | Tok::At | Tok::General | Tok::Date(_) => String::new(), - } + let mut s = String::new(); + self.push_literal(&mut s); + s } } @@ -128,6 +138,13 @@ pub(crate) struct Section { pub fraction: bool, pub currency: bool, pub fill: bool, + /// Date sections: the most digits of any `.0`/`.00`/`.000` part (at most 3), which decides how + /// the time of day is rounded. + pub sub_digits: u8, + /// Date sections: has a year, month or day part, so the serial's date is needed. + pub needs_date: bool, + /// Date sections: has `AM/PM` or `A/P`, so hours show on the 12-hour clock. + pub ampm: bool, pub calendar: Calendar, /// Excel rejects the whole code because of this section: a `B` and digit that is not a /// calendar prefix, or a prefix before a number format (`B20.00`). @@ -515,6 +532,9 @@ fn analyze(raw: Raw) -> Section { fraction: false, currency, fill: toks.iter().any(|t| matches!(t, Tok::Fill(_))), + sub_digits: 0, + needs_date: false, + ampm: false, calendar, invalid, }; @@ -573,6 +593,9 @@ fn analyze_date(mut toks: Vec, mut sec: Section) -> Section { } i += 1; } + sec.sub_digits = out.iter().filter_map(|t| if let Tok::Date(DatePart::SubSec(n)) = t { Some(*n) } else { None }).max().unwrap_or(0).min(3); + sec.needs_date = out.iter().any(|t| matches!(t, Tok::Date(d) if d.is_calendar())); + sec.ampm = out.iter().any(|t| matches!(t, Tok::Date(DatePart::AmPm | DatePart::AP(..)))); sec.toks = out; sec } diff --git a/crates/numfmt/src/render.rs b/crates/numfmt/src/render.rs index bfd31e56c..19864cdc5 100644 --- a/crates/numfmt/src/render.rs +++ b/crates/numfmt/src/render.rs @@ -1,5 +1,7 @@ //! Rendering a value with a parsed section. +use std::fmt::Write as _; + use gridcraft_core::DateSystem; use gridcraft_core::date::{datetime_from_serial, max_serial}; @@ -25,6 +27,10 @@ impl Out { self.fill = Some((c, self.text.len())); } } + /// Appends what `t` shows when it has no special meaning (no temporary string). + fn push_tok(&mut self, t: &Tok) { + t.push_literal(&mut self.text); + } pub(crate) fn prepend(&mut self, s: &str) { self.text.insert_str(0, s); if let Some((_, p)) = &mut self.fill { @@ -40,7 +46,7 @@ pub(crate) fn render_text(sec: &Section, text: &str) -> Out { match t { Tok::At => out.push(text), Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } out @@ -53,7 +59,7 @@ pub(crate) fn render_general(sec: &Section, v: f64, width: usize) -> Out { match t { Tok::General => out.push(&format_general_fit(v, width).unwrap_or_else(|| "#".repeat(width.clamp(1, 11)))), Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } out @@ -64,8 +70,7 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option if !v.is_finite() || !(0.0..max_serial(sys) as f64 + 1.0).contains(&v) { return None; } - let sub_digits = sec.toks.iter().filter_map(|t| if let Tok::Date(DatePart::SubSec(n)) = t { Some(*n) } else { None }).max().unwrap_or(0).min(3); - let unit: i64 = 10i64.pow(3 - sub_digits as u32); + let unit: i64 = 10i64.pow(3 - u32::from(sec.sub_digits.min(3))); let mut day = v.floor() as i64; let mut ms = (((v - v.floor()) * 86_400_000.0) / unit as f64).round() as i64 * unit; if ms >= 86_400_000 { @@ -82,38 +87,33 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option { return None; } - let needs_date = sec.toks.iter().any(|t| matches!(t, Tok::Date(d) if d.is_calendar())); - let dt = if needs_date { datetime_from_serial(sys, day as f64)? } else { datetime_from_serial(sys, 0.0).unwrap_or_default() }; - let ampm = sec.toks.iter().any(|t| matches!(t, Tok::Date(DatePart::AmPm | DatePart::AP(..)))); + let dt = if sec.needs_date { datetime_from_serial(sys, day as f64)? } else { datetime_from_serial(sys, 0.0).unwrap_or_default() }; + let ampm = sec.ampm; let hour = ms / 3_600_000; let minute = (ms / 60_000) % 60; let second = (ms / 1000) % 60; let milli = ms % 1000; - let pad = |n: i64, w: u8| if w >= 2 { format!("{:0w$}", n, w = w.min(20) as usize) } else { n.to_string() }; // No date in the section's calendar (serial 60 under `B2`): shown like an out-of-range date. let date = sec.calendar.date(&dt)?; let mut out = Out::default(); for t in &sec.toks { match t { Tok::Date(d) => { - let s = match *d { + let text = &mut out.text; + match *d { DatePart::Year(n) => { - if n <= 2 { - format!("{:02}", date.year.rem_euclid(100)) - } else { - format!("{:04}", date.year) - } + let _ = if n <= 2 { write!(text, "{:02}", date.year.rem_euclid(100)) } else { write!(text, "{:04}", date.year) }; } DatePart::Month(n) => match n { - 1 | 2 => pad(date.month as i64, n), - 3 => date.month_short.to_string(), - 5 => date.month_name.chars().take(1).collect(), - _ => date.month_name.to_string(), + 1 | 2 => push_padded(text, date.month as i64, n), + 3 => text.push_str(date.month_short), + 5 => text.extend(date.month_name.chars().take(1)), + _ => text.push_str(date.month_name), }, DatePart::Day(n) => match n { - 1 | 2 => pad(date.day as i64, n), - 3 => date.day_short.to_string(), - _ => date.day_name.to_string(), + 1 | 2 => push_padded(text, date.day as i64, n), + 3 => text.push_str(date.day_short), + _ => text.push_str(date.day_name), }, DatePart::Hour(n) => { let h = if ampm { @@ -122,78 +122,168 @@ pub(crate) fn render_date(sec: &Section, v: f64, sys: DateSystem) -> Option } else { hour }; - pad(h, n) + push_padded(text, h, n); } - DatePart::Minute(n) => pad(minute, n), - DatePart::Second(n) => pad(second, n), + DatePart::Minute(n) => push_padded(text, minute, n), + DatePart::Second(n) => push_padded(text, second, n), DatePart::SubSec(n) => { - let mut s = format!(".{:03}", milli); - s.truncate(1 + (n as usize).min(3)); - if n > 3 { - s.push_str(&"0".repeat(n as usize - 3)); + // `.` and the first `n` (at most 3) of the three millisecond digits, then + // zeros for the digits beyond milliseconds. + text.push('.'); + for (k, div) in [100, 10, 1].into_iter().enumerate() { + if k < n as usize { + text.push(char::from(b'0' + (milli / div % 10) as u8)); + } + } + for _ in 3..n { + text.push('0'); } - s } - DatePart::ElapsedH(n) => pad(total_ms / 3_600_000, n), - DatePart::ElapsedM(n) => pad(total_ms / 60_000, n), - DatePart::ElapsedS(n) => pad(total_ms / 1000, n), - DatePart::AmPm => (if hour >= 12 { "PM" } else { "AM" }).to_string(), - DatePart::AP(a, p) => (if hour >= 12 { p } else { a }).to_string(), - }; - out.push(&s); + DatePart::ElapsedH(n) => push_padded(text, total_ms / 3_600_000, n), + DatePart::ElapsedM(n) => push_padded(text, total_ms / 60_000, n), + DatePart::ElapsedS(n) => push_padded(text, total_ms / 1000, n), + DatePart::AmPm => text.push_str(if hour >= 12 { "PM" } else { "AM" }), + DatePart::AP(a, p) => text.push(if hour >= 12 { p } else { a }), + } } Tok::Fill(c) => out.fill(*c), - other => out.push(&other.literal_text()), + other => out.push_tok(other), } } Some(out) } +/// Appends `n`, zero padded to `w` digits (at most 20) when `w` is 2 or more. +fn push_padded(out: &mut String, n: i64, w: u8) { + let _ = if w >= 2 { write!(out, "{:0w$}", n, w = w.min(20) as usize) } else { write!(out, "{n}") }; +} + +/// Scratch text of one number rendering: every token's own output (`piece`, replaces the token's +/// text), the text inserted ahead of a token (`before`) and the replacement of a literal +/// (`lit_spaces`: fractions blank the literals of a hidden part) live in ONE buffer, addressed +/// by byte ranges, instead of one string per token. Writing a piece again appends the new text +/// and points the slot at it. +struct Parts { + buf: String, + slots: Vec, +} + +#[derive(Clone, Copy, Default)] +struct Slot { + piece: (usize, usize), + before: (usize, usize), + /// `Some(k)`: show `k` spaces instead of the literal. + lit_spaces: Option, +} + +impl Parts { + fn new(n: usize) -> Parts { + Parts { buf: String::new(), slots: vec![Slot::default(); n + 1] } + } + /// Start of a piece written straight into the buffer; finish it with [`Parts::end_piece`]. + fn mark(&self) -> usize { + self.buf.len() + } + fn end_piece(&mut self, ti: usize, start: usize) { + let end = self.buf.len(); + if let Some(sl) = self.slots.get_mut(ti) { + sl.piece = (start, end); + } + } + fn set_piece(&mut self, ti: usize, text: &str) { + let start = self.mark(); + self.buf.push_str(text); + self.end_piece(ti, start); + } + fn set_spaces(&mut self, ti: usize, k: usize) { + let start = self.mark(); + self.buf.extend(std::iter::repeat_n(' ', k)); + self.end_piece(ti, start); + } + fn set_before(&mut self, ti: usize, text: &str) { + let start = self.mark(); + self.buf.push_str(text); + let end = self.buf.len(); + if let Some(sl) = self.slots.get_mut(ti) { + sl.before = (start, end); + } + } + fn set_lit_spaces(&mut self, ti: usize, k: usize) { + if let Some(sl) = self.slots.get_mut(ti) { + sl.lit_spaces = Some(k); + } + } + fn text(&self, (a, b): (usize, usize)) -> &str { + self.buf.get(a..b).unwrap_or("") + } + fn piece(&self, ti: usize) -> &str { + self.slots.get(ti).map_or("", |sl| self.text(sl.piece)) + } + fn before(&self, ti: usize) -> &str { + self.slots.get(ti).map_or("", |sl| self.text(sl.before)) + } +} + /// Assigns integer digits right-to-left to the placeholders at `idxs`; the leftmost placeholder /// takes all remaining digits. `force_zero` shows `0` in the units placeholder when there are no /// digits. A blank `?` pads with a space, and so does the thousands separator it would carry. -fn fill_int(toks: &[Tok], idxs: &[usize], digits: &str, thousands: bool, force_zero: bool, pieces: &mut [String]) { - let ds: Vec = digits.chars().collect(); +/// +/// `digits` is ASCII (digit strings made by this crate), so bytes are characters. Each +/// placeholder's text is written left to right: a digit, then the separator that follows it when +/// its position counted from the right is a multiple of 3. +fn fill_int(toks: &[Tok], idxs: &[usize], digits: &str, thousands: bool, force_zero: bool, parts: &mut Parts) { + let ds = digits.as_bytes(); let mut rem = ds.len(); let mut pos = 0usize; let count = idxs.len(); + let sep = |p: usize| thousands && p > 0 && p.is_multiple_of(3); for (k, &ti) in idxs.iter().rev().enumerate() { let leftmost = k + 1 == count; let ph = match toks.get(ti) { Some(Tok::Digit(c)) => *c, _ => '0', }; - let mut rev = String::new(); - let push_digit = |c: char, rev: &mut String, pos: &mut usize| { - if thousands && *pos > 0 && pos.is_multiple_of(3) { - rev.push(','); - } - rev.push(c); - *pos += 1; - }; + let start = parts.mark(); let take = if leftmost { rem } else { rem.min(1) }; if take == 0 { match ph { - '0' => push_digit('0', &mut rev, &mut pos), - _ if force_zero && k == 0 => push_digit('0', &mut rev, &mut pos), + '0' => { + parts.buf.push('0'); + if sep(pos) { + parts.buf.push(','); + } + pos += 1; + } + _ if force_zero && k == 0 => { + parts.buf.push('0'); + if sep(pos) { + parts.buf.push(','); + } + pos += 1; + } '?' => { - if thousands && pos > 0 && pos.is_multiple_of(3) { - rev.push(' '); + parts.buf.push(' '); + if sep(pos) { + parts.buf.push(' '); } - rev.push(' '); pos += 1; } _ => {} } } else { - for _ in 0..take { - rem -= 1; - push_digit(ds.get(rem).copied().unwrap_or('0'), &mut rev, &mut pos); + // The `take` digits just left of `rem`, most significant first; the digit `s` places + // from the right end of the run sits at position `pos + s`. + for s in (0..take).rev() { + let d = ds.get(rem.saturating_sub(1 + s)).copied().unwrap_or(b'0'); + parts.buf.push(char::from(d)); + if sep(pos + s) { + parts.buf.push(','); + } } + rem -= take; + pos += take; } - if let Some(p) = pieces.get_mut(ti) { - *p = rev.chars().rev().collect(); - } + parts.end_piece(ti, start); } } @@ -213,26 +303,26 @@ fn with_commas(digits: &str, thousands: bool) -> String { } /// Fraction-digit placeholders: trailing zeros show as `0`, nothing (`#`) or a space (`?`). -fn fill_frac(toks: &[Tok], idxs: &[usize], frac: &str, pieces: &mut [String]) { - let fd: Vec = frac.chars().collect(); - let last_sig = fd.iter().rposition(|&c| c != '0'); +/// `frac` is ASCII digits. +fn fill_frac(toks: &[Tok], idxs: &[usize], frac: &str, parts: &mut Parts) { + let fd = frac.as_bytes(); + let last_sig = fd.iter().rposition(|&c| c != b'0'); for (i, &ti) in idxs.iter().enumerate() { let ph = match toks.get(ti) { Some(Tok::Digit(c)) => *c, _ => '0', }; - let s = if last_sig.is_some_and(|l| i <= l) { - fd.get(i).copied().unwrap_or('0').to_string() + let start = parts.mark(); + if last_sig.is_some_and(|l| i <= l) { + parts.buf.push(char::from(fd.get(i).copied().unwrap_or(b'0'))); } else { match ph { - '0' => "0".into(), - '?' => " ".into(), - _ => String::new(), + '0' => parts.buf.push('0'), + '?' => parts.buf.push(' '), + _ => {} } - }; - if let Some(p) = pieces.get_mut(ti) { - *p = s; } + parts.end_piece(ti, start); } } @@ -347,15 +437,12 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { } let toks = &sec.toks; let n = toks.len(); - // Pieces replace the token's own output; `before` is inserted ahead of a token; `lits` - // replaces a literal's text (fractions blank the literals of a hidden part). - let mut pieces: Vec = vec![String::new(); n]; - let mut before: Vec = vec![String::new(); n + 1]; - let mut lits: Vec> = Vec::new(); + // Pieces replace the token's own output; `before` is inserted ahead of a token; literal + // replacements blank the literals of a hidden fraction part (see [`Parts`]). + let mut parts = Parts::new(n); let mut zero = false; if sec.fraction { - lits = vec![None; n]; - render_fraction(sec, x, &mut pieces, &mut lits); + render_fraction(sec, x, &mut parts); } else { let NumberLayout { point, exp, int_idx, frac_idx } = number_layout(toks); let (int_s, frac_s) = if let Some(e_at) = exp { @@ -369,7 +456,6 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { if int_s.is_empty() && frac_s.chars().all(|c| c == '0') { e = 0; } - let head = if upper { "E" } else { "e" }; let sign = if e < 0 { "-" } else if plus { @@ -382,18 +468,16 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { // `E` (observed: `|0|e+|0|` shows `|1|e|+5|`); `?` pads the digits with spaces. match exp_idx.first() { Some(&first) => { - if let Some(p) = pieces.get_mut(e_at) { - *p = head.to_string(); - } - if let Some(b) = before.get_mut(first) { - *b = sign.to_string(); - } - fill_int(toks, &exp_idx, &digits, false, false, &mut pieces); + parts.set_piece(e_at, if upper { "E" } else { "e" }); + parts.set_before(first, sign); + fill_int(toks, &exp_idx, &digits, false, false, &mut parts); } None => { - if let Some(p) = pieces.get_mut(e_at) { - *p = format!("{head}{sign}{digits}"); - } + let start = parts.mark(); + parts.buf.push(if upper { 'E' } else { 'e' }); + parts.buf.push_str(sign); + parts.buf.push_str(&digits); + parts.end_piece(e_at, start); } } (int_s, frac_s) @@ -401,32 +485,32 @@ pub(crate) fn render_number(sec: &Section, v: f64) -> Out { fixed(x, frac_idx.len()) }; zero = int_s.chars().chain(frac_s.chars()).all(|c| c == '0'); - fill_int(toks, &int_idx, &int_s, sec.thousands, false, &mut pieces); + fill_int(toks, &int_idx, &int_s, sec.thousands, false, &mut parts); if int_idx.is_empty() && !int_s.is_empty() && let Some(p) = point.or(exp) - && let Some(b) = before.get_mut(p) { - *b = with_commas(&int_s, sec.thousands); + parts.set_before(p, &with_commas(&int_s, sec.thousands)); } - fill_frac(toks, &frac_idx, &frac_s, &mut pieces); - if let Some(p) = point - && let Some(s) = pieces.get_mut(p) - { - *s = ".".into(); + fill_frac(toks, &frac_idx, &frac_s, &mut parts); + if let Some(p) = point { + parts.set_piece(p, "."); } } let mut out = Out::default(); for (i, t) in toks.iter().enumerate() { - if let Some(b) = before.get(i) { - out.push(b); - } + out.push(parts.before(i)); match t { - Tok::Lit(s) => out.push(lits.get(i).and_then(Option::as_deref).unwrap_or(s)), + Tok::Lit(s) => match parts.slots.get(i).and_then(|sl| sl.lit_spaces) { + Some(k) => out.text.extend(std::iter::repeat_n(' ', k)), + None => out.push(s), + }, Tok::Percent => out.push("%"), Tok::Fill(c) => out.fill(*c), - Tok::At | Tok::General | Tok::Date(_) | Tok::Comma => {} - _ => out.push(pieces.get(i).map(String::as_str).unwrap_or("")), + // Every other token shows its piece. `At` (`0@`) is reachable here, `Comma`, `General` + // and `Date` are not (analysis turns commas into literals and routes sections with + // dates or General elsewhere); none of them is ever given a piece, so they print "". + _ => out.push(parts.piece(i)), } } out.zero = zero; @@ -482,22 +566,14 @@ fn shown_width(t: &Tok) -> usize { } /// Replaces the literals in `range` with spaces (`spaces`) or nothing; placeholders and the bar -/// in it get the same treatment through `pieces`. -fn blank_range(toks: &[Tok], range: std::ops::Range, spaces: bool, pieces: &mut [String], lits: &mut [Option]) { +/// in it get the same treatment as pieces. +fn blank_range(toks: &[Tok], range: std::ops::Range, spaces: bool, parts: &mut Parts) { for i in range { let Some(t) = toks.get(i) else { break }; - let s = if spaces { " ".repeat(shown_width(t)) } else { String::new() }; + let width = if spaces { shown_width(t) } else { 0 }; match t { - Tok::Lit(_) => { - if let Some(l) = lits.get_mut(i) { - *l = Some(s); - } - } - Tok::Digit(_) | Tok::Slash | Tok::FixedDenom(_) => { - if let Some(p) = pieces.get_mut(i) { - *p = s; - } - } + Tok::Lit(_) => parts.set_lit_spaces(i, width), + Tok::Digit(_) | Tok::Slash | Tok::FixedDenom(_) => parts.set_spaces(i, width), _ => {} } } @@ -581,7 +657,7 @@ fn fraction_parts(lay: &FractionLayout, x: f64) -> (f64, f64, u64) { /// becomes spaces when the whole part or the numerator has a `?`); when the fraction is zero and /// the numerator has no `0` placeholder, the separator, numerator, bar and denominator disappear /// (or become spaces when any part has a `?`) and the whole part shows at least `0`. -fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Option]) { +fn render_fraction(sec: &Section, x: f64, parts: &mut Parts) { let toks = &sec.toks; let Some(lay) = fraction_layout(toks) else { return }; let (whole, num, den) = fraction_parts(&lay, x); @@ -591,32 +667,28 @@ fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Opt let whole_s = if whole >= 1.0 { fixed(whole, 0).0 } else { String::new() }; let num_zero_ph = num_idx.iter().any(|&i| toks.get(i) == Some(&Tok::Digit('0'))); if has_whole && num == 0.0 && !num_zero_ph { - fill_int(toks, &whole_idx, &whole_s, sec.thousands, true, pieces); + fill_int(toks, &whole_idx, &whole_s, sec.thousands, true, parts); let spaces = has_q(&whole_idx) || has_q(&num_idx) || has_q(&den_idx); - blank_range(toks, sep_start..den_end, spaces, pieces, lits); + blank_range(toks, sep_start..den_end, spaces, parts); return; } if has_whole { // A zero value shows `0` in a `?` or `0` units place, nothing in a `#` one. let units_hash = whole_idx.last().is_some_and(|&i| toks.get(i) == Some(&Tok::Digit('#'))); let force = whole_s.is_empty() && num == 0.0 && !units_hash; - fill_int(toks, &whole_idx, &whole_s, sec.thousands, force, pieces); - let whole_shown = whole_idx.iter().any(|&i| pieces.get(i).is_some_and(|p| p.chars().any(|c| c.is_ascii_digit()))); + fill_int(toks, &whole_idx, &whole_s, sec.thousands, force, parts); + let whole_shown = whole_idx.iter().any(|&i| parts.piece(i).chars().any(|c| c.is_ascii_digit())); if !whole_shown { let spaces = has_q(&whole_idx) || has_q(&num_idx); - blank_range(toks, sep_start..num_start, spaces, pieces, lits); + blank_range(toks, sep_start..num_start, spaces, parts); } } let num_s = if num < 1e15 { (num as u64).to_string() } else { fixed(num, 0).0 }; - fill_int(toks, &num_idx, &num_s, false, true, pieces); - if let Some(p) = pieces.get_mut(slash) { - *p = "/".into(); - } + fill_int(toks, &num_idx, &num_s, false, true, parts); + parts.set_piece(slash, "/"); let den_s = den.to_string(); if fixed_den.is_some() { - if let Some(p) = pieces.get_mut(den_start) { - *p = den_s; - } + parts.set_piece(den_start, &den_s); } else { // Denominator digits are zero padded to the `0` placeholders, then left aligned; `?` // pads with spaces after them. @@ -641,9 +713,7 @@ fn render_fraction(sec: &Section, x: f64, pieces: &mut [String], lits: &mut [Opt } else { s }; - if let Some(p) = pieces.get_mut(ti) { - *p = s; - } + parts.set_piece(ti, &s); } } } diff --git a/crates/numfmt/src/tests.rs b/crates/numfmt/src/tests.rs index 0e9a169a5..d661d7acf 100644 --- a/crates/numfmt/src/tests.rs +++ b/crates/numfmt/src/tests.rs @@ -15,6 +15,9 @@ fn format_colors_resolve_named_and_indexed_rgb_safely() { const S: DateSystem = DateSystem::D1900; +/// Built-in format 44 (accounting with a currency symbol, two decimals). +const ACCOUNTING_2DP: &str = "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"; + fn f(code: &str, n: f64) -> String { format_value(&Value::Number(n), &NumberFormat::parse(code), S).text } @@ -256,7 +259,7 @@ fn literals_and_fill() -> Result<(), Box> { assert_eq!(f("(0)", 5.0), "(5)"); let r = full("*-0", Value::Number(5.0)); assert_eq!((r.text.as_str(), r.fill), ("5", Some(('-', 0)))); - let acct = builtin_format(44).ok_or("none")?; + let acct = ACCOUNTING_2DP; let r = full(acct, Value::Number(0.0)); assert_eq!((r.text.as_str(), r.fill), (" $- ", Some((' ', 2)))); let r = full(acct, Value::Number(1234.5)); @@ -356,7 +359,7 @@ fn kinds() -> Result<(), Box> { assert_eq!(NumberFormat::parse("0.00").kind(), FormatKind::Number); assert_eq!(NumberFormat::parse("#,##0.00;[Red](#,##0.00)").kind(), FormatKind::Number); assert_eq!(NumberFormat::parse("\"$\"#,##0.00").kind(), FormatKind::Currency); - assert_eq!(NumberFormat::parse(builtin_format(44).ok_or("none")?).kind(), FormatKind::Accounting); + assert_eq!(NumberFormat::parse(ACCOUNTING_2DP).kind(), FormatKind::Accounting); assert_eq!(NumberFormat::parse("m/d/yyyy").kind(), FormatKind::Date); assert_eq!(NumberFormat::parse("h:mm AM/PM").kind(), FormatKind::Time); assert_eq!(NumberFormat::parse("0%").kind(), FormatKind::Percentage); @@ -369,26 +372,11 @@ fn kinds() -> Result<(), Box> { } #[test] -fn builtins() -> Result<(), Box> { - assert_eq!(builtin_format(0), Some("General")); - assert_eq!(builtin_format(14), Some("m/d/yyyy")); - assert_eq!(builtin_format(49), Some("@")); - assert_eq!(builtin_format(30), None); - assert_eq!(builtin_format(50), None); - assert_eq!(builtin_id("m/d/yyyy"), Some(14)); - assert_eq!(builtin_id("0.00%"), Some(10)); - assert_eq!(builtin_id("GENERAL"), Some(0)); - assert_eq!(builtin_id("[h]:mm:ss"), Some(46)); - assert_eq!(builtin_id("#,##0.00_);(#,##0.00)"), None); - for id in 0..=60 { - if let Some(code) = builtin_format(id) { - assert_eq!(builtin_id(code), Some(id)); - } - } - assert_eq!(f(builtin_format(5).ok_or("none")?, -1234.0), "($1,234)"); - assert_eq!(f(builtin_format(12).ok_or("none")?, 0.5), " 1/2"); - assert_eq!(f(builtin_format(47).ok_or("none")?, 61.2 / 86400.0), "0101.2"); - Ok(()) +fn builtin_codes_render() { + // Built-in ids 5, 12 and 47 (the id tables live with the file formats, in gridcraft-xlsx). + assert_eq!(f("\"$\"#,##0_);\\(\"$\"#,##0\\)", -1234.0), "($1,234)"); + assert_eq!(f("# ?/?", 0.5), " 1/2"); + assert_eq!(f("mmss.0", 61.2 / 86400.0), "0101.2"); } #[test] diff --git a/crates/numfmt/tests/golden/codes.txt b/crates/numfmt/tests/golden/codes.txt new file mode 100644 index 000000000..118f8a767 --- /dev/null +++ b/crates/numfmt/tests/golden/codes.txt @@ -0,0 +1,152 @@ +1427c0507a69d2eb 79 "0.0000000000000000000000000" +c37039701ea91f71 79 "#,###,###,###,###,###,###,###,##0.00" +c886ef7cc0381aca 79 "0.0000000000000000000000000E+00" +16878498a86d6270 79 "# ?????/?????" +25810fe3aa348757 79 "General" +637698d5ec4840e7 79 "0" +f480db9c3f30f4f9 79 "0.00" +233cef56f68009ce 79 "#,##0" +d2fae9f7b5756366 79 "#,##0.00" +fcf13b3494133da6 79 "#.##" +93b3291c9e9847f7 79 "0.0##" +20ca0c941ad5c11f 79 "00000" +db86a5c476fc29cd 79 "000.000" +7cc10690ed9769ab 79 "#,###" +ea3312bb2d5c1342 79 "#,##0," +d14a7d6e48db33f0 79 "#,##0,," +e44a9c0ed1f49c8a 79 "0.0,,\"M\"" +3c70c4c8ff8af422 79 "?.??" +93f48e4f2d83fd29 79 "##0.0" +a06b56173759b328 79 "# " +5a35d2ec11553fa4 79 "#" +f3495305635eab2d 79 ".00" +1f2ae975e66d556b 79 ",0" +253e097c9bdaf643 79 "0,0" +ae00b79ca5b8d288 79 "0.00," +1b989416a1ec6e7c 79 "\"$\"#,##0.00" +f82256c8ac29c8f3 79 "\"$\"#,##0_);\\(\"$\"#,##0\\)" +7fc54d574b4ee62c 79 "#,##0.00;[Red](#,##0.00)" +7a81a86d7a4d3c69 79 "#,##0 ;(#,##0)" +57a11793e92c9715 79 "0;-0;;@" +a3d6c52367c35259 79 "0.00;;" +b45d3fdec1e8ff9a 79 ";;;" +11a021e5408029bb 79 "0_);(0)" +99b2e56004ca9b4a 79 "0\" units\"" +c9aaa3ef8150f3a6 79 "\\A0" +641306b16f0036fe 79 "(0)" +19af7c406782b02c 79 "00-00" +c298712a7d3815bd 79 "000-00-0000" +28e5bc8fc3d88c8e 79 "[<=9999999]###-####;(###) ###-####" +f494c7ff5ff0fd7a 79 "[$€-407]#,##0.00" +4e11e82e14ea8f05 79 "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)" +fc2d989610e136f3 79 "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)" +091f29e23e924f36 79 "*-0" +93217b43f4b7af91 79 "0*x" +6a7561a5ce77014d 79 "*0 0.0" +9a6620cdfa67c60b 79 "@*_" +0b8f6e85e6ff12f0 79 "0%" +d2f558040c30c4cc 79 "0.00%" +371df01b15432009 79 "#,##0%" +e5016314db39e07b 79 "0.0%%" +f603836d162018f0 79 "%0" +7d08aed42e8d71e0 79 "0%;[Red]-0%" +d7d44df9f81bff2e 79 "0.00E+00" +275678f83b4d4113 79 "##0.0E+0" +497033fffb9c0967 79 "0E+0" +9679af8d3aa45c26 79 "0.0e-0" +eafc341422b759ce 79 "0.00E+000" +67cf5d053575d3df 79 "E+0" +f9d5b40fa2875a1e 79 "0.00E+" +e633fc616f9a3ff3 79 "#.##E+00" +8ecbeef181fb381e 79 "0.0E+00 \"x\"" +7b74fb5636bda27e 79 "# ?/?" +aa283fbec589cdfe 79 "# ??/??" +9c9625146b582644 79 "# ???/???" +7d7dd7cefba77e8e 79 "?/?" +3babcf6e56373ec4 79 "# ?/8" +c2e84ddf50f1e21d 79 "# ??/16" +d5bbf7173f227e7a 79 "0/0" +6fe467468cd9a182 79 "# #/#" +a73c12b90cfc3656 79 "#/#" +d034fba50699f7ea 79 "?/100" +02ee892d135e01dc 79 "# ?/?;-# ?/?" +4bd7e5c1d9b3503f 79 "0 0/0" +2162452594679802 154 "m/d/yyyy" +d04b4c59e67bea74 154 "d-mmm-yy" +5dec91c67babe5f2 154 "d-mmm" +c384b64cedef18eb 154 "mmm-yy" +d7aa777afafbc668 154 "h:mm AM/PM" +b175c2057a22f6de 154 "h:mm:ss AM/PM" +aded84fe78136262 154 "h:mm" +5a79757ea5163c90 154 "h:mm:ss" +cd019c5ea6050d45 154 "m/d/yyyy h:mm" +54dbe04a04108479 154 "mm:ss" +e0718e196529a8f2 154 "[h]:mm:ss" +a38b480cf5c5a364 154 "[h]:mm" +2a913161d1313180 154 "[m]:ss" +76fdaf50d35fb2a2 154 "[s]" +ff9403ec80b0bc4f 154 "mmss.0" +425d4aa682fdc092 154 "yyyy-mm-dd" +058ce7638d05bbb9 154 "dddd, mmmm d, yyyy" +fe4bc250009a8329 154 "ddd mmm dd yy" +ccb6419ee78b2955 154 "mmmmm" +6c0289cc8797f513 154 "yy" +47cc3767e6229ec4 154 "h:mm:ss.000" +e6112ef1c8665ef8 154 "h:mm:ss.0000" +0bf1d0573ca9d704 154 "hh:mm:ss A/P" +e2a3d66078e1cf02 154 "[hh]:mm:ss" +8707505efd8a6284 154 "d" +7fa481a6092fa43c 154 "dd" +7d4b6020f781a3ba 154 "dddd" +949ed74d9b020535 154 "m" +0aa17a2f309cbecd 154 "mm" +c70a36183af563af 154 "s.00" +a8c8c009a7b0012f 154 "yyyy-mm-dd hh:mm:ss.00" +a70fc60b044d8b5e 154 "h AM/PM" +cf3e0c4fccbedb80 154 "[$-409]d-mmm-yy" +80b17497f3b9d5dd 154 "yyyy\"年\"m\"月\"" +e00bb34ae8360b4c 154 "[h]\" hrs\"" +8b75446f8310605f 154 "mmmm yyyy;@" +01123dabdab536a7 79 "@" +59fa7b60ca99b9b7 79 "\"x\"@" +b1eee7ca88961e38 79 "0;\"neg\"0;\"zero\";\"txt: \"@" +4d3f8a891532765e 79 "General;General;General;@" +59dc8b6bd14d719a 79 "[Red]0.0;[Blue]-0.0" +6a4d8666a78e97b3 79 "[Green]0;[Magenta]0;[Cyan]0;[Yellow]@" +15b794f754adc79d 79 "[Color3]0.00" +158526f0a5b736c5 79 "[>100]\"big\"0;[<-100]\"small\"0;0.0" +7a94ed36a6bb6573 79 "[=0]\"zero\";0.0" +30d048d3f229b1ba 79 "[>=1000]#,##0,\"K\";0" +2b2b1f13faeb468d 79 "[<1]0.00%;0.0" +b78758e73c416a1e 79 "[Red][>5]0;0.00" +8b2379e7de300595 79 "General\" kg\"" +73f3485fe971d2bd 79 "\"v\"General" +2d2ce0ee57d199a2 79 "[Red]General" +ea2f102aad8c3c1b 79 "0.0;[Red]General" +ef5fc7ae5e0cf36c 79 "[" +d80872ea795758e3 79 "0.0.0" +145e66eaa5b6a665 79 "0\"" +ddb909bb72f98e33 79 "\\" +cc44573b688554a4 154 "[h" +c78707c90ee7a1b6 79 "[$" +aaf7da8233ee4f96 154 "yyyy-mm-dd;" +b1bbc169427ccdf8 79 ";" +2cf377cde4d0586d 79 "0;0;0;0;0" +9293c3fbfd2434b1 79 "\"abc" +3a0ef807a097eb69 79 "[>x]0" +1ef168bf45be1582 79 "ggg" +b39c47241ef60000 79 "#?#0" +92199ef29ea54843 154 "0E" +fe0eba9c258b10e7 154 "E0" +c5adbc0bf74eded8 79 "/" +d77317d9cc005a8c 79 "0/" +86aefad6fd9ab034 79 "/0" +d08e2f5c1fb6d629 79 "*" +5257c26622f2cfe0 79 "_" +ef14d84a8bf4f1b4 79 "_0" +c1a24c12c4381ef2 154 "[Red" +1653b1ac403ed200 79 "0.00.00%" +aac2038aa4ae7651 154 "mm:ss.0.0" +00083314e9f89199 79 "[ABC]0" +a456ed739adcc87c 79 "[=1]0;[=2]0;0" +b0b3aa7b9f08690c 79 "0 ??/??;0" diff --git a/crates/numfmt/tests/golden/general.txt b/crates/numfmt/tests/golden/general.txt new file mode 100644 index 000000000..ab338965e --- /dev/null +++ b/crates/numfmt/tests/golden/general.txt @@ -0,0 +1,555 @@ +5fda7bf42a47c479 1 0e0 +e400479573b2baf2 1 -0e0 +57f99cd6daf9cc72 1 1e0 +0ddafbadc6bf1a04 1 -1e0 +882d8b786644b3c6 1 5e-1 +846783d06acc0a60 1 -5e-1 +eb4b115b5778e14e 1 2.5e-1 +d320cfc393d5b774 1 1.005e0 +e7497067a4de5fb0 1 2.675e0 +68aed91bbaac1534 1 9.999995e2 +96a3c4703fc3b01a 1 9.995e-1 +df98061474126a71 1 9.995e0 +5916478863984e9b 1 9.95e1 +13813099e41458c9 1 5e-2 +e6381aba4f09a980 1 5e-3 +6c09e99da5f3e27f 1 5e-4 +1e9840c7c9074f0a 1 1e-5 +b8e3e7ad44ea4900 1 1e-10 +a9d70b3df9c809d1 1 1e-300 +4c1d46df8304d5b7 1 5e-324 +c2a28fae0bcd7949 1 1.23456789e8 +c38f4a506cc3267c 1 1.2345678e3 +bde85249e1dcc206 1 1.2345678901e10 +f61e4f9f86db3eea 1 1e11 +72057cacab3997fa 1 1e15 +4183c6065332dc5d 1 1.23456789012345e14 +e1cb4037184e1fe4 1 1.234567890123455e15 +503e585193837caa 1 1e20 +a59688774ce6d2a1 1 1e100 +20a1bd96371a3de0 1 1.7976931348623157e308 +3b202f375efc1640 1 -1.2345e3 +816da9ca801294d6 1 -4e-3 +f8adad661b001d1f 1 -1e-10 +a8aaed4940499212 1 3.333333333333333e-1 +b146e4ee19146f95 1 6.666666666666666e-1 +7433fb99403da062 1 3.0000000000000004e-1 +1c0a0bcb4c0422b8 1 1e2 +4d9afc58f74b7ddb 1 1e3 +0d48b7819a8a4b8d 1 5.9e1 +01452d3bffb28025 1 6e1 +4921ebb52b8c73c1 1 6.1e1 +09ea6a1671f8c563 1 3.6526e4 +b63b4cad6f332f99 1 3.652675e4 +2bc41104f2e19271 1 4.38315e4 +4de1bb6ce4b9358b 1 4.5000123456e4 +ed7fda487b9bcf16 1 2.958465e6 +9059129e9a79299b 1 2.958465999999e6 +a32ca3ab1dbe969f 1 2.958466e6 +90b8f9a1e42bd0a4 1 -5e0 +7e564b2cabd8de16 1 9.9999999e-1 +8781fec461d58bfd 1 9.99e-4 +7c708dbcf3460659 1 4.199999999999e1 +20b2def65bc7174d 1 NaN +957f300224ed1cab 1 inf +4b2f7ba9aa3ce108 1 -inf +9ae51ec9ae4d5114 1 4.1666666666666664e-2 +2689dad0ffde687c 1 9.999999999e-1 +1599f472f24a81fa 1 1.0000000001e0 +5f1f3c268b6168ac 1 1.23456789e1 +c513d218d1599535 1 3.14159265358979e0 +8140c7ee3b5a5345 1 6.5535e4 +bd982ddec1e00aaa 1 1e5 +97614145ed7f215c 1 1.23456e5 +26cde2690b2bcbcc 1 4.5e0 +1a8add9df90d93c9 1 3.75e-1 +eef82337e19fa9f3 1 1e300 +884afff127bc9c8a 1 1.5e-8 +723565414e3a8eb5 1 9.99999999999999e14 +410dc1f39f0470e8 1 9.999999999999998e13 +abe0337311e85bc2 1 1.234e-5 +e05b983d81a775a8 1 1.2345678901234568e17 +f1b04c74292211ec 1 1e22 +8bd34401b876c52c 1 1e23 +8162c16a7c44dc93 1 1.234567890123456e-1 +5df098e551929df8 1 7.083333333333334e-4 +403536010a422332 1 -2.4456788999965284e-22 +5fda7bf42a47c479 1 0e0 +6f8d525947f4496b 1 5.078754219653861e-20 +5fda7bf42a47c479 1 0e0 +db9fc2cd4be272f9 1 4.762524884899572e-19 +5fda7bf42a47c479 1 0e0 +4e42d845bdfe1e05 1 -9.481878134872896e-18 +5fda7bf42a47c479 1 0e0 +e2b246d2b6df4843 1 4.677068358276083e-17 +5fda7bf42a47c479 1 0e0 +db3ef5385bc881e0 1 4.2817210912547903e-16 +5fda7bf42a47c479 1 0e0 +bbf8726d918fa10a 1 -1.2049587102547775e-16 +5fda7bf42a47c479 1 0e0 +c0efc68af4f6f086 1 1.3789513024013423e-14 +5fda7bf42a47c479 1 0e0 +790b21d90f49a509 1 6.059016384138212e-13 +5fda7bf42a47c479 1 0e0 +ce56d3a5c4486cd1 1 -2.0839028757975573e-12 +5fda7bf42a47c479 1 0e0 +975db8510513c040 1 8.895735060164848e-11 +5fda7bf42a47c479 1 0e0 +472bbe20b67f109c 1 2.924228743531876e-11 +5fda7bf42a47c479 1 0e0 +a9c44b1d793ee953 1 -2.790173620128004e-9 +5fda7bf42a47c479 1 0e0 +b65a4a2339333f83 1 3.047667157619239e-8 +5fda7bf42a47c479 1 0e0 +0c9b50a66593ab52 1 9.768724036381636e-7 +5fda7bf42a47c479 1 0e0 +deb83d6ed094b3b2 1 -6.624321624908681e-7 +5fda7bf42a47c479 1 0e0 +cfb05c2a70b35a2d 1 1.6894338933954122e-5 +5fda7bf42a47c479 1 0e0 +dfa7bf97c3a3f630 1 5.610159605226053e-4 +aa35c0cb97f7addc 1 1e-3 +6f2e8f1578d1b6f7 1 -4.5736943003751186e-3 +e6381aba4f09a980 1 5e-3 +618e160ad7d75e3f 1 1.2177853639931391e-2 +7b3b780b251205f7 1 1.2e-2 +bc71798a9a9fd283 1 5.376575791159676e-1 +552d6fe712db0701 1 5.38e-1 +30f24eae79fdbea7 1 -1.6906316761960716e0 +9a85be1470daf3e4 1 1.691e0 +428a08f2bbd5c129 1 6.804349057483373e1 +36f22379ff98da63 1 6.8043e1 +42c9ce8792ca41ca 1 5.180578200923046e2 +b753a84983c4b8fb 1 5.18058e2 +d48ddf21b575ae2e 1 -6.281034210276412e3 +335061600a21e480 1 6.281034e3 +93205c65ad863f72 1 3.435675535581831e4 +1abaaae29d8209f0 1 3.4356755e4 +d09ef10e029d4d19 1 6.613076865085881e5 +b1c40c1667e951b0 1 6.61307687e5 +8e6cdd6bc70fbd22 1 -4.48220238062414e6 +c083e5d00d97cef4 1 4.482202381e6 +e127f684d7698427 1 1.0000341372267152e8 +d4d94bdd00026ac9 1 1.00003413723e8 +0cab466bd5e1d8fa 1 1.5849160403504446e8 +d4c67910bde50f32 1 1.58491604035e8 +0e7f466ac9853aae 1 -1.4058601193934329e9 +a643a7677b4c5ac7 1 1.405860119393e9 +2776bab16cf335d4 1 2.8871069252525196e10 +fc7419911bfbaca8 1 2.8871069252525e10 +a76f1c3dce467478 1 9.690172366761895e11 +be689212e28414e9 1 9.6901723667619e11 +c4c5bd5055043ad5 1 -2.346417551608747e12 +93b654146869cfc6 1 2.346417551608747e12 +645dfd0b8d651920 1 6.228668439208695e12 +645dfd0b8d651920 1 6.228668439208695e12 +f147744d6ebe1c8c 1 2.2428112791069544e14 +f147744d6ebe1c8c 1 2.2428112791069544e14 +ba8da370ef0b6f07 1 -7.327413159633475e15 +76ca14ec7527c55d 1 7.327413159633475e15 +813a82002c70fe12 1 6.675488044574204e16 +813a82002c70fe12 1 6.675488044574204e16 +8a50e37fec07f962 1 6.706148330480582e17 +8a50e37fec07f962 1 6.706148330480582e17 +b2bcb6262b7ad869 1 -7.27445337914015e18 +a958885d3e964a90 1 7.27445337914015e18 +f8c71a4f984138a2 1 3.874586892728994e19 +f8c71a4f984138a2 1 3.874586892728994e19 +143942bc00828b3f 1 4.207699940806956e-21 +5fda7bf42a47c479 1 0e0 +cca09550d6d0af64 1 -6.321408874535118e-20 +5fda7bf42a47c479 1 0e0 +804ebfa29a23426e 1 3.689215801512328e-19 +5fda7bf42a47c479 1 0e0 +34d878721f2d74d3 1 9.392721997790782e-18 +5fda7bf42a47c479 1 0e0 +bf13de1377b6a335 1 -5.788399687409319e-17 +5fda7bf42a47c479 1 0e0 +9164e2fa13905b35 1 2.279056237090581e-16 +5fda7bf42a47c479 1 0e0 +4fc8a97fbf0232f5 1 3.463636269309186e-15 +5fda7bf42a47c479 1 0e0 +eb0a80785bd2f283 1 -4.489928391258454e-14 +5fda7bf42a47c479 1 0e0 +9f70d66705199764 1 7.703173974678871e-13 +5fda7bf42a47c479 1 0e0 +fb73018ae299a802 1 1.3090206448341722e-12 +5fda7bf42a47c479 1 0e0 +c064d37e034a6497 1 -6.338150789669849e-11 +5fda7bf42a47c479 1 0e0 +53be38791b9b3a71 1 3.809051908390393e-11 +5fda7bf42a47c479 1 0e0 +87e6cfab84040f20 1 1.0070431565221817e-9 +5fda7bf42a47c479 1 0e0 +5db583e96d56b61f 1 -5.266594942615038e-8 +5fda7bf42a47c479 1 0e0 +87730d5ab9b90358 1 2.039725677192573e-7 +5fda7bf42a47c479 1 0e0 +2c827aff0066cbce 1 4.851068050995432e-6 +5fda7bf42a47c479 1 0e0 +31b9628affbc1814 1 -7.75941473244553e-5 +5fda7bf42a47c479 1 0e0 +8be16fa85aa991e0 1 7.381056138192671e-4 +aa35c0cb97f7addc 1 1e-3 +6c3609330a25ae41 1 1.575376680923546e-4 +5fda7bf42a47c479 1 0e0 +02745e58c6fb95fe 1 -8.309622978077537e-2 +3ae24003b2396fd4 1 8.3e-2 +35cb86c8421094f5 1 5.938677033303984e-1 +cd17bcf30868b4d7 1 5.94e-1 +5fc071051c10dcf2 1 2.1055700940714903e0 +3301c223c3ee7087 1 2.106e0 +374453ca4e69570f 1 -4.979516792760796e0 +30ac305e01179f6c 1 4.98e0 +dc9118090ba75534 1 7.702392128251413e2 +5668bd8822fd37eb 1 7.70239e2 +bc0fea3db4020ccf 1 5.295197566581555e2 +6a1f8e82ef33f9d0 1 5.2952e2 +8f67ffd819348bfb 1 -9.175765278181084e4 +7431aae5707ad8ce 1 9.1757653e4 +e1050c33ba3c5491 1 9.482550236540656e5 +a129426c85bdbae0 1 9.48255024e5 +e6d83dc1bd22212c 1 5.347153721033019e6 +e026e88cae226949 1 5.347153721e6 +03a3405b7ce1bb74 1 -2.3568277599192787e7 +61da64c36fd8c5eb 1 2.3568277599e7 +b3b4116a1d7c332a 1 1.0018445295210177e8 +ad683901e4974657 1 1.00184452952e8 +0be5c818158d8aba 1 8.356382245461647e7 +9656312094a4767c 1 8.3563822455e7 +cf7cd1e0127e076f 1 -4.557698877252628e10 +d82db809145f4cb7 1 4.5576988772526e10 +e21e145b37008e47 1 5.270348510870472e11 +3eeab7eb66861c31 1 5.27034851087047e11 +d51733208c288c7c 1 5.782033227859766e12 +d51733208c288c7c 1 5.782033227859766e12 +f0e51f3bf67fc7a3 1 -5.931696066185405e13 +e15f77a9ff4fb907 1 5.931696066185405e13 +9a7a6b9b153d807e 1 5.1455458768478e14 +3cce9402217011dc 1 5.1455458768478006e14 +c9f98f1fa9b23bc5 1 1.3538071072562253e15 +c9f98f1fa9b23bc5 1 1.3538071072562253e15 +bbff8f21267563e7 1 -9.90253824456554e15 +1a63f8bd04651493 1 9.90253824456554e15 +64140d1f4b064459 1 8.527630873184316e17 +64140d1f4b064459 1 8.527630873184316e17 +c42d45691a199adf 1 3.2925484547629455e18 +c42d45691a199adf 1 3.2925484547629455e18 +6d966e6760bed43d 1 -3.91537728865695e19 +efdecd348456b99a 1 3.91537728865695e19 +51ff7d6fc5809c27 1 4.8757246809796105e-21 +5fda7bf42a47c479 1 0e0 +84bceb480eb2dcb6 1 2.3420748266718282e-20 +5fda7bf42a47c479 1 0e0 +de6b157cd8b3068b 1 -6.293170081812841e-19 +5fda7bf42a47c479 1 0e0 +fca52fc30bf77496 1 5.53513945740202e-18 +5fda7bf42a47c479 1 0e0 +fef0b8399d29f15b 1 9.77860691981301e-17 +5fda7bf42a47c479 1 0e0 +8956eb136a63fc19 1 -5.1566694357030435e-17 +5fda7bf42a47c479 1 0e0 +c8d0796283c1a680 1 5.364510629593392e-15 +5fda7bf42a47c479 1 0e0 +c10f295101314665 1 9.661665594611814e-14 +5fda7bf42a47c479 1 0e0 +71f69f42ed1abd5f 1 -3.925165717977516e-13 +5fda7bf42a47c479 1 0e0 +4b3b18313d353d45 1 4.634788713583143e-12 +5fda7bf42a47c479 1 0e0 +1714fa5b40a6cf7b 1 2.1289153323939538e-11 +5fda7bf42a47c479 1 0e0 +fbe658ebfc667eeb 1 -3.7731542867717375e-10 +5fda7bf42a47c479 1 0e0 +ed1059691b29437f 1 3.0793918014219435e-9 +5fda7bf42a47c479 1 0e0 +d68a671d72cbb7d4 1 1.398193907677646e-8 +5fda7bf42a47c479 1 0e0 +bf3150d59315bcd7 1 -5.243893844613249e-7 +5fda7bf42a47c479 1 0e0 +93b6b3ec8b06d2ff 1 9.40673096940851e-6 +5fda7bf42a47c479 1 0e0 +71583d853e2cc61b 1 9.755344390282753e-5 +5fda7bf42a47c479 1 0e0 +a2726686baf8eaa6 1 -7.322925001904205e-4 +aa35c0cb97f7addc 1 1e-3 +7b84f8a587116b61 1 2.20079478849153e-3 +97b75dab0686cb37 1 2e-3 +69efff5278fdc06b 1 5.4369720160556294e-2 +22dfdcd990abd9ff 1 5.4e-2 +bdd2e88716b365e0 1 -1.90566789746209e-1 +a419c77256cf893c 1 1.91e-1 +cfb6f41a39039f7f 1 1.1954616597037193e0 +8818e7a995d67595 1 1.195e0 +d936acae76796026 1 3.1198426146081694e1 +f7c4c11aa02f61b9 1 3.1198e1 +4d61587438c8539f 1 -1.7276546543447557e2 +31d3905e625db28b 1 1.72765e2 +592a0f53b05918db 1 3.716258271721481e3 +c5e0c130f048add8 1 3.716258e3 +9964fe1fd34a9e66 1 6.354064796316502e4 +a3273ac06e88377e 1 6.3540648e4 +9d1cc1cc40d1e375 1 -6.247169817552513e5 +6e32184b6303fecd 1 6.24716982e5 +a59dec2d400de042 1 9.672875989854455e6 +c6184456a5f3d6dc 1 9.67287599e6 +952a4c5d6d9f3cc7 1 5.101929774891923e7 +00398960ce567662 1 5.1019297749e7 +2985fb1731569c12 1 -7.868553457247695e8 +cbe14af9c8d6397f 1 7.86855345725e8 +6ddbb1af610f79cf 1 6.192364765951433e9 +e146c8288f6cd3d2 1 6.192364765951e9 +674dfcf3b4a569b6 1 5.0322395435787346e10 +4eaeab0011c8ca30 1 5.0322395435787e10 +2a2da0e805cf5409 1 -8.387392837998628e11 +6e45a8daf47f9904 1 8.38739283799863e11 +adea7a9a9dfe1129 1 3.475227374631468e12 +adea7a9a9dfe1129 1 3.475227374631468e12 +928198bdae5603d2 1 6.046260397573301e13 +928198bdae5603d2 1 6.046260397573301e13 +3b2d1a775477a31c 1 -9.366184270581025e14 +856529b379f79ef7 1 9.366184270581025e14 +ac839bfff6b8812f 1 9.330285659281354e15 +ac839bfff6b8812f 1 9.330285659281354e15 +73160e61af42cec9 1 3.539370193508612e16 +73160e61af42cec9 1 3.539370193508612e16 +dedb455241069fa2 1 -9.996650818086104e17 +ec679a20500b031d 1 9.996650818086104e17 +26215fcc4901f42b 1 7.905433521846371e18 +26215fcc4901f42b 1 7.905433521846371e18 +ec9afc963bc85c7b 1 2.961789170831489e19 +ec9afc963bc85c7b 1 2.961789170831489e19 +f2482a70bd7552a8 1 -4.148371208994727e-21 +5fda7bf42a47c479 1 0e0 +a55409a5a800406d 1 7.170663257742498e-20 +5fda7bf42a47c479 1 0e0 +17f938f9cc679cd3 1 3.9583585927085557e-20 +5fda7bf42a47c479 1 0e0 +9a113924c396bb44 1 -5.892919583042963e-18 +5fda7bf42a47c479 1 0e0 +f0c40ba08b4e8d1c 1 6.485392183833719e-17 +5fda7bf42a47c479 1 0e0 +da9719b7ced0671f 1 7.180575172217469e-16 +5fda7bf42a47c479 1 0e0 +90ac83c7b7ea92c2 1 -2.7801459081525537e-16 +5fda7bf42a47c479 1 0e0 +069d079990d92448 1 8.42511493280958e-14 +5fda7bf42a47c479 1 0e0 +1c765f9111408737 1 1.0841559011512436e-13 +5fda7bf42a47c479 1 0e0 +7530679d8e6abd87 1 -2.1480013377898877e-12 +5fda7bf42a47c479 1 0e0 +873ab178136be7f4 1 2.8019016665180154e-11 +5fda7bf42a47c479 1 0e0 +4f3329b0b1b9010d 1 4.740529408461166e-10 +5fda7bf42a47c479 1 0e0 +84f8fb1f92637dc1 1 -7.552388125782508e-9 +5fda7bf42a47c479 1 0e0 +3d7d836e4bdb867e 1 9.755292951606497e-9 +5fda7bf42a47c479 1 0e0 +3fa8b742b75bff54 1 3.8472751651678525e-7 +5fda7bf42a47c479 1 0e0 +d349c8520a8cc6de 1 -6.977907562349183e-6 +5fda7bf42a47c479 1 0e0 +e1df5d730caf3475 1 8.218407552134532e-5 +5fda7bf42a47c479 1 0e0 +3df6765577ebd93a 1 4.996897368127074e-4 +5fda7bf42a47c479 1 0e0 +81b26f24e3ef8f0d 1 -3.151243912693312e-3 +77147cf949266682 1 3e-3 +bd1c5f937c96ac6a 1 3.0347488452323525e-2 +fc87ec37c8733143 1 3e-2 +a73ab956230ebaef 1 6.889166586655288e-1 +5fa353279229150c 1 6.89e-1 +f568fd4cfe3142c6 1 -9.743654785321732e0 +1bc6257bb3848aa9 1 9.744e0 +0f5ba1866b7d2845 1 5.668388855766816e1 +ce72a327bacd2b00 1 5.6684e1 +b4efc6800c4b914d 1 2.628257089515055e2 +21784204a4be65c0 1 2.62826e2 +e67a2d6c6630643f 1 -1.6168861365096154e3 +47815c5434eded45 1 1.616886e3 +4e3b51240c986695 1 3.280045990086601e4 +6890cb1d3a7f9b75 1 3.280046e4 +cabedf8f491b936e 1 1.4504200063410145e5 +e3836c93597707d8 1 1.45042001e5 +4271771f51534d50 1 -7.313314211991392e6 +1fb3dc6157f32057 1 7.313314212e6 +46c2ec9b7d7dd831 1 2.618837245417135e7 +3b956a81d84ff17d 1 2.6188372454e7 +5a8bbd3fdc03e83f 1 2.2253015708835256e8 +394a2739b0947d1f 1 2.22530157088e8 +40691261266728fd 1 -9.870234060874973e9 +eb3d9a317a8874aa 1 9.870234060875e9 +6a990a269a5c74f0 1 1.867926582347318e10 +f2ca9284e3a4f179 1 1.8679265823473e10 +280ece39f3f50572 1 3.567370490275957e11 +c14e0f480d1a7640 1 3.56737049027596e11 +3789dd57fc5c815c 1 -7.912946695129458e12 +b5ae37bd691873c7 1 7.912946695129458e12 +9f1553246fdb40cb 1 4.524229340690999e12 +9f1553246fdb40cb 1 4.524229340690999e12 +b24a577eb86c3e84 1 2.3102331568926594e14 +b24a577eb86c3e84 1 2.3102331568926594e14 +cdce2c77867a3106 1 -4.547025311981324e15 +e947b96f57d17bec 1 4.547025311981324e15 +cf45a38c70fcf953 1 4.6288039353802424e16 +cf45a38c70fcf953 1 4.6288039353802424e16 +014ea8cd6a1290e2 1 6.162091464101659e17 +014ea8cd6a1290e2 1 6.162091464101659e17 +ce7e9262127eff44 1 -4.849757599353334e18 +e50f3fe044d209e7 1 4.849757599353334e18 +d85b0205e7941c69 1 6.452894445273737e19 +d85b0205e7941c69 1 6.452894445273737e19 +f2c90f259094e92e 1 4.7803298279157025e-21 +5fda7bf42a47c479 1 0e0 +9a93ee439db2dc35 1 -7.233406321955394e-20 +5fda7bf42a47c479 1 0e0 +d1e6a63ae1d3a9bb 1 2.950670879528398e-19 +5fda7bf42a47c479 1 0e0 +9230461e2e26d575 1 3.063434779812392e-18 +5fda7bf42a47c479 1 0e0 +314136843a1e646c 1 -5.4890433629545986e-17 +5fda7bf42a47c479 1 0e0 +6f3c47632e4d68b3 1 3.820800755734363e-16 +5fda7bf42a47c479 1 0e0 +6f9edd60cd32e0d5 1 5.829213571595438e-16 +5fda7bf42a47c479 1 0e0 +78eb51a008a64997 1 -2.4171609780651217e-14 +5fda7bf42a47c479 1 0e0 +44e17d26d950269c 1 1.1540630209261826e-13 +5fda7bf42a47c479 1 0e0 +8c1d1f942eaccbbf 1 1.84539975340409e-12 +5fda7bf42a47c479 1 0e0 +04d9cdb28aac62c3 1 -1.1081220338777577e-11 +5fda7bf42a47c479 1 0e0 +ed11aa9ba5e5353f 1 6.04223566612428e-10 +5fda7bf42a47c479 1 0e0 +f99d682b436ba383 1 5.35352701590592e-9 +5fda7bf42a47c479 1 0e0 +98b4918f9567bfd3 1 -8.711002748201745e-8 +5fda7bf42a47c479 1 0e0 +1ad36850fde48c38 1 6.114307847139126e-7 +5fda7bf42a47c479 1 0e0 +08f08db80439f666 1 7.209491487707519e-6 +5fda7bf42a47c479 1 0e0 +6a2bea77bad32684 1 -9.107581134395797e-5 +5fda7bf42a47c479 1 0e0 +a6e64a2db8e7c580 1 1.502958882607809e-4 +5fda7bf42a47c479 1 0e0 +bb4f7310114464cd 1 9.694573849985441e-3 +e40ccf007d65d503 1 1e-2 +99d400856d68de78 1 -6.734696325484274e-2 +6f6224f1a2f8d8f1 1 6.7e-2 +e1bf6652cb8fa67a 1 5.736804397820615e-1 +3503377f10431893 1 5.74e-1 +abde9a0944f0075c 1 8.385525171527297e-1 +bba484bf8ca102d3 1 8.39e-1 +f0cadc58f065b9d4 1 -7.281427491271538e1 +3c3889813ef35db4 1 7.2814e1 +977a2a510c565470 1 8.277766877236081e1 +1342b4ca96de7c68 1 8.2778e1 +7d0118272533a618 1 6.537925550437317e3 +eb730fb885709956 1 6.537926e3 +5cbc224dc75590af 1 -9.556341676004886e4 +651cd96eb2cd481d 1 9.5563417e4 +7454f12c9c69f37a 1 8.917218866870608e5 +66dff276a09c18ca 1 8.91721887e5 +6cd5d2549504ffe4 1 1.7117781599952288e6 +b1b8b8d63f96b6e3 1 1.71177816e6 +4a42c81491eef17c 1 -7.887393405242969e7 +84299b08c59543e1 1 7.8873934052e7 +e59890374c3b92fd 1 4.975792917240606e8 +744d7f2f33d43087 1 4.97579291724e8 +62bc9e9bf68188bf 1 2.1101063788838792e9 +d651f84aed764523 1 2.110106378884e9 +03d61121d089d8a5 1 -5.020627637227572e10 +e7dec7d4092686f3 1 5.0206276372276e10 +f17fe9bd3b512a80 1 6.810322205888846e11 +734531128a13110d 1 6.81032220588885e11 +9c1dec292ef7b37f 1 1.1358071611803188e12 +dd424abd6b3af684 1 1.135807161180319e12 +694f57c35b260310 1 -3.644307586777936e13 +13611f86ee78a0ed 1 3.644307586777936e13 +d2a2a06dea91c0a6 1 4.619231907602407e14 +d2a2a06dea91c0a6 1 4.619231907602407e14 +77901e2ac387e970 1 6.618779769599664e15 +77901e2ac387e970 1 6.618779769599664e15 +89f326d5337edbf9 1 -1.6799818086520645e15 +4522b28cd51ce9fc 1 1.6799818086520645e15 +f9e0155a978000b9 1 8.738754212610816e17 +f9e0155a978000b9 1 8.738754212610816e17 +4bd5f489ae1421a8 1 3.782517862984159e18 +4bd5f489ae1421a8 1 3.782517862984159e18 +40b6664a43480fc1 1 -2.4148772852069077e19 +9441359dbc20f6b6 1 2.4148772852069077e19 +76f2826b72e1fda6 1 8.877221241287098e-21 +5fda7bf42a47c479 1 0e0 +bb4a3ae1f5949c11 1 1.8341454532358146e-20 +5fda7bf42a47c479 1 0e0 +766ae042bb11d920 1 -9.887584766107361e-19 +5fda7bf42a47c479 1 0e0 +c959d5910bc866f9 1 9.166579699039185e-18 +5fda7bf42a47c479 1 0e0 +b2699aa224c131ce 1 3.2452783206646016e-18 +5fda7bf42a47c479 1 0e0 +e784da29c1b96c28 1 -4.798816460260819e-16 +5fda7bf42a47c479 1 0e0 +c3bc806511396a74 1 5.664578770036096e-15 +5fda7bf42a47c479 1 0e0 +6c133c6895703b73 1 8.829601615987267e-14 +5fda7bf42a47c479 1 0e0 +b7721097837aef20 1 -4.3573790293046475e-13 +5fda7bf42a47c479 1 0e0 +af58a97244a9c6c3 1 5.551460106733779e-12 +5fda7bf42a47c479 1 0e0 +4646a79cd8815ccf 1 6.050294303578548e-11 +5fda7bf42a47c479 1 0e0 +5a4ffdfaaac64359 1 -3.3879885578121597e-10 +5fda7bf42a47c479 1 0e0 +d30c3dba6f8b8d72 1 9.069160028722885e-9 +5fda7bf42a47c479 1 0e0 +35719d8fb5e8e209 1 3.1971165267170906e-8 +5fda7bf42a47c479 1 0e0 +8f3a36e216599613 1 -8.76974502693964e-7 +5fda7bf42a47c479 1 0e0 +4a7c13e5c2e949a0 1 4.680359748821511e-6 +5fda7bf42a47c479 1 0e0 +c58c1c5093cfd666 1 7.563237465046436e-5 +5fda7bf42a47c479 1 0e0 +71c0d2f679b366ab 1 -1.79232054129694e-4 +5fda7bf42a47c479 1 0e0 +cc39dc0b3f31ef1a 1 8.188147563049265e-3 +1e7c9f5936317e01 1 8e-3 +7810fc63819aa54f 1 4.8276882501172715e-2 +316f14fbacc0f6dc 1 4.8e-2 +91fcbed9b6c1e1eb 1 -4.4497532569195164e-1 +1ba135db6a0ed229 1 4.45e-1 +6a646bd946016bba 1 7.6485650387578605e0 +49f07016ba5d174a 1 7.649e0 +35aacb91d397e63c 1 6.863911531586618e1 +ad8b40222f9c0b67 1 6.8639e1 +b62458bc71ec92fd 1 -4.564364881857964e2 +d8bdf521d8767965 1 4.56436e2 +f0d014cf4e9edf7f 1 1.9163551023901937e3 +15c1eb9d0605d6b0 1 1.916355e3 +b051547190f1dead 1 1.850604973023119e4 +a8b5ae0e34ac6db2 1 1.850605e4 +57da0f4c75eb541f 1 -6.297725994049538e5 +41c4a242cd154e87 1 6.29772599e5 +3053c5c8720a172e 1 7.738395738921099e4 +aaeee63373ab5c58 1 7.7383957e4 +122c1b2c08853c0c 1 8.396373407043353e7 +7b7f558a837722ff 1 8.396373407e7 +ba34066cdc427569 1 -2.4288547122230375e8 +420f589afd6b0565 1 2.42885471222e8 +041703a36b497da8 1 2.812715976368404e9 +c889fbd6f49dfffc 1 2.812715976368e9 +72f6aaac4485ab61 1 5.522396329298781e10 +60ac6434d6875dde 1 5.5223963292988e10 +36803089910ba7e2 1 -7.073708200884093e11 +5afe826153ea2a19 1 7.07370820088409e11 +8a4f45dfac6a7733 1 3.73164417144063e12 +8a4f45dfac6a7733 1 3.73164417144063e12 +69bca16e7f65b43d 1 1.69380681630822e13 +69bca16e7f65b43d 1 1.69380681630822e13 diff --git a/crates/numfmt/tests/render_equality.rs b/crates/numfmt/tests/render_equality.rs new file mode 100644 index 000000000..12a3ab94d --- /dev/null +++ b/crates/numfmt/tests/render_equality.rs @@ -0,0 +1,225 @@ +//! Equality tests for the render/General cleanup: output over a large grid of format codes x +//! values (format_value and text_function in both date systems, then General at widths 1..=30 and a few larger ones) +//! is compared with golden digests recorded from the implementation BEFORE the cleanup +//! (`tests/golden/*.txt`, one FNV-1a digest per format code / per value, so the files stay +//! small). A failure names the code or value that differs; set `NUMFMT_DUMP_DIR` to write the +//! full text of the current output for diffing against a dump from the old code. To re-record +//! after an intended behaviour change run with `NUMFMT_BLESS=1`. + +#![allow(clippy::approx_constant, clippy::excessive_precision)] // values are test data, not constants + +use std::fmt::Write as _; + +use gridcraft_core::{DateSystem, Value}; +use gridcraft_numfmt::{NumberFormat, format_general_fit, format_value, text_function}; + +// Placeholders far beyond the 15 significant digits. +const LONG_DECIMALS: &str = "0.0000000000000000000000000"; +#[rustfmt::skip] +const CODES: &[&str] = &[ + LONG_DECIMALS, "#,###,###,###,###,###,###,###,##0.00", "0.0000000000000000000000000E+00", "# ?????/?????", + // numbers + "General", "0", "0.00", "#,##0", "#,##0.00", "#.##", "0.0##", "00000", "000.000", "#,###", "#,##0,", "#,##0,,", "0.0,,\"M\"", "?.??", "##0.0", "# ", "#", ".00", + ",0", "0,0", "0.00,", "\"$\"#,##0.00", "\"$\"#,##0_);\\(\"$\"#,##0\\)", "#,##0.00;[Red](#,##0.00)", "#,##0 ;(#,##0)", "0;-0;;@", "0.00;;", ";;;", + "0_);(0)", "0\" units\"", "\\A0", "(0)", "00-00", "000-00-0000", "[<=9999999]###-####;(###) ###-####", "[$€-407]#,##0.00", + // currency/accounting with fill + "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)", "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)", "*-0", "0*x", "*0 0.0", "@*_", + // percent + "0%", "0.00%", "#,##0%", "0.0%%", "%0", "0%;[Red]-0%", + // scientific + "0.00E+00", "##0.0E+0", "0E+0", "0.0e-0", "0.00E+000", "E+0", "0.00E+", "#.##E+00", "0.0E+00 \"x\"", + // fractions + "# ?/?", "# ??/??", "# ???/???", "?/?", "# ?/8", "# ??/16", "0/0", "# #/#", "#/#", "?/100", "# ?/?;-# ?/?", "0 0/0", + // dates/times + "m/d/yyyy", "d-mmm-yy", "d-mmm", "mmm-yy", "h:mm AM/PM", "h:mm:ss AM/PM", "h:mm", "h:mm:ss", "m/d/yyyy h:mm", "mm:ss", "[h]:mm:ss", "[h]:mm", "[m]:ss", "[s]", "mmss.0", + "yyyy-mm-dd", "dddd, mmmm d, yyyy", "ddd mmm dd yy", "mmmmm", "yy", "h:mm:ss.000", "h:mm:ss.0000", "hh:mm:ss A/P", "[hh]:mm:ss", "d", "dd", "dddd", "m", "mm", "s.00", + "yyyy-mm-dd hh:mm:ss.00", "h AM/PM", "[$-409]d-mmm-yy", "yyyy\"年\"m\"月\"", "[h]\" hrs\"", "mmmm yyyy;@", + // text sections, conditions, colours + "@", "\"x\"@", "0;\"neg\"0;\"zero\";\"txt: \"@", "General;General;General;@", "[Red]0.0;[Blue]-0.0", "[Green]0;[Magenta]0;[Cyan]0;[Yellow]@", "[Color3]0.00", + "[>100]\"big\"0;[<-100]\"small\"0;0.0", "[=0]\"zero\";0.0", "[>=1000]#,##0,\"K\";0", "[<1]0.00%;0.0", "[Red][>5]0;0.00", "General\" kg\"", "\"v\"General", + "[Red]General", "0.0;[Red]General", + // invalid / odd + "[", "0.0.0", "0\"", "\\", "[h", "[$", "yyyy-mm-dd;", ";", "0;0;0;0;0", "\"abc", "[>x]0", "ggg", "#?#0", "0E", "E0", "/", "0/", "/0", "*", "_", "_0", "[Red", "0.00.00%", "mm:ss.0.0", + "[ABC]0", "[=1]0;[=2]0;0", "0 ??/??;0", +]; + +fn values() -> Vec { + let mut v = vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + -0.5, + 0.25, + 1.005, + 2.675, + 999.9995, + 0.9995, + 9.995, + 99.5, + 0.05, + 0.005, + 0.0005, + 1e-5, + 1e-10, + 1e-300, + 5e-324, + 123456789.0, + 1234.5678, + 12345678901.0, + 1e11, + 1e15, + 123456789012345.0, + 1234567890123455.0, + 1e20, + 1e100, + 1.7976931348623157e308, + -1234.5, + -0.004, + -1e-10, + 1.0 / 3.0, + 2.0 / 3.0, + 0.1 + 0.2, + 100.0, + 1000.0, + 59.0, + 60.0, + 61.0, + 36526.0, + 36526.75, + 43831.5, + 45000.123456, + 2958465.0, + 2958465.999999, + 2958466.0, + -5.0, + 0.99999999, + 0.000999, + 41.99999999999, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + 0.041666666666666664, + 0.9999999999, + 1.0000000001, + 12.3456789, + 3.14159265358979, + 65535.0, + 1e5, + 123456.0, + 4.5, + 0.375, + 1e300, + 1.5e-8, + 999999999999999.0, + 99999999999999.99, + 0.00001234, + 123456789012345678.0, + 1e22, + 9.999999999999999e22, + 0.1234567890123456, + ]; + v.push(61.2 / 86400.0); + v +} + +fn fmt_formatted(f: &gridcraft_numfmt::Formatted) -> String { + format!("{:?}|{:?}|{:?}|{}", f.text, f.color, f.fill, f.numeric) +} + +fn dump_codes() -> Vec<(String, String)> { + assert!(CODES.len() >= 80 && values().len() >= 40); + let mut rows = Vec::new(); + let others = [Value::Text("abc".into()), Value::Text("12.5".into()), Value::Bool(true), Value::Empty]; + for code in CODES { + let mut out = String::new(); + let nf = NumberFormat::parse(code); + let systems: &[(DateSystem, &str)] = + if nf.is_date() { &[(DateSystem::D1900, "1900"), (DateSystem::D1904, "1904")] } else { &[(DateSystem::D1900, "1900")] }; + for (sys, name) in systems { + for n in values() { + let v = Value::Number(n); + let _ = + writeln!(out, "{code:?} {name} {n:e} => {} || {:?}", fmt_formatted(&format_value(&v, &nf, *sys)), text_function(&v, code, *sys)); + } + } + for v in &others { + let _ = writeln!( + out, + "{code:?} {v:?} => {} || {:?}", + fmt_formatted(&format_value(v, &nf, DateSystem::D1900)), + text_function(v, code, DateSystem::D1900) + ); + } + rows.push((format!("{code:?}"), out)); + } + rows +} + +fn dump_general() -> Vec<(String, String)> { + let mut vals = values(); + let mut x = 0.7f64; + // Deterministic spread over many magnitudes and digit patterns (no RNG dependency). + for i in 0..240 { + x = (x * 997.0 + 0.123_456_789).fract() + 1e-3; + let e = (i % 41) - 20; + let m = x * 10f64.powi(e); + vals.push(if i % 3 == 0 { -m } else { m }); + vals.push((m * 1000.0).round() / 1000.0); + } + assert!(vals.len() >= 300); + let mut rows = Vec::new(); + for n in vals { + let mut out = String::new(); + let _ = write!(out, "{n:e} =>"); + for w in (1..=30usize).chain([31, 40, 100, 399, 400, 401, 500, 1000]) { + let _ = write!(out, " {:?}", format_general_fit(n, w)); + } + out.push('\n'); + rows.push((format!("{n:e}"), out)); + } + rows +} + +fn fnv1a(s: &str) -> u64 { + s.bytes().fold(0xcbf2_9ce4_8422_2325, |h, b| (h ^ u64::from(b)).wrapping_mul(0x100_0000_01b3)) +} + +fn check(name: &str, rows: &[(String, String)], golden: &str, path: &str) { + if let Some(dir) = std::env::var_os("NUMFMT_DUMP_DIR") { + let all: String = rows.iter().map(|(_, b)| b.as_str()).collect(); + std::fs::write(std::path::Path::new(&dir).join(format!("{name}.txt")), all).expect("write dump"); + } + let actual: String = rows.iter().map(|(k, b)| format!("{:016x} {} {k}\n", fnv1a(b), b.lines().count())).collect(); + if std::env::var_os("NUMFMT_BLESS").is_some() { + std::fs::write(path, &actual).expect("write golden"); + return; + } + let bad: Vec = rows + .iter() + .zip(actual.lines().zip(golden.lines())) + .filter(|(_, (a, g))| a != g) + .take(3) + .map(|((_, body), (a, g))| { + format!(" new: {a}\n old: {g}\n new output of that row (first 12 lines):\n{}", body.lines().take(12).collect::>().join("\n")) + }) + .collect(); + assert!( + actual.lines().count() == golden.lines().count() && bad.is_empty(), + "{name}: output differs from the recorded golden ({} vs {} rows). If the change is intended, re-record with\n NUMFMT_BLESS=1 cargo test -p gridcraft-numfmt --test render_equality\n{}", + actual.lines().count(), + golden.lines().count(), + bad.join("\n") + ); +} + +#[test] +fn format_value_and_text_function_match_golden() { + check("codes", &dump_codes(), include_str!("golden/codes.txt"), concat!(env!("CARGO_MANIFEST_DIR"), "/tests/golden/codes.txt")); +} + +#[test] +fn general_fit_matches_golden() { + check("general", &dump_general(), include_str!("golden/general.txt"), concat!(env!("CARGO_MANIFEST_DIR"), "/tests/golden/general.txt")); +} diff --git a/crates/xlsx/src/tables.rs b/crates/xlsx/src/tables.rs index 3cfd84c2d..4b2c9b6ec 100644 --- a/crates/xlsx/src/tables.rs +++ b/crates/xlsx/src/tables.rs @@ -348,6 +348,29 @@ mod tests { assert_eq!(px_to_pt(20.0), 15.0); } + #[test] + fn builtin_format_ids() { + // The one table of built-in formats (gridcraft-numfmt has none: file formats do not depend on it). + assert_eq!((0..=60).filter(|&id| builtin_format(id).is_some()).count(), 36); + assert_eq!(builtin_format(0), Some("General")); + assert_eq!(builtin_format(22), Some("m/d/yyyy h:mm")); + assert_eq!(builtin_format(49), Some("@")); + // 23-36 are locale formats, above 49 is custom. + assert_eq!((23..=36).find_map(builtin_format), None); + assert_eq!(builtin_format(50), None); + // Writing: an exact (case-sensitive) code gives its id, except the locale-dependent currency and accounting formats 5-8, 42 and 44. + assert_eq!(builtin_id("General"), Some(0)); + assert_eq!(builtin_id("GENERAL"), None); + assert_eq!(builtin_id("[H]:MM:SS"), None); + assert_eq!(builtin_id("[h]:mm:ss"), Some(46)); + assert_eq!(builtin_id("#,##0.00_);(#,##0.00)"), None); + for id in 0..=60 { + let Some(code) = builtin_format(id) else { continue }; + let expected = if matches!(id, 5..=8 | 42 | 44) { None } else { Some(id) }; + assert_eq!(builtin_id(code), expected, "id {id}"); + } + } + #[test] fn functions() { assert_eq!(file_function_name("xlookup").as_deref(), Some("_xlfn.XLOOKUP")); From b7fd198e0644abb23bee084815db3bf35449b92a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 03:18:38 -0500 Subject: [PATCH 23/51] Conditional formatting: one scan budget per rule, no thresholds for unsupported colour scales (#3201) * Conditional formatting: one scan budget per rule, and no thresholds for unsupported colour scales numbers_in() scanned each of a rule's ranges separately with its own 2,000,000-cell limit, so a rule listing the same (or overlapping) large range many times multiplied the scan while painting a single cell. Colour scales also evaluated every stop's threshold, formulas included, before discarding scales that are not 2- or 3-colour. - A rule's statistics now share STATS_SCAN_LIMIT across all its ranges, and an exact repeat of a range is scanned once. - A colour scale with a stop count other than 2 or 3 paints nothing and evaluates none of its thresholds. * address review: count stored and spilled cells of sparse ranges, skip statistics for unsupported colour scales * address review: skip unrelated spills a row at a time in the statistics scan --- crates/engine/src/cf.rs | 137 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 133 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cf.rs b/crates/engine/src/cf.rs index 0af7a9df5..e5fe4dedc 100644 --- a/crates/engine/src/cf.rs +++ b/crates/engine/src/cf.rs @@ -1,7 +1,7 @@ //! Conditional formatting evaluation: what extra style, data bar, colour scale or icon a cell //! gets. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use gridcraft_core::{CellRef, RangeRef, Value, compare}; use gridcraft_model::{CfOperator, CfRule, CfValueKind, Color, CondFormat, PatternType, Sheet, Style, Underline, Workbook, default_icon_thresholds}; @@ -47,6 +47,8 @@ enum StatsNeed { fn stats_need(rule: &CfRule) -> Option { match rule { CfRule::Duplicate { .. } | CfRule::Unique { .. } => Some(StatsNeed::Counts), + // only two- and three-colour scales paint: no statistics for the others + CfRule::ColorScale { stops } if !matches!(stops.len(), 2 | 3) => None, CfRule::Top10 { .. } | CfRule::AboveAverage { .. } | CfRule::ColorScale { .. } | CfRule::DataBar { .. } | CfRule::IconSet { .. } => { Some(StatsNeed::Numbers) } @@ -70,15 +72,58 @@ thread_local! { static SCANNED: std::cell::Cell = const { std::cell::Cell::new(0) }; } +/// Most cells one rule's statistics visit, across all of its ranges together: a rule repeating +/// the same range many times must not multiply the scan. +const STATS_SCAN_LIMIT: usize = 2_000_000; + +/// Steps through the spilled cells inside `r`, row-major: `Some(cell)` for each, and `None` for a +/// jump past the spills left or right of the columns on a row (one step per row, however many +/// there are), so unrelated spills between a narrow range's corners cost little. +fn spill_steps(sh: &Sheet, r: RangeRef) -> impl Iterator> + '_ { + let mut cur = Some(r.start); + std::iter::from_fn(move || { + let at = cur.filter(|at| *at <= r.end)?; + let (&c, _) = sh.spill.range(at..=r.end).next()?; + if c.col < r.start.col { + cur = Some(CellRef::new(c.row, r.start.col)); + Some(None) + } else if c.col > r.end.col { + cur = c.row.checked_add(1).map(|row| CellRef::new(row, r.start.col)); + Some(None) + } else { + cur = c.col.checked_add(1).map(|col| CellRef::new(c.row, col)); + Some(Some(c)) + } + }) +} + fn numbers_in(sh: &Sheet, ranges: &[RangeRef], need: StatsNeed) -> (Vec, HashMap) { let mut nums = Vec::new(); let mut counts = HashMap::new(); + let mut budget = STATS_SCAN_LIMIT; + let mut seen: HashSet = HashSet::new(); + let used = sh.used_range(); for r in ranges { - let r = match sh.used_range().and_then(|u| r.intersection(&u)) { + if budget == 0 { + break; + } + let r = match used.and_then(|u| r.intersection(&u)) { Some(x) => x, None => continue, }; - for c in r.iter().take(2_000_000) { + // The same range listed twice counts its cells once. + if !seen.insert(r) { + continue; + } + // Only stored and spilled cells hold values: blank cells of a sparse range cost nothing, + // so a value far down a huge range is still counted. Every cell looked at is charged. + let stored = sh.cells.iter_range(r).map(|(c, _)| (Some(c), true)); + let spilled = spill_steps(sh, r).map(|c| (c, false)); + for (c, is_stored) in stored.chain(spilled).take(budget) { + budget -= 1; + // a skip over spills outside the columns (charged, not read), or a spilled value + // under a stored cell, which was read with it + let Some(c) = c.filter(|c| is_stored || sh.cells.get(*c).is_none()) else { continue }; #[cfg(test)] SCANNED.with(|n| n.set(n.get() + 1)); let v = sh.value(c); @@ -278,7 +323,8 @@ impl CfCache { _ => None, }, CfRule::ColorScale { stops } => { - if let Some(n) = v.as_f64() { + // Only two- and three-colour scales paint: evaluate no thresholds for others. + if let Some(n) = v.as_f64().filter(|_| matches!(stops.len(), 2 | 3)) { let pts: Vec<(f64, [u8; 3])> = stops .iter() .map(|(k, col)| (threshold(wb, sheet, c, st, k), col.resolve(&wb.theme).unwrap_or([255, 255, 255]))) @@ -585,6 +631,89 @@ mod tests { assert_eq!(SCANNED.with(|n| n.get()), 10_000); } + /// A rule listing the same range many times scans it once, and never more than the shared + /// limit however many ranges it holds. + #[test] + fn repeated_ranges_share_one_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2], [3]]})).unwrap(); + s.execute("range.setValues", json!({"range": "XFD1048576", "values": [[4]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let dense = RangeRef::parse("A1:XFD1048576").unwrap(); + let small = RangeRef::parse("A1:A3").unwrap(); + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &vec![small; 1000], StatsNeed::Numbers); + assert_eq!(st.sorted, vec![1.0, 2.0, 3.0]); + assert_eq!(SCANNED.with(|n| n.get()), 3); + let shifted: Vec = (0..50).map(|i| RangeRef::new(CellRef::new(i, 0), dense.end)).collect(); + SCANNED.with(|n| n.set(0)); + RuleStats::build(sh, &shifted, StatsNeed::Numbers); + assert!(SCANNED.with(|n| n.get()) <= STATS_SCAN_LIMIT); + } + + /// Blank cells of a sparse range are not charged to the scan budget: a value at the end of two + /// million-row columns still counts, so duplicates are found. + #[test] + fn sparse_ranges_keep_their_values_under_the_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[7]]})).unwrap(); + s.execute("range.setValues", json!({"range": "B1048576", "values": [[7]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let ranges = [RangeRef::parse("A1:A1048576").unwrap(), RangeRef::parse("B1:B1048576").unwrap()]; + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &ranges, StatsNeed::Counts); + assert_eq!(st.counts.get("7"), Some(&2)); + assert_eq!(SCANNED.with(|n| n.get()), 2); + } + + /// Spills left and right of a narrow range, between its corners, are skipped a row at a time: + /// they do not use the budget that the in-range spills need. + #[test] + fn spills_outside_a_narrow_range_do_not_use_its_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + let d = s.doc().unwrap(); + let mut sh = d.wb.sheet(0).unwrap().clone(); + for row in 0..1000 { + for col in [0u32, 5, 6, 7] { + sh.spill.insert(CellRef::new(row, col), Value::Number(1.0)); + } + } + let r = RangeRef::parse("F1:G1000").unwrap(); + let steps: Vec<_> = spill_steps(&sh, r).collect(); + assert_eq!(steps.iter().flatten().count(), 2000); + // column A is skipped one step a row, never one per cell + assert!(steps.len() <= 2000 + 1000, "{}", steps.len()); + let narrow = RangeRef::parse("C1:C1000").unwrap(); + assert_eq!(spill_steps(&sh, narrow).flatten().count(), 0); + } + + /// A colour scale with an unsupported number of stops paints nothing and evaluates none of them. + #[test] + fn colour_scales_with_other_stop_counts_paint_nothing() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [5]]})).unwrap(); + let mut wb: Workbook = (*s.doc().unwrap().wb).clone(); + let stop = (CfValueKind::Formula("=1".into()), Color::default()); + let rule = |n: usize| CondFormat { + ranges: vec![RangeRef::parse("A1:A2").unwrap()], + rule: CfRule::ColorScale { stops: vec![stop.clone(); n] }, + priority: 1, + stop_if_true: false, + }; + wb.sheet_mut(0).unwrap().cond_formats.push(rule(4)); + let mut cache = CfCache::new(); + assert_eq!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill), None); + wb.sheet_mut(0).unwrap().cond_formats = vec![rule(2)]; + let mut cache = CfCache::new(); + assert!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill).is_some()); + } + #[test] fn cell_value_operator_names() { let mut s = crate::Session::new(); From 5ce63e5fc24120c94aa14e5148c240128d81c750 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 03:48:32 -0500 Subject: [PATCH 24/51] Get Data: refuse an import whose dense rows would be too large (#3193) * Get Data: refuse an import whose dense rows would be too large data.getData and data.fromTextCsv built every row densely before writing it: JSON records were padded to the union of all keys (found with a quadratic Vec::contains scan), and CSV cells were expanded over the whole used rectangle. A few sparse records or one far-away CSV cell therefore allocated billions of slots. The importer now collects JSON keys with a HashSet, refuses more keys than a sheet has columns, and checks the projected rows x columns against MAX_IMPORT_CELLS (10,000,000) before building any row. * address review: stop collecting JSON keys at the column limit and reject oversized row counts before building rows * address review: enforce the CSV import cell limit while parsing --- crates/engine/src/cmd/extra/data.rs | 66 +++++++++++++++++++++++++++-- crates/engine/src/tests.rs | 12 ++++++ crates/xlsx/src/csv.rs | 15 +++++++ crates/xlsx/src/lib.rs | 5 ++- crates/xlsx/src/tests/csv_tests.rs | 15 +++++++ 5 files changed, 109 insertions(+), 4 deletions(-) diff --git a/crates/engine/src/cmd/extra/data.rs b/crates/engine/src/cmd/extra/data.rs index 9a1ae7417..5af7df0a8 100644 --- a/crates/engine/src/cmd/extra/data.rs +++ b/crates/engine/src/cmd/extra/data.rs @@ -42,6 +42,15 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result { }) } +/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely +/// before they are written, so a few sparse records or one far-away CSV cell must not expand +/// into a rectangle of billions of slots. +const MAX_IMPORT_CELLS: u64 = 10_000_000; + +fn too_many_cells() -> EngineError { + EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most).")) +} + pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let bytes = if let Some(t) = str_param(p, "text") { t.as_bytes().to_vec() @@ -57,15 +66,27 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { if text.trim_start().starts_with('[') { let arr: Vec = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?; let mut keys: Vec = Vec::new(); + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); for o in &arr { if let Some(m) = o.as_object() { for k in m.keys() { - if !keys.contains(k) { + if !seen.contains(k.as_str()) { + // Stop collecting at the column limit. + if keys.len() >= gridcraft_core::MAX_COLS as usize { + return Err(EngineError::Other("The data has more fields than a sheet has columns.".into())); + } + seen.insert(k.as_str()); keys.push(k.clone()); } } } } + if arr.len() as u64 + 1 > u64::from(gridcraft_core::MAX_ROWS) { + return Err(EngineError::Other("The data has more rows than a sheet has.".into())); + } + if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } let mut out = vec![keys.iter().map(|k| json!(k)).collect::>()]; for o in &arr { out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect()); @@ -75,10 +96,17 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let delim = str_param(p, "delimiter") .and_then(|d| d.chars().next()) .unwrap_or(if text.lines().next().is_some_and(|l| l.contains('\t')) { '\t' } else { ',' }); - let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, ..Default::default() }; - let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?; + let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, max_used_cells: Some(MAX_IMPORT_CELLS), ..Default::default() }; + // The cell limit is enforced while parsing, before an oversized file is stored. + let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| match e { + gridcraft_xlsx::IoError::TooLarge(_) => too_many_cells(), + e => EngineError::Other(e.to_string()), + })?; let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?; let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) }; + if u.count() > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } (u.start.row..=u.end.row) .map(|r| { (u.start.col..=u.end.col) @@ -129,3 +157,35 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result { .collect(), )) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::Session; + + fn s() -> Session { + let mut s = Session::new(); + s.new_workbook(); + s + } + + #[test] + fn sparse_imports_are_refused_before_their_rows_are_built() { + let mut s = s(); + let sheets = s.doc().unwrap().wb.sheets.len(); + // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field. + let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect(); + let text = serde_json::to_string(&records).unwrap(); + let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + // A CSV whose last line reaches 16,001 columns, 1,000 lines down. + let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000)); + let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + // Small imports still work. + let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap(); + assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}"); + } +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 08fc9119f..ed5e33945 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1350,6 +1350,18 @@ fn scripts_leave_file_access_working_after_they_run() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn get_data_refuses_json_with_too_many_fields_or_rows_before_building_rows() { + let mut s = s(); + let wide = format!("[{}]", (0..=gridcraft_core::MAX_COLS).map(|i| format!("{{\"k{i}\":1}}")).collect::>().join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": wide})).unwrap_err().to_string(); + assert!(err.contains("more fields"), "{err}"); + // MAX_ROWS objects plus the header row are one too many. + let tall = format!("[{}]", vec!["{}"; gridcraft_core::MAX_ROWS as usize].join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": tall})).unwrap_err().to_string(); + assert!(err.contains("more rows"), "{err}"); +} + #[test] fn scripts_cannot_reach_outside_the_workbook_even_when_trusted() { let dir = std::env::temp_dir().join(format!("gridcraft-script-deny-{}", std::process::id())); diff --git a/crates/xlsx/src/csv.rs b/crates/xlsx/src/csv.rs index a2e5493f1..254e903f0 100644 --- a/crates/xlsx/src/csv.rs +++ b/crates/xlsx/src/csv.rs @@ -496,6 +496,9 @@ pub fn read_csv_with_report(bytes: &[u8], opts: &CsvOptions) -> Result<(Workbook let mut styles = std::mem::take(&mut wb.styles); let mut fmt_cache: Vec<(&'static str, StyleId)> = vec![]; let mut row: u32 = 0; + // The bounding box of the non-empty cells, which is the sheet's used range. + let mut used: Option<(u32, u32, u32, u32)> = None; + let mut too_many = false; parse_records(&text, delim, quote, |rec| { if row >= MAX_ROWS { return false; @@ -509,6 +512,8 @@ pub fn read_csv_with_report(bytes: &[u8], opts: &CsvOptions) -> Result<(Workbook continue; } let pos = CellRef::new(row, col as u32); + let (r0, c0, c1) = used.map_or((row, col as u32, col as u32), |(r0, c0, _, c1)| (r0, c0.min(col as u32), c1.max(col as u32))); + used = Some((r0, c0, row, c1)); let cell = if opts.parse_values { let rule = date_columns.get(col).map_or(DateRule::NoDates, |(rule, _)| *rule); let p = read_field(&f, opts.date_system, rule); @@ -529,9 +534,19 @@ pub fn read_csv_with_report(bytes: &[u8], opts: &CsvOptions) -> Result<(Workbook }; sheet.cells.set(pos, cell); } + // Stop at the first record that takes the used range past the limit, before more is stored. + if let (Some(max), Some((r0, c0, r1, c1))) = (opts.max_used_cells, used) + && u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1) > max + { + too_many = true; + return false; + } row += 1; true }); + if too_many { + return Err(IoError::TooLarge(format!("the CSV's used range is over the limit of {} cells", opts.max_used_cells.unwrap_or(0)))); + } wb.styles = styles; wb.sheets = vec![Arc::new(sheet)]; Ok((wb, report)) diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 773e2895f..67694e72d 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -231,11 +231,14 @@ pub struct CsvOptions { pub date_system: DateSystem, /// Interpret fields like typed input (numbers, dates, booleans…); otherwise keep text. pub parse_values: bool, + /// Refuse (with [`IoError::TooLarge`], while reading) a file whose used range, from its first to + /// its last non-empty cell, would pass this many cells; `None` for no limit. + pub max_used_cells: Option, } impl Default for CsvOptions { fn default() -> Self { - CsvOptions { delimiter: b',', quote: b'"', date_system: DateSystem::D1900, parse_values: true } + CsvOptions { delimiter: b',', quote: b'"', date_system: DateSystem::D1900, parse_values: true, max_used_cells: None } } } diff --git a/crates/xlsx/src/tests/csv_tests.rs b/crates/xlsx/src/tests/csv_tests.rs index 585fb680f..ec4405f1b 100644 --- a/crates/xlsx/src/tests/csv_tests.rs +++ b/crates/xlsx/src/tests/csv_tests.rs @@ -406,3 +406,18 @@ fn parser_stops_collecting_fields_at_max_cols() { }); assert_eq!(lens, vec![MAX_COLS as usize]); } + +#[test] +fn csv_used_range_limit_is_enforced_while_reading() { + let opts = |max| CsvOptions { max_used_cells: Some(max), ..Default::default() }; + // 3 rows x 3 columns: exactly at the limit reads, one cell over is refused. + let dense = b"1,2,3\n4,5,6\n7,8,9\n"; + assert!(read_csv(dense, &opts(9)).is_ok()); + assert!(matches!(read_csv(dense, &opts(8)), Err(IoError::TooLarge(_)))); + // The limit is on the used range: empty leading rows, columns and trailing lines are free. + assert!(read_csv(b"\n\n,,x,y\n,,z,w\n\n\n", &opts(4)).is_ok()); + // A wide sparse row against a tall one: refused once the box passes the limit. + assert!(matches!(read_csv(b"a\nb\nc\n,,,,d\n", &opts(10)), Err(IoError::TooLarge(_)))); + // No limit by default. + assert!(read_csv(dense, &CsvOptions::default()).is_ok()); +} From ff9f4b8acebc7332d6a59e352fa1b7686c910b31 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 04:02:22 -0500 Subject: [PATCH 25/51] Find and Replace: bound the text a replacement builds (#3194) * Find and Replace: bound the text a replacement builds Each occurrence copied the whole replacement string, so a cell full of short matches and a long replacement expanded tens of kilobytes of input into about a gigabyte, in both the case-sensitive and the case-insensitive paths. Replacement now builds the new text only while it stays within Excel's 32,767 characters per cell (counted as UTF-8 bytes, 4 each at most) and within 256 MiB for the whole command, checking before each copy. Past either limit the command fails and changes nothing. * address review: enforce the 32,767-character limit on replacements and borrow the replacement text --- crates/engine/src/cmd/edit/find.rs | 70 ++++++++++++++++++++++++++--- crates/engine/src/cmd/edit/tests.rs | 44 ++++++++++++++++++ 2 files changed, 107 insertions(+), 7 deletions(-) diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index ddb9ecaf7..3dfbc6cbe 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -112,9 +112,19 @@ pub(super) fn find(s: &mut Session, p: &Json) -> Result { Ok(json!({"cell": c.a1(), "sheet": si, "value": t, "count": hits.len()})) } +/// Most characters a cell holds (Excel's limit). +const REPLACE_CELL_MAX_CHARS: usize = 32_767; +/// Longest text one replacement may give a cell, in bytes: the character limit, each up to four +/// bytes. Every occurrence copies the replacement text, so a short cell full of matches and a +/// long replacement would otherwise expand to gigabytes. The character limit is checked on what +/// fits in this. +const REPLACE_CELL_MAX_BYTES: usize = REPLACE_CELL_MAX_CHARS * 4; +/// Most bytes of replaced text one Replace All builds across all its cells. +const REPLACE_TOTAL_MAX_BYTES: usize = 256 * 1024 * 1024; + pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let what = str_param(p, "what").unwrap_or("").to_string(); - let with = str_param(p, "with").unwrap_or("").to_string(); + let with = str_param(p, "with").unwrap_or(""); if what.is_empty() { return Err(bad("edit.replace", "missing `what`")); } @@ -128,6 +138,7 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let n = edit(s, |cx| { let sheets: Vec = if workbook { (0..cx.wb.sheets.len()).collect() } else { vec![cx.wb.active_sheet()] }; let mut n = 0; + let mut total = 0usize; for si in sheets { let cells: Vec<(CellRef, String)> = cx.wb.sheet(si).map(|sh| sh.cells.iter().map(|(c, _)| (c, sh.input_text(c))).collect()).unwrap_or_default(); @@ -139,12 +150,20 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { continue; } let new = if whole { - with.clone() + (with.len() <= REPLACE_CELL_MAX_BYTES).then(|| with.to_string()) } else if case { - text.replace(&what, &with) + replace_cs_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) } else { - replace_ci(&text, &what, &with) + replace_ci_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) }; + let new = new.filter(|n| n.chars().count() <= REPLACE_CELL_MAX_CHARS); + let Some(new) = new else { + return Err(EngineError::Other(format!("The replacement makes the text of {} too long.", c.a1()))); + }; + total = total.saturating_add(new.len()); + if total > REPLACE_TOTAL_MAX_BYTES { + return Err(EngineError::Other("The replacements make too much text. Replace in a smaller range.".into())); + } // A Python cell's text is its code: the replacement edits the code and the cell // stays a Python cell. Code that would no longer fit in a PY formula is left as it was. let old_py = cx.wb.sheet(si).and_then(|sh| sh.cell(c)).and_then(|x| x.py.clone()); @@ -199,7 +218,37 @@ fn folds_in_place(s: &str) -> bool { }) } +#[cfg(test)] pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { + replace_ci_within(text, what, with, usize::MAX).unwrap_or_default() +} + +/// `text` with every `what` replaced by `with`, or `None` once the result passes `limit` bytes +/// (checked before each copy, so nothing larger is allocated). +pub(super) fn replace_cs_within(text: &str, what: &str, with: &str, limit: usize) -> Option { + if what.is_empty() { + return (text.len() <= limit).then(|| text.to_string()); + } + let mut out = String::new(); + let mut i = 0; + for (pos, m) in text.match_indices(what) { + if out.len() + (pos - i) + with.len() > limit { + return None; + } + out.push_str(text.get(i..pos).unwrap_or("")); + out.push_str(with); + i = pos + m.len(); + } + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) +} + +/// Case-insensitive [`replace_cs_within`]. +pub(super) fn replace_ci_within(text: &str, what: &str, with: &str, limit: usize) -> Option { let lower = text.to_lowercase(); let w = what.to_lowercase(); // Slicing `text` at positions found in `lower` is only sound when folding kept every byte @@ -207,15 +256,22 @@ pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { // boundaries (U+212A KELVIN SIGN shrinks 3→1 bytes, U+0130 grows 2→3), where the tail used // to be dropped. Those fall back to the case-sensitive replace, as length changes already do. if lower.len() != text.len() || w.is_empty() || !folds_in_place(text) || !folds_in_place(what) { - return text.replace(what, with); + return replace_cs_within(text, what, with, limit); } let mut out = String::new(); let mut i = 0; while let Some(pos) = lower.get(i..).and_then(|s| s.find(&w)) { + if out.len() + pos + with.len() > limit { + return None; + } out.push_str(text.get(i..i + pos).unwrap_or("")); out.push_str(with); i += pos + w.len(); } - out.push_str(text.get(i..).unwrap_or("")); - out + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) } diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index dd5fad636..0400e61e8 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -446,6 +446,50 @@ fn drag_and_drop_on_another_sheet_stays_on_that_sheet() { assert_eq!(wb.sheet(two).unwrap().value(CellRef::new(0, 1)), Value::Empty); } +#[test] +fn replace_refuses_text_that_would_grow_past_a_cell() { + // 10,000 matches × a 1,000-character replacement would make a 10 MB cell (and a few such + // cells gigabytes); the replacement is refused before that text is built. + let mut s = Session::new(); + s.new_workbook(); + let long = "a".repeat(10_000); + s.execute("range.setValues", json!({"range": "A1", "values": [[long.clone()], ["abc"]]})).unwrap(); + for case in [false, true] { + let err = s.execute("edit.replace", json!({"what": "a", "with": "x".repeat(1_000), "matchCase": case})).expect_err("too long"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!(long)); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!("abc")); + } + // A replacement that stays within a cell's limit still happens. + s.execute("edit.replace", json!({"what": "b", "with": "x".repeat(100)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!(format!("a{}c", "x".repeat(100)))); +} + +#[test] +fn replace_refuses_a_result_past_the_character_limit_though_it_fits_the_byte_limit() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["z"]]})).unwrap(); + for whole in [false, true] { + let err = s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_768), "wholeCell": whole})).expect_err("32,768 characters"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("z")); + } + s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_767)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("x".repeat(32_767))); +} + +#[test] +fn bounded_replace_stops_at_its_limit() { + use super::find::{replace_ci_within, replace_cs_within}; + assert_eq!(replace_cs_within("abab", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_cs_within("abab", "a", "xy", 5), None); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 5), None); + assert_eq!(replace_cs_within("abc", "z", "", 3).as_deref(), Some("abc")); + assert_eq!(replace_cs_within("abc", "z", "", 2), None); +} + #[test] fn replace_case_insensitive() { assert_eq!(replace_ci("Apple apple APPLE", "apple", "pear"), "pear pear pear"); From 79fe7b8eb6caf28e7e62235b245016f41db3a1ed Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 04:22:54 -0500 Subject: [PATCH 26/51] Forecast Sheet: count hindcast rows against the work limit (#3199) * Forecast Sheet: count hindcast rows against the work limit MAX_WORK bounded history rows x forecast rows, but a Forecast Start inside the history adds a hindcast row for every history point after it, each holding the same ETS formulas that refit the model over the history before the start. Those rows were not counted, so a start in the middle of 10,000 rows of history built millions of uncharged model fits per recalculation. The forecast row cap is now computed from the rows the model is fit to, after subtracting the hindcast rows; a start so early that the hindcast alone passes the limit is refused, and the default number of periods stays within the cap. * address review: base the in-history forecast cap on the slots the fit range spans * address review: document the hindcast-aware forecast work limit * address review: document the fitted-row minimum in the forecast work limit * address review: say the forecast start row counts toward the hindcast work limit --- crates/engine/src/cmd/forecast.rs | 76 +++++++++++++++++++++++++++++-- 1 file changed, 73 insertions(+), 3 deletions(-) diff --git a/crates/engine/src/cmd/forecast.rs b/crates/engine/src/cmd/forecast.rs index d0941a9ca..88de4b433 100644 --- a/crates/engine/src/cmd/forecast.rs +++ b/crates/engine/src/cmd/forecast.rs @@ -36,13 +36,28 @@ fn max_periods(history: usize) -> u32 { u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS) } +/// Most forecast rows when the model is fit to the first `fit` of `history` rows, which span +/// `slots` timeline slots once their gaps are filled. With a Forecast Start inside the history, +/// every row after it is forecast too (a hindcast row holds the same ETS formulas, each refitting +/// the model over the `slots` slots), so those rows (counted from the raw rows) count against +/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it. +fn max_periods_from(history: usize, fit: usize, slots: usize) -> Option { + if fit >= history { + return Some(max_periods(history)); + } + let f = u64::try_from(slots.max(fit)).unwrap_or(u64::MAX).max(1); + let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX); + let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?; + Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS)) +} + pub fn specs() -> Vec { vec![cmd!( "data.forecastSheet", "Forecast Sheet", ["Data", "Forecast"], None, - "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", + "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000, or with a forecastStart inside the history (hindcast rows are forecast too) (the greater of the fitted rows and fitted slots × (periods + rows from forecastStart on, that row included)) ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", has_doc, forecast_sheet )] @@ -257,13 +272,23 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result { } } }; - let cap = max_periods(rows.len()); + // The slots the model is fit over: the fit rows' span of the timeline, gaps included. + let slots = match (fit < rows.len(), times.get(fit.wrapping_sub(1)), times.first()) { + (true, Some(last_fit), Some(first)) => match (timeline.steps_after_last(*last_fit), timeline.steps_after_last(*first)) { + (Some(b), Some(a)) => ((b - a).round() + 1.0).clamp(0.0, 1e9) as usize, + _ => fit, + }, + _ => fit, + }; + let cap = max_periods_from(rows.len(), fit, slots).ok_or_else(|| { + EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into()) + })?; let periods = match end { Some(end) => steps_to(&timeline, end, cap)?, None => match number(p, "periods")? { Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32, Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))), - None => default_periods(rows.len()), + None => default_periods(rows.len()).min(cap), }, }; let future: Vec = @@ -532,6 +557,51 @@ mod tests { assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1); } + #[test] + fn hindcast_rows_count_against_the_work_limit() { + use super::{MAX_WORK, max_periods, max_periods_from}; + assert_eq!(max_periods_from(1_300, 1_300, 1_300), Some(max_periods(1_300))); + // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows. + assert_eq!(max_periods_from(2_000, 1_899, 1_899), Some(162)); + // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit. + assert_eq!(max_periods_from(2_000, 1_000, 1_000), None); + // 800 rows fit to the first 400, which span 571 slots (gaps filled): each formula refits + // 571 slots, so 475 forecast rows after the 400 hindcast rows, not 850. + assert_eq!(max_periods_from(800, 400, 571), Some(475)); + assert!((475 + 400) * 571 <= MAX_WORK as usize); + for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] { + if let Some(p) = max_periods_from(h, f, f) { + assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}"); + } + } + let mut s = Session::new(); + s.new_workbook(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let sheets = s.doc().unwrap().wb.sheets.len(); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long"); + assert!(err.to_string().contains("forecast start is too early"), "{err}"); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 162"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + } + + #[test] + fn the_hindcast_cap_follows_the_slots_the_fit_spans_not_its_rows() { + // 800 rows, 20% of the slots missing: the first 400 rows span 500 slots, so each ETS + // formula refits 500 slots and the cap is 1,000 - 400 = 600 (it was 850 by rows). + let mut s = Session::new(); + s.new_workbook(); + let times: Vec = (1u32..).filter(|t| !matches!(t % 10, 3 | 6)).take(800).collect(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend(times.iter().map(|t| json!([t, (t % 7) as f64 + f64::from(*t) * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let start = times[400]; + let err = s.execute("data.forecastSheet", json!({"range": "A1:B801", "forecastStart": start, "periods": 601})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 600"), "{err}"); + } + #[test] fn history_and_periods_are_capped() { use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods}; From aa4951fdf4695c71e67e02af15147181c9c657db Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 04:36:48 -0500 Subject: [PATCH 27/51] Security: WORKDAY, PIVOTBY, SORT and the D-functions stop rescanning their input (#3200) * functions: stop WORKDAY, PIVOTBY, SORT and the D-functions rescanning their input - WORKDAY/WORKDAY.INTL counted the holidays each step passed by scanning every holiday, once per step; a run of consecutive holidays made that quadratic. The working-day holidays are kept sorted and counted by binary search. - PIVOTBY totals rescanned the whole source (rebuilding keys) once per total row and column. The grouping pass now also indexes rows by row key and by column key, and totals read from those indexes in the same order. - SORT copied one key per sort_index entry for every row, so a long sort_index with repeated columns multiplied the source. Repeats can't break a tie their first use left, so only first uses are kept. - DSUM and the other database functions tested every record against every criteria row. A selection whose records x conditions exceed 5e8 is #NUM!, and each criteria label's database column is found once, not per cell. * address review: bounds-check SORT indices before narrowing; empty criteria rows are not charged --- crates/functions/src/database.rs | 59 ++++++++++++++++++++++++++------ crates/functions/src/date.rs | 23 ++++++++++++- crates/functions/src/dynamic.rs | 59 ++++++++++++++++++++++++-------- 3 files changed, 115 insertions(+), 26 deletions(-) diff --git a/crates/functions/src/database.rs b/crates/functions/src/database.rs index 37289d9fb..5d77eb350 100644 --- a/crates/functions/src/database.rs +++ b/crates/functions/src/database.rs @@ -70,27 +70,43 @@ fn cell_criterion(v: &Value) -> Option { /// when the label is unknown, criterion). type CriteriaRow = (usize, Vec<(usize, Option, Criterion)>); +/// Most condition tests one selection may make (records times conditions): every record can be +/// tested against every criteria row when none matches, so a modest database and criteria table +/// multiply into billions of tests. Past it the function is `#NUM!`, as lifted lookups are; 5e8 +/// is a few seconds of matching. +const SELECTION_WORK_BUDGET: usize = 500_000_000; + /// Rows (1-based within the database, excluding the header) of matching records. -fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> Vec { +fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> R> { + // The database column under each criteria label, found once per label rather than per cell. + let columns: Vec> = (0..crit.cols) + .map(|c| match crit.get(0, c) { + Some(Value::Text(t)) => find_column(db, t), + Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), + _ => None, + }) + .collect(); // Per criteria row: its index and a list of (criteria column, database column or None when // the label is unknown, criterion). let mut rows: Vec = Vec::new(); + let mut tests = 0usize; for r in 1..crit.rows { let mut conds = Vec::new(); for c in 0..crit.cols { let Some(cell) = crit.get(r, c) else { continue }; let Some(cr) = cell_criterion(cell) else { continue }; - let col = match crit.get(0, c) { - Some(Value::Text(t)) => find_column(db, t), - Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), - _ => None, - }; - conds.push((c, col, cr)); + conds.push((c, columns.get(c).copied().flatten(), cr)); } + tests = tests.saturating_add(conds.len().max(1)); rows.push((r, conds)); } - let all_match = rows.is_empty(); - (1..db.rows) + // An empty criteria row matches every record without testing a condition, so nothing is charged. + let all_match = rows.is_empty() || rows.iter().any(|(_, conds)| conds.is_empty()); + // Computed criteria (a formula evaluated per record) count like any other test. + if !all_match && db.rows.saturating_sub(1).saturating_mul(tests) > SELECTION_WORK_BUDGET { + return Err(CellError::Num); + } + Ok((1..db.rows) .filter(|&rec| { all_match || rows.iter().any(|(r, conds)| { @@ -102,7 +118,7 @@ fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> Vec { }) }) }) - .collect() + .collect()) } struct Selection<'a> { @@ -123,7 +139,7 @@ fn select<'a>(a: &'a [Arg], ctx: &mut dyn Ctx) -> R> { let db = as_array(dbv); let crit = as_array(critv); let field = field_index(&db, a)?; - let records = matching_records(&db, &crit, ctx); + let records = matching_records(&db, &crit, ctx)?; Ok(Selection { db, field, records }) } @@ -352,6 +368,27 @@ mod tests { is_err(d("DSUM", n(9.0)), CellError::Value); } + /// Security: records times criteria conditions past the selection budget is `#NUM!` before + /// any record is tested; a nonmatching criteria table used to test every record against + /// every criteria row. + #[test] + fn selection_work_is_bounded() { + use gridcraft_core::Array; + let column = |label: &str, len: usize, v: Value| { + let mut data = vec![tv(label)]; + data.extend(std::iter::repeat_n(v, len)); + Value::from(Array::column(data)) + }; + let db = column("Tree", 100_000, tv("Apple")); + let wide = column("Tree", 5_001, tv("=Plum")); + is_err(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(wide)]), CellError::Num); + let narrow = column("Tree", 2, tv("=Apple")); + close(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(narrow)]), 100_000.0); + // Empty criteria rows test no condition: the first one matches every record, so they cost nothing. + let empty_rows = column("Tree", 5_001, Value::Empty); + close(ev("DCOUNTA", vec![rf(db), t("Tree"), rf(empty_rows)]), 100_000.0); + } + #[test] fn simple_criteria() { let c = |cells: Vec| arr(vec![vec![tv("Tree"), tv("Height")], cells]); diff --git a/crates/functions/src/date.rs b/crates/functions/src/date.rs index 21d2a49c1..2060267c2 100644 --- a/crates/functions/src/date.rs +++ b/crates/functions/src/date.rs @@ -416,13 +416,17 @@ fn workday_core(a: &[Arg], c: &mut dyn Ctx, mask: [bool; 7], hol: Option<&Arg>) if days == 0 { return num_val(start as f64); } + // The holidays on working days, still sorted (`holidays` sorts them), so each step counts + // the ones it passed with two binary searches rather than a scan of every holiday: a run of + // consecutive holidays takes one step per holiday. + let hs: Vec = hs.into_iter().filter(|&h| !mask[weekday0(sys, h)]).collect(); let dir = days.signum(); let mut cur = start; let mut rem = days.abs(); while rem > 0 { let next = advance(sys, cur, rem, dir, &mask, per_week); let (lo, hi) = if dir > 0 { (cur + 1, next) } else { (next, cur - 1) }; - rem = hs.iter().filter(|&&h| h >= lo && h <= hi && !mask[weekday0(sys, h)]).count() as i64; + rem = hs.partition_point(|&h| h <= hi).saturating_sub(hs.partition_point(|&h| h < lo)) as i64; cur = next; if !(0..=MAX_SERIAL as i64).contains(&cur) { return Err(CellError::Num); @@ -755,6 +759,23 @@ mod tests { is_err(ev("TIMEVALUE", vec![t("x")]), CellError::Value); } + /// Security: WORKDAY counts the holidays each step passes by binary search, so a long run of + /// consecutive holidays is linear work, not one scan of every holiday per holiday. + #[test] + fn workday_over_many_consecutive_holidays() { + let start = d(2000.0, 1.0, 3.0); + let hol: Vec = (1..=200_000).map(|i| start + f64::from(i)).collect(); + let out = ev("WORKDAY.INTL", vec![n(start), n(1.0), n(11.0), av(col(&hol))]); + // Sundays are the only weekend (11): 200,000 days of holidays are skipped, landing on the + // first working day after them. + let after = start + 200_001.0; + let sunday = ev("WEEKDAY", vec![n(after), n(2.0)]) == nv(7.0); + close(out, if sunday { after + 1.0 } else { after }); + // Holidays on a weekend don't push the result out. + let fri = d(2026.0, 10.0, 9.0); + close(ev("WORKDAY", vec![n(fri), n(1.0), av(col(&[fri + 1.0, fri + 3.0]))]), fri + 4.0); + } + #[test] fn day_counts() { close(ev("DAYS", vec![n(d(2021.0, 3.0, 15.0)), n(d(2021.0, 2.0, 1.0))]), 42.0); diff --git a/crates/functions/src/dynamic.rs b/crates/functions/src/dynamic.rs index 816af8ea2..15e9ff7c9 100644 --- a/crates/functions/src/dynamic.rs +++ b/crates/functions/src/dynamic.rs @@ -171,13 +171,17 @@ fn sort(args: &[Arg], _c: &mut dyn Ctx) -> R { let a = if by_col { a.transpose() } else { a }; let idx = if has(args, 1) { int_list(&arg(args, 1)?.value)? } else { vec![1] }; let ord = if has(args, 2) { int_list(&arg(args, 2)?.value)? } else { vec![1] }; - if idx.is_empty() || idx.iter().any(|&i| i < 1 || i as usize > a.cols) || ord.iter().any(|&o| o != 1 && o != -1) { + if idx.is_empty() || idx.iter().any(|&i| i < 1 || i > a.cols as i64) || ord.iter().any(|&o| o != 1 && o != -1) { return Err(CellError::Value); } if ord.len() > 1 && ord.len() != idx.len() { return Err(CellError::Value); } let orders: Vec = (0..idx.len()).map(|i| *ord.get(i).or(ord.first()).unwrap_or(&1)).collect(); + // A column repeated in sort_index can't break a tie its first use left, so only first uses + // stay: the keys copied per row are then at most the row itself, however long sort_index is. + let mut seen = std::collections::HashSet::new(); + let (idx, orders): (Vec, Vec) = idx.into_iter().zip(orders).filter(|(i, _)| seen.insert(*i)).unzip(); let mut lines: Vec<(Vec, Vec)> = rows_of(&a).into_iter().map(|r| (idx.iter().map(|&i| r.get(i as usize - 1).cloned().unwrap_or(Value::Empty)).collect(), r)).collect(); lines.sort_by(|x, y| cmp_keys(&x.0, &y.0, &orders)); @@ -928,9 +932,16 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { let ckeys: Vec> = p.rows.iter().map(|&r| key_row(&cf, r)).collect(); let rk = distinct_keys(&rkeys, if row_sort.unsigned_abs() as usize <= rf.cols { row_sort } else { 0 }); let ck = distinct_keys(&ckeys, if col_sort.unsigned_abs() as usize <= cf.cols { col_sort } else { 0 }); + // Source rows of every row key, column key and pair, built in one pass, so a total reads + // its rows instead of rescanning the whole source once per total. let mut cell: HashMap<(String, String), Vec> = HashMap::new(); + let mut by_row: HashMap> = HashMap::new(); + let mut by_col: HashMap> = HashMap::new(); for ((r, c), &i) in rkeys.iter().zip(ckeys.iter()).zip(p.rows.iter()) { - cell.entry((row_key(r), row_key(c))).or_default().push(i); + let (r, c) = (row_key(r), row_key(c)); + by_row.entry(r.clone()).or_default().push(i); + by_col.entry(c.clone()).or_default().push(i); + cell.entry((r, c)).or_default().push(i); } let mut agg = |rows: &[usize], c: usize| -> Value { if rows.is_empty() { @@ -940,14 +951,12 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { aggregate_with(f, &vs, ctx) }; let rows_for = |r: Option<&Vec>, c: Option<&Vec>| -> Vec { - let rkey = r.map(|k| row_key(k)); - let ckey = c.map(|k| row_key(k)); - p.rows - .iter() - .zip(rkeys.iter().zip(ckeys.iter())) - .filter(|(_, (a, b))| rkey.as_ref().is_none_or(|k| *k == row_key(a)) && ckey.as_ref().is_none_or(|k| *k == row_key(b))) - .map(|(&i, _)| i) - .collect() + match (r, c) { + (Some(r), Some(c)) => cell.get(&(row_key(r), row_key(c))).cloned().unwrap_or_default(), + (Some(r), None) => by_row.get(&row_key(r)).cloned().unwrap_or_default(), + (None, Some(c)) => by_col.get(&row_key(c)).cloned().unwrap_or_default(), + (None, None) => p.rows.clone(), + } }; let mut cols: Vec>> = ck.iter().map(Some).collect(); if col_total > 0 { @@ -997,10 +1006,7 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { } }; for c in &cols { - let rows = match (r, c) { - (Some(rk), Some(ck)) => cell.get(&(row_key(rk), row_key(ck))).cloned().unwrap_or_default(), - _ => rows_for(r, *c), - }; + let rows = rows_for(r, *c); for vc in 0..vals.cols { line.push(agg(&rows, vc)); } @@ -1204,6 +1210,21 @@ mod tests { assert_eq!(rows_of(&r), vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]); } + /// Security: a column repeated in sort_index keys the sort once. The keys copied per row stay + /// within the row, however long sort_index is, and a repeat (even with the other order) can't + /// change the result, since its first use already ordered the rows. + #[test] + fn sort_drops_repeated_sort_indices() { + let m = || av(arr(vec![vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)], vec![nv(0.0), nv(5.0)]])); + let want = vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]; + let r = ev("SORT", vec![m(), av(row(&[1.0, 1.0, 2.0, 1.0])), av(row(&[1.0, -1.0, -1.0, 1.0]))]); + assert_eq!(rows_of(&r), want); + let idx = vec![1.0; 1_000_000]; + let tall = av(gridcraft_core::Array::filled(2_000, 1, nv(1.0)).into()); + let r = ev("SORT", vec![tall, av(row(&idx))]); + assert_eq!(rows_of(&r).len(), 2_000); + } + #[test] fn sortby() { let r = ev("SORTBY", vec![av(col(&[1.0, 2.0, 3.0])), av(col(&[30.0, 10.0, 20.0]))]); @@ -1447,6 +1468,16 @@ mod tests { ); } + /// Security: PIVOTBY totals read their source rows from indexes built in the grouping pass, + /// rather than rescanning the source once per total, so many distinct keys stay linear. + #[test] + fn pivotby_totals_with_many_groups() { + let keys: Vec = (0..20_000).map(f64::from).collect(); + let r = rows_of(&ev("PIVOTBY", vec![av(col(&keys)), av(col(&vec![1.0; 20_000])), av(col(&vec![2.0; 20_000])), t("SUM")])); + assert_eq!(r.get(1), Some(&vec![nv(0.0), nv(2.0), nv(2.0)])); + assert_eq!(r.last(), Some(&vec![tv("Total"), nv(40_000.0), nv(40_000.0)])); + } + #[test] fn pivotby_size_check_counts_column_header_rows() { // One row, n column fields and n value columns: n header rows + value-name row + body row + total row, From bbe082540474bec78058904fd6ba8b54654e8d2e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 04:55:21 -0500 Subject: [PATCH 28/51] Security: TEXTJOIN builds its result piece by piece within the text cap (#3202) * functions: TEXTJOIN builds its result piece by piece within the text cap TEXTJOIN converted every input and delimiter into an owned String before joining, so many references to one long shared text copied gigabytes before the joined length was ever checked. Errors are now found first without building text (so they still win in argument order), then each piece and delimiter is converted only as it is appended, stopping at the cap. * address review: check the cap before copying each text piece --- crates/functions/src/text.rs | 80 ++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 18 deletions(-) diff --git a/crates/functions/src/text.rs b/crates/functions/src/text.rs index 9d61340d2..53186bb0e 100644 --- a/crates/functions/src/text.rs +++ b/crates/functions/src/text.rs @@ -30,17 +30,27 @@ fn check_len(s: String) -> R { /// Pushes `piece` onto `out`, failing once the result exceeds the cell limit. fn push_capped(out: &mut String, piece: &str) -> R<()> { - out.push_str(piece); - if out.len() > MAX_TEXT * 4 { + // Checked before copying, so one oversized piece is refused without being appended. + if out.len().saturating_add(piece.len()) > MAX_TEXT * 4 { return Err(CellError::Value); } + out.push_str(piece); Ok(()) } +/// `v.to_text()` without copying text values. +fn piece(v: &Value) -> R> { + match v { + Value::Text(t) => Ok(std::borrow::Cow::Borrowed(t)), + Value::Array(a) => a.data.first().map_or(Ok(std::borrow::Cow::Borrowed("")), piece), + _ => v.to_text().map(std::borrow::Cow::Owned), + } +} + fn concatenate(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { - push_capped(&mut out, &scalar(x).to_text()?)?; + push_capped(&mut out, &piece(&scalar(x))?)?; } check_len(out) } @@ -49,31 +59,49 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { for v in as_array(&x.value).iter() { - push_capped(&mut out, &v.to_text()?)?; + push_capped(&mut out, &piece(v)?)?; } } check_len(out) } +/// The error `v.to_text()` would give, found without building any text. +fn text_error(v: &Value) -> Option { + match v { + Value::Error(e) => Some(*e), + Value::Array(a) => a.data.first().and_then(text_error), + _ => None, + } +} + fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R { - let delims: Vec = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::>()?; + // Errors are found first, in argument order, so they win as before; text is then built one + // piece at a time into the capped output, so many references to one long text (or a long + // delimiter list) stop at the cap instead of first copying every piece. + let delims = as_array(&arg(a, 0)?.value); + if let Some(e) = delims.iter().find_map(text_error) { + return Err(e); + } let ignore = boolean(a, 1)?; - let mut items = Vec::new(); - for x in a.iter().skip(2) { - for v in as_array(&x.value).iter() { - let s = v.to_text()?; - if ignore && s.is_empty() { - continue; - } - items.push(s); - } + let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect(); + if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) { + return Err(e); } let mut out = String::new(); - for (i, s) in items.iter().enumerate() { - if i > 0 && !delims.is_empty() { - push_capped(&mut out, &delims[(i - 1) % delims.len()])?; + let mut joined = 0usize; + for v in items.iter().flat_map(|x| x.iter()) { + let s = piece(v)?; + if ignore && s.is_empty() { + continue; } - push_capped(&mut out, s)?; + if joined > 0 + && !delims.data.is_empty() + && let Some(d) = delims.data.get((joined - 1) % delims.data.len()) + { + push_capped(&mut out, &piece(d)?)?; + } + push_capped(&mut out, &s)?; + joined += 1; } check_len(out) } @@ -1183,6 +1211,22 @@ mod tests { assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]); } + /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather + /// than first copying every input (100,000 references to one 32,767-character text were + /// 3 GB of copies). Errors among the inputs still win, in argument order. + #[test] + fn textjoin_stops_at_the_text_cap() { + use gridcraft_core::Array; + let long = tv(&"x".repeat(32_767)); + let many = || av(Array::filled(100_000, 1, long.clone()).into()); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value); + is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0); + is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA); + // Delimiters cycle and are converted only where used. + is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d"); + } + fn row_t(xs: &[&str]) -> Value { arr(vec![xs.iter().map(|x| tv(x)).collect()]) } From 67a0b7f32508770e27b2e69edeec582c45e78464 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 05:07:04 -0500 Subject: [PATCH 29/51] Security: chart rasterising, spline points and category labels stay bounded (#3204) * chart: bound scan conversion, spline points and category label bytes - The rasteriser tested every edge of a path at every sub-sample of every covered row. It now keeps an active-edge list (edges sorted by top, joining and leaving as the scan passes them), and one rasterize call has a total scan-work allowance after which remaining fills are skipped. - Smoothed lines budgeted interpolated points per run, so gap-separated short runs escaped both decimation and the cap. The budget is now shared by all runs of a line; once spent, runs are drawn through their data points. - Category labels copied every text value in full. Labels are clipped to 255 characters before copying, within an 8 MiB budget per series. * address review: chart raster and label budget fixes * address review: bound polyline point count before stroke_paths --- crates/chart/src/raster.rs | 91 +++++++++++++++++++++-- crates/chart/src/render/lines.rs | 40 +++++++++-- crates/chart/src/resolve.rs | 119 ++++++++++++++++++++++++++----- 3 files changed, 221 insertions(+), 29 deletions(-) diff --git a/crates/chart/src/raster.rs b/crates/chart/src/raster.rs index 0de2450b5..80042b970 100644 --- a/crates/chart/src/raster.rs +++ b/crates/chart/src/raster.rs @@ -7,6 +7,20 @@ use crate::{Prim, Rgba}; /// Largest image we allocate (pixels). const MAX_PIXELS: u64 = 64 * 1024 * 1024; const SUB: usize = 4; +/// Scan-conversion work one [`rasterize`] call may do: edge tests (an active edge at one +/// sub-sample) plus covered pixels blended. Edges are only tested on the rows they cross, but a +/// caller can still pass enough geometry to keep the rasteriser busy for minutes on a small +/// image; once the allowance is spent the remaining fills are skipped. 2^28 is about a second, +/// far beyond any chart. +const MAX_SCAN_WORK: u64 = 1 << 28; +/// Most points one filled path may have: more is skipped before any edge is built. Chart +/// renderers cap and decimate their paths far below this; it only bounds direct callers. +const MAX_FILL_POINTS: usize = 1 << 22; + +/// Most points one stroked polyline may have: more is skipped before any segment path is built +/// (each segment becomes its own quad). Charts hold at most 100,000 points per series and draw far +/// fewer; it only bounds direct callers. +const MAX_STROKE_POINTS: usize = 1 << 17; type Path = Vec>; @@ -15,6 +29,8 @@ struct Canvas { h: usize, buf: Vec, acc: Vec, + /// Scan-conversion work left (see [`MAX_SCAN_WORK`]). + work: u64, } impl Canvas { @@ -41,7 +57,12 @@ impl Canvas { /// Fills a multi-contour path with the even-odd rule. fn fill(&mut self, path: &Path, color: Rgba) { - let mut edges: Vec<[f32; 4]> = Vec::new(); + // Out of work, or more geometry than the allowance could ever scan: no edge list is built. + if self.work == 0 || path.iter().map(Vec::len).fold(0usize, usize::saturating_add) > MAX_FILL_POINTS { + return; + } + // Each edge as [top y, bottom y, x0, y0, x1, y1]. + let mut edges: Vec<[f32; 6]> = Vec::new(); let (mut minx, mut miny, mut maxx, mut maxy) = (f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY); for c in path { if c.len() < 3 || !c.iter().all(|p| p[0].is_finite() && p[1].is_finite()) { @@ -54,7 +75,7 @@ impl Canvas { miny = miny.min(p[1]); maxy = maxy.max(p[1]); if p[1] != q[1] { - edges.push([p[0], p[1], q[0], q[1]]); + edges.push([p[1].min(q[1]), p[1].max(q[1]), p[0], p[1], q[0], q[1]]); } } } @@ -72,8 +93,28 @@ impl Canvas { if self.acc.len() < bw + 1 { self.acc.resize(bw + 1, 0.0); } + // Active-edge list: edges join when the scan reaches their top and leave past their + // bottom, so each row tests only the edges crossing it instead of every edge. + edges.sort_by(|a, b| a[0].total_cmp(&b[0])); + let mut pending = edges.iter().peekable(); + let mut active: Vec<[f32; 6]> = Vec::new(); let mut xs: Vec = Vec::new(); for py in y0..y1 { + while let Some(e) = pending.next_if(|e| e[0] < py as f32 + 1.0) { + active.push(*e); + } + active.retain(|e| e[1] > py as f32); + if active.is_empty() { + continue; + } + // The row's edge tests and its `bw` accumulator clears and blends, charged before any of + // them, so rows with no coverage still count. + let cost = (active.len() * SUB) as u64 + bw as u64; + if cost > self.work { + self.work = 0; + return; + } + self.work -= cost; for v in self.acc.iter_mut().take(bw) { *v = 0.0; } @@ -81,10 +122,9 @@ impl Canvas { for s in 0..SUB { let sy = py as f32 + (s as f32 + 0.5) / SUB as f32; xs.clear(); - for e in &edges { - let (ya, yb) = if e[1] < e[3] { (e[1], e[3]) } else { (e[3], e[1]) }; - if sy >= ya && sy < yb { - xs.push(e[0] + (sy - e[1]) * (e[2] - e[0]) / (e[3] - e[1])); + for e in &active { + if sy >= e[0] && sy < e[1] { + xs.push(e[2] + (sy - e[3]) * (e[4] - e[2]) / (e[5] - e[3])); } } xs.sort_by(f32::total_cmp); @@ -154,6 +194,9 @@ fn rounded_rect(x: f32, y: f32, w: f32, h: f32, r: f32) -> Vec<[f32; 2]> { /// Thick polyline → one quad per segment (each a separate path so overlaps don't cancel). fn stroke_paths(pts: &[[f32; 2]], width: f32, dash: bool, closed: bool) -> Vec { + if pts.len() > MAX_STROKE_POINTS { + return Vec::new(); + } let width = width.clamp(1.0, 50.0); let mut segs: Vec<([f32; 2], [f32; 2])> = pts.windows(2).map(|w| (w[0], w[1])).collect(); if closed && let (Some(a), Some(b)) = (pts.last(), pts.first()) { @@ -212,7 +255,7 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { for _ in 0..w * h { buf.extend_from_slice(&background); } - let mut cv = Canvas { w, h, buf, acc: Vec::new() }; + let mut cv = Canvas { w, h, buf, acc: Vec::new(), work: MAX_SCAN_WORK }; for p in prims { match p { Prim::Rect { x, y, w, h, fill, stroke, radius } => { @@ -274,6 +317,40 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { mod tests { use super::*; + /// Security: each row tests only the edges crossing it. A polygon of 100,000 tiny edges over + /// 4,000 rows tested every edge at every sub-sample of every row (1.6 billion tests). + #[test] + fn many_short_edges_scan_only_their_rows() { + let mut pts: Vec<[f32; 2]> = (0..100_000).map(|i| [if i % 2 == 0 { 0.0 } else { 0.5 }, i as f32 * 0.04]).collect(); + pts.push([2.0, 4_000.0]); + pts.push([2.0, 0.0]); + let img = rasterize(&[Prim::Polygon { pts, fill: [255, 0, 0, 255], stroke: None }], 2, 4_000, [0, 0, 0, 0]); + // The right-hand pixel of every row is inside the polygon. + assert_eq!(img.get(4..8), Some(&[255, 0, 0, 255][..])); + assert_eq!(img.get(4 * (2 * 3_999 + 1)..4 * (2 * 3_999 + 2)), Some(&[255, 0, 0, 255][..])); + } + + /// Security: scan conversion stops once the call's work allowance is spent. + #[test] + fn fills_stop_when_the_work_allowance_is_spent() { + let square = vec![vec![[0.0, 0.0], [4.0, 0.0], [4.0, 4.0], [0.0, 4.0]]]; + let mut cv = Canvas { w: 4, h: 4, buf: vec![0; 64], acc: Vec::new(), work: 2 * SUB as u64 * 4 + 16 }; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 255)); + cv.buf = vec![0; 64]; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 0)); + } + + /// Security: a polyline over the point cap builds no segment paths at all, solid or dashed. + #[test] + fn oversized_polylines_build_no_segment_paths() { + let line = |n: usize| -> Vec<[f32; 2]> { (0..n).map(|i| [i as f32, (i % 2) as f32]).collect() }; + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, false, false).is_empty()); + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, true, false).is_empty()); + assert_eq!(stroke_paths(&line(MAX_STROKE_POINTS), 1.0, false, false).len(), MAX_STROKE_POINTS - 1); + } + #[test] fn dashed_geometry_stays_within_the_dash_budget() { // Forty very long dashed lines, each allowed 10_000 dashes per segment: before the diff --git a/crates/chart/src/render/lines.rs b/crates/chart/src/render/lines.rs index 770aabe02..678181326 100644 --- a/crates/chart/src/render/lines.rs +++ b/crates/chart/src/render/lines.rs @@ -136,7 +136,13 @@ pub(super) fn decimate_area(top: Vec<[f32; 2]>, base: Vec<[f32; 2]>, max: usize) /// Draws a polyline split at gaps (`None`), decimated when long. With `smooth` (the plot box), /// each run is interpolated with a Catmull-Rom spline so the line curves through the data points. +/// +/// The points of the smoothed runs (data points included) are budgeted across all the runs of the line, not per run: +/// short runs between gaps escape decimation, and 100 sub-segments each would turn a +/// 100,000-point series of three-point runs into millions of points. Past the budget the +/// remaining runs are drawn through their data points only. pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, width: f32, smooth: Option) { + let mut budget = MAX_LINE_PTS * 2; let mut run: Vec<[f32; 2]> = Vec::new(); for p in pts.iter().chain(std::iter::once(&None)) { match p { @@ -145,7 +151,11 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi if run.len() >= 2 { let r = decimate(std::mem::take(&mut run), MAX_LINE_PTS); let r = match smooth { - Some(plot) => smooth_run(r, plot), + Some(plot) => { + let r = smooth_run(r, plot, budget); + budget = budget.saturating_sub(r.len()); + r + } None => r, }; ctx.line(r, color, width); @@ -158,15 +168,15 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi } /// Catmull-Rom spline through `pts`: every segment gets the same number of sub-segments (up to -/// 100), chosen so the whole curve stays within `MAX_LINE_PTS * 2` points. Endpoints are +/// 100), chosen so the whole curve stays within `cap` points (one per data point when `cap` is +/// smaller than that). Endpoints are /// duplicated so the curve passes through the first and last points, and the curve is clamped /// to the `plot` box (a spline overshoots its data near sharp turns). -fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx) -> Vec<[f32; 2]> { +fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx, cap: usize) -> Vec<[f32; 2]> { let n = pts.len(); if n < 3 { return pts; } - let cap = MAX_LINE_PTS * 2; let steps = (cap / n).clamp(1, 100); let mut out: Vec<[f32; 2]> = Vec::with_capacity(n.saturating_mul(steps).min(cap) + 1); let at = |i: isize| -> [f32; 2] { @@ -219,7 +229,7 @@ mod smooth_tests { fn long_runs_are_smoothed_uniformly_within_the_cap() { let plot = Bx { x: 0.0, y: 0.0, w: 3000.0, h: 100.0 }; let pts: Vec<[f32; 2]> = (0..1500).map(|i| [i as f32 * 2.0, if i % 2 == 0 { 20.0 } else { 80.0 }]).collect(); - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() <= MAX_LINE_PTS * 2 + 1, "{}", out.len()); // Two sub-segments per segment all the way to the end, not 100 each and then a straight tail. assert_eq!(out.len(), 1499 * 2 + 1); @@ -231,11 +241,29 @@ mod smooth_tests { let plot = Bx { x: 0.0, y: 10.0, w: 300.0, h: 100.0 }; // A step right at the plot edges: Catmull-Rom overshoots past both. let pts = vec![[0.0, 110.0], [100.0, 110.0], [110.0, 10.0], [200.0, 10.0], [300.0, 10.0]]; - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() > 5); assert!(out.iter().all(|p| (0.0..=300.0).contains(&p[0]) && (10.0..=110.0).contains(&p[1]))); } + /// Security: smoothing budgets its added points across every run of a line. A series of + /// three-point runs between gaps used to get 100 sub-segments per run, 67 times its data. + #[test] + fn gap_separated_runs_share_one_smoothing_budget() { + let plot = Bx { x: 0.0, y: 0.0, w: 1e6, h: 100.0 }; + let pts: Vec> = + (0..100_000).map(|i| if i % 4 == 3 { None } else { Some([i as f32, if i % 2 == 0 { 20.0 } else { 80.0 }]) }).collect(); + let m = crate::ApproxMeasure; + let mut ctx = Ctx { out: Vec::new(), m: &m }; + draw_runs(&mut ctx, &pts, WHITE, 1.0, Some(plot)); + let total: usize = ctx.out.iter().map(|p| if let Prim::Line { pts, .. } = p { pts.len() } else { 0 }).sum(); + // 75,000 data points, plus at most the budget added by interpolation. + assert!(total <= 75_000 + MAX_LINE_PTS * 2, "{total}"); + assert_eq!(ctx.out.len(), 25_000); + // The first runs are still smoothed. + assert!(matches!(ctx.out.first(), Some(Prim::Line { pts, .. }) if pts.len() > 3)); + } + #[test] fn decimate_area_preserves_paired_vertical_strips() { let top: Vec<[f32; 2]> = (0..1000).map(|i| [i as f32 * 0.25, ((i * 37) % 100) as f32]).collect(); diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index 11213a5e7..baa0730ad 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -184,6 +184,53 @@ fn value_text(v: &Value) -> String { } } +/// Characters kept of one category label: an axis label never shows more (labels are fitted to +/// the axis), and copying more only costs memory. +const MAX_LABEL_CHARS: usize = 255; +/// Bytes of category label text kept per series. One long text broadcast by a short formula +/// (`IF(SEQUENCE(100000), long_text)`) is a single shared value, but turning every element into +/// its own label copied it 100,000 times (gigabytes). Labels past the budget are empty. +const MAX_CATEGORY_BYTES: usize = 8 << 20; + +/// At most [`MAX_LABEL_CHARS`] characters and `room` bytes of `s`, cut at a character boundary. +fn clip_label(s: &str, room: usize) -> String { + let mut end = 0; + for (n, (i, c)) in s.char_indices().enumerate() { + if n >= MAX_LABEL_CHARS || i + c.len_utf8() > room { + break; + } + end = i + c.len_utf8(); + } + s.get(..end).unwrap_or_default().to_string() +} + +/// The label of category `i` (value `v`) in at most `room` bytes: formatted with its source +/// cell's number format when it is a number. +fn category_label(wb: &Workbook, src: Option<(&gridcraft_model::Sheet, RangeRef)>, i: usize, v: &Value, room: usize) -> String { + if room == 0 { + return String::new(); // nothing fits: skip the number formatting + } + if let (Some((s, r)), Value::Number(_)) = (src, v) { + let w = r.width().max(1) as usize; + let at = r.start.offset((i / w) as i64, (i % w) as i64); + if let Some(c) = at { + let code = &wb.styles.get(s.style_id(c)).num_fmt.0; + if !code.eq_ignore_ascii_case("general") { + return clip_label(&format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text, room); + } + } + } + // Text is clipped before it is copied: the source may be one long shared string. + let mut v = v; + while let Value::Array(a) = v { + v = a.get(0, 0).unwrap_or(&Value::Empty); + } + match v { + Value::Text(t) => clip_label(t, room), + _ => clip_label(&value_text(v), room), + } +} + /// Category labels, formatted with each source cell's number format (dates, currency…), and /// the outer levels of a multi-level axis: a category range wider than a column of values (or /// taller than a row of values) groups its innermost labels, year beside month. Each outer level @@ -221,29 +268,29 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi None => eval_list_in(wb, sheet, formula, limit, 0), }; let src = cells.and_then(|(sh, r)| wb.sheet(sh).map(|s| (s, r))); + // The retained levels are known before any label is charged: a cell on a discarded level + // gets no label and spends none of the budget. `None`: a flat list (every cell is a label). + let layout = src.and_then(|(_, r)| { + let levels = levels?; + let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); + (vals.len() == w.saturating_mul(h)).then_some((levels, w, h)) // else truncated or not a plain block + }); + let kept = + |i: usize| layout.is_none_or(|(l, w, h)| if l.by_columns { i % w + 1 + l.outer as usize >= w } else { i / w + 1 + l.outer as usize >= h }); + let mut room = MAX_CATEGORY_BYTES; let texts: Vec = vals .iter() .enumerate() .map(|(i, v)| { - if let (Some((s, r)), Value::Number(_)) = (src, v) { - let w = r.width().max(1) as usize; - let at = r.start.offset((i / w) as i64, (i % w) as i64); - if let Some(c) = at { - let code = &wb.styles.get(s.style_id(c)).num_fmt.0; - if !code.eq_ignore_ascii_case("general") { - return format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text; - } - } + if !kept(i) { + return String::new(); } - value_text(v) + let label = category_label(wb, src, i, v, room); + room = room.saturating_sub(label.len()); + label }) .collect(); - let Some((_, r)) = src else { return (texts, vec![]) }; - let Some(levels) = levels else { return (texts, vec![]) }; - let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); - if texts.len() != w.saturating_mul(h) { - return (texts, vec![]); // truncated or not a plain block: a flat list - } + let Some((levels, w, h)) = layout else { return (texts, vec![]) }; let at = |row: usize, col: usize| texts.get(row * w + col).cloned().unwrap_or_default(); let column = |c: usize| (0..h).map(|row| at(row, c)).collect::>(); let row = |rr: usize| (0..w).map(|c| at(rr, c)).collect::>(); @@ -369,6 +416,46 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData { ChartData { categories, category_levels, series, palette } } +#[cfg(test)] +mod label_budget_tests { + use gridcraft_model::Workbook; + + /// Security: category labels are clipped before they are copied, with one byte budget per + /// series. A formula broadcasting one long text to 100,000 categories used to copy it + /// 100,000 times (3.3 GB). + #[test] + fn category_labels_share_one_byte_budget() { + let wb = Workbook::new(); + let (texts, _) = super::category_table(&wb, 0, r#"=IF(SEQUENCE(100000),REPT("x",32767))"#, "", super::MAX_POINTS); + assert_eq!(texts.len(), 100_000); + assert_eq!(texts.first().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert!(texts.iter().map(String::len).sum::() <= super::MAX_CATEGORY_BYTES); + // Labels are cut at character boundaries. + assert_eq!(super::clip_label("aé", 2), "a"); + assert_eq!(super::clip_label("aé", 3), "aé"); + } + + /// Cells on discarded outer levels (only the innermost labels and three outer levels are + /// kept) spend none of the byte budget. + #[test] + fn discarded_levels_do_not_spend_the_label_budget() { + let mut wb = Workbook::new(); + let rows = 8000u32; + if let Some(sh) = wb.sheet_mut(0) { + for r in 0..rows { + for c in 0..5 { + sh.set_value(gridcraft_core::CellRef::new(r, c), gridcraft_core::Value::text("x".repeat(255))); + } + } + } + // Five columns: column A is a fourth outer level and is discarded. + let (inner, outer) = super::category_table(&wb, 0, "=A1:E8000", "", super::MAX_POINTS); + assert_eq!(outer.len(), 3); + assert_eq!(inner.last().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert_eq!(outer.first().and_then(|l| l.last()).map(String::len), Some(super::MAX_LABEL_CHARS)); + } +} + #[cfg(test)] mod tests { use gridcraft_model::Workbook; From 542aa176cee615abcade390c701df28f6855884e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 05:19:24 -0500 Subject: [PATCH 30/51] Wildcards: match * and ? patterns in one pass over the text (#3205) * Wildcards: match * and ? patterns in one pass over the text After a mismatch the matcher restarted the pattern after the last star at the next text position, so a long repetitive cell against a near-matching pattern cost text x pattern comparisons. AutoFilter custom criteria, Find and Replace all call it once per cell. wildcard() now runs the pattern as a bit-parallel automaton: one pass over the text, one step per 64 pattern positions for each character. Results are unchanged; a test compares it with the old matcher on every short text and pattern over a small alphabet. * address review: build wildcard literal masks lazily with a bounded cache * address review: keep masks of the most frequent wildcard literals past the cache cap --- crates/engine/src/cmd/edit/find.rs | 100 ++++++++++++++++++++++------ crates/engine/src/cmd/edit/tests.rs | 97 +++++++++++++++++++++++++++ 2 files changed, 176 insertions(+), 21 deletions(-) diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index 3dfbc6cbe..3efcdaaf4 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -14,8 +14,13 @@ fn matches(hay: &str, what: &str, case: bool, whole: bool) -> bool { } /// `*`/`?` wildcard match with `~` escapes. +/// +/// The pattern runs as a bit-parallel automaton over the text: one pass, each character costing +/// one step per 64 pattern positions. Backtracking to the last `*` after a mismatch rescans the +/// text and costs text × pattern comparisons on a long, repetitive cell, for every cell a +/// filter or Find visits. pub(crate) fn wildcard(text: &str, pat: &str) -> bool { - let t: Vec = text.chars().collect(); + // The pattern as (character, escaped) items, runs of `*` folded into one. let mut p: Vec<(char, bool)> = Vec::new(); let mut it = pat.chars(); while let Some(c) = it.next() { @@ -23,32 +28,85 @@ pub(crate) fn wildcard(text: &str, pat: &str) -> bool { if let Some(n) = it.next() { p.push((n, true)); } - } else { + } else if c != '*' || p.last() != Some(&('*', false)) { p.push((c, false)); } } - let (mut ti, mut pi) = (0usize, 0usize); - let (mut star, mut mark) = (None, 0usize); - while ti < t.len() { - if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { - ti += 1; - pi += 1; - } else if pi < p.len() && p[pi] == ('*', false) { - star = Some(pi); - mark = ti; - pi += 1; - } else if let Some(sp) = star { - pi = sp + 1; - mark += 1; - ti = mark; - } else { - return false; + let words = (p.len() + 1).div_ceil(64); + let bit = |set: &mut [u64], i: usize| set[i / 64] |= 1 << (i % 64); + // Positions holding a `*` and a `?`, and where each literal character sits in the pattern. + let mut stars = vec![0u64; words]; + let mut any = vec![0u64; words]; + let mut at: std::collections::HashMap> = std::collections::HashMap::new(); + for (i, item) in p.iter().enumerate() { + match item { + ('*', false) => bit(&mut stars, i), + ('?', false) => bit(&mut any, i), + (c, _) => at.entry(*c).or_default().push(i), } } - while pi < p.len() && p[pi] == ('*', false) { - pi += 1; + // A literal's bit mask is built when the text first reads it, and kept for at most + // `MAX_CACHED_MASKS` literals: a long pattern of mostly distinct characters would otherwise + // allocate a mask of the pattern's width for each one up front. When there are more literals + // than that, the ones kept are those at the most pattern positions (the dearest to rebuild); + // each of the rest has at most pattern length / `MAX_CACHED_MASKS` positions, so rebuilding + // its mask in `scratch` costs less than one step of the scan. + const MAX_CACHED_MASKS: usize = 256; + let cacheable: Option> = (at.len() > MAX_CACHED_MASKS).then(|| { + let mut by_count: Vec<(&char, usize)> = at.iter().map(|(c, v)| (c, v.len())).collect(); + by_count.sort_unstable_by_key(|&(_, n)| std::cmp::Reverse(n)); + by_count.into_iter().take(MAX_CACHED_MASKS).map(|(c, _)| *c).collect() + }); + let mut masks: std::collections::HashMap> = std::collections::HashMap::new(); + let mut scratch = vec![0u64; words]; + // A set bit i: the text read so far matches the first i pattern items. A `*` matches nothing + // as well, so a state at a `*` is also at the item after it. + let close = |set: &mut Vec| { + let mut carry = 0u64; + for w in 0..words { + let s = set[w] & stars[w]; + let next = (s << 1) | carry; + carry = s >> 63; + set[w] |= next; + } + }; + let mut state = vec![0u64; words]; + bit(&mut state, 0); + close(&mut state); + let mut next = vec![0u64; words]; + for ch in text.chars() { + let lit: Option<&[u64]> = match at.get(&ch) { + None => None, + Some(positions) if cacheable.as_ref().is_none_or(|k| k.contains(&ch)) => { + let mask = masks.entry(ch).or_insert_with(|| { + let mut m = vec![0u64; words]; + positions.iter().for_each(|&i| bit(&mut m, i)); + m + }); + Some(mask) + } + Some(positions) => { + scratch.fill(0); + positions.iter().for_each(|&i| bit(&mut scratch, i)); + Some(&scratch) + } + }; + let mut carry = 0u64; + let mut alive = 0u64; + for w in 0..words { + // Items that read `ch` advance; a `*` reads it and stays. + let m = state[w] & (any[w] | lit.map_or(0, |l| l[w])); + next[w] = (m << 1) | carry | (state[w] & stars[w]); + carry = m >> 63; + alive |= next[w]; + } + if alive == 0 { + return false; + } + std::mem::swap(&mut state, &mut next); + close(&mut state); } - pi == p.len() + state[p.len() / 64] & (1 << (p.len() % 64)) != 0 } pub(super) fn find(s: &mut Session, p: &Json) -> Result { diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index 0400e61e8..160ce9157 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -324,6 +324,103 @@ fn wildcards() { assert!(wildcard("", "*")); } +/// The backtracking matcher `wildcard` replaced, as the reference for its results. +fn wildcard_backtracking(text: &str, pat: &str) -> bool { + let t: Vec = text.chars().collect(); + let mut p: Vec<(char, bool)> = Vec::new(); + let mut it = pat.chars(); + while let Some(c) = it.next() { + if c == '~' { + if let Some(n) = it.next() { + p.push((n, true)); + } + } else { + p.push((c, false)); + } + } + let (mut ti, mut pi) = (0usize, 0usize); + let (mut star, mut mark) = (None, 0usize); + while ti < t.len() { + if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { + ti += 1; + pi += 1; + } else if pi < p.len() && p[pi] == ('*', false) { + star = Some(pi); + mark = ti; + pi += 1; + } else if let Some(sp) = star { + pi = sp + 1; + mark += 1; + ti = mark; + } else { + return false; + } + } + while pi < p.len() && p[pi] == ('*', false) { + pi += 1; + } + pi == p.len() +} + +#[test] +fn wildcard_agrees_with_backtracking() { + // Every text up to 5 characters over a small alphabet against every pattern up to 4 items. + let alphabet = ['a', 'b', '*', '?', '~']; + let all = |n: usize, chars: &[char]| -> Vec { + let mut out = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..n { + level = level.iter().flat_map(|s| chars.iter().map(move |c| format!("{s}{c}"))).collect(); + out.extend(level.iter().cloned()); + } + out + }; + let texts = all(5, &['a', 'b', '*']); + for pat in all(4, &alphabet) { + for text in &texts { + assert_eq!(wildcard(text, &pat), wildcard_backtracking(text, &pat), "{text:?} ~ {pat:?}"); + } + } + // Patterns longer than one 64-bit word. + let long = format!("*{}*b?", "a".repeat(70)); + for text in [format!("x{}yb1", "a".repeat(70)), format!("{}b", "a".repeat(71)), "a".repeat(200)] { + assert_eq!(wildcard(&text, &long), wildcard_backtracking(&text, &long), "{text}"); + } +} + +#[test] +fn wildcard_with_many_distinct_literals_agrees_with_backtracking() { + // 600 distinct characters in the pattern: more than the masks kept, so the rest are rebuilt. + let chars: Vec = (0x4E00u32..0x4E00 + 600).filter_map(char::from_u32).collect(); + let word: String = chars.iter().collect(); + let pat = format!("*{word}?{}", chars[3]); + for text in [format!("x{word}yq{}", chars[3]), word.clone(), format!("{word}{word}"), chars.iter().rev().collect::()] { + assert_eq!(wildcard(&text, &pat), wildcard_backtracking(&text, &pat), "{} chars", text.chars().count()); + } + assert!(wildcard(&format!("{word}zz{}", chars[3]).replace("zz", "z"), &pat)); +} + +#[test] +fn wildcard_keeps_masks_of_frequent_literals_past_the_cache_limit() { + // 300 distinct characters come before the repeated `a`: past the cached masks, `a` (5,000 positions) + // still has to be read from a kept mask, or it is rebuilt for each of the 20,000 `a` in the text. + let rare: String = (0x4E00u32..0x4E00 + 300).filter_map(char::from_u32).collect(); + let pat = format!("*{rare}{}b", "a".repeat(5_000)); + let text = "a".repeat(20_000); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{rare}{}b", "a".repeat(5_000)), &pat)); + assert!(!wildcard(&format!("{rare}{}", "a".repeat(5_000)), &pat)); +} + +#[test] +fn wildcard_on_long_repetitive_text_is_linear() { + // The backtracking matcher compared about text × pattern characters here (10^8). + let text = "a".repeat(20_000); + let pat = format!("*{}b", "a".repeat(5_000)); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{text}b"), &pat)); +} + #[test] fn tsv() { assert_eq!(parse_tsv("a\tb\n1\t2\n"), vec![vec!["a", "b"], vec!["1", "2"]]); From e14f783503b397e59aca0f611b4847e73cdf7f31 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 05:40:31 -0500 Subject: [PATCH 31/51] Analysis ToolPak: stop building a report once it outgrows the output limit (#3207) * Analysis ToolPak: stop building a report once it outgrows the output limit Report builders filled their whole output map before write_report checked its size, so Descriptive Statistics grouped by rows over a tall, nearly empty column allocated about 16 entries for each of up to a million variables before refusing the report. Grid now tracks its extent as values are stored and keeps nothing once the report's rows x columns pass MAX_OUTPUT_CELLS; write_report refuses such a report. extent() no longer rescans every entry. * address review: stop the descriptive builder once the grid is too large * address review: descriptive statistics reads variables one at a time --- crates/engine/src/cmd/analysis.rs | 87 ++++++++++++++++++++++++------- 1 file changed, 67 insertions(+), 20 deletions(-) diff --git a/crates/engine/src/cmd/analysis.rs b/crates/engine/src/cmd/analysis.rs index 9cd10e50f..92699b75f 100644 --- a/crates/engine/src/cmd/analysis.rs +++ b/crates/engine/src/cmd/analysis.rs @@ -152,26 +152,32 @@ fn input_range(s: &Session, p: &Json, keys: &[&str]) -> Result<(usize, RangeRef) /// Splits a range into variables (columns, or rows when `by_rows`); with `labels` the first /// cell of each names it, else `default(i)` does. fn variables(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String) -> Result> { - let d = s.doc()?; - let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; - let (outer, inner) = if by_rows { (r.height(), r.width()) } else { (r.width(), r.height()) }; + let outer = if by_rows { r.height() } else { r.width() }; let mut out = Vec::with_capacity(outer as usize); for i in 0..outer { - let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; - let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); - let mut name = default(i as usize); - if labels && !vals.is_empty() { - let first = vals.remove(0); - let t = first.display(); - if !t.trim().is_empty() { - name = t; - } - } - out.push(Var { name, vals }); + out.push(variable(s, si, r, by_rows, labels, default, i)?); } Ok(out) } +/// Variable `i` of [`variables`], read alone so a caller that may stop early needn't build them all. +fn variable(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String, i: u32) -> Result { + let d = s.doc()?; + let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; + let inner = if by_rows { r.width() } else { r.height() }; + let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; + let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); + let mut name = default(i as usize); + if labels && !vals.is_empty() { + let first = vals.remove(0); + let t = first.display(); + if !t.trim().is_empty() { + name = t; + } + } + Ok(Var { name, vals }) +} + /// The numbers of a variable; blanks are skipped when `skip_blank`, anything else that isn't a /// number is an error. fn numbers(v: &[Value], skip_blank: bool) -> Result> { @@ -272,11 +278,25 @@ struct Grid { cells: BTreeMap<(u32, u32), Value>, /// Cells shown as percentages. pct: Vec<(u32, u32)>, + /// Rows and columns spanned by `cells`. + rows: u32, + cols: u32, + /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused. + too_large: bool, } impl Grid { + /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept, + /// so a builder that would produce millions of entries stops allocating them, and + /// [`write_report`] refuses the report. fn put(&mut self, r: u32, c: u32, v: Value) { if !v.is_empty() { + let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1))); + if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + self.too_large = true; + return; + } + (self.rows, self.cols) = (rows, cols); self.cells.insert((r, c), v); } } @@ -296,9 +316,7 @@ impl Grid { } /// (rows, cols) spanned. fn extent(&self) -> (u32, u32) { - let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0); - let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0); - (rows, cols) + (self.rows, self.cols) } } @@ -310,7 +328,7 @@ fn idx(i: usize) -> u32 { fn descriptive(s: &Session, p: &Json) -> Result { let (si, r) = input_range(s, p, &["range", "input"])?; - let vars = variables(s, si, r, by_rows(p), flag(p, "labels"), &|i| format!("Column{}", i + 1))?; + let (by_rows, labels) = (by_rows(p), flag(p, "labels")); let summary = p.get("summary").map(|_| flag(p, "summary")).unwrap_or(true); let conf = confidence(p, "confidence")?; let kl = num(p, "kthLargest").filter(|k| *k >= 1.0).map(|k| k as usize); @@ -319,7 +337,14 @@ fn descriptive(s: &Session, p: &Json) -> Result { return Err(bad(CMD, "choose at least one of summary, confidence, kthLargest or kthSmallest")); } let mut g = Grid::default(); - for (i, v) in vars.iter().enumerate() { + // Variables are read one at a time: a million-row range grouped by rows is a million variables. + for i in 0..(if by_rows { r.height() } else { r.width() }) { + // The report is refused once the grid is too large: the rest would only build discarded labels. + if g.too_large { + break; + } + let v = variable(s, si, r, by_rows, labels, &|i| format!("Column{}", i + 1), i)?; + let i = i as usize; let x = numbers(&v.vals, true)?; let c = idx(i).saturating_mul(2); g.txt(0, c, &v.name); @@ -1749,7 +1774,7 @@ fn sampling(s: &Session, p: &Json, seed: Option) -> Result { fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result { let (rows, cols) = g.extent(); let (rows, cols) = (rows.max(1), cols.max(1)); - if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { return Err(bad(CMD, "the report is too large")); } let d = s.doc()?; @@ -1914,6 +1939,28 @@ mod tests { x } + #[test] + fn a_report_stops_growing_past_the_output_limit() { + // Entries past the limit are not kept, and the report is refused. + let mut g = super::Grid::default(); + g.num(1999, 1999, 1.0); + assert!(!g.too_large); + g.num(0, 2000, 1.0); + assert!(g.too_large); + assert_eq!(g.cells.len(), 1); + assert_eq!(g.extent(), (2000, 2000)); + // Row-grouped statistics of a tall, almost empty column: one variable per row would make + // millions of report entries; the report is refused instead of built. + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap(); + s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap(); + let err = s + .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"})) + .expect_err("report too large"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(v(&s, "C1"), Value::Empty); + } + #[test] fn forecast_standard_errors_match_the_full_window_rescan() { // #1043: the running sum must give what re-adding every window gave, gaps included. From bc2dabc7ea5bc260e2fd393a21bb8b3d459a905d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 05:48:31 -0500 Subject: [PATCH 32/51] FORMULATEXT and the in-cell editor keep Hidden formulas hidden (#3298) * FORMULATEXT and the cell editor keep a hidden formula's source hidden * address review: recalculate formulas when sheet protection changes * address review: skip the full recalculation for a no-op unprotect * address review: recalculate on protect/unprotect without marking the edit structural --- crates/calc/src/eval/call.rs | 9 ++++++--- crates/engine/src/cmd.rs | 9 +++++++-- crates/engine/src/cmd/review.rs | 22 +++++++++++++++++++++- crates/engine/src/tests.rs | 17 +++++++++++++++++ crates/ui-egui/src/app/editing.rs | 5 +++++ 5 files changed, 56 insertions(+), 6 deletions(-) diff --git a/crates/calc/src/eval/call.rs b/crates/calc/src/eval/call.rs index 848762ab9..fe185e700 100644 --- a/crates/calc/src/eval/call.rs +++ b/crates/calc/src/eval/call.rs @@ -52,13 +52,16 @@ impl<'h> Evaluator<'h> { "ISFORMULA" | "FORMULATEXT" => { let Some(Ev::R(a)) = args.first().map(|a| self.eval(a)) else { return err(CellError::NA) }; let Some(a) = a.first() else { return err(CellError::NA) }; - let f = self.host.workbook().sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + let wb = self.host.workbook(); + let f = wb.sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + // A formula formatted Hidden on a protected sheet stays hidden from FORMULATEXT too. + let hidden = wb.sheet(a.sheet).is_some_and(|s| s.is_protected() && wb.styles.get(s.style_id(a.range.start)).protection.hidden); if name == "ISFORMULA" { Ev::V(Value::Bool(f.is_some())) } else { match f { - Some(f) => Ev::V(Value::text(format!("={}", f.text))), - None => err(CellError::NA), + Some(f) if !hidden => Ev::V(Value::text(format!("={}", f.text))), + _ => err(CellError::NA), } } } diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index da130fe2a..a59bf9b2a 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -366,6 +366,10 @@ pub struct Ctx<'a> { pub wb: Workbook, pub changed: Vec, pub structural: bool, + /// Recalculate every formula without the other effects of `structural` (table headers are not + /// resynchronized, array formulas are still checked): for edits that change only what a + /// function reads, like sheet protection. + pub recalc_all: bool, pub sel: &'a mut Selection, /// Rows whose automatic height should be recomputed (sheet, row). pub fit_rows: Vec<(usize, u32)>, @@ -400,13 +404,14 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) wb: (*d.wb).clone(), changed: Vec::new(), structural: false, + recalc_all: false, sel: &mut sel, fit_rows: Vec::new(), protection_checked: false, edits: Vec::new(), }; let r = f(&mut ctx)?; - let Ctx { mut wb, mut changed, structural, mut fit_rows, protection_checked, edits, .. } = ctx; + let Ctx { mut wb, mut changed, structural, recalc_all, mut fit_rows, protection_checked, edits, .. } = ctx; // Equations whose source changed (or that arrived without a layout, as from a paste) are // laid out here, so painting only ever reads the cache. wb.refresh_equations(); @@ -435,7 +440,7 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) if let Some(t0) = t0 { eprintln!("commit: edit {:?}, {} changed", since(t0), changed.len()); } - if structural { + if structural || recalc_all { d.calc.recalc_all(&mut wb); } else if !changed.is_empty() { d.calc.cells_changed(&mut wb, &changed); diff --git a/crates/engine/src/cmd/review.rs b/crates/engine/src/cmd/review.rs index de9c98ecd..7eafa26db 100644 --- a/crates/engine/src/cmd/review.rs +++ b/crates/engine/src/cmd/review.rs @@ -163,6 +163,8 @@ fn protect_sheet(s: &mut Session, p: &Json) -> Result { let what = d.wb.sheet(sheet).map(|sh| sh.name.clone()).unwrap_or_default(); let prot = SheetProtection { password: str_param(p, "password").filter(|x| !x.is_empty()).map(|pw| new_password_hash(d, &what, pw)), ..prot }; edit(s, |cx| { + // FORMULATEXT reads a sheet's protection: its results are recalculated. + cx.recalc_all = true; cx.sheet_mut(sheet)?.protection = Some(prot.clone()); Ok(Json::Null) }) @@ -182,7 +184,10 @@ fn unprotect_sheet(s: &mut Session, p: &Json) -> Result { } } edit(s, |cx| { - cx.sheet_mut(sheet)?.protection = None; + // Only removing protection changes what FORMULATEXT reads; a no-op unprotect skips the full recalculation. + if cx.sheet_mut(sheet)?.protection.take().is_some() { + cx.recalc_all = true; + } Ok(Json::Null) }) } @@ -404,6 +409,21 @@ mod tests { assert_eq!(at(&s), (0, "A1".to_string())); } + #[test] + fn protecting_and_unprotecting_leave_table_headers_alone() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["a", "b"], [1, 2]]})).unwrap(); + s.execute("insert.table", json!({"range": "A1:B2", "header": true})).unwrap(); + // An imported header can hold a number: only a structural edit would turn it back into text. + let d = s.doc_mut().unwrap(); + std::sync::Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap().set_value(gridcraft_core::CellRef::new(0, 0), gridcraft_core::Value::Number(7.0)); + s.execute("review.protectSheet", json!({})).unwrap(); + s.execute("review.unprotectSheet", json!({})).unwrap(); + let v = s.doc().unwrap().wb.sheet(0).unwrap().value(gridcraft_core::CellRef::new(0, 0)); + assert_eq!(v, gridcraft_core::Value::Number(7.0)); + } + #[test] fn protection_that_forbids_selecting_locked_cells_is_enforced_and_kept() { let mut s = book(); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index ed5e33945..55e795a01 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6480,6 +6480,23 @@ fn clearing_links_and_notes_keeps_allow_edit_ranges() { assert!(sh.hyperlinks.is_empty() && sh.comments.is_empty()); } +#[test] +fn formulatext_does_not_reveal_a_hidden_formula() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=1+1"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": false, "hidden": true}}})).unwrap(); + s.execute("home.formatCells", json!({"range": "B1", "style": {"protection": {"locked": false}}})).unwrap(); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "not protected yet"); + s.execute("review.protectSheet", json!({})).unwrap(); + // Without entering the formula again: protecting recalculates what reads the protection. + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("review.unprotectSheet", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "unprotecting shows it again without re-entering"); +} + #[test] fn protected_sheet_refuses_layout_commands_its_permissions_do_not_allow() { let mut s = s(); diff --git a/crates/ui-egui/src/app/editing.rs b/crates/ui-egui/src/app/editing.rs index 8682a435c..a5ac8a176 100644 --- a/crates/ui-egui/src/app/editing.rs +++ b/crates/ui-egui/src/app/editing.rs @@ -50,6 +50,11 @@ impl SheetApp { self.session.mode = gridcraft_engine::Mode::Edit; return; } + // A hidden formula's source isn't loaded into an edit (a blank edit would also write over + // it on commit): no edit starts. Typing still replaces the cell. + if text.is_none() && gridcraft_engine::display::formula_hidden(&d.wb, sh, at) { + return; + } let locale = self.ui.language.formula_locale(); let current = if sh.cell_pictures.contains_key(&at) { String::new() From 76e53085e30f36f059d67b022148c2630db09dd1 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 06:15:08 -0500 Subject: [PATCH 33/51] xlsx: keep the cached value of a follower whose shared-formula copy is refused (#3212) * xlsx: one hyperlink allowance per sheet, and a copy limit for unparsable shared masters Every element of a worksheet started a fresh cell and byte allowance although all of them fill the same map. The sheet now keeps one allowance across its elements. Followers of a shared master the parser refuses each copied its full text; a sheet full of followers of a long master multiplied that text without bound. The copies are now charged against a 64 MiB per-sheet limit, past which the followers keep their cached values. * address review: keep the cached value of a follower dropped for the shared-formula copy limit inside array ranges --- crates/xlsx/src/sheet_read.rs | 8 +++++++- crates/xlsx/src/tests/malformed.rs | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/crates/xlsx/src/sheet_read.rs b/crates/xlsx/src/sheet_read.rs index 40d4b1c36..f2ec76683 100644 --- a/crates/xlsx/src/sheet_read.rs +++ b/crates/xlsx/src/sheet_read.rs @@ -55,6 +55,9 @@ struct RowState { dynamic_cells: u64, /// Row-membership checks served so far, for the work cap above. dynamic_checks: u64, + /// The cell being read lost its shared formula to a copy limit: its cached value is kept, even + /// inside an array range (where a formula's cached values otherwise give way to the spill). + shared_copy_refused: bool, dropped: u64, rows: u64, } @@ -607,6 +610,7 @@ fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos return; } let dynamic = c.attr("cm").is_some(); + st.shared_copy_refused = false; let formula = c.child("f").and_then(|f| read_formula(cx, st, f, pos, dynamic)); // A `PY(index,0|1)` formula is a Python cell whose code is a script of the Python part. The // older `PY("…",0|1)` form is read from the formula's own text: the formula parser would fold @@ -632,7 +636,7 @@ fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos if formula.as_ref().is_some_and(|f| !f.dynamic && f.array.is_none() && f.expr_arc().is_some()) { cx.legacy_formulas.push(pos); } - let spilled_from = if formula.is_none() && py.is_none() { dynamic_range(st, pos) } else { None }; + let spilled_from = if formula.is_none() && py.is_none() && !st.shared_copy_refused { dynamic_range(st, pos) } else { None }; let python_anchor = spilled_from.is_some_and(|r| sheet.cells.get(r.start).is_some_and(|a| a.py.is_some())); let value = match spilled_from { // A Python cell's spilled values are kept: they are its result until it runs again. @@ -695,10 +699,12 @@ fn read_formula_text(cx: &mut Ctx<'_>, st: &mut RowState, f: &El, pos: CellRef, "a shared formula is longer than {} characters; kept the cached values of the cells that share it", gridcraft_formula::MAX_FORMULA_LEN )); + st.shared_copy_refused = true; return None; } if !charge(&mut cx.shared_formula_bytes, extra_shared_cost(sh.cost), MAX_SHARED_FORMULA_BYTES) { cx.warn("shared formulas expand to too much data; kept the cached values of the remaining cells that share them"); + st.shared_copy_refused = true; return None; } *unwrapped |= sh.unwrapped; diff --git a/crates/xlsx/src/tests/malformed.rs b/crates/xlsx/src/tests/malformed.rs index 9a533d9ca..870ce316d 100644 --- a/crates/xlsx/src/tests/malformed.rs +++ b/crates/xlsx/src/tests/malformed.rs @@ -539,3 +539,18 @@ fn outline_summary_directions_round_trip() { let xml = String::from_utf8(crate::package::Package::open(&bytes).unwrap().read("xl/worksheets/sheet1.xml").unwrap().unwrap()).unwrap(); assert!(!xml.contains("outlinePr"), "{xml}"); } + +/// A shared follower whose copy is refused keeps its cached value even inside an array range. +#[test] +fn refused_shared_copies_keep_their_values_inside_array_ranges() { + let master = format!("1+(\"{}", "a".repeat(gridcraft_formula::MAX_FORMULA_LEN)); + let rows = format!( + r#"11{master}17"# + ); + let (wb, rep) = read_xlsx(&minimal(&format!("{rows}"), &[], "", "")).unwrap(); + let s = wb.sheet(0).unwrap(); + let c2 = gridcraft_core::CellRef::new(1, 2); + assert!(s.cell(c2).is_some_and(|c| c.formula.is_none())); + assert_eq!(s.value(c2), gridcraft_core::Value::Number(7.0)); + assert!(rep.warnings.iter().any(|w| w.contains("longer than")), "{:?}", rep.warnings); +} From f79a425ca923b27a89cc34b7eb4cc78c9b73153c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 06:32:44 -0500 Subject: [PATCH 34/51] xlsx: find a reused picture through a content hash instead of comparing every media part (#3214) * xlsx: find a reused picture through a content hash instead of comparing every media part Out::image_part compared each new picture byte by byte with every media part written so far, so a workbook of many distinct pictures sharing a long prefix was exported in quadratic time. Media parts are now indexed by extension and SHA-256. * address review: index preserved media parts when they are added * address review: do not hash an image twice when it is first added --- crates/xlsx/src/tests/shared_media.rs | 18 ++++++++++++++++++ crates/xlsx/src/write.rs | 22 ++++++++++++++++------ 2 files changed, 34 insertions(+), 6 deletions(-) diff --git a/crates/xlsx/src/tests/shared_media.rs b/crates/xlsx/src/tests/shared_media.rs index a1d7bc0c7..0c37e584b 100644 --- a/crates/xlsx/src/tests/shared_media.rs +++ b/crates/xlsx/src/tests/shared_media.rs @@ -85,3 +85,21 @@ fn different_pictures_with_the_same_length_stay_separate() { let back = read_xlsx(&bytes).unwrap().0; assert_eq!(back.sheets[0].images.iter().map(|i| i.data.to_vec()).collect::>(), vec![a, b]); } + +/// Many distinct pictures that share a long prefix: each is matched against the earlier ones +/// through a content hash, not by comparing bytes with every one of them. +#[test] +fn many_distinct_pictures_with_a_common_prefix_are_each_written_once() { + let pictures: Vec> = (0..400u32) + .map(|i| { + let mut v = b"\x89PNG\r\n\x1a\n".to_vec(); + v.extend(std::iter::repeat_n(7u8, 65_536)); + v.extend(i.to_le_bytes()); + v + }) + .collect(); + let refs: Vec<(u32, &[u8])> = pictures.iter().enumerate().map(|(i, p)| (i as u32 + 1, p.as_slice())).collect(); + let wb = Workbook { sheets: vec![sheet_with("One", &refs), sheet_with("Two", &refs)], ..Default::default() }; + let bytes = write_xlsx(&wb).unwrap(); + assert_eq!(names(&bytes).iter().filter(|n| n.starts_with("xl/media/")).count(), 400); +} diff --git a/crates/xlsx/src/write.rs b/crates/xlsx/src/write.rs index 512144980..8f8c57105 100644 --- a/crates/xlsx/src/write.rs +++ b/crates/xlsx/src/write.rs @@ -6,6 +6,7 @@ use std::fmt::Write as _; use gridcraft_core::RangeRef; use gridcraft_formula::quote_sheet; use gridcraft_model::{CalcMode, CalcSettings, PhoneticString, ShowObjects, Style, Visibility, Workbook, WorkbookView}; +use sha2::Digest as _; use crate::package::ZipOut; use crate::xml::{esc, esc_attr}; @@ -112,6 +113,9 @@ pub struct Out { pub dxfs: Vec