diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index 2d1213a4e..758ecac06 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -27,6 +27,10 @@ jobs: name: Windows installation policy (no app build) runs-on: windows-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 2a4b5c96e..7dd388ccd 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -12,6 +12,7 @@ Generated-in-code art is original and has no file to list: the UI icon set (`cra | `epaint_default_fonts::HACK_REGULAR` (dependency font input) | Source Foundry Authors (Hack; DejaVu and Bitstream Vera foundations) | [egui 6d518fc](https://github.com/storytold/egui/tree/6d518fc799681571963842f95b55a6b68431b15a/crates/epaint_default_fonts/fonts), [Hack](https://github.com/source-foundry/Hack) | MIT + Bitstream Vera License (full upstream texts in `NOTICE`) | Hack-Regular.ttf is egui's bundled monospace face and GridCraft's last-resort symbol fallback. Dependency asset unmodified here, no font file copied into this repository. | | `epaint_default_fonts::NOTO_EMOJI_REGULAR` (dependency font input) | The Noto Project Authors | [egui 6d518fc](https://github.com/storytold/egui/tree/6d518fc799681571963842f95b55a6b68431b15a/crates/epaint_default_fonts/fonts), [Noto Emoji](https://fonts.google.com/noto/specimen/Noto+Emoji) | SIL Open Font License 1.1 (full upstream text in `NOTICE`) | NotoEmoji-Regular.ttf is enabled by egui's `monochrome_emoji_fonts` feature and used as the monochrome emoji fallback. Dependency asset unmodified here, no font file copied into this repository. | | `epaint_default_fonts::UBUNTU_LIGHT` (dependency font input) | Canonical Ltd. and the Ubuntu Font Family authors | [egui 6d518fc](https://github.com/storytold/egui/tree/6d518fc799681571963842f95b55a6b68431b15a/crates/epaint_default_fonts/fonts), [Ubuntu font](https://design.ubuntu.com/font) | Ubuntu Font Licence 1.0 (full upstream text in `NOTICE`) | Ubuntu-Light.ttf is egui's default proportional face. Dependency asset unmodified here, no font file copied into this repository. | +| Regional format data in `crates/core/src/region.rs` (data, no file of its own) | Unicode, Inc. (Unicode CLDR) | [cldr-json](https://github.com/unicode-org/cldr-json) (`cldr-numbers-full`, `cldr-dates-full`) and ICU's CLDR 48 data | Unicode License v3 (`Unicode-3.0`; full text in `NOTICE`) | Separators, month and weekday names, AM/PM words, currency symbols and date order of the built-in regions. No CLDR file is copied into this repository. | | `docs/brand/artcraft-logo.svg` | ArtCraft Team | ArtCraft brand kit | ArtCraft brand terms (`docs/brand/LICENSE-brand.txt`) | Trademark, not open source | | `docs/brand/artcraft-logo.png` | ArtCraft Team | ArtCraft brand kit | ArtCraft brand terms | Trademark | | `docs/brand/artcraft-logo-white.svg` | ArtCraft Team | ArtCraft brand kit | ArtCraft brand terms | Trademark | diff --git a/Cargo.lock b/Cargo.lock index a39620f96..92451aa83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -910,12 +910,12 @@ dependencies = [ [[package]] name = "craft-fonts-build" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" [[package]] name = "craft-i18n" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "fixed_decimal", "icu_decimal", @@ -938,7 +938,7 @@ dependencies = [ [[package]] name = "craft-print" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "craft-winprint-gdi", "log", @@ -952,7 +952,7 @@ dependencies = [ [[package]] name = "craft-ui" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "craft-fonts-build", "craft-i18n", @@ -972,7 +972,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "chrono", "dunce", @@ -982,6 +982,7 @@ dependencies = [ "image", "log", "plist", + "rustix 1.1.5", "semver", "serde", "serde_json", @@ -1018,7 +1019,7 @@ dependencies = [ [[package]] name = "craft-window" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c5a1cbfb5f9049f9270955caffb5bee5997aebf7#c5a1cbfb5f9049f9270955caffb5bee5997aebf7" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "egui", "objc2 0.6.5", @@ -1029,7 +1030,7 @@ dependencies = [ [[package]] name = "craft-winprint-gdi" version = "0.1.0" -source = "git+https://github.com/storytold/winprint.rs?rev=417ac3c4d14d3da5485a99c7d5ce8f05664ec312#417ac3c4d14d3da5485a99c7d5ce8f05664ec312" +source = "git+https://github.com/storytold/winprint.rs?rev=64e7afd4d1483512385012c340ecaaaf0f3beb7f#64e7afd4d1483512385012c340ecaaaf0f3beb7f" dependencies = [ "windows-sys 0.61.2", ] @@ -2022,7 +2023,7 @@ dependencies = [ [[package]] name = "gridcraft" -version = "0.16.0" +version = "0.19.1" dependencies = [ "arboard", "chrono", @@ -2034,6 +2035,7 @@ dependencies = [ "getrandom 0.4.3", "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "gridcraft-ui-egui", "image", "log", @@ -2049,7 +2051,7 @@ dependencies = [ [[package]] name = "gridcraft-calc" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "gridcraft-formula", @@ -2061,7 +2063,7 @@ dependencies = [ [[package]] name = "gridcraft-chart" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-calc", "gridcraft-core", @@ -2072,10 +2074,11 @@ dependencies = [ [[package]] name = "gridcraft-cli" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "gridcraft-mcp", + "gridcraft-sandbox", "log", "quick-xml 0.41.0", "serde_json", @@ -2084,7 +2087,7 @@ dependencies = [ [[package]] name = "gridcraft-core" -version = "0.16.0" +version = "0.19.1" dependencies = [ "chrono", "js-sys", @@ -2095,7 +2098,7 @@ dependencies = [ [[package]] name = "gridcraft-engine" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-calc", "gridcraft-chart", @@ -2118,7 +2121,7 @@ dependencies = [ [[package]] name = "gridcraft-formula" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "serde", @@ -2127,7 +2130,7 @@ dependencies = [ [[package]] name = "gridcraft-functions" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "regex", @@ -2135,7 +2138,7 @@ dependencies = [ [[package]] name = "gridcraft-math" -version = "0.16.0" +version = "0.19.1" dependencies = [ "ratex-font", "ratex-layout", @@ -2145,7 +2148,7 @@ dependencies = [ [[package]] name = "gridcraft-mcp" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "log", @@ -2154,7 +2157,7 @@ dependencies = [ [[package]] name = "gridcraft-model" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", "gridcraft-formula", @@ -2167,18 +2170,29 @@ dependencies = [ [[package]] name = "gridcraft-numfmt" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-core", ] [[package]] name = "gridcraft-pdf" -version = "0.16.0" +version = "0.19.1" + +[[package]] +name = "gridcraft-sandbox" +version = "0.19.1" +dependencies = [ + "landlock", + "libc", + "nix", + "seccompiler", + "thiserror 2.0.21", +] [[package]] name = "gridcraft-ui-egui" -version = "0.16.0" +version = "0.19.1" dependencies = [ "craft-ui", "craft-update", @@ -2197,6 +2211,7 @@ dependencies = [ "skrifa", "sys-locale", "toml", + "unicode-bidi", "web-sys", "web-time", "zbus", @@ -2204,7 +2219,7 @@ dependencies = [ [[package]] name = "gridcraft-web" -version = "0.16.0" +version = "0.19.1" dependencies = [ "craft-web-api", "craft-web-host", @@ -2223,7 +2238,7 @@ dependencies = [ [[package]] name = "gridcraft-xlsx" -version = "0.16.0" +version = "0.19.1" dependencies = [ "aes", "gridcraft-core", @@ -2872,6 +2887,17 @@ dependencies = [ "smallvec", ] +[[package]] +name = "landlock" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cca98e95f35b29d469dade6724c6f96cec9236640f745a0e99b0334ec320ab1" +dependencies = [ + "enumflags2", + "libc", + "thiserror 2.0.21", +] + [[package]] name = "lazy_static" version = "1.5.1" @@ -4543,6 +4569,15 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "seccompiler" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae55de56877481d112a559bbc12667635fdaf5e005712fd4e2b2fa50ffc884" +dependencies = [ + "libc", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -6485,7 +6520,7 @@ dependencies = [ [[package]] name = "winprint" version = "0.2.1" -source = "git+https://github.com/storytold/winprint.rs?rev=5f80b9baf6b58937c401656634e42d33f00ddf49#5f80b9baf6b58937c401656634e42d33f00ddf49" +source = "git+https://github.com/storytold/winprint.rs?rev=bc06a7c48b768eb27a828c7986f6d2868b245ee0#bc06a7c48b768eb27a828c7986f6d2868b245ee0" dependencies = [ "flate2", "fmt-derive", @@ -6500,6 +6535,7 @@ dependencies = [ "windows", "windows-core", "windows-implement", + "windows-numerics", "xml", ] @@ -6674,7 +6710,7 @@ checksum = "e450f9b2ed1dff33c94c12589a87338689467b9c4f5d8a5710bd09a847d2c8a7" [[package]] name = "xtask" -version = "0.16.0" +version = "0.19.1" dependencies = [ "gridcraft-engine", "serde_json", @@ -6935,7 +6971,6 @@ dependencies = [ "nix", "typed-path", "windows-sys 0.61.2", - "zopfli", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 594b29273..f244db53f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*", "apps/*", "xtask"] default-members = ["crates/*", "apps/gridcraft", "apps/gridcraft-cli", "xtask"] [workspace.package] -version = "0.16.0" +version = "0.19.1" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -14,6 +14,7 @@ repository = "https://github.com/storytold/gridcraft" # internal gridcraft-core = { path = "crates/core" } gridcraft-numfmt = { path = "crates/numfmt" } +gridcraft-sandbox = { path = "crates/sandbox" } gridcraft-formula = { path = "crates/formula" } gridcraft-functions = { path = "crates/functions" } gridcraft-model = { path = "crates/model" } @@ -26,9 +27,9 @@ gridcraft-mcp = { path = "crates/mcp" } gridcraft-pdf = { path = "crates/pdf" } gridcraft-math = { path = "crates/math" } # Every craft-libs crate (craft-update and craft-window here, craft-ui in ui-egui, craft-print in -# the app) comes from one storytold/craft-libs revision. +# the app) comes from one craft-libs revision. # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7", version = "=0.1.0", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", version = "=0.1.0", features = ["egui"] } # foundation thiserror = "2" @@ -71,7 +72,7 @@ skrifa = { version = "0.44", default-features = false, features = ["std"] } epaint_default_fonts = "0.37" # One native window per workbook: the window list, Window menu titles and the immediate-viewport # driver. Same craft-libs revision as craft-update above. -craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7", default-features = false, features = ["egui"] } +craft-window = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", default-features = false, features = ["egui"] } [workspace.lints.rust] unsafe_code = "forbid" diff --git a/NOTICE b/NOTICE index d859d99d3..64753f94a 100644 --- a/NOTICE +++ b/NOTICE @@ -275,6 +275,55 @@ DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM OTHER DEALINGS IN THE FONT SOFTWARE. + +The regional format data in crates/core/src/region.rs (separators, month and weekday names, AM/PM +words, currency symbols and date order) follows the Unicode Common Locale Data Repository (CLDR), +read from unicode-org/cldr-json and from ICU's CLDR 48 data, under the Unicode License v3. +Source: https://github.com/unicode-org/cldr-json +The following text is reproduced verbatim from cldr-json's LICENSE: + +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 2015-2024 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +SPDX-License-Identifier: Unicode-3.0 + Linux Wayland window decorations ------------------------------- sctk-adwaita 0.10.1 (https://github.com/PolyMeilex/sctk-adwaita, diff --git a/ROADMAP.md b/ROADMAP.md index 369fbd58a..6fc38cc37 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~25 points of "ready for real work" and ~160–240 h away -> **Last reviewed:** 2026-11-01 · **Last updated:** 2026-11-02 · **Change:** minor (form controls: buttons, sliders and spin buttons; earlier: RefEdit reference boxes in dialogs: Goal Seek, Solver and Data Analysis take ranges by selection on the grid; earlier: Python mode in the formula bar; earlier: Python cells stored and kept in files; earlier: equations: Insert › Equation, editor, gridcraft-math; earlier: What-if and analysis: Solver Sensitivity report for Simplex LP and GRG Nonlinear; earlier: floating objects: one selection, eight resize handles, keyboard delete; earlier: Chart data tables: live source values beneath column, bar, line, area and compatible combo charts; legend keys and borders; XLSX round-trip and print/PDF; earlier: Formulas › Function Library as labelled buttons with category menus; Recently Used and Insert Function categories; XLSX workbook-view selection, bounded tab ratios and sheet-edit regressions; readiness estimates unchanged; earlier: CSV and text open detects each column's date order, so day-first dates are not silently swapped) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-11-01 · **Last updated:** 2026-11-05 · **Change:** minor (CJK fallback text shares the Latin baseline; earlier: form controls: buttons, sliders and spin buttons; earlier: RefEdit reference boxes in dialogs: Goal Seek, Solver and Data Analysis take ranges by selection on the grid; earlier: Python mode in the formula bar; earlier: Python cells stored and kept in files; earlier: equations: Insert › Equation, editor, gridcraft-math; earlier: What-if and analysis: Solver Sensitivity report for Simplex LP and GRG Nonlinear; earlier: floating objects: one selection, eight resize handles, keyboard delete; earlier: Chart data tables: live source values beneath column, bar, line, area and compatible combo charts; legend keys and borders; XLSX round-trip and print/PDF; earlier: Formulas › Function Library as labelled buttons with category menus; Recently Used and Insert Function categories; XLSX workbook-view selection, bounded tab ratios and sheet-edit regressions; readiness estimates unchanged; earlier: CSV and text open detects each column's date order, so day-first dates are not silently swapped) · **Target:** Microsoft Excel (Microsoft 365) GridCraft aims at full Microsoft Excel parity, and at being better in places: faster, open (XLSX is the native format), fully scriptable by agents (MCP, control channel, CLI), and on the web. @@ -43,8 +43,7 @@ long `.xls` track, so about half parallelizes (≈30–50 wall-clock hours with Why alpha: all six core workflows in the [alpha gate](docs/roadmap.md#alpha-gate) pass end to end on macOS with save and reopen, and the core workflow works end to end (open an XLSX, enter formulas and data, format, sort and filter, chart, pivot, print to PDF, save XLSX), but users still hit blocking gaps on real -files: 100 MB workbooks don't open (#175), there's no `.xls` (#121), Arabic and Devanagari text -isn't shaped (Chinese text still reported in #142), charts lose their type and options on round trip, and +files: 100 MB workbooks don't open (#175), there's no `.xls` (#121), Devanagari text needs shaping verification (Chinese text still reported in #142), charts lose their type and options on round trip, and nobody has yet confirmed that Excel opens our output without a repair prompt across a real corpus. Beta needs those closed (see [beta gates](docs/roadmap.md#beta-gates)). @@ -105,7 +104,7 @@ Localized German formulas and numbers are in PR #163. Detail: [localization-pari | Simplified Chinese | zh-Hans | menus only | ~12% | | Spanish | es | none | 0% | | Hindi | hi | none | 0% | -| Arabic | ar | none (no RTL layout, no shaping) | 0% | +| Arabic | ar | catalog pending; shared shaping/bidi works | 0% | | French | fr | none | 0% | | Portuguese | pt-BR | menus, File view, dialogs | ~43% | | Indonesian | id | none | 0% | @@ -149,6 +148,9 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-31 | CSV and text open detects each column's date order: a value that reads only day first (`31/05/2026`) makes the column day first, a value that reads only month first (`12/31/2026`) makes it month first, an all-ambiguous column keeps month first with a warning, and a column with both kinds stays text with a warning (#2765) | | 2026-10-31 | Recently Used lists the functions most recently entered in a formula from any source (ribbon, control channel, MCP, scripts), kept with the preferences and returned by `formulas.recentlyUsed`; the Insert Function dialog takes Excel's title and a working category list (Most Recently Used, All, each category) | | 2026-10-31 | Formulas › Function Library as a big Insert Function button, the AutoSum split button and three columns of labelled small buttons, each category book in its own colour; category menus end with Insert Function… on that category; More Functions cascades into the remaining categories; Home › Editing shares the AutoSum menu; the MR key tip opens Recently Used | +| 2026-10-31 | System and embedded CJK fallback text shares the Latin baseline in cells, editors and window titles (#1265, #2621). Symbol and emoji alignment, manual row heights and clipping are unchanged | +| 2026-10-26 | Arabic bidi display/editing preserve logical source text; per-cell reading direction is editable and survives undo, XLSX and conditional-format priority | +| 2026-10-26 | Arabic text shows in cells and the interface through a system, fontconfig or craft-fonts Arabic face; workbook fonts keep their script fallbacks | | 2026-10-25 | AutoSum over a selection does what Excel does: a selected row gets one total to its right (in its last cell if that is empty), not a one-cell total under each cell; a block whose last column is empty gets each row's total there, one whose last row is empty each column's, and with both the corner gets the total of the block; otherwise totals go under each column as before | | 2026-10-24 | Macro interface ([macros.md](docs/macros.md#in-the-window)): an Automate › Macros group (Record Macro, which turns into Stop Recording, Use Relative References, Run Macro with recent macros); "Recording · N steps" with Stop in the status bar; a Record Macro dialog (checked name, free Ctrl+Shift shortcut, description); the Macros dialog lists macros beside Automate scripts and a VBA project, with Run, Step Into, Edit, Create, Delete and Options; Ctrl+Shift+letter macro shortcuts; questions that offer Run, Overwrite, Shift Down or Shift Right; a step editor with a breakpoint gutter, drag to reorder, a command picker, reference boxes that switch between relative tokens and absolute references, Inputs and Output tabs, a debug bar (Check, Run, Step, Continue, Run to Here, Stop with keep or undo) and grid overlays for inputs, writes, conflicts and the current step | | 2026-10-24 | Go To lists the named ranges of every sheet (`Sheet2!Name` for a name scoped to another sheet; hidden names, constants and formulas left out) and the references entered before, a double-click goes to an entry, and the Reference box takes several names or references separated by commas (`Price, Type`, `B14:C22,F19:G30`) | @@ -198,6 +200,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | Conditional formatting: a rule gathers range statistics only when it reads them (top/bottom, above average, duplicates, scales, bars, icons) and borrows them instead of copying them per cell, so the per-frame lookup of an expression or cell-value rule no longer visits its whole range (#1098, part 1) | | 2026-10-12 | Next/previous sheet navigation stops at the first/last visible worksheet instead of activating a hidden boundary sheet; the current selection is retained | | 2026-10-12 | PDF writer: objects are written straight to the output, identical pictures are stored once, style operators are emitted only when they change, `q` nesting is limited to 28 and polyline joins no longer leak into later strokes (#816) | +| 2026-10-12 | Number formatting: one table of built-in formats (xlsx); numbers, dates and General render with one scratch buffer instead of per-token strings; output unchanged, checked against digests recorded from the old code (#817) | | 2026-10-21 | Macro engine ([macros.md](docs/macros.md)): typed, declarative macros saved in the workbook (a JSON field; hidden names in xlsx); declared inputs checked with messages that name the cells; tokens (`Input.col(2)`, `Row`, `Output`, `Name:Total`, `{{…}}` in formulas); one undo step with full rollback; a footprint dry run with ask, fail, overwrite, shift down or shift right; a policy that keeps macros away from files and the app, with approval for macros in opened files; Repeat, If and Stop steps; a debugger; a relative recorder; `macro.*` commands beside the unchanged Automate scripts | | 2026-10-13 | Charts: work and memory are bounded by series x points (2,000,000 per chart: every series keeps its first points; 16 trendlines per chart), hairline columns and bars are merged per pixel column, series ranges are clipped before they are evaluated, the moving average is linear and rendering clean data no longer copies it (#1147) | | 2026-10-13 | Charts: one combo plot-kind rule for resolving and drawing; ticks of a scale off the step grid keep their own values (#1146) | @@ -265,6 +268,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | Change | Summary | |---|---|---| | 2026-11-06 | trivial | Progress log: scroll positions follow their sheet through sheet inserts, deletes and moves (#3485) | +| 2026-11-05 | minor | Reconcile upstream form-control progress with the CJK baseline update | | 2026-11-02 | minor | Form controls; catalog 277 / 292 → 281 / 296 (Developer › Controls); progress log row | | 2026-11-02 | minor | Progress log: RefEdit reference boxes in dialogs | | 2026-11-02 | minor | Progress log: Python mode in the formula bar | @@ -277,6 +281,9 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-31 | minor | File-page text fields keep the view open when Esc cancels an edit (#3035) | | 2026-10-31 | minor | Progress log: Recently Used and Insert Function categories | | 2026-10-31 | minor | Progress log: Function Library | +| 2026-10-31 | minor | Progress log: shared CJK/Latin baselines in cells, editors and window titles | +| 2026-10-26 | minor | Progress log: Arabic bidi display and editing | +| 2026-10-26 | minor | Progress log: Arabic font fallback | | 2026-10-25 | minor | Progress log: AutoSum over a selected row, and a selection's empty last row and column | | 2026-10-24 | minor | Macro interface on the Automate tab; progress log row | | 2026-10-24 | minor | Macro engine and `macro.*` commands; catalog 275 / 291 → 276 / 292 (View › Macros › Record Macro); progress log row | @@ -308,6 +315,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | 2026-10-13 | minor | Progress log: conditional-format lookups no longer scan the range for rules without statistics (#1098) | | 2026-10-12 | minor | Progress log: hidden-sheet navigation at workbook boundaries | | 2026-10-12 | minor | Progress log: PDF writer cleanup (#816) | +| 2026-10-12 | minor | Numfmt cleanup: one built-in format table (gridcraft-xlsx), fewer allocations rendering numbers, dates and General, equality test against the previous output (#817); progress log row | | 2026-10-12 | minor | The control server requires an auth token and closes bad connections (#777); progress log row | | 2026-10-12 | minor | Rows auto-fit oversized fonts on open and whole-row/column formatting (#833); progress log row | | 2026-10-12 | minor | Progress log: multi-area row/column delete (#758) | diff --git a/apps/gridcraft-cli/Cargo.toml b/apps/gridcraft-cli/Cargo.toml index 21e958005..83c45d0f2 100644 --- a/apps/gridcraft-cli/Cargo.toml +++ b/apps/gridcraft-cli/Cargo.toml @@ -14,6 +14,7 @@ name = "gridcraft-cli" path = "src/main.rs" [dependencies] +gridcraft-sandbox = { workspace = true } gridcraft-engine = { workspace = true } gridcraft-mcp = { workspace = true } serde_json = { workspace = true } diff --git a/apps/gridcraft-cli/src/main.rs b/apps/gridcraft-cli/src/main.rs index 5170ba0c5..2c49c6033 100644 --- a/apps/gridcraft-cli/src/main.rs +++ b/apps/gridcraft-cli/src/main.rs @@ -11,6 +11,7 @@ //! gridcraft-cli functions [--json] [--search X] [--category C] //! gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] //! gridcraft-cli send [json] +//! gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] //! gridcraft-cli version | --version //! ``` #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -30,6 +31,7 @@ macro_rules! outln { } mod check; +mod sandbox_check; use gridcraft_engine::Session; use serde_json::{Value, json}; @@ -49,6 +51,8 @@ const USAGE: &str = "usage: gridcraft-cli functions [--json] [--search X] [--category C] gridcraft-cli mcp [--connect PORT] [--in FILE | --sample NAME] gridcraft-cli send [json] + gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...] + whether this system confines programs (no program: a confined self-test) gridcraft-cli version"; /// stdout went away. A reader that stopped early (a closed pipe) ends the program quietly; any @@ -62,6 +66,11 @@ fn stdout_failed(e: std::io::Error) -> ! { } fn main() -> ExitCode { + // A confined launcher run (and its probes) comes first, before anything else starts. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + return ExitCode::from(u8::try_from(code).unwrap_or(1)); + } let args: Vec = std::env::args().skip(1).collect(); let rest = args.get(1..).unwrap_or_default(); let r = match args.first().map(String::as_str) { @@ -79,6 +88,7 @@ fn main() -> ExitCode { Some("cat") => cat(rest), Some("run") => run(rest), Some("check") => return check::main(rest), + Some("sandbox-check") => return sandbox_check::main(rest), Some("commands") => commands(rest), Some("functions") => functions(rest), Some("mcp") => mcp(rest), diff --git a/apps/gridcraft-cli/src/sandbox_check.rs b/apps/gridcraft-cli/src/sandbox_check.rs new file mode 100644 index 000000000..e40bb9ac5 --- /dev/null +++ b/apps/gridcraft-cli/src/sandbox_check.rs @@ -0,0 +1,98 @@ +//! `gridcraft-cli sandbox-check [--read PATH]... [--exec PATH]... [-- PROGRAM ARGS...]`: whether +//! this system confines programs, for a person or a reviewer checking a machine. +//! +//! Without a program it starts a confined copy of itself that tries what the sandbox must deny +//! (reading outside the policy, writing, the network, starting a process) and what it must still +//! allow (a thread, reading an allowed file), and prints one line per attempt. With a program it +//! runs that program confined, with the given read and executable paths and the system's shared +//! libraries, and passes its output and exit status through; its own lines then go to standard +//! error. Every path is absolute. Exit status: 0 when every attempt is +//! as required (or the program succeeded), 1 otherwise (a refused launch included), 2 when GridCraft +//! has no sandbox for this kind of system. + +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::ExitCode; + +use gridcraft_sandbox::{Canary, Policy, REFUSED_EXIT_CODE, REPORT_FLAG, Support}; + +pub fn main(args: &[String]) -> ExitCode { + match run(args) { + Ok(code) => code, + Err(e) => { + eprintln!("gridcraft-cli: sandbox-check: {e}"); + ExitCode::FAILURE + } + } +} + +fn run(args: &[String]) -> Result { + let mut policy = Policy::new().with_system_libraries(); + let mut it = args.iter(); + let mut program: Option<(PathBuf, Vec)> = None; + while let Some(a) = it.next() { + match a.as_str() { + "--read" => policy = policy.read(it.next().ok_or("--read needs a path")?), + "--exec" => policy = policy.exec(it.next().ok_or("--exec needs a path")?), + "--" => { + let p = it.next().ok_or("a program follows `--`")?; + program = Some((PathBuf::from(p), it.by_ref().map(OsString::from).collect())); + } + other => return Err(format!("unknown option `{other}`")), + } + } + // With a program, its standard output is passed through untouched, so this command's own + // lines go to standard error; the self-test's report is the output, on standard output. + let report = program.is_none(); + let say = |line: String| { + if report { + outln!("{line}"); + } else { + eprintln!("{line}"); + } + }; + match gridcraft_sandbox::support() { + Support::Available(how) => say(format!("sandbox: available ({how})")), + Support::Unsupported(why) => { + say(format!("sandbox: unsupported ({why})")); + return Ok(ExitCode::from(2)); + } + } + let canary = Canary::create().map_err(|e| e.to_string())?; + let exe = std::env::current_exe().map_err(|e| e.to_string())?; + let (program, program_args) = match program { + Some((p, a)) => (p, a), + // A confined copy of this program, reporting each attempt; it reads its own executable as + // the allowed file. + None => { + let home = std::env::var_os("HOME").unwrap_or_default(); + (exe.clone(), vec![REPORT_FLAG.into(), canary.file().into(), exe.clone().into(), home]) + } + }; + let policy = policy.exec(program.clone()); + let mut cmd = gridcraft_sandbox::launch_command(&policy, &canary, &program, &program_args).map_err(|e| e.to_string())?; + let status = cmd.status().map_err(|e| format!("couldn't start the launcher: {e}"))?; + drop(canary); + match status.code() { + Some(0) => { + say(if report { "result: every attempt is as the sandbox requires" } else { "result: the program ran confined and succeeded" }.into()); + Ok(ExitCode::SUCCESS) + } + Some(REFUSED_EXIT_CODE) => { + // The launcher's refusal and a program's own exit share the code; the launcher's reason, + // when it refused, is the `gridcraft-sandbox: refused:` line above. + say(format!( + "result: exit code {REFUSED_EXIT_CODE}: the launcher refused to run the program (its reason is above), or the program itself exited with {REFUSED_EXIT_CODE}" + )); + Ok(ExitCode::FAILURE) + } + other => { + say(format!( + "result: {} ({})", + if report { "some attempts are not as the sandbox requires" } else { "the program failed" }, + other.map_or("no exit code".into(), |c| format!("exit code {c}")) + )); + Ok(ExitCode::FAILURE) + } + } +} diff --git a/apps/gridcraft-cli/tests/it/main.rs b/apps/gridcraft-cli/tests/it/main.rs index 5906b3967..2078aedfd 100644 --- a/apps/gridcraft-cli/tests/it/main.rs +++ b/apps/gridcraft-cli/tests/it/main.rs @@ -1,3 +1,4 @@ //! The crate's integration tests, one module per file, built as one test binary. mod cli; +mod sandbox; mod text_to_columns_quotes; diff --git a/apps/gridcraft-cli/tests/it/sandbox.rs b/apps/gridcraft-cli/tests/it/sandbox.rs new file mode 100644 index 000000000..19861f8d8 --- /dev/null +++ b/apps/gridcraft-cli/tests/it/sandbox.rs @@ -0,0 +1,384 @@ +//! The confined launcher, through the real binary: what a confined program can't do, what it +//! still can, and that a launch whose check fails runs nothing. Linux and macOS confine; other +//! systems report that they can't. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; + +/// Whether this machine confines programs: a supported system whose self-test passes. On a Linux +/// or macOS machine that reports no sandbox (another architecture, no `sandbox-exec`) or whose +/// kernel doesn't enforce Landlock, the confinement tests have nothing to check and pass with a +/// note. On CI (`CI` set) the machine must confine, so a failure there is never skipped. +fn confines() -> bool { + static CONFINES: std::sync::OnceLock = std::sync::OnceLock::new(); + *CONFINES.get_or_init(|| { + if !cfg!(any(target_os = "linux", target_os = "macos")) { + return false; + } + let o = cli(&["sandbox-check"]); + let out = text(&o); + let missing = out.contains("sandbox: unsupported") || out.contains("Landlock is not enforced"); + if missing && std::env::var_os("CI").is_none() { + eprintln!("this machine doesn't confine programs; skipping:\n{out}"); + return false; + } + true + }) +} + +fn cli(args: &[&str]) -> Output { + Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).args(args).output().unwrap() +} + +fn text(o: &Output) -> String { + format!("{}{}", String::from_utf8_lossy(&o.stdout), String::from_utf8_lossy(&o.stderr)) +} + +/// A directory of the test's own in the temporary directory, with one file. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().canonicalize().unwrap().join(format!("gridcraft-sandbox-it-{name}-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("allowed.txt"), "allowed text\n").unwrap(); + dir +} + +/// `cat` as this system names it. +fn cat() -> &'static str { + if Path::new("/bin/cat").exists() { "/bin/cat" } else { "/usr/bin/cat" } +} + +#[test] +fn the_self_test_finds_every_denial() { + // A home directory of the test's own, so the home-directory attempt always runs. + let home = scratch("home"); + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")).arg("sandbox-check").env("HOME", &home).output().unwrap(); + let out = text(&o); + if !cfg!(any(target_os = "linux", target_os = "macos")) { + assert_eq!(o.status.code(), Some(2), "{out}"); + assert!(out.contains("sandbox: unsupported"), "{out}"); + return; + } + if !confines() { + return; + } + assert!(o.status.success(), "{out}"); + for line in [ + "denied: read the canary", + "denied: change the canary's permissions", + "denied: truncate the canary", + "denied: write next to the canary", + "denied: write in the temporary directory", + "denied: TCP connection", + "denied: start a process", + "denied: signal another process", + "denied: UDP socket", + "denied: list the home directory", + "works: start a thread", + "works: read an allowed file", + ] { + assert!(out.contains(line), "missing `{line}` in:\n{out}"); + } + std::fs::remove_dir_all(home).unwrap(); +} + +#[test] +fn a_confined_program_reads_only_what_it_is_given() { + if !confines() { + return; + } + let dir = scratch("read"); + let outside = scratch("outside"); + let d = dir.to_string_lossy().into_owned(); + let allowed = dir.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &allowed]); + assert!(o.status.success(), "{}", text(&o)); + // The program's output is passed through untouched; this command's own lines go to stderr. + assert_eq!(String::from_utf8_lossy(&o.stdout), "allowed text\n", "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stderr).contains("result: the program ran confined"), "{}", text(&o)); + // The same program, a file outside the policy: the read is refused and nothing is printed. + let other = outside.join("allowed.txt").to_string_lossy().into_owned(); + let o = cli(&["sandbox-check", "--read", &d, "--", cat(), &other]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(!String::from_utf8_lossy(&o.stdout).contains("allowed text"), "{}", text(&o)); + std::fs::remove_dir_all(dir).unwrap(); + std::fs::remove_dir_all(outside).unwrap(); +} + +#[test] +fn a_launch_whose_check_fails_runs_nothing() { + // A run that says it is already confined and isn't finds the canary readable and must refuse. + // On macOS that is the second stage started without `sandbox-exec`; on Linux the launcher never + // uses that flag, and the same refusal must hold. + let canary = scratch("canary"); + let echo = if Path::new("/bin/echo").exists() { "/bin/echo" } else { "/usr/bin/echo" }; + let probe = canary.join("allowed.txt").to_string_lossy().into_owned(); + // A dynamically linked `echo` needs the system's shared libraries and its loader. + // The second stage takes the process the first looked at: here, the parent of what `cli` starts. + let mut launch = vec![ + "--sandbox-confined".to_string(), + std::process::id().to_string(), + "--exec".into(), + echo.into(), + "--exec".into(), + env!("CARGO_BIN_EXE_gridcraft-cli").into(), + ]; + for lib in ["/lib", "/lib64", "/usr/lib", "/usr/lib64"].into_iter().filter(|d| Path::new(d).exists()) { + launch.extend(["--exec".to_string(), lib.into()]); + } + launch.extend(["--probe".to_string(), probe.clone(), "--".into(), echo.into(), "the program ran".into()]); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert_eq!(o.status.code(), Some(97), "{out}"); + assert!(!out.contains("the program ran"), "{out}"); + assert!(out.contains("gridcraft-sandbox: refused"), "{out}"); + assert!(out.contains("read the canary"), "the check refused, not the arguments: {out}"); + // Without that process, the second stage has nothing to try a signal on. + let without: Vec<&str> = std::iter::once("--sandbox-confined").chain(launch[2..].iter().map(String::as_str)).collect(); + let o = cli(&without); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(text(&o).contains("needs the process"), "{}", text(&o)); + // The normal path: the launcher confines itself first, and the program runs. + if confines() { + launch.splice(0..2, ["--sandbox-launch".to_string()]); + // The launcher may be under a folder in the list, on either system. + let mut by_folder = launch.clone(); + by_folder[4] = Path::new(env!("CARGO_BIN_EXE_gridcraft-cli")).parent().unwrap().to_string_lossy().into_owned(); + let o = cli(&by_folder.iter().map(String::as_str).collect::>()); + assert!(o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("the program ran"), "{}", text(&o)); + let o = cli(&launch.iter().map(String::as_str).collect::>()); + let out = text(&o); + assert!(o.status.success() && out.contains("the program ran"), "{out}"); + } + // A canary inside an allowed path is refused before anything runs, everywhere. + let dir = canary.to_string_lossy().into_owned(); + let o = cli(&["--sandbox-launch", "--read", &dir, "--exec", echo, "--probe", &probe, "--", echo, "the program ran"]); + assert_eq!(o.status.code(), Some(97), "{}", text(&o)); + assert!(!text(&o).contains("the program ran")); + std::fs::remove_dir_all(canary).unwrap(); +} + +/// A Python found on this system: the interpreter to run confined, and its installation, the path it +/// needs to read. `None` when there is no `python3`. +fn system_python() -> Option<(String, String)> { + // On macOS `/usr/bin/python3` is a stub that starts Xcode's Python, outside any read path given + // here; an installed Python comes first. + let candidates = if cfg!(target_os = "macos") { + ["/opt/homebrew/bin/python3", "/usr/local/bin/python3"] + } else { + ["/usr/bin/python3", "/usr/local/bin/python3"] + }; + let python = candidates.into_iter().find(|p| Path::new(p).exists())?; + let prefix = Command::new(python).args(["-c", "import sys; print(sys.base_prefix)"]).output().unwrap(); + let prefix = String::from_utf8_lossy(&prefix.stdout).trim().to_string(); + // A framework build's `bin/python3` only starts `Python.app`'s interpreter as a new process, + // which the sandbox refuses; the interpreter itself is run instead. + let framework = Path::new(&prefix).join("Resources/Python.app/Contents/MacOS/Python"); + let real = if framework.exists() { framework } else { std::fs::canonicalize(python).unwrap() }; + Some((real.to_string_lossy().into_owned(), prefix)) +} + +/// Confined, a program can't signal another process (this test, signal 0 delivering nothing), but can +/// still signal itself and its own threads. On Linux, where the filter names them, it also can't +/// change another process's limits or priority, or type into a terminal: no call here reaches a real +/// terminal, since its standard input isn't one. +#[test] +fn a_confined_program_leaves_other_processes_alone() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import os, signal, sys, threading +other = int(sys.argv[1]) +def refused(call, *args): + try: + call(*args) + except PermissionError: + return True + except OSError as e: + print("other error", call, args, e) + return False +assert refused(os.kill, other, 0), "a signal to another process" +os.kill(os.getpid(), 0) +seen = [] +signal.signal(signal.SIGUSR1, lambda *_: seen.append(1)) +signal.raise_signal(signal.SIGUSR1) +worker = threading.Thread(target=lambda: signal.pthread_kill(threading.main_thread().ident, signal.SIGUSR1)) +worker.start() +worker.join() +assert len(seen) >= 1, "a signal to itself" +if sys.platform == "linux": + import ctypes, errno, fcntl, resource, termios + libc = ctypes.CDLL(None, use_errno=True) + assert libc.shmget(0, 4096, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "System V shared memory" + assert libc.msgget(0, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V message queue" + assert libc.semget(0, 1, 0o600) == -1 and ctypes.get_errno() == errno.EPERM, "a System V semaphore" + if hasattr(libc, "mq_open"): + assert libc.mq_open(b"/gridcraft-check", os.O_RDONLY) == -1 and ctypes.get_errno() == errno.EPERM, "a POSIX message queue" + # macOS answers these for the processes the profile lets it signal, which are only itself. + assert refused(os.kill, -1, 0), "a signal to every process" + assert refused(os.kill, 0, 0), "a signal to the process group" + assert refused(resource.prlimit, other, resource.RLIMIT_NOFILE), "another process's limits" + assert refused(os.setpriority, os.PRIO_PROCESS, other, 0), "another process's priority" + assert refused(os.setpriority, os.PRIO_USER, 0, 0), "every process's priority" + assert refused(os.sched_setaffinity, other, {0}), "another process's processors" + assert refused(os.pidfd_open, other), "a handle on another process" + resource.prlimit(0, resource.RLIMIT_NOFILE) + resource.prlimit(os.getpid(), resource.RLIMIT_NOFILE) + os.setpriority(os.PRIO_PROCESS, 0, os.getpriority(os.PRIO_PROCESS, 0)) + assert refused(fcntl.ioctl, 0, termios.TIOCSTI, b"x"), "typing into a terminal" + assert refused(fcntl.ioctl, 0, termios.TIOCSTI | (1 << 32), b"x"), "the same with high bits set" + assert refused(fcntl.ioctl, 0, 0x541C, b"x"), "TIOCLINUX" +print("left alone") +"#; + let me = std::process::id().to_string(); + let o = cli(&["sandbox-check", "--read", &prefix, "--", &python, "-I", "-c", code, &me]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("left alone"), "{}", text(&o)); +} + +/// Confined, a program changes no file, whether the policy lets it read that file or not, by path or +/// by a descriptor: not its contents, permissions, owner, extended attributes, times or flags. +/// Skipped when there is no `python3`. +#[test] +fn a_confined_program_changes_no_file() { + if !confines() { + return; + } + let Some((python, prefix)) = system_python() else { + return; + }; + let code = r#" +import errno, fcntl, os, struct, sys +readable, outside = sys.argv[1:3] +def refused(what, call, *args): + try: + call(*args) + except PermissionError: + return + except OSError as e: + sys.exit(f"{what}: another error, not a refusal: {e}") + sys.exit(f"{what}: it worked") +fd = os.open(readable, os.O_RDONLY) +for target in (readable, outside, fd): + refused(f"permissions of {target}", os.chmod, target, 0) + refused(f"owner of {target}", os.chown, target, os.getuid(), os.getgid()) + refused(f"times of {target}", os.utime, target, (0, 0)) + if sys.platform == "linux": + refused(f"setting an attribute of {target}", os.setxattr, target, "user.gridcraft", b"1") + refused(f"removing an attribute of {target}", os.removexattr, target, "user.gridcraft") +for path in (readable, outside): + refused(f"truncate {path}", os.truncate, path, 0) + refused(f"open {path} with O_TRUNC", os.open, path, os.O_RDONLY | os.O_TRUNC) + refused(f"open {path} to write", os.open, path, os.O_WRONLY) +# A descriptor opened to read can't be truncated either way; the file's contents are checked after. +try: + os.ftruncate(fd, 0) +except OSError: + pass +if sys.platform == "linux": + import ctypes + libc = ctypes.CDLL(None, use_errno=True) + # Newer calls, which the kernel running this may not have: the filter answers before it would. + for name, number in (("fchmodat2", 452), ("setxattrat", 463), ("removexattrat", 466), ("file_setattr", 469)): + ctypes.set_errno(0) + result = libc.syscall(number, -100, readable.encode(), 0, 0) + assert result == -1 and ctypes.get_errno() == errno.EPERM, f"{name}: {result}, {ctypes.get_errno()}" + # On a descriptor opened only to read, these change the file's flags and attributes. + refused("FS_IOC_SETFLAGS", fcntl.ioctl, fd, 0x40086602, struct.pack("l", 0)) + refused("FS_IOC_FSSETXATTR", fcntl.ioctl, fd, 0x401C5820, bytes(28)) + refused("FS_IOC_SET_ENCRYPTION_POLICY", fcntl.ioctl, fd, 0x800C6613, bytes(12)) + # What a program still needs: its own descriptors' flags, and a file of its own in memory. + os.set_inheritable(fd, True) + os.set_inheritable(fd, False) + os.set_blocking(fd, False) + own = os.memfd_create("gridcraft-check") + os.ftruncate(own, 16) + assert os.fstat(own).st_size == 16 +print("unchanged") +"#; + let inside = scratch("unchanged-inside"); + let elsewhere = scratch("unchanged-outside"); + let (readable, outside) = (inside.join("allowed.txt"), elsewhere.join("allowed.txt")); + let state = |file: &Path| { + let m = std::fs::metadata(file).unwrap(); + (std::fs::read(file).unwrap(), m.permissions(), m.modified().unwrap()) + }; + let before = (state(&readable), state(&outside)); + let d = inside.to_string_lossy().into_owned(); + let (r, x) = (readable.to_string_lossy().into_owned(), outside.to_string_lossy().into_owned()); + let o = cli(&["sandbox-check", "--read", &prefix, "--read", &d, "--", &python, "-I", "-c", code, &r, &x]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("unchanged"), "{}", text(&o)); + assert_eq!((state(&readable), state(&outside)), before); + std::fs::remove_dir_all(inside).unwrap(); + std::fs::remove_dir_all(elsewhere).unwrap(); +} + +/// A Python found on this system starts and imports its standard library while confined, with +/// read access to its installation only. Skipped when there is no `python3`. +#[test] +fn python_starts_confined() { + if !confines() { + return; + } + let Some((real, prefix)) = system_python() else { + return; + }; + let o = + cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import json, math, datetime, decimal; print(json.dumps([math.pi > 3]))"]); + assert!(o.status.success(), "{}", text(&o)); + assert!(String::from_utf8_lossy(&o.stdout).contains("[true]"), "{}", text(&o)); + // Confined Python can't open the network or a file outside its installation. + let o = cli(&["sandbox-check", "--read", &prefix, "--", &real, "-I", "-c", "import socket; socket.create_connection(('127.0.0.1', 9), 2)"]); + assert!(!o.status.success(), "{}", text(&o)); + assert!(text(&o).contains("PermissionError") || text(&o).contains("Operation not permitted"), "{}", text(&o)); +} + +#[test] +fn a_confined_program_inherits_no_secrets() { + if !confines() { + return; + } + let env = if Path::new("/usr/bin/env").exists() { "/usr/bin/env" } else { "/bin/env" }; + let o = Command::new(env!("CARGO_BIN_EXE_gridcraft-cli")) + .args(["sandbox-check", "--", env]) + .env("GRIDCRAFT_CONTROL_TOKEN", "secret-token") + .env("AWS_SECRET_ACCESS_KEY", "secret-key") + .env("LANG", "en_US.UTF-8") + .output() + .unwrap(); + let out = text(&o); + assert!(o.status.success(), "{out}"); + assert!(!out.contains("secret-"), "the environment reached the program:\n{out}"); + assert!(out.contains("LANG=en_US.UTF-8"), "the locale is kept:\n{out}"); + assert!(out.contains("PATH=/usr/bin:/bin"), "{out}"); +} + +/// A descriptor the parent leaves open (here fd 9, opened by a shell on a file outside any policy) +/// would survive `exec` with access the sandbox doesn't revoke, so the launcher closes it: the +/// program runs, and can't read through it. (A desktop app's system libraries leave such files open, +/// so refusing to run would block Python there.) +#[test] +fn an_inherited_descriptor_never_reaches_the_program() { + if !confines() { + return; + } + let outside = scratch("leak"); + let leaked = outside.join("allowed.txt").to_string_lossy().into_owned(); + let cli = env!("CARGO_BIN_EXE_gridcraft-cli"); + // The shell opens fd 9 (not close-on-exec) and then becomes the command, which inherits it. The + // confined shell then tries to read through it, with a built-in: starting `cat` is refused. Not + // fd 3: on macOS the system's libraries hold a socket of their own there under the sandbox. + let bash = if cfg!(target_os = "macos") { "--exec /bin/bash" } else { "" }; + let program = "read line <&9 && echo \"leaked: $line\"; echo finished"; + let script = format!("exec 9<'{leaked}'; exec '{cli}' sandbox-check {bash} -- /bin/sh -c '{program}'"); + let o = Command::new("/bin/sh").args(["-c", &script]).output().unwrap(); + let out = text(&o); + assert!(out.contains("finished"), "the program ran:\n{out}"); + assert!(!out.contains("leaked:"), "the program read through the descriptor:\n{out}"); + std::fs::remove_dir_all(outside).unwrap(); +} diff --git a/apps/gridcraft/Cargo.toml b/apps/gridcraft/Cargo.toml index a7472821f..8704a6e39 100644 --- a/apps/gridcraft/Cargo.toml +++ b/apps/gridcraft/Cargo.toml @@ -13,6 +13,7 @@ workspace = true [dependencies] gridcraft-engine = { workspace = true } gridcraft-ui-egui = { workspace = true } +gridcraft-sandbox = { workspace = true } eframe = { workspace = true, features = ["default_fonts", "wgpu", "accesskit", "wayland", "x11", "clipboard"] } egui = { workspace = true } serde_json = { workspace = true } @@ -34,7 +35,7 @@ winit = { version = "0.30", default-features = false, features = ["x11", "waylan [target.'cfg(not(target_arch = "wasm32"))'.dependencies] # Same craft-libs revision as the workspace's craft-update and craft-window (see the root # Cargo.toml). -craft-print = { git = "https://github.com/storytold/craft-libs", rev = "c5a1cbfb5f9049f9270955caffb5bee5997aebf7" } +craft-print = { git = "https://github.com/storytold/craft-libs", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b" } craft-update = { workspace = true } rfd = { workspace = true } arboard = { workspace = true, features = ["image-data"] } diff --git a/apps/gridcraft/src/main.rs b/apps/gridcraft/src/main.rs index 2aa98b6eb..f05b421df 100644 --- a/apps/gridcraft/src/main.rs +++ b/apps/gridcraft/src/main.rs @@ -64,6 +64,9 @@ impl eframe::App for App { fn on_exit(&mut self) { print::finish_pending_jobs(); save_prefs(&self.0); + // A ready update installs once the process has exited. The macOS exit below skips + // dropping the app, so the helper is started here. + self.0.start_pending_update(); // macOS (#117): end the process here, before eframe tears the window down. AccessKit's // macOS adapter subclasses the window's content view and restores its class on teardown; // AppKit's Touch Bar support observes that view (KVO), so in the next display cycle @@ -238,6 +241,12 @@ fn icon() -> Option { } fn main() -> eframe::Result<()> { + // A confined launcher run (`gridcraft_sandbox::launch_command`) comes before everything else: + // it opens no window, writes no log, and becomes the program it confines. + let raw: Vec = std::env::args_os().skip(1).collect(); + if let Some(code) = gridcraft_sandbox::handle_args(&raw) { + std::process::exit(code); + } // First, so the panic hook and every start-up warning are recorded (`logging`). let logger = logging::install(); // Commands catch their panics (`Session::execute`), but the default hook only prints to diff --git a/contributors/people.toml b/contributors/people.toml index 1b8bcf996..395aecf6e 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -110,3 +110,6 @@ sync_github_name = true [people.maxm11] sync_github_name = true + +[people.RockerHX] +sync_github_name = true diff --git a/crates/calc/src/eval.rs b/crates/calc/src/eval.rs index 41ffca9c2..525440474 100644 --- a/crates/calc/src/eval.rs +++ b/crates/calc/src/eval.rs @@ -39,6 +39,15 @@ const MAX_DEPTH: usize = 400; /// every legitimate call; hostile work that grows past one dimension (a LAMBDA that recurses /// or nest `MAP` into `MAP`) then stops at the same bound instead of squaring. const LAMBDA_FUEL: u64 = MAX_CELLS; +/// Work one cell's formula may do: one unit per expression evaluated, per cell read from a +/// reference, per array value handed to a function, per pair of areas intersected and per +/// character of a defined name's formula parsed where the name is used. Each limit +/// above bounds one kind of step, but a compact formula could still repeat bounded steps +/// exponentially often (defined names that each use the one before twice, a computed criterion +/// whose database function reruns it, an aggregate over a large array inside every LAMBDA call). +/// Past this the formula is `#CALC!`. It is far above what a formula that reads every cell of a +/// full-size array a few times needs. +const EVAL_WORK: u64 = 16 * MAX_CELLS; #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub struct Area { @@ -145,6 +154,16 @@ pub struct Evaluator<'h> { /// runs out of work after the same number of calls that a legitimately wide one could /// make, instead of an amount of work that squares with array nesting (#2382). fuel: u64, + /// Work this cell still has left (see [`EVAL_WORK`]), shared by every branch of the formula + /// and by the formulas it computes on the way (names, computed criteria). + work: u64, + /// A spend found too little work left. A spend that leaves exactly nothing is fine, so this + /// can't be told from `work` alone; a database function checks it after its computed + /// criteria ran, whose failures it can't otherwise see. + exhausted: bool, + /// Computed criteria being evaluated, outermost first: one that is reached again from inside + /// itself counts as not met instead of repeating its work at every level. + active_criteria: Vec<(usize, CellRef)>, } struct FnCtx<'a, 'h> { @@ -174,6 +193,9 @@ impl Ctx for FnCtx<'_, '_> { fn computed_criterion(&mut self, row: usize, col: usize, db: &Array, record: usize) -> Option { let area = self.criteria?; let at = area.range.start.offset(row as i64, col as i64).filter(|c| area.range.contains(*c))?; + if self.ev.active_criteria.contains(&(area.sheet, at)) { + return Some(false); + } let expr = self.ev.host.workbook().sheet(area.sheet)?.cell(at)?.formula.as_ref()?.expr_arc()?; let shifted = gridcraft_formula::adjust::shift_relative((*expr).clone(), record as i64 - 1, 0); let env = (0..db.cols) @@ -193,12 +215,24 @@ impl Ctx for FnCtx<'_, '_> { calls: self.ev.calls, stack_top: self.ev.stack_top, fuel: self.ev.fuel, + work: self.ev.work, + exhausted: false, + active_criteria: std::mem::take(&mut self.ev.active_criteria), }; + ev.active_criteria.push((area.sheet, at)); let v = matches!(ev.value(&shifted).scalar().to_bool(), Ok(true)); // The child's spend counts against this cell's budget. self.ev.fuel = ev.fuel; + self.ev.work = ev.work; + self.ev.exhausted |= ev.exhausted; + let mut active = std::mem::take(&mut ev.active_criteria); + active.pop(); + self.ev.active_criteria = active; Some(v) } + fn exhausted(&self) -> bool { + self.ev.exhausted + } fn has_lambda(&self) -> bool { self.lambda.is_some() } @@ -216,7 +250,42 @@ impl Ctx for FnCtx<'_, '_> { impl<'h> Evaluator<'h> { pub fn new(host: &'h mut dyn Host, sheet: usize, at: CellRef) -> Self { - Evaluator { host, sheet, at, env: Vec::new(), depth: 0, calls: 0, stack_top: 0, fuel: LAMBDA_FUEL } + Evaluator { + host, + sheet, + at, + env: Vec::new(), + depth: 0, + calls: 0, + stack_top: 0, + fuel: LAMBDA_FUEL, + work: EVAL_WORK, + exhausted: false, + active_criteria: Vec::new(), + } + } + + /// Spends `units` of this cell's work; `false` (and nothing left) when there isn't enough. + pub(crate) fn spend(&mut self, units: u64) -> bool { + match self.work.checked_sub(units) { + Some(w) => { + self.work = w; + true + } + None => { + self.work = 0; + self.exhausted = true; + false + } + } + } + + /// Charges the values of an array (one unit per value; one for anything else). + pub(crate) fn spend_on(&mut self, v: &Value) -> bool { + match v { + Value::Array(a) => self.spend(a.data.len() as u64), + _ => self.spend(1), + } } /// Evaluates to a value (references are dereferenced; multi-cell → array). @@ -266,18 +335,27 @@ impl<'h> Evaluator<'h> { } // A reference keeps its size, so the array has its shape and its blanks count // (`=A1:A10`, `SUMPRODUCT(--(A1:A10=""))`). + let range = self.read_range(a); + let (h, w) = (range.height() as usize, range.width() as usize); + if (h as u64) * (w as u64) > MAX_CELLS { + return Value::Error(CellError::Num); + } + if !self.spend((h as u64) * (w as u64)) { + return Value::Error(CellError::Calc); + } + let data = self.host.range_values(a.sheet, range); + Array::new(h, w, data).map(Value::from).unwrap_or(Value::Error(CellError::Value)) + } + + /// The block that reading `a` gives: `a` itself, or when it is large (or runs to the sheet's + /// edge) the part inside the sheet's used range. + pub(crate) fn read_range(&mut self, a: &Area) -> RangeRef { let cells = u64::from(a.range.height()) * u64::from(a.range.width()); // `A:A`, `1:1` and `A2:A1048576` run to the edge of the sheet: they stand for "the rest of // the column", not for a block of a chosen size. let to_edge = a.range.end.row == MAX_ROWS - 1 || a.range.end.col == MAX_COLS - 1; let full_size = cells <= FULL_SIZE_CELLS || (!to_edge && cells <= FULL_SIZE_BOUNDED_CELLS); - let range = if full_size { a.range } else { self.trim(a) }; - let (h, w) = (range.height() as usize, range.width() as usize); - if (h as u64) * (w as u64) > MAX_CELLS { - return Value::Error(CellError::Num); - } - let data = self.host.range_values(a.sheet, range); - Array::new(h, w, data).map(Value::from).unwrap_or(Value::Error(CellError::Value)) + if full_size { a.range } else { self.trim(a) } } /// Trims the bottom/right of an area to the sheet's used range (keeps the top-left so @@ -310,6 +388,9 @@ impl<'h> Evaluator<'h> { self.host.stack_exhausted(); return err(CellError::Num); } + if !self.spend(1) { + return err(CellError::Calc); + } self.depth += 1; if self.depth > MAX_DEPTH { self.depth -= 1; @@ -382,6 +463,11 @@ impl<'h> Evaluator<'h> { if let Some(def) = wb.name(bare, scope_sheet) { let text = def.formula.clone(); let scope = def.scope.unwrap_or(self.sheet); + // Each use parses and evaluates the definition again: names that each use the one + // before twice double the work per name, so it is charged. + if !self.spend(text.len() as u64) { + return err(CellError::Calc); + } return match gridcraft_formula::parse_name(&text) { Ok(expr) => { // Relative references in a name are relative to A1: used here, they move by @@ -543,6 +629,11 @@ impl<'h> Evaluator<'h> { if r.height() == 1 && r.start.col <= self.at.col && self.at.col <= r.end.col { return Some(CellRef::new(r.start.row, self.at.col)); } + // A 2-D range intersects on both the formula's row and column (observed in Excel, on + // another sheet too); one that includes the formula's own cell is a circular reference. + if r.start.row <= self.at.row && self.at.row <= r.end.row && r.start.col <= self.at.col && self.at.col <= r.end.col { + return Some(self.at); + } None } @@ -574,6 +665,10 @@ impl<'h> Evaluator<'h> { let (x, y) = (self.eval(a), self.eval(b)); match (x, y) { (Ev::R(mut xa), Ev::R(ya)) => { + // A union reused through LET doubles at each step: its areas count. + if !self.spend((xa.len() as u64).saturating_add(ya.len() as u64)) { + return err(CellError::Calc); + } xa.extend(ya); Ev::R(xa) } @@ -585,6 +680,11 @@ impl<'h> Evaluator<'h> { let (x, y) = (self.eval(a), self.eval(b)); match (x, y) { (Ev::R(xa), Ev::R(ya)) => { + // Every pair is looked at, and each may be kept: charged before the loop, + // so unions squared through LET can't build an enormous list. + if !self.spend((xa.len() as u64).saturating_mul(ya.len() as u64)) { + return err(CellError::Calc); + } let mut out = Vec::new(); for p in &xa { for q in &ya { @@ -678,6 +778,113 @@ mod fuel_tests { ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))) } + /// Evaluates `src` in `wb` with `work` units of work, and the work left. + fn eval_work(wb: Workbook, work: u64, src: &str) -> (Value, u64) { + let mut host = Empty { wb }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(0, 0)); + ev.work = work; + let v = ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))); + (v, ev.work) + } + + fn is_calc(v: &Value) -> bool { + matches!(v, Value::Error(CellError::Calc)) + } + + /// Serves the workbook's stored values, for database functions to read. + struct Stored { + wb: Workbook, + } + impl Host for Stored { + fn workbook(&self) -> &Workbook { + &self.wb + } + fn cell_value(&mut self, sheet: usize, c: CellRef) -> Value { + self.wb.sheet(sheet).and_then(|s| s.cell(c)).map_or(Value::Empty, |cell| cell.value.clone()) + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + } + + /// A computed criterion that runs out of work counts as not met for its record, which the + /// database function can't see: it must answer #CALC!, not a count that leaves records out. + #[test] + fn a_computed_criterion_out_of_work_makes_the_database_function_calc() { + use gridcraft_model::{Cell, Formula}; + let mut wb = Workbook::new(); + let name = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(name("Step0".into(), "1".into())); + for i in 1..=60 { + wb.names.push(name(format!("Step{i}"), format!("Step{}+Step{}", i - 1, i - 1))); + } + let at = |s: &str| CellRef::parse(s).unwrap(); + let sheet = wb.sheet_mut(0).unwrap(); + for (r, (n, age)) in [("name", Value::text("age")), ("a", Value::number(1.0)), ("b", Value::number(2.0))].into_iter().enumerate() { + sheet.cells.set(at(&format!("A{}", r + 1)), Cell::value(Value::text(n))); + sheet.cells.set(at(&format!("B{}", r + 1)), Cell::value(age)); + } + sheet.cells.set(at("D1"), Cell::value(Value::text("Formula"))); + let host_wb = |criterion: &str| { + let mut wb = wb.clone(); + // A criteria cell is read by its value first (a blank one matches everything): give it + // the value a recalculation would have stored. + let mut cell = Cell::formula(Formula::new(criterion)); + cell.value = Value::Bool(true); + wb.sheet_mut(0).unwrap().cells.set(at("D2"), cell); + wb + }; + let run = |criterion: &str| { + let mut host = Stored { wb: host_wb(criterion) }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(9, 9)); + ev.work = 100_000; + ev.value(&gridcraft_formula::parser::parse("=DCOUNT(A1:B3,\"age\",D1:D2)").unwrap()) + }; + assert_eq!(run("=B2>0"), Value::number(2.0), "a cheap criterion counts every record"); + assert!(is_calc(&run("=Step60>0")), "{:?}", run("=Step60>0")); + } + + #[test] + fn names_that_double_at_each_step_run_out_of_work() { + // Step1 = Step0+Step0, Step2 = Step1+Step1, ...: Step60 would evaluate Step0 2^60 times. + let mut wb = Workbook::default(); + let name = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(name("Step0".into(), "1".into())); + for i in 1..=60 { + wb.names.push(name(format!("Step{i}"), format!("Step{}+Step{}", i - 1, i - 1))); + } + let (v, _) = eval_work(wb.clone(), EVAL_WORK, "=Step5"); + assert_eq!(v, Value::number(32.0)); + let (v, left) = eval_work(wb, 100_000, "=Step60"); + assert!(is_calc(&v), "{v:?}"); + assert_eq!(left, 0); + } + + #[test] + fn intersections_of_unions_are_charged_per_pair() { + let wb = Workbook::default(); + // 4 x 4 pairs, 4 of which meet. + let src = "=AREAS((A1,B1,C1,D1) (A1,B1,C1,D1))"; + let (v, _) = eval_work(wb.clone(), EVAL_WORK, src); + assert_eq!(v, Value::number(4.0)); + let (v, _) = eval_work(wb, 20, src); + assert!(is_calc(&v), "{v:?}"); + } + + #[test] + fn an_aggregate_inside_every_lambda_call_spends_work() { + let wb = Workbook::default(); + // 100 calls, each summing the 10,000 values of a captured array: about a million units. + let src = "=LET(big,SEQUENCE(10000),SUM(MAP(SEQUENCE(100),LAMBDA(x,SUM(big)))))"; + let (v, _) = eval_work(wb.clone(), 2_000_000, src); + assert_eq!(v, Value::number(100.0 * 50_005_000.0)); + let (v, _) = eval_work(wb, 200_000, src); + assert!(is_calc(&v), "{v:?}"); + } + #[test] fn lambda_calls_spend_fuel_and_run_out_quietly() { // Three applications, one charge each: 3 inside the budget. (A nested MAP wouldn't do diff --git a/crates/calc/src/eval/call.rs b/crates/calc/src/eval/call.rs index 8ac58c8d1..eb426398a 100644 --- a/crates/calc/src/eval/call.rs +++ b/crates/calc/src/eval/call.rs @@ -52,13 +52,16 @@ impl<'h> Evaluator<'h> { "ISFORMULA" | "FORMULATEXT" => { let Some(Ev::R(a)) = args.first().map(|a| self.eval(a)) else { return err(CellError::NA) }; let Some(a) = a.first() else { return err(CellError::NA) }; - let f = self.host.workbook().sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + let wb = self.host.workbook(); + let f = wb.sheet(a.sheet).and_then(|s| s.cell(a.range.start)).and_then(|c| c.formula.clone()); + // A formula formatted Hidden on a protected sheet stays hidden from FORMULATEXT too. + let hidden = wb.sheet(a.sheet).is_some_and(|s| s.is_protected() && wb.styles.get(s.style_id(a.range.start)).protection.hidden); if name == "ISFORMULA" { Ev::V(Value::Bool(f.is_some())) } else { match f { - Some(f) => Ev::V(Value::text(format!("={}", f.text))), - None => err(CellError::NA), + Some(f) if !hidden => Ev::V(Value::text(format!("={}", f.text))), + _ => err(CellError::NA), } } } @@ -203,9 +206,9 @@ impl<'h> Evaluator<'h> { let index = self.value(&args[0]); if let Value::Array(_) = index { let Some(spec) = gridcraft_functions::lookup("CHOOSE") else { return err(CellError::Name) }; - let mut evaluated = vec![Arg { value: index, from_ref: false }]; + let mut evaluated = vec![Arg::val(index)]; for argument in &args[1..] { - evaluated.push(Arg { value: self.value(argument), from_ref: false }); + evaluated.push(Arg::val(self.value(argument))); } let mut context = FnCtx { ev: self, criteria: None, lambda: None }; return gridcraft_functions::call(spec, &evaluated, &mut context).pipe(Ev::V); @@ -351,10 +354,25 @@ impl<'h> Evaluator<'h> { } ev => self.deref(ev), }; - evaluated.push(Arg { value, from_ref }); + // Only an argument left out (`f(a,,b)`) is omitted; one that computes a blank is given. + let blank_value = !from_ref && value == Value::Empty && lambda_at != Some(i) && !matches!(a, Expr::Missing); + evaluated.push(Arg { value, from_ref, blank_value }); } + // The function may go through every value it is given (a SUM inside each call of a + // LAMBDA over a large array): the values count against the cell's work. + let values = evaluated.iter().fold(0u64, |n, a| n.saturating_add(if let Value::Array(x) = &a.value { x.data.len() as u64 } else { 1 })); + if !self.spend(values) { + return err(CellError::Calc); + } + let database = criteria.is_some(); let mut ctx = FnCtx { ev: self, criteria, lambda }; - gridcraft_functions::call(spec, &evaluated, &mut ctx).pipe(Ev::V) + let v = gridcraft_functions::call(spec, &evaluated, &mut ctx); + // A computed criterion that ran out of work counted as not met: the result leaves + // records out. + if database && self.exhausted { + return err(CellError::Calc); + } + Ev::V(v) } /// Performs aggregation operations without copying its ranges: blocks are folded over the stored @@ -392,6 +410,12 @@ impl<'h> Evaluator<'h> { acc.count = acc.count.saturating_add(count); return Ok(()); } + // Folded without building an array, but still walked: the part inside the used + // range counts against the cell's work. + let r = self.read_range(a); + if !self.spend(u64::from(r.height()) * u64::from(r.width())) { + return Err(err(CellError::Calc)); + } let mut error = None; let walked = self.host.visit_range(a.sheet, a.range, &mut |v| match acc.add_cell(kind, v) { Ok(()) => true, @@ -411,10 +435,13 @@ impl<'h> Evaluator<'h> { /// Adds one argument's value by calling `SUM` and `COUNT` on it, so coercion and error rules /// stay in one place (`AVERAGE` of an argument is its `SUM` over its `COUNT`). fn aggregate_value(&mut self, kind: Agg, value: Value, from_ref: bool, acc: &mut Acc) -> Result<(), Ev> { + if !from_ref && !self.spend_on(&value) { + return Err(err(CellError::Calc)); + } let (Some(sum), Some(count)) = (gridcraft_functions::lookup("SUM"), gridcraft_functions::lookup("COUNT")) else { return Err(err(CellError::Name)); }; - let args = [Arg { value, from_ref }]; + let args = [Arg { value, from_ref, blank_value: false }]; if kind != Agg::Count { match gridcraft_functions::call(sum, &args, &mut FnCtx { ev: self, criteria: None, lambda: None }) { Value::Number(n) => acc.sum += n, diff --git a/crates/calc/src/eval/reference_fns.rs b/crates/calc/src/eval/reference_fns.rs index 4f5154c2d..6cec46deb 100644 --- a/crates/calc/src/eval/reference_fns.rs +++ b/crates/calc/src/eval/reference_fns.rs @@ -118,11 +118,29 @@ impl<'h> Evaluator<'h> { for c in 0..w { let pos = (num(&Some(ra.get_broadcast(r, c))), num(&Some(ca.get_broadcast(r, c)))); let v = match pos { + // A position that picks several cells is #CALC!. That is decided from the + // shape: reading the block first, for every position, let a short formula + // read one large range over and over only to discard it. + (Ok(row), Ok(col)) if index_picks_several_values(&src, row, col, two) => Value::Error(CellError::Calc), (Ok(row), Ok(col)) => { let one = self.index_one(src.clone(), row, col, area_n, two); - match self.deref(one) { - Value::Array(a) if a.rows * a.cols > 1 => Value::Error(CellError::Calc), - v => v.scalar(), + let cells = match &one { + Ev::R(areas) => match areas.as_slice() { + [a] => self.area_cells(a), + _ => 0, + }, + _ => 0, + }; + if cells > MAX_CELLS { + // As reading it would answer. + Value::Error(CellError::Num) + } else if cells > 1 { + Value::Error(CellError::Calc) + } else { + match self.deref(one) { + Value::Array(a) if a.rows * a.cols > 1 => Value::Error(CellError::Calc), + v => v.scalar(), + } } } (Err(e), _) | (_, Err(e)) => Value::Error(e), @@ -139,6 +157,16 @@ impl<'h> Evaluator<'h> { self.index_one(src, row, col, area_n, two) } + /// Cells that reading `a` gives: its own size, or for a large block the part inside the + /// sheet's used range (what [`Evaluator::area_value`] reads). + fn area_cells(&mut self, a: &Area) -> u64 { + if a.range.is_single() { + return 1; + } + let r = self.read_range(a); + u64::from(r.height()) * u64::from(r.width()) + } + /// INDEX for one row and column position (0 = the whole column or row). fn index_one(&mut self, src: Ev, row: i64, col: i64, area_n: i64, two_args: bool) -> Ev { if row < 0 || col < 0 { @@ -203,7 +231,7 @@ impl<'h> Evaluator<'h> { let horizontal = l.range.height() == 1 && l.range.width() > 1; let same_len = if vertical { r.range.height() == l.range.height() } else { r.range.width() == l.range.width() }; if !matches!(key, Value::Array(_)) && (vertical || horizontal) && same_len { - let mut xargs = vec![Arg { value: key, from_ref: false }, Arg { value: self.area_value(&l), from_ref: true }]; + let mut xargs = vec![Arg::val(key), Arg::reference(self.area_value(&l))]; // An empty match_mode before a given search_mode is the default (0), not the end. let given = |i: usize| args.get(i).is_some_and(|e| !matches!(e, Expr::Missing)); let last = if given(5) { @@ -215,10 +243,11 @@ impl<'h> Evaluator<'h> { }; for i in 4..=last { match args.get(i) { - Some(Expr::Missing) | None => xargs.push(Arg { value: Value::Number(0.0), from_ref: false }), + Some(Expr::Missing) | None => xargs.push(Arg::val(Value::Number(0.0))), Some(e) => { let value = self.value(e); - xargs.push(Arg { value, from_ref: false }); + let blank_value = value == Value::Empty; + xargs.push(Arg { value, from_ref: false, blank_value }); } } } @@ -407,3 +436,71 @@ impl<'h> Evaluator<'h> { Ev::V(gridcraft_functions::aggregate_values(func, &values)) } } + +/// Whether INDEX on an array picks a whole row or column of several values at position +/// (`row`, `col`), which the array-of-positions form answers with #CALC!. +fn index_picks_several_values(src: &Ev, row: i64, col: i64, two_args: bool) -> bool { + let Ev::V(Value::Array(arr)) = src else { return false }; + let (row, col) = if two_args && arr.rows == 1 { (1, row) } else { (row, col) }; + if row < 0 || col < 0 || row as u64 > arr.rows as u64 || col as u64 > arr.cols as u64 { + return false; + } + (row == 0 && arr.rows > 1) || (col == 0 && arr.cols > 1) +} + +#[cfg(test)] +mod tests { + use super::super::Host; + use super::*; + use gridcraft_model::Workbook; + + /// A sheet of blanks that counts the blocks it is asked for. + struct Counting { + wb: Workbook, + cells_read: usize, + } + impl Host for Counting { + fn workbook(&self) -> &Workbook { + &self.wb + } + fn cell_value(&mut self, _sheet: usize, _c: CellRef) -> Value { + Value::Empty + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + fn range_values(&mut self, _sheet: usize, range: RangeRef) -> Vec { + self.cells_read += range.count() as usize; + vec![Value::Empty; range.count() as usize] + } + } + + fn eval(src: &str) -> (Value, usize) { + let mut host = Counting { wb: Workbook::default(), cells_read: 0 }; + let mut ev = Evaluator::new(&mut host, 0, CellRef::new(0, 0)); + let v = ev.value(&gridcraft_formula::parser::parse(src).unwrap_or_else(|e| panic!("{e}"))); + (v, host.cells_read) + } + + fn all_calc(v: &Value, n: usize) -> bool { + matches!(v, Value::Array(a) if a.data.len() == n && a.data.iter().all(|x| *x == Value::Error(CellError::Calc))) + } + + #[test] + fn index_positions_that_pick_several_cells_are_calc_without_reading_them() { + // Each zero row position picks the whole of A1:A50000: #CALC!, decided from the shape. It + // used to read the 50,000 cells once per position before discarding them. + let (v, read) = eval("=INDEX(A1:A50000,{0,0,0,0},1)"); + assert!(all_calc(&v, 4), "{v:?}"); + assert_eq!(read, 0); + // The same for an array source: a whole column of a 3x3 array. + let (v, _) = eval("=INDEX({1,2,3;4,5,6;7,8,9},{0,0},2)"); + assert!(all_calc(&v, 2), "{v:?}"); + // Single cells are still read. + let (v, _) = eval("=INDEX({1,2,3;4,5,6;7,8,9},{1,3},2)"); + assert!(matches!(&v, Value::Array(a) if a.data == vec![Value::number(2.0), Value::number(8.0)]), "{v:?}"); + } +} diff --git a/crates/calc/src/lib.rs b/crates/calc/src/lib.rs index 2cc6bf4cf..e30cff29f 100644 --- a/crates/calc/src/lib.rs +++ b/crates/calc/src/lib.rs @@ -6,6 +6,7 @@ pub mod eval; mod precision; pub mod recalc; +mod spill_index; mod trimrange; pub use eval::{Area, SPECIAL_FUNCTIONS, is_known_function}; diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 5a3f27cc0..2b2ae7019 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -10,13 +10,24 @@ //! and the formulas that read one are evaluated again until the loop settles //! (`PassHost::iterate_loops`). -use std::collections::VecDeque; +use std::collections::{BTreeMap, BTreeSet, VecDeque}; use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; use gridcraft_formula::Expr; use gridcraft_model::{CalcMode, Workbook}; use crate::eval::{Evaluator, Host, precedents, stack_position}; +use crate::spill_index::{SpillIndex, indexing_pays}; + +/// Changed cells (in one `cells_changed`) from which a sheet's spill areas may be indexed. +const INDEX_SPILLS_FROM: usize = 16; + +/// Whether a sheet's spill areas are indexed for a batch of `batch` changed cells of which +/// `on_sheet` are on that sheet: the batch must be a paste, and registering the areas must cost +/// less than walking them once for each of the sheet's own changed cells. +fn index_spills<'a>(batch: usize, on_sheet: usize, areas: impl ExactSizeIterator) -> bool { + batch > INDEX_SPILLS_FROM && areas.len() > 0 && indexing_pays(areas, on_sheet) +} mod graph; mod pass_host; @@ -37,11 +48,61 @@ pub struct Calc { /// Cells whose formulas have errors from cycles. pub circular: Vec, pub last_recalc_cells: usize, + /// The cells whose value is `#SPILL!`, kept so that finding the formulas an emptied cell or a + /// freed spill area may unblock doesn't walk every formula or window of cells. Filled in + /// `rebuild`, kept up to date by `cells_changed` and by the write-back of a run; an entry is + /// always checked against the workbook before it is used. + spill_errors: SpillErrors, +} + +/// A set of cells by row, each row's columns in their own set: a window of rows and columns is +/// found by one column range per row that has an entry, so the cost follows the cells in the +/// window, not those to the side of it in the same rows. +#[derive(Clone, Debug, Default)] +struct SpillErrors { + rows: BTreeMap<(usize, u32), BTreeSet>, +} + +impl SpillErrors { + fn is_empty(&self) -> bool { + self.rows.is_empty() + } + + fn clear(&mut self) { + self.rows.clear(); + } + + fn insert(&mut self, (sheet, c): Key) { + self.rows.entry((sheet, c.row)).or_default().insert(c.col); + } + + fn remove(&mut self, (sheet, c): &Key) { + if let Some(cols) = self.rows.get_mut(&(*sheet, c.row)) { + cols.remove(&c.col); + if cols.is_empty() { + self.rows.remove(&(*sheet, c.row)); + } + } + } + + /// The cells of `sheet` in `rows` x `cols` (both inclusive), in row-major order. + fn range(&self, sheet: usize, rows: (u32, u32), cols: (u32, u32)) -> impl Iterator + '_ { + self.rows + .range((sheet, rows.0)..=(sheet, rows.1)) + .flat_map(move |(&(_, row), set)| set.range(cols.0..=cols.1).map(move |&col| CellRef::new(row, col))) + } } impl Default for Calc { fn default() -> Self { - Calc { graph: Graph::default(), rng: 0x9E37_79B9_7F4A_7C15, fixed_now: None, circular: vec![], last_recalc_cells: 0 } + Calc { + graph: Graph::default(), + rng: 0x9E37_79B9_7F4A_7C15, + fixed_now: None, + circular: vec![], + last_recalc_cells: 0, + spill_errors: SpillErrors::default(), + } } } @@ -125,8 +186,12 @@ impl Calc { pub fn rebuild(&mut self, wb: &Workbook) { self.graph = Graph::default(); let mut circ = Vec::new(); + self.spill_errors.clear(); for (si, sheet) in wb.sheets.iter().enumerate() { for (c, cell) in sheet.cells.iter() { + if cell.value == Value::Error(CellError::Spill) { + self.spill_errors.insert((si, c)); + } if let Some(f) = &cell.formula && let Some((areas, dynamic)) = f.with_expr(|e| precedents(wb, si, e)) { @@ -187,6 +252,20 @@ impl Calc { out } + /// The cells of `sheet` in `rows` x `cols` (inclusive, `cols` a lower and an upper column) that + /// show `#SPILL!`, in row-major order. + fn spill_errors_in(&self, wb: &Workbook, sheet: usize, rows: (u32, u32), cols: (u32, u32)) -> Vec { + let Some(sh) = wb.sheet(sheet) else { return Vec::new() }; + self.spill_errors.range(sheet, rows, cols).filter(|c| sh.cells.get(*c).is_some_and(|x| x.value == Value::Error(CellError::Spill))).collect() + } + + /// The formulas showing `#SPILL!` on `sheet` at or above and to the left of `end`: what a + /// cell emptied or a spill area freed there may unblock. + fn unblock_candidates(&self, wb: &Workbook, sheet: usize, end: CellRef) -> Vec { + let cells = self.spill_errors_in(wb, sheet, (0, end.row), (0, end.col)); + cells.into_iter().filter(|c| self.graph.nodes.contains_key(&(sheet, *c))).map(|c| (sheet, c)).collect() + } + /// Recalculates every formula (F9 / Ctrl+Alt+F9, and after loading). pub fn recalc_all(&mut self, wb: &mut Workbook) { self.rebuild(wb); @@ -208,6 +287,11 @@ impl Calc { pub fn cells_changed(&mut self, wb: &mut Workbook, changed: &[Key]) { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { + if wb.sheet(k.0).and_then(|s| s.cell(k.1)).is_some_and(|c| c.value == Value::Error(CellError::Spill)) { + self.spill_errors.insert(k); + } else { + self.spill_errors.remove(&k); + } let precedent = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.as_ref()).and_then(|f| f.with_expr(|e| precedents(wb, k.0, e))); let py = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.py.as_ref()); @@ -248,6 +332,7 @@ impl Calc { && cell.formula.is_some() { cell.value = Value::Error(CellError::Name); + self.spill_errors.remove(&k); } } } @@ -263,17 +348,40 @@ impl Calc { let mut emptied: Vec = Vec::new(); // Typing into (or clearing) a cell a dynamic array spills over (or would spill over) // re-evaluates the anchor, which may now be blocked or unblocked. + // With many changed cells (a paste) a sheet's spill areas are indexed once instead of being + // walked for each one, when registering them costs less than walking them would. + let mut areas = SpillIndex::default(); + let mut strategy: HashMap = HashMap::default(); + let mut per_sheet: HashMap = HashMap::default(); + for k in changed { + *per_sheet.entry(k.0).or_insert(0) += 1; + } + let mut at = Vec::new(); for &k in changed { if let Some(sh) = wb.sheet(k.0) { - for (anchor, r) in &sh.spill_ranges { - if *anchor != k.1 && r.contains(k.1) { - seeds.push((k.0, *anchor)); + let indexed = *strategy.entry(k.0).or_insert_with(|| { + let indexed = index_spills(changed.len(), per_sheet[&k.0], sh.spill_ranges.values()); + if indexed { + for (anchor, r) in &sh.spill_ranges { + areas.insert_rect((k.0, *anchor), *r, false); + } + } + indexed + }); + if indexed { + // Anchor order, as walking `spill_ranges` gives. + areas.meeting(k.0, RangeRef::new(k.1, k.1), &mut at); + seeds.extend(at.iter().filter(|a| **a != k.1).map(|a| (k.0, *a))); + } else { + for (anchor, r) in &sh.spill_ranges { + if *anchor != k.1 && r.contains(k.1) { + seeds.push((k.0, *anchor)); + } } } - for (c, cell) in sh.cells.iter_range(RangeRef::new(CellRef::new(k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)), k.1)) { - if c != k.1 && cell.value == Value::Error(CellError::Spill) { - seeds.push((k.0, c)); - } + if !self.spill_errors.is_empty() { + let (r0, c0) = (k.1.row.saturating_sub(64), k.1.col.saturating_sub(64)); + seeds.extend(self.spill_errors_in(wb, k.0, (r0, k.1.row), (c0, k.1.col)).into_iter().filter(|c| *c != k.1).map(|c| (k.0, c))); } if !sh.cells.has(k.1) { match emptied.iter_mut().find(|x| x.0 == k.0) { @@ -285,11 +393,7 @@ impl Calc { } // An emptied cell can unblock an anchor any distance above or to the left of it. for (si, end) in emptied { - for &(s, c) in self.graph.nodes.keys() { - if s == si && c.row <= end.row && c.col <= end.col && wb.sheet(s).is_some_and(|sh| sh.value(c) == Value::Error(CellError::Spill)) { - seeds.push((s, c)); - } - } + seeds.extend(self.unblock_candidates(wb, si, end)); } // Spill areas of changed anchors also change. They stay rectangles: one seed per spilled // cell listed the formulas reading the area once per cell. @@ -453,10 +557,15 @@ impl Calc { let mut freed = Vec::new(); for (key, value) in results { let Some(sheet) = wb.sheet_mut(key.0) else { continue }; - if let Some(cell) = sheet.cells.get_mut(key.1) - && cell.value != value - { - cell.value = value; + if let Some(cell) = sheet.cells.get_mut(key.1) { + if value == Value::Error(CellError::Spill) { + self.spill_errors.insert(key); + } else { + self.spill_errors.remove(&key); + } + if cell.value != value { + cell.value = value; + } } } for key in dirty { @@ -486,6 +595,7 @@ impl Calc { if blocked { if let Some(cell) = sheet.cells.get_mut(key.1) { cell.value = Value::Error(CellError::Spill); + self.spill_errors.insert(*key); } let mut blocked_dependents = Vec::new(); self.graph.dependents_of_range(key.0, range, &mut blocked_dependents); @@ -512,14 +622,7 @@ impl Calc { } } for (sheet_index, range) in freed { - for &(sheet, cell) in self.graph.nodes.keys() { - let was_blocked_cell = cell.row <= range.end.row - && cell.col <= range.end.col - && wb.sheet(sheet).is_some_and(|sheet| sheet.value(cell) == Value::Error(CellError::Spill)); - if sheet == sheet_index && was_blocked_cell { - spill_changes.push((sheet, cell)); - } - } + spill_changes.extend(self.unblock_candidates(wb, sheet_index, range.end)); } spill_changes } @@ -576,7 +679,7 @@ pub(crate) mod tests { use gridcraft_core::{CellRef, RangeRef, Value}; use gridcraft_model::{Cell, Formula, Workbook}; - use super::{Calc, HashMap, Host, Intervals, Key, PassHost}; + use super::{Calc, HashMap, Host, Intervals, Key, PassHost, index_spills}; /// xorshift64 for repeatable pseudo-random cases. fn rng(state: &mut u64) -> u64 { @@ -593,6 +696,19 @@ pub(crate) mod tests { v } + /// One changed cell on a sheet with many small spills does not index them just because the + /// batch (mostly on another sheet) is large; the sheet's own share of the batch decides. + #[test] + fn the_spill_index_is_chosen_from_the_sheets_own_changed_cells() { + let small: Vec = (0..1_000).map(|i| RangeRef::new(CellRef::new(i * 3, 0), CellRef::new(i * 3 + 2, 0))).collect(); + // Same areas, same batch of 5000: indexed when 5000 of them are on the sheet, walked when 1 is. + assert!(index_spills(5_000, 5_000, small.iter())); + assert!(!index_spills(5_000, 1, small.iter())); + // Not a paste, or nothing to index: never. + assert!(!index_spills(16, 16, small.iter())); + assert!(!index_spills(5_000, 5_000, [].iter())); + } + /// A 1,000 x 1,000 Python result laid out from A1 in a pass, with the same values already /// spilled on the sheet from the last calculation, as a full recalculation finds them. fn big_python_spill() -> (Workbook, HashMap) { @@ -635,6 +751,39 @@ pub(crate) mod tests { assert!(!host.visit_range(0, inside, &mut |_| true), "an overlapping read leaves the block to range_values"); } + /// A Python array laid out in this pass is seen by the indexed readers, whether the sheet + /// has no spill from before or an older, smaller one. + #[test] + fn a_python_spill_found_in_the_pass_is_read_through_the_index() { + for stale in [false, true] { + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let anchor = CellRef::new(0, 0); + let py = gridcraft_model::PyCode::new("grid"); + sh.set_cell(anchor, Cell { value: Value::Number(0.0), py: Some(std::sync::Arc::new(py)), ..Cell::default() }); + if stale { + // The last calculation spilled a 2 x 2 block of 100s. + sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(1, 1))); + for c in RangeRef::new(anchor, CellRef::new(1, 1)).iter().skip(1) { + sh.spill.insert(c, Value::Number(100.0)); + } + } + let data: Vec = (1..=9).map(|i| Value::Number(f64::from(i))).collect(); + let array = gridcraft_core::Array::new(3, 3, data).unwrap(); + let mut python = HashMap::default(); + python.insert((0, anchor), Value::Array(std::sync::Arc::new(array))); + let exprs = HashMap::default(); + let mut rng = 1u64; + let mut host = PassHost::new(&wb, &exprs, &mut rng, 0.0).with_python(python); + assert_eq!(host.cell_value(0, anchor), Value::Number(1.0)); + let all = RangeRef::parse("A1:C3").unwrap(); + let sum: f64 = host.range_values(0, all).iter().map(|v| if let Value::Number(n) = v { *n } else { 0.0 }).sum(); + assert_eq!(sum, 45.0, "stale={stale}: a SUM over the new 3 x 3 spill reads all of it"); + assert_eq!(host.cell_value(0, CellRef::new(2, 2)), Value::Number(9.0), "stale={stale}"); + assert!(!host.visit_range(0, all, &mut |_| true), "stale={stale}: the new spill leaves the block to range_values"); + } + } + #[test] fn intervals_match_a_linear_scan() { let mut iv: Intervals = Intervals::default(); diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 296d7ac81..1d7d1c38c 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -1,5 +1,6 @@ use super::graph::{HashMap, HashSet, Key}; use super::{Evaluator, Host, Workbook, stack_position}; +use crate::spill_index::{Cover, SpillIndex}; use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; use gridcraft_formula::Expr; @@ -24,6 +25,8 @@ pub(super) struct PassHost<'a> { in_progress: HashSet, /// New spills found during the pass: anchor → array. pub(super) spills: HashMap>, + /// Where the areas in `spills` are, so a read finds the ones it meets without scanning them all. + spill_index: SpillIndex, /// Used range per sheet: the workbook doesn't change during a pass, and working it out walks /// every row, so once per sheet rather than once per range read. used: HashMap>, @@ -93,6 +96,7 @@ impl<'a> PassHost<'a> { pending: exprs.keys().copied().collect(), in_progress: HashSet::default(), spills: HashMap::default(), + spill_index: SpillIndex::default(), used: HashMap::default(), rng, now, @@ -131,6 +135,30 @@ impl<'a> PassHost<'a> { } } +/// Most spill areas a range read meets before they are checked for overlapping one another by +/// walking all of the pass's spills instead of comparing pairs. +const MAX_UNORDERED_HITS: usize = 16; + +/// Whether an array at `anchor` covers `c`. +fn spill_covers(anchor: CellRef, arr: &Array, c: CellRef) -> bool { + c.row >= anchor.row && c.col >= anchor.col && ((c.row - anchor.row) as usize) < arr.rows && ((c.col - anchor.col) as usize) < arr.cols +} + +/// Whether two of the spill areas anchored at `hits` share a cell inside `range`. +fn spill_hits_overlap(index: &SpillIndex, sheet: usize, hits: &[CellRef], range: RangeRef) -> bool { + let rects: Vec = hits.iter().filter_map(|a| index.area(sheet, *a)).filter(|(_, empty)| !empty).map(|(r, _)| r).collect(); + rects.iter().enumerate().any(|(i, a)| { + rects.iter().skip(i + 1).any(|b| { + a.intersects(b) + && RangeRef::new( + CellRef::new(a.start.row.max(b.start.row), a.start.col.max(b.start.col)), + CellRef::new(a.end.row.min(b.end.row), a.end.col.min(b.end.col)), + ) + .intersects(&range) + }) + }) +} + /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, /// overlaps merged cells, holds other cells, or another formula's spill (unless that formula is /// recalculated in this pass, and so may spill elsewhere now). @@ -169,13 +197,6 @@ pub(super) fn spills_as_before(sheet: &gridcraft_model::Sheet, anchor: CellRef, .all(|(i, c)| c == anchor || (!sheet.cells.has(c) && sheet.spill.get(&c) == Some(arr.data.get(i).unwrap_or(&Value::Empty)))) } -/// The area an array anchored at `anchor` covers. -fn array_area(anchor: CellRef, a: &Array) -> RangeRef { - let end = - CellRef::new(anchor.row.saturating_add((a.rows as u32).saturating_sub(1)), anchor.col.saturating_add((a.cols as u32).saturating_sub(1))); - RangeRef::new(anchor, end) -} - /// The spilled values `sheet` stores inside `range`, passed to `f` until it returns false. Values /// outside the range are not walked: with fewer rows in the range than spilled values, each row /// is looked up on its own; otherwise one ordered scan visits each spilled value at most once. @@ -377,6 +398,7 @@ impl PassHost<'_> { // value -- a FILTER matching one row, a SORT or UNIQUE over one item -- so `C1#` // is the 1 x 1 range and `ROWS(C1#)` is 1, as in Excel (#1491). A formula that // returns a scalar never reaches this arm, so it does not become an anchor. + self.spill_index.insert(k, a.rows, a.cols); self.spills.insert(k, a.clone()); a.data.first().cloned().unwrap_or_default() } @@ -391,6 +413,7 @@ impl PassHost<'_> { let v = match self.spills.get(&k) { Some(a) if spill_blocked(self, k, a) => { self.spills.remove(&k); + self.spill_index.remove(k); Value::Error(CellError::Spill) } _ => v, @@ -410,6 +433,7 @@ impl PassHost<'_> { && a.cols >= 1 && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS); if sane { + self.spill_index.insert(k, a.rows, a.cols); self.spills.insert(k, a.clone()); a.data.first().cloned().unwrap_or_default() } else { @@ -421,6 +445,7 @@ impl PassHost<'_> { let v = match self.spills.get(&k) { Some(a) if spill_blocked(self, k, a) => { self.spills.remove(&k); + self.spill_index.remove(k); Value::Error(CellError::Spill) } _ => v, @@ -467,7 +492,11 @@ impl PassHost<'_> { while passes < max_iterations.min(max_passes) && !self.converged(max_change) { self.prev = again.iter().filter_map(|k| self.results.remove(k).map(|v| (*k, v))).collect(); self.pending = set.clone(); - self.spills.retain(|a, _| !set.contains(a)); + let gone: Vec = self.spills.keys().filter(|a| set.contains(a)).copied().collect(); + for a in gone { + self.spills.remove(&a); + self.spill_index.remove(a); + } for k in &again { self.calculate(*k); } @@ -551,17 +580,19 @@ impl Host for PassHost<'_> { if self.pending.contains(&k) || self.results.contains_key(&k) { return self.compute(k); } - // A cell covered by a spill computed in this pass. - for (anchor, arr) in &self.spills { - if anchor.0 == sheet && c.row >= anchor.1.row && c.col >= anchor.1.col { - let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize); - if dr < arr.rows && dc < arr.cols { - if self.iterate && self.tainted.contains(anchor) { - self.taint = true; - } - return arr.get(dr, dc).cloned().unwrap_or_default(); - } + // A cell covered by a spill computed in this pass. Where several overlap, the first in + // `spills`' iteration order counts, as it always has. + let covering = match self.spill_index.covering(sheet, c) { + Cover::None => None, + Cover::One(a) => self.spills.get_key_value(&(sheet, a)), + Cover::Many => self.spills.iter().find(|(a, arr)| a.0 == sheet && spill_covers(a.1, arr, c)), + }; + if let Some((anchor, arr)) = covering { + let (dr, dc) = ((c.row - anchor.1.row) as usize, (c.col - anchor.1.col) as usize); + if self.iterate && self.tainted.contains(anchor) { + self.taint = true; } + return arr.get(dr, dc).cloned().unwrap_or_default(); } self.wb.sheet(sheet).map(|s| s.value(c)).unwrap_or_default() } @@ -600,46 +631,44 @@ impl Host for PassHost<'_> { } } if !self.spills.is_empty() { - // Only the part of each array inside the range is walked: a large array beside a small - // read costs nothing. - let mut new: Vec<(usize, Value)> = Vec::new(); - for (a, arr) in self.spills.iter().filter(|(a, _)| a.0 == sheet) { - let area = array_area(a.1, arr); - if !area.intersects(&range) { - continue; - } - let (r0, r1) = (area.start.row.max(range.start.row), area.end.row.min(range.end.row)); - let (c0, c1) = (area.start.col.max(range.start.col), area.end.col.min(range.end.col)); - for row in r0..=r1 { - for col in c0..=c1 { - let c = CellRef::new(row, col); - if c == a.1 { - continue; - } - let v = arr.get((row - a.1.row) as usize, (col - a.1.col) as usize).cloned().unwrap_or_default(); - new.push(((row - range.start.row) as usize * w + (col - range.start.col) as usize, v)); - } - } - } - #[cfg(test)] - { - self.spill_visits += new.len(); - } + let mut hits = Vec::new(); + self.spill_index.meeting(sheet, range, &mut hits); // Iterating: an array spilled by a formula that read a previous value. - if self.iterate - && self.spills.iter().any(|(a, arr)| { - let end = CellRef::new( - a.1.row.saturating_add((arr.rows as u32).saturating_sub(1)), - a.1.col.saturating_add((arr.cols as u32).saturating_sub(1)), - ); - a.0 == sheet && self.tainted.contains(a) && RangeRef::new(a.1, end).intersects(&range) - }) - { + if self.iterate && hits.iter().any(|a| self.tainted.contains(&(sheet, *a))) { self.taint = true; } - for (i, v) in new { - if let Some(slot) = out.get_mut(i) { - *slot = v; + // Where areas overlap inside the range, the last in `spills`' iteration order wins, + // as it always has; otherwise the order doesn't matter. + let ordered = hits.len() > MAX_UNORDERED_HITS || spill_hits_overlap(&self.spill_index, sheet, &hits, range); + let mut put = |anchor: CellRef, arr: &Array| { + if arr.rows == 0 || arr.cols == 0 { + return; + } + let (r0, c0) = (range.start.row.max(anchor.row), range.start.col.max(anchor.col)); + let r1 = range.end.row.min(anchor.row.saturating_add((arr.rows as u32).saturating_sub(1))); + let c1 = range.end.col.min(anchor.col.saturating_add((arr.cols as u32).saturating_sub(1))); + for r in r0..=r1 { + for cc in c0..=c1 { + if (r, cc) != (anchor.row, anchor.col) + && let Some(slot) = out.get_mut((r - range.start.row) as usize * w + (cc - range.start.col) as usize) + { + *slot = arr.get((r - anchor.row) as usize, (cc - anchor.col) as usize).cloned().unwrap_or_default(); + } + } + } + }; + if ordered { + let wanted: HashSet = hits.iter().map(|a| (sheet, *a)).collect(); + for (a, arr) in &self.spills { + if wanted.contains(a) { + put(a.1, arr); + } + } + } else { + for a in &hits { + if let Some(arr) = self.spills.get(&(sheet, *a)) { + put(*a, arr); + } } } } @@ -658,7 +687,8 @@ impl Host for PassHost<'_> { } // Spilled values sit between the stored cells (and the formulas just calculated may have // produced some): leave such blocks to `range_values`. - let arrays = self.spills.iter().any(|(a, arr)| a.0 == sheet && array_area(a.1, arr).intersects(&range)); + let mut arrays = Vec::new(); + self.spill_index.meeting(sheet, range, &mut arrays); let mut stored = false; let _seen = stored_spill_in(sh, range, &mut |_, _| { stored = true; @@ -668,7 +698,7 @@ impl Host for PassHost<'_> { { self.spill_visits += _seen; } - if arrays || stored { + if !arrays.is_empty() || stored { return false; } for (c, cell) in sh.cells.iter_range(range) { diff --git a/crates/calc/src/spill_index.rs b/crates/calc/src/spill_index.rs new file mode 100644 index 000000000..d642bdee7 --- /dev/null +++ b/crates/calc/src/spill_index.rs @@ -0,0 +1,296 @@ +//! A spatial index over spill areas: which areas cover a cell, and which meet a range, without +//! looking at every area. Areas are registered in 16 x 16 cell blocks, so a lookup touches one +//! block (a cell) or the blocks a range covers (a range) rather than every spill on the sheet. +//! An area that would take more than `BIG_AREA_BLOCKS` blocks (a full-height column spill takes +//! 65,536) is not registered in blocks but kept in a short list that every lookup tests directly, +//! so registering costs what the area's size warrants only for small areas. + +use crate::recalc::{FxHasher, Key}; +use gridcraft_core::{CellRef, RangeRef}; +use std::hash::BuildHasherDefault; + +type HashMap = std::collections::HashMap>; + +/// Block edge in cells (rows and columns). +const BLOCK: u32 = 16; + +/// Areas spanning more blocks than this are tested directly instead of being registered. +const BIG_AREA_BLOCKS: u64 = 1024; + +/// One indexed area: its rectangle, and whether it covers no cells (an array with no rows or no +/// columns still meets the ranges its anchor is in, as a 1 x 1 area does for taint tracking). +#[derive(Clone, Copy)] +struct Entry { + rect: RangeRef, + empty: bool, +} + +#[derive(Default)] +struct SheetAreas { + entries: HashMap, + blocks: HashMap<(u32, u32), Vec>, + /// Anchors of the areas too big to register in blocks. + big: Vec, +} + +/// What covers a cell. +pub(crate) enum Cover { + None, + One(CellRef), + /// Several areas overlap here: the caller decides which one counts. + Many, +} + +#[derive(Default)] +pub(crate) struct SpillIndex { + sheets: HashMap, +} + +fn block_span(rect: &RangeRef) -> (u32, u32, u32, u32) { + (rect.start.row / BLOCK, rect.end.row / BLOCK, rect.start.col / BLOCK, rect.end.col / BLOCK) +} + +fn block_count(rect: &RangeRef) -> u64 { + let (r0, r1, c0, c1) = block_span(rect); + u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1) +} + +/// Whether registering `areas` in blocks costs less than checking each of them against each of +/// `lookups` cells would: a few block entries cost about one containment check each. A tall or +/// wide area is not registered at all (see `BIG_AREA_BLOCKS`) and so costs nothing here. +pub(crate) fn indexing_pays<'a>(areas: impl ExactSizeIterator, lookups: usize) -> bool { + const BLOCK_COST: u64 = 16; + let budget = (lookups as u64).saturating_mul(areas.len() as u64) / BLOCK_COST; + let mut blocks = 0u64; + for r in areas { + let n = block_count(r); + if n > BIG_AREA_BLOCKS { + continue; + } + blocks = blocks.saturating_add(n); + if blocks > budget { + return false; + } + } + true +} + +impl SpillIndex { + /// Whether any area is indexed on `sheet`. + #[cfg(test)] + pub(crate) fn has_sheet(&self, sheet: usize) -> bool { + self.sheets.get(&sheet).is_some_and(|s| !s.entries.is_empty()) + } + + /// Indexes the area an array of `rows` x `cols` anchored at `anchor` covers (replacing the + /// one already indexed for that anchor). + pub(crate) fn insert(&mut self, (sheet, anchor): Key, rows: usize, cols: usize) { + let end = + CellRef::new(anchor.row.saturating_add((rows as u32).saturating_sub(1)), anchor.col.saturating_add((cols as u32).saturating_sub(1))); + self.insert_rect((sheet, anchor), RangeRef::new(anchor, end), rows == 0 || cols == 0); + } + + /// Indexes an area given as a range. + pub(crate) fn insert_rect(&mut self, (sheet, anchor): Key, rect: RangeRef, empty: bool) { + self.remove((sheet, anchor)); + let areas = self.sheets.entry(sheet).or_default(); + if block_count(&rect) > BIG_AREA_BLOCKS { + areas.big.push(anchor); + } else { + let (r0, r1, c0, c1) = block_span(&rect); + for br in r0..=r1 { + for bc in c0..=c1 { + areas.blocks.entry((br, bc)).or_default().push(anchor); + } + } + } + areas.entries.insert(anchor, Entry { rect, empty }); + } + + pub(crate) fn remove(&mut self, (sheet, anchor): Key) { + let Some(areas) = self.sheets.get_mut(&sheet) else { return }; + let Some(old) = areas.entries.remove(&anchor) else { return }; + if block_count(&old.rect) > BIG_AREA_BLOCKS { + areas.big.retain(|a| *a != anchor); + return; + } + let (r0, r1, c0, c1) = block_span(&old.rect); + for br in r0..=r1 { + for bc in c0..=c1 { + if let Some(list) = areas.blocks.get_mut(&(br, bc)) { + list.retain(|a| *a != anchor); + if list.is_empty() { + areas.blocks.remove(&(br, bc)); + } + } + } + } + } + + /// The areas that cover `c`. + pub(crate) fn covering(&self, sheet: usize, c: CellRef) -> Cover { + let Some(areas) = self.sheets.get(&sheet) else { return Cover::None }; + let in_block = areas.blocks.get(&(c.row / BLOCK, c.col / BLOCK)).map_or(&[][..], |l| l.as_slice()); + let mut found = Cover::None; + for a in in_block.iter().chain(&areas.big) { + if let Some(e) = areas.entries.get(a) + && !e.empty + && e.rect.contains(c) + { + found = match found { + Cover::None => Cover::One(*a), + _ => return Cover::Many, + }; + } + } + found + } + + /// The anchors of the areas whose rectangle meets `range` (an empty area counts through its + /// anchor cell), sorted by anchor. + pub(crate) fn meeting(&self, sheet: usize, range: RangeRef, out: &mut Vec) { + out.clear(); + let Some(areas) = self.sheets.get(&sheet) else { return }; + let (r0, r1, c0, c1) = block_span(&range); + let touched = u64::from(r1 - r0 + 1) * u64::from(c1 - c0 + 1); + if touched > 4 * areas.entries.len() as u64 + 16 { + // A range wider than the areas are many: look at each area once. + out.extend(areas.entries.iter().filter(|(_, e)| e.rect.intersects(&range)).map(|(a, _)| *a)); + } else { + for br in r0..=r1 { + for bc in c0..=c1 { + if let Some(list) = areas.blocks.get(&(br, bc)) { + out.extend(list.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range)))); + } + } + } + out.extend(areas.big.iter().copied().filter(|a| areas.entries.get(a).is_some_and(|e| e.rect.intersects(&range)))); + } + out.sort_unstable(); + out.dedup(); + } + + /// The rectangle indexed for `anchor` and whether it covers no cells. + pub(crate) fn area(&self, sheet: usize, anchor: CellRef) -> Option<(RangeRef, bool)> { + self.sheets.get(&sheet)?.entries.get(&anchor).map(|e| (e.rect, e.empty)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn at(r: u32, c: u32) -> CellRef { + CellRef::new(r, c) + } + + #[test] + fn finds_the_areas_covering_a_cell_and_meeting_a_range() { + let mut ix = SpillIndex::default(); + ix.insert((0, at(0, 0)), 40, 3); + ix.insert((0, at(30, 1)), 1, 40); + ix.insert((1, at(0, 0)), 5, 5); + assert!(matches!(ix.covering(0, at(39, 2)), Cover::One(a) if a == at(0, 0))); + assert!(matches!(ix.covering(0, at(30, 2)), Cover::Many)); + assert!(matches!(ix.covering(0, at(40, 0)), Cover::None)); + assert!(matches!(ix.covering(2, at(0, 0)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(30, 30), at(50, 60)), &mut hits); + assert_eq!(hits, vec![at(30, 1)]); + ix.meeting(0, RangeRef::new(at(0, 0), at(1_000_000, 16_000)), &mut hits); + assert_eq!(hits, vec![at(0, 0), at(30, 1)]); + ix.remove((0, at(30, 1))); + assert!(matches!(ix.covering(0, at(30, 2)), Cover::One(_))); + assert!(ix.has_sheet(1) && !ix.has_sheet(2)); + ix.insert((0, at(0, 0)), 1, 1); + assert!(matches!(ix.covering(0, at(39, 2)), Cover::None)); + } + + #[test] + fn tall_or_wide_areas_are_not_registered_in_blocks() { + let column = RangeRef::new(at(0, 0), at(1_048_575, 0)); + assert!(block_count(&column) > BIG_AREA_BLOCKS); + let mut ix = SpillIndex::default(); + ix.insert_rect((0, at(0, 0)), column, false); + assert!(ix.sheets[&0].blocks.is_empty()); + assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::One(a) if a == at(0, 0))); + assert!(matches!(ix.covering(0, at(900_000, 1)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(5, 0), at(6, 3)), &mut hits); + assert_eq!(hits, vec![at(0, 0)]); + ix.remove((0, at(0, 0))); + assert!(matches!(ix.covering(0, at(900_000, 0)), Cover::None) && ix.sheets[&0].big.is_empty()); + // Registering a big area costs nothing (lookups test it directly), so a small paste still indexes next to many small areas. + assert!(indexing_pays([column].iter(), 17)); + // Many small areas pay back for a paste of a hundred cells, but not for a handful. + let small: Vec = (0..1_000).map(|i| RangeRef::new(at(i * 3, 0), at(i * 3 + 2, 0))).collect(); + assert!(indexing_pays(small.iter(), 100)); + assert!(!indexing_pays(small.iter(), 17)); + } + + /// The index against the old way: test every area for every lookup. Mixed shapes: small, + /// thin and tall, wide, big (kept out of the blocks), empty, on two sheets, replaced and removed. + #[test] + fn matches_a_scan_of_every_area_on_mixed_shapes() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = |n: u32| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed >> 11) as u32 % n + }; + let mut ix = SpillIndex::default(); + let mut model: std::collections::BTreeMap<(usize, CellRef), (RangeRef, bool)> = Default::default(); + for step in 0..3_000 { + let sheet = next(2) as usize; + let anchor = at(next(400) * 5, next(40) * 3); + if next(4) == 0 { + ix.remove((sheet, anchor)); + model.remove(&(sheet, anchor)); + } else { + let (rows, cols) = match next(8) { + 0 => (0, 1 + next(5) as usize), + 1 => (1 + next(5) as usize, 0), + 2 => (5_000 + next(100_000) as usize, 1), + 3 => (1, 1 + next(40_000) as usize), + 4 => (200 + next(300) as usize, 40 + next(40) as usize), + _ => (1 + next(60) as usize, 1 + next(40) as usize), + }; + ix.insert((sheet, anchor), rows, cols); + let end = at(anchor.row + (rows as u32).saturating_sub(1), anchor.col + (cols as u32).saturating_sub(1)); + model.insert((sheet, anchor), (RangeRef::new(anchor, end), rows == 0 || cols == 0)); + } + if step % 7 != 0 { + continue; + } + let sheet = next(3) as usize; + for _ in 0..20 { + let cell = at(next(2_100), next(130)); + let want: Vec = + model.iter().filter(|((s, _), (r, e))| *s == sheet && !e && r.contains(cell)).map(|((_, a), _)| *a).collect(); + match ix.covering(sheet, cell) { + Cover::None => assert!(want.is_empty(), "{cell:?}"), + Cover::One(a) => assert_eq!(want, vec![a], "{cell:?}"), + Cover::Many => assert!(want.len() > 1, "{cell:?}"), + } + let (r0, c0) = (next(2_100), next(130)); + let (tall, wide) = (if next(5) == 0 { 3_000 } else { 40 }, if next(5) == 0 { 130 } else { 20 }); + let range = RangeRef::new(at(r0, c0), at(r0 + next(tall), c0 + next(wide))); + let want: Vec = model.iter().filter(|((s, _), (r, _))| *s == sheet && r.intersects(&range)).map(|((_, a), _)| *a).collect(); + let mut got = Vec::new(); + ix.meeting(sheet, range, &mut got); + assert_eq!(got, want, "{range:?}"); + } + } + } + + #[test] + fn an_area_without_cells_covers_none_but_is_met_at_its_anchor() { + let mut ix = SpillIndex::default(); + ix.insert((0, at(3, 3)), 0, 4); + assert!(matches!(ix.covering(0, at(3, 3)), Cover::None)); + let mut hits = Vec::new(); + ix.meeting(0, RangeRef::new(at(3, 3), at(3, 3)), &mut hits); + assert_eq!(hits, vec![at(3, 3)]); + } +} diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 3f37196b2..7f6e753de 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -1826,6 +1826,54 @@ fn trim_references() { assert_eq!(t.get("K20"), Value::Error(CellError::Calc)); } +#[test] +fn lookup_of_an_empty_cell_stays_blank() { + // #374: B1 (the value for key "k") is empty. + let mut t = T::new(); + t.set("A1", "k"); + t.set("A2", "j"); + t.set("B2", "v"); + for (at, f, want) in [ + ("D2", "=\"x\"&VLOOKUP(\"k\",A1:B2,2,0)&\"y\"", Value::from("xy")), + ("D3", "=CONCATENATE(\"x\",VLOOKUP(\"k\",A1:B2,2,0),\"y\")", Value::from("xy")), + ("D4", "=ISBLANK(XLOOKUP(\"k\",A1:A2,B1:B2))", Value::Bool(true)), + ("D5", "=\"x\"&XLOOKUP(\"k\",A1:A2,B1:B2)&\"y\"", Value::from("xy")), + // A bare lookup still shows 0, like a reference to an empty cell. + ("D8", "=VLOOKUP(\"k\",A1:B2,2,0)", Value::Number(0.0)), + // Arithmetic still reads the blank as 0. + ("D9", "=VLOOKUP(\"k\",A1:B2,2,0)+1", Value::Number(1.0)), + // A blank lookup given as an argument is still given, not omitted. + ("D10", "=FILTER(1,FALSE,VLOOKUP(\"k\",A1:B2,2,FALSE))", Value::Number(0.0)), + ("D11", "=XLOOKUP(\"z\",A1:A2,B1:B2,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + // The same holds per element when the call is lifted over an array of blank lookups. + ("D12", "=CEILING.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D13", "=SUM(CEILING.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), + ("D14", "=FLOOR.MATH(6.7,VLOOKUP(\"k\",A1:B2,2,0))", Value::Number(0.0)), + ("D15", "=SUM(FLOOR.MATH(6.7,VLOOKUP({\"k\",\"k\"},A1:B2,2,0)))", Value::Number(0.0)), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} + +#[test] +fn blank_lookup_values_match_excel_windows_observation() { + // Excel 365 Windows 2609, build 16.0.20430.20032, black-box check 2026-10-13: + // A1="k", B1 and A2 genuinely empty. + // All three value lookups preserve blankness inside ISBLANK/concatenation. + let mut t = T::new(); + t.set("A1", "k"); + for (at, f, want) in [ + ("D1", "=ISBLANK(VLOOKUP(\"k\",A1:B1,2,FALSE))", Value::Bool(true)), + ("D2", "=ISBLANK(HLOOKUP(\"k\",A1:A2,2,FALSE))", Value::Bool(true)), + ("D3", "=ISBLANK(LOOKUP(\"k\",A1:A1,B1:B1))", Value::Bool(true)), + ("D4", "=\"x\"&HLOOKUP(\"k\",A1:A2,2,FALSE)&\"y\"", Value::from("xy")), + ] { + t.set(at, f); + assert_eq!(t.get(at), want, "{f}"); + } +} + #[test] fn cell_width_is_the_column_width_dialog_value_rounded() { // #735: Excel 365 values. Widths in px as `home.columnWidth {"chars"}` sets them @@ -1922,6 +1970,54 @@ fn offset_extreme_arguments_return_ref() { } } +/// `@` intersects a reference with the formula's row (a column range) or column (a row range); +/// a 2D range both (circular when it includes the formula's cell); a range the formula does not +/// line up with gives #VALUE!. An array gives its top-left value. Legacy formulas read from files use it where Excel intersects implicitly. +#[test] +fn implicit_intersection_operator() { + let mut t = T::new(); + for (at, v) in [("A1", "1"), ("A2", "2"), ("A3", "3"), ("A5", "4"), ("B5", "5"), ("C5", "6")] { + t.set(at, v); + } + t.set("B2", "=TANH(@A1:A3)"); + t.set("B7", "=TANH(@A5:C5)"); + t.set("B3", "=@A1:C3"); + t.set("B9", "=@A1:A3"); + t.set("E9", "=@A5:C5"); + t.set("D1", "=@{7,8;9,10}"); + t.set("D2", "=@A2"); + t.set("D3", "=SUM(@A1:A3)"); + assert_eq!(t.num("B2"), 2f64.tanh()); + assert_eq!(t.num("B7"), 5f64.tanh()); + for at in ["B9", "E9"] { + assert_eq!(t.get(at), Value::Error(CellError::Value), "{at}"); + } + // A 2D range that includes the formula's own cell is a circular reference (Excel shows 0). + assert_eq!(t.get("B3"), Value::Error(CellError::Circ)); + // From another sheet a 2D range intersects on both row and column; outside it is #VALUE!. + t.wb.sheets.push(std::sync::Arc::new(gridcraft_model::Sheet::new("Other"))); + for (at, input) in [("B2", "=@Sheet1!A1:C3"), ("C1", "=@Sheet1!A1:C3"), ("B9", "=@Sheet1!A1:C3"), ("E2", "=@Sheet1!A1:C3")] { + t.wb.sheet_mut(1).unwrap().cells.set(c(at), Cell::formula(Formula::new(input))); + } + t.calc.recalc_all(&mut t.wb); + let other = |at: &str| t.wb.sheet(1).unwrap().value(c(at)); + assert_eq!(other("B2"), Value::Number(2f64.tanh())); // Sheet1!B2 + assert_eq!(other("C1"), Value::Number(0.0)); + assert_eq!(other("B9"), Value::Error(CellError::Value)); + assert_eq!(other("E2"), Value::Error(CellError::Value)); + assert_eq!(t.num("D1"), 7.0); + assert_eq!(t.num("D2"), 2.0); + assert_eq!(t.num("D3"), 3.0); + // An array result shows its top-left value (T of a reference reads its top-left cell). + t.set("A11", "x"); + t.set("C11", "y"); + t.set("E1", "=@T(A11:C11)"); + assert_eq!(t.get("E1"), Value::text("x")); + // Without `@` (a dynamic-array formula) the same formula spills. + t.set("F1", "=TANH(A1:A3)"); + assert_eq!((t.num("F1"), t.num("F2"), t.num("F3")), (1f64.tanh(), 2f64.tanh(), 3f64.tanh())); +} + /// "Set precision as displayed": results are rounded to their cell's format before they are /// stored and before other formulas read them. Values observed in Excel (COM, /// `Workbook.PrecisionAsDisplayed = True`) with the same synthetic formulas. @@ -1987,7 +2083,6 @@ fn precision_as_displayed_rounds_results_by_cell_format() { t.calc.recalc_all(&mut t.wb); assert_eq!(t.num("A2"), (-130.0 + 657.41) * 10.0); } - #[test] fn analysis_toolpak_value_params_reject_multi_cell_references() { // Excel 365 (observed): EOMONTH, WORKDAY, COUPDAYS, BIN2DEC... give #VALUE! for a reference to more @@ -2072,7 +2167,6 @@ fn analysis_toolpak_value_params_reject_multi_cell_references() { t.set("H5", ""); } } - /// A formula reading one further down (or on a later sheet) is calculated on demand, inside the /// formula that reads it. Chains of thousands of such formulas, common in financial models, used /// to overflow the stack (or give `#CIRC!` past 2,000 levels); they now calculate in full. @@ -2379,6 +2473,201 @@ fn loan_schedule_residue_keeps_excels_sign() { assert_eq!(t.get("P7"), Value::Text("".into())); } +#[test] +fn clearing_a_cell_unblocks_the_arrays_above_and_left_of_it() { + let mut t = T::new(); + for r in 1..=50 { + t.set(&format!("D{r}"), &format!("={r}*2")); + } + t.set("B2", "7"); + t.set("B1", "=SEQUENCE(3)"); + t.set("C1", "=SEQUENCE(1,3)"); + t.set("A20", "=B1+1"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("A20"), Value::Error(CellError::Spill)); + // Emptied: the array above it spills. + t.wb.sheet_mut(0).unwrap().cells.remove(c("B2")); + t.calc.cells_changed(&mut t.wb, &[(0, c("B2"))]); + assert_eq!(t.num("B3"), 3.0); + assert_eq!(t.num("A20"), 2.0); + // Typing over the area blocks it again; far from it, nothing is disturbed. + t.set("B3", "9"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + t.set("D50", "1"); + assert_eq!(t.get("B1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("D49"), 98.0); + // Clearing it again frees the array. + t.set("B3", ""); + assert_eq!(t.num("B3"), 3.0); +} + +/// A sheet with `#SPILL!` cells in rows 0..64, columns 100..16384 (as blocked arrays leave), none +/// near column A. +fn spill_error_field() -> T { + let mut t = T::new(); + let sh = t.wb.sheet_mut(0).unwrap(); + for r in 0..64 { + for col in 100..16_384 { + sh.cells.set(CellRef::new(r, col), Cell::value(Value::Error(CellError::Spill))); + } + } + t.calc.recalc_all(&mut t.wb); + t +} + +#[test] +fn an_edit_far_left_of_many_spill_errors_does_not_disturb_them() { + let mut t = spill_error_field(); + t.set("A65", "5"); + assert_eq!(t.num("A65"), 5.0); + assert_eq!(t.get("CW1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("XFD64"), Value::Error(CellError::Spill)); + // A blocked formula inside the window is still found when a cell to its right is cleared. + t.set("DA66", "7"); + t.set("CZ66", "=SEQUENCE(1,3)"); + assert_eq!(t.get("CZ66"), Value::Error(CellError::Spill)); + t.wb.sheet_mut(0).unwrap().cells.remove(c("DA66")); + t.calc.cells_changed(&mut t.wb, &[(0, c("DA66"))]); + assert_eq!(t.num("DA66"), 2.0); +} + +/// Probe: single-cell edits in column A below a block of `#SPILL!` cells to its right. +/// `cargo test -p gridcraft-calc --release -- --ignored --nocapture spill_error_field_probe` +#[test] +#[ignore] +#[allow(clippy::disallowed_methods)] +fn spill_error_field_probe() { + let mut t = spill_error_field(); + let mut best = std::time::Duration::MAX; + for i in 0..9 { + let start = std::time::Instant::now(); + for _ in 0..100 { + t.set("A65", &format!("{i}")); + } + best = best.min(start.elapsed()); + } + eprintln!("100 edits of A65 beside 1,040,000 spill errors: best of 9 = {best:?}"); +} + +/// A full-height column spill (inside the 10,000,000-cell cap) and a paste of 17 cells elsewhere: +/// the paste must not register the spill's 65,536 blocks, and a paste over the spill must still +/// block it. +fn tall_spill_book() -> T { + let mut t = T::new(); + t.set("A1", "=SEQUENCE(1048576)"); + t.calc.recalc_all(&mut t.wb); + assert_eq!(t.num("A1048576"), 1_048_576.0); + t +} + +fn paste_into(t: &mut T, cells: &[(u32, u32)]) { + let changed: Vec<(usize, CellRef)> = cells.iter().map(|&(r, c)| (0, CellRef::new(r, c))).collect(); + for k in &changed { + t.wb.sheet_mut(0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0))); + } + t.calc.cells_changed(&mut t.wb, &changed); +} + +#[test] +fn a_paste_beside_a_tall_spill_leaves_it_alone_and_one_over_it_blocks_it() { + let mut t = tall_spill_book(); + let beside: Vec<(u32, u32)> = (0..17).map(|i| (i, 5)).collect(); + paste_into(&mut t, &beside); + assert_eq!(t.num("A1048576"), 1_048_576.0); + let over: Vec<(u32, u32)> = (0..17).map(|i| (500_000 + i, if i == 9 { 0 } else { 5 })).collect(); + paste_into(&mut t, &over); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill)); +} + +/// Probe: `cells_changed` for a 17-cell paste in column F beside the tall spill. +/// `cargo test -p gridcraft-calc --release -- --ignored --nocapture tall_spill_paste_probe` +#[test] +#[ignore] +#[allow(clippy::disallowed_methods)] +fn tall_spill_paste_probe() { + let mut t = tall_spill_book(); + let mut best = std::time::Duration::MAX; + for round in 0..9u32 { + let beside: Vec<(u32, u32)> = (0..17).map(|i| (round * 20 + i, 5)).collect(); + let start = std::time::Instant::now(); + paste_into(&mut t, &beside); + best = best.min(start.elapsed()); + } + eprintln!("tall spill, 17-cell paste beside it: best of 9 = {best:?}"); +} + +#[test] +fn many_spills_are_read_and_blocked_by_a_paste() { + let mut t = T::new(); + // 40 arrays of 3 rows side by side in rows 1-3, and a reader of a range over two of them. + for i in 0..40u32 { + let anchor = CellRef::new(0, 2 * i); + t.wb.sheet_mut(0).unwrap().cells.set(anchor, Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i)))); + } + t.set("A10", "=SUM(C1:F3)"); + t.set("A12", "=COUNT(C1:F3)"); + t.set("A11", "=C2*1"); + t.calc.recalc_all(&mut t.wb); + assert_eq!(t.num("A10"), (10 + 11 + 12 + 20 + 21 + 22) as f64); + assert_eq!(t.num("A11"), 11.0); + assert_eq!(t.num("A12"), 6.0); + // Pasting 20 cells (more than one spill's width worth, so the changed set is indexed) over + // the second row of every other array blocks those arrays; the rest are unchanged. + let changed: Vec<(usize, CellRef)> = (0..20u32).map(|i| (0, CellRef::new(1, 4 * i))).collect(); + for k in &changed { + t.wb.sheet_mut(0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0))); + } + t.calc.cells_changed(&mut t.wb, &changed); + assert_eq!(t.get("E1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("C1"), Value::Number(10.0)); + assert_eq!(t.get("G2"), Value::Number(31.0)); + assert_eq!(t.get("A10"), Value::Error(CellError::Spill)); + assert_eq!(t.num("A12"), 4.0); +} + +#[test] +fn a_paste_over_small_and_tall_spills_blocks_those_it_touches() { + let mut t = T::new(); + for i in 0..40u32 { + t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 2 * i), Cell::formula(Formula::new("=SEQUENCE(3)"))); + } + t.wb.sheet_mut(0).unwrap().cells.set(CellRef::new(0, 200), Cell::formula(Formula::new("=SEQUENCE(1048576)"))); + t.calc.recalc_all(&mut t.wb); + // 20 cells: the middle row of the small arrays in columns 0, 8, 16 ... 72 (columns 80 and up hit + // nothing), and one deep inside the tall one. + let mut cells: Vec<(u32, u32)> = (0..19u32).map(|i| (1, 8 * i)).collect(); + cells.push((700_000, 200)); + paste_into(&mut t, &cells); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill)); + assert_eq!(t.get("GS1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("C1"), 1.0); + assert_eq!(t.num("C3"), 3.0); + assert_eq!(t.get("I1"), Value::Error(CellError::Spill)); + assert_eq!(t.num("E3"), 3.0); +} + +#[test] +fn a_big_paste_on_one_sheet_and_one_cell_over_many_spills_on_another() { + let mut t = T::new(); + t.wb.sheets.push(std::sync::Arc::new(gridcraft_model::Sheet::new("Data"))); + for i in 0..1_500u32 { + t.wb.sheet_mut(1).unwrap().cells.set(CellRef::new(4 * i, 0), Cell::formula(Formula::new(&format!("=SEQUENCE(3,1,{})", 10 * i)))); + } + t.calc.recalc_all(&mut t.wb); + assert_eq!(t.wb.sheet(1).unwrap().value(CellRef::new(4 * 700 + 2, 0)), Value::Number(7002.0)); + // 5000 cells pasted on the first sheet, one over the middle of the 701st array on the second. + let mut changed: Vec<(usize, CellRef)> = (0..5_000u32).map(|i| (0, CellRef::new(i / 50, i % 50))).collect(); + changed.push((1, CellRef::new(4 * 700 + 1, 0))); + for k in &changed { + t.wb.sheet_mut(k.0).unwrap().cells.set(k.1, Cell::value(Value::Number(1.0))); + } + t.calc.cells_changed(&mut t.wb, &changed); + let data = t.wb.sheet(1).unwrap(); + assert_eq!(data.value(CellRef::new(4 * 700, 0)), Value::Error(CellError::Spill)); + assert_eq!(data.value(CellRef::new(4 * 699 + 2, 0)), Value::Number(6992.0)); + assert_eq!(data.value(CellRef::new(4 * 701 + 2, 0)), Value::Number(7012.0)); +} + /// `SUM`, `COUNT` and `AVERAGE` over blocks of constants share one pass-wide answer per block, and a /// running total extends the shorter block's. Every result must equal the left-to-right sum of the /// cells, bit for bit, with text, blanks, errors, formulas and a nonzero start in the mix. @@ -2591,3 +2880,19 @@ fn formulas_read_a_dirty_python_cell_after_it_is_laid_out() { assert_eq!((t.num("B1"), t.num("B2")), (5.0, 6.0)); assert_eq!((t.num("A1"), t.num("A2"), t.num("A3")), (5.0, 100.0, 111.0)); } + +/// A computed criterion whose formula runs a database function over the same criteria counts as +/// not met inside itself: before, each record's check ran the function again for every record, at +/// every level, so the work grew exponentially with the nesting allowed. +#[test] +fn computed_criterion_that_reenters_itself_is_not_met() { + let mut t = T::new(); + for (r, (name, age)) in [("name", "age"), ("a", "1"), ("b", "2"), ("c", "3"), ("d", "4"), ("e", "5")].iter().enumerate() { + t.set(&format!("F{}", r + 2), name); + t.set(&format!("G{}", r + 2), age); + } + t.set("N46", "Formula"); + t.set("N47", "=DCOUNT($F$2:$G$7,\"age\",$N$46:$N$47)>0"); + t.set("B60", "=DCOUNT($F$2:$G$7,\"age\",N46:N47)"); + assert_eq!(t.num("B60"), 0.0); +} diff --git a/crates/chart/src/raster.rs b/crates/chart/src/raster.rs index 0de2450b5..80042b970 100644 --- a/crates/chart/src/raster.rs +++ b/crates/chart/src/raster.rs @@ -7,6 +7,20 @@ use crate::{Prim, Rgba}; /// Largest image we allocate (pixels). const MAX_PIXELS: u64 = 64 * 1024 * 1024; const SUB: usize = 4; +/// Scan-conversion work one [`rasterize`] call may do: edge tests (an active edge at one +/// sub-sample) plus covered pixels blended. Edges are only tested on the rows they cross, but a +/// caller can still pass enough geometry to keep the rasteriser busy for minutes on a small +/// image; once the allowance is spent the remaining fills are skipped. 2^28 is about a second, +/// far beyond any chart. +const MAX_SCAN_WORK: u64 = 1 << 28; +/// Most points one filled path may have: more is skipped before any edge is built. Chart +/// renderers cap and decimate their paths far below this; it only bounds direct callers. +const MAX_FILL_POINTS: usize = 1 << 22; + +/// Most points one stroked polyline may have: more is skipped before any segment path is built +/// (each segment becomes its own quad). Charts hold at most 100,000 points per series and draw far +/// fewer; it only bounds direct callers. +const MAX_STROKE_POINTS: usize = 1 << 17; type Path = Vec>; @@ -15,6 +29,8 @@ struct Canvas { h: usize, buf: Vec, acc: Vec, + /// Scan-conversion work left (see [`MAX_SCAN_WORK`]). + work: u64, } impl Canvas { @@ -41,7 +57,12 @@ impl Canvas { /// Fills a multi-contour path with the even-odd rule. fn fill(&mut self, path: &Path, color: Rgba) { - let mut edges: Vec<[f32; 4]> = Vec::new(); + // Out of work, or more geometry than the allowance could ever scan: no edge list is built. + if self.work == 0 || path.iter().map(Vec::len).fold(0usize, usize::saturating_add) > MAX_FILL_POINTS { + return; + } + // Each edge as [top y, bottom y, x0, y0, x1, y1]. + let mut edges: Vec<[f32; 6]> = Vec::new(); let (mut minx, mut miny, mut maxx, mut maxy) = (f32::INFINITY, f32::INFINITY, f32::NEG_INFINITY, f32::NEG_INFINITY); for c in path { if c.len() < 3 || !c.iter().all(|p| p[0].is_finite() && p[1].is_finite()) { @@ -54,7 +75,7 @@ impl Canvas { miny = miny.min(p[1]); maxy = maxy.max(p[1]); if p[1] != q[1] { - edges.push([p[0], p[1], q[0], q[1]]); + edges.push([p[1].min(q[1]), p[1].max(q[1]), p[0], p[1], q[0], q[1]]); } } } @@ -72,8 +93,28 @@ impl Canvas { if self.acc.len() < bw + 1 { self.acc.resize(bw + 1, 0.0); } + // Active-edge list: edges join when the scan reaches their top and leave past their + // bottom, so each row tests only the edges crossing it instead of every edge. + edges.sort_by(|a, b| a[0].total_cmp(&b[0])); + let mut pending = edges.iter().peekable(); + let mut active: Vec<[f32; 6]> = Vec::new(); let mut xs: Vec = Vec::new(); for py in y0..y1 { + while let Some(e) = pending.next_if(|e| e[0] < py as f32 + 1.0) { + active.push(*e); + } + active.retain(|e| e[1] > py as f32); + if active.is_empty() { + continue; + } + // The row's edge tests and its `bw` accumulator clears and blends, charged before any of + // them, so rows with no coverage still count. + let cost = (active.len() * SUB) as u64 + bw as u64; + if cost > self.work { + self.work = 0; + return; + } + self.work -= cost; for v in self.acc.iter_mut().take(bw) { *v = 0.0; } @@ -81,10 +122,9 @@ impl Canvas { for s in 0..SUB { let sy = py as f32 + (s as f32 + 0.5) / SUB as f32; xs.clear(); - for e in &edges { - let (ya, yb) = if e[1] < e[3] { (e[1], e[3]) } else { (e[3], e[1]) }; - if sy >= ya && sy < yb { - xs.push(e[0] + (sy - e[1]) * (e[2] - e[0]) / (e[3] - e[1])); + for e in &active { + if sy >= e[0] && sy < e[1] { + xs.push(e[2] + (sy - e[3]) * (e[4] - e[2]) / (e[5] - e[3])); } } xs.sort_by(f32::total_cmp); @@ -154,6 +194,9 @@ fn rounded_rect(x: f32, y: f32, w: f32, h: f32, r: f32) -> Vec<[f32; 2]> { /// Thick polyline → one quad per segment (each a separate path so overlaps don't cancel). fn stroke_paths(pts: &[[f32; 2]], width: f32, dash: bool, closed: bool) -> Vec { + if pts.len() > MAX_STROKE_POINTS { + return Vec::new(); + } let width = width.clamp(1.0, 50.0); let mut segs: Vec<([f32; 2], [f32; 2])> = pts.windows(2).map(|w| (w[0], w[1])).collect(); if closed && let (Some(a), Some(b)) = (pts.last(), pts.first()) { @@ -212,7 +255,7 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { for _ in 0..w * h { buf.extend_from_slice(&background); } - let mut cv = Canvas { w, h, buf, acc: Vec::new() }; + let mut cv = Canvas { w, h, buf, acc: Vec::new(), work: MAX_SCAN_WORK }; for p in prims { match p { Prim::Rect { x, y, w, h, fill, stroke, radius } => { @@ -274,6 +317,40 @@ pub fn rasterize(prims: &[Prim], w: u32, h: u32, background: Rgba) -> Vec { mod tests { use super::*; + /// Security: each row tests only the edges crossing it. A polygon of 100,000 tiny edges over + /// 4,000 rows tested every edge at every sub-sample of every row (1.6 billion tests). + #[test] + fn many_short_edges_scan_only_their_rows() { + let mut pts: Vec<[f32; 2]> = (0..100_000).map(|i| [if i % 2 == 0 { 0.0 } else { 0.5 }, i as f32 * 0.04]).collect(); + pts.push([2.0, 4_000.0]); + pts.push([2.0, 0.0]); + let img = rasterize(&[Prim::Polygon { pts, fill: [255, 0, 0, 255], stroke: None }], 2, 4_000, [0, 0, 0, 0]); + // The right-hand pixel of every row is inside the polygon. + assert_eq!(img.get(4..8), Some(&[255, 0, 0, 255][..])); + assert_eq!(img.get(4 * (2 * 3_999 + 1)..4 * (2 * 3_999 + 2)), Some(&[255, 0, 0, 255][..])); + } + + /// Security: scan conversion stops once the call's work allowance is spent. + #[test] + fn fills_stop_when_the_work_allowance_is_spent() { + let square = vec![vec![[0.0, 0.0], [4.0, 0.0], [4.0, 4.0], [0.0, 4.0]]]; + let mut cv = Canvas { w: 4, h: 4, buf: vec![0; 64], acc: Vec::new(), work: 2 * SUB as u64 * 4 + 16 }; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 255)); + cv.buf = vec![0; 64]; + cv.fill(&square, [255; 4]); + assert!(cv.buf.iter().all(|&b| b == 0)); + } + + /// Security: a polyline over the point cap builds no segment paths at all, solid or dashed. + #[test] + fn oversized_polylines_build_no_segment_paths() { + let line = |n: usize| -> Vec<[f32; 2]> { (0..n).map(|i| [i as f32, (i % 2) as f32]).collect() }; + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, false, false).is_empty()); + assert!(stroke_paths(&line(MAX_STROKE_POINTS + 1), 1.0, true, false).is_empty()); + assert_eq!(stroke_paths(&line(MAX_STROKE_POINTS), 1.0, false, false).len(), MAX_STROKE_POINTS - 1); + } + #[test] fn dashed_geometry_stays_within_the_dash_budget() { // Forty very long dashed lines, each allowed 10_000 dashes per segment: before the diff --git a/crates/chart/src/render/lines.rs b/crates/chart/src/render/lines.rs index 770aabe02..678181326 100644 --- a/crates/chart/src/render/lines.rs +++ b/crates/chart/src/render/lines.rs @@ -136,7 +136,13 @@ pub(super) fn decimate_area(top: Vec<[f32; 2]>, base: Vec<[f32; 2]>, max: usize) /// Draws a polyline split at gaps (`None`), decimated when long. With `smooth` (the plot box), /// each run is interpolated with a Catmull-Rom spline so the line curves through the data points. +/// +/// The points of the smoothed runs (data points included) are budgeted across all the runs of the line, not per run: +/// short runs between gaps escape decimation, and 100 sub-segments each would turn a +/// 100,000-point series of three-point runs into millions of points. Past the budget the +/// remaining runs are drawn through their data points only. pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, width: f32, smooth: Option) { + let mut budget = MAX_LINE_PTS * 2; let mut run: Vec<[f32; 2]> = Vec::new(); for p in pts.iter().chain(std::iter::once(&None)) { match p { @@ -145,7 +151,11 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi if run.len() >= 2 { let r = decimate(std::mem::take(&mut run), MAX_LINE_PTS); let r = match smooth { - Some(plot) => smooth_run(r, plot), + Some(plot) => { + let r = smooth_run(r, plot, budget); + budget = budget.saturating_sub(r.len()); + r + } None => r, }; ctx.line(r, color, width); @@ -158,15 +168,15 @@ pub(super) fn draw_runs(ctx: &mut Ctx, pts: &[Option<[f32; 2]>], color: Rgba, wi } /// Catmull-Rom spline through `pts`: every segment gets the same number of sub-segments (up to -/// 100), chosen so the whole curve stays within `MAX_LINE_PTS * 2` points. Endpoints are +/// 100), chosen so the whole curve stays within `cap` points (one per data point when `cap` is +/// smaller than that). Endpoints are /// duplicated so the curve passes through the first and last points, and the curve is clamped /// to the `plot` box (a spline overshoots its data near sharp turns). -fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx) -> Vec<[f32; 2]> { +fn smooth_run(pts: Vec<[f32; 2]>, plot: Bx, cap: usize) -> Vec<[f32; 2]> { let n = pts.len(); if n < 3 { return pts; } - let cap = MAX_LINE_PTS * 2; let steps = (cap / n).clamp(1, 100); let mut out: Vec<[f32; 2]> = Vec::with_capacity(n.saturating_mul(steps).min(cap) + 1); let at = |i: isize| -> [f32; 2] { @@ -219,7 +229,7 @@ mod smooth_tests { fn long_runs_are_smoothed_uniformly_within_the_cap() { let plot = Bx { x: 0.0, y: 0.0, w: 3000.0, h: 100.0 }; let pts: Vec<[f32; 2]> = (0..1500).map(|i| [i as f32 * 2.0, if i % 2 == 0 { 20.0 } else { 80.0 }]).collect(); - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() <= MAX_LINE_PTS * 2 + 1, "{}", out.len()); // Two sub-segments per segment all the way to the end, not 100 each and then a straight tail. assert_eq!(out.len(), 1499 * 2 + 1); @@ -231,11 +241,29 @@ mod smooth_tests { let plot = Bx { x: 0.0, y: 10.0, w: 300.0, h: 100.0 }; // A step right at the plot edges: Catmull-Rom overshoots past both. let pts = vec![[0.0, 110.0], [100.0, 110.0], [110.0, 10.0], [200.0, 10.0], [300.0, 10.0]]; - let out = smooth_run(pts, plot); + let out = smooth_run(pts, plot, MAX_LINE_PTS * 2); assert!(out.len() > 5); assert!(out.iter().all(|p| (0.0..=300.0).contains(&p[0]) && (10.0..=110.0).contains(&p[1]))); } + /// Security: smoothing budgets its added points across every run of a line. A series of + /// three-point runs between gaps used to get 100 sub-segments per run, 67 times its data. + #[test] + fn gap_separated_runs_share_one_smoothing_budget() { + let plot = Bx { x: 0.0, y: 0.0, w: 1e6, h: 100.0 }; + let pts: Vec> = + (0..100_000).map(|i| if i % 4 == 3 { None } else { Some([i as f32, if i % 2 == 0 { 20.0 } else { 80.0 }]) }).collect(); + let m = crate::ApproxMeasure; + let mut ctx = Ctx { out: Vec::new(), m: &m }; + draw_runs(&mut ctx, &pts, WHITE, 1.0, Some(plot)); + let total: usize = ctx.out.iter().map(|p| if let Prim::Line { pts, .. } = p { pts.len() } else { 0 }).sum(); + // 75,000 data points, plus at most the budget added by interpolation. + assert!(total <= 75_000 + MAX_LINE_PTS * 2, "{total}"); + assert_eq!(ctx.out.len(), 25_000); + // The first runs are still smoothed. + assert!(matches!(ctx.out.first(), Some(Prim::Line { pts, .. }) if pts.len() > 3)); + } + #[test] fn decimate_area_preserves_paired_vertical_strips() { let top: Vec<[f32; 2]> = (0..1000).map(|i| [i as f32 * 0.25, ((i * 37) % 100) as f32]).collect(); diff --git a/crates/chart/src/resolve.rs b/crates/chart/src/resolve.rs index 11213a5e7..baa0730ad 100644 --- a/crates/chart/src/resolve.rs +++ b/crates/chart/src/resolve.rs @@ -184,6 +184,53 @@ fn value_text(v: &Value) -> String { } } +/// Characters kept of one category label: an axis label never shows more (labels are fitted to +/// the axis), and copying more only costs memory. +const MAX_LABEL_CHARS: usize = 255; +/// Bytes of category label text kept per series. One long text broadcast by a short formula +/// (`IF(SEQUENCE(100000), long_text)`) is a single shared value, but turning every element into +/// its own label copied it 100,000 times (gigabytes). Labels past the budget are empty. +const MAX_CATEGORY_BYTES: usize = 8 << 20; + +/// At most [`MAX_LABEL_CHARS`] characters and `room` bytes of `s`, cut at a character boundary. +fn clip_label(s: &str, room: usize) -> String { + let mut end = 0; + for (n, (i, c)) in s.char_indices().enumerate() { + if n >= MAX_LABEL_CHARS || i + c.len_utf8() > room { + break; + } + end = i + c.len_utf8(); + } + s.get(..end).unwrap_or_default().to_string() +} + +/// The label of category `i` (value `v`) in at most `room` bytes: formatted with its source +/// cell's number format when it is a number. +fn category_label(wb: &Workbook, src: Option<(&gridcraft_model::Sheet, RangeRef)>, i: usize, v: &Value, room: usize) -> String { + if room == 0 { + return String::new(); // nothing fits: skip the number formatting + } + if let (Some((s, r)), Value::Number(_)) = (src, v) { + let w = r.width().max(1) as usize; + let at = r.start.offset((i / w) as i64, (i % w) as i64); + if let Some(c) = at { + let code = &wb.styles.get(s.style_id(c)).num_fmt.0; + if !code.eq_ignore_ascii_case("general") { + return clip_label(&format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text, room); + } + } + } + // Text is clipped before it is copied: the source may be one long shared string. + let mut v = v; + while let Value::Array(a) = v { + v = a.get(0, 0).unwrap_or(&Value::Empty); + } + match v { + Value::Text(t) => clip_label(t, room), + _ => clip_label(&value_text(v), room), + } +} + /// Category labels, formatted with each source cell's number format (dates, currency…), and /// the outer levels of a multi-level axis: a category range wider than a column of values (or /// taller than a row of values) groups its innermost labels, year beside month. Each outer level @@ -221,29 +268,29 @@ fn category_table(wb: &Workbook, sheet: usize, formula: &str, values: &str, limi None => eval_list_in(wb, sheet, formula, limit, 0), }; let src = cells.and_then(|(sh, r)| wb.sheet(sh).map(|s| (s, r))); + // The retained levels are known before any label is charged: a cell on a discarded level + // gets no label and spends none of the budget. `None`: a flat list (every cell is a label). + let layout = src.and_then(|(_, r)| { + let levels = levels?; + let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); + (vals.len() == w.saturating_mul(h)).then_some((levels, w, h)) // else truncated or not a plain block + }); + let kept = + |i: usize| layout.is_none_or(|(l, w, h)| if l.by_columns { i % w + 1 + l.outer as usize >= w } else { i / w + 1 + l.outer as usize >= h }); + let mut room = MAX_CATEGORY_BYTES; let texts: Vec = vals .iter() .enumerate() .map(|(i, v)| { - if let (Some((s, r)), Value::Number(_)) = (src, v) { - let w = r.width().max(1) as usize; - let at = r.start.offset((i / w) as i64, (i % w) as i64); - if let Some(c) = at { - let code = &wb.styles.get(s.style_id(c)).num_fmt.0; - if !code.eq_ignore_ascii_case("general") { - return format_value(v, &NumberFormat::parse_cached(code), wb.date_system).text; - } - } + if !kept(i) { + return String::new(); } - value_text(v) + let label = category_label(wb, src, i, v, room); + room = room.saturating_sub(label.len()); + label }) .collect(); - let Some((_, r)) = src else { return (texts, vec![]) }; - let Some(levels) = levels else { return (texts, vec![]) }; - let (w, h) = (r.width().max(1) as usize, r.height().max(1) as usize); - if texts.len() != w.saturating_mul(h) { - return (texts, vec![]); // truncated or not a plain block: a flat list - } + let Some((levels, w, h)) = layout else { return (texts, vec![]) }; let at = |row: usize, col: usize| texts.get(row * w + col).cloned().unwrap_or_default(); let column = |c: usize| (0..h).map(|row| at(row, c)).collect::>(); let row = |rr: usize| (0..w).map(|c| at(rr, c)).collect::>(); @@ -369,6 +416,46 @@ pub fn resolve(wb: &Workbook, sheet: usize, chart: &Chart) -> ChartData { ChartData { categories, category_levels, series, palette } } +#[cfg(test)] +mod label_budget_tests { + use gridcraft_model::Workbook; + + /// Security: category labels are clipped before they are copied, with one byte budget per + /// series. A formula broadcasting one long text to 100,000 categories used to copy it + /// 100,000 times (3.3 GB). + #[test] + fn category_labels_share_one_byte_budget() { + let wb = Workbook::new(); + let (texts, _) = super::category_table(&wb, 0, r#"=IF(SEQUENCE(100000),REPT("x",32767))"#, "", super::MAX_POINTS); + assert_eq!(texts.len(), 100_000); + assert_eq!(texts.first().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert!(texts.iter().map(String::len).sum::() <= super::MAX_CATEGORY_BYTES); + // Labels are cut at character boundaries. + assert_eq!(super::clip_label("aé", 2), "a"); + assert_eq!(super::clip_label("aé", 3), "aé"); + } + + /// Cells on discarded outer levels (only the innermost labels and three outer levels are + /// kept) spend none of the byte budget. + #[test] + fn discarded_levels_do_not_spend_the_label_budget() { + let mut wb = Workbook::new(); + let rows = 8000u32; + if let Some(sh) = wb.sheet_mut(0) { + for r in 0..rows { + for c in 0..5 { + sh.set_value(gridcraft_core::CellRef::new(r, c), gridcraft_core::Value::text("x".repeat(255))); + } + } + } + // Five columns: column A is a fourth outer level and is discarded. + let (inner, outer) = super::category_table(&wb, 0, "=A1:E8000", "", super::MAX_POINTS); + assert_eq!(outer.len(), 3); + assert_eq!(inner.last().map(String::len), Some(super::MAX_LABEL_CHARS)); + assert_eq!(outer.first().and_then(|l| l.last()).map(String::len), Some(super::MAX_LABEL_CHARS)); + } +} + #[cfg(test)] mod tests { use gridcraft_model::Workbook; diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index b3f5dbd67..52b313b25 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,5 +1,5 @@ //! GridCraft core types: cell addresses, values and Excel's coercion rules, error values, -//! date serials and typed-input parsing. No dependencies on the rest of the workspace. +//! date serials, regional formats and typed-input parsing. No dependencies on the rest of the workspace. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] #![forbid(unsafe_code)] @@ -7,6 +7,7 @@ pub mod addr; pub mod color; pub mod date; pub mod parse; +pub mod region; pub mod value; pub use addr::{ @@ -14,4 +15,5 @@ pub use addr::{ split_areas, split_sheet, unquote_sheet, }; pub use date::DateSystem; +pub use region::Region; pub use value::{Array, CellError, Value, compare, compare_numbers, compare_text, number_to_text, sort_compare}; diff --git a/crates/core/src/region.rs b/crates/core/src/region.rs new file mode 100644 index 000000000..30632603c --- /dev/null +++ b/crates/core/src/region.rs @@ -0,0 +1,883 @@ +//! Regional formats: how a region writes numbers, dates, times and money. +//! +//! Excel takes these from the operating system's regional settings; they are independent of the +//! interface language and of the formula language. A [`Region`] changes what people *type* into +//! a cell (`1.234,56`, `25/09/2026`) and what cells *show* (separators, month and day names, +//! AM/PM, the system short and long dates). It never changes what is stored: values are numbers, +//! and the number format codes saved in a file stay in their canonical en-US spelling. +//! +//! [`EN_US`] is the default and reproduces GridCraft's original behaviour exactly. +//! +//! # Where the values come from +//! The separators, month and weekday names, AM/PM words, currency symbols and the order of day, +//! month and year are the Unicode CLDR conventions (`unicode-org/cldr-json`: +//! `cldr-numbers-full/main//numbers.json` and `cldr-dates-full/main//ca-gregorian.json`), +//! which is also what macOS, Android and ICU ship. They were read from those files for en-GB, +//! en-IN, cs and ru, read from ICU's CLDR 48 data for the Swiss regions and nl-NL, and written +//! from the same conventions for the other regions. CLDR data is under the Unicode License v3, +//! reproduced in `NOTICE` (row in `ATTRIBUTION.md`). +//! Deliberate choices where operating systems differ from CLDR: +//! - Short dates use a four-digit year (`dd/mm/yyyy`, where CLDR has `dd/MM/y` or `dd/MM/yy`): +//! Excel's short date is the system's, Windows writes four digits, and a two-digit year would +//! not read back to the same date. +//! - The long date is CLDR's *full* date (with the weekday), like Windows en-US, except where +//! Windows' long date differs: Russian, Japanese and Chinese have no weekday (`23 августа +//! 2005 г.`, `2005年8月23日`), as in Microsoft's National Language Support tables. +//! - Korean writes the short date `yyyy-mm-dd` (CLDR: `yy. M. d.`), also as in Microsoft's +//! National Language Support tables. +//! - Month names for `mmm` are CLDR's abbreviated names (so en-GB has `Sept`), except +//! Portuguese, which drops CLDR's period (`jan`, not `jan.`) as Excel in Brazil shows it. +//! - French and Portuguese (Portugal) group digits with U+00A0 (Windows), not CLDR's U+202F. +//! Parsing accepts every space-like grouping character in those regions. +//! - The Swiss regions (en-CH, de-CH, fr-CH, it-CH; Liechtenstein uses de-CH) group digits with +//! CLDR's ASCII apostrophe (`1'234.50`); parsing also accepts the typographic `’` that glibc +//! uses. French (Switzerland) writes a decimal point like the other Swiss regions and glibc's +//! `fr_CH`, where CLDR has a comma. +//! +//! Regions this module doesn't list (`de-AT`, `es-MX`, `en-CA`…) resolve to the main region of +//! their language through [`lookup`]; their own conventions (a Mexican decimal point) are not +//! modelled. + +/// How digits are grouped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Grouping { + /// Groups of three (`1,234,567`). + Thousands, + /// The Indian system: three digits, then groups of two (`12,34,567`). + Indian, +} + +/// The order of the parts of a short date. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DateOrder { + Mdy, + Dmy, + Ymd, +} + +/// One region's conventions. See the module documentation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Region { + /// BCP 47 tag, `pt-BR`. + pub tag: &'static str, + /// The region's name in its own language. + pub name: &'static str, + /// Windows locale id, used by `[$-409]`-style format tags. + pub lcid: u16, + pub decimal: char, + pub group: char, + pub grouping: Grouping, + pub date_order: DateOrder, + pub date_sep: char, + pub time_sep: char, + /// The system short date as a canonical format code (`dd/mm/yyyy`): built-in format 14. + pub short_date: &'static str, + /// The system long date (`[$-F800]`). + pub long_date: &'static str, + /// The system short time. It contains `AM/PM` in regions that use a 12-hour clock. + pub short_time: &'static str, + /// The system long time (`[$-F400]`). + pub long_time: &'static str, + /// Built-in format 22: the short date and a 24-hour time. + pub datetime: &'static str, + pub months: [&'static str; 12], + pub months_abbr: [&'static str; 12], + /// The month names a date with a day uses (Russian `1 января`, Czech `1. ledna`), where they + /// differ from `months`. + pub months_genitive: Option<[&'static str; 12]>, + /// Sunday first. + pub weekdays: [&'static str; 7], + pub weekdays_abbr: [&'static str; 7], + pub am_pm: [&'static str; 2], + pub currency: &'static str, + /// The currency format with the currency's usual decimals (two, or none for the yen and the + /// won), as a canonical code. Where it starts with a quote the symbol comes before the number, + /// otherwise after. + pub currency_format: &'static str, + /// The same with no decimals (for a typed whole amount). + pub currency_format_int: &'static str, +} + +const EN_MONTHS: [&str; 12] = + ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; +const EN_MONTHS_ABBR: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; +const EN_DAYS: [&str; 7] = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]; +const EN_DAYS_ABBR: [&str; 7] = ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]; +const AM_PM: [&str; 2] = ["AM", "PM"]; + +/// English (United States): the default. Identical to the formats before regions existed. +pub static EN_US: Region = Region { + tag: "en-US", + name: "English (United States)", + lcid: 0x0409, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Mdy, + date_sep: '/', + time_sep: ':', + short_date: "m/d/yyyy", + long_date: "dddd, mmmm dd, yyyy", + short_time: "h:mm AM/PM", + long_time: "h:mm:ss AM/PM", + datetime: "m/d/yyyy h:mm", + months: EN_MONTHS, + months_abbr: EN_MONTHS_ABBR, + months_genitive: None, + weekdays: EN_DAYS, + weekdays_abbr: EN_DAYS_ABBR, + am_pm: AM_PM, + currency: "$", + currency_format: "\"$\"#,##0.00", + currency_format_int: "\"$\"#,##0", +}; + +static EN_GB: Region = Region { + tag: "en-GB", + name: "English (United Kingdom)", + lcid: 0x0809, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd, d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd/mm/yyyy hh:mm", + months: EN_MONTHS, + months_abbr: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sept", "Oct", "Nov", "Dec"], + months_genitive: None, + weekdays: EN_DAYS, + weekdays_abbr: EN_DAYS_ABBR, + am_pm: AM_PM, + currency: "£", + currency_format: "\"£\"#,##0.00", + currency_format_int: "\"£\"#,##0", +}; + +static EN_IN: Region = Region { + tag: "en-IN", + name: "English (India)", + lcid: 0x4009, + decimal: '.', + group: ',', + grouping: Grouping::Indian, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd, d mmmm yyyy", + short_time: "h:mm AM/PM", + long_time: "h:mm:ss AM/PM", + datetime: "dd/mm/yyyy h:mm", + months: EN_MONTHS, + months_abbr: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sept", "Oct", "Nov", "Dec"], + months_genitive: None, + weekdays: EN_DAYS, + weekdays_abbr: EN_DAYS_ABBR, + am_pm: AM_PM, + currency: "₹", + currency_format: "\"₹\"#,##0.00", + currency_format_int: "\"₹\"#,##0", +}; + +static EN_CH: Region = Region { + tag: "en-CH", + name: "English (Switzerland)", + // English (Switzerland) has no LCID of its own: Windows reports LOCALE_CUSTOM_UNSPECIFIED. + lcid: 0x1000, + decimal: '.', + group: '\'', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd, d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: EN_MONTHS, + months_abbr: ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sept", "Oct", "Nov", "Dec"], + months_genitive: None, + weekdays: EN_DAYS, + weekdays_abbr: EN_DAYS_ABBR, + am_pm: AM_PM, + currency: "CHF", + currency_format: "\"CHF\" #,##0.00", + currency_format_int: "\"CHF\" #,##0", +}; + +static ES_ES: Region = Region { + tag: "es-ES", + name: "Español (España)", + lcid: 0x0C0A, + decimal: ',', + group: '.', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd, d \"de\" mmmm \"de\" yyyy", + short_time: "h:mm", + long_time: "h:mm:ss", + datetime: "dd/mm/yyyy h:mm", + months: ["enero", "febrero", "marzo", "abril", "mayo", "junio", "julio", "agosto", "septiembre", "octubre", "noviembre", "diciembre"], + months_abbr: ["ene", "feb", "mar", "abr", "may", "jun", "jul", "ago", "sep", "oct", "nov", "dic"], + months_genitive: None, + weekdays: ["domingo", "lunes", "martes", "miércoles", "jueves", "viernes", "sábado"], + weekdays_abbr: ["dom", "lun", "mar", "mié", "jue", "vie", "sáb"], + am_pm: ["a. m.", "p. m."], + currency: "€", + currency_format: "#,##0.00 \"€\"", + currency_format_int: "#,##0 \"€\"", +}; + +static DE_DE: Region = Region { + tag: "de-DE", + name: "Deutsch (Deutschland)", + lcid: 0x0407, + decimal: ',', + group: '.', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd, d. mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: ["Januar", "Februar", "März", "April", "Mai", "Juni", "Juli", "August", "September", "Oktober", "November", "Dezember"], + months_abbr: ["Jan", "Feb", "Mär", "Apr", "Mai", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dez"], + months_genitive: None, + weekdays: ["Sonntag", "Montag", "Dienstag", "Mittwoch", "Donnerstag", "Freitag", "Samstag"], + weekdays_abbr: ["So", "Mo", "Di", "Mi", "Do", "Fr", "Sa"], + am_pm: AM_PM, + currency: "€", + currency_format: "#,##0.00 \"€\"", + currency_format_int: "#,##0 \"€\"", +}; + +static DE_CH: Region = Region { + tag: "de-CH", + name: "Deutsch (Schweiz)", + lcid: 0x0807, + decimal: '.', + group: '\'', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd, d. mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: ["Januar", "Februar", "März", "April", "Mai", "Juni", "Juli", "August", "September", "Oktober", "November", "Dezember"], + months_abbr: ["Jan", "Feb", "Mär", "Apr", "Mai", "Jun", "Jul", "Aug", "Sep", "Okt", "Nov", "Dez"], + months_genitive: None, + weekdays: ["Sonntag", "Montag", "Dienstag", "Mittwoch", "Donnerstag", "Freitag", "Samstag"], + weekdays_abbr: ["So", "Mo", "Di", "Mi", "Do", "Fr", "Sa"], + am_pm: AM_PM, + currency: "CHF", + currency_format: "\"CHF\" #,##0.00", + currency_format_int: "\"CHF\" #,##0", +}; + +static FR_FR: Region = Region { + tag: "fr-FR", + name: "Français (France)", + lcid: 0x040C, + decimal: ',', + group: '\u{a0}', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd/mm/yyyy hh:mm", + months: ["janvier", "février", "mars", "avril", "mai", "juin", "juillet", "août", "septembre", "octobre", "novembre", "décembre"], + months_abbr: ["janv.", "févr.", "mars", "avr.", "mai", "juin", "juil.", "août", "sept.", "oct.", "nov.", "déc."], + months_genitive: None, + weekdays: ["dimanche", "lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi"], + weekdays_abbr: ["dim.", "lun.", "mar.", "mer.", "jeu.", "ven.", "sam."], + am_pm: AM_PM, + currency: "€", + currency_format: "#,##0.00 \"€\"", + currency_format_int: "#,##0 \"€\"", +}; + +static FR_CH: Region = Region { + tag: "fr-CH", + name: "Français (Suisse)", + lcid: 0x100C, + decimal: '.', + group: '\'', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd, d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: ["janvier", "février", "mars", "avril", "mai", "juin", "juillet", "août", "septembre", "octobre", "novembre", "décembre"], + months_abbr: ["janv.", "févr.", "mars", "avr.", "mai", "juin", "juil.", "août", "sept.", "oct.", "nov.", "déc."], + months_genitive: None, + weekdays: ["dimanche", "lundi", "mardi", "mercredi", "jeudi", "vendredi", "samedi"], + weekdays_abbr: ["dim.", "lun.", "mar.", "mer.", "jeu.", "ven.", "sam."], + am_pm: AM_PM, + currency: "CHF", + currency_format: "#,##0.00 \"CHF\"", + currency_format_int: "#,##0 \"CHF\"", +}; + +static IT_IT: Region = Region { + tag: "it-IT", + name: "Italiano (Italia)", + lcid: 0x0410, + decimal: ',', + group: '.', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd/mm/yyyy hh:mm", + months: ["gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"], + months_abbr: ["gen", "feb", "mar", "apr", "mag", "giu", "lug", "ago", "set", "ott", "nov", "dic"], + months_genitive: None, + weekdays: ["domenica", "lunedì", "martedì", "mercoledì", "giovedì", "venerdì", "sabato"], + weekdays_abbr: ["dom", "lun", "mar", "mer", "gio", "ven", "sab"], + am_pm: AM_PM, + currency: "€", + currency_format: "#,##0.00 \"€\"", + currency_format_int: "#,##0 \"€\"", +}; + +static IT_CH: Region = Region { + tag: "it-CH", + name: "Italiano (Svizzera)", + lcid: 0x0810, + decimal: '.', + group: '\'', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd, d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: ["gennaio", "febbraio", "marzo", "aprile", "maggio", "giugno", "luglio", "agosto", "settembre", "ottobre", "novembre", "dicembre"], + months_abbr: ["gen", "feb", "mar", "apr", "mag", "giu", "lug", "ago", "set", "ott", "nov", "dic"], + months_genitive: None, + weekdays: ["domenica", "lunedì", "martedì", "mercoledì", "giovedì", "venerdì", "sabato"], + weekdays_abbr: ["dom", "lun", "mar", "mer", "gio", "ven", "sab"], + am_pm: AM_PM, + currency: "CHF", + currency_format: "\"CHF\" #,##0.00", + currency_format_int: "\"CHF\" #,##0", +}; + +static NL_NL: Region = Region { + tag: "nl-NL", + name: "Nederlands (Nederland)", + lcid: 0x0413, + decimal: ',', + group: '.', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '-', + time_sep: ':', + short_date: "dd-mm-yyyy", + long_date: "dddd d mmmm yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd-mm-yyyy hh:mm", + months: ["januari", "februari", "maart", "april", "mei", "juni", "juli", "augustus", "september", "oktober", "november", "december"], + months_abbr: ["jan", "feb", "mrt", "apr", "mei", "jun", "jul", "aug", "sep", "okt", "nov", "dec"], + months_genitive: None, + weekdays: ["zondag", "maandag", "dinsdag", "woensdag", "donderdag", "vrijdag", "zaterdag"], + weekdays_abbr: ["zo", "ma", "di", "wo", "do", "vr", "za"], + am_pm: AM_PM, + currency: "€", + currency_format: "\"€\" #,##0.00", + currency_format_int: "\"€\" #,##0", +}; + +static PT_BR: Region = Region { + tag: "pt-BR", + name: "Português (Brasil)", + lcid: 0x0416, + decimal: ',', + group: '.', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd, d \"de\" mmmm \"de\" yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd/mm/yyyy hh:mm", + months: ["janeiro", "fevereiro", "março", "abril", "maio", "junho", "julho", "agosto", "setembro", "outubro", "novembro", "dezembro"], + months_abbr: ["jan", "fev", "mar", "abr", "mai", "jun", "jul", "ago", "set", "out", "nov", "dez"], + months_genitive: None, + weekdays: ["domingo", "segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", "sexta-feira", "sábado"], + weekdays_abbr: ["dom", "seg", "ter", "qua", "qui", "sex", "sáb"], + am_pm: AM_PM, + currency: "R$", + currency_format: "\"R$\" #,##0.00", + currency_format_int: "\"R$\" #,##0", +}; + +static PT_PT: Region = Region { + tag: "pt-PT", + name: "Português (Portugal)", + lcid: 0x0816, + decimal: ',', + group: '\u{a0}', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '/', + time_sep: ':', + short_date: "dd/mm/yyyy", + long_date: "dddd, d \"de\" mmmm \"de\" yyyy", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd/mm/yyyy hh:mm", + months: ["janeiro", "fevereiro", "março", "abril", "maio", "junho", "julho", "agosto", "setembro", "outubro", "novembro", "dezembro"], + months_abbr: ["jan", "fev", "mar", "abr", "mai", "jun", "jul", "ago", "set", "out", "nov", "dez"], + months_genitive: None, + weekdays: ["domingo", "segunda-feira", "terça-feira", "quarta-feira", "quinta-feira", "sexta-feira", "sábado"], + weekdays_abbr: ["dom", "seg", "ter", "qua", "qui", "sex", "sáb"], + am_pm: AM_PM, + currency: "€", + currency_format: "#,##0.00 \"€\"", + currency_format_int: "#,##0 \"€\"", +}; + +static RU_RU: Region = Region { + tag: "ru-RU", + name: "Русский (Россия)", + lcid: 0x0419, + decimal: ',', + group: '\u{a0}', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "d mmmm yyyy\" г.\"", + short_time: "hh:mm", + long_time: "hh:mm:ss", + datetime: "dd.mm.yyyy hh:mm", + months: ["январь", "февраль", "март", "апрель", "май", "июнь", "июль", "август", "сентябрь", "октябрь", "ноябрь", "декабрь"], + months_abbr: ["янв.", "февр.", "март", "апр.", "май", "июнь", "июль", "авг.", "сент.", "окт.", "нояб.", "дек."], + months_genitive: Some(["января", "февраля", "марта", "апреля", "мая", "июня", "июля", "августа", "сентября", "октября", "ноября", "декабря"]), + weekdays: ["воскресенье", "понедельник", "вторник", "среда", "четверг", "пятница", "суббота"], + weekdays_abbr: ["вс", "пн", "вт", "ср", "чт", "пт", "сб"], + am_pm: AM_PM, + currency: "₽", + currency_format: "#,##0.00\u{a0}\"₽\"", + currency_format_int: "#,##0\u{a0}\"₽\"", +}; + +static CS_CZ: Region = Region { + tag: "cs-CZ", + name: "Čeština (Česko)", + lcid: 0x0405, + decimal: ',', + group: '\u{a0}', + grouping: Grouping::Thousands, + date_order: DateOrder::Dmy, + date_sep: '.', + time_sep: ':', + short_date: "dd.mm.yyyy", + long_date: "dddd d. mmmm yyyy", + short_time: "h:mm", + long_time: "h:mm:ss", + datetime: "dd.mm.yyyy h:mm", + months: ["leden", "únor", "březen", "duben", "květen", "červen", "červenec", "srpen", "září", "říjen", "listopad", "prosinec"], + months_abbr: ["led", "úno", "bře", "dub", "kvě", "čvn", "čvc", "srp", "zář", "říj", "lis", "pro"], + months_genitive: Some(["ledna", "února", "března", "dubna", "května", "června", "července", "srpna", "září", "října", "listopadu", "prosince"]), + weekdays: ["neděle", "pondělí", "úterý", "středa", "čtvrtek", "pátek", "sobota"], + weekdays_abbr: ["ne", "po", "út", "st", "čt", "pá", "so"], + am_pm: ["dop.", "odp."], + currency: "Kč", + currency_format: "#,##0.00 \"Kč\"", + currency_format_int: "#,##0 \"Kč\"", +}; + +static JA_JP: Region = Region { + tag: "ja-JP", + name: "日本語 (日本)", + lcid: 0x0411, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Ymd, + date_sep: '/', + time_sep: ':', + short_date: "yyyy/m/d", + long_date: "yyyy\"年\"m\"月\"d\"日\"", + short_time: "h:mm", + long_time: "h:mm:ss", + datetime: "yyyy/m/d h:mm", + months: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], + months_abbr: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], + months_genitive: None, + weekdays: ["日曜日", "月曜日", "火曜日", "水曜日", "木曜日", "金曜日", "土曜日"], + weekdays_abbr: ["日", "月", "火", "水", "木", "金", "土"], + am_pm: ["午前", "午後"], + currency: "¥", + currency_format: "\"¥\"#,##0", + currency_format_int: "\"¥\"#,##0", +}; + +static KO_KR: Region = Region { + tag: "ko-KR", + name: "한국어 (대한민국)", + lcid: 0x0412, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Ymd, + date_sep: '-', + time_sep: ':', + short_date: "yyyy-mm-dd", + long_date: "yyyy\"년\" m\"월\" d\"일\" dddd", + short_time: "AM/PM h:mm", + long_time: "AM/PM h:mm:ss", + datetime: "yyyy-mm-dd hh:mm", + months: ["1월", "2월", "3월", "4월", "5월", "6월", "7월", "8월", "9월", "10월", "11월", "12월"], + months_abbr: ["1월", "2월", "3월", "4월", "5월", "6월", "7월", "8월", "9월", "10월", "11월", "12월"], + months_genitive: None, + weekdays: ["일요일", "월요일", "화요일", "수요일", "목요일", "금요일", "토요일"], + weekdays_abbr: ["일", "월", "화", "수", "목", "금", "토"], + am_pm: ["오전", "오후"], + currency: "₩", + currency_format: "\"₩\"#,##0", + currency_format_int: "\"₩\"#,##0", +}; + +static ZH_CN: Region = Region { + tag: "zh-CN", + name: "中文 (中国)", + lcid: 0x0804, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Ymd, + date_sep: '/', + time_sep: ':', + short_date: "yyyy/m/d", + long_date: "yyyy\"年\"m\"月\"d\"日\"", + short_time: "h:mm", + long_time: "h:mm:ss", + datetime: "yyyy/m/d h:mm", + months: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], + months_abbr: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], + months_genitive: None, + weekdays: ["星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"], + weekdays_abbr: ["周日", "周一", "周二", "周三", "周四", "周五", "周六"], + am_pm: ["上午", "下午"], + currency: "¥", + currency_format: "\"¥\"#,##0.00", + currency_format_int: "\"¥\"#,##0", +}; + +static ZH_TW: Region = Region { + tag: "zh-TW", + name: "中文 (台灣)", + lcid: 0x0404, + decimal: '.', + group: ',', + grouping: Grouping::Thousands, + date_order: DateOrder::Ymd, + date_sep: '/', + time_sep: ':', + short_date: "yyyy/m/d", + long_date: "yyyy\"年\"m\"月\"d\"日\"", + short_time: "AM/PM hh:mm", + long_time: "AM/PM hh:mm:ss", + datetime: "yyyy/m/d hh:mm", + months: ["一月", "二月", "三月", "四月", "五月", "六月", "七月", "八月", "九月", "十月", "十一月", "十二月"], + months_abbr: ["1月", "2月", "3月", "4月", "5月", "6月", "7月", "8月", "9月", "10月", "11月", "12月"], + months_genitive: None, + weekdays: ["星期日", "星期一", "星期二", "星期三", "星期四", "星期五", "星期六"], + weekdays_abbr: ["週日", "週一", "週二", "週三", "週四", "週五", "週六"], + am_pm: ["上午", "下午"], + currency: "NT$", + currency_format: "\"NT$\"#,##0.00", + currency_format_int: "\"NT$\"#,##0", +}; + +static ALL: [&Region; 20] = [ + &EN_US, &EN_GB, &EN_IN, &EN_CH, &ES_ES, &DE_DE, &DE_CH, &FR_FR, &FR_CH, &IT_IT, &IT_CH, &NL_NL, &PT_BR, &PT_PT, &RU_RU, &CS_CZ, &JA_JP, &KO_KR, + &ZH_CN, &ZH_TW, +]; + +/// Every region, English (United States) first. +pub fn all() -> &'static [&'static Region] { + &ALL +} + +/// The default region, English (United States). +pub fn default_region() -> &'static Region { + &EN_US +} + +/// The region with exactly this tag (`pt-BR`; case and `_` don't matter). +pub fn find(tag: &str) -> Option<&'static Region> { + let t = tag.trim().replace('_', "-"); + ALL.iter().copied().find(|r| r.tag.eq_ignore_ascii_case(&t)) +} + +/// The region for a tag as operating systems and browsers write it: `pt-BR`, `pt_BR.UTF-8`, +/// `de`, `en-AU`, `zh-Hant`, `sr_RS@latin`. An exact match wins; otherwise the language's main +/// region (with the usual variants: `en-AU`→`en-GB`, `pt-AO`→`pt-PT`, `zh-Hant`→`zh-TW`). +/// An unknown language is English (United States). +pub fn lookup(tag: &str) -> &'static Region { + if let Some(r) = find(tag) { + return r; + } + let base = tag.split(['.', '@']).next().unwrap_or(""); + let norm = base.trim().replace('_', "-"); + if let Some(r) = find(&norm) { + return r; + } + let mut parts = norm.split('-').filter(|p| !p.is_empty()); + let lang = parts.next().unwrap_or("").to_ascii_lowercase(); + let mut script = String::new(); + let mut country = String::new(); + for p in parts { + // An extension or private-use singleton (`-u-ca-gregory`, `-x-…`) ends the region part. + if p.len() == 1 { + break; + } + if p.len() == 4 && p.chars().all(|c| c.is_ascii_alphabetic()) { + script = p.to_ascii_lowercase(); + } else if (p.len() == 2 && p.chars().all(|c| c.is_ascii_alphabetic())) || (p.len() == 3 && p.chars().all(|c| c.is_ascii_digit())) { + country = p.to_ascii_uppercase(); + } + } + let tag = match lang.as_str() { + "en" => match country.as_str() { + "GB" | "AU" | "NZ" | "IE" => "en-GB", + "IN" => "en-IN", + "CH" | "LI" => "en-CH", + _ => "en-US", + }, + "pt" => match country.as_str() { + "AO" | "CH" | "CV" | "FR" | "GQ" | "GW" | "LU" | "MO" | "MZ" | "ST" | "TL" | "PT" => "pt-PT", + _ => "pt-BR", + }, + "zh" => match country.as_str() { + // An explicit territory wins over the script (`zh-Hant-CN` is China's conventions). + "CN" | "SG" => "zh-CN", + "TW" | "HK" | "MO" => "zh-TW", + _ if script == "hant" => "zh-TW", + _ => "zh-CN", + }, + "es" => "es-ES", + "de" => match country.as_str() { + "CH" | "LI" => "de-CH", + _ => "de-DE", + }, + "fr" => match country.as_str() { + "CH" | "LI" => "fr-CH", + _ => "fr-FR", + }, + "it" => match country.as_str() { + "CH" | "LI" => "it-CH", + _ => "it-IT", + }, + "nl" => "nl-NL", + "ru" => "ru-RU", + "cs" => "cs-CZ", + "ja" => "ja-JP", + "ko" => "ko-KR", + _ => "en-US", + }; + find(tag).unwrap_or(&EN_US) +} + +impl Region { + /// True for the characters that separate groups of digits in this region. A region that + /// groups with a space also accepts the other space-like characters (NBSP, narrow NBSP, thin + /// space), as people type and paste any of them; one that groups with an apostrophe accepts + /// both `'` and `’`. + pub fn is_group(&self, c: char) -> bool { + c == self.group || (is_space_like(self.group) && is_space_like(c)) || (is_apostrophe_like(self.group) && is_apostrophe_like(c)) + } + + /// True when the currency symbol comes before the number. + pub fn currency_first(&self) -> bool { + self.currency_format.starts_with('"') + } + + /// Month number (1–12) for a month name as typed: the full name, the abbreviation (with or + /// without its period) or any prefix of the full name of three or more letters (`Sept`), + /// ignoring case. `word` is one word: the genitive form used after a day (`января` in + /// `1 января`) matches too, and the caller reads the day itself. + pub fn month_from_name(&self, word: &str) -> Option { + let w = word.trim().to_lowercase(); + // Where the abbreviations end in a period (`сент.`, `janv.`), any abbreviation may be typed + // with one (`мар.`, `juin.`). Elsewhere a period is not part of a month name, so en-US + // `Mar. 4, 2026` stays text as before. + let dotted = self.months_abbr.iter().any(|a| a.ends_with('.')); + let stem = if dotted { w.trim_end_matches('.') } else { w.as_str() }; + if stem.is_empty() { + return None; + } + let long_enough = stem.chars().count() >= 3; + for (i, full) in self.months.iter().enumerate() { + let full_l = full.to_lowercase(); + let abbr_l = self.months_abbr.get(i).map(|a| a.to_lowercase()).unwrap_or_default(); + let gen_l = self.months_genitive.as_ref().and_then(|g| g.get(i)).map(|g| g.to_lowercase()); + let hit = w == full_l + || w == abbr_l + || stem == abbr_l.trim_end_matches('.') + || gen_l.as_deref() == Some(w.as_str()) + || (long_enough && full_l.starts_with(stem)) + || (long_enough && gen_l.as_deref().is_some_and(|g| g.starts_with(stem))); + if hit { + return Some(i as u32 + 1); + } + } + None + } + + /// The name of month `month` (1–12) for `mmmm`; `with_day` selects the genitive form where + /// the region has one. + pub fn month_name(&self, month: u32, with_day: bool) -> &'static str { + let i = (month as usize).wrapping_sub(1); + if with_day && let Some(g) = &self.months_genitive { + return g.get(i).copied().unwrap_or(""); + } + self.months.get(i).copied().unwrap_or("") + } + + /// True when the short time uses a 12-hour clock. + pub fn is_12_hour(&self) -> bool { + self.short_time.contains("AM/PM") + } +} + +fn is_space_like(c: char) -> bool { + matches!(c, ' ' | '\u{a0}' | '\u{202f}' | '\u{2009}') +} + +fn is_apostrophe_like(c: char) -> bool { + matches!(c, '\'' | '\u{2019}') +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lookup_accepts_system_and_browser_tags() { + let cases = [ + ("pt-BR", "pt-BR"), + ("pt_BR.UTF-8", "pt-BR"), + ("pt", "pt-BR"), + ("pt-PT", "pt-PT"), + ("pt-AO", "pt-PT"), + ("de", "de-DE"), + ("de-AT", "de-DE"), + ("de_CH.UTF-8", "de-CH"), + ("de-LI", "de-CH"), + ("fr-CH", "fr-CH"), + ("fr-LI", "fr-CH"), + ("fr-BE", "fr-FR"), + ("it_CH.UTF-8", "it-CH"), + ("it-LI", "it-CH"), + ("en-CH", "en-CH"), + ("nl_NL.UTF-8", "nl-NL"), + ("nl", "nl-NL"), + ("en-AU", "en-GB"), + ("en_GB.UTF-8", "en-GB"), + ("en-IN", "en-IN"), + ("en-CA", "en-US"), + ("zh-Hant", "zh-TW"), + ("zh-HK", "zh-TW"), + ("zh", "zh-CN"), + // Extensions don't replace the region or script; an explicit territory beats the script. + ("en-GB-u-ca-gregory", "en-GB"), + ("zh-Hant-u-nu-latn", "zh-TW"), + ("zh-Hant-CN", "zh-CN"), + ("zh-Hans-TW", "zh-TW"), + ("cs_CZ.UTF-8", "cs-CZ"), + ("C", "en-US"), + ("", "en-US"), + ("xx-YY", "en-US"), + ]; + for (tag, expected) in cases { + assert_eq!(lookup(tag).tag, expected, "{tag}"); + } + assert_eq!(find("DE-de").map(|r| r.tag), Some("de-DE")); + assert!(find("xx").is_none()); + } + + #[test] + fn every_region_is_consistent() { + let mut tags = std::collections::HashSet::new(); + for r in all() { + assert!(tags.insert(r.tag), "duplicate {}", r.tag); + assert_ne!(r.decimal, r.group, "{}", r.tag); + assert!(r.currency_format.contains(r.currency), "{}", r.tag); + assert!(r.months.iter().all(|m| !m.is_empty()) && r.months_abbr.iter().all(|m| !m.is_empty()), "{}", r.tag); + assert!(r.weekdays.iter().all(|m| !m.is_empty()) && r.weekdays_abbr.iter().all(|m| !m.is_empty()), "{}", r.tag); + assert_eq!(find(r.tag).map(|x| x.tag), Some(r.tag)); + } + } + + #[test] + fn month_names_match_loosely() { + let de = find("de-DE").unwrap(); + assert_eq!(de.month_from_name("März"), Some(3)); + assert_eq!(de.month_from_name("mär"), Some(3)); + assert_eq!(de.month_from_name("DEZ"), Some(12)); + assert_eq!(de.month_from_name("de"), None); + let gb = find("en-GB").unwrap(); + assert_eq!(gb.month_from_name("Sept"), Some(9)); + assert_eq!(gb.month_from_name("sep"), Some(9)); + let ru = find("ru-RU").unwrap(); + assert_eq!(ru.month_from_name("сентября"), Some(9)); + assert_eq!(ru.month_from_name("сент."), Some(9)); + // Abbreviations typed with a period the table doesn't carry. + assert_eq!(ru.month_from_name("мар."), Some(3)); + assert_eq!(ru.month_from_name("июн."), Some(6)); + assert_eq!(find("fr-FR").unwrap().month_from_name("juin."), Some(6)); + assert_eq!(EN_US.month_from_name("Mar."), None); + assert_eq!(de.month_from_name("."), None); + assert_eq!(EN_US.month_from_name("Ma"), None); + assert_eq!(EN_US.month_from_name("May"), Some(5)); + } + + #[test] + fn space_like_and_apostrophe_groups_are_interchangeable() { + let fr = find("fr-FR").unwrap(); + assert!(fr.is_group('\u{a0}') && fr.is_group(' ') && fr.is_group('\u{202f}')); + assert!(!find("de-DE").unwrap().is_group(' ')); + let ch = find("de-CH").unwrap(); + assert!(ch.is_group('\'') && ch.is_group('\u{2019}')); + assert!(!ch.is_group(' ') && !find("de-DE").unwrap().is_group('\'')); + } +} diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 0000028e9..a0b4fcad6 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -346,19 +346,91 @@ pub fn compare_numbers(a: f64, b: f64) -> Ordering { round15(a).partial_cmp(&round15(b)).unwrap_or(Ordering::Equal) } -/// Case-insensitive text order (Unicode lowercase). +/// Case-insensitive text order in three levels, as Excel's collation sorts: first by letters and +/// digits alone (Arabic marks, tatweel and invisible format characters skipped, Arabic-Indic digits +/// read as digits); then, letter by letter, by the marks on each letter (none first, then in +/// Excel's mark order) and the digit script (ASCII, Arabic-Indic, Extended Arabic-Indic); then, to +/// break a remaining tie, by every character (Unicode lowercase). Only text that is equal at the +/// last level compares equal. pub fn compare_text(a: &str, b: &str) -> Ordering { - let mut ai = a.chars().flat_map(char::to_lowercase); - let mut bi = b.chars().flat_map(char::to_lowercase); - loop { - match (ai.next(), bi.next()) { - (None, None) => return Ordering::Equal, - (None, Some(_)) => return Ordering::Less, - (Some(_), None) => return Ordering::Greater, - (Some(x), Some(y)) if x != y => return x.cmp(&y), - _ => {} + if !(a.is_ascii() && b.is_ascii()) { + let primary = primary_chars(a).cmp(primary_chars(b)); + if primary != Ordering::Equal { + return primary; + } + let secondary = secondary_weights(a).cmp(secondary_weights(b)); + if secondary != Ordering::Equal { + return secondary; } } + lowercase_chars(a).cmp(lowercase_chars(b)) +} + +fn lowercase_chars(s: &str) -> impl Iterator + '_ { + s.chars().flat_map(char::to_lowercase) +} + +/// Characters Excel's text order skips entirely: tatweel and invisible format characters (soft +/// hyphen, combining grapheme joiner, zero-width (non-)joiner, bidi marks, embeddings, overrides +/// and isolates, word joiner, byte order mark). The zero-width space is not one. +fn ignorable(c: char) -> bool { + matches!( + c, + '\u{00AD}' + | '\u{034F}' + | '\u{061C}' + | '\u{0640}' + | '\u{200C}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2060}' + | '\u{2066}'..='\u{2069}' + | '\u{FEFF}' + ) +} + +/// Arabic combining marks in the order Excel sorts a letter carrying them; an unmarked letter +/// sorts before all of them. Measured in Excel 16.0 (build 20527, en-US) with one mark on a letter. +/// U+06D6 and U+06D7 sort as letters in Excel, so they are not listed. +const ARABIC_MARK_ORDER: [char; 48] = [ + '\u{06EC}', '\u{0611}', '\u{0612}', '\u{0613}', '\u{0614}', '\u{06DB}', '\u{0656}', '\u{0657}', '\u{0610}', '\u{06EB}', '\u{0615}', '\u{06D9}', + '\u{06E4}', '\u{06E1}', '\u{06DA}', '\u{06DC}', '\u{06EA}', '\u{06E3}', '\u{06D8}', '\u{0655}', '\u{065F}', '\u{0654}', '\u{06E7}', '\u{0653}', + '\u{06E8}', '\u{06ED}', '\u{06E2}', '\u{0658}', '\u{064B}', '\u{064C}', '\u{064D}', '\u{064E}', '\u{064F}', '\u{0650}', '\u{0652}', '\u{0651}', + '\u{0670}', '\u{0659}', '\u{065A}', '\u{065B}', '\u{065C}', '\u{065D}', '\u{065E}', '\u{0616}', '\u{0617}', '\u{0618}', '\u{0619}', '\u{061A}', +]; + +/// Rank of an Arabic mark that letter order skips (its index in `ARABIC_MARK_ORDER`). +fn arabic_mark_rank(c: char) -> Option { + ARABIC_MARK_ORDER.iter().position(|&m| m == c).and_then(|i| u16::try_from(i).ok()) +} + +/// Digit script of `c` (0 ASCII, 1 Arabic-Indic, 2 Extended Arabic-Indic) and its ASCII digit. +fn digit(c: char) -> Option<(u16, char)> { + let (script, zero) = match c { + '0'..='9' => return Some((0, c)), + '\u{0660}'..='\u{0669}' => (1, 0x0660), + '\u{06F0}'..='\u{06F9}' => (2, 0x06F0), + _ => return None, + }; + char::from_u32(u32::from(c) - zero + u32::from('0')).map(|d| (script, d)) +} + +/// The characters that decide text order before any tie-break. +fn primary_chars(s: &str) -> impl Iterator + '_ { + lowercase_chars(s).filter_map(|c| if ignorable(c) || arabic_mark_rank(c).is_some() { None } else { Some(digit(c).map_or(c, |(_, d)| d)) }) +} + +/// The second level, for text whose primary characters are equal: per letter, its digit script and +/// then its marks, the letters' lists closed by 0 so that an unmarked letter sorts first. Marks +/// before the first letter form a list of their own. +fn secondary_weights(s: &str) -> impl Iterator + '_ { + lowercase_chars(s) + .filter(|&c| !ignorable(c)) + .flat_map(|c| match arabic_mark_rank(c) { + Some(rank) => [Some(rank.saturating_add(1)), None], + None => [Some(0), Some(digit(c).map_or(0, |(script, _)| script))], + }) + .flatten() + .chain(std::iter::once(0)) } /// Sorting order used by Sort (ascending): numbers, text, booleans, errors, then blanks last. @@ -457,6 +529,182 @@ mod tests { assert_eq!(sort_compare(&Value::Empty, &Value::Number(1.0)), Ordering::Greater); } + #[test] + fn arabic_diacritics_sort_with_their_base_letter() { + assert_eq!(compare_text("بَاب", "بيت"), Ordering::Less); + assert_eq!(compare_text("بيت", "بَاب"), Ordering::Greater); + assert_eq!(compare_text("كِتاب", "كتب"), Ordering::Less); + assert_eq!(compare_text("رحمٰن", "رحمي"), Ordering::Less); + assert_eq!(compare(&Value::from("بَاب"), &Value::from("بيت")), Ordering::Less); + } + + #[test] + fn arabic_indic_digits_sort_with_digits() { + assert_eq!(compare_text("١٢", "ب"), Ordering::Less); + assert_eq!(compare_text("۱۲", "ب"), Ordering::Less); + assert_eq!(compare_text("١", "a"), Ordering::Less); + assert_eq!(compare_text("١٢", "13"), Ordering::Less); + assert_eq!(compare_text("۱۴", "13"), Ordering::Greater); + } + + #[test] + fn tatweel_and_bidi_marks_do_not_move_sort_order() { + assert_eq!(compare_text("بـاب", "بت"), Ordering::Less); + assert_eq!(compare_text("\u{200F}apple", "banana"), Ordering::Less); + assert_eq!(compare_text("\u{200E}zebra", "apple"), Ordering::Greater); + assert_eq!(compare_text("\u{061C}a", "b"), Ordering::Less); + assert_eq!(compare_text("a\u{200C}b", "ac"), Ordering::Less); + assert_eq!(compare_text("a\u{200D}b", "ac"), Ordering::Less); + } + + #[test] + fn primary_ties_keep_equality_unchanged() { + assert_eq!(compare_text("ABC", "abc"), Ordering::Equal); + assert_eq!(compare_text("باب", "بَاب"), Ordering::Less); + assert_eq!(compare_text("12", "١٢"), Ordering::Less); + assert_eq!(compare_text("a", "\u{200F}a"), Ordering::Less); + let samples = [ + "", + "abc", + "ABC", + "abd", + "ab", + "باب", + "بَاب", + "بِاب", + "بـاب", + "بيت", + "١٢", + "۱۲", + "12", + "\u{200F}12", + "\u{200F}a", + "a", + "A\u{200C}", + "a\u{200F}", + "b", + "Straße", + "STRASSE", + "ΣΑΣ", + "σας", + "İ", + "i̇", + ]; + let lowered = |s: &str| s.chars().flat_map(char::to_lowercase).collect::(); + for a in samples { + for b in samples { + let order = compare_text(a, b); + assert_eq!(order.is_eq(), lowered(a) == lowered(b), "{a:?} vs {b:?}"); + assert_eq!(order, compare_text(b, a).reverse(), "{a:?} vs {b:?}"); + for c in samples { + if order.is_le() && compare_text(b, c).is_le() { + assert!(compare_text(a, c).is_le(), "{a:?} <= {b:?} <= {c:?}"); + } + } + } + } + } + + /// Sorts `expected` from reversed and rotated starts and checks each comes back as `expected`. + #[track_caller] + fn assert_sorts_to(expected: &[&str]) { + let mut reversed: Vec<&str> = expected.iter().rev().copied().collect(); + reversed.sort_by(|a, b| compare_text(a, b)); + assert_eq!(reversed, expected); + let mut rotated: Vec<&str> = expected.to_vec(); + rotated.rotate_left(expected.len() / 2); + rotated.sort_by(|a, b| compare_text(a, b)); + assert_eq!(rotated, expected); + } + + // The expected orders below are Excel's, observed with Range.Sort, =SORT() and the `<` + // operator in Excel 16.0 (build 20527, en-US). Strings Excel ranks equal are left out. + + #[test] + fn excel_orders_a_letter_by_its_mark() { + assert_sorts_to(&[ + "باب", + "ب\u{0611}اب", + "ب\u{0610}اب", + "ب\u{0654}اب", + "ب\u{064B}اب", + "ب\u{064C}اب", + "ب\u{064D}اب", + "ب\u{064E}اب", + "ب\u{064F}اب", + "ب\u{0650}اب", + "ب\u{0652}اب", + "ب\u{0651}اب", + "ب\u{0670}اب", + "ب\u{065E}اب", + "ب\u{061A}اب", + ]); + // Quranic marks are skipped by letter order like harakat (hamza sorts before alef). + assert_eq!(compare_text("ب\u{0610}ء", "باب"), Ordering::Less); + assert_eq!(compare_text("ب\u{06E1}ء", "باب"), Ordering::Less); + } + + #[test] + fn excel_compares_marks_letter_by_letter() { + assert_sorts_to(&[ + "بت", + "بت\u{064E}", + "بت\u{064F}", + "بت\u{0650}", + "ب\u{064E}ت", + "ب\u{064E}ت\u{064E}", + "ب\u{064E}ت\u{064F}", + "ب\u{064F}ت", + "ب\u{064F}ت\u{064E}", + "ب\u{0650}ت\u{0650}", + ]); + assert_sorts_to(&["مـحـمـد", "محم\u{0651}د", "م\u{064E}حمد", "م\u{064F}ح\u{064E}م\u{064E}\u{0651}د", "محمدي"]); + } + + #[test] + fn excel_weighs_marks_and_digit_scripts_by_position() { + // A mark on an earlier letter outranks the digit script or letter after it, whatever the + // code point of that next character. + assert_sorts_to(&[ + "ب", + "ب\u{064E}", + "ب1", + "ب١", + "ب۱", + "ب\u{064E}1", + "ب\u{064E}١", + "ب\u{064E}۱", + "بa", + "ب\u{064E}a", + "با", + "ب\u{064E}ا", + "ب\u{067E}", + "ب\u{064E}\u{067E}", + "ب\u{06CC}", + "ب\u{064E}\u{06CC}", + ]); + // And an earlier digit script outranks a later mark. + assert_sorts_to(&["12", "1٢", "١2", "١٢", "1ب", "1ب\u{064E}", "1ب\u{0650}", "١ب", "١ب\u{064E}", "١ب\u{0650}", "۱ب", "۱ب\u{064E}"]); + assert_sorts_to(&["ب1ب", "ب1ب\u{064E}", "ب١ب", "ب\u{064E}ب", "بب1", "بب١", "ب\u{064E}ب1"]); + } + + #[test] + fn excel_skips_invisible_format_characters() { + for c in ['\u{00AD}', '\u{034F}', '\u{061C}', '\u{200C}', '\u{200D}', '\u{200E}', '\u{200F}', '\u{202A}', '\u{202B}'] + .into_iter() + .chain(['\u{202C}', '\u{202D}', '\u{202E}', '\u{2060}', '\u{2066}', '\u{2067}', '\u{2068}', '\u{2069}', '\u{FEFF}']) + { + let s = format!("{c}b"); + assert_eq!(compare_text(&s, "a"), Ordering::Greater, "{s:?}"); + assert_eq!(compare_text(&s, "ba"), Ordering::Less, "{s:?}"); + assert_eq!(compare_text(&format!("ب{c}\u{064E}اب"), "باب\u{064B}"), Ordering::Greater, "{c:?}"); + } + // Not even as a tie-break: a leading mark doesn't push text past marked or other-script forms. + assert_sorts_to(&["\u{200F}باب", "ب\u{064B}اب", "ب\u{064E}اب", "ب\u{0670}اب"]); + assert_sorts_to(&["\u{200F}12", "١٢", "۱۲"]); + assert_sorts_to(&["ب\u{200F}ب\u{064E}", "\u{200F}ب\u{064E}ب"]); + } + #[test] fn errors() { for e in CellError::ALL { diff --git a/crates/engine/src/catalog.rs b/crates/engine/src/catalog.rs index 234e92e58..41d1157f7 100644 --- a/crates/engine/src/catalog.rs +++ b/crates/engine/src/catalog.rs @@ -118,7 +118,7 @@ pub const CATALOG: &[(&str, &str, &str, &str)] = &[ ("insert.picture", "Insert", "Illustrations", "Pictures"), ("insert.shape", "Insert", "Illustrations", "Shapes"), ("insert.icons", "Insert", "Illustrations", "Icons"), - ("insert.smartArt", "Insert", "Illustrations", "SmartArt"), + ("insert.smartArt", "Insert", "Illustrations", "Diagram"), ("insert.screenshot", "Insert", "Illustrations", "Screenshot"), ("insert.recommendedCharts", "Insert", "Charts", "Recommended Charts"), ("insert.chart", "Insert", "Charts", "Charts"), @@ -131,7 +131,7 @@ pub const CATALOG: &[(&str, &str, &str, &str)] = &[ ("review.newComment", "Insert", "Comments", "Comment"), ("insert.textBox", "Insert", "Text", "Text Box"), ("pageLayout.headerFooter", "Insert", "Text", "Header & Footer"), - ("insert.wordArt", "Insert", "Text", "WordArt"), + ("insert.wordArt", "Insert", "Text", "Decorative Text"), ("insert.signatureLine", "Insert", "Text", "Signature Line"), ("insert.object", "Insert", "Text", "Object"), ("insert.equation", "Insert", "Symbols", "Equation"), @@ -340,3 +340,15 @@ pub fn parity() -> (usize, usize) { let done = ids.iter().filter(|id| crate::find_command(id).is_some()).count(); (done, ids.len()) } + +#[cfg(test)] +mod feature_name_tests { + #[test] + fn own_feature_labels_keep_command_ids() { + for (id, expected) in [("insert.smartArt", "Diagram"), ("insert.wordArt", "Decorative Text")] { + let entries: Vec<_> = super::CATALOG.iter().filter(|entry| entry.0 == id).collect(); + assert_eq!(entries.len(), 1, "{id} appears once"); + assert_eq!(entries[0].3, expected); + } + } +} diff --git a/crates/engine/src/cf.rs b/crates/engine/src/cf.rs index 38f5bb1fe..e5fe4dedc 100644 --- a/crates/engine/src/cf.rs +++ b/crates/engine/src/cf.rs @@ -1,7 +1,7 @@ //! Conditional formatting evaluation: what extra style, data bar, colour scale or icon a cell //! gets. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use gridcraft_core::{CellRef, RangeRef, Value, compare}; use gridcraft_model::{CfOperator, CfRule, CfValueKind, Color, CondFormat, PatternType, Sheet, Style, Underline, Workbook, default_icon_thresholds}; @@ -47,6 +47,8 @@ enum StatsNeed { fn stats_need(rule: &CfRule) -> Option { match rule { CfRule::Duplicate { .. } | CfRule::Unique { .. } => Some(StatsNeed::Counts), + // only two- and three-colour scales paint: no statistics for the others + CfRule::ColorScale { stops } if !matches!(stops.len(), 2 | 3) => None, CfRule::Top10 { .. } | CfRule::AboveAverage { .. } | CfRule::ColorScale { .. } | CfRule::DataBar { .. } | CfRule::IconSet { .. } => { Some(StatsNeed::Numbers) } @@ -70,15 +72,58 @@ thread_local! { static SCANNED: std::cell::Cell = const { std::cell::Cell::new(0) }; } +/// Most cells one rule's statistics visit, across all of its ranges together: a rule repeating +/// the same range many times must not multiply the scan. +const STATS_SCAN_LIMIT: usize = 2_000_000; + +/// Steps through the spilled cells inside `r`, row-major: `Some(cell)` for each, and `None` for a +/// jump past the spills left or right of the columns on a row (one step per row, however many +/// there are), so unrelated spills between a narrow range's corners cost little. +fn spill_steps(sh: &Sheet, r: RangeRef) -> impl Iterator> + '_ { + let mut cur = Some(r.start); + std::iter::from_fn(move || { + let at = cur.filter(|at| *at <= r.end)?; + let (&c, _) = sh.spill.range(at..=r.end).next()?; + if c.col < r.start.col { + cur = Some(CellRef::new(c.row, r.start.col)); + Some(None) + } else if c.col > r.end.col { + cur = c.row.checked_add(1).map(|row| CellRef::new(row, r.start.col)); + Some(None) + } else { + cur = c.col.checked_add(1).map(|col| CellRef::new(c.row, col)); + Some(Some(c)) + } + }) +} + fn numbers_in(sh: &Sheet, ranges: &[RangeRef], need: StatsNeed) -> (Vec, HashMap) { let mut nums = Vec::new(); let mut counts = HashMap::new(); + let mut budget = STATS_SCAN_LIMIT; + let mut seen: HashSet = HashSet::new(); + let used = sh.used_range(); for r in ranges { - let r = match sh.used_range().and_then(|u| r.intersection(&u)) { + if budget == 0 { + break; + } + let r = match used.and_then(|u| r.intersection(&u)) { Some(x) => x, None => continue, }; - for c in r.iter().take(2_000_000) { + // The same range listed twice counts its cells once. + if !seen.insert(r) { + continue; + } + // Only stored and spilled cells hold values: blank cells of a sparse range cost nothing, + // so a value far down a huge range is still counted. Every cell looked at is charged. + let stored = sh.cells.iter_range(r).map(|(c, _)| (Some(c), true)); + let spilled = spill_steps(sh, r).map(|c| (c, false)); + for (c, is_stored) in stored.chain(spilled).take(budget) { + budget -= 1; + // a skip over spills outside the columns (charged, not read), or a spilled value + // under a stored cell, which was read with it + let Some(c) = c.filter(|c| is_stored || sh.cells.get(*c).is_none()) else { continue }; #[cfg(test)] SCANNED.with(|n| n.set(n.get() + 1)); let v = sh.value(c); @@ -278,7 +323,8 @@ impl CfCache { _ => None, }, CfRule::ColorScale { stops } => { - if let Some(n) = v.as_f64() { + // Only two- and three-colour scales paint: evaluate no thresholds for others. + if let Some(n) = v.as_f64().filter(|_| matches!(stops.len(), 2 | 3)) { let pts: Vec<(f64, [u8; 3])> = stops .iter() .map(|(k, col)| (threshold(wb, sheet, c, st, k), col.resolve(&wb.theme).unwrap_or([255, 255, 255]))) @@ -368,6 +414,9 @@ impl CfCache { /// Adds the parts `lower` sets that `style` leaves unset (differential styles: default parts /// are unset). fn layer_under(style: &mut Style, lower: &Style) { + if style.align.reading_order.is_none() { + style.align.reading_order = lower.align.reading_order; + } if style.fill.pattern == PatternType::None { style.fill = lower.fill; } @@ -417,6 +466,28 @@ mod tests { use super::*; + #[test] + fn reading_order_respects_conditional_rule_priority_and_explicit_context() { + for (higher, expected) in [ + (json!({"font": {"bold": true}}), "RightToLeft"), + (json!({"align": {"reading_order": "Context"}}), "Context"), + (json!({"align": {"reading_order": "LeftToRight"}}), "LeftToRight"), + ] { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("cell.set", json!({"cell": "A1", "input": "مرحبا 123 ABC"})).unwrap(); + for style in [json!({"align": {"reading_order": "RightToLeft"}}), higher] { + s.execute("home.conditionalFormat", json!({"range": "A1", "rule": {"type": "noBlanks", "style": style}})).unwrap(); + } + let wb = &s.doc().unwrap().wb; + let (reopened, _) = gridcraft_xlsx::read_xlsx(&gridcraft_xlsx::write_xlsx(wb).unwrap()).unwrap(); + for wb in [wb, &reopened] { + let style = CfCache::new().look(wb, 0, CellRef::new(0, 0)).unwrap().style.unwrap(); + assert_eq!(serde_json::to_value(style.align).unwrap()["reading_order"], expected); + } + } + } + #[test] fn every_matching_rule_applies() { let mut s = crate::Session::new(); @@ -560,6 +631,89 @@ mod tests { assert_eq!(SCANNED.with(|n| n.get()), 10_000); } + /// A rule listing the same range many times scans it once, and never more than the shared + /// limit however many ranges it holds. + #[test] + fn repeated_ranges_share_one_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2], [3]]})).unwrap(); + s.execute("range.setValues", json!({"range": "XFD1048576", "values": [[4]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let dense = RangeRef::parse("A1:XFD1048576").unwrap(); + let small = RangeRef::parse("A1:A3").unwrap(); + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &vec![small; 1000], StatsNeed::Numbers); + assert_eq!(st.sorted, vec![1.0, 2.0, 3.0]); + assert_eq!(SCANNED.with(|n| n.get()), 3); + let shifted: Vec = (0..50).map(|i| RangeRef::new(CellRef::new(i, 0), dense.end)).collect(); + SCANNED.with(|n| n.set(0)); + RuleStats::build(sh, &shifted, StatsNeed::Numbers); + assert!(SCANNED.with(|n| n.get()) <= STATS_SCAN_LIMIT); + } + + /// Blank cells of a sparse range are not charged to the scan budget: a value at the end of two + /// million-row columns still counts, so duplicates are found. + #[test] + fn sparse_ranges_keep_their_values_under_the_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[7]]})).unwrap(); + s.execute("range.setValues", json!({"range": "B1048576", "values": [[7]]})).unwrap(); + let d = s.doc().unwrap(); + let sh = d.wb.sheet(0).unwrap(); + let ranges = [RangeRef::parse("A1:A1048576").unwrap(), RangeRef::parse("B1:B1048576").unwrap()]; + SCANNED.with(|n| n.set(0)); + let st = RuleStats::build(sh, &ranges, StatsNeed::Counts); + assert_eq!(st.counts.get("7"), Some(&2)); + assert_eq!(SCANNED.with(|n| n.get()), 2); + } + + /// Spills left and right of a narrow range, between its corners, are skipped a row at a time: + /// they do not use the budget that the in-range spills need. + #[test] + fn spills_outside_a_narrow_range_do_not_use_its_scan_budget() { + let mut s = crate::Session::new(); + s.new_workbook(); + let d = s.doc().unwrap(); + let mut sh = d.wb.sheet(0).unwrap().clone(); + for row in 0..1000 { + for col in [0u32, 5, 6, 7] { + sh.spill.insert(CellRef::new(row, col), Value::Number(1.0)); + } + } + let r = RangeRef::parse("F1:G1000").unwrap(); + let steps: Vec<_> = spill_steps(&sh, r).collect(); + assert_eq!(steps.iter().flatten().count(), 2000); + // column A is skipped one step a row, never one per cell + assert!(steps.len() <= 2000 + 1000, "{}", steps.len()); + let narrow = RangeRef::parse("C1:C1000").unwrap(); + assert_eq!(spill_steps(&sh, narrow).flatten().count(), 0); + } + + /// A colour scale with an unsupported number of stops paints nothing and evaluates none of them. + #[test] + fn colour_scales_with_other_stop_counts_paint_nothing() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [5]]})).unwrap(); + let mut wb: Workbook = (*s.doc().unwrap().wb).clone(); + let stop = (CfValueKind::Formula("=1".into()), Color::default()); + let rule = |n: usize| CondFormat { + ranges: vec![RangeRef::parse("A1:A2").unwrap()], + rule: CfRule::ColorScale { stops: vec![stop.clone(); n] }, + priority: 1, + stop_if_true: false, + }; + wb.sheet_mut(0).unwrap().cond_formats.push(rule(4)); + let mut cache = CfCache::new(); + assert_eq!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill), None); + wb.sheet_mut(0).unwrap().cond_formats = vec![rule(2)]; + let mut cache = CfCache::new(); + assert!(cache.look(&wb, 0, CellRef::parse("A1").unwrap()).and_then(|l| l.scale_fill).is_some()); + } + #[test] fn cell_value_operator_names() { let mut s = crate::Session::new(); diff --git a/crates/engine/src/chart_svg.rs b/crates/engine/src/chart_svg.rs index be9e6d227..f5f43041e 100644 --- a/crates/engine/src/chart_svg.rs +++ b/crates/engine/src/chart_svg.rs @@ -10,6 +10,10 @@ use crate::io::esc; /// Widest or tallest chart written; larger anchors are clamped. const MAX_SIDE: f32 = 16_384.0; +/// Most bytes of SVG written for the charts of one web page together. A chart past what is +/// left makes the export fail rather than grow without bound or be dropped without a word. +pub(crate) const MAX_PAGE_SVG_BYTES: usize = 64 * 1024 * 1024; + struct HelveticaMeasure; impl gridcraft_chart::Measure for HelveticaMeasure { @@ -18,24 +22,40 @@ impl gridcraft_chart::Measure for HelveticaMeasure { } } -/// `chart` of sheet `sheet` as an `` element placed by `style`, or `None` for a degenerate size. -pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str) -> Option { +/// `chart` of sheet `sheet` as an `` element placed by `style`, `None` for a degenerate size, +/// or an error for an element longer than `budget` bytes. +pub(crate) fn chart_svg(wb: &Workbook, sheet: usize, chart: &Chart, style: &str, budget: usize) -> crate::Result> { let (w, h) = (chart.anchor.width.min(MAX_SIDE), chart.anchor.height.min(MAX_SIDE)); if !(w >= 1.0 && h >= 1.0) { - return None; + return Ok(None); } let data = gridcraft_chart::resolve(wb, sheet, chart); let prims = gridcraft_chart::render(chart, &data, w, h, &HelveticaMeasure); - Some(prims_svg(&prims, w, h, style)) + match prims_svg_within(&prims, w, h, style, budget) { + Some(svg) => Ok(Some(svg)), + None => Err(crate::EngineError::Other(format!( + "the charts would make the web page over {} MiB. Save as .xlsx or PDF instead.", + MAX_PAGE_SVG_BYTES / (1024 * 1024) + ))), + } } +#[cfg(test)] fn prims_svg(prims: &[Prim], w: f32, h: f32, style: &str) -> String { + prims_svg_within(prims, w, h, style, usize::MAX).unwrap_or_default() +} + +/// The `` element, or `None` as soon as it grows past `budget` bytes. +fn prims_svg_within(prims: &[Prim], w: f32, h: f32, style: &str, budget: usize) -> Option { let mut out = String::new(); let _ = write!( out, "" ); for p in prims { + if out.len() > budget { + return None; + } match p { Prim::Rect { x, y, w, h, fill, stroke, radius } => { let _ = write!(out, " String { } } out.push_str(""); - out + (out.len() <= budget).then_some(out) } /// Fill and stroke attributes of an open shape element; the caller closes it. @@ -149,6 +169,17 @@ fn num(v: f32) -> String { mod tests { use super::*; + #[test] + fn an_svg_past_its_byte_budget_is_left_out() { + let rect = Prim::Rect { x: 1.0, y: 2.0, w: 3.0, h: 4.0, fill: Some([255, 0, 0, 255]), stroke: None, radius: 0.0 }; + let prims = vec![rect; 10_000]; + let full = prims_svg(&prims, 100.0, 50.0, ""); + assert!(full.len() > 100_000); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len()), Some(full.clone())); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", full.len() - 1), None); + assert_eq!(prims_svg_within(&prims, 100.0, 50.0, "", 1_000), None); + } + #[test] fn every_primitive_becomes_an_svg_element() { let red = [255, 0, 0, 255]; diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index da130fe2a..0f79495fc 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -366,6 +366,10 @@ pub struct Ctx<'a> { pub wb: Workbook, pub changed: Vec, pub structural: bool, + /// Recalculate every formula without the other effects of `structural` (table headers are not + /// resynchronized, array formulas are still checked): for edits that change only what a + /// function reads, like sheet protection. + pub recalc_all: bool, pub sel: &'a mut Selection, /// Rows whose automatic height should be recomputed (sheet, row). pub fit_rows: Vec<(usize, u32)>, @@ -400,13 +404,14 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) wb: (*d.wb).clone(), changed: Vec::new(), structural: false, + recalc_all: false, sel: &mut sel, fit_rows: Vec::new(), protection_checked: false, edits: Vec::new(), }; let r = f(&mut ctx)?; - let Ctx { mut wb, mut changed, structural, mut fit_rows, protection_checked, edits, .. } = ctx; + let Ctx { mut wb, mut changed, structural, recalc_all, mut fit_rows, protection_checked, edits, .. } = ctx; // Equations whose source changed (or that arrived without a layout, as from a paste) are // laid out here, so painting only ever reads the cache. wb.refresh_equations(); @@ -435,7 +440,7 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) if let Some(t0) = t0 { eprintln!("commit: edit {:?}, {} changed", since(t0), changed.len()); } - if structural { + if structural || recalc_all { d.calc.recalc_all(&mut wb); } else if !changed.is_empty() { d.calc.cells_changed(&mut wb, &changed); @@ -446,8 +451,18 @@ pub(crate) fn commit(d: &mut DocState, f: impl FnOnce(&mut Ctx) -> Result) } fit_rows.sort_unstable(); fit_rows.dedup(); + let mut sweep: Option<(usize, MergeSweep)> = None; + let mut left = MERGE_FIT_WORK; for (si, row) in fit_rows { - auto_row_height(&mut wb, si, row); + if sweep.as_ref().is_none_or(|(at, _)| *at != si) { + if let Some((_, old)) = &sweep { + left = old.left(); + } + sweep = Some((si, MergeSweep::new(left))); + } + if let Some((_, sw)) = sweep.as_mut() { + fit_row_height(&mut wb, si, row, sw); + } } if let Some(t0) = t0 { eprintln!("commit: + rows {:?}", since(t0)); @@ -721,6 +736,94 @@ fn wrapped_line_count(line: &str, width: f32, size: f32, bold: bool) -> f32 { /// Line breaks in a cell's text count only when the cell wraps, as in Excel: with Wrap Text off /// a multi-line value shows on one line and AutoFit gives a one-line row. pub fn auto_row_height(wb: &mut Workbook, si: usize, row: u32) { + let mut sweep = MergeSweep::new(MERGE_FIT_WORK); + fit_row_height(wb, si, row, &mut sweep); +} + +/// Most merge comparisons one auto-fit pass (a commit, or opening a workbook) makes. Past it the +/// remaining rows keep the heights they have rather than stall on a sheet whose merges overlap +/// the fitted rows many times over. +pub(crate) const MERGE_FIT_WORK: usize = 50_000_000; + +/// The merged areas of one sheet that touch a row, for rows visited in ascending order: the +/// merges are sorted by first row once and swept, so fitting many rows never rescans every merge +/// per row. All comparisons count against [`MERGE_FIT_WORK`]. +pub(crate) struct MergeSweep { + /// Built on the first row that needs merge checks, so rows that return early never pay for it. + by_start: Option>, + next: usize, + active: Vec, + budget: usize, + /// The first row was answered by a plain scan: a pass that fits one row never pays for a sort. + scanned: bool, +} + +impl MergeSweep { + /// A sweep for one sheet's merges with `budget` comparisons left for the pass (carry + /// [`MergeSweep::left`] of the previous sheet's sweep into the next). Every call to + /// [`MergeSweep::at`] must pass that same sheet. + pub(crate) fn new(budget: usize) -> Self { + Self { by_start: None, next: 0, active: Vec::new(), budget, scanned: false } + } + + /// Comparisons left for the rest of the pass. + pub(crate) fn left(&self) -> usize { + self.budget + } + + /// Takes `n` comparisons from the budget; `false` once it is spent. + fn charge(&mut self, n: usize) -> bool { + match self.budget.checked_sub(n) { + Some(rest) => { + self.budget = rest; + true + } + None => { + self.budget = 0; + false + } + } + } + + /// The merges covering `row`, or `None` when the budget is spent. Rows must not decrease + /// between calls (skipping rows is fine). + fn at(&mut self, sh: &gridcraft_model::Sheet, row: u32) -> Option> { + if self.by_start.is_none() && !self.scanned { + self.scanned = true; + if !self.charge(sh.merges.len()) { + return None; + } + return Some(sh.merges.iter().copied().filter(|m| !m.is_single() && m.start.row <= row && row <= m.end.row).collect()); + } + if self.by_start.is_none() { + // Sorting is about n log2 n comparisons, charged before it starts. + let n = sh.merges.len(); + if !self.charge(n.saturating_mul((usize::BITS - n.leading_zeros()) as usize)) { + return None; + } + let mut v: Vec = sh.merges.iter().copied().filter(|m| !m.is_single()).collect(); + v.sort_unstable_by_key(|m| m.start.row); + self.by_start = Some(v); + } + let by_start = self.by_start.as_ref()?; + while let Some(m) = by_start.get(self.next).filter(|m| m.start.row <= row) { + self.active.push(*m); + self.next += 1; + } + if !self.charge(self.active.len()) { + return None; + } + self.active.retain(|m| m.end.row >= row); + Some(self.active.clone()) + } +} + +/// [`auto_row_height`] with the sheet's merges taken from `sweep`. +pub(crate) fn fit_row_height(wb: &mut Workbook, si: usize, row: u32, sweep: &mut MergeSweep) { + // Out of budget: the row keeps the height it has. + if sweep.left() == 0 { + return; + } let Some(sh) = wb.sheet(si) else { return }; if sh.rows.get(&row).is_some_and(|i| i.custom) { return; @@ -750,9 +853,14 @@ pub fn auto_row_height(wb: &mut Workbook, si: usize, row: u32) { // picked out once, and per cell the candidates are re-checked (normally none or a couple; // a sheet with many merges and many saved columns would otherwise scan all of them per // cell, rows × merges × columns). - let row_merges: Vec = - sh.merges.iter().copied().filter(|m| !m.is_single() && m.start.row <= row && row <= m.end.row).collect(); + // Rows are fitted in ascending order, so the merges touching this one come from a sweep over + // the sheet's merges sorted once, not from a scan of all of them. Out of budget: the row keeps + // the height it has. + let Some(row_merges) = sweep.at(sh, row) else { return }; for (c, cell) in sh.cells.row(row, 0, gridcraft_core::MAX_COLS - 1) { + if !sweep.charge(row_merges.len()) { + return; + } // Merged cells never auto-fit their rows, like Excel (column autofit skips them too). if row_merges.iter().any(|m| m.contains(CellRef::new(row, c))) { continue; @@ -796,3 +904,40 @@ impl Ctx<'_> { self.changed.push((sheet, c)); } } + +#[cfg(test)] +mod merge_sweep_tests { + use super::*; + + #[test] + fn the_budget_carries_from_one_sheets_sweep_to_the_next() { + let mut sheet = gridcraft_model::Sheet::new("S"); + for r in ["A1:B2", "C1:D2", "E1:F2"] { + sheet.merges.push(gridcraft_core::RangeRef::parse(r).unwrap()); + } + let mut first = MergeSweep::new(5); + assert!(first.at(&sheet, 0).is_some(), "the first row is a plain scan: three comparisons"); + assert!(first.at(&sheet, 1).is_none(), "the second row needs the sort: six comparisons, two are left"); + assert_eq!(first.left(), 0); + // The next sheet starts with what is left: nothing, so it fits nothing. + let mut second = MergeSweep::new(first.left()); + assert!(second.at(&sheet, 0).is_none()); + assert!(MergeSweep::new(MERGE_FIT_WORK).at(&sheet, 0).is_some()); + } + + #[test] + fn sorting_the_merges_is_charged() { + let mut sheet = gridcraft_model::Sheet::new("S"); + for r in ["A1:B2", "C1:D2", "E1:F2", "G1:H2"] { + sheet.merges.push(gridcraft_core::RangeRef::parse(r).unwrap()); + } + // The first row costs a plain scan (4); from the second on the merges are sorted, about + // 4 * 3 comparisons: 7 left is not enough, 12 plus the row's 4 is. + let mut sweep = MergeSweep::new(11); + assert!(sweep.at(&sheet, 0).is_some()); + assert!(sweep.at(&sheet, 1).is_none()); + assert_eq!(sweep.left(), 0); + let mut sweep = MergeSweep::new(20); + assert!(sweep.at(&sheet, 0).is_some() && sweep.at(&sheet, 1).is_some()); + } +} diff --git a/crates/engine/src/cmd/analysis.rs b/crates/engine/src/cmd/analysis.rs index 9cd10e50f..92699b75f 100644 --- a/crates/engine/src/cmd/analysis.rs +++ b/crates/engine/src/cmd/analysis.rs @@ -152,26 +152,32 @@ fn input_range(s: &Session, p: &Json, keys: &[&str]) -> Result<(usize, RangeRef) /// Splits a range into variables (columns, or rows when `by_rows`); with `labels` the first /// cell of each names it, else `default(i)` does. fn variables(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String) -> Result> { - let d = s.doc()?; - let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; - let (outer, inner) = if by_rows { (r.height(), r.width()) } else { (r.width(), r.height()) }; + let outer = if by_rows { r.height() } else { r.width() }; let mut out = Vec::with_capacity(outer as usize); for i in 0..outer { - let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; - let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); - let mut name = default(i as usize); - if labels && !vals.is_empty() { - let first = vals.remove(0); - let t = first.display(); - if !t.trim().is_empty() { - name = t; - } - } - out.push(Var { name, vals }); + out.push(variable(s, si, r, by_rows, labels, default, i)?); } Ok(out) } +/// Variable `i` of [`variables`], read alone so a caller that may stop early needn't build them all. +fn variable(s: &Session, si: usize, r: RangeRef, by_rows: bool, labels: bool, default: &dyn Fn(usize) -> String, i: u32) -> Result { + let d = s.doc()?; + let sh = d.wb.sheet(si).ok_or(EngineError::NoDocument)?; + let inner = if by_rows { r.width() } else { r.height() }; + let cell = |j: u32| if by_rows { CellRef::new(r.start.row + i, r.start.col + j) } else { CellRef::new(r.start.row + j, r.start.col + i) }; + let mut vals: Vec = (0..inner).map(|j| sh.value(cell(j))).collect(); + let mut name = default(i as usize); + if labels && !vals.is_empty() { + let first = vals.remove(0); + let t = first.display(); + if !t.trim().is_empty() { + name = t; + } + } + Ok(Var { name, vals }) +} + /// The numbers of a variable; blanks are skipped when `skip_blank`, anything else that isn't a /// number is an error. fn numbers(v: &[Value], skip_blank: bool) -> Result> { @@ -272,11 +278,25 @@ struct Grid { cells: BTreeMap<(u32, u32), Value>, /// Cells shown as percentages. pct: Vec<(u32, u32)>, + /// Rows and columns spanned by `cells`. + rows: u32, + cols: u32, + /// A cell was dropped because the report outgrew [`MAX_OUTPUT_CELLS`]: the report is refused. + too_large: bool, } impl Grid { + /// Stores a value. Past [`MAX_OUTPUT_CELLS`] (the most a report writes) nothing more is kept, + /// so a builder that would produce millions of entries stops allocating them, and + /// [`write_report`] refuses the report. fn put(&mut self, r: u32, c: u32, v: Value) { if !v.is_empty() { + let (rows, cols) = (self.rows.max(r.saturating_add(1)), self.cols.max(c.saturating_add(1))); + if self.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + self.too_large = true; + return; + } + (self.rows, self.cols) = (rows, cols); self.cells.insert((r, c), v); } } @@ -296,9 +316,7 @@ impl Grid { } /// (rows, cols) spanned. fn extent(&self) -> (u32, u32) { - let rows = self.cells.keys().map(|(r, _)| r + 1).max().unwrap_or(0); - let cols = self.cells.keys().map(|(_, c)| c + 1).max().unwrap_or(0); - (rows, cols) + (self.rows, self.cols) } } @@ -310,7 +328,7 @@ fn idx(i: usize) -> u32 { fn descriptive(s: &Session, p: &Json) -> Result { let (si, r) = input_range(s, p, &["range", "input"])?; - let vars = variables(s, si, r, by_rows(p), flag(p, "labels"), &|i| format!("Column{}", i + 1))?; + let (by_rows, labels) = (by_rows(p), flag(p, "labels")); let summary = p.get("summary").map(|_| flag(p, "summary")).unwrap_or(true); let conf = confidence(p, "confidence")?; let kl = num(p, "kthLargest").filter(|k| *k >= 1.0).map(|k| k as usize); @@ -319,7 +337,14 @@ fn descriptive(s: &Session, p: &Json) -> Result { return Err(bad(CMD, "choose at least one of summary, confidence, kthLargest or kthSmallest")); } let mut g = Grid::default(); - for (i, v) in vars.iter().enumerate() { + // Variables are read one at a time: a million-row range grouped by rows is a million variables. + for i in 0..(if by_rows { r.height() } else { r.width() }) { + // The report is refused once the grid is too large: the rest would only build discarded labels. + if g.too_large { + break; + } + let v = variable(s, si, r, by_rows, labels, &|i| format!("Column{}", i + 1), i)?; + let i = i as usize; let x = numbers(&v.vals, true)?; let c = idx(i).saturating_mul(2); g.txt(0, c, &v.name); @@ -1749,7 +1774,7 @@ fn sampling(s: &Session, p: &Json, seed: Option) -> Result { fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result { let (rows, cols) = g.extent(); let (rows, cols) = (rows.max(1), cols.max(1)); - if u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { + if g.too_large || u64::from(rows) * u64::from(cols) > MAX_OUTPUT_CELLS { return Err(bad(CMD, "the report is too large")); } let d = s.doc()?; @@ -1914,6 +1939,28 @@ mod tests { x } + #[test] + fn a_report_stops_growing_past_the_output_limit() { + // Entries past the limit are not kept, and the report is refused. + let mut g = super::Grid::default(); + g.num(1999, 1999, 1.0); + assert!(!g.too_large); + g.num(0, 2000, 1.0); + assert!(g.too_large); + assert_eq!(g.cells.len(), 1); + assert_eq!(g.extent(), (2000, 2000)); + // Row-grouped statistics of a tall, almost empty column: one variable per row would make + // millions of report entries; the report is refused instead of built. + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1]]})).unwrap(); + s.execute("range.setValues", json!({"range": "A1000000", "values": [[2]]})).unwrap(); + let err = s + .execute("data.analysisToolPak", json!({"tool": "descriptive", "range": "A1:A1000000", "groupedBy": "rows", "output": "C1"})) + .expect_err("report too large"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(v(&s, "C1"), Value::Empty); + } + #[test] fn forecast_standard_errors_match_the_full_window_rescan() { // #1043: the running sum must give what re-adding every window gave, gaps included. diff --git a/crates/engine/src/cmd/data/filter.rs b/crates/engine/src/cmd/data/filter.rs index 437bd7887..cda2160bf 100644 --- a/crates/engine/src/cmd/data/filter.rs +++ b/crates/engine/src/cmd/data/filter.rs @@ -317,5 +317,9 @@ pub fn filter_values(wb: &Workbook, sheet: usize, col: u32) -> Vec<(String, bool let key = if blank { sort_key(&Value::Empty) } else { sort_key(&v) }; seen.entry((key, t)).or_insert(on); } - seen.into_iter().map(|((_, t), on)| (t, on)).collect() + let mut values: Vec<_> = seen.into_iter().collect(); + values.sort_by(|((a, ta), _), ((b, tb), _)| { + a.0.cmp(&b.0).then(a.1.cmp(&b.1)).then_with(|| gridcraft_core::compare_text(&a.2, &b.2)).then_with(|| ta.cmp(tb)) + }); + values.into_iter().map(|((_, t), on)| (t, on)).collect() } diff --git a/crates/engine/src/cmd/data/tests.rs b/crates/engine/src/cmd/data/tests.rs index 3c2782444..e073e23a5 100644 --- a/crates/engine/src/cmd/data/tests.rs +++ b/crates/engine/src/cmd/data/tests.rs @@ -53,6 +53,17 @@ fn subtotal_puts_summary_rows_below_their_detail() { assert_eq!(sh.value(CellRef::new(3, 0)).display(), "West"); } +#[test] +fn filter_list_orders_arabic_like_sort() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["H"], ["بج"], ["بُت"], ["\u{200F}بب"]]})).unwrap(); + s.execute("data.filter", json!({"range": "A1:A4"})).unwrap(); + let doc = s.doc().unwrap(); + let list: Vec = filter_values(&doc.wb, 0, 0).into_iter().map(|(t, _)| t).collect(); + assert_eq!(list, ["\u{200F}بب", "بُت", "بج"]); +} + #[test] fn filter_by_huge_numeric_column_is_rejected_not_overflowed() { let mut s = Session::new(); diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index ddb9ecaf7..3efcdaaf4 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -14,8 +14,13 @@ fn matches(hay: &str, what: &str, case: bool, whole: bool) -> bool { } /// `*`/`?` wildcard match with `~` escapes. +/// +/// The pattern runs as a bit-parallel automaton over the text: one pass, each character costing +/// one step per 64 pattern positions. Backtracking to the last `*` after a mismatch rescans the +/// text and costs text × pattern comparisons on a long, repetitive cell, for every cell a +/// filter or Find visits. pub(crate) fn wildcard(text: &str, pat: &str) -> bool { - let t: Vec = text.chars().collect(); + // The pattern as (character, escaped) items, runs of `*` folded into one. let mut p: Vec<(char, bool)> = Vec::new(); let mut it = pat.chars(); while let Some(c) = it.next() { @@ -23,32 +28,85 @@ pub(crate) fn wildcard(text: &str, pat: &str) -> bool { if let Some(n) = it.next() { p.push((n, true)); } - } else { + } else if c != '*' || p.last() != Some(&('*', false)) { p.push((c, false)); } } - let (mut ti, mut pi) = (0usize, 0usize); - let (mut star, mut mark) = (None, 0usize); - while ti < t.len() { - if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { - ti += 1; - pi += 1; - } else if pi < p.len() && p[pi] == ('*', false) { - star = Some(pi); - mark = ti; - pi += 1; - } else if let Some(sp) = star { - pi = sp + 1; - mark += 1; - ti = mark; - } else { - return false; + let words = (p.len() + 1).div_ceil(64); + let bit = |set: &mut [u64], i: usize| set[i / 64] |= 1 << (i % 64); + // Positions holding a `*` and a `?`, and where each literal character sits in the pattern. + let mut stars = vec![0u64; words]; + let mut any = vec![0u64; words]; + let mut at: std::collections::HashMap> = std::collections::HashMap::new(); + for (i, item) in p.iter().enumerate() { + match item { + ('*', false) => bit(&mut stars, i), + ('?', false) => bit(&mut any, i), + (c, _) => at.entry(*c).or_default().push(i), } } - while pi < p.len() && p[pi] == ('*', false) { - pi += 1; + // A literal's bit mask is built when the text first reads it, and kept for at most + // `MAX_CACHED_MASKS` literals: a long pattern of mostly distinct characters would otherwise + // allocate a mask of the pattern's width for each one up front. When there are more literals + // than that, the ones kept are those at the most pattern positions (the dearest to rebuild); + // each of the rest has at most pattern length / `MAX_CACHED_MASKS` positions, so rebuilding + // its mask in `scratch` costs less than one step of the scan. + const MAX_CACHED_MASKS: usize = 256; + let cacheable: Option> = (at.len() > MAX_CACHED_MASKS).then(|| { + let mut by_count: Vec<(&char, usize)> = at.iter().map(|(c, v)| (c, v.len())).collect(); + by_count.sort_unstable_by_key(|&(_, n)| std::cmp::Reverse(n)); + by_count.into_iter().take(MAX_CACHED_MASKS).map(|(c, _)| *c).collect() + }); + let mut masks: std::collections::HashMap> = std::collections::HashMap::new(); + let mut scratch = vec![0u64; words]; + // A set bit i: the text read so far matches the first i pattern items. A `*` matches nothing + // as well, so a state at a `*` is also at the item after it. + let close = |set: &mut Vec| { + let mut carry = 0u64; + for w in 0..words { + let s = set[w] & stars[w]; + let next = (s << 1) | carry; + carry = s >> 63; + set[w] |= next; + } + }; + let mut state = vec![0u64; words]; + bit(&mut state, 0); + close(&mut state); + let mut next = vec![0u64; words]; + for ch in text.chars() { + let lit: Option<&[u64]> = match at.get(&ch) { + None => None, + Some(positions) if cacheable.as_ref().is_none_or(|k| k.contains(&ch)) => { + let mask = masks.entry(ch).or_insert_with(|| { + let mut m = vec![0u64; words]; + positions.iter().for_each(|&i| bit(&mut m, i)); + m + }); + Some(mask) + } + Some(positions) => { + scratch.fill(0); + positions.iter().for_each(|&i| bit(&mut scratch, i)); + Some(&scratch) + } + }; + let mut carry = 0u64; + let mut alive = 0u64; + for w in 0..words { + // Items that read `ch` advance; a `*` reads it and stays. + let m = state[w] & (any[w] | lit.map_or(0, |l| l[w])); + next[w] = (m << 1) | carry | (state[w] & stars[w]); + carry = m >> 63; + alive |= next[w]; + } + if alive == 0 { + return false; + } + std::mem::swap(&mut state, &mut next); + close(&mut state); } - pi == p.len() + state[p.len() / 64] & (1 << (p.len() % 64)) != 0 } pub(super) fn find(s: &mut Session, p: &Json) -> Result { @@ -112,9 +170,19 @@ pub(super) fn find(s: &mut Session, p: &Json) -> Result { Ok(json!({"cell": c.a1(), "sheet": si, "value": t, "count": hits.len()})) } +/// Most characters a cell holds (Excel's limit). +const REPLACE_CELL_MAX_CHARS: usize = 32_767; +/// Longest text one replacement may give a cell, in bytes: the character limit, each up to four +/// bytes. Every occurrence copies the replacement text, so a short cell full of matches and a +/// long replacement would otherwise expand to gigabytes. The character limit is checked on what +/// fits in this. +const REPLACE_CELL_MAX_BYTES: usize = REPLACE_CELL_MAX_CHARS * 4; +/// Most bytes of replaced text one Replace All builds across all its cells. +const REPLACE_TOTAL_MAX_BYTES: usize = 256 * 1024 * 1024; + pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let what = str_param(p, "what").unwrap_or("").to_string(); - let with = str_param(p, "with").unwrap_or("").to_string(); + let with = str_param(p, "with").unwrap_or(""); if what.is_empty() { return Err(bad("edit.replace", "missing `what`")); } @@ -128,6 +196,7 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let n = edit(s, |cx| { let sheets: Vec = if workbook { (0..cx.wb.sheets.len()).collect() } else { vec![cx.wb.active_sheet()] }; let mut n = 0; + let mut total = 0usize; for si in sheets { let cells: Vec<(CellRef, String)> = cx.wb.sheet(si).map(|sh| sh.cells.iter().map(|(c, _)| (c, sh.input_text(c))).collect()).unwrap_or_default(); @@ -139,12 +208,20 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { continue; } let new = if whole { - with.clone() + (with.len() <= REPLACE_CELL_MAX_BYTES).then(|| with.to_string()) } else if case { - text.replace(&what, &with) + replace_cs_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) } else { - replace_ci(&text, &what, &with) + replace_ci_within(&text, &what, with, REPLACE_CELL_MAX_BYTES) }; + let new = new.filter(|n| n.chars().count() <= REPLACE_CELL_MAX_CHARS); + let Some(new) = new else { + return Err(EngineError::Other(format!("The replacement makes the text of {} too long.", c.a1()))); + }; + total = total.saturating_add(new.len()); + if total > REPLACE_TOTAL_MAX_BYTES { + return Err(EngineError::Other("The replacements make too much text. Replace in a smaller range.".into())); + } // A Python cell's text is its code: the replacement edits the code and the cell // stays a Python cell. Code that would no longer fit in a PY formula is left as it was. let old_py = cx.wb.sheet(si).and_then(|sh| sh.cell(c)).and_then(|x| x.py.clone()); @@ -199,7 +276,37 @@ fn folds_in_place(s: &str) -> bool { }) } +#[cfg(test)] pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { + replace_ci_within(text, what, with, usize::MAX).unwrap_or_default() +} + +/// `text` with every `what` replaced by `with`, or `None` once the result passes `limit` bytes +/// (checked before each copy, so nothing larger is allocated). +pub(super) fn replace_cs_within(text: &str, what: &str, with: &str, limit: usize) -> Option { + if what.is_empty() { + return (text.len() <= limit).then(|| text.to_string()); + } + let mut out = String::new(); + let mut i = 0; + for (pos, m) in text.match_indices(what) { + if out.len() + (pos - i) + with.len() > limit { + return None; + } + out.push_str(text.get(i..pos).unwrap_or("")); + out.push_str(with); + i = pos + m.len(); + } + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) +} + +/// Case-insensitive [`replace_cs_within`]. +pub(super) fn replace_ci_within(text: &str, what: &str, with: &str, limit: usize) -> Option { let lower = text.to_lowercase(); let w = what.to_lowercase(); // Slicing `text` at positions found in `lower` is only sound when folding kept every byte @@ -207,15 +314,22 @@ pub(super) fn replace_ci(text: &str, what: &str, with: &str) -> String { // boundaries (U+212A KELVIN SIGN shrinks 3→1 bytes, U+0130 grows 2→3), where the tail used // to be dropped. Those fall back to the case-sensitive replace, as length changes already do. if lower.len() != text.len() || w.is_empty() || !folds_in_place(text) || !folds_in_place(what) { - return text.replace(what, with); + return replace_cs_within(text, what, with, limit); } let mut out = String::new(); let mut i = 0; while let Some(pos) = lower.get(i..).and_then(|s| s.find(&w)) { + if out.len() + pos + with.len() > limit { + return None; + } out.push_str(text.get(i..i + pos).unwrap_or("")); out.push_str(with); i += pos + w.len(); } - out.push_str(text.get(i..).unwrap_or("")); - out + let rest = text.get(i..).unwrap_or(""); + if out.len() + rest.len() > limit { + return None; + } + out.push_str(rest); + Some(out) } diff --git a/crates/engine/src/cmd/edit/go_to.rs b/crates/engine/src/cmd/edit/go_to.rs index 0cd7a3de9..3c632e3b9 100644 --- a/crates/engine/src/cmd/edit/go_to.rs +++ b/crates/engine/src/cmd/edit/go_to.rs @@ -17,11 +17,13 @@ pub(in super::super) fn go_to(s: &mut Session, p: &Json) -> Result { } let Some(reference) = str_param(p, "reference").or_else(|| str_param(p, "range")) else { return Err(bad("edit.goTo", "missing `reference`")) }; let d = s.doc()?; + let mut names = Resolver::new(&d.wb); let mut target: Option = None; let mut areas: Vec = Vec::new(); let mut named = false; for part in reference_parts(reference) { - let Some((sheet, found, by_name)) = go_to_target(&d.wb, d.wb.active_sheet(), part, 0) else { + let mut found = Vec::new(); + let Some((sheet, _, by_name)) = names.target(d.wb.active_sheet(), part, 0, Some(&mut found)) else { return Err(EngineError::Other("Reference isn't valid.".into())); }; // One selection holds the areas of one sheet. @@ -77,57 +79,114 @@ pub(super) fn reference_parts(text: &str) -> impl Iterator { }) } -/// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, its areas, -/// and whether a defined name or a table gave them. `Sheet2!Name` is the name scoped to Sheet2. -/// A name counts when what it refers to is a reference; a constant or a formula is not one. -fn go_to_target(wb: &gridcraft_model::Workbook, sheet: usize, part: &str, depth: usize) -> Option<(usize, Vec, bool)> { - if part.is_empty() || depth > GO_TO_NAME_DEPTH { - return None; - } - let (prefix, body) = split_sheet(part); - let on = match &prefix { - Some(n) => wb.sheet_index(n)?, - None => sheet, - }; - if let Some(r) = RangeRef::parse(&body.replace('$', "")) { - return Some((on, vec![r], false)); - } - let name = match prefix { - Some(_) => wb.names.iter().find(|n| n.scope == Some(on) && gridcraft_formula::same_name(&n.name, body)), - None => wb.name(body, sheet), - }; - if let Some(n) = name { - return name_areas(wb, n, sheet, depth).map(|(t, areas)| (t, areas, true)); - } - if prefix.is_none() - && let Some((si, ti)) = wb.table(body) - { - // As in Excel, a table's name stands for its data, without the header and totals rows. - let t = wb.sheet(si)?.tables.get(ti)?; - return Some((si, vec![t.data_range().unwrap_or(t.range)], true)); - } - None +/// Resolves Go To references against one workbook. Defined names are found through an index +/// built once, and whether a name refers to a reference is remembered: listing the names of a +/// workbook full of names that alias other names would otherwise scan every name for every alias +/// of every name. +struct Resolver<'a> { + wb: &'a gridcraft_model::Workbook, + /// Indexes into `wb.names` by scope and folded name, in workbook order. + by_name: std::collections::HashMap<(Option, String), Vec>, + /// [`Resolver::name_areas`] without areas, by name index, sheet and depth: the target sheet + /// and the number of areas. + memo: std::collections::HashMap<(usize, usize, usize), Option<(usize, usize)>>, } -/// The sheet and areas defined name `n` refers to, seen from sheet `sheet`, when that is a -/// reference. -fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, sheet: usize, depth: usize) -> Option<(usize, Vec)> { - let from = n.scope.unwrap_or(sheet); - let formula = n.formula.trim().trim_start_matches('='); - let mut target: Option = None; - let mut areas = Vec::new(); - for p in reference_parts(formula) { - let (s, found, _) = go_to_target(wb, from, p, depth + 1)?; - if target.is_some_and(|t| t != s) { - return None; - } - target = Some(s); - areas.extend(found); - if areas.len() > GO_TO_MAX_AREAS { - return None; +impl<'a> Resolver<'a> { + fn new(wb: &'a gridcraft_model::Workbook) -> Self { + let mut by_name: std::collections::HashMap<(Option, String), Vec> = std::collections::HashMap::new(); + for (i, n) in wb.names.iter().enumerate() { + by_name.entry((n.scope, gridcraft_formula::fold_name(&n.name))).or_default().push(i); } + Self { wb, by_name, memo: std::collections::HashMap::new() } + } + + /// The first name of scope `scope` that `same` accepts, as a scan of `wb.names` finds it. + /// Names `same` accepts fold alike, so only that bucket is checked. + fn find(&self, scope: Option, body: &str, same: impl Fn(&str, &str) -> bool) -> Option { + self.by_name + .get(&(scope, gridcraft_formula::fold_name(body)))? + .iter() + .copied() + .find(|&i| self.wb.names.get(i).is_some_and(|n| same(&n.name, body))) + } + + /// Where one part of a Go To reference points, seen from sheet `sheet`: the sheet, how many + /// areas (appended to `areas` when given), and whether a defined name or a table gave them. + /// `Sheet2!Name` is the name scoped to Sheet2. A name counts when what it refers to is a + /// reference; a constant or a formula is not one. + fn target(&mut self, sheet: usize, part: &str, depth: usize, areas: Option<&mut Vec>) -> Option<(usize, usize, bool)> { + let wb = self.wb; + if part.is_empty() || depth > GO_TO_NAME_DEPTH { + return None; + } + let (prefix, body) = split_sheet(part); + let on = match &prefix { + Some(n) => wb.sheet_index(n)?, + None => sheet, + }; + if let Some(r) = RangeRef::parse(&body.replace('$', "")) { + if let Some(a) = areas { + a.push(r); + } + return Some((on, 1, false)); + } + // As `Workbook::name`: the sheet's own name, else the workbook's. + let name = match prefix { + Some(_) => self.find(Some(on), body, gridcraft_formula::same_name), + None => self.find(Some(sheet), body, str::eq_ignore_ascii_case).or_else(|| self.find(None, body, str::eq_ignore_ascii_case)), + }; + if let Some(i) = name { + return self.name_areas(i, sheet, depth, areas).map(|(t, n)| (t, n, true)); + } + if prefix.is_none() + && let Some((si, ti)) = wb.table(body) + { + // As in Excel, a table's name stands for its data, without the header and totals rows. + let t = wb.sheet(si)?.tables.get(ti)?; + if let Some(a) = areas { + a.push(t.data_range().unwrap_or(t.range)); + } + return Some((si, 1, true)); + } + None + } + + /// The sheet and number of areas defined name `wb.names[i]` refers to, seen from sheet + /// `sheet`, when that is a reference; the areas are appended to `areas` when given. + fn name_areas(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let key = (i, sheet, depth); + if areas.is_none() + && let Some(known) = self.memo.get(&key) + { + return *known; + } + let out = self.name_areas_uncached(i, sheet, depth, areas.as_deref_mut()); + if areas.is_none() { + self.memo.insert(key, out); + } + out + } + + fn name_areas_uncached(&mut self, i: usize, sheet: usize, depth: usize, mut areas: Option<&mut Vec>) -> Option<(usize, usize)> { + let n = self.wb.names.get(i)?; + let from = n.scope.unwrap_or(sheet); + let formula = n.formula.trim().trim_start_matches('='); + let mut target: Option = None; + let mut count = 0usize; + for p in reference_parts(formula) { + let (s, found, _) = self.target(from, p, depth + 1, areas.as_deref_mut())?; + if target.is_some_and(|t| t != s) { + return None; + } + target = Some(s); + count += found; + if count > GO_TO_MAX_AREAS { + return None; + } + } + target.map(|t| (t, count)) } - target.map(|t| (t, areas)) } /// The names the Go To dialog lists, seen from sheet `sheet`: the named ranges of the workbook and @@ -137,11 +196,13 @@ fn name_areas(wb: &gridcraft_model::Workbook, n: &gridcraft_model::DefinedName, pub fn go_to_names(wb: &gridcraft_model::Workbook, sheet: usize) -> Vec { let own: std::collections::HashSet = wb.names.iter().filter(|n| n.scope == Some(sheet)).map(|n| gridcraft_formula::fold_name(&n.name)).collect(); + let mut names = Resolver::new(wb); let mut out: Vec = wb .names .iter() - .filter(|n| !n.hidden && name_areas(wb, n, sheet, 0).is_some()) - .filter_map(|n| match n.scope { + .enumerate() + .filter(|(i, n)| !n.hidden && names.name_areas(*i, sheet, 0, None).is_some()) + .filter_map(|(_, n)| match n.scope { None if own.contains(&gridcraft_formula::fold_name(&n.name)) => None, None => Some(n.name.clone()), Some(s) if s == sheet => Some(n.name.clone()), diff --git a/crates/engine/src/cmd/edit/tests.rs b/crates/engine/src/cmd/edit/tests.rs index 49317caa0..160ce9157 100644 --- a/crates/engine/src/cmd/edit/tests.rs +++ b/crates/engine/src/cmd/edit/tests.rs @@ -42,6 +42,28 @@ fn go_to_lists_the_named_ranges_of_every_sheet() { assert_eq!(list["names"], json!(["Areas", "Local", "Price", "Sheet1!Own", "type"]), "{list}"); } +/// Thousands of names that alias other names are listed with one lookup per alias, not a scan of +/// every name for each; aliases resolve through the chain like a single name does. +#[test] +fn go_to_lists_many_aliases_and_follows_them() { + let mut s = go_to_workbook(); + let wb = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb); + let alias = |name: String, formula: String| gridcraft_model::DefinedName { name, scope: None, formula, comment: String::new(), hidden: false }; + wb.names.push(alias("Chain0".into(), "Areas".into())); + for i in 1..20_000 { + wb.names.push(alias(format!("Chain{i}"), format!("Chain{},price", i - 1))); + } + let list = s.execute("edit.goTo", json!({"list": true})).unwrap(); + let names: Vec<&str> = list["names"].as_array().unwrap().iter().filter_map(|n| n.as_str()).collect(); + // Chain0..Chain6 are within the name depth Go To follows; longer chains are not ranges. + for i in 0..7 { + assert!(names.contains(&format!("Chain{i}").as_str()), "Chain{i} in {names:?}"); + } + assert!(!names.contains(&"Chain7"), "{names:?}"); + let out = s.execute("edit.goTo", json!({"reference": "chain1"})).unwrap(); + assert_eq!(out["selection"], "A1,C3:C4,B2:B5", "{out}"); +} + #[test] fn go_to_a_name_scoped_to_another_sheet_goes_to_that_sheet() { let mut s = go_to_workbook(); @@ -302,6 +324,103 @@ fn wildcards() { assert!(wildcard("", "*")); } +/// The backtracking matcher `wildcard` replaced, as the reference for its results. +fn wildcard_backtracking(text: &str, pat: &str) -> bool { + let t: Vec = text.chars().collect(); + let mut p: Vec<(char, bool)> = Vec::new(); + let mut it = pat.chars(); + while let Some(c) = it.next() { + if c == '~' { + if let Some(n) = it.next() { + p.push((n, true)); + } + } else { + p.push((c, false)); + } + } + let (mut ti, mut pi) = (0usize, 0usize); + let (mut star, mut mark) = (None, 0usize); + while ti < t.len() { + if pi < p.len() && (p[pi] == ('?', false) || (p[pi].0 == t[ti] && p[pi] != ('*', false))) { + ti += 1; + pi += 1; + } else if pi < p.len() && p[pi] == ('*', false) { + star = Some(pi); + mark = ti; + pi += 1; + } else if let Some(sp) = star { + pi = sp + 1; + mark += 1; + ti = mark; + } else { + return false; + } + } + while pi < p.len() && p[pi] == ('*', false) { + pi += 1; + } + pi == p.len() +} + +#[test] +fn wildcard_agrees_with_backtracking() { + // Every text up to 5 characters over a small alphabet against every pattern up to 4 items. + let alphabet = ['a', 'b', '*', '?', '~']; + let all = |n: usize, chars: &[char]| -> Vec { + let mut out = vec![String::new()]; + let mut level = vec![String::new()]; + for _ in 0..n { + level = level.iter().flat_map(|s| chars.iter().map(move |c| format!("{s}{c}"))).collect(); + out.extend(level.iter().cloned()); + } + out + }; + let texts = all(5, &['a', 'b', '*']); + for pat in all(4, &alphabet) { + for text in &texts { + assert_eq!(wildcard(text, &pat), wildcard_backtracking(text, &pat), "{text:?} ~ {pat:?}"); + } + } + // Patterns longer than one 64-bit word. + let long = format!("*{}*b?", "a".repeat(70)); + for text in [format!("x{}yb1", "a".repeat(70)), format!("{}b", "a".repeat(71)), "a".repeat(200)] { + assert_eq!(wildcard(&text, &long), wildcard_backtracking(&text, &long), "{text}"); + } +} + +#[test] +fn wildcard_with_many_distinct_literals_agrees_with_backtracking() { + // 600 distinct characters in the pattern: more than the masks kept, so the rest are rebuilt. + let chars: Vec = (0x4E00u32..0x4E00 + 600).filter_map(char::from_u32).collect(); + let word: String = chars.iter().collect(); + let pat = format!("*{word}?{}", chars[3]); + for text in [format!("x{word}yq{}", chars[3]), word.clone(), format!("{word}{word}"), chars.iter().rev().collect::()] { + assert_eq!(wildcard(&text, &pat), wildcard_backtracking(&text, &pat), "{} chars", text.chars().count()); + } + assert!(wildcard(&format!("{word}zz{}", chars[3]).replace("zz", "z"), &pat)); +} + +#[test] +fn wildcard_keeps_masks_of_frequent_literals_past_the_cache_limit() { + // 300 distinct characters come before the repeated `a`: past the cached masks, `a` (5,000 positions) + // still has to be read from a kept mask, or it is rebuilt for each of the 20,000 `a` in the text. + let rare: String = (0x4E00u32..0x4E00 + 300).filter_map(char::from_u32).collect(); + let pat = format!("*{rare}{}b", "a".repeat(5_000)); + let text = "a".repeat(20_000); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{rare}{}b", "a".repeat(5_000)), &pat)); + assert!(!wildcard(&format!("{rare}{}", "a".repeat(5_000)), &pat)); +} + +#[test] +fn wildcard_on_long_repetitive_text_is_linear() { + // The backtracking matcher compared about text × pattern characters here (10^8). + let text = "a".repeat(20_000); + let pat = format!("*{}b", "a".repeat(5_000)); + assert!(!wildcard(&text, &pat)); + assert!(wildcard(&format!("{text}b"), &pat)); +} + #[test] fn tsv() { assert_eq!(parse_tsv("a\tb\n1\t2\n"), vec![vec!["a", "b"], vec!["1", "2"]]); @@ -424,6 +543,50 @@ fn drag_and_drop_on_another_sheet_stays_on_that_sheet() { assert_eq!(wb.sheet(two).unwrap().value(CellRef::new(0, 1)), Value::Empty); } +#[test] +fn replace_refuses_text_that_would_grow_past_a_cell() { + // 10,000 matches × a 1,000-character replacement would make a 10 MB cell (and a few such + // cells gigabytes); the replacement is refused before that text is built. + let mut s = Session::new(); + s.new_workbook(); + let long = "a".repeat(10_000); + s.execute("range.setValues", json!({"range": "A1", "values": [[long.clone()], ["abc"]]})).unwrap(); + for case in [false, true] { + let err = s.execute("edit.replace", json!({"what": "a", "with": "x".repeat(1_000), "matchCase": case})).expect_err("too long"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!(long)); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!("abc")); + } + // A replacement that stays within a cell's limit still happens. + s.execute("edit.replace", json!({"what": "b", "with": "x".repeat(100)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A2"})).unwrap()["value"], json!(format!("a{}c", "x".repeat(100)))); +} + +#[test] +fn replace_refuses_a_result_past_the_character_limit_though_it_fits_the_byte_limit() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["z"]]})).unwrap(); + for whole in [false, true] { + let err = s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_768), "wholeCell": whole})).expect_err("32,768 characters"); + assert!(err.to_string().contains("too long"), "{err}"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("z")); + } + s.execute("edit.replace", json!({"what": "z", "with": "x".repeat(32_767)})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["value"], json!("x".repeat(32_767))); +} + +#[test] +fn bounded_replace_stops_at_its_limit() { + use super::find::{replace_ci_within, replace_cs_within}; + assert_eq!(replace_cs_within("abab", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_cs_within("abab", "a", "xy", 5), None); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 6).as_deref(), Some("xybxyb")); + assert_eq!(replace_ci_within("AbAb", "a", "xy", 5), None); + assert_eq!(replace_cs_within("abc", "z", "", 3).as_deref(), Some("abc")); + assert_eq!(replace_cs_within("abc", "z", "", 2), None); +} + #[test] fn replace_case_insensitive() { assert_eq!(replace_ci("Apple apple APPLE", "apple", "pear"), "pear pear pear"); diff --git a/crates/engine/src/cmd/extra/data.rs b/crates/engine/src/cmd/extra/data.rs index 9a1ae7417..5af7df0a8 100644 --- a/crates/engine/src/cmd/extra/data.rs +++ b/crates/engine/src/cmd/extra/data.rs @@ -42,6 +42,15 @@ pub(super) fn table_resize(s: &mut Session, p: &Json) -> Result { }) } +/// Most cells (rows × columns, blanks included) Get Data imports. The rows are built densely +/// before they are written, so a few sparse records or one far-away CSV cell must not expand +/// into a rectangle of billions of slots. +const MAX_IMPORT_CELLS: u64 = 10_000_000; + +fn too_many_cells() -> EngineError { + EngineError::Other(format!("The data is too large to import ({MAX_IMPORT_CELLS} cells at most).")) +} + pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let bytes = if let Some(t) = str_param(p, "text") { t.as_bytes().to_vec() @@ -57,15 +66,27 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { if text.trim_start().starts_with('[') { let arr: Vec = serde_json::from_str(&text).map_err(|e| bad("data.getData", e.to_string()))?; let mut keys: Vec = Vec::new(); + let mut seen: std::collections::HashSet<&str> = std::collections::HashSet::new(); for o in &arr { if let Some(m) = o.as_object() { for k in m.keys() { - if !keys.contains(k) { + if !seen.contains(k.as_str()) { + // Stop collecting at the column limit. + if keys.len() >= gridcraft_core::MAX_COLS as usize { + return Err(EngineError::Other("The data has more fields than a sheet has columns.".into())); + } + seen.insert(k.as_str()); keys.push(k.clone()); } } } } + if arr.len() as u64 + 1 > u64::from(gridcraft_core::MAX_ROWS) { + return Err(EngineError::Other("The data has more rows than a sheet has.".into())); + } + if (arr.len() as u64 + 1).saturating_mul(keys.len().max(1) as u64) > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } let mut out = vec![keys.iter().map(|k| json!(k)).collect::>()]; for o in &arr { out.push(keys.iter().map(|k| o.get(k).cloned().unwrap_or(Json::Null)).collect()); @@ -75,10 +96,17 @@ pub(super) fn from_csv(s: &mut Session, p: &Json) -> Result { let delim = str_param(p, "delimiter") .and_then(|d| d.chars().next()) .unwrap_or(if text.lines().next().is_some_and(|l| l.contains('\t')) { '\t' } else { ',' }); - let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, ..Default::default() }; - let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| EngineError::Other(e.to_string()))?; + let opts = gridcraft_xlsx::CsvOptions { delimiter: delim as u8, max_used_cells: Some(MAX_IMPORT_CELLS), ..Default::default() }; + // The cell limit is enforced while parsing, before an oversized file is stored. + let wb = gridcraft_xlsx::read_csv(&bytes, &opts).map_err(|e| match e { + gridcraft_xlsx::IoError::TooLarge(_) => too_many_cells(), + e => EngineError::Other(e.to_string()), + })?; let sh = wb.sheet(0).ok_or(EngineError::NoDocument)?; let Some(u) = sh.used_range() else { return Err(EngineError::Other("The file is empty.".into())) }; + if u.count() > MAX_IMPORT_CELLS { + return Err(too_many_cells()); + } (u.start.row..=u.end.row) .map(|r| { (u.start.col..=u.end.col) @@ -129,3 +157,35 @@ pub(super) fn queries(s: &mut Session, _: &Json) -> Result { .collect(), )) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use crate::Session; + + fn s() -> Session { + let mut s = Session::new(); + s.new_workbook(); + s + } + + #[test] + fn sparse_imports_are_refused_before_their_rows_are_built() { + let mut s = s(); + let sheets = s.doc().unwrap().wb.sheets.len(); + // 5,000 records with one field each: 5,001 × 5,000 slots once every record gets every field. + let records: Vec<_> = (0..5_000).map(|i| json!({ format!("k{i}"): i })).collect(); + let text = serde_json::to_string(&records).unwrap(); + let err = s.execute("data.getData", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + // A CSV whose last line reaches 16,001 columns, 1,000 lines down. + let text = format!("1{}{}x", "\n".repeat(999), ",".repeat(16_000)); + let err = s.execute("data.fromTextCsv", json!({"text": text})).expect_err("too many cells"); + assert!(err.to_string().contains("too large"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + // Small imports still work. + let out = s.execute("data.getData", json!({"text": r#"[{"a": 1}, {"b": 2}]"#})).unwrap(); + assert_eq!((out["rows"].as_u64(), out["cols"].as_u64()), (Some(3), Some(2)), "{out}"); + } +} diff --git a/crates/engine/src/cmd/forecast.rs b/crates/engine/src/cmd/forecast.rs index d0941a9ca..88de4b433 100644 --- a/crates/engine/src/cmd/forecast.rs +++ b/crates/engine/src/cmd/forecast.rs @@ -36,13 +36,28 @@ fn max_periods(history: usize) -> u32 { u32::try_from(MAX_WORK / h).unwrap_or(u32::MAX).clamp(1, MAX_PERIODS) } +/// Most forecast rows when the model is fit to the first `fit` of `history` rows, which span +/// `slots` timeline slots once their gaps are filled. With a Forecast Start inside the history, +/// every row after it is forecast too (a hindcast row holds the same ETS formulas, each refitting +/// the model over the `slots` slots), so those rows (counted from the raw rows) count against +/// [`MAX_WORK`] before any forecast row does. `None` when the hindcast alone exceeds it. +fn max_periods_from(history: usize, fit: usize, slots: usize) -> Option { + if fit >= history { + return Some(max_periods(history)); + } + let f = u64::try_from(slots.max(fit)).unwrap_or(u64::MAX).max(1); + let hindcast = u64::try_from(history - fit).unwrap_or(u64::MAX); + let rest = (MAX_WORK / f).checked_sub(hindcast).filter(|r| *r >= 1)?; + Some(u32::try_from(rest).unwrap_or(u32::MAX).min(MAX_PERIODS)) +} + pub fn specs() -> Vec { vec![cmd!( "data.forecastSheet", "Forecast Sheet", ["Data", "Forecast"], None, - "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", + "{timeline?: \"A2:A25\", values?: \"B2:B25\" (default: the first two columns of `range`, the selection or the current region; a text first cell is the header), end?: timeline value or date text | periods?: n (default: a third of the history, at least 1; ≤10,000 and history × periods ≤ 500,000, or with a forecastStart inside the history (hindcast rows are forecast too) (the greater of the fitted rows and fitted slots × (periods + rows from forecastStart on, that row included)) ≤ 500,000; history ≤ 10,000 rows), forecastStart?: a timeline value or date text on the history, before its last point (hindcast: the model is fit to the points before it, the forecast columns start there and run over the actuals; default and the last point: forecast after the history), confidenceInterval?: bool (default true), confidence?: 0.95 or 95, seasonality?: \"auto\" | 0 (none) | 2–8,760, fillMissing?: interpolate|zeros, aggregate?: average|count|counta|max|median|min|sum, statistics?: bool, chart?: line|column} → {sheet, table, chart, history, periods}. Inserts a sheet before the active one: timeline, values, Forecast (FORECAST.ETS), Lower/Upper Confidence Bound (FORECAST.ETS.CONFINT), optional FORECAST.ETS.STAT table, and a chart.", has_doc, forecast_sheet )] @@ -257,13 +272,23 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result { } } }; - let cap = max_periods(rows.len()); + // The slots the model is fit over: the fit rows' span of the timeline, gaps included. + let slots = match (fit < rows.len(), times.get(fit.wrapping_sub(1)), times.first()) { + (true, Some(last_fit), Some(first)) => match (timeline.steps_after_last(*last_fit), timeline.steps_after_last(*first)) { + (Some(b), Some(a)) => ((b - a).round() + 1.0).clamp(0.0, 1e9) as usize, + _ => fit, + }, + _ => fit, + }; + let cap = max_periods_from(rows.len(), fit, slots).ok_or_else(|| { + EngineError::Other("The forecast start is too early for this much history: choose a later forecast start or less history.".into()) + })?; let periods = match end { Some(end) => steps_to(&timeline, end, cap)?, None => match number(p, "periods")? { Some(n) if n >= 1.0 && n.ceil() <= f64::from(cap) => n.ceil() as u32, Some(_) => return Err(bad(CMD, format!("`periods` is from 1 to {cap} for {} rows of history", rows.len()))), - None => default_periods(rows.len()), + None => default_periods(rows.len()).min(cap), }, }; let future: Vec = @@ -532,6 +557,51 @@ mod tests { assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets - 1); } + #[test] + fn hindcast_rows_count_against_the_work_limit() { + use super::{MAX_WORK, max_periods, max_periods_from}; + assert_eq!(max_periods_from(1_300, 1_300, 1_300), Some(max_periods(1_300))); + // 2,000 rows fit to the first 1,899: 101 hindcast rows, then 162 forecast rows. + assert_eq!(max_periods_from(2_000, 1_899, 1_899), Some(162)); + // Fit to the first 1,000: the 1,000 hindcast rows alone are twice the limit. + assert_eq!(max_periods_from(2_000, 1_000, 1_000), None); + // 800 rows fit to the first 400, which span 571 slots (gaps filled): each formula refits + // 571 slots, so 475 forecast rows after the 400 hindcast rows, not 850. + assert_eq!(max_periods_from(800, 400, 571), Some(475)); + assert!((475 + 400) * 571 <= MAX_WORK as usize); + for (h, f) in [(2_000usize, 1_899usize), (10_000, 9_990), (100, 2), (60, 50)] { + if let Some(p) = max_periods_from(h, f, f) { + assert!((u64::from(p) + (h - f) as u64) * f as u64 <= MAX_WORK, "{h} {f}"); + } + } + let mut s = Session::new(); + s.new_workbook(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend((1..=2_000).map(|t| json!([t, (t % 7) as f64 + t as f64 * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let sheets = s.doc().unwrap().wb.sheets.len(); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_001})).expect_err("hindcast too long"); + assert!(err.to_string().contains("forecast start is too early"), "{err}"); + let err = s.execute("data.forecastSheet", json!({"range": "A1:B2001", "forecastStart": 1_900, "periods": 163})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 162"), "{err}"); + assert_eq!(s.doc().unwrap().wb.sheets.len(), sheets); + } + + #[test] + fn the_hindcast_cap_follows_the_slots_the_fit_spans_not_its_rows() { + // 800 rows, 20% of the slots missing: the first 400 rows span 500 slots, so each ETS + // formula refits 500 slots and the cap is 1,000 - 400 = 600 (it was 850 by rows). + let mut s = Session::new(); + s.new_workbook(); + let times: Vec = (1u32..).filter(|t| !matches!(t % 10, 3 | 6)).take(800).collect(); + let mut rows = vec![json!(["t", "y"])]; + rows.extend(times.iter().map(|t| json!([t, (t % 7) as f64 + f64::from(*t) * 0.5]))); + s.execute("range.setValues", json!({"range": "A1", "values": rows})).unwrap(); + let start = times[400]; + let err = s.execute("data.forecastSheet", json!({"range": "A1:B801", "forecastStart": start, "periods": 601})).expect_err("past the cap"); + assert!(err.to_string().contains("from 1 to 600"), "{err}"); + } + #[test] fn history_and_periods_are_capped() { use super::{MAX_HISTORY, MAX_PERIODS, MAX_WORK, default_periods, max_periods}; diff --git a/crates/engine/src/cmd/format.rs b/crates/engine/src/cmd/format.rs index 1066f48e2..aa2638801 100644 --- a/crates/engine/src/cmd/format.rs +++ b/crates/engine/src/cmd/format.rs @@ -85,7 +85,7 @@ pub fn specs() -> Vec { "Format Cells…", ["Home", "Cells", "Format"], Some("Cmd+1"), - "{range?, style: {font?, fill?, borders?, align?, numFmt?, protection?}} (partial Style JSON merged in)", + "{range?, style: {font?, fill?, borders?, align?: {reading_order?: Context|LeftToRight|RightToLeft|null, ...}, numFmt?, protection?}} (partial Style JSON merged in; null clears reading_order)", has_doc, format_cells ), @@ -870,6 +870,52 @@ mod tests { assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["style"]["num_fmt"], "0.00"); } + #[test] + fn arabic_vowel_and_direction_marks_add_no_width() { + let plain = approx_text_width("محمد يدرس في المدرسة", 11.0, false); + assert_eq!(approx_text_width("مُحَمَّد يَدْرُسُ فِي الْمَدْرَسَةِ", 11.0, false), plain); + assert_eq!(approx_text_width("\u{200F}محمد يدرس في المدرسة\u{200C}", 11.0, false), plain); + assert_eq!(approx_text_width("ب\u{0897}\u{0898}\u{089F}", 11.0, false), approx_text_width("ب", 11.0, false)); + assert!(approx_text_width("\u{08E2}", 11.0, false) > 0.0, "the disputed end of ayah is a visible sign"); + } + + #[test] + fn arabic_text_fits_a_column_about_as_wide_as_latin_text_of_the_same_length() { + let mut s = crate::Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["مُحَمَّد يَدْرُسُ فِي الْمَدْرَسَةِ", "Hello there in school"]]})).unwrap(); + s.execute("home.autofitColumnWidth", json!({"cols": "A:B"})).unwrap(); + let doc = s.doc().unwrap(); + let width = |c: u32| doc.wb.active().and_then(|sh| sh.cols.get(&c)).and_then(|e| e.size).unwrap(); + let ratio = width(0) / width(1); + assert!((0.75..=1.15).contains(&ratio), "Arabic {} vs Latin {}", width(0), width(1)); + } + + #[test] + fn arabic_word_ligatures_are_wider_than_letters() { + let em = |t: &str| approx_text_width(t, 72.0, false) / 96.0; + assert!(em("\u{FDFD}") > 8.0); + assert!(em("\u{FDFA}") > 1.0); + let honorifics = ['\u{FD40}', '\u{FD4B}', '\u{FD4C}', '\u{FD4F}', '\u{FDC8}', '\u{FDCF}', '\u{FDFE}', '\u{FDFF}']; + for honorific in honorifics.into_iter().chain(['\u{FBC3}', '\u{FBD2}', '\u{FD90}', '\u{FD91}']) { + assert!(em(&honorific.to_string()) > 1.0, "U+{:04X}", u32::from(honorific)); + } + assert!(em("ب") < 0.6); + } + + #[test] + fn arabic_digits_are_as_wide_as_ascii_digits_and_punctuation_as_a_letter() { + let width = |t: &str| approx_text_width(t, 11.0, false); + assert_eq!(width("١٢٣"), width("123")); + assert_eq!(width("۴۵۶"), width("456")); + assert_eq!(width("،؟"), width("بب")); + } + + #[test] + fn cjk_text_keeps_a_full_em_per_character() { + assert_eq!(approx_text_width("中文", 12.0, false), 2.0 * 12.0 * 96.0 / 72.0); + } + #[test] fn width_units() { assert_eq!(chars_to_points(8.43), 64.0); diff --git a/crates/engine/src/cmd/format/sizes.rs b/crates/engine/src/cmd/format/sizes.rs index c79327678..8b304c889 100644 --- a/crates/engine/src/cmd/format/sizes.rs +++ b/crates/engine/src/cmd/format/sizes.rs @@ -136,12 +136,55 @@ pub fn approx_text_width(text: &str, size: f32, bold: bool) -> f32 { c if c.is_ascii_uppercase() => 0.65, c if c.is_ascii_digit() => 0.56, c if c.is_ascii() => 0.52, - _ => 1.0, + c if is_zero_width(c) => 0.0, + c => arabic_em(c).unwrap_or(1.0), }; } w * size * if bold { 1.07 } else { 1.0 } * 96.0 / 72.0 } +/// Arabic vowel and Quranic marks, which sit on the previous letter, and invisible format +/// characters such as direction marks and zero-width joiners. +fn is_zero_width(c: char) -> bool { + matches!( + c, + '\u{0610}'..='\u{061A}' + | '\u{061C}' + | '\u{064B}'..='\u{065F}' + | '\u{0670}' + | '\u{06D6}'..='\u{06DC}' + | '\u{06DF}'..='\u{06E4}' + | '\u{06E7}'..='\u{06E8}' + | '\u{06EA}'..='\u{06ED}' + | '\u{0897}'..='\u{089F}' + | '\u{08CA}'..='\u{08E1}' + | '\u{08E3}'..='\u{08FF}' + | '\u{200B}'..='\u{200F}' + | '\u{202A}'..='\u{202E}' + | '\u{2060}'..='\u{2069}' + | '\u{FE20}'..='\u{FE2F}' + | '\u{FEFF}' + ) +} + +/// Width in ems of an Arabic-script character, measured from shaped Noto Sans Arabic, Noto Naskh +/// Arabic and DejaVu Sans: joined letters average about half an em, word ligatures such as ﷺ and the +/// rial sign about 1.2 em, the honorific ligatures (U+FD40–FD4B) about 1.6 em, and the bismillah +/// ligature about 11 em. The later honorific ligatures (U+FBC3–FBD2, U+FD4C–FD4F, U+FD90–FD91, +/// U+FDC8–FDCF) are not measured and get the 1.6 em of their siblings rather than the letter width. +/// Arabic-Indic digits are as wide as ASCII digits (0.53–0.57 em in Noto Sans Arabic, Arial, Tahoma +/// and Times New Roman); Arabic punctuation (0.2–0.5 em in the same fonts) keeps the letter width. +fn arabic_em(c: char) -> Option { + match c { + '\u{FDFD}' => Some(11.0), + '\u{FBC3}'..='\u{FBD2}' | '\u{FD40}'..='\u{FD4F}' | '\u{FD90}'..='\u{FD91}' | '\u{FDC8}'..='\u{FDCF}' => Some(1.6), + '\u{FDF0}'..='\u{FDFC}' | '\u{FDFE}'..='\u{FDFF}' => Some(1.2), + '\u{0660}'..='\u{0669}' | '\u{06F0}'..='\u{06F9}' => Some(0.56), + '\u{0600}'..='\u{06FF}' | '\u{0750}'..='\u{077F}' | '\u{0870}'..='\u{08FF}' | '\u{FB50}'..='\u{FDFF}' | '\u{FE70}'..='\u{FEFC}' => Some(0.48), + _ => None, + } +} + /// Distinct texts AutoFit measures exactly; past this many, `Session::text_width` is asked for a /// quick measure so a huge column doesn't lay out every value. const EXACT_TEXT_WIDTHS: usize = 2_000; diff --git a/crates/engine/src/cmd/pivot/tests.rs b/crates/engine/src/cmd/pivot/tests.rs index 3748d0705..c40ddf1be 100644 --- a/crates/engine/src/cmd/pivot/tests.rs +++ b/crates/engine/src/cmd/pivot/tests.rs @@ -209,6 +209,41 @@ fn show_values_as() { assert_eq!(text(&s, "B3"), "Biggest sale"); } +/// Running Total over nested rows adds up each child's value across the top-level items, in the +/// order they appear: the same value as summing the plain values of the earlier items' rows. +#[test] +fn running_total_over_nested_rows_adds_up_across_the_top_level_items() { + let mut s = sales(); + s.execute("insert.pivotTable", json!({"rows": ["Region", "Product"], "values": ["Sales"]})).unwrap(); + let r = pt(&s).last_range.unwrap(); + let rows: Vec = (r.start.row + 1..r.end.row).collect(); + let read = |s: &Session| -> Vec<(String, f64)> { + rows.iter().map(|row| (text(s, &CellRef::new(*row, 0).a1()), val(s, &CellRef::new(*row, 1).a1()).as_f64().unwrap_or(0.0))).collect() + }; + let before = read(&s); + s.execute("pivot.valueSettings", json!({"index": 0, "showAs": "runningTotal"})).unwrap(); + let after = read(&s); + // Region rows carry the report's top-level items (not indented); the others are products. + let indent = |s: &Session, row: u32| { + let d = s.doc().unwrap(); + let sh = d.wb.active().unwrap(); + d.wb.styles.get(sh.style_id(CellRef::new(row, 0))).align.indent + }; + let (mut regions, mut products) = (0.0, std::collections::HashMap::::new()); + for (i, row) in rows.iter().enumerate() { + let name = &before[i].0; + let total = if indent(&s, *row) == 0 { + regions += before[i].1; + regions + } else { + let t = products.entry(name.clone()).or_insert(0.0); + *t += before[i].1; + *t + }; + assert!((after[i].1 - total).abs() < 1e-9, "row {name}: {} vs {total}", after[i].1); + } +} + #[test] fn compact_vs_tabular_shapes() { let mut s = sales(); diff --git a/crates/engine/src/cmd/print/draw.rs b/crates/engine/src/cmd/print/draw.rs index 7a49b8e85..595b70d86 100644 --- a/crates/engine/src/cmd/print/draw.rs +++ b/crates/engine/src/cmd/print/draw.rs @@ -228,6 +228,82 @@ fn draw_border(pg: &mut Page, b: &BorderLine, wb: &Workbook, start: Point, end: pg.stroke_line_dashed(Point::new(x0, y0), Point::new(x1, y1), lw, color, &border_dash(b.style, k)); } +/// Merged areas active at one row of a downward sweep, kept disjoint: by first column, their last +/// column and last row. Adding, dropping and looking up a merge costs log(active merges). +#[derive(Default)] +struct MergeSweep { + cols: std::collections::BTreeMap, + /// (last row, first column) of the active merges, soonest ending first. + ends: std::collections::BinaryHeap>, +} + +impl MergeSweep { + /// Moves down to `row` (rows never decrease), dropping the merges that end above it. + fn advance(&mut self, row: u32) { + while let Some(std::cmp::Reverse((end, col))) = self.ends.peek().copied() { + if end >= row { + break; + } + self.ends.pop(); + self.cols.remove(&col); + } + } + + /// Does an active merge meet columns `c0..=c1`? The intervals are disjoint, so only the last + /// one starting at or before `c1` can. + fn meets(&self, c0: u32, c1: u32) -> bool { + self.cols.range(..=c1).next_back().is_some_and(|(_, (end, _))| *end >= c0) + } + + /// Adds merge `m`, at row `row`, unless it overlaps an active one. + fn add(&mut self, m: RangeRef, row: u32) -> bool { + self.advance(row); + if self.meets(m.start.col, m.end.col) { + return false; + } + self.cols.insert(m.start.col, (m.end.col, m.end.row)); + self.ends.push(std::cmp::Reverse((m.end.row, m.start.col))); + true + } +} + +/// The merged areas that meet `block`, ordered by the row where they enter it, then by column, +/// leaving out any that overlaps one kept before it (only a damaged file has overlapping merges; +/// Excel drops them when it repairs one). One sweep down the rows instead of a scan of every +/// merge per cell. +fn block_merges(sh: &Sheet, block: RangeRef) -> Vec { + let top = |m: &RangeRef| m.start.row.max(block.start.row); + let mut merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); + merges.sort_by_key(|m| (top(m), m.start.col)); + let mut sweep = MergeSweep::default(); + merges.retain(|m| sweep.add(*m, top(m))); + merges +} + +/// Answers "is this cell inside a merge?" for cells asked in row-major order, from the merges of +/// [`block_merges`]. +struct Covered<'a> { + merges: &'a [RangeRef], + top: u32, + next: usize, + sweep: MergeSweep, +} + +impl<'a> Covered<'a> { + fn new(merges: &'a [RangeRef], block: RangeRef) -> Self { + Self { merges, top: block.start.row, next: 0, sweep: MergeSweep::default() } + } + + fn covers(&mut self, c: CellRef) -> bool { + while let Some(m) = self.merges.get(self.next).filter(|m| m.start.row.max(self.top) <= c.row) { + self.sweep.add(*m, m.start.row.max(self.top)); + self.next += 1; + } + self.sweep.advance(c.row); + self.sweep.meets(c.col, c.col) + } +} + /// Draws the cells of a block: fills, gridlines, data bars, borders and text. pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Option) { let (bx, by) = (b.xs.first().copied().unwrap_or(0.0), b.ys.first().copied().unwrap_or(0.0)); @@ -237,17 +313,18 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt } let block = b.range(); let sh = b.sh; - let merges: Vec = sh.merges.iter().filter(|m| m.intersects(&block)).copied().collect(); - let covered = |c: CellRef| merges.iter().any(|m| m.contains(c)); - // Cells worth visiting. + let merges = block_merges(sh, block); + let by_start: std::collections::HashMap = merges.iter().map(|m| (m.start, *m)).collect(); + // Cells worth visiting, in row-major order (which `Covered` needs). + let mut covered = Covered::new(&merges, block); let cells: Vec = if block.count() <= DENSE_BLOCK { - block.iter().filter(|c| !covered(*c)).collect() + block.iter().filter(|c| !covered.covers(*c)).collect() } else { let mut set: Vec = sh.cells.iter_range(block).map(|(c, _)| c).collect(); set.extend(sh.spill.range(block.start..=block.end).map(|(c, _)| *c).filter(|c| block.contains(*c))); set.sort_unstable(); set.dedup(); - set.retain(|c| !covered(*c)); + set.retain(|c| !covered.covers(*c)); set }; pg.save(); @@ -277,14 +354,14 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt .filter(|(c, l)| { l.fill.is_some() || l.cf.is_some() - || merges.iter().any(|m| m.start == *c) + || by_start.contains_key(c) || !sh.value(*c).is_empty() || l.style.borders != Borders::default() || l.style.num_fmt.as_str() == "checkbox" }) .collect(); let rect_of = |c: CellRef| -> (f32, f32, f32, f32, bool) { - match merges.iter().find(|m| m.start == c) { + match by_start.get(&c) { Some(m) => { let (x, y, w, h) = b.rect(*m); (x, y, w, h, true) @@ -322,7 +399,7 @@ pub(super) fn draw_block(pg: &mut Page, b: &Block, gridlines: bool, cf: &mut Opt continue; } let bd = &l.style.borders; - let (right, bottom) = match merges.iter().find(|m| m.start == *c).filter(|_| merged) { + let (right, bottom) = match by_start.get(c).filter(|_| merged) { Some(m) => ( b.wb.styles.get(sh.style_id(CellRef::new(m.start.row, m.end.col))).borders.right, b.wb.styles.get(sh.style_id(CellRef::new(m.end.row, m.start.col))).borders.bottom, @@ -407,11 +484,12 @@ pub(super) struct PrintedBlock { pub(super) fn draw_block_links(pg: &mut Page, b: &Block, page: usize, pending: &mut Vec) { let range = b.range(); let (Some(&bx), Some(&by), Some(&right), Some(&bottom)) = (b.xs.first(), b.ys.first(), b.xs.last(), b.ys.last()) else { return }; + let by_start: std::collections::HashMap = block_merges(b.sh, range).into_iter().map(|m| (m.start, m)).collect(); for (cell, link) in b.sh.hyperlinks.range(range.start..=range.end) { if !range.contains(*cell) { continue; } - let cell_range = b.sh.merges.iter().find(|m| m.start == *cell).copied().unwrap_or(RangeRef::cell(*cell)); + let cell_range = by_start.get(cell).copied().unwrap_or(RangeRef::cell(*cell)); let (x, y, w, h) = b.rect(cell_range); let (x0, y0, x1, y1) = (x.max(bx), y.max(by), (x + w).min(right), (y + h).min(bottom)); if x1 <= x0 || y1 <= y0 { @@ -503,3 +581,44 @@ pub(super) fn make_block<'a>( } Block { wb, si, sh, rows, cols, xs, ys, k, scale: lay.scale } } + +#[cfg(test)] +mod merge_tests { + use super::{CellRef, Covered, RangeRef, Sheet, block_merges}; + + fn r(a: &str) -> RangeRef { + RangeRef::parse(a).unwrap() + } + + #[test] + fn block_merges_drop_overlaps_and_cover_like_a_scan() { + let mut sh = Sheet::new("S"); + sh.merges = vec![r("B2:C3"), r("E1:F8"), r("B2:B5"), r("A10:D10"), r("H3:H4"), r("Z1:Z2"), r("C3:D4"), r("A1:A1048576")]; + let block = r("B1:H12"); + let merges = block_merges(&sh, block); + // B2:B5 and C3:D4 overlap B2:C3, which comes first; Z1:Z2 is outside the block. + assert_eq!(merges, vec![r("E1:F8"), r("B2:C3"), r("H3:H4"), r("A10:D10")]); + let mut covered = Covered::new(&merges, block); + for c in block.iter() { + assert_eq!(covered.covers(c), merges.iter().any(|m| m.contains(c)), "{}", c.a1()); + } + // Cells asked sparsely, still in order. + let mut covered = Covered::new(&merges, block); + for (a, want) in [("C1", false), ("F5", true), ("G5", false), ("D10", true), ("H12", false)] { + assert_eq!(covered.covers(CellRef::parse(a).unwrap()), want, "{a}"); + } + } + + #[test] + fn many_stacked_merges_are_swept_once() { + // 50,000 merges over the same cells: one is kept, and the block is checked against it + // without comparing every cell with every merge. + let mut sh = Sheet::new("S"); + sh.merges = (0..50_000u32).map(|i| RangeRef::new(CellRef::new(0, 0), CellRef::new(199, 1 + i % 50))).collect(); + let block = r("A1:AZ200"); + let merges = block_merges(&sh, block); + assert_eq!(merges, vec![r("A1:B200")]); + let mut covered = Covered::new(&merges, block); + assert_eq!(block.iter().filter(|c| covered.covers(*c)).count(), 400); + } +} diff --git a/crates/engine/src/cmd/review.rs b/crates/engine/src/cmd/review.rs index de9c98ecd..7eafa26db 100644 --- a/crates/engine/src/cmd/review.rs +++ b/crates/engine/src/cmd/review.rs @@ -163,6 +163,8 @@ fn protect_sheet(s: &mut Session, p: &Json) -> Result { let what = d.wb.sheet(sheet).map(|sh| sh.name.clone()).unwrap_or_default(); let prot = SheetProtection { password: str_param(p, "password").filter(|x| !x.is_empty()).map(|pw| new_password_hash(d, &what, pw)), ..prot }; edit(s, |cx| { + // FORMULATEXT reads a sheet's protection: its results are recalculated. + cx.recalc_all = true; cx.sheet_mut(sheet)?.protection = Some(prot.clone()); Ok(Json::Null) }) @@ -182,7 +184,10 @@ fn unprotect_sheet(s: &mut Session, p: &Json) -> Result { } } edit(s, |cx| { - cx.sheet_mut(sheet)?.protection = None; + // Only removing protection changes what FORMULATEXT reads; a no-op unprotect skips the full recalculation. + if cx.sheet_mut(sheet)?.protection.take().is_some() { + cx.recalc_all = true; + } Ok(Json::Null) }) } @@ -404,6 +409,21 @@ mod tests { assert_eq!(at(&s), (0, "A1".to_string())); } + #[test] + fn protecting_and_unprotecting_leave_table_headers_alone() { + let mut s = Session::new(); + s.new_workbook(); + s.execute("range.setValues", json!({"range": "A1", "values": [["a", "b"], [1, 2]]})).unwrap(); + s.execute("insert.table", json!({"range": "A1:B2", "header": true})).unwrap(); + // An imported header can hold a number: only a structural edit would turn it back into text. + let d = s.doc_mut().unwrap(); + std::sync::Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap().set_value(gridcraft_core::CellRef::new(0, 0), gridcraft_core::Value::Number(7.0)); + s.execute("review.protectSheet", json!({})).unwrap(); + s.execute("review.unprotectSheet", json!({})).unwrap(); + let v = s.doc().unwrap().wb.sheet(0).unwrap().value(gridcraft_core::CellRef::new(0, 0)); + assert_eq!(v, gridcraft_core::Value::Number(7.0)); + } + #[test] fn protection_that_forbids_selecting_locked_cells_is_enforced_and_kept() { let mut s = book(); diff --git a/crates/engine/src/cmd/solver/reports.rs b/crates/engine/src/cmd/solver/reports.rs index 19500cab0..c2a5c6d84 100644 --- a/crates/engine/src/cmd/solver/reports.rs +++ b/crates/engine/src/cmd/solver/reports.rs @@ -52,14 +52,19 @@ pub(super) fn abs_ref(c: CellRef) -> String { format!("${}${}", gridcraft_core::col_to_letters(c.col), u64::from(c.row) + 1) } +/// How many cells to the left and above a cell its name is searched for. +const NAME_REACH: u32 = 1_000; + /// A cell's name in the reports: the nearest text to its left and the nearest text above it. pub(super) fn cell_name(sh: &Sheet, c: CellRef) -> String { let text = |c: CellRef| match sh.value(c) { Value::Text(t) if !t.trim().is_empty() => Some(t.to_string()), _ => None, }; - let left = (0..c.col).rev().find_map(|col| text(CellRef::new(c.row, col))); - let top = (0..c.row).rev().find_map(|row| text(CellRef::new(row, c.col))); + // Labels are next to their cells: looking further than this on a sparse sheet only costs time + // (a thousand cells near the last row would each probe a million empty coordinates). + let left = (c.col.saturating_sub(NAME_REACH)..c.col).rev().find_map(|col| text(CellRef::new(c.row, col))); + let top = (c.row.saturating_sub(NAME_REACH)..c.row).rev().find_map(|row| text(CellRef::new(row, c.col))); [left, top].into_iter().flatten().collect::>().join(" ") } @@ -438,3 +443,17 @@ pub(super) fn report(s: &mut Session, p: &Json) -> Result { } Ok(out) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_name_is_searched_only_a_bounded_distance() { + let mut sh = Sheet::new("S".to_string()); + sh.set_cell(CellRef::new(0, 0), gridcraft_model::Cell::value(Value::text("Far"))); + sh.set_cell(CellRef::new(5, 4), gridcraft_model::Cell::value(Value::text("Near"))); + assert_eq!(cell_name(&sh, CellRef::new(5, 5)), "Near"); + assert_eq!(cell_name(&sh, CellRef::new(0, NAME_REACH + 1)), ""); + } +} diff --git a/crates/engine/src/cmd/whatif/consolidate.rs b/crates/engine/src/cmd/whatif/consolidate.rs index 149b72bc7..702baa312 100644 --- a/crates/engine/src/cmd/whatif/consolidate.rs +++ b/crates/engine/src/cmd/whatif/consolidate.rs @@ -61,6 +61,24 @@ enum Key { const MAX_CONSOLIDATE_CELLS: u64 = 5_000_000; +/// The cells a consolidated table writes: every row key meets every column key, plus a label for +/// each key on the sides that have labels. +fn output_cells(rows: usize, cols: usize, top: bool, left: bool) -> u64 { + let (r, c) = (rows as u64, cols as u64); + r.saturating_mul(c).saturating_add(if top { c } else { 0 }).saturating_add(if left { r } else { 0 }) +} + +#[cfg(test)] +#[test] +fn the_output_limit_counts_label_cells() { + // A million row keys and four column keys are 4 million cells, 5 million with the row labels. + assert_eq!(output_cells(1_000_000, 4, false, false), 4_000_000); + assert_eq!(output_cells(1_000_000, 4, false, true), 5_000_000); + assert!(output_cells(1_000_000, 5, false, true) > MAX_CONSOLIDATE_CELLS); + assert_eq!(output_cells(3, 2, true, true), 6 + 2 + 3); + assert_eq!(output_cells(usize::MAX, usize::MAX, true, true), u64::MAX); +} + pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.consolidate"; let func = str_param(p, "function").unwrap_or("sum").to_string(); @@ -89,20 +107,20 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } None => (d.wb.active_sheet(), d.selection.active), }; + // Keys in first-seen order, each with its index: a lookup is one probe, not a scan. let mut row_keys: Vec = Vec::new(); let mut col_keys: Vec = Vec::new(); + let mut row_index: HashMap = HashMap::new(); + let mut col_index: HashMap = HashMap::new(); let mut row_label: HashMap = HashMap::new(); let mut col_label: HashMap = HashMap::new(); let mut aggs: HashMap<(usize, usize), Agg> = HashMap::new(); let mut total: u64 = 0; - let index_of = |keys: &mut Vec, k: Key| -> usize { - match keys.iter().position(|x| *x == k) { - Some(i) => i, - None => { - keys.push(k); - keys.len() - 1 - } - } + let index_of = |keys: &mut Vec, index: &mut HashMap, k: Key| -> usize { + *index.entry(k).or_insert_with_key(|k| { + keys.push(k.clone()); + keys.len() - 1 + }) }; for t in &srcs { let (si, r) = parse_range(&d.wb, t, CMD)?; @@ -131,7 +149,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(col - c0) }; - cols.push((col, index_of(&mut col_keys, k))); + cols.push((col, index_of(&mut col_keys, &mut col_index, k))); } for row in r0..=r.end.row { let k = if left { @@ -145,7 +163,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { } else { Key::Pos(row - r0) }; - let ri = index_of(&mut row_keys, k); + let ri = index_of(&mut row_keys, &mut row_index, k); for (col, ci) in &cols { aggs.entry((ri, *ci)).or_default().add(&sh.value(CellRef::new(row, *col))); } @@ -157,6 +175,9 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { { return Err(EngineError::Other("The consolidated data doesn't fit below and right of the destination.".into())); } + if output_cells(row_keys.len(), col_keys.len(), top, left) > MAX_CONSOLIDATE_CELLS { + return Err(bad(CMD, "the consolidated table would be too large")); + } let mut cells: Vec<(CellRef, Value)> = Vec::new(); if top { for (ci, k) in col_keys.iter().enumerate() { diff --git a/crates/engine/src/cmd/whatif/scenarios.rs b/crates/engine/src/cmd/whatif/scenarios.rs index 75bf147a0..9ebb1efb5 100644 --- a/crates/engine/src/cmd/whatif/scenarios.rs +++ b/crates/engine/src/cmd/whatif/scenarios.rs @@ -189,6 +189,9 @@ pub(super) fn scenario_delete(s: &mut Session, p: &Json) -> Result { Ok(json!({"deleted": removed.name})) } +/// Most changing-cell-by-scenario cells one Scenario Summary writes. +const MAX_SUMMARY_CELLS: usize = 1_000_000; + pub(super) fn scenario_summary(s: &mut Session, p: &Json) -> Result { const CMD: &str = "data.scenarioSummary"; let d = s.doc()?; @@ -211,17 +214,24 @@ pub(super) fn scenario_summary(s: &mut Session, p: &Json) -> Result { // Changing cells of every scenario, first-seen order; per scenario its values. let mut changing: Vec = Vec::new(); let mut per: Vec> = Vec::new(); + let mut seen: std::collections::HashSet = Default::default(); for sc in &scenarios { let (_, cells) = cell_list(&d.wb, &sc.changing, CMD, MAX_SCENARIO_CELLS)?; let mut m = HashMap::new(); for (c, v) in cells.iter().zip(sc.values.iter()) { - if !changing.contains(c) { + if seen.insert(*c) { changing.push(*c); } m.insert(*c, json_value(v, &d.wb)); } per.push(m); } + // The summary has a cell for every changing cell in every scenario, filled from the current + // value where a scenario doesn't set it, so small disjoint scenarios multiply. + // Each result cell is a row of the summary too, with a cell per scenario. + if changing.len().saturating_add(results.len()).saturating_mul(per.len()) > MAX_SUMMARY_CELLS { + return Err(bad(CMD, "the scenarios together change too many cells for a summary")); + } let current: Vec = changing.iter().map(|c| sh.value(*c)).collect(); let current_results: Vec = results.iter().map(|c| sh.value(*c)).collect(); let mut trial = Trial::new(d); diff --git a/crates/engine/src/cmd/whatif/tests.rs b/crates/engine/src/cmd/whatif/tests.rs index c1476e401..ec3f5262d 100644 --- a/crates/engine/src/cmd/whatif/tests.rs +++ b/crates/engine/src/cmd/whatif/tests.rs @@ -297,3 +297,38 @@ fn consolidate_by_position_and_label() { assert!(s.execute("data.consolidate", json!({"sources": []})).is_err()); assert!(s.execute("data.consolidate", json!({"sources": ["A1"], "function": "median"})).is_err()); } + +/// A hundred thousand positional rows take one key lookup each; a tall source and a wide one +/// whose keys would meet in a table of 160 million cells are refused before it is built. +#[test] +fn consolidate_keys_are_indexed_and_the_output_is_bounded() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [[1], [2]]})).unwrap(); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A100000"], "destination": "C1"})).unwrap(); + assert_eq!(r["rows"], 100_000); + assert_eq!(n(&s, "Sheet1!C2"), 2.0); + let r = s.execute("data.consolidate", json!({"sources": ["A1:A10000", "A1:XFC1"], "destination": "A1"})); + assert!(r.unwrap_err().to_string().contains("too large")); +} + +#[test] +fn scenario_summary_refuses_a_matrix_over_its_cell_limit() { + let mut s = s(); + // 200 scenarios of 32 disjoint cells: 6,400 changing cells x 200 scenarios = 1.28 million cells. + for i in 0..200 { + let (a, b) = (i * 32 + 1, i * 32 + 32); + s.execute("data.scenarioAdd", json!({"name": format!("S{i}"), "changing": format!("A{a}:A{b}")})).unwrap(); + } + assert!(s.execute("data.scenarioSummary", json!({})).is_err()); +} + +#[test] +fn scenario_summary_counts_result_rows_in_its_cell_limit() { + let mut s = s(); + // 900 scenarios of one distinct cell each are 810,000 cells; 255 result rows take it past the limit. + for i in 0..900 { + s.execute("data.scenarioAdd", json!({"name": format!("S{i}"), "changing": format!("A{}", i + 1)})).unwrap(); + } + let err = s.execute("data.scenarioSummary", json!({"resultCells": "C1:C255"})).unwrap_err().to_string(); + assert!(err.contains("too many cells"), "{err}"); +} diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index 06a0fa587..4dbdf46e8 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -8,8 +8,13 @@ use gridcraft_model::{Sheet, Workbook}; use gridcraft_numfmt::{Formatted, NumberFormat, format_value}; /// Parsed number formats are cached by code. -fn parsed(code: &str) -> NumberFormat { +pub(crate) fn parsed(code: &str) -> NumberFormat { static CACHE: std::sync::LazyLock>> = std::sync::LazyLock::new(|| Mutex::new(HashMap::new())); + // A code of more bytes than 4 × the character limit can't be within it: it shows as General + // without being hashed or scanned again for every cell that shares it. + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse("General"); + } let lock = || CACHE.lock().unwrap_or_else(std::sync::PoisonError::into_inner); if let Some(f) = lock().get(code) { return f.clone(); @@ -24,7 +29,12 @@ fn parsed(code: &str) -> NumberFormat { f } +/// The parsed format of `code`; one longer than the character limit is a General format that keeps +/// the code it was made from. The engine itself formats through `parsed`, which doesn't copy it. pub fn number_format(code: &str) -> NumberFormat { + if code.len() > 4 * gridcraft_numfmt::MAX_FORMAT_CODE_CHARS { + return NumberFormat::parse(code); + } parsed(code) } @@ -228,3 +238,31 @@ pub fn stats(sheet: &Sheet, ranges: &[RangeRef]) -> Stats { } s } + +#[cfg(test)] +mod tests { + use super::*; + + /// A format code far past Excel's 255 characters shows its values as General, for every cell + /// that shares it, without being hashed into the cache each time. + #[test] + fn an_overlong_code_formats_as_general() { + let wb = Workbook::new(); + let code = format!("0.00{}", "\"x\"".repeat(400_000)); + for _ in 0..1_000 { + assert_eq!(format(&Value::Number(1.5), &code, &wb).text, "1.5"); + } + assert_eq!(format(&Value::Number(1.5), "0.00", &wb).text, "1.50"); + } + + /// The public `number_format` keeps the code of an overlong format; it is General, but not the + /// same format as a real `General`. + #[test] + fn number_format_keeps_an_overlong_code() { + let code = format!("0.00{}", "\"x\"".repeat(2_000)); + let f = number_format(&code); + assert!(f.is_general()); + assert_eq!(f.code(), code); + assert_ne!(f, number_format("General")); + } +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 6cd39f839..894a58764 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -9,6 +9,13 @@ use crate::{EngineError, Result}; mod sylk; +/// The function library's answers to what the XLSX and XLS readers and the XLSX writer need to know about functions +/// (which names are built in, which parameters take a single value). +pub const XLSX_FUNCTIONS: gridcraft_xlsx::FunctionInfo = gridcraft_xlsx::FunctionInfo { + is_builtin: |name| gridcraft_functions::lookup(name).is_some(), + is_scalar_arg: |name, i| gridcraft_functions::lookup(name).is_none_or(|s| s.is_scalar_arg(i)), +}; + /// File formats GridCraft reads or writes. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FileKind { @@ -93,14 +100,18 @@ pub fn open_bytes_with(name: &str, bytes: &[u8], password: Option<&str>, csv_as_ /// line-height estimate can differ from Excel's, and shrinking files that already look /// right would be a visible regression. Custom-height rows are left alone entirely. fn fit_imported_rows(wb: &mut Workbook) { + // The whole pass shares one budget of merge comparisons: later sheets keep their heights once it is spent. + let mut left = crate::cmd::MERGE_FIT_WORK; for si in 0..wb.sheets.len() { let Some(sh) = wb.sheet(si) else { continue }; let mut rows: Vec = sh.cells.iter().map(|(c, _)| c.row).collect(); rows.sort_unstable(); rows.dedup(); + // One sweep over the sheet's merges for all of its rows, in ascending order. + let mut sweep = crate::cmd::MergeSweep::new(left); for row in rows { let before = wb.sheet(si).and_then(|sh| sh.rows.get(&row)).and_then(|i| i.size); - crate::cmd::auto_row_height(wb, si, row); + crate::cmd::fit_row_height(wb, si, row, &mut sweep); let Some(before) = before else { continue }; let after = wb.sheet(si).and_then(|sh| sh.rows.get(&row)).and_then(|i| i.size); if after.is_none_or(|a| before > a + 0.5) @@ -109,6 +120,7 @@ fn fit_imported_rows(wb: &mut Workbook) { sh.rows.entry(row).or_default().size = Some(before); } } + left = sweep.left(); } } @@ -264,7 +276,8 @@ fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, let kind = FileKind::from_path(name); let sniffed = gridcraft_xlsx::sniff(bytes); if sniffed == gridcraft_xlsx::Format::Xls { - let (wb, report) = gridcraft_xlsx::read_xls(bytes).map_err(|e| EngineError::Other(format!("We can't open '{name}': {e}")))?; + let (wb, report) = + gridcraft_xlsx::read_xls_with(bytes, XLSX_FUNCTIONS).map_err(|e| EngineError::Other(format!("We can't open '{name}': {e}")))?; return Ok((wb, report.warnings)); } if sniffed == gridcraft_xlsx::Format::LegacyBinary { @@ -289,12 +302,13 @@ fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, // Text named `.xlsx` is read as text. if sniffed == gridcraft_xlsx::Format::Xlsx || (kind == Some(FileKind::Xlsx) && sniffed != gridcraft_xlsx::Format::Csv) { // Google Sheets' export wrapper is read as the formula inside when GridCraft calculates it (#2871). - let (wb, report) = gridcraft_xlsx::read_xlsx_with(bytes, &gridcraft_calc::is_known_function).map_err(|e| { - EngineError::Other(match (&e, gridcraft_xlsx::oversized_part(bytes)) { - (gridcraft_xlsx::IoError::TooLarge(_), Some((part, size))) => part_too_large_message(name, &part, size), - _ => format!("We can't open '{name}': {e}"), - }) - })?; + let (wb, report) = gridcraft_xlsx::read_xlsx_with(bytes, (XLSX_FUNCTIONS, &gridcraft_calc::is_known_function as &dyn Fn(&str) -> bool)) + .map_err(|e| { + EngineError::Other(match (&e, gridcraft_xlsx::oversized_part(bytes)) { + (gridcraft_xlsx::IoError::TooLarge(_), Some((part, size))) => part_too_large_message(name, &part, size), + _ => format!("We can't open '{name}': {e}"), + }) + })?; return Ok((wb, report.warnings)); } if let Some(read) = read_html_table(name, kind, bytes, csv_as_text) { @@ -502,7 +516,7 @@ pub fn save_bytes_as(wb: &Workbook, path: &str, kind: FileKind) -> Result named, }; let book = std::path::Path::new(path).file_name().and_then(std::ffi::OsStr::to_str).unwrap_or_default(); - gridcraft_xlsx::write_xlsx_named(named.as_ref().unwrap_or(wb), gridcraft_xlsx::XlsxKind::from_path(path), book) + gridcraft_xlsx::write_xlsx_named_with(named.as_ref().unwrap_or(wb), gridcraft_xlsx::XlsxKind::from_path(path), book, XLSX_FUNCTIONS) .map_err(|e| EngineError::Other(e.to_string())) } FileKind::Xls => { @@ -567,7 +581,7 @@ pub fn csv_values_bytes(wb: &Workbook) -> Result> { let Some(sh) = wb.sheet(wb.active_sheet()) else { return Ok(Vec::new()) }; gridcraft_xlsx::write_csv(sh, wb.csv_delimiter.unwrap_or(b','), |c| match sh.value(c) { gridcraft_core::Value::Number(n) if n.is_finite() => { - let shape = crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); + let shape = crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).date_shape(n); match shape.zip(gridcraft_core::date::datetime_from_serial(wb.date_system, n)) { Some(((date, time), d)) => iso(&d, date, time), None => n.to_string(), @@ -860,8 +874,10 @@ pub fn to_html(wb: &Workbook, sheet: usize) -> Result { place(&im.anchor) ); } + let mut svg_budget = crate::chart_svg::MAX_PAGE_SVG_BYTES; for ch in &sh.charts { - if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor)) { + if let Some(svg) = crate::chart_svg::chart_svg(wb, sheet, ch, &place(&ch.anchor), svg_budget)? { + svg_budget -= svg.len(); out.push_str(&svg); } } @@ -879,12 +895,12 @@ thread_local! { /// While one is alive, [`read_file`] and [`write_file`] on this thread refuse every path: a /// stored script's steps run inside one (#2178), so no command reaches the file system from a /// script whatever its parameters are called. Dropping it (also on unwind) lifts the refusal. -pub(crate) struct ScriptSandbox(()); +pub struct ScriptSandbox(std::marker::PhantomData<*const ()>); impl ScriptSandbox { - pub(crate) fn enter() -> Self { + pub fn enter() -> Self { let _ = SCRIPT_SANDBOX.try_with(|c| c.set(c.get().saturating_add(1))); - ScriptSandbox(()) + ScriptSandbox(std::marker::PhantomData) } } @@ -912,6 +928,12 @@ const MAX_READ_BYTES: u64 = 2 * 1024 * 1024 * 1024; pub fn read_file(path: &str) -> Result> { use std::io::Read; script_sandbox_check(path)?; + // Checked before opening: opening a FIFO nobody writes to blocks the calling (GUI) thread. + // shortcut: a FIFO swapped in between this check and the open still blocks; close that with + // O_NONBLOCK when the crate takes a libc dependency. + if !std::fs::metadata(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?.is_file() { + return Err(EngineError::Other(format!("{path}: only regular files can be opened"))); + } let f = std::fs::File::open(path).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; // A symlink to an endless device reports length 0 but streams forever, so metadata // alone never bounds the read: require a real file and cap the bytes actually taken. @@ -943,18 +965,37 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { // link's target; and it inherits the destination's access restrictions, so saving a // private workbook doesn't widen who can read it. let tmp = format!("{path}.sctmp"); - let prior = std::fs::metadata(path).ok().map(|m| m.permissions()); + let prior_meta = std::fs::metadata(path).ok(); + let prior = prior_meta.as_ref().map(std::fs::Metadata::permissions); let mut opts = std::fs::OpenOptions::new(); opts.write(true).create_new(true); + // Over an existing file the replacement starts owner-only, whatever the umask: its group is + // still the saving user's primary one until it is changed below, and a member of that group + // must not be able to open it (and keep the descriptor) before then. #[cfg(unix)] - if let Some(p) = &prior { - use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; - opts.mode(p.mode()); + if prior.is_some() { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); } let mut f = opts.open(&tmp).map_err(|e| { let _ = std::fs::remove_file(&tmp); EngineError::Other(format!("{path}: {e}")) })?; + // The replacement belongs to the saving user's primary group: restore the old file's group and + // then its mode. When the group can't be restored its access is dropped rather than handed to a + // different group; failing to read that back is a failed save. + #[cfg(unix)] + if let Some(m) = &prior_meta { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + let _ = std::os::unix::fs::fchown(&f, None, Some(m.gid())); + let group_kept = f.metadata().map(|now| now.gid() == m.gid()).map_err(|e| { + let _ = std::fs::remove_file(&tmp); + EngineError::Other(format!("{path}: {e}")) + })?; + // Best effort: a file system that can't change modes keeps the owner-only one. Without the + // group, the old owner and other bits are kept and the group, setuid and setgid bits dropped. + let _ = f.set_permissions(std::fs::Permissions::from_mode(restored_mode(m.mode(), group_kept))); + } #[cfg(not(unix))] if let Some(p) = &prior { let _ = f.set_permissions(p.clone()); @@ -971,6 +1012,13 @@ pub fn write_file(path: &str, bytes: &[u8]) -> Result<()> { }) } +/// The mode a saved file gets from the file it replaces (`old`): all of it when its group was +/// restored; without the group, the owner and other bits only. +#[cfg(unix)] +fn restored_mode(old: u32, group_kept: bool) -> u32 { + if group_kept { old & 0o7777 } else { old & 0o707 } +} + #[cfg(target_arch = "wasm32")] pub fn write_file(path: &str, _bytes: &[u8]) -> Result<()> { script_sandbox_check(path)?; diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index ae5ea8832..c68c43f48 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -51,6 +51,14 @@ fn save_keeps_the_destination_access_restrictions() { write_file(dest.to_str().unwrap(), b"v2").unwrap(); let mode = std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777; assert_eq!(mode, 0o600); + // The mode is set outright from the old file's, not through the umask: group and other access + // it had stay as they were. + for old in [0o640, 0o664] { + std::fs::set_permissions(&dest, std::fs::Permissions::from_mode(old)).unwrap(); + write_file(dest.to_str().unwrap(), b"v3").unwrap(); + assert_eq!(std::fs::metadata(&dest).unwrap().permissions().mode() & 0o777, old); + } + assert!(!dir.join("private.xlsx.sctmp").exists()); // And a brand-new file is created with the restrictive, not permissive, default. let fresh = dir.join("new.xlsx"); write_file(fresh.to_str().unwrap(), b"v").unwrap(); @@ -59,6 +67,17 @@ fn save_keeps_the_destination_access_restrictions() { let _ = std::fs::remove_dir_all(&dir); } +/// Without the old group, a saved file keeps the old owner and other bits and nothing for a group. +#[cfg(not(target_arch = "wasm32"))] +#[cfg(unix)] +#[test] +fn restored_mode_drops_the_group_and_special_bits_when_the_group_is_lost() { + assert_eq!(super::restored_mode(0o100_664, true), 0o664); + assert_eq!(super::restored_mode(0o100_664, false), 0o604); + assert_eq!(super::restored_mode(0o102_444, false), 0o404, "an original read-only file stays read-only, setgid is dropped"); + assert_eq!(super::restored_mode(0o104_755, false), 0o705, "setuid is dropped"); +} + #[cfg(not(target_arch = "wasm32"))] #[test] fn open_refuses_directories_and_unknowable_lengths() { @@ -351,6 +370,19 @@ fn script_sandbox_refuses_file_access_until_dropped() { let _ = std::fs::remove_dir_all(&dir); } +/// The guard's drop lowers the counter of the thread it runs on, so it must not move between threads: +/// naming the trait's method is ambiguous (a compile error) if `ScriptSandbox` is `Send`. +#[test] +fn script_sandbox_is_not_send() { + trait AmbiguousIfSend { + fn check() {} + } + impl AmbiguousIfSend<()> for T {} + struct IsSend; + impl AmbiguousIfSend for T {} + >::check(); +} + /// A minimal compound (CFB/OLE2) file — FAT in sector 0, directory in sector 1 — holding a /// root entry and the given stream names. fn cfb(streams: &[&str]) -> Vec { @@ -761,3 +793,20 @@ fn sylk_survives_huge_relative_offsets_and_width_floods() { let (wb, _) = open_bytes("widths.slk", &flood).unwrap(); assert_eq!(wb.sheet(0).unwrap().value(CellRef::parse("A1").unwrap()), gridcraft_core::Value::Number(1.0)); } + +/// A FIFO with no writer would block an open forever: it is refused before it is opened. +#[cfg(unix)] +#[test] +fn reading_a_fifo_is_refused_without_blocking() { + let dir = std::env::temp_dir().join(format!("gridcraft-fifo-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let fifo = dir.join("pipe.xlsx"); + if !std::process::Command::new("mkfifo").arg(&fifo).status().is_ok_and(|s| s.success()) { + let _ = std::fs::remove_dir_all(&dir); + return; // no mkfifo here + } + let err = read_file(fifo.to_str().unwrap()).unwrap_err().to_string(); + assert!(err.contains("only regular files"), "{err}"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index c608a8e39..58979526c 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -237,6 +237,22 @@ const JOURNAL_MAX_ENTRIES: usize = 10_000; pub(crate) const JOURNAL_MAX_BYTES: usize = 64 * 1024 * 1024; const UNDO_BYTES_LIMIT: usize = 128 * 1024 * 1024; +/// Comment and hyperlink text of a sheet: a changed sheet's snapshot keeps its own copy of these +/// strings (an edit deep-copies them), so they count against the history budget with the cells. +fn sheet_text_bytes(sheet: &gridcraft_model::Sheet) -> usize { + let comments = sheet.comments.values().fold(0usize, |sum, c| { + // The vectors' own storage counts too: an empty reply still takes an element. + let replies = c.replies.iter().fold(0usize, |n, (a, t)| n.saturating_add(a.len()).saturating_add(t.len())); + let replies = replies.saturating_add(c.replies.capacity().saturating_mul(std::mem::size_of::<(String, String)>())); + let dates = c.dates.iter().fold(0usize, |n, d| n.saturating_add(d.len())); + let dates = dates.saturating_add(c.dates.capacity().saturating_mul(std::mem::size_of::())); + sum.saturating_add(std::mem::size_of::() + c.author.len() + c.text.len() + replies + dates) + }); + sheet.hyperlinks.values().fold(comments, |sum, h| { + sum.saturating_add(std::mem::size_of::() + h.target.len() + h.tooltip.as_ref().map_or(0, String::len)) + }) +} + fn history_retained_bytes(older: &Workbook, newer: &Workbook) -> usize { let mut bytes = std::mem::size_of::(); for (index, sheet) in older.sheets.iter().enumerate() { @@ -248,6 +264,7 @@ fn history_retained_bytes(older: &Workbook, newer: &Workbook) -> usize { let empty = gridcraft_model::CellStore::new(); let next = newer.sheets.get(index).map_or(&empty, |next| &next.cells); bytes = bytes.saturating_add(sheet.cells.retained_bytes_against(next)); + bytes = bytes.saturating_add(sheet_text_bytes(sheet)); } // Snapshots share the macro list until a change to it; a snapshot with its own list keeps it. if !older.macros.shares(&newer.macros) { diff --git a/crates/engine/src/macros/recorder.rs b/crates/engine/src/macros/recorder.rs index 337c41661..ef9be781f 100644 --- a/crates/engine/src/macros/recorder.rs +++ b/crates/engine/src/macros/recorder.rs @@ -440,7 +440,7 @@ pub fn infer_input(wb: &Workbook, sheet: usize, r: RangeRef) -> NamedInput { Some(match v { Value::Empty => return None, Value::Text(t) if t.is_empty() => return None, - Value::Number(_) if crate::display::number_format(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, + Value::Number(_) if crate::display::parsed(wb.styles.get(sh.style_id(c)).num_fmt.as_str()).is_date() => ColType::Date, Value::Number(_) => ColType::Number, Value::Text(_) => ColType::Text, Value::Bool(_) => ColType::Bool, diff --git a/crates/engine/src/macros/run.rs b/crates/engine/src/macros/run.rs index 9e31e3ca9..96c6f504a 100644 --- a/crates/engine/src/macros/run.rs +++ b/crates/engine/src/macros/run.rs @@ -335,7 +335,7 @@ fn check_input(wb: &Workbook, input: &gridcraft_model::NamedInput, place: Place) gridcraft_model::ColType::Number => v.is_number(), gridcraft_model::ColType::Text => v.is_text(), gridcraft_model::ColType::Bool => matches!(v, Value::Bool(_)), - gridcraft_model::ColType::Date => v.is_number() && crate::display::number_format(&date()).is_date(), + gridcraft_model::ColType::Date => v.is_number() && crate::display::parsed(&date()).is_date(), }; if !ok { wrong.push((c, holds(&v, ty))); diff --git a/crates/engine/src/pivot.rs b/crates/engine/src/pivot.rs index bfaccace4..1b5d48155 100644 --- a/crates/engine/src/pivot.rs +++ b/crates/engine/src/pivot.rs @@ -182,7 +182,7 @@ pub fn read_source(wb: &Workbook, text: &str, default_sheet: usize) -> Result Option Vec { @@ -64,6 +68,7 @@ pub fn chart_series(sheet: &str, origin: CellRef, out: &Output) -> Vec { let categories = out.label_col.and_then(|c| rows_ref(sheet, origin, c, &rows)); out.chart_cols .iter() + .take(MAX_CHART_SERIES) .filter_map(|(c, name)| { Some(Series { name: Some(format!("\"{}\"", name.replace('"', "\"\""))), @@ -106,3 +111,22 @@ pub fn clear_output(wb: &mut Workbook, sheet: usize, idx: usize) { p.last_range = None; } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_pivot_chart_gets_at_most_the_series_the_renderer_reads() { + let out = Output { + grid: vec![], + row_kinds: vec![], + body_offset: 0, + width: 1000, + chart_rows: vec![1, 2, 3], + chart_cols: (1..1000).map(|c| (c, format!("S{c}"))).collect(), + label_col: Some(0), + }; + assert_eq!(chart_series("Sheet1", CellRef::new(0, 0), &out).len(), MAX_CHART_SERIES); + } +} diff --git a/crates/engine/src/pivot/compute.rs b/crates/engine/src/pivot/compute.rs index 90e81408d..2bb6bdbc6 100644 --- a/crates/engine/src/pivot/compute.rs +++ b/crates/engine/src/pivot/compute.rs @@ -1,5 +1,7 @@ use super::items::{Acc, FieldItems, build_tree, collect_siblings, is_collapsed, item_label}; -use super::output::{AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, data_cols_exceed_sheet, flatten_cols}; +use super::output::{ + AggMap, CellKind, ColKind, ColSpec, Ctx, Grid, Line, LineKind, Output, RowKind, ValSrc, caption, data_cols_exceed_sheet, flatten_cols, +}; use super::{ CalcSet, CellError, CellRef, HashMap, HashSet, MAX_AXIS_FIELDS, MAX_COLS, MAX_VALUE_FIELDS, PivotDateGroup, PivotField, PivotFilter, PivotFunc, PivotLayout, PivotShowAs, PivotTable, PivotValue, Source, Value, Workbook, read_source, @@ -94,6 +96,10 @@ struct AggregateData { col_leaves: HashSet>, } +/// Most accumulators (groups times value columns) one report may hold: twice what the 5 million +/// group limit allowed at one value field. +const MAX_ACCUMULATORS: usize = 16_000_000; + fn aggregate_record( input: &AggregateInput<'_>, data: &mut AggregateData, @@ -152,7 +158,9 @@ fn aggregate_record( if !input.cols.is_empty() && data_cols_exceed_sheet(data.col_leaves.len(), input.pt.grand_totals_rows, input.vals.len()) { return Err("The PivotTable report will not fit on the sheet.".into()); } - if data.aggs.len() > 5_000_000 { + // Each entry holds one accumulator per value field and calculated-field source column, so the + // width counts too: a handful of groups over hundreds of fields is large as well. + if data.aggs.len() > 5_000_000 || data.aggs.len().saturating_mul(input.vals.len() + input.extra.len()) > MAX_ACCUMULATORS { return Err("The PivotTable report is too large.".into()); } Ok(()) @@ -259,8 +267,8 @@ fn write_headers(g: &mut Grid, input: HeaderInput<'_>) -> Result<(), String> { let lv = spec.path.len().saturating_sub(1); let label = spec.path.last().and_then(|it| col_items.get(lv).map(|fi| fi.label(*it))).unwrap_or_default(); let text = match vi { - Some(vi) if k > 1 => format!("{label} {}", val_name(*vi)), - _ => format!("{label} Total"), + Some(vi) if k > 1 => caption(&label, &val_name(*vi)), + _ => caption(&label, "Total"), }; g.set(first_level + lv, c, Value::text(text), CellKind::ColLabel, 0, None); } @@ -318,6 +326,7 @@ struct ShowContext<'a> { value_index: usize, level0: &'a [u32], siblings: &'a HashMap, Vec>, + cache: &'a ShowCache, } fn number(value: &Option) -> Option { @@ -334,42 +343,89 @@ fn ratio(a: Value, b: Option) -> Value { } } +/// What Running Total and Rank need from other cells, computed once per sibling set instead of +/// once per displayed value (which rescanned the whole set every time). +type ShowKey = (Vec, Vec, usize); +/// Cumulative values by top-level item, and the total of all of them. +type RunningTotals = (HashMap, f64); +/// Top-level items by (rest of a row path, column path). +type ItemsByRest = HashMap<(Vec, Vec), Vec>; + +#[derive(Default)] +struct ShowCache { + /// Ranked values' sibling numbers, sorted ascending without repeats, by (parent, column, value). + ranks: std::cell::RefCell>>, + /// Running totals by top-level item, with the total of all of them, by (rest of path, column, value). + running: std::cell::RefCell>, + /// The top-level items that have a row path ending in `rest` (with column path `col`), in + /// top-level order, by (`rest`, `col`): built once from the report's cells, so a running total + /// visits the items that can hold a number, not every top-level item per distinct `rest`. + items_by_rest: std::cell::RefCell>, +} + fn running_total(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { - let Some((first, _)) = context.row_path.split_first() else { return raw }; + let Some((first, rest)) = context.row_path.split_first() else { return raw }; if !matches!(raw, Value::Number(_)) { return raw; } - let mut total = 0.0; - for item in context.level0 { - let mut path = context.row_path.to_vec(); - if let Some(first_item) = path.first_mut() { - *first_item = *item; + let mut index = context.cache.items_by_rest.borrow_mut(); + let items_by_rest = index.get_or_insert_with(|| { + let position: HashMap = context.level0.iter().enumerate().rev().map(|(at, item)| (*item, at)).collect(); + let mut map = ItemsByRest::new(); + for (row, col) in lookup.aggs.keys() { + if let Some((item, rest)) = row.split_first() + && position.contains_key(item) + { + map.entry((rest.to_vec(), col.clone())).or_default().push(*item); + } } - total += number(&lookup.get(&path, context.col_path, context.value_index)).unwrap_or(0.0); - if item == first { - break; + for items in map.values_mut() { + items.sort_by_key(|item| position.get(item).copied().unwrap_or(usize::MAX)); } - } - Value::number(total) + map + }); + let mut cache = context.cache.running.borrow_mut(); + let (by_item, all) = cache.entry((rest.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut total = 0.0; + let mut by_item = HashMap::new(); + for item in items_by_rest.get(&(rest.to_vec(), context.col_path.to_vec())).into_iter().flatten() { + let mut path = Vec::with_capacity(rest.len() + 1); + path.push(*item); + path.extend_from_slice(rest); + // An item with no number here adds nothing and is not stored (the shown cell always has one). + let Some(value) = number(&lookup.get(&path, context.col_path, context.value_index)) else { continue }; + total += value; + by_item.entry(*item).or_insert(total); + } + (by_item, total) + }); + Value::number(by_item.get(first).copied().unwrap_or(*all)) } fn ranked_value(raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { let Some((_, parent)) = context.row_path.split_last() else { return Value::Empty }; let Some(me) = raw.as_f64() else { return raw }; - let siblings = context.siblings.get(parent).cloned().unwrap_or_default(); + let mut cache = context.cache.ranks.borrow_mut(); + let sorted = cache.entry((parent.to_vec(), context.col_path.to_vec(), context.value_index)).or_insert_with(|| { + let mut all: Vec = context + .siblings + .get(parent) + .into_iter() + .flatten() + .filter_map(|sibling| { + let mut path = parent.to_vec(); + path.push(*sibling); + number(&lookup.get(&path, context.col_path, context.value_index)) + }) + .collect(); + all.sort_by(f64::total_cmp); + all.dedup(); + all + }); // Excel's ranks are dense: ties share a rank and the next value takes the next one // (5, 4, 3, 3, 4 rank 1, 2, 3, 3, 2), so count the distinct larger values. - let mut larger: Vec = siblings - .into_iter() - .filter_map(|sibling| { - let mut path = parent.to_vec(); - path.push(sibling); - number(&lookup.get(&path, context.col_path, context.value_index)).filter(|value| *value > me) - }) - .collect(); - larger.sort_by(f64::total_cmp); - larger.dedup(); - Value::number(larger.len() as f64 + 1.0) + let larger = sorted.len() - sorted.partition_point(|value| *value <= me); + Value::number(larger as f64 + 1.0) } fn show_value(definition: &PivotValue, raw: Value, lookup: &ValueLookup<'_>, context: &ShowContext<'_>) -> Value { @@ -642,6 +698,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro }) .collect(); let lookup = ValueLookup { wb, source: src, calc: &calc, vals: &vals, extra: &extra, aggs: &aggs, value_count: k }; + let show_cache = ShowCache::default(); let mut chart_rows: Vec = Vec::new(); let chart_cols: Vec<(u32, String)> = data_cols .iter() @@ -699,7 +756,7 @@ pub(super) fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut pro let Some((_, vdef)) = vals.get(*vi) else { continue }; let cp = spec.path.as_slice(); let Some(raw) = lookup.get(rp, cp, *vi) else { continue }; - let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings }; + let context = ShowContext { row_path: rp, col_path: cp, value_index: *vi, level0: &level0, siblings: &siblings, cache: &show_cache }; let shown = show_value(vdef, raw, &lookup, &context); if let Some(pr) = probe.as_deref_mut() { if pr.at == Some((r, l + j)) { diff --git a/crates/engine/src/pivot/items.rs b/crates/engine/src/pivot/items.rs index 79b74da1a..6dee1dc5a 100644 --- a/crates/engine/src/pivot/items.rs +++ b/crates/engine/src/pivot/items.rs @@ -21,7 +21,7 @@ pub(super) struct FieldItems { } fn cmp_items(a: &Item, b: &Item) -> Ordering { - a.class.cmp(&b.class).then(a.num.total_cmp(&b.num)).then_with(|| a.text.cmp(&b.text)) + a.class.cmp(&b.class).then(a.num.total_cmp(&b.num)).then_with(|| gridcraft_core::compare_text(&a.text, &b.text)) } /// (identity key, item) for a source value under a field's date grouping. diff --git a/crates/engine/src/pivot/output.rs b/crates/engine/src/pivot/output.rs index c3d6ee88d..ca680335b 100644 --- a/crates/engine/src/pivot/output.rs +++ b/crates/engine/src/pivot/output.rs @@ -121,6 +121,18 @@ pub(super) enum ColKind { Grand, } +/// Longest item caption Excel keeps; a subtotal caption repeats its parent's label for every value +/// field at every level, so a long source label is cut rather than multiplied. +const MAX_CAPTION_LABEL_CHARS: usize = 255; + +/// `label` (cut to [`MAX_CAPTION_LABEL_CHARS`]) and `suffix`, as in "East Total". +pub(super) fn caption(label: &str, suffix: &str) -> String { + let mut out: String = label.chars().take(MAX_CAPTION_LABEL_CHARS).collect(); + out.push(' '); + out.push_str(suffix); + out +} + pub(super) struct ColSpec { pub(super) kind: ColKind, pub(super) path: Vec, @@ -162,7 +174,7 @@ impl Ctx<'_> { leaf: false, path: prefix.to_vec(), values: true, - labels: vec![(col, Value::text(format!("{label} {name}")), indent, None)], + labels: vec![(col, Value::text(caption(label, name)), indent, None)], vi: Some(vi), }); } @@ -249,7 +261,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), indent, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), indent, None)], vi, }); } @@ -316,7 +328,7 @@ impl Ctx<'_> { leaf: false, path: prefix.clone(), values: true, - labels: vec![(col, Value::text(format!("{label} Total")), 0, None)], + labels: vec![(col, Value::text(caption(&label, "Total")), 0, None)], vi, }); } @@ -438,3 +450,17 @@ pub(super) fn cell_style(lk: &Look, rk: RowKind, cell: &OutCell, last_header: bo } s } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn captions_cut_a_long_parent_label() { + assert_eq!(caption("East", "Total"), "East Total"); + let long = "é".repeat(100_000); + let c = caption(&long, "Sum of Sales"); + assert_eq!(c.chars().count(), MAX_CAPTION_LABEL_CHARS + " Sum of Sales".len()); + assert!(c.ends_with(" Sum of Sales")); + } +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index f97e1bc10..6555a1027 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -18,6 +18,44 @@ fn v(s: &Session, a: &str) -> Value { s.doc().unwrap().wb.active().unwrap().value(CellRef::parse(a).unwrap()) } +#[test] +fn format_cells_reading_order_survives_undo_and_xlsx() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "مرحبا 123 ABC"})).unwrap(); + s.execute("home.bold", json!({"range": "A1"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"align": {"reading_order": "RightToLeft"}}})).unwrap(); + let alignment = |s: &Session| { + let d = s.doc().unwrap(); + let style = d.wb.styles.get(d.wb.active().unwrap().style_id(CellRef::new(0, 0))); + assert!(style.font.bold); + serde_json::to_value(style.align).unwrap() + }; + assert_eq!(alignment(&s)["reading_order"], "RightToLeft"); + s.execute("edit.undo", json!({})).unwrap(); + assert!(alignment(&s)["reading_order"].is_null()); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(alignment(&s)["reading_order"], "RightToLeft"); + for order in [json!("invalid"), json!(2)] { + assert!(s.execute("home.formatCells", json!({"range": "A1", "style": {"align": {"reading_order": order}}})).is_err()); + assert_eq!(alignment(&s)["reading_order"], "RightToLeft"); + } + let wb = &s.doc().unwrap().wb; + let (reopened, _) = gridcraft_xlsx::read_xlsx(&gridcraft_xlsx::write_xlsx(wb).unwrap()).unwrap(); + let cell = CellRef::new(0, 0); + assert_eq!(reopened.styles.get(reopened.active().unwrap().style_id(cell)), wb.styles.get(wb.active().unwrap().style_id(cell))); + assert_eq!(reopened.active().unwrap().value(cell), Value::text("مرحبا 123 ABC")); + for order in ["LeftToRight", "Context"] { + s.execute("home.formatCells", json!({"range": "A1", "style": {"align": {"reading_order": order}}})).unwrap(); + assert_eq!(alignment(&s)["reading_order"], order); + } + s.execute("home.formatCells", json!({"range": "A1", "style": {"align": {"reading_order": null}}})).unwrap(); + assert!(alignment(&s)["reading_order"].is_null()); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(alignment(&s)["reading_order"], "Context"); + s.execute("edit.redo", json!({})).unwrap(); + assert!(alignment(&s)["reading_order"].is_null()); +} + #[test] fn entering_empty_cells_preserves_whole_sheet_formatting() { let mut s = s(); @@ -1312,6 +1350,18 @@ fn scripts_leave_file_access_working_after_they_run() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn get_data_refuses_json_with_too_many_fields_or_rows_before_building_rows() { + let mut s = s(); + let wide = format!("[{}]", (0..=gridcraft_core::MAX_COLS).map(|i| format!("{{\"k{i}\":1}}")).collect::>().join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": wide})).unwrap_err().to_string(); + assert!(err.contains("more fields"), "{err}"); + // MAX_ROWS objects plus the header row are one too many. + let tall = format!("[{}]", vec!["{}"; gridcraft_core::MAX_ROWS as usize].join(",")); + let err = s.execute("data.fromTextCsv", json!({"text": tall})).unwrap_err().to_string(); + assert!(err.contains("more rows"), "{err}"); +} + #[test] fn scripts_cannot_reach_outside_the_workbook_even_when_trusted() { let dir = std::env::temp_dir().join(format!("gridcraft-script-deny-{}", std::process::id())); @@ -1459,6 +1509,28 @@ fn case_sensitive_sort_puts_lowercase_first() { assert_eq!(col, ["a", "A", "aB", "Ab", "b", "B"].map(Value::from)); } +#[test] +fn data_sort_orders_vowelled_arabic_by_letters() { + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["بيت"], ["بَاب"], ["تمر"], ["\u{200F}بحر"]]})).unwrap(); + s.execute("data.sort", json!({"range": "A1:A4", "header": false, "keys": [{"column": "A"}]})).unwrap(); + let col: Vec = (1..=4).map(|r| v(&s, &format!("A{r}"))).collect(); + assert_eq!(col, ["بَاب", "\u{200F}بحر", "بيت", "تمر"].map(Value::from)); +} + +#[test] +fn data_sort_orders_marks_and_digit_scripts_like_excel() { + // Excel 16.0 (build 20527, en-US) sorts this column, as text, into exactly this order. + let want = ["١٢", "۱۲", "ب\u{064E}1", "ب\u{064E}١", "بa", "\u{200F}باب", "ب\u{064B}اب", "ب\u{064E}اب", "ب\u{0652}اب", "ب\u{0651}اب"]; + let mut s = s(); + s.execute("home.numberFormat", json!({"range": "A1:A10", "code": "@"})).unwrap(); + let input: Vec> = want.iter().rev().map(|t| vec![*t]).collect(); + s.execute("range.setValues", json!({"range": "A1", "values": input})).unwrap(); + s.execute("data.sort", json!({"range": "A1:A10", "header": false, "keys": [{"column": "A"}]})).unwrap(); + let col: Vec = (1..=10).map(|r| v(&s, &format!("A{r}"))).collect(); + assert_eq!(col, want.map(Value::from)); +} + #[test] fn sort_with_a_sheet_prefix_sorts_the_named_sheet() { let col = |s: &Session, sheet: &str| -> Vec { @@ -3604,6 +3676,79 @@ fn xls_import_keeps_formulas_and_never_saves_back_to_xls() { assert!(s.execute("file.saveBytes", json!({"format": "xlsx"})).is_ok()); } +#[test] +fn xls_formulas_calculate_with_legacy_implicit_intersection() { + // Data!A1 = 11, B1:G1 = 1, A2 = 5, A3 = 7. Report!B1:G1 are filled, so a formula in Report!A1 + // that spilled would show #SPILL!. Every cached result is 0, so the values below are GridCraft's. + fn rec(id: u16, payload: &[u8]) -> Vec { + [&id.to_le_bytes()[..], &(payload.len() as u16).to_le_bytes(), payload].concat() + } + let u16s = |v: &[u16]| v.iter().flat_map(|x| x.to_le_bytes()).collect::>(); + let bof = |dt: u16| rec(0x0809, &[&0x0600u16.to_le_bytes()[..], &dt.to_le_bytes(), &[0; 12]].concat()); + let number = |r: u16, c: u16, n: f64| rec(0x0203, &[&u16s(&[r, c, 0])[..], &n.to_le_bytes()].concat()); + let formula = |r: u16, c: u16, rgce: &[u8]| { + rec(0x0006, &[&u16s(&[r, c, 0])[..], &0f64.to_le_bytes(), &[0; 6], &(rgce.len() as u16).to_le_bytes(), rgce].concat()) + }; + let area3d = |r0: u16, r1: u16, c0: u16, c1: u16| [&[0x3Bu8, 0, 0][..], &u16s(&[r0, r1, c0, c1])].concat(); + let a2a3 = area3d(1, 2, 0, 0); + let mut data = bof(0x0010); + data.extend([number(0, 0, 11.0), number(1, 0, 5.0), number(2, 0, 7.0)].concat()); + (1..7).for_each(|c| data.extend(number(0, c, 1.0))); + data.extend(rec(0x000A, &[])); + let mut report = bof(0x0010); + (1..7).for_each(|c| report.extend(number(0, c, 0.0))); + // A1 = Data!A1:G1; B2 = Data!$A$2:$A$3*10; C2 = SQRT(Data!$A$2:$A$3) (row 2: SQRT(5)); + // H1 = {SUM(Data!$A$2:$A$3*2)}, a one-cell Ctrl+Shift+Enter array. + let cse = [&a2a3[..], &[0x1E, 2, 0, 0x05, 0x22, 1, 4, 0]].concat(); + report.extend(formula(0, 0, &area3d(0, 0, 0xC000, 6 | 0xC000))); + report.extend(formula(1, 1, &[&a2a3[..], &[0x1E, 10, 0, 0x05]].concat())); + report.extend(formula(1, 2, &[&a2a3[..], &[0x21, 20, 0]].concat())); + report.extend(formula(0, 7, &[1, 0, 0, 7, 0])); + report.extend(rec(0x0221, &[&u16s(&[0, 0])[..], &[7, 7], &[0; 6], &(cse.len() as u16).to_le_bytes(), &cse].concat())); + report.extend(rec(0x000A, &[])); + let boundsheet = |pos: u32, name: &str| rec(0x0085, &[&pos.to_le_bytes()[..], &[0, 0, name.len() as u8, 0], name.as_bytes()].concat()); + let globals = |p1: u32, p2: u32| { + [ + bof(0x0005), + boundsheet(p1, "Data"), + boundsheet(p2, "Report"), + rec(0x01AE, &u16s(&[2, 0x0401])), + rec(0x0017, &u16s(&[1, 0, 0, 0])), + rec(0x000A, &[]), + ] + .concat() + }; + let p1 = globals(0, 0).len() as u32; + let stream = [globals(p1, p1 + data.len() as u32), data, report].concat(); + let bytes = gridcraft_xlsx::testing::compound_file(&[("Workbook", &stream)]); + + let mut s = s(); + s.execute("file.open", json!({"name": "legacy.xls", "base64": crate::io::base64_encode(&bytes)})).unwrap(); + s.execute("formulas.calculateNow", json!({})).unwrap(); + let report = |s: &Session, a1: &str| s.doc().unwrap().wb.sheets[1].value(CellRef::parse(a1).unwrap()); + assert_eq!(report(&s, "A1"), Value::Number(11.0)); + assert_eq!(report(&s, "B2"), Value::Number(50.0)); + assert_eq!(report(&s, "C2"), Value::Number(5f64.sqrt())); + assert_eq!(report(&s, "H1"), Value::Number(24.0)); + // Nothing spilled out of A1 or B2. + assert_eq!(report(&s, "B1"), Value::Number(0.0)); + assert_eq!(report(&s, "B3"), Value::Empty); + // Saved as .xlsx, the formulas carry no dynamic-array metadata. + let r = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let saved = crate::io::base64_decode(r["base64"].as_str().unwrap()).unwrap(); + let (wb, _) = gridcraft_xlsx::read_xlsx(&saved).unwrap(); + let f = |a1: &str| wb.sheets[1].cell(CellRef::parse(a1).unwrap()).and_then(|c| c.formula.clone()).unwrap(); + assert_eq!(f("A1").text, "@Data!A1:G1"); + assert_eq!((f("H1").text.as_str(), f("H1").array), ("SUM(Data!$A$2:$A$3*2)", gridcraft_core::RangeRef::parse("H1"))); + let mut sheet = String::new(); + std::io::Read::read_to_string( + &mut zip::ZipArchive::new(std::io::Cursor::new(&saved)).unwrap().by_name("xl/worksheets/sheet2.xml").unwrap(), + &mut sheet, + ) + .unwrap(); + assert!(!sheet.contains("cm=") && !sheet.contains("SINGLE"), "{sheet}"); +} + #[test] fn malformed_xlsb_keeps_the_current_workbook() { let mut s = s(); @@ -6335,6 +6480,23 @@ fn clearing_links_and_notes_keeps_allow_edit_ranges() { assert!(sh.hyperlinks.is_empty() && sh.comments.is_empty()); } +#[test] +fn formulatext_does_not_reveal_a_hidden_formula() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=1+1"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": false, "hidden": true}}})).unwrap(); + s.execute("home.formatCells", json!({"range": "B1", "style": {"protection": {"locked": false}}})).unwrap(); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "not protected yet"); + s.execute("review.protectSheet", json!({})).unwrap(); + // Without entering the formula again: protecting recalculates what reads the protection. + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("cell.set", json!({"cell": "B1", "input": "=FORMULATEXT(A1)"})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Error(gridcraft_core::CellError::NA)); + s.execute("review.unprotectSheet", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::from("=1+1"), "unprotecting shows it again without re-entering"); +} + #[test] fn protected_sheet_refuses_layout_commands_its_permissions_do_not_allow() { let mut s = s(); @@ -6393,3 +6555,26 @@ fn pivot_new_sheet_destination_respects_workbook_structure_protection() { } assert!(s.execute("insert.pivotTable", json!({"source": "A1:B3", "location": "New"})).is_err()); } + +#[test] +fn history_charges_a_changed_sheets_comment_text() { + let mut s = s(); + s.execute("review.newNote", json!({"cell": "A1", "text": "n".repeat(30_000), "author": "Example"})).unwrap(); + let older = (*s.doc().unwrap().wb).clone(); + s.execute("cell.set", json!({"cell": "B2", "input": "x"})).unwrap(); + let newer = (*s.doc().unwrap().wb).clone(); + // The older snapshot owns the comment text now that the sheet changed under it. + assert!(crate::history_retained_bytes(&older, &newer) >= 30_000); +} + +#[test] +fn history_charges_the_storage_of_empty_comment_replies() { + let mut s = s(); + s.execute("review.newComment", json!({"cell": "A1", "text": "c", "author": "Ann"})).unwrap(); + for _ in 0..500 { + s.execute("review.replyComment", json!({"cell": "A1", "text": "", "author": ""})).unwrap(); + } + let sheet = s.doc().unwrap().wb.sheet(0).unwrap().clone(); + // 500 replies of no text are still 500 elements. + assert!(crate::sheet_text_bytes(&sheet) >= 500 * std::mem::size_of::<(String, String)>()); +} diff --git a/crates/engine/tests/it/row_autofit.rs b/crates/engine/tests/it/row_autofit.rs index 5bea5abca..438a39870 100644 --- a/crates/engine/tests/it/row_autofit.rs +++ b/crates/engine/tests/it/row_autofit.rs @@ -59,3 +59,31 @@ fn whole_row_format_keeps_hidden_rows_hidden() { assert!(info.size.unwrap_or(before) > before + 10.0, "hidden row still grows to its font: before={before}, size={:?}", info.size); assert!(info.hidden, "formatting a hidden row must not unhide it"); } + +#[test] +fn many_merges_fit_rows_the_same_and_skip_merged_cells() { + // Rows are fitted with one sweep over the sheet's merges; a merged cell still never grows + // its row, and rows next to thousands of merges still fit. + let mut s = gridcraft_engine::Session::new(); + s.new_workbook(); + let values: Vec<_> = (0..2_000).map(|i| json!([format!("v{i}")])).collect(); + s.execute("range.setValues", json!({"range": "A1", "values": values})).unwrap(); + { + let sh = sheet_mut(&mut s); + sh.merges.push(gridcraft_engine::core::RangeRef::parse("A3:B4").unwrap()); + for i in 0..20_000u32 { + let row = 4 + i % 1_990; + sh.merges.push(gridcraft_engine::core::RangeRef::new( + gridcraft_engine::core::CellRef::new(row, 3 + (i % 7) * 2), + gridcraft_engine::core::CellRef::new(row + 3, 4 + (i % 7) * 2), + )); + } + } + s.execute("home.fontSize", json!({"range": "A:A", "size": 20})).unwrap(); + let sh = s.doc().unwrap().wb.active().unwrap(); + let default = sh.default_row_height; + assert!(sh.row_height(0) > 30.0 && sh.row_height(1_999) > 30.0, "rows with big text grow"); + assert_eq!(sh.row_height(2), default, "a merged cell does not grow its row"); + assert_eq!(sh.row_height(3), default, "nor any row of the merge"); + assert!(sh.row_height(4) > 30.0 && sh.row_height(1_000) > 30.0, "rows beside other merges grow"); +} diff --git a/crates/engine/tests/legacy_array_functions.rs b/crates/engine/tests/legacy_array_functions.rs new file mode 100644 index 000000000..6e86f2e13 --- /dev/null +++ b/crates/engine/tests/legacy_array_functions.rs @@ -0,0 +1,366 @@ +//! Legacy formulas (no dynamic-array metadata) keep one value through GridCraft (issue #410), on +//! a calculated workbook: +//! - every function that returns several values: a legacy cell formula calling it opens with +//! Excel's implicit `@` and does not spill, and a typed `=@F(…)` saves and reopens with the same +//! single value. The library's functions are found by calling each with sample arguments, so +//! one added later is covered without editing this test. +//! - a legacy formula the analysis cannot finish does not spill and saves as read, without `cm`. + +use std::io::Write; + +use gridcraft_engine::Session; +use gridcraft_engine::core::{CellRef, DateSystem, Value}; +use gridcraft_functions::{Arg, Ctx, FnSpec}; +use serde_json::json; + +struct Probe; + +impl Ctx for Probe { + fn date_system(&self) -> DateSystem { + DateSystem::D1900 + } + fn now_serial(&self) -> f64 { + 45_000.0 + } + fn random(&mut self) -> f64 { + 0.5 + } +} + +/// The 2×2 block of numbers at `A1:B2` in the probe sheets. +fn block() -> Value { + Value::Array(std::sync::Arc::new( + gridcraft_engine::core::Array::new(2, 2, vec![Value::Number(1.0), Value::Number(2.0), Value::Number(3.0), Value::Number(4.0)]).unwrap(), + )) +} + +/// Sample arguments (formula text, value). Single values only at a position the function lifts +/// over, so an array result is the function's own, not lifting. +fn candidates(scalar: bool) -> Vec<(&'static str, Arg)> { + let mut out = vec![ + ("3", Arg::val(Value::Number(3.0))), + ("1", Arg::val(Value::Number(1.0))), + ("\"a1,b22\"", Arg::val(Value::text("a1,b22"))), + ("\",\"", Arg::val(Value::text(","))), + ("\"[0-9]+\"", Arg::val(Value::text("[0-9]+"))), + ("TRUE", Arg::val(Value::Bool(true))), + ]; + if !scalar { + out.insert(0, ("A1:B2", Arg::reference(block()))); + out.insert(1, ("{1,2;3,4}", Arg::val(block()))); + } + out +} + +/// The first sample call of `spec` that returns more than one value, as formula text. +fn array_call(spec: &FnSpec) -> Option { + let max = spec.max_args.min(spec.min_args.max(1) + 2).min(4).max(spec.min_args); + for n in spec.min_args..=max { + let per: Vec<_> = (0..n).map(|i| candidates(spec.is_scalar_arg(i))).collect(); + let per: Vec<_> = if n > 4 { per.into_iter().map(|c| c.into_iter().take(3).collect::>()).collect() } else { per }; + let mut idx = vec![0usize; n]; + let mut tries = 0; + loop { + let args: Vec = idx.iter().zip(&per).map(|(&i, c)| c[i].1.clone()).collect(); + if let Value::Array(a) = gridcraft_functions::call(spec, &args, &mut Probe) + && a.data.len() > 1 + { + let text: Vec<&str> = idx.iter().zip(&per).map(|(&i, c)| c[i].0).collect(); + return Some(format!("{}({})", spec.name, text.join(","))); + } + tries += 1; + // Next combination, last position fastest. + let mut k = n; + loop { + if k == 0 || tries > 3000 { + break; + } + k -= 1; + idx[k] += 1; + if idx[k] < per[k].len() { + break; + } + idx[k] = 0; + if k == 0 { + k = usize::MAX; + break; + } + } + if n == 0 || k == usize::MAX || tries > 3000 { + break; + } + } + } + None +} + +/// Every library function with an array result, as a sample call, and the functions the +/// evaluator implements itself that return several values or a range. +fn array_calls() -> Vec { + let mut calls: Vec = gridcraft_functions::all().iter().filter_map(array_call).collect(); + for found in ["MUNIT", "MODE.MULT", "TEXTSPLIT", "REGEXEXTRACT", "MMULT", "SEQUENCE"] { + assert!(calls.iter().any(|c| c.starts_with(&format!("{found}("))), "{found} not found: {calls:?}"); + } + calls.extend( + ["TRIMRANGE(A1:B2)", "OFFSET(A1:B2,0,0)", "INDIRECT(\"A1:B2\")", "INDEX(A1:B2,0,1)", "FILTERXML(\"12\",\"//b\")"] + .map(String::from), + ); + calls +} + +/// Where the `i`th sample call goes: column E, five rows apart, away from the data in `A1:B2`. +fn slot(i: usize) -> CellRef { + CellRef::new(9 + 5 * i as u32, 4) +} + +fn value(s: &Session, c: CellRef) -> Value { + s.doc().unwrap().wb.active().unwrap().value(c) +} + +/// The cells right of and below `c` are empty and `c` is not #SPILL!: the formula did not spill. +fn one_value(s: &Session, c: CellRef, what: &str) -> Result { + let v = value(s, c); + let right = value(s, CellRef::new(c.row, c.col + 1)); + let below = value(s, CellRef::new(c.row + 1, c.col)); + if right != Value::Empty || below != Value::Empty || v.display().contains("SPILL") { + return Err(format!("{what}: {} spilled ({right:?} right, {below:?} below)", v.display())); + } + Ok(v) +} + +#[test] +fn legacy_formulas_with_array_functions_show_one_value() { + let calls = array_calls(); + let formulas: Vec<_> = calls.iter().enumerate().map(|(i, f)| (slot(i).a1(), f.clone())).collect(); + let s = open(&legacy_xlsx(&formulas, "")); + let wrong: Vec<_> = calls.iter().enumerate().filter_map(|(i, f)| one_value(&s, slot(i), f).err()).collect(); + assert!(wrong.is_empty(), "{wrong:#?}"); +} + +#[test] +fn typed_at_of_array_functions_saves_one_value() { + let calls = array_calls(); + let mut s = session(); + for (a, v) in [("A1", 1), ("B1", 2), ("A2", 3), ("B2", 4)] { + s.execute("cell.set", json!({"cell": a, "input": v.to_string()})).unwrap(); + } + for (i, f) in calls.iter().enumerate() { + s.execute("cell.set", json!({"cell": slot(i).a1(), "input": format!("=@{f}")})).unwrap(); + } + let before: Vec<_> = (0..calls.len()).map(|i| value(&s, slot(i))).collect(); + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let mut back = Session::new(); + back.execute("file.open", json!({"name": "typed.xlsx", "base64": saved["base64"]})).unwrap(); + let wrong: Vec<_> = calls + .iter() + .enumerate() + .filter_map(|(i, f)| match one_value(&back, slot(i), f) { + Ok(v) if v == before[i] => None, + // A volatile function recalculates on open: one number again. + Ok(Value::Number(_)) if volatile(f) => None, + Ok(v) => Some(format!("@{f}: {} before saving, {} after", before[i].display(), v.display())), + Err(e) => Some(e), + }) + .collect(); + assert!(wrong.is_empty(), "{wrong:#?}"); +} + +fn volatile(call: &str) -> bool { + call.split('(').next().and_then(gridcraft_functions::lookup).is_some_and(|spec| spec.volatile) +} + +/// Formulas the analysis covers at Excel's nesting limits, and one it cannot finish (names too +/// large to analyse, whose ranges would spill three rows): calculated, none spills, and saving +/// writes each as read, as a plain ``. +#[test] +fn deep_and_unfinished_legacy_formulas_calculate_and_save_as_read() { + let nested = format!("{}A1:A3{}", "ABS(".repeat(200), ")".repeat(200)); + let parens = format!("{}A1:A3{}", "ABS((".repeat(64), "))".repeat(64)); + let parts = (1..=30).map(|i| format!("Part_{i}")).collect::>().join("+"); + let part = format!("Sheet1!$A$1:$A$3+{}", vec!["Sheet1!$B$1"; 599].join("+")); + let names: String = (1..=30).map(|i| format!(r#"{part}"#)).collect(); + let formulas = [("E10".to_string(), nested), ("E15".into(), parens), ("E20".into(), parts)]; + let mut s = open(&legacy_xlsx(&formulas, &names)); + for (a, _) in &formulas { + one_value(&s, CellRef::parse(a).unwrap(), a).unwrap(); + } + // Legacy Excel intersects A1:A3 with row 10 and 15: #VALUE!. The unfinished formula shows the + // top-left value of its sum, 1 + 599 × 2 per name. + assert_eq!(value(&s, CellRef::parse("E10").unwrap()).display(), "#VALUE!"); + assert_eq!(value(&s, CellRef::parse("E15").unwrap()).display(), "#VALUE!"); + assert_eq!(value(&s, CellRef::parse("E20").unwrap()), Value::Number(30.0 * (1.0 + 599.0 * 2.0))); + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let bytes = gridcraft_engine::io::base64_decode(saved["base64"].as_str().unwrap()).unwrap(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("xl/worksheets/sheet1.xml").unwrap(), &mut xml).unwrap(); + for (a, f) in &formulas { + assert!(xml.contains(&format!("{f}")), "{a} not saved as read: {xml}"); + } + assert!(!xml.contains("cm=") && !xml.contains("SINGLE") && !xml.contains("t=\"array\""), "{xml}"); +} + +/// Legacy unions where one value is expected, a name whose definition is IF of a range, and +/// multi-valued results in the file's own parentheses: calculated, none spills (legacy Excel +/// intersects a union to #VALUE!), and saving writes each as read, as a plain ``. +#[test] +fn legacy_unions_names_and_parentheses_calculate_and_save_as_read() { + let formulas: Vec<(String, String)> = ["SQRT((A1:A2,B1:B2))", "ABS(((A1:A2,B1:B2)))", "(A1:A2,B1:B2)+1", "SQRT(Flag)", "({1,2}+1)", "{1,2}+1"] + .iter() + .enumerate() + .map(|(i, f)| (slot(i).a1(), f.to_string())) + .collect(); + let names = r#"IF(Sheet1!$A$1:$A$3>1,1,0)"#; + let mut s = open(&legacy_xlsx(&formulas, names)); + for (a, _) in &formulas { + one_value(&s, CellRef::parse(a).unwrap(), a).unwrap(); + } + let shown: Vec = formulas.iter().map(|(a, _)| value(&s, CellRef::parse(a).unwrap()).display()).collect(); + assert_eq!(shown, ["#VALUE!", "#VALUE!", "#VALUE!", "0", "2", "2"]); + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let bytes = gridcraft_engine::io::base64_decode(saved["base64"].as_str().unwrap()).unwrap(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("xl/worksheets/sheet1.xml").unwrap(), &mut xml).unwrap(); + for (a, f) in &formulas { + assert!(xml.contains(&format!("{f}")), "{a} not saved as read: {xml}"); + } + assert!(!xml.contains("cm=") && !xml.contains("SINGLE") && !xml.contains("t=\"array\""), "{xml}"); +} + +/// A whole column in a scalar argument of a legacy formula meets the formula's row (issue #1636, +/// from PR #1660): `ISBLANK(B:B)` in row 4 tests `B4`, so the formula shows `C4` instead of +/// spilling into `G5`, and in row 6, where `B6` is blank, shows 0. +#[test] +fn whole_column_in_a_legacy_scalar_argument_meets_the_formulas_row() { + let formulas: Vec<(String, String)> = [ + ("B4", r#""Thierry""#), + ("C4", "7"), + ("G4", r#"IF(ISBLANK(B:B),0,IF(B4="Thierry",C4,0))"#), + ("G5", "1"), + ("G6", r#"IF(ISBLANK(B:B),0,IF(B6="Thierry",C6,0))"#), + ] + .iter() + .map(|(a, f)| (a.to_string(), f.to_string())) + .collect(); + let s = open(&legacy_xlsx(&formulas, "")); + let at = |a: &str| CellRef::parse(a).unwrap(); + assert_eq!(value(&s, at("G4")), Value::Number(7.0), "B:B meets row 4 instead of spilling"); + assert_eq!(value(&s, at("G5")), Value::Number(1.0), "nothing spills below G4"); + assert_eq!(one_value(&s, at("G6"), "G6").unwrap(), Value::Number(0.0), "B6 is blank"); +} + +/// One-cell ranges written as ranges (`B1:B1`) outside any `@`, and formulas whose reading +/// reaches the parser's nesting limit (254 nested functions, far past Excel's 64): saved as read, +/// without `@`. Only a one-cell range under an `@` counts as one an edit may have shrunk. All but +/// the last are calculated without spilling and saved as a plain ``. +#[test] +fn one_cell_ranges_and_formulas_at_the_nesting_limit_save_as_read() { + let deep = |inner: &str| format!("{}{inner}{}", "ABS(".repeat(254), ")".repeat(254)); + let formulas: Vec<(String, String)> = [ + "SQRT(A1:A3)+B1:B1".to_string(), + "A1:A3*B2:B2".into(), + "SQRT(A1:A3)+$B$2:$B$2".into(), + // Names with a one-cell range inside CHOOSE's index or IF's condition. + "SQRT(NC)".into(), + "SQRT(NI)".into(), + "NC+1".into(), + "NI*2".into(), + deep("A1:A3%"), + deep("-A1:A3"), + ] + .into_iter() + .enumerate() + .map(|(i, f)| (slot(i).a1(), f)) + .collect(); + let names = r#"CHOOSE(Sheet1!$B$1:$B$1,Sheet1!$A$1:$A$3,Sheet1!$A$1:$A$3)IF(Sheet1!$B$1:$B$1>0,Sheet1!$A$1:$A$3)"#; + let (last, rest) = formulas.split_last().unwrap(); + let mut s = open(&legacy_xlsx(&formulas, names)); + for (a, _) in rest { + one_value(&s, CellRef::parse(a).unwrap(), a).unwrap(); + } + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let bytes = gridcraft_engine::io::base64_decode(saved["base64"].as_str().unwrap()).unwrap(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("xl/worksheets/sheet1.xml").unwrap(), &mut xml).unwrap(); + for (a, f) in rest { + assert!(xml.contains(&format!("{f}")), "{a} not saved as read: {xml}"); + } + // A reading the parser cannot nest leaves the formula as read: it calculates as a + // dynamic-array formula would, but the file never gets an `@`. + let (a, f) = last; + assert!(xml.contains(&format!("{f}")), "{a} not saved as read: {xml}"); + assert!(!xml.contains('@') && !xml.contains("SINGLE"), "{xml}"); + assert_eq!(xml.matches("cm=").count(), 1, "{xml}"); +} + +/// Deleting rows shrinks the ranges names return (directly, through another name, CHOOSE's and +/// IF's results, parentheses) to one cell: the `@` the reader put on each name is still the one +/// legacy Excel implies, so the formulas save as legacy ones, without `_xlfn.SINGLE`. +#[test] +fn names_shrunk_by_deleting_rows_save_as_legacy() { + let formulas: Vec<(String, String)> = + ["SQRT(MM)", "SQRT(CHM)", "SQRT(NCH)", "SQRT(NIF)", "SQRT(PM)"].into_iter().enumerate().map(|(i, f)| (slot(i).a1(), f.to_string())).collect(); + let names = r#"Sheet1!$A$1:$A$3MMCHOOSE(1,Sheet1!$A$1:$A$3,Sheet1!$B$1:$B$3)IF(1,Sheet1!$A$1:$A$3,Sheet1!$B$1:$B$3)(Sheet1!$A$1:$A$3)"#; + let mut s = open(&legacy_xlsx(&formulas, names)); + s.execute("home.deleteRows", json!({"rows": "2:3"})).unwrap(); + let saved = s.execute("file.saveBytes", json!({"format": "xlsx"})).unwrap(); + let bytes = gridcraft_engine::io::base64_decode(saved["base64"].as_str().unwrap()).unwrap(); + let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + std::io::Read::read_to_string(&mut zip.by_name("xl/worksheets/sheet1.xml").unwrap(), &mut xml).unwrap(); + for (a, f) in &formulas { + assert!(xml.contains(&format!("{f}")), "{a} not saved as legacy {f}: {xml}"); + } + assert!(!xml.contains("SINGLE") && !xml.contains("cm="), "{xml}"); +} + +const CT: &str = r#""#; + +/// A one-sheet package: numbers 1–4 in `A1:B2` and `formulas` (address, file text) saved as +/// legacy formulas, without dynamic-array metadata; `names` is `` elements. +fn legacy_xlsx(formulas: &[(String, String)], names: &str) -> Vec { + let esc = |s: &str| s.replace('&', "&").replace('<', "<").replace('>', ">").replace('"', """); + let mut rows = + String::from(r#"1234"#); + // Cells in one row share its ``, in the order given. + let mut by_row: Vec<(String, String)> = Vec::new(); + for (a, f) in formulas { + let r: String = a.chars().filter(char::is_ascii_digit).collect(); + let c = format!(r#"{}"#, esc(f)); + match by_row.iter_mut().find(|(row, _)| *row == r) { + Some((_, cells)) => cells.push_str(&c), + None => by_row.push((r, c)), + } + } + for (r, cells) in by_row { + rows.push_str(&format!(r#"{cells}"#)); + } + let names = if names.is_empty() { String::new() } else { format!("{names}") }; + let parts: [(&str, String); 5] = [ + ("[Content_Types].xml", CT.into()), + ("_rels/.rels", r#""#.into()), + ("xl/workbook.xml", format!(r#"{names}"#)), + ("xl/_rels/workbook.xml.rels", r#""#.into()), + ("xl/worksheets/sheet1.xml", format!(r#"{rows}"#)), + ]; + let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (name, data) in parts { + z.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + z.write_all(data.as_bytes()).unwrap(); + } + z.finish().unwrap().into_inner() +} + +fn session() -> Session { + let mut s = Session::new(); + s.new_workbook(); + s +} + +fn open(bytes: &[u8]) -> Session { + let mut s = Session::new(); + s.execute("file.open", json!({"name": "legacy.xlsx", "base64": gridcraft_engine::io::base64_encode(bytes)})).unwrap(); + s +} diff --git a/crates/formula/src/parser.rs b/crates/formula/src/parser.rs index f906c4782..2b6f6d408 100644 --- a/crates/formula/src/parser.rs +++ b/crates/formula/src/parser.rs @@ -302,11 +302,15 @@ impl Parser { { return Ok(Expr::Unary(UnOp::Spill, Box::new(r))); } - // Files write `@x` as `_xlfn.SINGLE(x)`. + // Files write `@x` as `_xlfn.SINGLE(x)`; `SINGLE(A1:A3*2)` is `@(A1:A3*2)`. if name == "SINGLE" && args.len() == 1 && let Some(x) = args.pop() { + let x = match x { + Expr::Binary(..) | Expr::Unary(UnOp::Neg | UnOp::Plus | UnOp::Percent, _) => Expr::Paren(Box::new(x)), + x => x, + }; return Ok(Expr::Unary(UnOp::At, Box::new(x))); } // Files may spell a trim reference as a function call (`_xlfn._TRO_ALL(A1:E10)`). diff --git a/crates/formula/src/tests.rs b/crates/formula/src/tests.rs index 8b8215455..6e3829925 100644 --- a/crates/formula/src/tests.rs +++ b/crates/formula/src/tests.rs @@ -106,6 +106,15 @@ fn spill_operator() -> Result<(), Box> { Ok(()) } +/// Files spell `@x` as `_xlfn.SINGLE(x)`; an operation inside keeps its parentheses. +#[test] +fn single_is_the_implicit_intersection_operator() { + assert_eq!(rt("_xlfn.SINGLE(A1:A3)*2"), "@A1:A3*2"); + assert_eq!(rt("_xlfn.SINGLE(A1:A3*2)"), "@(A1:A3*2)"); + assert_eq!(rt("_xlfn.SINGLE(-A1:A3)"), "@(-A1:A3)"); + assert_eq!(rt("SQRT(_xlfn.SINGLE(Data[Qty]))"), "SQRT(@Data[Qty])"); +} + #[test] fn delete_sheet_shrinks_3d_span() -> Result<(), Box> { let order: Vec = ["Sheet1", "Sheet2", "Sheet3", "Results"].iter().map(|s| s.to_string()).collect(); diff --git a/crates/functions/src/database.rs b/crates/functions/src/database.rs index 37289d9fb..1ef713cdc 100644 --- a/crates/functions/src/database.rs +++ b/crates/functions/src/database.rs @@ -70,39 +70,59 @@ fn cell_criterion(v: &Value) -> Option { /// when the label is unknown, criterion). type CriteriaRow = (usize, Vec<(usize, Option, Criterion)>); +/// Most condition tests one selection may make (records times conditions): every record can be +/// tested against every criteria row when none matches, so a modest database and criteria table +/// multiply into billions of tests. Past it the function is `#NUM!`, as lifted lookups are; 5e8 +/// is a few seconds of matching. +const SELECTION_WORK_BUDGET: usize = 500_000_000; + /// Rows (1-based within the database, excluding the header) of matching records. -fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> Vec { +fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> R> { + // The database column under each criteria label, found once per label rather than per cell. + let columns: Vec> = (0..crit.cols) + .map(|c| match crit.get(0, c) { + Some(Value::Text(t)) => find_column(db, t), + Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), + _ => None, + }) + .collect(); // Per criteria row: its index and a list of (criteria column, database column or None when // the label is unknown, criterion). let mut rows: Vec = Vec::new(); + let mut tests = 0usize; for r in 1..crit.rows { let mut conds = Vec::new(); for c in 0..crit.cols { let Some(cell) = crit.get(r, c) else { continue }; let Some(cr) = cell_criterion(cell) else { continue }; - let col = match crit.get(0, c) { - Some(Value::Text(t)) => find_column(db, t), - Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)), - _ => None, - }; - conds.push((c, col, cr)); + conds.push((c, columns.get(c).copied().flatten(), cr)); } + tests = tests.saturating_add(conds.len().max(1)); rows.push((r, conds)); } - let all_match = rows.is_empty(); - (1..db.rows) + // An empty criteria row matches every record without testing a condition, so nothing is charged. + let all_match = rows.is_empty() || rows.iter().any(|(_, conds)| conds.is_empty()); + // Computed criteria (a formula evaluated per record) count like any other test. + if !all_match && db.rows.saturating_sub(1).saturating_mul(tests) > SELECTION_WORK_BUDGET { + return Err(CellError::Num); + } + Ok((1..db.rows) .filter(|&rec| { all_match - || rows.iter().any(|(r, conds)| { - conds.iter().all(|(c, col, cr)| match col { - Some(f) => cr.matches(db.get(rec, *f).unwrap_or(&Value::Empty)), - // Under a label that names no field, only a formula (a computed - // criterion) can hold. - None => ctx.computed_criterion(*r, *c, db, rec).unwrap_or(false), - }) - }) + // Out of work: a computed criterion would fail at once, so records are left out + // without scanning the criteria rows; the per-row check still stops mid-scan. + || (!ctx.exhausted() + && rows.iter().any(|(r, conds)| { + !ctx.exhausted() + && conds.iter().all(|(c, col, cr)| !ctx.exhausted() && match col { + Some(f) => cr.matches(db.get(rec, *f).unwrap_or(&Value::Empty)), + // Under a label that names no field, only a formula (a computed + // criterion) can hold. + None => ctx.computed_criterion(*r, *c, db, rec).unwrap_or(false), + }) + })) }) - .collect() + .collect()) } struct Selection<'a> { @@ -123,7 +143,7 @@ fn select<'a>(a: &'a [Arg], ctx: &mut dyn Ctx) -> R> { let db = as_array(dbv); let crit = as_array(critv); let field = field_index(&db, a)?; - let records = matching_records(&db, &crit, ctx); + let records = matching_records(&db, &crit, ctx)?; Ok(Selection { db, field, records }) } @@ -352,6 +372,63 @@ mod tests { is_err(d("DSUM", n(9.0)), CellError::Value); } + /// Computed criteria are no longer asked for once the host is out of work. + #[test] + fn matching_stops_asking_computed_criteria_when_exhausted() { + struct Spent { + asked: usize, + } + impl crate::Ctx for Spent { + fn date_system(&self) -> gridcraft_core::DateSystem { + gridcraft_core::DateSystem::D1900 + } + fn now_serial(&self) -> f64 { + 0.0 + } + fn random(&mut self) -> f64 { + 0.0 + } + fn computed_criterion(&mut self, _: usize, _: usize, _: &gridcraft_core::Array, _: usize) -> Option { + self.asked += 1; + Some(true) + } + fn exhausted(&self) -> bool { + self.asked > 0 + } + } + let criteria = arr(vec![vec![tv("Formula")], vec![nv(1.0)]]); + let mut ctx = Spent { asked: 0 }; + let spec = crate::lookup("DCOUNT").unwrap(); + crate::call(spec, &[rf(db()), t("Age"), rf(criteria)], &mut ctx); + assert_eq!(ctx.asked, 1); + // Within one criteria row, the conditions after the one that spent the work are skipped too. + let criteria = arr(vec![vec![tv("Formula"), tv("Other")], vec![nv(1.0), nv(1.0)]]); + let mut ctx = Spent { asked: 0 }; + crate::call(spec, &[rf(db()), t("Age"), rf(criteria)], &mut ctx); + assert_eq!(ctx.asked, 1); + } + + /// Security: records times criteria conditions past the selection budget is `#NUM!` before + /// any record is tested; a nonmatching criteria table used to test every record against + /// every criteria row. + #[test] + fn selection_work_is_bounded() { + use gridcraft_core::Array; + let column = |label: &str, len: usize, v: Value| { + let mut data = vec![tv(label)]; + data.extend(std::iter::repeat_n(v, len)); + Value::from(Array::column(data)) + }; + let db = column("Tree", 100_000, tv("Apple")); + let wide = column("Tree", 5_001, tv("=Plum")); + is_err(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(wide)]), CellError::Num); + let narrow = column("Tree", 2, tv("=Apple")); + close(ev("DCOUNTA", vec![rf(db.clone()), t("Tree"), rf(narrow)]), 100_000.0); + // Empty criteria rows test no condition: the first one matches every record, so they cost nothing. + let empty_rows = column("Tree", 5_001, Value::Empty); + close(ev("DCOUNTA", vec![rf(db), t("Tree"), rf(empty_rows)]), 100_000.0); + } + #[test] fn simple_criteria() { let c = |cells: Vec| arr(vec![vec![tv("Tree"), tv("Height")], cells]); diff --git a/crates/functions/src/date.rs b/crates/functions/src/date.rs index 21d2a49c1..2060267c2 100644 --- a/crates/functions/src/date.rs +++ b/crates/functions/src/date.rs @@ -416,13 +416,17 @@ fn workday_core(a: &[Arg], c: &mut dyn Ctx, mask: [bool; 7], hol: Option<&Arg>) if days == 0 { return num_val(start as f64); } + // The holidays on working days, still sorted (`holidays` sorts them), so each step counts + // the ones it passed with two binary searches rather than a scan of every holiday: a run of + // consecutive holidays takes one step per holiday. + let hs: Vec = hs.into_iter().filter(|&h| !mask[weekday0(sys, h)]).collect(); let dir = days.signum(); let mut cur = start; let mut rem = days.abs(); while rem > 0 { let next = advance(sys, cur, rem, dir, &mask, per_week); let (lo, hi) = if dir > 0 { (cur + 1, next) } else { (next, cur - 1) }; - rem = hs.iter().filter(|&&h| h >= lo && h <= hi && !mask[weekday0(sys, h)]).count() as i64; + rem = hs.partition_point(|&h| h <= hi).saturating_sub(hs.partition_point(|&h| h < lo)) as i64; cur = next; if !(0..=MAX_SERIAL as i64).contains(&cur) { return Err(CellError::Num); @@ -755,6 +759,23 @@ mod tests { is_err(ev("TIMEVALUE", vec![t("x")]), CellError::Value); } + /// Security: WORKDAY counts the holidays each step passes by binary search, so a long run of + /// consecutive holidays is linear work, not one scan of every holiday per holiday. + #[test] + fn workday_over_many_consecutive_holidays() { + let start = d(2000.0, 1.0, 3.0); + let hol: Vec = (1..=200_000).map(|i| start + f64::from(i)).collect(); + let out = ev("WORKDAY.INTL", vec![n(start), n(1.0), n(11.0), av(col(&hol))]); + // Sundays are the only weekend (11): 200,000 days of holidays are skipped, landing on the + // first working day after them. + let after = start + 200_001.0; + let sunday = ev("WEEKDAY", vec![n(after), n(2.0)]) == nv(7.0); + close(out, if sunday { after + 1.0 } else { after }); + // Holidays on a weekend don't push the result out. + let fri = d(2026.0, 10.0, 9.0); + close(ev("WORKDAY", vec![n(fri), n(1.0), av(col(&[fri + 1.0, fri + 3.0]))]), fri + 4.0); + } + #[test] fn day_counts() { close(ev("DAYS", vec![n(d(2021.0, 3.0, 15.0)), n(d(2021.0, 2.0, 1.0))]), 42.0); diff --git a/crates/functions/src/dynamic.rs b/crates/functions/src/dynamic.rs index 816af8ea2..15e9ff7c9 100644 --- a/crates/functions/src/dynamic.rs +++ b/crates/functions/src/dynamic.rs @@ -171,13 +171,17 @@ fn sort(args: &[Arg], _c: &mut dyn Ctx) -> R { let a = if by_col { a.transpose() } else { a }; let idx = if has(args, 1) { int_list(&arg(args, 1)?.value)? } else { vec![1] }; let ord = if has(args, 2) { int_list(&arg(args, 2)?.value)? } else { vec![1] }; - if idx.is_empty() || idx.iter().any(|&i| i < 1 || i as usize > a.cols) || ord.iter().any(|&o| o != 1 && o != -1) { + if idx.is_empty() || idx.iter().any(|&i| i < 1 || i > a.cols as i64) || ord.iter().any(|&o| o != 1 && o != -1) { return Err(CellError::Value); } if ord.len() > 1 && ord.len() != idx.len() { return Err(CellError::Value); } let orders: Vec = (0..idx.len()).map(|i| *ord.get(i).or(ord.first()).unwrap_or(&1)).collect(); + // A column repeated in sort_index can't break a tie its first use left, so only first uses + // stay: the keys copied per row are then at most the row itself, however long sort_index is. + let mut seen = std::collections::HashSet::new(); + let (idx, orders): (Vec, Vec) = idx.into_iter().zip(orders).filter(|(i, _)| seen.insert(*i)).unzip(); let mut lines: Vec<(Vec, Vec)> = rows_of(&a).into_iter().map(|r| (idx.iter().map(|&i| r.get(i as usize - 1).cloned().unwrap_or(Value::Empty)).collect(), r)).collect(); lines.sort_by(|x, y| cmp_keys(&x.0, &y.0, &orders)); @@ -928,9 +932,16 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { let ckeys: Vec> = p.rows.iter().map(|&r| key_row(&cf, r)).collect(); let rk = distinct_keys(&rkeys, if row_sort.unsigned_abs() as usize <= rf.cols { row_sort } else { 0 }); let ck = distinct_keys(&ckeys, if col_sort.unsigned_abs() as usize <= cf.cols { col_sort } else { 0 }); + // Source rows of every row key, column key and pair, built in one pass, so a total reads + // its rows instead of rescanning the whole source once per total. let mut cell: HashMap<(String, String), Vec> = HashMap::new(); + let mut by_row: HashMap> = HashMap::new(); + let mut by_col: HashMap> = HashMap::new(); for ((r, c), &i) in rkeys.iter().zip(ckeys.iter()).zip(p.rows.iter()) { - cell.entry((row_key(r), row_key(c))).or_default().push(i); + let (r, c) = (row_key(r), row_key(c)); + by_row.entry(r.clone()).or_default().push(i); + by_col.entry(c.clone()).or_default().push(i); + cell.entry((r, c)).or_default().push(i); } let mut agg = |rows: &[usize], c: usize| -> Value { if rows.is_empty() { @@ -940,14 +951,12 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { aggregate_with(f, &vs, ctx) }; let rows_for = |r: Option<&Vec>, c: Option<&Vec>| -> Vec { - let rkey = r.map(|k| row_key(k)); - let ckey = c.map(|k| row_key(k)); - p.rows - .iter() - .zip(rkeys.iter().zip(ckeys.iter())) - .filter(|(_, (a, b))| rkey.as_ref().is_none_or(|k| *k == row_key(a)) && ckey.as_ref().is_none_or(|k| *k == row_key(b))) - .map(|(&i, _)| i) - .collect() + match (r, c) { + (Some(r), Some(c)) => cell.get(&(row_key(r), row_key(c))).cloned().unwrap_or_default(), + (Some(r), None) => by_row.get(&row_key(r)).cloned().unwrap_or_default(), + (None, Some(c)) => by_col.get(&row_key(c)).cloned().unwrap_or_default(), + (None, None) => p.rows.clone(), + } }; let mut cols: Vec>> = ck.iter().map(Some).collect(); if col_total > 0 { @@ -997,10 +1006,7 @@ fn pivotby(args: &[Arg], ctx: &mut dyn Ctx) -> R { } }; for c in &cols { - let rows = match (r, c) { - (Some(rk), Some(ck)) => cell.get(&(row_key(rk), row_key(ck))).cloned().unwrap_or_default(), - _ => rows_for(r, *c), - }; + let rows = rows_for(r, *c); for vc in 0..vals.cols { line.push(agg(&rows, vc)); } @@ -1204,6 +1210,21 @@ mod tests { assert_eq!(rows_of(&r), vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]); } + /// Security: a column repeated in sort_index keys the sort once. The keys copied per row stay + /// within the row, however long sort_index is, and a repeat (even with the other order) can't + /// change the result, since its first use already ordered the rows. + #[test] + fn sort_drops_repeated_sort_indices() { + let m = || av(arr(vec![vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)], vec![nv(0.0), nv(5.0)]])); + let want = vec![vec![nv(0.0), nv(5.0)], vec![nv(1.0), nv(2.0)], vec![nv(1.0), nv(1.0)]]; + let r = ev("SORT", vec![m(), av(row(&[1.0, 1.0, 2.0, 1.0])), av(row(&[1.0, -1.0, -1.0, 1.0]))]); + assert_eq!(rows_of(&r), want); + let idx = vec![1.0; 1_000_000]; + let tall = av(gridcraft_core::Array::filled(2_000, 1, nv(1.0)).into()); + let r = ev("SORT", vec![tall, av(row(&idx))]); + assert_eq!(rows_of(&r).len(), 2_000); + } + #[test] fn sortby() { let r = ev("SORTBY", vec![av(col(&[1.0, 2.0, 3.0])), av(col(&[30.0, 10.0, 20.0]))]); @@ -1447,6 +1468,16 @@ mod tests { ); } + /// Security: PIVOTBY totals read their source rows from indexes built in the grouping pass, + /// rather than rescanning the source once per total, so many distinct keys stay linear. + #[test] + fn pivotby_totals_with_many_groups() { + let keys: Vec = (0..20_000).map(f64::from).collect(); + let r = rows_of(&ev("PIVOTBY", vec![av(col(&keys)), av(col(&vec![1.0; 20_000])), av(col(&vec![2.0; 20_000])), t("SUM")])); + assert_eq!(r.get(1), Some(&vec![nv(0.0), nv(2.0), nv(2.0)])); + assert_eq!(r.last(), Some(&vec![tv("Total"), nv(40_000.0), nv(40_000.0)])); + } + #[test] fn pivotby_size_check_counts_column_header_rows() { // One row, n column fields and n value columns: n header rows + value-name row + body row + total row, diff --git a/crates/functions/src/info.rs b/crates/functions/src/info.rs index 309f0323e..82eda5828 100644 --- a/crates/functions/src/info.rs +++ b/crates/functions/src/info.rs @@ -91,7 +91,7 @@ fn na(_a: &[Arg], _c: &mut dyn Ctx) -> R { fn n_fn(a: &[Arg], _c: &mut dyn Ctx) -> R { // N(range) reads the range's first cell, but a computed array (N(A1:A3>0), N({...})) is // converted element by element, so N can stand in for `--` inside SUMPRODUCT. - if let Some(Arg { value: Value::Array(arr), from_ref: false }) = a.first() { + if let Some(Arg { value: Value::Array(arr), from_ref: false, .. }) = a.first() { let data = arr.data.iter().map(|v| n_of(v.clone()).unwrap_or_else(Value::Error)).collect(); return Array::new(arr.rows, arr.cols, data).map(Value::from).ok_or(CellError::Calc); } diff --git a/crates/functions/src/lib.rs b/crates/functions/src/lib.rs index ccc61b026..dfca164a2 100644 --- a/crates/functions/src/lib.rs +++ b/crates/functions/src/lib.rs @@ -87,16 +87,19 @@ pub use stat::{aggregate_values, aggregate_values_k}; pub struct Arg { pub value: Value, pub from_ref: bool, + /// A written argument whose value came out blank (a lookup of an empty cell): given, not + /// omitted, although its value is `Value::Empty`. + pub blank_value: bool, } impl Arg { /// A literal (non-reference) argument. pub fn val(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: false } + Arg { value: value.into(), from_ref: false, blank_value: false } } /// An argument that came from a cell or range reference. pub fn reference(value: impl Into) -> Arg { - Arg { value: value.into(), from_ref: true } + Arg { value: value.into(), from_ref: true, blank_value: false } } } @@ -113,6 +116,11 @@ pub trait Ctx { fn computed_criterion(&mut self, _row: usize, _col: usize, _db: &Array, _record: usize) -> Option { None } + /// Whether the cell's work budget has run out: the database functions stop asking for computed + /// criteria then, as every one would fail at once. + fn exhausted(&self) -> bool { + false + } /// For GROUPBY/PIVOTBY: whether their function argument is a LAMBDA (its value is then not used). fn has_lambda(&self) -> bool { false @@ -308,6 +316,12 @@ mod tests { assert_eq!(ev("REGEXREPLACE", vec![t("2026-10-22"), t("(\\d+)-(\\d+)-(\\d+)"), t("$3/$2/$1")]), Value::text("22/10/2026")); assert_eq!(ev("REGEXREPLACE", vec![t("abc"), t("x*"), t("-")]), Value::text("-a-b-c-")); assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(2.0)]), Value::text("a1b#")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-2.0)]), Value::text("a#b2")); + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(-3.0)]), Value::text("a1b2")); + // 2^32 is no occurrence the text has (and on 32-bit targets it must not wrap to 0). + for occurrence in [4_294_967_296.0, -4_294_967_296.0] { + assert_eq!(ev("REGEXREPLACE", vec![t("a1b2"), t("[0-9]"), t("#"), n(occurrence)]), Value::text("a1b2")); + } // One match whose replacement repeats it 10,000 times: refused before it is built. assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000))]), Value::Error(CellError::Value)); assert_eq!(ev("REGEXREPLACE", vec![t(&many), t("^.*$"), t(&"$0".repeat(10_000)), n(1.0)]), Value::Error(CellError::Value)); diff --git a/crates/functions/src/lift.rs b/crates/functions/src/lift.rs index ea7f07fd9..d474bf12d 100644 --- a/crates/functions/src/lift.rs +++ b/crates/functions/src/lift.rs @@ -50,12 +50,7 @@ fn reject_multi_cell_refs(spec: &FnSpec, args: &[Arg]) -> Option> { if !args.iter().enumerate().any(|(i, a)| multi(i, a)) { return None; } - Some( - args.iter() - .enumerate() - .map(|(i, a)| if multi(i, a) { Arg { value: Value::Error(CellError::Value), from_ref: false } } else { a.clone() }) - .collect(), - ) + Some(args.iter().enumerate().map(|(i, a)| if multi(i, a) { Arg::val(Value::Error(CellError::Value)) } else { a.clone() }).collect()) } /// Lookups whose every call searches a lookup vector from its start, so a call lifted over `n` @@ -120,6 +115,8 @@ pub(crate) fn call_lifted(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx) -> Val { let v = arr.get_broadcast(r, c); slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v }; + // A blank element of a written array argument is given, as a scalar blank is. + slot.blank_value = !slot.from_ref && matches!(slot.value, Value::Empty); } } let out = (spec.imp)(&cell_args, ctx); diff --git a/crates/functions/src/lookup.rs b/crates/functions/src/lookup.rs index 17c177d15..6652e00b1 100644 --- a/crates/functions/src/lookup.rs +++ b/crates/functions/src/lookup.rs @@ -132,11 +132,6 @@ fn row(a: &Array, r: usize) -> Vec { (0..a.cols).map(|c| a.get(r, c).cloned().unwrap_or(Value::Empty)).collect() } -/// A looked-up cell value: blanks read as 0 like a reference to an empty cell. -fn cell_result(v: Value) -> Value { - if matches!(v, Value::Empty) { Value::Number(0.0) } else { v } -} - // --------------------------------------------------------------------------------------------- // VLOOKUP / HLOOKUP / LOOKUP / MATCH @@ -160,7 +155,9 @@ fn vh_lookup(args: &[Arg], vertical: bool) -> R { let pos = if approx { bsearch_last(&x, &keys, false) } else { find_exact(&x, &keys, true, false) }; let pos = pos.ok_or(CellError::NA)?; let v = if vertical { table.get(pos, idx) } else { table.get(idx, pos) }; - Ok(cell_result(v.cloned().unwrap_or(Value::Empty))) + // An empty target stays blank, as in Excel: ISBLANK sees it as blank and `&` / + // CONCATENATE treat it as ""; only a formula's final result displays 0 (#374). + Ok(v.cloned().unwrap_or(Value::Empty)) } fn vlookup(a: &[Arg], _c: &mut dyn Ctx) -> R { @@ -193,18 +190,18 @@ fn lookup(args: &[Arg], _c: &mut dyn Ctx) -> R { } else { None }; - return v.map(cell_result).ok_or(CellError::NA); + return v.ok_or(CellError::NA); } // Array form: search the first row of a wide array, else the first column; return the // matching entry from the last row / column. if la.cols > la.rows { let keys = row(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty))) + Ok(la.get(la.rows - 1, pos).cloned().unwrap_or(Value::Empty)) } else { let keys = column(&la, 0); let pos = bsearch_last(&x, &keys, false).ok_or(CellError::NA)?; - Ok(cell_result(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty))) + Ok(la.get(pos, la.cols - 1).cloned().unwrap_or(Value::Empty)) } } @@ -325,13 +322,13 @@ fn xlookup(args: &[Arg], _c: &mut dyn Ctx) -> R { if vertical { let r = row(&ra, p); if r.len() == 1 { - return Ok(cell_result(r.into_iter().next().unwrap_or(Value::Empty))); + return Ok(r.into_iter().next().unwrap_or(Value::Empty)); } array_val(1, r.len(), r) } else { let c = column(&ra, p); if c.len() == 1 { - return Ok(cell_result(c.into_iter().next().unwrap_or(Value::Empty))); + return Ok(c.into_iter().next().unwrap_or(Value::Empty)); } array_val(c.len(), 1, c) } @@ -595,7 +592,8 @@ mod tests { is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), b(false)]), CellError::NA); is_err(ev("VLOOKUP", vec![n(4.0), rf(table()), n(2.0), empty()]), CellError::NA); close(ev("VLOOKUP", vec![n(3.0), rf(table()), n(3.0), b(false)]), 30.0); - close(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), 0.0); + // An empty target stays blank (the cell displays 0 only as a final result) (#374). + assert_eq!(ev("VLOOKUP", vec![n(5.0), rf(table()), n(3.0), b(false)]), Value::Empty); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(4.0)]), CellError::Ref); is_err(ev("VLOOKUP", vec![n(3.0), rf(table()), n(0.0)]), CellError::Value); close(ev("VLOOKUP", vec![t("b*"), rf(names()), n(2.0), b(false)]), 2.0); diff --git a/crates/functions/src/text.rs b/crates/functions/src/text.rs index 24674a93c..53186bb0e 100644 --- a/crates/functions/src/text.rs +++ b/crates/functions/src/text.rs @@ -30,17 +30,27 @@ fn check_len(s: String) -> R { /// Pushes `piece` onto `out`, failing once the result exceeds the cell limit. fn push_capped(out: &mut String, piece: &str) -> R<()> { - out.push_str(piece); - if out.len() > MAX_TEXT * 4 { + // Checked before copying, so one oversized piece is refused without being appended. + if out.len().saturating_add(piece.len()) > MAX_TEXT * 4 { return Err(CellError::Value); } + out.push_str(piece); Ok(()) } +/// `v.to_text()` without copying text values. +fn piece(v: &Value) -> R> { + match v { + Value::Text(t) => Ok(std::borrow::Cow::Borrowed(t)), + Value::Array(a) => a.data.first().map_or(Ok(std::borrow::Cow::Borrowed("")), piece), + _ => v.to_text().map(std::borrow::Cow::Owned), + } +} + fn concatenate(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { - push_capped(&mut out, &scalar(x).to_text()?)?; + push_capped(&mut out, &piece(&scalar(x))?)?; } check_len(out) } @@ -49,31 +59,49 @@ fn concat(a: &[Arg], _c: &mut dyn Ctx) -> R { let mut out = String::new(); for x in a { for v in as_array(&x.value).iter() { - push_capped(&mut out, &v.to_text()?)?; + push_capped(&mut out, &piece(v)?)?; } } check_len(out) } +/// The error `v.to_text()` would give, found without building any text. +fn text_error(v: &Value) -> Option { + match v { + Value::Error(e) => Some(*e), + Value::Array(a) => a.data.first().and_then(text_error), + _ => None, + } +} + fn textjoin(a: &[Arg], _c: &mut dyn Ctx) -> R { - let delims: Vec = as_array(&arg(a, 0)?.value).iter().map(|v| v.to_text()).collect::>()?; + // Errors are found first, in argument order, so they win as before; text is then built one + // piece at a time into the capped output, so many references to one long text (or a long + // delimiter list) stop at the cap instead of first copying every piece. + let delims = as_array(&arg(a, 0)?.value); + if let Some(e) = delims.iter().find_map(text_error) { + return Err(e); + } let ignore = boolean(a, 1)?; - let mut items = Vec::new(); - for x in a.iter().skip(2) { - for v in as_array(&x.value).iter() { - let s = v.to_text()?; - if ignore && s.is_empty() { - continue; - } - items.push(s); - } + let items: Vec<_> = a.iter().skip(2).map(|x| as_array(&x.value)).collect(); + if let Some(e) = items.iter().flat_map(|x| x.iter()).find_map(text_error) { + return Err(e); } let mut out = String::new(); - for (i, s) in items.iter().enumerate() { - if i > 0 && !delims.is_empty() { - push_capped(&mut out, &delims[(i - 1) % delims.len()])?; + let mut joined = 0usize; + for v in items.iter().flat_map(|x| x.iter()) { + let s = piece(v)?; + if ignore && s.is_empty() { + continue; } - push_capped(&mut out, s)?; + if joined > 0 + && !delims.data.is_empty() + && let Some(d) = delims.data.get((joined - 1) % delims.data.len()) + { + push_capped(&mut out, &piece(d)?)?; + } + push_capped(&mut out, &s)?; + joined += 1; } check_len(out) } @@ -914,13 +942,25 @@ fn regexreplace(a: &[Arg], _c: &mut dyn Ctx) -> R { out.push_str(s.get(last..).unwrap_or_default()); return check_len(out); } - let caps: Vec = re.captures_iter(&s).collect(); - let k = occ.unsigned_abs() as usize; - let idx = if occ > 0 { k.checked_sub(1) } else { caps.len().checked_sub(k) }; - let Some(cap) = idx.and_then(|i| caps.get(i)) else { return text_val(s) }; + // An occurrence past what `usize` holds (32-bit targets) names no match: the text is returned as it was. + let Ok(k) = usize::try_from(occ.unsigned_abs()) else { return text_val(s) }; + // Keeps only the wanted match's captures: the nth from the front, or a window of the last `k`. + let cap = if occ > 0 { + re.captures_iter(&s).nth(k - 1) + } else { + let mut last = std::collections::VecDeque::new(); + for cap in re.captures_iter(&s) { + if last.len() == k { + last.pop_front(); + } + last.push_back(cap); + } + if last.len() == k { last.pop_front() } else { None } + }; + let Some(cap) = cap else { return text_val(s) }; let Some(m) = cap.get(0) else { return text_val(s) }; out.push_str(s.get(..m.start()).unwrap_or_default()); - expand_capped(cap, &rep, &mut out, limit).ok_or(CellError::Value)?; + expand_capped(&cap, &rep, &mut out, limit).ok_or(CellError::Value)?; out.push_str(s.get(m.end()..).unwrap_or_default()); check_len(out) } @@ -1171,6 +1211,22 @@ mod tests { assert_eq!(rows_of(&ev("CONCATENATE", vec![av(row_t(&["a", "b"])), t("x")])), vec![vec![tv("ax"), tv("bx")]]); } + /// Security: TEXTJOIN builds its result piece by piece and stops at the text cap, rather + /// than first copying every input (100,000 references to one 32,767-character text were + /// 3 GB of copies). Errors among the inputs still win, in argument order. + #[test] + fn textjoin_stops_at_the_text_cap() { + use gridcraft_core::Array; + let long = tv(&"x".repeat(32_767)); + let many = || av(Array::filled(100_000, 1, long.clone()).into()); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many()]), CellError::Value); + is_err(ev("TEXTJOIN", vec![av(Array::filled(1, 100_000, long.clone()).into()), b(false), t("a"), t("b")]), CellError::Value); + is_err(ev("TEXTJOIN", vec![t(","), b(true), many(), e(CellError::Div0)]), CellError::Div0); + is_err(ev("TEXTJOIN", vec![av(arr(vec![vec![tv(","), Value::Error(CellError::NA)]])), b(true), t("a")]), CellError::NA); + // Delimiters cycle and are converted only where used. + is_text(ev("TEXTJOIN", vec![av(arr(vec![vec![nv(1.0), tv(";")]])), b(false), t("a"), t("b"), t("c"), t("d")]), "a1b;c1d"); + } + fn row_t(xs: &[&str]) -> Value { arr(vec![xs.iter().map(|x| tv(x)).collect()]) } diff --git a/crates/functions/src/util.rs b/crates/functions/src/util.rs index 14c627937..4e7853d2c 100644 --- a/crates/functions/src/util.rs +++ b/crates/functions/src/util.rs @@ -51,7 +51,7 @@ pub(crate) fn arg(args: &[Arg], i: usize) -> R<&Arg> { /// Whether argument `i` was supplied (present and not an empty literal like `F(1,)`). pub(crate) fn has(args: &[Arg], i: usize) -> bool { - matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref) + matches!(args.get(i), Some(a) if !matches!(a.value, Value::Empty) || a.from_ref || a.blank_value) } /// Number coercion for a scalar parameter: empty = 0, TRUE = 1, numeric text parses. diff --git a/crates/math/src/lib.rs b/crates/math/src/lib.rs index 5e1e99a41..408afc413 100644 --- a/crates/math/src/lib.rs +++ b/crates/math/src/lib.rs @@ -3,7 +3,7 @@ //! without calling RaTeX again. //! //! RaTeX is young (0.1.x), so it only ever runs inside this crate's entry points ([`layout`], -//! [`to_omml`], [`check`]). Each one caps its input (4 KB, nesting 32) and runs under +//! [`to_omml`], [`check`]). Each one caps its input (4 KB, nesting 32, no macro definitions, at most [`MAX_CELLS`] matrix cells) and runs under //! `catch_unwind`, so a stray panic becomes a [`MathError`]. See the RaTeX gate in //! `docs/decisions.md` and `tests/gate.rs`. //! @@ -33,6 +33,9 @@ pub const MAX_DEPTH: usize = 32; pub const MAX_EM: f32 = 10_000.0; /// Most items one equation may produce. pub const MAX_ITEMS: usize = 50_000; +/// Most matrix cells one equation may have, counting the empty cells that pad a short row to the +/// width of the widest one. A ragged matrix costs rows times the widest row, not what is typed. +pub const MAX_CELLS: usize = 2_500; /// The error for a source over [`MAX_SOURCE_BYTES`]. It is one shared value so that a cache can /// answer for such a source by its length alone, without reading it. @@ -223,20 +226,61 @@ fn parse(latex: &str) -> Result, MathError> { }) } -/// The size and nesting caps, checked before RaTeX sees the source. +/// Control words that define or rebind macros. User macros have no use in an equation, and +/// RaTeX caps only how many expansions run, not how large they grow, so a self-referencing +/// definition whose argument doubles on every call keeps the expander busy without bound. They are +/// refused before RaTeX sees the source. +const MACRO_DEFINERS: &[&str] = + &["def", "gdef", "edef", "xdef", "let", "futurelet", "global", "long", "newcommand", "renewcommand", "providecommand"]; + +/// One open `\begin…\end` environment, counted from the source text. +struct Grid { + /// Rows ended so far (by `\\`, `\cr` or `\newline`). + rows: usize, + /// Cells in the current row. + cells: usize, + /// Cells in the widest row so far. + widest: usize, +} + +impl Grid { + fn new() -> Grid { + Grid { rows: 0, cells: 1, widest: 0 } + } + fn end_row(&mut self) { + self.widest = self.widest.max(self.cells); + self.rows += 1; + self.cells = 1; + } + /// The padded grid RaTeX lays out and `to_omml` writes: every row as wide as the widest. + fn padded_cells(&self) -> usize { + (self.rows + 1).saturating_mul(self.widest.max(self.cells)) + } +} + +/// The size, nesting, macro and matrix caps, checked before RaTeX sees the source. fn check_caps(latex: &str) -> Result<(), MathError> { if latex.len() > MAX_SOURCE_BYTES { return Err(SOURCE_TOO_LONG.clone()); } + // `%` starts a comment, except inside the address of `\href` and `\url`, where RaTeX reads it + // as a character. Which one it is decides what the rest of the line means, so the two are not + // allowed together. + if latex.contains('%') && (latex.contains("\\href") || latex.contains("\\url")) { + return Err(MathError::new("A % can't be combined with a link in an equation.", latex.find('%').map(|i| i..i + 1))); + } let mut depth = 0usize; + let mut grids: Vec = Vec::new(); + let mut cells_total = 0usize; let mut chars = latex.char_indices().peekable(); while let Some((i, c)) = chars.next() { let open = match c { '\\' => { - // A control word, or an escaped character (`\{` is not a group). + // A control word (letters and `@`, as RaTeX's lexer reads it), or an escaped + // character (`\{` is not a group). let mut word = String::new(); while let Some(&(_, n)) = chars.peek() { - if n.is_ascii_alphabetic() { + if n.is_ascii_alphabetic() || n == '@' { word.push(n); chars.next(); } else { @@ -244,15 +288,70 @@ fn check_caps(latex: &str) -> Result<(), MathError> { } } if word.is_empty() { - chars.next(); + if let Some((_, n)) = chars.next() + && n == '\\' + && let Some(g) = grids.last_mut() + { + g.end_row(); + } continue; } + if MACRO_DEFINERS.contains(&word.as_str()) { + return Err(MathError::new("Macro definitions are not supported in equations.", Some(i..i + 1 + word.len()))); + } match word.as_str() { - "left" | "begin" => Some(true), - "right" | "end" => Some(false), + "verb" => { + // Verbatim text runs to the next copy of its delimiter, whatever it + // holds: no group, comment, row or definition starts inside it. + if chars.peek().is_some_and(|&(_, n)| n == '*') { + chars.next(); + } + if let Some((_, delim)) = chars.next() { + for (_, n) in chars.by_ref() { + if n == delim { + break; + } + } + } + None + } + // RaTeX reads `\newline` as `\\`, so it ends a row like `\cr` does. + "cr" | "newline" => { + if let Some(g) = grids.last_mut() { + g.end_row(); + } + None + } + "begin" => { + if let Some(n) = alignat_columns(latex.get(i + 6..).unwrap_or_default()) + && n > MAX_ALIGNAT_PAIRS + { + return Err(too_many_cells(i..i + 6)); + } + grids.push(Grid::new()); + Some(true) + } + "end" => { + if let Some(g) = grids.pop() { + cells_total = cells_total.saturating_add(g.padded_cells()); + if cells_total > MAX_CELLS { + return Err(too_many_cells(i..i + 1 + word.len())); + } + } + Some(false) + } + "left" => Some(true), + "right" => Some(false), _ => None, } } + '&' | '@' => { + // `@` makes an arrow cell and an empty cell beside it in a `CD` diagram. + if let Some(g) = grids.last_mut() { + g.cells += if c == '@' { 2 } else { 1 }; + } + None + } '%' => { // A comment runs to the end of the line. for (_, n) in chars.by_ref() { @@ -277,9 +376,72 @@ fn check_caps(latex: &str) -> Result<(), MathError> { None => {} } } + // An environment left open is a syntax error for the parser to report, but it still counts. + for g in grids { + cells_total = cells_total.saturating_add(g.padded_cells()); + } + if cells_total > MAX_CELLS { + return Err(too_many_cells(latex.len()..latex.len())); + } Ok(()) } +/// Most column pairs `\begin{alignedat}{N}` may ask for. RaTeX allocates 2N column specs before +/// it lays anything out. +const MAX_ALIGNAT_PAIRS: usize = 50; + +/// N of `{alignat}{N}`, `{alignat*}{N}` or `{alignedat}{N}` at the start of `rest` (just after +/// `\begin`). `usize::MAX` when the count is present but is not a plain digit string RaTeX reads +/// the same way (too long, or hidden by a comment, inner spaces or a macro): refused, not skipped. +/// An environment name that is computed (contains `\` or `{`) is refused the same way. +/// +/// RaTeX reads the environment name and the count through its lexer, which drops comments and +/// spaces, so they are dropped here too before the two groups are compared. +fn alignat_columns(rest: &str) -> Option { + // Enough for the longest name and any count that fits `usize`; a longer count is refused. + const WINDOW: usize = 64; + let mut kept = String::new(); + let mut chars = rest.chars(); + while kept.len() < WINDOW { + match chars.next() { + None => break, + Some('%') => { + for n in chars.by_ref() { + if n == '\n' { + break; + } + } + } + Some(c) if c.is_whitespace() => {} + Some(c) => kept.push(c), + } + } + let group = kept.strip_prefix('{')?; + // A name that does not close inside the window cannot be read, so it is refused. + let Some((name, after)) = group.split_once('}') else { + return Some(usize::MAX); + }; + // A computed name (`align\@char{97}t` reads as `alignat`) cannot be resolved here: refused + // too. Every real environment name is letters, optionally with a trailing `*`. + if name.contains(['\\', '{']) { + return Some(usize::MAX); + } + if !matches!(name, "alignat" | "alignat*" | "alignedat") { + return None; + } + let Some(digits) = after.strip_prefix('{').and_then(|a| a.split_once('}')).map(|(d, _)| d) else { + return Some(usize::MAX); + }; + if digits.is_empty() || !digits.bytes().all(|b| b.is_ascii_digit()) { + return Some(usize::MAX); + } + Some(digits.parse().unwrap_or(usize::MAX)) +} + +fn too_many_cells(span: Range) -> MathError { + MathError::new(format!("An equation can have at most {MAX_CELLS} matrix cells (rows times the widest row)."), Some(span)) +} + fn color(c: &Color) -> Option<[u8; 3]> { let q = |v: f32| (v.clamp(0.0, 1.0) * 255.0).round() as u8; let rgb = [q(c.r), q(c.g), q(c.b)]; @@ -509,6 +671,116 @@ mod tests { } } + /// Runs `f` on its own thread and fails if it has not returned within `secs` seconds. + fn within(secs: u64, f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .stack_size(2 << 20) + .spawn(move || { + let _ = tx.send(f()); + }) + .unwrap(); + match rx.recv_timeout(std::time::Duration::from_secs(secs)) { + Ok(v) => v, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => panic!("did not return within {secs} s"), + Err(e) => panic!("the worker thread died: {e}"), + } + } + + #[test] + fn macro_definitions_are_refused_before_the_expander_runs() { + let defs = [ + "\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\gdef\\a#1{\\a{#1#1#1#1}}\\a{x}", + "\\edef\\a#1{\\a{#1#1}}\\a{x}", + "\\xdef\\a#1{{#1}{#1}}\\a{x}", + "\\newcommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\renewcommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\providecommand{\\a}[1]{\\a{#1#1}}\\a{x}", + "\\let\\a\\frac \\a{1}{2}", + "\\futurelet\\a\\frac \\a{1}{2}", + "\\global\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\long\\def\\a#1{\\a{#1#1}}\\a{x}", + "{\\def\\a#1{\\a{#1#1}}}\\a{x}", + ]; + for src in defs { + // `check_caps` alone, so this cannot hang even if the refusal regresses. + let e = check_caps(src).unwrap_err(); + assert!(e.message.contains("Macro definitions"), "{src}: {e}"); + assert_eq!(e.span.as_ref().map(|r| &src[r.clone()]).map(|w| w.starts_with('\\')), Some(true)); + let s = src.to_string(); + let (c, l, o) = within(10, move || (check(&s).is_err(), layout(&s).is_err(), to_omml(&s, 11.0, None).is_err())); + assert!(c && l && o, "{src}"); + } + // Names that merely start with a definer, escaped backslashes and comments are not macros. + for ok in ["\\default", "\\letter", "\\\\def", "x % \\def\\a\n+1", "\\longrightarrow"] { + assert!(check_caps(ok).is_ok(), "{ok}"); + } + assert!(layout("\\longrightarrow x").is_ok()); + // Text the lexer reads differently from a plain scan must not hide a definition: a + // verbatim `%` is no comment, a `%` in a link address is no comment, `@` joins a word. + for src in [ + "\\verb|%|\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\verb\\x\\\\def\\a#1{\\a{#1#1}}\\a{x}", + "%\\verb|\n\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\href{a%b}{x}\\def\\a#1{\\a{#1#1}}\\a{x}", + "\\url{a%}\\def\\a#1{\\a{#1#1}}\\a{x}", + ] { + assert!(check_caps(src).is_err(), "{src}"); + } + assert!(check_caps("\\verb|\\def| x").is_ok(), "verbatim text is not a definition"); + assert!(check_caps("\\def@x").is_ok(), "`\\def@x` is one unknown word to the lexer"); + } + + #[test] + fn ragged_matrices_are_capped_by_their_padded_size() { + let ragged = format!("\\begin{{matrix}}{}{}\\end{{matrix}}", "a\\\\".repeat(650), "a&".repeat(999) + "a"); + assert!(ragged.len() < MAX_SOURCE_BYTES); + let e = check_caps(&ragged).unwrap_err(); + assert!(e.message.contains("matrix cells"), "{e}"); + let r = ragged.clone(); + let (l, o) = within(10, move || (layout(&r).unwrap_err(), to_omml(&r, 11.0, None).unwrap_err())); + assert!(l.message.contains("matrix cells") && o.message.contains("matrix cells")); + // Without the environment words there is no grid to count; the parser rejects it. + // Ordinary and wide-but-regular matrices still work, and so does a trailing `\\`. + let wide = format!("\\begin{{pmatrix}}{}\\end{{pmatrix}}", vec!["a&".repeat(39) + "a"; 40].join("\\\\")); + assert!(wide.len() < MAX_SOURCE_BYTES); + assert!(layout(&wide).is_ok()); + assert!(to_omml(&wide, 11.0, None).is_ok()); + assert!(layout("\\begin{pmatrix} a & b \\\\ c & d \\\\ \\end{pmatrix}").is_ok()); + // Diagram arrows and `alignedat` column counts size a grid without any `&`. + let cd = format!("\\begin{{CD}}{}{}\\end{{CD}}", "A\\\\".repeat(900), "@>>>".repeat(300)); + assert!(cd.len() < MAX_SOURCE_BYTES); + assert!(check_caps(&cd).is_err_and(|e| e.message.contains("matrix cells"))); + let r = "\\begin{alignedat}{99999999}a\\end{alignedat}"; + assert!(check_caps(r).is_err_and(|e| e.message.contains("matrix cells"))); + assert!(check_caps("\\begin{alignedat}{2}a&b&c&d\\end{alignedat}").is_ok()); + // A comment, inner spaces or a macro in the count do not hide it from the cap. + for hidden in [ + "\\begin{alignedat}%\n{99999999}a\\end{alignedat}", + "\\begin %c\n {alignedat}{99999999}a\\end{alignedat}", + "\\begin{alignedat}{9 9}a\\end{alignedat}", + "\\begin{alignedat}{9%x\n9}a\\end{alignedat}", + "\\begin{align%x\nedat}{99}a\\end{alignedat}", + "\\begin{alignedat}{9\\relax9}a\\end{alignedat}", + ] { + assert!(check_caps(hidden).is_err_and(|e| e.message.contains("matrix cells")), "{hidden}"); + } + // A computed environment name reads as `alignat` to the parser, so it is capped too. + for name in ["align\\@char{97}t", "alignedat\\relax", "al\\@char{105}gnat*"] { + let over = format!("\\begin{{{name}}}{{51}}a\\end{{alignat}}"); + assert!(check_caps(&over).is_err_and(|e| e.message.contains("matrix cells")), "{over}"); + } + let long = format!("\\begin{{{}}}{{51}}a\\end{{alignat}}", "a".repeat(80)); + assert!(check_caps(&long).is_err_and(|e| e.message.contains("matrix cells"))); + assert!(check_caps("\\begin{alignat}{50}a\\end{alignat}").is_ok()); + assert!(check_caps("\\begin{alignedat} {2} % two pairs\na&b&c&d\\end{alignedat}").is_ok()); + assert!(layout("\\begin{CD}A @>f>> B \\\\ @VVV @VVV \\\\ C @>>> D\\end{CD}").is_ok()); + assert!(check_caps("\\operatorname{curl} 50\\%").is_ok()); + let two = format!("{0}{0}", "\\begin{matrix}a&b\\\\c&d\\end{matrix}"); + assert!(check_caps(&two).is_ok()); + } + #[test] fn an_injected_panic_is_an_error_not_a_crash() { INJECT_PANIC.with(|p| p.set(true)); @@ -530,4 +802,20 @@ mod tests { let l = layout("").unwrap(); assert!(l.items.is_empty()); } + + #[test] + fn newline_rows_count_toward_the_matrix_cap() { + // RaTeX reads `\newline` as `\\`, so it ends a row like `\cr` does. + for brk in ["\\newline ", "\\cr ", "\\\\"] { + let src = format!("\\begin{{matrix}}{}{}\\end{{matrix}}", format!("a{brk}").repeat(250), "a&".repeat(700) + "a"); + assert!(src.len() < MAX_SOURCE_BYTES); + let e = check_caps(&src).unwrap_err(); + assert!(e.message.contains("matrix cells"), "{brk}: {e}"); + let s = src.clone(); + let (l, o) = within(10, move || (layout(&s).unwrap_err(), to_omml(&s, 11.0, None).unwrap_err())); + assert!(l.message.contains("matrix cells") && o.message.contains("matrix cells"), "{brk}"); + } + assert!(check_caps("\\begin{matrix}a&b\\newline c&d\\end{matrix}").is_ok()); + assert!(layout("\\begin{matrix}a&b\\newline c&d\\end{matrix}").is_ok()); + } } diff --git a/crates/math/src/omml.rs b/crates/math/src/omml.rs index dc0141aed..7d5c66835 100644 --- a/crates/math/src/omml.rs +++ b/crates/math/src/omml.rs @@ -406,6 +406,9 @@ impl Writer { } ParseNode::Array { body, is_cd: None | Some(false), .. } => { let cols = body.iter().map(Vec::len).max().unwrap_or(0).max(1); + if body.len().saturating_mul(cols) > crate::MAX_CELLS { + return Err(MathError::new(format!("An equation can have at most {} matrix cells.", crate::MAX_CELLS), None)); + } self.out.push_str(&format!( "" )); diff --git a/crates/mcp/src/server.rs b/crates/mcp/src/server.rs index 75b82d8d8..0d262dbe7 100644 --- a/crates/mcp/src/server.rs +++ b/crates/mcp/src/server.rs @@ -1,6 +1,6 @@ //! JSON-RPC 2.0 framing and the MCP lifecycle / tools / resources / prompts methods. -use std::io::{BufRead, Write}; +use std::io::{BufRead, Read, Write}; use serde_json::{Value, json}; @@ -45,6 +45,45 @@ fn response(id: Value, result: Value) -> Value { json!({"jsonrpc": "2.0", "id": id, "result": result}) } +/// Reads the next newline-delimited message into `buf`, keeping at most `MAX_LINE` bytes of it: +/// a client sending no newline can't make the server allocate more (#2164). `None` at the end of +/// `input`; `Some(false)` for a line over the limit, whose rest is read and dropped so the next +/// message is read whole. +pub(crate) fn read_message(input: &mut impl BufRead, buf: &mut Vec) -> std::io::Result> { + buf.clear(); + // One byte past the limit tells an over-long line from one of exactly `MAX_LINE` bytes. + let limit = u64::try_from(MAX_LINE).unwrap_or(u64::MAX).saturating_add(1); + if input.take(limit).read_until(b'\n', buf)? == 0 { + return Ok(None); + } + if buf.len() > MAX_LINE && buf.last() != Some(&b'\n') { + skip_line(input)?; + buf.clear(); + return Ok(Some(false)); + } + Ok(Some(true)) +} + +/// Reads up to and including the next newline (or to the end of `input`) without keeping it. +fn skip_line(input: &mut impl BufRead) -> std::io::Result<()> { + loop { + let chunk = input.fill_buf()?; + if chunk.is_empty() { + return Ok(()); + } + match chunk.iter().position(|&b| b == b'\n') { + Some(i) => { + input.consume(i + 1); + return Ok(()); + } + None => { + let n = chunk.len(); + input.consume(n); + } + } + } +} + fn error(id: Value, code: i64, message: impl Into) -> Value { json!({"jsonrpc": "2.0", "id": id, "error": {"code": code, "message": message.into()}}) } @@ -68,14 +107,13 @@ impl Server { pub fn serve(&mut self, mut input: impl BufRead, mut output: impl Write) -> std::io::Result<()> { let mut buf = Vec::new(); loop { - buf.clear(); - let n = input.read_until(b'\n', &mut buf)?; - if n == 0 { - return Ok(()); - } - let reply = match std::str::from_utf8(&buf) { - Ok(line) => self.handle_line(line), - Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + let reply = match read_message(&mut input, &mut buf)? { + None => return Ok(()), + Some(false) => Some(error(Value::Null, INVALID_REQUEST, "message too large (16 MB max)").to_string()), + Some(true) => match std::str::from_utf8(&buf) { + Ok(line) => self.handle_line(line), + Err(_) => Some(error(Value::Null, PARSE_ERROR, "message is not valid UTF-8").to_string()), + }, }; if let Some(reply) = reply { output.write_all(reply.as_bytes())?; diff --git a/crates/mcp/src/tests.rs b/crates/mcp/src/tests.rs index 71631b5e8..5a5297246 100644 --- a/crates/mcp/src/tests.rs +++ b/crates/mcp/src/tests.rs @@ -359,6 +359,71 @@ fn serve_loop_over_buffers() { assert!(lines[2]["result"]["content"][0]["text"].as_str().unwrap().contains("42")); } +/// Streams `len` bytes of `a` from one reused buffer, then `tail`: a huge line costs no memory to +/// produce, so only the server can make it expensive. +struct HugeLine { + left: u64, + tail: &'static [u8], + chunk: Vec, +} + +impl std::io::Read for HugeLine { + fn read(&mut self, out: &mut [u8]) -> std::io::Result { + let buf = std::io::BufRead::fill_buf(self)?; + let n = buf.len().min(out.len()); + out[..n].copy_from_slice(&buf[..n]); + std::io::BufRead::consume(self, n); + Ok(n) + } +} + +impl std::io::BufRead for HugeLine { + fn fill_buf(&mut self) -> std::io::Result<&[u8]> { + if self.left > 0 { + let n = self.chunk.len().min(usize::try_from(self.left).unwrap_or(usize::MAX)); + return Ok(&self.chunk[..n]); + } + Ok(self.tail) + } + fn consume(&mut self, n: usize) { + if self.left > 0 { + self.left -= n as u64; + } else { + self.tail = &self.tail[n..]; + } + } +} + +#[test] +fn a_line_without_newline_keeps_at_most_the_limit_in_memory() { + // #2164: the whole line was buffered before its size was checked (4 GiB here; any size a + // client cares to send). + let mut input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut buf = Vec::new(); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(false)); + assert!(buf.capacity() <= 2 * (16 << 20) + 1, "kept {} bytes", buf.capacity()); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), Some(true)); + assert!(buf.starts_with(br#"{"jsonrpc""#)); + assert_eq!(crate::server::read_message(&mut input, &mut buf).unwrap(), None); +} + +#[test] +fn serve_loop_bounds_a_line_without_newline() { + // #2164: a 4 GiB line was read whole before its size was checked. Now the server keeps at + // most 16 MB of it, answers that it is too large, and reads the next message normally. + let input = + HugeLine { left: 4 << 30, tail: concat!("\n", r#"{"jsonrpc":"2.0","id":7,"method":"ping"}"#, "\n").as_bytes(), chunk: vec![b'a'; 1 << 20] }; + let mut out = Vec::new(); + server().serve(input, &mut out).unwrap(); + let lines: Vec = String::from_utf8(out).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(lines.len(), 2); + assert_eq!(lines[0]["error"]["code"], -32600); + assert!(lines[0]["error"]["message"].as_str().unwrap().contains("too large")); + assert_eq!(lines[1]["id"], 7); + assert!(lines[1].get("result").is_some(), "{}", lines[1]); +} + #[test] fn cell_set_without_input_is_error() { let mut s = server(); diff --git a/crates/model/src/features.rs b/crates/model/src/features.rs index 06e68839f..8ab74606a 100644 --- a/crates/model/src/features.rs +++ b/crates/model/src/features.rs @@ -230,6 +230,30 @@ pub enum CfRule { }, } +impl CfRule { + /// The differential style a matching rule applies; colour scales, data bars and icon sets have none. + pub fn style(&self) -> Option<&Style> { + match self { + CfRule::CellIs { style, .. } + | CfRule::Expression { style, .. } + | CfRule::ContainsText { style, .. } + | CfRule::NotContainsText { style, .. } + | CfRule::BeginsWith { style, .. } + | CfRule::EndsWith { style, .. } + | CfRule::Blanks { style } + | CfRule::NoBlanks { style } + | CfRule::Errors { style } + | CfRule::NoErrors { style } + | CfRule::Duplicate { style } + | CfRule::Unique { style } + | CfRule::Top10 { style, .. } + | CfRule::AboveAverage { style, .. } + | CfRule::TimePeriod { style, .. } => Some(style), + CfRule::ColorScale { .. } | CfRule::DataBar { .. } | CfRule::IconSet { .. } => None, + } + } +} + /// Number of icons in an icon set (`3Arrows` → 3, `5Quarters` → 5), always 3 to 5. pub fn icon_count(set: &str) -> usize { set.chars().next().and_then(|c| c.to_digit(10)).unwrap_or(3).clamp(3, 5) as usize diff --git a/crates/model/src/lib.rs b/crates/model/src/lib.rs index 35cfa315f..b1f875b3c 100644 --- a/crates/model/src/lib.rs +++ b/crates/model/src/lib.rs @@ -26,8 +26,8 @@ pub use python::{PyCode, PyOutput}; pub use sheet::{DEFAULT_COL_WIDTH, DEFAULT_ROW_HEIGHT, LineIndex, Sheet, SheetUid, SheetViewMode, Visibility, points_to_chars}; pub use store::CellStore; pub use style::{ - Alignment, BorderLine, BorderStyle, Borders, Color, Fill, Font, HAlign, NumFmt, PatternType, Style, StyleId, StyleTable, Theme, Underline, - VAlign, VertAlign, + Alignment, BorderLine, BorderStyle, Borders, Color, Fill, Font, HAlign, NumFmt, PatternType, ReadingOrder, Style, StyleId, StyleTable, Theme, + Underline, VAlign, VertAlign, }; pub use workbook::{ CalcMode, CalcSettings, DefinedName, DocProps, EQUATION_BUDGET_MESSAGE, MAX_EQUATION_LAYOUT_BYTES, PhoneticProps, PhoneticRun, PhoneticString, diff --git a/crates/model/src/sheet.rs b/crates/model/src/sheet.rs index 440647d42..409e93023 100644 --- a/crates/model/src/sheet.rs +++ b/crates/model/src/sheet.rs @@ -566,7 +566,9 @@ impl LineIndex { let mut prev = 0u32; for (&line, info) in custom { pos += (line - prev) as f64 * default; - let size = if info.hidden { 0.0 } else { info.size.map_or(default, |s| s as f64) }; + // A negative size (only a malformed file has one) counts as zero, so positions only + // grow and the searches below stay valid. + let size = if info.hidden { 0.0 } else { info.size.map_or(default, |s| s as f64).max(0.0) }; lines.push((line, pos, size)); pos += size; prev = line + 1; @@ -584,6 +586,44 @@ impl LineIndex { }, } } + /// The first line at or after `line` (and before the axis end) with a size above zero, or + /// `None` when every line from there on is hidden or zero-sized. O(log n): a run of hidden + /// lines all start where the next shown line starts, so one search steps over the whole run. + pub fn next_visible(&self, line: u32) -> Option { + if line >= self.max { + return None; + } + let i = self.lines.partition_point(|e| e.0 < line); + let custom = self.lines.get(i).is_some_and(|e| e.0 == line); + if !custom && self.default > 0.0 { + return Some(line); + } + // Every line from `line` up to the answer is zero-sized, so they all start at `p`; the + // last custom line starting at `p` either has a size (the answer) or is the end of the run. + let p = self.start(line); + let k = self.lines.partition_point(|e| e.1 <= p); + let &(last, _, size) = self.lines.get(k.checked_sub(1)?)?; + if last < line { + return None; + } + if size > 0.0 { + return Some(last).filter(|l| *l < self.max); + } + // The line after the run has no custom line (a custom one would start at `p` too and be + // part of the run): it is shown when the default size is. + let next = last.checked_add(1).filter(|n| *n < self.max)?; + (self.default > 0.0).then_some(next) + } + /// The lines from `first` to `last` (inclusive) with a size above zero, in order. Runs of + /// hidden lines cost one O(log n) step each, not one per line. + pub fn visible(&self, first: u32, last: u32) -> impl Iterator + '_ { + let mut next = self.next_visible(first); + std::iter::from_fn(move || { + let line = next.filter(|l| *l <= last)?; + next = line.checked_add(1).and_then(|n| self.next_visible(n)); + Some(line) + }) + } /// The line containing `pos`. pub fn at(&self, pos: f64) -> u32 { if pos <= 0.0 { @@ -644,6 +684,63 @@ mod tests { use super::*; use crate::cell::Formula; + /// `LineIndex::visible` lists exactly the lines a line-by-line size check shows, for hidden + /// runs at the start, in the middle and up to the end of the axis, custom sizes (zero and + /// negative included), lines with only a style, and a zero default size. + #[test] + fn visible_lines_match_a_line_by_line_check() { + let max = 400u32; + let mut seed = 0x2545_f491_4f6c_dd1du64; + let mut rnd = |n: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed % n + }; + for case in 0..300 { + let mut custom = BTreeMap::new(); + let runs = rnd(6); + for _ in 0..runs { + let start = rnd(u64::from(max)) as u32; + let len = rnd(if case % 3 == 0 { 400 } else { 30 }) as u32; + for l in start..(start + len).min(max) { + custom.insert(l, LineInfo { hidden: true, ..Default::default() }); + } + } + // Runs that start at the first line or end at the last one. + if case % 5 == 0 { + for l in 0..rnd(20) as u32 { + custom.insert(l, LineInfo { hidden: true, ..Default::default() }); + } + } + if case % 5 == 1 { + for l in max - rnd(20) as u32 - 1..max { + custom.insert(l, LineInfo { hidden: true, ..Default::default() }); + } + } + for _ in 0..rnd(8) { + let size = [Some(0.0), Some(3.0), Some(20.0), Some(-5.0), None][rnd(5) as usize]; + custom.insert(rnd(u64::from(max)) as u32, LineInfo { size, ..Default::default() }); + } + let default = if case % 7 == 0 { 0.0 } else { 15.0 }; + let size = |l: u32| match custom.get(&l) { + Some(i) if i.hidden => 0.0, + Some(LineInfo { size: Some(s), .. }) => s.max(0.0), + _ => default, + }; + let index = LineIndex::new(&custom, default, max); + for (first, last) in [(0, max - 1), (rnd(u64::from(max)) as u32, max - 1), (5, 5), (max - 1, max - 1)] { + let want: Vec = (first..=last).filter(|&l| size(l) > 0.0).collect(); + assert_eq!(index.visible(first, last).collect::>(), want, "case {case}, {first}..={last}"); + } + assert_eq!(index.next_visible(max), None); + } + // The row after a million hidden rows comes straight after the last shown one. + let custom: BTreeMap = (47..1_048_422).map(|r| (r, LineInfo { hidden: true, ..Default::default() })).collect(); + let index = LineIndex::new(&custom, 15.0, 1_048_576); + assert_eq!(index.visible(40, 1_048_424).collect::>(), [40, 41, 42, 43, 44, 45, 46, 1_048_422, 1_048_423, 1_048_424]); + } + #[test] fn a_sheet_keeps_its_identity_through_clones_and_a_new_sheet_gets_its_own() { let a = Sheet::new("A"); diff --git a/crates/model/src/style.rs b/crates/model/src/style.rs index d2f10780e..9ad61abec 100644 --- a/crates/model/src/style.rs +++ b/crates/model/src/style.rs @@ -350,6 +350,14 @@ pub enum VAlign { Distributed, } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum ReadingOrder { + #[default] + Context, + LeftToRight, + RightToLeft, +} + #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Alignment { pub h: HAlign, @@ -359,6 +367,9 @@ pub struct Alignment { pub indent: u8, /// Degrees -90..=90, or 255 for vertical stacked text. pub rotation: i16, + /// Unspecified inherits in differential styles and uses context for ordinary cells. + #[serde(default)] + pub reading_order: Option, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -488,6 +499,25 @@ static DEFAULT_STYLE: std::sync::LazyLock