diff --git a/Cargo.lock b/Cargo.lock index e320c2cf8..66accc510 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -369,6 +369,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bit_field" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb2907cc03d08cdb1c6724ad0670ce67c4a71e1c45a6d7635224011714cfc4b1" + [[package]] name = "bitflags" version = "1.3.2" @@ -926,7 +932,7 @@ version = "0.1.0" name = "craft-cms" version = "0.1.0" dependencies = [ - "md-5", + "md-5 0.10.6", "moxcms", "rayon", ] @@ -1025,16 +1031,27 @@ name = "craft-epub" version = "0.1.0" dependencies = [ "craft-xml", - "flate2", + "craft-zip", "thiserror 2.0.21", "zip", ] +[[package]] +name = "craft-ews" +version = "0.1.0" +dependencies = [ + "craft-xml", + "proptest", +] + [[package]] name = "craft-exr" version = "0.1.0" dependencies = [ + "exr", + "half", "serde_json", + "thiserror 2.0.21", ] [[package]] @@ -1180,16 +1197,32 @@ dependencies = [ "writeable", ] +[[package]] +name = "craft-ical" +version = "0.1.0" +dependencies = [ + "chrono", + "proptest", +] + [[package]] name = "craft-idml" version = "0.1.0" dependencies = [ "craft-xml", - "flate2", + "craft-zip", "thiserror 2.0.21", "zip", ] +[[package]] +name = "craft-jmap" +version = "0.1.0" +dependencies = [ + "proptest", + "serde_json", +] + [[package]] name = "craft-jpeg" version = "0.1.0" @@ -1275,6 +1308,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "craft-metadata" +version = "0.1.0" +dependencies = [ + "chrono", + "craft-tiff", + "proptest", + "quick-xml 0.41.0", + "serde", + "serde_json", +] + [[package]] name = "craft-metafile" version = "0.1.0" @@ -1371,6 +1416,7 @@ dependencies = [ name = "craft-pdf" version = "0.1.0" dependencies = [ + "aes", "bytemuck", "hayro", "hayro-cmap", @@ -1378,9 +1424,23 @@ dependencies = [ "hayro-syntax", "krilla", "kurbo 0.13.1", + "md-5 0.11.0", + "proptest", + "sha2 0.11.1", + "stringprep", "thiserror 2.0.21", "ttf-parser", "wasm-bindgen-test", + "zeroize", +] + +[[package]] +name = "craft-pdn" +version = "0.1.0" +dependencies = [ + "flate2", + "proptest", + "thiserror 2.0.21", ] [[package]] @@ -1538,7 +1598,9 @@ dependencies = [ name = "craft-rtf" version = "0.1.0" dependencies = [ + "craft-mime", "encoding_rs", + "proptest", ] [[package]] @@ -1552,6 +1614,10 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "craft-sheet" +version = "0.1.0" + [[package]] name = "craft-silhouette-studio" version = "0.1.0" @@ -1753,6 +1819,14 @@ dependencies = [ "web-sys", ] +[[package]] +name = "craft-webp" +version = "0.1.0" +dependencies = [ + "image", + "thiserror 2.0.21", +] + [[package]] name = "craft-window" version = "0.1.0" @@ -1771,11 +1845,43 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "craft-wordmodel" +version = "0.1.0" +dependencies = [ + "base64 0.23.1", + "log", + "proptest", + "serde", + "serde_json", + "thiserror 2.0.21", + "unicode-segmentation", +] + +[[package]] +name = "craft-xcf" +version = "0.1.0" +dependencies = [ + "flate2", + "thiserror 2.0.21", +] + [[package]] name = "craft-xls" version = "0.1.0" dependencies = [ "craft-cfb", + "craft-sheet", + "thiserror 2.0.21", +] + +[[package]] +name = "craft-xlsx" +version = "0.1.0" +dependencies = [ + "craft-opc", + "craft-sheet", + "craft-xml", "thiserror 2.0.21", ] @@ -2397,6 +2503,21 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.74.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb29d0be702adf92c645d9b2f93472906ba83d400e59f98adb3770ce5c18c2f6" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide 0.8.9", + "multiversion", + "smallvec", + "zune-inflate", +] + [[package]] name = "extended" version = "0.1.0" @@ -3980,6 +4101,12 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.190" @@ -4095,6 +4222,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -4194,6 +4331,28 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb8a68be21d0755f83bfe47e79b896cc732687cf1903f0e50585e8d6ecc53076" +dependencies = [ + "multiversion-macros", + "target-features", +] + +[[package]] +name = "multiversion-macros" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a05e0c3c3e4edaa21d87c645852ed0b0c3663752bcc6ec43353bd6189dce35e1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "target-features", +] + [[package]] name = "multiversion_no_op" version = "1.0.0" @@ -6643,6 +6802,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-features" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" + [[package]] name = "tempfile" version = "3.27.0" @@ -8477,6 +8642,15 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + [[package]] name = "zune-jpeg" version = "0.5.15" diff --git a/Cargo.toml b/Cargo.toml index 76238365d..a7304151b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ craft-aep = { path = "crates/aep" } craft-affinity = { path = "crates/affinity" } craft-ai-core = { path = "crates/ai-core" } craft-pdf = { path = "crates/pdf", version = "=0.1.0" } +craft-pdn = { path = "crates/pdn" } craft-plate-test = { path = "crates/plate-test" } krilla = { version = "=0.8.2", default-features = false, features = ["raster-images"] } hayro = { version = "=0.7.1", default-features = false } @@ -55,6 +56,8 @@ craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } craft-idml = { path = "crates/idml" } +craft-jmap = { path = "crates/jmap" } +craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } craft-kiwi = { path = "crates/kiwi" } craft-keymap = { path = "crates/keymap" } @@ -86,6 +89,7 @@ craft-pst = { path = "crates/pst" } craft-quit = { path = "crates/quit" } craft-rtf = { path = "crates/rtf" } craft-raw = { path = "crates/raw" } +craft-sheet = { path = "crates/sheet" } craft-sketch = { path = "crates/sketch" } craft-settings = { path = "crates/settings" } craft-silhouette-studio = { path = "crates/silhouette-studio" } @@ -96,20 +100,26 @@ craft-text = { path = "crates/text" } craft-tga = { path = "crates/tga" } craft-timeline = { path = "crates/timeline" } craft-tiff = { path = "crates/tiff" } +craft-metadata = { path = "crates/metadata" } craft-ui = { path = "crates/ui" } craft-update = { path = "crates/update" } craft-vcard = { path = "crates/vcard" } craft-vectorcraft-format = { path = "crates/vectorcraft-format" } craft-vectordrawable = { path = "crates/vectordrawable" } craft-xls = { path = "crates/xls" } +craft-wordmodel = { path = "crates/wordmodel" } +craft-xlsx = { path = "crates/xlsx" } +craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } craft-epub = { path = "crates/epub" } +craft-ews = { path = "crates/ews" } craft-video = { path = "crates/video" } craft-web-api = { path = "crates/web-api" } craft-window = { path = "crates/window" } craft-web-host = { path = "crates/web-host" } +craft-webp = { path = "crates/webp" } egui = { version = "0.37", default-features = false } md5 = { package = "md-5", version = "0.10", default-features = false } rayon = "1" diff --git a/NOTICE b/NOTICE index 3bca3ae0d..eb22134a2 100644 --- a/NOTICE +++ b/NOTICE @@ -93,7 +93,7 @@ crates/raw (craft-raw) is moved from LightCraft, with behaviour ported from two - LightCraft, crates/raw (src/lib.rs, dng.rs, dngwrite.rs, tiffraw.rs, unpack.rs, ljpeg.rs, jxl.rs, opcodes.rs, profile.rs, gaintable.rs, semantic.rs, preview.rs, vendor/crw.rs, vendor/hasselblad.rs, vendor.rs (shared level helpers), vendor/mrw.rs, vendor/erf.rs, vendor/mef.rs, vendor/kodak.rs, vendor/pef.rs, - vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), + vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, vendor/nef.rs, vendor/nefc.rs, vendor/nef_lens.rs, docs/nikon-lens-corrections.md, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), revision 5ff567a4a2e067bef1b6bce152c11e7f727ccbaf. https://github.com/storytold/lightcraft Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. diff --git a/README.md b/README.md index 87584f16f..0c2859e3a 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = "` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. +- [`craft-ical`](crates/ical/README.md): iCalendar (RFC 5545) building blocks, from MailCraft: content + lines, DATE/DATE-TIME values, durations, TEXT escaping, the common subset of recurrence rules, + and `VTIMEZONE` definitions read and written. No time-zone database; what events and to-dos + mean stays in each app. Bounded and panic-free on hostile input. - [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, CCITT Group 3 and 4 decoding and Group 4 encoding, JBIG2 decoding and lossless encoding. Every decoder is bounded and returns its partial output on damage. From PdfCraft; a PDF filter chain is built on it in `craft-pdf`. - [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft. +- [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write, + maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge + that keeps what another program wrote), GPX track logs, and the blocks that carry them in JPEG, PNG, + WebP, TIFF-based, RAF and CR3 files. One `Metadata` model for all of them. Built on `craft-tiff`. - [`craft-tiff`](crates/tiff/README.md): TIFF and BigTIFF IFD reader and writer, from LightCraft: every field type, the IFD chain, SubIFDs and Exif/GPS/Interop pointer IFDs, maker-note IFDs (Canon, Nikon, Sony, Fujifilm, Olympus, Panasonic, Pentax), strip and tile layout. Bounded and @@ -164,6 +177,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = " = { git = "https://github.com/storytold/craft-libs", rev = "` shape (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, `screenshot`) over the app's JSON control channel. - [`craft-web-host`](crates/web-host/README.md): the host messages between a Craft app's web build and the page that hosts it (`?host=parent` or `data-host="page"`): one contract and one bridge for every app. +- [`craft-webp`](crates/webp/src/lib.rs): the WebP container, from EffectCraft (reader) and + PhotoCraft (writer): canvas size, alpha and every animation frame's timing from the chunk + headers without decoding, and the RIFF container around an encoded VP8 frame with alpha, ICC, + EXIF and XMP chunks. The VP8 and VP8L codecs are not part of it. - [`craft-xls`](crates/xls/README.md): clean-room Excel 97-2003 workbook (`.xls`, BIFF8) reader: worksheets, cell values, formulas as A1 text (shared and array formulas included), cell formats with resolved palette colours, merged cells, column and row records, defined names, print areas and calculation settings, with limits on records, cells and text. +- [`craft-wordmodel`](crates/wordmodel/README.md): a word-processing document model (stories, paragraphs, + runs, tables, sections, styles, numbering, fields, notes, comments, revisions, drawings, math): what the + text-format crates read into and write from. WordCraft's model without its editing code. +- [`craft-xlsx`](crates/xlsx/README.md): Excel workbooks (`.xlsx`, `.xlsm`, SpreadsheetML, transitional and strict) and binary workbooks (`.xlsb`) on `craft-opc`: the workbook part, shared strings, the styles tables and worksheet rows streamed one at a time, with limits on the package, the XML and the binary records. - [`craft-ora`](crates/ora/README.md): OpenRaster documents (`.ora`): the ZIP archive and the `stack.xml` layer tree as written, read with bounded sizes and written. From PhotoCraft. - [`craft-y4m`](crates/y4m/src/lib.rs): YUV4MPEG2 (`.y4m`) reader, from FilmCraft: the stream header, colour space layouts at 8 to 16 bits, a frame index built from the `FRAME` lines without loading the file, and plane access. Bounded on hostile input. - [`craft-kra`](crates/kra/README.md): Krita documents (`.kra`, `.krz`), read: the ZIP archive, the `maindoc.xml` node tree as written, and the tiled pixel files with their LZF compression, a tile at a time. From PhotoCraft. +- [`craft-xcf`](crates/xcf/README.md): GIMP XCF images (`.xcf`, `.xcf.gz`), read: header, properties, parasites, layers with their item paths, channels, and tiled pixels (uncompressed, RLE, zlib) at every precision, under a caller's byte budget. From PhotoCraft. - [`craft-xml`](crates/xml/README.md): bounded XML for hostile input: an owned element tree with limits on size, depth, nodes and attributes, streaming of repeated elements, namespace handling (as written, by local name, or canonical prefixes by URI), escaping and a writer. No DTDs and no entity expansion. - [`craft-zip`](crates/zip/README.md): ZIP archives from untrusted files: an entry-count check before the archive is indexed (ZIP64 included), entry reads within per-entry and total budgets that count real bytes, duplicate-name detection, a strict structure preflight, and a deterministic stored/deflate writer. The shared ZIP layer for the package formats. - [Shared UI component audit](docs/ui-components-audit.md): source inventory, extraction priorities diff --git a/crates/aep/README.md b/crates/aep/README.md index 12ce8f9d6..95fcdd1a6 100644 --- a/crates/aep/README.md +++ b/crates/aep/README.md @@ -24,6 +24,13 @@ The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a XML tokenizer of its own); `parse_any` picks the reader by content, so the typed view takes either form. The mapping is in [`docs/aep-format.md`](docs/aep-format.md), section 1. +A reader that only looks does not need the owned tree. `TreeIndex` is the same tree as one flat, +read-only array in depth-first order, borrowed from the input: `TreeIndex::parse` for an `.aep`, +`index_aepx` and `TreeIndex::from_drafts` for an `.aepx` (decoded into one buffer first). It +counts the same chunks against the same `Limits` and returns the same errors as `parse`, and the +typed view reads through either (`Project::new` for the owned tree, `Project::from_index` for the +index). An importer should use the index; it is about twice as fast and allocates far less. + Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining input, nesting and chunk count are bounded (`Limits`), nothing is allocated from a declared size, diff --git a/crates/aep/docs/aep-format.md b/crates/aep/docs/aep-format.md index a9269bc10..3582276d7 100644 --- a/crates/aep/docs/aep-format.md +++ b/crates/aep/docs/aep-format.md @@ -121,6 +121,11 @@ counterpart (a name that is not 1 to 4 characters) are skipped with their conten with the byte offset. It runs without recursion; the nesting depth (`Limits::max_depth`) and the element count (`Limits::max_chunks`) are bounded, and everything it allocates is bounded by the size of the input. +- There are two readers. `read_aepx` builds the owned tree element by element. The importer, which + only reads, uses a second reader that decodes every body (hex, text, the alias JSON, the + numbers) into one buffer and records the chunks as a flat depth-first list whose bodies are + ranges of that buffer; it indexes that buffer (`TreeIndex`) as it indexes an `.aep`. The tests + hold both readers to the same tree and the same errors. - Tests: `tests/aepx.rs` compares the five fixture pairs chunk by chunk and byte by byte, compares the typed view of both forms, and runs truncated, bit-flipped, deeply nested and oversized inputs; `crates/aep/tests/it/import/aepx.rs` imports both forms of each pair and expects the same @@ -139,8 +144,8 @@ write on their own. |---|---| | `svap` (4 bytes, the first top-level chunk) is the version of the After Effects that **wrote** the file: `0f 11 0e 02` in all 36 (a down-saved copy keeps the 2026 word) and in all 1 192 other `.aep` files of the local corpus folders. After Effects opens a file without `svap` (`t-nosvap`). | All 36; corpus scan. | | `head` bytes 4 to 7 are the version the file is **written for**: `0f 11 0e 02` (= `svap`) in the 2026 saves, `0f 0b 06 61` in the 25.x copies, `0f 03 06 41` in the 24.x copies, the same word in every family. After Effects refuses a file whose word is zero ("created with an older version [0.0x0]", `u-zhead`). | All 36; `u-zhead`. | -| The word is packed (bit 31 first): bits 25-24 and 21-19 the major, 18-15 the minor, 14-11 the bug-fix, 10-8 a stage (6 in all), 7-0 the build; bits 31-26 (`000011`) and 23-22 (`00`) are the same in all samples and not decoded. `0f110e02` = **26.2.1x2**, which is what the saving After Effects reports for itself (`app.version` recorded by the corpus scripts) - this one sample fixes minor, bug-fix and build. The copies read as **25.6.0x97** and **24.6.0x65** (major **V** from three consecutive majors; minor, bug-fix and build of the copies **E**: one other word decides their positions). `effectcraft_aep::AeVersion`, `AepFile::file_version`, `AepFile::saved_by_version`. | Tests `decodes_the_sampled_words`, `the_container_of_a_copy_states_its_version_in_head_and_the_writer_in_svap`. | -| `head` bytes 0 to 1 (a `u16`) step with the major: 97 (`0x61`) in the 2026 saves, 96 in 25.x, 95 in 24.x; bytes 2 to 3 are 5, 9 and 6 (**?**). Bytes 8 to 15 differ between families, not between versions; bytes 16 to 19 are the save counter (one higher in each copy). Nothing is derived from bytes 0 to 3 (`AepFile::head_revision` exposes the `u16`). | All 36. | +| The word is packed (bit 31 first): bits 25-24 and 21-19 the major, 18-15 the minor, 14-11 the bug-fix, 10-8 a stage (6 in all), 7-0 the build; bits 31-26 (`000011`) and 23-22 (`00`) are the same in all samples and not decoded. `0f110e02` = **26.2.1x2**, which is what the saving After Effects reports for itself (`app.version` recorded by the corpus scripts) - this one sample fixes minor, bug-fix and build. The copies read as **25.6.0x97** and **24.6.0x65** (major **V** from three consecutive majors; minor, bug-fix and build of the copies **E**: one other word decides their positions). `craft_aep::AeVersion`, `AepFile::file_version`, `AepFile::saved_by_version`. | Tests `decodes_the_sampled_words`, `the_container_of_a_copy_states_its_version_in_head_and_the_writer_in_svap`. | +| `head` bytes 0 to 1 (a `u16`) step with the major: 97 (`0x61`) in the 2026 saves, 96 in 25.x, 95 in 24.x; bytes 2 to 3 are 5, 9 and 6 (**?**). Bytes 8 to 15 differ between families, not between versions; bytes 16 to 19 are the save counter (one higher in each copy). Nothing is derived from bytes 0 to 3, and the reader does not expose them. | All 36. | | The XMP packet after the container is no guide: its `CreatorTool` is "Adobe After Effects 2026" in the copies too. | All 36. | | **26 against 25.x** (per family, chunk tree aligned by item and layer id): besides `head`, `nhed` / `nnhd` differ in 6 bytes (`nhed` 20-23 and 26-27: `827fa930` / `0015` in the 2026 save, `00000000` / `011b` in both copies; session state, see section 4). Every **solid's** `Pin /CLRS` changes: the 2026 save has `dcui` (1 byte, 1), `apid` all `0xff` and an empty `Utf8`; the copies drop `dcui`, set `apid` to a GUID (`601e9973...`, the same in all) and embed a 935-byte JSON with an ICC profile (`baseColorProfile/colorProfileData`). A **footage** item that the 2026 save stores with an assigned profile (`mcsp` = 1 and a profile JSON) loses `mcsp` and that JSON in the copies, and its `epid` becomes all `0xff`. The rest is save noise: the footage alias `ascendcount_base` (the copies were saved in another folder), `sspc` byte 112 of a footage item (a modification time), the `oEXR` `opti` of an EXR item (the copies hold the display device name `\.\DISPLAY1` as UTF-16 at byte 126 and, in the 24.x copy, a few non-zero bytes from byte 2 102 that look like uninitialised memory; the 2026 save has zeros), the plug-ins' sequence data `sdat` of Lumetri Color (16 instances and the project's `EfdG`), the mask counter `mkif` byte 8. No chunk is added or reordered, and no `pard`, `cdat`, `ldta`, `cdta`, `tdb4`, keyframe, text or shape chunk differs. | All 12 families. | | **25.x against 24.x**: only `head` (bytes 1, 3, 5 and 7), and the session noise in `opti` and `sdat` above. | All 12 families. | @@ -218,12 +223,12 @@ the value from the corpus manifest (test file in parentheses at the end of each API: `LayerHeader` (`is_video_enabled`/`is_enabled`, `is_audio_enabled`, `is_effects_enabled`, `is_auto_oriented`, `light_type`, `blend_mode`, `stretch` / `stretch_den`, `stretch_ratio`, -`stretch_percent`, `in_point`, `out_point`); the same switches on `Layer`; `Layer::track_matte` +`in_point`, `out_point`); the same switches on `Layer`; `Layer::track_matte` (`TrackMatte`, `MatteMode`; test `tests/matte.rs`). `stretch` is now the numerator of the ratio (it was the whole multiplier before 50 % and 150 % were sampled). Tests: `tests/header.rs`, `tests/layer_switches.rs`. The layer name itself is the `Utf8` child of the `Layr` (empty while the name is the source's); the layer then shows the name of its source item -(`Project::layer_display_name`). +(`Project::layer_source(...)` then `Item::display_name`, the rule the importer uses). ### 3.2 Composition header (`Item/cdta`, 204 bytes; the decoder needs the first 176) and renderer @@ -248,9 +253,9 @@ numerator of the ratio (it was the whole multiplier before 50 % and 150 % were s The inert `ADBE Time Remapping` record of every AV layer has `tduM` = the source's duration in seconds: 5 for 5 s compositions (124, 257, 258), 3 and 2 for a 3 s and a 2 s composition (600, 601), 1 for the 1 s WAV (351), 0.1 for the three-frame sequence (353), 0 on solids and stills (000, 126). Every property's keyframe time scale (`tdb4` bytes 12 to 15) follows the composition (23 976 in every property of `L` after 229 and 230). The corpus manifest calls the classic renderer -`ADBE Advanced 3d`; the file stores `ADBE Escher` / `Classic 3D`. API: `CompHeader::frame_rate_exact` -(exact fraction), `duration_frames`, `shutter_phase`, `display_start`, `display_start_seconds`, -`display_start_frame`. Tests: `tests/header.rs`, `tests/layer_switches.rs`. +`ADBE Advanced 3d`; the file stores `ADBE Escher` / `Classic 3D`. API: `CompHeader::{time_base, +frame_rate_fixed, duration_units, shutter_phase, display_start}` (`display_start` is the exact +fraction). Tests: `tests/header.rs`, `tests/layer_switches.rs`. ### 3.3 Items, footage and precomps @@ -295,7 +300,8 @@ name. A footage layer shows the item's name while it has no name of its own (351 Test: `tests/header.rs` (precomp), `tests/text_markers_footage.rs` and `tests/batch3_objects.rs` (footage), `tests/layer_switches.rs` (solid colour). API: `Footage::{duration, native_frame_rate, -bits_per_pixel, file_size, audio_sample_rate, is_sequence, sequence_frame_count}`. +bits_per_pixel, file_size, audio_sample_rate, is_sequence}`; `Item::sequence_info` (with +`SequenceInfo::frame_count`). ### 3.4 Properties: sparse storage, static values and units @@ -404,8 +410,8 @@ out speed, at 40 out influence**). Scale key 1 of 305 (`n = 3`, in 0 / 33 %, out 15 / 66) is `0,0, .33,.33, 15,15, .66,.66`. Component `i` uses slot `i` of each block (**V** for the block layout and the units, Scale speed in fractions per second and Mask Feather speed in pixels per second; **E** that different values per dimension sit in the slot of their dimension, because -the tool that made the corpus applies one ease to all dimensions; **V** since the exporter wrote three different eases for the three Scale dimensions and After Effects read the same three back, aep-export.md R13). `Keyframe::plain_ease_slots` -returns the doubles as stored, `Keyframe::ease_component(i)` one component's `KeyEase`. Layout of +the tool that made the corpus applies one ease to all dimensions; **V** since the exporter wrote three different eases for the three Scale dimensions and After Effects read the same three back, aep-export.md R13). `Keyframe::ease_component(i)` +returns one component's `KeyEase`, `Keyframe::ease_components` all of them. Layout of the spatial template: `u32` at 8 (1 normally, 0 after the spatial auto-bezier and continuous switches were turned off together, 054), `f64` arc length of the segment to the next key at 16 (100 in 204 key 1, 0 on the last key and right after creation; safe to ignore), **ease at 24 in @@ -430,7 +436,7 @@ in tangent (`n` x `f64`, relative offset from the value) at `56 + 8 n`, out tang | **ease, colour switched to bezier** | spatial n=4 @24..55 | keyframe.set_interpolation bezier on two colour keys stores speed 0 and influence 0 on both sides, and After Effects interpolates them linearly (598 Start Color red to blue: 0.75 / 0.5 / 0.25 red at 0.5 / 1 / 1.5 s); a custom colour ease cannot be set by script | 598 | V | | **ease, mask path** | 64-byte record @24..55 | in speed, in influence, out speed, out influence as in the spatial template: 599 out / in influence 0.75, 700 0.2 / 0.5, speeds 0 (the path at 0.5 s is 5.42 % / 28.0 % of the way). The `f64` at 16 is 1 (093) or 2 (599, 700) on the first key and 0 on the last (not decoded) | 093, 599, 700 | V (influence), ? (speed unit) | -API: `Keyframe::{interpolation_in, interpolation_out, kind, value, ease, ease_component, ease_components, plain_ease_slots, color_distance, spatial}`; +API: `Keyframe::{interpolation_in, interpolation_out, kind, value, ease, ease_component, ease_components, spatial}` (the `f64` at byte 16 of a colour key is `SpatialKey::segment_length`); `Property::keys()` hands the property's template to its records. Marker record (16 bytes): time, `03 03` hold bytes, 8 stale bytes. Mask path record (64 bytes): the common fields and the ease at byte 24 (599, 700). Test: `tests/values.rs`, `tests/keyframes.rs`. @@ -767,7 +773,7 @@ pins are placed by hand or by a preset. 328), so mode and inverted are reported per mask. `mkif` byte 8 is the running mask id counter of the **layer** in the session (1, 2 for the two masks of 328; 329 then gets 3; undo does not reset it; a new layer starts at 1 again): exposed as `Mask::counter`, a session counter and not a stable id. API: -`Mask::{mode, inverted, counter}`, `Project::mask_vertices_px` (vertices in layer pixels), `Layer::mask_atoms` (`MaskAtom`: the mask with its atom group, its path property and its shapes by position, for the keys and expressions of the mask properties). Tests: +`Mask::{mode, inverted, counter}`, `ShapePath::vertices(size)` (vertices in layer pixels: `size` is the size of the layer's source, `Project::layer_source_size`; a text or shape layer has no source size and its mask boxes are already pixels, so the importer passes 1 x 1), `Layer::mask_atoms` (`MaskAtom`: the mask with its atom group, its path property and its shapes by position, for the keys and expressions of the mask properties). Tests: `tests/masks.rs` (with a synthetic check of the handle order), `tests/batch3_objects.rs`. ### 3.9 Shape layers (`tdgp[ADBE Root Vectors Group]`) @@ -895,7 +901,7 @@ groups as on other layers, with these conventions: | What | Path | Stored | Evidence | Conf. | |---|---|---|---|---| -| camera Position, default (auto state) | `ADBE Transform Group/ADBE Position`, `tdb4` 76 and 77 = 1 although no expression exists | `[0, 0, -2666.67]` (x and y are 0 whatever the composition size; z = -zoom). A **one-node** camera reports `[0, 0, -2666.67]` (251). **Correction (S16): a reopened two-node camera reports `[0, 0, -2666.67]` too (256 read back), and After Effects renders it there**: the default two-node camera of 537 above the 3D solid of 486 renders from `[0, 0, -2666.67]` towards the point of interest, not like the composition's default view (537 differs from 486 in 24 % of the grid points, and EffectCraft's import with the stored position matches 537 but for 4 edge points). The `[960, 540, ...]` of the batch-3 notes was not reproduced (535 and 537, created by script, read back the stored value too); `CameraPosition::reported` keeps the old rule and is marked as contradicted | 256, 251, 537 | V | +| camera Position, default (auto state) | `ADBE Transform Group/ADBE Position`, `tdb4` 76 and 77 = 1 although no expression exists | `[0, 0, -2666.67]` (x and y are 0 whatever the composition size; z = -zoom). A **one-node** camera reports `[0, 0, -2666.67]` (251). **Correction (S16): a reopened two-node camera reports `[0, 0, -2666.67]` too (256 read back), and After Effects renders it there**: the default two-node camera of 537 above the 3D solid of 486 renders from `[0, 0, -2666.67]` towards the point of interest, not like the composition's default view (537 differs from 486 in 24 % of the grid points, and EffectCraft's import with the stored position matches 537 but for 4 edge points). The `[960, 540, ...]` of the batch-3 notes was not reproduced (535 and 537, created by script, read back the stored value too); the reader keeps only the stored value (`CameraPosition::stored`) | 256, 251, 537 | V | | camera Position, set by the user | same, `tdb4` 76 and 77 = 0 | the absolute value `[100, 200, -1000]` | 253 | V | | camera point of interest | `ADBE Anchor Point` | `[960, 540, 0]` in **pixels** (not a fraction); the record's `tdsb` is 1 on a two-node and 3 on the one-node camera (not asserted) | 251, 253, 256 | V (value), E (`tdsb`) | | light Position, default | same as the camera | `[0, 0, -1333.33]`, as After Effects reports it (255 read back); the point of interest `[960, 540, 0]` | 254, 255 | V | @@ -911,10 +917,8 @@ groups as on other layers, with these conventions: A point light made by converting a spot light has a different property set (`Orientation`, `Rotate X/Y/Z`, `Cone Angle`, `Cone Feather 2` in the default state) that a spot light lacks (252); whether that -is an artefact of the conversion is untested. API: `Layer::camera_position`, -`CameraPosition::reported`, `Layer::camera_point_of_interest`, `Layer::light_intensity`, -`Layer::{camera_zoom, camera_depth_of_field, camera_focus_distance, camera_aperture, light_color, -light_cone_angle, light_cone_feather}`, `DEFAULT_CAMERA_ZOOM` (2 666.67 is verified for the 1920 px +is an artefact of the conversion is untested. API: `Layer::camera_position`; the camera and light +options are read by the importer's table (`import/camera.rs`); `DEFAULT_CAMERA_ZOOM` (2 666.67 is verified for the 1920 px composition only), `Layer::orientation` (`Orientation`; test `orientation_is_otst_otda` in `tests/batch5.rs`). Render checks against After Effects (20 px grid, 5 184 points): 486 1 point off, 535 2, 536 0, 537 4 (anti-aliased edges); **488 (Orientation `[10, 20, 30]`): 272 points diff --git a/crates/aep/src/aepx.rs b/crates/aep/src/aepx.rs index 9fad696ce..5267080fb 100644 --- a/crates/aep/src/aepx.rs +++ b/crates/aep/src/aepx.rs @@ -12,13 +12,23 @@ //! instructions. A document type declaration is refused (no entity expansion). It runs without //! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything //! it allocates is bounded by the size of the input. +//! +//! There are two readers. [`read_aepx`] builds the owned tree element by element +//! (`aepx/tree.rs`). The importer, which only reads, uses `index_aepx`: it decodes every body +//! (hex, text, the alias JSON, the numbers) into one buffer and records the chunks as entries of +//! the flat index ([`Draft`]s, in depth-first order, bodies as ranges of the buffer), which +//! [`crate::TreeIndex::from_drafts`] turns into an index that borrows the buffer. +//! The tests hold both to the same tree and the same errors. use std::borrow::Cow; -use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::chunk::{AepFile, LIST, NESTED_IDS}; use crate::error::AepError; +use crate::index::{Draft, Kind as EntryKind}; use crate::read::{Limits, parse}; +mod tree; + /// The root element of an `.aepx`. const ROOT: &[u8] = b"AfterEffectsProject"; /// The element that holds the XMP packet (the trailer of an `.aep`). @@ -68,17 +78,71 @@ pub fn read_aepx(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`read_aepx`]. pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + tree::read(data, limits) +} + +/// An `.aepx` read for the flat index: every body decoded into one buffer, and the chunks as +/// [`Draft`]s in depth-first order with their bodies as ranges of that buffer. +/// [`crate::TreeIndex::from_drafts`] makes the index, which borrows the buffer. +pub struct AepxDrafts { + /// The decoded bodies (hex, text, alias JSON, numbers) and the XMP packet. + pub buffer: Vec, + pub nodes: Vec, + /// The XMP packet: a range of `buffer` (empty without one). + pub trailer: (usize, usize), +} + +/// Reads an `.aepx` into [`AepxDrafts`]: the tree [`read_aepx_with_limits`] gives, with the same +/// errors, without one allocation per chunk. +/// +/// # Errors +/// See [`read_aepx_with_limits`]. When memory runs out, two errors come up in more cases than +/// there: [`AepError::TooLarge`] when the chunk list cannot grow (the owned-tree reader would +/// abort), and [`AepError::Xml`] "bdata is too large to hold in memory" also when the shared body +/// buffer cannot grow (the owned-tree reader reserves each value on its own). +pub fn index_aepx(data: &[u8], limits: Limits) -> Result { let text = match std::str::from_utf8(data) { Ok(t) => t.as_bytes(), Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), }; - Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() + // Every counted element takes at least 3 bytes (`` in a document cut short), so there are + // never more chunks than this. + let bound = (text.len() / 3).min(limits.max_chunks); + let mut nodes = Vec::new(); + let _ = nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + // Hex is most of an `.aepx` and decodes to half its length: one buffer of that size usually + // holds everything. A failed reservation is not an error here; the buffer grows as it goes. + let mut buffer = Vec::new(); + let _ = buffer.try_reserve_exact(text.len() / 2); + let reader = Reader { + data: text, + pos: 0, + limits, + count: 0, + stack: Vec::new(), + root_closed: false, + trailer: None, + buffer, + nodes, + bound, + attrs: Vec::new(), + }; + reader.run() } +/// The slots the chunk list starts with; it doubles from there (as the array of +/// [`crate::TreeIndex`] does). +const INITIAL_SLOTS: usize = 1024; + fn xml(offset: usize, what: &'static str) -> AepError { AepError::Xml { offset, what } } +/// The pad byte of a body of `len` bytes, as [`crate::Chunk::data`] sets it. +fn pad_of(len: usize) -> Option { + if len % 2 == 1 { Some(0) } else { None } +} + /// What an open element becomes when it closes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Kind { @@ -102,13 +166,14 @@ struct Frame { at: usize, /// The name as written, to match the end tag. name: (usize, usize), - /// The decoded `bdata` attribute. - bdata: Option>, + /// The chunk's entry in the list (for `Chunk`, `Utf8` and `Alias`). + entry: Option, + /// The buffer length when the element opened: its body starts here. + mark: usize, + /// The end of the decoded `bdata` attribute in the buffer. + bdata_end: Option, /// The attributes of a `fileReference`, in document order. attrs: Vec<(String, String)>, - /// Collected text (only for `string` and the XMP packet). - text: Vec, - children: Vec>, } struct Reader<'d> { @@ -117,13 +182,19 @@ struct Reader<'d> { limits: Limits, count: usize, stack: Vec, - /// The top-level chunks once the root element has closed. - top: Option>>, - trailer: Option>, + /// True once the root element has closed. + root_closed: bool, + trailer: Option<(usize, usize)>, + buffer: Vec, + nodes: Vec, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, + /// The attributes of the current start tag; kept to reuse its allocation. + attrs: Vec<((usize, usize), Cow<'d, [u8]>)>, } -impl Reader<'_> { - fn run(mut self) -> Result, AepError> { +impl<'d> Reader<'d> { + fn run(mut self) -> Result { if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { self.pos = 3; } @@ -137,11 +208,13 @@ impl Reader<'_> { if let Some(open) = self.stack.last() { return Err(xml(open.at, "an element is not closed before the end of the file")); } - let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; - Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + if !self.root_closed { + return Err(xml(self.data.len(), "no AfterEffectsProject root element")); + } + Ok(AepxDrafts { buffer: self.buffer, nodes: self.nodes, trailer: self.trailer.unwrap_or((0, 0)) }) } - fn rest(&self) -> &[u8] { + fn rest(&self) -> &'d [u8] { self.data.get(self.pos..).unwrap_or(&[]) } @@ -181,14 +254,13 @@ impl Reader<'_> { self.pos = start; let end = self.skip_past(b"]]>", "a CDATA section is not closed")?; let content = self.data.get(start..end).unwrap_or(&[]); - let at = start; - match self.stack.last_mut() { + match self.stack.last() { Some(frame) => { if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { - frame.text.extend_from_slice(content); + self.buffer.extend_from_slice(content); } } - None => return Err(xml(at, "CDATA outside the root element")), + None => return Err(xml(start, "CDATA outside the root element")), } } else if rest.starts_with(b" { self.pos = self.pos.saturating_add(n); } - fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + fn slice(&self, (a, b): (usize, usize)) -> &'d [u8] { self.data.get(a..b).unwrap_or(&[]) } - fn start_tag(&mut self) -> Result<(), AepError> { - let at = self.pos; - self.pos = self.pos.saturating_add(1); - let name = self.name()?; - let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); - let self_closing = loop { + /// Reads the attributes of a start tag into `attrs`; true when the tag closes itself. + fn attributes(&mut self, at: usize, attrs: &mut Vec<((usize, usize), Cow<'d, [u8]>)>) -> Result { + loop { self.skip_ws(); let rest = self.rest(); if rest.starts_with(b"/>") { self.pos = self.pos.saturating_add(2); - break true; + return Ok(true); } if rest.starts_with(b">") { self.pos = self.pos.saturating_add(1); - break false; + return Ok(false); } if rest.is_empty() { return Err(xml(at, "a start tag is not closed")); @@ -259,11 +328,27 @@ impl Reader<'_> { if raw.contains(&b'<') { return Err(xml(vstart, "`<` inside an attribute value")); } - let mut value = Vec::new(); - decode_entities(raw, vstart, &mut value)?; + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; self.pos = vend.saturating_add(1); attrs.push((key, value)); - }; + } + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = std::mem::take(&mut self.attrs); + attrs.clear(); + let self_closing = self.attributes(at, &mut attrs)?; let kind = self.kind_of(name, at)?; if kind != Kind::Root { self.count = self.count.saturating_add(1); @@ -275,15 +360,27 @@ impl Reader<'_> { return Err(AepError::TooDeep); } } - let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; - if matches!(kind, Kind::Chunk(_)) { - if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { - frame.bdata = Some(decode_hex(value, key.1)?); + let mark = self.buffer.len(); + let mut frame = Frame { kind, at, name, entry: None, mark, bdata_end: None, attrs: Vec::new() }; + match kind { + Kind::Chunk(id) => { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + decode_hex_into(value, key.1, MAX_BDATA, &mut self.buffer)?; + frame.bdata_end = Some(self.buffer.len()); + } + frame.entry = Some(self.open_entry(id)?); } - } else if kind == Kind::Alias { - frame.attrs = - attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + Kind::Utf8 => frame.entry = Some(self.open_entry(*b"Utf8")?), + Kind::Alias => { + frame.attrs = attrs + .iter() + .map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())) + .collect(); + frame.entry = Some(self.open_entry(*b"alas")?); + } + Kind::Root | Kind::Xmp | Kind::Skip => {} } + self.attrs = attrs; self.stack.push(frame); if self_closing { self.close()?; @@ -291,12 +388,38 @@ impl Reader<'_> { Ok(()) } + /// Appends the entry of a chunk whose element just opened; its body is filled in when it + /// closes. The list grows by its length (at least 16) but never past [`Self::bound`]; an + /// allocation the target cannot make is an error, not an abort. + fn open_entry(&mut self, id: [u8; 4]) -> Result { + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + u32::try_from(at.checked_add(1).ok_or(AepError::TooManyChunks)?).map_err(|_| AepError::TooManyChunks)?; + if at == self.nodes.capacity() { + // Past the bound (which only a wrong guess could reach) it keeps doubling. + let room = self.bound.saturating_sub(at); + let more = if room == 0 { at.max(16) } else { at.max(16).min(room) }; + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(Draft::placeholder(id)); + Ok(at) + } + + /// Fills in the entry at `at` once its element has closed; its subtree is what follows it. + fn finish(&mut self, at: usize, id: [u8; 4], kind: EntryKind, list_type: [u8; 4], body: (usize, usize), pad: Option) -> Result<(), AepError> { + let end = u32::try_from(self.nodes.len()).map_err(|_| AepError::TooManyChunks)?; + if let Some(d) = self.nodes.get_mut(at) { + *d = Draft { start: body.0, len: body.1.saturating_sub(body.0), id, list_type, end, kind, pad }; + } + Ok(()) + } + /// Classifies an element by its name and where it is. fn kind_of(&self, name: (usize, usize), at: usize) -> Result { let n = self.slice(name); let parent = match self.stack.last() { None => { - if self.top.is_some() { + if self.root_closed { return Err(xml(at, "a second root element")); } if n != ROOT { @@ -340,39 +463,104 @@ impl Reader<'_> { self.close() } - /// Closes the innermost element and hands its chunk to the parent. + /// Closes the innermost element and fills in its chunk's entry. fn close(&mut self) -> Result<(), AepError> { let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; - let chunk = match frame.kind { - Kind::Root => { - self.top = Some(frame.children); - return Ok(()); - } - Kind::Skip => return Ok(()), + match frame.kind { + Kind::Root => self.root_closed = true, + Kind::Skip => {} Kind::Xmp => { + // Only the first packet is kept; a later one is dropped from the buffer (its + // children are skipped, so nothing else was written after it opened). if self.trailer.is_none() { - self.trailer = Some(frame.text); + self.trailer = Some((frame.mark, self.buffer.len())); + } else { + self.buffer.truncate(frame.mark); } - return Ok(()); } - Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), - Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), - Kind::Chunk(id) => chunk_of(id, frame)?, - }; - if let Some(parent) = self.stack.last_mut() { - parent.children.push(chunk); + Kind::Utf8 => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let end = self.buffer.len(); + self.finish(at, *b"Utf8", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Alias => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + self.buffer.extend_from_slice(alias_json(&frame.attrs).as_bytes()); + let end = self.buffer.len(); + self.finish(at, *b"alas", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Chunk(id) => self.close_chunk(id, &frame)?, } Ok(()) } + /// The chunk of a closed element named by a chunk id (what the owned reader's `chunk_of` + /// built): `bdata` wins over children, `numS` / `ppSn` hold their number, a nested id holds its + /// children (or nothing), any other name is a `LIST` of that type. + fn close_chunk(&mut self, id: [u8; 4], frame: &Frame) -> Result<(), AepError> { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let after = at.checked_add(1).ok_or(AepError::TooManyChunks)?; + if let Some(bdata_end) = frame.bdata_end { + // The children were read (and counted, and checked) but are not part of the chunk. + self.nodes.truncate(after); + self.buffer.truncate(bdata_end); + let body = (frame.mark, bdata_end); + let pad = pad_of(bdata_end.saturating_sub(frame.mark)); + if OPAQUE_LIST_TYPES.contains(&id) { + return self.finish(at, LIST, EntryKind::OpaqueList, id, body, pad); + } + return self.finish(at, id, EntryKind::Data, [0; 4], body, pad); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let missing = || xml(frame.at, "a numS or ppSn element without its number"); + // Exactly one child chunk, a `Utf8`: the entry after this one, whose subtree runs to the end. + let only = self.nodes.get(after).copied().filter(|c| usize::try_from(c.end).ok() == Some(self.nodes.len())).ok_or_else(missing)?; + if only.id != *b"Utf8" { + return Err(missing()); + } + let text = match only.kind { + EntryKind::Data | EntryKind::OpaqueList => { + only.start.checked_add(only.len).and_then(|end| self.buffer.get(only.start..end)).unwrap_or(&[]) + } + EntryKind::List | EntryKind::Nested => &[], + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body: [u8; 8]; + let bytes: &[u8] = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + body = u64::from(n as u32).to_be_bytes(); + body.get(4..).unwrap_or(&[]) + } else { + body = text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes(); + &body + }; + self.nodes.truncate(after); + self.buffer.truncate(frame.mark); + self.buffer.extend_from_slice(bytes); + let end = self.buffer.len(); + return self.finish(at, id, EntryKind::Data, [0; 4], (frame.mark, end), pad_of(bytes.len())); + } + let empty = (frame.mark, frame.mark); + if NESTED_IDS.contains(&id) { + if self.nodes.len() == after { + return self.finish(at, id, EntryKind::Data, [0; 4], empty, None); + } + return self.finish(at, id, EntryKind::Nested, [0; 4], empty, None); + } + self.finish(at, LIST, EntryKind::List, id, empty, None) + } + fn text(&mut self) -> Result<(), AepError> { let start = self.pos; let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); let end = start.saturating_add(len); self.pos = end; let raw = self.data.get(start..end).unwrap_or(&[]); - match self.stack.last_mut() { - Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + match self.stack.last() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut self.buffer), Some(_) => Ok(()), None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), None => Err(xml(start, "text outside the root element")), @@ -393,41 +581,6 @@ fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { Some(id) } -/// The chunk of a closed element named by a chunk id. -fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { - if let Some(bytes) = frame.bdata { - if OPAQUE_LIST_TYPES.contains(&id) { - let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; - return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); - } - return Ok(Chunk::data(id, bytes)); - } - // Numbers written as text: `0` and `150.000000000`. - if &id == b"numS" || &id == b"ppSn" { - let text = match frame.children.as_slice() { - [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), - _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), - }; - let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); - let body = if &id == b"numS" { - // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. - let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); - let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; - (n as u32).to_be_bytes().to_vec() - } else { - text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() - }; - return Ok(Chunk::data(id, body)); - } - if NESTED_IDS.contains(&id) { - if frame.children.is_empty() { - return Ok(Chunk::data(id, Vec::new())); - } - return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); - } - Ok(Chunk::list(id, frame.children)) -} - /// Appends `raw` with the predefined entities and character references decoded. fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let mut i = 0; @@ -465,23 +618,74 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), A Ok(()) } -/// Decodes a `bdata` value (hex digits, two per byte). +/// Decodes a `bdata` value (hex digits, two per byte) into a new `Vec` of exactly its size. fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { - if text.len() % 2 == 1 { - return Err(xml(offset, "bdata has an odd number of hex digits")); - } + decode_hex_at_most(text, offset, MAX_BDATA) +} + +/// The most bytes one `bdata` may decode to: what the 32-bit size field of a chunk can hold (a +/// larger body could not be written back as an `.aep` either). +const MAX_BDATA: usize = u32::MAX as usize; + +/// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it). +fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result, AepError> { + // Checked before anything is reserved, then sized once (collecting into `Option` has no + // size hint and regrows step by step), and without aborting when the memory is not there. + let pairs = check_hex(text, offset, max)?; + let mut out = Vec::new(); + out.try_reserve_exact(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, &mut out)?; + Ok(out) +} + +/// Appends the bytes of a `bdata` value (hex digits, two per byte, at most `max` bytes) to `out`. +/// On an error `out` is unchanged. +fn decode_hex_into(text: &[u8], offset: usize, max: usize, out: &mut Vec) -> Result<(), AepError> { + let pairs = check_hex(text, offset, max)?; + // Grown at most once per value (amortised over a shared buffer), and without aborting when the + // memory is not there. + out.try_reserve(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, out) +} + +/// Appends the bytes of a `bdata` value that [`check_hex`] accepted. On an error (a digit the +/// check let through, which it does not) `out` is unchanged. +fn decode_checked(text: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let digit = |b: u8| match b { b'0'..=b'9' => Some(b - b'0'), b'a'..=b'f' => Some(b - b'a' + 10), b'A'..=b'F' => Some(b - b'A' + 10), _ => None, }; - let (pairs, _) = text.as_chunks::<2>(); - pairs - .iter() - .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) - .collect::>>() - .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + let mark = out.len(); + for [hi, lo] in text.as_chunks::<2>().0 { + match (digit(*hi), digit(*lo)) { + (Some(h), Some(l)) => out.push((h << 4) | l), + _ => { + out.truncate(mark); + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + } + } + Ok(()) +} + +/// Checks a `bdata` value before anything is reserved for it: an even number of hex digits, at +/// most `max` bytes. Returns the number of bytes it decodes to. +fn check_hex(text: &[u8], offset: usize, max: usize) -> Result { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let pairs = text.len() / 2; + if pairs > max { + return Err(AepError::TooLarge); + } + // One cheap pass checks every digit first. Blocks without an early exit inside, so the check + // vectorises; it stops at the first bad block. + if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) { + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + Ok(pairs) } /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document @@ -529,6 +733,7 @@ fn json_str(out: &mut String, s: &str) { #[cfg(test)] mod tests { use super::*; + use crate::index::TreeIndex; fn doc(body: &str) -> String { format!("\n{body}\n") @@ -629,6 +834,92 @@ mod tests { assert!(matches!(read_aepx(b"\xff"), Err(AepError::Xml { .. }))); } + /// Attribute values decode entities only where they have them. + #[test] + fn attribute_values_decode_entities_only_where_they_have_them() { + // An entity inside `bdata` still decodes before the hex does. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x01, 0xab][..])); + // A bad entity in any attribute, even of an element that is skipped, is still an error. + for bad in ["", "", ""] { + assert!(matches!(read_aepx(doc(bad).as_bytes()), Err(AepError::Xml { .. })), "{bad}"); + } + // Without `&` the value is taken as written. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x0a, 0xff][..])); + } + + /// A large bdata with a bad digit is refused before it is sized. + #[test] + fn a_large_bdata_with_a_bad_digit_is_refused_before_it_is_sized() { + // 10 MB of hex that is bad from the first pair (and once more at the very end). + let mut text = b"zz".to_vec(); + text.extend(std::iter::repeat_n(*b"00", 5_000_000).flatten()); + assert_eq!(decode_hex(&text, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let mut late = vec![b'0'; 10_000_000]; + late.extend_from_slice(b"0g"); + assert_eq!(decode_hex(&late, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let x = doc(&format!("", std::str::from_utf8(&text).unwrap())); + assert!(matches!(read_aepx(x.as_bytes()), Err(AepError::Xml { .. }))); + } + + /// Bdata decodes up to the size limit and not past it. + #[test] + fn bdata_decodes_up_to_the_size_limit_and_not_past_it() { + assert_eq!(decode_hex_at_most(b"00ff10", 0, 3), Ok(vec![0x00, 0xff, 0x10])); + assert_eq!(decode_hex_at_most(b"00ff10", 0, 2), Err(AepError::TooLarge)); + assert_eq!(decode_hex_at_most(b"", 0, 0), Ok(Vec::new())); + assert_eq!(decode_hex_at_most(b"00", 0, 0), Err(AepError::TooLarge)); + // The odd-length error still comes first. + assert_eq!(decode_hex_at_most(b"001", 0, 0), Err(xml(0, "bdata has an odd number of hex digits"))); + // The real limit is what a chunk's 32-bit size field holds. + assert_eq!(MAX_BDATA as u64, u64::from(u32::MAX)); + } + + /// Decode hex matches the collecting version. + #[test] + fn decode_hex_matches_the_collecting_version() { + /// The collecting decoder this one replaced, as the reference. + fn old(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + } + let alphabet = b"0123456789abcdefABCDEFgG \0"; + let mut seed = 0x2545_f491_4f6c_dd1du64; + for len in 0..200usize { + for _ in 0..20 { + let text: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Mostly hex digits, now and then a byte that is not one. + let i = (seed % 1000) as usize; + alphabet[if i < 990 { i % 22 } else { 22 + i % 4 }] + }) + .collect(); + let new = decode_hex(&text, 7); + assert_eq!(new, old(&text, 7)); + if let Ok(v) = new { + assert_eq!(v.capacity(), v.len()); + } + } + } + } + #[test] fn limits_hold() { let deep = format!("{}{}", "".repeat(100_000), "".repeat(100_000)); @@ -638,4 +929,293 @@ mod tests { let many = format!("{}", "".repeat(10)); assert_eq!(read_aepx_with_limits(many.as_bytes(), Limits { max_depth: 64, max_chunks: 9 }), Err(AepError::TooManyChunks)); } + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number. + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { usize::try_from(self.next() % n as u64).unwrap() } + } + + /// One of `items`. + fn pick<'s>(&mut self, items: &[&'s str]) -> &'s str { + items[self.below(items.len())] + } + } + + /// `n` random bytes as hex, upper or lower case. + fn hex(rng: &mut Rng, n: usize) -> String { + let upper = rng.below(4) == 0; + (0..n).map(|_| if upper { format!("{:02X}", rng.below(256)) } else { format!("{:02x}", rng.below(256)) }).collect() + } + + /// Random `.aepx` content: every element form the reader knows (data, lists, nested ids, + /// strings, numbers, `btdk`, file references, skipped elements, XMP packets), `bdata` next to + /// children, bodies that read as chunk sequences, entities, CDATA, comments and stray text. + fn content(rng: &mut Rng, depth: usize, budget: &mut usize, out: &mut String) { + for _ in 0..rng.below(6) { + if *budget == 0 { + return; + } + *budget -= 1; + match rng.below(20) { + 0..=3 => { + let id = rng.pick(&["cdat", "tdmn", "head", "Utf8", "idta", "LIST", "a"]); + let n = rng.below(9); + out.push_str(&format!("<{id} bdata=\"{}\"/>", hex(rng, n))); + } + 4..=6 if depth < 10 => { + let id = rng.pick(&["Fold", "tdgp", "Layr", "Pin", "LIST", "btdk", "sspc"]); + let attr = if rng.below(5) == 0 { format!(" bdata=\"{}\"", hex(rng, 3)) } else { String::new() }; + out.push_str(&format!("<{id}{attr}>")); + content(rng, depth + 1, budget, out); + out.push_str(&format!("")); + } + 7 => { + let id = rng.pick(&["tdsn", "pdnm", "fnam"]); + match rng.below(4) { + 0 => out.push_str(&format!("<{id}/>")), + 1 => out.push_str(&format!("<{id}>name & A")), + // A body that is itself a chunk sequence (`Utf8`, 1 byte, pad): still data. + 2 => out.push_str(&format!("<{id} bdata=\"557466380000000161{}\"/>", if rng.below(2) == 0 { "00" } else { "" })), + _ => out.push_str(&format!("<{id}>ab")), + } + } + 8 => { + let text = + rng.pick(&["plain", "A < B", "", "ab", "", "été", "xgoney", "\r\n"]); + out.push_str(&format!("{text}")); + } + 9 => { + let id = rng.pick(&["numS", "ppSn"]); + let inner = rng.pick(&[ + "0", + " 7 ", + "-2147483648", + "150.000000000", + "x", + "", + "12", + "", + "", + "34", + ]); + let attr = if rng.below(6) == 0 { " bdata=\"00000002\"" } else { "" }; + out.push_str(&format!("<{id}{attr}>{inner}")); + } + 10 => { + let n = rng.pick(&["", "41", "414243", "4c495354000000000000"]); + out.push_str(&format!("")); + } + 11 => { + let platform = rng.pick(&["Win", "1", "Mac"]); + out.push_str(&format!( + "", + rng.below(3) + )); + } + 12 if depth < 10 => { + out.push_str("text"); + content(rng, depth + 1, budget, out); + out.push_str(""); + } + 13 => { + let packet = rng.pick(&["x]]>", "plain & text", ""]); + out.push_str(&format!("{packet}")); + } + 14 => out.push_str(rng.pick(&["", "", "\n ", "stray"])), + _ => { + let n = rng.below(5); + out.push_str(&format!("", hex(rng, n))); + } + } + } + } + + /// The default limits and six tight ones (depth 0, 1 and 3; 0, 7 and 40 chunks). + fn limit_sets() -> [Limits; 7] { + [ + Limits { max_depth: 0, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 0 }, + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index reader and the owned-tree reader give the same `Result` (the tree, or the error), + /// and the index walks like the index of the owned tree. + fn same(bytes: &[u8], limits: Limits) { + let old = read_aepx_with_limits(bytes, limits); + let new = index_aepx(bytes, limits).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(new, old, "{}", String::from_utf8_lossy(bytes)); + if let Ok(file) = &old { + let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, limits).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let expected = TreeIndex::from_file(file).unwrap(); + assert_eq!(index.len(), expected.len()); + assert_eq!(index.trailer(), expected.trailer()); + let a: Vec = index.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + let b: Vec = expected.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + assert_eq!(a, b); + } + } + + /// The two readers give the same tree and errors on generated and damaged documents. + #[test] + fn the_two_readers_agree_on_generated_and_damaged_documents() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let noise = b"<>/=\"'&;#x0aZ \n![]-?"; + for _ in 0..600 { + let mut budget = 80; + let mut body = String::new(); + content(&mut rng, 0, &mut budget, &mut body); + let x = doc(&body); + for l in limit_sets() { + same(x.as_bytes(), l); + } + for _ in 0..4 { + let mut bad = x.clone().into_bytes(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(bad.len()); + bad[at] = noise[rng.below(noise.len())]; + } + for l in limit_sets() { + same(&bad, l); + } + } + let cut = rng.below(x.len()); + same(&x.as_bytes()[..cut], Limits::default()); + // Bytes that are not UTF-8, and a multi-byte character cut in half. + let mut bad = x.clone().into_bytes(); + let at = rng.below(bad.len()); + bad.insert(at, 0xff); + same(&bad, Limits::default()); + let open = format!("{}é", x.trim_end().trim_end_matches("")).into_bytes(); + same(&open[..open.len() - 1], Limits::default()); + } + } + + /// The flat chunk list never reserves more slots than the input or the chunk limit allows. + #[test] + fn the_chunk_list_is_bounded_by_the_input_and_the_limit() { + let many = format!("{}", "".repeat(10_000)); + let d = index_aepx(many.as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.nodes.len(), 10_000); + assert!(d.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", d.nodes.capacity()); + // The buffer is reserved once, at half the input, and holds nothing here. + assert!(d.buffer.is_empty() && d.buffer.capacity() <= many.len() / 2); + // At the limit the list never holds more slots than the limit. + let d = index_aepx(many.as_bytes(), Limits { max_depth: 64, max_chunks: 10_000 }).unwrap(); + assert!(d.nodes.capacity() <= 10_000); + let few = format!("{}", "".repeat(100)); + let d = index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 100 }).unwrap(); + assert!(d.nodes.capacity() <= 100, "{} slots for at most 100 chunks", d.nodes.capacity()); + assert_eq!(index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 99 }).err(), Some(AepError::TooManyChunks)); + // A tiny input cannot hold more than a third of its bytes in elements. + let tiny = doc(""); + assert!(index_aepx(tiny.as_bytes(), Limits::default()).unwrap().nodes.capacity() <= tiny.len() / 3); + } + + /// What a closed element drops is gone from the buffer and the chunk list, but was counted. + #[test] + fn dropped_content_leaves_nothing_behind_but_still_counts() { + // `bdata` wins over the children: they are read, counted and dropped. + let x = doc("xyzq"); + let d = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.nodes.len()), (&[1u8][..], 1)); + assert_eq!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 4 }).err(), Some(AepError::TooManyChunks)); + assert!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 5 }).is_ok()); + // A child's error is raised although the child would be dropped. + assert!(matches!(index_aepx(doc("").as_bytes(), Limits::default()), Err(AepError::Xml { .. }))); + // Only the first XMP packet stays in the buffer. + let packets = doc(&"packet".repeat(1000)); + let d = index_aepx(packets.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.trailer), (&b"packet"[..], (0, 6))); + // A number keeps its 4 or 8 bytes, not its text. + let d = index_aepx(doc(" 7 1.5").as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.buffer, [&[0, 0, 0, 7][..], &1.5f64.to_be_bytes()[..]].concat()); + assert_eq!(d.nodes.len(), 2); + } + + /// Bodies keep what the element says they are, even where reading the same bytes as an `.aep` + /// would decide otherwise: a nested id whose `bdata` is a chunk sequence stays data, an empty + /// `btdk` stays an opaque list, an empty list at the deepest level stays a list. + #[test] + fn bodies_keep_the_kind_the_element_gives_them() { + let x = doc(""); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let top: Vec<_> = index.chunks().collect(); + assert_eq!(top.len(), 3); + assert_eq!(top[0].bytes().map(<[u8]>::len), Some(11)); + assert!(top[0].children().next().is_none()); + assert!(top[1].is_opaque_list() && top[1].bytes() == Some(&[][..])); + assert!(top[2].is_opaque_list() && top[2].bytes().map(<[u8]>::len) == Some(12)); + // 64 levels: the innermost (empty) list is a list, not opaque. + let deep = format!("{}{}", "".repeat(64), "".repeat(64)); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(deep.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().all(|(_, n)| n.is_list(b"Fold") && !n.is_opaque_list())); + same(x.as_bytes(), Limits::default()); + same(deep.as_bytes(), Limits::default()); + } + + /// Hostile input is refused by both readers with the same errors. + #[test] + fn hostile_input_is_refused_alike_by_both_readers() { + let hostile = [ + format!("{}{}", "".repeat(100_000), "".repeat(100_000)), + format!("{}", "".repeat(1000)), + doc(&format!("", "00".repeat(100_000))), + doc(&format!("", "00".repeat(100_000))), + doc("4294967296"), + doc("1e400x"), + doc("�"), + doc("�"), + doc(&format!("{}", "".repeat(1000))), + format!("{}", "".repeat(10)), + " = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap(); + for l in limit_sets() { + let index = index_aepx(&bytes, l).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(index, read_aepx_with_limits(&bytes, l), "{}", path.display()); + } + } + } } diff --git a/crates/aep/src/aepx/tree.rs b/crates/aep/src/aepx/tree.rs new file mode 100644 index 000000000..6c0bc8435 --- /dev/null +++ b/crates/aep/src/aepx/tree.rs @@ -0,0 +1,362 @@ +//! The `.aepx` reader of the owned chunk tree ([`super::read_aepx`]): it builds the tree element by +//! element. The importer uses the index reader in `aepx.rs` instead; the tests compare the two on +//! generated, damaged and corpus input (the same tree and the same errors). + +use std::borrow::Cow; + +use super::{FILE_REFERENCE, OPAQUE_LIST_TYPES, ROOT, STRING, XMP, alias_json, chunk_id, decode_entities, decode_hex, xml}; +use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::error::AepError; +use crate::read::Limits; + +/// [`super::read_aepx_with_limits`]. +pub(super) fn read(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = + attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} diff --git a/crates/aep/src/arbitrary.rs b/crates/aep/src/arbitrary.rs index 45768ae44..70627139b 100644 --- a/crates/aep/src/arbitrary.rs +++ b/crates/aep/src/arbitrary.rs @@ -7,9 +7,9 @@ //! Hue/Saturation (444-446) or a Levels value (512-516) changes only bytes of this `aRbp` (and stale //! UI bytes of `pard`); the visible parameters of the same names keep no record. -use crate::Chunk; use crate::bytes::{be_f32, be_i32, be_u16, be_u32}; use crate::effects::Effect; +use crate::index::Node; impl<'f, 'a> Effect<'f, 'a> { /// The arbitrary data stored for the parameter with this match name (`-NNNN`): the @@ -17,13 +17,13 @@ impl<'f, 'a> Effect<'f, 'a> { /// none. pub fn arbitrary_data(&self, match_name: &str) -> Option<&'f [u8]> { let values = self.chunk().child_list(b"tdgp")?; - let kids = values.children(); - let at = kids.iter().position(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; - // The value record, then the data list (no match name of its own). - let rest = kids.get(at.checked_add(1)?..)?; - let mut it = rest.iter().take(3); + let mut kids = values.children(); + // Up to the parameter's match name, then the value record, then the data list (no match + // name of its own). + kids.find(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; + let mut it = kids.take(3); let _record = it.next().filter(|c| c.is_list(b"tdbs"))?; - it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Chunk::bytes) + it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Node::bytes) } } diff --git a/crates/aep/src/camera.rs b/crates/aep/src/camera.rs index 69cd8c2c8..934f36bd6 100644 --- a/crates/aep/src/camera.rs +++ b/crates/aep/src/camera.rs @@ -16,7 +16,6 @@ //! only when it is not the default 100. use crate::typed::{Layer, PropNode}; -use crate::values::{Tracked, ValueSource, read, read_opt}; /// The Position of a camera or light layer as stored. #[derive(Debug, Clone, Copy, PartialEq)] @@ -28,16 +27,6 @@ pub struct CameraPosition { pub auto_state: bool, } -impl CameraPosition { - /// The position as the batch-3 notes had After Effects report it: in the auto state of a - /// two-node camera (`two_node`) x and y are the composition centre; otherwise the stored - /// value. **Contradicted by 256 and 537 (session S16): After Effects reports and renders the - /// stored value**; use `stored` for what After Effects shows. - pub fn reported(&self, comp_width: u32, comp_height: u32, two_node: bool) -> [f64; 3] { - if self.auto_state && two_node { [f64::from(comp_width) / 2.0, f64::from(comp_height) / 2.0, self.stored[2]] } else { self.stored } - } -} - impl<'f, 'a> Layer<'f, 'a> { /// The Position record of a camera or light layer. pub fn camera_position(&self) -> Option { @@ -49,87 +38,9 @@ impl<'f, 'a> Layer<'f, 'a> { let derived = h.get(76) == Some(&1) && h.get(77) == Some(&1) && position.expression().is_none(); Some(CameraPosition { stored, auto_state: derived }) } - - /// The point of interest of a camera in pixels (its Anchor Point record); `None` when absent. - pub fn camera_point_of_interest(&self) -> Option<[f64; 3]> { - let Some(PropNode::Group(transform)) = self.properties()?.get("ADBE Transform Group") else { return None }; - let tracked = read(&transform, "ADBE Anchor Point", [0.0; 3], 1.0); - matches!(tracked.source, ValueSource::Stored | ValueSource::Keyframed).then_some(tracked.value) - } - - /// A light's intensity in percent; the default 100 when the property is absent. - pub fn light_intensity(&self) -> Option> { - let root = self.properties()?; - let Some(PropNode::Group(options)) = root.get("ADBE Light Options Group") else { return None }; - let t = read(&options, "ADBE Light Intensity", [100.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } } /// The camera zoom After Effects uses when `ADBE Camera Zoom` is absent: 2666.67 px for the 50 mm /// default camera in the 1920 px wide corpus composition. Whether it follows other composition /// widths was not sampled. pub const DEFAULT_CAMERA_ZOOM: f64 = 2_666.666_666_666_666_5; - -impl<'f, 'a> Layer<'f, 'a> { - fn camera_options(&self) -> Option> { - match self.properties()?.get("ADBE Camera Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - fn light_options(&self) -> Option> { - match self.properties()?.get("ADBE Light Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - /// A camera's zoom in pixels (`ADBE Camera Options Group/ADBE Camera Zoom`); absent = the - /// default, [`DEFAULT_CAMERA_ZOOM`] (341: 2000 is stored). The auto Position keeps `-2666.67` and - /// is not recomputed from the zoom. - pub fn camera_zoom(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Zoom", [DEFAULT_CAMERA_ZOOM], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// The depth of field switch (`ADBE Camera Depth of Field`, a checkbox; off by default, 342). - pub fn camera_depth_of_field(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Depth of Field", [0.0], 1.0); - Some(Tracked { value: t.value[0] != 0.0, source: t.source }) - } - - /// A camera's focus distance in pixels (`ADBE Camera Focus Distance`); `None` while absent (its - /// default is not known). - pub fn camera_focus_distance(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Focus Distance", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A camera's aperture (`ADBE Camera Aperture`); `None` while absent (its default is not known). - pub fn camera_aperture(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Aperture", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A light's colour as `[A, R, G, B]` fractions (`ADBE Light Options Group/ADBE Light Color`, - /// stored as doubles 0..255); white when absent (343 stores red). - pub fn light_color(&self) -> Option> { - Some(read(&self.light_options()?, "ADBE Light Color", [1.0; 4], 1.0 / 255.0)) - } - - /// A spot light's cone angle in degrees (`ADBE Light Cone Angle`); 90 when absent (344 stores 60). - /// A point light stores 90 with `tdsb` 3. - pub fn light_cone_angle(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Angle", [90.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A spot light's cone feather in percent (`ADBE Light Cone Feather 2`); 50 when absent (345 - /// stores 30). - pub fn light_cone_feather(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Feather 2", [50.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } -} diff --git a/crates/aep/src/effects.rs b/crates/aep/src/effects.rs index 25b7051ab..65f3c6125 100644 --- a/crates/aep/src/effects.rs +++ b/crates/aep/src/effects.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; -use crate::Chunk; use crate::bytes::{array, be_f32, be_f64, be_i32, be_u16, be_u32, cstr, text, until_nul}; +use crate::index::Node; use crate::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; /// The parameter type, the `u32` at byte 12 of `pard`. The numbering matches the plug-in @@ -132,7 +132,7 @@ pub enum LayerRef { #[derive(Debug, Clone)] pub struct Effect<'f, 'a> { match_name: String, - chunk: &'f Chunk<'a>, + chunk: Node<'f, 'a>, } /// Where the parameter definitions of an effect instance come from. @@ -178,13 +178,13 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The underlying `LIST sspc` chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } /// The instance name shown in the effect controls (`fnam`). pub fn name(&self) -> Option { - self.chunk.child(b"fnam")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text) + self.chunk.child(b"fnam")?.child(b"Utf8")?.bytes().map(text) } /// The name the user gave the instance: the `tdsn` of the instance's value group when it is @@ -192,7 +192,7 @@ impl<'f, 'a> Effect<'f, 'a> { /// `None` while the instance shows the effect's name. pub fn user_name(&self) -> Option { let values = self.chunk.child_list(b"tdgp")?; - let name = values.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; + let name = values.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (name != "-_0_/-" && !name.is_empty()).then_some(name) } @@ -250,7 +250,7 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The parameters described by `part` (a `LIST parT`) with the values of this instance. - fn params_from(&self, part: &'f Chunk<'a>, borrowed: bool) -> Vec> { + fn params_from(&self, part: Node<'f, 'a>, borrowed: bool) -> Vec> { let mut out: Vec> = Vec::new(); // The stored values by match name: the first entry of a name, as `PropGroup::get` finds it // (a group of that name means no stored value). Indexed once: a lookup per parameter @@ -271,8 +271,8 @@ impl<'f, 'a> Effect<'f, 'a> { out.push(EffectParam { index: out.len(), match_name, - kind: ParamKind::from_u32(be_u32(pard, 12).unwrap_or(u32::MAX)), - name: cstr(pard, 16, 32).unwrap_or_default(), + kind: Pard(pard).kind(), + name: Pard(pard).name().unwrap_or_default(), items: None, borrowed_definition: borrowed, pard, @@ -280,7 +280,7 @@ impl<'f, 'a> Effect<'f, 'a> { }); } else if c.is(b"pdnm") && let Some(last) = out.last_mut() - && let Some(u) = c.children().iter().find(|u| u.is(b"Utf8")) + && let Some(u) = c.child(b"Utf8") && let Some(b) = u.bytes() // An empty `pdnm` names nothing (some writers store one: the effect // supplies its items). @@ -293,8 +293,9 @@ impl<'f, 'a> Effect<'f, 'a> { } } -fn has_pard(part: &Chunk<'_>) -> bool { - part.children().iter().any(|c| c.is(b"pard")) +/// True when a `LIST parT` holds parameter definitions (a `pard`). +fn has_pard(part: Node<'_, '_>) -> bool { + part.children().any(|c| c.is(b"pard")) } impl<'f, 'a> Project<'f, 'a> { @@ -302,18 +303,18 @@ impl<'f, 'a> Project<'f, 'a> { /// name and the `LIST parT` of the copied instance, in file order (the stack order of the /// types, 365/366). The `EfDC` count before them is little-endian and is not needed to read /// them. A type whose copy has no definitions is left out. - pub fn effect_definitions(&self) -> Vec<(String, &'f Chunk<'a>)> { - let mut out: Vec<(String, &'f Chunk<'a>)> = Vec::new(); + pub fn effect_definitions(&self) -> Vec<(String, Node<'f, 'a>)> { + let mut out: Vec<(String, Node<'f, 'a>)> = Vec::new(); let Some(defs) = self.effect_defs else { return out }; let mut seen: HashSet = HashSet::new(); - for df in defs.children().iter().filter(|c| c.is_list(b"EfDf")) { + for df in defs.children().filter(|c| c.is_list(b"EfDf")) { let mut name: Option = None; for c in df.children() { if c.is(b"tdmn") { name = c.bytes().map(|b| text(until_nul(b))); } else if c.is_list(b"sspc") && let Some(n) = name.take() - && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(p)) + && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(*p)) && seen.insert(n.clone()) { out.push((n, part)); @@ -328,8 +329,8 @@ impl<'f, 'a> Project<'f, 'a> { /// For a type without a project-level copy it also keeps the own `parT` of the first instance /// in the file (items, layers and effects in file order) that has one. pub fn effect_definition_index(&self) -> EffectDefinitions<'f, 'a> { - let by_name: HashMap> = self.effect_definitions().into_iter().collect(); - let mut instances: HashMap> = HashMap::new(); + let by_name: HashMap> = self.effect_definitions().into_iter().collect(); + let mut instances: HashMap> = HashMap::new(); for item in self.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { @@ -337,7 +338,7 @@ impl<'f, 'a> Project<'f, 'a> { if by_name.contains_key(&fx.match_name) || instances.contains_key(&fx.match_name) { continue; } - if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(p)) { + if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(*p)) { instances.insert(fx.match_name, part); } } @@ -363,9 +364,9 @@ impl<'f, 'a> Project<'f, 'a> { #[derive(Debug, Clone, Default)] pub struct EffectDefinitions<'f, 'a> { /// The project-level copies (`EfdG`). - by_name: HashMap>, + by_name: HashMap>, /// For a type without a project-level copy: the own `parT` of the first instance that has one. - instances: HashMap>, + instances: HashMap>, } impl<'f, 'a> EffectDefinitions<'f, 'a> { @@ -392,7 +393,7 @@ impl<'f, 'a> EffectDefinitions<'f, 'a> { return effect.params(); } match self.by_name.get(effect.match_name()).or_else(|| self.instances.get(effect.match_name())) { - Some(part) => effect.params_from(part, true), + Some(part) => effect.params_from(*part, true), None => Vec::new(), } } @@ -484,26 +485,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { pub fn pard_default(&self) -> Option { match self.kind { ParamKind::Popup if self.borrowed_definition => None, - ParamKind::FloatSlider => be_f64(self.pard, 56).map(ParamValue::Number), - ParamKind::FixedSlider | ParamKind::Angle => be_i32(self.pard, 56).map(|v| ParamValue::Number(f64::from(v) / 65536.0)), - ParamKind::IntSlider => be_i32(self.pard, 56).map(|v| ParamValue::Number(f64::from(v))), - ParamKind::Checkbox => be_u32(self.pard, 56).map(|v| ParamValue::Bool(v != 0)), - ParamKind::Popup => be_u32(self.pard, 56).map(ParamValue::Index), - ParamKind::Point => { - let p = [f64::from(be_i32(self.pard, 56)?) / 65536.0, f64::from(be_i32(self.pard, 60)?) / 65536.0]; - // A fresh instance stores fractions of the layer (0.5, 0.5); After Effects' save of - // a project it opened stores percent (50, 50: the re-saves of read-back tests e1 and - // e2, where Gradient Ramp's Start and End and every other point default came back - // as 50 / 0 and 50 / 100). Point - // defaults lie inside the layer, so a component beyond 1 tells percent. - let percent = p.iter().any(|c| c.abs() > 1.0 + 1e-9); - Some(ParamValue::Point(if percent { p.map(|c| c / 100.0) } else { p })) - } - ParamKind::Color => { - let [a, r, g, b] = array::<4>(self.pard, 56)?.map(|c| f64::from(c) / 255.0); - Some(ParamValue::Argb([a, r, g, b])) - } - _ => None, + kind => Pard(self.pard).default(kind), } } @@ -557,22 +539,14 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// slider differs per parameter (Amount 37.5 % is stored 37.5, Blend 25 % is stored 0.25), so /// this is what tells a reader the scale. `None` for other types. pub fn valid_max(&self) -> Option { - match self.kind { - ParamKind::FixedSlider => be_i32(self.pard, 128).map(|v| f64::from(v) / 65536.0), - ParamKind::FloatSlider => be_f32(self.pard, 108).map(f64::from), - ParamKind::IntSlider => be_i32(self.pard, 128).map(f64::from), - _ => None, - } + Pard(self.pard).valid_max(self.kind) } /// The smallest valid value of a float slider, the `f32` at byte 104 of `pard` (Exposure /// -100, Offset -2, Gamma Correction 0.1: 389). `None` for other types (a fixed slider's /// minimum is not decoded). pub fn valid_min(&self) -> Option { - match self.kind { - ParamKind::FloatSlider => be_f32(self.pard, 104).map(f64::from), - _ => None, - } + Pard(self.pard).valid_min(self.kind) } /// The range of a float slider's slider (what the control's drag covers, inside the valid @@ -581,7 +555,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// for other types. pub fn slider_range(&self) -> Option<(f64, f64)> { match self.kind { - ParamKind::FloatSlider => Some((f64::from(be_f32(self.pard, 112)?), f64::from(be_f32(self.pard, 116)?))), + ParamKind::FloatSlider => Pard(self.pard).slider_range(self.kind), _ => None, } } @@ -591,7 +565,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { if self.kind != ParamKind::Popup { return None; } - be_u16(self.pard, 60) + Pard(self.pard).popup_item_count() } /// For a popup: the selected item index (1-based): the stored selection, else the default. @@ -619,6 +593,93 @@ impl<'f, 'a> EffectParam<'f, 'a> { } } +/// A parameter definition (`pard`, 148 bytes; aep-format.md 3.7): the one decoder of its fields. +/// The import reads them through [`EffectParam`]; the export writes them from its effect table and +/// its tests read them back through this. +#[derive(Clone, Copy, Debug)] +pub struct Pard<'f>(pub &'f [u8]); + +impl Pard<'_> { + /// The type code, the `u32` at byte 12. + pub fn code(self) -> Option { + be_u32(self.0, 12) + } + + /// The parameter type (an unreadable code is [`ParamKind::Other`]). + pub fn kind(self) -> ParamKind { + ParamKind::from_u32(self.code().unwrap_or(u32::MAX)) + } + + /// The parameter name at byte 16 (32 bytes). + pub fn name(self) -> Option { + cstr(self.0, 16, 32) + } + + /// The default at byte 56 as a parameter of type `kind` reads it (see + /// [`EffectParam::pard_default`]). + pub fn default(self, kind: ParamKind) -> Option { + let p = self.0; + match kind { + ParamKind::FloatSlider => be_f64(p, 56).map(ParamValue::Number), + ParamKind::FixedSlider | ParamKind::Angle => be_i32(p, 56).map(|v| ParamValue::Number(f64::from(v) / 65536.0)), + ParamKind::IntSlider => be_i32(p, 56).map(|v| ParamValue::Number(f64::from(v))), + ParamKind::Checkbox => be_u32(p, 56).map(|v| ParamValue::Bool(v != 0)), + ParamKind::Popup => be_u32(p, 56).map(ParamValue::Index), + ParamKind::Point => { + let pt = [f64::from(be_i32(p, 56)?) / 65536.0, f64::from(be_i32(p, 60)?) / 65536.0]; + // A fresh instance stores fractions of the layer (0.5, 0.5); After Effects' save of + // a project it opened stores percent (50, 50: the re-saves of read-back tests e1 and + // e2, where Gradient Ramp's Start and End and every other point default came back + // as 50 / 0 and 50 / 100). Point + // defaults lie inside the layer, so a component beyond 1 tells percent. + let percent = pt.iter().any(|c| c.abs() > 1.0 + 1e-9); + Some(ParamValue::Point(if percent { pt.map(|c| c / 100.0) } else { pt })) + } + ParamKind::Color => { + let [a, r, g, b] = array::<4>(p, 56)?.map(|c| f64::from(c) / 255.0); + Some(ParamValue::Argb([a, r, g, b])) + } + _ => None, + } + } + + /// The valid maximum of a slider (see [`EffectParam::valid_max`]). + pub fn valid_max(self, kind: ParamKind) -> Option { + match kind { + ParamKind::FixedSlider => be_i32(self.0, 128).map(|v| f64::from(v) / 65536.0), + ParamKind::FloatSlider => be_f32(self.0, 108).map(f64::from), + ParamKind::IntSlider => be_i32(self.0, 128).map(f64::from), + _ => None, + } + } + + /// The valid minimum of a float slider (see [`EffectParam::valid_min`]). + pub fn valid_min(self, kind: ParamKind) -> Option { + match kind { + ParamKind::FloatSlider => be_f32(self.0, 104).map(f64::from), + _ => None, + } + } + + /// A slider's range, the one its value records repeat as `tdum` / `tduM`: a float slider's + /// `f32` pair at 112 / 116, a fixed slider's 16.16 pair and an integer slider's `i32` pair at + /// 132 / 136. + pub fn slider_range(self, kind: ParamKind) -> Option<(f64, f64)> { + let p = self.0; + match kind { + ParamKind::FloatSlider => Some((f64::from(be_f32(p, 112)?), f64::from(be_f32(p, 116)?))), + ParamKind::FixedSlider => Some((f64::from(be_i32(p, 132)?) / 65536.0, f64::from(be_i32(p, 136)?) / 65536.0)), + ParamKind::IntSlider => Some((f64::from(be_i32(p, 132)?), f64::from(be_i32(p, 136)?))), + _ => None, + } + } + + /// A popup's item count, the `u16` at byte 60. + pub fn popup_item_count(self) -> Option { + be_u16(self.0, 60) + } +} + impl<'f, 'a> Layer<'f, 'a> { /// The effects on the layer, in stack order (`ADBE Effect Parade`). pub fn effects(&self) -> Vec> { @@ -638,6 +699,30 @@ impl<'f, 'a> Layer<'f, 'a> { #[cfg(test)] mod tests { use super::*; + use crate::Chunk; + + #[test] + fn a_parameter_definition_reads_its_fields_and_nothing_from_a_cut_record() { + let mut b = vec![0u8; 148]; + b[12..16].copy_from_slice(&7u32.to_be_bytes()); + b[16..22].copy_from_slice(b"Amount"); + b[56..64].copy_from_slice(&2.5f64.to_be_bytes()); + let p = Pard(&b); + assert_eq!((p.code(), p.name().as_deref()), (Some(7), Some("Amount"))); + assert_eq!(p.default(ParamKind::FloatSlider), Some(ParamValue::Number(2.5))); + // The same bytes as a checkbox and as a popup: the `u32` at 56, and the `u16` at 60. + b[56..60].copy_from_slice(&1u32.to_be_bytes()); + b[60..62].copy_from_slice(&3u16.to_be_bytes()); + let p = Pard(&b); + assert_eq!(p.default(ParamKind::Checkbox), Some(ParamValue::Bool(true))); + assert_eq!((p.default(ParamKind::Popup), p.popup_item_count()), (Some(ParamValue::Index(1)), Some(3))); + assert_eq!(p.default(ParamKind::IntSlider), Some(ParamValue::Number(1.0))); + // A record cut before a field has no value for it, and an unreadable code is `Other`. + let cut = Pard(&b[..12]); + assert_eq!((cut.code(), cut.kind(), cut.name(), cut.popup_item_count()), (None, ParamKind::Other(u32::MAX), None, None)); + assert_eq!(cut.default(ParamKind::FloatSlider), None); + assert_eq!(Pard(&b[..58]).default(ParamKind::Checkbox), None); + } fn tdmn(name: &str) -> Chunk<'static> { let mut b = name.as_bytes().to_vec(); @@ -670,11 +755,11 @@ mod tests { ("A".to_string(), Chunk::list(*b"tdbs", vec![Chunk::data(*b"cdat", vec![0u8; 8])])), ]; let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["A", "B", "C"]); let first = params.first().and_then(|p| p.stored).expect("A stored"); - assert!(first.chunk.children().is_empty(), "the first A (the second has a cdat)"); + assert!(first.chunk.children().next().is_none(), "the first A (the second has a cdat)"); assert!(params.get(1).is_some_and(|p| p.stored.is_none()), "a group of that name is no stored value"); assert!(params.get(2).is_some_and(|p| p.stored.is_none())); } @@ -687,7 +772,7 @@ mod tests { let names: Vec = (0..n).map(|i| format!("P{i}")).collect(); let stored: Vec<(String, Chunk<'static>)> = names.iter().map(|m| (m.clone(), tdbs())).collect(); let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.len(), n); assert!(params.iter().all(|p| p.stored.is_some())); @@ -707,19 +792,19 @@ mod tests { } let efdg = Chunk::list(*b"EfdG", defs); let fold = Chunk::list(*b"Fold", Vec::new()); - let project = Project { fold: &fold, effect_defs: Some(&efdg) }; + let project = Project { fold: Node::from(&fold), effect_defs: Some(Node::from(&efdg)) }; let all = project.effect_definitions(); assert_eq!(all.len(), n); let index = project.effect_definition_index(); let bare = Chunk::list(*b"sspc", vec![Chunk::list(*b"parT", Vec::new())]); for i in (0..n).step_by(50) { - let fx = Effect { match_name: format!("FX{i}"), chunk: &bare }; + let fx = Effect { match_name: format!("FX{i}"), chunk: Node::from(&bare) }; assert_eq!(index.source(&fx), DefinitionSource::Project); let params = index.params(&fx); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["R0"], "the first copy's definitions"); assert!(params.iter().all(|p| p.borrowed_definition)); } - let unknown = Effect { match_name: "nope".into(), chunk: &bare }; + let unknown = Effect { match_name: "nope".into(), chunk: Node::from(&bare) }; assert_eq!(index.source(&unknown), DefinitionSource::Missing); assert!(index.params(&unknown).is_empty()); assert_eq!(project.definition_source(&unknown), DefinitionSource::Missing); diff --git a/crates/aep/src/footage.rs b/crates/aep/src/footage.rs index 58a653a4a..e8c89b924 100644 --- a/crates/aep/src/footage.rs +++ b/crates/aep/src/footage.rs @@ -2,8 +2,8 @@ //! of a layer's source. use crate::bytes::{be_f32, be_f64, be_i32, be_u32, cstr, le_u32, text, until_nul}; -use crate::chunk::Chunk; use crate::header::Rational; +use crate::index::Node; use crate::typed::{Item, ItemKind, Layer, Project}; /// The footage data of an item (`LIST Pin ` with `sspc`, `opti` and, for files, `Als2/alas`). @@ -101,21 +101,6 @@ impl PsdLayer { } impl Footage { - /// The number of frames of an image sequence (`duration * native_frame_rate`: 3/30 s at 30 fps is - /// 3); `None` for anything else. - pub fn sequence_frame_count(&self) -> Option { - if !self.is_sequence { - return None; - } - let d = self.duration?; - let frames = i128::from(d.num).checked_mul(i128::from(self.native_frame_rate))?; - let den = i128::from(d.den); - if den <= 0 || frames % den != 0 { - return None; - } - u32::try_from(frames / den).ok() - } - /// The solid's colour as fractions R, G, B. pub fn solid_rgb(&self) -> Option<[f32; 3]> { let [_, r, g, b] = self.solid_argb?; @@ -269,14 +254,14 @@ impl SequenceInfo { /// is unreadable (empty). A name that is not a plain file name (holding `/`, `\`, `:` or a control /// character such as NUL, or being `.` or `..`) makes the whole list unreadable (empty): the names /// are joined to the folder. -fn sequence_file_list(list: &Chunk<'_>) -> Vec { +fn sequence_file_list(list: Node<'_, '_>) -> Vec { // After Effects writes the count first (the one sample); a list without a readable count is // not read rather than guessed. let Some(count) = list.child(b"StVS").and_then(|c| c.bytes()).and_then(|b| le_u32(b, 0)).and_then(|n| usize::try_from(n).ok()) else { return Vec::new(); }; let names: Vec = - list.children().iter().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).take(count).map(text).filter(|n| !n.is_empty()).collect(); + list.children().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).take(count).map(text).filter(|n| !n.is_empty()).collect(); if names.iter().any(|n| n.contains(|c: char| matches!(c, '/' | '\\' | ':') || c.is_control()) || n == "." || n == "..") { return Vec::new(); } @@ -308,9 +293,8 @@ impl<'f, 'a> Item<'f, 'a> { } let pin = self.chunk.child_list(b"Pin ")?; let sspc = pin.child(b"sspc")?.bytes()?; - let kids = pin.children(); - let after = kids.iter().position(|c| c.is_list(b"Als2"))?; - let naming = || kids.iter().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); + let after = pin.children().position(|c| c.is_list(b"Als2"))?; + let naming = || pin.children().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); let (start, end, digits) = (be_u32(sspc, 172)?, be_u32(sspc, 176)?, be_u32(sspc, 180)?); // Force alphabetical order: the file list replaces the numbering. if let Some(list) = naming().find(|c| c.is_list(b"StVc")) { @@ -320,7 +304,7 @@ impl<'f, 'a> Item<'f, 'a> { } return Some(SequenceInfo { prefix: String::new(), suffix: String::new(), start, end, digits, files }); } - let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).map(text).take(2).collect(); + let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).map(text).take(2).collect(); let (prefix, suffix) = match names.as_slice() { [prefix, suffix] => (prefix.clone(), suffix.clone()), // An empty prefix is not stored (`0001.png` has only `.png`). @@ -350,12 +334,6 @@ impl<'f, 'a> Item<'f, 'a> { self.chunk.child(b"idta")?.bytes()?.get(58).copied() } - /// The ids of the compositions this item uses as precomp layers: the little-endian `u32` of - /// each `idpi` chunk. - pub fn used_item_ids(&self) -> Vec { - self.chunk.children().iter().filter(|c| c.is(b"idpi")).filter_map(|c| le_u32(c.bytes()?, 0)).collect() - } - /// The footage data of a footage item. pub fn footage(&self) -> Option { if self.kind()? != ItemKind::Footage { @@ -405,11 +383,6 @@ impl<'f, 'a> Project<'f, 'a> { self.item_by_id(layer.header()?.source_id?) } - /// The name a layer shows: its own name, or its source's name when it has none. - pub fn layer_display_name(&self, layer: &Layer<'f, 'a>) -> Option { - layer.name().or_else(|| self.layer_source(layer)?.display_name()) - } - /// The pixel size of a layer's source: a footage item's width and height, or a precomp's /// composition size. pub fn layer_source_size(&self, layer: &Layer<'f, 'a>) -> Option<(u32, u32)> { diff --git a/crates/aep/src/header.rs b/crates/aep/src/header.rs index 8593a5633..ac869095f 100644 --- a/crates/aep/src/header.rs +++ b/crates/aep/src/header.rs @@ -237,12 +237,6 @@ impl LayerHeader { self.flag_bytes.get(byte).is_some_and(|b| b & mask != 0) } - /// Byte 37 bit `0x01`: the layer has a name set by the user (clear on a default-named camera or - /// light and on a precomp layer, whose name is the source's). - pub fn name_is_user_set(&self) -> bool { - self.flag(0, 0x01) - } - /// Byte 37 bit `0x02`: guide layer. pub fn is_guide(&self) -> bool { self.flag(0, 0x02) @@ -333,17 +327,6 @@ impl LayerHeader { Rational::new(i64::from(self.stretch), i64::from(self.stretch_den)) } - /// The time stretch in percent (`stretch / stretch_den * 100`); a zero denominator reads as 1. - pub fn stretch_percent(&self) -> f64 { - let den = if self.stretch_den == 0 { 1.0 } else { f64::from(self.stretch_den) }; - f64::from(self.stretch) / den * 100.0 - } - - /// The start time in seconds, `None` for a zero denominator. - pub fn start_seconds(&self) -> Option { - self.start.to_f64() - } - /// The absolute in point, `start + in * stretch`, exact and reduced. pub fn in_point(&self) -> Option { self.start.plus_scaled(&self.in_relative, &self.stretch_ratio()?) @@ -355,16 +338,6 @@ impl LayerHeader { pub fn out_point(&self) -> Option { self.start.plus_scaled(&self.out_relative, &self.stretch_ratio()?) } - - /// The absolute in point in seconds. - pub fn in_point_seconds(&self) -> Option { - self.in_point()?.to_f64() - } - - /// The absolute out point in seconds. - pub fn out_point_seconds(&self) -> Option { - self.out_point()?.to_f64() - } } impl<'f, 'a> Layer<'f, 'a> { @@ -464,64 +437,10 @@ impl CompHeader { self.time_base.frame_rate() } - /// The frame rate from the 16.16 fixed-point field (25 is `0x0019_0000`). - pub fn frame_rate_from_fixed(&self) -> f64 { - f64::from(self.frame_rate_fixed) / 65536.0 - } - /// The duration in seconds. pub fn duration_seconds(&self) -> Option { self.time_base.seconds(i32::try_from(self.duration_units).ok()?) } - - /// The work area start in seconds. - pub fn work_area_start_seconds(&self) -> Option { - self.time_base.seconds(i32::try_from(self.work_area_start_units).ok()?) - } - - /// The work area end in seconds; the composition duration when the work area is the whole - /// composition. - pub fn work_area_end_seconds(&self) -> Option { - match self.work_area_end_units { - Some(end) => self.time_base.seconds(i32::try_from(end).ok()?), - None => self.duration_seconds(), - } - } - - /// The background colour as fractions 0..1 (`byte / 255`). - pub fn background_rgb(&self) -> [f64; 3] { - self.background.map(|c| f64::from(c) / 255.0) - } - - /// The pixel aspect ratio as a number. - pub fn pixel_aspect_ratio(&self) -> Option { - self.pixel_aspect.to_f64() - } - - /// The display start time in seconds (the start timecode), `None` when absent or the - /// denominator is zero. - pub fn display_start_seconds(&self) -> Option { - self.display_start?.to_f64() - } - - /// The display start as a frame number (`display start * frame rate`, rounded); 100 at 25 fps - /// for a display start of 4 s. - pub fn display_start_frame(&self) -> Option { - let rate = self.frame_rate()?; - Some((self.display_start_seconds()? * rate).round() as i64) - } - - /// The frame rate as an exact fraction `timescale / frame_duration`, as typed: 29.97 fps is - /// `23976 / 800` (not `30000 / 1001`), 23.976 fps is `23976 / 1000`. - pub fn frame_rate_exact(&self) -> Option { - Rational::new(i64::from(self.time_base.timescale), i64::from(self.time_base.frame_duration)) - } - - /// The duration in whole frames (`duration / frame_duration`). After Effects snaps a duration - /// to a whole number of frames: 5 s at 29.97 fps becomes 150 frames (5.005 s). - pub fn duration_frames(&self) -> Option { - self.duration_units.checked_div(self.time_base.frame_duration) - } } impl<'f, 'a> Comp<'f, 'a> { diff --git a/crates/aep/src/index.rs b/crates/aep/src/index.rs new file mode 100644 index 000000000..76d5430da --- /dev/null +++ b/crates/aep/src/index.rs @@ -0,0 +1,1026 @@ +//! A read-only, flat index of the chunk tree for readers that only look (the importer). +//! +//! [`TreeIndex`] holds every chunk of a file in one `Vec` in depth-first order (a chunk, then its +//! whole subtree, then its next sibling). Each entry keeps its id, list type, pad byte, the bytes of +//! its body (borrowed from the input) and the index one past its subtree, so: +//! - a chunk's subtree is the index range `at + 1 .. end`, +//! - its direct children are found by stepping from `at + 1` to each child's `end`, +//! - the whole tree is one array, grown a few times while it is built and freed at once; it never +//! holds more slots than the input has room for chunk headers or the chunk limit allows. +//! +//! [`crate::parse`] builds the owned [`Chunk`] tree instead (one `Vec` per list), which +//! the exporter and editors need. [`TreeIndex::to_file`] and [`Node::to_chunk`] convert the index +//! into that tree where an owned one is needed. +//! +//! [`TreeIndex::parse_with_limits`] walks the input exactly as the [`Chunk`] parser does: the same +//! chunks are counted against [`Limits::max_chunks`] in the same order, the same lists are kept +//! opaque at [`Limits::max_depth`] or when damaged, and the same errors come back. A test compares +//! the two on the corpus and on generated and damaged trees. Two errors are its own, and both are +//! stricter than the [`Chunk`] parser (which would abort or could not address the tree): +//! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and +//! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. +//! +//! An `.aepx` is indexed by its own reader (`container/aepx.rs`), which decodes every body into +//! one buffer and fills the entries as it reads; the index then borrows that buffer +//! (`TreeIndex::from_drafts`). It gives the tree [`crate::read_aepx`] gives. +//! +//! [`Node`] is the handle the typed view ([`crate::Project`] and the types below it) +//! reads through: a chunk of either tree. The view works the same on both. + +use std::borrow::Cow; +use std::fmt; + +use crate::bytes::be_u32; +use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +use crate::error::AepError; +use crate::read::{Limits, container_end, id4}; + +/// What an entry's body is (the [`Body`] variants). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Data, + /// [`Body::Chunks`]. + Nested, + List, + OpaqueList, +} + +/// One chunk of a [`TreeIndex`]. +#[derive(Debug, Clone, Copy)] +struct Entry<'a> { + /// The body bytes of a data chunk, or the bytes after the list type of an opaque list; empty + /// for a list or nested chunk (their content is their children). + data: &'a [u8], + id: [u8; 4], + /// The list type of a `LIST` (parsed or opaque); zero otherwise. + list_type: [u8; 4], + /// One past the last entry of this chunk's subtree. + end: u32, + kind: Kind, + pad: Option, +} + +impl<'a> Entry<'a> { + /// An entry for a chunk whose body is not read yet. + fn placeholder(id: [u8; 4]) -> Self { + Self { data: &[], id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// An entry of an index whose bytes are still being written (the `.aepx` reader decodes into a +/// buffer that grows, so it keeps positions, not slices): the body is `start..start + len` of +/// that buffer. The fields are those of [`Entry`]; [`TreeIndex::from_drafts`] turns the list +/// into an index once the buffer is final. +#[derive(Debug, Clone, Copy)] +pub struct Draft { + pub(crate) start: usize, + pub(crate) len: usize, + pub(crate) id: [u8; 4], + pub(crate) list_type: [u8; 4], + pub(crate) end: u32, + pub(crate) kind: Kind, + pub(crate) pad: Option, +} + +impl Draft { + /// A data chunk with no body yet. + pub(crate) fn placeholder(id: [u8; 4]) -> Self { + Self { start: 0, len: 0, id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// The chunk tree of a file as one flat, read-only array (see the module documentation). +#[derive(Clone)] +pub struct TreeIndex<'a> { + nodes: Vec>, + trailer: &'a [u8], +} + +impl fmt::Debug for TreeIndex<'_> { + /// The sizes only: the chunk count and the trailer length. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TreeIndex").field("chunks", &self.nodes.len()).field("trailer", &self.trailer.len()).finish() + } +} + +/// An index that does not fit a `u32` (more than 4 billion chunks) is refused as too many chunks. +fn index_u32(at: usize) -> Result { + u32::try_from(at).map_err(|_| AepError::TooManyChunks) +} + +impl<'a> TreeIndex<'a> { + /// Indexes an `.aep` with the default [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::parse`], for the same inputs. + pub fn parse(data: &'a [u8]) -> Result { + Self::parse_with_limits(data, Limits::default()) + } + + /// Indexes an `.aep` with explicit [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::parse_with_limits`], for the same inputs and limits; also + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks and + /// [`AepError::TooLarge`] when the array cannot be allocated. + pub fn parse_with_limits(data: &'a [u8], limits: Limits) -> Result { + let end = container_end(data)?; + // Every chunk header takes 8 bytes of its own and is counted once, so the tree never has + // more chunks than this. + let bound = (end.saturating_sub(12) / 8).min(limits.max_chunks); + let mut builder = Builder { data, limits, count: 0, nodes: Vec::new(), bound }; + // A small start; [`Builder::push`] doubles it as chunks arrive, never past the bound, so + // the array stays proportional to the chunk count (a file of a few large chunks gets a few + // slots, not one per 32 bytes). `try_reserve_exact` so it fails nothing here. + let _ = builder.nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + builder.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(Self { nodes: builder.nodes, trailer }) + } + + /// The index of the entries `drafts` (in depth-first order, see [`Draft`]) whose bodies and + /// trailer are ranges of `buffer`. A range outside `buffer` reads as empty. The list is turned + /// into entries; a [`Draft`] and an [`Entry`] have the same size and alignment, so the standard + /// library reuses the list's allocation for them. + pub fn from_drafts(buffer: &'a [u8], drafts: Vec, trailer: (usize, usize)) -> Self { + let slice = |start: usize, len: usize| start.checked_add(len).and_then(|end| buffer.get(start..end)).unwrap_or(&[]); + let nodes = drafts + .into_iter() + .map(|d| Entry { data: slice(d.start, d.len), id: d.id, list_type: d.list_type, end: d.end, kind: d.kind, pad: d.pad }) + .collect(); + let trailer = buffer.get(trailer.0..trailer.1).unwrap_or(&[]); + Self { nodes, trailer } + } + + /// Indexes an already parsed (or built) chunk tree. The index borrows the file's bytes. + /// + /// # Errors + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. + pub fn from_file(file: &'a AepFile<'_>) -> Result { + // Sized once, to the chunks the tree holds. + let mut nodes: Vec> = Vec::new(); + nodes.try_reserve_exact(file.walk().count()).map_err(|_| AepError::TooLarge)?; + // Each frame: the siblings still to visit and the entry whose subtree they fill. + let mut stack: Vec<(std::slice::Iter<'a, Chunk<'_>>, Option)> = vec![(file.chunks.iter(), None)]; + while let Some((iter, parent)) = stack.last_mut() { + let parent = *parent; + match iter.next() { + Some(chunk) => { + let at = nodes.len(); + // The entry after this one must still have a `u32` index. + let next = index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + let (kind, data, list_type) = match &chunk.body { + Body::Data(d) => (Kind::Data, &d[..], [0; 4]), + Body::Chunks(_) => (Kind::Nested, &[][..], [0; 4]), + Body::List { list_type, .. } => (Kind::List, &[][..], *list_type), + Body::OpaqueList { list_type, data } => (Kind::OpaqueList, &data[..], *list_type), + }; + nodes.push(Entry { data, id: chunk.id, list_type, end: next, kind, pad: chunk.pad }); + let kids = chunk.children(); + if !kids.is_empty() { + stack.push((kids.iter(), Some(at))); + } + } + None => { + stack.pop(); + if let Some(p) = parent { + let end = index_u32(nodes.len())?; + if let Some(e) = nodes.get_mut(p) { + e.end = end; + } + } + } + } + } + Ok(Self { nodes, trailer: &file.trailer[..] }) + } + + /// The number of chunks in the tree (at every depth). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// True when the container holds no chunks. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// The bytes after the container's declared end (see [`AepFile::trailer`]). + pub fn trailer(&self) -> &'a [u8] { + self.trailer + } + + /// The top-level chunks. + pub fn chunks(&self) -> Nodes<'_, 'a> { + Nodes(NodesRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX) }) + } + + /// The first top-level chunk with this id. + pub fn chunk(&self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(&self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } + + /// Visits every chunk depth-first with its nesting depth (the top level is 1), as + /// [`AepFile::walk`] does. + pub fn walk(&self) -> NodeWalk<'_, 'a> { + NodeWalk(WalkRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX), open: Vec::new() }) + } + + /// The owned chunk tree (the bodies stay borrowed from the input): what + /// [`crate::parse`] gives for the same file. + pub fn to_file(&self) -> AepFile<'a> { + AepFile { chunks: self.build(0, u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)), trailer: Cow::Borrowed(self.trailer) } + } + + /// The entry at `at`, `None` past the end. + fn entry(&self, at: u32) -> Option<&Entry<'a>> { + self.nodes.get(usize::try_from(at).ok()?) + } + + /// The end of the subtree at `at`; never before `at + 1`, so walks always move on. + fn end_of(&self, at: u32) -> u32 { + self.entry(at).map_or(u32::MAX, |e| e.end).max(at.saturating_add(1)) + } + + /// The owned chunks for the siblings that start at `next` and fill `next..end`, without + /// recursion. + fn build(&self, next: u32, end: u32) -> Vec> { + struct Frame<'a> { + /// The list or nested chunk being filled; `None` for the outermost range. + at: Option, + kids: Vec>, + next: u32, + end: u32, + } + let mut stack = vec![Frame { at: None, kids: Vec::new(), next, end }]; + loop { + let Some(top) = stack.last_mut() else { return Vec::new() }; + if top.next < top.end { + let at = top.next; + top.next = self.end_of(at); + let Some(e) = self.entry(at).copied() else { + top.next = top.end; + continue; + }; + match e.kind { + Kind::List | Kind::Nested => { + let sub_end = self.end_of(at); + stack.push(Frame { at: Some(at), kids: Vec::new(), next: at.saturating_add(1), end: sub_end }); + } + Kind::Data => top.kids.push(Chunk { id: e.id, body: Body::Data(Cow::Borrowed(e.data)), pad: e.pad }), + Kind::OpaqueList => { + top.kids.push(Chunk { id: e.id, body: Body::OpaqueList { list_type: e.list_type, data: Cow::Borrowed(e.data) }, pad: e.pad }); + } + } + } else { + let Some(frame) = stack.pop() else { return Vec::new() }; + let Some(at) = frame.at else { return frame.kids }; + let Some(e) = self.entry(at).copied() else { continue }; + let body = match e.kind { + Kind::Nested => Body::Chunks(frame.kids), + Kind::List | Kind::Data | Kind::OpaqueList => Body::List { list_type: e.list_type, children: frame.kids }, + }; + let chunk = Chunk { id: e.id, body, pad: e.pad }; + match stack.last_mut() { + Some(parent) => parent.kids.push(chunk), + None => return vec![chunk], + } + } + } + } +} + +/// The slots a [`TreeIndex`] starts with (32 KiB); it doubles from there. +const INITIAL_SLOTS: usize = 1024; + +/// Builds a [`TreeIndex`]; mirrors `read::Parser` step for step (see the module documentation). +struct Builder<'a> { + data: &'a [u8], + limits: Limits, + count: usize, + nodes: Vec>, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, +} + +impl<'a> Builder<'a> { + /// Appends an entry. A full array grows by its length (at least 16), but never past + /// [`Self::bound`]; an allocation the target cannot make is an error, not an abort. + fn push(&mut self, entry: Entry<'a>) -> Result<(), AepError> { + let len = self.nodes.len(); + if len == self.nodes.capacity() { + let more = len.max(16).min(self.bound.saturating_sub(len)).max(1); + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(entry); + Ok(()) + } + + /// Appends the sibling chunks that exactly fill `start..end`, each followed by its subtree. + fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let mut pos = start; + while pos < end { + let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; + let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; + let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let avail = end - body_start; + if size > avail { + return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); + } + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + self.push(Entry::placeholder(id))?; + let body_end = body_start + size; + self.body(at, id, body_start, body_end, depth)?; + let mut next = body_end; + let mut pad = None; + if size % 2 == 1 && next < end { + pad = Some(self.data.get(next).copied().ok_or(AepError::Malformed { offset: next, what: "missing pad byte" })?); + next += 1; + } + let subtree_end = index_u32(self.nodes.len())?; + if let Some(e) = self.nodes.get_mut(at) { + e.pad = pad; + e.end = subtree_end; + } + pos = next; + } + Ok(()) + } + + /// Like [`Self::sequence`] but a structural problem yields `false` and removes what the + /// attempt appended (the caller keeps bytes). The chunk count keeps the attempt's chunks, as + /// the [`Chunk`] parser's does. + fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result { + let mark = self.nodes.len(); + match self.sequence(start, end, depth) { + Ok(()) => Ok(true), + Err(e) if e.is_structural() => { + self.nodes.truncate(mark); + Ok(false) + } + Err(e) => Err(e), + } + } + + /// The input bytes `start..end` (empty when out of range). + fn slice(&self, start: usize, end: usize) -> &'a [u8] { + self.data.get(start..end).unwrap_or(&[]) + } + + /// Fills in the body of the entry at `at` once the parser knows what it is. + fn set(&mut self, at: usize, kind: Kind, data: &'a [u8], list_type: [u8; 4]) { + if let Some(e) = self.nodes.get_mut(at) { + e.kind = kind; + e.data = data; + e.list_type = list_type; + } + } + + /// Fills in the entry at `at` for the body `start..end` and appends its subtree. + fn body(&mut self, at: usize, id: [u8; 4], start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let len = end - start; + if id == LIST && len >= 4 { + let list_type = id4(self.data, start).ok_or(AepError::Malformed { offset: start, what: "short read" })?; + if depth < self.limits.max_depth && self.tentative(start + 4, end, depth + 1)? { + self.set(at, Kind::List, &[], list_type); + } else { + let data = self.slice(start + 4, end); + self.set(at, Kind::OpaqueList, data, list_type); + } + return Ok(()); + } + if NESTED_IDS.contains(&id) && len >= 8 && depth < self.limits.max_depth { + let mark = self.nodes.len(); + if self.tentative(start, end, depth + 1)? && self.nodes.len() > mark { + self.set(at, Kind::Nested, &[], [0; 4]); + return Ok(()); + } + } + let data = self.slice(start, end); + self.set(at, Kind::Data, data, [0; 4]); + Ok(()) + } +} + +/// A chunk of either tree: an owned [`Chunk`] or an entry of a [`TreeIndex`]. The typed view +/// reads through it, so it works the same on both. Cheap to copy. +#[derive(Clone, Copy)] +pub struct Node<'f, 'a>(NodeRepr<'f, 'a>); + +#[derive(Clone, Copy)] +enum NodeRepr<'f, 'a> { + Chunk(&'f Chunk<'a>), + Index(&'f TreeIndex<'a>, u32), +} + +impl<'f, 'a> From<&'f Chunk<'a>> for Node<'f, 'a> { + /// The handle for this value. + fn from(chunk: &'f Chunk<'a>) -> Self { + Node(NodeRepr::Chunk(chunk)) + } +} + +/// Prints the chunk as [`Chunk`]'s `Debug` does, whichever tree it is in, without copying the +/// subtree. +impl fmt::Debug for Node<'_, '_> { + /// The chunk as `Chunk`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + NodeRepr::Chunk(c) => fmt::Debug::fmt(c, f), + NodeRepr::Index(..) => { + f.debug_struct("Chunk").field("id", &self.id()).field("body", &BodyDebug(*self)).field("pad", &self.pad()).finish() + } + } + } +} + +/// The body of an index [`Node`], printed as [`Body`]'s `Debug` prints it. +struct BodyDebug<'f, 'a>(Node<'f, 'a>); + +/// The children of a [`Node`], printed as a list. +struct ChildrenDebug<'f, 'a>(Node<'f, 'a>); + +impl fmt::Debug for BodyDebug<'_, '_> { + /// The body as `Body`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(e) = self.0.entry() else { return f.write_str("?") }; + match e.kind { + Kind::Data => f.debug_tuple("Data").field(&e.data).finish(), + Kind::Nested => f.debug_tuple("Chunks").field(&ChildrenDebug(self.0)).finish(), + Kind::List => f.debug_struct("List").field("list_type", &e.list_type).field("children", &ChildrenDebug(self.0)).finish(), + Kind::OpaqueList => f.debug_struct("OpaqueList").field("list_type", &e.list_type).field("data", &e.data).finish(), + } + } +} + +impl fmt::Debug for ChildrenDebug<'_, '_> { + /// The children as a list. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.children()).finish() + } +} + +impl<'f, 'a> Node<'f, 'a> { + /// The entry at `at`, `None` past the end. + fn entry(self) -> Option<&'f Entry<'a>> { + match self.0 { + NodeRepr::Index(tree, at) => tree.entry(at), + NodeRepr::Chunk(_) => None, + } + } + + /// The [`Chunk`] when this node is one of an owned tree; `None` for an index entry (see + /// [`Self::to_chunk`]). + pub fn as_chunk(self) -> Option<&'f Chunk<'a>> { + match self.0 { + NodeRepr::Chunk(c) => Some(c), + NodeRepr::Index(..) => None, + } + } + + /// The chunk and its subtree as an owned [`Chunk`] (bodies stay borrowed). + pub fn to_chunk(self) -> Chunk<'a> { + match self.0 { + NodeRepr::Chunk(c) => c.clone(), + NodeRepr::Index(tree, at) => tree.build(at, tree.end_of(at)).into_iter().next().unwrap_or_else(|| Chunk::data(self.id(), &[][..])), + } + } + + /// The 4-byte id (see [`Chunk::id`]). + pub fn id(self) -> [u8; 4] { + match self.0 { + NodeRepr::Chunk(c) => c.id, + NodeRepr::Index(..) => self.entry().map_or([0; 4], |e| e.id), + } + } + + /// The pad byte (see [`Chunk::pad`]). + pub fn pad(self) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.pad, + NodeRepr::Index(..) => self.entry().and_then(|e| e.pad), + } + } + + /// The raw bytes of a data chunk, or the bytes after the list type of an opaque list (see + /// [`Chunk::bytes`]). + pub fn bytes(self) -> Option<&'f [u8]> { + match self.0 { + NodeRepr::Chunk(c) => c.bytes(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::Data | Kind::OpaqueList)).map(|e| e.data), + } + } + + /// The child chunks of a list or nested chunk; none for data. + pub fn children(self) -> Nodes<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => Nodes(NodesRepr::Slice(c.children().iter())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + Nodes(NodesRepr::Index { tree, next: at.saturating_add(1), end }) + } + } + } + + /// The list type when this is a `LIST` (parsed or opaque). + pub fn list_type(self) -> Option<[u8; 4]> { + match self.0 { + NodeRepr::Chunk(c) => c.list_type(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::List | Kind::OpaqueList)).map(|e| e.list_type), + } + } + + /// True when this is a `LIST` of the given type. + pub fn is_list(self, list_type: &[u8; 4]) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_list(list_type), + NodeRepr::Index(..) => { + self.entry().is_some_and(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type) + } + } + } + + /// True when this is a `LIST` whose content could not be read as chunks. + pub fn is_opaque_list(self) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_opaque_list(), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.kind == Kind::OpaqueList), + } + } + + /// True when this chunk has the given id. + pub fn is(self, id: &[u8; 4]) -> bool { + self.id() == *id + } + + /// The first child chunk with this id. + pub fn child(self, id: &[u8; 4]) -> Option { + self.children().find(|c| c.is(id)) + } + + /// The first child `LIST` of this type. + pub fn child_list(self, list_type: &[u8; 4]) -> Option { + self.children().find(|c| c.is_list(list_type)) + } + + /// Every chunk below this one depth-first (children before later siblings), with the depth + /// counted from 1 for the children of this chunk. + pub fn descendants(self) -> NodeWalk<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => NodeWalk(WalkRepr::Chunk(c.descendants())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + NodeWalk(WalkRepr::Index { tree, next: at.saturating_add(1), end, open: Vec::new() }) + } + } + } + + /// The first descendant `LIST` of this type, searching depth-first. + pub fn find_list(self, list_type: &[u8; 4]) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.find_list(list_type).map(Node::from), + // The subtree is a contiguous range in depth-first order: a plain scan. + NodeRepr::Index(tree, at) => { + let first = usize::try_from(at).ok()?.checked_add(1)?; + let end = usize::try_from(tree.end_of(at)).ok()?.min(tree.nodes.len()); + let range = tree.nodes.get(first..end)?; + let i = range.iter().position(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type)?; + let found = u32::try_from(first.checked_add(i)?).ok()?; + Some(Node(NodeRepr::Index(tree, found))) + } + } + } +} + +/// The children of a [`Node`] (or the top-level chunks of a [`TreeIndex`]), in file order. +#[derive(Clone)] +pub struct Nodes<'f, 'a>(NodesRepr<'f, 'a>); + +#[derive(Clone)] +enum NodesRepr<'f, 'a> { + Slice(std::slice::Iter<'f, Chunk<'a>>), + Index { tree: &'f TreeIndex<'a>, next: u32, end: u32 }, +} + +impl<'f, 'a> Iterator for Nodes<'f, 'a> { + type Item = Node<'f, 'a>; + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + NodesRepr::Slice(it) => it.next().map(Node::from), + NodesRepr::Index { tree, next, end } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + *next = tree.end_of(at); + Some(Node(NodeRepr::Index(tree, at))) + } + } + } +} + +/// A depth-first walk over [`Node`]s with their depth; see [`Node::descendants`] and +/// [`TreeIndex::walk`]. +pub struct NodeWalk<'f, 'a>(WalkRepr<'f, 'a>); + +enum WalkRepr<'f, 'a> { + Chunk(Walk<'f, 'a>), + /// `open` holds the subtree ends of the entries above the next one (its ancestors inside the + /// walked range); their count is the next entry's depth minus one. + Index { + tree: &'f TreeIndex<'a>, + next: u32, + end: u32, + open: Vec, + }, +} + +impl<'f, 'a> Iterator for NodeWalk<'f, 'a> { + type Item = (usize, Node<'f, 'a>); + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + WalkRepr::Chunk(w) => w.next().map(|(d, c)| (d, Node::from(c))), + WalkRepr::Index { tree, next, end, open } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + while open.last().is_some_and(|e| *e <= at) { + open.pop(); + } + let depth = open.len().saturating_add(1); + let sub_end = tree.end_of(at); + if sub_end > at.saturating_add(1) { + open.push(sub_end); + } + *next = at.saturating_add(1); + Some((depth, Node(NodeRepr::Index(tree, at)))) + } + } + } +} + +/// The top level of either tree: what the project settings and the version are read from. +#[derive(Clone, Copy)] +pub enum Root<'f, 'a> { + File(&'f AepFile<'a>), + Index(&'f TreeIndex<'a>), +} + +impl<'f, 'a> From<&'f AepFile<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(file: &'f AepFile<'a>) -> Self { + Root::File(file) + } +} + +impl<'f, 'a> From<&'f TreeIndex<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(index: &'f TreeIndex<'a>) -> Self { + Root::Index(index) + } +} + +impl<'f, 'a> Root<'f, 'a> { + /// The top-level chunks. + pub fn chunks(self) -> Nodes<'f, 'a> { + match self { + Root::File(f) => Nodes(NodesRepr::Slice(f.chunks.iter())), + Root::Index(t) => t.chunks(), + } + } + + /// The first top-level chunk with this id. + pub fn chunk(self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{parse_with_limits, write}; + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift64*). + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: u64) -> u64 { + if n == 0 { 0 } else { self.next() % n } + } + } + + /// A random tree: lists, nested-id chunks holding a `Utf8`, data of odd and even sizes, opaque + /// lists, empty lists, deep chains. + fn tree(rng: &mut Rng, depth: usize, budget: &mut usize) -> Vec> { + let n = rng.below(6); + let mut out = Vec::new(); + for _ in 0..n { + if *budget == 0 { + break; + } + *budget -= 1; + let pick = rng.below(10); + let chunk = if depth < 12 && pick < 4 { + let types: [&[u8; 4]; 4] = [b"Fold", b"tdgp", b"tdbs", b"Layr"]; + let t = *types[usize::try_from(rng.below(4)).expect("small")]; + Chunk::list(t, tree(rng, depth + 1, budget)) + } else if pick == 4 { + let ids: [&[u8; 4]; 3] = [b"tdsn", b"pdnm", b"fnam"]; + let id = *ids[usize::try_from(rng.below(3)).expect("small")]; + let text = vec![b'a'; usize::try_from(rng.below(5)).expect("small")]; + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text)]), pad: None } + } else if pick == 5 { + let raw = vec![0x5a; usize::try_from(rng.below(9)).expect("small")]; + Chunk { id: LIST, body: Body::OpaqueList { list_type: *b"btdk", data: raw.into() }, pad: None } + } else { + let ids: [&[u8; 4]; 4] = [b"cdat", b"tdmn", b"Utf8", b"idta"]; + let id = *ids[usize::try_from(rng.below(4)).expect("small")]; + Chunk::data(id, vec![u8::try_from(rng.below(256)).expect("byte"); usize::try_from(rng.below(13)).expect("small")]) + }; + out.push(chunk); + } + out + } + + /// Both readers on the same bytes and limits: the same result, errors included; and the + /// index's walk visits the same chunks at the same depths as the tree's. + fn same(bytes: &[u8], limits: Limits) { + let old = parse_with_limits(bytes, limits); + let new = TreeIndex::parse_with_limits(bytes, limits); + match (&old, &new) { + (Ok(file), Ok(index)) => { + assert_eq!(&index.to_file(), file, "the index converts to the parsed tree"); + let a: Vec<(usize, Chunk<'_>)> = file.walk().map(|(d, c)| (d, c.clone())).collect(); + let b: Vec<(usize, Chunk<'_>)> = index.walk().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(a, b, "the walks agree"); + assert_eq!(index.len(), a.len()); + // Indexing the parsed tree gives the same index. + let again = TreeIndex::from_file(file).expect("index"); + assert_eq!(again.to_file(), *file); + } + (Err(a), Err(b)) => assert_eq!(a, b, "the same error"), + _ => panic!("one reader failed: {:?} vs {:?}", old.as_ref().err(), new.as_ref().err()), + } + } + + /// The default limits and four tight ones (depth 1 and 3, 7 and 40 chunks). + fn limits() -> [Limits; 5] { + [ + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index matches the chunk parser on generated and damaged trees. + #[test] + fn the_index_matches_the_chunk_parser_on_generated_and_damaged_trees() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for _ in 0..400 { + let mut budget = 120; + let chunks = tree(&mut rng, 0, &mut budget); + let file = AepFile { chunks, trailer: Cow::Borrowed(b"x") }; + let bytes = write(&file).expect("write"); + for l in limits() { + same(&bytes, l); + } + // Damage: overwrite a few bytes past the container header (sizes, ids, list types). + for _ in 0..4 { + let mut bad = bytes.clone(); + for _ in 0..1 + rng.below(3) { + let at = 12 + usize::try_from(rng.below(u64::try_from(bad.len().saturating_sub(12)).expect("len"))).expect("index"); + if let Some(b) = bad.get_mut(at) { + *b = u8::try_from(rng.below(256)).expect("byte"); + } + } + for l in limits() { + same(&bad, l); + } + } + // Truncated inputs (the container size then no longer fits). + let cut = bytes.len().saturating_sub(usize::try_from(rng.below(8)).expect("small") + 1); + same(&bytes[..cut], Limits::default()); + } + } + + /// Header errors are the same. + #[test] + fn header_errors_are_the_same() { + for bytes in + [&b""[..], b"RIFX", b"RIFX\0\0\0\x04Egg", b"RIFX\0\0\0\x04Fnord", b"RIFX\0\0\0\x08Egg!", b"RIFX\0\0\0\x03Egg!", b"RIFF\0\0\0\x04Egg!"] + { + same(bytes, Limits::default()); + } + } + + /// The typed accessors give the same answers through either tree. + #[test] + fn nodes_answer_alike_on_both_trees() { + let mut rng = Rng(42); + for _ in 0..100 { + let mut budget = 200; + let file = AepFile { chunks: tree(&mut rng, 0, &mut budget), trailer: Cow::Borrowed(&[]) }; + let bytes = write(&file).expect("write"); + let parsed = crate::parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let a: Vec> = parsed.walk().map(|(_, c)| Node::from(c)).collect(); + let b: Vec> = index.walk().map(|(_, n)| n).collect(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(&b) { + assert_eq!( + (x.id(), x.pad(), x.bytes(), x.list_type(), x.is_opaque_list()), + (y.id(), y.pad(), y.bytes(), y.list_type(), y.is_opaque_list()) + ); + assert_eq!(x.children().map(Node::to_chunk).collect::>(), y.children().map(Node::to_chunk).collect::>()); + for t in [b"Fold", b"tdgp", b"tdbs", b"Layr", b"btdk"] { + assert_eq!(x.find_list(t).map(Node::to_chunk), y.find_list(t).map(Node::to_chunk)); + assert_eq!(x.child_list(t).map(Node::to_chunk), y.child_list(t).map(Node::to_chunk)); + assert_eq!(x.is_list(t), y.is_list(t)); + } + for id in [b"cdat", b"Utf8", b"tdsn"] { + assert_eq!(x.child(id).map(Node::to_chunk), y.child(id).map(Node::to_chunk)); + } + let dx: Vec<(usize, Chunk<'_>)> = x.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + let dy: Vec<(usize, Chunk<'_>)> = y.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(dx, dy); + assert_eq!(format!("{x:?}"), format!("{y:?}")); + } + } + } + + /// The index never reserves more than the input or the limit allows. + #[test] + fn the_index_never_reserves_more_than_the_input_or_the_limit_allows() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("index"); + assert_eq!(index.len(), 1000); + // The start (1024 slots) is cut to what the input can hold. + assert!(index.nodes.capacity() <= 1000, "{} slots for at most 1000 headers", index.nodes.capacity()); + // A short tail or a header that runs past the end does not raise the bound. + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(TreeIndex::parse(&big).is_err() && crate::parse(&big).is_err()); + // Under a tighter limit the parse fails as the chunk parser does, and the guess never + // exceeds the limit. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(TreeIndex::parse_with_limits(&data, tight).err(), Some(AepError::TooManyChunks)); + let small = Limits { max_depth: 64, max_chunks: 1000 }; + let index = TreeIndex::parse_with_limits(&data, small).expect("exactly at the limit"); + assert_eq!(index.len(), 1000); + // A limit below what the input holds bounds the reservation: 100 chunks, never more slots. + let mut wide = b"RIFX\0\0\0\0Egg!".to_vec(); + wide.extend_from_slice(b"LIST"); + wide.extend_from_slice(&u32::try_from(4 + 8 * 99).expect("small").to_be_bytes()); + wide.extend_from_slice(b"Fold"); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 99).flatten()); + let declared = u32::try_from(wide.len() - 8).expect("small"); + wide[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse_with_limits(&wide, Limits { max_depth: 64, max_chunks: 100 }).expect("100 chunks"); + assert_eq!(index.len(), 100); + assert!(index.nodes.capacity() <= 100); + // `from_file` is sized exactly. + let file = crate::parse(&wide).expect("parse"); + assert_eq!(TreeIndex::from_file(&file).expect("index").nodes.capacity(), 100); + } + + /// A file of `n` data chunks of `size` bytes each. + fn flat(n: usize, size: usize) -> Vec { + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + for _ in 0..n { + data.extend_from_slice(b"cdat"); + data.extend_from_slice(&u32::try_from(size).expect("small").to_be_bytes()); + data.extend(std::iter::repeat_n(0u8, size)); + } + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + data + } + + /// The index grows with the chunk count not the byte size. + #[test] + fn the_index_grows_with_the_chunk_count_not_the_byte_size() { + // 4 MB in 4 chunks: a few slots, not one per 32 bytes (that would be 131 072 slots, 4 MB). + let big = flat(4, 1 << 20); + let index = TreeIndex::parse(&big).expect("index"); + assert_eq!(index.len(), 4); + assert!(index.nodes.capacity() <= INITIAL_SLOTS, "{} slots for 4 chunks", index.nodes.capacity()); + // Many small chunks: at most twice the count (it doubles). + let many = flat(10_000, 2); + let index = TreeIndex::parse(&many).expect("index"); + assert_eq!(index.len(), 10_000); + assert!(index.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", index.nodes.capacity()); + same(&big, Limits::default()); + same(&many, Limits::default()); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Failed attempts count once and leave nothing behind. + #[test] + fn failed_attempts_count_once_and_leave_nothing_behind() { + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let headers = 1 + 50 + 1000; + let index = TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }).err(), Some(AepError::TooManyChunks)); + // Only the outermost list is left, opaque: every failed attempt was rolled back. + assert_eq!(index.len(), 1); + let top = index.chunks().next().expect("one top-level chunk"); + assert!(top.is_opaque_list() && top.bytes().map(<[u8]>::len) == Some(level.len() - 12)); + assert_eq!(write(&index.to_file()).expect("write"), data); + same(&data, Limits::default()); + } + + /// Depth beyond the limit stays opaque without recursion past it. + #[test] + fn depth_beyond_the_limit_stays_opaque_without_recursion_past_it() { + // 5000 nested lists: far deeper than the limit; everything below depth 64 stays bytes. + let mut level = b"LIST\0\0\0\x04tdgp".to_vec(); + for _ in 0..5000 { + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(level.len() + 4).expect("small").to_be_bytes()); + out.extend_from_slice(b"tdgp"); + out.extend_from_slice(&level); + level = out; + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("parse"); + assert_eq!(index.len(), 64); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().last().is_some_and(|(_, n)| n.is_opaque_list())); + same(&data, Limits::default()); + // Converting and indexing a deep owned tree uses no recursion either. + let file = index.to_file(); + assert_eq!(TreeIndex::from_file(&file).expect("index").len(), 64); + } +} diff --git a/crates/aep/src/keys.rs b/crates/aep/src/keys.rs index 0c0c93ee6..6e107ad00 100644 --- a/crates/aep/src/keys.rs +++ b/crates/aep/src/keys.rs @@ -193,19 +193,10 @@ impl<'f> Keyframe<'f> { } } - /// The value of a 3-component spatial keyframe (Position), x, y and z in pixels. - pub fn spatial_value(&self) -> Option<[f64; 3]> { - let v = self.value()?; - if !matches!(self.kind(), KeyframeKind::Spatial { components: 3 }) { - return None; - } - Some([*v.first()?, *v.get(1)?, *v.get(2)?]) - } - /// The ease of the key: the four slots at byte 16 of a one-component plain record or at byte 24 /// of a spatial record or a mask path record. For linear and hold keys all four are 0 (a bezier key stores what /// `setTemporalEase` was given). Plain records with several components keep four slots per - /// component, see [`Keyframe::ease_component`] and [`Keyframe::plain_ease_slots`]. + /// component, see [`Keyframe::ease_component`]. pub fn ease(&self) -> Option { match self.kind() { KeyframeKind::Plain { components: 1 } => ease_at(self.record, 16), @@ -214,16 +205,6 @@ impl<'f> Keyframe<'f> { } } - /// The `4 n` ease `f64` of a plain record as stored (from byte `8 + 8 n`). With one component - /// they are [`Keyframe::ease`]; with several they are type-major, see - /// [`Keyframe::ease_component`]. - pub fn plain_ease_slots(&self) -> Option> { - match self.kind() { - KeyframeKind::Plain { components } => doubles(self.record, components.checked_mul(8)?.checked_add(8)?, components.checked_mul(4)?), - _ => None, - } - } - /// The ease of one value component. A plain record with `n` components stores the `4 n` ease /// doubles **type-major**: `[in speed x n][in influence x n][out speed x n][out influence x n]` /// starting at byte `8 + 8 n`, and component `i` uses slot `i` of each block (verified for the @@ -260,16 +241,6 @@ impl<'f> Keyframe<'f> { (0..n).map(|i| self.ease_component(i)).collect() } - /// The Euclidean ARGB distance to the next key that After Effects writes at byte 16 of a colour - /// (4-component spatial) record whose interpolation is not linear: 255 sqrt(2) = 360.6245 for - /// red to blue; 0 on the last key and on linear keys. `None` for other records. - pub fn color_distance(&self) -> Option { - match self.kind() { - KeyframeKind::Spatial { components: 4 } => be_f64(self.record, 16), - _ => None, - } - } - /// The decoded spatial record (Position: 128 bytes, colour: 152 bytes). pub fn spatial(&self) -> Option { let KeyframeKind::Spatial { components: n } = self.kind() else { return None }; @@ -300,11 +271,6 @@ pub struct ExpressionFlags { } impl ExpressionFlags { - /// True when bytes 76, 77 and 120 are all 1. - pub fn all_set(&self) -> bool { - self.byte_76 == 1 && self.byte_77 == 1 && self.byte_120 == 1 - } - /// True when byte 119 is 1. pub fn disabled(&self) -> bool { self.byte_119 == 1 @@ -324,11 +290,6 @@ impl<'f, 'a> Property<'f, 'a> { be_u32(self.chunk.child(b"tdsb")?.bytes()?, 0) } - /// True when `tdb4` byte 5 has bit 0 set: the property is static, it has no keyframes. - pub fn is_static_flag_set(&self) -> Option { - Some(self.header()?.get(5)? & 1 != 0) - } - /// The expression source: the raw UTF-8 text of the `Utf8` child that follows the value in the /// property (no length prefix, no engine tag; the engine is project-wide). pub fn expression(&self) -> Option { diff --git a/crates/aep/src/lib.rs b/crates/aep/src/lib.rs index a47cddfce..3e5dc2ceb 100644 --- a/crates/aep/src/lib.rs +++ b/crates/aep/src/lib.rs @@ -14,6 +14,8 @@ //! list) on damaged data instead of failing. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +// Ranges and borrowed bytes are passed around as plain tuples, as in EffectCraft, which allows this lint. +#![allow(clippy::type_complexity)] mod aepx; mod arbitrary; @@ -24,6 +26,7 @@ mod effects; mod error; mod footage; mod header; +pub mod index; mod keys; mod masks; mod matte; @@ -36,16 +39,17 @@ mod values; mod version; mod write; -pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; +pub use aepx::{AepxDrafts, index_aepx, is_aepx, parse_any, read_aepx, read_aepx_with_limits}; pub use arbitrary::{ CURVE_IDENTITY, CryptomatteData, CurvesData, ExtractorChannels, HueSaturationData, LevelsData, MeshData, curve_lut, identifier_channel_type, }; pub use camera::{CameraPosition, DEFAULT_CAMERA_ZOOM}; pub use chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; -pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue}; +pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue, Pard}; pub use error::AepError; pub use footage::{Footage, PsdLayer, SequenceInfo}; pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use index::{Draft, Node, NodeWalk, Nodes, Root, TreeIndex}; pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; pub use matte::{MatteMode, TrackMatte}; diff --git a/crates/aep/src/masks.rs b/crates/aep/src/masks.rs index 203ccefd5..42b52f5f4 100644 --- a/crates/aep/src/masks.rs +++ b/crates/aep/src/masks.rs @@ -6,9 +6,9 @@ //! property (`tdbs`, with a key list when animated) and `omks` with one `LIST shap` per keyframe (a //! single one when the path is static). -use crate::Chunk; use crate::bytes::{be_f32, be_u32, until_nul}; -use crate::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; +use crate::index::Node; +use crate::typed::{KeyList, Layer, PropGroup, PropNode, Property}; use crate::values::{Tracked, read}; /// The mask mode (`mkif` `u32` at byte 4). @@ -93,8 +93,10 @@ pub struct ShapePath { } impl ShapePath { - /// Decodes a `LIST shap`; `None` when `shph` or the point list is malformed. - pub fn from_chunk(shap: &Chunk<'_>) -> Option { + /// Decodes a `LIST shap` (a [`Chunk`](crate::Chunk) or a [`Node`]); `None` when + /// `shph` or the point list is malformed. + pub fn from_chunk<'f, 'a: 'f>(shap: impl Into>) -> Option { + let shap = shap.into(); let shph = shap.child(b"shph")?.bytes()?; let bbox = [be_f32(shph, 4)?, be_f32(shph, 8)?, be_f32(shph, 12)?, be_f32(shph, 16)?]; let closed_flag = *shph.get(20)?; @@ -187,22 +189,12 @@ fn mask_from_atom(atom: &PropGroup<'_, '_>, mkif: Option<&[u8]>) -> Mask { } } if let Some(omks) = oms.child_list(b"omks") { - mask.shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); + mask.shapes = omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); } } mask } -impl<'f, 'a> Project<'f, 'a> { - /// The vertices of every path shape of a mask in pixels of the layer: mask boxes are normalised - /// to the **layer** size (the size of its source), not the composition's. `None` when the layer - /// has no source with a size. - pub fn mask_vertices_px(&self, layer: &Layer<'f, 'a>, mask: &Mask) -> Option>> { - let (width, height) = self.layer_source_size(layer)?; - Some(mask.shapes.iter().map(|s| s.vertices((f64::from(width), f64::from(height)))).collect()) - } -} - /// One mask with the views of its records, for readers that need more than [`Mask`]: the /// keyframes and expressions of the mask properties (in [`MaskAtom::atom`]: `ADBE Mask Feather`, /// `ADBE Mask Opacity`, `ADBE Mask Offset`) and the path keyframes paired with their shapes. @@ -246,7 +238,7 @@ impl<'f, 'a> Layer<'f, 'a> { if let Some(PropNode::Other(oms)) = atom.get("ADBE Mask Shape") { path = oms.child_list(b"tdbs").and_then(Property::from_chunk); if let Some(omks) = oms.child_list(b"omks") { - path_shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); + path_shapes = omks.children().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); } } masks.push(MaskAtom { mask, atom, path, path_shapes }); diff --git a/crates/aep/src/read.rs b/crates/aep/src/read.rs index c769029e4..a969a3997 100644 --- a/crates/aep/src/read.rs +++ b/crates/aep/src/read.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; +use crate::bytes::{array, be_u32}; use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; use crate::error::AepError; @@ -36,10 +37,20 @@ pub fn parse(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`parse`]; also fails with [`AepError::TooManyChunks`]. pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let end = container_end(data)?; + let mut parser = Parser { data, limits, count: 0 }; + let chunks = parser.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) +} + +/// Checks the `RIFX`/`Egg!` header and returns where the container ends (its chunks fill +/// `12..end`; the trailer follows). Shared with [`crate::index::TreeIndex::parse`]. +pub(super) fn container_end(data: &[u8]) -> Result { if id4(data, 0) != Some(*b"RIFX") { return Err(AepError::NotRifx); } - let declared = u64::from(be32(data, 4).ok_or(AepError::NotRifx)?); + let declared = u64::from(be_u32(data, 4).ok_or(AepError::NotRifx)?); let form = id4(data, 8).ok_or(AepError::NotRifx)?; if &form != b"Egg!" { return Err(AepError::BadFormType(form)); @@ -49,20 +60,12 @@ pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, Aep return Err(AepError::Truncated { declared, available }); } // `declared <= available < data.len()`, so the end is inside the input. - let end = usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge)?; - let mut parser = Parser { data, limits, count: 0 }; - let chunks = parser.sequence(12, end, 1)?; - let trailer = data.get(end..).unwrap_or(&[]); - Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) + usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge) } -fn be32(data: &[u8], at: usize) -> Option { - id4(data, at).map(u32::from_be_bytes) -} - -fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { - let b = data.get(at..at.checked_add(4)?)?; - Some([*b.first()?, *b.get(1)?, *b.get(2)?, *b.get(3)?]) +/// The 4-byte id at `at`, `None` when the input is too short. +pub(super) fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { + array::<4>(data, at) } struct Parser<'a> { @@ -74,13 +77,16 @@ struct Parser<'a> { impl<'a> Parser<'a> { /// Parses the sibling chunks that exactly fill `start..end`. fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result>, AepError> { + // Sized once: a well-formed sequence fills exactly this many slots. `try_reserve` so a + // limit raised past what the target can allocate fails nothing here (the Vec then grows). let mut out = Vec::new(); + let _ = out.try_reserve_exact(self.capacity_for(start, end)); let mut pos = start; while pos < end { let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; - let size = be32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; let avail = end - body_start; if size > avail { return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); @@ -103,6 +109,26 @@ impl<'a> Parser<'a> { Ok(out) } + /// The slots to reserve for the sequence `start..end`: its header count, bounded by the input + /// (one per 8 bytes at most) and by the chunks the limit still allows. + fn capacity_for(&self, start: usize, end: usize) -> usize { + self.count_siblings(start, end).min(self.limits.max_chunks.saturating_sub(self.count)) + } + + /// The number of chunk headers that tile `start..end` the way [`Self::sequence`] walks them, + /// stopping at the first that does not fit. Never more than `(end - start) / 8`. + fn count_siblings(&self, start: usize, end: usize) -> usize { + let mut n = 0usize; + let mut pos = start; + while let Some(body_start) = pos.checked_add(8).filter(|b| *b <= end) { + let Some(size) = be_u32(self.data, pos.saturating_add(4)) else { break }; + let Some(next) = usize::try_from(size).ok().and_then(|s| body_start.checked_add(s)).filter(|e| *e <= end) else { break }; + n = n.saturating_add(1); + pos = if size % 2 == 1 && next < end { next.saturating_add(1) } else { next }; + } + n + } + /// Like [`Self::sequence`] but a structural problem yields `None` (the caller keeps bytes). fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result>>, AepError> { match self.sequence(start, end, depth) { @@ -140,3 +166,113 @@ impl<'a> Parser<'a> { Ok(Body::Data(Cow::Borrowed(self.slice(start, end)))) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every children `Vec` in the tree, depth first. + fn child_vecs<'f, 'a>(chunks: &'f [Chunk<'a>], out: &mut Vec<&'f Vec>>) { + for c in chunks { + match &c.body { + Body::List { children, .. } | Body::Chunks(children) => { + out.push(children); + child_vecs(children, out); + } + Body::Data(_) | Body::OpaqueList { .. } => {} + } + } + } + + /// Every sibling list is allocated once at its exact length. + #[test] + fn every_sibling_list_is_allocated_once_at_its_exact_length() { + let inner: Vec> = (0..37u8).map(|i| Chunk::data(*b"cdat", vec![i; usize::from(i % 5)])).collect(); + let middle: Vec> = (0..9).map(|_| Chunk::list(*b"tdbs", inner.clone())).collect(); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0; 3]), Chunk::list(*b"Fold", middle)], trailer: Cow::Borrowed(&[]) }; + let bytes = crate::write(&file).expect("write"); + let parsed = parse(&bytes).expect("parse"); + assert_eq!(crate::write(&parsed).expect("rewrite"), bytes); + let mut vecs = vec![&parsed.chunks]; + child_vecs(&parsed.chunks, &mut vecs); + assert_eq!(vecs.len(), 11); + // `try_reserve_exact` gives exactly the requested capacity with std's allocator today; a + // Vec grown push by push would have rounded up to a power of two (37 -> 64, 9 -> 16). + for v in vecs { + assert_eq!(v.capacity(), v.len(), "a sibling Vec grew past its length"); + } + } + + /// The preallocation is bounded by the input and the chunk limit. + #[test] + fn the_preallocation_is_bounded_by_the_input_and_the_chunk_limit() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, data.len()), 1000); + assert!(parser.count_siblings(12, data.len()) <= (data.len() - 12) / 8); + // A header whose size runs past the end stops the count; a stray tail is not counted. + assert_eq!(parser.count_siblings(12, data.len() - 3), 999); + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + let parser = Parser { data: &big, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, big.len()), 0); + let f = parse(&data).expect("parse"); + assert_eq!((f.chunks.len(), f.chunks.capacity()), (1000, 1000)); + // Under a tighter limit the parse fails as before, without sizing for more than allowed. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(parse_with_limits(&data, tight), Err(AepError::TooManyChunks)); + // The reservation never exceeds what the chunk limit still allows. + let mut parser = Parser { data: &data, limits: tight, count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 10); + parser.count = 7; + assert_eq!(parser.capacity_for(12, data.len()), 3); + parser.count = 11; + assert_eq!(parser.capacity_for(12, data.len()), 0); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 1000); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Nested lists with broken tails parse each header once. + #[test] + fn nested_lists_with_broken_tails_parse_each_header_once() { + // A wide, well-formed list at the bottom, then 50 levels that each fail and fall back to + // opaque: every level re-parses its child before giving up. + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + // 1 + 50 + 1000 headers in the file: the parse counts each exactly once (the count spans + // every attempt), so the work stays linear in the input. + let headers = 1 + 50 + 1000; + let f = parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }), Err(AepError::TooManyChunks)); + // The outermost list keeps its bytes, and the file writes back unchanged. + let [top] = f.chunks.as_slice() else { panic!("one top-level chunk") }; + assert!(matches!(&top.body, Body::OpaqueList { list_type, data } if list_type == b"Fold" && data.len() + 12 == level.len())); + assert_eq!(crate::write(&f).expect("write"), data); + } +} diff --git a/crates/aep/src/settings.rs b/crates/aep/src/settings.rs index 1193dbc39..440983805 100644 --- a/crates/aep/src/settings.rs +++ b/crates/aep/src/settings.rs @@ -2,8 +2,10 @@ //! linear-light switches, the time display and the frame count start. Cases 620-634: one change //! each against the base project 620; every field below is the only byte (or chunk) that changes. +use crate::bytes::be_u32; use crate::chunk::AepFile; use crate::footage::json_string; +use crate::index::{Node, Root, TreeIndex}; /// The time display style (`nhed` byte 8, feet + frames byte 11, film type byte 13). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -54,7 +56,17 @@ pub struct ProjectSettings { impl ProjectSettings { /// Reads the settings from the top-level chunks. pub fn read(file: &AepFile<'_>) -> ProjectSettings { - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); + Self::of(Root::from(file)) + } + + /// Reads the settings from the top-level chunks of an indexed file. + pub fn read_index(index: &TreeIndex<'_>) -> ProjectSettings { + Self::of(Root::from(index)) + } + + /// Reads the settings from the top-level chunks of either tree. + pub fn of(file: Root<'_, '_>) -> ProjectSettings { + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let byte = |i: usize| nhed.and_then(|b| b.get(i).copied()); let bits_per_channel = byte(15).and_then(|v| match v { 0 => Some(8), @@ -73,9 +85,9 @@ impl ProjectSettings { 1 => Some(1), _ => None, }); - let flag = |id: &[u8; 4]| file.chunk(id).and_then(|c| c.bytes()).and_then(|b| b.first().copied()) == Some(1); - let working_gamma_24 = file.chunk(b"dwga").and_then(|c| c.bytes()).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); - let scene_compensation = file.chunk(b"acer").and_then(|c| c.bytes()).and_then(|b| b.first().copied()).map(|v| v != 0); + let flag = |id: &[u8; 4]| file.chunk(id).and_then(Node::bytes).and_then(|b| b.first().copied()) == Some(1); + let working_gamma_24 = file.chunk(b"dwga").and_then(Node::bytes).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); + let scene_compensation = file.chunk(b"acer").and_then(Node::bytes).and_then(|b| b.first().copied()).map(|v| v != 0); ProjectSettings { bits_per_channel, working_space: working_space(file), @@ -90,9 +102,10 @@ impl ProjectSettings { } /// The working space: the `Utf8` right after `PwCs`. -fn working_space(file: &AepFile<'_>) -> Option { - let at = file.chunks.iter().position(|c| c.is(b"PwCs"))?; - let json = file.chunks.get(at.checked_add(1)?).filter(|c| c.is(b"Utf8"))?.bytes()?; +fn working_space(file: Root<'_, '_>) -> Option { + let mut chunks = file.chunks(); + chunks.find(|c| c.is(b"PwCs"))?; + let json = chunks.next().filter(|c| c.is(b"Utf8"))?.bytes()?; let trimmed: Vec = json.iter().copied().filter(|b| !b.is_ascii_whitespace() && *b != 0).collect(); if trimmed == b"{}" { return Some(WorkingSpace::None); @@ -130,7 +143,7 @@ fn base64_decode(text: &str) -> Option> { /// The profile description (ICC.1 tag `desc`): a v2 `desc` (ASCII) or a v4 `mluc` (its first /// record, UTF-16BE). `None` when the profile or the tag is cut short. fn icc_description(icc: &[u8]) -> Option { - let be32 = |at: usize| icc.get(at..at.checked_add(4)?).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes); + let be32 = |at: usize| be_u32(icc, at); let count = be32(128)?; let tag = (0..count.min(1024)).find_map(|i| { let at = 132usize.checked_add(usize::try_from(i).ok()?.checked_mul(12)?)?; @@ -141,13 +154,13 @@ fn icc_description(icc: &[u8]) -> Option { let body = icc.get(off..off.checked_add(len)?)?; let text = match body.get(..4)? { b"desc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(8..12)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 8)?).ok()?; let ascii = body.get(12..12usize.checked_add(n)?)?; String::from_utf8_lossy(ascii.split(|b| *b == 0).next().unwrap_or_default()).into_owned() } b"mluc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(20..24)?.try_into().ok()?)).ok()?; - let at = usize::try_from(u32::from_be_bytes(body.get(24..28)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 20)?).ok()?; + let at = usize::try_from(be_u32(body, 24)?).ok()?; let units: Vec = body.get(at..at.checked_add(n)?)?.as_chunks::<2>().0.iter().map(|p| u16::from_be_bytes(*p)).collect(); String::from_utf16_lossy(&units) } diff --git a/crates/aep/src/shapes.rs b/crates/aep/src/shapes.rs index 864cfa55b..28b572506 100644 --- a/crates/aep/src/shapes.rs +++ b/crates/aep/src/shapes.rs @@ -10,7 +10,7 @@ //! points are in pixels relative to the layer origin. use crate::masks::ShapePath; -use crate::typed::{Layer, PropGroup, PropNode, Property}; +use crate::typed::{Layer, PropGroup, PropNode}; use crate::values::{Tracked, read, read_opt}; /// The stroke line cap (`ADBE Vector Stroke Line Cap`, stored as a double; butt is the default and @@ -145,7 +145,7 @@ fn number(group: &PropGroup<'_, '_>, name: &str) -> Option> { fn paths_of(group: &PropGroup<'_, '_>) -> Vec { let Some(PropNode::Other(oms)) = group.get("ADBE Vector Shape") else { return Vec::new() }; let Some(omks) = oms.child_list(b"omks") else { return Vec::new() }; - omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() + omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() } fn item(match_name: String, group: &PropGroup<'_, '_>) -> ShapeItem { @@ -269,12 +269,3 @@ impl<'f, 'a> Layer<'f, 'a> { .collect() } } - -impl<'f, 'a> Property<'f, 'a> { - /// The ARGB fractions of a static colour property (4 components stored as doubles 0..255). - pub fn color_argb(&self) -> Option<[f64; 4]> { - let v = self.static_value()?; - let [a, r, g, b] = *v.as_slice().first_chunk::<4>()?; - Some([a / 255.0, r / 255.0, g / 255.0, b / 255.0]) - } -} diff --git a/crates/aep/src/textdoc.rs b/crates/aep/src/textdoc.rs index 373fd40db..1e7569b67 100644 --- a/crates/aep/src/textdoc.rs +++ b/crates/aep/src/textdoc.rs @@ -5,8 +5,11 @@ //! as `(` + BOM `fe ff` + UTF-16BE + `)` with `\` escapes. [`parse_text_dict`] reads it (depth //! limited, never panics) and [`Layer::text_document`] pulls out the verified fields. -use crate::Chunk; +use std::borrow::Cow; +use std::sync::LazyLock; + use crate::bytes::{be_u32, text}; +use crate::index::Node; use crate::typed::{Comp, Layer, Property}; /// How deep dictionaries and arrays may nest before the parser gives up. @@ -73,15 +76,99 @@ impl TextValue { _ => None, } } +} - fn key_text(&self) -> String { +/// The text of a key the way the dictionary stores it: a `/name` as written, any other value in +/// its canonical form (`06` and `+6` are `6`, `1.0` is `1`, a string its decoded text). +fn owned_key(value: &V<'_>) -> String { + match value { + V::Name(s) | V::Token(s) => text(s), + V::Str(utf16) => decode_utf16be(utf16), + V::Int(i) => i.to_string(), + V::Float(f) => f.to_string(), + V::Bool(b) => b.to_string(), + V::Bytes(b) => text(b), + V::Dict(_) | V::Array(_) => String::new(), + } +} + +/// A value of the text dictionary as [`TextDocument::parse`] reads it: it borrows names, bare +/// words and strings from the body, so a document is read without a `String` per token. A key is +/// borrowed when it is a `/name` (every key After Effects writes); its text is the same as the +/// [`TextValue`] key's. [`V::into_owned`] gives the public [`TextValue`]. +#[derive(Debug, Clone)] +enum V<'b> { + Dict(Vec<(Cow<'b, str>, V<'b>)>), + Array(Vec>), + /// A string that started with the byte order mark: the UTF-16BE bytes after it, unescaped. + Str(Cow<'b, [u8]>), + Bytes(Cow<'b, [u8]>), + Name(&'b [u8]), + Int(i64), + Float(f64), + Bool(bool), + Token(&'b [u8]), +} + +impl<'b> V<'b> { + /// The value of a dictionary key (the last one when a key repeats). + fn get(&self, key: &str) -> Option<&V<'b>> { match self { - Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), - Self::Int(i) => i.to_string(), - Self::Float(f) => f.to_string(), - Self::Bool(b) => b.to_string(), - Self::Bytes(b) => text(b), - Self::Dict(_) | Self::Array(_) => String::new(), + Self::Dict(pairs) => pairs.iter().rev().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + fn at(&self, index: usize) -> Option<&V<'b>> { + match self { + Self::Array(items) => items.get(index), + _ => None, + } + } + + /// The decoded text of a string value. + fn string(&self) -> Option { + match self { + Self::Str(utf16) => Some(decode_utf16be(utf16)), + _ => None, + } + } + + fn as_f64(&self) -> Option { + match self { + Self::Int(i) => Some(*i as f64), + Self::Float(f) => Some(*f), + _ => None, + } + } + + fn as_i64(&self) -> Option { + match self { + Self::Int(i) => Some(*i), + _ => None, + } + } + + /// The key a value makes when it is written in a key position. + fn into_key(self) -> Cow<'b, str> { + match self { + Self::Name(s) => String::from_utf8_lossy(s), + other => Cow::Owned(owned_key(&other)), + } + } + + /// The public form. Recursion is bounded: a parsed value nests at most [`MAX_DEPTH`] + 1 deep. + fn into_owned(self) -> TextValue { + match self { + Self::Dict(pairs) => TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.into_owned(), v.into_owned())).collect()), + Self::Array(items) => TextValue::Array(items.into_iter().map(V::into_owned).collect()), + Self::Str(utf16) => TextValue::Str(decode_utf16be(&utf16)), + Self::Bytes(b) => TextValue::Bytes(b.into_owned()), + Self::Name(s) => TextValue::Name(text(s)), + Self::Int(i) => TextValue::Int(i), + Self::Float(f) => TextValue::Float(f), + Self::Bool(b) => TextValue::Bool(b), + Self::Token(s) => TextValue::Token(text(s)), } } } @@ -89,32 +176,44 @@ impl TextValue { struct Parser<'b> { b: &'b [u8], i: usize, + /// The pairs of the dictionaries being read, innermost last: a dictionary's pairs are moved + /// into a vector of their exact size when it closes (one allocation per dictionary instead of + /// one per doubling). + pairs: Vec<(Cow<'b, str>, V<'b>)>, + /// The same for the items of arrays. + items: Vec>, } +/// Whitespace between tokens. fn is_ws(c: u8) -> bool { matches!(c, b' ' | b'\t' | b'\r' | b'\n') } +/// A byte that ends a bare word. fn is_delimiter(c: u8) -> bool { is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') } -impl Parser<'_> { +impl<'b> Parser<'b> { + /// The next byte, `None` at the end. fn peek(&self) -> Option { self.b.get(self.i).copied() } + /// Moves past whitespace. fn skip_ws(&mut self) { while self.peek().is_some_and(is_ws) { self.i = self.i.saturating_add(1); } } + /// True when the unread bytes start with `s`. fn starts_with(&self, s: &[u8]) -> bool { self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) } - fn word(&mut self) -> &[u8] { + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &'b [u8] { let start = self.i; while self.peek().is_some_and(|c| !is_delimiter(c)) { self.i = self.i.saturating_add(1); @@ -122,79 +221,106 @@ impl Parser<'_> { self.b.get(start..self.i).unwrap_or(&[]) } - fn parse(&mut self, depth: usize) -> Option { + /// The body of a `( ... )` string after the `(`, with `\` escapes resolved: borrowed when it + /// has none. `None` when the string or an escape is not closed. + fn string_body(&mut self) -> Option> { + let rest = self.b.get(self.i..)?; + let stop = rest.iter().position(|c| matches!(c, b')' | b'\\'))?; + let head = rest.get(..stop)?; + self.i = self.i.saturating_add(stop); + if self.peek()? == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Borrowed(head)); + } + let mut out = head.to_vec(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Owned(out)); + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option> { self.skip_ws(); if depth > MAX_DEPTH { return None; } if self.starts_with(b"<<") { self.i = self.i.saturating_add(2); - let mut pairs = Vec::new(); + // On an error the pairs read so far stay on the stack: the error ends the whole parse. + let mark = self.pairs.len(); loop { self.skip_ws(); if self.starts_with(b">>") { self.i = self.i.saturating_add(2); - return Some(TextValue::Dict(pairs)); + return Some(V::Dict(self.pairs.drain(mark..).collect())); } let key = self.parse(depth.saturating_add(1))?; let value = self.parse(depth.saturating_add(1))?; - pairs.push((key.key_text(), value)); + self.pairs.push((key.into_key(), value)); } } match self.peek()? { b'[' => { self.i = self.i.saturating_add(1); - let mut items = Vec::new(); + let mark = self.items.len(); loop { self.skip_ws(); if self.peek()? == b']' { self.i = self.i.saturating_add(1); - return Some(TextValue::Array(items)); + return Some(V::Array(self.items.drain(mark..).collect())); } - items.push(self.parse(depth.saturating_add(1))?); + let item = self.parse(depth.saturating_add(1))?; + self.items.push(item); } } b'(' => { self.i = self.i.saturating_add(1); - let mut out = Vec::new(); - loop { - let c = self.peek()?; - if c == b'\\' { - out.push(*self.b.get(self.i.saturating_add(1))?); - self.i = self.i.saturating_add(2); - } else if c == b')' { - self.i = self.i.saturating_add(1); - break; - } else { - out.push(c); - self.i = self.i.saturating_add(1); - } - } - Some(match out.strip_prefix(&[0xfe, 0xff]) { - Some(utf16) => TextValue::Str(decode_utf16be(utf16)), - None => TextValue::Bytes(out), + let body = self.string_body()?; + Some(if body.starts_with(&[0xfe, 0xff]) { + V::Str(match body { + Cow::Borrowed(b) => Cow::Borrowed(b.get(2..).unwrap_or(&[])), + Cow::Owned(mut v) => { + let n = v.len().min(2); + v.drain(..n); + Cow::Owned(v) + } + }) + } else { + V::Bytes(body) }) } b'/' => { self.i = self.i.saturating_add(1); - Some(TextValue::Name(text(self.word()))) + Some(V::Name(self.word())) } _ => { let word = self.word(); if word.is_empty() { return None; } - let w = text(word); - Some(match w.as_str() { - "true" => TextValue::Bool(true), - "false" => TextValue::Bool(false), + // A word that is not UTF-8 reads as text with replacement characters, which is + // neither a boolean nor a number: a token. + let Ok(w) = std::str::from_utf8(word) else { return Some(V::Token(word)) }; + Some(match w { + "true" => V::Bool(true), + "false" => V::Bool(false), _ => match w.parse::() { - Ok(i) => TextValue::Int(i), + Ok(i) => V::Int(i), // `parse` also takes `nan`, `inf` and overflowing exponents; those are not // numbers a text document holds, so they stay tokens and read as missing. Err(_) => match w.parse::() { - Ok(f) if f.is_finite() => TextValue::Float(f), - _ => TextValue::Token(w), + Ok(f) if f.is_finite() => V::Float(f), + _ => V::Token(word), }, }, }) @@ -203,6 +329,7 @@ impl Parser<'_> { } } +/// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). fn decode_utf16be(bytes: &[u8]) -> String { let (pairs, rest) = bytes.as_chunks::<2>(); let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); @@ -213,11 +340,9 @@ fn decode_utf16be(bytes: &[u8]) -> String { s } -/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). -/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was -/// read before is returned. -pub fn parse_text_dict(body: &[u8]) -> TextValue { - let mut parser = Parser { b: body, i: 0 }; +/// The top-level `key value` pairs of a `btdk` body, borrowed (see [`parse_text_dict`]). +fn parse_dict(body: &[u8]) -> V<'_> { + let mut parser = Parser { b: body, i: 0, pairs: Vec::new(), items: Vec::new() }; let mut pairs = Vec::new(); loop { parser.skip_ws(); @@ -226,13 +351,20 @@ pub fn parse_text_dict(body: &[u8]) -> TextValue { } let Some(key) = parser.parse(0) else { break }; let Some(value) = parser.parse(0) else { break }; - pairs.push((key.key_text(), value)); + pairs.push((key.into_key(), value)); } - TextValue::Dict(pairs) + V::Dict(pairs) +} + +/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). +/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was +/// read before is returned. +pub fn parse_text_dict(body: &[u8]) -> TextValue { + parse_dict(body).into_owned() } /// The run style slots (`RUN/6/0[n]/0/0/6/`) that are decoded; every other slot is compared -/// with its default (`style_defaults`) and listed in +/// with its default (`STYLE_DEFAULTS`) and listed in /// [`TextDocument::undecoded_style_fields`] when it differs. pub const DECODED_STYLE_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12", "13", "18", "35", "36", "52", "53", "54", "56", "57", "58", "62", "63", "70"]; @@ -241,25 +373,27 @@ pub const DECODED_STYLE_SLOTS: &[&str] = pub const DECODED_PARAGRAPH_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "8", "9", "21", "29", "33"]; /// A number slot. -fn val(v: f64) -> TextValue { - TextValue::Float(v) +fn val(v: f64) -> V<'static> { + V::Float(v) } -fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { - TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) +/// A dictionary value with these keys, for the built-in defaults. +fn dict(pairs: Vec<(&'static str, V<'static>)>) -> V<'static> { + V::Dict(pairs.into_iter().map(|(k, v)| (Cow::Borrowed(k), v)).collect()) } -fn numbers(v: &[f64]) -> TextValue { - TextValue::Array(v.iter().map(|x| val(*x)).collect()) +/// An array of numbers, for the built-in defaults. +fn numbers(v: &[f64]) -> V<'static> { + V::Array(v.iter().map(|x| val(*x)).collect()) } /// The default of every run style slot that is not decoded and not neutral (zero, `false`, /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. -fn style_defaults() -> TextValue { - let paint = - |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); - let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); +/// Built once. +static STYLE_DEFAULTS: LazyLock> = LazyLock::new(|| { + let paint = |a: f64| dict(vec![("99", V::Name(b"SimplePaint")), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &'static [u8]| dict(vec![("99", V::Name(name))]); dict(vec![ ("37", val(2.0)), ("41", val(2.0)), @@ -269,24 +403,24 @@ fn style_defaults() -> TextValue { ("46", val(7.0)), ("49", val(-1.0)), ("50", val(-1.0)), - ("55", class("SimpleBlender")), + ("55", class(b"SimpleBlender")), ("64", val(4.0)), ("71", val(4.0)), - ("77", TextValue::Bool(true)), - ("78", TextValue::Bool(true)), + ("77", V::Bool(true)), + ("78", V::Bool(true)), ("79", paint(0.0)), ("82", val(3.0)), ("83", val(3.0)), - ("86", class("SimpleCustomFeature")), + ("86", class(b"SimpleCustomFeature")), ("87", val(100.0)), - ("88", TextValue::Bool(true)), + ("88", V::Bool(true)), ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), - ("96", TextValue::Bool(true)), + ("96", V::Bool(true)), ]) -} +}); -/// The same for the paragraph slots (110, 243-245). -fn paragraph_defaults() -> TextValue { +/// The same for the paragraph slots (110, 243-245). Built once. +static PARAGRAPH_DEFAULTS: LazyLock> = LazyLock::new(|| { dict(vec![ ("6", val(1.0)), ("7", val(1.2)), @@ -294,30 +428,35 @@ fn paragraph_defaults() -> TextValue { ("11", val(2.0)), ("12", val(2.0)), ("14", val(36.0)), - ("15", TextValue::Bool(true)), + ("15", V::Bool(true)), ("16", val(0.5)), ("17", numbers(&[0.8, 1.0, 1.33])), ("18", numbers(&[0.0, 0.0, 0.0])), ("19", numbers(&[1.0, 1.0, 1.0])), ("20", val(6.0)), ("22", val(1.0)), - ("23", TextValue::Bool(true)), + ("23", V::Bool(true)), ("24", val(1.0)), - ("26", TextValue::Bool(true)), + ("26", V::Bool(true)), ("31", val(36.0)), ("35", val(1.0)), ("40", val(2.0)), ]) -} +}); -/// Equal values; numbers compare by value (`0` equals `0.0`). -fn same(a: &TextValue, b: &TextValue) -> bool { +/// Equal values; numbers compare by value (`0` equals `0.0`), names and words by their text, +/// strings by their decoded text. +fn same(a: &V<'_>, b: &V<'_>) -> bool { match (a, b) { - (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), - (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Dict(x), V::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (V::Array(x), V::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Str(x), V::Str(y)) => x == y || decode_utf16be(x) == decode_utf16be(y), + (V::Bytes(x), V::Bytes(y)) => x == y, + (V::Name(x), V::Name(y)) | (V::Token(x), V::Token(y)) => x == y || text(x) == text(y), + (V::Bool(x), V::Bool(y)) => x == y, _ => match (a.as_f64(), b.as_f64()) { (Some(p), Some(q)) => p == q, - _ => a == b, + _ => false, }, } } @@ -326,12 +465,12 @@ fn same(a: &TextValue, b: &TextValue) -> bool { /// listed when it holds something else (a neutral value carries nothing that could be lost). /// `/nil` is how an untouched document stores a slot without a value (244: paragraph `/27`, `/28`, /// `/36`). -fn is_neutral(v: &TextValue) -> bool { +fn is_neutral(v: &V<'_>) -> bool { match v { - TextValue::Name(n) => n == "nil", - TextValue::Bool(b) => !b, - TextValue::Array(a) => a.is_empty(), - TextValue::Dict(d) => d.is_empty(), + V::Name(n) => *n == b"nil", + V::Bool(b) => !b, + V::Array(a) => a.is_empty(), + V::Dict(d) => d.is_empty(), other => other.as_f64() == Some(0.0), } } @@ -356,13 +495,13 @@ pub const MAX_UNDECODED_FIELDS: usize = 64; /// Adds to `out` the slots of `actual` that are not decoded and differ from `sampled`, each once /// and at most [`MAX_UNDECODED_FIELDS`] in all. -fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { - let TextValue::Dict(pairs) = actual else { return }; +fn undecoded_fields(actual: &V<'_>, sampled: Option<&V<'_>>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let V::Dict(pairs) = actual else { return }; for (key, value) in pairs { if out.len() >= MAX_UNDECODED_FIELDS { return; } - if decoded.contains(&key.as_str()) || out.contains(key) { + if decoded.contains(&key.as_ref()) || out.iter().any(|o| o == key) { continue; } if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { @@ -373,7 +512,7 @@ fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[ None => !is_neutral(value), }; if differs { - out.push(key.clone()); + out.push(key.to_string()); } } } @@ -433,13 +572,13 @@ pub struct KernRun { /// The manual kerning list of a text run dictionary (`RUN/8/0`): entries `<< /0 << /0 amount >> /1 /// length >>` (554 kerns each of `HHHH` by 50; the closing `\r` has an entry without an amount). -fn kern_runs(run: &TextValue) -> Vec { - let Some(TextValue::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; +fn kern_runs(run: &V<'_>) -> Vec { + let Some(V::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; let runs: Vec = items .iter() .filter_map(|e| { let length = e.get("1").and_then(usize_of)?; - let amount = e.get("0").and_then(|v| v.get("0")).and_then(TextValue::as_f64).filter(|v| v.is_finite()); + let amount = e.get("0").and_then(|v| v.get("0")).and_then(V::as_f64).filter(|v| v.is_finite()); Some(KernRun { length, amount }) }) .collect(); @@ -602,7 +741,7 @@ pub struct StyleRun { impl StyleRun { /// The style of the sampled untouched document (244), used for a damaged first run. - pub(crate) const SAMPLED: StyleRun = StyleRun { + pub const SAMPLED: StyleRun = StyleRun { length: 0, font_size: 100.0, fill_argb: [1.0; 4], @@ -645,24 +784,26 @@ pub struct TextBox { pub height: f64, } -fn colour_of(style: &TextValue, slot: &str) -> Option<[f64; 4]> { +/// The RGBA colour in a style slot (`slot/0/1`), `None` when it is not four numbers. +fn colour_of(style: &V<'_>, slot: &str) -> Option<[f64; 4]> { let c = style.get(slot)?.get("0")?.get("1")?; Some([c.at(0)?.as_f64()?, c.at(1)?.as_f64()?, c.at(2)?.as_f64()?, c.at(3)?.as_f64()?]) } -fn usize_of(v: &TextValue) -> Option { +/// A non-negative integer value as `usize`. +fn usize_of(v: &V<'_>) -> Option { usize::try_from(v.as_i64()?).ok() } /// A finite number slot of a style, `default` when it is missing or not a finite number. -fn num(style: &TextValue, slot: &str, default: f64) -> f64 { - style.get(slot).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(default) +fn num(style: &V<'_>, slot: &str, default: f64) -> f64 { + style.get(slot).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(default) } /// A boolean slot of a style, `default` when it is missing or not a boolean. -fn flag(style: &TextValue, slot: &str, default: bool) -> bool { +fn flag(style: &V<'_>, slot: &str, default: bool) -> bool { match style.get(slot) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, } } @@ -676,9 +817,9 @@ struct RunNotes { } /// The paragraph runs of a text run dictionary (`RUN/5/0`). -fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(paragraph_defaults()); +fn paragraph_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*PARAGRAPH_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -686,7 +827,7 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { continue; }; let para = e.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("5")); - let justification = match para.and_then(|p| p.get("0")).and_then(TextValue::as_i64) { + let justification = match para.and_then(|p| p.get("0")).and_then(V::as_i64) { Some(0) => Justification::Left, Some(1) => Justification::Right, Some(2) => Justification::Center, @@ -701,12 +842,12 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } }; if let Some(p) = para { - undecoded_fields(p, sampled.as_ref(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); + undecoded_fields(p, sampled, DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); } - let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); - let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(TextValue::as_i64).unwrap_or(default); + let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); + let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(V::as_i64).unwrap_or(default); let flag = |slot: &str, default: bool| match para.and_then(|p| p.get(slot)) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, }; out.push(ParagraphRun { @@ -728,9 +869,9 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } /// The style runs of a text run dictionary (`RUN/6/0`). -fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(style_defaults()); +fn style_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*STYLE_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -743,31 +884,31 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { out.push(StyleRun { length, ..prev }); continue; }; - let font_size = style.get("1").and_then(TextValue::as_f64); + let font_size = style.get("1").and_then(V::as_f64); let fill_argb = colour_of(style, "53"); let font_index = style.get("0").and_then(usize_of); if font_size.is_none() || fill_argb.is_none() || font_index.is_none() { notes.incomplete = notes.incomplete.saturating_add(1); } - undecoded_fields(style, sampled.as_ref(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); + undecoded_fields(style, sampled, DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); out.push(StyleRun { length, font_size: font_size.unwrap_or(prev.font_size), fill_argb: fill_argb.unwrap_or(prev.fill_argb), stroke_argb: colour_of(style, "54"), - tracking: style.get("8").and_then(TextValue::as_f64).unwrap_or(0.0), - faux_bold: matches!(style.get("2"), Some(TextValue::Bool(true))), - faux_italic: matches!(style.get("3"), Some(TextValue::Bool(true))), - caps: match style.get("12").and_then(TextValue::as_i64) { + tracking: style.get("8").and_then(V::as_f64).unwrap_or(0.0), + faux_bold: matches!(style.get("2"), Some(V::Bool(true))), + faux_italic: matches!(style.get("3"), Some(V::Bool(true))), + caps: match style.get("12").and_then(V::as_i64) { None | Some(0) => Caps::Normal, Some(1) => Caps::Small, Some(2) => Caps::All, Some(other) => Caps::Other(other), }, - auto_leading: matches!(style.get("4"), Some(TextValue::Bool(true))), - leading: style.get("5").and_then(TextValue::as_f64), + auto_leading: matches!(style.get("4"), Some(V::Bool(true))), + leading: style.get("5").and_then(V::as_f64), font_index: font_index.unwrap_or(prev.font_index), - baseline: match style.get("13").and_then(TextValue::as_i64) { + baseline: match style.get("13").and_then(V::as_i64) { None | Some(0) => FontBaseline::Normal, Some(1) => FontBaseline::Superscript, Some(2) => FontBaseline::Subscript, @@ -776,22 +917,22 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { horizontal_scale: num(style, "6", 1.0), vertical_scale: num(style, "7", 1.0), baseline_shift: num(style, "9", 0.0), - auto_kern: match style.get("11").and_then(TextValue::as_i64) { + auto_kern: match style.get("11").and_then(V::as_i64) { Some(0) => AutoKern::None, None | Some(1) => AutoKern::Metrics, Some(2) => AutoKern::Optical, Some(other) => AutoKern::Other(other), }, ligatures: flag(style, "18", false), - baseline_direction: style.get("35").and_then(TextValue::as_i64).unwrap_or(1), + baseline_direction: style.get("35").and_then(V::as_i64).unwrap_or(1), tsume: num(style, "36", 0.0), no_break: flag(style, "52", false), apply_fill: flag(style, "56", true), apply_stroke: flag(style, "57", false), stroke_over_fill: flag(style, "58", true), - line_join: style.get("62").and_then(TextValue::as_i64).unwrap_or(0), + line_join: style.get("62").and_then(V::as_i64).unwrap_or(0), stroke_width: num(style, "63", 1.0), - digit_set: style.get("70").and_then(TextValue::as_i64).unwrap_or(0), + digit_set: style.get("70").and_then(V::as_i64).unwrap_or(0), }); } out @@ -799,13 +940,13 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { /// The box of paragraph text from the frame dictionary `/0/8/0[0]/0`: key `/1/0` holds 16 x, y pairs /// that trace the rectangle, and `/2/0` is 1; both keys are absent for point text. -fn text_box(top: &TextValue) -> Option { +fn text_box(top: &V<'_>) -> Option { let frame = top.get("0")?.get("8")?.get("0")?.at(0)?.get("0")?; if frame.get("2")?.get("0")?.as_i64()? != 1 { return None; } - let TextValue::Array(coords) = frame.get("1")?.get("0")? else { return None }; - let numbers: Vec = coords.iter().map(TextValue::as_f64).collect::>()?; + let V::Array(coords) = frame.get("1")?.get("0")? else { return None }; + let numbers: Vec = coords.iter().map(V::as_f64).collect::>()?; let (pairs, _) = numbers.as_chunks::<2>(); if pairs.is_empty() { return None; @@ -819,18 +960,16 @@ fn text_box(top: &TextValue) -> Option { impl TextDocument { /// Decodes the body of a `btdk` list. pub fn parse(body: &[u8]) -> Option { - let top = parse_text_dict(body); + let top = parse_dict(body); let run = top.get("1")?.get("1")?.at(0)?.get("0")?; - let raw_text = run.get("0")?.as_str()?.to_string(); + let raw_text = run.get("0")?.string()?; let style = run.get("6")?.get("0")?.at(0)?.get("0")?.get("0")?.get("6")?; let fill_argb = colour_of(style, "53")?; let font_index = usize::try_from(style.get("0")?.as_i64()?).ok()?; let mut fonts = Vec::new(); - if let Some(TextValue::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { - fonts = list - .iter() - .map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(TextValue::as_str).unwrap_or_default().to_string()) - .collect(); + if let Some(V::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { + fonts = + list.iter().map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(V::string).unwrap_or_default()).collect(); } let mut notes = RunNotes::default(); let paragraphs = paragraph_runs(run, &mut notes); @@ -858,7 +997,7 @@ impl TextDocument { .and_then(|v| v.get("0")) .and_then(|v| v.get("2")) .and_then(|v| v.get("1")) - .and_then(TextValue::as_i64), + .and_then(V::as_i64), }) } @@ -882,23 +1021,32 @@ impl TextDocument { } impl<'f, 'a> Layer<'f, 'a> { - /// The text document of a text layer (`LIST btdk` below `ADBE Text Properties`). + /// The Source Text record `LIST btds` (`ADBE Text Document`). It is a direct child of the + /// `ADBE Text Properties` group, which is a direct child of the layer's root `tdgp` + /// (`Layr/tdgp/tdgp/btds` in every sample, and where the exporter writes it), so only that + /// level is searched rather than the whole layer. + fn source_text_record(&self) -> Option> { + let root = self.chunk.child_list(b"tdgp")?; + root.children().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) + } + + /// The text document of a text layer: the first `LIST btdk` in the Source Text record `btds` + /// (the same documents [`Layer::text_document_count`] counts). pub fn text_document(&self) -> Option { - TextDocument::parse(self.chunk.find_list(b"btdk")?.bytes()?) + TextDocument::parse(self.source_text_record()?.find_list(b"btdk")?.bytes()?) } /// The Source Text property record: the `tdbs` in `btds[ADBE Text Document]` (a placeholder /// in every sample: no keys, no expression). Source Text keyframes and expressions were not /// sampled; a `list` (keys) or an expression in it marks them. pub fn source_text_property(&self) -> Option> { - let btds = self.chunk.descendants().map(|(_, c)| c).find(|c| c.is_list(b"btds"))?; - Property::from_chunk(btds.child_list(b"tdbs")?) + Property::from_chunk(self.source_text_record()?.child_list(b"tdbs")?) } - /// The number of text documents (`LIST btdk`) of the layer: 1 in every sample; more would be - /// one per Source Text keyframe (not sampled). + /// The number of text documents (`LIST btdk`) in the Source Text record: 1 in every sample; + /// more would be one per Source Text keyframe (not sampled). pub fn text_document_count(&self) -> usize { - self.chunk.descendants().filter(|(_, c)| c.is_list(b"btdk")).count() + self.source_text_record().map_or(0, |btds| btds.descendants().filter(|(_, c)| c.is_list(b"btdk")).count()) } } @@ -943,10 +1091,24 @@ pub enum CueType { Other(u32), } +impl<'f, 'a> Node<'f, 'a> { + /// The marker list `LIST mrst` of a marker holder: a layer (`LIST Layr`) or a composition's + /// `SecL` list. A layer's `mrst` is a direct child of its root `tdgp` (`ADBE Marker`, where + /// After Effects and the exporter write it), so only that place is looked at; most layers have + /// none, and searching their whole property tree for it was a large part of an import. Any + /// other holder is searched depth-first. + pub fn marker_list(self) -> Option> { + if self.is_list(b"Layr") { + return self.child_list(b"tdgp")?.child_list(b"mrst"); + } + self.find_list(b"mrst") + } +} + /// Reads the markers below `holder` (a layer or the composition's `SecL` list): the `mrst` list has a /// property whose 16-byte records give the times and one `Nmrd` list per marker with the strings. -fn markers_in(holder: &Chunk<'_>) -> Vec { - let Some(mrst) = holder.find_list(b"mrst") else { return Vec::new() }; +fn markers_in(holder: Node<'_, '_>) -> Vec { + let Some(mrst) = holder.marker_list() else { return Vec::new() }; let Some(prop) = mrst.child_list(b"tdbs").and_then(Property::from_chunk) else { return Vec::new() }; let Some(keys) = prop.keys() else { return Vec::new() }; let scale = prop.time_scale().unwrap_or(0); @@ -956,11 +1118,11 @@ fn markers_in(holder: &Chunk<'_>) -> Vec { .filter_map(|(k, nmrd)| { let time_units = k.time_units()?; // The five strings, in the order comment, chapter, URL, frame target, cue point name. - let mut strings = nmrd.children().iter().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); + let mut strings = nmrd.children().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); let mut next = || strings.next().unwrap_or_default(); let (comment, chapter, url, frame_target, cue_point_name) = (next(), next(), next(), next(), next()); let extras: Vec = strings.collect(); - let header = nmrd.child(b"NmHd").and_then(Chunk::bytes); + let header = nmrd.child(b"NmHd").and_then(Node::bytes); let duration_seconds = header.and_then(|h| { let (duration, per_second) = (be_u32(h, 8)?, be_u32(h, 12)?); (per_second != 0).then(|| f64::from(duration) / f64::from(per_second)) @@ -1003,9 +1165,647 @@ impl<'f, 'a> Comp<'f, 'a> { } } +/// The dictionary parser and the defaults as they were before they borrowed from the body (#3276), +/// kept as the reference the differential tests compare with. +#[cfg(test)] +mod oracle { + use super::{MAX_DEPTH, MAX_UNDECODED_FIELDS, TextValue}; + use crate::bytes::text; + + pub(super) fn same(a: &TextValue, b: &TextValue) -> bool { + match (a, b) { + (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + _ => match (a.as_f64(), b.as_f64()) { + (Some(p), Some(q)) => p == q, + _ => a == b, + }, + } + } + + pub(super) fn is_neutral(v: &TextValue) -> bool { + match v { + TextValue::Name(n) => n == "nil", + TextValue::Bool(b) => !b, + TextValue::Array(a) => a.is_empty(), + TextValue::Dict(d) => d.is_empty(), + other => other.as_f64() == Some(0.0), + } + } + + pub(super) fn undecoded_fields( + actual: &TextValue, + sampled: Option<&TextValue>, + decoded: &[&str], + neutral: &[(&str, &[i64])], + out: &mut Vec, + ) { + let TextValue::Dict(pairs) = actual else { return }; + for (key, value) in pairs { + if out.len() >= MAX_UNDECODED_FIELDS { + return; + } + if decoded.contains(&key.as_str()) || out.contains(key) { + continue; + } + if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { + continue; + } + let differs = match sampled.and_then(|s| s.get(key)) { + Some(default) => !same(value, default), + None => !is_neutral(value), + }; + if differs { + out.push(key.clone()); + } + } + } + + impl TextValue { + /// The canonical text of a key, as the owning parser built it. + fn key_text(&self) -> String { + match self { + Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), + Self::Int(i) => i.to_string(), + Self::Float(f) => f.to_string(), + Self::Bool(b) => b.to_string(), + Self::Bytes(b) => text(b), + Self::Dict(_) | Self::Array(_) => String::new(), + } + } + } + + struct Parser<'b> { + b: &'b [u8], + i: usize, + } + + /// Whitespace between tokens. + fn is_ws(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\r' | b'\n') + } + + /// A byte that ends a bare word. + fn is_delimiter(c: u8) -> bool { + is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') + } + + impl Parser<'_> { + /// The next byte, `None` at the end. + fn peek(&self) -> Option { + self.b.get(self.i).copied() + } + + /// Moves past whitespace. + fn skip_ws(&mut self) { + while self.peek().is_some_and(is_ws) { + self.i = self.i.saturating_add(1); + } + } + + /// True when the unread bytes start with `s`. + fn starts_with(&self, s: &[u8]) -> bool { + self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) + } + + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &[u8] { + let start = self.i; + while self.peek().is_some_and(|c| !is_delimiter(c)) { + self.i = self.i.saturating_add(1); + } + self.b.get(start..self.i).unwrap_or(&[]) + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option { + self.skip_ws(); + if depth > MAX_DEPTH { + return None; + } + if self.starts_with(b"<<") { + self.i = self.i.saturating_add(2); + let mut pairs = Vec::new(); + loop { + self.skip_ws(); + if self.starts_with(b">>") { + self.i = self.i.saturating_add(2); + return Some(TextValue::Dict(pairs)); + } + let key = self.parse(depth.saturating_add(1))?; + let value = self.parse(depth.saturating_add(1))?; + pairs.push((key.key_text(), value)); + } + } + match self.peek()? { + b'[' => { + self.i = self.i.saturating_add(1); + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek()? == b']' { + self.i = self.i.saturating_add(1); + return Some(TextValue::Array(items)); + } + items.push(self.parse(depth.saturating_add(1))?); + } + } + b'(' => { + self.i = self.i.saturating_add(1); + let mut out = Vec::new(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + break; + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + Some(match out.strip_prefix(&[0xfe, 0xff]) { + Some(utf16) => TextValue::Str(decode_utf16be(utf16)), + None => TextValue::Bytes(out), + }) + } + b'/' => { + self.i = self.i.saturating_add(1); + Some(TextValue::Name(text(self.word()))) + } + _ => { + let word = self.word(); + if word.is_empty() { + return None; + } + let w = text(word); + Some(match w.as_str() { + "true" => TextValue::Bool(true), + "false" => TextValue::Bool(false), + _ => match w.parse::() { + Ok(i) => TextValue::Int(i), + // `parse` also takes `nan`, `inf` and overflowing exponents; those are not + // numbers a text document holds, so they stay tokens and read as missing. + Err(_) => match w.parse::() { + Ok(f) if f.is_finite() => TextValue::Float(f), + _ => TextValue::Token(w), + }, + }, + }) + } + } + } + } + + /// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). + fn decode_utf16be(bytes: &[u8]) -> String { + let (pairs, rest) = bytes.as_chunks::<2>(); + let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); + let mut s: String = char::decode_utf16(units).map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER)).collect(); + if !rest.is_empty() { + s.push(char::REPLACEMENT_CHARACTER); + } + s + } + + /// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). + /// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was + /// read before is returned. + pub(super) fn parse_text_dict(body: &[u8]) -> TextValue { + let mut parser = Parser { b: body, i: 0 }; + let mut pairs = Vec::new(); + loop { + parser.skip_ws(); + if parser.i >= body.len() { + break; + } + let Some(key) = parser.parse(0) else { break }; + let Some(value) = parser.parse(0) else { break }; + pairs.push((key.key_text(), value)); + } + TextValue::Dict(pairs) + } + + fn val(v: f64) -> TextValue { + TextValue::Float(v) + } + + /// A dictionary value with these keys, for the built-in defaults. + fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { + TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) + } + + /// An array of numbers, for the built-in defaults. + fn numbers(v: &[f64]) -> TextValue { + TextValue::Array(v.iter().map(|x| val(*x)).collect()) + } + + /// The default of every run style slot that is not decoded and not neutral (zero, `false`, + /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot + /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. + pub(super) fn style_defaults() -> TextValue { + let paint = |a: f64| { + dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]) + }; + let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); + dict(vec![ + ("37", val(2.0)), + ("41", val(2.0)), + ("43", dict(vec![("0", val(0.5))])), + ("44", val(2.0)), + ("45", val(2.0)), + ("46", val(7.0)), + ("49", val(-1.0)), + ("50", val(-1.0)), + ("55", class("SimpleBlender")), + ("64", val(4.0)), + ("71", val(4.0)), + ("77", TextValue::Bool(true)), + ("78", TextValue::Bool(true)), + ("79", paint(0.0)), + ("82", val(3.0)), + ("83", val(3.0)), + ("86", class("SimpleCustomFeature")), + ("87", val(100.0)), + ("88", TextValue::Bool(true)), + ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), + ("96", TextValue::Bool(true)), + ]) + } + + /// The same for the paragraph slots (110, 243-245). + pub(super) fn paragraph_defaults() -> TextValue { + dict(vec![ + ("6", val(1.0)), + ("7", val(1.2)), + ("10", val(6.0)), + ("11", val(2.0)), + ("12", val(2.0)), + ("14", val(36.0)), + ("15", TextValue::Bool(true)), + ("16", val(0.5)), + ("17", numbers(&[0.8, 1.0, 1.33])), + ("18", numbers(&[0.0, 0.0, 0.0])), + ("19", numbers(&[1.0, 1.0, 1.0])), + ("20", val(6.0)), + ("22", val(1.0)), + ("23", TextValue::Bool(true)), + ("24", val(1.0)), + ("26", TextValue::Bool(true)), + ("31", val(36.0)), + ("35", val(1.0)), + ("40", val(2.0)), + ]) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::Chunk; + + /// A `LIST` of this type holding `kids`. + fn list(t: &[u8; 4], kids: Vec>) -> Chunk<'static> { + Chunk::list(*t, kids) + } + + /// A document with every kind of value, close to what After Effects writes: names, numbers, + /// a UTF-16 string with escapes, a plain string, nested dictionaries and arrays, an undecoded + /// style slot, a font list and a box frame. + const DOC: &[u8] = b"/0 << /1 << /0 [ << /0 << /0 << /0 (\xfe\xff\0M\0y\0r) >> /1 1 >> >> << /0 << /0 << /0 (\xfe\xff\0A\0\\)\0b) >> >> >> ] >> \ + /8 << /0 [ << /0 << /1 << /0 [ 0 0 10 0 10 5 0 5 ] >> /2 << /0 1 /1 2 >> >> >> ] >> >> \ + /1 << /1 [ << /0 << /0 (\xfe\xff\0H\0i\0\r) /5 << /0 [ << /0 << /0 << /5 << /0 2 /9 false /24 0 /34 7 /17 [ .8 1.0 1.33 ] >> >> >> /1 3 >> ] >> \ + /6 << /0 [ << /0 << /0 << /6 << /0 1 /1 36.5 /53 << /0 << /0 1 /1 [ 1 0.5 0 1 ] >> >> /79 << /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >> /68 1 /90 /x /91 (plain) /92 word >> >> >> /1 3 >> ] >> \ + /8 << /0 [ << /0 << /0 50 >> /1 2 >> << /1 1 >> ] >> >> >> ] >>"; + + /// A deterministic xorshift generator: the tests need varied input, not a dependency. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift). + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + usize::try_from(self.next() % (n as u64)).unwrap_or(0) + } + } + + /// The borrowing parser reads what the owning one read. + #[test] + fn the_borrowing_parser_reads_what_the_owning_one_read() { + // Every key form the owned keys canonicalise (`06`, `+6`, `-0`, `1.0`, `1e2`, strings, + // booleans, containers), names that are not UTF-8, escapes, unclosed strings and escapes, + // non-finite numbers, and bare words. + let parts: &[&[u8]] = &[ + b"<<", + b">>", + b"[", + b"]", + b"/0", + b"/1", + b"/nil", + b"/SimplePaint", + b"/\xff\xfe", + b"/\xc3\xa9", + b"/", + b"06", + b"+6", + b"-0", + b"1.0", + b"1.50", + b"1e2", + b"-2.5", + b".8", + b"inf", + b"nan", + b"1e999", + b"9223372036854775808", + b"true", + b"false", + b"(abc)", + b"(\xfe\xff\0A\0B)", + b"(\xfe\xff\0A\0)", + b"(a\\)b)", + b"(\\", + b"(\xfe\xff\\)\0)", + b"(\\\xfe\xff\0C)", + b"()", + b"(\xfe\xff)", + b"(unclosed", + b"word", + b"\xc3\x28", + b"\xff", + b" ", + b"\n", + b"\t\r", + b"<", + b">", + b")", + ]; + // A value: mostly well formed, with a random part in its place now and then. + /// Appends one generated value to `out`: mostly well formed, now and then a random part. + fn value(rng: &mut Rng, parts: &[&[u8]], depth: usize, out: &mut Vec) { + let pick = |rng: &mut Rng| parts.get(rng.below(parts.len())).copied().unwrap_or_default(); + match rng.below(if depth > 6 { 5 } else { 10 }) { + 0 => out.extend_from_slice(pick(rng)), + 1..=4 => out.extend_from_slice([&b"/0"[..], b"1", b"(\xfe\xff\0z)", b"0.5", b"/nil"].get(rng.below(5)).copied().unwrap_or_default()), + 5..=7 => { + out.extend_from_slice(b"<< "); + for _ in 0..rng.below(5) { + let key = if rng.below(4) == 0 { + pick(rng) + } else { + [&b"/0"[..], b"/1", b"/6", b"/53"].get(rng.below(4)).copied().unwrap_or_default() + }; + out.extend_from_slice(key); + out.push(b' '); + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.extend_from_slice(b">>"); + } + } + _ => { + out.extend_from_slice(b"[ "); + for _ in 0..rng.below(5) { + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.push(b']'); + } + } + } + } + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let (mut read, mut nested) = (0, 0); + for case in 0..40_000 { + let mut body = Vec::new(); + if case % 2 == 0 { + for _ in 0..rng.below(60) { + body.extend_from_slice(parts.get(rng.below(parts.len())).copied().unwrap_or_default()); + if rng.below(3) != 0 { + body.push(b' '); + } + } + } else { + for _ in 0..1 + rng.below(4) { + body.extend_from_slice(b"/0 "); + value(&mut rng, parts, 0, &mut body); + body.push(b'\n'); + } + } + let parsed = parse_text_dict(&body); + assert_eq!(parsed, oracle::parse_text_dict(&body), "case {case}: {}", String::from_utf8_lossy(&body)); + let _ = TextDocument::parse(&body); + if let TextValue::Dict(pairs) = &parsed { + read += usize::from(!pairs.is_empty()); + nested += usize::from(pairs.iter().any(|(_, v)| matches!(v, TextValue::Dict(d) if !d.is_empty()))); + } + } + // The input is not all rejected at the first token. + assert!(read > 25_000 && nested > 3_000, "{read} cases read something, {nested} a dictionary"); + // Every cut of a realistic document, and nesting past the limit. + for end in 0..=DOC.len() { + let body = DOC.get(..end).unwrap_or_default(); + assert_eq!(parse_text_dict(body), oracle::parse_text_dict(body), "cut at {end}"); + let _ = TextDocument::parse(body); + } + for depth in [63, 64, 65, 66, 200] { + let body = [b"/k ".to_vec(), b"<< /a ".repeat(depth), b"1".to_vec(), b" >>".repeat(depth), b" /z 2".to_vec()].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "depth {depth}"); + let body = [b"/k ".to_vec(), b"[ ".repeat(depth), b"]".repeat(depth)].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "array depth {depth}"); + } + // Dictionaries and arrays mixed (`<< /a [ << /a [ ...`), closed, and left open exactly + // at the limit. + for depth in 60..=68 { + let open: Vec = (0..depth).flat_map(|i| if i % 2 == 0 { &b"<< /a "[..] } else { &b"[ "[..] }).copied().collect(); + let close: Vec = (0..depth).rev().flat_map(|i| if i % 2 == 0 { &b" >>"[..] } else { &b" ]"[..] }).copied().collect(); + for body in + [[b"/k ".to_vec(), open.clone(), b"1".to_vec(), close, b" /z 2".to_vec()].concat(), [b"/k ".to_vec(), open, b"1".to_vec()].concat()] + { + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "mixed depth {depth}"); + let _ = TextDocument::parse(&body); + } + } + } + + /// A realistic document is read in full. + #[test] + fn a_realistic_document_is_read_in_full() { + let doc = TextDocument::parse(DOC).expect("a document"); + assert_eq!((doc.text.as_str(), doc.font_size, doc.fill_argb, doc.font.as_deref()), ("Hi", 36.5, [1.0, 0.5, 0.0, 1.0], Some("A)b"))); + assert_eq!(doc.fonts, ["Myr", "A)b"]); + assert_eq!(doc.paragraphs.first().map(|p| (p.length, p.justification, p.auto_hyphenate)), Some((3, Justification::Center, false))); + // `/68 1` and `/34 7` are neutral; `/90 /x`, `/91 (plain)` and `/92 word` differ from the + // (missing, neutral) default; `/79` equals its default. + assert_eq!(doc.undecoded_style_fields, ["90", "91", "92"]); + assert!(doc.undecoded_paragraph_fields.is_empty()); + assert_eq!(doc.kerning, [KernRun { length: 2, amount: Some(50.0) }, KernRun { length: 1, amount: None }]); + assert_eq!(doc.text_box, Some(TextBox { x: 0.0, y: 0.0, width: 10.0, height: 5.0 })); + assert_eq!(doc.line_orientation, Some(2)); + } + + /// The defaults are the sampled ones and built once. + #[test] + fn the_defaults_are_the_sampled_ones_and_built_once() { + assert_eq!(STYLE_DEFAULTS.clone().into_owned(), oracle::style_defaults()); + assert_eq!(PARAGRAPH_DEFAULTS.clone().into_owned(), oracle::paragraph_defaults()); + assert!(std::ptr::eq(&*STYLE_DEFAULTS, &*STYLE_DEFAULTS)); + } + + /// Names keys and strings are borrowed from the body. + #[test] + fn names_keys_and_strings_are_borrowed_from_the_body() { + let V::Dict(pairs) = parse_dict(b"/a /b /c (\xfe\xff\0x) /d (plain) /e word /f 1.5 /g (\\(esc) 07 1") else { panic!("a dict") }; + // Every `/name` key, name, string without escapes and bare word points into the body. + assert!(pairs.iter().take(6).all(|(k, _)| matches!(k, Cow::Borrowed(_)))); + assert!(matches!(pairs.first().map(|p| &p.1), Some(V::Name(b"b")))); + assert!(matches!(pairs.get(1).map(|p| &p.1), Some(V::Str(Cow::Borrowed(b"\0x"))))); + assert!(matches!(pairs.get(2).map(|p| &p.1), Some(V::Bytes(Cow::Borrowed(b"plain"))))); + assert!(matches!(pairs.get(3).map(|p| &p.1), Some(V::Token(b"word")))); + // An escaped string is copied; a number key is written in its canonical form. + assert!(matches!(pairs.get(5).map(|p| &p.1), Some(V::Bytes(Cow::Owned(b))) if b == b"(esc")); + assert_eq!(pairs.get(6).map(|p| p.0.as_ref()), Some("7")); + // Containers hold what was read. + let V::Dict(outer) = parse_dict(b"/0 << /a 1 /b 2 /c 3 /d 4 /e 5 >> /1 [ 1 2 3 4 5 6 7 8 9 ]") else { panic!("a dict") }; + assert!(matches!(outer.first().map(|p| &p.1), Some(V::Dict(d)) if d.len() == 5 && d.capacity() >= d.len())); + assert!(matches!(outer.get(1).map(|p| &p.1), Some(V::Array(a)) if a.len() == 9 && a.capacity() >= a.len())); + } + + /// Slot checks agree with the owned ones. + #[test] + fn slot_checks_agree_with_the_owned_ones() { + // Slots: defaults, decoded, neutral-listed and unknown ones. Values: each default as + // stored (also as `Int` where it is `2.0`), near misses, neutral values and every kind. + let slots = ["0", "6", "7", "17", "24", "34", "35", "37", "43", "55", "68", "77", "79", "86", "95", "96", "200", "x"]; + let values: &[&[u8]] = &[ + b"2", + b"2.0", + b"7", + b"1", + b"0", + b"-1", + b"0.5", + b"1.2", + b"36", + b"true", + b"false", + b"/nil", + b"/x", + b"/SimpleBlender", + b"[ ]", + b"<< >>", + b"[ .8 1 1.33 ]", + b"[ 0.8 1.0 ]", + b"[ 1 1 1 ]", + b"<< /0 .5 >>", + b"<< /0 0 /1 0 >>", + b"<< /1 0 /0 0 >>", + b"<< /99 /SimpleBlender >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 1 ] >> >>", + b"(plain)", + b"(\xfe\xff\0a)", + b"word", + b"/\xff", + ]; + let mut rng = Rng(0x2545_f491_4f6c_dd1d); + let mut listed = 0; + for case in 0..20_000 { + let mut body = b"/s << ".to_vec(); + for _ in 0..rng.below(12) { + body.push(b'/'); + body.extend_from_slice(slots.get(rng.below(slots.len())).copied().unwrap_or_default().as_bytes()); + body.push(b' '); + body.extend_from_slice(values.get(rng.below(values.len())).copied().unwrap_or_default()); + body.push(b' '); + } + body.extend_from_slice(b">>"); + let (new, old) = (parse_dict(&body), oracle::parse_text_dict(&body)); + let (Some(new), Some(old)) = (new.get("s"), old.get("s")) else { panic!("case {case}: a dict") }; + for (sampled, old_sampled, decoded, neutral) in [ + (&*STYLE_DEFAULTS, oracle::style_defaults(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES), + (&*PARAGRAPH_DEFAULTS, oracle::paragraph_defaults(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES), + ] { + let (mut a, mut b) = (Vec::new(), Vec::new()); + undecoded_fields(new, Some(sampled), decoded, neutral, &mut a); + oracle::undecoded_fields(old, Some(&old_sampled), decoded, neutral, &mut b); + assert_eq!(a, b, "case {case}: {}", String::from_utf8_lossy(&body)); + listed += a.len(); + } + // Every pair of values compares and reads as neutral alike. + if let (V::Dict(x), TextValue::Dict(y)) = (new, old) { + for ((_, p), (_, q)) in x.iter().zip(y) { + assert_eq!(is_neutral(p), oracle::is_neutral(q), "case {case}"); + for ((_, r), (_, t)) in x.iter().zip(y) { + assert_eq!(same(p, r), oracle::same(q, t), "case {case}"); + } + } + } + } + assert!(listed > 10_000, "{listed} slots listed"); + } + + /// Values compare by their text. + #[test] + fn values_compare_by_their_text() { + // Two names that are not UTF-8 read as the same text with replacement characters, as the + // owned values did. + assert!(same(&V::Name(b"\xff"), &V::Name(b"\xfe"))); + assert!(!same(&V::Name(b"a"), &V::Name(b"b"))); + assert!(same(&V::Str(Cow::Borrowed(b"\xd8\x00")), &V::Str(Cow::Owned(b"\xd8\x01".to_vec())))); + assert!(same(&V::Int(0), &V::Float(0.0)) && !same(&V::Int(0), &V::Bool(false)) && !same(&V::Name(b"x"), &V::Token(b"x"))); + assert!( + is_neutral(&V::Name(b"nil")) && !is_neutral(&V::Token(b"nil")) && is_neutral(&V::Float(0.0)) && !is_neutral(&V::Str(Cow::Borrowed(b""))) + ); + } + + /// A layers marker list is the one in its root group. + #[test] + fn a_layers_marker_list_is_the_one_in_its_root_group() { + let marked = |tag: u8| list(b"mrst", vec![Chunk::data(*b"test", vec![tag])]); + // The stored place: `Layr/tdgp/mrst`. + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), marked(1)])]); + assert_eq!(Node::from(&layer).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[1u8][..])); + // A list deeper in the layer is not the layer's marker list (the old depth-first search + // found it); a layer without a marker list in its root group has no markers. + let deep = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(2)])])]); + assert!(Node::from(&deep).marker_list().is_none()); + assert!(Layer { chunk: Node::from(&deep) }.markers().is_empty()); + // Any other holder (a composition's `SecL`) is still searched depth-first. + let secl = list(b"SecL", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(3)])])]); + assert_eq!(Node::from(&secl).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[3u8][..])); + } + + /// The source text record is found in the text group. + #[test] + fn the_source_text_record_is_found_in_the_text_group() { + let doc = b"/1 << /1 [ << /0 << /0 (\xfe\xff\0H\0\r) /6 << /0 [ << /0 << /0 << /6 << /0 0 /1 50 /53 << /0 << /1 [ 1 1 0 0 ] >> >> >> >> >> /1 2 >> ] >> >> >> ] >>"; + // `btdk` is an opaque list (its body is not a chunk sequence), as the reader makes it. + let btdk = || Chunk { id: *b"LIST", body: crate::Body::OpaqueList { list_type: *b"btdk", data: doc.to_vec().into() }, pad: None }; + let btds_real = |n: usize| list(b"btds", std::iter::once(list(b"tdbs", vec![])).chain((0..n).map(|_| btdk())).collect()); + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), list(b"tdgp", vec![btds_real(2)])])]); + let view = Layer { chunk: Node::from(&layer) }; + assert_eq!(view.text_document().map(|d| d.text), Some("H".to_string())); + assert!(view.source_text_property().is_some()); + assert_eq!(view.text_document_count(), 2); + // A record anywhere else in the layer is not its Source Text. + let elsewhere = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![list(b"tdgp", vec![btds_real(1)])])])]); + let view = Layer { chunk: Node::from(&elsewhere) }; + assert!(view.text_document().is_none() && view.source_text_property().is_none()); + assert_eq!(view.text_document_count(), 0); + } #[test] fn non_finite_numbers_are_not_numbers() { @@ -1018,13 +1818,13 @@ mod tests { fn undecoded_fields_are_capped_and_linear() { // 300 000 distinct undecoded slots that differ from a sampled default: before, each was // checked against every slot recorded so far (4.5e10 compares) and all were listed. - let actual = TextValue::Dict((0..300_000).map(|i| (format!("{}", 9000 + i), TextValue::Int(1))).collect()); + let actual = V::Dict((0..300_000).map(|i| (Cow::Owned(format!("{}", 9000 + i)), V::Int(1))).collect()); let mut out = Vec::new(); undecoded_fields(&actual, None, &["0"], &[], &mut out); assert_eq!(out.len(), MAX_UNDECODED_FIELDS); assert_eq!(out.first().map(String::as_str), Some("9000")); // A slot listed once is not listed again. - let again = TextValue::Dict(vec![("9000".into(), TextValue::Int(1))]); + let again = V::Dict(vec![(Cow::Borrowed("9000"), V::Int(1))]); let mut short = vec!["9000".to_string()]; undecoded_fields(&again, None, &[], &[], &mut short); assert_eq!(short, ["9000"]); diff --git a/crates/aep/src/typed.rs b/crates/aep/src/typed.rs index 16fcda33c..9334b45cc 100644 --- a/crates/aep/src/typed.rs +++ b/crates/aep/src/typed.rs @@ -4,42 +4,57 @@ //! //! Only facts that were established from our own one-change-per-file corpus and are checked by //! tests against it are decoded (see `docs/aep-format.md` for the evidence). Everything else -//! stays reachable through the raw [`Chunk`] tree. Views borrow the file and never fail loudly: -//! a chunk that does not have the expected shape yields `None` or is skipped. +//! stays reachable through the raw chunk tree ([`Node`]). Views borrow the file and never fail +//! loudly: a chunk that does not have the expected shape yields `None` or is skipped. +//! +//! A view reads either tree: the owned [`Chunk`](crate::Chunk) tree +//! ([`Project::new`]) or the flat [`TreeIndex`] the importer builds ([`Project::from_index`]); +//! both give the same answers. use crate::bytes::{be_f64, be_u16, be_u32, text, until_nul}; -use crate::chunk::{AepFile, Chunk}; +use crate::chunk::AepFile; +use crate::index::{Node, Root, TreeIndex}; /// The name a user gave a property or group: the `Utf8` inside its `tdsn` chunk. `None` for the /// sentinel `-_0_/-` ("no custom name") and for empty names. -pub(crate) fn user_name(chunk: &Chunk<'_>) -> Option { - let name = chunk.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; +pub(crate) fn user_name(chunk: Node<'_, '_>) -> Option { + let name = chunk.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (!name.is_empty() && name != "-_0_/-").then_some(name) } /// The content of a `Utf8` child as text (not trimmed). -pub(crate) fn utf8_child(chunk: &Chunk<'_>) -> Option { - chunk.child(b"Utf8").and_then(Chunk::bytes).map(text) +pub(crate) fn utf8_child(chunk: Node<'_, '_>) -> Option { + chunk.child(b"Utf8").and_then(Node::bytes).map(text) } /// The project: the top-level `LIST Fold`, which holds the project items. #[derive(Debug, Clone, Copy)] pub struct Project<'f, 'a> { - pub(crate) fold: &'f Chunk<'a>, + pub(crate) fold: Node<'f, 'a>, /// The top-level `LIST EfdG` (project-level effect definitions), when there is one. - pub(crate) effect_defs: Option<&'f Chunk<'a>>, + pub(crate) effect_defs: Option>, } impl<'f, 'a> Project<'f, 'a> { /// The project view of a file, or `None` when it has no `LIST Fold`. pub fn new(file: &'f AepFile<'a>) -> Option { - file.list(b"Fold").map(|fold| Self { fold, effect_defs: file.list(b"EfdG") }) + Self::of(Root::from(file)) + } + + /// The project view of an indexed file ([`TreeIndex`]), or `None` when it has no `LIST Fold`. + pub fn from_index(index: &'f TreeIndex<'a>) -> Option { + Self::of(Root::from(index)) + } + + /// The project view of either tree: its `LIST Fold` and `LIST EfdG`. + pub fn of(root: Root<'f, 'a>) -> Option { + root.list(b"Fold").map(|fold| Self { fold, effect_defs: root.list(b"EfdG") }) } /// The top-level items (the project panel's root; folders hold their children, see /// [`Item::children`]). pub fn items(&self) -> impl Iterator> + 'f { - self.fold.children().iter().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.fold.children().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// True when the `LIST Fold` could not be read as chunks (damaged): its items cannot be seen. @@ -78,12 +93,12 @@ pub enum ItemKind { /// A project item (`LIST Item`). #[derive(Debug, Clone, Copy)] pub struct Item<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Item<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -106,7 +121,7 @@ impl<'f, 'a> Item<'f, 'a> { /// The items inside a folder (`LIST Sfdr`). pub fn children(&self) -> impl Iterator> + 'f { - self.chunk.child_list(b"Sfdr").into_iter().flat_map(|sfdr| sfdr.children().iter()).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.chunk.child_list(b"Sfdr").into_iter().flat_map(Node::children).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// Whether the item has the `LIST Sfdr` of a folder's contents (After Effects writes an empty @@ -117,7 +132,7 @@ impl<'f, 'a> Item<'f, 'a> { /// True when the folder's `LIST Sfdr` is there but could not be read as chunks (damaged). pub fn folder_contents_unreadable(&self) -> bool { - self.chunk.child_list(b"Sfdr").is_some_and(Chunk::is_opaque_list) + self.chunk.child_list(b"Sfdr").is_some_and(Node::is_opaque_list) } /// The composition view when this item has composition data (`cdta`). @@ -135,6 +150,10 @@ pub struct TimeBase { } impl TimeBase { + /// The NTSC rates as `(numerator, denominator)` and the value After Effects stores for each (a + /// float32 of exactly these decimals): the one table an importer and an exporter use. + pub const NTSC: [((u32, u32), f32); 4] = [((24000, 1001), 23.976), ((30000, 1001), 29.97), ((60000, 1001), 59.94), ((120000, 1001), 119.88)]; + /// Frames per second (`timescale / frame_duration`), `None` for a zero duration. pub fn frame_rate(&self) -> Option { (self.frame_duration != 0).then(|| f64::from(self.timescale) / f64::from(self.frame_duration)) @@ -149,7 +168,7 @@ impl TimeBase { /// A composition item (an [`Item`] with `cdta`). #[derive(Debug, Clone, Copy)] pub struct Comp<'f, 'a> { - pub(crate) item: &'f Chunk<'a>, + pub(crate) item: Node<'f, 'a>, } impl<'f, 'a> Comp<'f, 'a> { @@ -163,19 +182,19 @@ impl<'f, 'a> Comp<'f, 'a> { /// The layers, top layer first (`LIST Layr`; the `DLay`/`SLay`/`CLay`/`SecL` lists next to /// them are not layers). pub fn layers(&self) -> impl Iterator> + 'f { - self.item.children().iter().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) + self.item.children().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) } } /// A layer (`LIST Layr`). #[derive(Debug, Clone, Copy)] pub struct Layer<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Layer<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -205,12 +224,12 @@ pub enum PropNode<'f, 'a> { /// A property (`LIST tdbs`). Property(Property<'f, 'a>), /// Any other list that follows a match name (observed: `otst`, the effect instance `sspc`). - Other(&'f Chunk<'a>), + Other(Node<'f, 'a>), } impl<'f, 'a> PropNode<'f, 'a> { /// The node a `LIST` after a match name is. - fn of(c: &'f Chunk<'a>) -> Self { + fn of(c: Node<'f, 'a>) -> Self { if c.is_list(b"tdgp") { PropNode::Group(PropGroup { chunk: c }) } else if c.is_list(b"tdbs") { @@ -233,17 +252,18 @@ pub struct PropEntry<'f, 'a> { /// `ADBE Group End` closes the group and names nothing. #[derive(Debug, Clone, Copy)] pub struct PropGroup<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> PropGroup<'f, 'a> { - /// A group view of a `LIST tdgp` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A group view of a `LIST tdgp` chunk (a [`Chunk`](crate::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdgp").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -264,7 +284,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes().map(match_name); - } else if c.id == crate::chunk::LIST + } else if c.is(&crate::chunk::LIST) && let Some(name) = pending.take() { out.push(PropEntry { match_name: name, node: PropNode::of(c) }); @@ -285,9 +305,9 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes(); - } else if c.id == crate::chunk::LIST + } else if c.is(&crate::chunk::LIST) && let Some(body) = pending.take() - && match_name(body) == name + && match_name_is(body, name) { return Some(PropNode::of(c)); } @@ -314,21 +334,30 @@ fn match_name(body: &[u8]) -> String { text(until_nul(body)) } +/// `match_name(body) == name` without building the `String`: equal bytes match; otherwise only +/// a body that is not valid UTF-8 can still match (through its U+FFFD replacements), and only a +/// name that holds U+FFFD. +fn match_name_is(body: &[u8], name: &str) -> bool { + let bytes = until_nul(body); + bytes == name.as_bytes() || (name.contains('\u{FFFD}') && String::from_utf8_lossy(bytes) == name) +} + /// A property (`LIST tdbs`): `tdsb`, `tdsn`, the 124-byte header `tdb4`, then either the static /// value `cdat` or a keyframe `LIST list`. #[derive(Debug, Clone, Copy)] pub struct Property<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Property<'f, 'a> { - /// A property view of a `LIST tdbs` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A property view of a `LIST tdbs` chunk (a [`Chunk`](crate::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdbs").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -436,12 +465,13 @@ pub struct KeyLayout { impl<'f> KeyList<'f> { /// A view of a `LIST list` chunk. `None` when `lhd3` is not 52 bytes or `ldat` is not exactly /// `count * record_size` bytes (a list without `ldat` must have no records). - pub fn from_chunk<'a>(chunk: &'f Chunk<'a>) -> Option { + pub fn from_chunk<'a: 'f>(chunk: impl Into>) -> Option { + let chunk = chunk.into(); if !chunk.is_list(b"list") { return None; } let header = ListHeader::parse(chunk.child(b"lhd3")?.bytes()?)?; - let ldat: &[u8] = chunk.child(b"ldat").and_then(Chunk::bytes).unwrap_or(&[]); + let ldat: &[u8] = chunk.child(b"ldat").and_then(Node::bytes).unwrap_or(&[]); let expected = usize::try_from(header.count).ok()?.checked_mul(usize::try_from(header.record_size).ok()?)?; (ldat.len() == expected).then_some(Self { header, ldat, layout: None }) } @@ -500,6 +530,7 @@ impl<'f> Keyframe<'f> { #[cfg(test)] mod tests { use super::*; + use crate::Chunk; fn key_list(count: u32, record_size: u32, ldat: &[u8]) -> Chunk<'static> { let mut lhd3 = vec![0u8; 52]; @@ -543,4 +574,31 @@ mod tests { assert!(g.get("C").is_none()); assert_eq!(g.names(), ["A", "B", "A"]); } + + /// Match name is agrees with comparing the decoded name. + #[test] + fn match_name_is_agrees_with_comparing_the_decoded_name() { + let bodies: [&[u8]; 9] = [ + b"ADBE Position\0\0\0", + b"ADBE Position", + b"ADBE Pos", + b"", + b"\0ADBE", + b"ADBE \xff Bad\0", + b"\xff", + b"\xef\xbf\xbd", + "Gr\u{fc}n\0".as_bytes(), + ]; + let names = ["ADBE Position", "ADBE Pos", "", "ADBE \u{FFFD} Bad", "\u{FFFD}", "Gr\u{fc}n", "ADBE Position\0", "ADBE \u{FFFD}"]; + for body in bodies { + for name in names { + assert_eq!(match_name_is(body, name), match_name(body) == name, "{body:?} vs {name:?}"); + } + } + // Names read back from a group find their entries, a damaged one included. + let group = Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdmn", b"X\xffY\0".to_vec()), Chunk::list(*b"tdbs", Vec::new())]); + let g = PropGroup::from_chunk(&group).expect("group"); + let names = g.names(); + assert!(g.get(&names[0]).is_some()); + } } diff --git a/crates/aep/src/values.rs b/crates/aep/src/values.rs index 5613e07a3..b6857165a 100644 --- a/crates/aep/src/values.rs +++ b/crates/aep/src/values.rs @@ -8,6 +8,7 @@ //! 100 %. These defaults are verified for AV layers (solids); cameras and lights use other //! conventions (see the `camera` module). +use crate::bytes::be_f64; use crate::typed::{Layer, Project, PropGroup, PropNode}; /// Where a [`Tracked`] value came from. @@ -192,8 +193,7 @@ impl<'f, 'a> Layer<'f, 'a> { let otda = otst.child_list(b"otky")?.child(b"otda")?.bytes()?; let mut degrees = [0.0; 3]; for (i, slot) in degrees.iter_mut().enumerate() { - let b = otda.get(i * 8..i * 8 + 8)?; - *slot = f64::from_be_bytes(b.try_into().ok()?); + *slot = be_f64(otda, i * 8)?; } Some(Orientation { degrees, @@ -223,12 +223,10 @@ impl<'f, 'a> Layer<'f, 'a> { .child_list(b"otky") .map(|k| { k.children() - .iter() .filter(|c| c.is(b"otda")) .map(|c| { let b = c.bytes()?; - let at = |i: usize| b.get(i * 8..i * 8 + 8).and_then(|s| s.try_into().ok()).map(f64::from_be_bytes); - Some([at(0)?, at(1)?, at(2)?]) + Some([be_f64(b, 0)?, be_f64(b, 8)?, be_f64(b, 16)?]) }) .collect() }) diff --git a/crates/aep/src/version.rs b/crates/aep/src/version.rs index 53de0834a..023793037 100644 --- a/crates/aep/src/version.rs +++ b/crates/aep/src/version.rs @@ -8,8 +8,9 @@ use std::fmt; -use crate::bytes::{be_u16, be_u32}; +use crate::bytes::be_u32; use crate::chunk::AepFile; +use crate::index::{Root, TreeIndex}; /// An After Effects version as packed in `head` / `svap`: `26.2.1x2` is major 26, minor 2, bug-fix /// 1, build 2 (the form of the scripting API's `app.version`). @@ -71,21 +72,38 @@ impl AepFile<'_> { /// version there, and the older version for File ▸ Save a Copy As 25.x / 24.x. `None` when the /// chunk is missing, short or holds no version. pub fn file_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + Root::from(self).file_version() } /// The version of the After Effects that wrote the file: `svap` bytes 0 to 3 (the same word in a /// down-saved copy as in a normal save). `None` when the chunk is missing (After Effects opens such /// a file), short or holds no version. pub fn saved_by_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) + Root::from(self).saved_by_version() + } +} + +impl TreeIndex<'_> { + /// [`AepFile::file_version`] of the indexed file. + pub fn file_version(&self) -> Option { + Root::from(self).file_version() } - /// The `u16` at `head` byte 0: 97 (`0x61`) in a 2026 file, 96 in a copy saved as 25.x, 95 as - /// 24.x. It steps with the major version of [`AepFile::file_version`]; what it counts is not - /// known, so nothing is derived from it. - pub fn head_revision(&self) -> Option { - be_u16(self.chunk(b"head")?.bytes()?, 0) + /// [`AepFile::saved_by_version`] of the indexed file. + pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl Root<'_, '_> { + /// The version the file is written for (`head` bytes 4 to 7). + pub fn file_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + } + + /// The version of the application that wrote the file (`svap`). + pub fn saved_by_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) } } diff --git a/crates/aep/src/write.rs b/crates/aep/src/write.rs index 1adf3126d..cadd84f0d 100644 --- a/crates/aep/src/write.rs +++ b/crates/aep/src/write.rs @@ -17,6 +17,18 @@ const MAX_WRITE_DEPTH: usize = 256; /// [`AepError::TooDeep`] for an absurdly deep hand-built tree. pub fn write(file: &AepFile<'_>) -> Result, AepError> { let mut out = Vec::new(); + // Sized once to the exact output. A tree past the 32-bit container size fails at once (the + // same error the write below would give after serialising it). `try_reserve` so a hand-built + // tree that repeats one large borrowed body many times cannot abort on the allocation; the + // writer then grows as it goes, as before. + if let Some(len) = sequence_len(&file.chunks, 1) { + if u32::try_from(len.saturating_add(4)).is_err() { + return Err(AepError::TooLarge); + } + if let Some(total) = len.checked_add(12).and_then(|n| n.checked_add(file.trailer.len())) { + let _ = out.try_reserve_exact(total); + } + } out.extend_from_slice(b"RIFX"); out.extend_from_slice(&[0; 4]); out.extend_from_slice(b"Egg!"); @@ -56,6 +68,28 @@ fn write_sequence(out: &mut Vec, chunks: &[Chunk<'_>], depth: usize) -> Resu Ok(()) } +/// The number of bytes [`write_sequence`] produces for `chunks`; `None` past the depth limit or +/// on overflow (both make the write fail). +fn sequence_len(chunks: &[Chunk<'_>], depth: usize) -> Option { + if depth > MAX_WRITE_DEPTH { + return None; + } + let count = chunks.len(); + let mut total = 0usize; + for (i, chunk) in chunks.iter().enumerate() { + let body = match &chunk.body { + Body::Data(d) => d.len(), + Body::Chunks(c) => sequence_len(c, depth + 1)?, + Body::List { children, .. } => sequence_len(children, depth + 1)?.checked_add(4)?, + Body::OpaqueList { data, .. } => data.len().checked_add(4)?, + }; + // The same pad rule as `write_chunk`. + let pad = usize::from(body % 2 == 1 && (chunk.pad.is_some() || i + 1 != count)); + total = total.checked_add(8)?.checked_add(body)?.checked_add(pad)?; + } + Some(total) +} + fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize) -> Result<(), AepError> { out.extend_from_slice(&chunk.id); let size_at = out.len(); @@ -84,3 +118,62 @@ fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize } Ok(()) } + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::*; + + /// The output is sized once to its exact length. + #[test] + fn the_output_is_sized_once_to_its_exact_length() { + let odd = |n: u8| Chunk::data(*b"cdat", vec![n; usize::from(n)]); + let mut padded = odd(3); + padded.pad = Some(0x7f); + let trees = [ + AepFile { chunks: Vec::new(), trailer: Cow::Borrowed(&[]) }, + AepFile { + chunks: vec![ + odd(1), + Chunk::list(*b"Fold", vec![odd(2), odd(3), Chunk::list(*b"tdgp", vec![odd(5)])]), + Chunk { id: *b"LIST", body: Body::OpaqueList { list_type: *b"btdk", data: Cow::Owned(vec![1, 2, 3]) }, pad: None }, + Chunk { id: *b"tdmn", body: Body::Chunks(vec![odd(7), padded]), pad: None }, + odd(9), + ], + trailer: Cow::Borrowed(b""), + }, + ]; + for file in &trees { + let out = write(file).expect("write"); + assert_eq!(sequence_len(&file.chunks, 1).map(|n| n + 12 + file.trailer.len()), Some(out.len())); + // Exact with std's allocator today; grown push by push it would be rounded up. + assert_eq!(out.capacity(), out.len(), "the output grew past its size"); + } + } + + /// A tree past the container size fails before serialising. + #[cfg(target_pointer_width = "64")] + #[test] + fn a_tree_past_the_container_size_fails_before_serialising() { + // 4097 chunks borrowing one 1 MiB body: over 4 GiB of output, none of it written. + let body = vec![0u8; 1 << 20]; + let chunks: Vec> = (0..4097).map(|_| Chunk::data(*b"cdat", &body[..])).collect(); + let file = AepFile { chunks, trailer: Cow::Borrowed(&[]) }; + assert_eq!(write(&file), Err(AepError::TooLarge)); + // Just under the limit still sizes, and fails nothing up front. + assert!(sequence_len(&file.chunks[..4000], 1).is_some_and(|n| u32::try_from(n + 4).is_ok())); + } + + /// A tree past the depth limit has no size. + #[test] + fn a_tree_past_the_depth_limit_has_no_size() { + let mut deep = Chunk::data(*b"cdat", vec![0]); + for _ in 0..=MAX_WRITE_DEPTH { + deep = Chunk::list(*b"tdgp", vec![deep]); + } + let file = AepFile { chunks: vec![deep], trailer: Cow::Borrowed(&[]) }; + assert_eq!(sequence_len(&file.chunks, 1), None); + assert_eq!(write(&file), Err(AepError::TooDeep)); + } +} diff --git a/crates/aep/tests/it/common/mod.rs b/crates/aep/tests/it/common/mod.rs index 791c3bbc9..b5605bb95 100644 --- a/crates/aep/tests/it/common/mod.rs +++ b/crates/aep/tests/it/common/mod.rs @@ -128,6 +128,13 @@ pub fn resolve(name: &str) -> Option { locate(name).map(|s| s.path().clone()) } +/// The ids of the compositions an item uses as precomposition layers, as the file stores them: +/// the little-endian `u32` of each `idpi` chunk (the importer does not read them; the exporter +/// writes them). +pub fn used_item_ids(item: &craft_aep::Item<'_, '_>) -> Vec { + item.chunk().children().filter(|c| c.is(b"idpi")).filter_map(|c| Some(u32::from_le_bytes(c.bytes()?.get(..4)?.try_into().ok()?))).collect() +} + /// The bytes of a case (see the module comment), or `None` when it is missing. pub fn try_fixture(name: &str) -> Option> { match locate(name)? { diff --git a/crates/aep/tests/it/container.rs b/crates/aep/tests/it/container.rs index d937d296c..514bbde1c 100644 --- a/crates/aep/tests/it/container.rs +++ b/crates/aep/tests/it/container.rs @@ -25,5 +25,6 @@ mod shapes; mod text_fields; mod text_markers_footage; mod text_style_slots; +mod tree_index; mod typed; mod values; diff --git a/crates/aep/tests/it/container/batch3_core.rs b/crates/aep/tests/it/container/batch3_core.rs index 8c2518a9f..83bd942b8 100644 --- a/crates/aep/tests/it/container/batch3_core.rs +++ b/crates/aep/tests/it/container/batch3_core.rs @@ -117,8 +117,8 @@ fn a_transform_set_to_its_default_adds_no_record() { assert_eq!(t.opacity_percent.source, ValueSource::Default); assert!(layer.properties().unwrap().find(&["ADBE Transform Group", "ADBE Position"]).is_none()); // The two cases are layers of one file: they differ in the layer id only. - let a = without_layer_id(layer.chunk()); - skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(other.chunk())))); + let a = without_layer_id(&layer.chunk().to_chunk()); + skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(&other.chunk().to_chunk())))); })); } @@ -140,16 +140,13 @@ fn scale_keys_keep_their_ease_type_major() { } assert_near(&keys[0].value().unwrap(), &[1.0, 1.0, 1.0], 1e-9); assert_near(&keys[1].value().unwrap(), &[2.0, 1.5, 1.0], 1e-9); - // The raw block: [in speed x3][in influence x3][out speed x3][out influence x3]. - assert_near(&keys[0].plain_ease_slots().unwrap(), &[0.0, 0.0, 0.0, 0.33, 0.33, 0.33, 0.2, 0.2, 0.2, 0.66, 0.66, 0.66], 1e-9); - assert_near(&keys[1].plain_ease_slots().unwrap(), &[0.1, 0.1, 0.1, 0.66, 0.66, 0.66, 0.0, 0.0, 0.0, 0.33, 0.33, 0.33], 1e-9); + // The raw block is [in speed x3][in influence x3][out speed x3][out influence x3]. for i in 0..3 { assert!(same_ease(&keys[0].ease_component(i).unwrap(), &ease(0.0, 0.33, 0.2, 0.66)), "key 1 component {i}"); assert!(same_ease(&keys[1].ease_component(i).unwrap(), &ease(0.1, 0.66, 0.0, 0.33)), "key 2 component {i}"); } assert!(keys[0].ease_component(3).is_none()); assert_eq!(keys[0].ease_components().unwrap().len(), 3); - assert!(keys[0].color_distance().is_none()); })); } @@ -172,7 +169,6 @@ fn mask_feather_keys_keep_their_ease_type_major() { } assert_near(&keys[0].value().unwrap(), &[0.0, 0.0], 1e-9); assert_near(&keys[1].value().unwrap(), &[50.0, 30.0], 1e-9); - assert_near(&keys[0].plain_ease_slots().unwrap(), &[0.0, 0.0, 0.33, 0.33, 15.0, 15.0, 0.66, 0.66], 1e-9); for i in 0..2 { assert!(same_ease(&keys[0].ease_component(i).unwrap(), &ease(0.0, 0.33, 15.0, 0.66)), "key 1 component {i}"); assert!(same_ease(&keys[1].ease_component(i).unwrap(), &ease(8.0, 0.66, 0.0, 0.33)), "key 2 component {i}"); @@ -213,14 +209,15 @@ fn bezier_colour_keys_carry_the_distance_to_the_next_key() { assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Bezier), Some(Interpolation::Bezier))); assert!(same_ease(&k.ease_component(0).unwrap(), &ease(0.0, 0.0, 0.0, 0.0))); } - assert!(near(keys[0].color_distance().unwrap(), 255.0 * 2f64.sqrt(), 1e-3), "{:?}", keys[0].color_distance()); - assert!(near(keys[0].color_distance().unwrap(), 360.6245, 1e-3)); - assert_eq!(keys[1].color_distance(), Some(0.0)); + let distance = |k: &craft_aep::Keyframe<'_>| k.spatial().unwrap().segment_length; + assert!(near(distance(&keys[0]), 255.0 * 2f64.sqrt(), 1e-3), "{}", distance(&keys[0])); + assert!(near(distance(&keys[0]), 360.6245, 1e-3)); + assert_eq!(distance(&keys[1]), 0.0); })); skip!(with_layer("210-key-fill-color-2key-linear.aep", L, |_, layer| { let p = layer.effects()[0].params().into_iter().find(|p| p.name == "Color").unwrap(); let k = p.stored().unwrap().keys().unwrap().keyframes().next().unwrap(); - assert_eq!(k.color_distance(), Some(0.0)); + assert_eq!(k.spatial().unwrap().segment_length, 0.0); })); } @@ -243,21 +240,19 @@ fn time_stretch_is_an_exact_ratio() { let (h, in_p, out_p, scale) = skip!(stretch_facts("316-layer-stretch50.aep")); assert_eq!((h.stretch, h.stretch_den), (1, 2)); assert_eq!(h.stretch_ratio(), Some(rat(1, 2))); - assert_eq!(h.stretch_percent(), 50.0); assert_eq!(h.out_relative, rat(128000, 25600)); assert_eq!((in_p, out_p, scale), (Some(rat(0, 1)), Some(rat(5, 2)), 25600)); // 317: 150 %: 3/2, ends at 7.5 s, the property time scale is 25600 x 3/2. let (h, in_p, out_p, scale) = skip!(stretch_facts("317-layer-stretch150.aep")); assert_eq!((h.stretch, h.stretch_den), (3, 2)); assert_eq!(h.stretch_ratio(), Some(rat(3, 2))); - assert_eq!(h.stretch_percent(), 150.0); assert_eq!((in_p, out_p, scale), (Some(rat(0, 1)), Some(rat(15, 2)), 38400)); // 018: 200 %: 2/1, ends at 10 s, time scale 51200. The base is 1/1. let (h, _, out_p, scale) = skip!(stretch_facts("018-layer-stretch200.aep")); assert_eq!((h.stretch, h.stretch_den, h.stretch_ratio()), (2, 1, Some(rat(2, 1)))); assert_eq!((out_p, scale), (Some(rat(10, 1)), 51200)); let (h, _, out_p, scale) = skip!(stretch_facts("000-base.aep")); - assert_eq!((h.stretch, h.stretch_den, h.stretch_percent()), (1, 1, 100.0)); + assert_eq!((h.stretch, h.stretch_den, h.stretch_ratio()), (1, 1, Some(rat(1, 1)))); assert_eq!((out_p, scale), (Some(rat(5, 1)), 25600)); } @@ -269,9 +264,9 @@ fn fractional_times_are_exact_multiples_of_one_over_25600() { assert_eq!(h.in_relative, rat(8532, 25600)); assert_eq!(h.out_relative, rat(136532, 25600)); assert_eq!(h.start, rat(0, 25600)); + // 0.33328125 s and 5.33328125 s. assert_eq!(h.in_point(), Some(rat(2133, 6400))); - assert_eq!(h.in_point_seconds(), Some(0.33328125)); - assert_eq!(h.out_point_seconds(), Some(5.33328125)); + assert_eq!(h.out_point(), Some(rat(34133, 6400))); assert_eq!(h.stretch_ratio(), Some(rat(1, 1))); } @@ -364,18 +359,17 @@ fn pixel_aspect_shutter_phase_and_display_start() { // 325: pixelAspect 0.9091 reads back 0.90909090909091 = the exact ratio 10/11. let h = skip!(comp_header("325-comp-pixelaspect-0.9091.aep")); assert_eq!(h.pixel_aspect.reduced(), Some(rat(10, 11))); - assert!(near(h.pixel_aspect_ratio().unwrap(), 0.90909090909091, 1e-12)); // 326: shutterPhase -90, the shutter angle stays 180. let h = skip!(comp_header("326-comp-shutterphase-m90.aep")); assert_eq!((h.shutter_phase, h.shutter_angle), (Some(-90), 180)); // 327: displayStartFrame 100 at 25 fps = 4 s = 102400/25600; the shutter phase is back to 0. let h = skip!(comp_header("327-comp-displaystartframe-100.aep")); assert_eq!(h.display_start, Some(rat(102400, 25600))); - assert_eq!((h.display_start_seconds(), h.display_start_frame(), h.shutter_phase), (Some(4.0), Some(100), Some(0))); + assert_eq!(h.shutter_phase, Some(0)); // The base: no phase, start 0. let h = skip!(comp_header("000-base.aep")); - assert_eq!((h.shutter_phase, h.display_start_seconds(), h.display_start_frame()), (Some(0), Some(0.0), Some(0))); - assert_eq!(h.pixel_aspect_ratio(), Some(1.0)); + assert_eq!((h.shutter_phase, h.display_start.and_then(|r| r.reduced())), (Some(0), Some(rat(0, 1)))); + assert_eq!(h.pixel_aspect.reduced(), Some(rat(1, 1))); // A short chunk still decodes the old fields. let bytes = [0u8; 176]; let h = craft_aep::CompHeader::parse(&bytes).unwrap(); diff --git a/crates/aep/tests/it/container/batch3_objects.rs b/crates/aep/tests/it/container/batch3_objects.rs index bbb7040b4..c78525a00 100644 --- a/crates/aep/tests/it/container/batch3_objects.rs +++ b/crates/aep/tests/it/container/batch3_objects.rs @@ -148,19 +148,21 @@ fn a_mask_box_is_normalised_to_the_layer_size() { for (got, want) in m.shapes[0].bbox.iter().zip(want_box) { assert!(near(f64::from(*got), want, 1e-6), "{:?}", m.shapes[0].bbox); } - let shapes = project.mask_vertices_px(&layer, m).unwrap(); - assert_eq!(shapes.len(), 1); - let got: Vec<[f64; 2]> = shapes[0].iter().map(|v| v.position).collect(); + // The box scales by the size of the layer's source (the importer takes that size from the + // source item it matched, the same item `layer_source_size` reads). + let (w, h) = project.layer_source_size(&layer).unwrap(); + assert_eq!(m.shapes.len(), 1); + let got: Vec<[f64; 2]> = m.shapes[0].vertices((f64::from(w), f64::from(h))).iter().map(|v| v.position).collect(); for (g, w) in got.iter().zip([[20.0, 30.0], [80.0, 30.0], [80.0, 70.0], [20.0, 70.0]]) { assert_near(g, &w, 1e-3); } assert_eq!(got.len(), 4); - // A layer without a sized source has no pixel vertices. + // A layer without a sized source has no size to scale by. let bytes = fixture!("333-shape-group-two-rects.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let shape = layer_named(&first_comp(&project), "Shape"); - assert!(project.mask_vertices_px(&shape, m).is_none()); + assert!(project.layer_source_size(&shape).is_none()); } #[test] @@ -178,12 +180,14 @@ fn one_mkif_per_mask_and_the_mode_none() { assert_eq!((m.mode, m.inverted, m.counter), (Some(MaskMode::Add), Some(false), Some(counter))); assert_eq!(m.shapes.len(), 1); } - let v1 = project.mask_vertices_px(&layer, &masks[0]).unwrap(); + let (w, h) = project.layer_source_size(&layer).unwrap(); + let px = |m: &craft_aep::Mask| m.shapes.iter().map(|s| s.vertices((f64::from(w), f64::from(h)))).collect::>(); + let v1 = px(&masks[0]); for (v, w) in v1[0].iter().zip(ellipse_vertices()) { assert_near(&v.position, &w, 1e-2); } // Mask 2 has its own box (500..900 x 350..650) normalised to the layer. - let v2 = project.mask_vertices_px(&layer, &masks[1]).unwrap(); + let v2 = px(&masks[1]); for (v, w) in v2[0].iter().zip([[500.0, 350.0], [900.0, 350.0], [900.0, 650.0], [500.0, 650.0]]) { assert_near(&v.position, &w, 1e-2); } @@ -384,81 +388,16 @@ fn the_last_style_run_absorbs_the_final_carriage_return() { // ------------------------------------------------------------------ camera and light (341-345) +/// Camera zoom keeps the auto position. #[test] -fn camera_zoom_and_depth_of_field() { - let camera = |name: &str| { - Some({ - let bytes = crate::common::try_fixture(name)?; - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Camera); - ( - layer.camera_zoom(), - layer.camera_depth_of_field(), - layer.camera_focus_distance(), - layer.camera_aperture(), - layer.camera_position().map(|p| (p.stored, p.auto_state)), - ) - }) - }; - // 341: Zoom 2000 is stored; the auto Position keeps [0, 0, -2666.67]. - let (zoom, dof, focus, aperture, pos) = skip!(camera("341-camera-zoom-2000.aep")); - assert_eq!(zoom.map(|z| (z.value, z.source)), Some((2000.0, ValueSource::Stored))); - assert_eq!((dof.map(|d| (d.value, d.source)), focus, aperture), (Some((false, ValueSource::Default)), None, None)); - let (stored, auto) = pos.unwrap(); - assert!(auto && near(stored[2], -2666.67, 0.01) && stored[0] == 0.0, "{stored:?}"); - // 342: DOF on, Focus Distance 800, Aperture 30; Zoom stays the default. - let (zoom, dof, focus, aperture, _) = skip!(camera("342-camera-dof-800-30.aep")); - assert_eq!(zoom.map(|z| (z.source, near(z.value, 2666.67, 0.01))), Some((ValueSource::Default, true))); - assert_eq!(dof.map(|d| (d.value, d.source)), Some((true, ValueSource::Stored))); - assert_eq!(focus.map(|f| (f.value, f.source)), Some((800.0, ValueSource::Stored))); - assert_eq!(aperture.map(|f| (f.value, f.source)), Some((30.0, ValueSource::Stored))); - // The default camera (256) has nothing but the default zoom. - let (zoom, dof, ..) = skip!(camera("256-camera-two-node-default.aep")); - assert_eq!((zoom.unwrap().source, dof.unwrap().value), (ValueSource::Default, false)); - assert_eq!(zoom.unwrap().value, craft_aep::DEFAULT_CAMERA_ZOOM); -} - -#[test] -fn light_colour_cone_angle_and_feather() { - let light = |name: &str| { - Some({ - let bytes = crate::common::try_fixture(name)?; - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Light); - assert_eq!(layer.header().unwrap().light_type(), Some(craft_aep::LightType::Spot), "{name}"); - let t = |x: Option>| x.map(|x| (x.value, x.source)); - (layer.light_color().map(|c| (c.value, c.source)), t(layer.light_cone_angle()), t(layer.light_cone_feather())) - }) - }; - // The default spot light: white, 90, 50, all absent. - let (colour, angle, feather) = skip!(light("254-light-spot-intensity-50.aep")); - assert_eq!( - (colour, angle, feather), - (Some(([1.0; 4], ValueSource::Default)), Some((90.0, ValueSource::Default)), Some((50.0, ValueSource::Default))) - ); - // 343: colour [1,0,0,1] = ARGB doubles [255,255,0,0]. - let (colour, angle, feather) = skip!(light("343-light-spot-color-red.aep")); - assert_eq!(colour, Some(([1.0, 1.0, 0.0, 0.0], ValueSource::Stored))); - assert_eq!((angle, feather), (Some((90.0, ValueSource::Default)), Some((50.0, ValueSource::Default)))); - // 344: Cone Angle 60. 345: Cone Feather 30. - let (colour, angle, feather) = skip!(light("344-light-spot-coneangle-60.aep")); - assert_eq!((colour.unwrap().1, angle, feather), (ValueSource::Default, Some((60.0, ValueSource::Stored)), Some((50.0, ValueSource::Default)))); - let (_, angle, feather) = skip!(light("345-light-spot-conefeather-30.aep")); - assert_eq!((angle, feather), (Some((90.0, ValueSource::Default)), Some((30.0, ValueSource::Stored)))); - // A point light (252) stores Cone Angle 90 / Feather 50 in their default state. - let bytes = fixture!("252-light-point.aep"); +fn camera_zoom_keeps_the_auto_position() { + // 341: Zoom 2000 is stored (checked on import); the auto Position keeps [0, 0, -2666.67]. + let bytes = fixture!("341-camera-zoom-2000.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Light); - assert_eq!(layer.light_cone_angle().map(|a| (a.value, a.source)), Some((90.0, ValueSource::Stored))); - // Layers without light options have no light accessors. - let bytes = fixture!("000-base.aep"); - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let solid = base_layer(&first_comp(&project)); - assert!(solid.light_color().is_none() && solid.camera_zoom().is_none() && solid.camera_depth_of_field().is_none()); + let layer = layer_of_type(&first_comp(&project), LayerType::Camera); + let pos = layer.camera_position().unwrap(); + assert!(pos.auto_state && near(pos.stored[2], -2666.67, 0.01) && pos.stored[0] == 0.0, "{:?}", pos.stored); } // ------------------------------------------------------------------ marker data (347-349, 249) @@ -508,7 +447,7 @@ fn audio_footage_item() { assert_eq!((f.width, f.height, f.is_solid), (0, 0, false)); assert_eq!(f.duration, Some(rat(48000, 48000))); assert_eq!((f.audio_sample_rate, f.file_size, f.bits_per_pixel, f.native_frame_rate), (Some(48000.0), 96044, 0, 0)); - assert!(!f.is_sequence && f.sequence_frame_count().is_none()); + assert!(!f.is_sequence && item.sequence_info().is_none()); // The combined file stores no display name for the item (the single-case file had // "test-tone.wav"); the path names the file. assert_eq!(item.label(), Some(7)); @@ -532,7 +471,7 @@ fn image_sequence_is_one_item() { assert_eq!(f.duration, Some(rat(3, 30)), "{name}"); assert_eq!((f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate), (30, 24, 504, None), "{name}"); assert!(f.is_sequence, "{name}"); - assert_eq!(f.sequence_frame_count(), Some(3), "{name}"); + assert_eq!(item.sequence_info().map(|s| s.frame_count()), Some(3), "{name}"); // The first file name sits in opti; the alias points at the sequence folder. assert_eq!(f.name.as_deref(), Some("seq_001.png"), "{name}"); assert!(f.path.as_deref().is_some_and(|p| p.ends_with("test-seq")), "{name}"); diff --git a/crates/aep/tests/it/container/cases_600.rs b/crates/aep/tests/it/container/cases_600.rs index 210fb0916..43df3052a 100644 --- a/crates/aep/tests/it/container/cases_600.rs +++ b/crates/aep/tests/it/container/cases_600.rs @@ -38,7 +38,7 @@ fn a_composition_counts_the_compositions_it_shows() { let id = |n: &str| items.iter().find(|i| i.display_name().as_deref() == Some(n)).and_then(|i| i.id()).expect(n); let c = items.iter().find(|i| i.display_name().as_deref() == Some("C")).expect("C"); // The list holds each used composition once (in no particular order). - let mut used = c.used_item_ids(); + let mut used = crate::common::used_item_ids(c); used.sort(); let mut want = vec![id("P30"), id("Q")]; want.sort(); @@ -60,8 +60,8 @@ fn an_image_sequence_knows_its_numbering() { let item = project.all_items().into_iter().find(|i| i.footage().is_some_and(|f| !f.is_solid)).expect("sequence"); let f = item.footage().expect("footage"); assert_eq!((f.file_type.as_str(), f.width, f.height, f.native_frame_rate, f.is_sequence), ("oEXR", 64, 48, 30, true)); - assert_eq!(f.sequence_frame_count(), Some(3)); let s = item.sequence_info().expect("numbering"); + assert_eq!(s.frame_count(), 3); assert_eq!((s.prefix.as_str(), s.suffix.as_str(), s.start, s.end, s.digits), ("shot_", ".exr", 101, 103, 4)); assert_eq!(s.file_name(101), "shot_0101.exr"); // A still has no numbering (000: the solid; the base has no other footage). diff --git a/crates/aep/tests/it/container/header.rs b/crates/aep/tests/it/container/header.rs index f78b0bba9..f9252f7fd 100644 --- a/crates/aep/tests/it/container/header.rs +++ b/crates/aep/tests/it/container/header.rs @@ -55,7 +55,6 @@ fn assert_plain(h: &LayerHeader) { assert_eq!(h.label, 1); // No name was set, so the copy of the name at byte 64 is empty (the layer shows its source name). assert_eq!(h.name_copy, ""); - assert!(!h.name_is_user_set()); assert!(!h.is_null_object()); assert_eq!(h.parent_id, None); assert_eq!(h.source_id, Some(0x1a)); @@ -67,16 +66,13 @@ fn base_layer_header() { assert_plain(&h); assert_eq!(h.quality, Quality::Best); assert_eq!(h.blend_mode, BlendMode::Normal); - assert_eq!(h.stretch, 1); - assert_eq!(h.stretch_percent(), 100.0); + assert_eq!((h.stretch, h.stretch_den), (1, 1)); assert_eq!(switches(&h), NONE); // start 0 s, in 0 s, out 5 s; denominator = the composition timescale. assert_eq!(h.start, Rational { num: 0, den: 25600 }); assert_eq!(h.in_relative, Rational { num: 0, den: 25600 }); assert_eq!(h.out_relative, Rational { num: 128_000, den: 25600 }); - assert_eq!(h.start_seconds(), Some(0.0)); - assert_eq!(h.in_point_seconds(), Some(0.0)); - assert_eq!(h.out_point_seconds(), Some(5.0)); + assert_eq!(h.in_point(), Rational::new(0, 1)); assert_eq!(h.out_point(), Rational::new(5, 1)); } @@ -98,7 +94,7 @@ fn one_switch_per_case() { assert_eq!(switches(&h), expected, "{name}"); assert_plain(&h); assert_eq!((h.quality, h.blend_mode, h.stretch), (Quality::Best, BlendMode::Normal, 1), "{name}"); - assert_eq!(h.out_point_seconds(), Some(5.0), "{name}"); + assert_eq!(h.out_point(), Rational::new(5, 1), "{name}"); } } @@ -106,7 +102,6 @@ fn one_switch_per_case() { fn name_blend_mode_and_quality() { let h = skip!(l_header("010-layer-name.aep")); assert_eq!(h.name_copy, "Renamed"); - assert!(h.name_is_user_set()); let bytes = fixture!("010-layer-name.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); @@ -114,7 +109,6 @@ fn name_blend_mode_and_quality() { // The layer name is the user's; the source footage keeps its own name. assert_eq!(layer.name().as_deref(), Some("Renamed")); assert_eq!(project.layer_source(&layer).and_then(|i| i.display_name()).as_deref(), Some("L")); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("Renamed")); assert_eq!(skip!(l_header("016-layer-blend-multiply.aep")).blend_mode, BlendMode::Multiply); assert_eq!(skip!(l_header("023-layer-quality-draft.aep")).quality, Quality::Draft); @@ -127,7 +121,7 @@ fn display_name_falls_back_to_the_source_name() { let project = Project::new(&file).unwrap(); let layer = base_layer(&first_comp(&project)); assert_eq!(layer.name(), None); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("L")); + assert_eq!(project.layer_source(&layer).and_then(|i| i.display_name()).as_deref(), Some("L")); assert_eq!(project.layer_source_size(&layer), Some((1920, 1080))); let source = project.layer_source(&layer).unwrap(); assert_eq!(source.id(), Some(0x1a)); @@ -139,26 +133,23 @@ fn time_stretch_and_the_in_out_start_convention() { // 018: stretch 100 -> 200: the stored out point stays 5 s, the layer ends at 10 s. let h = skip!(l_header("018-layer-stretch200.aep")); assert_eq!(h.stretch, 2); - assert_eq!(h.stretch_percent(), 200.0); + assert_eq!(h.stretch_ratio(), Rational::new(2, 1)); assert_eq!(h.out_relative, Rational { num: 128_000, den: 25600 }); - assert_eq!(h.out_point_seconds(), Some(10.0)); - assert_eq!(h.in_point_seconds(), Some(0.0)); + assert_eq!(h.out_point(), Rational::new(10, 1)); + assert_eq!(h.in_point(), Rational::new(0, 1)); // 019: inPoint 0 -> 1; AE moved the out point to 6 s and left the start at 0. let h = skip!(l_header("019-layer-inpoint1.aep")); - assert_eq!(h.start_seconds(), Some(0.0)); + assert_eq!(h.start.num, 0); assert_eq!(h.in_relative, Rational { num: 25600, den: 25600 }); assert_eq!(h.in_point(), Rational::new(1, 1)); - assert_eq!(h.out_point_seconds(), Some(6.0)); + assert_eq!(h.out_point(), Rational::new(6, 1)); // 021: startTime 0 -> 0.5: in 0.5 s and out 5.5 s although the stored in point stays 0. let h = skip!(l_header("021-layer-starttime0.5.aep")); assert_eq!(h.start, Rational { num: 12800, den: 25600 }); - assert_eq!(h.start_seconds(), Some(0.5)); assert_eq!(h.in_relative.num, 0); assert_eq!(h.in_point(), Rational::new(1, 2)); - assert_eq!(h.in_point_seconds(), Some(0.5)); - assert_eq!(h.out_point_seconds(), Some(5.5)); assert_eq!(h.out_point(), Rational::new(11, 2)); } @@ -175,7 +166,6 @@ fn parent_link_and_null_object() { let nh = null.header().unwrap(); assert_eq!(null.name().as_deref(), Some("Null")); assert!(nh.is_null_object()); - assert!(nh.name_is_user_set()); assert_eq!(nh.layer_type, LayerType::Av); assert_eq!(nh.parent_id, None); assert_eq!(project.layer_source(null).and_then(|i| i.display_name()).as_deref(), Some("Null 1")); @@ -225,17 +215,14 @@ fn base_comp_header() { assert_eq!((h.width, h.height), (1920, 1080)); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 25600)); assert_eq!(h.frame_rate(), Some(25.0)); - assert_eq!(h.frame_rate_from_fixed(), 25.0); + assert_eq!(h.frame_rate_fixed, 25 << 16); assert_eq!(h.duration_units, 128_000); assert_eq!(h.duration_seconds(), Some(5.0)); assert_eq!(h.background, [0, 0, 0]); - assert_eq!(h.background_rgb(), [0.0, 0.0, 0.0]); assert_eq!(h.shutter_angle, 180); - assert_eq!(h.work_area_start_seconds(), Some(0.0)); + assert_eq!(h.work_area_start_units, 0); assert_eq!(h.work_area_end_units, None); - assert_eq!(h.work_area_end_seconds(), Some(5.0)); assert_eq!(h.pixel_aspect, Rational { num: 1, den: 1 }); - assert_eq!(h.pixel_aspect_ratio(), Some(1.0)); assert!(!h.motion_blur); // The manifest calls the classic renderer `ADBE Advanced 3d`; the file stores `ADBE Escher` // and the display name `Classic 3D`. @@ -255,7 +242,7 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("031-comp-fps30.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 30720)); assert_eq!(h.frame_rate(), Some(30.0)); - assert_eq!(h.frame_rate_from_fixed(), 30.0); + assert_eq!(h.frame_rate_fixed, 30 << 16); assert_eq!(h.duration_units, 153_600); assert_eq!(h.duration_seconds(), Some(5.0)); @@ -263,12 +250,11 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("032-comp-duration10.aep")); assert_eq!(h.duration_units, 256_000); assert_eq!(h.duration_seconds(), Some(10.0)); - assert_eq!(h.work_area_end_seconds(), Some(10.0)); + assert_eq!(h.work_area_end_units, None); // 033: bgColor [0,0,0] -> [0,1,0]. let (h, _, _) = skip!(comp_header("033-comp-bg-green.aep")); assert_eq!(h.background, [0, 255, 0]); - assert_eq!(h.background_rgb(), [0.0, 1.0, 0.0]); // 034: shutterAngle 180 -> 90. let (h, _, _) = skip!(comp_header("034-comp-shutter90.aep")); @@ -279,8 +265,6 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("035-comp-workarea-1-2.aep")); assert_eq!(h.work_area_start_units, 25600); assert_eq!(h.work_area_end_units, Some(76_800)); - assert_eq!(h.work_area_start_seconds(), Some(1.0)); - assert_eq!(h.work_area_end_seconds(), Some(3.0)); } #[test] @@ -290,7 +274,6 @@ fn one_comp_change_per_case_aspect_blur_renderer() { // 037: pixelAspect 1 -> 2. let (h, _, _) = skip!(comp_header("037-comp-pixelaspect2.aep")); assert_eq!(h.pixel_aspect, Rational { num: 2, den: 1 }); - assert_eq!(h.pixel_aspect_ratio(), Some(2.0)); // 038: comp motion blur on. let (h, _, _) = skip!(comp_header("038-comp-motionblur.aep")); @@ -328,17 +311,16 @@ fn precomp_item_ids_and_labels() { assert_eq!(h.source_id, pre_item.id()); assert_eq!(project.layer_source(layer).unwrap().id(), pre_item.id()); assert_eq!(h.label, 15); - assert!(!h.name_is_user_set()); assert_eq!(layer.name(), None); - assert_eq!(project.layer_display_name(layer).as_deref(), Some("Pre")); + assert_eq!(project.layer_source(layer).and_then(|i| i.display_name()).as_deref(), Some("Pre")); assert_eq!(project.layer_source_size(layer), Some((1920, 1080))); // The precomp layer's start time has the denominator 1; the Pre comp is 1920x1080, 25 fps, 5 s. assert_eq!(h.start.den, 1); let ph = pre.header().unwrap(); assert_eq!((ph.width, ph.height, ph.frame_rate(), ph.duration_seconds()), (1920, 1080, Some(25.0), Some(5.0))); // C records that it uses the comp Pre (chunk idpi, a little-endian u32). - assert_eq!(c_item.used_item_ids(), [pre_item.id().unwrap()]); - assert!(pre_item.used_item_ids().is_empty()); + assert_eq!(crate::common::used_item_ids(c_item), [pre_item.id().unwrap()]); + assert!(crate::common::used_item_ids(pre_item).is_empty()); assert_eq!(pre_item.label(), Some(15)); // The layer inside Pre is a new layer made from the solid L (moving the layers into a precomp // creates new layers). @@ -347,7 +329,7 @@ fn precomp_item_ids_and_labels() { let ih = inner[0].header().unwrap(); assert_ne!(ih.id, L); assert_eq!(ih.source_id, Some(0x1a)); - assert_eq!(project.layer_display_name(&inner[0]).as_deref(), Some("L")); + assert_eq!(project.layer_source(&inner[0]).and_then(|i| i.display_name()).as_deref(), Some("L")); } #[test] @@ -356,9 +338,8 @@ fn a_negative_stretch_is_a_reversed_layer() { // point 0 and out point 5 s relative, so the layer runs from 5 s back to 0 s; 983: -50 % = -1/2. let h = skip!(l_header("981-layer-stretch-minus100-start5.aep")); assert_eq!((h.stretch, h.stretch_den), (-1, 1)); - assert_eq!(h.stretch_percent(), -100.0); - assert_eq!((h.in_point_seconds(), h.out_point_seconds()), (Some(5.0), Some(0.0))); + assert_eq!((h.in_point(), h.out_point()), (Rational::new(5, 1), Rational::new(0, 1))); let h = skip!(l_header("983-layer-stretch-minus50-opacity-keys.aep")); assert_eq!((h.stretch, h.stretch_den), (-1, 2)); - assert_eq!((h.in_point_seconds(), h.out_point_seconds()), (Some(5.0), Some(2.5))); + assert_eq!((h.in_point(), h.out_point()), (Rational::new(5, 1), Rational::new(5, 2))); } diff --git a/crates/aep/tests/it/container/helpers.rs b/crates/aep/tests/it/container/helpers.rs index b6ab11432..af2282ab8 100644 --- a/crates/aep/tests/it/container/helpers.rs +++ b/crates/aep/tests/it/container/helpers.rs @@ -379,7 +379,8 @@ pub fn layer_named<'f, 'a>(comp: &Comp<'f, 'a>, name: &str) -> Layer<'f, 'a> { /// The layer that shows this name: its own name, or its source's while it has none. pub fn layer_shown_as<'f, 'a>(project: &Project<'f, 'a>, comp: &Comp<'f, 'a>, name: &str) -> Layer<'f, 'a> { - comp.layers().find(|l| project.layer_display_name(l).as_deref() == Some(name)).unwrap_or_else(|| panic!("no layer shown as {name}")) + let shown = |l: &Layer<'f, 'a>| l.name().or_else(|| project.layer_source(l)?.display_name()); + comp.layers().find(|l| shown(l).as_deref() == Some(name)).unwrap_or_else(|| panic!("no layer shown as {name}")) } /// The first layer of this type in a composition. diff --git a/crates/aep/tests/it/container/keyframes.rs b/crates/aep/tests/it/container/keyframes.rs index 56ec367cc..27b8073ab 100644 --- a/crates/aep/tests/it/container/keyframes.rs +++ b/crates/aep/tests/it/container/keyframes.rs @@ -19,6 +19,9 @@ fn prop_of(name: &str, path: &[&str], f: impl FnOnce(Property<'_, '_>) -> R) const POSITION: &[&str] = &["ADBE Transform Group", "ADBE Position"]; const OPACITY: &[&str] = &["ADBE Transform Group", "ADBE Opacity"]; +/// The ease of a linear key: all four slots 0. +const NO_EASE: KeyEase = KeyEase { in_speed: 0.0, in_influence: 0.0, out_speed: 0.0, out_influence: 0.0 }; + fn ease(in_speed: f64, in_influence: f64, out_speed: f64, out_influence: f64) -> KeyEase { KeyEase { in_speed, in_influence, out_speed, out_influence } } @@ -33,7 +36,7 @@ fn in_and_out_interpolation_are_separate_bytes() { assert_eq!((keys[0].interpolation_in(), keys[0].interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Hold))); assert_eq!((keys[1].interpolation_in(), keys[1].interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Linear))); assert_eq!(keys.iter().map(|k| k.seconds(25600).unwrap()).collect::>(), [0.0, 1.0]); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); })); // 203: key 1 inType HOLD, outType LINEAR (bytes 03 01): byte 4 is the incoming side. skip!(prop_of("203-key-in-hold-out-linear.aep", POSITION, |p| { @@ -76,7 +79,7 @@ fn opacity_ease_speed_is_in_stored_units_per_second() { assert_near(&[e0.in_speed, e0.in_influence, e0.out_speed, e0.out_influence], &[0.05, 0.2, 0.2, 0.8], 1e-12); let e1 = keys[1].ease().unwrap(); assert_near(&[e1.in_speed, e1.in_influence, e1.out_speed, e1.out_influence], &[0.3, 0.6, 0.09, 0.4], 1e-12); - assert_eq!(keys[0].plain_ease_slots().unwrap().len(), 4); + assert_eq!(keys[0].ease_components(), Some(vec![e0])); for k in &keys { assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Bezier), Some(Interpolation::Bezier))); } @@ -94,7 +97,7 @@ fn scale_keys_use_the_plain_template_with_three_components() { // A 128-byte record is not always spatial: the header's template flag decides. assert_eq!(k.kind(), KeyframeKind::Plain { components: 3 }); assert_eq!(k.spatial(), None); - assert_eq!(k.plain_ease_slots(), Some(vec![0.0; 12])); + assert_eq!(k.ease_components(), Some(vec![NO_EASE; 3])); assert_eq!(k.ease(), None); assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Linear))); } @@ -125,7 +128,7 @@ fn two_dimensional_mask_feather_keys() { assert!(keys.iter().all(|k| k.raw().len() == 88 && k.kind() == KeyframeKind::Plain { components: 2 })); assert_eq!(keys.iter().map(|k| k.value().unwrap()).collect::>(), [vec![0.0, 0.0], vec![40.0, 20.0]]); assert_eq!(keys.iter().map(|k| k.seconds(25600).unwrap()).collect::>(), [0.0, 1.0]); - assert_eq!(keys[0].plain_ease_slots(), Some(vec![0.0; 8])); + assert_eq!(keys[0].ease_components(), Some(vec![NO_EASE; 2])); let mask = &layer.masks()[0]; assert!(mask.feather.source == ValueSource::Keyframed && mask.feather.value == [0.0, 0.0]); } diff --git a/crates/aep/tests/it/container/layer_switches.rs b/crates/aep/tests/it/container/layer_switches.rs index 5e2179d53..05e939c38 100644 --- a/crates/aep/tests/it/container/layer_switches.rs +++ b/crates/aep/tests/it/container/layer_switches.rs @@ -2,7 +2,7 @@ //! colours, checked against the values the corpus was made with (its MANIFEST, batch 2). use crate::container::helpers::{assert_near, base_layer, first_comp, layer_of_type, layer_shown_as, open}; -use craft_aep::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Project, Rational, ValueSource}; +use craft_aep::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Project}; fn l_header(name: &str) -> Option { Some({ @@ -80,22 +80,24 @@ fn ntsc_frame_rates_are_exact_fractions() { // 229: frameRate 29.97 -> read back 29.9700012207031; the duration snaps from 5 s to 150 frames (5.005005 s). let h = skip!(comp_header("229-comp-fps-29.97.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (800, 23976)); - assert_eq!(h.frame_rate_exact(), Some(Rational { num: 2997, den: 100 })); assert!((h.frame_rate().unwrap() - 29.97).abs() < 1e-12); - assert!((h.frame_rate_from_fixed() - 29.970_001_220_703_1).abs() < 1e-7); - assert_eq!((h.duration_units, h.duration_frames()), (120_000, Some(150))); + // The 16.16 rate holds the float32 29.97 (29.9700012207031). + assert_eq!(h.frame_rate_fixed, 1_964_114); + // 150 frames of 800 units. + assert_eq!(h.duration_units, 120_000); assert!((h.duration_seconds().unwrap() - 5.005_005_005_005_01).abs() < 1e-9); // 230: frameRate 23.976 -> read back 23.9759979248047; 120 frames. let h = skip!(comp_header("230-comp-fps-23.976.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1000, 23976)); - assert_eq!(h.frame_rate_exact(), Some(Rational { num: 2997, den: 125 })); assert!((h.frame_rate().unwrap() - 23.976).abs() < 1e-12); - assert!((h.frame_rate_from_fixed() - 23.975_997_924_804_7).abs() < 1e-7); - assert_eq!((h.duration_units, h.duration_frames()), (120_000, Some(120))); + // The 16.16 rate holds the float32 23.976 (23.9759979248047). + assert_eq!(h.frame_rate_fixed, 1_571_291); + // 120 frames of 1000 units. + assert_eq!(h.duration_units, 120_000); assert!((h.duration_seconds().unwrap() - 5.005_005_005_005_01).abs() < 1e-9); // 25 fps stays 25. let h = skip!(comp_header("000-base.aep")); - assert_eq!((h.frame_rate_exact(), h.duration_frames()), (Some(Rational { num: 25, den: 1 }), Some(125))); + assert_eq!((h.time_base.timescale, h.time_base.frame_duration, h.duration_units), (25600, 1024, 128_000)); } #[test] @@ -128,7 +130,6 @@ fn background_is_8_bits_per_channel() { // 231: bgColor [0.5, 0.5, 0.5] is stored as 128 and reads back as 128/255 = 0.50196. let h = skip!(comp_header("231-comp-bg-grey-0.5.aep")); assert_eq!(h.background, [128, 128, 128]); - assert_near(&h.background_rgb(), &[0.501_960_784_313_73; 3], 1e-12); // The frame rate stays 25 even though an earlier edit of the AE session left other numbers in the cache-like fields. assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 25600)); } @@ -143,12 +144,10 @@ fn cameras_auto_orient_and_position() { let camera = layer_of_type(&comp, LayerType::Camera); let h = camera.header().unwrap(); assert_eq!((h.flag_bytes[1], h.is_auto_oriented(), h.is_three_d(), h.source_id, h.light_type()), (0x44, true, true, None, None)); - // The default Position is an "auto state" record that stores [0, 0, -2666.67]; the two-node camera reports the comp centre. + // The default Position is an "auto state" record that stores [0, 0, -2666.67]. let pos = camera.camera_position().unwrap(); assert!(pos.auto_state); assert_near(&pos.stored, &[0.0, 0.0, -2666.67], 0.01); - assert_near(&pos.reported(1920, 1080, true), &[960.0, 540.0, -2666.67], 0.01); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); let bytes = fixture!("251-camera-one-node.aep"); let file = open(&bytes); @@ -159,10 +158,9 @@ fn cameras_auto_orient_and_position() { // AE reports [0, 0, -2666.67] for the one-node camera: the stored value. let pos = camera.camera_position().unwrap(); assert!(pos.auto_state); - assert_near(&pos.reported(1920, 1080, h.is_auto_oriented()), &[0.0, 0.0, -2666.67], 0.01); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); + assert_near(&pos.stored, &[0.0, 0.0, -2666.67], 0.01); - // 253: Position set to [100, 200, -1000]: stored as given, no auto state; the point of interest stays [960, 540, 0]. + // 253: Position set to [100, 200, -1000]: stored as given, no auto state. let bytes = fixture!("253-camera-position-100-200-m1000.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); @@ -170,35 +168,28 @@ fn cameras_auto_orient_and_position() { let pos = camera.camera_position().unwrap(); assert!(!pos.auto_state); assert_eq!(pos.stored, [100.0, 200.0, -1000.0]); - assert_eq!(pos.reported(1920, 1080, true), [100.0, 200.0, -1000.0]); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); - // A camera has no light intensity. - assert_eq!(camera.light_intensity(), None); } +/// Lights type and position. #[test] -fn lights_type_and_intensity() { - // 254: a spot light, Intensity 100 -> 50 (stored as 50.0 percent, absent at the default 100). +fn lights_type_and_position() { + // 254: a spot light (its Intensity 50 is checked on import). let bytes = fixture!("254-light-spot-intensity-50.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let light = layer_of_type(&first_comp(&project), LayerType::Light); let h = light.header().unwrap(); assert_eq!((h.light_type(), h.light_type_code, h.source_id, h.is_auto_oriented(), h.is_three_d()), (Some(LightType::Spot), 1, None, true, true)); - let intensity = light.light_intensity().unwrap(); - assert_eq!((intensity.value, intensity.source), (50.0, ValueSource::Stored)); assert!(light.camera_position().unwrap().auto_state); assert_near(&light.camera_position().unwrap().stored, &[0.0, 0.0, -1333.33], 0.01); - // 252: a point light (light type 2) at the default intensity, which is not stored. + // 252: a point light (light type 2). let bytes = fixture!("252-light-point.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let light = layer_of_type(&first_comp(&project), LayerType::Light); let h = light.header().unwrap(); assert_eq!((h.light_type(), h.light_type_code), (Some(LightType::Point), 2)); - let intensity = light.light_intensity().unwrap(); - assert_eq!((intensity.value, intensity.source), (100.0, ValueSource::Default)); // The solid under it is no light. let solid = base_layer(&first_comp(&project)).header().unwrap(); assert_eq!((solid.light_type(), solid.light_type_code), (None, 0)); @@ -220,6 +211,6 @@ fn solid_colour_is_four_float32_argb() { // L is still red. let red = project.layer_source(&base_layer(&comp)).unwrap().footage().unwrap(); assert_eq!(red.solid_rgb(), Some([1.0, 0.0, 0.0])); - // The new layer has no user-set name flag (it shows its source's name). - assert!(!blue.header().unwrap().name_is_user_set()); + // The new layer has no name of its own (it shows its source's name). + assert_eq!(blue.name(), None); } diff --git a/crates/aep/tests/it/container/matte.rs b/crates/aep/tests/it/container/matte.rs index 97c8aad74..c05045e39 100644 --- a/crates/aep/tests/it/container/matte.rs +++ b/crates/aep/tests/it/container/matte.rs @@ -13,7 +13,7 @@ fn layers(name: &str) -> Option, u32)>> { comp.layers() .map(|l| { let h = l.header().expect("header"); - (project.layer_display_name(&l).unwrap_or_default(), h.is_video_enabled(), l.track_matte(), h.id) + (l.name().or_else(|| project.layer_source(&l)?.display_name()).unwrap_or_default(), h.is_video_enabled(), l.track_matte(), h.id) }) .collect() }) diff --git a/crates/aep/tests/it/container/robustness.rs b/crates/aep/tests/it/container/robustness.rs index f27f53d25..0f26cdd4c 100644 --- a/crates/aep/tests/it/container/robustness.rs +++ b/crates/aep/tests/it/container/robustness.rs @@ -154,16 +154,15 @@ use craft_aep::{AepFile, Body, Chunk, Comp, Layer, Project, PropGroup, PropNode, /// Touches the property accessors; none may panic on a damaged property. fn exercise_property(p: &Property<'_, '_>) { - let _ = (p.header(), p.dims(), p.is_animated(), p.static_value(), p.time_scale(), p.flags(), p.is_static_flag_set()); - let _ = (p.expression(), p.expression_flags().map(|f| (f.all_set(), f.disabled())), p.expression_enabled(), p.user_name(), p.first_key_value()); - let _ = p.color_argb(); + let _ = (p.header(), p.dims(), p.is_animated(), p.static_value(), p.time_scale(), p.flags()); + let _ = (p.expression(), p.expression_flags().map(|f| f.disabled()), p.expression_enabled(), p.user_name(), p.first_key_value()); if let Some(keys) = p.keys() { let scale = p.time_scale().unwrap_or(0); for k in keys.keyframes() { - let _ = (k.time_units(), k.spatial_value(), k.raw().len(), k.kind(), k.seconds(scale), k.seconds(0)); + let _ = (k.time_units(), k.raw().len(), k.kind(), k.seconds(scale), k.seconds(0)); let _ = (k.interpolation_in(), k.interpolation_out(), k.flags(), k.label(), k.is_roving()); - let _ = (k.scalar_value(), k.ease(), k.spatial(), k.components(), k.value(), k.plain_ease_slots()); - let _ = (k.ease_components(), k.color_distance()); + let _ = (k.scalar_value(), k.ease(), k.spatial(), k.components(), k.value()); + let _ = k.ease_components(); for component in [0, 1, 2, 3, 7, 255, usize::MAX] { let _ = k.ease_component(component); } @@ -193,10 +192,9 @@ fn exercise_group(g: &PropGroup<'_, '_>, depth: usize) { fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Layer<'_, '_>) { let _ = (layer.name(), layer.is_three_d(), layer.id(), layer.track_matte()); if let Some(h) = layer.header() { - let _ = (h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds(), h.start_seconds(), h.stretch_percent()); - let _ = (h.stretch_ratio(), h.stretch_den); + let _ = (h.in_point(), h.out_point(), h.stretch_ratio(), h.stretch_den); let _ = (h.is_shy(), h.is_locked(), h.is_solo(), h.is_three_d(), h.is_motion_blur(), h.is_adjustment(), h.is_guide()); - let _ = (h.is_null_object(), h.collapses_transformations(), h.name_is_user_set(), h.quality, h.blend_mode, h.layer_type); + let _ = (h.is_null_object(), h.collapses_transformations(), h.quality, h.blend_mode, h.layer_type); let _ = ( h.is_enabled(), h.is_video_enabled(), @@ -208,10 +206,8 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye ); } let _ = (layer.is_enabled(), layer.is_audio_enabled(), layer.is_effects_enabled()); - let _ = (layer.camera_position().map(|p| p.reported(comp_size.0, comp_size.1, true)), layer.camera_point_of_interest(), layer.light_intensity()); + let _ = layer.camera_position().map(|p| (p.stored, p.auto_state)); let _ = project.layer_anchor_px(layer); - let _ = (layer.camera_zoom(), layer.camera_depth_of_field(), layer.camera_focus_distance(), layer.camera_aperture()); - let _ = (layer.light_color(), layer.light_cone_angle(), layer.light_cone_feather()); let _ = layer .time_remap() .map(|t| (t.enabled, t.keys.iter().fold(0.0, |acc, k| acc + k.value_seconds + k.time_seconds.unwrap_or(0.0) + f64::from(k.time_units)))); @@ -238,7 +234,8 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye } } } - let _ = (project.layer_source(layer).map(|i| i.id()), project.layer_display_name(layer), project.layer_source_size(layer)); + let _ = + (project.layer_source(layer).map(|i| i.id()), project.layer_source(layer).and_then(|i| i.display_name()), project.layer_source_size(layer)); if let Some(t) = layer.transform(comp_size.0, comp_size.1) { let _ = (t.anchor_px(f64::from(comp_size.0), f64::from(comp_size.1)), t.position.value, t.opacity_percent.value); } @@ -271,7 +268,6 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye } for m in layer.masks() { let _ = (m.name.clone(), m.mode, m.inverted, m.counter, m.feather, m.opacity_percent, m.expansion, m.path_animated, m.path_key_seconds.len()); - let _ = project.mask_vertices_px(layer, &m).map(|shapes| shapes.iter().map(Vec::len).sum::()); for s in &m.shapes { let _ = (s.vertex_count(), s.vertices((1920.0, 1080.0)).len(), s.vertices((f64::NAN, f64::INFINITY)).len(), s.open); } @@ -302,10 +298,9 @@ fn exercise_comp(project: &Project<'_, '_>, comp: &Comp<'_, '_>) { let mut size = (1920, 1080); if let Some(h) = comp.header() { size = (u32::from(h.width), u32::from(h.height)); - let _ = (h.frame_rate(), h.frame_rate_from_fixed(), h.duration_seconds(), h.work_area_start_seconds(), h.work_area_end_seconds()); - let _ = (h.frame_rate_exact(), h.duration_frames()); - let _ = (h.background_rgb(), h.pixel_aspect_ratio(), h.pixel_aspect.reduced(), h.shutter_angle, h.motion_blur); - let _ = (h.shutter_phase, h.display_start, h.display_start_seconds(), h.display_start_frame()); + let _ = (h.frame_rate(), h.frame_rate_fixed, h.duration_seconds(), h.work_area_start_units, h.work_area_end_units); + let _ = (h.background, h.pixel_aspect.to_f64(), h.pixel_aspect.reduced(), h.shutter_angle, h.motion_blur); + let _ = (h.shutter_phase, h.display_start.map(|r| r.to_f64())); } for layer in comp.layers() { exercise_layer(project, size, &layer); @@ -316,10 +311,8 @@ fn exercise_comp(project: &Project<'_, '_>, comp: &Comp<'_, '_>) { fn exercise(file: &AepFile<'_>) { let Some(project) = Project::new(file) else { return }; for item in project.all_items() { - let _ = (item.kind(), item.name(), item.id(), item.label(), item.used_item_ids(), item.footage(), item.display_name()); - let _ = item - .footage() - .map(|f| (f.duration, f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate, f.is_sequence, f.sequence_frame_count())); + let _ = (item.kind(), item.name(), item.id(), item.label(), item.footage(), item.display_name()); + let _ = item.footage().map(|f| (f.duration, f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate, f.is_sequence)); let _ = item.id().and_then(|id| project.item_by_id(id)); if let Some(comp) = item.comp() { exercise_comp(&project, &comp); @@ -572,12 +565,12 @@ fn extreme_values_in_the_decoded_headers() { use craft_aep::{CompHeader, LayerHeader, Rational, ShapePath}; // Every field at its maximum: no overflow, no division by zero, no panic. let h = LayerHeader::parse(&[0xff; 164]).unwrap(); - let _ = (h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds(), h.start_seconds(), h.stretch_percent()); + let _ = (h.in_point(), h.out_point(), h.start.to_f64(), h.stretch_ratio()); let mut zero_den = [0u8; 164]; zero_den[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); zero_den[12..16].copy_from_slice(&i32::MIN.to_be_bytes()); let h = LayerHeader::parse(&zero_den).unwrap(); - assert_eq!((h.in_point(), h.out_point(), h.start_seconds()), (None, None, None)); + assert_eq!((h.in_point(), h.out_point(), h.start.to_f64()), (None, None, None)); let mut wide = [0u8; 164]; wide[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); wide[12..16].copy_from_slice(&i32::MIN.to_be_bytes()); @@ -585,14 +578,12 @@ fn extreme_values_in_the_decoded_headers() { wide[20..24].copy_from_slice(&i32::MAX.to_be_bytes()); wide[24..28].copy_from_slice(&u32::MAX.to_be_bytes()); let h = LayerHeader::parse(&wide).unwrap(); - let _ = (h.in_point(), h.in_point_seconds()); + let _ = h.in_point(); let c = CompHeader::parse(&[0xff; 204]).unwrap(); - assert_eq!((c.duration_seconds(), c.work_area_start_seconds(), c.work_area_end_units), (None, None, None)); - let _ = (c.frame_rate(), c.frame_rate_from_fixed(), c.background_rgb(), c.pixel_aspect_ratio(), c.frame_rate_exact()); - assert_eq!(c.duration_frames(), Some(1)); + assert_eq!((c.duration_seconds(), c.work_area_end_units), (None, None)); + let _ = (c.frame_rate(), c.pixel_aspect.to_f64()); let zero = CompHeader::parse(&[0; 204]).unwrap(); - assert_eq!((zero.frame_rate(), zero.pixel_aspect_ratio(), zero.duration_seconds()), (None, None, None)); - assert_eq!((zero.frame_rate_exact(), zero.duration_frames()), (None, None)); + assert_eq!((zero.frame_rate(), zero.pixel_aspect.to_f64(), zero.duration_seconds()), (None, None, None)); let all = LayerHeader::parse(&[0xff; 164]).unwrap(); let _ = (all.is_enabled(), all.is_auto_oriented(), all.light_type(), all.blend_mode.code()); assert_eq!(LayerHeader::parse(&[0u8; 164]).unwrap().light_type(), None); @@ -691,7 +682,7 @@ fn hostile_text_dictionaries_and_marker_headers_never_panic() { #[test] fn extreme_values_in_the_batch_3_fields() { - use craft_aep::{CompHeader, Footage, LayerHeader, Rational}; + use craft_aep::{CompHeader, LayerHeader}; // Time stretch: zero and maximal numerators and denominators (bytes 8 and 108), with extreme times. for (num, den) in [(0u32, 0u32), (1, 0), (0, 1), (u32::MAX, 1), (1, u32::MAX), (u32::MAX, u32::MAX), (3, 2)] { let mut ldta = [0u8; 164]; @@ -704,7 +695,7 @@ fn extreme_values_in_the_batch_3_fields() { ldta[28..32].copy_from_slice(&i32::MIN.to_be_bytes()); ldta[32..36].copy_from_slice(&1u32.to_be_bytes()); let h = LayerHeader::parse(&ldta).unwrap(); - let _ = (h.stretch_ratio(), h.stretch_percent(), h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds()); + let _ = (h.stretch_ratio(), h.in_point(), h.out_point()); if den == 0 { assert_eq!((h.stretch_ratio(), h.in_point(), h.out_point()), (None, None, None)); } @@ -712,34 +703,8 @@ fn extreme_values_in_the_batch_3_fields() { // Display start and shutter phase from extreme bytes; a missing tail leaves them unknown. for fill in [0x00u8, 0x7f, 0x80, 0xff] { let c = CompHeader::parse(&[fill; 204]).unwrap(); - let _ = (c.shutter_phase, c.display_start, c.display_start_seconds(), c.display_start_frame()); + let _ = (c.shutter_phase, c.display_start.map(|r| r.to_f64())); } - let zero_rate = CompHeader::parse(&[0; 204]).unwrap(); - assert_eq!((zero_rate.display_start_seconds(), zero_rate.display_start_frame()), (None, None)); - // Footage durations: zero denominator, negative and maximal numerators, a huge frame rate. - let base = Footage { - file_type: String::new(), - width: 0, - height: 0, - name: None, - is_solid: false, - solid_argb: None, - path: None, - duration: None, - native_frame_rate: 0, - bits_per_pixel: 0, - file_size: 0, - audio_sample_rate: None, - is_sequence: true, - psd_layer: None, - }; - for (num, den, rate) in - [(3i64, 30i64, 30u32), (3, 0, 30), (-3, 30, 30), (i64::MAX, 1, u32::MAX), (i64::MIN, -1, u32::MAX), (1, i64::MAX, 1), (0, 1, 0)] - { - let f = Footage { duration: Some(Rational { num, den }), native_frame_rate: rate, ..base.clone() }; - let _ = f.sequence_frame_count(); - } - assert_eq!(Footage { duration: Some(Rational { num: 3, den: 30 }), native_frame_rate: 30, ..base.clone() }.sequence_frame_count(), Some(3)); - assert_eq!(Footage { duration: Some(Rational { num: 3, den: 0 }), native_frame_rate: 30, ..base.clone() }.sequence_frame_count(), None); - assert_eq!(Footage { duration: None, ..base }.sequence_frame_count(), None); + // A zero denominator has no value. + assert_eq!(CompHeader::parse(&[0; 204]).unwrap().display_start.and_then(|r| r.to_f64()), None); } diff --git a/crates/aep/tests/it/container/text_markers_footage.rs b/crates/aep/tests/it/container/text_markers_footage.rs index 6d666fbee..f6f81eed1 100644 --- a/crates/aep/tests/it/container/text_markers_footage.rs +++ b/crates/aep/tests/it/container/text_markers_footage.rs @@ -13,7 +13,6 @@ fn text_layer_document(name: &str) -> Option { let layer = comp.layers().find(|l| l.name().as_deref() == Some("T")).expect("the text layer T"); let h = layer.header().unwrap(); assert_eq!(h.layer_type, LayerType::Text); - assert!(h.name_is_user_set()); layer.text_document().expect("text document") }) } @@ -192,7 +191,6 @@ fn png_footage_layer() { // The path comes from the alias JSON (the fixture holds a neutral replacement of the original). assert!(f.path.as_deref().is_some_and(|p| p.ends_with("test8x8.png")), "{:?}", f.path); assert_eq!(project.layer_source_size(&layer), Some((8, 8))); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("png")); // L is still there below it. assert_eq!(comp.layers().count(), 2); } diff --git a/crates/aep/tests/it/container/tree_index.rs b/crates/aep/tests/it/container/tree_index.rs new file mode 100644 index 000000000..c1bed0163 --- /dev/null +++ b/crates/aep/tests/it/container/tree_index.rs @@ -0,0 +1,231 @@ +//! The flat tree index the importer reads ([`TreeIndex`]) against the owned chunk tree, on every +//! file of the fixture directory: the same tree, and the same answers from the typed view. + +use std::fmt::Write as _; + +use craft_aep::index::TreeIndex; +use craft_aep::{ + AepFile, Chunk, CryptomatteData, CurvesData, Effect, HueSaturationData, LevelsData, Limits, MeshData, Project, PropGroup, PropNode, is_aepx, + parse, parse_with_limits, read_aepx, write, +}; + +/// Every `.aep` and `.aepx` in the fixture directory (the real projects included), sorted. +fn every_file() -> Vec { + let Some(dir) = crate::common::dir() else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(&dir) else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let mut out: Vec = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aep" || x == "aepx")).collect(); + out.sort(); + out +} + +/// What the typed view reads from a project, as text: items, compositions, layers with their +/// headers, transforms, masks, effects with every parameter value, text, markers, the whole +/// property tree with static values and key records, settings and versions. +fn dump(view: &Project<'_, '_>) -> String { + let mut s = String::new(); + let defs = view.effect_definition_index(); + for item in view.all_items() { + let _ = writeln!(s, "item {:?} {:?} {:?} {:?} {:?}", item.kind(), item.name(), item.display_name(), item.id(), item.label()); + let _ = writeln!(s, " footage {:?} {:?} {:?}", item.footage(), item.sequence_info(), item.conform_frame_rate()); + let _ = writeln!(s, " folder {} {}", item.has_folder_contents(), item.folder_contents_unreadable()); + let Some(comp) = item.comp() else { continue }; + let _ = writeln!(s, " comp {:?} {:?} {:?}", comp.time_base(), comp.header(), comp.markers()); + for layer in comp.layers() { + let _ = writeln!(s, " layer {:?} {:?} {:?} {:?}", layer.name(), layer.id(), layer.header(), layer.track_matte()); + let _ = writeln!(s, " transform {:?}", layer.transform(1920, 1080)); + let _ = writeln!(s, " time remap {:?} orientation {:?} {:?}", layer.time_remap(), layer.orientation(), layer.orientation_keys()); + let _ = writeln!(s, " masks {:?}", layer.mask_atoms().iter().map(|m| (&m.mask, &m.path_shapes)).collect::>()); + let _ = writeln!(s, " shapes {:?}", layer.shape_groups()); + let _ = writeln!( + s, + " text {:?} {} {:?}", + layer.text_document(), + layer.text_document_count(), + layer.source_text_property().map(|p| p.is_animated()) + ); + let _ = writeln!(s, " markers {:?} source {:?}", layer.markers(), view.layer_source(&layer).map(|i| i.id())); + for fx in layer.effects() { + let _ = writeln!( + s, + " fx {} {:?} {:?} {:?} {:?}", + fx.match_name(), + fx.display_name(), + fx.enabled(), + defs.source(&fx), + fx.value_record_names() + ); + let _ = + writeln!(s, " names {:?} {:?} {:?} {:?}", fx.name(), fx.user_name(), fx.declared_param_count(), fx.has_own_definitions()); + for p in defs.params(&fx) { + let _ = writeln!(s, " {} {:?} {:?} {:?} {:?}", p.match_name, p.kind, p.name, p.items, p.value()); + arbitrary(&fx, &p.match_name, &mut s); + } + for name in fx.value_record_names() { + arbitrary(&fx, &name, &mut s); + } + } + if let Some(root) = layer.properties() { + tree(&root, 0, &mut s); + } + } + } + s +} + +/// The arbitrary data stored for a parameter and what each decoder reads from it. +fn arbitrary(fx: &Effect<'_, '_>, name: &str, s: &mut String) { + let Some(data) = fx.arbitrary_data(name) else { return }; + let _ = writeln!(s, " arbitrary {name} {data:?}"); + let _ = writeln!(s, " {:?} {:?}", HueSaturationData::parse(data), LevelsData::parse(data)); + let _ = writeln!(s, " {:?} {:?} {:?}", CurvesData::parse(data), MeshData::parse(data), CryptomatteData::parse(data)); +} + +/// Appends a group and everything below it to the dump, depth-limited. +fn tree(group: &PropGroup<'_, '_>, depth: usize, s: &mut String) { + if depth > 32 { + return; + } + let _ = writeln!(s, "{:depth$}group {:?} {:?}", "", group.user_name(), group.flags()); + for e in group.entries() { + match e.node { + PropNode::Group(g) => tree(&g, depth + 1, s), + PropNode::Property(p) => { + let keys: Option>> = p.keys().map(|k| k.records().map(<[u8]>::to_vec).collect()); + let _ = writeln!(s, "{:depth$} {} {:?} {:?} {:?} {:?}", "", e.match_name, p.user_name(), p.header(), p.static_value(), keys); + } + PropNode::Other(c) => { + let _ = writeln!(s, "{:depth$} {} other {:?} {}", "", e.match_name, c.list_type(), c.descendants().count()); + } + } + } +} + +/// The index is the parsed tree and the view reads the same on every file. +#[test] +fn the_index_is_the_parsed_tree_and_the_view_reads_the_same_on_every_file() { + for path in every_file() { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + // `.aepx` is read into the chunk tree and indexed from it; `.aep` is indexed in place. + // A file that does not read (a damaged sample) must fail the index the same way. + let file: AepFile<'_> = match if is_aepx(&bytes) { read_aepx(&bytes) } else { parse(&bytes) } { + Ok(file) => file, + Err(e) => { + if !is_aepx(&bytes) { + assert_eq!(TreeIndex::parse(&bytes).err(), Some(e), "{name}"); + } + continue; + } + }; + let index = (if is_aepx(&bytes) { TreeIndex::from_file(&file) } else { TreeIndex::parse(&bytes) }).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(index.to_file() == file, "{name}: the index is not the parsed tree"); + assert_eq!(index.len(), file.walk().count(), "{name}"); + assert_eq!(index.file_version(), file.file_version(), "{name}"); + assert_eq!(index.saved_by_version(), file.saved_by_version(), "{name}"); + assert_eq!(craft_aep::ProjectSettings::read_index(&index), craft_aep::ProjectSettings::read(&file), "{name}"); + let (Some(a), Some(b)) = (Project::new(&file), Project::from_index(&index)) else { + assert!(Project::new(&file).is_none() && Project::from_index(&index).is_none(), "{name}"); + continue; + }; + let (da, db) = (dump(&a), dump(&b)); + assert!(da == db, "{name}: the view reads differently through the index"); + } +} + +/// The index and the parser fail alike under tight limits. +#[test] +fn the_index_and_the_parser_fail_alike_under_tight_limits() { + for path in every_file().into_iter().filter(|p| p.extension().is_some_and(|x| x == "aep")) { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + let total = parse(&bytes).map(|f| f.walk().count()).unwrap_or(0); + for limits in [ + Limits { max_depth: 2, max_chunks: 10_000_000 }, + Limits { max_depth: 6, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: total }, + Limits { max_depth: 64, max_chunks: total.saturating_sub(1) }, + Limits { max_depth: 64, max_chunks: total / 2 }, + ] { + let old = parse_with_limits(&bytes, limits); + let new = TreeIndex::parse_with_limits(&bytes, limits); + match (old, new) { + (Ok(f), Ok(i)) => assert!(i.to_file() == f, "{name}: {limits:?}"), + (Err(a), Err(b)) => assert_eq!(a, b, "{name}: {limits:?}"), + (a, b) => panic!("{name}: {limits:?}: {:?} vs {:?}", a.err(), b.err()), + } + } + } +} + +/// A 40-byte NUL-padded match name. +fn tdmn(name: &str) -> Chunk<'static> { + let mut b = name.as_bytes().to_vec(); + b.resize(40, 0); + Chunk::data(*b"tdmn", b) +} + +/// A nested chunk `id` holding one `Utf8` with `text`. +fn utf8(id: [u8; 4], text: &str) -> Chunk<'static> { + Chunk { id, body: craft_aep::Body::Chunks(vec![Chunk::data(*b"Utf8", text.as_bytes().to_vec())]), pad: None } +} + +/// A project with one composition, one layer and one effect whose five parameters carry arbitrary +/// data in the sizes the decoders read (Hue/Saturation, Levels, Curves, a 2 x 2 mesh, Cryptomatte), +/// so the arbitrary-data path runs without the fixture directory. +fn synthetic() -> Vec { + let mut mesh = vec![0u8; 180]; + mesh[..8].copy_from_slice(&[0, 0, 0, 2, 0, 0, 0, 2]); + let mut cry = vec![0u8; 200]; + cry[..4].copy_from_slice(b"cry1"); + cry[32..37].copy_from_slice(b"Layer"); + let mut levels = vec![0u8; 108]; + levels[8..16].copy_from_slice(&1.0f64.to_be_bytes()); + let payloads: [Vec; 5] = [vec![7u8; 180], levels, vec![0u8; CurvesData::SIZE], mesh, cry]; + let (mut part, mut values) = + (vec![Chunk::data(*b"parn", 6u32.to_be_bytes().to_vec())], vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), utf8(*b"tdsn", "Mine")]); + for (i, data) in payloads.into_iter().enumerate() { + let name = format!("FX-{:04}", i + 1); + let mut pard = vec![0u8; 148]; + pard[12..16].copy_from_slice(&11u32.to_be_bytes()); + part.extend([tdmn(&name), Chunk::data(*b"pard", pard)]); + let record = Chunk::list(*b"tdbs", vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), Chunk::data(*b"cdat", vec![0, 0, 0, 1])]); + values.extend([tdmn(&name), record, Chunk::list(*b"aRbs", vec![Chunk::data(*b"aRbp", data)])]); + } + values.push(tdmn("ADBE Group End")); + let sspc = Chunk::list(*b"sspc", vec![utf8(*b"fnam", "Effect"), Chunk::list(*b"parT", part), Chunk::list(*b"tdgp", values)]); + let parade = Chunk::list(*b"tdgp", vec![tdmn("FX"), sspc, tdmn("ADBE Group End")]); + let root = Chunk::list(*b"tdgp", vec![tdmn("ADBE Effect Parade"), parade, tdmn("ADBE Group End")]); + let layer = Chunk::list(*b"Layr", vec![Chunk::data(*b"ldta", vec![0u8; 160]), Chunk::data(*b"Utf8", b"Layer".to_vec()), root]); + let mut idta = vec![0u8; 64]; + idta[..2].copy_from_slice(&4u16.to_be_bytes()); + idta[16..20].copy_from_slice(&2u32.to_be_bytes()); + let mut cdta = vec![0u8; 200]; + cdta[4..12].copy_from_slice(&[0, 0, 0, 1, 0, 0, 0, 30]); + let item = Chunk::list(*b"Item", vec![Chunk::data(*b"idta", idta), Chunk::data(*b"Utf8", b"Comp".to_vec()), Chunk::data(*b"cdta", cdta), layer]); + let file = + AepFile { chunks: vec![Chunk::data(*b"head", vec![0u8; 20]), Chunk::list(*b"Fold", vec![item])], trailer: std::borrow::Cow::Borrowed(&[]) }; + write(&file).expect("write") +} + +/// The view reads arbitrary effect data the same through the index. +#[test] +fn the_view_reads_arbitrary_effect_data_the_same_through_the_index() { + let bytes = synthetic(); + let file = parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let (a, b) = (Project::new(&file).expect("project"), Project::from_index(&index).expect("project")); + let (da, db) = (dump(&a), dump(&b)); + assert_eq!(da, db); + // The synthetic effect really reaches every decoder (the dump is not empty of them). + assert_eq!(da.matches("arbitrary FX-").count(), 10, "{da}"); + assert!(da.contains("Some(HueSaturationData") && da.contains("Some(LevelsData") && da.contains("Some(CurvesData"), "{da}"); + assert!(da.contains("Some(MeshData") && da.contains("Some(CryptomatteData"), "{da}"); + assert!(da.contains("Some(\"Mine\")"), "the user name is read: {da}"); +} diff --git a/crates/aep/tests/it/container/typed.rs b/crates/aep/tests/it/container/typed.rs index 7299043e9..9be97f3fd 100644 --- a/crates/aep/tests/it/container/typed.rs +++ b/crates/aep/tests/it/container/typed.rs @@ -111,7 +111,7 @@ fn match_names_come_from_the_tdmn_chunks() { assert!(!names.iter().any(|n| n == "ADBE Group End")); assert!(root.find(&["ADBE Transform Group", "ADBE Nonexistent"]).is_none()); // The raw tdmn body is 40 bytes, NUL padded. - let raw = root.chunk().children().iter().find(|c| c.is(b"tdmn")).unwrap(); + let raw = root.chunk().children().find(|c| c.is(b"tdmn")).unwrap(); assert_eq!(raw.bytes().unwrap().len(), 40); } @@ -150,8 +150,8 @@ fn effect_parameter_value() { assert!(instance.is_list(b"sspc")); // The effect's name is in a nested fnam chunk. let fnam = instance.child(b"fnam").unwrap(); - assert!(matches!(fnam.body, Body::Chunks(_))); - assert_eq!(fnam.children()[0].bytes(), Some(&b"Slider Control"[..])); + assert!(matches!(fnam.to_chunk().body, Body::Chunks(_))); + assert_eq!(fnam.children().next().unwrap().bytes(), Some(&b"Slider Control"[..])); // The parameters are a property group inside the instance. let params = PropGroup::from_chunk(instance.child_list(b"tdgp").unwrap()).unwrap(); assert_eq!(params.names(), ["ADBE Slider Control-0000", "ADBE Slider Control-0001", "ADBE Effect Built In Params"]); @@ -186,7 +186,7 @@ fn position_keyframes() { assert_eq!(times, [0, 25_600, 64_000]); let seconds: Vec = times.iter().map(|t| tb.seconds(*t).unwrap()).collect(); assert_eq!(seconds, [0.0, 1.0, 2.5]); - let values: Vec<[f64; 3]> = keys.keyframes().map(|k| k.spatial_value().unwrap()).collect(); + let values: Vec> = keys.keyframes().map(|k| k.value().unwrap()).collect(); assert_eq!(values, [[0.0, 0.0, 0.0], [100.0, 0.0, 0.0], [200.0, 100.0, 0.0]]); } diff --git a/crates/aep/tests/it/container/values.rs b/crates/aep/tests/it/container/values.rs index 17fccf62f..6c3616012 100644 --- a/crates/aep/tests/it/container/values.rs +++ b/crates/aep/tests/it/container/values.rs @@ -119,7 +119,7 @@ fn linear_position_keys() { let values: Vec<_> = keys.iter().map(|k| k.spatial().unwrap().value).collect(); assert_eq!(values, [[0.0, 0.0, 0.0], [100.0, 0.0, 0.0], [200.0, 100.0, 0.0]]); assert_eq!(p.first_key_value(), Some(vec![0.0, 0.0, 0.0])); - assert_eq!(p.is_static_flag_set(), Some(false)); + assert!(p.is_animated()); })); // The transform of the layer reports the animated Position as keyframed (value at the first key). let (t, _) = skip!(transform("055-key-position-3key-inserted-t2.5.aep")); @@ -181,7 +181,7 @@ fn roving_and_label_flags() { assert_eq!(roving, [false, true, false]); assert_eq!(keys[1].time_units(), Some(26509)); assert_eq!(keys[1].seconds(25600), Some(1.0355078125)); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); assert_eq!(keys[1].flags(), Some(0x0027)); assert_eq!(keys[1].label(), Some(0)); })); @@ -193,7 +193,7 @@ fn roving_and_label_flags() { assert_eq!(keys[1].flags(), Some(0x0247)); assert_eq!(keys[0].flags(), Some(0x0007)); assert_eq!(keys[1].is_roving(), Some(false)); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); })); } @@ -204,7 +204,6 @@ fn scalar_opacity_keys() { assert!(p.is_animated()); assert_eq!(p.dims(), Some(1)); assert_eq!(p.time_scale(), Some(25600)); - assert_eq!(p.is_static_flag_set(), Some(false)); let list = p.keys().unwrap(); assert_eq!(list.header.record_size, 48); let keys: Vec<_> = list.keyframes().collect(); @@ -243,10 +242,8 @@ fn expression_text_and_flag_bytes() { assert_eq!(p.expression(), None); let flags = p.expression_flags().unwrap(); assert_eq!((flags.byte_76, flags.byte_77, flags.byte_120), (0, 0, 0)); - assert!(!flags.all_set()); // A written static property: tdsb 1, byte 5 of tdb4 has bit 0 set (15 for Position). assert_eq!(p.flags(), Some(1)); - assert_eq!(p.is_static_flag_set(), Some(true)); assert_eq!(p.header().and_then(|h| h.get(5).copied()), Some(15)); })); } diff --git a/crates/audio/src/aac/mod.rs b/crates/audio/src/aac/mod.rs index eb1c7d40a..34b081ea6 100644 --- a/crates/audio/src/aac/mod.rs +++ b/crates/audio/src/aac/mod.rs @@ -113,6 +113,26 @@ pub fn wav_order(channel_config: u8) -> Option<&'static [usize]> { }) } +/// The channel configuration whose speakers a stream has: its own, or for configuration 0 the +/// configuration 3–6 that its program config element spells out. A program config element lists +/// front, side, back and LFE elements in that order, so one that opens with a centre and a front +/// pair and goes on with exactly the elements of a configuration has that configuration's speakers +/// (and [`wav_order`]). Any other list has no speaker order that can be told, and gives 0: ffmpeg's +/// 5.1(side), for one, lists the pair first and already decodes in WAV order. +#[cfg(feature = "aac-decode")] +pub fn speaker_config(asc: &AudioSpecificConfig) -> u8 { + if asc.channel_config != 0 { + return asc.channel_config; + } + let centre_then_pair = asc.pce.as_ref().is_some_and(|p| p.front.iter().map(|e| e.0).eq([false, true])); + if !centre_then_pair { + return 0; + } + let elements = asc.layout(); + let same = |config: &u8| config_layout(*config).is_some_and(|l| l.iter().map(|e| e.0).eq(elements.iter().map(|e| e.0))); + (3..=6).find(same).unwrap_or(0) +} + /// The order the [`Encoder`] takes its input in, from WAV order: for each encoder input channel /// (AAC element order, see [`EncoderConfig::channels`]), the WAV-order channel to feed it. The /// inverse of [`wav_order`] for 3 to 7 channels; `None` for mono and stereo (already in order) and @@ -279,6 +299,30 @@ mod tests { mod order_tests { use super::*; + /// A program config element in the shape of a channel configuration has that configuration's + /// speakers, with the centre first. ffmpeg's 5.1(side) lists its pair first and already decodes + /// in WAV order, so it stays as it is. + #[cfg(feature = "aac-decode")] + #[test] + fn a_program_config_element_takes_the_configuration_it_matches() { + let with = |pce| AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let centre_first = ProgramConfig { front: vec![(false, 0), (true, 0)], back: vec![(true, 1)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(centre_first)), 6); + let three = ProgramConfig { front: vec![(false, 0), (true, 0)], ..Default::default() }; + assert_eq!(speaker_config(&with(three)), 3); + let ffmpeg = ProgramConfig { front: vec![(true, 0), (false, 0)], side: vec![(false, 1)], back: vec![(true, 1)], ..Default::default() }; + assert_eq!(speaker_config(&with(ffmpeg)), 0); + // the same element types with no front pair: the side pair is not left and right + let no_front_pair = + ProgramConfig { front: vec![(false, 0)], side: vec![(true, 0)], back: vec![(true, 1)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(no_front_pair)), 0); + // 7.1 in a program config element is not one of the unambiguous configurations + let wide = + ProgramConfig { front: vec![(false, 0), (true, 0)], side: vec![(true, 1)], back: vec![(true, 2)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(wide)), 0); + assert_eq!(speaker_config(&AudioSpecificConfig::lc(3, 6)), 6); + } + /// `aac_order` undoes `wav_order` for each channel count the encoder maps to a configuration /// (3–6 channels: configurations 3–6; 7: configuration 11). #[test] diff --git a/crates/audio/src/track.rs b/crates/audio/src/track.rs index 3d9ed7dfb..af984d718 100644 --- a/crates/audio/src/track.rs +++ b/crates/audio/src/track.rs @@ -156,7 +156,8 @@ impl TrackDecoder { let codec = if up.is_some() { CodecId::HeAac } else { CodecId::Aac }; // Only twice the core rate is upsampled to; any other signalled rate plays the core. let out = if up.is_some() { out } else { dec.sample_rate() }; - let cfg = dec.config().channel_config; + // a program config element that spells out a configuration has its speakers + let cfg = aac::speaker_config(dec.config()); let layout = aac_layout(cfg, dec.channels()); let order = layout.is_ordered().then(|| aac::wav_order(cfg)).flatten(); (Inner::Aac { dec: Box::new(dec), up, order, adts_format }, codec, out, layout, SampleFormat::Compressed) @@ -1079,6 +1080,8 @@ mod tests { let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); assert_eq!(dec.channels(), 3); assert_eq!(dec.decode(&frame).unwrap().channels.len(), 3); + // its element lists a centre and a front pair, as configuration 3 does: L R C, in WAV order + assert_eq!(dec.layout(), ChannelLayout::SURROUND_3_0); // no program config element where one is needed let bare = aac::AdtsHeader::write(2, 3, 0, 1, 0x7FF).unwrap(); let bare = [&bare[..], &[0xE0]].concat(); diff --git a/crates/audio/tests/codecs/routing_oracle.rs b/crates/audio/tests/codecs/routing_oracle.rs index c35a89002..6472979b3 100644 --- a/crates/audio/tests/codecs/routing_oracle.rs +++ b/crates/audio/tests/codecs/routing_oracle.rs @@ -669,10 +669,14 @@ fn adts_with_a_program_config_element_decodes() { assert_eq!(bytes[3] >> 6 | (bytes[2] & 1) << 2, 0, "the fixture signals its layout in a PCE"); let (info, buf) = decode(&bytes, None).unwrap(); assert_eq!((info.container, info.codec, buf.channels.len()), (c.container, c.codec, c.channels)); - assert_eq!(info.layout, c.layout); + // A program config element that lists a centre, a front pair and then the elements of 5.1 has + // 5.1's speakers, in WAV order like ffmpeg's decode; any other list keeps the decoded order, + // which ffmpeg versions write differently. + let ordered = info.layout.is_ordered(); + assert_eq!(info.layout, if ordered { ChannelLayout::SURROUND_5_1_SIDE } else { c.layout }); assert_eq!(buf.frames(), reference(&ff, &path).len() / c.channels); - // A program config's channels come in its own order, and which order ffmpeg writes differs by - // version: each of ours must match a different one of ffmpeg's. + // Each of our channels must match a different one of ffmpeg's: the same one when the layout is + // ordered. let refr = reference(&ff, &path); let mut matched = Vec::new(); for plane in &buf.channels { @@ -686,7 +690,9 @@ fn adts_with_a_program_config_element_decodes() { assert!(s >= c.min_snr, "channel {} matches ffmpeg's {best} at only {s:.1} dB", matched.len()); matched.push(best); } - matched.sort_unstable(); + if !ordered { + matched.sort_unstable(); + } assert_eq!(matched, (0..c.channels).collect::>()); } diff --git a/crates/cms/README.md b/crates/cms/README.md index 55bbc6b82..71f3146d5 100644 --- a/crates/cms/README.md +++ b/crates/cms/README.md @@ -30,6 +30,8 @@ Buffers are checked once per call, before any pixel is written. * `Transform::proof(src, proof, display, intent, bpc, simulate_paper)`: soft-proof chain. * `GamutCheck::new(src, proof, ΔE)`: out-of-gamut test (gamut warning). + `check.mask_rgba(&rgba, &mut mask)?` fills a whole mask (255 where alpha > 0 and out of gamut) on + the thread pool and returns the count; it needs an RGB source profile. * `Lut3d::from_transform(&t, 33)?.to_rgba16f_bytes()`: display transform as a 3D texture; `Lut3d::from_fn(size, f)` samples any RGB function. * `Transform::gpu_matrix_shaper()`: accurate RGB input curves + matrix + output curves for diff --git a/crates/cms/src/gamut.rs b/crates/cms/src/gamut.rs index 247b183f2..11dd17732 100644 --- a/crates/cms/src/gamut.rs +++ b/crates/cms/src/gamut.rs @@ -2,13 +2,18 @@ use crate::clut::Clut; use crate::pipeline::{Pipeline, Stage}; -use crate::profile::Profile; +use crate::profile::{ColorSpace, Profile}; use crate::transform::link_stages; -use crate::{CmsError, Intent, check_len}; +use crate::{CmsError, Intent, check_len, invalid_arg}; /// Default ΔE76 threshold above which a colour counts as out of gamut. pub const DEFAULT_THRESHOLD: f32 = 4.0; +#[cfg(not(target_arch = "wasm32"))] +const PAR_MIN_PIXELS: usize = 16 * 1024; +#[cfg(not(target_arch = "wasm32"))] +const PAR_CHUNK_PIXELS: usize = 16 * 1024; + /// Precomputed out-of-gamut test for colours of `src` against `proof`. /// /// A colour is out of gamut when sending it to the proof device (relative colorimetric, @@ -17,6 +22,7 @@ pub const DEFAULT_THRESHOLD: f32 = 4.0; #[derive(Debug)] pub struct GamutCheck { inputs: usize, + source_space: ColorSpace, grid: Option, exact: (Pipeline, Pipeline), pub threshold: f32, @@ -44,7 +50,7 @@ impl GamutCheck { 4 => Some(Clut::sample(vec![17; 4], 1, de)?), _ => None, }; - Ok(GamutCheck { inputs: n, grid, exact: (direct, via), threshold }) + Ok(GamutCheck { inputs: n, source_space: src.color_space, grid, exact: (direct, via), threshold }) } pub fn inputs(&self) -> usize { @@ -76,4 +82,84 @@ impl GamutCheck { pub fn out_of_gamut(&self, px: &[f32]) -> Result { Ok(self.delta_e(px)? > self.threshold) } + + /// Out-of-gamut mask for RGBA pixels: `255` where the pixel has alpha > 0 and its RGB + /// channels are out of gamut, `0` elsewhere. Returns the number of flagged pixels. Only + /// `min(px.len(), mask.len())` pixels are processed. Runs on the rayon pool (not on wasm); + /// the result is identical to calling [`Self::out_of_gamut`] per pixel. + /// + /// # Errors + /// [`CmsError::InvalidArgument`] when the check was not built from an RGB source profile: an + /// RGBA pixel carries only RGB colour channels, so it cannot be tested against a CMYK, gray + /// or Lab source. Nothing is written to `mask` in that case. + pub fn mask_rgba(&self, px: &[[f32; 4]], mask: &mut [u8]) -> Result { + if self.source_space != ColorSpace::Rgb || self.inputs != 3 { + return Err(invalid_arg(format!("gamut mask needs an RGB source profile, got {:?} with {} channels", self.source_space, self.inputs))); + } + let n = px.len().min(mask.len()); + let (Some(px), Some(mask)) = (px.get(..n), mask.get_mut(..n)) else { return Ok(0) }; + let run = |p: &[[f32; 4]], m: &mut [u8]| { + let mut count = 0; + for (p, m) in p.iter().zip(m.iter_mut()) { + // The source has three channels (checked above), so the check cannot fail. + let hit = p[3] > 0.0 && self.out_of_gamut(&[p[0], p[1], p[2]]).unwrap_or(false); + *m = if hit { 255 } else { 0 }; + count += usize::from(hit); + } + count + }; + #[cfg(not(target_arch = "wasm32"))] + if n >= PAR_MIN_PIXELS { + use rayon::prelude::*; + return Ok(px.par_chunks(PAR_CHUNK_PIXELS).zip(mask.par_chunks_mut(PAR_CHUNK_PIXELS)).map(|(p, m)| run(p, m)).sum()); + } + Ok(run(px, mask)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Builtin; + + fn image(n: usize) -> Vec<[f32; 4]> { + let mut s = 0x9E37_79B9u32; + let mut rnd = move || { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s >> 8) as f32 / (1u32 << 24) as f32 + }; + // Includes out-of-range values and zero alpha. + (0..n).map(|i| [rnd() * 1.2 - 0.1, rnd(), rnd(), if i % 7 == 0 { 0.0 } else { rnd() }]).collect() + } + + #[test] + fn mask_rgba_matches_per_pixel_check() { + let src = Builtin::Srgb.profile(); + // Sizes straddle the parallel threshold and the chunk size, odd ones included. + for (proof, threshold) in [(Builtin::CoatedCmyk, 4.0), (Builtin::CoatedCmyk, 1.5), (Builtin::GrayGamma22, 4.0), (Builtin::DisplayP3, 0.5)] { + let check = GamutCheck::new(src, proof.profile(), threshold).unwrap(); + for n in [0, 1, 1023, 16 * 1024 - 1, 16 * 1024, 16 * 1024 + 1, 50_001] { + let px = image(n); + let want: Vec = px.iter().map(|p| if p[3] > 0.0 && check.out_of_gamut(&p[..3]).unwrap() { 255 } else { 0 }).collect(); + let mut got = vec![7u8; n]; + let count = check.mask_rgba(&px, &mut got).unwrap(); + assert_eq!(got, want, "{proof:?} threshold {threshold} n {n}"); + assert_eq!(count, want.iter().filter(|&&m| m == 255).count()); + } + } + } + + #[test] + fn mask_rgba_rejects_non_rgb_source() { + // A CMYK (4-channel), gray (1-channel) and Lab (3-channel, not RGB) source cannot be tested from RGBA pixels. + for src in [Builtin::CoatedCmyk, Builtin::GrayGamma22, Builtin::LabD50] { + let check = GamutCheck::new(src.profile(), Builtin::Srgb.profile(), 4.0).unwrap(); + let px = image(64); + let mut mask = vec![9u8; 64]; + assert!(matches!(check.mask_rgba(&px, &mut mask), Err(CmsError::InvalidArgument(_))), "{src:?}"); + assert!(mask.iter().all(|&m| m == 9), "mask must stay untouched"); + } + } } diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index c2261325d..404b5a315 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -135,7 +135,8 @@ pub fn parse_as(format: Format, bytes: &[u8]) -> Result { } } -/// UTF-16 with a byte-order mark (Photoshop loads such `.look` files, #9118), else UTF-8. +/// UTF-16 with a byte-order mark (Photoshop loads such `.look` files, #9118), else UTF-8 +/// (a UTF-8 BOM is stripped). fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { let utf16 = |rest: &[u8], from: fn([u8; 2]) -> u16| { let units: Vec = rest.as_chunks::<2>().0.iter().map(|&c| from(c)).collect(); @@ -144,6 +145,7 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { match bytes { [0xFF, 0xFE, rest @ ..] => utf16(rest, u16::from_le_bytes), [0xFE, 0xFF, rest @ ..] => utf16(rest, u16::from_be_bytes), + [0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest), _ => String::from_utf8_lossy(bytes), } } @@ -155,14 +157,19 @@ pub fn parse_cube(text: &str) -> Result { let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]); let mut rows: Vec = Vec::new(); for line in text.lines() { - // A '#' inside a quoted TITLE is part of the title, not a comment. - let raw = line.trim_start(); - let skip = if raw.starts_with("TITLE") { raw.find('"').and_then(|q| raw[q + 1..].find('"').map(|e| q + e + 2)) } else { None }.unwrap_or(0); - let line = match raw[skip..].find('#') { - Some(i) => &raw[..skip + i], - None => raw, + // Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an + // unquoted title's comment must not turn that comment into part of the title. + if let Some(quoted) = line + .trim_start() + .strip_prefix("TITLE") + .filter(|rest| rest.starts_with(char::is_whitespace)) + .and_then(|rest| rest.trim_start().strip_prefix('"')) + { + let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; + title = value.to_string(); + continue; } - .trim(); + let line = line.split('#').next().unwrap_or("").trim(); if line.is_empty() { continue; } @@ -187,7 +194,9 @@ pub fn parse_cube(text: &str) -> Result { } k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => { let mut v = vec![k.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?]; - v.extend(it.filter_map(|s| s.parse::().ok())); + for value in it { + v.push(value.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?); + } if v.len() != 3 { return err(format!("bad data line `{line}`")); } @@ -209,6 +218,15 @@ pub fn parse_cube(text: &str) -> Result { if rows.len() != expected_rows { return err("no LUT_3D_SIZE in .cube file"); } + // #8913: the resample below bakes the curve onto a fixed 33-point grid, so a steep + // curve over a narrow domain cannot survive the resample — the baked table silently + // shifts colours (measured 42 RGB8 levels for a two-node ramp over 0.24..0.26, where + // Photoshop renders the ramp exactly). Rather than proxy the risk by domain width (a + // 0.1..0.17 ramp spans two grid cells yet still loses ~28 levels), measure the + // resample's actual error against the exact curve and refuse only tables whose baked + // output shifts by more than a couple of RGB8 levels. Default-domain tables keep the + // existing bake untouched; gentle curves over any domain bake within tolerance and + // stay accepted. // The 1D table is resampled onto a 0..1 cube anyway, so its input domain // (`DOMAIN_MIN`/`MAX`, `LUT_1D_INPUT_RANGE`) is baked into the resample: an input `v` // reads the table at `(v - min) / (max - min)`, saturating outside. @@ -221,6 +239,36 @@ pub fn parse_cube(text: &str) -> Result { }; let mut l = LutFile::from_fn(&title, 33, |c| [curve(0, c[0]), curve(1, c[1]), curve(2, c[2])])?; l.title = title; + let non_default = dmin.iter().any(|v| *v != 0.0) || dmax.iter().any(|v| *v != 1.0); + if non_default { + // Worst baked-vs-exact error per channel, in RGB8 levels: the baked cube is read + // by linear interpolation between grid points (red fastest, so channel `ch` sits + // at stride 33^ch), while `curve` is what the file actually says. + const MAX_BAKE_ERROR: f32 = 2.0 / 255.0; + for ch in 0..3 { + let stride = [1usize, 33, 33 * 33][ch]; + let mut worst = 0.0f32; + for k in 0..=256 { + let v = k as f32 / 256.0; + let x = v * 32.0; + let i = (x as usize).min(31); + let f = x - i as f32; + let baked = l.data[i * stride * 3 + ch] * (1.0 - f) + l.data[(i + 1).min(32) * stride * 3 + ch] * f; + worst = worst.max((baked - curve(ch, v)).abs()); + } + if worst > MAX_BAKE_ERROR { + return err(format!( + "1D LUT domain on {} is {min}..{max}: the curve changes faster than the 33-point grid can \ + hold (baking shifts colours by up to {levels:.0} RGB8 levels); export the look as a 3D LUT \ + or widen DOMAIN_MIN/DOMAIN_MAX", + ["red", "green", "blue"][ch], + min = dmin[ch], + max = dmax[ch], + levels = worst * 255.0, + )); + } + } + } return l.check(); } err("no LUT_3D_SIZE in .cube file") @@ -554,6 +602,48 @@ pub fn builtin(id: &str) -> Option { mod tests { use super::*; + #[test] + fn cube_title_hashes_and_comment_quotes_are_distinct() { + let rows = "LUT_1D_SIZE 2\n0 0 0 # low\n1 1 1 # high\n"; + for (line, title) in [ + (r#"TITLE "Look #3" # comment "ignored""#, "Look #3"), + (r#"TITLE Look # "draft""#, "Look"), + (" TITLE\t\"Été #2\"", "Été #2"), + (r#"TITLE "" # empty"#, ""), + ] { + assert_eq!(parse_cube(&format!("{line}\n{rows}")).unwrap().title, title); + } + let lut = LutFile::from_fn("Look #3", 2, |c| c).unwrap(); + assert_eq!(parse_cube(&write_cube(&lut)).unwrap().title, "Look #3"); + assert!(parse_cube(&format!("TITLE \"unterminated # title\n{rows}")).is_err()); + } + + #[test] + fn cube_rejects_invalid_numeric_fields() { + let text = "LUT_3D_SIZE 2\n0.25 nope 0.5 0.75\n0 0 1\n0 1 0\n0 1 1\n1 0 0\n1 0 1\n1 1 0\n1 1 1\n"; + assert!(parse_cube(text).is_err(), "invalid numeric field must not be ignored"); + } + + #[test] + fn utf8_bom_is_stripped() { + let l = LutFile::identity(3).unwrap(); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(write_cube(&l).as_bytes()); + let p = parse("a.cube", &bytes).unwrap(); + assert_eq!(p.size, l.size); + assert_eq!(p.title, l.title); + let titled = LutFile::from_fn("Look", 3, |c| c).unwrap(); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(write_cube(&titled).as_bytes()); + assert_eq!(parse("a.cube", &bytes).unwrap().title, "Look"); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(b"LUT_3D_SIZE 2\n"); + for i in 0..8 { + bytes.extend_from_slice(format!("{} {} {}\n", i & 1, (i >> 1) & 1, i >> 2).as_bytes()); + } + assert_eq!(parse("a.cube", &bytes).unwrap().size, 2); + } + /// An identity `.3dl` of edge `n` (12-bit nodes, blue fastest) under `shaper`. fn identity_3dl(shaper: &[u32], n: usize) -> String { let mut t = shaper.iter().map(u32::to_string).collect::>().join(" ") + "\n"; @@ -811,6 +901,49 @@ DOMAIN_MAX 0.5 1 1 assert!(e.0.contains("below DOMAIN_MAX"), "{e}"); } + #[test] + fn cube_1d_narrow_domain_is_refused() { + // #8913: the issue's fixture — a two-node ramp over 0.24..0.26. Baking it onto the + // 33-point grid rendered grey-65 as 148 where Photoshop renders 190 (42-level error). + // It must now be an actionable parse error rather than a silently wrong table. + let e = parse_cube("LUT_1D_SIZE 2\nDOMAIN_MIN 0.24 0.24 0.24\nDOMAIN_MAX 0.26 0.26 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster than the 33-point grid"), "{e}"); + // The same domain via LUT_1D_INPUT_RANGE is refused identically. + let e = parse_cube("LUT_1D_SIZE 2\nLUT_1D_INPUT_RANGE 0.24 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster"), "{e}"); + // A per-channel narrow domain names the offending channel. + let e = parse_cube("LUT_1D_SIZE 2\nDOMAIN_MIN 0 0 0.24\nDOMAIN_MAX 1 1 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("blue"), "{e}"); + } + + #[test] + fn cube_1d_wide_domain_still_bakes() { + // A steep ramp over 0.1..0.17 spans two grid cells but still loses ~28 RGB8 levels + // to the resample, so the measured-error check refuses it even though the old width + // check let it through. + let e = parse_cube("LUT_1D_SIZE 2\nLUT_1D_INPUT_RANGE 0.1 0.17\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster"), "steep ramp over 0.1..0.17 refused: {e}"); + // A gentle (here linear) curve over a grid-aligned domain bakes exactly and stays + // accepted — the check measures the bake's error, not the domain's width. Value + // assertions, not just a successful parse: the baked table must reproduce the curve. + // Table row k sits at t = k/32 of the domain 0.25..0.75; the curve y = 0.4 + 0.4t + // is gentle (a half-level per grid step), so the bake must reproduce it exactly. + let mut table = String::from("LUT_1D_SIZE 33\nLUT_1D_INPUT_RANGE 0.25 0.75\n"); + for k in 0..33 { + let y = 0.4 + 0.4 * (k as f32 / 32.0); + table.push_str(&format!("{y} {y} {y}\n")); + } + let l = parse_cube(&table).expect("gentle curve over a non-default domain still bakes"); + let at = |r: usize| l.data[r * 3]; + // Grid entry r is input r/32: 0.5 is the middle of the domain (t=0.5) -> 0.6. + assert!((at(16) - 0.6).abs() < 1.0 / 255.0, "baked value at 0.5 tracks the curve: {}", at(16)); + // Domain top (input 0.75 = grid 24, t=1) -> 0.8. + assert!((at(24) - 0.8).abs() < 1.0 / 255.0, "baked value at the domain top tracks the curve: {}", at(24)); + // Below the domain saturates to the first row. + assert!((at(8) - 0.4).abs() < 1.0 / 255.0, "below the domain saturates: {}", at(8)); + parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1\n").expect("default domain"); + } + #[test] fn cube_1d_domain_is_baked_into_the_resample() { // An identity ramp over the domain 0.25..0.75: 0.25 maps to 0, 0.5 to 0.5, 0.75 to 1. @@ -1042,11 +1175,14 @@ DOMAIN_MAX 0.5 1 1 } #[test] - fn look_hex_floats() { - let id = LutFile::identity(2).unwrap(); - let hex: String = id.data.iter().flat_map(|v| v.to_le_bytes()).map(|b| format!("{b:02X}")).collect(); + fn look_data_is_little_endian_float32() { + // Bytes written out by hand rather than by the encoder the parser mirrors, so a wrong byte + // order fails: 0.25, 0.5, 1.0 (3E800000, 3F000000, 3F800000 stored low byte first), then zeros. + let hex = format!("0000803E 0000003F 0000803F {}", "00000000 ".repeat(21)); let xml = format!("\"2\"\"{hex}\""); - assert_eq!(parse("a.look", xml.as_bytes()).unwrap().data, id.data); + let data = parse("a.look", xml.as_bytes()).unwrap().data; + assert_eq!(data[..3], [0.25, 0.5, 1.0]); + assert!(data[3..].iter().all(|&v| v == 0.0)); } #[test] diff --git a/crates/drawingml/src/lib.rs b/crates/drawingml/src/lib.rs index 47d6ee107..1e47301dc 100644 --- a/crates/drawingml/src/lib.rs +++ b/crates/drawingml/src/lib.rs @@ -11,7 +11,7 @@ //! - [`warp`]: preset text warps (`a:prstTxWarp`). //! - The model, without readers and writers yet: [`style`] (colour references, fills, lines, //! effects), [`text`] (text bodies, paragraphs, runs), [`table`], [`chart`] with [`numfmt`] -//! (number format codes) and [`diagram`] (SmartArt outlines). Media are named by [`MediaId`]; +//! (number format codes), [`theme`] (colour, font and format schemes) and [`diagram`] (SmartArt outlines). Media are named by [`MediaId`]; //! the bytes stay with the caller. //! //! Nothing here opens a package: readers take a [`craft_xml::El`] and writers a @@ -27,6 +27,7 @@ pub mod preset; pub mod style; pub mod table; pub mod text; +pub mod theme; mod trendline; pub mod warp; diff --git a/crates/drawingml/src/theme.rs b/crates/drawingml/src/theme.rs new file mode 100644 index 000000000..d4416e2f1 --- /dev/null +++ b/crates/drawingml/src/theme.rs @@ -0,0 +1,183 @@ +//! Themes: colour scheme, font scheme and format scheme (the fill/line/effect "style matrix"). + +use crate::color::{ColorScheme, ColorTransform, Rgba, SchemeSlot}; +use serde::{Deserialize, Serialize}; + +use crate::style::{ColorRef, Effects, Fill, Gradient, GradientShape, GradientStop, Line, Shadow}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FontSet { + pub latin: String, + pub ea: String, + pub cs: String, + /// Per-script typefaces (`a:font script="Arab" typeface="…"`), in document order. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub scripts: Vec, +} + +/// A theme font for one script (ISO 15924 tag, e.g. `Arab`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ScriptFont { + pub script: String, + pub typeface: String, +} + +impl FontSet { + /// The typeface the theme gives `script` (`"Arab"`), if any. + pub fn script(&self, script: &str) -> Option<&str> { + self.scripts.iter().find(|f| f.script.eq_ignore_ascii_case(script) && !f.typeface.is_empty()).map(|f| f.typeface.as_str()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FontScheme { + pub name: String, + /// Headings. + pub major: FontSet, + /// Body. + pub minor: FontSet, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FormatScheme { + pub name: String, + /// Subtle, moderate, intense (style index 1–3). + pub fills: Vec, + pub lines: Vec, + pub effects: Vec, + /// Background fills (index 1001–1003). + pub bg_fills: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Theme { + pub name: String, + pub colors: ColorScheme, + pub fonts: FontScheme, + pub format: FormatScheme, + /// Raw theme XML extras kept for round-trip (object defaults, extra colour lists). + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_extra: Option, +} + +impl Default for Theme { + /// A plain theme: black and white, a slate and a warm grey, six accents, no font names (the + /// writer puts `Arial` where a Latin face is required) and [`default_format`]. + fn default() -> Self { + let hex = |s: &str| Rgba::from_hex(s).unwrap_or(Rgba::BLACK); + let colors = ["000000", "FFFFFF", "1E2B3C", "EDEBE7", "2E6FD8", "F26B1D", "2EA36F", "8A5CD8", "D83F6B", "E3B21C", "2E6FD8", "7A5BA6"]; + Theme { + name: "Theme".into(), + colors: ColorScheme { name: "Default".into(), colors: colors.map(hex) }, + fonts: FontScheme::default(), + format: default_format(), + raw_extra: None, + } + } +} + +impl Theme { + /// Resolve a theme font reference (`+mj-lt`, `+mn-ea`…) to a family name. + pub fn font(&self, name: &str) -> String { + match name { + "+mj-lt" => self.fonts.major.latin.clone(), + "+mn-lt" => self.fonts.minor.latin.clone(), + "+mj-ea" => self.fonts.major.ea.clone(), + "+mn-ea" => self.fonts.minor.ea.clone(), + "+mj-cs" => self.fonts.major.cs.clone(), + "+mn-cs" => self.fonts.minor.cs.clone(), + other => other.to_string(), + } + } +} + +fn ph() -> ColorRef { + ColorRef::scheme(SchemeSlot::PhClr) +} + +/// The format scheme used where a theme has none, and to fill a short style list: flat solid fills, a soft gradient, and a gentle shadow for "intense". +pub fn default_format() -> FormatScheme { + let grad = |a: i32, b: i32| { + Fill::Gradient(Gradient { + tile: [0.0; 4], + stops: vec![ + GradientStop { pos: 0.0, color: ph().with(ColorTransform::LumMod(a)).with(ColorTransform::SatMod(103000)) }, + GradientStop { pos: 1.0, color: ph().with(ColorTransform::LumMod(b)) }, + ], + shape: GradientShape::Linear { angle: 90.0, scaled: false }, + rotate_with_shape: true, + flip: None, + }) + }; + FormatScheme { + name: "Default".into(), + fills: vec![Fill::solid(ph()), grad(110000, 92000), grad(105000, 80000)], + lines: vec![Line::solid(ph(), 0.75), Line::solid(ph(), 1.0), Line::solid(ph(), 1.5)], + effects: vec![ + Effects::default(), + Effects::default(), + Effects { + outer_shadow: Some(Shadow { + color: ColorRef::rgb(Rgba::BLACK).with(ColorTransform::Alpha(40000)), + blur: 6.0, + dist: 3.0, + dir: 90.0, + inner: false, + sx: 1.0, + sy: 1.0, + kx: 0.0, + ky: 0.0, + align: "t".into(), + rotate_with_shape: false, + }), + ..Default::default() + }, + ], + bg_fills: vec![ + Fill::solid(ph()), + Fill::solid(ph().with(ColorTransform::Tint(95000)).with(ColorTransform::SatMod(170000))), + grad(102000, 85000), + ], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_default_theme_has_a_full_style_matrix_and_no_font_names() { + let t = Theme::default(); + assert_eq!((t.format.fills.len(), t.format.lines.len(), t.format.effects.len(), t.format.bg_fills.len()), (3, 3, 3, 3)); + assert_eq!(t.colors.get(SchemeSlot::Lt1), Rgba::WHITE); + assert_eq!(t.fonts, FontScheme::default()); + } + + #[test] + fn theme_font_references_resolve_and_other_names_pass_through() { + let mut t = Theme::default(); + t.fonts.major.latin = "Head".into(); + t.fonts.minor.cs = "Body CS".into(); + t.fonts.minor.scripts.push(ScriptFont { script: "Arab".into(), typeface: "Naskh".into() }); + assert_eq!(t.font("+mj-lt"), "Head"); + assert_eq!(t.font("+mn-cs"), "Body CS"); + assert_eq!(t.font("Georgia"), "Georgia"); + assert_eq!(t.fonts.minor.script("arab"), Some("Naskh")); + assert_eq!(t.fonts.minor.script("Hebr"), None); + } + + #[test] + fn a_theme_survives_json_and_missing_fields_take_the_default() { + let t = Theme::default(); + let json = serde_json::to_string(&t).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), t); + let partial: Theme = serde_json::from_str(r#"{"name":"X"}"#).unwrap(); + assert_eq!(partial.colors, t.colors); + assert_eq!(partial.name, "X"); + } +} diff --git a/crates/dxf/src/model.rs b/crates/dxf/src/model.rs index 55fc1848c..667024a9e 100644 --- a/crates/dxf/src/model.rs +++ b/crates/dxf/src/model.rs @@ -3,8 +3,8 @@ //! [`Document::read`] indexes a tag list (from [`crate::parse`]) into its header variables, //! symbol tables, blocks, entities and objects. Nothing is copied: every record borrows its tags, //! so a caller can still read a group this module gives no name to (its own extended data, a -//! newer group). The typed forms ([`Layer`], [`Linetype`], [`TextStyle`], [`Layout`]...) are made -//! on request. +//! newer group). The typed forms ([`Layer`], [`Linetype`], [`TextStyle`], [`Layout`], and +//! [`Entity`] for an entity record) are made on request. //! //! The values are the file's: points in the coordinate system the format stores them in (the //! object coordinate system for planar entities, see [`ocs_to_wcs`]), angles in the unit the @@ -18,10 +18,18 @@ use crate::Tag; pub mod aci; mod document; +mod entity; +mod hatch; mod tables; +pub mod text; pub use document::{Block, Dictionary, Document, EntityRecord, Header, Layout, Record, entity_records, records}; +pub use entity::{ + Attrib, Common, Dimension, Entity, EntityKind, Image, Insert, Leader, LwPolyline, LwVertex, Polyline, Spline, Text, Vertex, Viewport, +}; +pub use hatch::{Gradient, Hatch, HatchEdge, HatchLoop, HatchPath, PatternLine}; pub use tables::{AppId, BlockRecord, DashElement, DimStyle, Layer, Linetype, TextStyle, Ucs, View, Vport, dim_var_code, dim_var_name}; +pub use text::{MText, MTextColumns}; /// A point or vector: x, y, z. pub type Point = [f64; 3]; diff --git a/crates/dxf/src/model/entity.rs b/crates/dxf/src/model/entity.rs new file mode 100644 index 000000000..6b05f2748 --- /dev/null +++ b/crates/dxf/src/model/entity.rs @@ -0,0 +1,669 @@ +//! Entities, typed: the properties every entity has ([`Common`]) and the geometry of each type. +//! +//! Values are as the file stores them. Planar entities (CIRCLE, ARC, LWPOLYLINE, 2D POLYLINE, +//! TEXT, INSERT, SOLID, TRACE, HATCH) have their points in the object coordinate system of +//! [`Common::extrusion`] ([`super::ocs_to_wcs`]); LINE, POINT, ELLIPSE, SPLINE, RAY, XLINE, +//! 3DFACE, MTEXT, LEADER and DIMENSION definition points are world coordinates. Angles are in +//! degrees, except an ELLIPSE's parameters, which the format stores in radians. + +use super::document::{EntityRecord, Record}; +use super::hatch::Hatch; +use super::text::{MText, decode_line, split_embedded}; +use super::{Color, Groups, Lineweight, Point, Transparency, outside_app, parse_handle}; +use crate::Tag; + +/// The properties every entity has. +#[derive(Clone, Debug, PartialEq)] +pub struct Common { + pub handle: Option, + /// Group 330: the block record (or other object) that owns the entity. + pub owner: Option, + /// Group 8 (`0` when absent). + pub layer: String, + /// Group 6; `None` is ByLayer. + pub linetype: Option, + pub color: Color, + pub lineweight: Lineweight, + /// Group 48 (1 when absent or not positive). + pub ltscale: f64, + /// Group 60 is not 1. + pub visible: bool, + /// Group 67 is 1. + pub paper_space: bool, + pub transparency: Transparency, + /// Group 39. + pub thickness: f64, + /// Groups 210/220/230 (the world Z axis when absent). + pub extrusion: Point, + /// Group 390: the plot style name object. + pub plot_style: Option, +} + +/// The groups of the AcDbEntity subclass: the record up to the entity's own subclass marker (all +/// of it in files without markers, and in records with no AcDbEntity marker, whose layer and +/// colour may follow the entity's marker), or up to an embedded object (101). Later groups reuse +/// the codes: an MTEXT's column width is 48. Markers inside an application-defined group do not +/// count. +fn entity_groups(tags: &[Tag]) -> &[Tag] { + let is_entity = |t: &Tag| t.str_ref().is_some_and(|s| s == "AcDbEntity"); + let marked = outside_app(tags).any(|(_, t)| t.code == 100 && is_entity(t)); + let end = outside_app(tags).find(|(_, t)| (marked && t.code == 100 && !is_entity(t)) || t.code == 101).map_or(tags.len(), |(i, _)| i); + tags.get(..end).unwrap_or(tags) +} + +impl Common { + pub fn read(tags: &[Tag]) -> Common { + let (all, e) = (Groups(tags), Groups(entity_groups(tags))); + Common { + handle: e.handle(5), + owner: outside_app(tags).find(|(_, t)| t.code == 330).and_then(|(_, t)| parse_handle(&t.str())), + layer: e.name(8).unwrap_or_else(|| "0".into()), + linetype: e.name(6).filter(|l| !l.eq_ignore_ascii_case("ByLayer")), + color: Color::of(&e), + lineweight: e.i(370).map_or(Lineweight::ByLayer, Lineweight::from_dxf), + ltscale: e.f(48).filter(|s| *s > 0.0).unwrap_or(1.0), + visible: e.i(60) != Some(1), + paper_space: e.i(67) == Some(1), + transparency: e.i(440).map_or(Transparency::ByLayer, Transparency::from_dxf), + thickness: all.f_or(39, 0.0), + extrusion: [all.f_or(210, 0.0), all.f_or(220, 0.0), all.f_or(230, 1.0)], + plot_style: e.handle(390).filter(|h| *h != 0), + } + } +} + +/// A vertex of an LWPOLYLINE. +#[derive(Clone, Copy, Debug, PartialEq, Default)] +pub struct LwVertex { + pub x: f64, + pub y: f64, + /// Group 42: the tangent of a quarter of the arc to the next vertex (0: straight). + pub bulge: f64, + pub start_width: f64, + pub end_width: f64, +} + +/// An LWPOLYLINE. +#[derive(Clone, Debug, PartialEq)] +pub struct LwPolyline { + pub vertices: Vec, + /// Flags (70) bit 1. + pub closed: bool, + /// Flags bit 128: the linetype runs around the vertices. + pub plinegen: bool, + /// Group 43. + pub const_width: f64, + /// Group 38. + pub elevation: f64, +} + +/// A VERTEX of a POLYLINE. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Vertex { + pub location: Point, + pub bulge: f64, + pub start_width: f64, + pub end_width: f64, + /// Group 70: 1 added by curve fitting, 8 added by spline fitting, 16 a spline frame control + /// point, 32 a 3D polyline vertex, 64 a mesh vertex, 128 a polyface vertex (128 without 64: + /// a face record). + pub flags: i64, + /// Groups 71–74: the vertex numbers of a polyface face record (1-based; negative: the edge + /// from it is invisible; 0: none). + pub face: [i64; 4], +} + +impl Vertex { + /// A face record of a polyface mesh, which holds vertex numbers and no point. + pub fn is_face(&self) -> bool { + self.flags & 128 != 0 && self.flags & 64 == 0 + } + /// A control point of the frame a spline-fit polyline was made from: not on the curve. + pub fn is_spline_frame(&self) -> bool { + self.flags & 16 != 0 + } +} + +/// A POLYLINE with its VERTEX records: a 2D or 3D polyline, a polygon mesh or a polyface mesh. +#[derive(Clone, Debug, PartialEq)] +pub struct Polyline { + /// Group 70: 1 closed (a mesh: closed in M), 2 curve-fit, 4 spline-fit, 8 a 3D polyline, 16 a + /// polygon mesh, 32 a mesh closed in N, 64 a polyface mesh, 128 the linetype runs around the + /// vertices. + pub flags: i64, + /// The z of the POLYLINE's own point: a 2D polyline's elevation. + pub elevation: f64, + /// Groups 40 and 41: the default widths. + pub start_width: f64, + pub end_width: f64, + /// Groups 71 and 72: a mesh's M and N vertex counts (a polyface mesh: vertices and faces). + pub m: i64, + pub n: i64, + /// Group 75: the smooth surface type (5 quadratic, 6 cubic, 8 Bezier). + pub smooth: i64, + pub vertices: Vec, +} + +impl Polyline { + pub fn closed(&self) -> bool { + self.flags & 1 != 0 + } + pub fn is_3d(&self) -> bool { + self.flags & 8 != 0 + } + pub fn is_mesh(&self) -> bool { + self.flags & 16 != 0 + } + pub fn is_polyface(&self) -> bool { + self.flags & 64 != 0 + } +} + +/// A SPLINE (NURBS), in world coordinates. +#[derive(Clone, Debug, PartialEq)] +pub struct Spline { + /// Group 71 (3 when absent). + pub degree: i64, + /// Group 70: 1 closed, 2 periodic, 4 rational, 8 planar, 16 linear. + pub flags: i64, + pub knots: Vec, + pub control: Vec, + /// Group 41: one per control point when rational. + pub weights: Vec, + pub fit: Vec, + /// Groups 12/22/32 and 13/23/33. + pub start_tangent: Option, + pub end_tangent: Option, + /// Group 44. + pub fit_tolerance: f64, +} + +/// TEXT, and the text of an ATTRIB or ATTDEF. +#[derive(Clone, Debug, PartialEq)] +pub struct Text { + /// Group 10: the first alignment point. + pub insert: Point, + /// Group 11: the second alignment point, which places text that is not left-baseline. + pub align: Option, + pub height: f64, + /// The string, decoded ([`decode_line`]); `%%` codes are left as written. + pub value: String, + /// Group 50, in degrees. + pub rotation: f64, + /// Group 41 (1 when absent). + pub width_factor: f64, + /// Group 51, in degrees. + pub oblique: f64, + /// Group 7 (`Standard` when absent). + pub style: String, + /// Group 71: 2 backwards, 4 upside down. + pub generation: i64, + /// Group 72: 0 left, 1 centre, 2 right, 3 aligned, 4 middle, 5 fit. + pub halign: i64, + /// Group 73 (74 in an ATTRIB or ATTDEF): 0 baseline, 1 bottom, 2 middle, 3 top. + pub valign: i64, +} + +impl Text { + pub fn read(g: &Groups<'_>) -> Text { + Text { + insert: g.point(10), + align: g.point_opt(11), + height: g.f_or(40, 0.0), + value: decode_line(&g.s(1).unwrap_or_default()), + rotation: g.f_or(50, 0.0), + width_factor: g.f_or(41, 1.0), + oblique: g.f_or(51, 0.0), + style: g.name(7).unwrap_or_else(|| "Standard".into()), + generation: g.i_or(71, 0), + halign: g.i_or(72, 0), + valign: g.i(73).or_else(|| g.i(74)).unwrap_or(0), + } + } +} + +/// An ATTRIB (an attribute of a block reference) or ATTDEF (its definition in the block). +#[derive(Clone, Debug, PartialEq)] +pub struct Attrib { + /// Group 2. + pub tag: String, + pub text: Text, + /// Group 3 of an ATTDEF, decoded; empty in an ATTRIB. + pub prompt: String, + /// Group 70: 1 invisible, 2 constant, 4 verify, 8 preset. + pub flags: i64, + /// The 280 after the tag in the attribute's subclass (one before the tag is the subclass + /// version). + pub lock_position: bool, + /// The embedded MTEXT of a multiline attribute. + pub multiline: Option, +} + +impl Attrib { + pub fn read(kind: &str, tags: &[Tag]) -> Attrib { + // A multiline attribute has its text as embedded MTEXT after its own groups. + let (own, embedded) = split_embedded(tags); + let g = Groups(own); + let sub = own.iter().position(|t| t.code == 100 && t.str_ref().is_some_and(|s| s.trim().starts_with("AcDbAttribute"))).unwrap_or(0); + let rest = own.get(sub..).unwrap_or_default(); + let lock_position = rest.iter().position(|t| t.code == 2).is_some_and(|at| rest.iter().skip(at + 1).any(|t| t.code == 280 && t.i64() != 0)); + Attrib { + tag: g.s(2).unwrap_or_default(), + text: Text::read(&g), + prompt: if kind == "ATTDEF" { decode_line(&g.s(3).unwrap_or_default()) } else { String::new() }, + flags: g.i_or(70, 0), + lock_position, + multiline: embedded.filter(|e| !e.is_empty()).map(MText::read), + } + } +} + +/// An INSERT: a block reference, with its attributes. +#[derive(Clone, Debug, PartialEq)] +pub struct Insert { + /// Group 2. + pub block: String, + pub insert: Point, + /// Groups 41, 42 and 43 (1 when absent). + pub scale: Point, + /// Group 50, in degrees. + pub rotation: f64, + /// Groups 70 and 71 (1 when absent or less). + pub columns: i64, + pub rows: i64, + /// Groups 44 and 45. + pub column_spacing: f64, + pub row_spacing: f64, + /// The ATTRIB records that follow, each with its own properties. + pub attribs: Vec<(Common, Attrib)>, +} + +/// A DIMENSION. The geometry it is drawn with is in its block ([`Dimension::block`]). +#[derive(Clone, Debug, PartialEq)] +pub struct Dimension { + /// Group 70: the type in the low bits (0 rotated, 1 aligned, 2 angular, 3 diameter, 4 radius, + /// 5 angular 3-point, 6 ordinate), 32 the block is this dimension's alone, 64 an ordinate's + /// X type, 128 the text is at a user position. + pub flags: i64, + /// Group 2: the block that holds the dimension's picture. + pub block: Option, + /// Group 3 (`Standard` when absent). + pub style: String, + /// Group 10: the definition point. + pub defpt: Point, + /// Group 11: the middle of the text, in the object coordinate system. + pub text_mid: Point, + /// Group 12: the insertion point for clones (baseline and continue). + pub insertion: Point, + /// Groups 13 to 16: the definition points of the type. + pub p13: Point, + pub p14: Point, + pub p15: Point, + pub p16: Point, + /// Group 1, decoded: empty or `<>` for the measurement, a space for none. + pub text: String, + /// Group 42: the measurement, when the file states it. + pub measurement: Option, + /// Group 50: the angle of a rotated dimension, in degrees. + pub rotation: f64, + /// Group 52: the oblique angle of the extension lines, in degrees. + pub oblique: f64, + /// Group 53: the text's rotation, in degrees. + pub text_rotation: f64, + /// Group 51: the horizontal direction, in degrees. + pub horizontal: f64, + /// Group 40: the leader length of a radius or diameter dimension. + pub leader_length: f64, +} + +impl Dimension { + /// The type: the low four bits of the flags. + pub fn kind(&self) -> i64 { + self.flags & 0x0f + } +} + +/// A LEADER. +#[derive(Clone, Debug, PartialEq)] +pub struct Leader { + pub vertices: Vec, + /// Group 71 is not 0. + pub arrow: bool, + /// Group 72 is not 0: a spline path. + pub spline: bool, + /// Group 3 (`Standard` when absent). + pub style: String, + /// Group 73: what the leader was made with (0 text, 1 a tolerance, 2 a block, 3 nothing). + pub annotation_type: i64, + /// Group 340: the annotation. + pub annotation: Option, +} + +/// A paper space VIEWPORT. +#[derive(Clone, Debug, PartialEq)] +pub struct Viewport { + /// Group 10: the centre on the sheet. + pub center: Point, + /// Groups 40 and 41: the size on the sheet. + pub width: f64, + pub height: f64, + /// Group 69: the id (1 is the sheet's own view). Other programs write -1 for unknown. + pub id: i64, + /// Group 90: the status flags (16384 locked, 65536 clipped by the entity of group 340, + /// 131072 off). + pub status: i64, + /// Groups 12/22: the view centre in display coordinates, whose origin is the target. + pub view_center: [f64; 2], + /// Groups 16/26/36 and 17/27/37. + pub view_direction: Point, + pub view_target: Point, + /// Group 45. + pub view_height: f64, + /// Group 51: the twist, in degrees. + pub twist: f64, + /// Group 340: the entity that clips the viewport. + pub clip: Option, + /// Groups 331 and 341: the layers frozen in the viewport. + pub frozen_layers: Vec, +} + +/// An IMAGE or WIPEOUT: a raster's place and clip. The file is named by the IMAGEDEF object. +#[derive(Clone, Debug, PartialEq)] +pub struct Image { + /// Group 10: the lower-left corner. + pub insert: Point, + /// Groups 11 and 12: the vectors of one pixel along the image's width and height. + pub u: Point, + pub v: Point, + /// Groups 13/23: the size in pixels. + pub size: [f64; 2], + /// Group 340: the IMAGEDEF. + pub image_def: Option, + /// Group 70: 1 show, 2 show when not aligned with the screen, 4 use the clip, 8 transparency. + pub display: i64, + /// Group 280 is not 0. + pub clipping: bool, + /// Groups 281, 282 and 283 (50, 50 and 0 when absent). + pub brightness: i64, + pub contrast: i64, + pub fade: i64, + /// Group 71: 1 a rectangle (two corners), 2 a polygon. + pub clip_type: i64, + /// Groups 14/24: the clip boundary in pixels, with the origin at the image's top-left corner + /// and y down: a point `q` is at `insert + u * (q.x + 0.5) + v * (0.5 - q.y)`. + pub clip: Vec<[f64; 2]>, + /// Group 290 is not 0: the clip keeps the outside. + pub clip_inverted: bool, +} + +/// The geometry of an entity. +#[derive(Clone, Debug, PartialEq)] +pub enum EntityKind { + Line { + start: Point, + end: Point, + }, + /// `angle` is group 50: the X axis of the point's symbol, in degrees. + Point { + location: Point, + angle: f64, + }, + Circle { + center: Point, + radius: f64, + }, + Arc { + center: Point, + radius: f64, + start_angle: f64, + end_angle: f64, + }, + /// `major` is the end of the major axis from the centre; `start` and `end` are parameters in + /// radians (0 and 2π for a full ellipse). + Ellipse { + center: Point, + major: Point, + ratio: f64, + start: f64, + end: f64, + }, + LwPolyline(LwPolyline), + Polyline(Polyline), + Spline(Spline), + Ray { + base: Point, + direction: Point, + }, + XLine { + base: Point, + direction: Point, + }, + Text(Text), + MText(MText), + AttDef(Attrib), + Insert(Insert), + Dimension(Dimension), + Leader(Leader), + /// The fourth corner repeats the third when the file has three. + Solid { + corners: [Point; 4], + }, + Trace { + corners: [Point; 4], + }, + /// `hidden_edges` is group 70: bit n hides edge n. + Face3d { + corners: [Point; 4], + hidden_edges: i64, + }, + Hatch(Hatch), + Viewport(Viewport), + Image(Image), + Wipeout(Image), + /// `text` is the feature control frame's string, as written. + Tolerance { + insert: Point, + direction: Point, + style: String, + text: String, + }, + /// A type this module does not model (or a HATCH with no boundary data): read it from the + /// record's tags. + Other, +} + +/// An entity: its properties and geometry. +#[derive(Clone, Debug, PartialEq)] +pub struct Entity { + pub common: Common, + pub kind: EntityKind, +} + +fn lwpolyline(tags: &[Tag], g: &Groups<'_>) -> LwPolyline { + let mut vertices: Vec = Vec::new(); + for t in tags { + if t.code == 10 { + vertices.push(LwVertex { x: t.f64(), ..LwVertex::default() }); + } else if let Some(v) = vertices.last_mut() { + match t.code { + 20 => v.y = t.f64(), + 40 => v.start_width = t.f64(), + 41 => v.end_width = t.f64(), + 42 => v.bulge = t.f64(), + _ => {} + } + } + } + let flags = g.i_or(70, 0); + LwPolyline { vertices, closed: flags & 1 != 0, plinegen: flags & 128 != 0, const_width: g.f_or(43, 0.0), elevation: g.f_or(38, 0.0) } +} + +fn polyline(g: &Groups<'_>, followers: &[Record<'_>]) -> Polyline { + let vertices = followers + .iter() + .map(|r| { + let v = r.groups(); + Vertex { + location: v.point(10), + bulge: v.f_or(42, 0.0), + start_width: v.f_or(40, 0.0), + end_width: v.f_or(41, 0.0), + flags: v.i_or(70, 0), + face: [71, 72, 73, 74].map(|c| v.i_or(c, 0)), + } + }) + .collect(); + Polyline { + flags: g.i_or(70, 0), + elevation: g.f_or(30, 0.0), + start_width: g.f_or(40, 0.0), + end_width: g.f_or(41, 0.0), + m: g.i_or(71, 0), + n: g.i_or(72, 0), + smooth: g.i_or(75, 0), + vertices, + } +} + +fn spline(g: &Groups<'_>) -> Spline { + Spline { + degree: g.i_or(71, 3), + flags: g.i_or(70, 0), + knots: g.all_f(40), + control: g.points(10), + weights: g.all_f(41), + fit: g.points(11), + start_tangent: g.point_opt(12), + end_tangent: g.point_opt(13), + fit_tolerance: g.f_or(44, 0.0), + } +} + +fn insert(g: &Groups<'_>, followers: &[Record<'_>]) -> Insert { + Insert { + block: g.s(2).unwrap_or_default(), + insert: g.point(10), + scale: [g.f_or(41, 1.0), g.f_or(42, 1.0), g.f_or(43, 1.0)], + rotation: g.f_or(50, 0.0), + columns: g.i_or(70, 1).max(1), + rows: g.i_or(71, 1).max(1), + column_spacing: g.f_or(44, 0.0), + row_spacing: g.f_or(45, 0.0), + attribs: followers.iter().map(|r| (Common::read(r.tags), Attrib::read("ATTRIB", r.tags))).collect(), + } +} + +fn dimension(g: &Groups<'_>) -> Dimension { + Dimension { + flags: g.i_or(70, 0), + block: g.name(2), + style: g.name(3).unwrap_or_else(|| "Standard".into()), + defpt: g.point(10), + text_mid: g.point(11), + insertion: g.point(12), + p13: g.point(13), + p14: g.point(14), + p15: g.point(15), + p16: g.point(16), + text: decode_line(&g.s(1).unwrap_or_default()), + measurement: g.f(42), + rotation: g.f_or(50, 0.0), + oblique: g.f_or(52, 0.0), + text_rotation: g.f_or(53, 0.0), + horizontal: g.f_or(51, 0.0), + leader_length: g.f_or(40, 0.0), + } +} + +fn viewport(tags: &[Tag], g: &Groups<'_>) -> Viewport { + Viewport { + center: g.point(10), + width: g.f_or(40, 0.0), + height: g.f_or(41, 0.0), + id: g.i_or(69, 0), + status: g.i_or(90, 0), + view_center: [g.f_or(12, 0.0), g.f_or(22, 0.0)], + view_direction: g.point_opt(16).unwrap_or([0.0, 0.0, 1.0]), + view_target: g.point(17), + view_height: g.f_or(45, 0.0), + twist: g.f_or(51, 0.0), + clip: g.handle(340).filter(|h| *h != 0), + frozen_layers: tags.iter().filter(|t| matches!(t.code, 331 | 341)).filter_map(|t| parse_handle(&t.str())).collect(), + } +} + +fn image(g: &Groups<'_>) -> Image { + Image { + insert: g.point(10), + u: g.point(11), + v: g.point(12), + size: [g.f_or(13, 0.0), g.f_or(23, 0.0)], + image_def: g.handle(340).filter(|h| *h != 0), + display: g.i_or(70, 0), + clipping: g.i_or(280, 0) != 0, + brightness: g.i_or(281, 50), + contrast: g.i_or(282, 50), + fade: g.i_or(283, 0), + clip_type: g.i_or(71, 1), + clip: g.points(14).into_iter().map(|p| [p[0], p[1]]).collect(), + clip_inverted: g.i_or(290, 0) != 0, + } +} + +fn corners(g: &Groups<'_>) -> [Point; 4] { + [g.point(10), g.point(11), g.point(12), g.point_opt(13).unwrap_or_else(|| g.point(12))] +} + +impl Entity { + /// The typed form of an entity record. + pub fn read(e: &EntityRecord<'_>) -> Entity { + let g = e.groups(); + let line = |g: &Groups<'_>| (g.point(10), g.point(11)); + let kind = match e.kind { + "LINE" => EntityKind::Line { start: line(&g).0, end: line(&g).1 }, + "POINT" => EntityKind::Point { location: g.point(10), angle: g.f_or(50, 0.0) }, + "CIRCLE" => EntityKind::Circle { center: g.point(10), radius: g.f_or(40, 0.0) }, + "ARC" => EntityKind::Arc { center: g.point(10), radius: g.f_or(40, 0.0), start_angle: g.f_or(50, 0.0), end_angle: g.f_or(51, 360.0) }, + "ELLIPSE" => EntityKind::Ellipse { + center: g.point(10), + major: g.point(11), + ratio: g.f_or(40, 1.0), + start: g.f_or(41, 0.0), + end: g.f_or(42, std::f64::consts::TAU), + }, + "LWPOLYLINE" => EntityKind::LwPolyline(lwpolyline(e.tags, &g)), + "POLYLINE" => EntityKind::Polyline(polyline(&g, &e.followers)), + "SPLINE" => EntityKind::Spline(spline(&g)), + "RAY" => EntityKind::Ray { base: line(&g).0, direction: line(&g).1 }, + "XLINE" => EntityKind::XLine { base: line(&g).0, direction: line(&g).1 }, + "TEXT" => EntityKind::Text(Text::read(&g)), + "MTEXT" => EntityKind::MText(MText::read(e.tags)), + "ATTDEF" => EntityKind::AttDef(Attrib::read("ATTDEF", e.tags)), + "INSERT" => EntityKind::Insert(insert(&g, &e.followers)), + "DIMENSION" => EntityKind::Dimension(dimension(&g)), + "LEADER" => EntityKind::Leader(Leader { + vertices: g.points(10), + arrow: g.i_or(71, 1) != 0, + spline: g.i_or(72, 0) != 0, + style: g.name(3).unwrap_or_else(|| "Standard".into()), + annotation_type: g.i_or(73, 3), + annotation: g.handle(340).filter(|h| *h != 0), + }), + "SOLID" => EntityKind::Solid { corners: corners(&g) }, + "TRACE" => EntityKind::Trace { corners: corners(&g) }, + "3DFACE" => EntityKind::Face3d { corners: corners(&g), hidden_edges: g.i_or(70, 0) }, + "HATCH" => Hatch::read(e.tags).map_or(EntityKind::Other, EntityKind::Hatch), + "VIEWPORT" => EntityKind::Viewport(viewport(e.tags, &g)), + "IMAGE" => EntityKind::Image(image(&g)), + "WIPEOUT" => EntityKind::Wipeout(image(&g)), + "TOLERANCE" => EntityKind::Tolerance { + insert: g.point(10), + direction: g.point_opt(11).unwrap_or([1.0, 0.0, 0.0]), + style: g.name(3).unwrap_or_else(|| "Standard".into()), + text: g.s(1).unwrap_or_default(), + }, + _ => EntityKind::Other, + }; + Entity { common: Common::read(e.tags), kind } + } +} diff --git a/crates/dxf/src/model/hatch.rs b/crates/dxf/src/model/hatch.rs new file mode 100644 index 000000000..ac3425622 --- /dev/null +++ b/crates/dxf/src/model/hatch.rs @@ -0,0 +1,355 @@ +//! HATCH: the boundary loops (polylines or edges), the pattern definition, the seed points and +//! the gradient, as the file holds them. Boundary points are 2D, in the object coordinate system +//! of the hatch at its elevation. + +use super::{Color, Groups, parse_handle}; +use crate::Tag; + +/// One edge of a boundary loop made of edges. +#[derive(Clone, Debug, PartialEq)] +pub enum HatchEdge { + Line { + start: [f64; 2], + end: [f64; 2], + }, + /// Angles in degrees; a clockwise arc (`ccw` false) has them measured clockwise. + Arc { + center: [f64; 2], + radius: f64, + start_angle: f64, + end_angle: f64, + ccw: bool, + }, + /// `major` is the end of the major axis from the centre, `ratio` the minor axis's length + /// over the major's; angles in degrees, as for an arc. + Ellipse { + center: [f64; 2], + major: [f64; 2], + ratio: f64, + start_angle: f64, + end_angle: f64, + ccw: bool, + }, + Spline { + degree: i64, + rational: bool, + periodic: bool, + knots: Vec, + control: Vec<[f64; 2]>, + /// One per control point when rational; else empty. + weights: Vec, + fit: Vec<[f64; 2]>, + start_tangent: Option<[f64; 2]>, + end_tangent: Option<[f64; 2]>, + }, +} + +/// The path of a boundary loop. +#[derive(Clone, Debug, PartialEq)] +pub enum HatchPath { + /// Vertices as x, y and bulge. + Polyline { + closed: bool, + vertices: Vec<[f64; 3]>, + }, + Edges(Vec), +} + +/// A boundary loop. +#[derive(Clone, Debug, PartialEq)] +pub struct HatchLoop { + /// Group 92: 1 external, 2 a polyline, 4 derived, 8 a text box, 16 outermost. + pub flags: i64, + pub path: HatchPath, + /// Groups 330 after the loop's 97: the entities an associative hatch follows. + pub sources: Vec, +} + +/// One line family of a pattern definition, as drawn: scaled and rotated with the hatch. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct PatternLine { + /// Group 53, in degrees. + pub angle: f64, + /// Groups 43/44: a point the first line passes through. + pub base: [f64; 2], + /// Groups 45/46: the step from one line to the next, as a vector. + pub offset: [f64; 2], + /// Groups 49: dash (positive), gap (negative) and dot (0) lengths; empty for a solid line. + pub dashes: Vec, +} + +/// A gradient fill (groups 450–470). +#[derive(Clone, Debug, PartialEq)] +pub struct Gradient { + /// Group 470 (`LINEAR`, `CYLINDER`, `SPHERICAL`...). + pub name: String, + /// Group 460, in radians. + pub angle: f64, + /// Group 461: 0 centred, 1 shifted up and left. + pub shift: f64, + /// Group 452 is 1: one colour, shaded by `tint` (group 462). + pub single: bool, + pub tint: f64, + /// The colour of each 463 record: its true colour (421), else its index (63). + pub colors: Vec, +} + +/// A HATCH. +#[derive(Clone, Debug, PartialEq)] +pub struct Hatch { + /// Group 2, as written (`SOLID` when absent). + pub pattern: String, + /// Group 70 is 1. + pub solid: bool, + /// Group 71 is not 0. + pub associative: bool, + /// Group 75: 0 normal (odd parity), 1 outer, 2 ignore islands. + pub style: i64, + /// Group 76: 0 user-defined, 1 predefined, 2 custom. + pub pattern_type: i64, + /// Group 52, in degrees. + pub angle: f64, + /// Group 41 (1 when absent). + pub scale: f64, + /// Group 77 is not 0: a user-defined pattern is doubled (crossed). + pub double: bool, + /// The z of the elevation point (the first group 30). + pub elevation: f64, + pub loops: Vec, + pub pattern_lines: Vec, + /// Groups 10/20 after the 98 count. + pub seeds: Vec<[f64; 2]>, + pub gradient: Option, +} + +/// A cursor over a record's tags, for the boundary data, which is read in order. +struct Cur<'a> { + tags: &'a [Tag], + at: usize, +} + +impl<'a> Cur<'a> { + fn peek(&self) -> Option<&'a Tag> { + self.tags.get(self.at) + } + /// The next tag when its code is `code`, consumed. + fn take(&mut self, code: i32) -> Option<&'a Tag> { + let t = self.peek().filter(|t| t.code == code)?; + self.at += 1; + Some(t) + } + fn f(&mut self, code: i32) -> f64 { + self.take(code).map_or(0.0, Tag::f64) + } + fn xy(&mut self, code: i32) -> [f64; 2] { + [self.f(code), self.f(code + 10)] + } + /// Move to the next `code`; false when there is none. + fn seek(&mut self, code: i32) -> bool { + while self.peek().is_some_and(|t| t.code != code) { + self.at += 1; + } + self.peek().is_some() + } +} + +/// A declared count: never trusted for an allocation, only as the most items to read. +fn count(t: &Tag) -> usize { + usize::try_from(t.i64()).unwrap_or(0) +} + +fn polyline_path(c: &mut Cur<'_>) -> Option { + // 72 has bulges, 73 closed, 93 the vertex count. + let mut closed = false; + while let Some(t) = c.peek().filter(|t| t.code != 93) { + if t.code == 73 { + closed = t.i64() != 0; + } else if t.code != 72 { + return None; + } + c.at += 1; + } + let n = count(c.take(93)?); + let mut vertices = Vec::new(); + while vertices.len() < n && c.peek().is_some_and(|t| t.code == 10) { + let [x, y] = c.xy(10); + vertices.push([x, y, c.f(42)]); + } + Some(HatchPath::Polyline { closed, vertices }) +} + +fn spline_edge(c: &mut Cur<'_>) -> HatchEdge { + let (mut degree, mut rational, mut periodic, mut knot_count, mut control_count) = (3, false, false, 0, 0); + while let Some(t) = c.peek().filter(|t| matches!(t.code, 94 | 73 | 74 | 95 | 96)) { + match t.code { + 94 => degree = t.i64(), + 73 => rational = t.i64() != 0, + 74 => periodic = t.i64() != 0, + 95 => knot_count = count(t), + _ => control_count = count(t), + } + c.at += 1; + } + let mut knots = Vec::new(); + while knots.len() < knot_count && c.peek().is_some_and(|t| t.code == 40) { + knots.push(c.f(40)); + } + let (mut control, mut weights) = (Vec::new(), Vec::new()); + while control.len() < control_count && c.peek().is_some_and(|t| t.code == 10) { + control.push(c.xy(10)); + weights.extend(c.take(42).map(Tag::f64)); + } + if weights.len() != control.len() { + weights.clear(); + } + // A 97 here is the edge's fit point count (R2010 and later) when fit points follow it, or + // when it is 0 and the loop's own 97 comes next; otherwise it is the loop's source count. + let mut fit = Vec::new(); + let next = c.tags.get(c.at + 1).map(|t| t.code); + let fit_count = c.peek().filter(|t| t.code == 97 && (next == Some(11) || (t.i64() == 0 && next == Some(97)))); + if let Some(n) = fit_count.map(count) { + c.at += 1; + while fit.len() < n && c.peek().is_some_and(|t| t.code == 11) { + fit.push(c.xy(11)); + } + } + let tangent = |c: &mut Cur<'_>, code: i32| c.peek().is_some_and(|t| t.code == code).then(|| c.xy(code)); + let start_tangent = tangent(c, 12); + let end_tangent = tangent(c, 13); + HatchEdge::Spline { degree, rational, periodic, knots, control, weights, fit, start_tangent, end_tangent } +} + +fn edge_path(c: &mut Cur<'_>) -> Option { + if !c.seek(93) { + return None; + } + let n = count(c.take(93)?); + let mut edges = Vec::new(); + while edges.len() < n { + // Groups this reader does not know may sit between edges; the loop's own end may not. + while c.peek().is_some_and(|t| !matches!(t.code, 72 | 97 | 92 | 75)) { + c.at += 1; + } + if c.peek().is_none_or(|t| t.code != 72) { + break; + } + let kind = c.take(72).map_or(0, Tag::i64); + // 73 is counter-clockwise unless it says 0. + let ccw = |c: &mut Cur<'_>| c.take(73).is_none_or(|t| t.i64() != 0); + edges.push(match kind { + 1 => HatchEdge::Line { start: c.xy(10), end: c.xy(11) }, + 2 => HatchEdge::Arc { center: c.xy(10), radius: c.f(40), start_angle: c.f(50), end_angle: c.f(51), ccw: ccw(c) }, + 3 => HatchEdge::Ellipse { center: c.xy(10), major: c.xy(11), ratio: c.f(40), start_angle: c.f(50), end_angle: c.f(51), ccw: ccw(c) }, + 4 => spline_edge(c), + // An edge type the format does not have: its groups are passed over above. + _ => continue, + }); + } + Some(HatchPath::Edges(edges)) +} + +fn pattern_lines(tags: &[Tag]) -> Vec { + let Some(at) = tags.iter().position(|t| t.code == 78) else { return Vec::new() }; + let mut lines: Vec = Vec::new(); + for t in tags.iter().skip(at + 1) { + if t.code == 53 { + lines.push(PatternLine { angle: t.f64(), ..PatternLine::default() }); + continue; + } + let Some(l) = lines.last_mut() else { break }; + match t.code { + 43 => l.base[0] = t.f64(), + 44 => l.base[1] = t.f64(), + 45 => l.offset[0] = t.f64(), + 46 => l.offset[1] = t.f64(), + 79 => {} + 49 => l.dashes.push(t.f64()), + _ => break, + } + } + lines +} + +/// Groups 450–470, when 450 says gradient. Each 463 starts a colour record whose 63 (index) and +/// 421 (true colour) follow. +fn gradient(tags: &[Tag]) -> Option { + let start = tags.iter().position(|t| t.code == 450)?; + if tags.get(start)?.i64() != 1 { + return None; + } + let mut g = Gradient { name: String::new(), angle: 0.0, shift: 0.0, single: false, tint: 0.0, colors: Vec::new() }; + for t in tags.iter().skip(start + 1).take_while(|t| t.code != 1001) { + match t.code { + 452 => g.single = t.i64() == 1, + 460 => g.angle = t.f64(), + 461 => g.shift = t.f64(), + 462 => g.tint = t.f64(), + 463 => g.colors.push(Color::ByLayer), + 63 => { + // A true colour already read for this record wins over its index. + if let Some(c) = g.colors.last_mut().filter(|c| !matches!(c, Color::Rgb(_))) { + *c = Color::from_index(t.i64()); + } + } + 421 => { + if let Some(c) = g.colors.last_mut() { + *c = Color::from_true(t.i64()); + } + } + 470 => g.name = t.str(), + _ => {} + } + } + Some(g) +} + +impl Hatch { + /// The typed form of a HATCH record; `None` when it has no boundary data (no group 91) or + /// the data ends inside a loop. + pub fn read(tags: &[Tag]) -> Option { + let g = Groups(tags); + let n_loops = count(tags.iter().find(|t| t.code == 91)?); + let mut c = Cur { tags, at: tags.iter().position(|t| t.code == 91)? + 1 }; + let mut loops = Vec::new(); + while loops.len() < n_loops { + if !c.seek(92) { + return None; + } + let flags = c.take(92)?.i64(); + let path = if flags & 2 != 0 { polyline_path(&mut c)? } else { edge_path(&mut c)? }; + // 97 counts the source entities (330) that follow. + let mut sources = Vec::new(); + if let Some(n) = c.take(97).map(count) { + while sources.len() < n + && let Some(t) = c.take(330) + { + sources.extend(parse_handle(&t.str()).filter(|h| *h != 0)); + } + } + loops.push(HatchLoop { flags, path, sources }); + } + let mut seeds = Vec::new(); + if c.seek(98) { + let n = c.take(98).map_or(0, count); + while seeds.len() < n && c.peek().is_some_and(|t| t.code == 10) { + seeds.push(c.xy(10)); + } + } + let pattern = g.name(2).unwrap_or_else(|| "SOLID".into()); + Some(Hatch { + solid: g.i_or(70, 0) == 1, + associative: g.i_or(71, 0) != 0, + style: g.i_or(75, 0), + pattern_type: g.i_or(76, 1), + angle: g.f_or(52, 0.0), + scale: g.f_or(41, 1.0), + double: g.i_or(77, 0) != 0, + elevation: g.f_or(30, 0.0), + loops, + pattern_lines: pattern_lines(tags), + seeds, + gradient: gradient(tags), + pattern, + }) + } +} diff --git a/crates/dxf/src/model/text.rs b/crates/dxf/src/model/text.rs new file mode 100644 index 000000000..ffafae96d --- /dev/null +++ b/crates/dxf/src/model/text.rs @@ -0,0 +1,280 @@ +//! Text in DXF: the caret encoding of control characters in strings, and MTEXT with its columns. +//! +//! Control characters are written as a caret and a letter (`^I` a tab, `^J` a line feed, `^ ` a +//! caret). TEXT, ATTDEF, ATTRIB and DIMENSION strings use the same encoding as MTEXT contents, +//! without line feeds: AutoCAD reads a bare `^2` in `10^2` as a control character. +//! +//! MTEXT columns are groups 75 (type), 76 (count), 78 (flow reversed), 79 (auto height), 48 +//! (width), 49 (gutter) and 50 (column heights, after the column type; before it, 50 is the +//! rotation) before R2018, and an embedded object (group 101) from R2018 on. + +use super::{Groups, Point, outside_app}; +use crate::Tag; + +/// Decode MTEXT contents: `^I` (tab), `^J` (line feed) and `^ ` (caret). Other carets stay as +/// they are (older files write stacked tolerances `\S+0.1^-0.1;` with a bare caret). +pub fn decode_contents(s: &str) -> String { + decode(s, true) +} + +/// A single-line string as read: [`decode_contents`] without line feeds (`^J` stays as it is). +pub fn decode_line(s: &str) -> String { + decode(s, false) +} + +fn decode(s: &str, line_feeds: bool) -> String { + if !s.contains('^') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut it = s.chars().peekable(); + while let Some(c) = it.next() { + if c != '^' { + out.push(c); + continue; + } + match it.peek().copied() { + Some(' ') => out.push('^'), + Some('I') => out.push('\t'), + Some('J') if line_feeds => out.push('\n'), + _ => { + out.push('^'); + continue; + } + } + it.next(); + } + out +} + +/// A single-line string (TEXT, ATTDEF/ATTRIB, DIMENSION) for a DXF file: a caret as `^ ` and a +/// tab as `^I`, read back by [`decode_line`]. It must hold no line break. +pub fn encode_line(s: &str) -> String { + s.replace('^', "^ ").replace('\t', "^I") +} + +/// Encode MTEXT contents: a tab as `^I`, a line feed as `^J`, a caret as `^ `; other control +/// characters (which draw nothing) are dropped. +pub fn encode_contents(s: &str) -> String { + if !s.chars().any(|c| c == '^' || c.is_control()) { + return s.to_string(); + } + let mut out = String::with_capacity(s.len() + 8); + for c in s.chars() { + match c { + '^' => out.push_str("^ "), + '\t' => out.push_str("^I"), + '\n' => out.push_str("^J"), + c if c.is_control() => {} + c => out.push(c), + } + } + out +} + +/// Apply `f` ([`encode_line`] or [`decode_line`]) to the groups of a `kind` record that hold an +/// encoded string: the text (1), and an ATTDEF's prompt (3). An embedded MTEXT (from group 101) +/// has its own and is left alone. +pub fn map_text_groups(kind: &str, tags: &mut [Tag], f: fn(&str) -> String) { + let kind = kind.to_ascii_uppercase(); + if !matches!(kind.as_str(), "TEXT" | "ATTRIB" | "ATTDEF" | "DIMENSION") { + return; + } + for t in tags.iter_mut().take_while(|t| t.code != 101) { + if t.code == 1 || (t.code == 3 && kind == "ATTDEF") { + *t = Tag::s(t.code, f(&t.str())); + } + } +} + +/// A record's groups split at its embedded object (`101 Embedded Object`): its own groups and, +/// when it has one, the object's. +pub fn split_embedded(tags: &[Tag]) -> (&[Tag], Option<&[Tag]>) { + match tags.iter().position(|t| t.code == 101) { + Some(i) => (tags.get(..i).unwrap_or(tags), tags.get(i + 1..)), + None => (tags, None), + } +} + +/// Upper bound on the column heights read from one MTEXT. +pub const MAX_COLUMNS: usize = 1_000; + +/// The columns of an MTEXT. +#[derive(Clone, Debug, PartialEq)] +pub struct MTextColumns { + /// Dynamic columns (type 2): the text flows into as many as it needs. Otherwise static + /// (type 1). + pub dynamic: bool, + pub count: u16, + /// One column's width (the MTEXT's own width is the total in some writers' files). + pub width: f64, + pub gutter: f64, + /// The column height: the tallest for manual heights. + pub height: f64, + pub auto_height: bool, + /// Each column's height, for dynamic columns with manual heights; else empty. + pub heights: Vec, + pub flow_reversed: bool, +} + +fn pos(v: f64) -> f64 { + v.max(0.0) +} + +impl MTextColumns { + /// The columns of an MTEXT record, from the legacy groups or the embedded object. + pub fn read(tags: &[Tag]) -> Option { + let (own, embedded) = split_embedded(tags); + // An application-defined group in the record may hold any code, 50 included. + let own_tags: Vec<&Tag> = outside_app(own).map(|(_, t)| t).collect(); + let Some(start) = own_tags.iter().position(|t| t.code == 75) else { return Self::read_embedded(own, embedded?) }; + let dynamic = match own_tags.get(start).map(|t| t.i64()) { + Some(1) => false, + Some(2) => true, + _ => return None, + }; + let cols = own_tags.get(start..).unwrap_or_default(); + let first = |code: i32| cols.iter().find(|t| t.code == code).copied(); + // The first 50 is the number of heights that follow, unless the file has the heights + // of group 76 alone. + let fifties = cols.iter().filter(|t| t.code == 50); + let counted = first(76).is_some_and(|t| i64::try_from(fifties.clone().count()).is_ok_and(|n| n - 1 == t.i64())); + let heights: Vec = fifties.skip(usize::from(counted)).take(MAX_COLUMNS).map(|t| pos(t.f64())).collect(); + let count = first(76).map_or(heights.len().max(1), |t| usize::try_from(t.i64()).unwrap_or(1)).clamp(1, MAX_COLUMNS); + let auto_height = first(79).is_some_and(|t| t.i64() != 0); + let defined = own_tags.iter().find(|t| t.code == 46).map_or(0.0, |t| pos(t.f64())); + let manual = dynamic && !auto_height; + let height = + if manual { heights.iter().copied().fold(0.0, f64::max) } else { heights.first().copied().filter(|h| *h > 0.0).unwrap_or(defined) }; + Some(MTextColumns { + dynamic, + count: u16::try_from(count).unwrap_or(1), + width: first(48).map_or(0.0, |t| pos(t.f64())), + gutter: first(49).map_or(0.0, |t| pos(t.f64())), + height, + auto_height, + heights: if manual { heights } else { Vec::new() }, + flow_reversed: first(78).is_some_and(|t| t.i64() != 0), + }) + } + + /// The columns of an R2018 MTEXT, from its embedded object, whose groups reuse MTEXT's + /// codes: 71 type, 72 count, 44 column width, 45 gutter, 73 auto height, 74 flow reversed, + /// 46 heights. + #[expect(clippy::cast_possible_truncation, reason = "the rounded count is clamped to 1..=MAX_COLUMNS before use")] + fn read_embedded(own: &[Tag], obj: &[Tag]) -> Option { + let first = |code: i32| obj.iter().find(|t| t.code == code); + let dynamic = match first(71).map(Tag::i64) { + Some(1) => false, + Some(2) => true, + _ => return None, + }; + let heights: Vec = obj.iter().filter(|t| t.code == 46).take(MAX_COLUMNS).map(|t| pos(t.f64())).collect(); + let auto_height = first(73).is_some_and(|t| t.i64() != 0); + let manual = dynamic && !auto_height; + // The column height: the entity's own 46 (defined height), else the object's 41 or first 46. + let own_46 = own.iter().find(|t| t.code == 46).map(|t| pos(t.f64())); + let defined = [own_46, first(41).map(Tag::f64), heights.first().copied()].into_iter().flatten().find(|h| *h > 0.0).unwrap_or(0.0); + // A dynamic auto-height object stores 0 columns in 72: the count follows from the total + // width (42 = n * 44 + (n - 1) * 45). + let [width, gutter, total] = [44, 45, 42].map(|g| first(g).map_or(0.0, Tag::f64)); + let count = match first(72).map(Tag::i64) { + Some(n) if n > 0 => n, + Some(_) if width + gutter > 0.0 && total > 0.0 => ((total + gutter) / (width + gutter)).round().clamp(1.0, MAX_COLUMNS as f64) as i64, + _ => 1, + }; + Some(MTextColumns { + dynamic, + count: u16::try_from(count.clamp(1, MAX_COLUMNS as i64)).unwrap_or(1), + width: pos(width), + gutter: pos(gutter), + height: if manual { heights.iter().copied().fold(0.0, f64::max) } else { defined }, + auto_height, + heights: if manual { heights } else { Vec::new() }, + flow_reversed: first(74).is_some_and(|t| t.i64() != 0), + }) + } +} + +/// Multiline text: an MTEXT entity, or the one embedded in a multiline attribute. +#[derive(Clone, Debug, PartialEq)] +pub struct MText { + /// Group 10. + pub insert: Point, + /// Group 40: the text height. + pub height: f64, + /// Group 41: the reference rectangle's width (0: none). With columns, see + /// [`MTextColumns::width`]. + pub width: f64, + /// Group 71: 1–9, top left to bottom right. + pub attach: u8, + /// Group 72: 1 left to right, 3 top to bottom, 5 by style. + pub drawing_direction: i64, + /// Groups 11/21/31: the X axis direction, which wins over the rotation when given and not + /// zero. + pub direction: Option, + /// Group 50, in degrees. + pub rotation: f64, + /// Group 7 (`Standard` when absent). + pub style: String, + /// The contents (the group 3 chunks, then group 1), decoded ([`decode_contents`]); the + /// formatting codes (`\P`, `\f...;`) are left as written. + pub contents: String, + /// Group 44: the line spacing factor (1 when absent). + pub line_spacing: f64, + /// Group 73 is 2: the spacing is exact, not "at least". + pub line_spacing_exact: bool, + /// Group 90: the background fill flags (1 a fill colour, 2 the window's colour, 16 a + /// frame), with the fill colour (63, 421) and the border offset factor (45). + pub background: Option<(i64, super::Color, f64)>, + pub columns: Option, +} + +impl MText { + /// The typed form of MTEXT groups. + pub fn read(tags: &[Tag]) -> MText { + let columns = MTextColumns::read(tags); + // The MTEXT's own groups: an embedded object reuses their codes (44 is its column + // width, not the line spacing factor). + let (own, _) = split_embedded(tags); + let g = Groups(own); + let mut contents = String::new(); + for t in own.iter().filter(|t| t.code == 3) { + contents.push_str(&t.str()); + } + contents.push_str(&g.s(1).unwrap_or_default()); + // The direction and rotation come from before the column groups, where 50 is a height. + let head = Groups(own.get(..own.iter().position(|t| t.code == 75).unwrap_or(own.len())).unwrap_or(own)); + let background = g.i(90).filter(|f| *f != 0).map(|f| { + let color = match (g.i(421), g.i(63)) { + (Some(tc), _) => super::Color::from_true(tc), + (None, Some(v)) => super::Color::from_index(v), + (None, None) => super::Color::ByLayer, + }; + (f, color, g.f_or(45, 1.5)) + }); + MText { + insert: g.point(10), + height: g.f_or(40, 0.0), + width: pos(g.f_or(41, 0.0)), + attach: u8::try_from(g.i_or(71, 1).clamp(1, 9)).unwrap_or(1), + drawing_direction: g.i_or(72, 1), + direction: head.point_opt(11).filter(|d| d[0] != 0.0 || d[1] != 0.0 || d[2] != 0.0), + rotation: head.f_or(50, 0.0), + style: g.name(7).unwrap_or_else(|| "Standard".into()), + contents: decode_contents(&contents), + line_spacing: g.f_or(44, 1.0), + line_spacing_exact: g.i(73) == Some(2), + background, + columns, + } + } + /// The angle of the text's X axis in radians: of the direction when the record has one, + /// else the rotation. + pub fn angle(&self) -> f64 { + match self.direction { + Some(d) if d[0] != 0.0 || d[1] != 0.0 => d[1].atan2(d[0]), + _ => self.rotation.to_radians(), + } + } +} diff --git a/crates/dxf/tests/model_entities.rs b/crates/dxf/tests/model_entities.rs new file mode 100644 index 000000000..5de59761b --- /dev/null +++ b/crates/dxf/tests/model_entities.rs @@ -0,0 +1,393 @@ +//! The drawing model's entities: common properties, geometry, text, hatches, dimensions. + +use craft_dxf::model::text::{MTextColumns, decode_contents, decode_line, encode_contents, encode_line, map_text_groups}; +use craft_dxf::model::{Color, Document, Entity, EntityKind, HatchEdge, HatchPath, Lineweight, Transparency, entity_records, records}; +use craft_dxf::{Tag, parse}; + +fn tags(text: &str) -> Vec { + let lines: Vec<&str> = text.split(['\n', '|']).map(str::trim).filter(|l| !l.is_empty()).map(|l| if l == "~" { "" } else { l }).collect(); + parse((lines.join("\n") + "\n").as_bytes()).expect("the test drawing parses") +} + +/// The entities of `text`, the records of an ENTITIES section. +fn entities(text: &str) -> Vec { + let t = tags(text); + let (recs, strays) = entity_records(&records(&t)); + assert_eq!(strays, 0); + recs.iter().map(Entity::read).collect() +} + +fn one(text: &str) -> Entity { + let mut e = entities(text); + assert_eq!(e.len(), 1); + e.remove(0) +} + +#[test] +fn common_properties() { + let c = one("0|LINE|5|2A|330|1F|100|AcDbEntity|8|Walls|6|DASHED|62|1|420|65280|370|50|48|2.5|60|1|67|1|440|33554687|390|B|100|AcDbLine|39|1.5|210|0.0|220|0.0|230|-1.0").common; + assert_eq!((c.handle, c.owner, c.layer.as_str(), c.linetype.as_deref()), (Some(0x2A), Some(0x1F), "Walls", Some("DASHED"))); + assert_eq!((c.color, c.lineweight, c.ltscale), (Color::Rgb([0, 255, 0]), Lineweight::Hundredths(50), 2.5)); + assert!(!c.visible && c.paper_space); + assert_eq!((c.transparency, c.thickness, c.extrusion, c.plot_style), (Transparency::Alpha(255), 1.5, [0.0, 0.0, -1.0], Some(0xB))); + let d = one("0|LINE").common; + assert_eq!((d.handle, d.owner, d.layer.as_str(), d.linetype.as_deref(), d.color), (None, None, "0", None, Color::ByLayer)); + assert_eq!((d.lineweight, d.ltscale, d.visible, d.paper_space, d.extrusion), (Lineweight::ByLayer, 1.0, true, false, [0.0, 0.0, 1.0])); + assert_eq!(one("0|LINE|6|BYLAYER|48|-1.0|62|-2").common, d, "ByLayer by name, a negative scale and a negative colour are the defaults"); +} + +#[test] +fn entity_properties_are_not_read_from_the_entitys_own_subclass() { + // An MTEXT's 48 is its column width, not the linetype scale, and a marker in an + // application's braces is the application's: it does not end the entity's groups. A record with no AcDbEntity marker has its + // layer wherever it is. + let c = one("0|MTEXT|102|{APP|100|AcDbMText|102|}|100|AcDbEntity|8|A|100|AcDbMText|48|7.0|62|3").common; + assert_eq!((c.ltscale, c.color, c.layer.as_str()), (1.0, Color::ByLayer, "A")); + let c = one("0|LINE|100|AcDbLine|8|B|62|3").common; + assert_eq!((c.color, c.layer.as_str()), (Color::Index(3), "B")); + // An embedded object ends the entity's groups. + assert_eq!(one("0|ATTDEF|8|C|101|Embedded Object|62|5").common.color, Color::ByLayer); +} + +#[test] +fn curves() { + let k = |text: &str| one(text).kind; + assert_eq!(k("0|LINE|10|1.0|20|2.0|30|3.0|11|4.0|21|5.0"), EntityKind::Line { start: [1.0, 2.0, 3.0], end: [4.0, 5.0, 0.0] }); + assert_eq!(k("0|POINT|10|1.0|50|30.0"), EntityKind::Point { location: [1.0, 0.0, 0.0], angle: 30.0 }); + assert_eq!(k("0|CIRCLE|10|1.0|20|2.0|40|3.0"), EntityKind::Circle { center: [1.0, 2.0, 0.0], radius: 3.0 }); + assert_eq!(k("0|ARC|10|1.0|40|3.0|50|10.0"), EntityKind::Arc { center: [1.0, 0.0, 0.0], radius: 3.0, start_angle: 10.0, end_angle: 360.0 }); + assert_eq!( + k("0|ELLIPSE|10|1.0|11|2.0|21|0.0|40|0.5|41|0.0|42|3.0"), + EntityKind::Ellipse { center: [1.0, 0.0, 0.0], major: [2.0, 0.0, 0.0], ratio: 0.5, start: 0.0, end: 3.0 } + ); + assert_eq!(k("0|RAY|10|1.0|11|0.0|21|1.0"), EntityKind::Ray { base: [1.0, 0.0, 0.0], direction: [0.0, 1.0, 0.0] }); + assert_eq!(k("0|XLINE|10|1.0|11|0.0|21|1.0"), EntityKind::XLine { base: [1.0, 0.0, 0.0], direction: [0.0, 1.0, 0.0] }); + let EntityKind::Spline(s) = k( + "0|SPLINE|70|8|71|2|40|0.0|40|0.0|40|1.0|10|0.0|20|0.0|10|1.0|20|1.0|30|5.0|10|2.0|41|1.0|41|2.0|41|1.0|11|9.0|21|9.0|12|1.0|22|0.0|44|0.01", + ) else { + panic!("a spline") + }; + assert_eq!((s.degree, s.flags, s.knots.len(), s.weights.as_slice(), s.fit_tolerance), (2, 8, 3, [1.0, 2.0, 1.0].as_slice(), 0.01)); + assert_eq!(s.control, [[0.0, 0.0, 0.0], [1.0, 1.0, 5.0], [2.0, 0.0, 0.0]], "the third point has no y or z of its own"); + assert_eq!((s.fit.as_slice(), s.start_tangent, s.end_tangent), ([[9.0, 9.0, 0.0]].as_slice(), Some([1.0, 0.0, 0.0]), None)); +} + +#[test] +fn polylines() { + let EntityKind::LwPolyline(p) = one("0|LWPOLYLINE|90|3|70|129|43|0.5|38|2.0|10|0.0|20|0.0|42|1.0|10|4.0|20|0.0|40|0.1|41|0.2|10|4.0|20|3.0").kind + else { + panic!("a lightweight polyline") + }; + assert!(p.closed && p.plinegen); + assert_eq!((p.const_width, p.elevation, p.vertices.len()), (0.5, 2.0, 3)); + assert_eq!((p.vertices[0].bulge, p.vertices[1].start_width, p.vertices[1].end_width, p.vertices[2].y), (1.0, 0.1, 0.2, 3.0)); + // A polyface mesh: three vertices and a face record that names them. + let e = one( + "0|POLYLINE|66|1|70|64|71|3|72|1|10|0.0|20|0.0|30|7.0|0|VERTEX|10|0.0|20|0.0|30|0.0|70|192|0|VERTEX|10|1.0|70|192|0|VERTEX|10|1.0|20|1.0|70|192|0|VERTEX|70|128|71|1|72|2|73|-3|0|SEQEND", + ); + let EntityKind::Polyline(m) = e.kind else { panic!("a polyline") }; + assert!(m.is_polyface() && !m.is_mesh() && !m.is_3d() && !m.closed()); + assert_eq!((m.m, m.n, m.elevation, m.vertices.len()), (3, 1, 7.0, 4)); + assert!(m.vertices[3].is_face() && !m.vertices[0].is_face()); + assert_eq!(m.vertices[3].face, [1, 2, -3, 0]); + let EntityKind::Polyline(h) = one("0|POLYLINE|70|4|40|0.5|41|0.25|0|VERTEX|10|1.0|42|0.5|40|0.1|41|0.2|70|16|0|SEQEND").kind else { + panic!("a polyline") + }; + assert_eq!((h.start_width, h.end_width), (0.5, 0.25)); + let v = h.vertices[0]; + assert!(v.is_spline_frame()); + assert_eq!((v.bulge, v.start_width, v.end_width), (0.5, 0.1, 0.2)); +} + +#[test] +fn text_and_attributes() { + let EntityKind::Text(t) = one("0|TEXT|10|1.0|20|2.0|11|3.0|21|4.0|40|2.5|1|10^ 2^Im|50|90.0|41|0.8|51|15.0|7|Notes|71|2|72|1|73|2").kind else { + panic!("text") + }; + assert_eq!((t.insert, t.align, t.height, t.value.as_str()), ([1.0, 2.0, 0.0], Some([3.0, 4.0, 0.0]), 2.5, "10^2\tm")); + assert_eq!((t.rotation, t.width_factor, t.oblique, t.style.as_str(), t.generation, t.halign, t.valign), (90.0, 0.8, 15.0, "Notes", 2, 1, 2)); + let EntityKind::Text(d) = one("0|TEXT|1|x").kind else { panic!("text") }; + assert_eq!((d.align, d.width_factor, d.style.as_str(), d.halign, d.valign), (None, 1.0, "Standard", 0, 0)); + + let EntityKind::AttDef(a) = + one("0|ATTDEF|100|AcDbText|10|1.0|1|default|100|AcDbAttributeDefinition|280|0|3|Enter^Ia|2|TAG|70|12|74|3|280|1").kind + else { + panic!("an attribute definition") + }; + assert_eq!((a.tag.as_str(), a.prompt.as_str(), a.flags, a.text.valign, a.lock_position), ("TAG", "Enter\ta", 12, 3, true)); + assert!(a.multiline.is_none()); + + // A block reference with two attributes; the second is multiline. + let EntityKind::Insert(i) = one("0|INSERT|66|1|2|DOOR|10|1.0|20|2.0|41|2.0|43|-1.0|50|45.0|70|3|71|0|44|5.0|45|6.0 + 0|ATTRIB|8|Tags|62|2|1|one|2|A|70|1|280|0 + 0|ATTRIB|1|~|2|B|101|Embedded Object|10|7.0|40|2.0|71|5|3|line one^J|1|line two + 0|SEQEND") + .kind + else { + panic!("an insert") + }; + assert_eq!((i.block.as_str(), i.insert, i.scale, i.rotation), ("DOOR", [1.0, 2.0, 0.0], [2.0, 1.0, -1.0], 45.0)); + assert_eq!((i.columns, i.rows, i.column_spacing, i.row_spacing), (3, 1, 5.0, 6.0)); + assert_eq!(i.attribs.len(), 2); + let (c, a) = &i.attribs[0]; + assert_eq!( + (c.layer.as_str(), c.color, a.tag.as_str(), a.text.value.as_str(), a.flags, a.lock_position), + ("Tags", Color::Index(2), "A", "one", 1, false) + ); + assert!(a.prompt.is_empty(), "an ATTRIB has no prompt"); + let m = i.attribs[1].1.multiline.as_ref().expect("embedded MTEXT"); + assert_eq!((m.contents.as_str(), m.insert, m.height, m.attach), ("line one\nline two", [7.0, 0.0, 0.0], 2.0, 5)); +} + +#[test] +fn caret_encoding() { + assert_eq!(decode_contents("a^Ib^Jc^ d^2^"), "a\tb\nc^d^2^"); + assert_eq!(decode_line("a^Ib^Jc^ d"), "a\tb^Jc^d", "a single line has no line feed"); + assert_eq!(encode_contents("a\tb\nc^d\u{7}"), "a^Ib^Jc^ d", "other control characters are dropped"); + assert_eq!(encode_line("10^2\t"), "10^ 2^I"); + for s in ["", "plain", "10^2", "tab\there", "^^ ^I"] { + assert_eq!(decode_line(&encode_line(s)), s); + assert_eq!(decode_contents(&encode_contents(s)), s); + } + let mut t = vec![Tag::s(1, "a^ b"), Tag::s(3, "p^ q"), Tag::i(101, 0), Tag::s(1, "e^ f")]; + map_text_groups("attdef", &mut t, decode_line); + assert_eq!(t.iter().map(Tag::str).collect::>(), ["a^b", "p^q", "0", "e^ f"], "the embedded object's text is left alone"); + let mut t = vec![Tag::s(1, "a^ b"), Tag::s(3, "p^ q")]; + map_text_groups("TEXT", &mut t, decode_line); + assert_eq!(t[1].str(), "p^ q", "only an ATTDEF's 3 is a prompt"); + map_text_groups("LINE", &mut t, |_| String::new()); + assert_eq!(t[0].str(), "a^b"); +} + +#[test] +fn mtext() { + let EntityKind::MText(m) = one("0|MTEXT|10|1.0|20|2.0|40|2.5|41|40.0|71|5|72|1|3|first ^Ipart, |3|second, |1|last\\Pline|7|Notes|11|0.0|21|1.0|50|30.0|44|1.2|73|2|90|1|63|4|45|1.25").kind else { + panic!("multiline text") + }; + assert_eq!((m.insert, m.height, m.width, m.attach, m.style.as_str()), ([1.0, 2.0, 0.0], 2.5, 40.0, 5, "Notes")); + assert_eq!(m.contents, "first \tpart,second,last\\Pline"); + assert_eq!((m.direction, m.rotation, m.line_spacing, m.line_spacing_exact), (Some([0.0, 1.0, 0.0]), 30.0, 1.2, true)); + assert!((m.angle() - std::f64::consts::FRAC_PI_2).abs() < 1e-12, "the direction wins over the rotation"); + assert_eq!(m.background, Some((1, Color::Index(4), 1.25))); + assert!(m.columns.is_none()); + let EntityKind::MText(r) = one("0|MTEXT|1|x|50|90.0|11|0.0|21|0.0|71|99").kind else { panic!("multiline text") }; + assert!((r.angle() - std::f64::consts::FRAC_PI_2).abs() < 1e-12, "a zero direction is none"); + assert_eq!((r.attach, r.background, r.line_spacing), (9, None, 1.0)); +} + +#[test] +fn mtext_columns() { + // Legacy groups: static, 2 columns, the first 50 after 75 counts the heights; the 50 before + // 75 is the rotation. + let t = tags("0|MTEXT|10|0.0|50|15.0|1|x|75|1|76|2|78|1|79|0|48|30.0|49|5.0|50|2|50|80.0|50|80.0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.width, c.gutter, c.height, c.auto_height, c.flow_reversed), (false, 2, 30.0, 5.0, 80.0, false, true)); + assert!(c.heights.is_empty(), "only manual dynamic columns keep each height"); + let EntityKind::MText(m) = one("0|MTEXT|10|0.0|50|15.0|1|x|75|1|76|2|48|30.0|49|5.0|50|2|50|80.0|50|80.0").kind else { panic!("multiline text") }; + assert_eq!(m.rotation, 15.0); + // Dynamic with manual heights, and no count before them. + let t = tags("0|MTEXT|1|x|75|2|76|3|79|0|48|30.0|49|5.0|50|10.0|50|40.0|50|20.0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.height, c.heights.as_slice()), (true, 3, 40.0, [10.0, 40.0, 20.0].as_slice())); + // R2018: the embedded object; 72 is 0 for dynamic auto-height, the count follows the widths. + let t = tags("0|MTEXT|1|x|46|50.0|101|Embedded Object|70|1|10|1.0|11|0.0|40|0.0|41|0.0|42|65.0|43|0.0|71|2|72|0|44|30.0|45|5.0|73|1|74|0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.width, c.gutter, c.height, c.auto_height), (true, 2, 30.0, 5.0, 50.0, true)); + // No columns: type 0, or no column groups at all. + assert!(MTextColumns::read(&tags("0|MTEXT|1|x|75|0|76|0")[1..]).is_none()); + assert!(MTextColumns::read(&tags("0|MTEXT|1|x|101|Embedded Object|71|0")[1..]).is_none()); +} + +#[test] +fn the_column_height_limit() { + let mut text = String::from("0|MTEXT|1|x|75|2|76|5000|79|0"); + for _ in 0..5000 { + text.push_str("|50|1.0"); + } + let t = tags(&text); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.heights.len(), c.count), (1000, 1000)); + let mut text = String::from("0|MTEXT|1|x|101|Embedded Object|71|2|72|40000|73|0"); + for _ in 0..5000 { + text.push_str("|46|1.0"); + } + let t = tags(&text); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.heights.len(), c.count), (1000, 1000)); + // A total width that asks for a count beyond any number. + let t = tags("0|MTEXT|1|x|101|Embedded Object|71|2|72|0|73|1|42|1e300|44|1e-300|45|0.0"); + assert_eq!(MTextColumns::read(&t[1..]).expect("columns").count, 1000); +} + +#[test] +fn dimensions_and_leaders() { + let EntityKind::Dimension(d) = + one("0|DIMENSION|2|*D1|3|ISO-25|10|1.0|20|2.0|11|3.0|21|4.0|12|0.5|13|5.0|14|6.0|15|7.0|16|8.0|70|161|1|<> ^Im|42|12.5|50|30.0|52|60.0|53|45.0|40|2.0").kind + else { + panic!("a dimension") + }; + assert_eq!((d.kind(), d.flags & 128, d.flags & 32), (1, 128, 32)); + assert_eq!((d.block.as_deref(), d.style.as_str(), d.text.as_str(), d.measurement), (Some("*D1"), "ISO-25", "<> \tm", Some(12.5))); + assert_eq!((d.defpt, d.text_mid, d.insertion[0]), ([1.0, 2.0, 0.0], [3.0, 4.0, 0.0], 0.5)); + assert_eq!((d.p13[0], d.p14[0], d.p15[0], d.p16[0]), (5.0, 6.0, 7.0, 8.0)); + assert_eq!((d.rotation, d.oblique, d.text_rotation, d.leader_length), (30.0, 60.0, 45.0, 2.0)); + let EntityKind::Dimension(n) = one("0|DIMENSION|70|6").kind else { panic!("a dimension") }; + assert_eq!((n.block.as_deref(), n.style.as_str(), n.measurement, n.kind()), (None, "Standard", None, 6)); + + // The second vertex has no z: it does not take the third's. + let EntityKind::Leader(l) = one("0|LEADER|3|Arrows|71|0|72|1|73|0|340|4F|10|1.0|20|2.0|30|3.0|10|4.0|20|5.0|10|6.0|20|7.0|30|8.0").kind else { + panic!("a leader") + }; + assert_eq!(l.vertices, [[1.0, 2.0, 3.0], [4.0, 5.0, 0.0], [6.0, 7.0, 8.0]]); + assert!(!l.arrow && l.spline); + assert_eq!((l.style.as_str(), l.annotation_type, l.annotation), ("Arrows", 0, Some(0x4F))); +} + +#[test] +fn faces_viewports_images_and_tolerances() { + let k = |text: &str| one(text).kind; + let three = "|10|0.0|11|1.0|12|1.0|22|1.0"; + assert_eq!(k(&format!("0|SOLID{three}")), EntityKind::Solid { corners: [[0.0; 3], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [1.0, 1.0, 0.0]] }); + assert!(matches!(k(&format!("0|TRACE{three}|13|9.0")), EntityKind::Trace { corners } if corners[3] == [9.0, 0.0, 0.0])); + assert!(matches!(k(&format!("0|3DFACE{three}|70|5")), EntityKind::Face3d { hidden_edges: 5, .. })); + let EntityKind::Viewport(v) = + k("0|VIEWPORT|10|5.0|20|4.0|40|8.0|41|6.0|69|2|90|81920|12|1.0|22|2.0|17|9.0|45|30.0|51|90.0|340|A1|331|B1|331|B2|341|B3") + else { + panic!("a viewport") + }; + assert_eq!((v.center, v.width, v.height, v.id, v.status & 16384, v.status & 131_072), ([5.0, 4.0, 0.0], 8.0, 6.0, 2, 16384, 0)); + assert_eq!((v.view_center, v.view_target, v.view_direction, v.view_height, v.twist), ([1.0, 2.0], [9.0, 0.0, 0.0], [0.0, 0.0, 1.0], 30.0, 90.0)); + assert_eq!((v.clip, v.frozen_layers.as_slice()), (Some(0xA1), [0xB1, 0xB2, 0xB3].as_slice())); + let EntityKind::Image(i) = + k("0|IMAGE|10|1.0|11|0.5|12|0.0|22|0.5|13|640.0|23|480.0|340|C1|70|7|280|1|281|60|71|2|14|0.0|24|0.0|14|10.0|24|0.0|14|10.0|24|10.0|290|1") + else { + panic!("an image") + }; + assert_eq!((i.insert[0], i.u[0], i.v[1], i.size, i.image_def), (1.0, 0.5, 0.5, [640.0, 480.0], Some(0xC1))); + assert_eq!((i.display, i.clipping, i.brightness, i.contrast, i.fade, i.clip_type, i.clip_inverted), (7, true, 60, 50, 0, 2, true)); + assert_eq!(i.clip, [[0.0, 0.0], [10.0, 0.0], [10.0, 10.0]]); + assert!(matches!(k("0|WIPEOUT|10|1.0|14|-0.5|24|-0.5|14|0.5|24|0.5"), EntityKind::Wipeout(w) if w.clip.len() == 2 && w.clip_type == 1)); + assert_eq!( + k("0|TOLERANCE|3|Tol|10|1.0|1|{\\Fgdt;j}%%v0.1"), + EntityKind::Tolerance { insert: [1.0, 0.0, 0.0], direction: [1.0, 0.0, 0.0], style: "Tol".into(), text: "{\\Fgdt;j}%%v0.1".into() } + ); + assert_eq!(k("0|MULTILEADER|10|1.0"), EntityKind::Other, "a type the model does not have is read from its tags"); +} + +#[test] +fn hatch_with_a_polyline_loop_and_a_pattern() { + let EntityKind::Hatch(h) = one("0|HATCH|10|0.0|20|0.0|30|2.5|2|ANSI31|70|0|71|1|91|1 + 92|7|72|1|73|1|93|3|10|0.0|20|0.0|42|0.5|10|4.0|20|0.0|42|0.0|10|4.0|20|3.0|42|0.0|97|2|330|A1|330|A2 + 75|1|76|1|52|45.0|41|2.0|77|0|78|2 + 53|45.0|43|0.0|44|0.0|45|-1.4|46|1.4|79|0 + 53|135.0|43|1.0|44|2.0|45|1.0|46|1.0|79|2|49|3.0|49|-1.5 + 98|2|10|1.0|20|1.0|10|2.0|20|2.0") + .kind + else { + panic!("a hatch") + }; + assert_eq!((h.pattern.as_str(), h.solid, h.associative, h.style, h.pattern_type), ("ANSI31", false, true, 1, 1)); + assert_eq!((h.angle, h.scale, h.double, h.elevation), (45.0, 2.0, false, 2.5)); + assert_eq!(h.loops.len(), 1); + let l = &h.loops[0]; + assert_eq!((l.flags, l.sources.as_slice()), (7, [0xA1, 0xA2].as_slice())); + assert_eq!(l.path, HatchPath::Polyline { closed: true, vertices: vec![[0.0, 0.0, 0.5], [4.0, 0.0, 0.0], [4.0, 3.0, 0.0]] }); + assert_eq!(h.pattern_lines.len(), 2); + assert_eq!((h.pattern_lines[0].angle, h.pattern_lines[0].offset, h.pattern_lines[0].dashes.len()), (45.0, [-1.4, 1.4], 0)); + assert_eq!((h.pattern_lines[1].base, h.pattern_lines[1].dashes.as_slice()), ([1.0, 2.0], [3.0, -1.5].as_slice())); + assert_eq!(h.seeds, [[1.0, 1.0], [2.0, 2.0]]); + assert!(h.gradient.is_none()); +} + +#[test] +fn hatch_with_edge_loops_and_a_gradient() { + let EntityKind::Hatch(h) = one( + "0|HATCH|2|SOLID|70|1|71|0|91|2 + 92|1|93|4 + 72|1|10|0.0|20|0.0|11|4.0|21|0.0 + 72|2|10|4.0|20|1.0|40|1.0|50|270.0|51|360.0|73|1 + 72|3|10|2.0|20|2.0|11|2.0|21|0.0|40|0.5|50|0.0|51|180.0|73|0 + 72|4|94|3|73|1|74|0|95|8|96|4|40|0.0|40|0.0|40|0.0|40|0.0|40|1.0|40|1.0|40|1.0|40|1.0|10|0.0|20|2.0|42|1.0|10|0.0|20|1.5|42|2.0|10|0.0|20|0.5|42|2.0|10|0.0|20|0.0|42|1.0|97|2|11|0.0|21|2.0|11|0.0|21|0.0|12|0.0|22|-1.0|13|0.0|23|-1.0 + 97|0 + 92|16|93|1 + 72|2|10|2.0|20|1.0|40|0.25|50|0.0|51|360.0|73|1 + 97|1|330|B1 + 75|0|76|1|98|0 + 450|1|451|0|460|0.785|461|0.0|452|0|462|0.0|453|2|463|0.0|63|5|421|255|463|1.0|63|2|470|LINEAR + 1001|ACAD|1010|0.0", + ) + .kind + else { + panic!("a hatch") + }; + assert!(h.solid && !h.associative); + assert_eq!(h.loops.len(), 2); + let HatchPath::Edges(e) = &h.loops[0].path else { panic!("edges") }; + assert_eq!(e.len(), 4); + assert_eq!(e[0], HatchEdge::Line { start: [0.0, 0.0], end: [4.0, 0.0] }); + assert_eq!(e[1], HatchEdge::Arc { center: [4.0, 1.0], radius: 1.0, start_angle: 270.0, end_angle: 360.0, ccw: true }); + assert_eq!(e[2], HatchEdge::Ellipse { center: [2.0, 2.0], major: [2.0, 0.0], ratio: 0.5, start_angle: 0.0, end_angle: 180.0, ccw: false }); + let HatchEdge::Spline { degree, rational, periodic, knots, control, weights, fit, start_tangent, end_tangent } = &e[3] else { + panic!("a spline edge") + }; + assert_eq!((*degree, *rational, *periodic, knots.len(), control.len()), (3, true, false, 8, 4)); + assert_eq!( + (weights.as_slice(), fit.len(), *start_tangent, *end_tangent), + ([1.0, 2.0, 2.0, 1.0].as_slice(), 2, Some([0.0, -1.0]), Some([0.0, -1.0])) + ); + // The spline's own 97 (fit points) is not the loop's source count. + assert!(h.loops[0].sources.is_empty()); + assert_eq!((h.loops[1].flags, h.loops[1].sources.as_slice()), (16, [0xB1].as_slice())); + let g = h.gradient.expect("a gradient"); + assert_eq!((g.name.as_str(), g.angle, g.shift, g.single), ("LINEAR", 0.785, 0.0, false)); + assert_eq!(g.colors, [Color::Rgb([0, 0, 255]), Color::Index(2)], "a true colour wins over its index"); +} + +#[test] +fn damaged_hatches() { + let k = |text: &str| one(text).kind; + assert_eq!(k("0|HATCH|2|SOLID"), EntityKind::Other, "no boundary data"); + assert_eq!(k("0|HATCH|91|2|92|2|93|1|10|0.0|20|0.0"), EntityKind::Other, "the second loop is missing"); + assert_eq!(k("0|HATCH|91|1|92|2|8|x|93|1"), EntityKind::Other, "a foreign group in the loop header"); + // Counts larger than the data: what is there is read, nothing is reserved for the rest. + let EntityKind::Hatch(h) = k("0|HATCH|91|1|92|2|93|2000000000|10|1.0|20|2.0|97|2000000000|330|A|98|2000000000|10|5.0|20|6.0") else { + panic!("a hatch") + }; + assert_eq!(h.loops[0].path, HatchPath::Polyline { closed: false, vertices: vec![[1.0, 2.0, 0.0]] }); + assert_eq!((h.loops[0].sources.as_slice(), h.seeds.as_slice()), ([0xA].as_slice(), [[5.0, 6.0]].as_slice())); + let EntityKind::Hatch(h) = k("0|HATCH|91|1|92|0|93|9|72|9|10|1.0|72|1|10|1.0|20|1.0|11|2.0|21|2.0|72|4|95|-5|96|2000000000|10|1.0|20|1.0") else { + panic!("a hatch") + }; + let HatchPath::Edges(e) = &h.loops[0].path else { panic!("edges") }; + assert_eq!(e.len(), 2, "an edge type the format does not have is passed over"); + assert!(matches!(&e[1], HatchEdge::Spline { knots, control, .. } if knots.is_empty() && control.len() == 1)); + assert_eq!(k("0|HATCH|91|-4").clone(), k("0|HATCH|91|0"), "a negative loop count is none"); + // A 450 that says no gradient. + assert!(matches!(k("0|HATCH|91|0|450|0|463|0.0|63|1"), EntityKind::Hatch(h) if h.gradient.is_none())); +} + +#[test] +fn entities_of_a_whole_drawing() { + let t = tags( + "0|SECTION|2|BLOCKS|0|BLOCK|2|B|0|CIRCLE|40|2.0|0|ENDBLK|0|ENDSEC|0|SECTION|2|ENTITIES|0|INSERT|2|B|0|POLYLINE|0|VERTEX|10|1.0|0|SEQEND|0|ENDSEC|0|EOF", + ); + let d = Document::read(&t); + let kinds: Vec = d.entities.iter().chain(&d.blocks[0].entities).map(|e| Entity::read(e).kind).collect(); + assert!(matches!(kinds.as_slice(), [EntityKind::Insert(_), EntityKind::Polyline(p), EntityKind::Circle { .. }] if p.vertices.len() == 1)); +} + +#[test] +fn a_spline_edge_without_fit_data_leaves_the_loops_source_count() { + // Before R2010 a spline edge has no 97 of its own: the one after it counts the loop's sources. + let EntityKind::Hatch(h) = one( + "0|HATCH|91|1|92|1|93|1|72|4|94|1|73|0|74|0|95|4|96|2|40|0.0|40|0.0|40|1.0|40|1.0|10|0.0|20|0.0|10|1.0|20|1.0|97|2|330|A1|330|A2|75|0|98|0", + ) + .kind + else { + panic!("a hatch") + }; + assert_eq!(h.loops[0].sources, [0xA1, 0xA2]); + assert!( + matches!(&h.loops[0].path, HatchPath::Edges(e) if matches!(&e[0], HatchEdge::Spline { fit, control, .. } if fit.is_empty() && control.len() == 2)) + ); +} diff --git a/crates/epub/Cargo.toml b/crates/epub/Cargo.toml index 690a89778..67875fe89 100644 --- a/crates/epub/Cargo.toml +++ b/crates/epub/Cargo.toml @@ -13,8 +13,9 @@ workspace = true [dependencies] craft-xml = { workspace = true } thiserror = { workspace = true } -# `deflate-flate2` with flate2's default miniz_oxide backend, as craft-update pins it (zip's own -# `deflate` feature would switch a whole app's flate2 to zlib-rs). +craft-zip = { workspace = true } + +[dev-dependencies] +# The tests build hostile archives by hand. zip = { version = "9", default-features = false, features = ["deflate-flate2"] } -# The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +craft-zip = { workspace = true, features = ["test-support"] } diff --git a/crates/epub/README.md b/crates/epub/README.md index c15c13876..c9ba7d134 100644 --- a/crates/epub/README.md +++ b/crates/epub/README.md @@ -94,7 +94,8 @@ for path in &spine { | `max_total_bytes` (all entries read from one package) | 256 MiB | `Error::TooLarge` | | `max_manifest`, `max_spine` | 10,000 each | further items are ignored | -Sizes are checked against what an entry really expands to, not what the archive declares. The two +The ZIP layer is craft-zip: the declared entry count is checked before the archive is indexed, and +sizes are checked against what an entry really expands to, not what the archive declares. The two XML documents are read with craft-xml's limits (depth 256, 2,000,000 nodes) and `max_entry_bytes`. ## Where it came from diff --git a/crates/epub/src/read.rs b/crates/epub/src/read.rs index ac93c5b0e..0540e70f8 100644 --- a/crates/epub/src/read.rs +++ b/crates/epub/src/read.rs @@ -1,7 +1,6 @@ //! An EPUB 2 or 3 package, read: container, package document, and bounded entry reads. use std::collections::HashMap; -use std::io::{Cursor, Read}; use craft_xml::{El, Local, Options}; @@ -224,10 +223,8 @@ pub fn parse_package(xml: &[u8], opf_dir: &str, limits: &Limits) -> Result { - zip: zip::ZipArchive>, + zip: craft_zip::Archive<'a>, limits: Limits, - /// Bytes the entries still to be read may expand to. - left: u64, /// The archive path of the package document. pub opf_path: String, pub package: Package, @@ -242,11 +239,16 @@ impl<'a> Epub<'a> { /// Reads the container and the package document. The `mimetype` entry is not required: /// packages without one, or with it compressed, are common. pub fn open_with(bytes: &'a [u8], limits: Limits) -> Result { - let zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| Error::NotEpub(e.to_string()))?; - if zip.len() > limits.max_entries { - return Err(Error::TooManyEntries { found: zip.len(), limit: limits.max_entries }); - } - let mut epub = Epub { zip, limits, left: limits.max_total_bytes, opf_path: String::new(), package: Package::default() }; + let zip = + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: limits.max_total_bytes }) + .map_err(|e| match e { + craft_zip::Error::TooManyEntries { declared, limit } => Error::TooManyEntries { + found: usize::try_from(declared).unwrap_or(usize::MAX), + limit: usize::try_from(limit).unwrap_or(usize::MAX), + }, + e => Error::NotEpub(e.to_string()), + })?; + let mut epub = Epub { zip, limits, opf_path: String::new(), package: Package::default() }; let container = epub.read("META-INF/container.xml")?.ok_or_else(|| Error::NotEpub("META-INF/container.xml is missing".into()))?; let opf_path = parse_container(&container)?.ok_or_else(|| Error::NotEpub("container.xml names no package document".into()))?; // The container gives a path from the archive's root; resolving it also drops a leading `/`. @@ -262,28 +264,20 @@ impl<'a> Epub<'a> { /// Does the archive have an entry `path`? pub fn contains(&self, path: &str) -> bool { - self.zip.index_for_name(path).is_some() + self.zip.contains(path) } /// The bytes of archive entry `path`, or `None` when the archive has no such entry. An entry /// past the limits is an error, and so is one that expands to more than it declared. pub fn read(&mut self, path: &str) -> Result>, Error> { - let Some(index) = self.zip.index_for_name(path).filter(|_| !path.ends_with('/')) else { return Ok(None) }; - let part = |msg: String| Error::Part { part: path.to_string(), msg }; - let f = self.zip.by_index(index).map_err(|e| part(e.to_string()))?; - let limit = self.limits.max_entry_bytes.min(self.left); - let too_large = || Error::TooLarge { part: path.to_string(), limit }; - if f.size() > limit { - return Err(too_large()); + if path.ends_with('/') { + return Ok(None); } - let mut buf = Vec::new(); - f.take(limit.saturating_add(1)).read_to_end(&mut buf).map_err(|e| part(e.to_string()))?; - let n = buf.len() as u64; - if n > limit { - return Err(too_large()); - } - self.left = self.left.saturating_sub(n); - Ok(Some(buf)) + let limit = self.limits.max_entry_bytes.min(self.zip.budget().remaining()); + self.zip.read(path, self.limits.max_entry_bytes).map_err(|e| match e { + craft_zip::Error::EntryTooLarge { .. } | craft_zip::Error::BudgetSpent { .. } => Error::TooLarge { part: path.to_string(), limit }, + e => Error::Part { part: path.to_string(), msg: e.to_string() }, + }) } /// [`Epub::read`] for what a file of the package refers to: `href` as written in the file at @@ -295,6 +289,6 @@ impl<'a> Epub<'a> { /// Bytes the entries still to be read may expand to. pub fn budget_left(&self) -> u64 { - self.left + self.zip.budget().remaining() } } diff --git a/crates/epub/src/write.rs b/crates/epub/src/write.rs index 0c77951a8..fbc741b55 100644 --- a/crates/epub/src/write.rs +++ b/crates/epub/src/write.rs @@ -3,7 +3,6 @@ use std::borrow::Cow; use std::collections::HashSet; use std::fmt::Write as _; -use std::io::Write as _; use craft_xml::{esc, esc_attr, is_name}; @@ -316,10 +315,8 @@ pub fn write(book: &Book) -> Result, Error> { w3cdtf(&book.metadata.modified).ok_or_else(|| Error::Invalid(format!("`{}` is not a modification date", book.metadata.modified)))?; let opf = package_document(book, &modified); let nav = nav_document(book); - let fail = |e: &dyn std::fmt::Display| Error::Invalid(format!("the archive could not be written: {e}")); - let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - let deflated = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let fail = |e: craft_zip::Error| Error::Invalid(e.to_string()); + let mut z = craft_zip::Writer::new(); let own = [ ("mimetype".to_string(), MIMETYPE.as_bytes(), true), ("META-INF/container.xml".to_string(), CONTAINER.as_bytes(), false), @@ -328,10 +325,7 @@ pub fn write(book: &Book) -> Result, Error> { ]; let resources = book.resources.iter().map(|r| (format!("{ROOT}/{}", r.path), r.data.as_ref(), r.stored)); for (name, data, store) in own.into_iter().chain(resources) { - // More than 4 GiB needs the ZIP64 form. - let options = if store { stored } else { deflated }.large_file(data.len() >= u32::MAX as usize); - z.start_file(name, options).map_err(|e| fail(&e))?; - z.write_all(data).map_err(|e| fail(&e))?; + z.add(&name, data, if store { craft_zip::Compression::Stored } else { craft_zip::Compression::Deflated }).map_err(fail)?; } - z.finish().map(std::io::Cursor::into_inner).map_err(|e| fail(&e)) + z.finish().map_err(fail) } diff --git a/crates/epub/tests/read.rs b/crates/epub/tests/read.rs index 88fb27df6..dd1999ce4 100644 --- a/crates/epub/tests/read.rs +++ b/crates/epub/tests/read.rs @@ -252,3 +252,11 @@ fn a_package_document_past_the_entry_limit_is_refused() { let limits = Limits { max_entry_bytes: (xml.len() - 1) as u64, ..Limits::DEFAULT }; assert!(matches!(parse_package(&xml, "", &limits), Err(Error::Part { .. }))); } + +/// A ZIP64 end record can declare millions of entries in a small file; the count is refused on +/// the raw bytes, before the archive is indexed. +#[test] +fn an_archive_declaring_too_many_entries_is_refused_before_it_is_indexed() { + let bomb = craft_zip::zip64_entry_count_sample(100_001); + assert_eq!(Epub::open(&bomb).err(), Some(Error::TooManyEntries { found: 100_001, limit: 100_000 })); +} diff --git a/crates/ews/Cargo.toml b/crates/ews/Cargo.toml new file mode 100644 index 000000000..91afc504d --- /dev/null +++ b/crates/ews/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-ews" +description = "Exchange Web Services SOAP layer without sockets: the request envelope and an escaping element builder, and the answer envelope, faults and response messages read with craft-xml." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-xml = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/ews/README.md b/crates/ews/README.md new file mode 100644 index 000000000..1442eaaa0 --- /dev/null +++ b/crates/ews/README.md @@ -0,0 +1,40 @@ +# craft-ews + +The SOAP layer of Exchange Web Services, without sockets. + +- **Requests**: `Element` builds an operation's elements (`Element::new("m:GetFolder").child(…)`) + and escapes everything it is given: attribute values lose every control character, text keeps + its tabs and line feeds and writes a carriage return as ` `, names keep only what a + qualified name is made of. `envelope(Version, &operation)` writes the SOAP 1.1 envelope with + `RequestServerVersion`. +- **Answers**: `Answer::parse` reads the envelope with `craft-xml`. A fault is the error it names; + `Answer::messages("GetFolder")` gives the `GetFolderResponseMessage`s with their + `ResponseClass` and `ResponseCode`, and `ResponseMessage::ok` the element or the error. A busy + server (`ErrorServerBusy`, or that code inside `ErrorInternalServerError`) gives its wait, + clamped to 1 s to 5 min. + +Names in an answer carry fixed prefixes whatever the server used: `s:` envelope, `m:` messages, +`t:` types, `e:` errors. + +Written from Microsoft's public documentation: the EWS reference, [MS-OXWSCDATA] and +[MS-OXWSCORE]. + +## Hostile input + +A document type declaration, malformed markup, more than 40 levels, more than 100,000 nodes or +more than 64 MiB is `Error::Malformed`; an entity XML does not define is left out. Errors hold no +text a server wrote except a response code that is a short identifier (64 letters and digits at +most). `Debug` output of an answer or a message shows nothing a server chose. + +## What stays in the apps + +HTTP and its status codes, Autodiscover, NTLM and Basic authentication, the operations (which +elements to send, what to make of what comes back), sync state and the mapping onto a mailbox. + +## Origin + +MailCraft `crates/ews` (`soap.rs`, `reply.rs`, the response codes of `error.rs`, the escaping of +`xml.rs`) at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors (MIT OR Apache-2.0), with the tests of that layer. +MailCraft's own bounded XML reader (`xml.rs`) is replaced by `craft-xml`. diff --git a/crates/ews/src/answer.rs b/crates/ews/src/answer.rs new file mode 100644 index 000000000..562ec63d5 --- /dev/null +++ b/crates/ews/src/answer.rs @@ -0,0 +1,210 @@ +//! Reading an answer: the SOAP envelope, a fault, and the response messages of an operation with +//! their classes and codes. +//! +//! The answer is read with `craft-xml` into a tree whose names carry fixed prefixes whatever the +//! document used: `s:` for the envelope ([`SOAP`]), `m:` for the messages ([`MESSAGES`]), `t:` for +//! the types ([`TYPES`]), `e:` for the errors ([`ERRORS`]); a name in any other namespace starts +//! with `?:`. Elements are found by namespace and local name, never by the document's prefix; +//! everything else in an answer (an element not asked about, a header, a remark) is skipped. + +use std::fmt; + +use craft_xml::{Canonical, Doctype, El, Limits, Local, Options, UnknownEntity}; + +use crate::codes::{DEFAULT_BACK_OFF_MS, MAX_BACK_OFF_MS, MIN_BACK_OFF_MS, ResponseClass, ResponseCode}; +use crate::soap::{ERRORS, MESSAGES, SOAP, TYPES}; + +/// Deepest the elements of an answer may nest. +pub const MAX_DEPTH: usize = 40; +/// Most nodes (elements and stretches of text) in one answer. +pub const MAX_NODES: usize = 100_000; +/// Most bytes of an answer: a `GetItem` answer carries whole messages in base64. +pub const MAX_BYTES: usize = 64 * 1024 * 1024; + +const NAMESPACES: [(&str, &str); 4] = [("s", SOAP), ("m", MESSAGES), ("t", TYPES), ("e", ERRORS)]; +const CANONICAL: Canonical<'static> = Canonical::new(&NAMESPACES); + +/// Why an answer says nothing an operation can use. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Error { + /// Not XML, or XML past the limits, or with a document type declaration. + Malformed, + /// XML, but not the envelope or the answer that was asked for. + Unexpected, + /// The server is busy: ask again after this many milliseconds. + Busy { back_off_ms: u32 }, + /// The server said no, with this code. + Server(ResponseCode), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Malformed => f.write_str("the answer is not well-formed XML within the limits"), + Error::Unexpected => f.write_str("the answer is not the one asked for"), + Error::Busy { back_off_ms } => write!(f, "the server is busy: ask again in {back_off_ms} ms"), + Error::Server(code) => write!(f, "the server answered {code}"), + } + } +} + +impl std::error::Error for Error {} + +/// Read `bytes` into a tree with the fixed prefixes (see the module's documentation). No document +/// type declaration is read, and a reference to an entity XML does not define is left out. +pub fn parse_xml(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err(Error::Malformed); + } + let limits = Limits { max_input: MAX_BYTES, max_depth: MAX_DEPTH, max_nodes: MAX_NODES, ..Limits::DEFAULT }; + let options = + Options { doctype: Doctype::Reject, unknown_entity: UnknownEntity::Drop, ..Options::canonical(&CANONICAL).strict().with_limits(limits) }; + options.parse(bytes).map_err(|_| Error::Malformed) +} + +/// The body of a SOAP answer that was not a fault. +pub struct Answer { + body: El, +} + +impl fmt::Debug for Answer { + /// Nothing of the answer: it may hold someone's mail. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("Answer(..)") + } +} + +/// One response message of an operation's answer. +pub struct ResponseMessage<'a> { + class: ResponseClass, + code: ResponseCode, + /// The server says it is busy (`ErrorServerBusy`, or inside `ErrorInternalServerError`). + busy: bool, + back_off_ms: Option, + el: &'a El, +} + +impl fmt::Debug for ResponseMessage<'_> { + /// The class and the code: the rest is the server's. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ResponseMessage").field("class", &self.class).field("code", &self.code).finish() + } +} + +impl Answer { + /// Reads the document `bytes`: a SOAP envelope whose body is not a fault. A fault is the + /// error it names. + pub fn parse(bytes: &[u8]) -> Result { + let root = parse_xml(bytes)?; + if root.name != "s:Envelope" { + return Err(Error::Unexpected); + } + let body = root.child("s:Body").ok_or(Error::Unexpected)?; + if let Some(fault) = body.child("s:Fault") { + return Err(read_fault(fault)); + } + Ok(Self { body: body.clone() }) + } + + /// The `s:Body` element. + pub fn body(&self) -> &El { + &self.body + } + + /// The busy error, when any response message of the answer says the server is busy: it is + /// the whole request that is to be made again, whatever the operation, and a busy server + /// answers one message however many questions there were. + pub fn busy(&self) -> Option { + let response = self.body.elements().next()?; + let all = response.child("m:ResponseMessages")?; + all.elements().find_map(|child| ResponseMessage::read(child).ok()?.ok().err().filter(|error| matches!(error, Error::Busy { .. }))) + } + + /// The response messages of the answer to `operation` (`GetFolder`: the + /// `GetFolderResponseMessage`s inside `GetFolderResponse`), in order. + pub fn messages(&self, operation: &str) -> Result>, Error> { + let response = self.body.child(format!("m:{operation}Response").as_str()).ok_or(Error::Unexpected)?; + let all = response.child("m:ResponseMessages").ok_or(Error::Unexpected)?; + let name = format!("m:{operation}ResponseMessage"); + all.elements().filter(|child| child.name == name).map(ResponseMessage::read).collect() + } +} + +impl<'a> ResponseMessage<'a> { + fn read(el: &'a El) -> Result { + let class = el.attr("ResponseClass").and_then(ResponseClass::from_name).ok_or(Error::Unexpected)?; + let code = el.child("m:ResponseCode").map(|code| ResponseCode::from_name(&code.text())).ok_or(Error::Unexpected)?; + // (`MessageXml` and its values are told by their local names: a server puts them in the + // messages' namespace or the types'.) + let wait = el.child(Local("BackOffMilliseconds")).map(|wait| wait.text()); + let (busy, back_off_ms) = busy_of(&code, el.child(Local("MessageXml")), wait.as_deref()); + Ok(Self { class, code, busy, back_off_ms, el }) + } + + /// How the server says it went. + pub fn class(&self) -> ResponseClass { + self.class + } + + /// The code the server gave. + pub fn code(&self) -> &ResponseCode { + &self.code + } + + /// The element, for whoever reads what it holds. + pub fn el(&self) -> &'a El { + self.el + } + + /// The element when the message says it was done; the error it names when it was not. + pub fn ok(&self) -> Result<&'a El, Error> { + if self.class != ResponseClass::Error { + return Ok(self.el); + } + Err(if self.busy { Error::Busy { back_off_ms: self.back_off_ms.unwrap_or(DEFAULT_BACK_OFF_MS) } } else { Error::Server(self.code.clone()) }) + } +} + +/// The text of the `Value` called `name` inside a `MessageXml`. +fn value_of(message_xml: &El, name: &str) -> Option { + message_xml.children(Local("Value")).find(|value| value.attr("Name") == Some(name)).map(|value| value.text().trim().to_string()) +} + +/// Whether a message with this `code` and `MessageXml` says the server is busy, and the wait +/// it asks for. Busy is `ErrorServerBusy`, and also `ErrorInternalServerError` whose inner +/// error is `ErrorServerBusy` (Exchange's throttling says so in a 200 answer). The wait is a +/// `BackOffMilliseconds` of its own or a `Value` of that name. +fn busy_of(code: &ResponseCode, message_xml: Option<&El>, direct: Option<&str>) -> (bool, Option) { + let inner = message_xml.and_then(|xml| value_of(xml, "InnerErrorResponseCode")).map(|c| ResponseCode::from_name(&c)); + let busy = *code == ResponseCode::ServerBusy || (*code == ResponseCode::InternalServerError && inner == Some(ResponseCode::ServerBusy)); + let wait = direct.map(back_off_ms).or_else(|| message_xml.and_then(|xml| value_of(xml, "BackOffMilliseconds")).map(|w| back_off_ms(&w))); + (busy, wait) +} + +/// A wait in milliseconds, as a server writes it: digits, at least [`MIN_BACK_OFF_MS`] and at +/// most [`MAX_BACK_OFF_MS`]; anything else is the default. +pub fn back_off_ms(text: &str) -> u32 { + let text = text.trim(); + if text.is_empty() || !text.bytes().all(|b| b.is_ascii_digit()) { + return DEFAULT_BACK_OFF_MS; + } + // (A number of any length: too many digits is simply the most.) + let ms = text.parse::().unwrap_or(u64::MAX); + u32::try_from(ms.clamp(u64::from(MIN_BACK_OFF_MS), u64::from(MAX_BACK_OFF_MS))).unwrap_or(MAX_BACK_OFF_MS) +} + +/// The error a `Fault` names: the response code of its `detail`, and for a busy server the wait. +/// (`ResponseCode` and `MessageXml` are told by their local names, and `Value` too: the service +/// writes `MessageXml` in the types' namespace as often as the errors'.) +fn read_fault(fault: &El) -> Error { + let detail = fault.child(Local("detail")); + let code = detail + .and_then(|detail| detail.child(Local("ResponseCode"))) + .map_or_else(|| ResponseCode::Other(String::new()), |code| ResponseCode::from_name(&code.text())); + let (busy, wait) = busy_of(&code, detail.and_then(|detail| detail.child(Local("MessageXml"))), None); + if busy { Error::Busy { back_off_ms: wait.unwrap_or(DEFAULT_BACK_OFF_MS) } } else { Error::Server(code) } +} + +#[cfg(test)] +#[path = "answer_tests.rs"] +mod tests; diff --git a/crates/ews/src/answer_tests.rs b/crates/ews/src/answer_tests.rs new file mode 100644 index 000000000..bdca77e34 --- /dev/null +++ b/crates/ews/src/answer_tests.rs @@ -0,0 +1,234 @@ +//! Answers and requests, from the tests of the SOAP layer of the source. + +use proptest::prelude::*; + +use super::*; +use crate::codes::{DEFAULT_BACK_OFF_MS, MAX_BACK_OFF_MS, MIN_BACK_OFF_MS}; +use crate::soap::{Element, Version, envelope, escape}; + +fn reply_with(code: &str, extra: &str) -> String { + format!( + "\ + {code}{extra}\ + " + ) +} + +/// The error the first message of a `GetFolder` answer names. +fn folder_error(body: &str) -> Error { + let answer = Answer::parse(body.as_bytes()).unwrap(); + answer.messages("GetFolder").unwrap()[0].ok().unwrap_err() +} + +#[test] +fn a_busy_server_says_how_long_and_is_believed_only_so_far() { + assert_eq!( + folder_error(&reply_with("ErrorServerBusy", "soon")), + Error::Busy { back_off_ms: DEFAULT_BACK_OFF_MS } + ); + assert_eq!( + folder_error(&reply_with("ErrorServerBusy", "7000")), + Error::Busy { back_off_ms: 7_000 } + ); + assert_eq!( + folder_error(&reply_with("ErrorServerBusy", "9999999999999999999999")), + Error::Busy { back_off_ms: MAX_BACK_OFF_MS } + ); + assert_eq!(folder_error(&reply_with("ErrorServerBusy", "")), Error::Busy { back_off_ms: DEFAULT_BACK_OFF_MS }); + // A wait of nothing is a wait. + assert_eq!( + folder_error(&reply_with("ErrorServerBusy", "0")), + Error::Busy { back_off_ms: MIN_BACK_OFF_MS } + ); + // The whole answer says so. + let answer = Answer::parse(reply_with("ErrorServerBusy", "7000").as_bytes()).unwrap(); + assert_eq!(answer.busy(), Some(Error::Busy { back_off_ms: 7_000 })); + assert_eq!(Answer::parse(reply_with("ErrorAccessDenied", "").as_bytes()).unwrap().busy(), None); +} + +#[test] +fn a_fault_is_the_error_it_names() { + let fault = |code: &str| { + format!( + "s:Client\ + {code}" + ) + }; + assert_eq!(Answer::parse(fault("ErrorSchemaValidation").as_bytes()).unwrap_err(), Error::Server(ResponseCode::SchemaValidation)); + assert_eq!(Answer::parse(fault("ErrorInvalidRequest").as_bytes()).unwrap_err(), Error::Server(ResponseCode::InvalidRequest)); + assert_eq!(Answer::parse(fault("not a code").as_bytes()).unwrap_err(), Error::Server(ResponseCode::Other(String::new()))); +} + +#[test] +fn prefixes_are_the_documents_own_choice() { + let body = "\ + \ + NoErrorInbox\ + "; + let answer = Answer::parse(body.as_bytes()).unwrap(); + let messages = answer.messages("GetFolder").unwrap(); + assert_eq!((messages[0].class(), messages[0].code()), (ResponseClass::Success, &ResponseCode::NoError)); + let folder = messages[0].ok().unwrap().path(&["m:Folders", "t:Folder"]).unwrap(); + assert_eq!(folder.child_attr("t:FolderId", "Id"), Some("AbC+/=-_")); + assert_eq!(folder.child_text("t:DisplayName").as_deref(), Some("Inbox")); +} + +#[test] +fn an_element_in_the_wrong_namespace_is_not_the_one_asked_for() { + let body = reply_with("NoError", "").replace("services/2006/messages", "services/2006/elsewhere"); + assert_eq!(Answer::parse(body.as_bytes()).unwrap().messages("GetFolder").unwrap_err(), Error::Unexpected); + // Nor is another operation's answer. + assert_eq!(Answer::parse(reply_with("NoError", "").as_bytes()).unwrap().messages("GetItem").unwrap_err(), Error::Unexpected); +} + +#[test] +fn unknown_elements_are_skipped() { + let extra = "1"; + assert_eq!(folder_error(&reply_with("ErrorAccessDenied", extra)), Error::Server(ResponseCode::AccessDenied)); +} + +#[test] +fn response_codes_are_kept_only_as_short_identifiers() { + assert_eq!(ResponseCode::from_name(" ErrorServerBusy "), ResponseCode::ServerBusy); + assert_eq!(ResponseCode::from_name("ErrorSomethingNew"), ResponseCode::Other("ErrorSomethingNew".into())); + assert_eq!(ResponseCode::from_name("bad code\r\nX: y"), ResponseCode::Other(String::new())); + assert_eq!(ResponseCode::from_name(&"A".repeat(500)), ResponseCode::Other(String::new())); + assert_eq!(ResponseCode::from_name("bad\r\nX: y").to_string(), "an unnamed error"); +} + +#[test] +fn a_message_without_a_class_or_a_code_is_unexpected() { + let no_class = reply_with("NoError", "").replace(" ResponseClass=\"Error\"", ""); + assert_eq!(Answer::parse(no_class.as_bytes()).unwrap().messages("GetFolder").unwrap_err(), Error::Unexpected); + let odd_class = reply_with("NoError", "").replace("\"Error\"", "\"Maybe\""); + assert_eq!(Answer::parse(odd_class.as_bytes()).unwrap().messages("GetFolder").unwrap_err(), Error::Unexpected); + let no_code = reply_with("NoError", "").replace("NoError", ""); + assert_eq!(Answer::parse(no_code.as_bytes()).unwrap().messages("GetFolder").unwrap_err(), Error::Unexpected); +} + +#[test] +fn hostile_answers_are_errors_never_panics() { + for body in [ + "]>".to_string(), + "".to_string(), + "not xml at all".to_string(), + format!( + "{}{}", + "".repeat(MAX_DEPTH), + "".repeat(MAX_DEPTH) + ), + format!("{}", "".repeat(MAX_NODES)), + ] { + assert_eq!(Answer::parse(body.as_bytes()).unwrap_err(), Error::Malformed, "{}", &body[..body.len().min(80)]); + } + assert_eq!(Answer::parse(&vec![b' '; MAX_BYTES + 1]).unwrap_err(), Error::Malformed); + assert_eq!(Answer::parse(b"Sign in").unwrap_err(), Error::Unexpected); + assert_eq!(Answer::parse(b"").unwrap_err(), Error::Unexpected); + // An entity XML does not define is left out, and the rest is read. + let body = reply_with("Error&nonsense;AccessDenied", ""); + assert_eq!(folder_error(&body), Error::Server(ResponseCode::AccessDenied)); +} + +#[test] +fn a_busy_fault_with_its_wait_in_the_types_namespace_is_read() { + let fault = |wait: &str| { + format!( + "s:Server\ + ErrorServerBusy\ + {wait}\ + " + ) + }; + assert_eq!(Answer::parse(fault("297749").as_bytes()).unwrap_err(), Error::Busy { back_off_ms: 297_749 }); + assert_eq!(Answer::parse(fault("9999999").as_bytes()).unwrap_err(), Error::Busy { back_off_ms: MAX_BACK_OFF_MS }); +} + +#[test] +fn a_busy_server_inside_an_internal_server_error_is_busy() { + let inner = "ErrorServerBusy4000"; + assert_eq!(folder_error(&reply_with("ErrorInternalServerError", inner)), Error::Busy { back_off_ms: 4_000 }); + // Without the inner error it is the server's own error, and not a reason to wait. + let body = + reply_with("ErrorInternalServerError", "4000"); + assert_eq!(folder_error(&body), Error::Server(ResponseCode::InternalServerError)); + // The same in a fault. + let fault = "s:Server\ + ErrorInternalServerError\ + ErrorServerBusy2500"; + assert_eq!(Answer::parse(fault.as_bytes()).unwrap_err(), Error::Busy { back_off_ms: 2_500 }); +} + +#[test] +fn debug_output_shows_nothing_a_server_chose() { + let body = "\ + NoErrorSecret Name\ + "; + let answer = Answer::parse(body.as_bytes()).unwrap(); + let messages = answer.messages("GetFolder").unwrap(); + let shown = format!("{answer:?} {messages:?} {:?}", Element::new("t:DisplayName").attr("Id", "SecretId").text("Secret Name")); + assert!(!shown.contains("Secret") && !shown.contains("Folders"), "{shown}"); +} + +// ---- the request ---- + +#[test] +fn the_request_is_a_soap_envelope_with_the_version_and_one_operation() { + let operation = Element::new("m:GetFolder").child(Element::new("m:FolderIds").child(Element::new("t:DistinguishedFolderId").attr("Id", "inbox"))); + let text = envelope(Version::Exchange2016, &operation); + let tree = parse_xml(text.as_bytes()).unwrap(); + assert_eq!(tree.name, "s:Envelope"); + assert_eq!(tree.path(&["s:Header", "t:RequestServerVersion"]).and_then(|v| v.attr("Version")), Some("Exchange2016")); + let body = tree.child("s:Body").unwrap(); + assert_eq!(body.elements().map(|e| e.name.as_str()).collect::>(), ["m:GetFolder"]); + assert!(envelope(Version::Exchange2010Sp2, &operation).contains("Exchange2010_SP2")); + assert_eq!(Version::NAMES.len(), 9); +} + +#[test] +fn nothing_written_can_break_out_of_its_place() { + let nasty = "a\r\nX-Injected: 1\0\"'&"; + let element = Element::new("m:Op\r\n") && !text.contains("")); + assert_eq!(op.elements().count(), 1); +} + +#[test] +fn a_text_keeps_its_lines() { + let text = "first\r\nsecond\tindented\nthird\0\u{1}\u{FFFE}<&>"; + let written = Element::new("t:Body").text(text).write(); + assert_eq!(written, "first \nsecond\tindented\nthird<&>"); + assert!(!written.contains(['\r', '\0'])); + let tree = parse_xml(envelope(Version::default(), &Element::new("m:Op").text(text)).as_bytes()).unwrap(); + let op = tree.child("s:Body").unwrap().elements().next().unwrap(); + assert_eq!(op.text(), "first\r\nsecond\tindented\nthird<&>"); + // An attribute, or a header, keeps none of it. + assert_eq!(escape("a\r\nb\tc\0"), "abc"); +} + +proptest! { + #[test] + fn arbitrary_bytes_never_panic(bytes in proptest::collection::vec(any::(), 0..4096)) { + if let Ok(answer) = Answer::parse(&bytes) { + let _ = (answer.busy(), answer.messages("GetFolder").map(|m| m.len())); + } + } + + #[test] + fn any_text_written_reads_back_as_itself(text in "\\PC{0,200}", name in "[a-zA-Z][a-zA-Z0-9]{0,10}") { + let tree = parse_xml(envelope(Version::default(), &Element::new(&format!("m:{name}")).attr("A", &text).text(&text)).as_bytes()).unwrap(); + let op = tree.child("s:Body").unwrap().elements().next().unwrap(); + let kept: String = text.chars().filter(|c| !c.is_control() && *c != '\u{FFFE}' && *c != '\u{FFFF}').collect(); + prop_assert_eq!(op.attr("A"), Some(kept.as_str())); + prop_assert_eq!(op.text().into_owned(), kept); + } +} diff --git a/crates/ews/src/codes.rs b/crates/ews/src/codes.rs new file mode 100644 index 000000000..09d6cf757 --- /dev/null +++ b/crates/ews/src/codes.rs @@ -0,0 +1,126 @@ +//! How a response message says it went, and why not ([MS-OXWSCDATA]), and the waits a busy +//! server asks for. + +use std::fmt; + +/// The most a server may ask the client to wait, in milliseconds (five minutes); more is +/// taken as this. +pub const MAX_BACK_OFF_MS: u32 = 300_000; +/// The shortest wait a busy server is given (one second): a server that says 0 is not to be +/// asked again at once. +pub const MIN_BACK_OFF_MS: u32 = 1_000; +/// How long to wait when a server is busy and does not say for how long (one second). +pub const DEFAULT_BACK_OFF_MS: u32 = 1_000; +/// The longest response code kept. +const MAX_CODE: usize = 64; + +/// How a response message says it went (`ResponseClass`, [MS-OXWSCDATA]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ResponseClass { + /// It was done. + Success, + /// It was done, with a remark. + Warning, + /// It was not done. + Error, +} + +impl ResponseClass { + /// The class a document names; none for a word the schema does not have. + pub fn from_name(name: &str) -> Option { + match name { + "Success" => Some(Self::Success), + "Warning" => Some(Self::Warning), + "Error" => Some(Self::Error), + _ => None, + } + } +} + +/// Why a response message says no (`ResponseCodeType`, [MS-OXWSCDATA]): the ones the client +/// acts on by name, and any other as the identifier it was. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum ResponseCode { + /// `NoError`. + NoError, + /// `ErrorServerBusy`: try again after the back-off. + ServerBusy, + /// `ErrorAccessDenied`. + AccessDenied, + /// `ErrorFolderNotFound`. + FolderNotFound, + /// `ErrorItemNotFound`. + ItemNotFound, + /// `ErrorNonExistentMailbox`. + NonExistentMailbox, + /// `ErrorInvalidSyncStateData`: the sync state is no longer known; read afresh. + InvalidSyncState, + /// `ErrorInvalidPropertyRequest`: the server doesn't know a property that was asked for. + InvalidPropertyRequest, + /// `ErrorSchemaValidation`: the request did not fit the schema (a bug here). + SchemaValidation, + /// `ErrorInvalidServerVersion`: the server does not know the version asked for. + InvalidServerVersion, + /// `ErrorInvalidRequest`. + InvalidRequest, + /// `ErrorInternalServerError` or `ErrorInternalServerTransientError`. + InternalServerError, + /// `ErrorMailboxStoreUnavailable` or `ErrorMailboxMoveInProgress`: the mailbox cannot be + /// reached for now. + MailboxUnavailable, + /// `ErrorTimeoutExpired`. + TimeoutExpired, + /// Any other code, as the short identifier it was. + Other(String), +} + +impl ResponseCode { + /// The code a document names. A name that is not a short identifier (letters and digits) + /// is no code and reads as `Other("")`. + pub fn from_name(name: &str) -> Self { + match name.trim() { + "NoError" => Self::NoError, + "ErrorServerBusy" => Self::ServerBusy, + "ErrorAccessDenied" => Self::AccessDenied, + "ErrorFolderNotFound" => Self::FolderNotFound, + "ErrorItemNotFound" => Self::ItemNotFound, + "ErrorNonExistentMailbox" => Self::NonExistentMailbox, + "ErrorInvalidSyncStateData" => Self::InvalidSyncState, + "ErrorInvalidPropertyRequest" => Self::InvalidPropertyRequest, + "ErrorSchemaValidation" => Self::SchemaValidation, + "ErrorInvalidServerVersion" => Self::InvalidServerVersion, + "ErrorInvalidRequest" => Self::InvalidRequest, + "ErrorInternalServerError" | "ErrorInternalServerTransientError" => Self::InternalServerError, + "ErrorMailboxStoreUnavailable" | "ErrorMailboxMoveInProgress" => Self::MailboxUnavailable, + "ErrorTimeoutExpired" => Self::TimeoutExpired, + other if !other.is_empty() && other.len() <= MAX_CODE && other.bytes().all(|b| b.is_ascii_alphanumeric()) => { + Self::Other(other.to_owned()) + } + _ => Self::Other(String::new()), + } + } +} + +impl fmt::Display for ResponseCode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NoError => f.write_str("NoError"), + Self::ServerBusy => f.write_str("ErrorServerBusy"), + Self::AccessDenied => f.write_str("ErrorAccessDenied"), + Self::FolderNotFound => f.write_str("ErrorFolderNotFound"), + Self::ItemNotFound => f.write_str("ErrorItemNotFound"), + Self::NonExistentMailbox => f.write_str("ErrorNonExistentMailbox"), + Self::InvalidSyncState => f.write_str("ErrorInvalidSyncStateData"), + Self::InvalidPropertyRequest => f.write_str("ErrorInvalidPropertyRequest"), + Self::SchemaValidation => f.write_str("ErrorSchemaValidation"), + Self::InvalidServerVersion => f.write_str("ErrorInvalidServerVersion"), + Self::InvalidRequest => f.write_str("ErrorInvalidRequest"), + Self::InternalServerError => f.write_str("ErrorInternalServerError"), + Self::MailboxUnavailable => f.write_str("ErrorMailboxStoreUnavailable"), + Self::TimeoutExpired => f.write_str("ErrorTimeoutExpired"), + Self::Other(code) if code.is_empty() => f.write_str("an unnamed error"), + Self::Other(code) => f.write_str(code), + } + } +} diff --git a/crates/ews/src/lib.rs b/crates/ews/src/lib.rs new file mode 100644 index 000000000..12590250d --- /dev/null +++ b/crates/ews/src/lib.rs @@ -0,0 +1,25 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! Exchange Web Services: the SOAP layer, without sockets. +//! +//! - [`soap`]: the request envelope ([`envelope`]) with the schema version asked for ([`Version`]), +//! and [`Element`], a builder of the operation's elements that escapes everything it is given, so +//! nothing a caller hands in can end an element, open another or break a line of the protocol. +//! - [`answer`]: the answer envelope read with `craft-xml` ([`Answer::parse`]): a fault as the error +//! it names, the response messages of an operation ([`Answer::messages`]) with their class and code +//! ([`codes`]), and a busy server's wait. +//! +//! Written from Microsoft's public documentation: the EWS reference and the Open Specifications +//! [MS-OXWSCDATA] (types) and [MS-OXWSCORE] (operations and their messages). +//! +//! What stays with the caller: HTTP, Autodiscover, authentication, the operations themselves (which +//! elements to send, what to make of the elements that come back) and any sync state. + +pub mod answer; +pub mod codes; +pub mod soap; + +pub use answer::{Answer, Error, ResponseMessage}; +pub use codes::{ResponseClass, ResponseCode}; +pub use soap::{Element, Version, envelope}; diff --git a/crates/ews/src/soap.rs b/crates/ews/src/soap.rs new file mode 100644 index 000000000..8cdb4d78d --- /dev/null +++ b/crates/ews/src/soap.rs @@ -0,0 +1,225 @@ +//! The SOAP envelope of a request, and the elements it carries. +//! +//! Everything written here is escaped: attribute values pass through [`escape`] and +//! lose every control character (CR, LF and NUL among them); text passes through +//! [`escape_text`], which keeps its tabs and line feeds (a body of text keeps its +//! lines), writes a carriage return as a character reference and drops NUL and the other +//! controls XML forbids; and a name keeps only the characters a qualified name is made of. +//! Nothing a caller hands in can therefore end an element, open another, or break a line of the +//! protocol. + +/// The SOAP 1.1 envelope's namespace. +pub const SOAP: &str = "http://schemas.xmlsoap.org/soap/envelope/"; +/// The namespace of the types ([MS-OXWSCDATA]). +pub const TYPES: &str = "http://schemas.microsoft.com/exchange/services/2006/types"; +/// The namespace of the messages (the operations and their answers). +pub const MESSAGES: &str = "http://schemas.microsoft.com/exchange/services/2006/messages"; +/// The namespace of the errors a fault carries. +pub const ERRORS: &str = "http://schemas.microsoft.com/exchange/services/2006/errors"; + +/// The version of the schema a request asks the server to answer in (`RequestServerVersion`, +/// [MS-OXWSCDATA] `ExchangeVersionType`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Version { + /// Exchange 2007. + Exchange2007, + /// Exchange 2007 with service pack 1. + Exchange2007Sp1, + /// Exchange 2010. + Exchange2010, + /// Exchange 2010 with service pack 1. + Exchange2010Sp1, + /// Exchange 2010 with service pack 2. + Exchange2010Sp2, + /// Exchange 2013: what a request asks for unless told otherwise, as every server that + /// speaks the service today understands it. + #[default] + Exchange2013, + /// Exchange 2013 with service pack 1. + Exchange2013Sp1, + /// Exchange 2015. + Exchange2015, + /// Exchange 2016. + Exchange2016, +} + +impl Version { + /// Every name the schema has for a version. + pub const NAMES: [&'static str; 9] = [ + "Exchange2007", + "Exchange2007_SP1", + "Exchange2010", + "Exchange2010_SP1", + "Exchange2010_SP2", + "Exchange2013", + "Exchange2013_SP1", + "Exchange2015", + "Exchange2016", + ]; + + /// The name the schema gives it. + pub fn as_str(self) -> &'static str { + match self { + Self::Exchange2007 => "Exchange2007", + Self::Exchange2007Sp1 => "Exchange2007_SP1", + Self::Exchange2010 => "Exchange2010", + Self::Exchange2010Sp1 => "Exchange2010_SP1", + Self::Exchange2010Sp2 => "Exchange2010_SP2", + Self::Exchange2013 => "Exchange2013", + Self::Exchange2013Sp1 => "Exchange2013_SP1", + Self::Exchange2015 => "Exchange2015", + Self::Exchange2016 => "Exchange2016", + } + } +} + +/// A name as a qualified name may be written: letters, digits, `_`, `-`, `.` and one `:` +/// between the prefix and the local name; anything else is dropped. +fn clean_name(name: &str) -> String { + let mut colon = false; + name.chars() + .filter(|c| match c { + ':' if !colon => { + colon = true; + true + } + c => c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'), + }) + .collect() +} + +/// One element of a request: a name (with its prefix, `m:GetFolder`), attributes, and text and +/// children. +#[derive(Clone, Default, PartialEq, Eq)] +pub struct Element { + name: String, + attributes: Vec<(String, String)>, + text: String, + children: Vec, +} + +impl std::fmt::Debug for Element { + /// The name and how much is inside: the text and the attributes may be somebody's mail. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Element") + .field("name", &self.name) + .field("attributes", &self.attributes.len()) + .field("children", &self.children.len()) + .finish() + } +} + +impl Element { + /// An element called `name`. + pub fn new(name: &str) -> Self { + Self { name: clean_name(name), ..Self::default() } + } + + /// With the attribute `name` set to `value`. + #[must_use] + pub fn attr(mut self, name: &str, value: &str) -> Self { + self.attributes.push((clean_name(name), escape(value))); + self + } + + /// With `text` inside it. + #[must_use] + pub fn text(mut self, text: &str) -> Self { + self.text = escape_text(text); + self + } + + /// With `child` inside it, after the ones before. + #[must_use] + pub fn child(mut self, child: Element) -> Self { + self.children.push(child); + self + } + + /// The element as text. + pub fn write(&self) -> String { + let mut out = String::new(); + self.write_into(&mut out); + out + } + + fn write_into(&self, out: &mut String) { + out.push('<'); + out.push_str(&self.name); + for (name, value) in &self.attributes { + out.push(' '); + out.push_str(name); + out.push_str("=\""); + out.push_str(value); + out.push('"'); + } + if self.text.is_empty() && self.children.is_empty() { + out.push_str("/>"); + return; + } + out.push('>'); + out.push_str(&self.text); + for child in &self.children { + child.write_into(out); + } + out.push_str("'); + } +} + +/// The whole request: a SOAP envelope whose header names `version` and whose body holds +/// `operation` (an element of the messages' namespace, `m:GetFolder`). +pub fn envelope(version: Version, operation: &Element) -> String { + let header = Element::new("t:RequestServerVersion").attr("Version", version.as_str()); + format!( + "\ + {}{}", + header.write(), + operation.write() + ) +} + +/// `text` as it is written inside an attribute (or anywhere a line of a protocol could end): the +/// five characters XML has names for by those names, and nothing that could end a line or a +/// header: every control character (CR, LF, NUL and the tab too), and the characters XML 1.0 +/// forbids (U+FFFE, U+FFFF). +pub fn escape(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '"' => out.push_str("""), + '\'' => out.push_str("'"), + c if c.is_control() || c == '\u{FFFE}' || c == '\u{FFFF}' => {} + c => out.push(c), + } + } + out +} + +/// `text` as it is written inside an element: as [`escape`], except that the tab and the line +/// feed stay (a body of text keeps its lines) and a carriage return is written as ` `, which +/// is not one on the wire. NUL and the other C0 controls, U+FFFE and U+FFFF are dropped. +/// (A `&str` holds no unpaired surrogate.) +pub fn escape_text(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for c in text.chars() { + match c { + '&' => out.push_str("&"), + '<' => out.push_str("<"), + '>' => out.push_str(">"), + '\r' => out.push_str(" "), + c if allowed(c) => out.push(c), + _ => {} + } + } + out +} + +/// Whether XML 1.0 can carry `c` in a text. +fn allowed(c: char) -> bool { + !((c as u32) < 0x20 && !matches!(c, '\t' | '\n' | '\r')) && c != '\u{FFFE}' && c != '\u{FFFF}' +} diff --git a/crates/exr/Cargo.toml b/crates/exr/Cargo.toml index 943402ef8..0c0989bad 100644 --- a/crates/exr/Cargo.toml +++ b/crates/exr/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "craft-exr" -description = "OpenEXR pieces shared by the Craft apps: channel and layer naming, colour channel selection and Cryptomatte (IDs, manifests, rank channels)." +description = "OpenEXR for the Craft apps: channel and layer naming, colour channel selection, Cryptomatte, and (feature `decode`) bounded part listing, flat decoding and writing on top of the exr crate." version.workspace = true edition.workspace = true license.workspace = true @@ -10,5 +10,13 @@ repository.workspace = true [lints] workspace = true +[features] +# Reading and writing pixels through the `exr` crate (BSD-3-Clause). The naming and Cryptomatte +# modules need none of it. +decode = ["dep:exr", "dep:half", "dep:thiserror"] + [dependencies] serde_json = { workspace = true } +exr = { version = "1.73", default-features = false, optional = true } +half = { version = "2", optional = true } +thiserror = { workspace = true, optional = true } diff --git a/crates/exr/src/decode.rs b/crates/exr/src/decode.rs new file mode 100644 index 000000000..c5e906829 --- /dev/null +++ b/crates/exr/src/decode.rs @@ -0,0 +1,546 @@ +//! Bounded OpenEXR reading and writing on top of the `exr` crate (feature `decode`). +//! +//! - [`parts`] lists every part from the headers alone (size, windows, channels, compression, +//! text attributes), after checking the declared sizes against [`Limits`] and the file's size. +//! - [`pick_part`] chooses the part a plain picture decode shows: the best colour channel group +//! ([`crate::channels::color_channels`]), the earliest part on a tie. +//! - [`decode_part`] decodes the channels of one flat part that a filter keeps, in their stored +//! sample type, over the part's data window. Every compression the `exr` crate reads is read, +//! scan lines and tiles alike (the largest resolution level). Other parts' chunks are skipped +//! unread, and channels the filter drops are never allocated. +//! - [`write`] writes one scan-line part. +//! +//! The `exr` crate sizes buffers from headers before it reads pixels, so everything that reaches +//! it is checked first: preview attributes against their own size, each part's samples against +//! [`Limits`], all parts together against the budget, and the samples against what a file of +//! this size can decompress to. + +use std::io::Cursor; + +use exr::block::UncompressedBlock; +use exr::block::chunk::TileCoordinates; +use exr::image::read::image::ReadImage; +use exr::image::read::layers::{ChannelsReader, ReadAllLayers, ReadChannels}; +use exr::meta::MetaData; +use exr::meta::attribute::{AttributeValue, SampleType as ExrSample}; +use exr::meta::header::Header; +use exr::prelude::{ + AnyChannel, AnyChannels, Blocks, Compression as ExrCompression, Encoding, FlatSamples, Layer, LayerAttributes, LineOrder, WritableImage, +}; +pub use half::f16; + +use crate::channels::color_channels; + +/// Why a file could not be read or written. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The file is damaged or not OpenEXR. + #[error("OpenEXR: {0}")] + Malformed(String), + /// A valid file this reader does not cover (deep data in a flat read, HTJ2K, subsampled + /// channels kept by the filter). + #[error("unsupported OpenEXR: {0}")] + Unsupported(String), + /// The file declares more than [`Limits`] allow. + #[error("OpenEXR exceeds a limit: {0}")] + LimitExceeded(String), + /// The picture handed to [`write`] cannot be stored. + #[error("OpenEXR: cannot write: {0}")] + Encode(String), +} + +/// Result of this module's functions. +pub type Result = std::result::Result; + +fn malformed(e: impl std::fmt::Display) -> Error { + Error::Malformed(e.to_string()) +} + +/// An `exr`-crate error, its `NotSupported` kind kept apart from damaged data. +fn map_exr(e: exr::error::Error) -> Error { + match e { + exr::error::Error::NotSupported(message) => Error::Unsupported(message.to_string()), + e => malformed(e), + } +} + +/// Caps on what a file may ask the reader to allocate. `#[non_exhaustive]`: start from +/// [`Limits::default`] and set the fields you change. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub struct Limits { + /// Largest data window width and height of a part. + pub max_width: u32, + pub max_height: u32, + /// Largest data window of a part, in pixels. + pub max_pixels: u64, + /// Most bytes of samples all parts together may declare, counting 4 bytes a sample. A decode + /// allocates at most this for the channels it keeps. + pub max_bytes: u64, + /// The most a compressed byte is taken to expand to (DWA parts get 4 times as much). A file + /// declaring more samples than its size times this, plus 1 MiB, is refused before reading. + pub max_ratio: u64, +} + +impl Default for Limits { + fn default() -> Self { + let max_bytes = if cfg!(target_pointer_width = "64") { 8 << 30 } else { 1 << 30 }; + Limits { max_width: 1 << 18, max_height: 1 << 18, max_pixels: 1 << 30, max_bytes, max_ratio: 1 << 16 } + } +} + +/// How a channel's samples are stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SampleType { + F16, + F32, + U32, +} + +impl SampleType { + fn of(t: ExrSample) -> Self { + match t { + ExrSample::F16 => SampleType::F16, + ExrSample::F32 => SampleType::F32, + ExrSample::U32 => SampleType::U32, + } + } +} + +/// A channel as the header declares it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ChannelInfo { + /// The name as stored (`diffuse.R`); the part name is not prefixed. + pub name: String, + pub sample: SampleType, + /// Horizontal and vertical sampling (1, 1 for a full-resolution channel). + pub sampling: (usize, usize), +} + +/// A window: position and size. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Window { + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, +} + +/// Header facts of one part. +#[derive(Debug, Clone, PartialEq)] +pub struct PartInfo { + /// Position in the file. + pub index: usize, + /// The part's name (`name` attribute), the prefix renderers give its channels. + pub name: Option, + /// The `view` attribute of a stereo part. + pub view: Option, + pub data_window: Window, + pub display_window: Window, + pub deep: bool, + pub tiled: bool, + /// The compression's name as the `exr` crate spells it (`ZIP16`, `PIZ`, `DWAA(None)`, …). + pub compression: String, + pub channels: Vec, + /// Every text attribute of the part and of the shared header (`cryptomatte/…/manifest`, + /// `owner`, …), name and value, sorted by name. + pub text_attributes: Vec<(String, String)>, +} + +impl PartInfo { + /// The value of the text attribute called `name`. + pub fn text(&self, name: &str) -> Option<&str> { + self.text_attributes.iter().find(|(k, _)| k == name).map(|(_, v)| v.as_str()) + } + + /// How much the part looks like a colour picture: 4 RGBA, 3 RGB, 2 YA, 1 Y (or a lone + /// channel), 0 anything else. + pub fn color_rank(&self) -> usize { + color_channels(self.channels.iter().map(|c| c.name.as_str())).map_or(0, |(layout, _)| layout.channels()) + } +} + +/// Refuses headers whose `preview` attributes declare more pixels than their value holds. +/// +/// The `exr` crate sizes a preview's buffer from its declared dimensions (up to 4 MiB, zero +/// filled) before it finds the attribute too short, and in its non-pedantic mode then skips it +/// and goes on: thousands of tiny malformed previews would each cost that allocation. This walks +/// the raw headers and checks each preview against its own size. Anything else odd is left to +/// the crate to report. +fn check_previews(b: &[u8]) -> Result<()> { + if b.get(..4) != Some(&[0x76, 0x2f, 0x31, 0x01][..]) { + return Ok(()); + } + let Some(flags) = b.get(4..8).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_le_bytes) else { return Ok(()) }; + let multipart = flags & 0x1000 != 0; + let cstr = |at: usize| -> Option<(&[u8], usize)> { + let rest = b.get(at..)?; + let n = rest.iter().position(|&c| c == 0)?; + Some((rest.get(..n)?, at + n + 1)) + }; + let u32_at = |at: usize| b.get(at..at.checked_add(4)?).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_le_bytes); + let mut at = 8; + loop { + // One header's attributes: name, type, size, value; an empty name ends the header. + loop { + let Some((name, next)) = cstr(at) else { return Ok(()) }; + at = next; + if name.is_empty() { + break; + } + let Some((kind, next)) = cstr(at) else { return Ok(()) }; + let Some(size) = u32_at(next).and_then(|n| i32::try_from(n).ok()).and_then(|n| usize::try_from(n).ok()) else { return Ok(()) }; + let value = next + 4; + let end = match value.checked_add(size) { + Some(end) if end <= b.len() => end, + _ => return Ok(()), + }; + if kind == b"preview" { + let declared = u32_at(value) + .zip(u32_at(value + 4)) + .and_then(|(w, h)| u64::from(w).checked_mul(u64::from(h))?.checked_mul(4)) + .filter(|_| size >= 8); + if declared.is_none_or(|n| n > (size - 8) as u64) { + return Err(malformed("a preview attribute declares more pixels than it holds")); + } + } + at = end; + } + // A multi-part header list ends with an empty header (a lone NUL). + if !multipart || b.get(at) == Some(&0) { + return Ok(()); + } + } +} + +fn window_size(size: exr::math::Vec2) -> (u32, u32) { + (u32::try_from(size.0).unwrap_or(u32::MAX), u32::try_from(size.1).unwrap_or(u32::MAX)) +} + +/// Reads the headers and checks every part's declared size, alone and together, before any pixel +/// allocation. Headers are read without the crate's validation, which refuses deep parts. +fn read_meta(bytes: &[u8], limits: &Limits) -> Result { + check_previews(bytes)?; + let meta = MetaData::read_from_buffered(Cursor::new(bytes), false).map_err(map_exr)?; + let (mut total, mut dwa) = (0u64, 0u64); + for header in meta.headers.iter() { + let (w, h) = window_size(header.layer_size); + if w > limits.max_width || h > limits.max_height { + return Err(Error::LimitExceeded(format!("a {w}x{h} part exceeds {}x{}", limits.max_width, limits.max_height))); + } + let pixels = u64::from(w) * u64::from(h); + if pixels > limits.max_pixels { + return Err(Error::LimitExceeded(format!("a part of {pixels} pixels exceeds {}", limits.max_pixels))); + } + let bytes = pixels.saturating_mul(header.channels.list.len().max(1) as u64 * 4); + total = total.saturating_add(bytes); + if matches!(header.compression, ExrCompression::DWAA(_) | ExrCompression::DWAB(_)) { + dwa = dwa.saturating_add(bytes); + } + } + if total > limits.max_bytes { + return Err(Error::LimitExceeded(format!("{total} bytes across {} parts exceed {}", meta.headers.len(), limits.max_bytes))); + } + // The crate allocates a flat part's samples before reading its chunks, so the file must be + // able to hold them. DWA's run-length channels are zlib-packed on top and reach about + // 132000:1, so DWA parts are counted at a quarter. Deep parts size their samples from the + // file's own counts. + if !meta.requirements.has_deep_data { + let declared = total.saturating_sub(dwa).saturating_add(dwa / 4); + let capacity = (bytes.len() as u64).saturating_mul(limits.max_ratio).saturating_add(1 << 20); + if declared > capacity { + return Err(Error::LimitExceeded(format!("a {} byte file cannot hold {declared} bytes of samples", bytes.len()))); + } + } + Ok(meta) +} + +fn text_attributes(header: &Header) -> Vec<(String, String)> { + let mut out: Vec<(String, String)> = header + .shared_attributes + .other + .iter() + .chain(header.own_attributes.other.iter()) + .filter_map(|(k, v)| match v { + AttributeValue::Text(t) => Some((k.to_string(), t.to_string())), + _ => None, + }) + .collect(); + out.sort(); + out.dedup_by(|a, b| a.0 == b.0); + out +} + +fn part_info(index: usize, header: &Header) -> PartInfo { + let (w, h) = window_size(header.layer_size); + let display = &header.shared_attributes.display_window; + let (dw, dh) = window_size(display.size); + PartInfo { + index, + name: header.own_attributes.layer_name.as_ref().map(|n| n.to_string()), + view: header.own_attributes.view_name.as_ref().map(|v| v.to_string()), + data_window: Window { x: header.own_attributes.layer_position.0, y: header.own_attributes.layer_position.1, width: w, height: h }, + display_window: Window { x: display.position.0, y: display.position.1, width: dw, height: dh }, + deep: header.deep, + tiled: matches!(header.blocks, exr::meta::BlockDescription::Tiles(_)), + compression: format!("{:?}", header.compression), + channels: header + .channels + .list + .iter() + .map(|c| ChannelInfo { name: c.name.to_string(), sample: SampleType::of(c.sample_type), sampling: (c.sampling.0, c.sampling.1) }) + .collect(), + text_attributes: text_attributes(header), + } +} + +/// Lists every part of the file from its headers, after checking the declared sizes. +pub fn parts(bytes: &[u8], limits: &Limits) -> Result> { + let meta = read_meta(bytes, limits)?; + Ok(meta.headers.iter().enumerate().map(|(i, h)| part_info(i, h)).collect()) +} + +/// The part a plain picture decode shows: the highest [`PartInfo::color_rank`], the earliest on a +/// tie. `None` when no part has colour channels. +pub fn pick_part(parts: &[PartInfo]) -> Option { + let mut best: Option<(usize, usize)> = None; + for p in parts { + let rank = p.color_rank(); + if rank > 0 && best.is_none_or(|(_, r)| rank > r) { + best = Some((p.index, rank)); + } + } + best.map(|(i, _)| i) +} + +/// The samples of a decoded channel, in their stored type. +#[derive(Debug, Clone, PartialEq)] +pub enum Samples { + F16(Vec), + F32(Vec), + U32(Vec), +} + +impl Samples { + pub fn len(&self) -> usize { + match self { + Samples::F16(v) => v.len(), + Samples::F32(v) => v.len(), + Samples::U32(v) => v.len(), + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Sample `i` as `f32` (U32 IDs exact up to 2^24). 0 past the end. + pub fn get_f32(&self, i: usize) -> f32 { + match self { + Samples::F16(v) => v.get(i).map_or(0.0, |s| s.to_f32()), + Samples::F32(v) => v.get(i).copied().unwrap_or(0.0), + Samples::U32(v) => v.get(i).map_or(0.0, |&s| s as f32), + } + } + + /// All samples as `f32`. + pub fn to_f32(&self) -> Vec { + match self { + Samples::F16(v) => v.iter().map(|s| s.to_f32()).collect(), + Samples::F32(v) => v.clone(), + Samples::U32(v) => v.iter().map(|&s| s as f32).collect(), + } + } +} + +/// A decoded channel. +#[derive(Debug, Clone, PartialEq)] +pub struct Channel { + /// The name as stored. + pub name: String, + /// `width × height` samples of the part's data window, top row first. + pub samples: Samples, +} + +/// A decoded flat part: its header facts and the channels the filter kept, in file order. +#[derive(Debug, Clone, PartialEq)] +pub struct Part { + pub info: PartInfo, + pub channels: Vec, +} + +impl Part { + /// The kept channel called `name`. + pub fn channel(&self, name: &str) -> Option<&Channel> { + self.channels.iter().find(|c| c.name == name) + } +} + +/// Places a data-window plane in a display window: samples outside the display window are cut, +/// the display window's pixels the data window does not cover are `fill`. `None` when the plane +/// is not `data.width × data.height` samples or the display window has more than `max_pixels` +/// pixels. +pub fn to_display_window(plane: &[f32], data: Window, display: Window, fill: f32, max_pixels: u64) -> Option> { + let (w, h) = (data.width as usize, data.height as usize); + if plane.len() != w.checked_mul(h)? || u64::from(display.width) * u64::from(display.height) > max_pixels { + return None; + } + let (dw, dh) = (display.width as usize, display.height as usize); + let mut out = vec![fill; dw.checked_mul(dh)?]; + let (ox, oy) = (i64::from(data.x) - i64::from(display.x), i64::from(data.y) - i64::from(display.y)); + for (y, row) in plane.chunks_exact(w.max(1)).enumerate() { + let Some(ty) = usize::try_from(y as i64 + oy).ok().filter(|ty| *ty < dh) else { continue }; + let skip = usize::try_from(-ox).unwrap_or(0).min(w); + let tx = usize::try_from(ox.max(0)).unwrap_or(usize::MAX); + let take = (w - skip).min(dw.saturating_sub(tx)); + if take == 0 { + continue; + } + let at = ty * dw + tx; + if let (Some(dst), Some(src)) = (out.get_mut(at..at + take), row.get(skip..skip + take)) { + dst.copy_from_slice(src); + } + } + Some(out) +} + +/// Which channels of which part a decode keeps. +struct Select<'k> { + index: usize, + keep: &'k dyn Fn(&str) -> bool, + seen: std::cell::Cell, +} + +/// One part's kept channels while it is decoded: each line goes straight into its channel's plane. +struct SelectReader { + /// Per channel of the part: its plane, `None` when not kept. + planes: Vec>, + width: usize, + height: usize, +} + +impl<'s> ReadChannels<'s> for Select<'_> { + type Reader = SelectReader; + + fn create_channels_reader(&'s self, header: &Header) -> exr::error::Result { + let position = self.seen.get(); + self.seen.set(position.saturating_add(1)); + let (width, height) = (header.layer_size.0, header.layer_size.1); + if position != self.index { + // Another part: keep nothing, so its chunks are skipped unread. + return Ok(SelectReader { planes: header.channels.list.iter().map(|_| None).collect(), width, height }); + } + let n = width.checked_mul(height).ok_or(exr::error::Error::Invalid("part size".into()))?; + let mut planes = Vec::with_capacity(header.channels.list.len()); + for c in header.channels.list.iter() { + if !(self.keep)(&c.name.to_string()) { + planes.push(None); + continue; + } + if c.sampling.0 != 1 || c.sampling.1 != 1 { + return Err(exr::error::Error::NotSupported("subsampled channels".into())); + } + planes.push(Some(match c.sample_type { + ExrSample::F16 => Samples::F16(vec![f16::ZERO; n]), + ExrSample::F32 => Samples::F32(vec![0.0; n]), + ExrSample::U32 => Samples::U32(vec![0; n]), + })); + } + Ok(SelectReader { planes, width, height }) + } +} + +impl ChannelsReader for SelectReader { + type Channels = Vec>; + + fn filter_block(&self, tile: TileCoordinates) -> bool { + tile.is_largest_resolution_level() && self.planes.iter().any(Option::is_some) + } + + fn read_block(&mut self, header: &Header, block: UncompressedBlock) -> exr::error::UnitResult { + for line in block.lines(&header.channels) { + let Some(Some(plane)) = self.planes.get_mut(line.location.channel) else { continue }; + let (x, y, count) = (line.location.position.0, line.location.position.1, line.location.sample_count); + if y >= self.height || x.checked_add(count).is_none_or(|end| end > self.width) { + continue; + } + let at = y * self.width + x; + match plane { + Samples::F16(v) => line.read_samples_into_slice(v.get_mut(at..at + count).unwrap_or_default())?, + Samples::F32(v) => line.read_samples_into_slice(v.get_mut(at..at + count).unwrap_or_default())?, + Samples::U32(v) => line.read_samples_into_slice(v.get_mut(at..at + count).unwrap_or_default())?, + } + } + Ok(()) + } + + fn into_channels(self) -> Vec> { + self.planes + } +} + +/// Decodes the channels of flat part `index` that `keep` accepts (by the name as stored), in +/// their stored type. A file with deep parts, or a kept channel that is subsampled, is +/// [`Error::Unsupported`]. +pub fn decode_part(bytes: &[u8], index: usize, keep: &dyn Fn(&str) -> bool, limits: &Limits) -> Result { + let meta = read_meta(bytes, limits)?; + if meta.requirements.has_deep_data { + return Err(Error::Unsupported("deep data (use the deep reader)".into())); + } + let header = meta.headers.get(index).ok_or_else(|| malformed(format!("part {index} of {}", meta.headers.len())))?; + if matches!(header.compression, ExrCompression::HTJ2K32 | ExrCompression::HTJ2K256) { + return Err(Error::Unsupported("HTJ2K compression".into())); + } + let info = part_info(index, header); + let select = Select { index, keep, seen: std::cell::Cell::new(0) }; + let image = ReadImage::new(ReadAllLayers { read_channels: select }, |_| {}).from_buffered(Cursor::new(bytes)).map_err(map_exr)?; + let planes = image.layer_data.into_iter().nth(index).map(|l| l.channel_data).ok_or_else(|| malformed("the part was not read"))?; + let channels = info.channels.iter().zip(planes).filter_map(|(c, p)| Some(Channel { name: c.name.clone(), samples: p? })).collect(); + Ok(Part { info, channels }) +} + +/// Compression for [`write`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum Compression { + None, + Rle, + Zip1, + #[default] + Zip16, + Piz, +} + +/// Writes one scan-line part of `width × height` with these channels (each `width × height` +/// samples, top row first; the order given does not matter, the file stores them sorted). +pub fn write(width: u32, height: u32, channels: &[Channel], compression: Compression) -> Result> { + let enc = |m: &str| Error::Encode(m.into()); + if width == 0 || height == 0 || channels.is_empty() { + return Err(enc("empty picture")); + } + let n = (width as usize).checked_mul(height as usize).ok_or_else(|| enc("picture too large"))?; + if channels.iter().any(|c| c.samples.len() != n) { + return Err(enc("a channel has the wrong number of samples")); + } + let mut list = Vec::with_capacity(channels.len()); + for c in channels { + let name = exr::meta::attribute::Text::new_or_none(c.name.as_str()).ok_or_else(|| enc("a channel name is not valid OpenEXR text"))?; + let samples = match &c.samples { + Samples::F16(v) => FlatSamples::F16(v.clone()), + Samples::F32(v) => FlatSamples::F32(v.clone()), + Samples::U32(v) => FlatSamples::U32(v.clone()), + }; + list.push(AnyChannel::new(name, samples)); + } + let compression = match compression { + Compression::None => ExrCompression::Uncompressed, + Compression::Rle => ExrCompression::RLE, + Compression::Zip1 => ExrCompression::ZIP1, + Compression::Zip16 => ExrCompression::ZIP16, + Compression::Piz => ExrCompression::PIZ, + }; + let encoding = Encoding { compression, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }; + let layer = Layer::new((width as usize, height as usize), LayerAttributes::default(), encoding, AnyChannels::sort(list.into_iter().collect())); + let mut out = Cursor::new(Vec::new()); + exr::image::Image::from_layer(layer).write().to_buffered(&mut out).map_err(|e| Error::Encode(e.to_string()))?; + Ok(out.into_inner()) +} diff --git a/crates/exr/src/decode_tests.rs b/crates/exr/src/decode_tests.rs new file mode 100644 index 000000000..69d8d07ac --- /dev/null +++ b/crates/exr/src/decode_tests.rs @@ -0,0 +1,200 @@ +//! The preview check is PhotoCraft's test; the rest builds files with the `exr` crate. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use exr::meta::attribute::{Preview, Text}; +use exr::prelude::{ + AnyChannel, AnyChannels, Blocks, Encoding, FlatSamples, ImageAttributes, IntegerBounds, Layer, LayerAttributes, LineOrder, WritableImage, +}; + +use super::decode::*; + +fn ramp(n: usize, k: f32) -> Vec { + (0..n).map(|i| i as f32 * k).collect() +} + +/// A layer of `w × h` with these channels of f32 ramps, scan lines or 4 × 4 tiles. +fn layer( + name: Option<&str>, + w: usize, + h: usize, + channels: &[&str], + tiles: bool, + compression: exr::prelude::Compression, +) -> Layer> { + let mut attrs = name.map_or_else(LayerAttributes::default, LayerAttributes::named); + attrs.other.insert(Text::from("cryptomatte/abc1234/name"), exr::meta::attribute::AttributeValue::Text(Text::from("CryptoObject"))); + let list = channels.iter().enumerate().map(|(i, n)| AnyChannel::new(*n, FlatSamples::F32(ramp(w * h, i as f32 + 1.0)))).collect(); + let blocks = if tiles { Blocks::Tiles(exr::math::Vec2(4, 4)) } else { Blocks::ScanLines }; + Layer::new((w, h), attrs, Encoding { compression, blocks, line_order: LineOrder::Increasing }, AnyChannels::sort(list)) +} + +fn file(layers: Vec>>, display: (usize, usize)) -> Vec { + let image = exr::image::Image::from_layers(ImageAttributes::new(IntegerBounds::new((0, 0), display)), layers); + let mut out = std::io::Cursor::new(Vec::new()); + image.write().to_buffered(&mut out).unwrap(); + out.into_inner() +} + +fn all(_: &str) -> bool { + true +} + +#[test] +fn every_compression_scan_lines_and_tiles_decode_exactly() { + use exr::prelude::Compression::*; + for compression in [Uncompressed, RLE, ZIP1, ZIP16, PIZ] { + for tiles in [false, true] { + let bytes = file(vec![layer(None, 9, 7, &["R", "G", "B"], tiles, compression)], (9, 7)); + let part = decode_part(&bytes, 0, &all, &Limits::default()).unwrap(); + assert_eq!((part.info.tiled, part.info.data_window.width, part.info.data_window.height), (tiles, 9, 7)); + // Each channel kept its own ramp, whatever order the file stores them in. + for (name, k) in [("R", 1.0), ("G", 2.0), ("B", 3.0)] { + assert_eq!(part.channel(name).unwrap().samples, Samples::F32(ramp(63, k)), "{compression:?} tiles {tiles} {name}"); + } + } + } + // Lossy ones decode too (values not exact). + for compression in [PXR24, B44, B44A, DWAA(None), DWAB(None)] { + let bytes = file(vec![layer(None, 16, 16, &["R"], false, compression)], (16, 16)); + assert_eq!(decode_part(&bytes, 0, &all, &Limits::default()).unwrap().channels[0].samples.len(), 256, "{compression:?}"); + } +} + +#[test] +fn multi_part_files_list_and_decode_one_part() { + let bytes = file( + vec![ + layer(Some("depth"), 4, 2, &["Z"], false, exr::prelude::Compression::ZIP1), + layer(Some("beauty"), 4, 2, &["A", "B", "G", "R"], true, exr::prelude::Compression::PIZ), + ], + (4, 2), + ); + let parts = parts(&bytes, &Limits::default()).unwrap(); + assert_eq!(parts.len(), 2); + assert_eq!((parts[0].name.as_deref(), parts[1].name.as_deref(), parts[1].tiled), (Some("depth"), Some("beauty"), true)); + assert_eq!(parts[0].channels, [ChannelInfo { name: "Z".into(), sample: SampleType::F32, sampling: (1, 1) }]); + assert_eq!((parts[0].color_rank(), parts[1].color_rank(), parts[1].compression.as_str()), (1, 4, "PIZ")); + assert_eq!(parts[1].text("cryptomatte/abc1234/name"), Some("CryptoObject")); + assert_eq!(pick_part(&parts), Some(1)); + // Only the kept channels of the chosen part come back. + let part = decode_part(&bytes, 1, &|n| n == "R" || n == "A", &Limits::default()).unwrap(); + assert_eq!(part.channels.iter().map(|c| c.name.as_str()).collect::>(), ["A", "R"]); + let z = decode_part(&bytes, 0, &all, &Limits::default()).unwrap(); + assert_eq!(z.channels[0].samples.to_f32(), ramp(8, 1.0)); + assert!(matches!(decode_part(&bytes, 2, &all, &Limits::default()), Err(Error::Malformed(_)))); + assert_eq!(pick_part(&[]), None); +} + +#[test] +fn written_files_read_back() { + let h: Vec = (0..6).map(|i| f16::from_f32(i as f32 / 4.0)).collect(); + let channels = vec![ + Channel { name: "Y".into(), samples: Samples::F16(h.clone()) }, + Channel { name: "A".into(), samples: Samples::F32(ramp(6, 0.5)) }, + Channel { name: "id".into(), samples: Samples::U32((0..6).collect()) }, + ]; + for c in [Compression::None, Compression::Rle, Compression::Zip1, Compression::Zip16, Compression::Piz] { + let bytes = write(3, 2, &channels, c).unwrap(); + let part = decode_part(&bytes, 0, &all, &Limits::default()).unwrap(); + assert_eq!(part.channel("Y").unwrap().samples, Samples::F16(h.clone())); + assert_eq!(part.channel("A").unwrap().samples, Samples::F32(ramp(6, 0.5))); + assert_eq!(part.channel("id").unwrap().samples, Samples::U32((0..6).collect())); + assert_eq!(part.channel("id").unwrap().samples.get_f32(5), 5.0); + } + assert!(matches!(write(0, 2, &channels, Compression::None), Err(Error::Encode(_)))); + assert!(matches!(write(2, 2, &channels, Compression::None), Err(Error::Encode(_)))); + assert!(matches!(write(3, 2, &[], Compression::None), Err(Error::Encode(_)))); +} + +#[test] +fn display_window_placement() { + let data = Window { x: 1, y: -1, width: 2, height: 2 }; + let display = Window { x: 0, y: 0, width: 3, height: 2 }; + // Row -1 is cut; row 0 lands at display row 0 from column 1. + assert_eq!(to_display_window(&[1.0, 2.0, 3.0, 4.0], data, display, 9.0, 100).unwrap(), [9.0, 3.0, 4.0, 9.0, 9.0, 9.0]); + assert_eq!(to_display_window(&[1.0], data, display, 0.0, 100), None); + assert_eq!(to_display_window(&[1.0, 2.0, 3.0, 4.0], data, display, 0.0, 5), None); + let far = Window { x: -10, y: 0, width: 2, height: 2 }; + assert_eq!(to_display_window(&[1.0, 2.0, 3.0, 4.0], far, display, 0.0, 100).unwrap(), [0.0; 6]); +} + +/// A 2x2 single-part file whose layer carries a 2x1 preview. +fn with_preview() -> Vec { + let mut attrs = LayerAttributes::named("rgb"); + attrs.preview = Some(Preview { size: exr::math::Vec2(2, 1), pixel_data: vec![7; 8] }); + let list = ["R", "G", "B"].into_iter().map(|n| AnyChannel::new(n, FlatSamples::F32(vec![0.5; 4]))).collect(); + let layer = Layer::new( + (2, 2), + attrs, + Encoding { compression: exr::prelude::Compression::Uncompressed, blocks: Blocks::ScanLines, line_order: LineOrder::Increasing }, + AnyChannels::sort(list), + ); + file(vec![layer], (2, 2)) +} + +#[test] +fn a_preview_declaring_more_pixels_than_it_holds_is_refused_before_the_crate_reads_it() { + let good = with_preview(); + assert_eq!(decode_part(&good, 0, &all, &Limits::default()).unwrap().channels.len(), 3); + // Patch the preview to 1024x1024 (4 MiB) while its value still holds 8 bytes. + let tag = b"preview\0preview\0"; + let at = good.windows(tag.len()).position(|w| w == tag).unwrap() + tag.len() + 4; + let mut bad = good.clone(); + bad[at..at + 4].copy_from_slice(&1024u32.to_le_bytes()); + bad[at + 4..at + 8].copy_from_slice(&1024u32.to_le_bytes()); + for r in [parts(&bad, &Limits::default()).map(|_| ()), decode_part(&bad, 0, &all, &Limits::default()).map(|_| ())] { + assert!(matches!(&r, Err(Error::Malformed(m)) if m.contains("preview")), "{r:?}"); + } + // A header cut off mid-attribute is left to the crate to report. + assert!(parts(&good[..at], &Limits::default()).is_err()); +} + +#[test] +fn limits_are_checked_from_the_headers() { + let bytes = file(vec![layer(None, 8, 4, &["R", "G"], false, exr::prelude::Compression::ZIP16)], (8, 4)); + let ok = |l: Limits| parts(&bytes, &l).is_ok(); + let d = Limits::default(); + assert!(ok(Limits { max_width: 8, max_height: 4, max_pixels: 32, max_bytes: 8 * 4 * 2 * 4, ..d })); + assert!(matches!(parts(&bytes, &Limits { max_width: 7, ..d }), Err(Error::LimitExceeded(_)))); + assert!(matches!(parts(&bytes, &Limits { max_height: 3, ..d }), Err(Error::LimitExceeded(_)))); + assert!(matches!(parts(&bytes, &Limits { max_pixels: 31, ..d }), Err(Error::LimitExceeded(_)))); + assert!(matches!(decode_part(&bytes, 0, &all, &Limits { max_bytes: 255, ..d }), Err(Error::LimitExceeded(_)))); + // A file too small for what it declares: 256 bytes of samples against a ratio of 0 and no slack + // is over; the 1 MiB slack always admits small pictures. + assert!(ok(Limits { max_ratio: 0, ..d })); + // A header claiming a huge data window over a few bytes of file is refused by the ratio. + let big = file(vec![layer(None, 4096, 4096, &["R"], false, exr::prelude::Compression::ZIP16)], (4096, 4096)); + assert!(ok_big(&big)); + assert!(matches!(parts(&big, &Limits { max_ratio: 1, ..d }), Err(Error::LimitExceeded(_)))); +} + +fn ok_big(b: &[u8]) -> bool { + parts(b, &Limits::default()).is_ok() +} + +#[test] +fn hostile_files_never_panic() { + let good = file( + vec![ + layer(Some("a"), 5, 3, &["R", "G", "B"], false, exr::prelude::Compression::ZIP1), + layer(Some("b"), 5, 3, &["Y"], true, exr::prelude::Compression::RLE), + ], + (5, 3), + ); + let limits = Limits { max_bytes: 1 << 24, ..Limits::default() }; + for n in 0..good.len() { + let _ = parts(&good[..n], &limits); + let _ = decode_part(&good[..n], 1, &all, &limits); + } + for i in 0..good.len() { + for v in [0u8, 1, 0x7F, 0xFF] { + let mut b = good.clone(); + b[i] = v; + if let Ok(p) = decode_part(&b, 0, &all, &limits) { + let n = p.info.data_window.width as usize * p.info.data_window.height as usize; + assert!(p.channels.iter().all(|c| c.samples.len() == n)); + } + } + } + assert!(parts(b"not exr", &limits).is_err()); +} diff --git a/crates/exr/src/lib.rs b/crates/exr/src/lib.rs index 8075f2516..c9e642364 100644 --- a/crates/exr/src/lib.rs +++ b/crates/exr/src/lib.rs @@ -10,7 +10,14 @@ //! - [`cryptomatte`]: Cryptomatte (specification 1.2) IDs, metadata keys, manifests and rank //! channel names. //! -//! Nothing here reads pixels or depends on a decoder: the functions take names and attribute text. +//! Those take names and attribute text and depend on no decoder. With the `decode` feature: +//! +//! - [`decode`]: bounded part listing, flat decoding (scan lines and tiles, every compression the +//! `exr` crate reads) and writing, on top of the `exr` crate. pub mod channels; pub mod cryptomatte; +#[cfg(feature = "decode")] +pub mod decode; +#[cfg(all(test, feature = "decode"))] +mod decode_tests; diff --git a/crates/font-catalog/src/catalog.rs b/crates/font-catalog/src/catalog.rs index c20753fb9..10663e6af 100644 --- a/crates/font-catalog/src/catalog.rs +++ b/crates/font-catalog/src/catalog.rs @@ -22,6 +22,19 @@ const MAX_SCAN_READ: u64 = 256 << 20; /// The largest font file loaded. const MAX_LOAD: u64 = 512 << 20; +/// The bytes of the file at `path`, at most `max`: the size was checked before, but a file can +/// grow between that check and the read, so the read itself stops one byte past the limit. +pub(crate) fn read_capped(path: &Path, max: u64) -> Result, FontFileError> { + use std::io::Read as _; + let mut bytes = Vec::new(); + std::fs::File::open(path)?.take(max.saturating_add(1)).read_to_end(&mut bytes)?; + let size = bytes.len() as u64; + if size > max { + return Err(FontFileError::TooLarge { size, max }); + } + Ok(bytes) +} + /// Why a font file or font data wasn't added or loaded. #[derive(Debug, thiserror::Error)] pub enum FontFileError { @@ -544,7 +557,7 @@ impl Catalog { if size > MAX_LOAD { return Err(FontFileError::TooLarge { size, max: MAX_LOAD }); } - let bytes: Arc<[u8]> = std::fs::read(path)?.into(); + let bytes: Arc<[u8]> = read_capped(path, MAX_LOAD)?.into(); let mut files = lock(&self.files); files.retain(|_, w| w.strong_count() > 0); files.insert(path.to_path_buf(), Arc::downgrade(&bytes)); diff --git a/crates/font-catalog/src/tests.rs b/crates/font-catalog/src/tests.rs index c061ca2aa..5137a4295 100644 --- a/crates/font-catalog/src/tests.rs +++ b/crates/font-catalog/src/tests.rs @@ -556,3 +556,15 @@ fn repeated_name_records_are_bounded_before_pairing() { assert!(!face.keys.legacy.is_empty()); assert_eq!(face.keys.legacy.len(), 256, "the pair budget counts distinct pairs"); } + +/// A read stops at its cap, whatever the file has grown to since its size was checked. +#[test] +fn a_read_is_capped_where_the_file_has_grown() { + use crate::catalog::read_capped; + let path = std::env::temp_dir().join(format!("craft-font-catalog-cap-{}", std::process::id())); + std::fs::write(&path, b"0123456789").unwrap(); + assert_eq!(read_capped(&path, 10).unwrap(), b"0123456789"); + assert!(matches!(read_capped(&path, 9), Err(FontFileError::TooLarge { size: 10, max: 9 }))); + assert!(matches!(read_capped(&path.with_extension("missing"), 9), Err(FontFileError::Io(_)))); + let _ = std::fs::remove_file(&path); +} diff --git a/crates/ical/Cargo.toml b/crates/ical/Cargo.toml new file mode 100644 index 000000000..b533c1386 --- /dev/null +++ b/crates/ical/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-ical" +description = "iCalendar (RFC 5545) building blocks: content lines, values, recurrence rules and VTIMEZONE definitions, read and written." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +chrono = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/ical/README.md b/crates/ical/README.md new file mode 100644 index 000000000..1781ffc3b --- /dev/null +++ b/crates/ical/README.md @@ -0,0 +1,56 @@ +# craft-ical + +iCalendar (RFC 5545) building blocks, read and written. Times are `chrono` values; instants are +`i64` seconds since 1970-01-01T00:00:00Z. + +- **`lines`**: `Lines` unfolds a text (CRLF, LF or CR; a byte order mark is skipped); + `ContentLine::parse` splits `NAME;PARAM="quoted",x:value` into an upper-cased name, + parameters (RFC 6868 caret escapes undone) and the value as written; `fold` writes a line in + 75-octet pieces without splitting a character; `param_value` quotes and escapes a parameter. +- **`values`**: `When` (`DATE`, floating or zoned `DATE-TIME`, UTC `DATE-TIME`, with the ISO 8601 + punctuation some writers add), `parse_duration`, `escape_text` / `unescape_text`, `split_list`, + `format_date` / `format_local` / `format_utc`, and the wall-clock helpers they rest on. +- **`rrule`**: `Recurrence`, the subset of RRULE calendars show and edit (daily, weekly, monthly, + yearly; interval; weekdays; the nth or last weekday; count; until), `parse_rrule` and + `format_rrule`, and `format_for` / `format_zoned`, which spell out the start's weekday and + month and give `UNTIL` the start's value type. A rule outside the subset is read as the nearest + it holds. +- **`zones`**: `TimeZones::collect` reads a text's `VTIMEZONE`s (yearly rules by weekday or day + of the month, `UNTIL`, `COUNT`, `RDATE`s); a `VTimezone` answers `offset_at(instant)`, + `offset_for_wall(wall)` and `instant(wall)`, with RFC 5545's reading of skipped and repeated + hours. `write_vtimezone` writes a zone from the caller's `ZoneRules` (offset at an instant, + transitions): one `DAYLIGHT` and one `STANDARD` part with yearly rules where the zone keeps one + rule, else change by change. + +No time-zone database is here: matching a `TZID` to an IANA or Windows zone, and which zone an +event is shown in, are the caller's. So is what events and to-dos mean: an app reads them from +`ContentLine`s with these values and maps them onto its own model. + +## Hostile input + +Nothing panics on any text. + +| Limit | Value | +|---|---| +| unfolded line (`lines::MAX_LINE`) | 1 MiB | +| parameters on a line, values of a parameter | 64 each | +| recurrence rule | 4096 bytes, 64 parts | +| durations | about 10,000 years | +| zones per text (`zones::MAX_ZONES`) | 64 | +| parts per zone, dates per part | 64 each | +| `TZID` kept (`zones::MAX_TZID`) | 128 bytes | +| offsets | at most 18 hours | +| work on zone definitions per text (`zones::WORK_BUDGET`) | 20,000,000 units, spent by the caller through `TimeZones::spend` | + +Every date and time written is in the years 1 to 9999; a value outside them is moved to the +nearest end. + +## Origin + +MailCraft `crates/ical` (`lines.rs`, `values.rs`, `rrule.rs`, `zones.rs`) at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors (MIT OR Apache-2.0), with their tests. MailCraft's +own date types are replaced by `chrono`, its time-zone database by the `ZoneRules` trait (the +writer's tests use rule zones built in the test), and its recurrence model by `Recurrence`. +MailCraft's event and to-do reader and writer (`read.rs`, `write.rs`), and the matching of a +`TZID` against its zone database, stay in MailCraft as the mapping onto its model. diff --git a/crates/ical/proptest-regressions/lines.txt b/crates/ical/proptest-regressions/lines.txt new file mode 100644 index 000000000..bd55905b3 --- /dev/null +++ b/crates/ical/proptest-regressions/lines.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 960cb45b7c8511f655aea177ca4d647864fc5cf713e5fbdd611ddba6ed077047 # shrinks to line = "\u{feff}\u{feff}" diff --git a/crates/ical/src/lib.rs b/crates/ical/src/lib.rs new file mode 100644 index 000000000..be77be9e7 --- /dev/null +++ b/crates/ical/src/lib.rs @@ -0,0 +1,33 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! iCalendar (RFC 5545): the layers a calendar program reads and writes `.ics` text with. +//! +//! - [`lines`]: content lines. [`Lines`] unfolds a text, [`ContentLine::parse`] splits a line +//! into its name, parameters (RFC 6868 caret escapes undone) and value, [`fold`] and +//! [`param_value`] write them. +//! - [`values`]: DATE and DATE-TIME values ([`When`]), durations, TEXT escaping, lists. +//! - [`rrule`]: the common subset of recurrence rules ([`Recurrence`]), read and written, with the +//! start's weekday and month spelled out where a rule leans on them. +//! - [`zones`]: `VTIMEZONE` definitions a text brings ([`TimeZones`]), which answer the offset at +//! an instant and the instant of a wall-clock time, and [`write_vtimezone`], which writes a +//! definition from the caller's zone rules ([`ZoneRules`]). +//! +//! Times are `chrono` values; instants are seconds since 1970-01-01T00:00:00Z. There is no +//! time-zone database here: what a `TZID` means beyond the text is the caller's. +//! +//! What events and to-dos mean (which properties become which fields, which zone a time is shown +//! in) is the caller's too: it reads them from [`ContentLine`]s with these values. +//! +//! Input is hostile. Nothing here panics on any text: lines, parameters, rules, zones, parts and +//! dates are capped, and work on zone definitions is budgeted ([`TimeZones::spend`]). + +pub mod lines; +pub mod rrule; +pub mod values; +pub mod zones; + +pub use lines::{ContentLine, Lines, Param, fold, param_value}; +pub use rrule::{Freq, Recurrence, format_rrule, parse_rrule}; +pub use values::When; +pub use zones::{TimeZones, Transition, VTimezone, ZoneRules, write_vtimezone}; diff --git a/crates/ical/src/lines.rs b/crates/ical/src/lines.rs new file mode 100644 index 000000000..9c82a7c94 --- /dev/null +++ b/crates/ical/src/lines.rs @@ -0,0 +1,321 @@ +//! Content lines (RFC 5545 §3.1–3.2): unfolding on the way in, folding on the way out, and the +//! `NAME;PARAM=value;PARAM="quoted":value` shape in between. + +/// Longest logical line kept, in bytes; the rest of a longer line is dropped. +pub const MAX_LINE: usize = 1 << 20; +/// Most parameters read from one line, and most values of one parameter. +const MAX_PARAMS: usize = 64; +/// Octets per physical line when writing. +const FOLD_AT: usize = 75; + +/// The longest prefix of `s` of at most `max` bytes that ends on a char boundary. +pub fn prefix(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + s.get(..end).unwrap_or("") +} + +/// The logical (unfolded) lines of a text. Lines end at CRLF, LF or CR; a line that starts with a +/// space or a tab continues the one before. +pub struct Lines<'a> { + rest: &'a str, +} + +impl<'a> Lines<'a> { + pub fn new(text: &'a str) -> Lines<'a> { + Lines { rest: text.strip_prefix('\u{feff}').unwrap_or(text) } + } + + /// The next physical line, without its line break. + fn physical(&mut self) -> Option<&'a str> { + if self.rest.is_empty() { + return None; + } + let bytes = self.rest.as_bytes(); + let end = bytes.iter().position(|b| matches!(b, b'\r' | b'\n')).unwrap_or(bytes.len()); + let skip = match (bytes.get(end), bytes.get(end.saturating_add(1))) { + (Some(b'\r'), Some(b'\n')) => 2, + (Some(_), _) => 1, + (None, _) => 0, + }; + let line = self.rest.get(..end).unwrap_or(""); + self.rest = self.rest.get(end.saturating_add(skip)..).unwrap_or(""); + Some(line) + } +} + +impl Iterator for Lines<'_> { + type Item = String; + + fn next(&mut self) -> Option { + let mut line = String::new(); + push_capped(&mut line, self.physical()?); + while self.rest.starts_with([' ', '\t']) { + let Some(more) = self.physical() else { break }; + push_capped(&mut line, more.get(1..).unwrap_or("")); + } + Some(line) + } +} + +fn push_capped(line: &mut String, more: &str) { + line.push_str(prefix(more, MAX_LINE.saturating_sub(line.len()))); +} + +/// One property parameter: `CN="Ada Lovelace"`, `ROLE=OPT-PARTICIPANT`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Param { + /// Upper-cased. + pub name: String, + pub values: Vec, +} + +/// One property: `ATTENDEE;CN=Ada:mailto:ada@example.org`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ContentLine { + /// Upper-cased. + pub name: String, + pub params: Vec, + /// As written: still escaped. + pub value: String, +} + +impl ContentLine { + /// Parse an unfolded line; `None` for a line without a name or without a `:`. + pub fn parse(line: &str) -> Option { + // Every delimiter is ASCII, so the byte positions found here are char boundaries. + let bytes = line.as_bytes(); + let stop = |from: usize, at: &[u8]| { + bytes.get(from..).and_then(|b| b.iter().position(|c| at.contains(c))).map_or(bytes.len(), |i| from.saturating_add(i)) + }; + let mut pos = stop(0, b";:"); + let name = line.get(..pos)?.trim(); + if name.is_empty() { + return None; + } + let mut params = Vec::new(); + while bytes.get(pos) == Some(&b';') { + let start = pos.saturating_add(1); + pos = stop(start, b"=;:"); + let param = line.get(start..pos)?.trim().to_ascii_uppercase(); + let mut values = Vec::new(); + while matches!(bytes.get(pos), Some(b'=' | b',')) { + let from = pos.saturating_add(1); + let value = if bytes.get(from) == Some(&b'"') { + let open = from.saturating_add(1); + let close = stop(open, b"\""); + // A quote that never closes: the line has no value. + bytes.get(close)?; + pos = stop(close, b",;:"); + line.get(open..close)? + } else { + pos = stop(from, b",;:"); + line.get(from..pos)? + }; + if values.len() < MAX_PARAMS { + values.push(caret_decode(value)); + } + } + if !param.is_empty() && params.len() < MAX_PARAMS { + params.push(Param { name: param, values }); + } + } + if bytes.get(pos) != Some(&b':') { + return None; + } + Some(ContentLine { name: name.to_ascii_uppercase(), params, value: line.get(pos.saturating_add(1)..)?.to_string() }) + } + + /// The first value of the parameter `name` (upper case). + pub fn param(&self, name: &str) -> Option<&str> { + self.params.iter().find(|p| p.name == name).and_then(|p| p.values.first()).map(String::as_str) + } +} + +/// Undo RFC 6868: `^n` is a line break, `^'` a double quote, `^^` a caret. +fn caret_decode(s: &str) -> String { + if !s.contains('^') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + let decoded = match (c, chars.peek()) { + ('^', Some('n')) => Some('\n'), + ('^', Some('\'')) => Some('"'), + ('^', Some('^')) => Some('^'), + _ => None, + }; + match decoded { + Some(d) => { + out.push(d); + chars.next(); + } + None => out.push(c), + } + } + out +} + +/// A parameter value ready to write: RFC 6868 escapes for what a parameter can't hold, other +/// control characters dropped, and quotes around it when it needs them. +pub fn param_value(s: &str) -> String { + let mut out = String::with_capacity(s.len().saturating_add(2)); + for c in s.chars() { + match c { + '^' => out.push_str("^^"), + '"' => out.push_str("^'"), + '\n' => out.push_str("^n"), + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + } + if out.contains([';', ':', ',']) || out.starts_with(' ') || out.ends_with(' ') { format!("\"{out}\"") } else { out } +} + +/// Append `line` to `out` with CRLF, folded so no physical line is longer than 75 octets and no +/// character is split. +pub fn fold(out: &mut String, line: &str) { + let mut rest = line; + let mut room = FOLD_AT; + while rest.len() > room { + let head = prefix(rest, room); + if head.is_empty() { + break; + } + out.push_str(head); + out.push_str("\r\n "); + rest = rest.get(head.len()..).unwrap_or(""); + // The leading space counts. + room = FOLD_AT - 1; + } + out.push_str(rest); + out.push_str("\r\n"); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(text: &str) -> Vec { + Lines::new(text).collect() + } + + #[test] + fn unfolds_every_kind_of_line_break() { + assert_eq!(lines("A:1\r\nB:2\nC:3\rD:4"), ["A:1", "B:2", "C:3", "D:4"]); + assert_eq!(lines("SUMMARY:Lunch \r\n with Ada\r\n"), ["SUMMARY:Lunch with Ada"]); + assert_eq!(lines("SUMMARY:Lun\n\tch\n twice\n"), ["SUMMARY:Lunch twice"]); + // Only the one space is the fold. + assert_eq!(lines("X:a\r\n b"), ["X:a b"]); + assert_eq!(lines("X:a \r\nY:b"), ["X:a ", "Y:b"]); + assert!(lines("").is_empty()); + } + + #[test] + fn a_byte_order_mark_and_blank_lines_are_harmless() { + assert_eq!(lines("\u{feff}BEGIN:VCALENDAR\r\n\r\nEND:VCALENDAR"), ["BEGIN:VCALENDAR", "", "END:VCALENDAR"]); + assert_eq!(ContentLine::parse(""), None); + } + + #[test] + fn parses_names_parameters_and_values() { + let l = ContentLine::parse("dtstart;tzid=Example/Riverton;Value=DATE-TIME:20261012T090000").unwrap(); + assert_eq!(l.name, "DTSTART"); + assert_eq!(l.param("TZID"), Some("Example/Riverton")); + assert_eq!(l.param("VALUE"), Some("DATE-TIME")); + assert_eq!(l.param("CN"), None); + assert_eq!(l.value, "20261012T090000"); + let plain = ContentLine::parse("SUMMARY:Budget: phase two; hiring").unwrap(); + assert_eq!((plain.name.as_str(), plain.params.len(), plain.value.as_str()), ("SUMMARY", 0, "Budget: phase two; hiring")); + assert_eq!(ContentLine::parse("DESCRIPTION:").unwrap().value, ""); + } + + #[test] + fn quoted_parameters_hide_delimiters() { + let l = ContentLine::parse("ATTENDEE;CN=\"Okafor, Nadia: lead; planning\";ROLE=REQ-PARTICIPANT:mailto:nadia@example.org").unwrap(); + assert_eq!(l.param("CN"), Some("Okafor, Nadia: lead; planning")); + assert_eq!(l.param("ROLE"), Some("REQ-PARTICIPANT")); + assert_eq!(l.value, "mailto:nadia@example.org"); + let several = ContentLine::parse("X-TEST;MEMBER=\"mailto:a@example.org\",\"mailto:b@example.org\",c:v").unwrap(); + assert_eq!(several.params[0].values, ["mailto:a@example.org", "mailto:b@example.org", "c"]); + // A parameter without a value, and an empty one. + let bare = ContentLine::parse("X-TEST;FLAG;EMPTY=:v").unwrap(); + assert_eq!(bare.params.len(), 2); + assert_eq!(bare.param("FLAG"), None); + assert_eq!(bare.param("EMPTY"), Some("")); + } + + #[test] + fn broken_lines_are_refused() { + for bad in ["no colon here", ":value without a name", ";CN=x:y", "ATTENDEE;CN=\"never closed:mailto:a@example.org", "ATTENDEE;CN=x", " "] { + assert_eq!(ContentLine::parse(bad), None, "{bad}"); + } + } + + #[test] + fn parameter_values_escape_and_quote() { + assert_eq!(param_value("Ada Lovelace"), "Ada Lovelace"); + assert_eq!(param_value("Lovelace, Ada"), "\"Lovelace, Ada\""); + assert_eq!(param_value("Ada \"the Countess\" ^1\nLine\r\u{7}"), "Ada ^'the Countess^' ^^1^nLine"); + for name in ["Ada", "Lovelace, Ada", "a\"b^c\nd", "x:y;z", "^n^'^^", "Zoë — 東京"] { + let l = ContentLine::parse(&format!("ORGANIZER;CN={}:mailto:a@example.org", param_value(name))).unwrap(); + assert_eq!(l.param("CN"), Some(name)); + } + } + + #[test] + fn folds_at_75_octets_without_splitting_characters() { + let line = format!("SUMMARY:{}", "é東🎉x".repeat(40)); + let mut out = String::new(); + fold(&mut out, &line); + assert!(out.ends_with("\r\n")); + let physical: Vec<&str> = out.trim_end_matches("\r\n").split("\r\n").collect(); + assert!(physical.len() > 3); + assert!(physical.iter().all(|l| l.len() <= 75), "{physical:?}"); + assert!(physical.iter().skip(1).all(|l| l.starts_with(' '))); + assert_eq!(lines(&out), [line]); + // A short line is left alone, and an exact fit isn't folded. + let mut short = String::new(); + fold(&mut short, &"x".repeat(75)); + assert_eq!(short, format!("{}\r\n", "x".repeat(75))); + let mut one_more = String::new(); + fold(&mut one_more, &"x".repeat(76)); + assert_eq!(one_more, format!("{}\r\n x\r\n", "x".repeat(75))); + } + + #[test] + fn enormous_lines_are_capped() { + let text = format!("SUMMARY:{}\r\n {}\r\nUID:after", "é".repeat(MAX_LINE), "x".repeat(1000)); + let got = lines(&text); + assert_eq!(got.len(), 2); + assert!(got[0].len() <= MAX_LINE); + assert_eq!(got[1], "UID:after"); + assert_eq!(prefix("héllo", 2), "h"); + assert_eq!(prefix("héllo", 0), ""); + assert_eq!(prefix("héllo", 99), "héllo"); + } + + proptest::proptest! { + #[test] + fn folding_then_unfolding_gives_the_line_back(line in "[^\r\n]{0,400}") { + let mut out = String::new(); + fold(&mut out, &line); + proptest::prop_assert!(out.split("\r\n").all(|l| l.len() <= 75)); + // A byte order mark at the very start of a text isn't part of its first line. + proptest::prop_assert_eq!(Lines::new(&out).collect::>(), vec![line.strip_prefix('\u{feff}').unwrap_or(&line).to_string()]); + } + + #[test] + fn any_line_parses_or_is_refused(line in proptest::prelude::any::()) { + let _ = ContentLine::parse(&line); + let _ = Lines::new(&line).count(); + } + } +} diff --git a/crates/ical/src/rrule.rs b/crates/ical/src/rrule.rs new file mode 100644 index 000000000..2833e5a29 --- /dev/null +++ b/crates/ical/src/rrule.rs @@ -0,0 +1,516 @@ +//! Recurrence rules (RFC 5545 §3.3.10): the common subset a calendar keeps. + +use chrono::{Datelike, NaiveDate, NaiveDateTime, Weekday}; + +use crate::lines::prefix; +use crate::values::{When, format_date, valid_time}; + +type Timestamp = i64; + +/// How often a rule repeats. `SECONDLY`, `MINUTELY` and `HOURLY` are not held. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Freq { + Daily, + #[default] + Weekly, + Monthly, + Yearly, +} + +impl Freq { + /// From `DAILY`, `weekly`…: any letter case. + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_uppercase().as_str() { + "DAILY" => Some(Freq::Daily), + "WEEKLY" => Some(Freq::Weekly), + "MONTHLY" => Some(Freq::Monthly), + "YEARLY" => Some(Freq::Yearly), + _ => None, + } + } + + /// `DAILY`, `WEEKLY`, `MONTHLY`, `YEARLY`. + pub fn as_str(self) -> &'static str { + match self { + Freq::Daily => "DAILY", + Freq::Weekly => "WEEKLY", + Freq::Monthly => "MONTHLY", + Freq::Yearly => "YEARLY", + } + } +} + +/// How an event repeats: the subset of RRULE that calendars show and edit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Recurrence { + pub freq: Freq, + /// Every `interval` days/weeks/months/years; 0 reads as 1. + pub interval: u32, + /// Weekly: the days of the week (none: the start's own). Daily: only on these days. Monthly and + /// yearly with `nth`: the weekday(s) counted. + pub by_day: Vec, + /// Monthly and yearly: the `nth` `by_day` weekday of the month (1–5, or -1 for the last) + /// instead of the start's day of the month. + pub nth: Option, + /// Ends after this many occurrences. + pub count: Option, + /// Ends on this day (inclusive). + pub until: Option, +} + +impl Recurrence { + /// Every day, week, month or year. + pub fn every(freq: Freq) -> Recurrence { + Recurrence { freq, interval: 1, ..Default::default() } + } +} + +/// The days of the week, Monday first. +pub const WEEKDAYS: [Weekday; 7] = [Weekday::Mon, Weekday::Tue, Weekday::Wed, Weekday::Thu, Weekday::Fri, Weekday::Sat, Weekday::Sun]; + +/// `MO`, `TU`… as iCalendar writes a weekday. +pub fn weekday_code(day: Weekday) -> &'static str { + match day { + Weekday::Mon => "MO", + Weekday::Tue => "TU", + Weekday::Wed => "WE", + Weekday::Thu => "TH", + Weekday::Fri => "FR", + Weekday::Sat => "SA", + Weekday::Sun => "SU", + } +} + +/// The weekday of `MO`, `tu`…; `None` for anything else. +pub fn weekday_of_code(s: &str) -> Option { + WEEKDAYS.into_iter().find(|d| weekday_code(*d).eq_ignore_ascii_case(s.trim())) +} + +/// Longest rule read, in bytes, and most parts (and most `BYDAY` days) of one rule. +const MAX_RULE: usize = 4096; +const MAX_PARTS: usize = 64; + +/// `FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE` as a [`Recurrence`]; `None` for a rule the model can't hold +/// (`SECONDLY`, `MINUTELY`, `HOURLY`, junk). +/// +/// `UNTIL` gives its date, whether it is written as a date or a date-time. `BYDAY=2TU`, `BYDAY=-1FR` +/// and `BYDAY=TU;BYSETPOS=2` give `nth`. Parts that only restate the start (`BYMONTH`, +/// `BYMONTHDAY`, `WKST`) and parts the model has no place for are ignored. +pub fn parse_rrule(rule: &str) -> Option { + parse_with(rule, None) +} + +/// [`parse_rrule`] for a rule read from a file: with `offset_at` (minutes east of UTC at an +/// instant), a UTC `UNTIL` gives its local day. +pub fn parse_with(rule: &str, offset_at: Option<&dyn Fn(Timestamp) -> i32>) -> Option { + let rule = prefix(rule.trim(), MAX_RULE); + let rule = match rule.get(..6) { + Some(name) if name.eq_ignore_ascii_case("RRULE:") => rule.get(6..).unwrap_or(""), + _ => rule, + }; + let mut freq = None; + let mut interval = 1u32; + let (mut count, mut until, mut set_pos) = (None, None, None); + let mut days: Vec<(Option, Weekday)> = Vec::new(); + // Parts that pick days in a way the model can't: with them, a bare BYDAY means something else. + let mut limited = false; + for part in rule.split(';').take(MAX_PARTS) { + let Some((key, value)) = part.split_once('=') else { continue }; + let value = value.trim(); + match key.trim().to_ascii_uppercase().as_str() { + "FREQ" => freq = Some(Freq::parse(value)?), + "INTERVAL" => interval = value.parse().ok().filter(|n| *n > 0).unwrap_or(1), + "COUNT" => count = value.parse::().ok(), + "UNTIL" => { + until = When::parse(value).map(|w| match offset_at { + Some(offset_at) => w.local(offset_at).date(), + None => w.wall().date(), + }) + } + "BYDAY" => days = value.split(',').take(MAX_PARTS).filter_map(by_day).collect(), + "BYSETPOS" => set_pos = value.parse::().ok(), + "BYMONTH" | "BYMONTHDAY" | "BYYEARDAY" | "BYWEEKNO" => limited = true, + _ => {} + } + } + let mut freq = freq?; + let by_month = matches!(freq, Freq::Monthly | Freq::Yearly); + // `1MO,3MO` isn't one position: the first one is kept. + let ordinal = days.iter().find_map(|(ordinal, _)| *ordinal); + if by_month && ordinal.is_some() { + days.retain(|(o, _)| *o == ordinal); + } + let mut by_day: Vec = Vec::new(); + for (_, day) in &days { + if !by_day.contains(day) { + by_day.push(*day); + } + } + let mut nth = None; + if by_month { + nth = match (ordinal, set_pos) { + (Some(ordinal), _) => position(ordinal), + // A position in a set of several weekdays ("the last working day") isn't a weekday's position. + (None, Some(pos)) if by_day.len() == 1 || (by_day.is_empty() && !limited) => position(pos), + _ => None, + }; + if nth.is_none() { + if freq == Freq::Monthly && interval == 1 && !by_day.is_empty() && ordinal.is_none() && set_pos.is_none() && !limited { + // Every Monday of every month is every Monday. + freq = Freq::Weekly; + } else { + by_day.clear(); + } + } + } + Some(Recurrence { freq, interval, by_day, nth, count, until }) +} + +/// `MO`, `2TU`, `-1FR`: the position, when there is one, and the day. +fn by_day(item: &str) -> Option<(Option, Weekday)> { + let item = item.trim(); + let split = item.len().checked_sub(2)?; + let day = weekday_of_code(item.get(split..)?)?; + let ordinal = item.get(..split)?; + Some((if ordinal.is_empty() { None } else { Some(ordinal.parse::().ok()?) }, day)) +} + +/// A position the model holds: the first to the fifth, or the last. +fn position(n: i32) -> Option { + if (1..=5).contains(&n) || n == -1 { i8::try_from(n).ok() } else { None } +} + +/// A [`Recurrence`] as an RRULE value: `FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE`. +/// +/// An `nth` without a `by_day` means "the start's weekday", which a rule alone can't say: it is +/// written as a bare `BYSETPOS`, which [`parse_rrule`] reads back but other programs may not. +/// [`format_for`] knows the start and writes the weekday (and the month of a yearly rule) out. For monthly and yearly rules without an `nth` the model ignores `by_day`, so it isn't +/// written. +pub fn format_rrule(r: &Recurrence) -> String { + format(r, None, None) +} + +/// The rule of an event that starts at `start`: the start's weekday and month are spelled out +/// where the rule leans on them, and `UNTIL` has the start's value type. +pub fn format_for(r: &Recurrence, start: NaiveDateTime, all_day: bool) -> String { + format(r, Some((valid_time(start), all_day)), None) +} + +/// The rule of a timed event whose start is written in a zone: as [`format_for`], with `UNTIL` +/// as the instant `until` in UTC, which is how RFC 5545 has it beside such a start. +pub fn format_zoned(r: &Recurrence, start: NaiveDateTime, until: Option) -> String { + format(r, Some((valid_time(start), false)), until) +} + +fn format(r: &Recurrence, start: Option<(NaiveDateTime, bool)>, until_at: Option) -> String { + let mut out = format!("FREQ={}", r.freq.as_str()); + if r.interval > 1 { + out.push_str(&format!(";INTERVAL={}", r.interval)); + } + let mut days: Vec = Vec::new(); + for day in &r.by_day { + if !days.contains(day) { + days.push(*day); + } + } + let list = |days: &[Weekday], position: &str| days.iter().map(|d| format!("{position}{}", weekday_code(*d))).collect::>().join(","); + match (r.freq, r.nth.filter(|n| *n != 0)) { + (Freq::Daily | Freq::Weekly, _) => { + if !days.is_empty() { + out.push_str(&format!(";BYDAY={}", list(&days, ""))); + } + } + (_, None) => {} + (freq, Some(nth)) => { + // Any negative position is "the last" to the model. + let nth = nth.max(-1); + if let Some((start, _)) = start { + if days.is_empty() { + days.push(start.date().weekday()); + } + if freq == Freq::Yearly { + out.push_str(&format!(";BYMONTH={}", start.date().month())); + } + } + if days.is_empty() { + out.push_str(&format!(";BYSETPOS={nth}")); + } else { + out.push_str(&format!(";BYDAY={}", list(&days, &nth.to_string()))); + } + } + } + if let Some(count) = r.count { + out.push_str(&format!(";COUNT={count}")); + } + if let (Some(_), Some(at)) = (r.until, until_at) { + out.push_str(&format!(";UNTIL={}", crate::values::format_utc(at))); + } else if let Some(until) = r.until { + out.push_str(&format!(";UNTIL={}", format_date(until))); + // The whole last day of a timed series. + if matches!(start, Some((_, false))) { + out.push_str("T235959"); + } + } + out +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + use crate::values::at_midnight; + + fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() + } + + fn rule(freq: Freq) -> Recurrence { + Recurrence::every(freq) + } + + #[test] + fn parses_the_common_rules() { + assert_eq!(parse_rrule("FREQ=DAILY"), Some(rule(Freq::Daily))); + assert_eq!( + parse_rrule("FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE"), + Some(Recurrence { interval: 2, by_day: vec![Weekday::Mon, Weekday::Wed], ..rule(Freq::Weekly) }) + ); + assert_eq!( + parse_rrule("freq=weekly;byday=fr,mo;count=10;wkst=su"), + Some(Recurrence { by_day: vec![Weekday::Fri, Weekday::Mon], count: Some(10), ..rule(Freq::Weekly) }) + ); + assert_eq!( + parse_rrule("FREQ=DAILY;BYDAY=MO,TU,WE,TH,FR"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Tue, Weekday::Wed, Weekday::Thu, Weekday::Fri], ..rule(Freq::Daily) }) + ); + assert_eq!(parse_rrule("FREQ=MONTHLY;INTERVAL=3"), Some(Recurrence { interval: 3, ..rule(Freq::Monthly) })); + assert_eq!(parse_rrule("FREQ=YEARLY"), Some(rule(Freq::Yearly))); + // The property name and stray spaces are let through. + assert_eq!(parse_rrule(" RRULE:FREQ=DAILY; INTERVAL = 4 ;"), Some(Recurrence { interval: 4, ..rule(Freq::Daily) })); + } + + #[test] + fn positions_come_from_ordinals_or_bysetpos() { + let second_tue = Recurrence { by_day: vec![Weekday::Tue], nth: Some(2), ..rule(Freq::Monthly) }; + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=2TU"), Some(second_tue.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=+2TU"), Some(second_tue.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=TU;BYSETPOS=2"), Some(second_tue)); + let last_fri = Recurrence { by_day: vec![Weekday::Fri], nth: Some(-1), ..rule(Freq::Monthly) }; + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=-1FR"), Some(last_fri.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYSETPOS=-1;BYDAY=FR"), Some(last_fri)); + // The fourth Thursday of November: the month restates the start. + assert_eq!( + parse_rrule("FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"), + Some(Recurrence { by_day: vec![Weekday::Thu], nth: Some(4), ..rule(Freq::Yearly) }) + ); + assert_eq!( + parse_rrule("FREQ=MONTHLY;BYDAY=1MO,1FR"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Fri], nth: Some(1), ..rule(Freq::Monthly) }) + ); + // Positions mean nothing in a weekly rule. + assert_eq!(parse_rrule("FREQ=WEEKLY;BYDAY=2TU"), Some(Recurrence { by_day: vec![Weekday::Tue], ..rule(Freq::Weekly) })); + } + + #[test] + fn rules_the_model_cannot_hold_fall_back() { + // Two positions: the first is kept. + assert_eq!( + parse_rrule("FREQ=MONTHLY;BYDAY=1MO,3MO,1WE"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Wed], nth: Some(1), ..rule(Freq::Monthly) }) + ); + // The last working day, the second-to-last Friday, Friday the 13th and the like: plain monthly. + for r in [ + "FREQ=MONTHLY;BYDAY=MO,TU,WE,TH,FR;BYSETPOS=-1", + "FREQ=MONTHLY;BYDAY=-2FR", + "FREQ=MONTHLY;BYDAY=FR;BYMONTHDAY=13", + "FREQ=MONTHLY;BYMONTHDAY=15", + "FREQ=MONTHLY;BYMONTHDAY=1,2,3;BYSETPOS=1", + ] { + assert_eq!(parse_rrule(r), Some(rule(Freq::Monthly)), "{r}"); + } + // The 20th Monday of the year. + assert_eq!(parse_rrule("FREQ=YEARLY;BYDAY=20MO"), Some(rule(Freq::Yearly))); + // Every Monday and Thursday of every month is weekly; every other month isn't. + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=MO,TH"), Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Thu], ..rule(Freq::Weekly) })); + assert_eq!(parse_rrule("FREQ=MONTHLY;INTERVAL=2;BYDAY=MO"), Some(Recurrence { interval: 2, ..rule(Freq::Monthly) })); + assert_eq!(parse_rrule("FREQ=YEARLY;BYMONTH=3,6,9"), Some(rule(Freq::Yearly))); + } + + #[test] + fn until_is_a_date_whatever_its_form() { + let until = |r: &str| parse_rrule(r).and_then(|r| r.until); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231T235959"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231T030000Z"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=someday"), None); + // Read from a file, a UTC time is a local day. + let west = parse_with("FREQ=DAILY;UNTIL=20261231T030000Z", Some(&|_| -300)).unwrap(); + assert_eq!(west.until, Some(d(2026, 12, 30))); + let east = parse_with("FREQ=DAILY;UNTIL=20261230T220000Z", Some(&|_| 600)).unwrap(); + assert_eq!(east.until, Some(d(2026, 12, 31))); + assert_eq!(parse_with("FREQ=DAILY;UNTIL=20261231", Some(&|_| -300)).unwrap().until, Some(d(2026, 12, 31))); + } + + #[test] + fn junk_and_fast_rules_are_none() { + for bad in [ + "", + "FREQ=", + "FREQ=HOURLY", + "FREQ=MINUTELY;INTERVAL=5", + "FREQ=SECONDLY", + "FREQ=SOMETIMES", + "INTERVAL=2;BYDAY=MO", + "weekly", + ";;;===", + "RRULE:", + ] { + assert_eq!(parse_rrule(bad), None, "{bad}"); + } + } + + #[test] + fn junk_parts_are_ignored() { + let got = parse_rrule("FREQ=WEEKLY;INTERVAL=0;COUNT=many;BYDAY=MO,XX,,7,MO,é;BYSETPOS=x;X-THING=1;novalue").unwrap(); + assert_eq!(got, Recurrence { by_day: vec![Weekday::Mon], ..rule(Freq::Weekly) }); + assert_eq!(parse_rrule("FREQ=DAILY;INTERVAL=99999999999;COUNT=99999999999").unwrap(), rule(Freq::Daily)); + assert_eq!(parse_rrule("FREQ=DAILY;COUNT=0").unwrap().count, Some(0)); + let long = format!("FREQ=WEEKLY;BYDAY={}", "MO,".repeat(100_000)); + assert_eq!(parse_rrule(&long).unwrap().by_day, [Weekday::Mon]); + } + + #[test] + fn formats_rules() { + assert_eq!(format_rrule(&rule(Freq::Daily)), "FREQ=DAILY"); + assert_eq!( + format_rrule(&Recurrence { interval: 2, by_day: vec![Weekday::Mon, Weekday::Wed, Weekday::Mon], ..rule(Freq::Weekly) }), + "FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE" + ); + assert_eq!(format_rrule(&Recurrence { count: Some(10), ..rule(Freq::Yearly) }), "FREQ=YEARLY;COUNT=10"); + assert_eq!(format_rrule(&Recurrence { until: Some(d(2026, 12, 31)), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;UNTIL=20261231"); + assert_eq!(format_rrule(&Recurrence { by_day: vec![Weekday::Tue], nth: Some(2), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;BYDAY=2TU"); + assert_eq!( + format_rrule(&Recurrence { by_day: vec![Weekday::Mon, Weekday::Fri], nth: Some(-1), ..rule(Freq::Monthly) }), + "FREQ=MONTHLY;BYDAY=-1MO,-1FR" + ); + assert_eq!(format_rrule(&Recurrence { nth: Some(3), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;BYSETPOS=3"); + // Interval 0 reads as 1, and the model ignores the days of a monthly rule without a position. + assert_eq!(format_rrule(&Recurrence { interval: 0, by_day: vec![Weekday::Mon], nth: Some(0), ..rule(Freq::Monthly) }), "FREQ=MONTHLY"); + } + + #[test] + fn an_event_s_rule_spells_out_what_the_start_implies() { + let tuesday = d(2026, 10, 13).and_hms_opt(9, 0, 0).unwrap(); + let r = Recurrence { nth: Some(2), until: Some(d(2027, 6, 30)), ..rule(Freq::Monthly) }; + assert_eq!(format_for(&r, tuesday, false), "FREQ=MONTHLY;BYDAY=2TU;UNTIL=20270630T235959"); + assert_eq!(format_for(&r, tuesday, true), "FREQ=MONTHLY;BYDAY=2TU;UNTIL=20270630"); + let thanksgiving = Recurrence { by_day: vec![Weekday::Thu], nth: Some(4), ..rule(Freq::Yearly) }; + assert_eq!(format_for(&thanksgiving, at_midnight(d(2026, 11, 26)), true), "FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"); + assert_eq!(format_for(&Recurrence { nth: Some(i8::MIN), ..rule(Freq::Yearly) }, tuesday, false), "FREQ=YEARLY;BYMONTH=10;BYDAY=-1TU"); + // What it writes reads back as the same series. + assert_eq!(parse_rrule("FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"), Some(thanksgiving)); + // A start in a year iCalendar can't write is moved into the years it can. + let written = format_for(&Recurrence { nth: Some(1), until: Some(NaiveDate::MAX), ..rule(Freq::Yearly) }, NaiveDateTime::MAX, true); + assert_eq!(written, "FREQ=YEARLY;BYMONTH=12;BYDAY=1FR;UNTIL=99991231"); + } + + /// Every shape the model gives a meaning: weekly and daily rules with any days, monthly and + /// yearly rules with a position (and then any days) or without. + fn shapes() -> Vec { + let day_sets: [&[Weekday]; 5] = [&[], &[Weekday::Tue], &[Weekday::Sun, Weekday::Mon], &[Weekday::Fri, Weekday::Mon, Weekday::Wed], &WEEKDAYS]; + let mut out = Vec::new(); + for freq in [Freq::Daily, Freq::Weekly, Freq::Monthly, Freq::Yearly] { + for interval in [1, 2, 13, u32::MAX] { + for count in [None, Some(0), Some(1), Some(u32::MAX)] { + for until in [None, Some(d(2030, 1, 31)), Some(d(1, 1, 1)), Some(d(9999, 12, 31))] { + for days in day_sets { + let positions: &[Option] = if matches!(freq, Freq::Monthly | Freq::Yearly) { + &[None, Some(1), Some(2), Some(3), Some(4), Some(5), Some(-1)] + } else { + &[None] + }; + for nth in positions { + if nth.is_none() && !days.is_empty() && matches!(freq, Freq::Monthly | Freq::Yearly) { + continue; + } + out.push(Recurrence { freq, interval, by_day: days.to_vec(), nth: *nth, count, until }); + } + } + } + } + } + } + out + } + + #[test] + fn every_shape_the_model_supports_round_trips() { + let shapes = shapes(); + assert!(shapes.len() > 1000); + for r in shapes { + let text = format_rrule(&r); + assert_eq!(parse_rrule(&text), Some(r.clone()), "{text}"); + // And as an event writes it: the start is a Tuesday, so a missing day reads back as Tuesday. + for all_day in [false, true] { + let text = format_for(&r, d(2026, 10, 13).and_hms_opt(9, 30, 0).unwrap(), all_day); + let mut want = r.clone(); + if want.nth.is_some() && want.by_day.is_empty() { + want.by_day = vec![Weekday::Tue]; + } + assert_eq!(parse_rrule(&text), Some(want), "{text}"); + } + } + } + + #[test] + fn shapes_without_a_meaning_are_normalised() { + let normal = |r: &Recurrence| parse_rrule(&format_rrule(r)).unwrap(); + // Interval 0 is 1, duplicate days are one, position 0 is none, any negative position is the last. + assert_eq!(normal(&Recurrence { interval: 0, ..rule(Freq::Weekly) }).interval, 1); + assert_eq!(normal(&Recurrence { by_day: vec![Weekday::Mon; 100], ..rule(Freq::Weekly) }).by_day, [Weekday::Mon]); + assert_eq!(normal(&Recurrence { nth: Some(0), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }), rule(Freq::Monthly)); + assert_eq!(normal(&Recurrence { nth: Some(-7), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }).nth, Some(-1)); + // A position no month has isn't read back. + assert_eq!(normal(&Recurrence { nth: Some(9), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }), rule(Freq::Monthly)); + // A position means nothing in a weekly rule. + assert_eq!(normal(&Recurrence { nth: Some(2), ..rule(Freq::Weekly) }), rule(Freq::Weekly)); + } + + fn recurrence() -> impl Strategy { + let freq = prop_oneof![Just(Freq::Daily), Just(Freq::Weekly), Just(Freq::Monthly), Just(Freq::Yearly)]; + let days = proptest::sample::subsequence(WEEKDAYS.to_vec(), 0..=7).prop_shuffle(); + let nth = proptest::option::of(prop_oneof![1i8..=5, Just(-1i8)]); + let until = proptest::option::of((1i32..=9999, 1u32..=12, 1u32..=28).prop_map(|(y, m, day)| d(y, m, day))); + (freq, 1u32.., days, nth, proptest::option::of(any::()), until).prop_map(|(freq, interval, by_day, nth, count, until)| { + let monthly = matches!(freq, Freq::Monthly | Freq::Yearly); + let nth = if monthly { nth } else { None }; + let by_day = if monthly && nth.is_none() { Vec::new() } else { by_day }; + Recurrence { freq, interval, by_day, nth, count, until } + }) + } + + proptest! { + #[test] + fn any_supported_rule_round_trips(r in recurrence()) { + prop_assert_eq!(parse_rrule(&format_rrule(&r)), Some(r)); + } + + #[test] + fn any_rule_at_all_formats_and_reads_back_stably(freq in 0u8..4, interval in any::(), days in proptest::collection::vec(0usize..7, 0..12), nth in proptest::option::of(any::()), count in proptest::option::of(any::())) { + let freq = [Freq::Daily, Freq::Weekly, Freq::Monthly, Freq::Yearly][usize::from(freq)]; + let r = Recurrence { freq, interval, by_day: days.into_iter().filter_map(|i| WEEKDAYS.get(i).copied()).collect(), nth, count, until: None }; + let text = format_rrule(&r); + let back = parse_rrule(&text).unwrap(); + // Reading normalises; a normal rule then round-trips. + prop_assert_eq!(parse_rrule(&format_rrule(&back)), Some(back)); + } + + #[test] + fn rule_text_never_panics(s in any::(), parts in proptest::collection::vec("(FREQ|INTERVAL|COUNT|UNTIL|BYDAY|BYSETPOS|BYMONTH|X)=[-+A-Z0-9,]{0,12}", 0..8)) { + let _ = parse_rrule(&s); + let _ = parse_rrule(&parts.join(";")); + let _ = parse_with(&parts.join(";"), Some(&|_| i32::MIN)); + } + } +} diff --git a/crates/ical/src/values.rs b/crates/ical/src/values.rs new file mode 100644 index 000000000..1f75a804d --- /dev/null +++ b/crates/ical/src/values.rs @@ -0,0 +1,419 @@ +//! Property values: dates and times, durations and escaped text. +//! +//! Times are `chrono` values. An instant is an `i64` of seconds since 1970-01-01T00:00:00Z, and an +//! offset of a zone is minutes east of UTC where this crate takes one from a caller and seconds +//! where a `VTIMEZONE` gives one. Every date and time this crate makes lies in the years 1 to 9999, +//! the years iCalendar can write. + +use chrono::{DateTime, NaiveDate, NaiveDateTime}; + +/// Seconds in a day. +pub const DAY: i64 = 86_400; + +/// Longest duration read, in seconds (some 10 000 years): sums of it can't overflow. +const MAX_DURATION: i64 = 10_000 * 366 * DAY; + +/// The first and the last second iCalendar can write, as seconds since 1970 on the same clock. +const MIN_CIVIL: i64 = -62_135_596_800; +const MAX_CIVIL: i64 = 253_402_300_799; + +/// Seconds since 1970-01-01T00:00:00 on the same wall clock (no zone applied). +pub fn civil_seconds(t: NaiveDateTime) -> i64 { + t.and_utc().timestamp() +} + +/// The wall-clock time `secs` seconds after 1970-01-01T00:00:00, kept in the years 1 to 9999. +pub fn from_civil_seconds(secs: i64) -> NaiveDateTime { + DateTime::from_timestamp(secs.clamp(MIN_CIVIL, MAX_CIVIL), 0).map(|t| t.naive_utc()).unwrap_or_default() +} + +/// The instant a wall clock shows in a zone `offset_minutes` east of UTC. +pub fn to_timestamp(t: NaiveDateTime, offset_minutes: i32) -> i64 { + civil_seconds(t).saturating_sub(i64::from(offset_minutes).saturating_mul(60)) +} + +/// The wall clock at the instant `ts` in a zone `offset_minutes` east of UTC. +pub fn from_timestamp(ts: i64, offset_minutes: i32) -> NaiveDateTime { + from_civil_seconds(ts.saturating_add(i64::from(offset_minutes).saturating_mul(60))) +} + +/// Midnight at the start of `d`. +pub fn at_midnight(d: NaiveDate) -> NaiveDateTime { + d.and_time(chrono::NaiveTime::MIN) +} + +/// A date and time in the years iCalendar can write: one outside them is moved to the nearest end. +pub(crate) fn valid_time(t: NaiveDateTime) -> NaiveDateTime { + from_civil_seconds(civil_seconds(t)) +} + +/// A date from `YYYY-MM-DD` or `YYYYMMDD`, years 1 to 9999. +fn parse_iso_date(s: &str) -> Option { + let s = s.trim(); + let digits: String = s.chars().filter(char::is_ascii_digit).collect(); + if digits.len() != 8 || s.chars().any(|c| !c.is_ascii_digit() && c != '-') { + return None; + } + let year: i32 = digits.get(0..4)?.parse().ok()?; + if year < 1 { + return None; + } + NaiveDate::from_ymd_opt(year, digits.get(4..6)?.parse().ok()?, digits.get(6..8)?.parse().ok()?) +} + +/// A DATE or DATE-TIME value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum When { + /// `20261012`: a whole day. + Date(NaiveDate), + /// `20261012T090000`: a wall-clock time, floating or in a named zone. + Local(NaiveDateTime), + /// `20261012T090000Z`. + Utc(NaiveDateTime), +} + +impl When { + /// From `YYYYMMDD`, `YYYYMMDDTHHMMSS` or `YYYYMMDDTHHMMSSZ`; the seconds may be missing, and + /// the ISO 8601 punctuation some writers add (`2026-10-12T09:00:00Z`) is let through. + pub fn parse(s: &str) -> Option { + let s = s.trim(); + if s.len() > 32 || !s.is_ascii() { + return None; + } + let (date, time) = match s.find(['T', 't']) { + Some(i) => (s.get(..i)?, Some(s.get(i.saturating_add(1)..)?)), + None => (s, None), + }; + let date = parse_iso_date(date)?; + let Some(time) = time else { return Some(When::Date(date)) }; + let (clock, utc) = match time.strip_suffix(['Z', 'z']) { + Some(clock) => (clock, true), + None => (time, false), + }; + let digits: String = clock.chars().filter(|c| *c != ':').collect(); + if !matches!(digits.len(), 4 | 6) || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let part = |at: usize| digits.get(at..at.saturating_add(2)).and_then(|d| d.parse::().ok()); + // A leap second (60) is read as the last second of the minute. + let second = if digits.len() == 6 { part(4)?.min(59) } else { 0 }; + let t = date.and_hms_opt(part(0)?, part(2)?, second)?; + Some(if utc { When::Utc(t) } else { When::Local(t) }) + } + + /// The date and time as written, whatever the zone. + pub fn wall(self) -> NaiveDateTime { + match self { + When::Date(d) => at_midnight(d), + When::Local(t) | When::Utc(t) => t, + } + } + + /// The local wall-clock time: a UTC time is converted, anything else is taken as written. + /// `offset_at(instant)` is minutes east of UTC. + pub fn local(self, offset_at: &dyn Fn(i64) -> i32) -> NaiveDateTime { + match self { + When::Utc(t) => { + let ts = to_timestamp(t, 0); + from_timestamp(ts, offset_at(ts)) + } + other => other.wall(), + } + } + + /// The instant: anything but a UTC time is read in the local zone. + pub fn timestamp(self, offset_at: &dyn Fn(i64) -> i32) -> i64 { + match self { + When::Utc(t) => to_timestamp(t, 0), + other => { + let t = other.wall(); + // Asked twice, so a time near a daylight-saving change still gets its own offset. + let guess = to_timestamp(t, offset_at(to_timestamp(t, 0))); + to_timestamp(t, offset_at(guess)) + } + } + } +} + +/// A duration in seconds from `PT15M`, `-PT1H30M`, `P1D`, `P1DT12H`, `P2W`; `None` for junk. +pub fn parse_duration(s: &str) -> Option { + let s = s.trim(); + let (negative, body) = match s.strip_prefix('-') { + Some(body) => (true, body), + None => (false, s.strip_prefix('+').unwrap_or(s)), + }; + let body = body.strip_prefix(['P', 'p'])?; + let (mut total, mut number, mut in_time, mut any) = (0i64, None::, false, false); + for c in body.chars() { + match c.to_ascii_uppercase() { + '0'..='9' => { + let digit = i64::from(c.to_digit(10)?); + number = Some(number.unwrap_or(0).checked_mul(10)?.checked_add(digit)?); + } + 'T' if number.is_none() => in_time = true, + unit => { + let seconds = match (unit, in_time) { + ('W', false) => 7 * DAY, + ('D', false) => DAY, + ('H', true) => 3600, + ('M', true) => 60, + ('S', true) => 1, + _ => return None, + }; + total = total.checked_add(number.take()?.checked_mul(seconds)?)?; + any = true; + } + } + } + if number.is_some() || !any || total > MAX_DURATION { + return None; + } + Some(if negative { -total } else { total }) +} + +/// `20261012`. +pub fn format_date(d: NaiveDate) -> String { + valid_time(at_midnight(d)).format("%Y%m%d").to_string() +} + +/// `20261012T090000` (floating). +pub fn format_local(t: NaiveDateTime) -> String { + valid_time(t).format("%Y%m%dT%H%M%S").to_string() +} + +/// `20261012T130000Z`. +pub fn format_utc(ts: i64) -> String { + format!("{}Z", format_local(from_timestamp(ts, 0))) +} + +/// A TEXT value ready to write: `\`, `;`, `,` and line breaks escaped (`\r\n` and `\r` count as +/// one line break), other control characters except tab dropped. +pub fn escape_text(s: &str) -> String { + let mut out = String::with_capacity(s.len().saturating_add(8)); + let mut after_cr = false; + for c in s.chars() { + let cr = c == '\r'; + match c { + '\n' if after_cr => {} + '\r' | '\n' => out.push_str("\\n"), + '\\' => out.push_str("\\\\"), + ';' => out.push_str("\\;"), + ',' => out.push_str("\\,"), + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + after_cr = cr; + } + out +} + +/// Undo [`escape_text`]: `\n` and `\N` are line breaks, `\\`, `\;`, `\,` (and the `\:` and `\"` +/// some writers add) their second character; any other backslash is kept. Control characters +/// except tab are dropped. +pub fn unescape_text(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.next() { + Some('n' | 'N') => out.push('\n'), + Some(x @ ('\\' | ';' | ',' | ':' | '"')) => out.push(x), + Some(x) => { + out.push('\\'); + if !x.is_control() || x == '\t' { + out.push(x); + } + } + None => out.push('\\'), + }, + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + } + out +} + +/// The first `max` items of a list value, split on its unescaped commas; the items are still +/// escaped. +pub fn split_list(s: &str, max: usize) -> Vec<&str> { + let mut out = Vec::new(); + let (mut start, mut escaped) = (0usize, false); + for (i, c) in s.char_indices() { + if out.len() >= max { + return out; + } + match c { + _ if escaped => escaped = false, + '\\' => escaped = true, + ',' => { + out.push(s.get(start..i).unwrap_or("")); + start = i.saturating_add(1); + } + _ => {} + } + } + if out.len() < max { + out.push(s.get(start..).unwrap_or("")); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() + } + + fn at(y: i32, m: u32, day: u32, h: u32, min: u32, s: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, s).unwrap() + } + + #[test] + fn dates_and_times() { + assert_eq!(When::parse("20261012"), Some(When::Date(d(2026, 10, 12)))); + assert_eq!(When::parse("20261012T090000"), Some(When::Local(at(2026, 10, 12, 9, 0, 0)))); + assert_eq!(When::parse(" 20261012T133015Z "), Some(When::Utc(at(2026, 10, 12, 13, 30, 15)))); + assert_eq!(When::parse("20261012t0900z"), Some(When::Utc(at(2026, 10, 12, 9, 0, 0)))); + assert_eq!(When::parse("2026-10-12T09:00:00Z"), Some(When::Utc(at(2026, 10, 12, 9, 0, 0)))); + // A leap second. + assert_eq!(When::parse("20161231T235960Z"), Some(When::Utc(at(2016, 12, 31, 23, 59, 59)))); + } + + #[test] + fn junk_dates_are_none() { + for bad in [ + "", + "tomorrow", + "2026101", + "202610123", + "20261312", + "20260230", + "00000101", + "20261012T", + "20261012T25", + "20261012T240000", + "20261012T0960", + "20261012T090000+0200", + "20261012", + "20261012T09000000000000000000000000000000000000", + ] { + assert_eq!(When::parse(bad), None, "{bad}"); + } + } + + #[test] + fn utc_times_become_local_and_local_times_become_instants() { + let utc = When::parse("20261012T130000Z").unwrap(); + assert_eq!(utc.local(&|_| -240), at(2026, 10, 12, 9, 0, 0)); + assert_eq!(utc.local(&|_| 330), at(2026, 10, 12, 18, 30, 0)); + assert_eq!(utc.wall(), at(2026, 10, 12, 13, 0, 0)); + // Across midnight. + assert_eq!(When::parse("20261012T020000Z").unwrap().local(&|_| -300).date(), d(2026, 10, 11)); + let local = When::parse("20261012T090000").unwrap(); + assert_eq!(local.local(&|_| -240), at(2026, 10, 12, 9, 0, 0)); + assert_eq!(local.timestamp(&|_| -240), utc.timestamp(&|_| 600)); + assert_eq!(When::Date(d(1970, 1, 2)).timestamp(&|_| 60), DAY - 3600); + // A zone that changes its offset at an instant: each side gets its own. + let change = to_timestamp(at(2026, 11, 1, 6, 0, 0), 0); + let zone = |ts: i64| if ts < change { -240 } else { -300 }; + assert_eq!(When::parse("20261101T000000").unwrap().timestamp(&zone), to_timestamp(at(2026, 11, 1, 4, 0, 0), 0)); + assert_eq!(When::parse("20261101T120000").unwrap().timestamp(&zone), to_timestamp(at(2026, 11, 1, 17, 0, 0), 0)); + // A hostile offset is survived. + let _ = utc.local(&|_| i32::MIN); + let _ = local.timestamp(&|_| i32::MAX); + } + + #[test] + fn durations() { + assert_eq!(parse_duration("PT15M"), Some(900)); + assert_eq!(parse_duration("-PT15M"), Some(-900)); + assert_eq!(parse_duration("+PT1H30M"), Some(5400)); + assert_eq!(parse_duration("P1D"), Some(DAY)); + assert_eq!(parse_duration("P1DT12H"), Some(DAY + 12 * 3600)); + assert_eq!(parse_duration("P2W"), Some(14 * DAY)); + assert_eq!(parse_duration("pt0s"), Some(0)); + assert_eq!(parse_duration("-PT0M"), Some(0)); + assert_eq!(parse_duration("PT1H2M3S"), Some(3723)); + for bad in ["", "P", "PT", "15M", "PT15", "P15M", "PT1D", "PTM", "P1X", "P-1D", "P99999999999999999999D", "P9999999999W", "PT1H T"] { + assert_eq!(parse_duration(bad), None, "{bad}"); + } + } + + #[test] + fn formats_and_keeps_dates_in_writable_years() { + assert_eq!(format_date(d(2026, 1, 2)), "20260102"); + assert_eq!(format_date(d(987, 6, 5)), "09870605"); + assert_eq!(format_local(at(2026, 10, 12, 9, 5, 0)), "20261012T090500"); + assert_eq!(format_utc(0), "19700101T000000Z"); + assert_eq!(format_utc(1_760_082_785), "20251010T075305Z"); + assert_eq!(format_utc(i64::MAX), "99991231T235959Z"); + assert_eq!(format_utc(i64::MIN), "00010101T000000Z"); + // Years iCalendar can't write are moved to the nearest it can. + for hostile in [NaiveDate::MIN, NaiveDate::MAX, d(0, 1, 1), d(-4, 2, 29), d(10_000, 2, 29)] { + assert!(When::parse(&format_date(hostile)).is_some(), "{hostile:?}"); + assert!(When::parse(&format_local(at_midnight(hostile))).is_some(), "{hostile:?}"); + } + assert_eq!(format_date(NaiveDate::MAX), "99991231"); + assert_eq!(format_local(NaiveDateTime::MIN), "00010101T000000"); + // Civil seconds are the same clock read as UTC, and are kept in the years 1 to 9999. + assert_eq!(civil_seconds(at(1970, 1, 2, 0, 0, 1)), DAY + 1); + assert_eq!(from_civil_seconds(DAY + 1), at(1970, 1, 2, 0, 0, 1)); + assert_eq!(from_civil_seconds(i64::MAX), at(9999, 12, 31, 23, 59, 59)); + assert_eq!(from_civil_seconds(i64::MIN), at(1, 1, 1, 0, 0, 0)); + assert_eq!(from_timestamp(0, -300), at(1969, 12, 31, 19, 0, 0)); + assert_eq!(to_timestamp(at(1969, 12, 31, 19, 0, 0), -300), 0); + } + + #[test] + fn text_escapes() { + assert_eq!(escape_text("Budget, phase two; \\ done\nNext"), "Budget\\, phase two\\; \\\\ done\\nNext"); + assert_eq!(escape_text("a\r\nb\rc\n\nd\u{0}\u{7f}\te"), "a\\nb\\nc\\n\\nd\te"); + assert_eq!(unescape_text("Budget\\, phase two\\; \\\\ done\\nNext\\NLast"), "Budget, phase two; \\ done\nNext\nLast"); + // What other writers escape without being asked to, and what isn't an escape at all. + assert_eq!(unescape_text("10\\:30 \\\"quoted\\\" C:\\Users\\x tail\\"), "10:30 \"quoted\" C:\\Users\\x tail\\"); + assert_eq!(unescape_text("a\u{0}b\u{1b}c\td"), "abc\td"); + } + + #[test] + fn lists_split_on_unescaped_commas() { + assert_eq!(split_list("Work,Travel\\, abroad,,Home\\\\,x", 10), ["Work", "Travel\\, abroad", "", "Home\\\\", "x"]); + assert_eq!(split_list("", 10), [""]); + assert_eq!(split_list("a,b,c,d", 2), ["a", "b"]); + assert_eq!(split_list("a,b", 2), ["a", "b"]); + assert!(split_list("a,b", 0).is_empty()); + } + + proptest::proptest! { + #[test] + fn clean_text_survives_escaping(s in "(\\PC|[\n\t;,\\\\:\"]){0,80}") { + proptest::prop_assert_eq!(unescape_text(&escape_text(&s)), s); + } + + #[test] + fn any_text_escapes_to_one_clean_line(s in proptest::prelude::any::()) { + let escaped = escape_text(&s); + proptest::prop_assert!(!escaped.contains(['\r', '\n'])); + // Escaping what was read gives the same line again. + proptest::prop_assert_eq!(escape_text(&unescape_text(&escaped)), escaped); + } + + #[test] + fn values_never_panic(s in proptest::prelude::any::(), max in 0usize..5) { + let _ = When::parse(&s); + let _ = parse_duration(&s); + let _ = unescape_text(&s); + let _ = split_list(&s, max); + } + + #[test] + fn any_civil_second_reads_back_in_writable_years(secs in proptest::prelude::any::()) { + let t = from_civil_seconds(secs); + proptest::prop_assert_eq!(civil_seconds(t), secs.clamp(MIN_CIVIL, MAX_CIVIL)); + proptest::prop_assert!(When::parse(&format_local(t)).is_some()); + } + } +} diff --git a/crates/ical/src/zones.rs b/crates/ical/src/zones.rs new file mode 100644 index 000000000..9d01d93b8 --- /dev/null +++ b/crates/ical/src/zones.rs @@ -0,0 +1,498 @@ +//! Time zones in iCalendar text (RFC 5545 §3.6.5): the `VTIMEZONE` definitions a text brings with +//! it, read ([`TimeZones`], [`VTimezone`]) and written ([`write_vtimezone`]). +//! +//! This crate has no time-zone database. A definition read from a text answers for itself (the +//! offset at an instant, the instant of a wall-clock time); what a `TZID` means beyond the text +//! (an IANA name, a Windows name, which database zone keeps the same clock) is the caller's. To +//! write a definition, the caller hands over its zone's rules through [`ZoneRules`]. +//! +//! Definitions are hostile text: offsets of more than eighteen hours are not offsets, and the +//! zones, their parts and their dates are all capped. [`TimeZones::spend`] keeps a budget of the +//! work a caller does with them. + +use std::cell::Cell; + +use chrono::{Datelike, NaiveDate, NaiveDateTime, Weekday}; + +use crate::lines::{ContentLine, Lines, fold, prefix}; +use crate::rrule::{weekday_code, weekday_of_code}; +use crate::values::{When, at_midnight, civil_seconds, format_local, from_civil_seconds, from_timestamp, to_timestamp}; + +/// Most zones one text defines, most parts (`STANDARD`, `DAYLIGHT`) of one zone, and most dates +/// (`RDATE`) of one part. +pub const MAX_ZONES: usize = 64; +pub const MAX_PARTS: usize = 64; +pub const MAX_DATES: usize = 64; +/// The work one text's times may cost, in units of one look at one part or one date of a +/// definition ([`VTimezone::cost`]). Past it, [`TimeZones::spend`] says no. +pub const WORK_BUDGET: u64 = 20_000_000; +/// The furthest from UTC an offset may be, in seconds. +const MAX_OFFSET: i32 = 18 * 3600; +/// Longest `TZID` (and `X-LIC-LOCATION`) kept, in bytes. +pub const MAX_TZID: usize = 128; + +/// On which day of each year a part of a zone begins. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Yearly { + /// The `nth` (1–5, or negative from the end) `weekday` of `month`. + Nth { month: u32, nth: i8, weekday: Weekday }, + /// Day `day` of `month`. + Day { month: u32, day: u32 }, +} + +/// The `nth` (1–5, or negative from the end) `weekday` of a month, when the month has one. +fn nth_weekday(year: i32, month: u32, weekday: Weekday, nth: i8) -> Option { + let first = NaiveDate::from_ymd_opt(year, month, 1)?; + if nth > 0 { + let offset = (i64::from(weekday.num_days_from_monday()) - i64::from(first.weekday().num_days_from_monday())).rem_euclid(7); + NaiveDate::from_ymd_opt(year, month, u32::try_from(1 + offset + 7 * (i64::from(nth) - 1)).ok()?) + } else { + let last = first.checked_add_months(chrono::Months::new(1))?.pred_opt()?; + let back = (i64::from(last.weekday().num_days_from_monday()) - i64::from(weekday.num_days_from_monday())).rem_euclid(7); + let day = last.checked_sub_days(chrono::Days::new(u64::try_from(back + 7 * (i64::from(nth.saturating_neg()) - 1)).ok()?))?; + (day.month() == month).then_some(day) + } +} + +impl Yearly { + fn date(self, year: i32) -> Option { + match self { + Yearly::Day { month, day } => NaiveDate::from_ymd_opt(year, month, day), + Yearly::Nth { month, nth, weekday } => nth_weekday(year, month, weekday, nth), + } + } +} + +/// One `STANDARD` or `DAYLIGHT` part of a definition: from `start` on (a wall-clock time by the +/// offset `from`), and then again every year or on the listed dates, the offset is `to`. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Part { + start: NaiveDateTime, + /// Seconds east of UTC before and after. + from: i32, + to: i32, + yearly: Option, + /// The rule holds up to this instant, and for this many beginnings. + until: Option, + count: Option, + dates: Vec, +} + +impl Part { + /// The wall-clock second from which `to` holds for a beginning written as `start`: a time + /// the clocks skip is still by the offset before, and so is the first of an hour that comes + /// twice (RFC 5545 §3.3.5). + fn wall_from(&self, start: NaiveDateTime) -> i64 { + civil_seconds(start).saturating_add(i64::from(self.to.saturating_sub(self.from).max(0))) + } + + fn instant_of(&self, start: NaiveDateTime) -> i64 { + civil_seconds(start).saturating_sub(i64::from(self.from)) + } + + /// The beginning of the rule's year `year`, when the rule has one then. + fn in_year(&self, year: i32) -> Option { + let rule = self.yearly?; + let first = self.start.year(); + if year < first || self.count.is_some_and(|count| i64::from(year) - i64::from(first) >= i64::from(count)) { + return None; + } + let begins = rule.date(year)?.and_time(self.start.time()); + (begins >= self.start && self.until.is_none_or(|until| self.instant_of(begins) <= until)).then_some(begins) + } + + /// The last beginning at or before `limit`, as `key` measures beginnings. + fn latest(&self, limit: i64, year: i32, key: &dyn Fn(&Part, NaiveDateTime) -> i64) -> Option { + let mut best: Option = None; + let mut take = |begins: NaiveDateTime| { + let at = key(self, begins); + if at <= limit && best.is_none_or(|have| at > have) { + best = Some(at); + } + }; + take(self.start); + self.dates.iter().copied().for_each(&mut take); + // The year of the time asked about and the years beside it; and where the rule ends. + let last = self.until.map(|until| from_timestamp(until, 0).year()); + let counted = self.count.map(|count| self.start.year().saturating_add(i32::try_from(count).unwrap_or(i32::MAX)).saturating_sub(1)); + for year in [Some(year.saturating_add(1)), Some(year), Some(year.saturating_sub(1)), last, last.map(|y| y.saturating_sub(1)), counted] + .into_iter() + .flatten() + { + if let Some(begins) = self.in_year(year) { + take(begins); + } + } + best + } +} + +/// A zone as a text defines it (`VTIMEZONE`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct VTimezone { + tzid: String, + /// `X-LIC-LOCATION`: the place the writer says the zone is (an IANA name, as a rule). + place: String, + parts: Vec, +} + +impl VTimezone { + /// The `TZID`, at most [`MAX_TZID`] bytes. + pub fn tzid(&self) -> &str { + &self.tzid + } + + /// `X-LIC-LOCATION`, empty when the text gives none. + pub fn place(&self) -> &str { + &self.place + } + + /// What one look at the definition costs, in units of [`WORK_BUDGET`]. + pub fn cost(&self) -> u64 { + self.parts.iter().fold(0u64, |sum, part| sum.saturating_add(8).saturating_add(part.dates.len() as u64)) + } + + fn offset_by(&self, limit: i64, year: i32, key: &dyn Fn(&Part, NaiveDateTime) -> i64) -> i32 { + let mut best: Option<(i64, i32)> = None; + for part in &self.parts { + if let Some(at) = part.latest(limit, year, key) + && best.is_none_or(|(have, _)| at >= have) + { + best = Some((at, part.to)); + } + } + match best { + Some((_, to)) => to, + // Before the first beginning: the offset the first one changes from. + None => self.parts.iter().min_by_key(|part| part.start).map_or(0, |part| part.from), + } + } + + /// Seconds east of UTC for a wall-clock time written in the zone. A time the clocks skip is + /// read by the offset before the change; a time that comes twice is the first of the two. + pub fn offset_for_wall(&self, wall: NaiveDateTime) -> i32 { + self.offset_by(civil_seconds(wall), wall.year(), &|part, begins| part.wall_from(begins)) + } + + /// Seconds east of UTC at an instant (seconds since 1970-01-01T00:00:00Z). + pub fn offset_at(&self, instant: i64) -> i32 { + self.offset_by(instant, from_timestamp(instant, 0).year(), &|part, begins| part.instant_of(begins)) + } + + /// The instant of a wall-clock time written in the zone. + pub fn instant(&self, wall: NaiveDateTime) -> i64 { + civil_seconds(wall).saturating_sub(i64::from(self.offset_for_wall(wall))) + } +} + +/// `+0100`, `-0500`, `+053000`: seconds east of UTC; `None` for anything else and for more than +/// an offset can be. +pub fn parse_utc_offset(text: &str) -> Option { + let text = text.trim(); + let (sign, digits) = match (text.strip_prefix('+'), text.strip_prefix('-')) { + (Some(rest), _) => (1, rest), + (None, Some(rest)) => (-1, rest), + (None, None) => return None, + }; + let digits: String = digits.chars().filter(|c| *c != ':').collect(); + if !matches!(digits.len(), 4 | 6) || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let part = |at: usize| digits.get(at..at.saturating_add(2)).and_then(|d| d.parse::().ok()); + let (hours, minutes, seconds) = (part(0)?, part(2)?, if digits.len() == 6 { part(4)? } else { 0 }); + let total = hours * 3600 + minutes * 60 + seconds; + (minutes < 60 && seconds < 60 && total <= MAX_OFFSET).then_some(sign * total) +} + +/// An offset in seconds as `+0100`, `-0930`, `+045602`. +pub fn format_utc_offset(seconds: i32) -> String { + let sign = if seconds < 0 { '-' } else { '+' }; + let a = seconds.unsigned_abs(); + if a.is_multiple_of(60) { + format!("{sign}{:02}{:02}", a / 3600, a / 60 % 60) + } else { + format!("{sign}{:02}{:02}{:02}", a / 3600, a / 60 % 60, a % 60) + } +} + +/// The rule of a part: `FREQ=YEARLY;BYMONTH=3;BYDAY=2SU`, with the day of the month or the +/// part's own day when no weekday is named. `None` for a rule that isn't yearly. +fn yearly(rule: &str, start: NaiveDateTime) -> Option<(Yearly, Option, Option)> { + let (mut yearly, mut month, mut by_day, mut month_day) = (false, None, None, None); + let (mut until, mut count) = (None, None); + for part in prefix(rule, 512).split(';').take(16) { + let Some((key, value)) = part.split_once('=') else { continue }; + let value = value.trim(); + match key.trim().to_ascii_uppercase().as_str() { + "FREQ" => yearly = value.eq_ignore_ascii_case("YEARLY"), + "INTERVAL" if value != "1" => return None, + "BYMONTH" => month = value.parse::().ok().filter(|m| (1..=12).contains(m)), + "BYMONTHDAY" => month_day = value.parse::().ok().filter(|d| (1..=31).contains(d)), + "BYDAY" => { + let split = value.len().checked_sub(2)?; + let weekday = weekday_of_code(value.get(split..)?)?; + let nth = match value.get(..split)? { + "" => return None, + nth => nth.parse::().ok().filter(|n| (1..=5).contains(n) || (-5..=-1).contains(n))?, + }; + by_day = Some((nth, weekday)); + } + "UNTIL" => until = When::parse(value).map(|at| to_timestamp(at.wall(), 0)), + "COUNT" => count = value.parse::().ok(), + _ => {} + } + } + if !yearly { + return None; + } + let month = month.unwrap_or(start.month()); + let rule = match (by_day, month_day) { + (Some((nth, weekday)), _) => Yearly::Nth { month, nth, weekday }, + (None, day) => Yearly::Day { month, day: day.unwrap_or(start.day()) }, + }; + Some((rule, until, count)) +} + +/// A `STANDARD` or `DAYLIGHT` part being read. +#[derive(Default)] +struct PartBuilder { + start: Option, + from: Option, + to: Option, + rule: Option, + dates: Vec, +} + +impl PartBuilder { + fn finish(self) -> Option { + let (start, from, to) = (self.start?, self.from?, self.to?); + let (yearly, until, count) = match self.rule.as_deref().and_then(|rule| yearly(rule, start)) { + Some((rule, until, count)) => (Some(rule), until, count), + None => (None, None, None), + }; + Some(Part { start, from, to, yearly, until, count, dates: self.dates }) + } +} + +/// The zones a text defines, and the budget of work spent on them. +#[derive(Debug, Default)] +pub struct TimeZones { + zones: Vec, + /// Work spent so far (see [`WORK_BUDGET`]), spends granted so far, and whether the budget ran out. + spent: Cell, + granted: Cell, + stopped: Cell, +} + +impl TimeZones { + /// Read the `VTIMEZONE`s of a text: at most [`MAX_ZONES`], the first of each `TZID`. A zone + /// without a `TZID` or without a part that has a start and both offsets is not one. + pub fn collect(text: &str) -> TimeZones { + let mut out = TimeZones::default(); + // Nothing to look for in most texts. + let named = text.as_bytes().windows(9).any(|bytes| bytes.eq_ignore_ascii_case(b"VTIMEZONE")); + if !named { + return out; + } + let mut zone: Option = None; + let mut part: Option = None; + for raw in Lines::new(text) { + let Some(line) = ContentLine::parse(&raw) else { continue }; + let value = prefix(line.value.trim(), MAX_TZID); + match (line.name.as_str(), value.to_ascii_uppercase().as_str()) { + ("BEGIN", "VTIMEZONE") => { + if out.zones.len() >= MAX_ZONES { + break; + } + zone = Some(VTimezone::default()); + part = None; + } + ("END", "VTIMEZONE") => { + part = None; + if let Some(definition) = zone.take() + && !definition.tzid.is_empty() + && !definition.parts.is_empty() + && !out.zones.iter().any(|have| have.tzid == definition.tzid) + { + out.zones.push(definition); + } + } + ("BEGIN", "STANDARD" | "DAYLIGHT") if zone.is_some() => part = Some(PartBuilder::default()), + ("END", "STANDARD" | "DAYLIGHT") => { + if let (Some(definition), Some(built)) = (&mut zone, part.take().and_then(PartBuilder::finish)) + && definition.parts.len() < MAX_PARTS + { + definition.parts.push(built); + } + } + // Anything else that begins inside a zone is not a part of it. + ("BEGIN", _) => part = None, + (name, _) => match (&mut zone, &mut part) { + (Some(_), Some(part)) => match name { + "DTSTART" => part.start = part.start.or_else(|| When::parse(&line.value).map(When::wall)), + "TZOFFSETFROM" => part.from = part.from.or_else(|| parse_utc_offset(&line.value)), + "TZOFFSETTO" => part.to = part.to.or_else(|| parse_utc_offset(&line.value)), + "RRULE" if part.rule.is_none() => part.rule = Some(prefix(&line.value, 512).to_string()), + "RDATE" => { + let room = MAX_DATES.saturating_sub(part.dates.len()); + part.dates.extend(line.value.split(',').take(room).filter_map(When::parse).map(When::wall)); + } + _ => {} + }, + (Some(definition), None) => match name { + "TZID" if definition.tzid.is_empty() => definition.tzid = value.to_string(), + "X-LIC-LOCATION" if definition.place.is_empty() => definition.place = value.to_string(), + _ => {} + }, + _ => {} + }, + } + } + out + } + + /// The definition of `tzid`: the one of that name, else one whose name differs only in letter + /// case. The name is looked at to [`MAX_TZID`] bytes. + pub fn get(&self, tzid: &str) -> Option<&VTimezone> { + let tzid = prefix(tzid.trim(), MAX_TZID); + self.zones.iter().find(|zone| zone.tzid == tzid).or_else(|| self.zones.iter().find(|zone| zone.tzid.eq_ignore_ascii_case(tzid))) + } + + /// Every definition, in the order of the text. + pub fn iter(&self) -> impl Iterator { + self.zones.iter() + } + + pub fn is_empty(&self) -> bool { + self.zones.is_empty() + } + + /// Spend `units` of the text's work budget; `false` (and the budget is out for good) when + /// there is not enough left. + pub fn spend(&self, units: u64) -> bool { + let total = self.spent.get().saturating_add(units); + if self.stopped.get() || total > WORK_BUDGET { + self.stopped.set(true); + return false; + } + self.spent.set(total); + self.granted.set(self.granted.get().saturating_add(1)); + true + } + + /// When the budget ran out: how many spends were granted before it did. + pub fn stopped_after(&self) -> Option { + self.stopped.get().then(|| self.granted.get()) + } +} + +// ---- writing ---- + +/// A change of a zone's offset: at the instant `at` (seconds since 1970-01-01T00:00:00Z), from +/// `before` to `after` seconds east of UTC. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Transition { + pub at: i64, + pub before: i32, + pub after: i32, +} + +/// What [`write_vtimezone`] needs to know of a zone: the caller's time-zone database. +pub trait ZoneRules { + /// Seconds east of UTC at an instant. + fn offset_at(&self, instant: i64) -> i32; + /// The changes of offset from `from` up to `to` (instants), in order. + fn transitions(&self, from: i64, to: i64) -> Vec; +} + +/// The rule a change of clocks follows when it falls on the same weekday of the same month each +/// year at the same time of day: the `nth` weekday (the last, when it is in the month's last week +/// in each of the years). +fn yearly_rule(changes: &[(NaiveDateTime, i32, i32)]) -> Option { + let (first, first_from, first_to) = changes.first().copied()?; + let alike = changes.iter().all(|(at, from, to)| { + at.month() == first.month() && at.weekday() == first.weekday() && at.time() == first.time() && (*from, *to) == (first_from, first_to) + }); + if !alike { + return None; + } + let nth = |at: &NaiveDateTime| (at.day() - 1) / 7 + 1; + let last = |at: &NaiveDateTime| at.date().checked_add_days(chrono::Days::new(7)).is_none_or(|week_on| week_on.month() != at.month()); + let (month, weekday) = (first.month(), first.weekday()); + if changes.iter().all(|(at, _, _)| nth(at) == nth(&first)) && nth(&first) < 5 && !changes.iter().all(|(at, _, _)| last(at)) { + Some(Yearly::Nth { month, nth: i8::try_from(nth(&first)).unwrap_or(1), weekday }) + } else if changes.iter().all(|(at, _, _)| last(at)) { + Some(Yearly::Nth { month, nth: -1, weekday }) + } else { + None + } +} + +fn write_part(out: &mut String, kind: &str, start: NaiveDateTime, from: i32, to: i32, rule: Option) { + fold(out, &format!("BEGIN:{kind}")); + fold(out, &format!("DTSTART:{}", format_local(start))); + fold(out, &format!("TZOFFSETFROM:{}", format_utc_offset(from))); + fold(out, &format!("TZOFFSETTO:{}", format_utc_offset(to))); + if let Some(Yearly::Nth { month, nth, weekday }) = rule { + fold(out, &format!("RRULE:FREQ=YEARLY;BYMONTH={month};BYDAY={nth}{}", weekday_code(weekday))); + } + fold(out, &format!("END:{kind}")); +} + +/// Write the definition of a zone named `tzid` (control characters dropped, at most [`MAX_TZID`] +/// bytes) for times in the years `from`–`to` (at most 120 +/// years are written; the years are kept in 2 to 9990). +/// +/// A zone that changes its clocks twice a year by one rule is written as most programs write it, +/// and all of them read it: one `STANDARD` and one `DAYLIGHT` part, each with its yearly rule. A +/// zone that doesn't change is one `STANDARD` part. Any other (a year in which the rule changed, +/// a zone that follows the moon) is written out change by change for those years. +pub fn write_vtimezone(out: &mut String, tzid: &str, zone: &dyn ZoneRules, from: i32, to: i32) { + let (from, to) = (from.clamp(2, 9_990), to.clamp(2, 9_990).max(from.clamp(2, 9_990))); + // A few hundred changes at most are looked at. + let to = to.min(from.saturating_add(120)); + let instant = |year: i32| NaiveDate::from_ymd_opt(year, 1, 1).map_or(0, |day| to_timestamp(at_midnight(day), 0)); + let before = zone.offset_at(instant(from - 1)); + // Each change as it is written: the wall-clock time by the offset before it. + let changes: Vec<(NaiveDateTime, i32, i32)> = zone + .transitions(instant(from - 1), instant(to + 1)) + .into_iter() + .map(|change| (from_civil_seconds(change.at.saturating_add(i64::from(change.before))), change.before, change.after)) + .collect(); + let midnight = |year: i32| NaiveDate::from_ymd_opt(year, 1, 1).map(at_midnight).unwrap_or_default(); + fold(out, "BEGIN:VTIMEZONE"); + // A name can't start another line. + let tzid: String = prefix(tzid, MAX_TZID).chars().filter(|c| !c.is_control()).collect(); + fold(out, &format!("TZID:{tzid}")); + if changes.is_empty() { + write_part(out, "STANDARD", midnight(1601), before, before, None); + fold(out, "END:VTIMEZONE"); + return; + } + // Clocks forward is the beginning of daylight time, clocks back the end of it. + let (forward, back): (Vec<_>, Vec<_>) = changes.iter().copied().partition(|(_, from, to)| to > from); + let years = usize::try_from(i64::from(to) - i64::from(from) + 2).unwrap_or(0); + let regular = forward.len() == years && back.len() == years; + match (yearly_rule(&forward).filter(|_| regular), yearly_rule(&back).filter(|_| regular)) { + (Some(spring), Some(autumn)) => { + for (kind, list, rule) in [("DAYLIGHT", &forward, spring), ("STANDARD", &back, autumn)] { + if let Some((start, from, to)) = list.first().copied() { + write_part(out, kind, start, from, to, Some(rule)); + } + } + } + _ => { + // What holds before the first change, then each change. + let first = changes.first().map_or(NaiveDateTime::default(), |(at, _, _)| *at); + write_part(out, "STANDARD", midnight(first.year().saturating_sub(1).max(1)), before, before, None); + for (start, from, to) in changes { + write_part(out, if to > from { "DAYLIGHT" } else { "STANDARD" }, start, from, to, None); + } + } + } + fold(out, "END:VTIMEZONE"); +} + +#[cfg(test)] +#[path = "zones_tests.rs"] +mod tests; diff --git a/crates/ical/src/zones_tests.rs b/crates/ical/src/zones_tests.rs new file mode 100644 index 000000000..5c1ec9eba --- /dev/null +++ b/crates/ical/src/zones_tests.rs @@ -0,0 +1,293 @@ +//! `VTIMEZONE` reading and writing. The zones written are rule zones built here (the United States' +//! rules, the European Union's, Sydney's, a zone without changes, the United States' change of +//! rule in 2007), standing in for a caller's time-zone database. + +use proptest::prelude::*; + +use super::*; + +fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() +} + +fn at(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, 0).unwrap() +} + +fn utc(t: NaiveDateTime) -> i64 { + to_timestamp(t, 0) +} + +/// A zone by its list of changes. +struct Rules { + before: i32, + changes: Vec, +} + +impl ZoneRules for Rules { + fn offset_at(&self, instant: i64) -> i32 { + self.changes.iter().rev().find(|c| c.at <= instant).map_or(self.before, |c| c.after) + } + + fn transitions(&self, from: i64, to: i64) -> Vec { + self.changes.iter().copied().filter(|c| c.at >= from && c.at < to).collect() + } +} + +/// The change on the `nth` `weekday` of `month` at `hour` UTC (`day_before`: the evening before). +fn change(year: i32, month: u32, nth: i8, hour: u32, day_before: bool, before: i32, after: i32) -> Transition { + let day = nth_weekday(year, month, Weekday::Sun, nth).unwrap(); + let day = if day_before { day.pred_opt().unwrap() } else { day }; + Transition { at: utc(day.and_hms_opt(hour, 0, 0).unwrap()), before, after } +} + +const H: i32 = 3600; + +/// The United States' rule since 2007 (and, with `history`, the one before it up to 2006). +fn us_eastern(years: std::ops::RangeInclusive, history: bool) -> Rules { + let mut changes = Vec::new(); + for y in years { + if history && y < 2007 { + changes.push(change(y, 4, 1, 7, false, -5 * H, -4 * H)); + changes.push(change(y, 10, -1, 6, false, -4 * H, -5 * H)); + } else { + changes.push(change(y, 3, 2, 7, false, -5 * H, -4 * H)); + changes.push(change(y, 11, 1, 6, false, -4 * H, -5 * H)); + } + } + Rules { before: -5 * H, changes } +} + +fn berlin(years: std::ops::RangeInclusive) -> Rules { + let changes = years.flat_map(|y| [change(y, 3, -1, 1, false, H, 2 * H), change(y, 10, -1, 1, false, 2 * H, H)]).collect(); + Rules { before: H, changes } +} + +fn sydney(years: std::ops::RangeInclusive) -> Rules { + let changes = years.flat_map(|y| [change(y, 4, 1, 16, true, 11 * H, 10 * H), change(y, 10, 1, 16, true, 10 * H, 11 * H)]).collect(); + Rules { before: 11 * H, changes } +} + +fn fixed(seconds: i32) -> Rules { + Rules { before: seconds, changes: Vec::new() } +} + +/// The zone as a big mail program writes it: today's rule, from the year 1601. +const EASTERN: &str = "BEGIN:VTIMEZONE\r\nTZID:Eastern Standard Time\r\nBEGIN:STANDARD\r\nDTSTART:16011104T020000\r\nRRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11\r\n\ +TZOFFSETFROM:-0400\r\nTZOFFSETTO:-0500\r\nEND:STANDARD\r\nBEGIN:DAYLIGHT\r\nDTSTART:16010311T020000\r\nRRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3\r\n\ +TZOFFSETFROM:-0500\r\nTZOFFSETTO:-0400\r\nEND:DAYLIGHT\r\nEND:VTIMEZONE\r\n"; + +fn only(text: &str) -> VTimezone { + let zones = TimeZones::collect(text); + assert_eq!(zones.iter().count(), 1, "{text}"); + zones.iter().next().unwrap().clone() +} + +#[test] +fn a_definition_gives_each_time_its_offset() { + let eastern = only(EASTERN); + assert_eq!(eastern.tzid(), "Eastern Standard Time"); + assert_eq!(eastern.offset_for_wall(at(2026, 1, 15, 9, 0)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 7, 15, 9, 0)), -4 * 3600); + // March 8, 2026: 1:59 is winter time, 3:00 summer time, and 2:30 (which no clock shows) + // is still read by the offset before. + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 1, 59)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 2, 30)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 3, 0)), -4 * 3600); + // November 1: 1:30 comes twice, and is the first of the two. + assert_eq!(eastern.offset_for_wall(at(2026, 11, 1, 1, 30)), -4 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 11, 1, 2, 0)), -5 * 3600); + // By instants, every half hour of two years agrees with the rules. + let rules = us_eastern(2025..=2028, false); + let mut wall = at(2026, 1, 1, 0, 0); + while wall < at(2028, 1, 1, 0, 0) { + let instant = utc(wall); + assert_eq!(eastern.offset_at(instant), rules.offset_at(instant), "{wall}"); + wall += chrono::Duration::minutes(30); + } + assert_eq!(eastern.instant(at(2026, 7, 15, 9, 0)), utc(at(2026, 7, 15, 13, 0))); + // Before there were any rules: what the first change changes from. + assert_eq!(eastern.offset_for_wall(at(1500, 6, 1, 0, 0)), -5 * 3600); +} + +#[test] +fn definitions_of_every_shape_are_read() { + // Listed dates, a rule that ends, a rule by the day of the month, a part with no rule. + let text = "BEGIN:VTIMEZONE\nTZID:Example/Odd\nX-LIC-LOCATION:Europe/Berlin\n\ +BEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +BEGIN:DAYLIGHT\nDTSTART:20200329T020000\nRDATE:20210328T020000,20220327T020000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0200\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20201025T030000\nRRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=20221030T010000Z\nTZOFFSETFROM:+0200\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +BEGIN:DAYLIGHT\nDTSTART:20240401T000000\nRRULE:FREQ=YEARLY;BYMONTHDAY=1;BYMONTH=4;COUNT=2\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0130\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20241001T000000\nRRULE:FREQ=YEARLY\nTZOFFSETFROM:+0130\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +END:VTIMEZONE\n"; + let odd = only(text); + let offset = |wall: NaiveDateTime| odd.offset_for_wall(wall) / 60; + assert_eq!(offset(at(2019, 7, 1, 12, 0)), 60); + assert_eq!(offset(at(2020, 7, 1, 12, 0)), 120); + assert_eq!(offset(at(2021, 1, 1, 12, 0)), 60); + assert_eq!(offset(at(2022, 7, 1, 12, 0)), 120); + assert_eq!(offset(at(2023, 7, 1, 12, 0)), 60); + // Two Aprils of half an hour, and no third. + assert_eq!(offset(at(2024, 7, 1, 12, 0)), 90); + assert_eq!(offset(at(2025, 7, 1, 12, 0)), 90); + assert_eq!(offset(at(2025, 12, 1, 12, 0)), 60); + assert_eq!(offset(at(2026, 7, 1, 12, 0)), 60); + // It says where it is, and is found by its name in any letter case. + let zones = TimeZones::collect(text); + assert_eq!(zones.get("Example/Odd").map(VTimezone::place), Some("Europe/Berlin")); + assert!(zones.get("example/odd").is_some()); + assert!(zones.get("Example/Even").is_none()); +} + +#[test] +fn nonsense_definitions_are_no_definitions() { + for text in [ + "BEGIN:VTIMEZONE\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:Empty\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + // Offsets of days, and no offsets at all. + "BEGIN:VTIMEZONE\nTZID:Far\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+9900\nTZOFFSETTO:-7200\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:Far\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+1801\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:None\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:soon\nTZOFFSETTO:later\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:NoStart\nBEGIN:STANDARD\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + // A part outside a zone, and a zone that never ends. + "BEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + "BEGIN:VTIMEZONE\nTZID:Open\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + ] { + assert!(TimeZones::collect(text).is_empty(), "{text}"); + } + assert_eq!(parse_utc_offset("+053000"), Some(5 * 3600 + 30 * 60)); + assert_eq!(parse_utc_offset("-00:30"), Some(-30 * 60)); + for bad in ["", "0100", "+1", "+01", "+0160", "+010060", "+2400", "+١٢٠٠", "++0100"] { + assert_eq!(parse_utc_offset(bad), None, "{bad}"); + } + assert_eq!((format_utc_offset(-570 * 60), format_utc_offset(0), format_utc_offset(16_562)), ("-0930".into(), "+0000".into(), "+043602".into())); + // A megabyte of name, hundreds of zones, thousands of parts and dates: all bounded. + let long = format!( + "BEGIN:VTIMEZONE\nTZID:{}\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + "Z".repeat(1 << 20) + ); + let zones = TimeZones::collect(&long); + assert!(only(&long).tzid().len() <= MAX_TZID); + assert!(zones.get(&"Z".repeat(1 << 20)).is_some()); + let many: String = (0..500) + .map(|n| format!("BEGIN:VTIMEZONE\nTZID:Zone{n}\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n")) + .collect(); + assert_eq!(TimeZones::collect(&many).iter().count(), MAX_ZONES); + let part = format!( + "BEGIN:STANDARD\nDTSTART:19700101T000000\nRDATE:{}\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + "19800101T000000,".repeat(5_000) + ); + let crowded = only(&format!("BEGIN:VTIMEZONE\nTZID:Crowded\n{}END:VTIMEZONE\n", part.repeat(1_000))); + assert_eq!((crowded.parts.len(), crowded.parts[0].dates.len()), (MAX_PARTS, MAX_DATES)); + assert_eq!(crowded.cost(), (MAX_PARTS * (8 + MAX_DATES)) as u64); + let _ = crowded.offset_for_wall(at(2026, 1, 1, 0, 0)); + // Hostile times ask nothing that panics. + let eastern = only(EASTERN); + for wall in [NaiveDateTime::MAX, NaiveDateTime::MIN, NaiveDateTime::default()] { + let _ = (eastern.offset_for_wall(wall), eastern.instant(wall)); + } + let _ = (eastern.offset_at(i64::MIN), eastern.offset_at(i64::MAX)); +} + +#[test] +fn the_budget_runs_out_for_good() { + let zones = TimeZones::collect(EASTERN); + assert_eq!(zones.stopped_after(), None); + assert!(zones.spend(WORK_BUDGET - 10)); + assert!(zones.spend(10)); + assert!(!zones.spend(1)); + // Once out, out: even a spend of nothing is refused. + assert!(!zones.spend(0)); + assert_eq!(zones.stopped_after(), Some(2)); +} + +/// What a written definition says, read back. +fn written(name: &str, zone: &dyn ZoneRules, from: i32, to: i32) -> (String, VTimezone) { + let mut out = String::new(); + write_vtimezone(&mut out, name, zone, from, to); + let read = only(&out); + (out, read) +} + +#[test] +fn written_definitions_are_the_usual_two_rules_where_a_zone_has_them() { + let (text, _) = written("America/New_York", &us_eastern(2025..=2027, false), 2026, 2026); + assert_eq!( + text.replace("\r\n", "\n"), + "BEGIN:VTIMEZONE\nTZID:America/New_York\n\ +BEGIN:DAYLIGHT\nDTSTART:20250309T020000\nTZOFFSETFROM:-0500\nTZOFFSETTO:-0400\nRRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20251102T020000\nTZOFFSETFROM:-0400\nTZOFFSETTO:-0500\nRRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU\nEND:STANDARD\n\ +END:VTIMEZONE\n" + ); + let (text, _) = written("Europe/Berlin", &berlin(2025..=2031), 2026, 2030); + assert!(text.contains("RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU") && text.contains("RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU"), "{text}"); + assert!(text.contains("DTSTART:20250330T020000") && text.contains("DTSTART:20251026T030000"), "{text}"); + // The south: summer time begins in October. + let (text, _) = written("Australia/Sydney", &sydney(2025..=2027), 2026, 2026); + assert!(text.contains("RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU") && text.contains("RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=1SU"), "{text}"); + // No changes: one part. An offset is a zone too. + let (text, _) = written("Asia/Kolkata", &fixed(5 * H + 1800), 2026, 2026); + assert_eq!( + text.replace("\r\n", "\n"), + "BEGIN:VTIMEZONE\nTZID:Asia/Kolkata\nBEGIN:STANDARD\nDTSTART:16010101T000000\nTZOFFSETFROM:+0530\nTZOFFSETTO:+0530\nEND:STANDARD\nEND:VTIMEZONE\n" + ); + let (text, _) = written("UTC-09:30", &fixed(-570 * 60), 2026, 2026); + assert!(text.contains("TZID:UTC-09:30") && text.contains("TZOFFSETTO:-0930"), "{text}"); + // The year the United States changed its rule: written out change by change. + let history = us_eastern(2004..=2009, true); + let (text, read) = written("America/New_York", &history, 2006, 2007); + assert!(!text.contains("RRULE") && text.contains("DTSTART:20060402T020000") && text.contains("DTSTART:20070311T020000"), "{text}"); + assert_eq!(read.instant(at(2006, 3, 20, 9, 0)), utc(at(2006, 3, 20, 14, 0))); + assert_eq!(read.instant(at(2007, 3, 20, 9, 0)), utc(at(2007, 3, 20, 13, 0))); + // Hostile years are years, and a hostile name can't start a line. + for (from, to) in [(i32::MIN, i32::MAX), (9_999, 1), (0, 0)] { + let _ = written("Europe/Berlin", &berlin(2020..=2030), from, to); + } + let (text, read) = written("Evil\r\nEND:VTIMEZONE\r\nBEGIN:VEVENT", &fixed(0), 2026, 2026); + assert_eq!(read.tzid(), "EvilEND:VTIMEZONEBEGIN:VEVENT"); + assert_eq!(text.matches("VTIMEZONE").count(), 3, "{text}"); +} + +fn any_zone() -> impl Strategy { + (0usize..6, -12 * 60i32..=14 * 60) +} + +fn zone_of(which: usize, minutes: i32) -> Rules { + match which { + 0 => us_eastern(1960..=2100, false), + 1 => us_eastern(1960..=2100, true), + 2 => berlin(1960..=2100), + 3 => sydney(1960..=2100), + 4 => fixed(minutes * 60), + _ => fixed(0), + } +} + +proptest! { + #[test] + fn a_written_definition_reads_as_the_zone_it_was_written_for((which, minutes_east) in any_zone(), year in 1971i32..2090, span in 0i32..4, minutes in 0i64..365 * 24 * 60) { + let zone = zone_of(which, minutes_east); + let (text, read) = written("Example/Zone", &zone, year, year + span); + // Any time of those years is the instant the rules say. + let wall = at(year, 1, 1, 0, 0) + chrono::Duration::minutes(minutes) + chrono::Duration::days(i64::from(span) * 365); + let instant = utc(wall); + prop_assert_eq!(read.offset_at(instant), zone.offset_at(instant), "{} at {}\n{}", which, wall, text); + // And a wall-clock time is the instant it shows, where the clock shows it once. + let wall_instant = read.instant(wall); + if read.offset_at(wall_instant) == zone.offset_at(wall_instant) && zone.offset_at(wall_instant - 2 * i64::from(H)) == zone.offset_at(wall_instant + 2 * i64::from(H)) { + prop_assert_eq!(wall_instant + i64::from(zone.offset_at(wall_instant)), civil_seconds(wall)); + } + } + + #[test] + fn any_text_is_definitions_or_none(text in proptest::prelude::any::(), parts in proptest::collection::vec("(BEGIN|END|DTSTART|TZOFFSETFROM|TZOFFSETTO|RRULE|RDATE|TZID):[-+A-Z0-9=;,]{0,24}", 0..40), wall in proptest::num::i64::ANY) { + let _ = TimeZones::collect(&text); + let zones = TimeZones::collect(&format!("BEGIN:VTIMEZONE\nTZID:X\n{}\nEND:VTIMEZONE\n", parts.join("\n"))); + let wall = from_civil_seconds(wall); + if let Some(definition) = zones.get("X") { + let _ = (definition.offset_for_wall(wall), definition.offset_at(utc(wall)), definition.instant(wall)); + } + let _ = zones.get(&text); + } +} diff --git a/crates/idml/Cargo.toml b/crates/idml/Cargo.toml index 660456229..41df76d7e 100644 --- a/crates/idml/Cargo.toml +++ b/crates/idml/Cargo.toml @@ -13,8 +13,9 @@ workspace = true [dependencies] craft-xml = { workspace = true } thiserror = { workspace = true } -# `deflate-flate2` with flate2's default miniz_oxide backend, as craft-update pins it (zip's own -# `deflate` feature would switch a whole app's flate2 to zlib-rs). +craft-zip = { workspace = true } + +[dev-dependencies] +# The tests build hostile archives by hand. zip = { version = "9", default-features = false, features = ["deflate-flate2"] } -# The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +craft-zip = { workspace = true, features = ["test-support"] } diff --git a/crates/idml/README.md b/crates/idml/README.md index 0ef741bf2..16435a361 100644 --- a/crates/idml/README.md +++ b/crates/idml/README.md @@ -53,7 +53,8 @@ for line in &doc.warnings { /* a part designmap.xml lists and the package lacks | `xml.max_depth` | 256 | `Error::Part` | | `xml.max_nodes`, `xml.max_attrs_total` (one part) | 8,000,000 and 32,000,000 | `Error::Part` | -Sizes are checked against what a part really expands to, not what the archive declares. +The ZIP layer is craft-zip: the declared entry count is checked before the archive is indexed, and +sizes are checked against what a part really expands to, not what the archive declares. ## Writing diff --git a/crates/idml/src/package.rs b/crates/idml/src/package.rs index c8b6ebec2..dab9d3626 100644 --- a/crates/idml/src/package.rs +++ b/crates/idml/src/package.rs @@ -1,7 +1,6 @@ //! The ZIP container: bounded part reads, lenient part names, designmap includes, and the writer. use std::collections::HashMap; -use std::io::{Cursor, Read, Write}; use craft_xml::El; @@ -34,12 +33,10 @@ impl Default for Limits { /// An IDML package whose parts are decompressed only when asked for, within byte budgets that /// don't trust the sizes the archive claims. pub struct Package<'a> { - zip: zip::ZipArchive>, + zip: craft_zip::Archive<'a>, /// Normalised part name → entry name, first entry wins (zip order, so the choice is stable). lenient: HashMap, limits: Limits, - /// Bytes the parts still to be read may expand to. - left: u64, } /// `designmap.xml` with the parts it includes. @@ -63,46 +60,40 @@ impl<'a> Package<'a> { Self::open_with(bytes, Limits::DEFAULT) } + /// The declared entry count is checked against [`Limits::max_entries`] before the archive + /// is indexed. pub fn open_with(bytes: &'a [u8], limits: Limits) -> Result { - let zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| Error::NotIdml(e.to_string()))?; - if zip.len() > limits.max_entries { - return Err(Error::TooManyEntries { found: zip.len(), limit: limits.max_entries }); - } + let zip = + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: limits.max_total_bytes }) + .map_err(|e| match e { + craft_zip::Error::TooManyEntries { declared, limit } => Error::TooManyEntries { + found: usize::try_from(declared).unwrap_or(usize::MAX), + limit: usize::try_from(limit).unwrap_or(usize::MAX), + }, + e => Error::NotIdml(e.to_string()), + })?; let mut lenient: HashMap = HashMap::new(); - for i in 0..zip.len() { - let Some(Ok(name)) = zip.name_for_index(i) else { continue }; - if name.ends_with('/') { - continue; - } + for name in zip.names().filter(|n| !n.ends_with('/')) { lenient.entry(normalize_part_name(&name)).or_insert_with(|| name.into_owned()); } - Ok(Package { zip, lenient, limits, left: limits.max_total_bytes }) + Ok(Package { zip, lenient, limits }) } /// The bytes of part `name` (matched exactly, then leniently: writers differ in /// percent-encoding, case, path separators and a leading `./` or `/`), or `None` if the /// package has no such part. pub fn part(&mut self, name: &str) -> Result>> { - let exact = self.zip.index_for_name(name).filter(|_| !name.ends_with('/')); - let Some(i) = exact.or_else(|| self.lenient.get(&normalize_part_name(name)).and_then(|n| self.zip.index_for_name(n))) else { + let exact = (self.zip.contains(name) && !name.ends_with('/')).then_some(name); + let Some(stored) = exact.or_else(|| self.lenient.get(&normalize_part_name(name)).map(String::as_str)) else { return Ok(None); }; - let part = self.zip.name_for_index(i).and_then(|n| n.ok()).map_or_else(|| name.to_string(), |n| n.into_owned()); - let f = self.zip.by_index(i).map_err(|e| Error::NotIdml(e.to_string()))?; - let limit = self.limits.max_part_bytes.min(self.left); - if f.size() > limit { - return Err(Error::TooLarge { part, limit }); - } - let mut buf = Vec::new(); - if let Err(e) = f.take(limit.saturating_add(1)).read_to_end(&mut buf) { - return Err(Error::Part { part, msg: e.to_string() }); - } - let n = buf.len() as u64; - if n > limit { - return Err(Error::TooLarge { part, limit }); - } - self.left = self.left.saturating_sub(n); - Ok(Some(buf)) + let limit = self.limits.max_part_bytes.min(self.zip.budget().remaining()); + let stored = stored.to_string(); + self.zip.read(&stored, self.limits.max_part_bytes).map_err(|e| match e { + craft_zip::Error::EntryTooLarge { .. } | craft_zip::Error::BudgetSpent { .. } => Error::TooLarge { part: stored.clone(), limit }, + craft_zip::Error::Entry { msg, .. } => Error::Part { part: stored.clone(), msg }, + e => Error::NotIdml(e.to_string()), + }) } /// Part `name` as an element tree ([`crate::parse_part`]). @@ -114,7 +105,7 @@ impl<'a> Package<'a> { /// Bytes the parts still to be read may expand to. pub fn budget_left(&self) -> u64 { - self.left + self.zip.budget().remaining() } /// Reads `designmap.xml` and every part it includes. @@ -173,19 +164,14 @@ fn normalize_part_name(name: &str) -> String { } /// Zips an IDML package: the stored `mimetype` first, then `parts` in order as (name, bytes). +/// Equal parts give equal bytes. pub fn write_package, D: AsRef<[u8]>>(parts: &[(N, D)]) -> Result> { - use zip::write::SimpleFileOptions; - let fail = |e: &dyn std::fmt::Display| Error::Write(e.to_string()); - let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); - let stored = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - let deflated = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); - w.start_file("mimetype", stored).map_err(|e| fail(&e))?; - w.write_all(MIMETYPE.as_bytes()).map_err(|e| fail(&e))?; + use craft_zip::Compression; + let fail = |e: craft_zip::Error| Error::Write(e.to_string()); + let mut w = craft_zip::Writer::new(); + w.add("mimetype", MIMETYPE.as_bytes(), Compression::Stored).map_err(fail)?; for (name, data) in parts { - let data = data.as_ref(); - // More than 4 GiB needs the ZIP64 form. - w.start_file(name.as_ref(), deflated.large_file(data.len() >= u32::MAX as usize)).map_err(|e| fail(&e))?; - w.write_all(data).map_err(|e| fail(&e))?; + w.add(name.as_ref(), data.as_ref(), Compression::Deflated).map_err(fail)?; } - w.finish().map(Cursor::into_inner).map_err(|e| fail(&e)) + w.finish().map_err(fail) } diff --git a/crates/idml/tests/package.rs b/crates/idml/tests/package.rs index 1c7ffd353..03a42d088 100644 --- a/crates/idml/tests/package.rs +++ b/crates/idml/tests/package.rs @@ -244,3 +244,16 @@ fn parts_are_written_one_element_per_line_without_touching_text() { ); assert!(with_props(El::new("A"), Vec::new()).children.is_empty()); } + +/// A ZIP64 end record can declare millions of entries in a small file; the count is refused on +/// the raw bytes, before the archive is indexed. +#[test] +fn a_package_declaring_too_many_entries_is_refused_before_it_is_indexed() { + let bomb = craft_zip::zip64_entry_count_sample(100_001); + assert_eq!(Package::open(&bomb).err(), Some(Error::TooManyEntries { found: 100_001, limit: 100_000 })); +} + +#[test] +fn equal_parts_give_equal_bytes() { + assert_eq!(sample(), sample()); +} diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml new file mode 100644 index 000000000..5b597ba13 --- /dev/null +++ b/crates/jmap/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-jmap" +description = "JMAP (RFC 8620) protocol core without sockets: session, requests of method calls with result references, responses, ids, address templates and the judgement of HTTP answers, over any HTTP implementation." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/jmap/README.md b/crates/jmap/README.md new file mode 100644 index 000000000..5da2a8c7b --- /dev/null +++ b/crates/jmap/README.md @@ -0,0 +1,37 @@ +# craft-jmap + +The JMAP protocol core (RFC 8620) without sockets: a program brings an HTTP implementation +(`HttpExchange`), this crate describes each request and judges each answer. + +- **Session**: `Client`, `fetch_session` / `read_session`, `Session`, `Account`, `CoreLimits`, + the capabilities (`CAPABILITY_CORE`, `CAPABILITY_MAIL`). +- **Requests**: `ApiRequest` of method calls with `CallId`s and `ResultReference`s + (back-references), checked against the session's limits. +- **Responses**: `ApiResponse`, `MethodResponse`, `MethodError` (RFC 8620 §3.6.2), per call. +- **Addresses**: `Address` (absolute `http(s)` URLs, RFC 3986 and the WHATWG URL Standard), the + transport rule (`https`, or plain `http` to loopback only), `session_address` for what a person + types, `AddressTemplate` (RFC 6570 level 1, the download and upload templates). +- **HTTP**: `HttpRequest`, `HttpResponse`, `judge_response` (status codes, problem details, + JSON), `RetryAfter`, `Credentials` (Basic, Bearer), `Redactor` (excerpts of what a server says, + with the credentials and addresses taken out). +- `Id`: the JMAP id alphabet. + +Method calls' arguments and results are `serde_json::Value`s: what a program asks +(`Mailbox/get`, `Email/query`…) and what it makes of the answers stays in the program. + +## Hostile input + +Every size, count, nesting depth and string length a server sends is bounded; ids are held to +the JMAP alphabet; nothing indexes into what came over the wire, and the protocol modules are +checked with `clippy::indexing_slicing` and `clippy::arithmetic_side_effects`. Credentials and +the addresses of the session and the account never reach error text or `Debug` output. + +## Origin + +MailCraft `crates/jmap` (the protocol core: `address`, `classify`, `client`, `credentials`, +`discovery`, `error`, `host`, `http`, `id`, `percent`, `redact`, `request`, `response`, `retry`, +`session`, `template`) at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors (MIT OR Apache-2.0), with all of its tests. The mail +client that sits on it (the mailbox and email reads into MailCraft's sync model, the fetch of +whole messages, the in-process test server) stays in MailCraft. diff --git a/crates/jmap/src/address.rs b/crates/jmap/src/address.rs new file mode 100644 index 000000000..b9547b7f6 --- /dev/null +++ b/crates/jmap/src/address.rs @@ -0,0 +1,533 @@ +//! Absolute `http` and `https` addresses (RFC 3986, RFC 9110 §4.2), reference resolution (RFC 3986 §5.2), +//! origins, and the transport rule that decides whether an address may carry credentials. +//! +//! An [`Address`] is always absolute, always `http` or `https`, always has a host, never has user information +//! or a fragment, and is kept in a normal form (RFC 3986 §6.2.2 and §6.2.3, RFC 9110 §4.2.3): lower-case scheme +//! and host, no default port, percent-escapes in upper case with escaped unreserved characters decoded, no +//! dot-segments, and `/` for an empty path. +//! +//! Its `Debug` form shows the origin only, so that a path or a query holding something private never reaches +//! a log by accident. [`Address::as_str`] gives the full form, for the request itself. + +use std::fmt; + +use crate::host::Host; +use crate::percent::{self, is_sub_delim, is_unreserved}; + +/// The longest address accepted, in octets. +pub const MAX_ADDRESS_LEN: usize = 8 * 1024; + +/// An address scheme this crate can use. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Scheme { + /// `http` (RFC 9110 §4.2.1); only usable towards a loopback host. + Http, + /// `https` (RFC 9110 §4.2.2). + Https, +} + +impl Scheme { + /// The scheme's name, lower case. + pub fn as_str(self) -> &'static str { + match self { + Self::Http => "http", + Self::Https => "https", + } + } + + /// The TCP port used when an address names none (RFC 9110 §4.2.1 and §4.2.2). + pub fn default_port(self) -> u16 { + match self { + Self::Http => 80, + Self::Https => 443, + } + } +} + +/// Why a text is not a usable address. +/// +/// No variant carries the text that was refused: it may hold a password or a token. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum AddressError { + /// Nothing was given. + Empty, + /// Longer than [`MAX_ADDRESS_LEN`]. + TooLong, + /// A space, a control character, a character outside ASCII, or a character no URI may contain. + ForbiddenCharacter, + /// A backslash, which browsers read as a slash and RFC 3986 does not allow. + Backslash, + /// No scheme where an absolute address was needed. + MissingScheme, + /// A scheme other than `http` or `https`. + UnsupportedScheme, + /// No `//` and authority after the scheme. + MissingAuthority, + /// The host is empty (RFC 9110 §4.2.1). + EmptyHost, + /// User information (`name@` or `name:password@`) before the host (RFC 9110 §4.2.4). + UserInfo, + /// The host is not a usable name. + InvalidHost, + /// The host is outside ASCII; it must be given in its `xn--` form. + InternationalHost, + /// A numeric host that is not a plain dotted IPv4 address (`127.1`, `0x7f.0.0.1`, …). + AmbiguousNumericHost, + /// A bracketed IPv6 literal that does not parse. + InvalidIpv6, + /// The port is not a number from 1 to 65535. + InvalidPort, + /// The path holds a character or an escape that RFC 3986 §3.3 does not allow. + InvalidPath, + /// The query holds a character or an escape that RFC 3986 §3.4 does not allow. + InvalidQuery, + /// The fragment holds a character or an escape that RFC 3986 §3.5 does not allow. + InvalidFragment, + /// Plain `http` to a host that is not loopback. + InsecureTransport, + /// An address on this computer or its local network, given by a host farther away (see + /// [`HostScope`](crate::host::HostScope)). + LocalFromRemote, +} + +impl fmt::Display for AddressError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Empty => "no address was entered; type the JMAP server's address, for example example.com", + Self::TooLong => "the address is longer than 8 KiB (8192 characters), more than any server address needs; check that nothing else was pasted with it", + Self::ForbiddenCharacter => "the address contains a space, a control character, a non-ASCII character or one of \" < > ^ ` { | }; remove it or percent-encode it", + Self::Backslash => "the address contains a backslash (\\); addresses use forward slashes (/)", + Self::MissingScheme => "the address does not say how to connect; start it with https://", + Self::UnsupportedScheme => "the address uses a scheme other than https; enter an https:// address (http is only for this computer)", + Self::MissingAuthority => "the scheme is not followed by // and a server name; write it as https://server.example/path", + Self::EmptyHost => "the address has no server name after https://; add the server's name", + Self::UserInfo => "the address contains a user name followed by @ before the server name; remove it and enter the user name in the sign-in fields", + Self::InvalidHost => "the server name contains characters a DNS name cannot hold, or an empty or overlong label; check its spelling", + Self::InternationalHost => "the server name has non-ASCII letters; enter its ASCII form, which starts with xn--", + Self::AmbiguousNumericHost => "the server name looks numeric but is not a plain IPv4 address; write four numbers such as 192.0.2.1, without leading zeros, or use the server's name", + Self::InvalidIpv6 => "the IPv6 address in brackets cannot be read; zone suffixes such as %eth0 and IPvFuture forms are not supported", + Self::InvalidPort => "the port after the server name is not a number from 1 to 65535", + Self::InvalidPath => "the path contains a character or a % sign that addresses do not allow there; percent-encode it", + Self::InvalidQuery => "the part after ? contains a character or a % sign that addresses do not allow there; percent-encode it", + Self::InvalidFragment => "the part after # contains a character or a % sign that addresses do not allow there", + Self::InsecureTransport => "this address would send the password unencrypted over plain http to another computer; use https:// (plain http is only accepted for localhost, 127.0.0.0/8 and [::1])", + Self::LocalFromRemote => "the server tried to send this program to this computer or its local network, which a remote server may not do; its configuration may be wrong or tampered with", + }) + } +} + +impl std::error::Error for AddressError {} + +/// An absolute `http` or `https` address in normal form. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Address { + text: String, + scheme: Scheme, + host: Host, + port: Option, + path_start: usize, + query_start: Option, +} + +/// The origin of an address: scheme, host and port (the tuple origin of the WHATWG URL Standard §4.7 for +/// `http` and `https`). Its `Display` form is `scheme://host` with `:port` only when it is not the default. +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct Origin { + scheme: Scheme, + host: Host, + port: Option, +} + +impl Origin { + /// The scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// The host. + pub fn host(&self) -> &Host { + &self.host + } + + /// The port, the scheme's default when none was given. + pub fn port(&self) -> u16 { + self.port.unwrap_or_else(|| self.scheme.default_port()) + } +} + +impl fmt::Display for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}://{}", self.scheme.as_str(), self.host)?; + if let Some(port) = self.port { + write!(f, ":{port}")?; + } + Ok(()) + } +} + +impl fmt::Debug for Origin { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(self, f) + } +} + +/// The parts of a URI reference (RFC 3986 Appendix B), the fragment already set aside. +struct Parts<'a> { + scheme: Option<&'a str>, + authority: Option<&'a str>, + path: &'a str, + query: Option<&'a str>, +} + +fn refuse_unusable_characters(input: &str) -> Result<(), AddressError> { + if input.is_empty() { + return Err(AddressError::Empty); + } + if input.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + for b in input.bytes() { + if b == b'\\' { + return Err(AddressError::Backslash); + } + if !(0x21..=0x7E).contains(&b) || matches!(b, b'"' | b'<' | b'>' | b'^' | b'`' | b'{' | b'|' | b'}') { + return Err(AddressError::ForbiddenCharacter); + } + } + Ok(()) +} + +fn looks_like_scheme(text: &str) -> bool { + let mut bytes = text.bytes(); + matches!(bytes.next(), Some(b) if b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) +} + +/// Splits a reference into its components (RFC 3986 Appendix B); the fragment is dropped, as RFC 9110 §7.1 +/// leaves it out of every target. +fn reference_parts(reference: &str) -> Result, AddressError> { + let (without_fragment, fragment) = match reference.split_once('#') { + Some((head, fragment)) => (head, Some(fragment)), + None => (reference, None), + }; + if let Some(fragment) = fragment + && !chars_allowed(fragment, true) + { + return Err(AddressError::InvalidFragment); + } + let (scheme, rest) = match without_fragment.find([':', '/', '?']) { + Some(index) if without_fragment.get(index..).is_some_and(|tail| tail.starts_with(':')) && index > 0 => { + let (scheme, rest) = without_fragment.split_at_checked(index).ok_or(AddressError::MissingScheme)?; + (Some(scheme), rest.get(1..).unwrap_or("")) + } + _ => (None, without_fragment), + }; + let (authority, rest) = match rest.strip_prefix("//") { + Some(after) => { + let end = after.find(['/', '?']).unwrap_or(after.len()); + let (authority, rest) = after.split_at_checked(end).ok_or(AddressError::MissingAuthority)?; + (Some(authority), rest) + } + None => (None, rest), + }; + let (path, query) = match rest.split_once('?') { + Some((path, query)) => (path, Some(query)), + None => (rest, None), + }; + Ok(Parts { scheme, authority, path, query }) +} + +/// Whether every character of a path (`allow_question` false) or a query/fragment (`allow_question` true) is +/// allowed by RFC 3986 §3.3–§3.5, escapes included. +fn chars_allowed(text: &str, allow_question: bool) -> bool { + let mut bytes = text.bytes(); + while let Some(b) = bytes.next() { + let ok = match b { + b'%' => bytes.next().and_then(percent::hex_value).is_some() && bytes.next().and_then(percent::hex_value).is_some(), + b':' | b'@' | b'/' => true, + b'?' => allow_question, + _ => is_unreserved(b) || is_sub_delim(b), + }; + if !ok { + return false; + } + } + true +} + +fn port_number(text: &str, scheme: Scheme) -> Result, AddressError> { + if text.is_empty() { + return Ok(None); + } + let mut value: u32 = 0; + for b in text.bytes() { + let digit = match b { + b'0'..=b'9' => u32::from(b.saturating_sub(b'0')), + _ => return Err(AddressError::InvalidPort), + }; + value = value.checked_mul(10).and_then(|v| v.checked_add(digit)).filter(|v| *v <= u32::from(u16::MAX)).ok_or(AddressError::InvalidPort)?; + } + let port = u16::try_from(value).map_err(|_| AddressError::InvalidPort)?; + if port == 0 { + return Err(AddressError::InvalidPort); + } + Ok(if port == scheme.default_port() { None } else { Some(port) }) +} + +/// Separates an authority's host text (brackets kept for IPv6) from its port text, empty when absent. +fn separate_port(authority: &str) -> Result<(&str, &str), AddressError> { + if authority.starts_with('[') { + let close = authority.find(']').ok_or(AddressError::InvalidIpv6)?; + let end = close.checked_add(1).ok_or(AddressError::InvalidIpv6)?; + let (host, rest) = authority.split_at_checked(end).ok_or(AddressError::InvalidIpv6)?; + if rest.is_empty() { + return Ok((host, "")); + } + let port = rest.strip_prefix(':').ok_or(AddressError::InvalidHost)?; + return Ok((host, port)); + } + if authority.contains(['[', ']']) { + return Err(AddressError::InvalidHost); + } + Ok(authority.split_once(':').unwrap_or((authority, ""))) +} + +/// Removes the `.` and `..` segments of a path (RFC 3986 §5.2.4). +pub(crate) fn collapse_dot_segments(path: &str) -> String { + let mut input = path; + let mut output = String::with_capacity(path.len()); + while !input.is_empty() { + if let Some(rest) = input.strip_prefix("../") { + input = rest; + } else if let Some(rest) = input.strip_prefix("./") { + input = rest; + } else if input.starts_with("/./") { + input = input.get(2..).unwrap_or(""); + } else if input == "/." { + input = "/"; + } else if input.starts_with("/../") { + input = input.get(3..).unwrap_or(""); + drop_last_segment(&mut output); + } else if input == "/.." { + input = "/"; + drop_last_segment(&mut output); + } else if input == "." || input == ".." { + input = ""; + } else { + let end = input.get(1..).and_then(|after_first| after_first.find('/')).and_then(|index| index.checked_add(1)).unwrap_or(input.len()); + let (segment, rest) = input.split_at_checked(end).unwrap_or((input, "")); + output.push_str(segment); + input = rest; + } + } + output +} + +fn drop_last_segment(output: &mut String) { + match output.rfind('/') { + Some(index) => output.truncate(index), + None => output.clear(), + } +} + +/// Merges a relative path with the path of a base that has an authority (RFC 3986 §5.2.3). +fn base_directory_plus(base_path: &str, reference_path: &str) -> String { + let directory = match base_path.rfind('/') { + Some(index) => base_path.get(..=index).unwrap_or("/"), + None => "/", + }; + let mut merged = String::with_capacity(directory.len().saturating_add(reference_path.len())); + merged.push_str(directory); + merged.push_str(reference_path); + merged +} + +impl Address { + /// Parses an absolute `http` or `https` address. + /// + /// This checks the syntax only; [`Address::parse_secure`] also applies the transport rule. A fragment is + /// accepted and dropped (RFC 9110 §7.1). + pub fn parse(input: &str) -> Result { + refuse_unusable_characters(input)?; + let parts = reference_parts(input)?; + let scheme_text = parts.scheme.ok_or(AddressError::MissingScheme)?; + if !looks_like_scheme(scheme_text) { + return Err(AddressError::MissingScheme); + } + let scheme = match scheme_text.to_ascii_lowercase().as_str() { + "http" => Scheme::Http, + "https" => Scheme::Https, + _ => return Err(AddressError::UnsupportedScheme), + }; + let authority = parts.authority.ok_or(AddressError::MissingAuthority)?; + if authority.contains('@') { + return Err(AddressError::UserInfo); + } + let (host_text, port_text) = separate_port(authority)?; + let host = Host::parse(host_text)?; + let port = port_number(port_text, scheme)?; + if !chars_allowed(parts.path, false) { + return Err(AddressError::InvalidPath); + } + let path = percent::normalise_escapes(parts.path).ok_or(AddressError::InvalidPath)?; + let mut path = collapse_dot_segments(&path); + if path.is_empty() { + path.push('/'); + } + let query = match parts.query { + Some(query) => { + if !chars_allowed(query, true) { + return Err(AddressError::InvalidQuery); + } + Some(percent::normalise_escapes(query).ok_or(AddressError::InvalidQuery)?) + } + None => None, + }; + Self::assemble(scheme, host, port, &path, query.as_deref()) + } + + /// Parses an absolute address and applies the transport rule ([`Address::require_secure_transport`]). + pub fn parse_secure(input: &str) -> Result { + let address = Self::parse(input)?; + address.require_secure_transport()?; + Ok(address) + } + + fn assemble(scheme: Scheme, host: Host, port: Option, path: &str, query: Option<&str>) -> Result { + let origin = Origin { scheme, host, port }; + let mut text = origin.to_string(); + let path_start = text.len(); + text.push_str(path); + let query_start = match query { + Some(query) => { + let start = text.len(); + text.push('?'); + text.push_str(query); + Some(start) + } + None => None, + }; + if text.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + let Origin { scheme, host, port } = origin; + Ok(Self { text, scheme, host, port, path_start, query_start }) + } + + /// Resolves a reference, relative or absolute, against this address (RFC 3986 §5.2.2, strict). + /// + /// The result is checked like [`Address::parse`]; the transport rule is not applied. + pub fn resolve(&self, reference: &str) -> Result { + if reference.is_empty() { + return Ok(self.clone()); + } + refuse_unusable_characters(reference)?; + let parts = reference_parts(reference)?; + if parts.scheme.is_some() { + return Self::parse(reference); + } + let without_fragment = reference.split_once('#').map_or(reference, |(head, _)| head); + if parts.authority.is_some() { + return Self::parse(&format!("{}:{}", self.scheme.as_str(), without_fragment)); + } + let (path, query) = if parts.path.is_empty() { + (self.path().to_owned(), parts.query.or(self.query())) + } else if parts.path.starts_with('/') { + (parts.path.to_owned(), parts.query) + } else { + (base_directory_plus(self.path(), parts.path), parts.query) + }; + // Dot-segment removal (§5.2.2, §5.2.4) runs in `parse`, after escapes are normalised (§6.2.2.2), so that + // `%2e` segments are treated the same way whether the reference was relative or absolute. + let mut target = self.origin().to_string(); + target.push_str(&path); + if let Some(query) = query { + target.push('?'); + target.push_str(query); + } + Self::parse(&target) + } + + /// The full address, for the request line. Do not log it: use [`Address::origin`] for diagnostics. + pub fn as_str(&self) -> &str { + &self.text + } + + /// The scheme. + pub fn scheme(&self) -> Scheme { + self.scheme + } + + /// The host. + pub fn host(&self) -> &Host { + &self.host + } + + /// The port, the scheme's default when the address names none. + pub fn port(&self) -> u16 { + self.port.unwrap_or_else(|| self.scheme.default_port()) + } + + /// The path, starting with `/`. + pub fn path(&self) -> &str { + let end = self.query_start.unwrap_or(self.text.len()); + self.text.get(self.path_start..end).unwrap_or("/") + } + + /// The query without its `?`, if there is one. + pub fn query(&self) -> Option<&str> { + let start = self.query_start?.checked_add(1)?; + self.text.get(start..) + } + + /// The origin: scheme, host and port. + pub fn origin(&self) -> Origin { + Origin { scheme: self.scheme, host: self.host.clone(), port: self.port } + } + + /// Whether both addresses have the same origin. + pub fn same_origin(&self, other: &Self) -> bool { + self.scheme == other.scheme && self.host == other.host && self.port == other.port + } + + /// Whether this address may carry credentials: `https`, or `http` to a loopback host + /// (see [`Host::is_loopback`]). + pub fn uses_secure_transport(&self) -> bool { + self.scheme == Scheme::Https || self.host.is_loopback() + } + + /// The transport rule as a `Result`. + pub fn require_secure_transport(&self) -> Result<(), AddressError> { + if self.uses_secure_transport() { Ok(()) } else { Err(AddressError::InsecureTransport) } + } + + /// Checks an address that `self` handed out (a session's endpoint, a template's expansion, a redirect's + /// target): it must satisfy the transport rule, and it may not be closer to this computer than `self` + /// ([`Host::scope`]), so that a remote server cannot point the program, credentials and all, at a service + /// on this computer or its local network. Only the address text is judged: a name that resolves to such an + /// address can only be caught by the program performing the request. + pub fn check_link(&self, target: &Self) -> Result<(), AddressError> { + target.require_secure_transport()?; + if target.host.scope() > self.host.scope() { + return Err(AddressError::LocalFromRemote); + } + Ok(()) + } + + /// The same origin with another path (and no query). + pub(crate) fn with_path(&self, path: &str) -> Result { + let mut text = self.origin().to_string(); + text.push_str(path); + Self::parse(&text) + } +} + +impl fmt::Debug for Address { + /// Shows the origin only (requirement 1.8): a path or query may hold something private. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Address({})", self.origin()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/address/tests.rs b/crates/jmap/src/address/tests.rs new file mode 100644 index 000000000..392f64641 --- /dev/null +++ b/crates/jmap/src/address/tests.rs @@ -0,0 +1,243 @@ +use super::*; + +fn parsed(input: &str) -> String { + Address::parse(input).unwrap().as_str().to_owned() +} + +// RFC 3986 §3 and RFC 9110 §4.2.1–§4.2.2: the components of http and https addresses. +#[test] +fn parses_components() { + let address = Address::parse("https://jmap.example.com:8443/api/v1?x=1&y=2").unwrap(); + assert_eq!(address.scheme(), Scheme::Https); + assert_eq!(address.host(), &Host::Name("jmap.example.com".into())); + assert_eq!(address.port(), 8443); + assert_eq!(address.path(), "/api/v1"); + assert_eq!(address.query(), Some("x=1&y=2")); + assert_eq!(address.as_str(), "https://jmap.example.com:8443/api/v1?x=1&y=2"); +} + +// RFC 3986 §6.2.2 and §6.2.3, RFC 9110 §4.2.3: the three equivalent forms normalise to one. +#[test] +fn normal_form() { + assert_eq!(parsed("http://example.com:80/~smith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("http://EXAMPLE.com/%7Esmith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("http://EXAMPLE.com:/%7esmith/home.html"), "http://example.com/~smith/home.html"); + assert_eq!(parsed("HTTPS://Example.ORG"), "https://example.org/"); + assert_eq!(parsed("https://example.org:443/a/./b/../c"), "https://example.org/a/c"); + assert_eq!(parsed("https://example.org/a%2fb?q=%2f%41"), "https://example.org/a%2Fb?q=%2FA"); + assert_eq!(parsed("https://example.org/a/%2e%2E/b"), "https://example.org/b"); +} + +// RFC 9110 §7.1: the fragment is not part of the target. +#[test] +fn drops_fragment() { + assert_eq!(parsed("https://example.com/a?b#c"), "https://example.com/a?b"); + assert_eq!(Address::parse("https://example.com/#a b"), Err(AddressError::ForbiddenCharacter)); + assert_eq!(Address::parse("https://example.com/#a#b"), Err(AddressError::InvalidFragment)); +} + +// Requirement 1.1: what is not a usable absolute http/https address is refused. +#[test] +fn refuses_unusable_addresses() { + let cases: &[(&str, AddressError)] = &[ + ("", AddressError::Empty), + ("https://exa mple.com/", AddressError::ForbiddenCharacter), + ("https://example.com/\u{7f}", AddressError::ForbiddenCharacter), + ("https://example.com/\n", AddressError::ForbiddenCharacter), + ("https://example.com/é", AddressError::ForbiddenCharacter), + ("https://example.com/{x}", AddressError::ForbiddenCharacter), + ("https://example.com\\@example.org/", AddressError::Backslash), + ("https:\\\\example.com/", AddressError::Backslash), + ("//example.com/", AddressError::MissingScheme), + ("example.com/path", AddressError::MissingScheme), + ("ftp://example.com/", AddressError::UnsupportedScheme), + ("javascript:alert(1)", AddressError::UnsupportedScheme), + ("https:example.com", AddressError::MissingAuthority), + ("https:///path", AddressError::EmptyHost), + ("https://:443/", AddressError::EmptyHost), + ("https://user@example.com/", AddressError::UserInfo), + ("https://user:secret@example.com/", AddressError::UserInfo), + ("https://[::1/", AddressError::InvalidIpv6), + ("https://[::g]/", AddressError::InvalidIpv6), + ("https://[::1]x/", AddressError::InvalidHost), + ("https://example.com:65536/", AddressError::InvalidPort), + ("https://example.com:0/", AddressError::InvalidPort), + ("https://example.com:-1/", AddressError::InvalidPort), + ("https://example.com:12a/", AddressError::InvalidPort), + ("https://example.com:1:2/", AddressError::InvalidPort), + ("https://example.com/%zz", AddressError::InvalidPath), + ("https://example.com/[x]", AddressError::InvalidPath), + ("https://example.com/?%", AddressError::InvalidQuery), + ("https://127.1/", AddressError::AmbiguousNumericHost), + ]; + for (input, expected) in cases { + assert_eq!(Address::parse(input).as_ref().map(Address::as_str), Err(expected), "{input:?}"); + } + let long = format!("https://example.com/{}", "a".repeat(MAX_ADDRESS_LEN)); + assert_eq!(Address::parse(&long), Err(AddressError::TooLong)); +} + +// RFC 3986 §3.2.3: an empty port means the default one; leading zeros are still the same number. +#[test] +fn ports() { + assert_eq!(parsed("https://example.com:/"), "https://example.com/"); + assert_eq!(parsed("https://example.com:00443/"), "https://example.com/"); + assert_eq!(parsed("http://localhost:8080"), "http://localhost:8080/"); + assert_eq!(Address::parse("https://example.com:65535/").unwrap().port(), 65535); + assert_eq!(parsed("https://[::1]:8443/x"), "https://[::1]:8443/x"); +} + +// Requirement 1.2: https always; http only to localhost, 127.0.0.0/8 or [::1]. +#[test] +fn transport_rule() { + for ok in ["https://example.com/", "http://localhost/", "http://LocalHost:8080/", "http://127.0.0.1/", "http://127.9.8.7:1/", "http://[::1]/"] { + assert!(Address::parse_secure(ok).is_ok(), "{ok}"); + } + for refused in [ + "http://example.com/", + "http://sub.localhost/", + "http://localhost./", + "http://192.0.2.1/", + "http://[2001:db8::2]/", + "http://[::ffff:127.0.0.1]/", + ] { + assert_eq!(Address::parse_secure(refused), Err(AddressError::InsecureTransport), "{refused}"); + } +} + +// Requirement 1.2 hardened, RFC 9110 §4.2.1 and RFC 8620 §8.3: a remote host cannot hand out a loopback address. +#[test] +fn links_from_remote_to_loopback_are_refused() { + let remote = Address::parse("https://jmap.example.com/").unwrap(); + let local = Address::parse("http://127.0.0.1:8080/").unwrap(); + let other = Address::parse("https://api.example.net/").unwrap(); + let plain = Address::parse("http://example.org/").unwrap(); + assert_eq!(remote.check_link(&local), Err(AddressError::LocalFromRemote)); + for sneaky in [ + "https://localhost./", + "https://x.localhost/", + "https://0.0.0.0/", + "https://[::]/", + "https://[::ffff:7f00:1]/", + "https://192.168.1.10/", + "https://[fe80::1]/", + ] { + let target = Address::parse(sneaky).unwrap(); + assert_eq!(remote.check_link(&target), Err(AddressError::LocalFromRemote), "{sneaky}"); + } + let lan = Address::parse("https://192.168.1.10/jmap").unwrap(); + assert_eq!(lan.check_link(&Address::parse("https://192.168.1.11/api").unwrap()), Ok(())); + assert_eq!(lan.check_link(&Address::parse("https://localhost/api").unwrap()), Err(AddressError::LocalFromRemote)); + assert_eq!(remote.check_link(&plain), Err(AddressError::InsecureTransport)); + assert_eq!(remote.check_link(&other), Ok(())); + assert_eq!(local.check_link(&local), Ok(())); + assert_eq!(local.check_link(&remote), Ok(())); +} + +// RFC 3986 §5.4.1: the normal examples, hosts moved under example domains (the non-http "g:h" is +// refused, as this crate only knows http and https). +#[test] +fn resolves_normal_examples() { + let base = Address::parse("http://a.example.com/b/c/d;p?q").unwrap(); + let cases = [ + ("g", "http://a.example.com/b/c/g"), + ("./g", "http://a.example.com/b/c/g"), + ("g/", "http://a.example.com/b/c/g/"), + ("/g", "http://a.example.com/g"), + ("//g.example.org", "http://g.example.org/"), + ("?y", "http://a.example.com/b/c/d;p?y"), + ("g?y", "http://a.example.com/b/c/g?y"), + ("#s", "http://a.example.com/b/c/d;p?q"), + ("g#s", "http://a.example.com/b/c/g"), + ("g?y#s", "http://a.example.com/b/c/g?y"), + (";x", "http://a.example.com/b/c/;x"), + ("g;x", "http://a.example.com/b/c/g;x"), + ("g;x?y#s", "http://a.example.com/b/c/g;x?y"), + ("", "http://a.example.com/b/c/d;p?q"), + (".", "http://a.example.com/b/c/"), + ("./", "http://a.example.com/b/c/"), + ("..", "http://a.example.com/b/"), + ("../", "http://a.example.com/b/"), + ("../g", "http://a.example.com/b/g"), + ("../..", "http://a.example.com/"), + ("../../", "http://a.example.com/"), + ("../../g", "http://a.example.com/g"), + ]; + for (reference, expected) in cases { + assert_eq!(base.resolve(reference).unwrap().as_str(), expected, "{reference:?}"); + } + assert_eq!(base.resolve("g:h"), Err(AddressError::UnsupportedScheme)); +} + +// RFC 3986 §5.4.2: the abnormal examples (hosts under example domains), strict for "http:g". +#[test] +fn resolves_abnormal_examples() { + let base = Address::parse("http://a.example.com/b/c/d;p?q").unwrap(); + let cases = [ + ("../../../g", "http://a.example.com/g"), + ("../../../../g", "http://a.example.com/g"), + ("/./g", "http://a.example.com/g"), + ("/../g", "http://a.example.com/g"), + ("g.", "http://a.example.com/b/c/g."), + (".g", "http://a.example.com/b/c/.g"), + ("g..", "http://a.example.com/b/c/g.."), + ("..g", "http://a.example.com/b/c/..g"), + ("./../g", "http://a.example.com/b/g"), + ("./g/.", "http://a.example.com/b/c/g/"), + ("g/./h", "http://a.example.com/b/c/g/h"), + ("g/../h", "http://a.example.com/b/c/h"), + ("g;x=1/./y", "http://a.example.com/b/c/g;x=1/y"), + ("g;x=1/../y", "http://a.example.com/b/c/y"), + ("g?y/./x", "http://a.example.com/b/c/g?y/./x"), + ("g?y/../x", "http://a.example.com/b/c/g?y/../x"), + ("g#s/./x", "http://a.example.com/b/c/g"), + ("g#s/../x", "http://a.example.com/b/c/g"), + ]; + for (reference, expected) in cases { + assert_eq!(base.resolve(reference).unwrap().as_str(), expected, "{reference:?}"); + } + assert_eq!(base.resolve("http:g"), Err(AddressError::MissingAuthority)); + // RFC 3986 §6.2.2.2: an escaped dot is a dot, whether the reference is relative or absolute. + assert_eq!(base.resolve("/a/%2e%2e/../b"), Address::parse("http://a.example.com/a/%2e%2e/../b")); + assert_eq!(base.resolve("/a/%2e%2e/../b").unwrap().path(), "/b"); +} + +// RFC 3986 §5.2.4: the two worked examples of dot-segment removal. +#[test] +fn removes_dot_segments() { + assert_eq!(collapse_dot_segments("/a/b/c/./../../g"), "/a/g"); + assert_eq!(collapse_dot_segments("mid/content=5/../6"), "mid/6"); + assert_eq!(collapse_dot_segments("/.."), "/"); + assert_eq!(collapse_dot_segments("//x/../y"), "//y"); +} + +// Requirement 1.4: the origin elides the default port; the full form keeps everything else. +#[test] +fn origin_and_full_form() { + let address = Address::parse("https://example.com:443/session?token=abc").unwrap(); + assert_eq!(address.origin().to_string(), "https://example.com"); + assert_eq!(address.origin().port(), 443); + let address = Address::parse("http://[::1]:8080/x").unwrap(); + assert_eq!(address.origin().to_string(), "http://[::1]:8080"); + assert!(address.same_origin(&Address::parse("http://[0::1]:8080/y").unwrap())); + assert!(!address.same_origin(&Address::parse("http://[::1]:8081/x").unwrap())); +} + +// Requirement 1.8: diagnostics never show more than the origin. +#[test] +fn debug_shows_origin_only() { + let address = Address::parse("https://example.com/private/path?token=s3cr3t").unwrap(); + let shown = format!("{address:?}"); + assert_eq!(shown, "Address(https://example.com)"); + assert!(!shown.contains("s3cr3t") && !shown.contains("private")); +} + +// RFC 3986 §6.2.2: normalisation is idempotent, so parsing the full form gives the same address. +#[test] +fn reparse_is_stable() { + for input in ["https://EXAMPLE.com:443/a/../b/%7e?Q=%2f", "http://[0:0::1]:80//x/./y", "https://example.org"] { + let first = Address::parse(input).unwrap(); + let second = Address::parse(first.as_str()).unwrap(); + assert_eq!(first, second); + } +} diff --git a/crates/jmap/src/classify.rs b/crates/jmap/src/classify.rs new file mode 100644 index 000000000..ad674e59e --- /dev/null +++ b/crates/jmap/src/classify.rs @@ -0,0 +1,199 @@ +//! Judging a response: status codes (RFC 9110 §15), problem details (RFC 9457, which obsoletes RFC 7807 and +//! which JMAP's request-level errors use, RFC 8620 §3.6.1), content types (RFC 9110 §8.3) and JSON (RFC 8259). + +use serde_json::{Map, Value}; + +use crate::error::{Error, LimitName, RequestProblem, Unusable}; +use crate::http::{HttpRequest, HttpResponse, PROBLEM_BODY_LIMIT}; +use crate::redact::{Excerpt, Redactor}; +use crate::retry::RetryAfter; + +/// The URN prefix of JMAP's request-level problem types (RFC 8620 §3.6.1, §9.3). +const JMAP_PROBLEM_PREFIX: &str = "urn:ietf:params:jmap:error:"; +/// The longest JMAP problem name kept. +const MAX_PROBLEM_NAME_LEN: usize = 64; + +/// Whether a `Content-Type` value names `expected` (RFC 9110 §8.3.1: type and subtype are case-insensitive; +/// parameters such as `charset` are ignored, JSON being UTF-8 by RFC 8259 §8.1). +pub(crate) fn media_type_is(content_type: Option<&str>, expected: &str) -> bool { + content_type.is_some_and(|value| { + let essence = value.split(';').next().unwrap_or("").trim_matches([' ', '\t']); + essence.eq_ignore_ascii_case(expected) + }) +} + +/// The most JSON values a body may hold before it is parsed into memory. +pub const MAX_JSON_VALUES: usize = 1_000_000; +/// The most JSON objects a body may hold before it is parsed into memory. +pub const MAX_JSON_OBJECTS: usize = 100_000; + +/// Counts, without building anything, an upper bound of the values and the objects of a JSON text, so that a +/// small text cannot expand into a huge tree (RFC 8259 §9 lets a parser limit what it accepts; RFC 8620 §8.4). +/// Every value but the first follows `[`, `,` or `:`; strings are skipped with their escapes. +fn within_budget(body: &[u8]) -> bool { + let mut in_string = false; + let mut escaped = false; + let mut values = 1usize; + let mut objects = 0usize; + for &b in body { + if in_string { + if escaped { + escaped = false; + } else if b == b'\\' { + escaped = true; + } else if b == b'"' { + in_string = false; + } + continue; + } + match b { + b'"' => in_string = true, + b'{' => objects = objects.saturating_add(1), + b'[' | b',' | b':' => values = values.saturating_add(1), + _ => {} + } + if values > MAX_JSON_VALUES || objects > MAX_JSON_OBJECTS { + return false; + } + } + true +} + +/// Parses a JSON text (RFC 8259), skipping a leading byte order mark (RFC 8259 §8.1 allows ignoring one). +/// Nesting is bounded by `serde_json`'s recursion limit (128); the number of values and objects by +/// [`MAX_JSON_VALUES`] and [`MAX_JSON_OBJECTS`]. +pub(crate) fn parse_json(body: &[u8]) -> Result { + let body = body.strip_prefix(b"\xEF\xBB\xBF").unwrap_or(body); + if !within_budget(body) { + return Err(Unusable::TooComplex); + } + serde_json::from_slice(body).map_err(|_| Unusable::InvalidJson) +} + +/// What was understood of a problem details object. +#[derive(Debug, Default)] +struct Problem { + jmap: Option, + status: Option, + limit: Option, + text: Option, + names_bad_token: bool, +} + +fn jmap_problem(kind: &str) -> Option { + let prefix = kind.get(..JMAP_PROBLEM_PREFIX.len())?; + if !prefix.eq_ignore_ascii_case(JMAP_PROBLEM_PREFIX) { + return None; + } + let name = kind.get(JMAP_PROBLEM_PREFIX.len()..)?; + Some(match name { + "unknownCapability" => RequestProblem::UnknownCapability, + "notJSON" => RequestProblem::NotJson, + "notRequest" => RequestProblem::NotRequest, + "limit" => RequestProblem::Limit, + other if !other.is_empty() && other.len() <= MAX_PROBLEM_NAME_LEN && other.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') => { + RequestProblem::Other(other.to_owned()) + } + _ => return None, + }) +} + +/// Reads a problem details object (RFC 9457 §3.1). Members of the wrong type are ignored, as §3.1 requires. +fn read_problem(response: &HttpResponse, limit: usize) -> Option { + let body = response.body(); + if body.is_empty() || body.len() > PROBLEM_BODY_LIMIT.min(limit) { + return None; + } + let content_type = response.content_type(); + if content_type.is_some() && !media_type_is(content_type, "application/problem+json") && !media_type_is(content_type, "application/json") { + return None; + } + let value = parse_json(body).ok()?; + let object: &Map = value.as_object()?; + let mut problem = Problem::default(); + if let Some(kind) = object.get("type").and_then(Value::as_str) { + problem.jmap = jmap_problem(kind); + let last = kind.rsplit([':', '/', '#']).next().unwrap_or(""); + problem.names_bad_token = last.eq_ignore_ascii_case("invalid_token") || last.eq_ignore_ascii_case("insufficient_scope"); + } + problem.status = object.get("status").and_then(Value::as_u64).and_then(|status| u16::try_from(status).ok()); + problem.limit = object.get("limit").and_then(Value::as_str).and_then(LimitName::new); + problem.text = object.get("detail").and_then(Value::as_str).or_else(|| object.get("title").and_then(Value::as_str)).map(str::to_owned); + Some(problem) +} + +fn excerpt(problem: Option<&Problem>, redactor: &Redactor) -> Option { + let text = problem?.text.as_deref()?; + let excerpt = redactor.excerpt(text); + if excerpt.as_str().is_empty() { None } else { Some(excerpt) } +} + +/// Judges a response to `request`. +/// +/// `Ok(())` means the body may be read as the request's kind expects (for session and API requests: a 2xx +/// status, `application/json`, and a body within the size limit). Everything else becomes the [`Error`] +/// variant the mail client can act on. Server text appears only as an [`Excerpt`] cleaned by `redactor`. +pub fn judge_response(request: &HttpRequest, response: &HttpResponse, redactor: &Redactor) -> Result<(), Error> { + let status = response.status(); + if (200..=299).contains(&status) { + if response.body().len() > request.max_response_bytes() { + return Err(Error::Unusable(Unusable::BodyTooLarge)); + } + if request.kind().expects_json() && !media_type_is(response.content_type(), "application/json") { + return Err(Error::Unusable(Unusable::WrongContentType)); + } + return Ok(()); + } + if (300..=399).contains(&status) { + return Err(judge_redirect(request, response)); + } + let problem = read_problem(response, request.max_response_bytes()); + let detail = excerpt(problem.as_ref(), redactor); + let retry_after = || response.retry_after().and_then(RetryAfter::parse); + // A server-chosen identifier is shown in messages: drop it when it echoes a secret. + let jmap = problem.as_ref().and_then(|problem| problem.jmap.clone()).map(|jmap| match jmap { + RequestProblem::Other(name) if redactor.mentions_secret(&name) => RequestProblem::Other(String::new()), + other => other, + }); + let limit = problem.as_ref().and_then(|problem| problem.limit.clone()).filter(|limit| !redactor.mentions_secret(limit.as_str())); + // RFC 9457 §3.1.2: the status member tells what the origin server meant when an intermediary changed it. + let says_unauthorised = problem.as_ref().is_some_and(|problem| problem.status == Some(401) || problem.names_bad_token); + Err(match status { + 401 => Error::CredentialsRefused, + 403 if says_unauthorised => Error::CredentialsRefused, + 429 | 503 => Error::Busy { retry_after: retry_after() }, + 413 => Error::TooLarge { limit }, + _ if jmap == Some(RequestProblem::Limit) => match limit { + Some(limit) if limit.is_concurrency() => Error::Busy { retry_after: retry_after() }, + limit => Error::TooLarge { limit }, + }, + 403 => Error::Forbidden { detail }, + 404 | 410 => Error::NotFound, + 400..=499 if jmap.is_some() => Error::Refused { problem: jmap, detail }, + 400 | 405 | 406 | 415 | 422 => Error::Refused { problem: None, detail }, + _ => Error::Http { status, detail }, + }) +} + +/// A redirect (RFC 9110 §15.4) is never followed here: its target is checked and handed to the caller. +fn judge_redirect(request: &HttpRequest, response: &HttpResponse) -> Error { + let status = response.status(); + let Some(location) = response.location() else { + return Error::Http { status, detail: None }; + }; + if !matches!(status, 301 | 302 | 303 | 307 | 308) { + return Error::Http { status, detail: None }; + } + let from = request.address(); + let target = match from.resolve(location) { + Ok(target) => target, + Err(error) => return Error::RedirectRefused(error), + }; + if let Err(error) = from.check_link(&target) { + return Error::RedirectRefused(error); + } + Error::Redirected { same_origin: from.same_origin(&target), target } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/classify/tests.rs b/crates/jmap/src/classify/tests.rs new file mode 100644 index 000000000..9a179bc0b --- /dev/null +++ b/crates/jmap/src/classify/tests.rs @@ -0,0 +1,199 @@ +use super::*; +use crate::address::{Address, AddressError}; +use crate::credentials::Credentials; +use crate::http::{API_RESPONSE_LIMIT, HttpRequest, Method, RequestKind}; +use std::time::Duration; + +fn request(kind: RequestKind) -> HttpRequest { + let credentials = Credentials::bearer("token-abcdef").unwrap(); + let address = Address::parse("https://jmap.example.com/api/").unwrap(); + HttpRequest::new(Method::Post, address, kind, credentials.authorization()) +} + +fn judge(response: HttpResponse) -> Result<(), Error> { + let mut redactor = Redactor::new(); + redactor.add("token-abcdef"); + judge_response(&request(RequestKind::Api), &response, &redactor) +} + +fn problem(status: u16, body: &str) -> HttpResponse { + HttpResponse::new(status, body.as_bytes().to_vec()).with_field("Content-Type", "application/problem+json") +} + +// RFC 8620 §3.1: a successful API response is application/json. +#[test] +fn success_needs_json() { + let ok = HttpResponse::new(200, b"{}".to_vec()).with_field("Content-Type", "Application/JSON; charset=utf-8"); + assert_eq!(judge(ok), Ok(())); + let html = HttpResponse::new(200, b"".to_vec()).with_field("Content-Type", "text/html"); + assert_eq!(judge(html), Err(Error::Unusable(Unusable::WrongContentType))); + assert_eq!(judge(HttpResponse::new(200, b"{}".to_vec())), Err(Error::Unusable(Unusable::WrongContentType))); + let download = HttpResponse::new(200, vec![1, 2, 3]).with_field("Content-Type", "application/pdf"); + assert_eq!(judge_response(&request(RequestKind::Download), &download, &Redactor::new()), Ok(())); +} + +// Requirement 2.2: a limit the caller set for one request is enforced too, at the octet. +#[test] +fn per_request_body_limit() { + let limited = request(RequestKind::Download).with_response_limit(10); + let exact = HttpResponse::new(200, vec![0; 10]); + assert_eq!(judge_response(&limited, &exact, &Redactor::new()), Ok(())); + let over = HttpResponse::new(200, vec![0; 11]); + assert_eq!(judge_response(&limited, &over, &Redactor::new()), Err(Error::Unusable(Unusable::BodyTooLarge))); +} + +// Requirement 2.2: the crate refuses a body above the limit it states for the kind of request. +#[test] +fn success_body_limit() { + let big = HttpResponse::new(200, vec![b' '; API_RESPONSE_LIMIT + 1]).with_field("Content-Type", "application/json"); + assert_eq!(judge(big), Err(Error::Unusable(Unusable::BodyTooLarge))); +} + +// RFC 9110 §15.5.2 and §15.5.4, RFC 9457 §3.1.2: 401, and 403 when the problem says so. +#[test] +fn credentials_refused() { + assert_eq!(judge(HttpResponse::new(401, Vec::new())), Err(Error::CredentialsRefused)); + assert_eq!(judge(problem(403, r#"{"status":401,"title":"Unauthorized"}"#)), Err(Error::CredentialsRefused)); + assert_eq!(judge(problem(403, r#"{"type":"https://example.com/probs/invalid_token"}"#)), Err(Error::CredentialsRefused)); + assert!(matches!(judge(problem(403, r#"{"status":403,"detail":"no access to this account"}"#)), Err(Error::Forbidden { detail: Some(_) }))); + assert!(matches!(judge(HttpResponse::new(403, Vec::new())), Err(Error::Forbidden { detail: None }))); +} + +// Requirement 2.4 (429 and 503), RFC 9110 §15.6.4 and §10.2.3: busy, with the delay asked for. +#[test] +fn busy() { + let response = HttpResponse::new(503, Vec::new()).with_field("Retry-After", "120"); + assert_eq!(judge(response), Err(Error::Busy { retry_after: Some(RetryAfter::Delay(Duration::from_secs(120))) })); + let response = HttpResponse::new(429, Vec::new()).with_field("Retry-After", "soon"); + assert_eq!(judge(response), Err(Error::Busy { retry_after: None })); + let concurrency = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxConcurrentRequests","status":400}"#); + assert_eq!(judge(concurrency), Err(Error::Busy { retry_after: None })); +} + +// RFC 9110 §15.5.14 and RFC 8620 §3.6.1: 413 and the JMAP limit problem with its limit name. +#[test] +fn too_large() { + assert_eq!(judge(HttpResponse::new(413, Vec::new())), Err(Error::TooLarge { limit: None })); + let limit = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"maxSizeRequest","status":400,"detail":"too big"}"#); + assert_eq!(judge(limit), Err(Error::TooLarge { limit: LimitName::new("maxSizeRequest") })); + let odd = problem(400, r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"max size\n"}"#); + assert_eq!(judge(odd), Err(Error::TooLarge { limit: None })); +} + +// RFC 8620 §3.6.1: the other request-level problems are recognised; RFC 9110 §15.5.5 and §15.5.11. +#[test] +fn refused_and_not_found() { + let unknown = problem( + 400, + r#"{"type":"urn:ietf:params:jmap:error:unknownCapability","status":400,"detail":"The Request object used capability 'https://example.com/apis/foobar', which is not supported by this server."}"#, + ); + match judge(unknown) { + Err(Error::Refused { problem: Some(RequestProblem::UnknownCapability), detail: Some(detail) }) => { + assert!(detail.as_str().starts_with("The Request object used capability")); + } + other => panic!("{other:?}"), + } + assert!(matches!( + judge(problem(400, r#"{"type":"urn:ietf:params:jmap:error:notJSON"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::NotJson), .. }) + )); + assert!(matches!( + judge(problem(400, r#"{"type":"urn:ietf:params:jmap:error:notRequest"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::NotRequest), .. }) + )); + assert!(matches!( + judge(problem(400, r#"{"type":"URN:IETF:params:jmap:error:somethingNew"}"#)), + Err(Error::Refused { problem: Some(RequestProblem::Other(_)), .. }) + )); + assert!(matches!(judge(HttpResponse::new(400, b"not json".to_vec())), Err(Error::Refused { problem: None, detail: None }))); + assert_eq!(judge(HttpResponse::new(404, Vec::new())), Err(Error::NotFound)); + assert_eq!(judge(HttpResponse::new(410, Vec::new())), Err(Error::NotFound)); +} + +// Requirement 3.5: a server-chosen identifier that echoes a secret is not shown. +#[test] +fn identifiers_echoing_secrets_are_dropped() { + let mut redactor = Redactor::new(); + redactor.add("hunter22"); + let judge = |body: &str| judge_response(&request(RequestKind::Api), &problem(400, body), &redactor); + assert_eq!( + judge(r#"{"type":"urn:ietf:params:jmap:error:hunter22"}"#), + Err(Error::Refused { problem: Some(RequestProblem::Other(String::new())), detail: None }) + ); + assert_eq!(judge(r#"{"type":"urn:ietf:params:jmap:error:limit","limit":"xhunter22"}"#), Err(Error::TooLarge { limit: None })); +} + +// RFC 9457 §3.1: a member of the wrong type is ignored. +#[test] +fn wrong_member_types_are_ignored() { + let response = problem(400, r#"{"type":42,"status":"401","limit":[],"detail":{"x":1},"title":"Bad"}"#); + match judge(response) { + Err(Error::Refused { problem: None, detail: Some(detail) }) => assert_eq!(detail.as_str(), "Bad"), + other => panic!("{other:?}"), + } +} + +// Requirement 2.4: server text is a short, clean excerpt without secrets. +#[test] +fn detail_is_cleaned() { + let body = format!(r#"{{"detail":"bad token token-abcdef\n\u001b[2J{}"}}"#, "x".repeat(500)); + match judge(problem(500, &body)) { + Err(Error::Http { status: 500, detail: Some(detail) }) => { + assert!(detail.as_str().starts_with("bad token [redacted] [2J")); + assert!(!detail.as_str().contains('\n') && !detail.as_str().contains('\u{1b}')); + assert!(detail.as_str().chars().count() <= crate::redact::EXCERPT_MAX_CHARS + 1); + } + other => panic!("{other:?}"), + } + let html = HttpResponse::new(502, b"

Bad gateway

".to_vec()).with_field("Content-Type", "text/html"); + assert_eq!(judge(html), Err(Error::Http { status: 502, detail: None })); +} + +// RFC 9110 §10.2.2 and §15.4: redirects are judged, not followed; http and loopback targets are refused. +#[test] +fn redirects() { + let response = HttpResponse::new(307, Vec::new()).with_field("Location", "/api/v2/"); + match judge(response) { + Err(Error::Redirected { target, same_origin: true }) => assert_eq!(target.as_str(), "https://jmap.example.com/api/v2/"), + other => panic!("{other:?}"), + } + let response = HttpResponse::new(301, Vec::new()).with_field("Location", "https://jmap.example.net/api/"); + assert!(matches!(judge(response), Err(Error::Redirected { same_origin: false, .. }))); + let response = HttpResponse::new(302, Vec::new()).with_field("Location", "http://jmap.example.com/api/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::InsecureTransport))); + let response = HttpResponse::new(308, Vec::new()).with_field("Location", "http://127.0.0.1:8080/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::LocalFromRemote))); + let response = HttpResponse::new(302, Vec::new()).with_field("Location", "https://user@example.com/"); + assert_eq!(judge(response), Err(Error::RedirectRefused(AddressError::UserInfo))); + assert_eq!(judge(HttpResponse::new(302, Vec::new())), Err(Error::Http { status: 302, detail: None })); + assert_eq!(judge(HttpResponse::new(304, Vec::new())), Err(Error::Http { status: 304, detail: None })); +} + +// RFC 9110 §15: anything else is an HTTP error with its status. +#[test] +fn other_statuses() { + assert_eq!(judge(HttpResponse::new(500, Vec::new())), Err(Error::Http { status: 500, detail: None })); + assert_eq!(judge(HttpResponse::new(418, Vec::new())), Err(Error::Http { status: 418, detail: None })); + assert_eq!(judge(HttpResponse::new(100, Vec::new())), Err(Error::Http { status: 100, detail: None })); + assert_eq!(judge(HttpResponse::new(0, Vec::new())), Err(Error::Http { status: 0, detail: None })); +} + +// RFC 8259 §8.1: a byte order mark may be ignored. +#[test] +fn json_with_bom() { + assert!(parse_json(b"\xEF\xBB\xBF{\"a\":1}").is_ok()); + assert_eq!(parse_json(b"{\"a\":"), Err(Unusable::InvalidJson)); + let deep = "[".repeat(10_000); + assert_eq!(parse_json(deep.as_bytes()), Err(Unusable::InvalidJson)); +} + +// RFC 8259 §9 and RFC 8620 §8.4: a text that would expand into too many values is refused before parsing. +#[test] +fn json_budget() { + let many_objects = format!("[{}{{}}]", "{},".repeat(MAX_JSON_OBJECTS)); + assert_eq!(parse_json(many_objects.as_bytes()), Err(Unusable::TooComplex)); + let many_values = format!("[{}0]", "0,".repeat(MAX_JSON_VALUES)); + assert_eq!(parse_json(many_values.as_bytes()), Err(Unusable::TooComplex)); + let in_strings = format!("[\"{}\"]", "{},:[\\\"".repeat(200_000)); + assert!(parse_json(in_strings.as_bytes()).is_ok()); +} diff --git a/crates/jmap/src/client.rs b/crates/jmap/src/client.rs new file mode 100644 index 000000000..43ee983af --- /dev/null +++ b/crates/jmap/src/client.rs @@ -0,0 +1,245 @@ +//! Fetching the session and talking to the API through the program's HTTP exchange. +//! +//! Each operation exists in two forms: a synchronous pair (build the [`HttpRequest`], then read the +//! [`HttpResponse`]) for programs that drive the exchange themselves, and an `async` helper that does both +//! through an [`HttpExchange`]. Neither follows redirects: a redirect comes back as +//! [`Error::Redirected`] with a checked target, and the caller decides. + +use std::fmt; + +use crate::address::Address; +use crate::classify::judge_response; +use crate::credentials::Credentials; +use crate::error::{BuildError, Error}; +use crate::http::{HttpExchange, HttpRequest, HttpResponse, Method, RequestKind}; +use crate::id::Id; +use crate::redact::{Excerpt, Redactor}; +use crate::request::ApiRequest; +use crate::response::ApiResponse; +use crate::session::{Account, CAPABILITY_MAIL, CoreLimits, Session}; + +/// A redactor knowing the credentials and the session address. +fn base_redactor(address: &Address, credentials: &Credentials) -> Redactor { + let mut redactor = Redactor::new(); + credentials.register_secrets(&mut redactor); + redactor.add_personal(address.as_str()); + if let Some(query) = address.query() { + redactor.add_personal(query); + // A query parameter may carry a token: each value, as written and decoded. + for pair in query.split('&') { + let value = pair.split_once('=').map_or(pair, |(_, value)| value); + redactor.add_personal(value); + if let Ok(decoded) = crate::percent::decode(value) { + let decoded = crate::credentials::Secret::new(decoded); + redactor.add_personal(decoded.expose()); + } + } + } + redactor +} + +/// Builds the request that fetches the session object (RFC 8620 §2). The address must satisfy the transport +/// rule, since the credentials travel with the request. +pub fn session_request(address: &Address, credentials: &Credentials) -> Result { + address.require_secure_transport()?; + Ok(HttpRequest::new(Method::Get, address.clone(), RequestKind::Session, credentials.authorization())) +} + +/// Reads the response to a [`session_request`]. +pub fn read_session(request: &HttpRequest, response: &HttpResponse, credentials: &Credentials) -> Result { + let redactor = base_redactor(request.address(), credentials); + judge_response(request, response, &redactor)?; + Session::parse(response.body(), request.address()) +} + +/// Fetches and reads the session object through `exchange`. +pub async fn fetch_session(exchange: &E, address: &Address, credentials: &Credentials) -> Result { + let request = session_request(address, credentials)?; + let response = exchange.exchange(&request).await?; + read_session(&request, &response, credentials) +} + +/// A JMAP client for one session and one mail account. +#[derive(Clone)] +pub struct Client { + session: Session, + credentials: Credentials, + account: Id, + redactor: Redactor, +} + +impl Client { + /// A client for the session's mail account ([`Session::mail_account`]). + pub fn new(session: Session, credentials: Credentials) -> Result { + let account = session.mail_account().cloned().ok_or(Error::NoMailAccount)?; + Self::for_account(session, credentials, account) + } + + /// A client for a chosen account of the session. + pub fn for_account(session: Session, credentials: Credentials, account: Id) -> Result { + let account_name = session.account(&account).ok_or(Error::NoMailAccount)?.name().to_owned(); + let mut redactor = base_redactor(session.address(), &credentials); + redactor.add_personal(session.username()); + redactor.add_personal(&account_name); + Ok(Self { session, credentials, account, redactor }) + } + + /// The session. + pub fn session(&self) -> &Session { + &self.session + } + + /// The account id in use. + pub fn account_id(&self) -> &Id { + &self.account + } + + /// The account in use. + pub fn account(&self) -> Option<&Account> { + self.session.account(&self.account) + } + + /// The core limits, defaulted and capped. + pub fn limits(&self) -> &CoreLimits { + self.session.core_limits() + } + + /// Whether the server advertises `capability`. + pub fn has_capability(&self, capability: &str) -> bool { + self.session.has_capability(capability) + } + + /// Whether the account in use offers `capability`. + pub fn account_has_capability(&self, capability: &str) -> bool { + self.account().is_some_and(|account| account.has_capability(capability)) + } + + /// Whether the account offers mail. + pub fn has_mail(&self) -> bool { + self.account_has_capability(CAPABILITY_MAIL) + } + + /// Cleans a server's text for an error message, with every secret this client knows replaced. + pub fn excerpt(&self, text: &str) -> Excerpt { + self.redactor.excerpt(text) + } + + /// The redactor holding this client's secrets. + pub fn redactor(&self) -> &Redactor { + &self.redactor + } + + /// Whether `response` reports a session state other than the one this client holds (RFC 8620 §3.4): the + /// session should then be fetched again. + pub fn is_session_stale(&self, response: &ApiResponse) -> bool { + response.session_state() != self.session.state() + } + + /// The address to download a blob (RFC 8620 §6.2), with `media_type` as `type` and `name` as `name`. + pub fn download_address(&self, blob: &Id, media_type: &str, name: &str) -> Result { + let values = [("accountId", self.account.as_str()), ("blobId", blob.as_str()), ("type", media_type), ("name", name)]; + Ok(self.session.download_template().expand(&values)?) + } + + /// The address to upload a blob to the account (RFC 8620 §6.1). + pub fn upload_address(&self) -> Result { + Ok(self.session.upload_template().expand(&[("accountId", self.account.as_str())])?) + } + + /// The request that fetches the session again. + pub fn session_request(&self) -> Result { + session_request(self.session.address(), &self.credentials) + } + + /// The HTTP request for an API request (RFC 8620 §3.1). Checked first: the capabilities used must be + /// advertised, the number of calls must be within `maxCallsInRequest`, and the body within + /// `maxSizeRequest`. + pub fn api_request(&self, request: &ApiRequest) -> Result { + if request.capabilities().any(|capability| !self.session.has_capability(capability)) { + return Err(BuildError::UnsupportedCapability.into()); + } + let calls = u64::try_from(request.len()).unwrap_or(u64::MAX); + let max_calls = self.limits().max_calls_in_request(); + if calls > max_calls { + return Err(BuildError::TooManyCalls { limit: max_calls }.into()); + } + let body = request.to_json()?; + let max_size = self.limits().max_size_request(); + if u64::try_from(body.len()).unwrap_or(u64::MAX) > max_size { + return Err(BuildError::TooLarge { limit: max_size }.into()); + } + let address = self.session.api_address().clone(); + let http = HttpRequest::new(Method::Post, address, RequestKind::Api, self.credentials.authorization()).with_json_body(body); + Ok(match request.response_limit() { + Some(limit) => http.with_response_limit(limit), + None => http, + }) + } + + /// Reads the response to an [`Client::api_request`]. + pub fn read_api_response(&self, request: &HttpRequest, response: &HttpResponse) -> Result { + judge_response(request, response, &self.redactor)?; + ApiResponse::parse(response.body(), &self.redactor) + } + + /// The HTTP request that downloads a blob (RFC 8620 §6.2). + pub fn download_request(&self, blob: &Id, media_type: &str, name: &str) -> Result { + let address = self.download_address(blob, media_type, name)?; + Ok(HttpRequest::new(Method::Get, address, RequestKind::Download, self.credentials.authorization())) + } + + /// Reads the response to a [`Client::download_request`] and hands back the blob's bytes. + pub fn read_download(&self, request: &HttpRequest, response: HttpResponse) -> Result, Error> { + judge_response(request, &response, &self.redactor)?; + Ok(response.into_body()) + } + + /// Sends an API request through `exchange` and reads the response. + pub async fn call(&self, exchange: &E, request: &ApiRequest) -> Result { + let http = self.api_request(request)?; + let response = exchange.exchange(&http).await?; + self.read_api_response(&http, &response) + } + + /// Downloads a blob through `exchange`. + /// + /// `max_bytes`, when given, lowers the largest body accepted for this download below + /// [`DOWNLOAD_RESPONSE_LIMIT`](crate::http::DOWNLOAD_RESPONSE_LIMIT); the exchange sees it before + /// reading the body. + pub async fn download( + &self, + exchange: &E, + blob: &Id, + media_type: &str, + name: &str, + max_bytes: Option, + ) -> Result, Error> { + let mut http = self.download_request(blob, media_type, name)?; + if let Some(limit) = max_bytes { + http = http.with_response_limit(limit); + } + let response = exchange.exchange(&http).await?; + self.read_download(&http, response) + } + + /// Fetches the session again through `exchange`, for when [`Client::is_session_stale`] says so. + pub async fn refetch_session(&self, exchange: &E) -> Result { + fetch_session(exchange, self.session.address(), &self.credentials).await + } +} + +impl fmt::Debug for Client { + /// Only what says nothing about the person: the authentication scheme, how many capabilities the + /// server offers, and the core limits. No account id, no address or origin (a personal domain names its + /// owner), no user name, no account name, no secret. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Client") + .field("scheme", &self.credentials.scheme_name()) + .field("capabilities", &self.session.capabilities().count()) + .field("limits", self.session.core_limits()) + .finish_non_exhaustive() + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/client/tests.rs b/crates/jmap/src/client/tests.rs new file mode 100644 index 000000000..b398dbaf2 --- /dev/null +++ b/crates/jmap/src/client/tests.rs @@ -0,0 +1,245 @@ +use super::*; +use crate::address::AddressError; +use crate::error::Unusable; +use crate::http::TransportError; +use crate::retry::RetryAfter; +use crate::session::tests::{EXAMPLE, session_address}; +use crate::test_support::{FakeExchange, block_on, made_secret}; +use serde_json::{Map, Value, json}; +use std::time::Duration; + +fn credentials() -> Credentials { + Credentials::basic("john@example.com", &made_secret("open-sesame-42")).unwrap() +} + +fn json_response(status: u16, body: &Value) -> HttpResponse { + HttpResponse::new(status, body.to_string().into_bytes()).with_field("Content-Type", "application/json") +} + +fn client() -> Client { + let session = Session::parse(EXAMPLE.as_bytes(), &session_address()).unwrap(); + Client::new(session, credentials()).unwrap() +} + +// RFC 8620 §2: the session is fetched with an authenticated GET and parsed. +#[test] +fn fetches_session() { + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(200, EXAMPLE.as_bytes().to_vec()).with_field("Content-Type", "application/json"))]); + let session = block_on(fetch_session(&exchange, &session_address(), &credentials())).unwrap(); + assert_eq!(session.state(), "75128aab4b1b"); + let sent = exchange.sent(); + assert_eq!(sent.len(), 1); + assert_eq!(sent[0].method(), Method::Get); + assert_eq!(sent[0].url(), "https://jmap.example.com/.well-known/jmap"); + assert!(sent[0].fields().any(|(name, value)| name == "Authorization" && value == "Basic am9obkBleGFtcGxlLmNvbTpvcGVuLXNlc2FtZS00Mg==")); +} + +// Requirement 1.2: credentials are never sent over plain http to a remote host. +#[test] +fn refuses_insecure_session_address() { + let exchange = FakeExchange::new(Vec::new()); + let address = Address::parse("http://jmap.example.com/").unwrap(); + assert_eq!(block_on(fetch_session(&exchange, &address, &credentials())), Err(Error::Address(AddressError::InsecureTransport))); + assert!(exchange.sent().is_empty()); +} + +// RFC 8620 §2.2 ("following any redirects") with requirement 3.1: the redirect comes back to the caller. +#[test] +fn session_redirect_is_returned() { + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(301, Vec::new()).with_field("Location", "https://mail.example.net/jmap/session"))]); + match block_on(fetch_session(&exchange, &session_address(), &credentials())) { + Err(Error::Redirected { target, same_origin: false }) => assert_eq!(target.as_str(), "https://mail.example.net/jmap/session"), + other => panic!("{other:?}"), + } + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(302, Vec::new()).with_field("Location", "http://mail.example.net/"))]); + assert_eq!(block_on(fetch_session(&exchange, &session_address(), &credentials())), Err(Error::RedirectRefused(AddressError::InsecureTransport))); +} + +// Requirement 3.2: the account, limits, capabilities, download and upload addresses. +#[test] +fn client_exposes_what_mail_code_needs() { + let client = client(); + assert_eq!(client.account_id().as_str(), "A13824"); + assert_eq!(client.limits().max_calls_in_request(), 32); + assert!(client.has_capability(CAPABILITY_MAIL)); + assert!(client.has_mail()); + assert!(!client.has_capability("urn:ietf:params:jmap:calendars")); + let blob = Id::new("Gb1").unwrap(); + assert_eq!( + client.download_address(&blob, "image/png", "photo 1.png").unwrap().as_str(), + "https://jmap.example.com/download/A13824/Gb1/photo%201.png?accept=image%2Fpng" + ); + assert_eq!(client.upload_address().unwrap().as_str(), "https://jmap.example.com/upload/A13824/"); + assert!(matches!(client.download_address(&blob, "image/png", ".."), Err(Error::Template(_)))); +} + +// RFC 8620 §3.1, §3.3, §3.4 and §3.7: a request with a result reference, and its per-call results. +#[test] +fn calls_the_api() { + let client = client(); + let mut request = ApiRequest::new(); + request.using(CAPABILITY_MAIL).unwrap(); + let query = request.add_call("Email/query", Map::from_iter([("accountId".to_owned(), json!("A13824"))])).unwrap(); + let ids = request.reference(&query, "Email/query", "/ids").unwrap(); + let get = request.add_call_with_references("Email/get", Map::from_iter([("accountId".to_owned(), json!("A13824"))]), vec![("ids", ids)]).unwrap(); + let answer = json!({ + "methodResponses": [ + ["Email/query", {"accountId": "A13824", "ids": ["M1"], "position": 0}, "c0"], + ["error", {"type": "invalidResultReference"}, "c1"] + ], + "sessionState": "changed" + }); + let exchange = FakeExchange::new(vec![Ok(json_response(200, &answer))]); + let mut response = block_on(client.call(&exchange, &request)).unwrap(); + assert!(client.is_session_stale(&response)); + assert_eq!(response.take_named(&query, "Email/query").unwrap()["ids"], json!(["M1"])); + assert!(matches!(response.take(&get), Err(crate::response::CallFailure::Method(_)))); + let sent = &exchange.sent()[0]; + assert_eq!(sent.method(), Method::Post); + assert_eq!(sent.url(), "https://jmap.example.com/api/"); + let body: Value = serde_json::from_slice(sent.body()).unwrap(); + assert_eq!(body["using"], json!(["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"])); +} + +// RFC 8620 §2 limits: maxCallsInRequest and maxSizeRequest are enforced before sending. +#[test] +fn enforces_limits_before_sending() { + let client = client(); + let mut request = ApiRequest::new(); + for _ in 0..33 { + request.add_call("Core/echo", Map::new()).unwrap(); + } + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::TooManyCalls { limit: 32 })); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::from_iter([("blob".to_owned(), json!("x".repeat(10_000_001)))])).unwrap(); + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::TooLarge { limit: 10_000_000 })); + let mut request = ApiRequest::new(); + request.using("urn:ietf:params:jmap:calendars").unwrap(); + request.add_call("Core/echo", Map::new()).unwrap(); + assert_eq!(client.api_request(&request).unwrap_err(), Error::Build(BuildError::UnsupportedCapability)); +} + +// Requirement 2.4 through the client: busy, credentials, transport overload, unusable answers. +#[test] +fn classifies_api_failures() { + let client = client(); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + let cases: Vec<(Result, Error)> = vec![ + ( + Ok(HttpResponse::new(429, Vec::new()).with_field("Retry-After", "7")), + Error::Busy { retry_after: Some(RetryAfter::Delay(Duration::from_secs(7))) }, + ), + (Ok(HttpResponse::new(401, Vec::new())), Error::CredentialsRefused), + (Err(TransportError::Overloaded), Error::Busy { retry_after: None }), + (Err(TransportError::TimedOut), Error::Transport(TransportError::TimedOut)), + (Ok(HttpResponse::new(200, b"".to_vec()).with_field("Content-Type", "text/html")), Error::Unusable(Unusable::WrongContentType)), + (Ok(HttpResponse::new(200, b"{\"x\":".to_vec()).with_field("Content-Type", "application/json")), Error::Unusable(Unusable::InvalidJson)), + ]; + for (answer, expected) in cases { + let exchange = FakeExchange::new(vec![answer]); + assert_eq!(block_on(client.call(&exchange, &request)).unwrap_err(), expected); + } +} + +// Requirement 3.5 through the client: the user name, the account name and the password never show. +#[test] +fn errors_hide_secrets() { + let client = client(); + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + let problem = json!({"status": 500, "detail": "user john@example.com with password open-sesame-42 at https://jmap.example.com/.well-known/jmap"}); + let response = HttpResponse::new(500, problem.to_string().into_bytes()).with_field("Content-Type", "application/problem+json"); + let exchange = FakeExchange::new(vec![Ok(response)]); + let error = block_on(client.call(&exchange, &request)).unwrap_err(); + let shown = error.to_string(); + assert!(!shown.contains("john") && !shown.contains("open-sesame") && !shown.contains("well-known")); +} + +// Requirement 2.3: the client's debug form shows no secret, no user name, no account id and no address. +#[test] +fn debug_shows_nothing_personal() { + let client = client(); + let debug = format!("{client:?}"); + for hidden in ["john", "jane", "open-sesame", "A13824", "jmap.example.com", "75128aab4b1b", "am9obk"] { + assert!(!debug.contains(hidden), "{hidden} shows in the client's debug form"); + } + assert!(debug.starts_with("Client { scheme: \"Basic\"")); +} + +// Requirement 3.5: a token in the session address's query is redacted on its own, and a short server-chosen +// user name does not blank every message. +#[test] +fn session_address_parts_are_secrets() { + let address = Address::parse("https://jmap.example.com/.well-known/jmap?apikey=SECRETKEY123").unwrap(); + let text = EXAMPLE.replace(r#""username": "john@example.com""#, r#""username": "jo""#); + let session = Session::parse(text.as_bytes(), &address).unwrap(); + let client = Client::new(session, Credentials::bearer("token-0123456789").unwrap()).unwrap(); + assert_eq!(client.excerpt("invalid key SECRETKEY123 for jo").as_str(), "invalid key [redacted] for jo"); + assert_eq!(client.excerpt("bad token-0123456789").as_str(), "bad [redacted]"); +} + +// RFC 8620 §6.2: a download is an authenticated GET of the expanded template; the body comes back as is. +#[test] +fn downloads() { + let client = client(); + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(200, vec![0, 159, 146, 150]).with_field("Content-Type", "application/octet-stream"))]); + let bytes = block_on(client.download(&exchange, &Id::new("B1").unwrap(), "application/octet-stream", "a.bin", None)).unwrap(); + assert_eq!(bytes, vec![0, 159, 146, 150]); + assert_eq!(exchange.sent()[0].url(), "https://jmap.example.com/download/A13824/B1/a.bin?accept=application%2Foctet-stream"); + let exchange = FakeExchange::new(vec![Ok(HttpResponse::new(404, Vec::new()))]); + assert_eq!(block_on(client.download(&exchange, &Id::new("B1").unwrap(), "", "a.bin", None)), Err(Error::NotFound)); +} + +// Requirement 2.2: a limit the caller sets for one request reaches the exchange before the body is read, +// and a body above it is refused, whether the exchange stops early or hands the whole body over. +#[test] +fn per_request_limits() { + let client = client(); + let blob = Id::new("B1").unwrap(); + let body = || Ok(HttpResponse::new(200, vec![7; 5])); + let careful = FakeExchange::capping(vec![body()]); + assert_eq!(block_on(client.download(&careful, &blob, "", "a.bin", Some(4))), Err(Error::Unusable(Unusable::BodyTooLarge))); + assert_eq!(careful.sent()[0].max_response_bytes(), 4); + let careless = FakeExchange::new(vec![body()]); + assert_eq!(block_on(client.download(&careless, &blob, "", "a.bin", Some(4))), Err(Error::Unusable(Unusable::BodyTooLarge))); + let fits = FakeExchange::capping(vec![body()]); + assert_eq!(block_on(client.download(&fits, &blob, "", "a.bin", Some(5))).unwrap(), vec![7; 5]); + let unlimited = FakeExchange::new(vec![body()]); + block_on(client.download(&unlimited, &blob, "", "a.bin", None)).unwrap(); + assert_eq!(unlimited.sent()[0].max_response_bytes(), crate::http::DOWNLOAD_RESPONSE_LIMIT); + + let mut request = ApiRequest::new(); + request.add_call("Core/echo", Map::new()).unwrap(); + request.set_response_limit(64); + assert_eq!(client.api_request(&request).unwrap().max_response_bytes(), 64); + let answer = json!({"methodResponses": [["Core/echo", {"pad": "x".repeat(100)}, "c0"]], "sessionState": "s"}); + let exchange = FakeExchange::capping(vec![Ok(json_response(200, &answer))]); + assert_eq!(block_on(client.call(&exchange, &request)).unwrap_err(), Error::Unusable(Unusable::BodyTooLarge)); + request.set_response_limit(usize::MAX); + assert_eq!(client.api_request(&request).unwrap().max_response_bytes(), crate::http::API_RESPONSE_LIMIT); +} + +// Requirement 3.2: no mail account, no client. +#[test] +fn needs_a_mail_account() { + let text = EXAMPLE.replace("urn:ietf:params:jmap:mail\": {", "urn:example:other\": {"); + let session = Session::parse(text.as_bytes(), &session_address()).unwrap(); + assert_eq!(session.mail_account(), None); + assert_eq!(Client::new(session, credentials()).unwrap_err(), Error::NoMailAccount); +} + +// Requirement 2.6 context (RFC 8620 §3.1 calls made through the caller's exchange): the helpers' futures are +// Send when the exchange's are, so native programs can spawn them. +#[test] +fn futures_are_send() { + fn assert_send(_: &T) {} + let client = client(); + let exchange = FakeExchange::new(Vec::new()); + let request = ApiRequest::new(); + let future = client.call(&exchange, &request); + assert_send(&future); + let address = session_address(); + let credentials = credentials(); + assert_send(&fetch_session(&exchange, &address, &credentials)); +} diff --git a/crates/jmap/src/credentials.rs b/crates/jmap/src/credentials.rs new file mode 100644 index 000000000..5585602e4 --- /dev/null +++ b/crates/jmap/src/credentials.rs @@ -0,0 +1,245 @@ +//! Credentials: HTTP Basic (RFC 7617) and Bearer tokens (RFC 6750). +//! +//! Credentials live in memory only. Their `Debug` form names the scheme and nothing else: no secret and no +//! user name, which is usually an e-mail address and so personal data. The memory holding a secret is +//! overwritten when it is dropped; this is best effort (the allocator, the operating system or an earlier +//! copy made by the caller are out of reach without `unsafe`), not a guarantee. + +use std::fmt; + +use crate::redact::Redactor; + +/// The longest user name, password or token accepted, in octets. +pub const MAX_CREDENTIAL_LEN: usize = 8 * 1024; + +/// A piece of text that must not be shown, and that is overwritten when dropped. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct Secret(String); + +impl Secret { + pub(crate) fn new(text: String) -> Self { + Self(text) + } + + pub(crate) fn expose(&self) -> &str { + &self.0 + } +} + +impl Drop for Secret { + fn drop(&mut self) { + let mut bytes = std::mem::take(&mut self.0).into_bytes(); + bytes.iter_mut().for_each(|b| *b = 0); + std::hint::black_box(&bytes); + } +} + +impl fmt::Debug for Secret { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("") + } +} + +/// Why credentials were refused. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum CredentialsError { + /// The user name is empty. + EmptyUser, + /// The token is empty. + EmptyToken, + /// The user name holds a colon, which Basic cannot carry (RFC 7617 §2). + ColonInUser, + /// A control character in the user name or password (RFC 7617 §2). + ControlCharacter, + /// The token holds a character RFC 6750 §2.1 does not allow. + InvalidToken, + /// Longer than [`MAX_CREDENTIAL_LEN`]. + TooLong, +} + +impl fmt::Display for CredentialsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::EmptyUser => "the user name is empty; enter the user name of the mail account", + Self::EmptyToken => "the access token is empty; sign in again to obtain one", + Self::ColonInUser => { + "the user name contains a colon (:), which password sign-in cannot carry; use a token instead, or the user name without it" + } + Self::ControlCharacter => "the user name or password contains an invisible control character, often from a paste; type it again", + Self::InvalidToken => { + "the access token contains a space, a comma or another character a token cannot hold; copy it again without extra characters" + } + Self::TooLong => "the user name, password or token is longer than 8 KiB, more than this program accepts; check what was pasted", + }) + } +} + +impl std::error::Error for CredentialsError {} + +#[derive(Clone)] +enum Kind { + Basic { user: Secret, password: Secret }, + Bearer { token: Secret }, +} + +/// Credentials for the JMAP server. +#[derive(Clone)] +pub struct Credentials { + kind: Kind, + authorization: Secret, +} + +const BASE64_ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +fn base64_char(index: u8) -> char { + char::from(BASE64_ALPHABET.get(usize::from(index & 0x3F)).copied().unwrap_or(b'A')) +} + +/// Base64 with padding (RFC 4648 §4), written into `out`. +fn base64_into(input: &[u8], out: &mut String) { + for chunk in input.chunks(3) { + let (a, b, c) = match *chunk { + [a, b, c] => (a, Some(b), Some(c)), + [a, b] => (a, Some(b), None), + [a] => (a, None, None), + _ => continue, + }; + let b0 = b.unwrap_or(0); + let c0 = c.unwrap_or(0); + out.push(base64_char(a >> 2)); + out.push(base64_char(((a & 0x03) << 4) | (b0 >> 4))); + out.push(if b.is_some() { base64_char(((b0 & 0x0F) << 2) | (c0 >> 6)) } else { '=' }); + out.push(if c.is_some() { base64_char(c0 & 0x3F) } else { '=' }); + } +} + +/// The part an octet may play in a bearer token (RFC 6750 §2.1, `b64token`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum TokenOctet { + /// A letter, a digit, or one of `-` `.` `_` `~` `+` `/`: allowed anywhere before the padding. + Body, + /// `=`: allowed only at the end, any number of times. + Padding, + /// Anything else, spaces, commas, quotes, controls and non-ASCII octets among them. + Refused, +} + +/// Sorts one octet into its [`TokenOctet`] part. +const fn token_octet(octet: u8) -> TokenOctet { + match octet { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' => TokenOctet::Body, + b'-' | b'.' | b'_' | b'~' | b'+' | b'/' => TokenOctet::Body, + b'=' => TokenOctet::Padding, + _ => TokenOctet::Refused, + } +} + +/// Reads a bearer token in one pass: at least one body octet, then nothing but padding once padding began. +fn is_bearer_token(token: &str) -> bool { + let mut body_octets = 0usize; + let mut padding_started = false; + for octet in token.bytes() { + match (token_octet(octet), padding_started) { + (TokenOctet::Body, false) => body_octets = body_octets.saturating_add(1), + (TokenOctet::Padding, _) => padding_started = true, + (TokenOctet::Body, true) | (TokenOctet::Refused, _) => return false, + } + } + body_octets > 0 +} + +impl Credentials { + /// Basic credentials (RFC 7617 §2): `user:password` in UTF-8, base64-encoded. + /// + /// The user name may not be empty or hold a colon; neither part may hold a control character. The text is + /// sent as given: Unicode normalisation (RFC 7617 §2.1 mentions NFC) is left to the caller. + pub fn basic(user: &str, password: &str) -> Result { + if user.is_empty() { + return Err(CredentialsError::EmptyUser); + } + if user.len() > MAX_CREDENTIAL_LEN || password.len() > MAX_CREDENTIAL_LEN { + return Err(CredentialsError::TooLong); + } + if user.contains(':') { + return Err(CredentialsError::ColonInUser); + } + if user.chars().chain(password.chars()).any(char::is_control) { + return Err(CredentialsError::ControlCharacter); + } + let mut pair = Vec::with_capacity(user.len().saturating_add(password.len()).saturating_add(1)); + pair.extend_from_slice(user.as_bytes()); + pair.push(b':'); + pair.extend_from_slice(password.as_bytes()); + let encoded_len = pair.len().div_ceil(3).saturating_mul(4); + let mut header = String::with_capacity(encoded_len.saturating_add(6)); + header.push_str("Basic "); + base64_into(&pair, &mut header); + pair.iter_mut().for_each(|b| *b = 0); + std::hint::black_box(&pair); + Ok(Self { + kind: Kind::Basic { user: Secret::new(user.to_owned()), password: Secret::new(password.to_owned()) }, + authorization: Secret::new(header), + }) + } + + /// Bearer credentials (RFC 6750 §2.1). The token must match `b64token`. + pub fn bearer(token: &str) -> Result { + if token.is_empty() { + return Err(CredentialsError::EmptyToken); + } + if token.len() > MAX_CREDENTIAL_LEN { + return Err(CredentialsError::TooLong); + } + if !is_bearer_token(token) { + return Err(CredentialsError::InvalidToken); + } + let mut header = String::with_capacity(token.len().saturating_add(7)); + header.push_str("Bearer "); + header.push_str(token); + Ok(Self { kind: Kind::Bearer { token: Secret::new(token.to_owned()) }, authorization: Secret::new(header) }) + } + + /// The authentication scheme's name: `Basic` or `Bearer`. + pub fn scheme_name(&self) -> &'static str { + match self.kind { + Kind::Basic { .. } => "Basic", + Kind::Bearer { .. } => "Bearer", + } + } + + /// The value of the `Authorization` header field. + pub(crate) fn authorization(&self) -> &Secret { + &self.authorization + } + + /// Teaches a redactor every secret these credentials hold. + pub fn register_secrets(&self, redactor: &mut Redactor) { + match &self.kind { + Kind::Basic { user, password } => { + redactor.add_personal(user.expose()); + redactor.add(password.expose()); + } + Kind::Bearer { token } => redactor.add(token.expose()), + } + let header = self.authorization.expose(); + let value = header.split_once(' ').map_or(header, |(_, value)| value); + redactor.add(value); + if let Kind::Basic { .. } = self.kind { + // The base64 forms a server might echo: without padding, and in the URL-safe alphabet (RFC 4648 §5). + let unpadded = value.trim_end_matches('='); + redactor.add(unpadded); + let url_safe = Secret::new(unpadded.replace('+', "-").replace('/', "_")); + redactor.add(url_safe.expose()); + } + } +} + +impl fmt::Debug for Credentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Credentials({}, )", self.scheme_name()) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/credentials/tests.rs b/crates/jmap/src/credentials/tests.rs new file mode 100644 index 000000000..2b2973a2a --- /dev/null +++ b/crates/jmap/src/credentials/tests.rs @@ -0,0 +1,107 @@ +use super::*; +use crate::test_support::made_secret; + +// RFC 7617 §2: the "Aladdin" / "open sesame" example. +#[test] +fn basic_example() { + let credentials = Credentials::basic("Aladdin", &made_secret("open sesame")).unwrap(); + assert_eq!(credentials.authorization().expose(), "Basic QWxhZGRpbjpvcGVuIHNlc2FtZQ=="); +} + +// RFC 7617 §2.1: the UTF-8 example ("test" / "123£"). +#[test] +fn basic_utf8_example() { + let credentials = Credentials::basic("test", &made_secret("123\u{A3}")).unwrap(); + assert_eq!(credentials.authorization().expose(), "Basic dGVzdDoxMjPCow=="); +} + +// RFC 4648 §10: the base64 test vectors. +#[test] +fn base64_vectors() { + for (input, expected) in + [("", ""), ("f", "Zg=="), ("fo", "Zm8="), ("foo", "Zm9v"), ("foob", "Zm9vYg=="), ("fooba", "Zm9vYmE="), ("foobar", "Zm9vYmFy")] + { + let mut out = String::new(); + base64_into(input.as_bytes(), &mut out); + assert_eq!(out, expected); + } +} + +// RFC 7617 §2: no colon in the user name, no control characters anywhere. +#[test] +fn basic_refusals() { + assert_eq!(Credentials::basic("", &made_secret("x")).unwrap_err(), CredentialsError::EmptyUser); + assert_eq!(Credentials::basic("a:b", &made_secret("x")).unwrap_err(), CredentialsError::ColonInUser); + assert_eq!(Credentials::basic("user", &made_secret("pa\nss")).unwrap_err(), CredentialsError::ControlCharacter); + assert_eq!(Credentials::basic("us\u{7f}er", &made_secret("pass")).unwrap_err(), CredentialsError::ControlCharacter); + assert!(Credentials::basic("user", &made_secret("with:colon")).is_ok()); +} + +// RFC 6750 §2.1: the specification's example token, and an empty token. +#[test] +fn bearer_tokens() { + let credentials = Credentials::bearer("mF_9.B5f-4.1JqM").unwrap(); + assert_eq!(credentials.authorization().expose(), "Bearer mF_9.B5f-4.1JqM"); + assert_eq!(Credentials::bearer("").unwrap_err(), CredentialsError::EmptyToken); +} + +// RFC 6750 §2.1, rule 1: the body is made of ALPHA, DIGIT and the six marks - . _ ~ + /. +#[test] +fn token_body_octets() { + for token in ["AZaz09", "a-b", "a.b", "a_b", "a~b", "a+b", "a/b", "-._~+/"] { + assert!(is_bearer_token(token), "{token}"); + } + for octet in 0u8..=255 { + let expected = octet.is_ascii_alphanumeric() || b"-._~+/".contains(&octet); + assert_eq!(token_octet(octet) == TokenOctet::Body, expected, "{octet}"); + } +} + +// RFC 6750 §2.1, rule 2: at least one body octet ("1*"). +#[test] +fn token_needs_a_body() { + for token in ["", "=", "==="] { + assert!(!is_bearer_token(token), "{token:?}"); + } + assert_eq!(Credentials::bearer("==").unwrap_err(), CredentialsError::InvalidToken); +} + +// RFC 6750 §2.1, rule 3: "=" only at the end, any number of times ("*\"=\""). +#[test] +fn token_padding_only_at_the_end() { + for token in ["abc=", "abc==", "abc===="] { + assert!(is_bearer_token(token), "{token}"); + } + for token in ["=abc", "a=b", "ab=c=", "ab==c"] { + assert!(!is_bearer_token(token), "{token}"); + } +} + +// RFC 6750 §2.1, rule 4: nothing else, which also rules out header injection (RFC 9110 §5.5). +#[test] +fn token_refused_octets() { + for token in ["a b", "a\r\nX-Injected: 1", "a,b", "a\"b", "a:b", "tok\u{e9}n", "a\tb", "a;b"] { + assert!(!is_bearer_token(token), "{token:?}"); + assert_eq!(Credentials::bearer(token).unwrap_err(), CredentialsError::InvalidToken, "{token:?}"); + } +} + +// Requirement 2.3: no secret and no user name in the debug form. +#[test] +fn debug_shows_no_secret() { + let basic = Credentials::basic("john@example.com", &made_secret("hunter2-long")).unwrap(); + let shown = format!("{basic:?}"); + assert_eq!(shown, "Credentials(Basic, )"); + let bearer = Credentials::bearer("tok-123456").unwrap(); + assert!(!format!("{bearer:?}").contains("tok-123456")); +} + +// Requirement 3.5: the redactor learns the user name, the password and the encoded header value. +#[test] +fn registers_secrets() { + let credentials = Credentials::basic("john@example.com", &made_secret("open sesame")).unwrap(); + let mut redactor = Redactor::new(); + credentials.register_secrets(&mut redactor); + let excerpt = redactor.excerpt("john@example.com open sesame am9obkBleGFtcGxlLmNvbTpvcGVuIHNlc2FtZQ=="); + assert_eq!(excerpt.as_str(), "[redacted] [redacted] [redacted]"); +} diff --git a/crates/jmap/src/discovery.rs b/crates/jmap/src/discovery.rs new file mode 100644 index 000000000..65ca76041 --- /dev/null +++ b/crates/jmap/src/discovery.rs @@ -0,0 +1,60 @@ +//! Turning what a person types into the JMAP session address to fetch (requirement 1.6, RFC 8620 §2.2). +//! +//! A person may type `example.com`, `example.com/jmap/session`, `example.com:8443`, `https://example.com` or a +//! full address. The result defaults to `https`, and to [`WELL_KNOWN_PATH`] when only a host was given. Plain +//! `http` is kept only when it was typed and the host is loopback (requirement 1.2); it is never chosen by +//! this function, and `http` typed for a remote host is refused rather than silently upgraded, so that the +//! person sees that what they typed cannot be used. + +use crate::address::{Address, AddressError, MAX_ADDRESS_LEN}; + +/// The well-known path of the JMAP session resource (RFC 8620 §2.2, registered by §9.2). +pub const WELL_KNOWN_PATH: &str = "/.well-known/jmap"; + +/// Whether `text` begins with `scheme://`, the scheme being well formed (RFC 3986 §3.1). +fn has_explicit_scheme(text: &str) -> bool { + match text.split_once("://") { + Some((scheme, _)) => { + let mut bytes = scheme.bytes(); + matches!(bytes.next(), Some(b) if b.is_ascii_alphabetic()) && bytes.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'+' | b'-' | b'.')) + } + None => false, + } +} + +/// Whether `text` starts with `http:` or `https:` without the `//` that must follow (`https:example.com`). +fn has_scheme_without_slashes(text: &str) -> bool { + let lower = text.get(..6).unwrap_or(text).to_ascii_lowercase(); + (lower.starts_with("http:") || lower.starts_with("https:")) && !text.contains("://") +} + +/// Turns a session address typed by a person into the address to fetch. +/// +/// Surrounding white space is ignored; anything else that is not a valid address is refused with the same +/// errors as [`Address::parse`]. The result always satisfies the transport rule. +pub fn session_address(input: &str) -> Result { + let trimmed = input.trim_matches(|c: char| c.is_ascii_whitespace()); + if trimmed.is_empty() { + return Err(AddressError::Empty); + } + if trimmed.len() > MAX_ADDRESS_LEN { + return Err(AddressError::TooLong); + } + if has_scheme_without_slashes(trimmed) { + return Err(AddressError::MissingAuthority); + } + let address = if has_explicit_scheme(trimmed) { + Address::parse(trimmed)? + } else { + let mut with_scheme = String::with_capacity(trimmed.len().saturating_add(8)); + with_scheme.push_str("https://"); + with_scheme.push_str(trimmed); + Address::parse(&with_scheme)? + }; + let address = if address.path() == "/" && address.query().is_none() { address.with_path(WELL_KNOWN_PATH)? } else { address }; + address.require_secure_transport()?; + Ok(address) +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/discovery/tests.rs b/crates/jmap/src/discovery/tests.rs new file mode 100644 index 000000000..f38cdeaa6 --- /dev/null +++ b/crates/jmap/src/discovery/tests.rs @@ -0,0 +1,53 @@ +use super::*; + +fn fetched(input: &str) -> String { + session_address(input).unwrap().as_str().to_owned() +} + +// RFC 8620 §2.2: a bare host means https and the well-known path. +#[test] +fn bare_host_gets_well_known_path() { + assert_eq!(fetched("example.com"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched(" Example.COM \n"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("example.com/"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("example.com:8443"), "https://example.com:8443/.well-known/jmap"); + assert_eq!(fetched("https://example.com"), "https://example.com/.well-known/jmap"); + assert_eq!(fetched("[::1]:8080"), "https://[::1]:8080/.well-known/jmap"); +} + +// Requirement 1.6: a host with a path keeps the path; https is the default scheme. +#[test] +fn host_with_path_keeps_it() { + assert_eq!(fetched("example.com/jmap/session"), "https://example.com/jmap/session"); + assert_eq!(fetched("jmap.example.org:8443/api/session?x=1"), "https://jmap.example.org:8443/api/session?x=1"); + assert_eq!(fetched("HTTPS://example.net/jmap/"), "https://example.net/jmap/"); + assert_eq!(fetched("example.com/?a=b"), "https://example.com/?a=b"); +} + +// Requirement 1.2 and 1.6: never a downgrade; http only as typed, and only to loopback. +#[test] +fn never_downgrades() { + assert_eq!(fetched("localhost:8080"), "https://localhost:8080/.well-known/jmap"); + assert_eq!(fetched("http://localhost:8080"), "http://localhost:8080/.well-known/jmap"); + assert_eq!(fetched("http://127.0.0.1:8080/jmap"), "http://127.0.0.1:8080/jmap"); + assert_eq!(session_address("http://example.com"), Err(AddressError::InsecureTransport)); + assert_eq!(session_address("http://sub.localhost/"), Err(AddressError::InsecureTransport)); +} + +// Requirement 1.1 applied to typed input: what is not an address is refused. +#[test] +fn refuses_junk() { + assert_eq!(session_address(""), Err(AddressError::Empty)); + assert_eq!(session_address(" "), Err(AddressError::Empty)); + assert_eq!(session_address("https:example.com"), Err(AddressError::MissingAuthority)); + assert_eq!(session_address("http:localhost"), Err(AddressError::MissingAuthority)); + assert_eq!(session_address("ftp://example.com"), Err(AddressError::UnsupportedScheme)); + assert_eq!(session_address("user@example.com"), Err(AddressError::UserInfo)); + assert_eq!(session_address("exa mple.com"), Err(AddressError::ForbiddenCharacter)); + assert_eq!(session_address("example.com\\jmap"), Err(AddressError::Backslash)); + assert_eq!(session_address("bücher.example.com"), Err(AddressError::ForbiddenCharacter)); + assert_eq!( + session_address("example.com/path?u=http://example.org"), + Ok(Address::parse("https://example.com/path?u=http://example.org").unwrap()) + ); +} diff --git a/crates/jmap/src/error.rs b/crates/jmap/src/error.rs new file mode 100644 index 000000000..4db0f26c6 --- /dev/null +++ b/crates/jmap/src/error.rs @@ -0,0 +1,412 @@ +//! The crate's error type, arranged by what the mail client can do about each case. + +use std::fmt; + +use crate::address::{Address, AddressError}; +use crate::credentials::CredentialsError; +use crate::http::TransportError; +use crate::id::IdError; +use crate::redact::Excerpt; +use crate::retry::RetryAfter; +use crate::template::TemplateError; + +/// The longest limit name kept from a server. +const MAX_LIMIT_NAME_LEN: usize = 64; + +/// The name of a JMAP limit a server says was exceeded (`maxSizeRequest`, `maxCallsInRequest`, …), checked to +/// be a short identifier (letters, digits and `_`). +#[derive(Clone, PartialEq, Eq, Hash)] +pub struct LimitName(String); + +impl LimitName { + /// Checks a limit name received from a server. + pub fn new(text: &str) -> Option { + if text.is_empty() || text.len() > MAX_LIMIT_NAME_LEN || !text.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') { + return None; + } + Some(Self(text.to_owned())) + } + + /// The name. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Whether the limit is about concurrency (`maxConcurrentRequests`, `maxConcurrentUpload`) rather than size. + pub fn is_concurrency(&self) -> bool { + self.0.starts_with("maxConcurrent") + } +} + +impl fmt::Display for LimitName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl fmt::Debug for LimitName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "LimitName({})", self.0) + } +} + +/// A request-level JMAP problem (RFC 8620 §3.6.1), the `urn:ietf:params:jmap:error:` prefix removed. +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RequestProblem { + /// `unknownCapability`: the request used a capability the server does not support. + UnknownCapability, + /// `notJSON`: the request was not `application/json` or not I-JSON. + NotJson, + /// `notRequest`: the request did not have the shape of a Request object. + NotRequest, + /// `limit`: a request limit was exceeded. + Limit, + /// Another JMAP problem type, checked to be a short identifier. + Other(String), +} + +impl fmt::Display for RequestProblem { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::UnknownCapability => f.write_str("it uses a capability this server does not support"), + Self::NotJson => f.write_str("the server could not read it as JSON"), + Self::NotRequest => f.write_str("its JSON is not shaped like a JMAP request"), + Self::Limit => f.write_str("it goes beyond one of the server's request limits"), + Self::Other(name) if name.is_empty() => f.write_str("the server reported an unnamed JMAP problem"), + Self::Other(name) => write!(f, "the server reported the JMAP problem {name}"), + } + } +} + +/// Why a server's successful-looking answer cannot be used. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Unusable { + /// The content type is not the one expected (`application/json`). + WrongContentType, + /// The body is larger than the request's limit ([`HttpRequest::max_response_bytes`](crate::HttpRequest::max_response_bytes): + /// the kind's, or the lower one the caller set), whether the crate or the transport found it. + BodyTooLarge, + /// The body is not valid JSON (RFC 8259), or nests too deeply. + InvalidJson, + /// The body holds more JSON values or objects than the crate accepts. + TooComplex, + /// The JSON does not have the expected shape; the text says which part, in the crate's own words. + Shape(&'static str), + /// An address the server gave cannot be used; the text names the field. + Address(&'static str, AddressError), + /// An address template the server gave cannot be used; the text names the field. + Template(&'static str, TemplateError), +} + +impl fmt::Display for Unusable { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::WrongContentType => f.write_str("its content type is not application/json"), + Self::BodyTooLarge => f.write_str("it is larger than the size limit set for this request"), + Self::InvalidJson => f.write_str("it is not valid JSON, or nests more than 128 levels deep"), + Self::TooComplex => f.write_str("it holds more JSON values or objects than this program reads at once"), + Self::Shape(what) => write!(f, "{what}"), + Self::Address(field, error) => write!(f, "its {field} cannot be used: {error}"), + Self::Template(field, error) => write!(f, "its {field} cannot be used: {error}"), + } + } +} + +/// A mistake in a request being built, found before anything is sent. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum BuildError { + /// A method name that is empty, too long, or not printable ASCII. + InvalidMethodName, + /// A capability identifier that is empty, too long, or not printable ASCII. + InvalidCapability, + /// A capability the session does not advertise. + UnsupportedCapability, + /// An argument given both plainly and as a result reference (RFC 8620 §3.7). + DuplicateArgument, + /// An argument name that is empty or starts with `#`. + InvalidArgumentName, + /// A result reference to a call that is not earlier in this request (RFC 8620 §3.7). + UnknownCall, + /// A result reference path that is not a JSON Pointer (RFC 6901, with JMAP's `*`). + InvalidPath, + /// More calls than the server accepts (`maxCallsInRequest`) or than the crate builds. + TooManyCalls { + /// The limit that applies. + limit: u64, + }, + /// A request body larger than the server accepts (`maxSizeRequest`). + TooLarge { + /// The limit that applies, in octets. + limit: u64, + }, + /// A request without any method call. + NoCalls, + /// The request could not be written as JSON. + Serialisation, +} + +impl fmt::Display for BuildError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidMethodName => f.write_str("the method name is empty, longer than 255 characters, or holds a space or non-ASCII character"), + Self::InvalidCapability => { + f.write_str("the capability identifier is empty, longer than 256 characters, or holds a space or non-ASCII character") + } + Self::UnsupportedCapability => { + f.write_str("the request uses a capability this server's session does not offer; remove it or check the server") + } + Self::DuplicateArgument => f.write_str("an argument is given both directly and as a result reference (#name); keep only one"), + Self::InvalidArgumentName => { + f.write_str("a result reference argument name is empty or starts with #; give the plain name, the # is added") + } + Self::UnknownCall => f.write_str("a result reference names a call that is not earlier in this same request"), + Self::InvalidPath => { + f.write_str("a result reference path is not a JSON pointer: it must be empty or start with /, and use ~ only as ~0 or ~1") + } + Self::TooManyCalls { limit } => { + write!(f, "the request holds more than {limit} method calls, the server's limit; send it in several requests") + } + Self::TooLarge { limit } => { + write!(f, "the request body is larger than the server's limit of {limit} octets; send it in several requests") + } + Self::NoCalls => f.write_str("the request holds no method call, so there is nothing to send"), + Self::Serialisation => f.write_str("the request could not be written out as JSON"), + } + } +} + +/// Everything that can go wrong, arranged by what the mail client can do about it. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum Error { + /// The program's HTTP exchange failed before a response came (offline, timeout, TLS, …). + Transport(TransportError), + /// The server is busy or limiting the rate (429, 503, connection-level overload, a concurrency limit): + /// try again later, after at least the delay it asked for when there is one. + Busy { + /// What the server asked for, if anything usable. + retry_after: Option, + }, + /// The credentials were refused (401, or 403 when the problem details say so): ask for them again. + CredentialsRefused, + /// The request was too large for the server (413, or a JMAP `limit` problem): split it. + TooLarge { + /// The limit the server named, if any. + limit: Option, + }, + /// The server refuses access to this resource (403). + Forbidden { + /// What the server said, cleaned. + detail: Option, + }, + /// Nothing at that address (404, 410). + NotFound, + /// The server rejected the request as malformed or unsupported (400 and other client errors, or a JMAP + /// request-level problem). + Refused { + /// The JMAP problem, if one was recognised. + problem: Option, + /// What the server said, cleaned. + detail: Option, + }, + /// The server redirected; the crate follows nothing. The target has passed the transport rule and the + /// loopback check; whether to follow (and whether to send the credentials to another origin) is the + /// caller's decision. + Redirected { + /// Where the server points. + target: Address, + /// Whether the target has the same origin as the request. + same_origin: bool, + }, + /// The server redirected to an address that may not be used (plain http, loopback from a remote host, + /// malformed). + RedirectRefused(AddressError), + /// The server answered something that cannot be used. + Unusable(Unusable), + /// Any other HTTP status. + Http { + /// The status code. + status: u16, + /// What the server said, cleaned. + detail: Option, + }, + /// An address given to the crate is not usable. + Address(AddressError), + /// An address template could not be expanded. + Template(TemplateError), + /// Credentials given to the crate are not usable. + Credentials(CredentialsError), + /// An id given to the crate is not valid. + Id(IdError), + /// A request being built is wrong. + Build(BuildError), + /// The session offers no account for mail (RFC 8621 `urn:ietf:params:jmap:mail`). + NoMailAccount, +} + +impl Error { + /// The delay before trying again, for [`Error::Busy`]. + pub fn retry_after(&self) -> Option<&RetryAfter> { + match self { + Self::Busy { retry_after } => retry_after.as_ref(), + _ => None, + } + } +} + +fn with_detail(f: &mut fmt::Formatter<'_>, text: &str, detail: Option<&Excerpt>) -> fmt::Result { + f.write_str(text)?; + if let Some(detail) = detail { + write!(f, " (server said: \"{detail}\")")?; + } + Ok(()) +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Transport(error) => write!(f, "the request did not complete: {error}"), + Self::Busy { + retry_after: Some(RetryAfter::Delay(delay)), + } => write!( + f, + "the server is busy and asks to wait {} s before the next attempt", + delay.as_secs() + ), + Self::Busy { + retry_after: Some(RetryAfter::At(_)), + } => f.write_str("the server is busy and named a time before which not to try again"), + Self::Busy { retry_after: None } => { + f.write_str("the server is busy or limiting requests; wait a little before trying again") + } + Self::CredentialsRefused => f.write_str( + "the server rejected the user name and password (or the access token) for this account; check them, or sign in again if they were changed or have expired", + ), + Self::TooLarge { limit: Some(limit) } => write!( + f, + "the request went beyond the server's limit {limit}; send it in smaller parts" + ), + Self::TooLarge { limit: None } => { + f.write_str("the request was larger than the server accepts; send it in smaller parts") + } + Self::Forbidden { detail } => with_detail( + f, + "the server does not let this account reach that resource", + detail.as_ref(), + ), + Self::NotFound => f.write_str( + "the server has nothing at that address; it may have been moved or deleted, so refresh and try again", + ), + Self::Refused { + problem: Some(problem), + detail, + } => with_detail( + f, + &format!("the server rejected the request: {problem}"), + detail.as_ref(), + ), + Self::Refused { + problem: None, + detail, + } => with_detail(f, "the server rejected the request as invalid", detail.as_ref()), + Self::Redirected { + target, + same_origin: true, + } => write!( + f, + "the server moved this resource to another path on {}; it was not followed automatically", + target.origin() + ), + Self::Redirected { target, .. } => write!( + f, + "the server sent this request on to {}; confirm before following, since the credentials would go there too", + target.origin() + ), + Self::RedirectRefused(error) => write!( + f, + "the server redirected to an address that cannot be used: {error}" + ), + Self::Unusable(why) => write!(f, "the server's answer cannot be used: {why}"), + Self::Http { status, detail } => with_detail( + f, + &format!("the server answered with HTTP status {status}, which this program cannot act on"), + detail.as_ref(), + ), + Self::Address(error) => fmt::Display::fmt(error, f), + Self::Template(error) => fmt::Display::fmt(error, f), + Self::Credentials(error) => fmt::Display::fmt(error, f), + Self::Id(error) => fmt::Display::fmt(error, f), + Self::Build(error) => fmt::Display::fmt(error, f), + Self::NoMailAccount => f.write_str( + "the server offers no mail account to these credentials; check that mail is enabled for this account", + ), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Transport(error) => Some(error), + Self::RedirectRefused(error) | Self::Address(error) => Some(error), + Self::Template(error) => Some(error), + Self::Credentials(error) => Some(error), + Self::Id(error) => Some(error), + _ => None, + } + } +} + +impl From for Error { + /// A connection-level overload is the server being busy; a body cut at the request's limit is the same + /// failure as one the crate refuses after reading; anything else stays a transport failure. + fn from(error: TransportError) -> Self { + match error { + TransportError::Overloaded => Self::Busy { retry_after: None }, + TransportError::BodyTooLarge => Self::Unusable(Unusable::BodyTooLarge), + other => Self::Transport(other), + } + } +} + +impl From for Error { + fn from(error: AddressError) -> Self { + Self::Address(error) + } +} + +impl From for Error { + fn from(error: TemplateError) -> Self { + Self::Template(error) + } +} + +impl From for Error { + fn from(error: CredentialsError) -> Self { + Self::Credentials(error) + } +} + +impl From for Error { + fn from(error: IdError) -> Self { + Self::Id(error) + } +} + +impl From for Error { + fn from(error: BuildError) -> Self { + Self::Build(error) + } +} + +impl From for Error { + fn from(why: Unusable) -> Self { + Self::Unusable(why) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/jmap/src/error/tests.rs b/crates/jmap/src/error/tests.rs new file mode 100644 index 000000000..3a451c91e --- /dev/null +++ b/crates/jmap/src/error/tests.rs @@ -0,0 +1,39 @@ +use super::*; +use std::time::Duration; + +// RFC 8620 §3.6.1: the limit property names the limit; only short identifiers are kept. +#[test] +fn limit_names() { + assert_eq!(LimitName::new("maxSizeRequest").unwrap().as_str(), "maxSizeRequest"); + assert!(LimitName::new("maxConcurrentRequests").unwrap().is_concurrency()); + assert!(!LimitName::new("maxCallsInRequest").unwrap().is_concurrency()); + for bad in ["", "max size", "x\ny", &"a".repeat(65), "", + "file:///etc/passwd", + "ms-msdt:/id", + // Protocol-relative and UNC references name another machine (NTLM leak on Windows). + "//attacker.example/share/x", + "\\\\attacker.example\\share\\x", + "/\\attacker.example\\x", + " //attacker.example/x", + // Characters the exporters drop on writing don't hide a scheme. + "file\u{85}:///etc/passwd", + "java\u{9F}script:alert(1)", + "/\u{FFFE}/attacker.example/x", + "", + ] { + assert!(!is_safe_href(bad), "{bad:?}"); + } + } +} diff --git a/crates/wordmodel/src/locale_number.rs b/crates/wordmodel/src/locale_number.rs new file mode 100644 index 000000000..336269f64 --- /dev/null +++ b/crates/wordmodel/src/locale_number.rs @@ -0,0 +1,262 @@ +//! Numbers written the way a language writes them: `1 250,50` in Polish, `1.250,50` in German, +//! `1,250.50` in English. Used by table sort and table formulas. + +use crate::collate::language; + +/// The decimal separator of language `lang` (a BCP 47 tag; `None` means English). +pub fn decimal_separator(lang: Option<&str>) -> char { + let tag = lang.unwrap_or("").trim().to_ascii_lowercase().replace('_', "-"); + // Regions that write a decimal point where the language usually writes a comma. + const POINT_REGIONS: &[&str] = + &["de-ch", "de-li", "it-ch", "es-mx", "es-us", "es-419", "es-pr", "es-do", "es-gt", "es-hn", "es-ni", "es-pa", "es-sv", "es-cu", "es-pe"]; + if POINT_REGIONS.iter().any(|r| tag == *r || tag.starts_with(&format!("{r}-"))) { + return '.'; + } + const COMMA: &[&str] = &[ + "af", "az", "be", "bg", "bs", "ca", "cs", "da", "de", "el", "es", "et", "eu", "fi", "fo", "fr", "gl", "hr", "hu", "hy", "id", "is", "it", + "ka", "kk", "ky", "lb", "lt", "lv", "mk", "mn", "nb", "nl", "nn", "no", "pl", "pt", "ro", "ru", "sk", "sl", "sq", "sr", "sv", "tr", "uk", + "uz", "vi", + ]; + if COMMA.contains(&language(lang).as_str()) { ',' } else { '.' } +} + +const SPACES: &[char] = &[' ', '\u{a0}', '\u{202f}', '\u{2009}']; +const CURRENCY: &[char] = &['$', '€', '£', '¥', '¢', '₽', '₴', '₺', '₹', '₩', '₪', '₫', '฿', '₱', '₸', '₼', '₾']; +/// Currency signs spelled with more than one character: the Panamanian balboa (`B/.`). +const CURRENCY_SYMBOLS: &[&str] = &["B/."]; + +/// Trim spaces and single-character currency signs from either end. +fn trim_currency(s: &str) -> &str { + s.trim_matches(|c: char| c.is_whitespace() || CURRENCY.contains(&c)) +} + +/// Strip a currency symbol or a short currency word (`zł`, `kr`, `PLN`, `B/.`) from either end. +/// As Word does, a multi-character sign is a word of its own (`B/. 1,250.50`, `1,250.50 B/.`); +/// `B/.1,250.50` and `B/ 1,250.50` are not numbers. +fn strip_currency(s: &str) -> &str { + let mut s = trim_currency(s); + if let Some(rest) = + CURRENCY_SYMBOLS.iter().find_map(|sym| s.strip_prefix(sym).filter(|rest| rest.chars().next().is_some_and(char::is_whitespace))) + { + s = trim_currency(rest); + } + if let Some(rest) = + CURRENCY_SYMBOLS.iter().find_map(|sym| s.strip_suffix(sym).filter(|rest| rest.chars().next_back().is_some_and(char::is_whitespace))) + { + s = trim_currency(rest); + } + if let Some((head, word)) = s.rsplit_once(SPACES) + && (1..=3).contains(&word.chars().count()) + && word.chars().all(|c| c.is_alphabetic() || c == '.') + { + s = head; + } + if let Some((word, tail)) = s.split_once(SPACES) + && word.chars().count() == 3 + && word.chars().all(|c| c.is_ascii_uppercase()) + { + s = tail; + } + s.trim_matches(|c: char| c.is_whitespace() || CURRENCY.contains(&c)) +} + +/// Digits of an integer part with group separators `groups`: the first group has 1–3 digits, +/// every later one exactly 3. +fn grouped_digits(int: &str, groups: &[char]) -> Option { + if int.chars().all(|c| c.is_ascii_digit()) { + return Some(int.to_string()); + } + let parts: Vec<&str> = int.split(groups).collect(); + let (first, rest) = parts.split_first()?; + // A first group of `0` is never grouping (`0.125` is a fraction, not 125). + let ok = (1..=3).contains(&first.len()) + && !first.starts_with('0') + && first.chars().all(|c| c.is_ascii_digit()) + && rest.iter().all(|g| g.len() == 3 && g.chars().all(|c| c.is_ascii_digit())); + ok.then(|| parts.concat()) +} + +/// Parse `s` as a number written in language `lang`, or `None` when it isn't one. The decimal +/// separator is the language's; group separators are spaces (and `.` or `'` where the decimal +/// separator is a comma, `,` or `'` where it is a point) between groups of three digits. A +/// currency sign or short currency word is ignored, and so is a trailing `%` (`12 %` is 12, as +/// table sort reads it). In comma languages a lone `.` that cannot be a group separator (`1.5`) +/// is read as a decimal point. A trailing separator with no digits after it (`1.`, an ordinal +/// in a numbered column) is the whole number. A minus may be `-`, `−` or an en dash. +pub fn parse_number(s: &str, lang: Option<&str>) -> Option { + let dec = decimal_separator(lang); + let s = s.trim(); + let s = s.strip_suffix('%').map_or(s, str::trim_end); + let mut body = strip_currency(s); + let mut neg = false; + if let Some(rest) = body.strip_prefix(['-', '\u{2212}', '\u{2013}']) { + neg = true; + body = strip_currency(rest); + } else if let Some(rest) = body.strip_prefix('+') { + body = strip_currency(rest); + } + if body.is_empty() || body.len() > 400 { + return None; + } + let mut groups: Vec = SPACES.to_vec(); + groups.extend(['\'', '\u{2019}']); + groups.push(if dec == ',' { '.' } else { ',' }); + let (int, frac) = match body.split_once(dec) { + Some((i, f)) => (i, Some(f)), + None => (body, None), + }; + let int_digits = match grouped_digits(int, &groups) { + Some(d) => d, + None if dec == ',' && frac.is_none() && int.matches('.').count() == 1 => { + return parse_number(&format!("{}{}", if neg { "-" } else { "" }, int.replace('.', ",")), lang); + } + None => return None, + }; + let frac_digits = match frac { + Some(f) if f.chars().all(|c| c.is_ascii_digit()) => f, + Some(_) => return None, + None => "", + }; + if int_digits.is_empty() && frac_digits.is_empty() { + return None; + } + let v: f64 = format!("0{int_digits}.{frac_digits}0").parse().ok()?; + if !v.is_finite() { + return None; + } + Some(if neg { -v } else { v }) +} + +/// Write `v` the way language `lang` writes a formula result: whole numbers without decimals, +/// others with two decimals and the language's decimal separator (`5,75` in Polish). +pub fn format_number(v: f64, lang: Option<&str>) -> String { + if !v.is_finite() { + return "0".into(); + } + if v.fract() == 0.0 && v.abs() < 9.2e18 { + return format!("{}", v as i64); + } + let s = format!("{v:.2}"); + let s = if s == "-0.00" { "0.00".to_string() } else { s }; + if decimal_separator(lang) == ',' { s.replace('.', ",") } else { s } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn polish_numbers() { + let pl = Some("pl-PL"); + assert_eq!(parse_number("1,5", pl), Some(1.5)); + assert_eq!(parse_number("3,50", pl), Some(3.5)); + assert_eq!(parse_number("1 250,00", pl), Some(1250.0)); + assert_eq!(parse_number("1\u{a0}250,00 zł", pl), Some(1250.0)); + assert_eq!(parse_number("1.250,00", pl), Some(1250.0)); + assert_eq!(parse_number("1.5", pl), Some(1.5)); + assert_eq!(parse_number("-2,25", pl), Some(-2.25)); + assert_eq!(parse_number("PLN 12", pl), Some(12.0)); + assert_eq!(parse_number("-B/. 1 250,50", pl), Some(-1250.5)); + assert_eq!(parse_number("abc", pl), None); + assert_eq!(parse_number("1,2,3", pl), None); + // From review: ordinals, a leading zero, percentages and an en dash minus. + assert_eq!(parse_number("1.", pl), Some(1.0), "a numbered column: 1."); + assert_eq!(parse_number("10.", pl), Some(10.0)); + assert_eq!(parse_number("0.125", pl), Some(0.125), "a leading 0 is never a group"); + assert_eq!(parse_number("0,125", pl), Some(0.125)); + assert_eq!(parse_number("12 %", pl), Some(12.0)); + assert_eq!(parse_number("12%", pl), Some(12.0)); + assert_eq!(parse_number("\u{2013}2,5", pl), Some(-2.5)); + assert_eq!(parse_number("%", pl), None); + assert_eq!(parse_number(".", pl), None); + assert_eq!(format_number(5.75, pl), "5,75"); + assert_eq!(format_number(1350.0, pl), "1350"); + assert_eq!(format_number(-0.001, pl), "0,00", "a rounded zero has no sign"); + } + + #[test] + fn overflowing_digit_strings_are_not_numbers() { + assert_eq!(parse_number(&"1".repeat(350), None), None); + assert_eq!(parse_number(&"9".repeat(400), Some("pl-PL")), None); + assert_eq!(parse_number(&format!("-{}", "1".repeat(350)), None), None); + let big = parse_number(&"1".repeat(300), None); + assert!(big.is_some_and(f64::is_finite), "{big:?}"); + } + + #[test] + fn cuban_and_peruvian_numbers_use_decimal_points() { + for lang in [Some("es-CU"), Some("es-PE")] { + assert_eq!(parse_number("1.234", lang), Some(1.234)); + assert_eq!(parse_number("1,234.50", lang), Some(1234.5)); + assert_eq!(format_number(5.75, lang), "5.75"); + } + assert_eq!(parse_number("1.234,50", Some("es-ES")), Some(1234.5)); + assert_eq!(format_number(5.75, Some("es-ES")), "5,75"); + } + + #[test] + fn balboa_numbers() { + let en = Some("en-US"); + // Word 16.0 reads `B/.` as a word of its own, before or after the number. + assert_eq!(parse_number("B/. 1,234.50", en), Some(1234.5)); + assert_eq!(parse_number("1,234.50 B/.", en), Some(1234.5)); + assert_eq!(parse_number("B/.\u{a0}1,234.50", en), Some(1234.5), "a no-break space after the sign"); + assert_eq!(parse_number("B/. 1,234.50 B/.", en), Some(1234.5), "the sign at both ends"); + assert_eq!(parse_number("-B/. 1,234.50", en), Some(-1234.5), "a minus before the sign"); + assert_eq!(parse_number("B/. 1,234.50", Some("es-PA")), Some(1234.5), "the balboa's own region"); + // The sign has no space of its own in Word: touching the number is text. + assert_eq!(parse_number("B/.1,234.50", en), None); + assert_eq!(parse_number("1,234.50B/.", en), None); + assert_eq!(parse_number("B/ 1,234.50", en), None, "the balboa is B/. in Word"); + assert_eq!(parse_number("B/.", en), None); + // The slash and the full stop make `B/.` ambiguous with a fraction: the sign is matched at + // the ends of the text only, so a fraction is never read as a balboa amount and a sign in + // the middle is not a sign. + assert_eq!(parse_number("1/.5", en), None, "a fraction is not a price"); + assert_eq!(parse_number("1/.5", Some("es-PA")), None); + assert_eq!(parse_number("1/.5 B/.", en), None, "the sign at the end does not rescue a fraction"); + assert_eq!(parse_number("B/. 1/.5", en), None); + assert_eq!(parse_number("2 B/. 1,234.50", en), None, "the sign sits before the number"); + assert_eq!(parse_number("1,234.50 B/. 1,234.50", en), None, "the sign sits between two numbers"); + assert_eq!(parse_number("1,234.50/2", en), None, "a division is not a balboa amount"); + assert_eq!(parse_number("B/. 1,234.50,50", en), None, "a second decimal separator"); + } + + #[test] + fn english_numbers() { + let en = Some("en-US"); + assert_eq!(parse_number("1,250.50", en), Some(1250.5)); + assert_eq!(parse_number("$1,250", en), Some(1250.0)); + assert_eq!(parse_number("PAB 1,250.50", en), Some(1250.5), "the ISO code"); + assert_eq!(parse_number("1.5", en), Some(1.5)); + assert_eq!(parse_number("12", None), Some(12.0)); + assert_eq!(parse_number(".5", None), Some(0.5)); + assert_eq!(parse_number("1,5", en), None); + assert_eq!(parse_number("1.", en), Some(1.0)); + assert_eq!(parse_number("0,125", en), None, "never 125"); + assert_eq!(format_number(5.75, en), "5.75"); + assert_eq!(format_number(-0.001, en), "0.00", "a rounded zero has no sign"); + assert_eq!(format_number(-0.0, None), "0"); + assert_eq!(decimal_separator(Some("de-CH")), '.'); + assert_eq!(decimal_separator(Some("de-DE")), ','); + } + + #[test] + fn hostile_numbers_never_panic() { + for s in ["", "-", "+", ",", ".", "1e400", "inf", "NaN", "--1", "₽", "1 ,", " zł", "\u{2212}", "9".repeat(500).as_str()] { + for l in [None, Some("pl"), Some("en"), Some("")] { + let _ = parse_number(s, l); + } + } + assert_eq!(format_number(f64::NAN, None), "0"); + } + + #[test] + fn large_whole_numbers_have_no_decimals() { + assert_eq!(format_number(1e15, None), "1000000000000000"); + assert_eq!(format_number(2.5e15, None), "2500000000000000"); + assert_eq!(format_number(-1e15, None), "-1000000000000000"); + assert_eq!(format_number(9.0e18, None), "9000000000000000000"); + assert_eq!(format_number(1e15 + 0.5, None), "1000000000000000.50"); + } +} diff --git a/crates/wordmodel/src/math.rs b/crates/wordmodel/src/math.rs new file mode 100644 index 000000000..db9875fdb --- /dev/null +++ b/crates/wordmodel/src/math.rs @@ -0,0 +1,1138 @@ +//! Equations: the Office Math object model (OMML, ECMA-376 Part 1 §22.1) as a tree, its linear +//! format (UnicodeMath, the plain text Word shows for an equation) and a parser for that format. +//! +//! An equation is a list of [`MNode`]s (an *argument*); structures hold arguments of their own +//! (a fraction's numerator and denominator, a radical's degree and base…). Layout reads the tree; +//! DOCX import builds it from OMML and keeps the source XML so a saved file carries the equation +//! exactly as Word wrote it. + +use serde::{Deserialize, Serialize}; + +use crate::props::Rgb; + +/// A sequence of math nodes (OMML's `m:e`, `m:num`, `m:sub`…). +pub type Arg = Vec; + +/// Deepest structure nesting kept (parsers stop here; layout draws nothing deeper). +pub const MAX_DEPTH: usize = 64; + +/// Most matrix columns kept (writers pad every row to the widest, so this bounds a row's cost). +pub const MAX_MATRIX_COLS: usize = 64; + +/// Most matrix rows kept. +pub const MAX_MATRIX_ROWS: usize = 1000; + +/// An equation's structure, plus what's needed to write it back. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +pub struct Math { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub nodes: Arg, + /// The source `m:oMath` element (DOCX import), written back unchanged on save. Empty for + /// equations made here. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub omml: String, + /// Justification of a display equation. + #[serde(default, skip_serializing_if = "is_default")] + pub jc: MathJc, + /// Shown as its linear-format text (Equation › Convert › Linear) instead of built up. + #[serde(default, skip_serializing_if = "is_default")] + pub linear: bool, +} + +fn is_default(v: &T) -> bool { + *v == T::default() +} + +impl Math { + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// Rough heap bytes a clone copies: every node, its argument lists and its run text. + /// Iterative, so it costs at most the size of the structure. + pub fn heap_bytes(&self) -> usize { + let mut n = self.omml.len(); + let mut todo: Vec<&MNode> = self.nodes.iter().collect(); + while let Some(node) = todo.pop() { + n = n.saturating_add(std::mem::size_of::()); + if let MNode::Run(r) = node { + n = n.saturating_add(r.text.len()).saturating_add(r.font.as_ref().map_or(0, String::len)); + } + each_arg(node, &mut |a| { + n = n.saturating_add(std::mem::size_of::()); + todo.extend(a); + }); + } + n + } +} + +/// Display equation justification (`m:oMathParaPr/m:jc`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MathJc { + #[default] + CenterGroup, + Center, + Left, + Right, +} + +/// Where a display equation too wide for its line breaks at a binary operator (`m:brkBin`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum BrkBin { + /// The operator starts the next line. + #[default] + Before, + /// The operator ends the line. + After, + /// The operator ends the line and starts the next one again. + Repeat, +} + +/// How a minus repeats across a break when [`BrkBin::Repeat`] (`m:brkBinSub`): the operator +/// ending the line, then the one starting the next. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum BrkBinSub { + /// `--` + #[default] + MinusMinus, + /// `-+` + MinusPlus, + /// `+-` + PlusMinus, +} + +/// Document-wide equation options (`m:mathPr` in the settings part) that layout uses: how long +/// display equations wrap. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct MathProps { + pub brk_bin: BrkBin, + pub brk_bin_sub: BrkBinSub, + /// Indent of a wrapped equation's continuation lines, points (`m:wrapIndent`). + pub wrap_indent: f32, + /// Continuation lines align right instead (`m:wrapRight`). + pub wrap_right: bool, +} + +/// `m:wrapIndent`'s default: 1440 twips, one inch. +pub const DEFAULT_WRAP_INDENT: f32 = 72.0; + +impl Default for MathProps { + fn default() -> Self { + MathProps { brk_bin: BrkBin::Before, brk_bin_sub: BrkBinSub::MinusMinus, wrap_indent: DEFAULT_WRAP_INDENT, wrap_right: false } + } +} + +/// Math run style (`m:sty`): plain, bold, italic, bold italic. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MSty { + Plain, + Bold, + Italic, + BoldItalic, +} + +/// Math alphabet (`m:scr`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum MScr { + #[default] + Roman, + Script, + Fraktur, + DoubleStruck, + SansSerif, + Monospace, +} + +/// A run of math text (`m:r`). +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +pub struct MRun { + pub text: String, + /// `None`: math default (letters italic, the rest upright). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sty: Option, + #[serde(default, skip_serializing_if = "is_default")] + pub scr: MScr, + /// Normal (non-math) text. + #[serde(default, skip_serializing_if = "is_default")] + pub nor: bool, + /// Literal: no automatic operator handling. + #[serde(default, skip_serializing_if = "is_default")] + pub lit: bool, + /// Font size, points. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, + /// Font for normal text. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub font: Option, + /// A manual line break before this run in a display equation (`m:brk`): `Some(0)` wraps as + /// usual, `Some(n)` aligns the new line at the `n`th operator of the first line (`m:alnAt`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub brk: Option, +} + +impl MRun { + pub fn new(text: impl Into) -> MRun { + MRun { text: text.into(), ..Default::default() } + } + pub fn plain(text: impl Into) -> MRun { + MRun { text: text.into(), sty: Some(MSty::Plain), ..Default::default() } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FracKind { + /// Stacked with a rule. + #[default] + Bar, + /// Skewed (num ⁄ den). + Skewed, + /// Linear (num/den). + Linear, + /// Stacked without a rule. + NoBar, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ScriptKind { + Sup, + Sub, + SubSup, + /// Pre-scripts (`m:sPre`). + Pre, +} + +/// Where an n-ary operator's limits go. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum LimLoc { + /// Under and over the operator. + UndOvr, + /// As sub- and superscripts. + SubSup, +} + +/// Column justification in a matrix. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ColJc { + #[default] + Center, + Left, + Right, +} + +/// A math node. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "t", rename_all = "camelCase")] +pub enum MNode { + Run(MRun), + /// `m:f`. + Frac { + #[serde(default)] + kind: FracKind, + num: Arg, + den: Arg, + }, + /// `m:sSup`, `m:sSub`, `m:sSubSup`, `m:sPre`. + Script { + kind: ScriptKind, + base: Arg, + #[serde(default)] + sub: Arg, + #[serde(default)] + sup: Arg, + }, + /// `m:rad`. + Rad { + #[serde(default)] + deg: Arg, + #[serde(default)] + deg_hide: bool, + e: Arg, + }, + /// `m:nary`: ∑ ∫ ∏ … with limits and an operand. + Nary { + chr: char, + /// `None`: Word's default (integrals take sub/sup limits, the rest under/over). + #[serde(default)] + lim_loc: Option, + #[serde(default)] + grow: bool, + #[serde(default)] + sub_hide: bool, + #[serde(default)] + sup_hide: bool, + #[serde(default)] + sub: Arg, + #[serde(default)] + sup: Arg, + e: Arg, + }, + /// `m:d`: delimiters around elements separated by `sep`. + Delim { + /// `None` = no delimiter. + beg: Option, + end: Option, + #[serde(default)] + sep: Option, + /// Delimiters grow to the content (Word's default). + #[serde(default = "yes")] + grow: bool, + /// Grow to match the content exactly rather than centred on the math axis. + #[serde(default)] + shp_match: bool, + elems: Vec, + }, + /// `m:func`: function name applied to an argument (sin x). + Func { + name: Arg, + e: Arg, + }, + /// `m:limLow` / `m:limUpp`. + Lim { + upper: bool, + e: Arg, + lim: Arg, + }, + /// `m:acc`: accent over the base. + Acc { + chr: char, + e: Arg, + }, + /// `m:bar`: overbar or underbar. + Bar { + top: bool, + e: Arg, + }, + /// `m:borderBox`. `hide`: top, bottom, left, right. `strike`: horizontal, vertical, + /// bottom-left→top-right, top-left→bottom-right. + BorderBox { + #[serde(default)] + hide: [bool; 4], + #[serde(default)] + strike: [bool; 4], + e: Arg, + }, + /// `m:box`. + Boxed { + e: Arg, + }, + /// `m:groupChr`: a stretched character (brace, arrow) under or over the base. + GroupChr { + chr: char, + top: bool, + e: Arg, + }, + /// `m:eqArr`: rows aligned at `&`. + EqArr { + rows: Vec, + }, + /// `m:m`. + Matrix { + rows: Vec>, + #[serde(default)] + col_jc: Vec, + }, + /// `m:phant`. + Phant { + #[serde(default = "yes")] + show: bool, + #[serde(default)] + zero_wid: bool, + #[serde(default)] + zero_asc: bool, + #[serde(default)] + zero_desc: bool, + e: Arg, + }, +} + +fn yes() -> bool { + true +} + +/// A row holds an equation-number mark (`#`) in a top-level run. +pub fn has_number_mark(row: &[MNode]) -> bool { + row.iter().any(|n| matches!(n, MNode::Run(r) if !r.lit && !r.nor && r.text.contains('#'))) +} + +/// Integral signs (their limits default to sub/sup positions). +pub fn is_integral(c: char) -> bool { + matches!(c, '∫' | '∬' | '∭' | '∮' | '∯' | '∰' | '∱' | '∲' | '∳' | '⨌') +} + +/// Characters that take n-ary operator form in the linear format. +pub fn is_nary_char(c: char) -> bool { + is_integral(c) || matches!(c, '∑' | '∏' | '∐' | '⋃' | '⋂' | '⋁' | '⋀' | '⨁' | '⨂' | '⨀' | '⨄' | '⨆') +} + +/// Mathematical alphanumeric for `c` in a style and alphabet (Unicode block U+1D400). +pub fn math_alnum(c: char, bold: bool, italic: bool, scr: MScr) -> char { + let off = |base: u32, k: u32| char::from_u32(base + k).unwrap_or(c); + let upper = c.is_ascii_uppercase(); + let lower = c.is_ascii_lowercase(); + if upper || lower { + let k = if upper { c as u32 - 'A' as u32 } else { c as u32 - 'a' as u32 }; + // Letters that live in the Letterlike Symbols block. + let hole = |table: &[(char, char)]| table.iter().find(|(f, _)| *f == c).map(|(_, t)| *t); + let (base_u, base_l) = match (scr, bold, italic) { + (MScr::Roman, false, false) => return c, + (MScr::Roman, true, false) => (0x1D400, 0x1D41A), + (MScr::Roman, false, true) => { + if c == 'h' { + return '\u{210E}'; + } + (0x1D434, 0x1D44E) + } + (MScr::Roman, true, true) => (0x1D468, 0x1D482), + (MScr::Script, false, _) => { + if let Some(t) = hole(&[ + ('B', 'ℬ'), + ('E', 'ℰ'), + ('F', 'ℱ'), + ('H', 'ℋ'), + ('I', 'ℐ'), + ('L', 'ℒ'), + ('M', 'ℳ'), + ('R', 'ℛ'), + ('e', 'ℯ'), + ('g', 'ℊ'), + ('o', 'ℴ'), + ]) { + return t; + } + (0x1D49C, 0x1D4B6) + } + (MScr::Script, true, _) => (0x1D4D0, 0x1D4EA), + (MScr::Fraktur, false, _) => { + if let Some(t) = hole(&[('C', 'ℭ'), ('H', 'ℌ'), ('I', 'ℑ'), ('R', 'ℜ'), ('Z', 'ℨ')]) { + return t; + } + (0x1D504, 0x1D51E) + } + (MScr::Fraktur, true, _) => (0x1D56C, 0x1D586), + (MScr::DoubleStruck, _, _) => { + if let Some(t) = hole(&[('C', 'ℂ'), ('H', 'ℍ'), ('N', 'ℕ'), ('P', 'ℙ'), ('Q', 'ℚ'), ('R', 'ℝ'), ('Z', 'ℤ')]) { + return t; + } + (0x1D538, 0x1D552) + } + (MScr::SansSerif, false, false) => (0x1D5A0, 0x1D5BA), + (MScr::SansSerif, true, false) => (0x1D5D4, 0x1D5EE), + (MScr::SansSerif, false, true) => (0x1D608, 0x1D622), + (MScr::SansSerif, true, true) => (0x1D63C, 0x1D656), + (MScr::Monospace, _, _) => (0x1D670, 0x1D68A), + }; + return off(if upper { base_u } else { base_l }, k); + } + if c.is_ascii_digit() { + let k = c as u32 - '0' as u32; + return match (scr, bold) { + (MScr::Roman, true) => off(0x1D7CE, k), + (MScr::DoubleStruck, _) => off(0x1D7D8, k), + (MScr::SansSerif, false) => off(0x1D7E2, k), + (MScr::SansSerif, true) => off(0x1D7EC, k), + (MScr::Monospace, _) => off(0x1D7F6, k), + _ => c, + }; + } + // Greek (roman alphabet only). + if scr == MScr::Roman && (bold || italic) { + let (cap, small) = match (bold, italic) { + (true, false) => (0x1D6A8, 0x1D6C2), + (false, true) => (0x1D6E2, 0x1D6FC), + _ => (0x1D71C, 0x1D736), + }; + let u = c as u32; + if (0x391..=0x3A9).contains(&u) && u != 0x3A2 { + return off(cap, u - 0x391); + } + if (0x3B1..=0x3C9).contains(&u) { + return off(small, u - 0x3B1); + } + let extra = match c { + '∂' => Some(25), + 'ϵ' => Some(26), + 'ϑ' => Some(27), + 'ϰ' => Some(28), + 'ϕ' => Some(29), + 'ϱ' => Some(30), + 'ϖ' => Some(31), + _ => None, + }; + if let Some(e) = extra + && italic + { + return off(small, e); + } + } + c +} + +// --------------------------------------------------------------------------------------------- +// Linear format out. + +/// The equation as linear-format text (`x=(-b±√(b^2-4ac))/2a`). +pub fn to_linear(nodes: &[MNode]) -> String { + let mut s = String::new(); + lin_arg(nodes, &mut s, 0); + s +} + +fn lin_arg(a: &[MNode], s: &mut String, depth: usize) { + if depth > MAX_DEPTH { + return; + } + for (i, n) in a.iter().enumerate() { + lin_node(n, s, depth + 1); + // A bare script or limit ends at a space: a letter, digit or period after it is its own item. + if matches!(n, MNode::Script { kind, .. } if *kind != ScriptKind::Pre) || matches!(n, MNode::Lim { .. }) { + let ends_bare = s.chars().next_back().is_some_and(|c| c.is_alphanumeric() || c == '.'); + if ends_bare && a.get(i + 1).and_then(first_lin_char).is_some_and(|c| c.is_alphanumeric() || c == '.') { + s.push(' '); + } + } + } +} + +/// The first character `n` is written with, when that is cheap to tell (letters and digits). +fn first_lin_char(n: &MNode) -> Option { + match n { + MNode::Run(r) => r.text.chars().next(), + MNode::Func { name, .. } => name.first().and_then(first_lin_char), + MNode::Script { kind, base, .. } if *kind != ScriptKind::Pre => base.first().and_then(first_lin_char), + MNode::Lim { e, .. } => e.first().and_then(first_lin_char), + MNode::Frac { num, .. } => num.first().and_then(first_lin_char), + _ => None, + } +} + +/// An argument as an operand: bare when it's a single simple atom, parenthesised otherwise. +fn lin_operand(a: &[MNode], s: &mut String, depth: usize) { + let mut inner = String::new(); + lin_arg(a, &mut inner, depth); + let simple = match a { + [MNode::Run(r)] => { + let mut cs = r.text.chars(); + match (cs.next(), cs.next()) { + (Some(c), None) => !"+-−=/^_ ".contains(c), + _ => r.text.chars().all(|c| c.is_alphanumeric() || c == '.'), + } + } + [MNode::Delim { .. }] => true, + _ => false, + }; + if simple && !inner.is_empty() { + s.push_str(&inner); + } else { + s.push('('); + s.push_str(&inner); + s.push(')'); + } +} + +/// A script or limit operand. The parser reads a run of letters and digits up to the next space or +/// operator, so `V_dd` is written bare as Word's Linear format shows it (`V_(dd)` reads back the same). +fn lin_script(a: &[MNode], s: &mut String, depth: usize) { + // Bare only when the parser reads the whole run back, in the same style. + if let [MNode::Run(r)] = a + && r.text.chars().nth(1).is_some() + { + let upright_name = r.sty == Some(MSty::Plain) && crate::math_linear::FUNCTION_NAMES.contains(&r.text.as_str()); + let number = r.text.starts_with(|c: char| c.is_ascii_digit()) && r.text.chars().all(|c| c.is_alphanumeric() || c == '.'); + let word = r.text.starts_with(char::is_alphabetic) && r.text.chars().all(char::is_alphanumeric); + let upright_other = r.sty == Some(MSty::Plain) && !upright_name; + if upright_other || (!number && !word) { + s.push('('); + s.push_str(&r.text); + s.push(')'); + return; + } + } + lin_operand(a, s, depth); +} + +fn lin_node(n: &MNode, s: &mut String, depth: usize) { + match n { + MNode::Run(r) => s.push_str(&r.text), + MNode::Frac { kind, num, den } => { + lin_operand(num, s, depth); + s.push(match kind { + FracKind::Skewed => '⁄', + FracKind::Linear => '∕', + FracKind::NoBar => '¦', + FracKind::Bar => '/', + }); + lin_operand(den, s, depth); + } + MNode::Script { kind, base, sub, sup } => { + if *kind == ScriptKind::Pre { + s.push('_'); + lin_script(sub, s, depth); + s.push('^'); + lin_script(sup, s, depth); + s.push('▒'); + lin_operand(base, s, depth); + return; + } + lin_operand(base, s, depth); + if matches!(kind, ScriptKind::Sub | ScriptKind::SubSup) { + s.push('_'); + lin_script(sub, s, depth); + } + if matches!(kind, ScriptKind::Sup | ScriptKind::SubSup) { + s.push('^'); + lin_script(sup, s, depth); + } + } + MNode::Rad { deg, deg_hide, e } => { + if *deg_hide || deg.is_empty() { + s.push('√'); + lin_operand(e, s, depth); + } else { + s.push_str("√("); + lin_arg(deg, s, depth); + s.push('&'); + lin_arg(e, s, depth); + s.push(')'); + } + } + MNode::Nary { chr, sub_hide, sup_hide, sub, sup, e, .. } => { + s.push(*chr); + if !sub_hide && !sub.is_empty() { + s.push('_'); + lin_script(sub, s, depth); + } + if !sup_hide && !sup.is_empty() { + s.push('^'); + lin_script(sup, s, depth); + } + s.push('▒'); + lin_operand(e, s, depth); + } + MNode::Delim { beg, end, elems, .. } => { + // A standard pair writes as itself; anything else uses explicit `├X … Y┤`. + let pair = matches!((beg, end), (Some(b), Some(e)) if crate::math_linear::closer(*b) == Some(*e)); + let invisible = beg.is_none() && end.is_none(); + if pair { + s.extend(*beg); + } else if invisible { + s.push('〖'); + } else { + s.push('├'); + s.extend(*beg); + } + for (i, e) in elems.iter().enumerate() { + if i > 0 { + s.push('│'); + } + lin_arg(e, s, depth); + } + if pair { + s.extend(*end); + } else if invisible { + s.push('〗'); + } else { + s.extend(*end); + s.push('┤'); + } + } + MNode::Func { name, e } => { + lin_arg(name, s, depth); + s.push('\u{2061}'); + lin_operand(e, s, depth); + } + MNode::Lim { upper, e, lim } => { + lin_operand(e, s, depth); + s.push(if *upper { '┴' } else { '┬' }); + lin_script(lim, s, depth); + } + MNode::Acc { chr, e } => { + lin_operand(e, s, depth); + s.push(*chr); + } + MNode::Bar { top, e } => { + s.push(if *top { '¯' } else { '▁' }); + lin_operand(e, s, depth); + } + MNode::BorderBox { e, .. } => { + s.push('▭'); + lin_operand(e, s, depth); + } + MNode::Boxed { e } => { + s.push('□'); + lin_operand(e, s, depth); + } + MNode::GroupChr { chr, e, .. } => { + s.push(*chr); + lin_operand(e, s, depth); + } + // A numbered equation (`E=mc^2#(1)`) is written as its one row. + MNode::EqArr { rows } if rows.len() == 1 && rows.first().is_some_and(|r| has_number_mark(r)) => { + if let Some(r) = rows.first() { + lin_arg(r, s, depth); + } + } + MNode::EqArr { rows } => { + s.push_str("█("); + for (i, r) in rows.iter().enumerate() { + if i > 0 { + s.push('@'); + } + lin_arg(r, s, depth); + } + s.push(')'); + } + MNode::Matrix { rows, .. } => { + s.push_str("■("); + for (i, r) in rows.iter().enumerate() { + if i > 0 { + s.push('@'); + } + for (j, c) in r.iter().enumerate() { + if j > 0 { + s.push('&'); + } + lin_arg(c, s, depth); + } + } + s.push(')'); + } + MNode::Phant { e, .. } => { + s.push('⟡'); + lin_operand(e, s, depth); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Linear format in. + +/// Parse linear-format text into nodes ([`crate::math_linear`]). Never fails: what isn't +/// understood stays as text. +pub fn parse_linear(s: &str) -> Arg { + crate::math_linear::parse(s) +} + +pub use crate::math_linear::FUNCTION_NAMES; + +/// Spacing class of a math character (TeX's atom classes). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum MClass { + #[default] + Ord, + Op, + Bin, + Rel, + Open, + Close, + Punct, + Inner, +} + +/// The spacing class of a character in an equation. +pub fn math_class(c: char) -> MClass { + match c { + '+' | '−' | '-' | '±' | '∓' | '×' | '÷' | '·' | '∙' | '∗' | '*' | '∘' | '⊕' | '⊖' | '⊗' | '⊘' | '⊙' | '⊚' | '⊛' | '∪' | '∩' | '∧' | '∨' + | '∖' | '⋅' | '⋆' | '⊎' | '⊓' | '⊔' | '⋄' | '⊞' | '⊟' | '⊠' | '⊡' | '⋉' | '⋊' | '⋋' | '⋌' | '⋎' | '⋏' | '⋒' | '⋓' | '∔' | '∸' | '≀' | '⨿' + | '⊼' | '⊻' | '⊽' => MClass::Bin, + '=' | '<' | '>' | '≤' | '≥' | '≠' | '≈' | '≡' | '≢' | '∼' | '≃' | '≅' | '≐' | '∝' | '≪' | '≫' | '≺' | '≻' | '≼' | '≽' | '⊂' | '⊃' | '⊆' + | '⊇' | '⊄' | '⊅' | '⊈' | '⊉' | '⊊' | '⊋' | '∈' | '∉' | '∋' | '∌' | '⊥' | '∥' | '∦' | '∣' | '∤' | '≔' | '≕' | '≝' | '≜' | '≞' | '≟' | '→' + | '←' | '↔' | '⇒' | '⇐' | '⇔' | '↦' | '⟼' | '⟶' | '⟵' | '⟷' | '⟹' | '⟸' | '⟺' | '↑' | '↓' | '↕' | '⇑' | '⇓' | '⇕' | '↗' | '↘' | '↙' | '↖' + | '⊢' | '⊣' | '⊨' | '⊩' | '⊪' | '⊬' | '⊭' | '⊮' | '⊯' | '≲' | '≳' | '≍' | '⊏' | '⊐' | '⊑' | '⊒' | '⋢' | '⋣' | '≶' | '≷' | '≮' | '≯' | '≰' + | '≱' | '≦' | '≧' | '⩽' | '⩾' | ':' | '∶' | '∷' | '∺' | '∻' | '≗' | '≙' | '≚' | '≑' | '≒' | '≓' | '≖' | '⇌' | '⇋' | '⇄' | '⇆' | '⇇' | '⇉' + | '⇈' | '⇊' | '↪' | '↩' | '↼' | '↽' | '⇀' | '⇁' | '↿' | '↾' | '⇃' | '⇂' | '↶' | '↷' | '↺' | '↻' | '⊸' | '↜' | '↝' | '↞' | '↠' | '↢' | '↣' + | '↫' | '↬' | '↭' | '⇚' | '⇛' | '↰' | '↱' | '⇝' | '≁' | '≄' | '≉' | '≇' | '≭' | '≨' | '≩' | '⊀' | '⊁' | '⋠' | '⋡' | '⋦' | '⋧' | '⋨' | '⋩' + | '⋪' | '⋫' | '⋬' | '⋭' | '⋈' | '∴' | '∵' | '≏' | '≎' | '⋘' | '⋙' | '⊲' | '⊳' | '⊴' | '⊵' | '⋐' | '⋑' => { + MClass::Rel + } + '(' | '[' | '{' | '⟨' | '⌈' | '⌊' | '⟦' | '〈' => MClass::Open, + ')' | ']' | '}' | '⟩' | '⌉' | '⌋' | '⟧' | '〉' | '!' => MClass::Close, + ',' | ';' => MClass::Punct, + c if is_nary_char(c) => MClass::Op, + _ => MClass::Ord, + } +} + +/// Merge adjacent runs with the same formatting; recurse into every structure. +pub fn merge_runs(a: Arg) -> Arg { + let mut out: Arg = Vec::with_capacity(a.len()); + for n in a { + let n = match n { + MNode::Run(r) => { + if let Some(MNode::Run(prev)) = out.last_mut() + && prev.sty == r.sty + && prev.scr == r.scr + && prev.nor == r.nor + && prev.lit == r.lit + && prev.size == r.size + && prev.color == r.color + && prev.font == r.font + && r.brk.is_none() + { + prev.text.push_str(&r.text); + continue; + } + MNode::Run(r) + } + other => crate::math_linear::map_args(other, &merge_runs), + }; + out.push(n); + } + out +} + +/// Deepest node (a leaf counts 1) the parsers wrap further. Half of [`MAX_DEPTH`]: serde_json +/// stops reading at 128 nested containers and a node costs two (object and array). +pub const WRAP_DEPTH: usize = MAX_DEPTH / 2; + +/// Calls `f` with every argument of `n`. +fn each_arg<'a>(n: &'a MNode, f: &mut impl FnMut(&'a [MNode])) { + match n { + MNode::Run(_) => {} + MNode::Frac { num, den, .. } => { + f(num); + f(den); + } + MNode::Script { base, sub, sup, .. } => { + f(base); + f(sub); + f(sup); + } + MNode::Rad { deg, e, .. } => { + f(deg); + f(e); + } + MNode::Nary { sub, sup, e, .. } => { + f(sub); + f(sup); + f(e); + } + MNode::Delim { elems, .. } => elems.iter().for_each(|a| f(a)), + MNode::Func { name, e } => { + f(name); + f(e); + } + MNode::Lim { e, lim, .. } => { + f(e); + f(lim); + } + MNode::Acc { e, .. } + | MNode::Bar { e, .. } + | MNode::BorderBox { e, .. } + | MNode::Boxed { e } + | MNode::GroupChr { e, .. } + | MNode::Phant { e, .. } => f(e), + MNode::EqArr { rows } => rows.iter().for_each(|a| f(a)), + MNode::Matrix { rows, .. } => rows.iter().flatten().for_each(|a| f(a)), + } +} + +/// `n` is already nested too deeply to be wrapped in another structure. The parsers use this so +/// `x^1^1^1…` stays shallow (a left-nested tree 10 000 deep overflows the stack in `Clone`, +/// `Drop`, `PartialEq` and serde). Iterative, stops at the first too-deep path; costs at most the +/// size of `n`, which the parsers' input caps bound. +pub fn too_deep(n: &MNode) -> bool { + let mut stack: Vec<(&MNode, usize)> = vec![(n, 1)]; + while let Some((n, d)) = stack.pop() { + if d >= WRAP_DEPTH { + return true; + } + each_arg(n, &mut |a| stack.extend(a.iter().map(|c| (c, d + 1)))); + } + false +} + +/// Any of `a` is too deep to be wrapped ([`too_deep`]). +pub fn any_too_deep(a: &[MNode]) -> bool { + a.iter().any(too_deep) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(t: &str) -> MNode { + MNode::Run(MRun::new(t)) + } + + #[test] + fn quadratic_formula_parses_into_structures() { + let n = parse_linear("x=(-b±√(b^2-4ac))/2a"); + assert_eq!(n.len(), 2, "{n:#?}"); + assert_eq!(n[0], run("x=")); + let MNode::Frac { num, den, .. } = &n[1] else { panic!("{n:#?}") }; + assert_eq!(den, &vec![run("2a")]); + assert!(num.iter().any(|m| matches!(m, MNode::Rad { .. })), "{num:#?}"); + let MNode::Rad { e, .. } = num.iter().find(|m| matches!(m, MNode::Rad { .. })).unwrap() else { panic!() }; + assert!(matches!(e.first(), Some(MNode::Script { kind: ScriptKind::Sup, .. })), "{e:#?}"); + } + + #[test] + fn scripts_and_nary() { + let n = parse_linear("∑_(i=1)^n▒i^2"); + let MNode::Nary { chr, sub, sup, e, .. } = &n[0] else { panic!("{n:#?}") }; + assert_eq!(*chr, '∑'); + assert_eq!(sub, &vec![run("i=1")]); + assert_eq!(sup, &vec![run("n")]); + assert!(matches!(e.first(), Some(MNode::Script { .. }))); + let n = parse_linear("x_i^2"); + assert!(matches!(&n[0], MNode::Script { kind: ScriptKind::SubSup, .. }), "{n:#?}"); + } + + #[test] + fn functions_and_matrices() { + let n = parse_linear("sin x"); + assert!(matches!(&n[0], MNode::Func { .. }), "{n:#?}"); + let n = parse_linear("■(a&b@c&d)"); + let MNode::Matrix { rows, .. } = &n[0] else { panic!("{n:#?}") }; + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].len(), 2); + } + + #[test] + fn linear_round_trip_keeps_structure() { + for s in [ + "x=(-b±√(b^2-4ac))/2a", + "∑_(i=1)^n▒i", + "a/b+c", + "E=mc^2", + "√(3&x)", + "├]a[┤", + "{█(x@y)┤", + "|x|+‖v‖", + "⏞(a+b)┴k", + "_1^n▒Y", + "sin^(−1)⁡x", + "lim┬(n→∞)⁡a_n", + "A⃗+x̂", + "▭(a)+¯(b)+▁(c)", + "(a│b)", + "E=mc^2#(1)", + ] { + let n = parse_linear(s); + let back = parse_linear(&to_linear(&n)); + assert_eq!(n, back, "{s} → {}", to_linear(&n)); + } + } + + //T1 + /// Deepest nesting of `a` (a leaf counts 1), without recursion. + fn depth_of(a: &[MNode]) -> usize { + let mut best = 0; + let mut stack: Vec<(&MNode, usize)> = a.iter().map(|n| (n, 1)).collect(); + while let Some((n, d)) = stack.pop() { + best = best.max(d); + each_arg(n, &mut |a| stack.extend(a.iter().map(|c| (c, d + 1)))); + } + best + } + + /// Run `f` on a thread with a stack far below the 8 MB of a main thread (#1456). + fn on_small_stack(f: impl FnOnce() + Send + 'static) { + std::thread::Builder::new().stack_size(512 * 1024).spawn(f).unwrap().join().unwrap(); + } + + #[test] + fn deeply_chained_input_stays_shallow() { + on_small_stack(|| { + let cases: Vec<(&str, Arg)> = vec![ + ("x^1^1…", parse_linear(&format!("x{}", "^1".repeat(9999)))), + ("a/a/a…", parse_linear(&format!("a{}", "/a".repeat(9999)))), + ("accents", parse_linear(&format!("x{}", "\u{302}".repeat(9999)))), + ("limits", parse_linear(&format!("x{}", "┬a".repeat(9000)))), + ("(a b)^1^1…", parse_linear(&format!("(a b){}", "^1".repeat(9000)))), + ("latex x^1^1…", crate::math_latex::parse_latex(&format!("x{}", "^1".repeat(9999)))), + ("latex \\sum…", crate::math_latex::parse_latex(&"\\sum".repeat(5000))), + ("latex \\sin x^1…", crate::math_latex::parse_latex(&format!("\\sin x{}", "^1".repeat(9000)))), + ]; + for (name, n) in cases { + assert!(!n.is_empty(), "{name}"); + assert!(depth_of(&n) <= 70, "{name}: depth {}", depth_of(&n)); + let copy = n.clone(); + assert_eq!(copy, n, "{name}"); + let _ = to_linear(&n); + let _ = crate::math_latex::to_latex(&n); + let json = serde_json::to_string(&n).unwrap(); + let back: Arg = serde_json::from_str(&json).unwrap(); + assert_eq!(back, n, "{name}"); + } + }); + } + + #[test] + fn ordinary_nesting_is_still_built() { + // Well below the cap: scripts, fractions and operators still wrap. + let n = parse_linear(&format!("x{}", "^1".repeat(10))); + assert_eq!(depth_of(&n), 11, "{n:#?}"); + let n = crate::math_latex::parse_latex(r"\sum_{i}\sum_{j}a_{ij}"); + assert!(matches!(&n[0], MNode::Nary { e, .. } if matches!(e.first(), Some(MNode::Nary { .. }))), "{n:#?}"); + } + + //T2 + #[test] + fn latex_text_escapes_what_latex_reads_as_markup() { + for text in ["a}b\\c", "{x}", "50% of #1", "a_b&c$d~e^f", "\\", "\\alpha", "\\{"] { + let n = vec![MNode::Run(MRun { text: text.into(), nor: true, sty: Some(MSty::Plain), ..Default::default() })]; + let tex = crate::math_latex::to_latex(&n); + assert_eq!(crate::math_latex::parse_latex(&tex), n, "{text:?} → {tex}"); + } + let n = vec![MNode::Run(MRun::new("a\\b"))]; + assert_eq!(crate::math_latex::parse_latex(&crate::math_latex::to_latex(&n)), n); + } + + //T3 + #[test] + fn latex_runs_merge_inside_every_structure() { + use crate::math_latex::parse_latex; + for (tex, lin) in [(r"\overline{ab}", "¯(ab)"), (r"\underline{ab}", "▁(ab)"), (r"\boxed{ab}", "▭(ab)"), (r"\phantom{ab}", "⟡(ab)")] { + assert_eq!(parse_latex(tex), parse_linear(lin), "{tex}"); + } + let n = parse_latex(r"\begin{matrix}a b&c d\\e f&g\end{matrix}"); + let MNode::Matrix { rows, .. } = &n[0] else { panic!("{n:#?}") }; + assert!(rows.iter().flatten().all(|c| c.len() == 1), "{rows:#?}"); + let n = parse_latex(r"\hat{ab}\underbrace{ab}\underset{ab}{cd}"); + let runs = |a: &Arg| a.iter().filter(|n| matches!(n, MNode::Run(_))).count(); + assert!(matches!(&n[0], MNode::Acc { e, .. } if runs(e) == 1), "{n:#?}"); + assert!(matches!(&n[1], MNode::GroupChr { e, .. } if runs(e) == 1), "{n:#?}"); + assert!(matches!(&n[2], MNode::Lim { e, lim, .. } if runs(e) == 1 && runs(lim) == 1), "{n:#?}"); + } + + #[test] + fn multi_letter_scripts_round_trip() { + // Word for Mac 16.113.4 (#1609): a script is the whole run up to a space or operator, and Word's + // Linear format shows it bare (`V_dd`); the parenthesised form reads back identically. + let script = |kind, sub: &str, sup: &str| MNode::Script { kind, base: vec![run("V")], sub: parse_linear(sub), sup: parse_linear(sup) }; + for n in [ + vec![script(ScriptKind::Sub, "dd", "")], + vec![script(ScriptKind::Sup, "", "ab")], + vec![script(ScriptKind::SubSup, "in", "2")], + vec![MNode::Nary { + chr: '∑', + lim_loc: None, + grow: false, + sub_hide: false, + sup_hide: false, + sub: vec![run("ij")], + sup: vec![run("n")], + e: vec![run("x")], + }], + vec![MNode::Lim { upper: false, e: vec![run("⟶")], lim: vec![run("yields")] }], + ] { + assert_eq!(parse_linear(&to_linear(&n)), n, "{}", to_linear(&n)); + } + // Numbers and single letters stay bare. + assert_eq!(to_linear(&[script(ScriptKind::SubSup, "10", "2.5")]), "V_10^2.5"); + assert_eq!(to_linear(&[script(ScriptKind::Sub, "d", "")]), "V_d"); + assert_eq!(to_linear(&[script(ScriptKind::Sub, "dd", "")]), "V_dd"); + assert_eq!(to_linear(&[script(ScriptKind::Sup, "", "ab")]), "V^ab"); + } + + #[test] + fn multi_letter_script_operands_follow_word() { + // #1609, Word for Mac 16.113.4: `V_dd` and `V_(dd)` build up the same; `dd` is all subscript. + let sub_dd = MNode::Script { kind: ScriptKind::Sub, base: vec![run("V")], sub: vec![run("dd")], sup: vec![] }; + assert_eq!(parse_linear("V_dd"), vec![sub_dd.clone()]); + assert_eq!(parse_linear("V_(dd)"), vec![sub_dd]); + let sup_ab = MNode::Script { kind: ScriptKind::Sup, base: vec![run("x")], sub: vec![], sup: vec![run("ab")] }; + assert_eq!(parse_linear("x^ab"), vec![sup_ab.clone()]); + assert_eq!(parse_linear("x^(ab)"), vec![sup_ab]); + // Delimiters still end the operand. + let both = MNode::Script { kind: ScriptKind::SubSup, base: vec![run("x")], sub: vec![run("i")], sup: vec![run("2")] }; + assert_eq!(parse_linear("x_i^2"), vec![both]); + let a_i = MNode::Script { kind: ScriptKind::Sub, base: vec![run("a")], sub: vec![run("i")], sup: vec![] }; + assert_eq!(parse_linear("a_i b"), vec![a_i, run("b")]); + let both2 = MNode::Script { kind: ScriptKind::SubSup, base: vec![run("x")], sub: vec![run("ij")], sup: vec![run("kl")] }; + assert_eq!(parse_linear("x_ij^kl"), vec![both2]); + // Round trip is Word's bare form. + assert_eq!(to_linear(&parse_linear("V_(dd)")), "V_dd"); + } + + #[test] + fn a_bare_script_does_not_absorb_what_follows() { + // #1609 review: `x_i` then `y` must not read back as x with subscript `iy`. + let sc = + |kind, base: &str, sub: &str, sup: &str| MNode::Script { kind, base: vec![run(base)], sub: parse_linear(sub), sup: parse_linear(sup) }; + let sin_x = MNode::Func { name: vec![MNode::Run(MRun::plain("sin"))], e: vec![run("x")] }; + let lim = MNode::Lim { upper: false, e: vec![run("a")], lim: vec![run("n")] }; + for n in [ + vec![sc(ScriptKind::Sub, "x", "i", ""), run("y")], + vec![sc(ScriptKind::Sub, "a", "i", ""), sc(ScriptKind::Sub, "b", "j", "")], + vec![sc(ScriptKind::Sup, "x", "", "2"), run("y")], + vec![sc(ScriptKind::Sup, "x", "", "2"), run("3")], + vec![sc(ScriptKind::Sub, "x", "i", ""), sin_x], + vec![lim.clone(), run("y")], + vec![sc(ScriptKind::Sub, "x", "ab", ""), run("y")], + vec![sc(ScriptKind::Sub, "x", "i", ""), run(".5")], + ] { + let lin = to_linear(&n); + assert_eq!(parse_linear(&lin), n, "{lin}"); + } + assert_eq!(to_linear(&[sc(ScriptKind::Sub, "x", "i", ""), run("y")]), "x_i y"); + // Nothing extra when the script already ends in a bracket or a symbol follows. + assert_eq!(to_linear(&[sc(ScriptKind::Sub, "x", "i", ""), run("+")]), "x_i+"); + } + + #[test] + fn a_function_name_in_a_script_stays_upright() { + // Word for Mac (#1609): `x_max` and `x_(max)` both give an upright function name. + let up = vec![MNode::Script { kind: ScriptKind::Sub, base: vec![run("x")], sub: vec![MNode::Run(MRun::plain("max"))], sup: vec![] }]; + assert_eq!(parse_linear("x_max"), up); + assert_eq!(parse_linear("x_(max)"), up); + assert_eq!(parse_linear(&to_linear(&up)), up, "{}", to_linear(&up)); + // Other upright text is not a function name: it keeps its parentheses. + let other = vec![MNode::Script { kind: ScriptKind::Sub, base: vec![run("x")], sub: vec![MNode::Run(MRun::plain("ab"))], sup: vec![] }]; + assert_eq!(to_linear(&other), "x_(ab)"); + } + + #[test] + fn digits_then_letters_continue_a_script() { + // Word for Mac (#1609): a script runs to a space or operator, so `x_2a` is subscript "2a". + let sub = |t: &str| vec![MNode::Script { kind: ScriptKind::Sub, base: vec![run("x")], sub: vec![run(t)], sup: vec![] }]; + assert_eq!(parse_linear("x_2a"), sub("2a")); + assert_eq!(parse_linear("x_a2"), sub("a2")); + assert_eq!(parse_linear("x_αβ"), sub("αβ")); + assert_eq!(parse_linear("x_10"), sub("10")); + assert_eq!(parse_linear("x_2.5"), sub("2.5")); + assert_eq!(to_linear(&sub("2a")), "x_2a"); + assert_eq!(parse_linear(&to_linear(&sub("2a"))), sub("2a")); + } + + #[test] + fn hostile_linear_never_panics() { + for s in ["", "/", "^", "_", "√", "∑", "((((", "))))", "■(", "█(@@&&", "a/", "/b", "^^^__", "√(&)", "∫_^▒", "lim_", "sin"] { + let _ = to_linear(&parse_linear(s)); + } + let deep = "(".repeat(5000) + &")".repeat(5000); + let _ = parse_linear(&deep); + let deep = "√".repeat(3000); + let _ = parse_linear(&deep); + } + + #[test] + fn a_long_run_of_letters_parses_in_linear_time() { + // Each letter that doesn't start a function name used to rescan (and copy) the rest of + // the run, so a 20,000-letter equation did 200 million steps. (No wall-clock assert: a + // loaded runner would fail it; a regression shows as a slow test.) + for _ in 0..5 { + let e = parse_linear(&"x".repeat(30_000)); + assert!(matches!(e.as_slice(), [MNode::Run(r)] if r.text.len() == 20_000), "{:?}", e.len()); + } + // Function names are still found at the start of a run, and only as whole runs. + assert!(matches!(parse_linear("sin x").as_slice(), [MNode::Func { .. }])); + assert!(matches!(parse_linear("arcsin x").as_slice(), [MNode::Func { .. }])); + assert!(matches!(parse_linear("sinx").as_slice(), [MNode::Run(_)])); + assert!(matches!(parse_linear("arcsinhx").as_slice(), [MNode::Run(_)])); + } +} diff --git a/crates/wordmodel/src/math_latex.rs b/crates/wordmodel/src/math_latex.rs new file mode 100644 index 000000000..92979534c --- /dev/null +++ b/crates/wordmodel/src/math_latex.rs @@ -0,0 +1,1242 @@ +//! LaTeX ↔ math tree (Word's LaTeX equation input and conversion). +//! +//! Input covers the commands people type in equations: `\frac`, `\sqrt[n]{}`, scripts, big +//! operators with limits, `\left…\middle…\right`, matrix and `cases`/`aligned` environments, +//! accents, `\overbrace`/`\underbrace`, `\text`/`\mathrm`/`\mathbf`/`\mathbb`…, function names, +//! spacing and every named symbol in [`crate::math_symbols`]. Unknown commands stay as text. + +use crate::math::{ + Arg, ColJc, FracKind, MAX_DEPTH, MNode, MRun, MScr, MSty, ScriptKind, WRAP_DEPTH, any_too_deep, is_nary_char, math_class, merge_runs, too_deep, +}; +use crate::math_linear::{FUNCTION_NAMES, map_args}; +use crate::math_symbols::{name_of, symbol}; + +#[derive(Clone, Debug, PartialEq)] +enum Tok { + Cmd(String), + Open, + Close, + Sup, + Sub, + Amp, + Row, + Ch(char), +} + +/// Commands whose `{…}` argument is text, where spaces are part of the content. +const TEXT_COMMANDS: &[&str] = &["text", "textrm", "mbox", "textnormal", "textbf", "textit"]; + +fn tokenize(s: &str) -> Vec { + let cs: Vec = s.chars().take(20_000).collect(); + let mut out = Vec::new(); + let mut i = 0; + // A text command was just read: its `{` starts text, where spaces count (#448). + let mut text_next = false; + // Brace depth inside a text argument. + let mut text_depth: Option = None; + while let Some(&c) = cs.get(i) { + i += 1; + if text_depth.is_some() && matches!(c, ' ' | '\t' | '\n' | '\r') { + // A run of white space is one space, as in LaTeX. + if out.last() != Some(&Tok::Ch(' ')) { + out.push(Tok::Ch(' ')); + } + continue; + } + let was_text_next = std::mem::take(&mut text_next); + match c { + '\\' => { + let start = i; + while cs.get(i).is_some_and(char::is_ascii_alphabetic) { + i += 1; + } + if i == start { + match cs.get(i) { + Some('\\') => { + i += 1; + out.push(Tok::Row); + } + Some(&c) => { + i += 1; + out.push(Tok::Cmd(c.to_string())); + } + None => {} + } + } else { + let name: String = cs.get(start..i).unwrap_or(&[]).iter().collect(); + text_next = text_depth.is_none() && TEXT_COMMANDS.contains(&name.as_str()); + out.push(Tok::Cmd(name)); + } + } + '{' => { + text_depth = match text_depth { + Some(d) => Some(d.saturating_add(1)), + None if was_text_next => Some(0), + None => None, + }; + out.push(Tok::Open); + } + '}' => { + text_depth = match text_depth { + Some(0) | None => None, + Some(d) => Some(d.saturating_sub(1)), + }; + out.push(Tok::Close); + } + '^' => out.push(Tok::Sup), + '_' => out.push(Tok::Sub), + '&' => out.push(Tok::Amp), + // White space between a text command and its `{` doesn't end the wait. + ' ' | '\t' | '\n' | '\r' => text_next = was_text_next, + '~' => out.push(Tok::Ch('\u{A0}')), + c => out.push(Tok::Ch(c)), + } + } + out +} + +/// Parse LaTeX math into nodes. Never fails. +pub fn parse_latex(s: &str) -> Arg { + let mut p = Lp { t: tokenize(s), i: 0, depth: 0 }; + let mut out = Vec::new(); + while p.i < p.t.len() { + out.extend(p.seq(false)); + // A stray `}` or `&`/`\\` at the top level. + match p.t.get(p.i) { + Some(Tok::Amp) => out.push(run("&")), + Some(Tok::Row) => {} + Some(Tok::Close) => {} + _ => {} + } + p.i += 1; + } + let out = merge_runs(out); + if out.iter().any(|n| matches!(n, MNode::Run(r) if !r.nor && r.text.contains('#'))) { + return vec![MNode::EqArr { rows: vec![out] }]; + } + out +} + +fn run(t: &str) -> MNode { + MNode::Run(MRun::new(t)) +} + +struct Lp { + t: Vec, + i: usize, + depth: usize, +} + +impl Lp { + fn peek(&self) -> Option<&Tok> { + self.t.get(self.i) + } + + /// A sequence up to `}` (in a group), `&`, `\\`, `\right`, `\middle` or `\end`. + fn seq(&mut self, in_group: bool) -> Arg { + self.depth += 1; + let mut out: Arg = Vec::new(); + if self.depth > MAX_DEPTH { + self.i = self.t.len(); + self.depth -= 1; + return out; + } + while let Some(tok) = self.peek().cloned() { + match tok { + Tok::Close if in_group => break, + Tok::Close | Tok::Amp | Tok::Row => break, + Tok::Cmd(ref c) if matches!(c.as_str(), "right" | "middle" | "end") => break, + Tok::Sup | Tok::Sub => { + self.i += 1; + let s = self.arg(); + if out.last().is_some_and(too_deep) { + // Too deep to wrap (`x^1^1^1…`): the operator is dropped, the script kept. + out.extend(s); + } else { + let base = out.pop(); + out.push(script(base, tok == Tok::Sup, s)); + } + } + Tok::Cmd(ref c) if c == "limits" || c == "nolimits" || c == "displaystyle" || c == "textstyle" => { + self.i += 1; + if c == "nolimits" + && let Some(MNode::Nary { lim_loc, .. }) = out.last_mut() + { + *lim_loc = Some(crate::math::LimLoc::SubSup); + } + if c == "limits" + && let Some(MNode::Nary { lim_loc, .. }) = out.last_mut() + { + *lim_loc = Some(crate::math::LimLoc::UndOvr); + } + } + _ => { + let a = self.atom(); + out.extend(a); + } + } + } + self.depth -= 1; + // Big operators take the following term as their operand. + nary_operands(out, self.depth) + } + + /// A command argument: a `{group}` or one token. + fn arg(&mut self) -> Arg { + match self.peek() { + Some(Tok::Open) => { + self.i += 1; + let g = self.seq(true); + if self.peek() == Some(&Tok::Close) { + self.i += 1; + } + g + } + Some(_) => self.atom(), + None => Vec::new(), + } + } + + /// `[optional]` argument. + fn opt(&mut self) -> Option { + if self.peek() != Some(&Tok::Ch('[')) { + return None; + } + self.i += 1; + let mut toks = Vec::new(); + let mut level = 0i32; + while let Some(t) = self.peek().cloned() { + self.i += 1; + match t { + Tok::Ch(']') if level == 0 => break, + Tok::Open => level += 1, + Tok::Close => level -= 1, + _ => {} + } + toks.push(t); + } + let mut p = Lp { t: toks, i: 0, depth: self.depth }; + Some(p.seq(false)) + } + + /// The text of a `{…}` argument, verbatim. + fn raw_arg(&mut self) -> String { + let mut s = String::new(); + if self.peek() != Some(&Tok::Open) { + if let Some(Tok::Ch(c)) = self.peek().cloned() { + self.i += 1; + s.push(c); + } + return s; + } + self.i += 1; + let mut level = 0i32; + // The previous token was a control word (`\backslash{}`: the empty braces end its name). + let mut after_word = false; + while let Some(t) = self.peek().cloned() { + self.i += 1; + let was_word = std::mem::replace(&mut after_word, matches!(&t, Tok::Cmd(c) if c.chars().all(|c| c.is_ascii_alphabetic()))); + match t { + Tok::Close if level == 0 => break, + Tok::Open if was_word && self.peek() == Some(&Tok::Close) => self.i += 1, + Tok::Open => { + level += 1; + s.push('{'); + } + Tok::Close => { + level -= 1; + s.push('}'); + } + Tok::Ch(c) => s.push(c), + Tok::Cmd(c) => { + if c.chars().all(|c| c.is_ascii_alphabetic()) { + match symbol(&c) { + Some(sym) => s.push(sym), + None => s.push_str(&c), + } + } else { + s.push_str(&c); + } + if c == " " { + s.push(' '); + } + } + Tok::Sup => s.push('^'), + Tok::Sub => s.push('_'), + Tok::Amp => s.push('&'), + Tok::Row => s.push('\n'), + } + } + s + } + + /// A delimiter after `\left`, `\right`, `\middle`, `\big…`. + fn delim(&mut self) -> Option { + let t = self.peek().cloned()?; + self.i += 1; + match t { + Tok::Ch('.') => None, + Tok::Ch(c) => Some(c), + Tok::Cmd(c) => match c.as_str() { + "{" => Some('{'), + "}" => Some('}'), + "|" => Some('‖'), + other => symbol(other), + }, + _ => None, + } + } + + fn atom(&mut self) -> Arg { + if self.depth > MAX_DEPTH { + self.i = self.t.len(); + return Vec::new(); + } + self.depth += 1; + let out = self.atom_inner(); + self.depth -= 1; + out + } + + fn atom_inner(&mut self) -> Arg { + let Some(t) = self.peek().cloned() else { return Vec::new() }; + self.i += 1; + match t { + Tok::Open => { + let g = self.seq(true); + if self.peek() == Some(&Tok::Close) { + self.i += 1; + } + g + } + Tok::Ch(c) => vec![run(&c.to_string())], + Tok::Close | Tok::Amp | Tok::Row | Tok::Sup | Tok::Sub => Vec::new(), + Tok::Cmd(c) => self.command(&c), + } + } + + fn command(&mut self, c: &str) -> Arg { + let styled = |a: Arg, sty: Option, scr: MScr, nor: bool| -> Arg { restyle(a, sty, scr, nor) }; + let one = |n: MNode| vec![n]; + match c { + "frac" | "dfrac" | "tfrac" | "cfrac" => { + let num = self.arg(); + let den = self.arg(); + one(MNode::Frac { kind: FracKind::Bar, num, den }) + } + "genfrac" => { + let beg = delim_arg(&self.arg()); + let end = delim_arg(&self.arg()); + let thickness = self.arg(); + let _style = self.arg(); + let num = self.arg(); + let den = self.arg(); + let kind = if is_zero_length(&thickness) { FracKind::NoBar } else { FracKind::Bar }; + let frac = MNode::Frac { kind, num, den }; + if beg.is_none() && end.is_none() { + one(frac) + } else { + one(MNode::Delim { beg, end, sep: None, grow: true, shp_match: false, elems: vec![vec![frac]] }) + } + } + "binom" | "dbinom" | "tbinom" => { + let n = self.arg(); + let k = self.arg(); + one(MNode::Delim { + beg: Some('('), + end: Some(')'), + sep: None, + grow: true, + shp_match: false, + elems: vec![vec![MNode::Frac { kind: FracKind::NoBar, num: n, den: k }]], + }) + } + "sqrt" => { + let deg = self.opt().unwrap_or_default(); + let e = self.arg(); + one(MNode::Rad { deg_hide: deg.is_empty(), deg, e }) + } + "left" => { + let beg = self.delim(); + let mut elems = vec![self.seq(false)]; + let mut sep = None; + let mut end = None; + while let Some(Tok::Cmd(k)) = self.peek().cloned() { + self.i += 1; + if k == "middle" { + sep = self.delim().or(Some('|')); + elems.push(self.seq(false)); + } else { + if k == "right" { + end = self.delim(); + } + break; + } + } + one(MNode::Delim { beg, end, sep, grow: true, shp_match: false, elems }) + } + "bigl" | "Bigl" | "biggl" | "Biggl" | "bigr" | "Bigr" | "biggr" | "Biggr" | "big" | "Big" | "bigg" | "Bigg" => { + self.delim().map(|d| vec![run(&d.to_string())]).unwrap_or_default() + } + "begin" => { + let env = self.raw_arg(); + self.environment(env.trim()) + } + "text" | "textrm" | "mbox" | "textnormal" => { + let t = self.raw_arg(); + one(MNode::Run(MRun { text: t, nor: true, sty: Some(MSty::Plain), ..Default::default() })) + } + "textbf" => { + let t = self.raw_arg(); + one(MNode::Run(MRun { text: t, nor: true, sty: Some(MSty::Bold), ..Default::default() })) + } + "textit" => { + let t = self.raw_arg(); + one(MNode::Run(MRun { text: t, nor: true, sty: Some(MSty::Italic), ..Default::default() })) + } + "operatorname" => { + let starred = self.peek() == Some(&Tok::Ch('*')); + if starred { + self.i += 1; + } + let t = self.raw_arg(); + let name = vec![MNode::Run(MRun::plain(t))]; + if starred && self.peek() == Some(&Tok::Sub) { + self.i += 1; + let lim = self.arg(); + vec![MNode::Lim { upper: false, e: name, lim }] + } else { + self.apply(name) + } + } + "tag" => { + let starred = self.peek() == Some(&Tok::Ch('*')); + if starred { + self.i += 1; + } + let t = self.raw_arg(); + let num_str = if starred { format!("#{t}") } else { format!("# ({t})") }; + let mut r = MRun::new(num_str); + r.lit = false; + vec![MNode::Run(r)] + } + "mathrm" | "rm" => { + let a = self.arg(); + styled(a, Some(MSty::Plain), MScr::Roman, false) + } + "mathbf" | "bf" => { + let a = self.arg(); + styled(a, Some(MSty::Bold), MScr::Roman, false) + } + "mathit" => { + let a = self.arg(); + styled(a, Some(MSty::Italic), MScr::Roman, false) + } + "boldsymbol" | "bm" => { + let a = self.arg(); + styled(a, Some(MSty::BoldItalic), MScr::Roman, false) + } + "mathbb" => { + let a = self.arg(); + styled(a, None, MScr::DoubleStruck, false) + } + "mathcal" | "mathscr" => { + let a = self.arg(); + styled(a, None, MScr::Script, false) + } + "mathfrak" => { + let a = self.arg(); + styled(a, None, MScr::Fraktur, false) + } + "mathsf" => { + let a = self.arg(); + styled(a, None, MScr::SansSerif, false) + } + "mathtt" => { + let a = self.arg(); + styled(a, None, MScr::Monospace, false) + } + "overline" => one(MNode::Bar { top: true, e: self.arg() }), + "underline" => one(MNode::Bar { top: false, e: self.arg() }), + "boxed" | "fbox" => one(MNode::BorderBox { hide: [false; 4], strike: [false; 4], e: self.arg() }), + "phantom" => one(MNode::Phant { show: false, zero_wid: false, zero_asc: false, zero_desc: false, e: self.arg() }), + "overbrace" | "underbrace" | "overparen" | "underparen" | "overbracket" | "underbracket" => { + let top = c.starts_with("over"); + let chr = match c { + "overbrace" => '⏞', + "underbrace" => '⏟', + "overparen" => '⏜', + "underparen" => '⏝', + "overbracket" => '⎴', + _ => '⎵', + }; + let e = self.arg(); + let g = MNode::GroupChr { chr, top, e }; + let want = if top { Tok::Sup } else { Tok::Sub }; + if self.peek() == Some(&want) { + self.i += 1; + let lim = self.arg(); + one(MNode::Lim { upper: top, e: vec![g], lim }) + } else { + one(g) + } + } + "stackrel" | "overset" => { + let over = self.arg(); + let base = self.arg(); + one(MNode::Lim { upper: true, e: base, lim: over }) + } + "underset" => { + let under = self.arg(); + let base = self.arg(); + one(MNode::Lim { upper: false, e: base, lim: under }) + } + "hat" | "widehat" | "tilde" | "widetilde" | "dot" | "ddot" | "dddot" | "check" | "breve" | "acute" | "grave" | "vec" | "bar" + | "overrightarrow" | "overleftarrow" | "overleftrightarrow" | "mathring" => { + let chr = match c { + "hat" | "widehat" => '\u{302}', + "tilde" | "widetilde" => '\u{303}', + "dot" => '\u{307}', + "ddot" => '\u{308}', + "dddot" => '\u{20DB}', + "check" => '\u{30C}', + "breve" => '\u{306}', + "acute" => '\u{301}', + "grave" => '\u{300}', + "bar" => '\u{305}', + "overleftarrow" => '\u{20D6}', + "overleftrightarrow" => '\u{20E1}', + "mathring" => '\u{30A}', + _ => '\u{20D7}', + }; + one(MNode::Acc { chr, e: self.arg() }) + } + "," | "thinspace" => vec![run("\u{2009}")], + ":" | ">" | "medspace" => vec![run("\u{205F}")], + ";" | "thickspace" => vec![run("\u{2005}")], + "!" | "negthinspace" => Vec::new(), + " " => vec![run(" ")], + "qquad" => vec![run("\u{2003}\u{2003}")], + "{" | "}" | "%" | "$" | "#" | "&" | "_" => { + // Escaped characters (a literal `#` must not number the equation). + let mut r = MRun::new(c); + r.lit = c == "#"; + vec![MNode::Run(r)] + } + "|" => vec![run("‖")], + name if FUNCTION_NAMES.contains(&name) => { + let plain = vec![MNode::Run(MRun::plain(name))]; + let limits = matches!(name, "lim" | "liminf" | "limsup" | "max" | "min" | "sup" | "inf" | "det" | "gcd" | "Pr"); + let named = if limits && self.peek() == Some(&Tok::Sub) { + self.i += 1; + let lim = self.arg(); + vec![MNode::Lim { upper: false, e: plain, lim }] + } else if matches!(self.peek(), Some(Tok::Sup) | Some(Tok::Sub)) { + let sup = self.peek() == Some(&Tok::Sup); + self.i += 1; + let s = self.arg(); + vec![script(Some(MNode::Run(MRun::plain(name))), sup, s)] + } else { + plain + }; + self.apply(named) + } + name => match symbol(name) { + Some(ch) if is_nary_char(ch) => vec![MNode::Nary { + chr: ch, + lim_loc: None, + grow: false, + sub_hide: true, + sup_hide: true, + sub: Vec::new(), + sup: Vec::new(), + e: Vec::new(), + }], + Some(ch) => vec![run(&ch.to_string())], + None => vec![run(&format!("\\{name}"))], + }, + } + } + + /// A function name applied to the next atom (with its scripts). + fn apply(&mut self, name: Arg) -> Arg { + let mut e = self.atom(); + while let Some(t @ (Tok::Sup | Tok::Sub)) = self.peek().cloned() { + self.i += 1; + let s = self.arg(); + if any_too_deep(&e) { + e.extend(s); + continue; + } + let base = if e.len() == 1 { e.pop() } else { Some(group(std::mem::take(&mut e))) }; + e.push(script(base, t == Tok::Sup, s)); + } + if e.is_empty() { + return name; + } + vec![MNode::Func { name, e }] + } + + fn environment(&mut self, env: &str) -> Arg { + let base = env.trim_end_matches('*'); + let mut col_jc = Vec::new(); + if base == "array" || base == "tabular" { + let _ = self.opt(); + let spec = self.raw_arg(); + for ch in spec.chars() { + match ch { + 'l' | 'L' => col_jc.push(ColJc::Left), + 'r' | 'R' => col_jc.push(ColJc::Right), + 'c' | 'C' => col_jc.push(ColJc::Center), + _ => {} + } + } + } else if base == "aligned" { + let _ = self.opt(); + } + // Rows of `&`-separated cells up to `\end{…}`. + let mut rows: Vec> = vec![vec![]]; + loop { + let cell = self.seq(false); + if let Some(r) = rows.last_mut() { + r.push(cell); + } + match self.peek().cloned() { + Some(Tok::Amp) => self.i += 1, + Some(Tok::Row) => { + self.i += 1; + rows.push(vec![]); + } + Some(Tok::Cmd(k)) if k == "end" => { + self.i += 1; + let _ = self.raw_arg(); + break; + } + Some(Tok::Close) => self.i += 1, + Some(_) => self.i += 1, + None => break, + } + } + // A trailing `\\` leaves an empty last row. + if rows.len() > 1 && rows.last().is_some_and(|r| r.iter().all(|c| c.is_empty())) { + rows.pop(); + } + let delims = match base { + "pmatrix" => Some((Some('('), Some(')'))), + "bmatrix" => Some((Some('['), Some(']'))), + "Bmatrix" => Some((Some('{'), Some('}'))), + "vmatrix" => Some((Some('|'), Some('|'))), + "Vmatrix" => Some((Some('‖'), Some('‖'))), + _ => None, + }; + let wrap = |inner: MNode, beg: Option, end: Option| MNode::Delim { + beg, + end, + sep: None, + grow: true, + shp_match: false, + elems: vec![vec![inner]], + }; + match base { + "matrix" | "pmatrix" | "bmatrix" | "Bmatrix" | "vmatrix" | "Vmatrix" | "smallmatrix" | "array" | "tabular" => { + let m = MNode::Matrix { rows, col_jc }; + match delims { + Some((b, e)) => vec![wrap(m, b, e)], + None => vec![m], + } + } + "cases" | "dcases" => { + let eq = MNode::EqArr { rows: rows.into_iter().map(join_cells).collect() }; + vec![wrap(eq, Some('{'), None)] + } + "rcases" => { + let eq = MNode::EqArr { rows: rows.into_iter().map(join_cells).collect() }; + vec![wrap(eq, None, Some('}'))] + } + _ => vec![MNode::EqArr { rows: rows.into_iter().map(join_cells).collect() }], + } + } +} + +/// Cells of an aligned row joined with `&` alignment marks. +fn join_cells(cells: Vec) -> Arg { + let mut out = Vec::new(); + for (i, c) in cells.into_iter().enumerate() { + if i > 0 { + out.push(run("&")); + } + out.extend(c); + } + merge_runs(out) +} + +fn group(a: Arg) -> MNode { + MNode::Delim { beg: None, end: None, sep: None, grow: true, shp_match: false, elems: vec![a] } +} + +fn script(base: Option, sup: bool, s: Arg) -> MNode { + match base { + Some(MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub, sup: sp, e }) if e.is_empty() => { + if sup { + MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide: false, sub, sup: s, e } + } else { + MNode::Nary { chr, lim_loc, grow, sub_hide: false, sup_hide, sub: s, sup: sp, e } + } + } + Some(MNode::Script { kind: ScriptKind::Sub, base, sub, .. }) if sup => MNode::Script { kind: ScriptKind::SubSup, base, sub, sup: s }, + Some(MNode::Script { kind: ScriptKind::Sup, base, sup: sp, .. }) if !sup => MNode::Script { kind: ScriptKind::SubSup, base, sub: s, sup: sp }, + b => { + let base = b.map(|b| vec![b]).unwrap_or_default(); + if sup { + MNode::Script { kind: ScriptKind::Sup, base, sub: Vec::new(), sup: s } + } else { + MNode::Script { kind: ScriptKind::Sub, base, sub: s, sup: Vec::new() } + } + } + } +} + +/// Big operators without an operand take the atoms after them up to a relation or binary operator. +/// `depth` is how deep the operators already sit; past [`WRAP_DEPTH`] they stay without operand +/// (`\sum\sum\sum…` must not nest once per operator). +fn nary_operands(a: Arg, depth: usize) -> Arg { + let mut out: Arg = Vec::with_capacity(a.len()); + let mut it = a.into_iter().peekable(); + while let Some(n) = it.next() { + match n { + MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub, sup, e } if e.is_empty() && depth < WRAP_DEPTH => { + let mut body = Vec::new(); + while let Some(next) = it.peek() { + let stop = match next { + MNode::Run(r) => r.text.chars().next().is_some_and(|c| { + matches!(math_class(c), crate::math::MClass::Rel | crate::math::MClass::Bin | crate::math::MClass::Punct) || c == '&' + }), + _ => false, + }; + if stop { + break; + } + if let Some(x) = it.next() { + body.push(x); + } + } + let body = merge_runs(nary_operands(body, depth + 1)); + if any_too_deep(&body) { + // No room to wrap the operand: it follows the operator. + out.push(MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub, sup, e }); + out.extend(body); + } else { + out.push(MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub, sup, e: body }); + } + } + other => out.push(other), + } + } + out +} + +/// Set a style on every run inside. +fn restyle(a: Arg, sty: Option, scr: MScr, nor: bool) -> Arg { + a.into_iter() + .map(|n| match n { + MNode::Run(mut r) => { + if sty.is_some() { + r.sty = sty; + } + if scr != MScr::Roman { + r.scr = scr; + } + r.nor |= nor; + MNode::Run(r) + } + other => map_args(other, &|a| restyle(a, sty, scr, nor)), + }) + .collect() +} + +// --------------------------------------------------------------------------------------------- +// Out. + +/// The equation as LaTeX. +pub fn to_latex(nodes: &[MNode]) -> String { + let mut s = String::new(); + arg(nodes, &mut s, 0); + s.trim().to_string() +} + +fn arg(a: &[MNode], s: &mut String, depth: usize) { + if depth > MAX_DEPTH { + return; + } + for n in a { + node(n, s, depth + 1); + } +} + +fn braced(a: &[MNode], s: &mut String, depth: usize) { + s.push('{'); + arg(a, s, depth); + s.push('}'); +} + +fn chars(text: &str, s: &mut String) { + for c in text.chars() { + match c { + '{' | '}' | '%' | '$' | '#' | '&' | '_' => { + s.push('\\'); + s.push(c); + } + '\\' => s.push_str("\\backslash "), + '−' => s.push('-'), + c if c.is_ascii() => s.push(c), + c => match name_of(c) { + Some(n) if !matches!(n, "sqrt" | "matrix" | "of" | "above" | "below" | "box" | "rect") => { + s.push('\\'); + s.push_str(n); + s.push(' '); + } + _ => s.push(c), + }, + } + } +} + +/// Text inside `\text{…}`: only what LaTeX would read as markup is escaped. +fn text_chars(text: &str, s: &mut String) { + for c in text.chars() { + match c { + '{' | '}' | '%' | '$' | '#' | '&' | '_' | '~' => { + s.push('\\'); + s.push(c); + } + // `{}` ends the command name without adding a space (spaces count in text). + '\\' => s.push_str("\\backslash{}"), + c => s.push(c), + } + } +} + +/// The delimiter of a `\genfrac` argument: one character, or none for `{}` and anything else. +fn delim_arg(a: &[MNode]) -> Option { + let [MNode::Run(r)] = a else { return None }; + let mut cs = r.text.chars(); + match (cs.next(), cs.next()) { + (Some(c), None) if c != '.' => Some(c), + _ => None, + } +} + +/// Whether a `\genfrac` thickness is a zero length such as `0pt`, `0.0pt` or `0 em`. An empty +/// thickness means the default rule and anything that is not a plain number and unit is not zero. +fn is_zero_length(a: &[MNode]) -> bool { + let mut text = String::new(); + for n in a { + let MNode::Run(r) = n else { return false }; + text.extend(r.text.chars().filter(|c| !c.is_whitespace() && !matches!(c, '\u{2009}' | '\u{205F}' | '\u{2005}'))); + } + let number = text.trim_end_matches(|c: char| c.is_ascii_alphabetic()); + let unit = &text[number.len()..]; + !number.is_empty() + && matches!(unit, "pt" | "em" | "ex" | "mm" | "cm" | "in" | "bp" | "pc" | "sp" | "mu") + && number.parse::().is_ok_and(|v| v == 0.0) +} + +fn delim_tex(c: Option) -> String { + match c { + None => ".".into(), + Some('{') => "\\{".into(), + Some('}') => "\\}".into(), + Some('‖') => "\\|".into(), + Some(c) if c.is_ascii() => c.to_string(), + Some(c) => name_of(c).map(|n| format!("\\{n} ")).unwrap_or_else(|| c.to_string()), + } +} + +fn group_chr_tex(chr: char, top: bool) -> &'static str { + match chr { + '⏜' => "\\overparen", + '⏝' => "\\underparen", + '⎴' => "\\overbracket", + '⎵' => "\\underbracket", + _ if top => "\\overbrace", + _ => "\\underbrace", + } +} + +fn node(node: &MNode, output: &mut String, depth: usize) { + match node { + MNode::Run(run) => run_node(run, output), + MNode::Frac { kind, num, den } => fraction_node(*kind, num, den, output, depth), + MNode::Script { kind, base, sub, sup } => script_node(*kind, base, sub, sup, output, depth), + MNode::Rad { deg, deg_hide, e } => radical_node(deg, *deg_hide, e, output, depth), + MNode::Nary { .. } => nary_node(node, output, depth), + MNode::Delim { beg, end, sep, elems, .. } => delimiter_node(*beg, *end, *sep, elems, output, depth), + MNode::Func { name, e } => { + arg(name, output, depth); + braced(e, output, depth); + } + MNode::Lim { upper, e, lim } => limit_node(*upper, e, lim, output, depth), + MNode::Acc { chr, e } => accent_node(*chr, e, output, depth), + MNode::Bar { top, e } => { + output.push_str(if *top { "\\overline" } else { "\\underline" }); + braced(e, output, depth); + } + MNode::BorderBox { e, .. } => { + output.push_str("\\boxed"); + braced(e, output, depth); + } + MNode::Boxed { e } => braced(e, output, depth), + MNode::GroupChr { chr, top, e } => { + output.push_str(group_chr_tex(*chr, *top)); + braced(e, output, depth); + } + MNode::EqArr { rows } => equation_array_node(rows, output, depth), + MNode::Matrix { rows, .. } => matrix_node(rows, output, depth), + MNode::Phant { e, .. } => { + output.push_str("\\phantom"); + braced(e, output, depth); + } + } +} + +fn run_node(run: &MRun, output: &mut String) { + let (open, close) = if run.nor { + ("\\text{", "}") + } else { + match (run.sty, run.scr) { + (_, MScr::DoubleStruck) => ("\\mathbb{", "}"), + (_, MScr::Script) => ("\\mathcal{", "}"), + (_, MScr::Fraktur) => ("\\mathfrak{", "}"), + (_, MScr::SansSerif) => ("\\mathsf{", "}"), + (_, MScr::Monospace) => ("\\mathtt{", "}"), + (Some(MSty::Bold), _) => ("\\mathbf{", "}"), + (Some(MSty::BoldItalic), _) => ("\\boldsymbol{", "}"), + (Some(MSty::Plain), _) if run.text.chars().any(char::is_alphabetic) => { + if FUNCTION_NAMES.contains(&run.text.as_str()) { + output.push('\\'); + output.push_str(&run.text); + output.push(' '); + return; + } + ("\\mathrm{", "}") + } + _ => ("", ""), + } + }; + output.push_str(open); + if run.nor { + text_chars(&run.text, output) + } else { + chars(&run.text, output) + } + output.push_str(close); +} + +fn fraction_node(kind: FracKind, numerator: &[MNode], denominator: &[MNode], output: &mut String, depth: usize) { + match kind { + FracKind::NoBar => { + output.push_str("\\genfrac{}{}{0pt}{}"); + braced(numerator, output, depth); + braced(denominator, output, depth); + } + FracKind::Linear | FracKind::Skewed => { + braced(numerator, output, depth); + output.push('/'); + braced(denominator, output, depth); + } + FracKind::Bar => { + output.push_str("\\frac"); + braced(numerator, output, depth); + braced(denominator, output, depth); + } + } +} + +fn script_node(kind: ScriptKind, base: &[MNode], sub: &[MNode], sup: &[MNode], output: &mut String, depth: usize) { + if kind == ScriptKind::Pre { + output.push_str("{}_"); + braced(sub, output, depth); + output.push('^'); + braced(sup, output, depth); + braced(base, output, depth); + return; + } + braced(base, output, depth); + if matches!(kind, ScriptKind::Sub | ScriptKind::SubSup) { + output.push('_'); + braced(sub, output, depth); + } + if matches!(kind, ScriptKind::Sup | ScriptKind::SubSup) { + output.push('^'); + braced(sup, output, depth); + } +} + +fn radical_node(degree: &[MNode], degree_hidden: bool, expression: &[MNode], output: &mut String, depth: usize) { + output.push_str("\\sqrt"); + if !degree_hidden && !degree.is_empty() { + output.push('['); + arg(degree, output, depth); + output.push(']'); + } + braced(expression, output, depth); +} + +fn nary_node(node: &MNode, output: &mut String, depth: usize) { + let MNode::Nary { chr, lim_loc, sub_hide, sup_hide, sub, sup, e, .. } = node else { return }; + match name_of(*chr) { + Some(name) => { + output.push('\\'); + output.push_str(name); + } + None => output.push(*chr), + } + match lim_loc { + Some(crate::math::LimLoc::UndOvr) => output.push_str("\\limits"), + Some(crate::math::LimLoc::SubSup) if !crate::math::is_integral(*chr) => output.push_str("\\nolimits"), + _ => {} + } + if !sub_hide && !sub.is_empty() { + output.push('_'); + braced(sub, output, depth); + } + if !sup_hide && !sup.is_empty() { + output.push('^'); + braced(sup, output, depth); + } + output.push(' '); + braced(e, output, depth); +} + +fn delimiter_node(begin: Option, end: Option, separator: Option, elements: &[Arg], output: &mut String, depth: usize) { + output.push_str("\\left"); + output.push_str(&delim_tex(begin)); + let middle = separator.map(|c| format!("\\middle{}", delim_tex(Some(c)))).unwrap_or_else(|| "\\middle|".to_string()); + for (index, element) in elements.iter().enumerate() { + if index > 0 { + output.push_str(&middle); + } + arg(element, output, depth); + } + output.push_str("\\right"); + output.push_str(&delim_tex(end)); +} + +fn limit_node(upper: bool, expression: &[MNode], limit: &[MNode], output: &mut String, depth: usize) { + let simple = matches!(expression, [MNode::Run(run)] if run.sty == Some(MSty::Plain) && FUNCTION_NAMES.contains(&run.text.as_str())); + if simple && !upper { + arg(expression, output, depth); + output.push('_'); + braced(limit, output, depth); + } else if let [MNode::GroupChr { chr, top, e: inner }] = expression { + output.push_str(group_chr_tex(*chr, *top)); + braced(inner, output, depth); + output.push(if upper { '^' } else { '_' }); + braced(limit, output, depth); + } else { + output.push_str(if upper { "\\overset" } else { "\\underset" }); + braced(limit, output, depth); + braced(expression, output, depth); + } +} + +fn accent_node(character: char, expression: &[MNode], output: &mut String, depth: usize) { + let command = match character { + '\u{302}' => "hat", + '\u{303}' => "tilde", + '\u{307}' => "dot", + '\u{308}' => "ddot", + '\u{20DB}' => "dddot", + '\u{30C}' => "check", + '\u{306}' => "breve", + '\u{301}' => "acute", + '\u{300}' => "grave", + '\u{305}' | '\u{AF}' => "bar", + '\u{20D6}' => "overleftarrow", + '\u{20E1}' => "overleftrightarrow", + _ => "vec", + }; + output.push('\\'); + output.push_str(command); + braced(expression, output, depth); +} + +fn equation_array_node(rows: &[Arg], output: &mut String, depth: usize) { + if rows.len() == 1 + && let Some(row) = rows.first() + && crate::math::has_number_mark(row) + { + arg(row, output, depth); + return; + } + output.push_str("\\begin{aligned}"); + for (index, row) in rows.iter().enumerate() { + if index > 0 { + output.push_str("\\\\"); + } + for child_node in row { + match child_node { + MNode::Run(run) if !run.lit && run.text.contains('&') => { + for (index, part) in run.text.split('&').enumerate() { + if index > 0 { + output.push('&'); + } + let mut piece = run.clone(); + piece.text = part.to_string(); + run_node(&piece, output); + } + } + other => node(other, output, depth + 1), + } + } + } + output.push_str("\\end{aligned}"); +} + +fn matrix_node(rows: &[Vec], output: &mut String, depth: usize) { + output.push_str("\\begin{matrix}"); + for (row_index, row) in rows.iter().enumerate() { + if row_index > 0 { + output.push_str("\\\\"); + } + for (column, cell) in row.iter().enumerate() { + if column > 0 { + output.push('&'); + } + arg(cell, output, depth); + } + } + output.push_str("\\end{matrix}"); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::math::{LimLoc, MathJc}; + + #[test] + fn latex_structures() { + let n = parse_latex(r"x=\frac{-b\pm\sqrt{b^2-4ac}}{2a}"); + assert_eq!(crate::math::to_linear(&n), "x=(-b±√(b^2-4ac))/2a"); + let n = parse_latex(r"\sum_{i=1}^{n} i^2 = \frac{n(n+1)(2n+1)}{6}"); + let MNode::Nary { chr, sub, sup, e, .. } = &n[0] else { panic!("{n:#?}") }; + assert_eq!((*chr, sub.len(), sup.len()), ('∑', 1, 1)); + assert!(matches!(e.first(), Some(MNode::Script { .. })), "{e:#?}"); + let n = parse_latex(r"\begin{pmatrix} a & b \\ c & d \end{pmatrix}"); + assert!( + matches!(&n[0], MNode::Delim { beg: Some('('), elems, .. } if matches!(elems[0][0], MNode::Matrix { ref rows, .. } if rows.len() == 2)) + ); + let n = parse_latex(r"f(x)=\begin{cases} x & x\ge 0\\ -x & x<0\end{cases}"); + assert!(n.iter().any(|m| matches!(m, MNode::Delim { beg: Some('{'), end: None, .. })), "{n:#?}"); + let n = parse_latex(r"\lim_{n\to\infty}\left(1+\frac{1}{n}\right)^n"); + assert!(matches!(&n[0], MNode::Func { .. }), "{n:#?}"); + let n = parse_latex(r"\mathbb{R}\hat{x}\text{ if }\overbrace{a+b}^{k}"); + assert!(matches!(&n[0], MNode::Run(r) if r.scr == MScr::DoubleStruck)); + let _ = (LimLoc::UndOvr, MathJc::Center); + } + + #[test] + fn latex_binom_round_trip() { + let a = parse_latex(r"\binom{n}{k}"); + let l = to_latex(&a); + assert_eq!(parse_latex(&l), a, "exported as {l}"); + let nobar = |n: &str, k: &str| MNode::Frac { kind: FracKind::NoBar, num: parse_latex(n), den: parse_latex(k) }; + let b = parse_latex(r"\genfrac{}{}{0pt}{}{n}{k}"); + assert_eq!(b, vec![nobar("n", "k")]); + assert_eq!(parse_latex(&to_latex(&b)), b); + // Other spellings of a zero thickness. + for z in ["0 pt", "0.0pt", r"0\,pt", "0em"] { + let c = parse_latex(&format!(r"\genfrac{{}}{{}}{{{z}}}{{}}{{n}}{{k}}")); + assert_eq!(c, vec![nobar("n", "k")], "thickness {z}"); + } + // A non-zero or default thickness keeps the bar. + for t in ["1pt", "0.4pt", ""] { + let c = parse_latex(&format!(r"\genfrac{{}}{{}}{{{t}}}{{}}{{n}}{{k}}")); + let bar = MNode::Frac { kind: FracKind::Bar, num: parse_latex("n"), den: parse_latex("k") }; + assert_eq!(c, vec![bar], "thickness {t:?}"); + } + // Delimiters are kept. + let d = parse_latex(r"\genfrac(){0pt}{}{n}{k}"); + assert_eq!(d, a); + let e = parse_latex(r"\genfrac[]{0pt}{}{n}{k}"); + assert!(matches!(&e[..], [MNode::Delim { beg: Some('['), end: Some(']'), .. }]), "{e:#?}"); + assert_eq!(parse_latex(&to_latex(&e)), e); + } + + #[test] + fn latex_round_trip() { + for lin in ["x=(-b±√(b^2-4ac))/2a", "∑_(i=1)^n▒i^2", "√(3&x)", "a⃗+x̂", "■(a&b@c&d)", "sin⁡x+cos⁡y", "e^(-x^2)"] { + let n = crate::math::parse_linear(lin); + let tex = to_latex(&n); + let back = parse_latex(&tex); + assert_eq!(crate::math::to_linear(&back), crate::math::to_linear(&n), "{lin} → {tex}"); + } + } + + #[test] + fn group_character_latex_roundtrip() { + for cmd in ["\\overparen{a+b}", "\\underparen{a+b}", "\\overbracket{x}", "\\underbracket{x}"] { + let n = parse_latex(cmd); + let tex = to_latex(&n); + assert_eq!(tex, cmd); + } + } + + #[test] + fn hostile_latex_never_panics() { + for s in ["", "\\", "{", "}", "^", "_", "\\frac", "\\sqrt[", "\\left(", "\\right)", "\\begin{matrix", "&&\\\\", "{{{{", "\\end{x}", "a^^b__c"] + { + let _ = to_latex(&parse_latex(s)); + } + let deep = "{".repeat(5000); + let _ = parse_latex(&deep); + let deep = "\\frac{".repeat(3000); + let _ = parse_latex(&deep); + let deep = "\\sqrt".repeat(5000); + let _ = parse_latex(&deep); + let deep = "\\hat".repeat(5000) + "x"; + let _ = parse_latex(&deep); + } + + #[test] + fn array_column_specs_and_aligned_optional_args() { + let n = parse_latex(r"\begin{array}{lcr} a & b & c \\ d & e & f \end{array}"); + let MNode::Matrix { rows, col_jc } = &n[0] else { panic!("{n:#?}") }; + assert_eq!(col_jc, &[ColJc::Left, ColJc::Center, ColJc::Right]); + assert_eq!(rows.len(), 2); + assert_eq!(crate::math::to_linear(&rows[0][0]), "a"); + assert_eq!(crate::math::to_linear(&rows[0][1]), "b"); + assert_eq!(crate::math::to_linear(&rows[0][2]), "c"); + + let n = parse_latex(r"\begin{aligned}[t] x &= 1 \\ y &= 2 \end{aligned}"); + assert!(matches!(&n[0], MNode::EqArr { rows } if rows.len() == 2)); + } + + #[test] + fn operatorname_starred_and_tag() { + let n = parse_latex(r"\operatorname*{argmin}_{x} f(x)"); + assert!(matches!(&n[0], MNode::Lim { upper: false, .. })); + + let n = parse_latex(r"E=mc^2 \tag{1}"); + let MNode::EqArr { rows } = &n[0] else { panic!("{n:#?}") }; + assert!(crate::math::has_number_mark(&rows[0])); + + let n = parse_latex(r"E=mc^2 \tag*{(A)}"); + let MNode::EqArr { rows } = &n[0] else { panic!("{n:#?}") }; + assert!(crate::math::has_number_mark(&rows[0])); + } + + #[test] + fn delim_preserves_custom_separator() { + let delim = MNode::Delim { + beg: Some('{'), + end: Some('}'), + sep: Some('‖'), + grow: true, + shp_match: false, + elems: vec![vec![MNode::Run(MRun::new("x"))], vec![MNode::Run(MRun::new("y"))]], + }; + let tex = to_latex(&[delim]); + assert!(tex.contains(r"\middle\|"), "Expected \\middle\\|, got: {tex}"); + } +} diff --git a/crates/wordmodel/src/math_linear.rs b/crates/wordmodel/src/math_linear.rs new file mode 100644 index 000000000..39a25ab5d --- /dev/null +++ b/crates/wordmodel/src/math_linear.rs @@ -0,0 +1,762 @@ +//! The linear format (UnicodeMath, the plain text Word builds equations from) → math tree. +//! +//! Grammar, informally: an expression is a sequence of atoms and operators. Atoms are characters, +//! bracket groups (`(…)`, `[…]`, `{…}`, `|…|`, `〖…〗` invisible, `├…┤` one-sided, `│` +//! separators), prefix structures (`√x`, `∑_a^b▒x`, `█(…)`, `■(…)`, `▭x`, `□x`, `¯x`, `▁x`, +//! `⏞x`, `⏟x`, `⟡x`), function names (`sin x`, `log_2 x`, `lim┬(n→∞) x`) and placeholders (`⬚`). +//! Postfix operators apply to the atom before them: `^` `_` scripts, combining accents, `┬` `┴` +//! limits. `/` `⁄` `∕` `¦` take the operands on either side (an operand is a run of atoms up to +//! an operator or space; a parenthesised operand loses its parentheses). `^`/`_` with nothing +//! before them are pre-scripts. A top-level `#` numbers the equation (`E=mc^2#(1)`). + +use crate::math::{Arg, FracKind, MAX_DEPTH, MClass, MNode, MRun, ScriptKind, any_too_deep, is_nary_char, math_class, merge_runs, too_deep}; + +/// Function names set upright and applied to the following operand. +pub const FUNCTION_NAMES: &[&str] = &[ + "arccos", "arccot", "arccsc", "arcsec", "arcsin", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc", "csch", "deg", "det", "dim", "exp", + "gcd", "hom", "inf", "ker", "lg", "lim", "liminf", "limsup", "ln", "log", "max", "min", "mod", "Pr", "sec", "sech", "sin", "sinh", "sup", "tan", + "tanh", +]; + +/// Length of the longest of [`FUNCTION_NAMES`]. +const MAX_NAME: usize = { + let (mut max, mut i) = (0, 0); + while i < FUNCTION_NAMES.len() { + if FUNCTION_NAMES[i].len() > max { + max = FUNCTION_NAMES[i].len(); + } + i += 1; + } + max +}; + +/// Function names whose subscripts go underneath (`lim_(n→∞)`). +const LIMIT_NAMES: &[&str] = &["lim", "liminf", "limsup", "max", "min", "sup", "inf", "det", "gcd", "Pr"]; + +/// The empty-argument placeholder. +pub const PLACEHOLDER: char = '⬚'; + +/// Parse linear-format text. Never fails: what isn't understood stays as text. +pub fn parse(s: &str) -> Arg { + let mut p = P { c: s.chars().take(20_000).collect(), i: 0, depth: 0, no_limits: 0, steps: 0 }; + let mut out = Vec::new(); + // Stray closers at the top level are text. + while p.i < p.c.len() { + out.extend(p.expr(&[])); + if let Some(c) = p.peek() { + p.i += 1; + out.push(text(c)); + } + } + let mut out = clean(out); + // `expr#(n)`: a numbered equation. + if out.iter().any(|n| matches!(n, MNode::Run(r) if r.text.contains('#'))) { + out = vec![MNode::EqArr { rows: vec![out] }]; + } + out +} + +fn text(c: char) -> MNode { + MNode::Run(MRun::new(c.to_string())) +} + +/// Closing bracket for an opening one. +pub fn closer(c: char) -> Option { + Some(match c { + '(' => ')', + '[' => ']', + '{' => '}', + '⟨' => '⟩', + '〈' => '〉', + '⌈' => '⌉', + '⌊' => '⌋', + '⟦' => '⟧', + '〖' => '〗', + _ => return None, + }) +} + +fn is_closer(c: char) -> bool { + matches!(c, ')' | ']' | '}' | '⟩' | '〉' | '⌉' | '⌋' | '⟧' | '〗' | '┤') +} + +/// Prefix characters that make a structure from the operand after them. +fn is_prefix(c: char) -> bool { + matches!(c, '√' | '∛' | '∜' | '█' | '■' | '▭' | '□' | '¯' | '▁' | '⟡' | '⏞' | '⏟' | '⏜' | '⏝' | '⎴' | '⎵') || is_nary_char(c) +} + +/// Combining marks used as accents. +pub fn is_combining(c: char) -> bool { + matches!(c as u32, 0x300..=0x36F | 0x20D0..=0x20FF) +} + +/// Characters that end an operand. +fn is_boundary(c: char) -> bool { + matches!(c, ' ' | '&' | '@' | '#' | '▒' | '│' | '/' | '⁄' | '∕' | '¦' | '\u{2061}') + || is_closer(c) + || matches!(math_class(c), MClass::Bin | MClass::Rel | MClass::Punct) +} + +struct P { + c: Vec, + i: usize, + depth: usize, + /// Inside a brace's operand: limits are left for the brace. + no_limits: usize, + /// Work done (bounds hostile input). + steps: usize, +} + +const MAX_STEPS: usize = 400_000; + +impl P { + fn peek(&self) -> Option { + self.c.get(self.i).copied() + } + fn peek_at(&self, k: usize) -> Option { + self.c.get(self.i + k).copied() + } + fn tired(&mut self) -> bool { + self.steps += 1; + self.steps > MAX_STEPS || self.depth > MAX_DEPTH + } + /// The rest of the input as text (when too deep or too long). + fn rest(&mut self) -> Arg { + let t: String = self.c.get(self.i..).unwrap_or(&[]).iter().collect(); + self.i = self.c.len(); + if t.is_empty() { Vec::new() } else { vec![MNode::Run(MRun::new(t))] } + } + + /// A sequence up to one of `stops` (not consumed) or the end. + fn expr(&mut self, stops: &[char]) -> Arg { + self.depth += 1; + let mut out: Arg = Vec::new(); + if self.tired() { + self.depth -= 1; + return self.rest(); + } + while let Some(c) = self.peek() { + if stops.contains(&c) || (is_closer(c) && !stops.is_empty()) { + break; + } + if self.tired() { + out.extend(self.rest()); + break; + } + match c { + '/' | '⁄' | '∕' | '¦' => { + self.i += 1; + let start = operand_start(&out); + // A numerator that is already deep stays as it is: `/` is text then. + if out.get(start..).is_some_and(any_too_deep) { + out.push(text(c)); + continue; + } + let num = strip_parens(out.drain(start..).collect()); + let den = self.operand(stops); + let kind = match c { + '⁄' => FracKind::Skewed, + '∕' => FracKind::Linear, + '¦' => FracKind::NoBar, + _ => FracKind::Bar, + }; + out.push(MNode::Frac { kind, num, den }); + } + '^' | '_' => { + let pre = match out.last() { + None => true, + Some(MNode::Run(r)) => r.text == " " || r.text.chars().all(|c| is_boundary(c) && c != '#'), + _ => false, + }; + if pre { + out.push(self.prescript(stops)); + } else { + self.i += 1; + let s = self.script_operand(stops); + if out.last().is_some_and(too_deep) { + out.push(text(c)); + out.extend(s); + } else { + let base = out.pop(); + out.push(attach_script(base, c == '^', s)); + } + } + } + '┬' | '┴' => { + self.i += 1; + let lim = self.script_operand(stops); + if out.last().is_some_and(too_deep) { + out.push(text(c)); + out.extend(lim); + } else { + let e = out.pop().map(|n| vec![n]).unwrap_or_default(); + out.push(MNode::Lim { upper: c == '┴', e: strip_parens(e), lim }); + } + } + c if is_combining(c) => { + self.i += 1; + if out.last().is_some_and(too_deep) { + out.push(text(c)); + } else { + let e = out.pop().map(|n| strip_parens(vec![n])).unwrap_or_default(); + out.push(MNode::Acc { chr: c, e }); + } + } + '▒' | '\u{2061}' => self.i += 1, + ' ' => { + self.i += 1; + out.push(text(' ')); + } + _ => { + let atom = self.atom(stops); + out.extend(atom); + } + } + } + self.depth -= 1; + out + } + + /// One atom (a character, a bracket group, a structure or a function application). + fn atom(&mut self, stops: &[char]) -> Arg { + let Some(c) = self.peek() else { return Vec::new() }; + if let Some(close) = closer(c) { + self.i += 1; + return vec![self.group(Some(c), &[close])]; + } + if c == '├' { + self.i += 1; + if let Some(d) = self.explicit_group() { + return vec![d]; + } + return vec![self.group(None, &[')', ']', '}', '⟩', '⌉', '⌋', '|', '‖'])]; + } + if (c == '|' || c == '‖') && self.c.get(self.i + 1..).is_some_and(|r| r.contains(&c)) { + self.i += 1; + return vec![self.group(Some(c), &[c])]; + } + if is_prefix(c) { + self.i += 1; + return vec![self.prefix(c, stops)]; + } + if c.is_ascii_alphabetic() { + return self.word(stops); + } + self.i += 1; + vec![text(c)] + } + + /// A bracket group after its opening character; `closers` end it. + fn group(&mut self, open: Option, closers: &[char]) -> MNode { + let mut stops: Vec = closers.to_vec(); + stops.extend(['┤', '│']); + let mut elems = Vec::new(); + let mut end = None; + loop { + let e = self.expr(&stops); + elems.push(e); + match self.peek() { + Some('│') => self.i += 1, + Some('┤') => { + self.i += 1; + break; + } + Some(c) if closers.contains(&c) => { + self.i += 1; + end = Some(c); + break; + } + // Another closer: the group ends unclosed there. + _ => break, + } + if self.tired() { + break; + } + } + let beg = open.filter(|c| *c != '〖'); + let end = end.filter(|c| *c != '〗'); + let sep = if elems.len() > 1 { Some('|') } else { None }; + MNode::Delim { beg, end, sep, grow: true, shp_match: false, elems } + } + + /// `├X … Y┤`: brackets given explicitly (any bracket characters, either side optional), the + /// cursor just after `├`. `None` (cursor unchanged) when there's no matching `┤`. + fn explicit_group(&mut self) -> Option { + let mut level = 1usize; + let mut k = self.i; + while let Some(c) = self.c.get(k) { + match c { + '├' => level += 1, + '┤' => { + level -= 1; + if level == 0 { + break; + } + } + _ => {} + } + k += 1; + } + if level != 0 { + return None; + } + let bracket = |c: &char| matches!(c, '(' | ')' | '[' | ']' | '{' | '}' | '⟨' | '⟩' | '⌈' | '⌉' | '⌊' | '⌋' | '|' | '‖' | '⟦' | '⟧'); + let mut inner: Vec = self.c.get(self.i..k).unwrap_or(&[]).to_vec(); + self.i = k + 1; + let beg = if inner.first().is_some_and(bracket) { Some(inner.remove(0)) } else { None }; + // A closing bracket at the end that pairs with an opening one inside is content. + let paired = |inner: &[char]| -> bool { + let Some((last, body)) = inner.split_last() else { return false }; + let opener = match last { + ')' => '(', + ']' => '[', + '}' => '{', + '⟩' => '⟨', + '⌉' => '⌈', + '⌋' => '⌊', + '⟧' => '⟦', + _ => return false, + }; + body.iter().filter(|c| **c == opener).count() > body.iter().filter(|c| *c == last).count() + }; + let end = if !inner.is_empty() && inner.last().is_some_and(bracket) && !paired(&inner) { inner.pop() } else { None }; + let mut sub = P { c: inner, i: 0, depth: self.depth + 1, no_limits: 0, steps: self.steps }; + let mut elems = vec![Vec::new()]; + while sub.i < sub.c.len() { + if let Some(e) = elems.last_mut() { + e.extend(sub.expr(&['│'])); + } + match sub.peek() { + Some('│') => { + sub.i += 1; + elems.push(Vec::new()); + } + Some(c) => { + sub.i += 1; + if let Some(e) = elems.last_mut() { + e.push(text(c)); + } + } + None => {} + } + if sub.tired() { + break; + } + } + self.steps = sub.steps; + let sep = if elems.len() > 1 { Some('|') } else { None }; + Some(MNode::Delim { beg, end, sep, grow: true, shp_match: false, elems }) + } + + /// A prefix structure after its character. + fn prefix(&mut self, c: char, stops: &[char]) -> MNode { + match c { + '√'..='∜' => { + let mut deg = match c { + '∛' => vec![text('3')], + '∜' => vec![text('4')], + _ => Vec::new(), + }; + let mut e = self.operand(stops); + if let Some(k) = e.iter().position(|n| matches!(n, MNode::Run(r) if r.text == "&")) { + let rest = e.split_off(k + 1); + e.pop(); + deg = e; + e = rest; + } + MNode::Rad { deg_hide: deg.is_empty(), deg, e } + } + '█' | '■' => { + if self.peek() == Some('(') { + self.i += 1; + let inner = self.expr(&[')']); + if self.peek() == Some(')') { + self.i += 1; + } + grid(c == '■', inner) + } else { + text(c) + } + } + '▭' => MNode::BorderBox { hide: [false; 4], strike: [false; 4], e: self.operand(stops) }, + '□' => MNode::Boxed { e: self.operand(stops) }, + '¯' => MNode::Bar { top: true, e: self.operand(stops) }, + '▁' => MNode::Bar { top: false, e: self.operand(stops) }, + '⟡' => MNode::Phant { show: false, zero_wid: false, zero_asc: false, zero_desc: false, e: self.operand(stops) }, + // A label (`┴`/`┬`) after the operand belongs to the whole brace. + '⏞' | '⏜' | '⎴' | '⏟' | '⏝' | '⎵' => { + self.no_limits += 1; + let e = self.operand(stops); + self.no_limits -= 1; + MNode::GroupChr { chr: c, top: matches!(c, '⏞' | '⏜' | '⎴'), e } + } + _ => { + // N-ary operator: limits, then the operand. + let (mut sub, mut sup) = (Vec::new(), Vec::new()); + let (mut has_sub, mut has_sup) = (false, false); + loop { + match self.peek() { + Some('_') => { + self.i += 1; + sub = self.script_operand(stops); + has_sub = true; + } + Some('^') => { + self.i += 1; + sup = self.script_operand(stops); + has_sup = true; + } + _ => break, + } + } + if self.peek() == Some('▒') { + self.i += 1; + } + let e = if self.peek().is_some_and(|c| c == ' ') { Vec::new() } else { self.operand(stops) }; + MNode::Nary { chr: c, lim_loc: None, grow: false, sub_hide: !has_sub, sup_hide: !has_sup, sub, sup, e } + } + } + } + + /// Letters: a function application when they spell a function name, else one letter. + fn word(&mut self, stops: &[char]) -> Arg { + let start = self.i; + let mut end = start; + // Look no further than the longest name (and one past it, to see the run is longer): a + // scan to the end of every run made a long run of letters quadratic (CWE-407). + while end - start <= MAX_NAME && self.c.get(end).is_some_and(char::is_ascii_alphabetic) { + end += 1; + } + let word: String = if end - start > MAX_NAME { String::new() } else { self.c.get(start..end).unwrap_or(&[]).iter().collect() }; + if !FUNCTION_NAMES.contains(&word.as_str()) { + self.i = start + 1; + return vec![text(self.c.get(start).copied().unwrap_or(' '))]; + } + self.i = end; + let plain = vec![MNode::Run(MRun::plain(word.clone()))]; + // Limits and scripts on the name. + let name = match self.peek() { + Some('┬') | Some('┴') => { + let upper = self.peek() == Some('┴'); + self.i += 1; + let lim = self.script_operand(stops); + vec![MNode::Lim { upper, e: plain, lim }] + } + Some('_') if LIMIT_NAMES.contains(&word.as_str()) => { + self.i += 1; + let lim = self.script_operand(stops); + vec![MNode::Lim { upper: false, e: plain, lim }] + } + Some(c @ ('^' | '_')) => { + self.i += 1; + let s = self.script_operand(stops); + let mut n = attach_script(Some(MNode::Run(MRun::plain(word.clone()))), c == '^', s); + if let Some(c2 @ ('^' | '_')) = self.peek() { + self.i += 1; + let s2 = self.script_operand(stops); + n = attach_script(Some(n), c2 == '^', s2); + } + vec![n] + } + _ => plain, + }; + // The operand, after an optional space or function-application mark. + if matches!(self.peek(), Some(' ') | Some('\u{2061}') | Some('▒')) { + self.i += 1; + } + let next = self.peek(); + let applies = next.is_some_and(|c| !stops.contains(&c) && !is_closer(c) && (!is_boundary(c) || c == '(' || matches!(c, '−' | '-' | '+'))); + if !applies { + return name; + } + let e = self.operand(stops); + if e.is_empty() { + return name; + } + vec![MNode::Func { name, e }] + } + + /// Pre-scripts (`_a^b x`): the scripts, then the base. + fn prescript(&mut self, stops: &[char]) -> MNode { + let (mut sub, mut sup) = (Vec::new(), Vec::new()); + while let Some(c @ ('^' | '_')) = self.peek() { + self.i += 1; + let s = self.script_operand(stops); + if c == '^' { + sup = s; + } else { + sub = s; + } + } + if self.peek() == Some('▒') { + self.i += 1; + } + let base = self.operand(stops); + MNode::Script { kind: ScriptKind::Pre, base, sub, sup } + } + + /// An operand: a parenthesised group (parentheses dropped) or a run of atoms with their + /// postfix scripts, accents and limits, up to an operator or space. + fn operand(&mut self, stops: &[char]) -> Arg { + if self.tired() { + return self.rest(); + } + self.depth += 1; + let mut out: Arg = Vec::new(); + if self.peek() == Some('(') { + self.i += 1; + let g = self.group(Some('('), &[')']); + let inner = match g { + MNode::Delim { elems, end: Some(')'), .. } if elems.len() == 1 => elems.into_iter().next().unwrap_or_default(), + other => vec![other], + }; + out = inner; + self.postfix(&mut out, stops, true); + } else if self.peek() == Some('〖') { + self.i += 1; + let g = self.group(Some('〖'), &['〗']); + out = match g { + MNode::Delim { elems, .. } if elems.len() == 1 => elems.into_iter().next().unwrap_or_default(), + other => vec![other], + }; + self.postfix(&mut out, stops, true); + } else { + // A signed operand (`e^-x`, `−b`). + if let Some(c @ ('−' | '-' | '+' | '±' | '∓')) = self.peek() + && self.peek_at(1).is_some_and(|n| !is_boundary(n)) + { + self.i += 1; + out.push(text(c)); + } + while let Some(c) = self.peek() { + if stops.contains(&c) || is_boundary(c) || self.tired() { + break; + } + let a = self.atom(stops); + if a.is_empty() { + break; + } + out.extend(a); + self.postfix(&mut out, stops, false); + } + if out.is_empty() + && let Some(c) = self.peek() + && c != ' ' + && !stops.contains(&c) + && !is_closer(c) + && !matches!(c, '/' | '⁄' | '∕' | '¦' | '&' | '@' | '#' | '│' | '▒') + { + // A lone operator as operand. + self.i += 1; + out.push(text(c)); + } + } + self.depth -= 1; + out + } + + /// Postfix operators on the last atom of `out` (`whole`: on all of `out` as one group). + fn postfix(&mut self, out: &mut Arg, stops: &[char], whole: bool) { + while let Some(c) = self.peek() { + if self.tired() { + break; + } + let base = |out: &mut Arg| -> Option { + if whole && out.len() != 1 { + if out.is_empty() { + return None; + } + return Some(MNode::Delim { beg: None, end: None, sep: None, grow: true, shp_match: false, elems: vec![std::mem::take(out)] }); + } + out.pop() + }; + if !(matches!(c, '^' | '_') || matches!(c, '┬' | '┴') && self.no_limits == 0 || is_combining(c)) { + break; + } + // The base is already deep: the operator is text (see `too_deep`). + let deep = if whole { any_too_deep(out) } else { out.last().is_some_and(too_deep) }; + self.i += 1; + if deep { + let s = if matches!(c, '^' | '_' | '┬' | '┴') { self.script_operand(stops) } else { Vec::new() }; + out.push(text(c)); + out.extend(s); + continue; + } + match c { + '^' | '_' => { + let s = self.script_operand(stops); + let b = base(out); + out.push(attach_script(b, c == '^', s)); + } + '┬' | '┴' => { + let lim = self.script_operand(stops); + let b = base(out); + out.push(MNode::Lim { upper: c == '┴', e: b.map(|n| vec![n]).unwrap_or_default(), lim }); + } + _ => { + let b = base(out); + out.push(MNode::Acc { chr: c, e: b.map(|n| strip_parens(vec![n])).unwrap_or_default() }); + } + } + } + } + + /// A script or limit: a parenthesised group, a structure, a number, a signed operand, a run of + /// letters (and digits) or one other character. + fn script_operand(&mut self, stops: &[char]) -> Arg { + if self.tired() { + return self.rest(); + } + self.depth += 1; + let out = match self.peek() { + Some('(') => { + self.i += 1; + match self.group(Some('('), &[')']) { + MNode::Delim { elems, end: Some(')'), .. } if elems.len() == 1 => elems.into_iter().next().unwrap_or_default(), + other => vec![other], + } + } + Some('〖') => { + self.i += 1; + match self.group(Some('〖'), &['〗']) { + MNode::Delim { elems, .. } if elems.len() == 1 => elems.into_iter().next().unwrap_or_default(), + other => vec![other], + } + } + Some(c) if is_prefix(c) || closer(c).is_some() => self.atom(stops), + Some(c) if c.is_ascii_digit() => { + let start = self.i; + while self.peek().is_some_and(|k| k.is_alphanumeric() || k == '.') { + self.i += 1; + } + let t: String = self.c.get(start..self.i).unwrap_or(&[]).iter().collect(); + vec![MNode::Run(MRun::new(t))] + } + Some(c @ ('−' | '-' | '+' | '±' | '∓')) if self.peek_at(1).is_some_and(|n| !is_boundary(n)) => { + self.i += 1; + let mut v = vec![text(c)]; + v.extend(self.script_operand(stops)); + v + } + Some(c) if c.is_alphabetic() => { + // Word (#1609): the whole run of letters and digits up to a space or operator, so + // `V_dd` is V with a subscript `dd`, exactly like `V_(dd)`. + let start = self.i; + while self.peek().is_some_and(char::is_alphanumeric) { + self.i += 1; + } + let t: String = self.c.get(start..self.i).unwrap_or(&[]).iter().collect(); + // A known function name stays upright, as Word builds it. + vec![MNode::Run(if FUNCTION_NAMES.contains(&t.as_str()) { MRun::plain(t) } else { MRun::new(t) })] + } + Some(c) if !stops.contains(&c) && c != ' ' && !is_closer(c) => { + self.i += 1; + vec![text(c)] + } + _ => Vec::new(), + }; + self.depth -= 1; + out + } +} + +/// Where the trailing operand of `out` starts (after the last operator or space). +fn operand_start(out: &[MNode]) -> usize { + let mut k = out.len(); + while k > 0 { + match out.get(k - 1) { + Some(MNode::Run(r)) if r.text.chars().all(is_boundary) => break, + _ => k -= 1, + } + } + k +} + +/// A single parenthesised group used as an operand loses its parentheses. +pub fn strip_parens(mut a: Arg) -> Arg { + if a.len() == 1 + && let Some(MNode::Delim { beg: Some('('), end: Some(')'), elems, .. } | MNode::Delim { beg: None, end: None, elems, .. }) = a.first_mut() + && elems.len() == 1 + { + return elems.pop().unwrap_or_default(); + } + a +} + +fn attach_script(base: Option, sup: bool, s: Arg) -> MNode { + match base { + Some(MNode::Script { kind: ScriptKind::Sub, base, sub, .. }) if sup => MNode::Script { kind: ScriptKind::SubSup, base, sub, sup: s }, + Some(MNode::Script { kind: ScriptKind::Sup, base, sup: sp, .. }) if !sup => MNode::Script { kind: ScriptKind::SubSup, base, sub: s, sup: sp }, + b => { + let base = b.map(|b| vec![b]).unwrap_or_default(); + if sup { + MNode::Script { kind: ScriptKind::Sup, base, sub: Vec::new(), sup: s } + } else { + MNode::Script { kind: ScriptKind::Sub, base, sub: s, sup: Vec::new() } + } + } + } +} + +/// `█(a@b)` equation array / `■(a&b@c&d)` matrix from the parsed inside. +fn grid(matrix: bool, inner: Arg) -> MNode { + let mut rows: Vec> = vec![vec![Vec::new()]]; + for n in inner { + match &n { + MNode::Run(r) if r.text == "@" => rows.push(vec![Vec::new()]), + MNode::Run(r) if r.text == "&" && matrix => { + if let Some(row) = rows.last_mut() { + row.push(Vec::new()); + } + } + _ => { + if let Some(cell) = rows.last_mut().and_then(|r| r.last_mut()) { + cell.push(n); + } + } + } + } + if matrix { + MNode::Matrix { rows, col_jc: Vec::new() } + } else { + MNode::EqArr { rows: rows.into_iter().map(|r| r.into_iter().flatten().collect()).collect() } + } +} + +/// Drop placeholders and build-up spaces, merge runs; recursively. +fn clean(a: Arg) -> Arg { + let mut out = Vec::with_capacity(a.len()); + for n in a { + match n { + MNode::Run(r) if r.text == " " || r.text.chars().all(|c| c == PLACEHOLDER) => {} + other => out.push(map_args(other, &clean)), + } + } + merge_runs(out) +} + +/// Apply `f` to every argument of a node. +pub fn map_args(n: MNode, f: &dyn Fn(Arg) -> Arg) -> MNode { + match n { + MNode::Run(r) => MNode::Run(r), + MNode::Frac { kind, num, den } => MNode::Frac { kind, num: f(num), den: f(den) }, + MNode::Script { kind, base, sub, sup } => MNode::Script { kind, base: f(base), sub: f(sub), sup: f(sup) }, + MNode::Rad { deg, deg_hide, e } => MNode::Rad { deg: f(deg), deg_hide, e: f(e) }, + MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub, sup, e } => { + MNode::Nary { chr, lim_loc, grow, sub_hide, sup_hide, sub: f(sub), sup: f(sup), e: f(e) } + } + MNode::Delim { beg, end, sep, grow, shp_match, elems } => { + MNode::Delim { beg, end, sep, grow, shp_match, elems: elems.into_iter().map(f).collect() } + } + MNode::Func { name, e } => MNode::Func { name: f(name), e: f(e) }, + MNode::Lim { upper, e, lim } => MNode::Lim { upper, e: f(e), lim: f(lim) }, + MNode::Acc { chr, e } => MNode::Acc { chr, e: f(e) }, + MNode::Bar { top, e } => MNode::Bar { top, e: f(e) }, + MNode::BorderBox { hide, strike, e } => MNode::BorderBox { hide, strike, e: f(e) }, + MNode::Boxed { e } => MNode::Boxed { e: f(e) }, + MNode::GroupChr { chr, top, e } => MNode::GroupChr { chr, top, e: f(e) }, + MNode::EqArr { rows } => MNode::EqArr { rows: rows.into_iter().map(f).collect() }, + MNode::Matrix { rows, col_jc } => MNode::Matrix { rows: rows.into_iter().map(|r| r.into_iter().map(f).collect()).collect(), col_jc }, + MNode::Phant { show, zero_wid, zero_asc, zero_desc, e } => MNode::Phant { show, zero_wid, zero_asc, zero_desc, e: f(e) }, + } +} diff --git a/crates/wordmodel/src/math_symbols.rs b/crates/wordmodel/src/math_symbols.rs new file mode 100644 index 000000000..28b3a7f83 --- /dev/null +++ b/crates/wordmodel/src/math_symbols.rs @@ -0,0 +1,390 @@ +//! Named math characters: Math AutoCorrect while typing in an equation (`\alpha` + space → α) +//! and LaTeX conversion. Names are the usual TeX names plus the linear-format operators +//! (`\sqrt` → √, `\above` → ┴, `\matrix` → ■…). + +/// (name without the backslash, character). When a character has several names the first one +/// listed is the one LaTeX output uses. +pub const SYMBOLS: &[(&str, char)] = &[ + // Greek. + ("alpha", 'α'), + ("beta", 'β'), + ("gamma", 'γ'), + ("delta", 'δ'), + ("epsilon", 'ϵ'), + ("varepsilon", 'ε'), + ("zeta", 'ζ'), + ("eta", 'η'), + ("theta", 'θ'), + ("vartheta", 'ϑ'), + ("iota", 'ι'), + ("kappa", 'κ'), + ("varkappa", 'ϰ'), + ("lambda", 'λ'), + ("mu", 'μ'), + ("nu", 'ν'), + ("xi", 'ξ'), + ("omicron", 'ο'), + ("pi", 'π'), + ("varpi", 'ϖ'), + ("rho", 'ρ'), + ("varrho", 'ϱ'), + ("sigma", 'σ'), + ("varsigma", 'ς'), + ("tau", 'τ'), + ("upsilon", 'υ'), + ("phi", 'ϕ'), + ("varphi", 'φ'), + ("chi", 'χ'), + ("psi", 'ψ'), + ("omega", 'ω'), + ("Alpha", 'Α'), + ("Beta", 'Β'), + ("Gamma", 'Γ'), + ("Delta", 'Δ'), + ("Epsilon", 'Ε'), + ("Zeta", 'Ζ'), + ("Eta", 'Η'), + ("Theta", 'Θ'), + ("Iota", 'Ι'), + ("Kappa", 'Κ'), + ("Lambda", 'Λ'), + ("Mu", 'Μ'), + ("Nu", 'Ν'), + ("Xi", 'Ξ'), + ("Omicron", 'Ο'), + ("Pi", 'Π'), + ("Rho", 'Ρ'), + ("Sigma", 'Σ'), + ("Tau", 'Τ'), + ("Upsilon", 'Υ'), + ("Phi", 'Φ'), + ("Chi", 'Χ'), + ("Psi", 'Ψ'), + ("Omega", 'Ω'), + ("digamma", 'ϝ'), + // Binary operators. + ("pm", '±'), + ("mp", '∓'), + ("times", '×'), + ("div", '÷'), + ("cdot", '⋅'), + ("bullet", '∙'), + ("ast", '∗'), + ("star", '⋆'), + ("circ", '∘'), + ("oplus", '⊕'), + ("ominus", '⊖'), + ("otimes", '⊗'), + ("oslash", '⊘'), + ("odot", '⊙'), + ("cup", '∪'), + ("cap", '∩'), + ("uplus", '⊎'), + ("sqcup", '⊔'), + ("sqcap", '⊓'), + ("vee", '∨'), + ("wedge", '∧'), + ("setminus", '∖'), + ("backslash", '\\'), + ("wr", '≀'), + ("diamond", '⋄'), + ("dagger", '†'), + ("ddagger", '‡'), + ("amalg", '⨿'), + ("dotplus", '∔'), + ("ltimes", '⋉'), + ("rtimes", '⋊'), + // Relations. + ("le", '≤'), + ("leq", '≤'), + ("ge", '≥'), + ("geq", '≥'), + ("ne", '≠'), + ("neq", '≠'), + ("approx", '≈'), + ("equiv", '≡'), + ("sim", '∼'), + ("simeq", '≃'), + ("cong", '≅'), + ("propto", '∝'), + ("ll", '≪'), + ("gg", '≫'), + ("prec", '≺'), + ("succ", '≻'), + ("preceq", '≼'), + ("succeq", '≽'), + ("subset", '⊂'), + ("supset", '⊃'), + ("subseteq", '⊆'), + ("supseteq", '⊇'), + ("sqsubseteq", '⊑'), + ("sqsupseteq", '⊒'), + ("in", '∈'), + ("ni", '∋'), + ("notin", '∉'), + ("perp", '⊥'), + ("parallel", '∥'), + ("mid", '∣'), + ("nmid", '∤'), + ("asymp", '≍'), + ("doteq", '≐'), + ("models", '⊨'), + ("vdash", '⊢'), + ("dashv", '⊣'), + ("bowtie", '⋈'), + ("coloneq", '≔'), + ("defeq", '≝'), + ("triangleq", '≜'), + ("nless", '≮'), + ("ngtr", '≯'), + ("nleq", '≰'), + ("ngeq", '≱'), + ("nsim", '≁'), + ("ncong", '≇'), + ("nequiv", '≢'), + ("nsubset", '⊄'), + ("nsupset", '⊅'), + ("nsubseteq", '⊈'), + ("nsupseteq", '⊉'), + ("subsetneq", '⊊'), + ("supsetneq", '⊋'), + ("lesssim", '≲'), + ("gtrsim", '≳'), + ("therefore", '∴'), + ("because", '∵'), + // Arrows. + ("rightarrow", '→'), + ("to", '→'), + ("leftarrow", '←'), + ("gets", '←'), + ("leftrightarrow", '↔'), + ("uparrow", '↑'), + ("downarrow", '↓'), + ("updownarrow", '↕'), + ("Rightarrow", '⇒'), + ("Leftarrow", '⇐'), + ("Leftrightarrow", '⇔'), + ("Uparrow", '⇑'), + ("Downarrow", '⇓'), + ("Updownarrow", '⇕'), + ("longrightarrow", '⟶'), + ("longleftarrow", '⟵'), + ("longleftrightarrow", '⟷'), + ("Longrightarrow", '⟹'), + ("Longleftarrow", '⟸'), + ("Longleftrightarrow", '⟺'), + ("implies", '⟹'), + ("iff", '⟺'), + ("mapsto", '↦'), + ("longmapsto", '⟼'), + ("nearrow", '↗'), + ("searrow", '↘'), + ("swarrow", '↙'), + ("nwarrow", '↖'), + ("hookleftarrow", '↩'), + ("hookrightarrow", '↪'), + ("leftharpoonup", '↼'), + ("leftharpoondown", '↽'), + ("rightharpoonup", '⇀'), + ("rightharpoondown", '⇁'), + ("rightleftharpoons", '⇌'), + ("leftrightharpoons", '⇋'), + ("circlearrowleft", '↺'), + ("circlearrowright", '↻'), + // Big operators. + ("sum", '∑'), + ("prod", '∏'), + ("coprod", '∐'), + ("int", '∫'), + ("iint", '∬'), + ("iiint", '∭'), + ("oint", '∮'), + ("oiint", '∯'), + ("oiiint", '∰'), + ("bigcup", '⋃'), + ("bigcap", '⋂'), + ("bigvee", '⋁'), + ("bigwedge", '⋀'), + ("bigoplus", '⨁'), + ("bigotimes", '⨂'), + ("bigodot", '⨀'), + ("biguplus", '⨄'), + ("bigsqcup", '⨆'), + // Brackets. + ("langle", '⟨'), + ("rangle", '⟩'), + ("lceil", '⌈'), + ("rceil", '⌉'), + ("lfloor", '⌊'), + ("rfloor", '⌋'), + ("lbrack", '['), + ("rbrack", ']'), + ("lbrace", '{'), + ("rbrace", '}'), + ("Vert", '‖'), + ("norm", '‖'), + ("vert", '|'), + ("lbbrack", '⟦'), + ("rbbrack", '⟧'), + // Miscellaneous. + ("infty", '∞'), + ("partial", '∂'), + ("nabla", '∇'), + ("forall", '∀'), + ("exists", '∃'), + ("nexists", '∄'), + ("neg", '¬'), + ("lnot", '¬'), + ("emptyset", '∅'), + ("varnothing", '∅'), + ("hbar", 'ℏ'), + ("ell", 'ℓ'), + ("Re", 'ℜ'), + ("Im", 'ℑ'), + ("wp", '℘'), + ("aleph", 'ℵ'), + ("beth", 'ℶ'), + ("gimel", 'ℷ'), + ("daleth", 'ℸ'), + ("degree", '°'), + ("angle", '∠'), + ("measuredangle", '∡'), + ("sphericalangle", '∢'), + ("rightangle", '∟'), + ("triangle", '△'), + ("square", '□'), + ("blacksquare", '■'), + ("qed", '∎'), + ("top", '⊤'), + ("bot", '⊥'), + ("cdots", '⋯'), + ("ldots", '…'), + ("dots", '…'), + ("vdots", '⋮'), + ("ddots", '⋱'), + ("adots", '⋰'), + ("prime", '′'), + ("dprime", '″'), + ("trprime", '‴'), + ("dd", 'ⅆ'), + ("Dd", 'ⅅ'), + ("ee", 'ⅇ'), + ("ii", 'ⅈ'), + ("jj", 'ⅉ'), + ("imath", 'ı'), + ("jmath", 'ȷ'), + ("complement", '∁'), + ("Angstrom", 'Å'), + ("mho", '℧'), + ("eth", 'ð'), + ("flat", '♭'), + ("natural", '♮'), + ("sharp", '♯'), + ("clubsuit", '♣'), + ("diamondsuit", '♢'), + ("heartsuit", '♡'), + ("spadesuit", '♠'), + // Spaces. + ("quad", '\u{2003}'), + ("emsp", '\u{2003}'), + ("ensp", '\u{2002}'), + ("thicksp", '\u{2005}'), + ("medsp", '\u{205F}'), + ("thinsp", '\u{2009}'), + ("hairsp", '\u{200A}'), + ("nbsp", '\u{A0}'), + ("zwsp", '\u{200B}'), + // Linear-format operators (build-up). + ("sqrt", '√'), + ("cbrt", '∛'), + ("qdrt", '∜'), + ("matrix", '■'), + ("eqarray", '█'), + ("above", '┴'), + ("below", '┬'), + ("of", '▒'), + ("naryand", '▒'), + ("funcapply", '\u{2061}'), + ("box", '□'), + ("rect", '▭'), + ("overbar", '¯'), + ("underbar", '▁'), + ("phantom", '⟡'), + ("overbrace", '⏞'), + ("underbrace", '⏟'), + ("overparen", '⏜'), + ("underparen", '⏝'), + ("atop", '¦'), + ("over", '/'), + ("ldiv", '∕'), + ("sdiv", '⁄'), + ("begin", '〖'), + ("end", '〗'), + ("left", '├'), + ("right", '┤'), + ("eqno", '#'), + ("placeholder", '⬚'), + // Accents (combining, typed after the base). + ("hat", '\u{302}'), + ("check", '\u{30C}'), + ("tilde", '\u{303}'), + ("acute", '\u{301}'), + ("grave", '\u{300}'), + ("dot", '\u{307}'), + ("ddot", '\u{308}'), + ("dddot", '\u{20DB}'), + ("breve", '\u{306}'), + ("bar", '\u{305}'), + ("Bar", '\u{33F}'), + ("vec", '\u{20D7}'), + ("tvec", '\u{20E1}'), + ("lhvec", '\u{20D0}'), + ("hvec", '\u{20D1}'), +]; + +/// The character for a name (`alpha` → α), including `doubleR`, `scriptL`, `frakturg`. +pub fn symbol(name: &str) -> Option { + if let Some((_, c)) = SYMBOLS.iter().find(|(n, _)| *n == name) { + return Some(*c); + } + for (prefix, scr) in [("double", crate::math::MScr::DoubleStruck), ("script", crate::math::MScr::Script), ("fraktur", crate::math::MScr::Fraktur)] + { + if let Some(rest) = name.strip_prefix(prefix) { + let mut cs = rest.chars(); + if let (Some(c), None) = (cs.next(), cs.next()) + && c.is_ascii_alphanumeric() + { + let m = crate::math::math_alnum(c, false, false, scr); + if m != c { + return Some(m); + } + } + } + } + None +} + +/// The (first) name of a character. +pub fn name_of(c: char) -> Option<&'static str> { + SYMBOLS.iter().find(|(_, s)| *s == c).map(|(n, _)| *n) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_resolve() { + assert_eq!(symbol("alpha"), Some('α')); + assert_eq!(symbol("doubleR"), Some('ℝ')); + assert_eq!(symbol("scriptL"), Some('ℒ')); + assert_eq!(symbol("nonesuch"), None); + assert_eq!(name_of('≤'), Some("le")); + // No name is listed twice. + let mut names: Vec<&str> = SYMBOLS.iter().map(|(n, _)| *n).collect(); + names.sort_unstable(); + let before = names.len(); + names.dedup(); + assert_eq!(before, names.len()); + } +} diff --git a/crates/wordmodel/src/math_tree.rs b/crates/wordmodel/src/math_tree.rs new file mode 100644 index 000000000..45d2c603a --- /dev/null +++ b/crates/wordmodel/src/math_tree.rs @@ -0,0 +1,104 @@ +//! Structural access to an equation tree: the arguments of a node, and every run in a tree. + +use crate::math::{Arg, MAX_DEPTH, MNode, MRun}; + +/// Number of child arguments of a node. +pub fn child_count(n: &MNode) -> usize { + match n { + MNode::Run(_) => 0, + MNode::Frac { .. } | MNode::Func { .. } | MNode::Lim { .. } => 2, + MNode::Rad { .. } => 2, + MNode::Script { .. } | MNode::Nary { .. } => 3, + MNode::Delim { elems, .. } => elems.len(), + MNode::EqArr { rows } => rows.len(), + MNode::Matrix { rows, .. } => rows.iter().map(|r| r.len()).sum(), + MNode::Acc { .. } | MNode::Bar { .. } | MNode::BorderBox { .. } | MNode::Boxed { .. } | MNode::GroupChr { .. } | MNode::Phant { .. } => 1, + } +} + +/// Child argument `i` of a node (see [`child_count`] for the numbering: fraction num 0 / den 1; +/// script base 0 / sub 1 / sup 2; radical degree 0 / base 1; n-ary sub 0 / sup 1 / operand 2; +/// function name 0 / argument 1; limit base 0 / limit 1; matrices row by row). +pub fn child(n: &MNode, i: usize) -> Option<&Arg> { + match (n, i) { + (MNode::Frac { num, .. }, 0) => Some(num), + (MNode::Frac { den, .. }, 1) => Some(den), + (MNode::Script { base, .. }, 0) => Some(base), + (MNode::Script { sub, .. }, 1) => Some(sub), + (MNode::Script { sup, .. }, 2) => Some(sup), + (MNode::Rad { deg, .. }, 0) => Some(deg), + (MNode::Rad { e, .. }, 1) => Some(e), + (MNode::Nary { sub, .. }, 0) => Some(sub), + (MNode::Nary { sup, .. }, 1) => Some(sup), + (MNode::Nary { e, .. }, 2) => Some(e), + (MNode::Delim { elems, .. }, i) => elems.get(i), + (MNode::Func { name, .. }, 0) => Some(name), + (MNode::Func { e, .. }, 1) => Some(e), + (MNode::Lim { e, .. }, 0) => Some(e), + (MNode::Lim { lim, .. }, 1) => Some(lim), + ( + MNode::Acc { e, .. } + | MNode::Bar { e, .. } + | MNode::BorderBox { e, .. } + | MNode::Boxed { e } + | MNode::GroupChr { e, .. } + | MNode::Phant { e, .. }, + 0, + ) => Some(e), + (MNode::EqArr { rows }, i) => rows.get(i), + (MNode::Matrix { rows, .. }, i) => rows.iter().flatten().nth(i), + _ => None, + } +} + +pub fn child_mut(n: &mut MNode, i: usize) -> Option<&mut Arg> { + match (n, i) { + (MNode::Frac { num, .. }, 0) => Some(num), + (MNode::Frac { den, .. }, 1) => Some(den), + (MNode::Script { base, .. }, 0) => Some(base), + (MNode::Script { sub, .. }, 1) => Some(sub), + (MNode::Script { sup, .. }, 2) => Some(sup), + (MNode::Rad { deg, .. }, 0) => Some(deg), + (MNode::Rad { e, .. }, 1) => Some(e), + (MNode::Nary { sub, .. }, 0) => Some(sub), + (MNode::Nary { sup, .. }, 1) => Some(sup), + (MNode::Nary { e, .. }, 2) => Some(e), + (MNode::Delim { elems, .. }, i) => elems.get_mut(i), + (MNode::Func { name, .. }, 0) => Some(name), + (MNode::Func { e, .. }, 1) => Some(e), + (MNode::Lim { e, .. }, 0) => Some(e), + (MNode::Lim { lim, .. }, 1) => Some(lim), + ( + MNode::Acc { e, .. } + | MNode::Bar { e, .. } + | MNode::BorderBox { e, .. } + | MNode::Boxed { e } + | MNode::GroupChr { e, .. } + | MNode::Phant { e, .. }, + 0, + ) => Some(e), + (MNode::EqArr { rows }, i) => rows.get_mut(i), + (MNode::Matrix { rows, .. }, i) => rows.iter_mut().flatten().nth(i), + _ => None, + } +} + +/// Call `f` on every run in the tree. +pub fn each_run_mut(root: &mut Arg, f: &mut dyn FnMut(&mut MRun)) { + fn go(a: &mut Arg, f: &mut dyn FnMut(&mut MRun), depth: usize) { + if depth > MAX_DEPTH { + return; + } + for n in a.iter_mut() { + if let MNode::Run(r) = n { + f(r); + } + for c in 0..child_count(n) { + if let Some(ch) = child_mut(n, c) { + go(ch, f, depth + 1); + } + } + } + } + go(root, f, 0); +} diff --git a/crates/wordmodel/src/missing.rs b/crates/wordmodel/src/missing.rs new file mode 100644 index 000000000..d60d93c14 --- /dev/null +++ b/crates/wordmodel/src/missing.rs @@ -0,0 +1,143 @@ +//! Pictures a reader could not bring in or an exporter could not draw, for strict export (#6005). +//! Kept beside the document while it is open; never saved, never drawn, and not part of the +//! document's content (`==` ignores it). + +use serde::{Deserialize, Serialize}; + +/// Most pictures remembered per document; past it [`MissingPictures::truncated`] is set. +pub const MAX_MISSING_PICTURES: usize = 256; + +/// Most characters of a picture's part or story name a record keeps (names come from the file). +pub const MAX_PICTURE_NAME_CHARS: usize = 1024; + +/// Why a picture is not in the document or not drawn. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum MissingPictureReason { + /// The relationship names a part the file does not contain, or no relationship at all. + MissingPart, + /// The target is outside the file (`TargetMode="External"`); it is never fetched. + ExternalTarget, + /// The picture only links to a file (`r:link`) and holds no embedded copy. + LinkOnly, + /// The bytes are a picture format we know but they do not decode. + Undecodable, + /// The bytes are not a picture format we can draw. + UnsupportedFormat, +} + +impl MissingPictureReason { + /// The words used in reports (`"missing part"`, `"external target"`, `"link only"`, + /// `"undecodable"`, `"unsupported format"`). + pub fn label(self) -> &'static str { + match self { + Self::MissingPart => "missing part", + Self::ExternalTarget => "external target", + Self::LinkOnly => "link only", + Self::Undecodable => "undecodable", + Self::UnsupportedFormat => "unsupported format", + } + } + + /// The reason a [`code`](Self::code) names. + pub fn from_code(code: &str) -> Option { + [Self::MissingPart, Self::ExternalTarget, Self::LinkOnly, Self::Undecodable, Self::UnsupportedFormat].into_iter().find(|r| r.code() == code) + } + + /// The stable snake_case code the JSON report carries (`"missing_part"`, `"external_target"`, + /// `"link_only"`, `"undecodable"`, `"unsupported_format"`); never reworded. + pub fn code(self) -> &'static str { + match self { + Self::MissingPart => "missing_part", + Self::ExternalTarget => "external_target", + Self::LinkOnly => "link_only", + Self::Undecodable => "undecodable", + Self::UnsupportedFormat => "unsupported_format", + } + } +} + +/// One picture that is missing. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct MissingPicture { + /// The target as written, or the part name it resolved to (a media key for a decoded one). + pub part: String, + /// The story part it is in (`word/document.xml`, `word/header2.xml`), as the source file + /// names it; empty when not known. + pub story: String, + pub reason: MissingPictureReason, +} + +/// A capped list of [`MissingPicture`]s, with the source names a report needs to say where a +/// picture is (a reader fills them; a document made or opened another way leaves them empty). +#[derive(Clone, Debug, Default)] +pub struct MissingPictures { + pub list: Vec, + /// More than [`MAX_MISSING_PICTURES`] were found; the rest are not listed. + pub truncated: bool, + /// Pictures found, those past the cap included (saturating). Each distinct one counts once. + pub total: u64, + /// The main story's part name in the source file (`word/document.xml`). + pub main: String, + /// A header, footer, note or comment story's part id ([`crate::Document::parts`]) to the + /// part it was read from (`word/header2.xml`). Ids freed and reused after opening may name + /// the old part. + pub part_names: std::collections::BTreeMap>, + /// A [`crate::Document::media`] key to the part its bytes were read from + /// (`word/media/image2.bin`). + pub media_parts: std::collections::BTreeMap, + /// What `list` holds, for an O(1) test of "already there" (a hostile file has many). + seen: std::collections::HashSet, + /// Hashes of the distinct pictures past the cap (8 bytes each), so the total counts them once. + unlisted: std::collections::HashSet, + /// The chunk parts whose unlisted pictures [`Self::add_unlisted`] has counted, so a chunk + /// imported twice counts them once. + counted_chunks: std::collections::HashSet, +} + +impl MissingPictures { + /// Add `n` pictures of the chunk part `chunk` that its reader counted but did not list. The + /// same chunk part counts once however often it is imported (its listed pictures dedupe + /// through [`Self::push`]). + pub fn add_unlisted(&mut self, chunk: &str, n: u64) { + if n > 0 && self.counted_chunks.insert(chunk.to_string()) { + self.total = self.total.saturating_add(n); + } + } + + /// Add one (the same part, story and reason once). + pub fn push(&mut self, mut item: MissingPicture) { + for name in [&mut item.part, &mut item.story] { + if name.len() > MAX_PICTURE_NAME_CHARS { + *name = name.chars().take(MAX_PICTURE_NAME_CHARS).collect(); + } + } + if self.seen.contains(&item) { + return; + } + if self.list.len() >= MAX_MISSING_PICTURES { + self.truncated = true; + if self.unlisted.insert(picture_hash(&item)) { + self.total = self.total.saturating_add(1); + } + return; + } + self.total = self.total.saturating_add(1); + self.seen.insert(item.clone()); + self.list.push(item); + } +} + +/// A 64-bit hash of a record, to tell the ones past a cap apart without keeping them. +pub fn picture_hash(item: &MissingPicture) -> u64 { + use std::hash::{Hash, Hasher}; + let mut h = std::collections::hash_map::DefaultHasher::new(); + item.hash(&mut h); + h.finish() +} + +impl PartialEq for MissingPictures { + /// Not part of the document's content. + fn eq(&self, _: &Self) -> bool { + true + } +} diff --git a/crates/wordmodel/src/numbering.rs b/crates/wordmodel/src/numbering.rs new file mode 100644 index 000000000..5ab94ea87 --- /dev/null +++ b/crates/wordmodel/src/numbering.rs @@ -0,0 +1,776 @@ +//! Lists: abstract numbering definitions (9 levels each), numbering instances, and the counter +//! state used to compute list labels. + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::props::{Align, CharProps}; +use crate::section::NumFormat; +use crate::{DocError, free_id}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum LevelSuffix { + #[default] + Tab, + Space, + Nothing, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Level { + pub start: u32, + pub format: NumFormat, + /// Label template: `%1.`, `%1.%2.`, or the bullet character. + pub text: String, + pub align: Align, + /// Left indent of the text, points. + pub indent: f32, + /// Hanging indent (label sits this far left of `indent`), points. + pub hanging: f32, + pub suffix: LevelSuffix, + /// Label formatting (bullet font etc.). + pub chr: CharProps, + /// Restart after a higher level (true = after any higher level, the default). + pub restart: bool, + /// Restart only after this level (1-based, Word's `w:lvlRestart`): a level 3 that restarts + /// after level 1 keeps counting across level 2 items. `None` restarts after the level just + /// above. Ignored when `restart` is false. + #[serde(skip_serializing_if = "Option::is_none")] + pub restart_after: Option, + /// The tab stop that follows the number (Word's "Add tab stop at"), points from the + /// paragraph's left edge. + #[serde(skip_serializing_if = "Option::is_none")] + pub tab: Option, + /// Show all levels as decimal (legal numbering). + pub legal: bool, + /// Paragraph style linked to this level (e.g. Heading 1 for outline numbering). + pub style: Option, +} + +impl Default for Level { + fn default() -> Self { + Level { + start: 1, + format: NumFormat::Decimal, + text: "%1.".into(), + align: Align::Left, + indent: 36.0, + hanging: 18.0, + suffix: LevelSuffix::Tab, + chr: CharProps::default(), + restart: true, + restart_after: None, + tab: None, + legal: false, + style: None, + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct AbstractNum { + pub id: u32, + pub name: Option, + pub levels: Vec, + /// Word's list identity (`w:nsid`, 8 hex digits), which it uses to match lists across + /// documents (paste, compare); kept from the file. + #[serde(skip_serializing_if = "Option::is_none")] + pub nsid: Option, + /// The list template the definition came from (`w:tmpl`); kept from the file. + #[serde(skip_serializing_if = "Option::is_none")] + pub tmpl: Option, + /// `w:multiLevelType` from the file (`singleLevel`, `multilevel`, `hybridMultilevel`); + /// `None`: derived from the levels. + #[serde(skip_serializing_if = "Option::is_none")] + pub multi_level_type: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Num { + pub id: u32, + pub abstract_id: u32, + /// Per-level start overrides (level, start). + pub start_overrides: Vec<(u8, u32)>, + /// Levels this list defines itself instead of taking them from its abstract list (level, + /// definition): Word's `w:lvlOverride/w:lvl`. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub level_overrides: Vec<(u8, Level)>, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct Numbering { + pub abstracts: Vec, + pub nums: Vec, +} + +/// Built-in list kinds for the Bullets / Numbering / Multilevel galleries. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ListKind { + /// • ○ ▪ repeating. + Bullet, + /// A specific bullet character. + BulletChar(char), + /// 1. a. i. + Numbered, + /// 1) a) i) + NumberedParen, + /// I. A. 1. + Outline, + /// A. B. C. + UpperLetter, + /// a) b) c) + LowerLetter, + /// a. b. c. + LowerLetterDot, + /// i. ii. iii. + LowerRoman, + /// 1. 1.1. 1.1.1. + Legal, +} + +impl ListKind { + pub fn parse(s: &str) -> Option { + Some(match s { + "bullet" => ListKind::Bullet, + "numbered" | "number" => ListKind::Numbered, + "numberedParen" => ListKind::NumberedParen, + "outline" => ListKind::Outline, + "upperLetter" => ListKind::UpperLetter, + "lowerLetter" => ListKind::LowerLetter, + "lowerLetterDot" => ListKind::LowerLetterDot, + "lowerRoman" => ListKind::LowerRoman, + "legal" | "multilevel" => ListKind::Legal, + s if s.chars().count() == 1 => ListKind::BulletChar(s.chars().next()?), + _ => return None, + }) + } + pub fn is_bullet(self) -> bool { + matches!(self, ListKind::Bullet | ListKind::BulletChar(_)) + } +} + +/// The bullet a level of a plain bulleted list shows: • ○ ▪ repeating (`level` is zero-based). +pub fn default_bullet(level: usize) -> char { + ['•', '○', '▪'][level % 3] +} + +fn bullet_level(i: usize, c: char) -> Level { + Level { + format: NumFormat::Bullet, + text: c.to_string(), + indent: 36.0 * (i as f32 + 1.0), + hanging: 18.0, + chr: CharProps { + font: match c { + '•' | '▪' | '○' => Some("Arial".into()), + // Plain text characters (a dash bullet) use the paragraph's font. + c if c.is_ascii() || c == '–' => None, + _ => Some("Segoe UI Symbol".into()), + }, + ..CharProps::default() + }, + ..Level::default() + } +} + +/// Nine levels for a list kind. +pub fn levels_for(kind: ListKind) -> Vec { + (0..9) + .map(|i| { + let ind = 36.0 * (i as f32 + 1.0); + let cyc = |a: NumFormat, b: NumFormat, c: NumFormat| match i % 3 { + 0 => a, + 1 => b, + _ => c, + }; + let lvl = |format: NumFormat, text: String| Level { format, text, indent: ind, hanging: 18.0, ..Level::default() }; + let n = i + 1; + match kind { + ListKind::Bullet => bullet_level(i, default_bullet(i)), + ListKind::BulletChar(c) => bullet_level(i, if i == 0 { c } else { default_bullet(i) }), + ListKind::Numbered => lvl(cyc(NumFormat::Decimal, NumFormat::LowerLetter, NumFormat::LowerRoman), format!("%{n}.")), + ListKind::NumberedParen => lvl(cyc(NumFormat::Decimal, NumFormat::LowerLetter, NumFormat::LowerRoman), format!("%{n})")), + ListKind::UpperLetter => lvl(cyc(NumFormat::UpperLetter, NumFormat::LowerLetter, NumFormat::LowerRoman), format!("%{n}.")), + ListKind::LowerLetter => lvl(cyc(NumFormat::LowerLetter, NumFormat::LowerRoman, NumFormat::Decimal), format!("%{n})")), + ListKind::LowerLetterDot => lvl(cyc(NumFormat::LowerLetter, NumFormat::LowerRoman, NumFormat::Decimal), format!("%{n}.")), + ListKind::LowerRoman => lvl(cyc(NumFormat::LowerRoman, NumFormat::LowerLetter, NumFormat::Decimal), format!("%{n}.")), + ListKind::Outline => { + let f = [NumFormat::UpperRoman, NumFormat::UpperLetter, NumFormat::Decimal, NumFormat::LowerLetter, NumFormat::LowerRoman] + .get(i) + .copied() + .unwrap_or(NumFormat::Decimal); + lvl(f, format!("%{n}.")) + } + ListKind::Legal => { + let text: String = (1..=n).map(|k| format!("%{k}.")).collect(); + Level { format: NumFormat::Decimal, text, indent: 18.0 + 25.0 * i as f32, hanging: 18.0 + 7.0 * i as f32, ..Level::default() } + } + } + }) + .collect() +} + +impl Numbering { + pub fn num(&self, id: u32) -> Option<&Num> { + self.nums.iter().find(|n| n.id == id) + } + pub fn abstract_of(&self, num: u32) -> Option<&AbstractNum> { + let n = self.num(num)?; + self.abstracts.iter().find(|a| a.id == n.abstract_id) + } + /// Whether a paragraph in list `num` at `level` gets a label ([`Counters::next_label`] gives + /// one), without counting it. + pub fn shows_label(&self, num: u32, level: u8) -> bool { + self.abstract_of(num).is_some() && self.level(num, level.min(8)).is_some() + } + pub fn level(&self, num: u32, level: u8) -> Option<&Level> { + if let Some((_, l)) = self.num(num)?.level_overrides.iter().find(|(i, _)| *i == level) { + return Some(l); + } + self.abstract_of(num)?.levels.get(level as usize) + } + /// Create a new list of `kind`; returns the `num` id. List ids run out only past `u32::MAX` + /// lists, more than memory holds; then nothing is added and the id is 0, "no list". + pub fn add_list(&mut self, kind: ListKind) -> u32 { + self.try_add_list(kind).unwrap_or(0) + } + /// [`Numbering::add_list`], or an error, adding nothing, when list ids are all taken. + pub fn try_add_list(&mut self, kind: ListKind) -> crate::Result { + self.try_add_levels(None, levels_for(kind)) + } + /// A new list definition (`name`, `levels`) and a `num` for it; returns the `num` id. Both ids + /// are [`free_id`]s, so the largest ids a file can hold don't make them collide with its lists; + /// an error, adding nothing, when they are all taken. + pub fn try_add_levels(&mut self, name: Option, levels: Vec) -> crate::Result { + let aid = free_id(self.abstracts.iter().map(|a| a.id).max(), self.abstracts.iter().map(|a| a.id), 0); + let (Some(aid), Some(nid)) = (aid, self.free_num_id()) else { return Err(DocError::Invalid("no free list id".into())) }; + self.abstracts.push(AbstractNum { id: aid, name, levels, ..Default::default() }); + self.nums.push(Num { id: nid, abstract_id: aid, ..Default::default() }); + Ok(nid) + } + /// A `num` id no list has (`num` 0 means "no list"), `None` when they are all taken. + fn free_num_id(&self) -> Option { + free_id(self.nums.iter().map(|n| n.id).max(), self.nums.iter().map(|n| n.id), 1) + } + /// A new `num` that restarts numbering of an existing list (`None`: no such list, or list ids + /// are all taken). + pub fn restart(&mut self, num: u32) -> Option { + let source = self.num(num)?; + let a = source.abstract_id; + let level_overrides = source.level_overrides.clone(); + let nid = self.free_num_id()?; + // Every level starts again at its own effective start, so a level this list defines itself keeps its start. + let levels = self.abstracts.iter().find(|x| x.id == a).map_or(0, |x| x.levels.len()); + let start_overrides = (0..levels).filter_map(|i| self.level(num, i as u8).map(|l| (i as u8, l.start))).collect(); + self.nums.push(Num { id: nid, abstract_id: a, start_overrides, level_overrides }); + Some(nid) + } + /// Find an existing list whose first level matches `kind`, so consecutive bullet clicks reuse it. + pub fn find_kind(&self, kind: ListKind) -> Option { + let want = levels_for(kind); + let first = want.first()?; + self.nums + .iter() + .find(|n| self.abstract_of(n.id).and_then(|a| a.levels.first()).is_some_and(|l| l.format == first.format && l.text == first.text)) + .map(|n| n.id) + } +} + +/// Running list counters while walking a document in order. +#[derive(Default, Debug, Clone)] +pub struct Counters { + /// Per abstract list: current value per level. + state: HashMap, + /// Which levels of which nums have started: a num's start override for a level applies + /// once, when that level of that num is first used (ECMA-376 Part 1, 17.9.27 startOverride). + seen: HashMap<(u32, u8), bool>, + /// Per abstract list: whether each level has counted yet (a value of 0 can be a real count when a + /// level starts at 0). + flags: HashMap, + /// The last explicitly rendered label and template at each abstract-list level. + labels: HashMap<(u32, u8), CountedLabel>, +} + +#[derive(Debug, Clone)] +struct CountedLabel { + label: Option, + levels: Vec, +} + +/// Whether a level has counted yet, and the value its next item takes if a start override says so. +#[derive(Default, Debug, Clone, Copy)] +struct LevelFlags { + started: bool, + first: Option, +} + +impl Counters { + /// The current value of each level of `num`'s list (0 for a level not counted yet). + pub fn values(&self, numbering: &Numbering, num: u32) -> Option<[u32; 9]> { + let abs = numbering.abstract_of(num)?; + Some(self.state.get(&abs.id).copied().unwrap_or([0; 9])) + } + + /// Advance the counter for `num`/`level` and return the label (`"1."`, `"a)"`, `"•"`). + pub fn next_label(&mut self, numbering: &Numbering, num: u32, level: u8) -> Option<(String, Level)> { + let n = numbering.num(num)?; + let abs = numbering.abstract_of(num)?; + let lv = (level as usize).min(8); + // This list's own definition of a level wins over its abstract list's. + let level_def = |k: usize| numbering.level(num, k as u8); + // A "none" level (Word's numFmt none) still counts and restarts the levels below it; only + // its number reads as nothing, so its literal text ("RESPONSE:") shows (#4985). + let def = level_def(lv)?.clone(); + let st = self.state.entry(abs.id).or_insert([0; 9]); + let fl = self.flags.entry(abs.id).or_default(); + // Only this level's override, not the num's others: the counters are shared by every + // num of the abstract list, so restarting its other levels here would renumber + // paragraphs of the other nums. A heading style can use one num for levels 1-4 and + // another, with start overrides on every level, for 5-9; its first level-5 heading + // must not restart the chapter count. + if let std::collections::hash_map::Entry::Vacant(e) = self.seen.entry((num, lv as u8)) { + e.insert(true); + if let Some((_, start)) = n.start_overrides.iter().find(|(l, _)| *l as usize == lv) + && let (Some(s), Some(f)) = (st.get_mut(lv), fl.get_mut(lv)) + { + *s = 0; + *f = LevelFlags { started: false, first: Some(*start) }; + } + } + // A level above that hasn't counted yet counts its first number here, as in Word: after + // 1.1 and 1.2 with no level-1 item before them, the next level-1 item is 2, whether or not + // the deeper level's text shows it. + for (k, (v, f)) in st.iter_mut().zip(fl.iter_mut()).enumerate().take(lv) { + if f.started { + continue; + } + let start = match self.seen.entry((num, k as u8)) { + std::collections::hash_map::Entry::Vacant(e) => { + e.insert(true); + n.start_overrides.iter().find(|(l, _)| *l as usize == k).map(|(_, s)| *s) + } + std::collections::hash_map::Entry::Occupied(_) => None, + }; + *v = start.or(f.first).unwrap_or_else(|| level_def(k).map_or(1, |l| l.start)); + *f = LevelFlags { started: true, first: None }; + } + if let (Some(v), Some(f)) = (st.get_mut(lv), fl.get_mut(lv)) { + *v = if f.started { v.saturating_add(1) } else { f.first.unwrap_or(def.start) }; + *f = LevelFlags { started: true, first: None }; + } + // Deeper levels restart, unless they never restart or only restart after a level above + // this one (Word's lvlRestart). + for (d, (deeper, f)) in st.iter_mut().zip(fl.iter_mut()).enumerate().skip(lv + 1) { + let restarts = level_def(d).is_none_or(|l| l.restart && l.restart_after.is_none_or(|k| lv < k as usize)); + if restarts { + *deeper = 0; + *f = LevelFlags::default(); + self.labels.remove(&(abs.id, d as u8)); + } + } + let vals = *st; + let started = *fl; + let numeric = def.format != NumFormat::Bullet; + let label = if numeric { render_label(numbering, num, &def, &vals, &started) } else { def.text.clone() }; + // A "none" level shows its literal text, but has no number of its own for a reference to + // show (#4985): it is kept as one that shows none. + let counted = if def.format == NumFormat::None { + CountedLabel { label: None, levels: Vec::new() } + } else { + CountedLabel { label: numeric.then(|| label.clone()), levels: template_levels(&def.text) } + }; + self.labels.insert((abs.id, lv as u8), counted); + Some((label, def)) + } + /// Which levels of `num`'s list have counted a paragraph yet (a level never used has no + /// number of its own to show in a reference). + pub fn started(&self, numbering: &Numbering, num: u32) -> Option<[bool; 9]> { + let abs = numbering.abstract_of(num)?; + Some(std::array::from_fn(|k| self.labels.get(&(abs.id, k as u8)).is_some_and(|last| last.label.is_some()))) + } + + /// The label `level` of `num`'s list shows at this point of the walk, without counting: what + /// the last paragraph at that level showed (`"14."` while walking clause 14's sub-clauses). + /// `None` for a bullet or "none" level, or a level the list doesn't define. + pub fn current_label(&self, numbering: &Numbering, num: u32, level: u8) -> Option { + let abs = numbering.abstract_of(num)?; + if let Some(last) = self.labels.get(&(abs.id, level.min(8))) { + return last.label.clone(); + } + // A deeper item can count a level implicitly without rendering a paragraph there. + let def = numbering.level(num, level.min(8))?; + if matches!(def.format, NumFormat::None | NumFormat::Bullet) { + return None; + } + let vals = self.state.get(&abs.id).copied().unwrap_or([0; 9]); + let started = self.flags.get(&abs.id).copied().unwrap_or_default(); + Some(render_label(numbering, num, def, &vals, &started)) + } + + /// Levels shown by the last rendered label, or the current definition for an implicit level. + pub fn current_template_levels(&self, numbering: &Numbering, num: u32, level: u8) -> Vec { + if let Some(abs) = numbering.abstract_of(num) + && let Some(last) = self.labels.get(&(abs.id, level.min(8))) + { + return last.levels.clone(); + } + numbering.level(num, level.min(8)).map(|def| template_levels(&def.text)).unwrap_or_default() + } +} + +/// The zero-based levels a label template shows (`"%1.%2"` → `[0, 1]`, `"(%3)"` → `[2]`). +pub fn template_levels(text: &str) -> Vec { + let mut out = Vec::new(); + let mut chars = text.chars().take(MAX_LEVEL_TEXT).peekable(); + while let Some(c) = chars.next() { + if c == '%' + && let Some(d) = chars.peek().and_then(|d| d.to_digit(10)) + { + chars.next(); + let k = (d as usize).saturating_sub(1).min(8); + if !out.contains(&k) { + out.push(k); + } + } + } + out +} + +/// Render a list label without advancing counters, retaining the existing hostile-input caps. +fn render_label(numbering: &Numbering, num: u32, def: &Level, vals: &[u32; 9], started: &[LevelFlags; 9]) -> String { + let level_def = |k: usize| numbering.level(num, k as u8); + let mut label = String::new(); + // Word's level text is short; .docx and the lists command already cap it at 64. + let mut chars = def.text.chars().take(MAX_LEVEL_TEXT).peekable(); + while let Some(c) = chars.next() { + if c == '%' + && let Some(d) = chars.peek().and_then(|d| d.to_digit(10)) + { + chars.next(); + let k = (d as usize).saturating_sub(1).min(8); + let val = vals.get(k).copied().unwrap_or(0); + let counted = started.get(k).is_some_and(|f| f.started); + let val = if counted { val } else { level_def(k).map(|l| l.start).unwrap_or(1) }; + // Legal numbering (w:isLgl) shows every level in Arabic digits, whatever its own + // format (二 of a Chinese outline becomes 2 in "2.1"); a "none" level stays empty. + let fmt = match level_def(k).map(|l| l.format).unwrap_or(NumFormat::Decimal) { + f @ (NumFormat::DecimalZero | NumFormat::None) => f, + _ if def.legal => NumFormat::Decimal, + f => f, + }; + label.push_str(&fmt.format(val)); + if label.len() > MAX_LABEL_BYTES { + break; + } + } else { + label.push(c); + } + } + label +} + +/// Longest level text (in characters) expanded into a list label. +const MAX_LEVEL_TEXT: usize = 64; +/// Labels stop growing once they pass this many bytes. +const MAX_LABEL_BYTES: usize = 256; + +#[cfg(test)] +mod tests { + use super::*; + + /// Read-only counter APIs retain the renderer's bounds and never advance a list. + #[test] + fn reference_labels_are_bounded_and_do_not_advance_counters() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.level_overrides = vec![(0, Level { text: "%1".repeat(30_000), ..Level::default() })]; + } + let mut c = Counters::default(); + let label = c.next_label(&n, id, 0).unwrap().0; + let before = c.values(&n, id); + assert_eq!(c.current_label(&n, id, 0).as_deref(), Some(label.as_str())); + assert_eq!(c.values(&n, id), before); + assert!(label.len() <= 512); + assert_eq!(template_levels(&"%1%9".repeat(30_000)), [0, 8]); + assert!(c.started(&n, id).unwrap()[0]); + } + + #[test] + fn last_ancestor_label_keeps_the_instance_that_rendered_it() { + let mut numbering = Numbering::default(); + let first = numbering.add_list(ListKind::Numbered); + numbering.abstracts[0].levels[0].text = "Clause %1.".into(); + let mut second = numbering.num(first).unwrap().clone(); + second.id = first + 1; + second.level_overrides = vec![ + (0, Level { text: "Part %1.%2.".into(), ..Level::default() }), + (1, Level { format: NumFormat::LowerLetter, text: "(%2)".into(), ..Level::default() }), + ]; + let second_id = second.id; + numbering.nums.push(second); + let mut counters = Counters::default(); + assert_eq!(counters.next_label(&numbering, first, 0).unwrap().0, "Clause 1."); + assert_eq!(counters.next_label(&numbering, second_id, 1).unwrap().0, "(a)"); + assert_eq!(counters.current_label(&numbering, second_id, 0).as_deref(), Some("Clause 1.")); + assert_eq!(counters.current_template_levels(&numbering, second_id, 0), vec![0]); + assert_eq!(counters.started(&numbering, second_id).unwrap()[..2], [true, true]); + } + + #[test] + fn a_huge_level_text_gives_a_bounded_label() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let huge = Level { format: NumFormat::UpperRoman, text: "%1".repeat(30_000), start: u32::MAX, ..Level::default() }; + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.level_overrides = vec![(0, huge)]; + } + let mut c = Counters::default(); + let label = c.next_label(&n, id, 0).unwrap().0; + assert!(label.len() <= 512, "label is {} bytes", label.len()); + assert!(label.starts_with("MMMM")); + } + + #[test] + fn a_list_s_own_level_definitions_win() { + // A specification's article numbers: level 1 counts parts with no label of its own, level 2 + // shows "part.article" with a zero-padded article (Word's lvlOverride/lvl). + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let lvl = |format, text: &str| Level { format, text: text.into(), ..Level::default() }; + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.level_overrides = vec![(1, lvl(NumFormat::Decimal, "")), (2, lvl(NumFormat::DecimalZero, "%2.%3"))]; + } + let mut c = Counters::default(); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, ""); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, "1.01"); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, "1.02"); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, ""); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, "2.01"); + // Levels it doesn't define still come from the abstract list; level 0 took its first number + // with the first level-1 item, so this is its second. + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "2."); + assert_eq!(n.level(id, 2).map(|l| l.format), Some(NumFormat::DecimalZero)); + } + + #[test] + fn a_none_level_shows_its_text_and_still_counts() { + // As Word shows them (#4985): a "none" level's literal text is its label, its own number + // reads as nothing (also in legal numbering), and it restarts the levels below it. + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let lvl = |format, text: &str| Level { format, text: text.into(), ..Level::default() }; + let legal = Level { legal: true, ..lvl(NumFormat::Decimal, "%2.%3") }; + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.level_overrides = vec![(0, lvl(NumFormat::None, "RESPONSE:")), (1, lvl(NumFormat::None, "[%2]")), (2, legal)]; + } + assert!(n.shows_label(id, 0)); + let mut c = Counters::default(); + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "RESPONSE:"); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "[]"); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, ".1"); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, ".2"); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "[]"); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, ".1"); + } + + #[test] + fn a_start_override_restarts_only_its_own_level_of_a_shared_list() { + // One abstract list numbers the headings through two nums: one for levels 1-4, and one + // restarting every level (startOverride 1 on 0..8) for levels 5-9, first used deep in + // chapter 2. Chapter 3 is still chapter 3, and the deep heading counts from 1. + let mut n = Numbering::default(); + let chapters = n.add_list(ListKind::Legal); + let abs = n.abstract_of(chapters).map(|a| a.id).unwrap(); + let deep = chapters + 1; + n.nums.push(Num { id: deep, abstract_id: abs, start_overrides: (0..9).map(|l| (l, 1)).collect(), ..Num::default() }); + let mut c = Counters::default(); + let mut l = |num, lv| c.next_label(&n, num, lv).unwrap().0; + assert_eq!(l(chapters, 0), "1."); + assert_eq!(l(chapters, 0), "2."); + assert_eq!(l(chapters, 1), "2.1."); + assert_eq!(l(chapters, 2), "2.1.1."); + assert_eq!(l(chapters, 3), "2.1.1.1."); + assert_eq!(l(deep, 4), "2.1.1.1.1.", "the deep num restarts only level 5"); + assert_eq!(l(deep, 4), "2.1.1.1.2."); + assert_eq!(l(chapters, 0), "3.", "and not the chapter count"); + assert_eq!(l(chapters, 1), "3.1."); + } + + #[test] + fn levels_restart_only_after_their_restart_level() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Legal); + if let Some(a) = n.abstracts.first_mut() { + // Level 3 restarts only after level 1; level 2 never restarts. + a.levels[2].restart_after = Some(1); + a.levels[2].text = "(%3)".into(); + a.levels[1].restart = false; + } + let mut c = Counters::default(); + let mut l = |lv| c.next_label(&n, id, lv).unwrap().0; + assert_eq!(l(0), "1."); + assert_eq!(l(1), "1.1."); + assert_eq!(l(2), "(1)"); + assert_eq!(l(1), "1.2."); + assert_eq!(l(2), "(2)", "level 2 doesn't restart level 3"); + assert_eq!(l(0), "2."); + assert_eq!(l(1), "2.3.", "level 2 never restarts"); + assert_eq!(l(2), "(1)", "level 1 restarts level 3"); + } + + #[test] + fn a_deeper_item_first_counts_the_levels_above_it() { + // Measured in Word: a level above that hasn't counted yet takes its first number at the + // first deeper item, whether or not that item's text shows it. + let labels_with = |starts: [u32; 3], texts: [&str; 3], overrides: &[(u8, u32)], seq: &[u8]| { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Legal); + if let Some(a) = n.abstracts.first_mut() { + for (l, (s, t)) in a.levels.iter_mut().zip(starts.into_iter().zip(texts)) { + l.start = s; + l.text = t.into(); + } + } + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.start_overrides = overrides.to_vec(); + } + let mut c = Counters::default(); + seq.iter().map(|lv| c.next_label(&n, id, *lv).unwrap().0).collect::>() + }; + let labels = |starts: [u32; 3], texts: [&str; 3], seq: &[u8]| labels_with(starts, texts, &[], seq); + let std = ["%1.", "%1.%2.", "%1.%2.%3."]; + assert_eq!(labels([1, 1, 1], std, &[1, 1, 0, 0]), ["1.1.", "1.2.", "2.", "3."]); + assert_eq!(labels([10, 1, 1], std, &[1, 1, 1, 1, 0, 0]), ["10.1.", "10.2.", "10.3.", "10.4.", "11.", "12."]); + assert_eq!(labels([1, 1, 1], ["%1.", "(%2)", "%1.%2.%3."], &[1, 1, 0, 0]), ["(1)", "(2)", "2.", "3."]); + assert_eq!(labels([1, 1, 1], std, &[2, 1, 0, 1]), ["1.1.1.", "1.2.", "2.", "2.1."]); + assert_eq!(labels([1, 1, 1], ["%1.", "%1.%2.", "(%3)"], &[2, 1, 0]), ["(1)", "1.2.", "2."]); + assert_eq!(labels([1, 1, 1], std, &[0, 2, 1, 0]), ["1.", "1.1.1.", "1.2.", "2."]); + // The num's own start for the level above counts in its place, as the level's start does. + assert_eq!(labels_with([1, 1, 1], std, &[(0, 10)], &[1, 1, 0]), ["10.1.", "10.2.", "11."]); + assert_eq!(labels_with([1, 1, 1], std, &[(0, 5)], &[1, 0, 1]), ["5.1.", "6.", "6.1."]); + // A level-1 item after the first deeper ones restarts level 2 below it, as usual. + assert_eq!(labels([1, 1, 1], std, &[1, 1, 0, 1, 1, 0, 2]), ["1.1.", "1.2.", "2.", "2.1.", "2.2.", "3.", "3.1.1."]); + } + + #[test] + fn a_level_starting_at_zero_counts_up() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + if let Some(a) = n.abstracts.first_mut() { + a.levels[0].start = 0; + } + let mut c = Counters::default(); + let labels: Vec = (0..4).map(|_| c.next_label(&n, id, 0).unwrap().0).collect(); + assert_eq!(labels, ["0.", "1.", "2.", "3."]); + } + + #[test] + fn a_start_override_of_one_or_zero_is_honoured() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + if let Some(a) = n.abstracts.first_mut() { + a.levels[0].start = 5; + } + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.start_overrides = vec![(0, 1)]; + } + let mut c = Counters::default(); + let labels: Vec = (0..2).map(|_| c.next_label(&n, id, 0).unwrap().0).collect(); + assert_eq!(labels, ["1.", "2."]); + + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.start_overrides = vec![(0, 0)]; + } + let mut c = Counters::default(); + let labels: Vec = (0..3).map(|_| c.next_label(&n, id, 0).unwrap().0).collect(); + assert_eq!(labels, ["0.", "1.", "2."]); + } + + #[test] + fn numbered_labels() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let mut c = Counters::default(); + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "1."); + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "2."); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "a."); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "b."); + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "3."); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "a."); + assert!(c.next_label(&n, 999, 0).is_none()); + } + + #[test] + fn east_asian_labels_and_legal_numbering() { + // A Chinese outline: 第一章, then "一、" items, and a legal level showing 1.1. + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + if let Some(a) = n.abstracts.first_mut() { + a.levels[0] = Level { format: NumFormat::ChineseCounting, text: "第%1章".into(), ..Level::default() }; + a.levels[1] = Level { format: NumFormat::ChineseCounting, text: "%2、".into(), ..Level::default() }; + a.levels[2] = Level { format: NumFormat::ChineseCounting, text: "%1.%2.%3".into(), legal: true, ..Level::default() }; + a.levels[3] = Level { format: NumFormat::ChineseCounting, text: "(%4)".into(), ..Level::default() }; + } + let mut c = Counters::default(); + let mut l = |lv| c.next_label(&n, id, lv).unwrap().0; + assert_eq!(l(0), "第一章"); + assert_eq!(l(1), "一、"); + assert_eq!(l(1), "二、"); + assert_eq!(l(1), "三、"); + assert_eq!(l(2), "1.3.1", "legal numbering is decimal on every level"); + assert_eq!(l(3), "(一)"); + for _ in 0..9 { + l(3); + } + assert_eq!(l(3), "(十一)"); + assert_eq!(l(0), "第二章"); + } + + #[test] + fn legal_labels() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Legal); + let mut c = Counters::default(); + assert_eq!(c.next_label(&n, id, 0).unwrap().0, "1."); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "1.1."); + assert_eq!(c.next_label(&n, id, 2).unwrap().0, "1.1.1."); + assert_eq!(c.next_label(&n, id, 1).unwrap().0, "1.2."); + } + + #[test] + fn restart_and_bullets() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let mut c = Counters::default(); + c.next_label(&n, id, 0); + c.next_label(&n, id, 0); + let r = n.restart(id).unwrap(); + assert_eq!(c.next_label(&n, r, 0).unwrap().0, "1."); + let b = n.add_list(ListKind::Bullet); + assert_eq!(c.next_label(&n, b, 0).unwrap().0, "•"); + assert_eq!(n.find_kind(ListKind::Bullet), Some(b)); + } + + #[test] + fn restart_keeps_the_list_s_own_level_definitions() { + let mut n = Numbering::default(); + let id = n.add_list(ListKind::Numbered); + let own = Level { format: NumFormat::UpperRoman, text: "Part %1".into(), indent: 90.0, hanging: 40.0, start: 5, ..Level::default() }; + if let Some(num) = n.nums.iter_mut().find(|x| x.id == id) { + num.level_overrides.push((0, own.clone())); + } + let r = n.restart(id).unwrap(); + assert_eq!(n.num(r).unwrap().level_overrides, vec![(0, own.clone())]); + assert_eq!(n.level(r, 0), Some(&own)); + let mut c = Counters::default(); + assert_eq!(c.next_label(&n, r, 0).unwrap().0, "Part V"); + } +} diff --git a/crates/wordmodel/src/para.rs b/crates/wordmodel/src/para.rs new file mode 100644 index 000000000..0ed90870b --- /dev/null +++ b/crates/wordmodel/src/para.rs @@ -0,0 +1,809 @@ +//! Paragraphs: text with formatting runs and inline objects. +//! +//! A paragraph's text is one `String`. Formatting is a list of [`Run`]s whose byte lengths sum to +//! the text length. Inline objects (pictures, fields, note references, bookmarks, comment +//! anchors…) sit in the text as U+FFFC; the k-th U+FFFC is `objects[k]`. Special characters: +//! `\t` tab, `\n` manual line break, U+000C page break, U+000E column break. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use serde::{Deserialize, Serialize}; + +use crate::props::{CharProps, ParaProps}; +use crate::section::SectionProps; +use crate::{DocError, Result}; + +mod float; +mod inline_kinds; +mod inline_object; +mod run; + +pub use float::{AltText, Anchor, Float, FloatAlign, MAX_ALT_TEXT, Wrap, WrapText, cap_alt}; +pub use inline_kinds::{NoteKind, PTabAlign, PTabBase, RubyAlign, ShapeKind, ruby_offset, ruby_raise}; +pub use inline_object::{CanvasStyle, GroupChild, InlineObject, MAX_GROUP_CHILDREN}; +pub use run::{COLUMN_BREAK, LINE_BREAK, NB_HYPHEN, NBSP, OBJ, PAGE_BREAK, Run, SOFT_HYPHEN, next_rev}; + +/// A paragraph. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Paragraph { + pub text: String, + pub runs: Vec, + #[serde(default, skip_serializing_if = "ParaProps::is_empty")] + pub props: ParaProps, + /// Formatting of the paragraph mark (used when typing into an empty paragraph). + #[serde(default, skip_serializing_if = "CharProps::is_empty")] + pub mark: CharProps, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub objects: Vec, + /// Set when this paragraph ends a section (OOXML keeps `w:sectPr` on the last paragraph). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub section: Option>, + /// Change counter for caches (not part of equality). + #[serde(skip, default = "next_rev")] + pub rev: u64, +} + +impl PartialEq for Paragraph { + fn eq(&self, o: &Self) -> bool { + self.text == o.text + && self.runs == o.runs + && self.props == o.props + && self.mark == o.mark + && self.objects == o.objects + && self.section == o.section + } +} + +impl Default for Paragraph { + fn default() -> Self { + Paragraph::new() + } +} + +impl Paragraph { + pub fn new() -> Self { + Paragraph { + text: String::new(), + runs: Vec::new(), + props: ParaProps::default(), + mark: CharProps::default(), + objects: Vec::new(), + section: None, + rev: next_rev(), + } + } + /// A paragraph of plain text in one run. + pub fn with_text(text: &str, props: CharProps) -> Self { + let mut p = Paragraph::new(); + let clean: String = text.chars().filter(|c| *c != OBJ).collect(); + if !clean.is_empty() { + p.runs.push(Run { len: clean.len(), props: props.clone() }); + } + p.text = clean; + p.mark = props; + p + } + pub fn styled(mut self, style: &str) -> Self { + self.props.style = Some(style.to_string()); + self + } + pub fn touch(&mut self) { + self.rev = next_rev(); + } + pub fn len(&self) -> usize { + self.text.len() + } + pub fn is_empty(&self) -> bool { + self.text.is_empty() + } + /// The text with objects removed or replaced by their text (fields → result). + pub fn plain_text(&self) -> String { + let mut out = String::with_capacity(self.text.len()); + let mut k = 0; + for c in self.text.chars() { + if c == OBJ { + if let Some(o) = self.objects.get(k) { + out.push_str(o.plain_text()); + } + k += 1; + } else { + out.push(c); + } + } + out + } + /// [`Self::plain_text`] as it reads on screen: characters in runs set in a symbol font + /// (Symbol, Wingdings…) become their standard Unicode equivalents + /// ([`crate::symbol_fonts`]). For copying as text and plain-text export; the stored text is + /// unchanged. + pub fn shown_text(&self) -> String { + let mut out = String::with_capacity(self.text.len()); + let mut k = 0; + let mut runs = self.run_ranges().peekable(); + for (i, c) in self.text.char_indices() { + while runs.peek().is_some_and(|(r, _)| r.end <= i) { + runs.next(); + } + if c == OBJ { + if let Some(o) = self.objects.get(k) { + out.push_str(o.plain_text()); + } + k += 1; + continue; + } + let font = runs.peek().filter(|(r, _)| r.contains(&i)).and_then(|(_, p)| p.shown_font()); + out.push(font.and_then(|f| crate::symbol_fonts::symbol_char(f, c)).unwrap_or(c)); + } + out + } + /// [`Self::plain_text`] without tracked deletions: the text as it reads with every change accepted. + pub fn final_text(&self) -> String { + self.text_without(&self.deleted_ranges()) + } + /// The byte ranges of the tracked deletions. + pub fn deleted_ranges(&self) -> Vec> { + self.run_ranges().filter(|(_, c)| c.del.is_some()).map(|(r, _)| r).collect() + } + /// [`Self::plain_text`] without the text in the byte ranges `dropped` (in any order, overlapping + /// or not). + pub fn text_without(&self, dropped: &[std::ops::Range]) -> String { + let mut sorted = dropped.to_vec(); + sorted.sort_unstable_by_key(|r| r.start); + let mut next = sorted.iter().peekable(); + let mut out = String::with_capacity(self.text.len()); + let mut k = 0; + for (i, c) in self.text.char_indices() { + // By start: once the ranges that end by `i` are skipped, `i` is dropped if the next has begun. + while next.next_if(|r| r.end <= i).is_some() {} + let keep = next.peek().is_none_or(|r| r.start > i); + if c == OBJ { + if keep && let Some(o) = self.objects.get(k) { + out.push_str(o.plain_text()); + } + k += 1; + } else if keep { + out.push(c); + } + } + out + } + + fn check(&self, off: usize) -> Result<()> { + if off > self.text.len() || !self.text.is_char_boundary(off) { + return Err(DocError::BadOffset(off)); + } + Ok(()) + } + + /// Clamp `off` to the text and back to a char boundary. + pub fn clamp(&self, off: usize) -> usize { + let mut o = off.min(self.text.len()); + while o > 0 && !self.text.is_char_boundary(o) { + o -= 1; + } + o + } + + /// Index of the run containing byte `off` and the offset where that run starts. At a run + /// boundary the earlier run wins (typing continues the formatting to the left). + fn run_at(&self, off: usize) -> Option<(usize, usize)> { + let mut start = 0; + for (i, r) in self.runs.iter().enumerate() { + let end = start + r.len; + if off < end || (off == end && off > start) { + return Some((i, start)); + } + start = end; + } + None + } + + /// Formatting in effect at `off` (what typing there would get): the run to the left, or the + /// first run at offset 0, or the paragraph mark when empty. + pub fn props_at(&self, off: usize) -> &CharProps { + if off == 0 { + return self.runs.first().map(|r| &r.props).unwrap_or(&self.mark); + } + self.run_at(off).and_then(|(i, _)| self.runs.get(i)).map(|r| &r.props).unwrap_or(&self.mark) + } + + /// Formatting of the character starting at `off`. + pub fn props_of_char(&self, off: usize) -> &CharProps { + let mut start = 0; + for r in &self.runs { + if off < start + r.len { + return &r.props; + } + start += r.len; + } + &self.mark + } + + /// Runs as (byte range, props). + pub fn run_ranges(&self) -> impl Iterator, &CharProps)> { + let mut s = 0; + self.runs.iter().map(move |r| { + let a = s; + s += r.len; + (a..s, &r.props) + }) + } + + /// Number of objects in `text[..off]`. + fn objects_before(&self, off: usize) -> usize { + self.text.get(..off).map(|t| t.chars().inspect(|_| crate::work::steps(1)).filter(|c| *c == OBJ).count()).unwrap_or(0) + } + + /// The object at byte `off` (which must hold U+FFFC). + pub fn object_at(&self, off: usize) -> Option<&InlineObject> { + if !self.text.get(off..)?.starts_with(OBJ) { + return None; + } + self.objects.get(self.objects_before(off)) + } + pub fn object_at_mut(&mut self, off: usize) -> Option<&mut InlineObject> { + if !self.text.get(off..)?.starts_with(OBJ) { + return None; + } + let k = self.objects_before(off); + self.objects.get_mut(k) + } + /// Byte offsets of all objects, in order. + pub fn object_offsets(&self) -> Vec { + self.text.char_indices().inspect(|_| crate::work::steps(1)).filter(|(_, c)| *c == OBJ).map(|(i, _)| i).collect() + } + /// (byte offset, object) for every object, in text order, in one pass. Prefer it to + /// [`Paragraph::object_offsets`] + [`Paragraph::object_at`], which counts the objects before + /// each offset again, so a loop over a paragraph's objects is quadratic (#3614). + pub fn objects_with_offsets(&self) -> impl Iterator + '_ { + self.text.char_indices().inspect(|_| crate::work::steps(1)).filter(|(_, c)| *c == OBJ).map(|(i, _)| i).zip(self.objects.iter()) + } + + /// Insert plain text (U+FFFC is dropped) at `off` with `props`. Returns the inserted length. + pub fn insert_text(&mut self, off: usize, s: &str, props: &CharProps) -> Result { + self.check(off)?; + let clean: std::borrow::Cow = if s.contains(OBJ) { s.chars().filter(|c| *c != OBJ).collect::().into() } else { s.into() }; + if clean.is_empty() { + return Ok(0); + } + self.text.insert_str(off, &clean); + self.insert_run(off, clean.len(), props.clone()); + self.touch(); + Ok(clean.len()) + } + + /// Insert an object at `off`. + pub fn insert_object(&mut self, off: usize, obj: InlineObject, props: &CharProps) -> Result<()> { + self.check(off)?; + let k = self.objects_before(off); + self.text.insert(off, OBJ); + self.objects.insert(k.min(self.objects.len()), obj); + self.insert_run(off, OBJ.len_utf8(), props.clone()); + self.touch(); + Ok(()) + } + + fn insert_run(&mut self, off: usize, len: usize, props: CharProps) { + // Split the run at `off`, insert, merge neighbours. + let mut start = 0; + let mut idx = self.runs.len(); + for (i, r) in self.runs.iter().enumerate() { + if off <= start { + idx = i; + break; + } + if off < start + r.len { + // split + let left = off - start; + let right = r.len - left; + let p = r.props.clone(); + if let Some(r) = self.runs.get_mut(i) { + r.len = left; + } + self.runs.insert(i + 1, Run { len: right, props: p }); + idx = i + 1; + break; + } + start += r.len; + } + self.runs.insert(idx.min(self.runs.len()), Run { len, props }); + self.normalize(); + } + + /// Merge adjacent equal runs and drop empty ones. + pub fn normalize(&mut self) { + // Runs must end on character boundaries and cover the text exactly (never expected, but + // files lie): clamp each end to the text, push it past a split character, pad the tail. + let n = self.text.len(); + let mut end = 0usize; + let mut out: Vec = Vec::with_capacity(self.runs.len()); + for r in self.runs.drain(..) { + let mut e = end.saturating_add(r.len).min(n); + while !self.text.is_char_boundary(e) { + e += 1; + } + if e == end { + continue; + } + match out.last_mut() { + Some(l) if l.props == r.props => l.len += e - end, + _ => out.push(Run { len: e - end, props: r.props }), + } + end = e; + } + if end < n { + match out.last_mut() { + Some(l) => l.len += n - end, + None => out.push(Run { len: n, props: self.mark.clone() }), + } + } + self.runs = out; + } + + /// Delete `a..b` (bytes). Objects inside are removed. + pub fn delete(&mut self, a: usize, b: usize) -> Result<()> { + let (a, b) = (a.min(b), a.max(b)); + self.check(a)?; + self.check(b)?; + if a == b { + return Ok(()); + } + if self.runs.is_empty() || a == 0 { + // Keep the formatting of what's deleted for the mark when emptying. + if b == self.text.len() { + // Formatting only: the mark is a tracked change of its own or it is not, whatever + // revision the deleted text was part of. + let revisions = (self.mark.ins, self.mark.del, self.mark.fmt_change.clone()); + let mut mark = self.props_of_char(a).clone(); + (mark.ins, mark.del, mark.fmt_change) = revisions; + self.mark = mark; + } + } + let k0 = self.objects_before(a); + let k1 = self.objects_before(b); + if k1 > k0 && k1 <= self.objects.len() { + self.objects.drain(k0..k1); + } + self.text.replace_range(a..b, ""); + let mut start = 0; + for r in &mut self.runs { + let (rs, re) = (start, start + r.len); + start = re; + let ov = re.min(b).saturating_sub(rs.max(a)); + r.len -= ov.min(r.len); + } + self.normalize(); + self.touch(); + Ok(()) + } + + /// Delete `a..b` except the one-character objects at the offsets in `keep` (control + /// markers a deletion leaves in place). + pub fn delete_except(&mut self, a: usize, b: usize, keep: &[usize]) -> Result<()> { + let (a, b) = (a.min(b), a.max(b)); + let mut keep: Vec = + keep.iter().copied().filter(|k| *k >= a && k + OBJ.len_utf8() <= b && self.text.get(*k..).is_some_and(|t| t.starts_with(OBJ))).collect(); + if keep.is_empty() { + return self.delete(a, b); + } + keep.sort_unstable(); + keep.dedup(); + // Pieces between the kept objects, last first so earlier offsets stay valid. + let mut pieces = Vec::with_capacity(keep.len() + 1); + let mut from = a; + for k in &keep { + pieces.push((from, *k)); + from = k + OBJ.len_utf8(); + } + pieces.push((from, b)); + for (x, y) in pieces.into_iter().rev() { + if y > x { + self.delete(x, y)?; + } + } + Ok(()) + } + + /// Apply `f` to the formatting of `a..b`. + pub fn format(&mut self, a: usize, b: usize, f: &dyn Fn(&mut CharProps)) -> Result<()> { + let (a, b) = (a.min(b), a.max(b)); + self.check(a)?; + self.check(b)?; + if a == b { + return Ok(()); + } + let mut out = Vec::with_capacity(self.runs.len() + 2); + let mut start = 0; + for r in self.runs.drain(..) { + let (rs, re) = (start, start + r.len); + start = re; + let cut = [rs, a.clamp(rs, re), b.clamp(rs, re), re]; + for w in cut.windows(2) { + let (x, y) = (w[0], w[1]); + if y <= x { + continue; + } + let mut p = r.props.clone(); + if x >= a && y <= b { + f(&mut p); + } + out.push(Run { len: y - x, props: p }); + } + } + self.runs = out; + self.normalize(); + self.touch(); + Ok(()) + } + + /// Split at `off`: `self` keeps `..off`, the returned paragraph gets `off..` with the same + /// paragraph properties (the section break, if any, moves to the second half). + pub fn split_off(&mut self, off: usize) -> Result { + self.check(off)?; + let k = self.objects_before(off); + let tail_text = self.text.split_off(off); + let tail_objs = if k <= self.objects.len() { self.objects.split_off(k) } else { Vec::new() }; + let mut tail_runs = Vec::new(); + let mut start = 0; + let mut keep = Vec::new(); + for r in self.runs.drain(..) { + let (rs, re) = (start, start + r.len); + start = re; + if re <= off { + keep.push(r); + } else if rs >= off { + tail_runs.push(r); + } else { + keep.push(Run { len: off - rs, props: r.props.clone() }); + tail_runs.push(Run { len: re - off, props: r.props }); + } + } + self.runs = keep; + // The new paragraph's mark takes the formatting at the split, but never a hyperlink. + let carry = self.props_at(off).unlinked(); + let mut tail = Paragraph { + text: tail_text, + runs: tail_runs, + props: self.props.clone(), + mark: if self.text.is_empty() && off == 0 { self.mark.clone() } else { carry }, + objects: tail_objs, + section: self.section.take(), + rev: next_rev(), + }; + if tail.text.is_empty() { + tail.mark = self.props_at(off).unlinked(); + } + self.normalize(); + tail.normalize(); + self.touch(); + Ok(tail) + } + + /// Append `other`'s content (its paragraph properties are dropped; its section break kept). + pub fn append(&mut self, other: Paragraph) { + let Paragraph { text, runs, objects, section, .. } = other; + self.text.push_str(&text); + self.runs.extend(runs); + self.objects.extend(objects); + if section.is_some() { + self.section = section; + } + self.normalize(); + self.touch(); + } + + /// Previous caret stop before `off` (0 at start): a grapheme boundary that doesn't split a + /// complex-script syllable ([`crate::cluster`]). + pub fn prev_boundary(&self, off: usize) -> usize { + crate::cluster::prev_cluster(&self.text, self.clamp(off)) + } + /// Next caret stop after `off` (len at end), see [`Self::prev_boundary`]. + pub fn next_boundary(&self, off: usize) -> usize { + crate::cluster::next_cluster(&self.text, self.clamp(off)) + } + /// Where Backspace at `off` deletes from: one Arabic combining mark or one character in the + /// Brahmic scripts (Myanmar, Khmer, Devanagari, Thai…), otherwise the previous caret stop. + pub fn backspace_from(&self, off: usize) -> usize { + let off = self.clamp(off); + match self.text.get(..off).and_then(|t| t.char_indices().next_back()) { + Some((i, c)) if crate::cluster::backspaces_by_char(c) => i, + _ => self.prev_boundary(off), + } + } + /// Start of the word at or before `off` (Ctrl+Left). + pub fn word_start(&self, off: usize) -> usize { + let off = self.clamp(off); + let Some(before) = self.text.get(..off) else { return 0 }; + let mut idx = off; + let mut seen_word = false; + for (i, c) in before.char_indices().rev() { + let w = crate::bidi::is_word_char(c); + if w { + seen_word = true; + idx = i; + } else if seen_word { + break; + } else { + idx = i; + } + } + idx + } + /// End of the word after `off`, including trailing spaces (Ctrl+Right, like Word). + pub fn word_end(&self, off: usize) -> usize { + let off = self.clamp(off); + let Some(after) = self.text.get(off..) else { return self.text.len() }; + let it = after.char_indices().peekable(); + let mut end = self.text.len(); + let first_word = after.chars().next().is_some_and(|c| crate::bidi::is_word_char(c) && c != '\''); + let mut in_space = false; + for (i, c) in it { + let w = crate::bidi::is_word_char(c); + if c == ' ' || c == NBSP { + in_space = true; + continue; + } + if in_space || (first_word && !w) || (!first_word && i > 0) { + end = off + i; + break; + } + } + end + } + /// The word around `off` (double-click): (start, end) without trailing space. + pub fn word_at(&self, off: usize) -> (usize, usize) { + let off = self.clamp(off); + let is_w = |c: char| crate::bidi::is_word_char(c); + let mut a = off; + for (i, c) in self.text.get(..off).unwrap_or("").char_indices().rev() { + if !is_w(c) { + break; + } + a = i; + } + let mut b = off; + for (i, c) in self.text.get(off..).unwrap_or("").char_indices() { + if !is_w(c) { + if i == 0 && a == off { + b = off + c.len_utf8(); + } + break; + } + b = off + i + c.len_utf8(); + } + // Word also selects the trailing spaces. + let mut e = b; + for (i, c) in self.text.get(b..).unwrap_or("").char_indices() { + if c != ' ' { + break; + } + e = b + i + 1; + } + (a, e) + } + /// Sentence around `off` (Ctrl+click): ends after `.`, `!`, `?` and following spaces. + pub fn sentence_at(&self, off: usize) -> (usize, usize) { + let off = self.clamp(off); + let t = &self.text; + let mut a = 0; + let mut prev_end = false; + for (i, c) in t.char_indices() { + if i > off { + break; + } + if prev_end && c != ' ' { + a = i; + } + if matches!(c, '.' | '!' | '?') { + prev_end = true; + } else if c != ' ' { + prev_end = false; + } + } + let mut b = t.len(); + // A caret on the space right after a sentence's end belongs to that sentence; a terminator + // directly followed by text ("Hello.World") doesn't end the sentence the caret is in. + let mut ended = t.get(..off).unwrap_or("").ends_with(['.', '!', '?']) && t.get(off..).unwrap_or("").starts_with(' '); + for (i, c) in t.get(off..).unwrap_or("").char_indices() { + if ended && c != ' ' { + b = off + i; + break; + } + if matches!(c, '.' | '!' | '?') { + ended = true; + } + } + (a, b) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sentence_at_sentence_start() { + let p = Paragraph::with_text("Hello. World", CharProps::default()); + assert_eq!(p.sentence_at(0), (0, 7)); + assert_eq!(p.sentence_at(6), (0, 7)); + assert_eq!(p.sentence_at(7), (7, 12)); + assert_eq!(p.sentence_at(8), (7, 12)); + assert_eq!(p.sentence_at(99), (7, 12)); + let p = Paragraph::with_text("Hello.World", CharProps::default()); + assert_eq!(p.sentence_at(6), (6, 11)); + assert_eq!(p.sentence_at(8), (6, 11)); + } + + fn bold() -> CharProps { + CharProps { bold: Some(true), ..Default::default() } + } + + #[test] + fn text_without_takes_ranges_in_any_order() { + // Multi-byte chars at range edges, an object, and ranges unsorted, overlapping, nested and empty. + let mut p = Paragraph::with_text("añb€c", CharProps::default()); + p.insert_object( + 3, + InlineObject::Field { instr: "PAGE".into(), result: "7".into(), locked: false, dirty: false, code: false, form: String::new() }, + &CharProps::default(), + ) + .unwrap(); + let text = p.text.clone(); + let naive = |dropped: &[std::ops::Range]| { + let mut out = String::new(); + let mut k = 0; + for (i, c) in text.char_indices() { + let keep = !dropped.iter().any(|r| r.contains(&i)); + if c == OBJ { + if keep && let Some(o) = p.objects.get(k) { + out.push_str(o.plain_text()); + } + k += 1; + } else if keep { + out.push(c); + } + } + out + }; + let n = text.len(); + let cases: Vec>> = + vec![vec![], vec![1..3], vec![5..n, 0..1], vec![0..4, 2..3, 3..6], vec![2..9, 1..2, 4..4, 0..n], vec![6..6, 9..n, 0..0], vec![3..n + 10]]; + for dropped in cases { + assert_eq!(p.text_without(&dropped), naive(&dropped), "{dropped:?} of {text:?}"); + } + assert_eq!(p.text_without(std::slice::from_ref(&(1..3))), "a7b€c"); + } + + #[test] + fn insert_and_runs() { + let mut p = Paragraph::with_text("Hello world", CharProps::default()); + p.insert_text(5, ",", &CharProps::default()).unwrap(); + assert_eq!(p.text, "Hello, world"); + assert_eq!(p.runs.len(), 1); + p.insert_text(0, "Oh ", &bold()).unwrap(); + assert_eq!(p.runs.len(), 2); + assert_eq!(p.runs[0].len, 3); + assert!(p.insert_text(100, "x", &bold()).is_err()); + } + + #[test] + fn insert_rejects_mid_char() { + let mut p = Paragraph::with_text("é", CharProps::default()); + assert!(p.insert_text(1, "x", &bold()).is_err()); + } + + #[test] + fn format_splits_runs() { + let mut p = Paragraph::with_text("abcdef", CharProps::default()); + p.format(2, 4, &|c| c.bold = Some(true)).unwrap(); + assert_eq!(p.runs.iter().map(|r| r.len).collect::>(), vec![2, 2, 2]); + assert_eq!(p.props_of_char(2).bold, Some(true)); + p.format(0, 6, &|c| c.bold = Some(true)).unwrap(); + assert_eq!(p.runs.len(), 1); + } + + #[test] + fn delete_merges() { + let mut p = Paragraph::with_text("abcdef", CharProps::default()); + p.format(2, 4, &|c| c.bold = Some(true)).unwrap(); + p.delete(2, 4).unwrap(); + assert_eq!(p.text, "abef"); + assert_eq!(p.runs.len(), 1); + p.delete(0, 4).unwrap(); + assert!(p.runs.is_empty()); + } + + #[test] + fn objects_track_text() { + let mut p = Paragraph::with_text("ab", CharProps::default()); + p.insert_object( + 1, + InlineObject::Field { instr: "PAGE".into(), result: "1".into(), locked: false, dirty: false, code: false, form: String::new() }, + &CharProps::default(), + ) + .unwrap(); + p.insert_object(0, InlineObject::BookmarkStart { name: "x".into() }, &CharProps::default()).unwrap(); + assert_eq!(p.objects.len(), 2); + assert!(matches!(p.object_at(0), Some(InlineObject::BookmarkStart { .. }))); + let off = p.object_offsets()[1]; + assert!(matches!(p.object_at(off), Some(InlineObject::Field { .. }))); + // The one-pass walk agrees with offsets + object_at. + let one_pass: Vec<(usize, &InlineObject)> = p.objects_with_offsets().collect(); + let each: Vec<(usize, &InlineObject)> = p.object_offsets().into_iter().filter_map(|o| Some((o, p.object_at(o)?))).collect(); + assert_eq!(one_pass, each); + assert_eq!(one_pass.len(), 2); + assert_eq!(p.plain_text(), "a1b"); + p.delete(0, 3).unwrap(); + assert_eq!(p.objects.len(), 1); + let tail = p.split_off(p.object_offsets()[0]).unwrap(); + assert!(p.objects.is_empty()); + assert_eq!(tail.objects.len(), 1); + } + + #[test] + fn split_and_append() { + let mut p = Paragraph::with_text("Hello world", bold()); + let t = p.split_off(5).unwrap(); + assert_eq!(p.text, "Hello"); + assert_eq!(t.text, " world"); + assert_eq!(t.runs[0].props.bold, Some(true)); + p.append(t); + assert_eq!(p.text, "Hello world"); + assert_eq!(p.runs.len(), 1); + let t = p.split_off(p.len()).unwrap(); + assert!(t.is_empty()); + assert_eq!(t.mark.bold, Some(true)); + } + + #[test] + fn emptying_a_paragraph_keeps_the_formatting_but_not_the_revisions_of_its_text() { + let mut props = bold(); + props.ins = Some(3); + props.del = Some(4); + let mut p = Paragraph::with_text("Hello", props); + p.mark = bold(); + p.delete(0, 5).unwrap(); + assert!(p.is_empty()); + assert_eq!(p.mark.bold, Some(true)); + assert_eq!((p.mark.ins, p.mark.del), (None, None)); + // A mark that is itself a tracked change keeps its own revision. + let mut p = Paragraph::with_text("Hello", bold()); + p.mark.del = Some(7); + p.delete(0, 5).unwrap(); + assert_eq!((p.mark.ins, p.mark.del), (None, Some(7))); + } + + #[test] + fn words() { + let p = Paragraph::with_text("The quick brown fox.", CharProps::default()); + assert_eq!(p.word_end(0), 4); + assert_eq!(p.word_end(4), 11); + assert_eq!(p.word_start(10), 4); + assert_eq!(p.word_start(11), 4); + assert_eq!(p.word_at(5), (4, 11)); + assert_eq!(p.sentence_at(3), (0, 21)); + assert_eq!(p.next_boundary(0), 1); + assert_eq!(p.prev_boundary(0), 0); + } + + proptest::proptest! { + #[test] + fn edits_keep_runs_consistent(ops in proptest::collection::vec((0u8..4, 0usize..40, 0usize..40, "[a-zé ]{0,5}"), 0..40)) { + let mut p = Paragraph::with_text("seed text", CharProps::default()); + for (k, a, b, s) in ops { + let a = p.clamp(a); + let b = p.clamp(b); + let _ = match k { + 0 => p.insert_text(a, &s, &bold()).map(|_| ()), + 1 => p.delete(a, b), + 2 => p.format(a, b, &|c| c.italic = Some(true)), + _ => p.split_off(a).map(|t| p.append(t)), + }; + let total: usize = p.runs.iter().map(|r| r.len).sum(); + proptest::prop_assert_eq!(total, p.text.len()); + proptest::prop_assert!(p.runs.iter().all(|r| r.len > 0)); + proptest::prop_assert_eq!(p.text.matches(OBJ).count(), p.objects.len()); + } + } + } +} diff --git a/crates/wordmodel/src/para/float.rs b/crates/wordmodel/src/para/float.rs new file mode 100644 index 000000000..315e0e042 --- /dev/null +++ b/crates/wordmodel/src/para/float.rs @@ -0,0 +1,285 @@ +use super::{Deserialize, Serialize}; + +/// How a floating object is placed. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Wrap { + /// In line with text (not floating). + #[default] + Inline, + Square, + Tight, + Through, + TopAndBottom, + BehindText, + InFrontOfText, +} + +/// Which sides of a Square, Tight or Through object text may flow on (`@wrapText`, ECMA-376 +/// Part 1, §20.4.3.7 `ST_WrapText`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum WrapText { + /// Both sides of the object. + #[default] + BothSides, + /// Only the left side: the band to the object's right stays empty. + Left, + /// Only the right side. + Right, + /// Only whichever side has more room, decided per line. + Largest, +} + +impl WrapText { + /// From the OOXML value (`bothSides`, `left`, `right`, `largest`). + pub fn from_ooxml(v: &str) -> Option { + let v = v.trim(); + if v.eq_ignore_ascii_case("bothSides") || v.eq_ignore_ascii_case("both") { + Some(WrapText::BothSides) + } else if v.eq_ignore_ascii_case("left") { + Some(WrapText::Left) + } else if v.eq_ignore_ascii_case("right") { + Some(WrapText::Right) + } else if v.eq_ignore_ascii_case("largest") { + Some(WrapText::Largest) + } else { + None + } + } + + /// The OOXML value. + pub fn ooxml(self) -> &'static str { + match self { + WrapText::BothSides => "bothSides", + WrapText::Left => "left", + WrapText::Right => "right", + WrapText::Largest => "largest", + } + } + + pub fn is_both(&self) -> bool { + *self == WrapText::BothSides + } +} + +/// What a floating object's offset is relative to. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Anchor { + #[default] + Column, + Margin, + Page, + Paragraph, + /// The page's left margin area (from the page's left edge to the text). + LeftMargin, + /// The page's right margin area (from the text to the page's right edge). + RightMargin, + /// The page's top margin area (from the page's top edge to the text). + TopMargin, + /// The page's bottom margin area (from the text to the page's bottom edge). + BottomMargin, + /// The inside margin (left on odd pages, right on even pages when mirrored). + InsideMargin, + /// The outside margin. + OutsideMargin, + /// The anchor character (horizontal only). + Character, + /// The anchor line (vertical only). + Line, +} + +/// Alignment of a floating object within its [`Anchor`] area, instead of an offset. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FloatAlign { + /// Left or top. + Start, + Center, + /// Right or bottom. + End, + Inside, + Outside, +} + +impl FloatAlign { + /// From an OOXML alignment (`wp:align`, `w:tblpXSpec`, `w:tblpYSpec`). + pub fn from_ooxml(v: &str) -> Option { + match v { + "left" | "top" => Some(FloatAlign::Start), + "center" => Some(FloatAlign::Center), + "right" | "bottom" => Some(FloatAlign::End), + "inside" => Some(FloatAlign::Inside), + "outside" => Some(FloatAlign::Outside), + _ => None, + } + } + + /// The OOXML name on the horizontal (`left`, `right`) or vertical (`top`, `bottom`) axis. + pub fn ooxml(self, horizontal: bool) -> &'static str { + match (self, horizontal) { + (FloatAlign::Start, true) => "left", + (FloatAlign::Start, false) => "top", + (FloatAlign::Center, _) => "center", + (FloatAlign::End, true) => "right", + (FloatAlign::End, false) => "bottom", + (FloatAlign::Inside, _) => "inside", + (FloatAlign::Outside, _) => "outside", + } + } +} + +/// Floating placement (ignored when `wrap` is `Inline`). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Float { + pub wrap: Wrap, + /// The sides text wraps on (Square, Tight and Through). + #[serde(skip_serializing_if = "WrapText::is_both")] + pub wrap_text: WrapText, + pub h_rel: Anchor, + pub v_rel: Anchor, + /// Offsets, points (used when the matching alignment is `None`). + pub x: f32, + pub y: f32, + /// Alignments within the anchor area; they take precedence over the offsets. + #[serde(skip_serializing_if = "Option::is_none")] + pub h_align: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub v_align: Option, + /// Distance from surrounding text at the left and right (points). + pub dist: f32, + /// Distance from surrounding text above and below (points). + pub dist_top: f32, + pub dist_bottom: f32, + /// Room around the object for effects such as shadows (left, top, right, bottom, points), + /// inline or floating: lines and surrounding text keep clear of it. Rotation's own overhang + /// is not in here: it follows from [`Float::rot`] (see [`Float::spin_pad`]). + pub effect: [f32; 4], + /// Rotation clockwise about the object's centre, degrees (DrawingML `a:xfrm/@rot`). Offsets + /// and the size stay those of the unrotated frame, as in Word; see [`Float::spin`]. + #[serde(skip_serializing_if = "is_zero")] + pub rot: f32, + /// Mirrored left to right / top to bottom (`a:xfrm/@flipH`, `@flipV`). + #[serde(skip_serializing_if = "is_false")] + pub flip_h: bool, + #[serde(skip_serializing_if = "is_false")] + pub flip_v: bool, + /// The outline Tight and Through wrapping follow (`wp:wrapPolygon`), in the object's own + /// (unrotated) frame; `None` = derived from the object (see [`crate::wrap`]). + #[serde(skip_serializing_if = "Option::is_none")] + pub wrap_polygon: Option>, + /// What the drawing is, for screen readers and exports (`wp:docPr` `@descr`, `@title` and the + /// decorative mark). See [`InlineObject::alt_text`]. + #[serde(skip_serializing_if = "AltText::is_empty")] + pub alt: AltText, + /// Hidden (`wp:docPr/@hidden`, VML `visibility:hidden`): kept and saved, but neither drawn nor + /// laid out, as Word for Windows does (Word for Mac draws hidden VML shapes). + #[serde(skip_serializing_if = "is_false")] + pub hidden: bool, + /// Anchored in a table cell but placed on the page, not in the cell (`wp:anchor/@layoutInCell` + /// off, any VML shape in a header or footer): a picture watermark in a header that starts with a table. + #[serde(skip_serializing_if = "is_false")] + pub outside_cell: bool, + /// Stacking order among floating objects (`wp:anchor/@relativeHeight`): higher is drawn on + /// top. 0 = unknown, drawn in document order (objects inserted here, or from other formats). + #[serde(skip_serializing_if = "is_zero_u32")] + pub z: u32, + /// The page a drawing anchored to a page is on, from 1 (ODF's `text:anchor-page-number`, + /// §19.872); 0 = anchored to its paragraph or not said. Kept for saving: the layout places a + /// drawing on its anchor paragraph's page. + #[serde(skip_serializing_if = "is_zero_u32")] + pub page: u32, + /// Turned, but the file keeps no room for the turn (`wp:effectExtent` missing or zero, as + /// files from other tools have): Word then keeps an inline one's line to its frame and lets + /// the turned object overhang it (see [`Float::line_pad`]). Cleared by [`Float::set_spin`]. + #[serde(skip_serializing_if = "is_false")] + pub bare_turn: bool, +} + +/// Longest alternative text kept (characters): hostile files and commands are capped. +pub const MAX_ALT_TEXT: usize = 10_000; + +/// A drawing's alternative text (Picture/Shape Format › Accessibility › Alt Text). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct AltText { + /// The description of a shape, text box or group. Pictures and charts keep theirs in their + /// own `alt`; read it with [`InlineObject::alt_text`], which knows where it is. + #[serde(skip_serializing_if = "String::is_empty")] + pub text: String, + /// The short title (`wp:docPr/@title`). + #[serde(skip_serializing_if = "String::is_empty")] + pub title: String, + /// Decorative: it adds nothing to the content, so screen readers skip it and the + /// accessibility checker doesn't ask for a description. + #[serde(skip_serializing_if = "is_false")] + pub decorative: bool, +} + +impl AltText { + pub fn is_empty(&self) -> bool { + self.text.is_empty() && self.title.is_empty() && !self.decorative + } +} + +/// `s` cut to [`MAX_ALT_TEXT`] characters. +pub fn cap_alt(s: &str) -> String { + s.chars().take(MAX_ALT_TEXT).collect() +} + +fn is_zero(v: &f32) -> bool { + *v == 0.0 +} + +fn is_false(v: &bool) -> bool { + !*v +} + +fn is_zero_u32(v: &u32) -> bool { + *v == 0 +} + +impl Float { + /// The rotation and flips, the angle normalised to `0..360` (a NaN angle is no rotation). + pub fn spin(&self) -> crate::geom::Spin { + crate::geom::Spin::new(self.rot, self.flip_h, self.flip_v) + } + /// Set the rotation and flips (the angle normalised). + pub fn set_spin(&mut self, s: crate::geom::Spin) { + self.rot = crate::geom::normalize_degrees(s.deg); + self.flip_h = s.flip_h; + self.flip_v = s.flip_v; + self.bare_turn = false; + } + /// How far a `w` × `h` frame's rotated bounds reach past it on each side (x, y): the room a + /// rotated object takes beyond its frame (zero unrotated; negative where the turned object + /// is narrower than its frame, as a wide picture turned 90° is). + pub fn spin_pad(&self, w: f32, h: f32) -> (f32, f32) { + let (w, h) = (crate::geom::finite(w).clamp(0.0, 4000.0), crate::geom::finite(h).clamp(0.0, 4000.0)); + let (bw, bh) = self.spin().extent(w, h); + ((bw - w) / 2.0, (bh - h) / 2.0) + } + /// A [`Float::bare_turn`] still inline and without effects room: made floating, or given room + /// for a shadow or glow since, it saves, and so lays out, with the turn's room again. + pub fn is_bare_turn(&self) -> bool { + self.bare_turn && self.wrap == Wrap::Inline && self.effect_extent() == [0.0; 4] + } + /// The room an inline `w` × `h` object's line keeps past its frame on each side (x, y) for its + /// turn: its rotated bounds ([`Float::spin_pad`]) — or, for a [`Float::is_bare_turn`], none, except + /// that Word 2013+ layout (`modern`) exchanges the frame's width and height from 45° up to 135° + /// and from 225° up to 315°. + pub fn line_pad(&self, w: f32, h: f32, modern: bool) -> (f32, f32) { + if !self.is_bare_turn() { + return self.spin_pad(w, h); + } + let (w, h) = (crate::geom::finite(w).clamp(0.0, 4000.0), crate::geom::finite(h).clamp(0.0, 4000.0)); + let deg = crate::geom::normalize_degrees(self.rot); + if modern && ((45.0..135.0).contains(°) || (225.0..315.0).contains(°)) { ((h - w) / 2.0, (w - h) / 2.0) } else { (0.0, 0.0) } + } + /// The effect extents, finite and clamped (left, top, right, bottom). + pub fn effect_extent(&self) -> [f32; 4] { + self.effect.map(|v| if v.is_finite() { v.clamp(0.0, 1584.0) } else { 0.0 }) + } +} diff --git a/crates/wordmodel/src/para/inline_kinds.rs b/crates/wordmodel/src/para/inline_kinds.rs new file mode 100644 index 000000000..686089f7f --- /dev/null +++ b/crates/wordmodel/src/para/inline_kinds.rs @@ -0,0 +1,146 @@ +use super::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum ShapeKind { + #[default] + Rectangle, + RoundedRectangle, + Ellipse, + /// Flowchart OR preset: an ellipse with horizontal and vertical spokes. + FlowchartOr, + Triangle, + /// A right triangle (DrawingML's `rtTriangle`): the right angle at the bottom left. + RightTriangle, + Diamond, + Line, + /// A block arrow pointing right (`rightArrow`; also stands for the double-headed arrows). + Arrow, + /// A block arrow pointing left (`leftArrow`). + ArrowLeft, + /// A block arrow pointing up (`upArrow`). + ArrowUp, + /// A block arrow pointing down (`downArrow`). + ArrowDown, + Star, + Heart, + /// A text box (rectangle with a text story). + TextBox, + /// A shape drawn point by point, such as ink (its geometry is the shape's `freeform`). + Freeform, + /// Connectors: lines whose ends can be glued to other shapes (see [`crate::connector`]). + StraightConnector, + ElbowConnector, + CurvedConnector, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum NoteKind { + #[default] + Footnote, + Endnote, +} + +/// Where a positional tab ([`InlineObject::PositionalTab`]) lands on its line, and how the text +/// after it lines up there (ECMA-376 `ST_PTabAlignment`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum PTabAlign { + #[default] + Left, + Center, + Right, +} + +impl PTabAlign { + pub const ALL: [PTabAlign; 3] = [PTabAlign::Left, PTabAlign::Center, PTabAlign::Right]; + + /// The OOXML name (`left`, `center`, `right`). + pub fn ooxml(self) -> &'static str { + match self { + PTabAlign::Left => "left", + PTabAlign::Center => "center", + PTabAlign::Right => "right", + } + } + + pub fn from_ooxml(v: &str) -> Option { + PTabAlign::ALL.into_iter().find(|a| a.ooxml() == v) + } +} + +/// What a positional tab's place is measured against (ECMA-376 `ST_PTabRelativeTo`): the text +/// margins, or the paragraph's indents. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum PTabBase { + #[default] + Margin, + Indent, +} + +impl PTabBase { + pub const ALL: [PTabBase; 2] = [PTabBase::Margin, PTabBase::Indent]; + + /// The OOXML name (`margin`, `indent`). + pub fn ooxml(self) -> &'static str { + match self { + PTabBase::Margin => "margin", + PTabBase::Indent => "indent", + } + } + + pub fn from_ooxml(v: &str) -> Option { + PTabBase::ALL.into_iter().find(|a| a.ooxml() == v) + } +} + +/// How ruby (phonetic guide) text lines up over its base text (ECMA-376 `ST_RubyAlign`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum RubyAlign { + #[default] + Center, + /// Letters spread evenly from edge to edge. + DistributeLetter, + /// Letters spread evenly with half a gap at each edge. + DistributeSpace, + Left, + Right, + /// Right-aligned beside vertical text (laid out like `Right` in horizontal text). + RightVertical, +} + +impl RubyAlign { + pub const ALL: [RubyAlign; 6] = + [RubyAlign::Center, RubyAlign::DistributeLetter, RubyAlign::DistributeSpace, RubyAlign::Left, RubyAlign::Right, RubyAlign::RightVertical]; + + /// The OOXML name (`center`, `distributeLetter`…), also used by commands. + pub fn ooxml(self) -> &'static str { + match self { + RubyAlign::Center => "center", + RubyAlign::DistributeLetter => "distributeLetter", + RubyAlign::DistributeSpace => "distributeSpace", + RubyAlign::Left => "left", + RubyAlign::Right => "right", + RubyAlign::RightVertical => "rightVertical", + } + } + + pub fn from_ooxml(v: &str) -> Option { + RubyAlign::ALL.into_iter().find(|a| a.ooxml() == v) + } +} + +/// Phonetic Guide's Offset for a ruby `raise` (baseline to baseline, points): the gap from the top +/// of the base text to the bottom of the ruby text, taking letters as rising 0.8 em above their +/// baseline and dropping 0.2 em below it. +pub fn ruby_offset(raise: f32, base_size: f32, size: f32) -> f32 { + raise - 0.8 * base_size - 0.2 * size +} + +/// The ruby raise (baseline to baseline, points) for a Phonetic Guide Offset (see [`ruby_offset`]). +pub fn ruby_raise(offset: f32, base_size: f32, size: f32) -> f32 { + offset + 0.8 * base_size + 0.2 * size +} diff --git a/crates/wordmodel/src/para/inline_object.rs b/crates/wordmodel/src/para/inline_object.rs new file mode 100644 index 000000000..4c40e14a8 --- /dev/null +++ b/crates/wordmodel/src/para/inline_object.rs @@ -0,0 +1,438 @@ +use super::float::{Float, Wrap, cap_alt}; +use super::inline_kinds::{NoteKind, PTabAlign, PTabBase, RubyAlign, ShapeKind}; +use super::{CharProps, Deserialize, Serialize}; + +/// An inline object anchored at a U+FFFC in the paragraph text. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "camelCase")] +pub enum InlineObject { + Image { + /// Key in `Document::media`. + media: String, + /// Display size, points. + w: f32, + h: f32, + #[serde(default)] + alt: String, + #[serde(default)] + float: Float, + /// Crop (left, top, right, bottom) fractions 0..1. + #[serde(default)] + crop: [f32; 4], + /// Brightness, contrast and transparency read from the file (see [`crate::picfx`]). + #[serde(default, skip_serializing_if = "crate::picfx::PicFx::is_identity")] + fx: crate::picfx::PicFx, + /// The OLE object (embedded or linked file) or Word signature line this picture shows, as + /// read from a file: saving writes the object back, not only its picture. See + /// [`crate::graphic::Embedded`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + ole: Option>, + }, + /// A chart or SmartArt diagram (see [`crate::graphic`]). + Graphic { + w: f32, + h: f32, + #[serde(default)] + alt: String, + #[serde(default)] + float: Float, + graphic: std::sync::Arc, + }, + Shape { + kind: ShapeKind, + w: f32, + h: f32, + fill: Option, + stroke: Option, + #[serde(default)] + stroke_width: f32, + #[serde(default)] + float: Float, + /// Text box content: `Document::parts` id. + #[serde(default)] + story: Option, + /// A freeform's geometry (`ShapeKind::Freeform`): its paths, or an ink stroke. + #[serde(default, skip_serializing_if = "Option::is_none")] + freeform: Option>, + /// Shadow, glow and soft edges. + #[serde(default, skip_serializing_if = "crate::effects::ShapeEffects::is_empty")] + effects: crate::effects::ShapeEffects, + /// Drawing id, text warp, connector ends and arrowheads. + #[serde(default, skip_serializing_if = "crate::connector::ShapeExtra::is_empty")] + extra: crate::connector::ShapeExtra, + }, + /// Pictures, shapes and text boxes grouped into one object (Layout › Arrange › Group): it + /// moves, wraps and resizes as one. Its members are laid out in the group's own coordinate + /// space, `ch_w` × `ch_h`, which is stretched over the group's `w` × `h` (DrawingML's + /// `a:chExt` and `a:ext`), so resizing the group scales them. + Group { + w: f32, + h: f32, + #[serde(default)] + float: Float, + /// Size of the members' coordinate space (their offsets and sizes are in it). + ch_w: f32, + ch_h: f32, + children: Vec, + /// A Drawing Canvas (Insert › Shapes › New Drawing Canvas, DrawingML's `wpc:wpc`) rather + /// than a group: it has its own background and outline, and its frame is independent of + /// its members: resizing it doesn't scale them (`ch_w` × `ch_h` follow `w` × `h`), and + /// what lies outside the frame is clipped. + #[serde(default, skip_serializing_if = "Option::is_none")] + canvas: Option, + }, + /// A field: `instr` is the field code (`PAGE`, `NUMPAGES`, `DATE \@ "M/d/yyyy"`, `TOC \o "1-3"`…); + /// `result` the cached display text. A `locked` field keeps its result when fields update. + Field { + instr: String, + #[serde(default)] + result: String, + #[serde(default)] + locked: bool, + /// `w:dirty` (ECMA-376 §17.16.5.35): the cached result is stale — another program that + /// couldn't compute it set the flag so Word refreshes the field on open. Kept through + /// saves; cleared when the app computes a result, like Word's F9. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + dirty: bool, + /// Shows its code (`{ PAGE }`) instead of its result, the other way round from the + /// document-wide View › Field Codes setting (`fields.toggleCode`, Shift+F9). A view + /// toggle: never written to files. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + code: bool, + /// A legacy form field's definition (`FORMCHECKBOX`, `FORMTEXT`, `FORMDROPDOWN`): the + /// source's `w:ffData`, raw OOXML kept for round trip. Empty for other fields. + #[serde(default, skip_serializing_if = "String::is_empty")] + form: String, + }, + /// The start of a field whose result is ordinary content — formatted, possibly spanning + /// paragraphs — up to the matching [`InlineObject::FieldEnd`]. Imported TOCs and citation + /// managers' `ADDIN` fields are kept this way; `instr` is the field code. See + /// [`crate::fields`]. + FieldStart { + instr: String, + #[serde(default)] + locked: bool, + /// `w:dirty` on the begin marker, as [`InlineObject::Field::dirty`]. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + dirty: bool, + }, + /// The end of the innermost open [`InlineObject::FieldStart`]. + FieldEnd, + /// The start of a content control whose content is what follows, up to the matching + /// [`InlineObject::ControlEnd`]. See [`crate::control`]. + ControlStart { + control: Box, + }, + /// The end of the innermost open [`InlineObject::ControlStart`]. + ControlEnd, + NoteRef { + kind: NoteKind, + /// `Document::parts` id of the note's story. + id: u32, + /// Custom mark (empty = automatic number). + #[serde(default)] + custom: String, + }, + BookmarkStart { + name: String, + }, + BookmarkEnd { + name: String, + }, + CommentStart { + id: u32, + }, + CommentEnd { + id: u32, + }, + /// The start of a range that stays editable while the document is protected (Restrict + /// Editing exceptions, `w:permStart`): for everyone (`group` = `everyone`), another editing + /// group, or one editor (`editor`, a user name or address). + PermStart { + id: u32, + #[serde(default, skip_serializing_if = "String::is_empty")] + group: String, + #[serde(default, skip_serializing_if = "String::is_empty")] + editor: String, + }, + /// The end of the editable range [`InlineObject::PermStart`] with the same `id` opened. + PermEnd { + id: u32, + }, + /// An equation: its linear format (`x=(-b±√(b^2-4ac))/2a`, for plain text) and structure. + /// A display equation sits on a line of its own. + Equation { + linear: String, + #[serde(default)] + display: bool, + /// The structure; empty = parse `linear`. + #[serde(default, skip_serializing_if = "crate::math::Math::is_empty")] + math: crate::math::Math, + }, + /// Ruby (Phonetic Guide): small `ruby` text (furigana, pinyin…) set over `base` text. The base + /// text takes the formatting of the object's run; it reads as the paragraph's text. + Ruby { + base: String, + ruby: String, + #[serde(default)] + align: RubyAlign, + /// Ruby text size, points (`w:hps`). + size: f32, + /// From the base text's baseline up to the ruby text's baseline, points (`w:hpsRaise`). + raise: f32, + /// The base text size the ruby was set for, points (`w:hpsBaseText`). + #[serde(default)] + base_size: f32, + /// Language of the ruby (`w:lid`, e.g. `ja-JP`); empty = unspecified. + #[serde(default, skip_serializing_if = "String::is_empty")] + lang: String, + /// Formatting of the ruby text over the run's (font, colour…); its size is `size`. + #[serde(default, skip_serializing_if = "CharProps::is_empty")] + props: Box, + }, + /// A positional tab (Word's alignment tab, `w:ptab`): it goes to the left edge, centre or + /// right edge of the margins or of the indents, whatever the paragraph's tab stops, and the + /// text after it lines up there. Text already past that place sends it to the next line. + PositionalTab { + #[serde(default)] + align: PTabAlign, + #[serde(default)] + relative_to: PTabBase, + #[serde(default)] + leader: crate::props::TabLeader, + }, + /// Something we don't model, kept for round-trip (raw XML of the source format). + Opaque { + format: String, + xml: String, + #[serde(default)] + text: String, + }, +} + +/// One member of an [`InlineObject::Group`]: a picture or shape (its own `w`, `h` and story), at +/// `x`, `y` in the group's coordinate space. Its `float` is unused. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GroupChild { + pub x: f32, + pub y: f32, + pub obj: InlineObject, +} + +/// Most members a group keeps (hostile files). +pub const MAX_GROUP_CHILDREN: usize = 1000; + +/// The background fill and outline of a Drawing Canvas (an [`InlineObject::Group`] with `canvas`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CanvasStyle { + pub fill: Option, + pub stroke: Option, + /// Outline width, points. + pub stroke_width: f32, +} + +impl InlineObject { + /// A picture, chart or diagram, shape, text box or group: something drawn with a frame, sized and wrapped. + pub fn is_drawing(&self) -> bool { + matches!(self, InlineObject::Image { .. } | InlineObject::Graphic { .. } | InlineObject::Shape { .. } | InlineObject::Group { .. }) + } + /// A drawing's size and placement. + pub fn frame(&self) -> Option<(f32, f32, &Float)> { + match self { + InlineObject::Image { w, h, float, .. } + | InlineObject::Graphic { w, h, float, .. } + | InlineObject::Shape { w, h, float, .. } + | InlineObject::Group { w, h, float, .. } => Some((*w, *h, float)), + _ => None, + } + } + /// A drawing's placement, to change. + pub fn float_mut(&mut self) -> Option<&mut Float> { + match self { + InlineObject::Image { float, .. } + | InlineObject::Graphic { float, .. } + | InlineObject::Shape { float, .. } + | InlineObject::Group { float, .. } => Some(float), + _ => None, + } + } + /// Resize a drawing (a group's members scale with it; a canvas's keep their size and place). + pub fn set_size(&mut self, nw: f32, nh: f32) { + if let InlineObject::Image { w, h, .. } + | InlineObject::Graphic { w, h, .. } + | InlineObject::Shape { w, h, .. } + | InlineObject::Group { w, h, .. } = self + { + *w = nw; + *h = nh; + } + if let InlineObject::Group { ch_w, ch_h, canvas: Some(_), .. } = self { + *ch_w = nw; + *ch_h = nh; + } + } + /// A Drawing Canvas's background and outline (`None` for anything else). + pub fn canvas_style(&self) -> Option<&CanvasStyle> { + match self { + InlineObject::Group { canvas, .. } => canvas.as_ref(), + _ => None, + } + } + /// Whether this is a Drawing Canvas. + pub fn is_canvas(&self) -> bool { + self.canvas_style().is_some() + } + /// The text box story this object shows, if it is a text box. + pub fn text_box(&self) -> Option { + match self { + InlineObject::Shape { story, .. } => *story, + _ => None, + } + } + /// The text box stories this object shows: a text box's, or its group members'. + pub fn text_boxes(&self) -> Vec { + match self { + InlineObject::Shape { story: Some(id), .. } => vec![*id], + InlineObject::Group { children, .. } => children.iter().filter_map(|c| c.obj.text_box()).collect(), + _ => Vec::new(), + } + } + /// The text box story slots of this object and its group members, to repoint. + pub fn text_box_slots(&mut self) -> Vec<&mut Option> { + match self { + InlineObject::Shape { story, .. } => vec![story], + InlineObject::Group { children, .. } => children + .iter_mut() + .filter_map(|c| match &mut c.obj { + InlineObject::Shape { story, .. } => Some(story), + _ => None, + }) + .collect(), + _ => Vec::new(), + } + } + /// A group's members placed in a `w` × `h` box at (`x`, `y`): each member's rectangle + /// (x, y, w, h) and the member. Empty for anything else. + pub fn group_rects(&self, x: f32, y: f32, w: f32, h: f32) -> Vec<([f32; 4], &InlineObject)> { + let InlineObject::Group { children, .. } = self else { return Vec::new() }; + self.member_rects(x, y, w, h).into_iter().filter_map(|(i, r)| Some((r, &children.get(i)?.obj))).collect() + } + /// [`InlineObject::group_rects`] with each member's index in `children` instead of the member. + pub fn member_rects(&self, x: f32, y: f32, w: f32, h: f32) -> Vec<(usize, [f32; 4])> { + let InlineObject::Group { children, .. } = self else { return Vec::new() }; + let (sx, sy) = self.group_scale(w, h); + let fin = |v: f32| if v.is_finite() { v } else { 0.0 }; + children + .iter() + .take(MAX_GROUP_CHILDREN) + .enumerate() + .filter_map(|(i, c)| { + let (cw, chh, _) = c.obj.frame()?; + let r = [x + fin(c.x) * sx, y + fin(c.y) * sy, (fin(cw) * sx).clamp(0.0, 4000.0), (fin(chh) * sy).clamp(0.0, 4000.0)]; + (!matches!(c.obj, InlineObject::Group { .. })).then_some((i, r)) + }) + .collect() + } + /// How much a group drawn `w` × `h` scales its members' space (x, y): 1 for a canvas, whose + /// members keep their size. + pub fn group_scale(&self, w: f32, h: f32) -> (f32, f32) { + let InlineObject::Group { ch_w, ch_h, .. } = self else { return (1.0, 1.0) }; + let fin = |v: f32| if v.is_finite() { v } else { 0.0 }; + let sx = if fin(*ch_w) > 0.0 { fin(w) / ch_w } else { 1.0 }; + let sy = if fin(*ch_h) > 0.0 { fin(h) / ch_h } else { 1.0 }; + (sx, sy) + } + /// A legacy form field's definition (`FORMCHECKBOX`, `FORMTEXT`, `FORMDROPDOWN`), read from + /// the `w:ffData` it keeps; `None` for anything else. + pub fn legacy_form(&self) -> Option { + match self { + InlineObject::Field { form, .. } if !form.is_empty() => crate::legacy_form::LegacyForm::parse(form), + _ => None, + } + } + /// Zero-width markers don't take part in layout. + pub fn is_marker(&self) -> bool { + matches!( + self, + InlineObject::BookmarkStart { .. } + | InlineObject::BookmarkEnd { .. } + | InlineObject::CommentStart { .. } + | InlineObject::CommentEnd { .. } + | InlineObject::PermStart { .. } + | InlineObject::PermEnd { .. } + | InlineObject::FieldStart { .. } + | InlineObject::FieldEnd + | InlineObject::ControlStart { .. } + | InlineObject::ControlEnd + ) + } + /// A drawing's description (alternative text), wherever it is kept. + pub fn alt_text(&self) -> &str { + match self { + InlineObject::Image { alt, .. } | InlineObject::Graphic { alt, .. } => alt, + InlineObject::Shape { float, .. } | InlineObject::Group { float, .. } => &float.alt.text, + _ => "", + } + } + /// Set a drawing's description (capped at [`MAX_ALT_TEXT`] characters); `false` if this is no drawing. + pub fn set_alt_text(&mut self, text: &str) -> bool { + let text = cap_alt(text); + match self { + InlineObject::Image { alt, .. } | InlineObject::Graphic { alt, .. } => *alt = text, + InlineObject::Shape { float, .. } | InlineObject::Group { float, .. } => float.alt.text = text, + _ => return false, + } + true + } + /// A drawing marked decorative. + pub fn is_decorative(&self) -> bool { + self.frame().is_some_and(|(_, _, f)| f.alt.decorative) + } + pub fn is_floating(&self) -> bool { + match self { + InlineObject::Image { float, .. } + | InlineObject::Graphic { float, .. } + | InlineObject::Shape { float, .. } + | InlineObject::Group { float, .. } => float.wrap != Wrap::Inline, + _ => false, + } + } + /// The geometry of an ink stroke (a freeform shape drawn with a pen), if this is one. + pub fn ink(&self) -> Option<&crate::freeform::Freeform> { + match self { + InlineObject::Shape { freeform: Some(f), .. } if f.is_ink() => Some(f), + _ => None, + } + } + /// The line a footnote or endnote separator draws (ECMA-376 §17.11: the `w:separator` and + /// `w:continuationSeparator` run elements): a short line, or with `continuation` one across + /// the column. Kept as the run element itself. + pub fn note_separator(continuation: bool) -> InlineObject { + let el = if continuation { "w:continuationSeparator" } else { "w:separator" }; + InlineObject::Opaque { format: "docx".into(), xml: format!("<{el}/>"), text: String::new() } + } + /// For a [`InlineObject::note_separator`]: whether it is the continuation separator. + pub fn as_note_separator(&self) -> Option { + match self { + InlineObject::Opaque { format, xml, .. } if format == "docx" => match xml.as_str() { + "" => Some(false), + "" => Some(true), + _ => None, + }, + _ => None, + } + } + /// The text this object contributes to plain-text extraction. + pub fn plain_text(&self) -> &str { + match self { + InlineObject::Field { result, .. } => result, + InlineObject::Equation { linear, .. } => linear, + InlineObject::Ruby { base, .. } => base, + InlineObject::PositionalTab { .. } => "\t", + InlineObject::Opaque { text, .. } => text, + _ => "", + } + } +} diff --git a/crates/wordmodel/src/para/run.rs b/crates/wordmodel/src/para/run.rs new file mode 100644 index 000000000..10ff4f09e --- /dev/null +++ b/crates/wordmodel/src/para/run.rs @@ -0,0 +1,37 @@ +use super::{AtomicU64, CharProps, Deserialize, Ordering, Serialize}; + +/// Object replacement character: marks an inline object in paragraph text. +pub const OBJ: char = '\u{FFFC}'; + +/// Manual line break (Shift+Enter). +pub const LINE_BREAK: char = '\n'; + +/// Page break (Ctrl+Enter). +pub const PAGE_BREAK: char = '\u{000C}'; + +/// Column break (Ctrl+Shift+Enter). +pub const COLUMN_BREAK: char = '\u{000E}'; + +/// Optional hyphen (Ctrl+-). +pub const SOFT_HYPHEN: char = '\u{00AD}'; + +/// Nonbreaking hyphen (Ctrl+Shift+-). +pub const NB_HYPHEN: char = '\u{2011}'; + +/// Nonbreaking space (Ctrl+Shift+Space). +pub const NBSP: char = '\u{00A0}'; + +static REV: AtomicU64 = AtomicU64::new(1); + +/// A fresh revision number (process-unique), used by layout caches to detect edits. +pub fn next_rev() -> u64 { + REV.fetch_add(1, Ordering::Relaxed) +} + +/// A formatting run: `len` bytes of text sharing `props`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Run { + pub len: usize, + #[serde(default, skip_serializing_if = "CharProps::is_empty")] + pub props: CharProps, +} diff --git a/crates/wordmodel/src/parts.rs b/crates/wordmodel/src/parts.rs new file mode 100644 index 000000000..830aa0430 --- /dev/null +++ b/crates/wordmodel/src/parts.rs @@ -0,0 +1,133 @@ +use super::block::Blocks; +use super::position::Pos; +use super::{Deserialize, Serialize, props}; + +/// A free id, `min` or more, for something new (a part, a list, a comment): one past `largest`, the +/// largest id in use, so ids keep growing; once that would pass `u32::MAX` (a file can use any id), +/// the lowest one `used` (every id in use) doesn't hold. `None` when every id from `min` up is taken. +pub fn free_id(largest: Option, used: impl Iterator, min: u32) -> Option { + let Some(largest) = largest else { return Some(min) }; + if let Some(next) = largest.checked_add(1) { + return Some(next.max(min)); + } + let mut ids: Vec = used.filter(|&i| i >= min).collect(); + ids.sort_unstable(); + ids.dedup(); + let mut want = min; + for i in ids { + if i > want { + break; + } + want = want.checked_add(1)?; + } + Some(want) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum PartKind { + #[default] + Header, + Footer, + Footnote, + Endnote, + Comment, + TextBox, +} + +/// A document's own separators and continuation notice for its footnotes or its endnotes +/// (ECMA-376 §17.11: the `w:footnote`/`w:endnote` entries of type `separator`, +/// `continuationSeparator` and `continuationNotice`). `None` = the default: a short line above +/// the notes, a line across the column above notes continued from the page before, no notice. +/// The lines themselves are [`InlineObject::note_separator`] objects in the paragraphs. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct NoteSeparators { + /// Above the notes on a page. + pub separator: Option, + /// Above the rest of a note continued from the page before. + pub continuation_separator: Option, + /// Below the part of a note that continues on the next page. + pub continuation_notice: Option, +} + +impl NoteSeparators { + /// No separator of the document's own: the defaults apply. + pub fn is_empty(&self) -> bool { + self.separator.is_none() && self.continuation_separator.is_none() && self.continuation_notice.is_none() + } +} + +/// A secondary story. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +pub struct Part { + pub kind: PartKind, + pub blocks: Blocks, + /// A text box's text direction, alignment and link (unused by other parts). + #[serde(default, skip_serializing_if = "TextBody::is_default")] + pub body: TextBody, +} + +/// How a text box shows its text (Shape Format › Text): which way it runs, where it sits between +/// the box's top and bottom, and the box its overflow continues in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct TextBody { + pub vert: props::TextVert, + pub anchor: props::VAlign, + /// Create Link: the text box (another `TextBox` part, shown by its own shape) the text + /// continues in when it doesn't fit this one. The linked box shows the rest of this box's + /// story; its own blocks are unused. See [`Document::text_box_chains`]. + pub next: Option, + /// Internal margins (left, top, right, bottom), EMU, when the file gives its own + /// (`wps:bodyPr` `lIns`… ECMA-376 Part 1 §21.1.2.1.1, or VML `v:textbox` `inset`); + /// `None` is Word's defaults. + pub inset: Option<[u32; 4]>, + /// Resize shape to fit text (`a:spAutoFit`, VML `mso-fit-shape-to-text`): Word grows the + /// box to its text, so none of the text is cut. + pub auto_fit: bool, + /// The colour the shape's style gives its text (`wps:style/a:fontRef`): what automatic text + /// colour means inside this box, instead of black. Text whose colour comes from its runs, + /// styles or the document defaults keeps it (Word, observed with synthetic documents). + #[serde(skip_serializing_if = "Option::is_none")] + pub auto_color: Option, +} + +/// Word's default text box margins (left, top, right, bottom), EMU: 0.1" and 0.05". +pub const TEXT_BOX_INSET: [u32; 4] = [91_440, 45_720, 91_440, 45_720]; + +/// Widest text box margin kept, EMU (22", the largest page side). +pub const MAX_TEXT_BOX_INSET: u32 = 20_116_800; + +impl TextBody { + pub fn is_default(&self) -> bool { + *self == TextBody::default() + } + + /// The internal margins (left, top, right, bottom) in points. + pub fn insets_pt(&self) -> [f32; 4] { + self.inset.unwrap_or(TEXT_BOX_INSET).map(|v| v.min(MAX_TEXT_BOX_INSET) as f32 / 12_700.0) + } +} + +/// Most boxes one chain of linked text boxes runs through (hostile files). +pub const MAX_LINKED_BOXES: usize = 256; + +/// A range that stays editable while the document is protected; see [`Document::perm_ranges`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PermRange { + pub id: u32, + /// Editing group (`everyone`, `editors`…); empty when the range names an editor. + pub group: String, + /// The one editor the range is for; empty for a group. + pub editor: String, + pub start: Pos, + pub end: Pos, +} + +impl PermRange { + /// Can everyone edit it (rather than one person or group)? + pub fn everyone(&self) -> bool { + self.group.eq_ignore_ascii_case("everyone") + } +} diff --git a/crates/wordmodel/src/picfx.rs b/crates/wordmodel/src/picfx.rs new file mode 100644 index 000000000..21d5bcf5b --- /dev/null +++ b/crates/wordmodel/src/picfx.rs @@ -0,0 +1,190 @@ +//! Picture colour corrections read from a file: brightness, contrast and transparency +//! (DrawingML `a:lum` and `a:alphaModFix` inside `a:blip`, ECMA-376 §20.1.8.42 and §20.1.8.4; +//! VML `v:imagedata` `gain` and `blacklevel`, which Word's picture watermark "Washout" sets). +//! +//! Values are kept in OOXML units: thousandths of a percent, so 100 000 is 100 %. All zero is +//! the picture as it is. +//! +//! The pixel formula was measured on Word (#1711, #1766): Word bakes the corrected pixels into +//! the picture it embeds in a PDF export, so a synthetic grey and colour ramp exported with +//! 20 `a:lum` combinations and 6 VML `gain`/`blacklevel` pairs gives the curve. Every channel is +//! mapped on its own (no hue or luminance coupling) by one straight line, clamped to 0..=255 +//! only at the end: +//! +//! `y = s·(x − 127.5) + 127.5 + 127.5·b·(1 + s)`, with `b = bright / 100 000`, `c = contrast / +//! 100 000`, `s = 1 + c` when `c ≤ 0` and `s = 1 / (1 − c)` when `c > 0` +//! +//! (as if half the brightness shift were applied before the contrast and half after). Measured +//! slope and intercept of Word's output for grey input `x` (PNG pictures, exact to ±1 level): +//! bright 20 000 → 1.00, +51; bright −40 000 → 1.00, −103; contrast 80 000 → 5.00, −510; +//! contrast −80 000 → 0.20, +101.4; (40 000, −40 000) → 0.60, +132.2; (80 000, −80 000) → +//! 0.20, +223.8 (also the #1766 header picture: fitted 0.197..0.201, +223.6..224.0); +//! (−40 000, 40 000) → 1.667, −221.7; (30 000, 60 000) → 2.50, −57.8. +//! +//! VML: `gain` is the slope itself and `blacklevel` half the brightness — Word draws `gain` 0.3, +//! `blacklevel` 0.35 (Washout) as 0.30, +205.3, the same as `a:lum` (70 000, −70 000), and +//! `gain` 1, `blacklevel` 0.2 as the very same picture as `bright="40000"`. + +use serde::{Deserialize, Serialize}; + +/// The largest magnitude of a correction (100 %). +pub const FX_MAX: i32 = 100_000; + +/// Brightness, contrast and transparency of a picture (see the module docs). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(default)] +pub struct PicFx { + /// −100 000 (black) ..= 100 000 (white). + pub bright: i32, + /// −100 000 (flat grey) ..= 100 000 (hard threshold). + pub contrast: i32, + /// 0 (opaque) ..= 100 000 (invisible). + pub transparency: i32, +} + +impl PicFx { + /// Corrections clamped to their ranges. + pub fn new(bright: i32, contrast: i32, transparency: i32) -> PicFx { + PicFx { bright: bright.clamp(-FX_MAX, FX_MAX), contrast: contrast.clamp(-FX_MAX, FX_MAX), transparency: transparency.clamp(0, FX_MAX) } + } + + /// The same corrections, each clamped to its range (for values that came from anywhere). + pub fn clamped(self) -> PicFx { + PicFx::new(self.bright, self.contrast, self.transparency) + } + + /// No correction at all. + pub fn is_identity(&self) -> bool { + *self == PicFx::default() + } + + /// Brightness or contrast changes the pixels (transparency alone doesn't). + pub fn changes_pixels(&self) -> bool { + let f = self.clamped(); + f.bright != 0 || f.contrast != 0 + } + + /// The picture's opacity, 0..=1 (1 − transparency). + pub fn opacity(&self) -> f32 { + 1.0 - self.transparency.clamp(0, FX_MAX) as f32 / FX_MAX as f32 + } + + /// The corrections a VML `v:imagedata` gives with `gain` and `blacklevel` (already parsed to + /// plain numbers, 1 and 0 when missing): contrast from the slope `gain`, brightness twice the + /// black level (see the module docs). Non-finite values count as missing. + pub fn from_vml(gain: f64, blacklevel: f64) -> PicFx { + let gain = if gain.is_finite() { gain.clamp(0.0, 1.0e6) } else { 1.0 }; + let black = if blacklevel.is_finite() { blacklevel.clamp(-0.5, 0.5) } else { 0.0 }; + // s = 1 + c (c ≤ 0) or 1 / (1 − c) (c > 0). + let c = if gain <= 1.0 { gain - 1.0 } else { 1.0 - 1.0 / gain }; + let fx = |v: f64| (v * FX_MAX as f64).round().clamp(-(FX_MAX as f64), FX_MAX as f64) as i32; + PicFx::new(fx(2.0 * black), fx(c), 0) + } + + /// `gain` and `blacklevel` for VML (the inverse of [`PicFx::from_vml`]). + pub fn to_vml(&self) -> (f64, f64) { + (self.slope(), self.clamped().bright as f64 / FX_MAX as f64 / 2.0) + } + + /// The contrast slope `s` (capped at 255: contrast 100 % is a threshold). + fn slope(&self) -> f64 { + let c = self.contrast.clamp(-FX_MAX, FX_MAX) as f64 / FX_MAX as f64; + if c <= 0.0 { 1.0 + c } else { (1.0 / (1.0 - c).max(1.0 / 255.0)).min(255.0) } + } + + /// The lookup table every colour channel goes through. + pub fn table(&self) -> [u8; 256] { + let s = self.slope(); + let b = self.bright.clamp(-FX_MAX, FX_MAX) as f64 / FX_MAX as f64; + let mut t = [0u8; 256]; + for (x, slot) in t.iter_mut().enumerate() { + let y = s * (x as f64 - 127.5) + 127.5 + 127.5 * b * (1.0 + s); + *slot = if y.is_finite() { y.round().clamp(0.0, 255.0) as u8 } else { 0 }; + } + t + } + + /// Apply brightness and contrast to straight (not premultiplied) RGBA8 pixels in place. The + /// alpha byte is left alone (transparency is drawn as an opacity, see [`PicFx::opacity`]); a + /// trailing partial pixel is ignored. + pub fn apply_rgba(&self, rgba: &mut [u8]) { + if !self.changes_pixels() { + return; + } + let t = self.table(); + for px in rgba.as_chunks_mut::<4>().0 { + for c in px.iter_mut().take(3) { + *c = t.get(usize::from(*c)).copied().unwrap_or(*c); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Word's own output for grey input 0, 128 and 255 (PNG pictures in a PDF export). + #[test] + fn matches_word() { + let cases: [(PicFx, [u8; 3]); 6] = [ + (PicFx::new(40_000, -40_000, 0), [132, 209, 255]), + (PicFx::new(80_000, -80_000, 0), [224, 249, 255]), + (PicFx::new(-40_000, 40_000, 0), [0, 0, 203]), + (PicFx::new(0, 80_000, 0), [0, 130, 255]), + (PicFx::new(20_000, 0, 0), [51, 179, 255]), + (PicFx::from_vml(19661.0 / 65536.0, 22938.0 / 65536.0), [205, 244, 255]), + ]; + for (fx, want) in cases { + let t = fx.table(); + for (x, w) in [0usize, 128, 255].into_iter().zip(want) { + let got = t[x]; + assert!(got.abs_diff(w) <= 2, "{fx:?}: {x} → {got}, Word {w}"); + } + } + } + + #[test] + fn channels_apart_alpha_kept() { + let fx = PicFx::new(80_000, -80_000, 0); + let mut px = [255, 0, 0, 77, 0, 128, 255, 255]; + fx.apply_rgba(&mut px); + // Word: red (255, 0, 0) → (255, 224, 224); 128 → 249 (the formula gives 250.0). + assert_eq!(px, [255, 224, 224, 77, 224, 250, 255, 255]); + let mut odd = [0u8; 6]; + fx.apply_rgba(&mut odd); + assert_eq!(odd[4..], [0, 0]); + } + + #[test] + fn vml_mapping() { + // 16.16 fixed point: 19661f and 22938f are 0.3 and 0.35 to within 1e-5. + let washout = PicFx::from_vml(19661.0 / 65536.0, 22938.0 / 65536.0); + assert!((washout.bright - 70_000).abs() <= 2 && (washout.contrast + 70_000).abs() <= 2, "{washout:?}"); + assert_eq!(PicFx::from_vml(1.0, 0.2), PicFx::new(40_000, 0, 0)); + assert_eq!(PicFx::from_vml(2.0, 0.0), PicFx::new(0, 50_000, 0)); + assert!(PicFx::from_vml(1.0, 0.0).is_identity()); + for (g, b) in [(f64::NAN, f64::INFINITY), (-1.0e30, 1.0e30), (1.0e300, -7.0), (0.0, 0.0)] { + let fx = PicFx::from_vml(g, b); + assert_eq!(fx, fx.clamped(), "{g} {b}"); + let _ = fx.table(); + } + } + + #[test] + fn hostile_values_clamp() { + for v in [i32::MIN, -100_001, 0, 100_001, i32::MAX] { + let fx = PicFx { bright: v, contrast: v, transparency: v }; + let _ = fx.table(); + assert!((0.0..=1.0).contains(&fx.opacity())); + let mut px = [1u8, 2, 3, 4]; + fx.apply_rgba(&mut px); + assert_eq!(px[3], 4); + } + assert_eq!(PicFx::new(0, 0, 25_000).opacity(), 0.75); + assert!(!PicFx::new(0, 0, 25_000).changes_pixels()); + // Contrast 100 %: a threshold at mid-grey, no NaN. + let t = PicFx::new(0, FX_MAX, 0).table(); + assert_eq!((t[0], t[127], t[128], t[255]), (0, 0, 255, 255)); + } +} diff --git a/crates/wordmodel/src/position.rs b/crates/wordmodel/src/position.rs new file mode 100644 index 000000000..800197720 --- /dev/null +++ b/crates/wordmodel/src/position.rs @@ -0,0 +1,73 @@ +use super::{Deserialize, Serialize}; + +/// Which story a position is in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum StoryRef { + #[default] + Body, + Part(u32), +} + +/// Path to a paragraph: `[block]` in a story, `[block, row, cell, block]` inside a table cell, +/// three more indexes per nesting level. Lexicographic order = document order. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default)] +pub struct Path(pub Vec); + +impl Path { + pub fn top(i: usize) -> Path { + Path(vec![i as u32]) + } + /// The block index within its container. + pub fn last(&self) -> usize { + self.0.last().copied().unwrap_or(0) as usize + } + pub fn with_last(&self, i: usize) -> Path { + let mut p = self.clone(); + if let Some(l) = p.0.last_mut() { + *l = i as u32; + } + p + } + /// The container prefix (everything but the last index). + pub fn parent(&self) -> &[u32] { + self.0.split_last().map(|(_, r)| r).unwrap_or(&[]) + } + /// Nesting depth (0 = top-level story block). + pub fn depth(&self) -> usize { + self.0.len().saturating_sub(1) / 3 + } + /// The enclosing table cell (table block path, row, cell) if any. + pub fn cell(&self) -> Option<(Path, usize, usize)> { + if self.0.len() < 4 { + return None; + } + let n = self.0.len(); + let t = Path(self.0.get(..n - 3)?.to_vec()); + Some((t, *self.0.get(n - 3)? as usize, *self.0.get(n - 2)? as usize)) + } +} + +impl std::fmt::Display for Path { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s: Vec = self.0.iter().map(ToString::to_string).collect(); + write!(f, "{}", s.join("/")) + } +} + +/// A position in a document. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize, Default)] +pub struct Pos { + pub story: StoryRef, + pub path: Path, + pub off: usize, +} + +impl Pos { + pub fn new(story: StoryRef, path: Path, off: usize) -> Pos { + Pos { story, path, off } + } + pub fn body(block: usize, off: usize) -> Pos { + Pos { story: StoryRef::Body, path: Path::top(block), off } + } +} diff --git a/crates/wordmodel/src/props.rs b/crates/wordmodel/src/props.rs new file mode 100644 index 000000000..38feba083 --- /dev/null +++ b/crates/wordmodel/src/props.rs @@ -0,0 +1,203 @@ +//! Character, paragraph, table and cell properties. +//! +//! Every field is an `Option`: `None` means "inherit" (from the style chain, then document +//! defaults). [`CharProps::overlay`] and friends apply a patch on top of a base. + +use serde::{Deserialize, Serialize}; + +macro_rules! overlay_fields { + ($dst:ident, $src:ident; $($f:ident),*) => { + $( if $src.$f.is_some() { $dst.$f = $src.$f.clone(); } )* + }; +} + +macro_rules! drop_same_fields { + ($dst:ident, $base:ident; $($f:ident),*) => { + $( if $dst.$f == $base.$f { $dst.$f = None; } )* + }; +} + +mod char_props; +mod color; +mod para_props; +mod table_props; + +pub use char_props::{ + CharProps, CombineBrackets, DEFAULT_FONT_SIZE, EastAsianLayout, EmphasisMark, Highlight, Ligatures, NumChange, PropChange, Underline, VertAlign, +}; +pub use color::{Rgb, TextColor}; +pub use para_props::{ + Align, Border, BorderStyle, Borders, LineSpacing, NumRef, ParaProps, ParagraphTextDirection, TAB_POS_EPS, TabAlign, TabLeader, TabStop, + TextAlignment, TextFrame, +}; +pub use table_props::{ + CellPicture, CellProps, HeightRule, PictureRepeat, RowProps, TableFloat, TableLook, TableProps, TextDirection, TextVert, VAlign, VMerge, + anchor_ooxml, valign_from_anchor, +}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rgb_hex() { + assert_eq!(Rgb(255, 0, 16).hex(), "FF0010"); + assert_eq!(Rgb::parse("#FF0010"), Some(Rgb(255, 0, 16))); + assert_eq!(Rgb::parse("zzzzzz"), None); + assert_eq!(Rgb::parse("é1234"), None); + } + + #[test] + fn rgb_parse_rejects_non_hex() { + assert_eq!(Rgb::parse("+1+2+3"), None); + assert_eq!(Rgb::parse("-1-2-3"), None); + assert_eq!(Rgb::parse("##ff0000"), None); + assert_eq!(Rgb::parse("ff00 0"), None); + assert_eq!(Rgb::parse("#ff0000"), Some(Rgb(255, 0, 0))); + assert_eq!(Rgb::parse("ff0000"), Some(Rgb(255, 0, 0))); + } + + #[test] + fn overlay_keeps_unset() { + let base = CharProps { bold: Some(true), size: Some(12.0), ..Default::default() }; + let patch = CharProps { size: Some(14.0), italic: Some(true), ..Default::default() }; + let r = base.overlaid(&patch); + assert_eq!(r.bold, Some(true)); + assert_eq!(r.size, Some(14.0)); + assert_eq!(r.italic, Some(true)); + } + + #[test] + fn borders_overlay_per_side() { + let thin = Border::single(0.5); + let thick = Border::single(2.0); + let nil = Border { style: BorderStyle::None, width: 0.0, color: None, space: 0.0 }; + let mut b = Borders::all(thin); + b.overlay(&Borders { top: Some(thick), ..Default::default() }); + assert_eq!(b.top, Some(thick)); + assert_eq!(b.bottom, Some(thin)); + b.overlay(&Borders::box_(nil)); + assert_eq!(b.left, Some(nil)); // explicit nil overrides that side + assert_eq!(b.between, Some(thin)); // untouched sides survive + b.overlay(&Borders::default()); + assert_eq!(b.left, Some(nil)); // empty patch changes nothing + } + + #[test] + fn ooxml_names_round_trip() { + for u in Underline::ALL { + assert_eq!(Underline::from_ooxml(u.ooxml()), u); + } + for h in Highlight::ALL { + assert_eq!(Highlight::from_ooxml(h.ooxml()), h); + } + } + + /// The 16 `w14:ligatures` values of [MS-DOCX], each to its flags and back. + #[test] + fn ligature_names() { + let l = |standard, contextual, historical, discretional| Ligatures { standard, contextual, historical, discretional }; + for (name, flags) in [ + ("none", l(false, false, false, false)), + ("standard", l(true, false, false, false)), + ("contextual", l(false, true, false, false)), + ("historical", l(false, false, true, false)), + ("discretional", l(false, false, false, true)), + ("standardContextual", l(true, true, false, false)), + ("standardHistorical", l(true, false, true, false)), + ("contextualHistorical", l(false, true, true, false)), + ("standardDiscretional", l(true, false, false, true)), + ("contextualDiscretional", l(false, true, false, true)), + ("historicalDiscretional", l(false, false, true, true)), + ("standardContextualHistorical", l(true, true, true, false)), + ("standardContextualDiscretional", l(true, true, false, true)), + ("standardHistoricalDiscretional", l(true, false, true, true)), + ("contextualHistoricalDiscretional", l(false, true, true, true)), + ("all", l(true, true, true, true)), + ] { + assert_eq!(Ligatures::from_ooxml(name), Some(flags), "{name}"); + assert_eq!(flags.ooxml(), name); + } + assert_eq!(Ligatures::all().count(), 16); + for bad in ["", "Standard", "contextualStandard", "standardstandard", "allx"] { + assert_eq!(Ligatures::from_ooxml(bad), None, "{bad:?}"); + } + } + + #[test] + fn text_direction_names() { + let mut d = TextDirection::Horizontal; + for _ in 0..3 { + assert_eq!(TextDirection::from_ooxml(d.ooxml()), d); + d = d.next(); + } + assert_eq!(d, TextDirection::Horizontal, "the button cycles through three directions"); + // Strict names, East Asian vertical layouts and junk. + assert_eq!(TextDirection::from_ooxml("rl"), TextDirection::Down); + assert_eq!(TextDirection::from_ooxml("lr"), TextDirection::Up); + assert_eq!(TextDirection::from_ooxml("tbRlV"), TextDirection::Down); + assert_eq!(TextDirection::from_ooxml("tb"), TextDirection::Horizontal); + assert_eq!(TextDirection::from_ooxml("lrTbV"), TextDirection::Horizontal); + assert_eq!(TextDirection::from_ooxml("é?"), TextDirection::Horizontal); + } + + /// #865: a colour reads as `format.color` takes it (`RRGGBB`, `#RRGGBB`, `auto`), while what + /// documents and settings already hold reads and writes exactly as before. + #[test] + fn text_color_reads_hex_and_keeps_its_serialized_forms() { + let read = |j: &str| serde_json::from_str::(j); + let ink = TextColor::Rgb(Rgb(0x1B, 0x16, 0x11)); + assert_eq!(read(r#""1B1611""#).unwrap(), ink); + assert_eq!(read(r##""#1b1611""##).unwrap(), ink); + assert_eq!(read(r#""auto""#).unwrap(), TextColor::Auto); + assert_eq!(read(r#""Automatic""#).unwrap(), TextColor::Auto); + for (c, j) in [(TextColor::Auto, r#""Auto""#), (ink, r#"{"Rgb":[27,22,17]}"#)] { + assert_eq!(serde_json::to_string(&c).unwrap(), j, "written as before"); + assert_eq!(read(j).unwrap(), c); + } + for bad in [r#""1B161""#, r#""zzzzzz""#, r#""""#, r#"{"Rgb":"1B16"}"#, r#"{"Rgb":[1,2]}"#, r#"{"Hex":[1,2,3]}"#, "7", "[27,22,17]"] { + assert!(read(bad).is_err(), "{bad}"); + } + assert_eq!(read(r#"{"Rgb":"1B1611"}"#).unwrap(), ink); + let p: CharProps = serde_json::from_value(serde_json::json!({"color": "#1B1611", "bold": true})).unwrap(); + assert_eq!((p.color, p.bold), (Some(ink), Some(true))); + } + + /// #865: an [`Rgb`] reads `RRGGBB` / `#RRGGBB` as well as the `[r, g, b]` it still writes. + #[test] + fn rgb_reads_hex_and_keeps_its_serialized_form() { + let read = |j: &str| serde_json::from_str::(j); + let ink = Rgb(0x1B, 0x16, 0x11); + assert_eq!(serde_json::to_string(&ink).unwrap(), "[27,22,17]", "written as before"); + for j in ["[27,22,17]", r#""1B1611""#, r##""#1b1611""##] { + assert_eq!(read(j).unwrap(), ink, "{j}"); + } + for bad in ["[27,22]", "[27,22,17,0]", "[27,22,300]", "[-1,0,0]", r#""auto""#, r#""1B161""#, r#""""#, "7", "null", r#"{"r":1}"#] { + assert!(read(bad).is_err(), "{bad}"); + } + } + + #[test] + fn props_serde_skip_none() { + let p = CharProps { bold: Some(true), ..Default::default() }; + let s = serde_json::to_string(&p).unwrap(); + assert_eq!(s, r#"{"bold":true}"#); + let back: CharProps = serde_json::from_str(&s).unwrap(); + assert_eq!(back, p); + } + + #[test] + fn symbol_font_is_optional_in_json_and_survives_font_changes() { + // Older documents (no `symFont`) still load. + let old: CharProps = serde_json::from_str(r#"{"font":"Arial"}"#).unwrap(); + assert_eq!(old.sym_font, None); + let p = CharProps { font: Some("Arial".into()), sym_font: Some("Wingdings".into()), ..Default::default() }; + let s = serde_json::to_string(&p).unwrap(); + assert!(s.contains(r#""symFont":"Wingdings""#), "{s}"); + assert_eq!(serde_json::from_str::(&s).unwrap(), p); + // A font change keeps the symbol's own font; Clear Formatting too. + let changed = p.clone().overlaid(&CharProps { font: Some("Times New Roman".into()), ..Default::default() }); + assert_eq!(changed.sym_font.as_deref(), Some("Wingdings")); + assert_eq!(p.cleared().sym_font.as_deref(), Some("Wingdings")); + } +} diff --git a/crates/wordmodel/src/props/char_props.rs b/crates/wordmodel/src/props/char_props.rs new file mode 100644 index 000000000..3cde0b132 --- /dev/null +++ b/crates/wordmodel/src/props/char_props.rs @@ -0,0 +1,584 @@ +use super::color::{Rgb, TextColor}; +use super::para_props::Border; +use super::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Underline { + #[default] + None, + Single, + Words, + Double, + Thick, + Dotted, + Dash, + DotDash, + DotDotDash, + Wave, + DoubleWave, +} + +impl Underline { + pub const ALL: [Underline; 11] = [ + Underline::None, + Underline::Single, + Underline::Words, + Underline::Double, + Underline::Thick, + Underline::Dotted, + Underline::Dash, + Underline::DotDash, + Underline::DotDotDash, + Underline::Wave, + Underline::DoubleWave, + ]; + /// The OOXML `w:u/@w:val` name. + pub fn ooxml(self) -> &'static str { + match self { + Underline::None => "none", + Underline::Single => "single", + Underline::Words => "words", + Underline::Double => "double", + Underline::Thick => "thick", + Underline::Dotted => "dotted", + Underline::Dash => "dash", + Underline::DotDash => "dotDash", + Underline::DotDotDash => "dotDotDash", + Underline::Wave => "wave", + Underline::DoubleWave => "wavyDouble", + } + } + pub fn from_ooxml(s: &str) -> Underline { + Underline::ALL.iter().copied().find(|u| u.ooxml() == s).unwrap_or(match s { + "thickDash" | "dashLong" | "dashedHeavy" | "dashLongHeavy" => Underline::Dash, + "dottedHeavy" => Underline::Dotted, + "wavyHeavy" => Underline::Wave, + "dashDotHeavy" => Underline::DotDash, + "dashDotDotHeavy" => Underline::DotDotDash, + _ => Underline::Single, + }) + } +} + +/// The OpenType ligatures a run asks for ([MS-DOCX] `w14:ligatures`): standard (`liga`), +/// contextual (`clig`), historical (`hlig`) and discretional (`dlig`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Ligatures { + pub standard: bool, + pub contextual: bool, + pub historical: bool, + pub discretional: bool, +} + +impl Ligatures { + /// Every combination of the four flags. + pub fn all() -> impl Iterator { + (0u8..16).map(|b| Ligatures { standard: b & 1 != 0, contextual: b & 2 != 0, historical: b & 4 != 0, discretional: b & 8 != 0 }) + } + /// The `w14:ligatures/@w14:val` name: `none`, `all`, or the set flags joined in camelCase + /// (`standardContextual`). + pub fn ooxml(self) -> String { + let on: Vec<&str> = + [(self.standard, "standard"), (self.contextual, "contextual"), (self.historical, "historical"), (self.discretional, "discretional")] + .into_iter() + .filter_map(|(on, n)| on.then_some(n)) + .collect(); + match on.len() { + 0 => "none".into(), + 4 => "all".into(), + _ => on + .iter() + .enumerate() + .flat_map(|(i, n)| n.chars().enumerate().map(move |(j, c)| if i > 0 && j == 0 { c.to_ascii_uppercase() } else { c })) + .collect(), + } + } + /// `None` for a name [MS-DOCX] doesn't define. + pub fn from_ooxml(s: &str) -> Option { + Ligatures::all().find(|l| l.ooxml() == s) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum VertAlign { + #[default] + Baseline, + Superscript, + Subscript, +} + +/// Word's 16 highlighter colours (plus none). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Highlight { + #[default] + None, + Yellow, + BrightGreen, + Turquoise, + Pink, + Blue, + Red, + DarkBlue, + Teal, + Green, + Violet, + DarkRed, + DarkYellow, + Gray50, + Gray25, + Black, + White, +} + +impl Highlight { + pub const ALL: [Highlight; 17] = [ + Highlight::None, + Highlight::Yellow, + Highlight::BrightGreen, + Highlight::Turquoise, + Highlight::Pink, + Highlight::Blue, + Highlight::Red, + Highlight::DarkBlue, + Highlight::Teal, + Highlight::Green, + Highlight::Violet, + Highlight::DarkRed, + Highlight::DarkYellow, + Highlight::Gray50, + Highlight::Gray25, + Highlight::Black, + Highlight::White, + ]; + pub fn rgb(self) -> Option { + Some(match self { + Highlight::None => return None, + Highlight::Yellow => Rgb(255, 255, 0), + Highlight::BrightGreen => Rgb(0, 255, 0), + Highlight::Turquoise => Rgb(0, 255, 255), + Highlight::Pink => Rgb(255, 0, 255), + Highlight::Blue => Rgb(0, 0, 255), + Highlight::Red => Rgb(255, 0, 0), + Highlight::DarkBlue => Rgb(0, 0, 128), + Highlight::Teal => Rgb(0, 128, 128), + Highlight::Green => Rgb(0, 128, 0), + Highlight::Violet => Rgb(128, 0, 128), + Highlight::DarkRed => Rgb(128, 0, 0), + Highlight::DarkYellow => Rgb(128, 128, 0), + Highlight::Gray50 => Rgb(128, 128, 128), + Highlight::Gray25 => Rgb(192, 192, 192), + Highlight::Black => Rgb(0, 0, 0), + Highlight::White => Rgb(255, 255, 255), + }) + } + /// OOXML `w:highlight/@w:val`. + pub fn ooxml(self) -> &'static str { + match self { + Highlight::None => "none", + Highlight::Yellow => "yellow", + Highlight::BrightGreen => "green", + Highlight::Turquoise => "cyan", + Highlight::Pink => "magenta", + Highlight::Blue => "blue", + Highlight::Red => "red", + Highlight::DarkBlue => "darkBlue", + Highlight::Teal => "darkCyan", + Highlight::Green => "darkGreen", + Highlight::Violet => "darkMagenta", + Highlight::DarkRed => "darkRed", + Highlight::DarkYellow => "darkYellow", + Highlight::Gray50 => "darkGray", + Highlight::Gray25 => "lightGray", + Highlight::Black => "black", + Highlight::White => "white", + } + } + pub fn from_ooxml(s: &str) -> Highlight { + Highlight::ALL.iter().copied().find(|h| h.ooxml() == s).unwrap_or(Highlight::None) + } + pub fn name(self) -> &'static str { + match self { + Highlight::None => "No Color", + Highlight::Yellow => "Yellow", + Highlight::BrightGreen => "Bright Green", + Highlight::Turquoise => "Turquoise", + Highlight::Pink => "Pink", + Highlight::Blue => "Blue", + Highlight::Red => "Red", + Highlight::DarkBlue => "Dark Blue", + Highlight::Teal => "Teal", + Highlight::Green => "Green", + Highlight::Violet => "Violet", + Highlight::DarkRed => "Dark Red", + Highlight::DarkYellow => "Dark Yellow", + Highlight::Gray50 => "Gray-50%", + Highlight::Gray25 => "Gray-25%", + Highlight::Black => "Black", + Highlight::White => "White", + } + } +} + +/// The text size, in points, of a run whose size no style or default sets. +pub const DEFAULT_FONT_SIZE: f32 = 11.0; + +/// Character formatting (direct or in a style). `None` = inherit. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CharProps { + /// Character style id. + #[serde(skip_serializing_if = "Option::is_none")] + pub style: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub font: Option, + /// The symbol's own font (OOXML `w:sym/@w:font`, ECMA-376 §17.3.3.30), set on a symbol + /// character from a `w:sym` or Insert › Symbol: it is shown in this font whatever `font` is, + /// so changing the text's font keeps the symbol. `None` = the run's font. + #[serde(skip_serializing_if = "Option::is_none")] + pub sym_font: Option, + /// Points. + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bold: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub italic: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub underline: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub underline_color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub strike: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub double_strike: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub color: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub highlight: Option, + /// Character shading (background fill). + #[serde(skip_serializing_if = "Option::is_none")] + pub shading: Option, + /// Character border (`w:bdr`); `style: None` = explicitly no border. + #[serde(skip_serializing_if = "Option::is_none")] + pub border: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub vert_align: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub caps: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub small_caps: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub hidden: Option, + /// Hidden in Web Layout (`w:webHidden`); distinct from `w:vanish` (`hidden`). + #[serde(skip_serializing_if = "Option::is_none")] + pub web_hidden: Option, + /// Snap characters to the document grid (`w:snapToGrid`). Unset inherits; Word's default is on. + #[serde(skip_serializing_if = "Option::is_none")] + pub snap_to_grid: Option, + /// Character spacing (expanded/condensed), points. + #[serde(skip_serializing_if = "Option::is_none")] + pub spacing: Option, + /// Horizontal scale, percent (100 = normal). + #[serde(skip_serializing_if = "Option::is_none")] + pub scale: Option, + /// Raised (+) / lowered (−) position, points. + #[serde(skip_serializing_if = "Option::is_none")] + pub position: Option, + /// Kerning for fonts at or above this size (points); 0 = off. + #[serde(skip_serializing_if = "Option::is_none")] + pub kern: Option, + /// A mark over (or under) each character (`w:em`, Font › Emphasis mark), as East Asian text + /// stresses words; `Some(EmphasisMark::None)` turns off one a style sets. + #[serde(skip_serializing_if = "Option::is_none")] + pub emphasis: Option, + /// OpenType ligatures (`w14:ligatures`). `None` leaves the app's default: standard and + /// contextual ligatures. + #[serde(skip_serializing_if = "Option::is_none")] + pub ligatures: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub outline: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub shadow: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub emboss: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub engrave: Option, + /// WordArt text effects: fill, outline, shadow, glow, reflection ([MS-DOCX] `w14:` run + /// properties). Overlaid effect by effect. + #[serde(skip_serializing_if = "Option::is_none")] + pub text_effects: Option>, + /// BCP 47 language tag (proofing). + #[serde(skip_serializing_if = "Option::is_none")] + pub lang: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_proof: Option, + /// Right-to-left run (OOXML `w:rtl`): its characters are complex script and read right to left. + #[serde(skip_serializing_if = "Option::is_none")] + pub rtl: Option, + /// Format the whole run with the complex-script properties below (OOXML `w:cs`). + #[serde(skip_serializing_if = "Option::is_none")] + pub cs: Option, + /// Complex-script font (Arabic, Persian, Hebrew…; OOXML `w:rFonts/@w:cs`). `None` = `font`. + #[serde(skip_serializing_if = "Option::is_none")] + pub font_cs: Option, + /// East Asian font (Chinese, Japanese and Korean text; OOXML `w:rFonts/@w:eastAsia`). `None` = + /// inherit, then `font`. + #[serde(skip_serializing_if = "Option::is_none")] + pub font_ea: Option, + /// Complex-script size, points (`w:szCs`). `None` leaves the inherited value unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + pub size_cs: Option, + /// Complex-script bold / italic (`w:bCs`, `w:iCs`). `None` leaves the inherited value unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + pub bold_cs: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub italic_cs: Option, + /// BCP 47 language of complex-script text (`w:lang/@w:bidi`, e.g. `fa-IR`). + #[serde(skip_serializing_if = "Option::is_none")] + pub lang_bidi: Option, + /// BCP 47 language of East Asian text (`w:lang/@w:eastAsia`, e.g. `zh-CN`, `ja-JP`, `ko-KR`). + #[serde(skip_serializing_if = "Option::is_none")] + pub lang_ea: Option, + /// East Asian typography: two lines in one and horizontal in vertical (`w:eastAsianLayout`). + /// Kept for round trip; layout doesn't apply it yet. + #[serde(skip_serializing_if = "Option::is_none")] + pub east_asian_layout: Option, + /// Hyperlink target: a URL, or `#bookmark` for an internal link. + #[serde(skip_serializing_if = "Option::is_none")] + pub link: Option, + /// Tracked insertion: index into `Document::revisions`. + #[serde(skip_serializing_if = "Option::is_none")] + pub ins: Option, + /// Tracked deletion: index into `Document::revisions`. + #[serde(skip_serializing_if = "Option::is_none")] + pub del: Option, + /// Tracked formatting change (`w:rPrChange`): the formatting before it. + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, +} + +/// East Asian typography settings of a run (ISO/IEC 29500-1 §17.3.2.10, `w:eastAsianLayout`): +/// *two lines in one* sets the run's text on two sub-lines inside one line (warichu), *horizontal +/// in vertical* turns it 90° to read across a vertical line (tate-chu-yoko). Each attribute is +/// `None` when the file doesn't say. +#[derive(Clone, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct EastAsianLayout { + /// East Asian typography run id (`w:id`). + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + /// Two lines in one (`w:combine`). + #[serde(skip_serializing_if = "Option::is_none")] + pub combine: Option, + /// Brackets around two lines in one (`w:combineBrackets`). + #[serde(skip_serializing_if = "Option::is_none")] + pub combine_brackets: Option, + /// Horizontal in vertical (`w:vert`). + #[serde(skip_serializing_if = "Option::is_none")] + pub vert: Option, + /// Compress horizontal-in-vertical text to the line height (`w:vertCompress`). + #[serde(skip_serializing_if = "Option::is_none")] + pub vert_compress: Option, +} + +/// A mark over or under each character (`ST_Em`, ECMA-376 §17.18.24): Word's Font › Emphasis +/// mark. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum EmphasisMark { + #[default] + None, + /// A dot above. + Dot, + /// A comma (sesame) above. + Comma, + /// A hollow circle above. + Circle, + /// A dot below. + UnderDot, +} + +impl EmphasisMark { + pub const ALL: [EmphasisMark; 5] = [EmphasisMark::None, EmphasisMark::Dot, EmphasisMark::Comma, EmphasisMark::Circle, EmphasisMark::UnderDot]; + /// OOXML `w:em` value. + pub fn ooxml(self) -> &'static str { + match self { + EmphasisMark::None => "none", + EmphasisMark::Dot => "dot", + EmphasisMark::Comma => "comma", + EmphasisMark::Circle => "circle", + EmphasisMark::UnderDot => "underDot", + } + } + /// The mark an OOXML `w:em` value names; unknown values are none. + pub fn from_ooxml(v: &str) -> EmphasisMark { + EmphasisMark::ALL.into_iter().find(|m| m.ooxml() == v.trim()).unwrap_or_default() + } +} + +/// Brackets around two lines in one (`ST_CombineBrackets`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CombineBrackets { + None, + Round, + Square, + Angle, + Curly, +} + +impl CombineBrackets { + pub const ALL: [CombineBrackets; 5] = + [CombineBrackets::None, CombineBrackets::Round, CombineBrackets::Square, CombineBrackets::Angle, CombineBrackets::Curly]; + /// OOXML `w:combineBrackets` value. + pub fn ooxml(self) -> &'static str { + match self { + CombineBrackets::None => "none", + CombineBrackets::Round => "round", + CombineBrackets::Square => "square", + CombineBrackets::Angle => "angle", + CombineBrackets::Curly => "curly", + } + } + /// The value for `s`, `None` for one the schema doesn't have. + pub fn from_ooxml(s: &str) -> Option { + CombineBrackets::ALL.into_iter().find(|b| b.ooxml() == s) + } +} + +/// A tracked formatting change (ECMA-376 §17.13.5: `w:rPrChange`, `w:pPrChange`, +/// `w:tblPrChange`, `w:trPrChange`, `w:tcPrChange`, `w:sectPrChange`): the properties as they +/// were before the change, and the revision (author, date) that made it. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PropChange { + /// Index into `Document::revisions`. + pub rev: u32, + /// The properties before the change (formatting only: no revision or link marks). + pub old: T, +} + +impl PropChange { + pub fn boxed(rev: u32, old: T) -> Option>> { + Some(Box::new(PropChange { rev, old })) + } +} + +/// A tracked change of a paragraph's list numbering (`w:numberingChange`, kept for round trip): +/// `original` is the number as it was shown before the change. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct NumChange { + /// Index into `Document::revisions`. + pub rev: u32, + pub original: String, +} + +impl CharProps { + /// The same formatting outside a hyperlink: no link target, and no "Hyperlink" character + /// style that came with it. What a paragraph mark or text typed next to a link gets. + pub fn unlinked(&self) -> CharProps { + let mut c = self.clone(); + if c.link.take().is_some() && c.style.as_deref() == Some("Hyperlink") { + c.style = None; + } + c + } + + /// The fields of `self` that differ from `base` (equal ones unset): what `self` adds when + /// overlaid on `base`. + pub fn minus(mut self, base: &CharProps) -> CharProps { + self.minus_text_style(base); + self.minus_run_visibility(base); + self.minus_script_style(base); + self.minus_document_marks(base); + self + } + + fn minus_text_style(&mut self, base: &CharProps) { + drop_same_fields!(self, base; style, font, sym_font, size, bold, italic, underline, underline_color, strike, double_strike, color, highlight, + shading, border, vert_align, caps, small_caps, hidden, spacing, scale, position, kern, outline, shadow, emboss, engrave, lang, no_proof, rtl); + } + + fn minus_run_visibility(&mut self, base: &CharProps) { + drop_same_fields!(self, base; web_hidden, snap_to_grid, emphasis); + } + + fn minus_script_style(&mut self, base: &CharProps) { + drop_same_fields!(self, base; cs, font_cs, font_ea, size_cs, bold_cs, italic_cs, lang_bidi, lang_ea, east_asian_layout); + } + + fn minus_document_marks(&mut self, base: &CharProps) { + drop_same_fields!(self, base; link, ins, del, ligatures); + } + + /// Apply every `Some` field of `patch` on top of `self`. + /// + /// Plain and complex-script properties inherit independently. Commands that intend to + /// format both scripts must set both values explicitly. + pub fn overlay(&mut self, patch: &CharProps) { + self.overlay_text_style(patch); + self.overlay_run_visibility(patch); + self.overlay_script_style(patch); + self.overlay_document_marks(patch); + if let Some(p) = &patch.text_effects { + self.text_effects.get_or_insert_with(Default::default).overlay(p); + } + } + + fn overlay_text_style(&mut self, patch: &CharProps) { + overlay_fields!(self, patch; style, font, sym_font, size, bold, italic, underline, underline_color, strike, double_strike, color, highlight, + shading, border, vert_align, caps, small_caps, hidden, spacing, scale, position, kern, outline, shadow, emboss, engrave, lang, no_proof, rtl); + } + + fn overlay_run_visibility(&mut self, patch: &CharProps) { + overlay_fields!(self, patch; web_hidden, snap_to_grid, emphasis); + } + + fn overlay_script_style(&mut self, patch: &CharProps) { + overlay_fields!(self, patch; cs, font_cs, font_ea, size_cs, bold_cs, italic_cs, lang_bidi, lang_ea, east_asian_layout); + } + + fn overlay_document_marks(&mut self, patch: &CharProps) { + overlay_fields!(self, patch; link, ins, del, fmt_change, ligatures); + } + pub fn overlaid(mut self, patch: &CharProps) -> CharProps { + self.overlay(patch); + self + } + pub fn is_empty(&self) -> bool { + *self == CharProps::default() + } + /// The font a character of this run is shown in: the symbol's own font when it has one + /// ([`Self::sym_font`]), else the run's font. + pub fn shown_font(&self) -> Option<&str> { + self.sym_font.as_deref().filter(|f| !f.trim().is_empty()).or(self.font.as_deref()) + } + /// The same formatting for text typed or painted next to a symbol: without the symbol's own + /// font. + pub fn without_symbol(mut self) -> CharProps { + self.sym_font = None; + self + } + /// Clear direct formatting but keep what isn't "formatting" (links, revisions, character style + /// is cleared too, as Word's Clear Formatting does). + pub fn cleared(&self) -> CharProps { + CharProps { + link: self.link.clone(), + ins: self.ins, + del: self.del, + fmt_change: self.fmt_change.clone(), + // A symbol stays the symbol it is. + sym_font: self.sym_font.clone(), + ..Default::default() + } + } + /// Just the formatting: without the link, revision marks and tracked formatting change. + pub fn formatting(&self) -> CharProps { + CharProps { link: None, ins: None, del: None, fmt_change: None, ..self.clone() } + } +} diff --git a/crates/wordmodel/src/props/color.rs b/crates/wordmodel/src/props/color.rs new file mode 100644 index 000000000..baf2d948d --- /dev/null +++ b/crates/wordmodel/src/props/color.rs @@ -0,0 +1,171 @@ +use super::{Deserialize, Serialize}; + +/// An sRGB colour. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Default)] +pub struct Rgb(pub u8, pub u8, pub u8); + +impl<'de> Deserialize<'de> for Rgb { + /// Reads what it writes (`[r, g, b]`) and also a string [`Rgb::parse`] reads, so the colours in + /// `CharProps` / `ParaProps` take the `RRGGBB` that `para.borders` and `table.shading` take (#865). + fn deserialize>(d: D) -> Result { + use serde::de::{Error, SeqAccess, Unexpected, Visitor}; + struct V; + impl<'de> Visitor<'de> for V { + type Value = Rgb; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("`[r, g, b]`, `RRGGBB` or `#RRGGBB`") + } + fn visit_str(self, s: &str) -> Result { + Rgb::parse(s).ok_or_else(|| E::invalid_value(Unexpected::Str(s), &self)) + } + fn visit_seq>(self, mut a: A) -> Result { + let mut c = [0u8; 3]; + for (i, x) in c.iter_mut().enumerate() { + *x = a.next_element()?.ok_or_else(|| A::Error::invalid_length(i, &self))?; + } + if a.next_element::()?.is_some() { + return Err(A::Error::invalid_length(4, &self)); + } + Ok(Rgb(c[0], c[1], c[2])) + } + } + d.deserialize_any(V) + } +} + +impl Rgb { + pub const BLACK: Rgb = Rgb(0, 0, 0); + pub const WHITE: Rgb = Rgb(255, 255, 255); + /// `RRGGBB` (no `#`). + pub fn hex(self) -> String { + format!("{:02X}{:02X}{:02X}", self.0, self.1, self.2) + } + /// Parse `RRGGBB` or `#RRGGBB`. + pub fn parse(s: &str) -> Option { + let s = s.trim(); + let s = s.strip_prefix('#').unwrap_or(s); + if s.len() != 6 || !s.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let p = |i: usize| s.get(i..i + 2).and_then(|h| u8::from_str_radix(h, 16).ok()); + Some(Rgb(p(0)?, p(2)?, p(4)?)) + } + /// This colour with the DrawingML colour modifier `name` (`lumMod`, `lumOff`, `tint`, + /// `shade`, `satMod`, ECMA-376 §20.1.2.3) at `v` (1.0 = 100%); other names leave it + /// unchanged. + pub fn modified(self, name: &str, v: f64) -> Rgb { + match name { + "lumMod" => adjust_lum(self, |l| l * v), + "lumOff" => adjust_lum(self, |l| l + v), + "tint" => mix(self, 255.0, v), + "shade" => mix(self, 0.0, v), + "satMod" => { + let (h, sat, l) = to_hsl(self); + from_hsl(h, (sat * v).clamp(0.0, 1.0), l) + } + _ => self, + } + } + /// The colour with hue `h` (0..1 of a turn), saturation `s` and lightness `l` (0..1). + pub fn from_hsl(h: f64, s: f64, l: f64) -> Rgb { + from_hsl(h, s, l) + } + /// This colour with its HSL luminance changed by `f`. + pub fn with_lum(self, f: impl Fn(f64) -> f64) -> Rgb { + adjust_lum(self, f) + } + pub fn luma(self) -> f32 { + 0.299 * self.0 as f32 + 0.587 * self.1 as f32 + 0.114 * self.2 as f32 + } +} + +/// A text colour: automatic (black on light backgrounds, white on dark) or a fixed colour. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Default)] +pub enum TextColor { + #[default] + Auto, + Rgb(Rgb), +} + +impl TextColor { + /// `auto` or `automatic` (any case), or a colour [`Rgb::parse`] reads. + pub fn parse(s: &str) -> Option { + if s.eq_ignore_ascii_case("auto") || s.eq_ignore_ascii_case("automatic") { Some(TextColor::Auto) } else { Rgb::parse(s).map(TextColor::Rgb) } + } +} + +impl<'de> Deserialize<'de> for TextColor { + /// Reads what it writes (`"Auto"`, `{"Rgb": [r, g, b]}`) and also a string [`TextColor::parse`] + /// reads, so `styles.modify` and `format.set` take the colours `format.color` takes. + fn deserialize>(d: D) -> Result { + use serde::de::{Error, Unexpected}; + #[derive(Deserialize)] + enum Written { + Auto, + Rgb(Rgb), + } + match serde_json::Value::deserialize(d)? { + serde_json::Value::String(s) => { + TextColor::parse(&s).ok_or_else(|| D::Error::invalid_value(Unexpected::Str(&s), &"`auto`, `RRGGBB` or `#RRGGBB`")) + } + v => Ok(match serde_json::from_value(v).map_err(D::Error::custom)? { + Written::Auto => TextColor::Auto, + Written::Rgb(c) => TextColor::Rgb(c), + }), + } + } +} + +/// `tint` (towards `to` = 255) and `shade` (towards 0): `v` is the share of the original colour kept. +fn mix(c: Rgb, to: f64, v: f64) -> Rgb { + let f = |x: u8| (x as f64 * v + to * (1.0 - v)).round().clamp(0.0, 255.0) as u8; + Rgb(f(c.0), f(c.1), f(c.2)) +} + +/// `c` with its HSL luminance changed by `f`. +fn adjust_lum(c: Rgb, f: impl Fn(f64) -> f64) -> Rgb { + let (h, s, l) = to_hsl(c); + from_hsl(h, s, f(l).clamp(0.0, 1.0)) +} + +fn to_hsl(c: Rgb) -> (f64, f64, f64) { + let (r, g, b) = (c.0 as f64 / 255.0, c.1 as f64 / 255.0, c.2 as f64 / 255.0); + let (max, min) = (r.max(g).max(b), r.min(g).min(b)); + let l = (max + min) / 2.0; + if max == min { + return (0.0, 0.0, l); + } + let d = max - min; + let s = if l > 0.5 { d / (2.0 - max - min) } else { d / (max + min) }; + let h = if max == r { + (g - b) / d + if g < b { 6.0 } else { 0.0 } + } else if max == g { + (b - r) / d + 2.0 + } else { + (r - g) / d + 4.0 + }; + (h / 6.0, s, l) +} + +fn from_hsl(h: f64, s: f64, l: f64) -> Rgb { + let byte = |v: f64| (v * 255.0).round().clamp(0.0, 255.0) as u8; + if s == 0.0 { + return Rgb(byte(l), byte(l), byte(l)); + } + let q = if l < 0.5 { l * (1.0 + s) } else { l + s - l * s }; + let p = 2.0 * l - q; + let channel = |t: f64| { + let t = t.rem_euclid(1.0); + let v = if t < 1.0 / 6.0 { + p + (q - p) * 6.0 * t + } else if t < 0.5 { + q + } else if t < 2.0 / 3.0 { + p + (q - p) * (2.0 / 3.0 - t) * 6.0 + } else { + p + }; + byte(v) + }; + Rgb(channel(h + 1.0 / 3.0), channel(h), channel(h - 1.0 / 3.0)) +} diff --git a/crates/wordmodel/src/props/para_props.rs b/crates/wordmodel/src/props/para_props.rs new file mode 100644 index 000000000..a48540248 --- /dev/null +++ b/crates/wordmodel/src/props/para_props.rs @@ -0,0 +1,552 @@ +use super::char_props::{NumChange, PropChange}; +use super::color::Rgb; +use super::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Align { + #[default] + Left, + Center, + Right, + Justify, + Distribute, +} + +impl Align { + /// Alignment is logical (`Left` is the start edge, ISO 29500 `start`): the alignment as seen + /// on the page, or the logical one for a visual choice, in a paragraph that reads right to + /// left when `rtl` (Left and Right swap; the swap is its own inverse). + pub fn visual(self, rtl: bool) -> Align { + match self { + Align::Left if rtl => Align::Right, + Align::Right if rtl => Align::Left, + a => a, + } + } +} + +/// Paragraph-level text flow (`w:pPr/w:textDirection`, ECMA-376 §17.3.1.41). +/// +/// Kept separate from [`super::TextDirection`], whose three values model the table-cell direction +/// control. These values preserve the exact OOXML direction through JSON and DOCX saves. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ParagraphTextDirection { + LrTb, + LrTbV, + Tb, + TbRl, + TbRlV, + TbLr, + TbLrV, + BtLr, + TbV, + Rl, + RlV, + Lr, + LrV, +} + +impl ParagraphTextDirection { + pub fn from_ooxml(value: &str) -> Option { + match value { + "lrTb" => Some(Self::LrTb), + "lrTbV" => Some(Self::LrTbV), + "tb" => Some(Self::Tb), + "tbRl" => Some(Self::TbRl), + "tbRlV" => Some(Self::TbRlV), + "tbLr" => Some(Self::TbLr), + "tbLrV" => Some(Self::TbLrV), + "btLr" => Some(Self::BtLr), + "tbV" => Some(Self::TbV), + "rl" => Some(Self::Rl), + "rlV" => Some(Self::RlV), + "lr" => Some(Self::Lr), + "lrV" => Some(Self::LrV), + _ => None, + } + } + + pub fn ooxml(self) -> &'static str { + match self { + Self::LrTb => "lrTb", + Self::LrTbV => "lrTbV", + Self::Tb => "tb", + Self::TbRl => "tbRl", + Self::TbRlV => "tbRlV", + Self::TbLr => "tbLr", + Self::TbLrV => "tbLrV", + Self::BtLr => "btLr", + Self::TbV => "tbV", + Self::Rl => "rl", + Self::RlV => "rlV", + Self::Lr => "lr", + Self::LrV => "lrV", + } + } +} + +/// Vertical alignment of characters of different sizes on a line (`w:textAlignment`, +/// ECMA-376 §17.3.1.39). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TextAlignment { + Auto, + Top, + Center, + Baseline, + Bottom, +} + +impl TextAlignment { + pub fn ooxml(self) -> &'static str { + match self { + TextAlignment::Auto => "auto", + TextAlignment::Top => "top", + TextAlignment::Center => "center", + TextAlignment::Baseline => "baseline", + TextAlignment::Bottom => "bottom", + } + } + pub fn from_ooxml(s: &str) -> Option { + match s { + "auto" => Some(TextAlignment::Auto), + "top" => Some(TextAlignment::Top), + "center" => Some(TextAlignment::Center), + "baseline" => Some(TextAlignment::Baseline), + "bottom" => Some(TextAlignment::Bottom), + _ => None, + } + } +} + +/// Line spacing rule. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "rule", content = "value")] +pub enum LineSpacing { + /// Multiple of single spacing (1.0 = single, 1.15, 1.5, 2.0 = double). + Multiple(f32), + /// At least this many points. + AtLeast(f32), + /// Exactly this many points. + Exactly(f32), +} + +impl Default for LineSpacing { + fn default() -> Self { + LineSpacing::Multiple(1.0) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TabAlign { + #[default] + Left, + Center, + Right, + Decimal, + Bar, + /// Clears an inherited stop at this position. + Clear, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TabLeader { + #[default] + None, + Dot, + Hyphen, + Underscore, + MiddleDot, +} + +impl TabLeader { + pub fn char(self) -> Option { + match self { + TabLeader::None => None, + TabLeader::Dot => Some('.'), + TabLeader::Hyphen => Some('-'), + TabLeader::Underscore => Some('_'), + TabLeader::MiddleDot => Some('·'), + } + } +} + +/// Two tab positions closer than this (points) are the same stop: `para.tabs` replaces at this +/// tolerance, and the Tabs dialog and the ruler merge and detect no-ops with it. +pub const TAB_POS_EPS: f32 = 0.5; + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +pub struct TabStop { + /// Points from the paragraph's left indent origin (the text column's left edge). + pub pos: f32, + #[serde(default)] + pub align: TabAlign, + #[serde(default)] + pub leader: TabLeader, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum BorderStyle { + #[default] + None, + Single, + Double, + Dotted, + Dashed, + Thick, + Triple, + DotDash, + Wave, + /// The compound lines of thin and thick strokes (`thinThickSmallGap` to + /// `thinThickThinLargeGap`), kept so a file writes back the line it read; drawn as a single + /// line of the border's width. + ThinThickSmallGap, + ThickThinSmallGap, + ThinThickThinSmallGap, + ThinThickMediumGap, + ThickThinMediumGap, + ThinThickThinMediumGap, + ThinThickLargeGap, + ThickThinLargeGap, + ThinThickThinLargeGap, +} + +impl BorderStyle { + pub fn ooxml(self) -> &'static str { + match self { + BorderStyle::None => "nil", + BorderStyle::Single => "single", + BorderStyle::Double => "double", + BorderStyle::Dotted => "dotted", + BorderStyle::Dashed => "dashed", + BorderStyle::Thick => "thick", + BorderStyle::Triple => "triple", + BorderStyle::DotDash => "dotDash", + BorderStyle::Wave => "wave", + BorderStyle::ThinThickSmallGap => "thinThickSmallGap", + BorderStyle::ThickThinSmallGap => "thickThinSmallGap", + BorderStyle::ThinThickThinSmallGap => "thinThickThinSmallGap", + BorderStyle::ThinThickMediumGap => "thinThickMediumGap", + BorderStyle::ThickThinMediumGap => "thickThinMediumGap", + BorderStyle::ThinThickThinMediumGap => "thinThickThinMediumGap", + BorderStyle::ThinThickLargeGap => "thinThickLargeGap", + BorderStyle::ThickThinLargeGap => "thickThinLargeGap", + BorderStyle::ThinThickThinLargeGap => "thinThickThinLargeGap", + } + } + pub fn from_ooxml(s: &str) -> BorderStyle { + match s { + "nil" | "none" => BorderStyle::None, + "double" => BorderStyle::Double, + "dotted" => BorderStyle::Dotted, + "dashed" | "dashSmallGap" => BorderStyle::Dashed, + "thick" => BorderStyle::Thick, + "triple" => BorderStyle::Triple, + "dotDash" | "dotDotDash" => BorderStyle::DotDash, + "wave" | "doubleWave" => BorderStyle::Wave, + "thinThickSmallGap" => BorderStyle::ThinThickSmallGap, + "thickThinSmallGap" => BorderStyle::ThickThinSmallGap, + "thinThickThinSmallGap" => BorderStyle::ThinThickThinSmallGap, + "thinThickMediumGap" => BorderStyle::ThinThickMediumGap, + "thickThinMediumGap" => BorderStyle::ThickThinMediumGap, + "thinThickThinMediumGap" => BorderStyle::ThinThickThinMediumGap, + "thinThickLargeGap" => BorderStyle::ThinThickLargeGap, + "thickThinLargeGap" => BorderStyle::ThickThinLargeGap, + "thinThickThinLargeGap" => BorderStyle::ThinThickThinLargeGap, + _ => BorderStyle::Single, + } + } +} + +/// One border line. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct Border { + pub style: BorderStyle, + /// Line width, points. + pub width: f32, + pub color: Option, + /// Distance from the text, points. + pub space: f32, +} + +impl Border { + pub fn single(width: f32) -> Border { + Border { style: BorderStyle::Single, width, color: None, space: 0.0 } + } + pub fn is_visible(&self) -> bool { + self.style != BorderStyle::None && self.width > 0.0 + } +} + +/// Borders on the four sides (plus inner borders for tables and runs of bordered paragraphs). +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct Borders { + pub top: Option, + pub left: Option, + pub bottom: Option, + pub right: Option, + /// Between paragraphs / inside horizontal (tables). + pub between: Option, + /// Inside vertical (tables). + pub inside_v: Option, +} + +impl Borders { + pub fn all(b: Border) -> Borders { + Borders { top: Some(b), left: Some(b), bottom: Some(b), right: Some(b), between: Some(b), inside_v: Some(b) } + } + pub fn box_(b: Border) -> Borders { + Borders { top: Some(b), left: Some(b), bottom: Some(b), right: Some(b), between: None, inside_v: None } + } + pub fn any_visible(&self) -> bool { + [self.top, self.left, self.bottom, self.right, self.between, self.inside_v].iter().flatten().any(Border::is_visible) + } + /// A side is drawn, or a paragraph side without a line holds padding (ODF's, which keeps + /// its room in LibreOffice's layout). + pub fn takes_room(&self) -> bool { + self.any_visible() || [self.top, self.left, self.bottom, self.right].iter().flatten().any(|b| b.space.is_finite() && b.space > 0.0) + } + /// Apply every `Some` side of `patch` on top of `self`. + pub fn overlay(&mut self, patch: &Borders) { + if patch.top.is_some() { + self.top = patch.top; + } + if patch.left.is_some() { + self.left = patch.left; + } + if patch.bottom.is_some() { + self.bottom = patch.bottom; + } + if patch.right.is_some() { + self.right = patch.right; + } + if patch.between.is_some() { + self.between = patch.between; + } + if patch.inside_v.is_some() { + self.inside_v = patch.inside_v; + } + } +} + +/// A reference to a numbering definition and level. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct NumRef { + /// `Numbering::nums` id; 0 removes numbering inherited from a style. + pub num: u32, + pub level: u8, +} + +/// A text frame's placement (`w:framePr`, ECMA-376 §17.3.1.11), in points. Offsets apply when +/// the matching alignment is unset; anchors and the wrap keep OOXML's names. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TextFrame { + /// Width; `None` fits the text. + pub w: Option, + /// Height, with `h_exact` exact rather than at least; `None` fits the text. + pub h: Option, + pub h_exact: bool, + pub x: f32, + pub y: f32, + /// `left`, `center`, `right`, `inside`, `outside`. + pub x_align: Option, + /// `top`, `center`, `bottom`, `inside`, `outside` (`inline` is no alignment). + pub y_align: Option, + /// `text` (the column), `margin` or `page`. + pub h_anchor: String, + /// `text` (the paragraph), `margin` or `page`. + pub v_anchor: String, + /// `around`, `auto`, `notBeside`, `none`, `tight` or `through`. + pub wrap: String, + /// Whether the frame stays fixed to its page anchor when its paragraph moves. + #[serde(skip_serializing_if = "Option::is_none")] + pub anchor_lock: Option, + /// Distance from the text left and right, and above and below. + pub h_space: f32, + pub v_space: f32, +} + +/// Paragraph formatting. `None` = inherit. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ParaProps { + /// Paragraph style id. + #[serde(skip_serializing_if = "Option::is_none")] + pub style: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub align: Option, + /// Left indent, points. + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_left: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_right: Option, + /// First-line indent, points; negative = hanging. + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_first: Option, + /// Left (start) indent in characters (`w:leftChars` / `w:startChars`, stored by Word in + /// hundredths). A non-zero value wins over `indent_left`; one character is the size of the + /// paragraph mark's font. `Some(0.0)` cancels an inherited value. Common in East Asian documents. + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_left_chars: Option, + /// Right (end) indent in characters (`w:rightChars` / `w:endChars`), like `indent_left_chars`. + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_right_chars: Option, + /// First-line indent in characters (`w:firstLineChars`; negative = `w:hangingChars`), like + /// `indent_left_chars`. + #[serde(skip_serializing_if = "Option::is_none")] + pub indent_first_chars: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub space_before: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub space_after: Option, + /// HTML-style automatic space before (`w:beforeAutospacing`): 14 pt, none at the top of the + /// document or of a table cell, and collapsing with the space after the paragraph above. + #[serde(skip_serializing_if = "Option::is_none")] + pub space_before_auto: Option, + /// HTML-style automatic space after (`w:afterAutospacing`): 14 pt, none at the end of a cell. + #[serde(skip_serializing_if = "Option::is_none")] + pub space_after_auto: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub line_spacing: Option, + /// Ignore space before/after between paragraphs of the same style. + #[serde(skip_serializing_if = "Option::is_none")] + pub contextual_spacing: Option, + /// `w:mirrorIndents` (ECMA-376 §17.3.1.18): the left and right indents are the inside and + /// outside ones, so they swap sides on even (left-hand) pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub mirror_indents: Option, + /// `w:adjustRightInd`: automatically adjust the right indent for the document grid when enabled; keep the specified right indent when disabled. + #[serde(skip_serializing_if = "Option::is_none")] + pub adjust_right_indent: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub keep_next: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub keep_lines: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub page_break_before: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub widow_control: Option, + /// 0..=8 = Level 1..9; 9 = body text. + #[serde(skip_serializing_if = "Option::is_none")] + pub outline_level: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub numbering: Option, + /// Tab stops (replaces the inherited list; `TabAlign::Clear` stops remove inherited ones). + #[serde(skip_serializing_if = "Option::is_none")] + pub tabs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub shading: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub borders: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub suppress_hyphens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub suppress_line_numbers: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bidi: Option, + /// Paragraph text direction (`w:textDirection`); `None` inherits from the paragraph style. + #[serde(skip_serializing_if = "Option::is_none")] + pub text_direction: Option, + /// Drop cap: lines to drop (0 = none) — applies to the first character(s). + #[serde(skip_serializing_if = "Option::is_none")] + pub drop_cap: Option, + /// Drop cap "In margin" (`w:dropCap="margin"`): the letter hangs in the left margin and the + /// text keeps its indent. Unset = dropped into the text. + #[serde(skip_serializing_if = "Option::is_none")] + pub drop_cap_margin: Option, + /// The drop cap's distance from the text, in points (`w:hSpace`). Unset = 0. + #[serde(skip_serializing_if = "Option::is_none")] + pub drop_cap_distance: Option, + /// The drop cap letter's font family (the cap run's `w:rFonts`). Unset = the text's font. + #[serde(skip_serializing_if = "Option::is_none")] + pub drop_cap_font: Option, + /// A text frame (`w:framePr` other than a drop cap): the paragraph is placed apart from the + /// text, which may wrap around it — a footer's page number in a frame, for instance. + #[serde(skip_serializing_if = "Option::is_none")] + pub frame: Option, + /// Asian typography (`w:kinsoku`): East Asian line-breaking rules — no line starts with + /// closing punctuation such as 、。」 or ends with opening punctuation such as 「(. + /// Unset = on. + #[serde(skip_serializing_if = "Option::is_none")] + pub kinsoku: Option, + /// `w:wordWrap`: Latin words wrap whole; `false` lets them break at any character (Word's + /// "Allow Latin text to wrap in the middle of a word"). Unset = on. + #[serde(skip_serializing_if = "Option::is_none")] + pub word_wrap: Option, + /// `w:overflowPunct`: punctuation may hang past the line end. Unset = on. + #[serde(skip_serializing_if = "Option::is_none")] + pub overflow_punct: Option, + /// `w:topLinePunct`: compress punctuation at the start of a line. Unset = off. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_line_punct: Option, + /// `w:autoSpaceDE`: automatic space between Asian and Latin text. Unset = on. + #[serde(rename = "autoSpaceDE", skip_serializing_if = "Option::is_none")] + pub auto_space_de: Option, + /// `w:autoSpaceDN`: automatic space between Asian text and numbers. Unset = on. + #[serde(rename = "autoSpaceDN", skip_serializing_if = "Option::is_none")] + pub auto_space_dn: Option, + /// Tracked change of the list numbering (`w:numberingChange`). + #[serde(skip_serializing_if = "Option::is_none")] + pub num_change: Option>, + /// Tracked formatting change (`w:pPrChange`): the paragraph properties before it. + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, + /// `w:snapToGrid`: lines snap to the section's document grid. Unset = on. + #[serde(skip_serializing_if = "Option::is_none")] + pub snap_to_grid: Option, + /// `w:textAlignment`: how characters of different sizes line up on a line. Kept for the + /// file (not laid out yet); Word sizes a line by it, so dropping it changes Word's pages. + #[serde(skip_serializing_if = "Option::is_none")] + pub text_alignment: Option, +} + +impl ParaProps { + pub fn overlay(&mut self, patch: &ParaProps) { + self.overlay_layout(patch); + self.overlay_pagination(patch); + self.overlay_script_layout(patch); + } + + fn overlay_layout(&mut self, patch: &ParaProps) { + overlay_fields!(self, patch; style, align, indent_left, indent_right, indent_first, indent_left_chars, indent_right_chars, indent_first_chars, + space_before, space_after, space_before_auto, space_after_auto, line_spacing, contextual_spacing, mirror_indents, adjust_right_indent, tabs, shading, borders); + } + + fn overlay_pagination(&mut self, patch: &ParaProps) { + overlay_fields!(self, patch; keep_next, keep_lines, page_break_before, widow_control, outline_level, numbering, suppress_hyphens, suppress_line_numbers, + drop_cap, drop_cap_margin, drop_cap_distance, drop_cap_font, frame, snap_to_grid); + } + + fn overlay_script_layout(&mut self, patch: &ParaProps) { + overlay_fields!(self, patch; bidi, text_direction, kinsoku, word_wrap, overflow_punct, top_line_punct, auto_space_de, auto_space_dn); + overlay_fields!(self, patch; bidi, kinsoku, word_wrap, overflow_punct, top_line_punct, auto_space_de, auto_space_dn, text_alignment); + } + pub fn overlaid(mut self, patch: &ParaProps) -> ParaProps { + self.overlay(patch); + self + } + pub fn is_empty(&self) -> bool { + *self == ParaProps::default() + } + /// After setting indents in points: keep the indents in characters (left, right, first line) + /// an inherited style may set from winning over them, as Word does (`w:firstLineChars="0"`). + pub fn cancel_char_indents(&mut self, left: bool, right: bool, first: bool) { + for (on, chars) in [(left, &mut self.indent_left_chars), (right, &mut self.indent_right_chars), (first, &mut self.indent_first_chars)] { + if on { + *chars = Some(0.0); + } + } + } + /// Just the formatting: without the tracked changes. + pub fn formatting(&self) -> ParaProps { + ParaProps { num_change: None, fmt_change: None, ..self.clone() } + } +} diff --git a/crates/wordmodel/src/props/table_props.rs b/crates/wordmodel/src/props/table_props.rs new file mode 100644 index 000000000..312644d8b --- /dev/null +++ b/crates/wordmodel/src/props/table_props.rs @@ -0,0 +1,346 @@ +use super::char_props::PropChange; +use super::color::Rgb; +use super::para_props::{Align, Borders}; +use super::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum VAlign { + #[default] + Top, + Center, + Bottom, +} + +/// Which way a table cell's text runs (`w:textDirection`, ECMA-376 §17.4.72). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TextDirection { + /// Left to right, lines stacking downwards. + #[default] + Horizontal, + /// Turned 90° clockwise: lines read top to bottom and stack from right to left (`tbRl`). + Down, + /// Turned 90° counter-clockwise: lines read bottom to top and stack from left to right (`btLr`). + Up, +} + +impl TextDirection { + /// The direction an OOXML `ST_TextDirection` value names (transitional and strict names). + /// The East Asian vertical layouts turn like `Down`; unknown values are horizontal. + pub fn from_ooxml(v: &str) -> TextDirection { + match v { + "tbRl" | "tbRlV" | "tbLrV" | "rl" | "rlV" | "lrV" => TextDirection::Down, + "btLr" | "lr" => TextDirection::Up, + _ => TextDirection::Horizontal, + } + } + /// The transitional OOXML name. + pub fn ooxml(self) -> &'static str { + match self { + TextDirection::Horizontal => "lrTb", + TextDirection::Down => "tbRl", + TextDirection::Up => "btLr", + } + } + /// Word's Text Direction button: horizontal → down → up → horizontal. + pub fn next(self) -> TextDirection { + match self { + TextDirection::Horizontal => TextDirection::Down, + TextDirection::Down => TextDirection::Up, + TextDirection::Up => TextDirection::Horizontal, + } + } + pub fn is_turned(self) -> bool { + self != TextDirection::Horizontal + } +} + +/// Which way a text box's text runs (Shape Format › Text › Text Direction; DrawingML +/// `a:bodyPr/@vert`, ECMA-376 §21.1.10.83 `ST_TextVerticalType`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum TextVert { + /// Upright, lines stacking downwards (`horz`). + #[default] + Horz, + /// Turned 90° clockwise (`vert`). + Vert, + /// Turned 270° (`vert270`). + Vert270, + /// Upright letters stacked one above the other (`wordArtVert`). + Stacked, +} + +impl TextVert { + /// The direction an `ST_TextVerticalType` value names; the East Asian and Mongolian vertical + /// layouts turn like `vert`, unknown values are horizontal. + pub fn from_ooxml(v: &str) -> TextVert { + match v { + "vert" | "eaVert" | "mongolianVert" => TextVert::Vert, + "vert270" => TextVert::Vert270, + "wordArtVert" | "wordArtVertRtl" => TextVert::Stacked, + _ => TextVert::Horz, + } + } + pub fn ooxml(self) -> &'static str { + match self { + TextVert::Horz => "horz", + TextVert::Vert => "vert", + TextVert::Vert270 => "vert270", + TextVert::Stacked => "wordArtVert", + } + } + /// How its lines turn on the page (stacked text stays upright). + pub fn turn(self) -> TextDirection { + match self { + TextVert::Vert => TextDirection::Down, + TextVert::Vert270 => TextDirection::Up, + TextVert::Horz | TextVert::Stacked => TextDirection::Horizontal, + } + } +} + +/// Where a text box's text sits between its top and bottom (Align Text; `a:bodyPr/@anchor`, +/// `ST_TextAnchoringType`): reuses [`VAlign`]. Justified and distributed anchoring read as top +/// and middle. +pub fn valign_from_anchor(v: &str) -> VAlign { + match v { + "ctr" | "dist" => VAlign::Center, + "b" => VAlign::Bottom, + _ => VAlign::Top, + } +} + +/// The `ST_TextAnchoringType` value for `a`. +pub fn anchor_ooxml(a: VAlign) -> &'static str { + match a { + VAlign::Top => "t", + VAlign::Center => "ctr", + VAlign::Bottom => "b", + } +} + +/// Table-wide properties. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TableProps { + pub style: Option, + /// Preferred width: points (`Some`) or automatic (`None`). + pub width: Option, + /// Preferred width as a percentage of the text column (overrides `width`). + pub width_pct: Option, + pub align: Option, + /// Indent from the left margin, points. + pub indent: Option, + pub borders: Option, + /// Default cell margins (top, left, bottom, right), points. + pub cell_margins: Option<[f32; 4]>, + /// Fixed column widths (no auto-fit). + pub fixed: bool, + /// Table style options ("look"): header row, total row, banded rows, first column, last + /// column, banded columns. + pub look: TableLook, + pub shading: Option, + /// Alternative text: the title (`w:tblCaption`). + pub caption: Option, + /// Alternative text: the description (`w:tblDescription`). + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + /// Floating placement (`w:tblpPr`); `None` for a table in the text flow. + #[serde(skip_serializing_if = "Option::is_none")] + pub float: Option, + /// Tracked formatting change (`w:tblPrChange`). + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, + /// Right-to-left table (`w:bidiVisual`): the first logical column stands at the right edge, + /// cell margins and left/right borders mirror, and the alignment and indent are measured from + /// the right margin. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub bidi_visual: bool, + /// Space above and below a table in the text flow, points (ODF's table `fo:margin-top` and + /// `fo:margin-bottom`; Word has none). Added to the spacing of the paragraphs around it. + #[serde(skip_serializing_if = "Option::is_none")] + pub space_before: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub space_after: Option, +} + +/// Where a floating table sits; the text after it wraps around it. +#[derive(Clone, Copy, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TableFloat { + /// What `x` is measured from: the column (`Column`), the margin or the page. + pub h_rel: crate::para::Anchor, + /// What `y` is measured from: the text where the table stands (`Paragraph`), the margin or + /// the page. + pub v_rel: crate::para::Anchor, + /// Offsets, points (used when the matching alignment is `None`). + pub x: f32, + pub y: f32, + pub h_align: Option, + pub v_align: Option, + /// Distance from surrounding text: left, top, right, bottom (points). + pub dist: [f32; 4], + /// Whether it may overlap other floating tables (`w:tblOverlap`); Word's default is yes. + pub overlap: bool, +} + +impl TableFloat { + /// The distances from text, finite and clamped (left, top, right, bottom). + pub fn dist_from_text(&self) -> [f32; 4] { + self.dist.map(|v| if v.is_finite() { v.clamp(0.0, 1584.0) } else { 0.0 }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TableLook { + pub header_row: bool, + pub total_row: bool, + pub banded_rows: bool, + pub first_column: bool, + pub last_column: bool, + pub banded_columns: bool, +} + +impl Default for TableLook { + fn default() -> Self { + TableLook { header_row: true, total_row: false, banded_rows: true, first_column: true, last_column: false, banded_columns: false } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum HeightRule { + #[default] + Auto, + AtLeast, + Exact, +} + +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct RowProps { + /// Points (meaning depends on `height_rule`). + pub height: Option, + pub height_rule: HeightRule, + /// Repeat as a header row at the top of each page. + pub header: bool, + /// Don't let the row break across pages. + pub cant_split: bool, + /// Table properties this row overrides (`w:tblPrEx`, ECMA-376 §17.4.61): width, alignment, + /// indent, borders, shading, layout and cell margins. `Some` when the file has the element. + #[serde(skip_serializing_if = "Option::is_none")] + pub exceptions: Option>, + /// Grid columns skipped before the row's first cell (`w:gridBefore`), and the preferred + /// width of that gap in points (`w:wBefore`). + #[serde(skip_serializing_if = "is_zero")] + pub grid_before: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub w_before: Option, + /// Grid columns left after the row's last cell (`w:gridAfter`), and their width in points + /// (`w:wAfter`). + #[serde(skip_serializing_if = "is_zero")] + pub grid_after: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub w_after: Option, + /// Tracked formatting change (`w:trPrChange`). + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, +} + +fn is_zero(v: &u32) -> bool { + *v == 0 +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum VMerge { + #[default] + None, + /// First cell of a vertically merged range. + Restart, + /// Continues the merge from the cell above. + Continue, +} + +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CellProps { + /// Preferred width, points. + pub width: Option, + /// Preferred width as a percentage of the table's (`w:tcW w:type="pct"`). + #[serde(skip_serializing_if = "Option::is_none")] + pub width_pct: Option, + /// Number of grid columns spanned (1 = no horizontal merge). + pub span: u32, + pub vmerge: VMerge, + pub shading: Option, + pub borders: Option, + /// Vertical alignment (`w:vAlign`); `None` follows the table style, else top. An explicit + /// `Top` beats the style. + #[serde(skip_serializing_if = "Option::is_none")] + pub valign: Option, + /// Cell margins override (top, left, bottom, right), points. + pub margins: Option<[f32; 4]>, + /// Per-side overrides when `margins` is absent; omitted sides inherit table defaults. + #[serde(skip_serializing_if = "Option::is_none")] + pub partial_margins: Option<[Option; 4]>, + /// Which way the cell's text runs. + pub text_direction: TextDirection, + /// The file states the horizontal direction (`lrTb`) outright; saving writes it back. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub text_direction_set: bool, + pub no_wrap: bool, + /// The cell's last paragraph takes no room when it is empty (`w:hideMark`): a row of such + /// empty cells shrinks to its borders. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub hide_mark: bool, + /// Tracked formatting change (`w:tcPrChange`). + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, + /// A picture behind the cell's content (an ODF cell's `style:background-image`; Word has + /// none). + #[serde(skip_serializing_if = "Option::is_none")] + pub picture: Option>, +} + +/// A cell's background picture: drawn at its own size where `align` puts it, tiled, or stretched +/// over the cell, and cut off at the cell's edges. +#[derive(Clone, Debug, PartialEq, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct CellPicture { + /// Key into [`crate::Document::media`]. + pub media: String, + /// Its own size, points. + pub size: [f32; 2], + pub repeat: PictureRepeat, + /// Where an unrepeated picture sits: 0 the left (top) edge, 0.5 the middle, 1 the right + /// (bottom) edge. + pub align: [f32; 2], +} + +/// How a background picture fills its area (ODF §20.348 `style:repeat`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum PictureRepeat { + /// Once, at its own size. + #[default] + None, + /// Repeated across the area. + Tile, + /// Stretched over the area. + Stretch, +} + +impl CellProps { + /// Resolve top, left, bottom and right margins against the current table defaults. + /// A complete override takes precedence over per-side overrides. + pub fn resolved_margins(&self, defaults: [f32; 4]) -> [f32; 4] { + if let Some(margins) = self.margins { + return margins; + } + self.partial_margins.map_or(defaults, |sides| std::array::from_fn(|i| sides[i].unwrap_or(defaults[i]))) + } +} diff --git a/crates/wordmodel/src/resolve.rs b/crates/wordmodel/src/resolve.rs new file mode 100644 index 000000000..40449e82c --- /dev/null +++ b/crates/wordmodel/src/resolve.rs @@ -0,0 +1,629 @@ +//! Style resolution: document defaults → paragraph style chain → character style chain → direct +//! formatting, into concrete values for layout. + +use serde::{Deserialize, Serialize}; + +use crate::props::{ + Align, Border, Borders, CharProps, Highlight, LineSpacing, NumRef, ParaProps, Rgb, TabAlign, TabStop, TextColor, Underline, VertAlign, +}; +use crate::styles::{StyleKind, StyleSheet}; + +/// Automatic (HTML-style) paragraph spacing, points: what Word for Windows gives `w:beforeAutospacing` +/// and `w:afterAutospacing` at any font size (measured). +pub const AUTO_SPACING: f32 = 14.0; +/// The largest drop cap distance from the text, in points (22", as Word's dialog allows). +pub const MAX_DROP_CAP_DISTANCE: f32 = 1584.0; + +/// Concrete character formatting. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResolvedChar { + pub font: String, + pub size: f32, + pub bold: bool, + pub italic: bool, + pub underline: Underline, + pub underline_color: Option, + pub strike: bool, + pub double_strike: bool, + pub color: TextColor, + pub highlight: Option, + pub shading: Option, + /// Visible character border only (`nil`/`none` resolve to `None`). + pub border: Option, + pub vert_align: VertAlign, + pub caps: bool, + pub small_caps: bool, + pub hidden: bool, + pub spacing: f32, + pub scale: f32, + pub position: f32, + /// Kerning: the smallest font size, in points, that is kerned (`w:kern`); 0 = never kerned. + #[serde(default)] + pub kern: f32, + /// A mark over or under each character (Font › Emphasis mark). + #[serde(default)] + pub emphasis: crate::props::EmphasisMark, + /// OpenType ligatures (`w14:ligatures`); `None` = the shaper's default (standard and + /// contextual). + #[serde(default)] + pub ligatures: Option, + pub outline: bool, + pub shadow: bool, + pub emboss: bool, + pub engrave: bool, + /// WordArt text effects (sanitized; `None` when there are none). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text_effects: Option>, + pub link: Option, + pub ins: Option, + pub del: Option, + /// The revision of a tracked formatting change (`w:rPrChange`) of the run. + pub fmt: Option, + pub no_proof: bool, + pub lang: Option, + /// Right-to-left run (`w:rtl`). + pub rtl: bool, + /// The whole run uses the complex-script properties (`w:cs`). + pub cs: bool, + /// Complex-script font, size, bold and italic (Arabic, Persian, Hebrew… text). + pub font_cs: String, + /// East Asian font (Chinese, Japanese and Korean text), when one is set. + #[serde(skip_serializing_if = "Option::is_none")] + pub font_ea: Option, + pub size_cs: f32, + pub bold_cs: bool, + pub italic_cs: bool, +} + +impl ResolvedChar { + fn from(c: &CharProps) -> ResolvedChar { + ResolvedChar { + // A symbol character is shown in its own font (`w:sym`), whatever the run's font. + font: c.shown_font().map(str::to_string).unwrap_or_else(|| crate::styles::BODY_FONT.to_string()), + size: c.size.unwrap_or(crate::props::DEFAULT_FONT_SIZE).clamp(1.0, 1638.0), + bold: c.bold.unwrap_or(false), + italic: c.italic.unwrap_or(false), + underline: c.underline.unwrap_or_default(), + underline_color: c.underline_color, + strike: c.strike.unwrap_or(false), + double_strike: c.double_strike.unwrap_or(false), + color: c.color.unwrap_or_default(), + highlight: c.highlight.and_then(Highlight::rgb), + shading: c.shading, + border: c.border.filter(Border::is_visible), + vert_align: c.vert_align.unwrap_or_default(), + caps: c.caps.unwrap_or(false), + small_caps: c.small_caps.unwrap_or(false), + hidden: c.hidden.unwrap_or(false), + spacing: c.spacing.unwrap_or(0.0).clamp(-100.0, 100.0), + scale: c.scale.unwrap_or(100.0).clamp(1.0, 600.0), + position: c.position.unwrap_or(0.0).clamp(-800.0, 800.0), + kern: c.kern.filter(|k| k.is_finite()).unwrap_or(0.0).clamp(0.0, 1638.0), + emphasis: c.emphasis.unwrap_or_default(), + ligatures: c.ligatures, + outline: c.outline.unwrap_or(false), + shadow: c.shadow.unwrap_or(false), + emboss: c.emboss.unwrap_or(false), + engrave: c.engrave.unwrap_or(false), + text_effects: c.text_effects.as_deref().map(crate::wordart::TextEffects::sanitized).filter(|t| !t.is_empty()).map(Box::new), + link: c.link.clone(), + ins: c.ins, + del: c.del, + fmt: c.fmt_change.as_ref().map(|f| f.rev), + no_proof: c.no_proof.unwrap_or(false), + lang: c.lang.clone(), + rtl: c.rtl.unwrap_or(false), + cs: c.cs.unwrap_or(false), + font_cs: c.font_cs.clone().or_else(|| c.font.clone()).unwrap_or_else(|| crate::styles::BODY_FONT.to_string()), + font_ea: c.font_ea.clone(), + size_cs: c.size_cs.or(c.size).unwrap_or(crate::props::DEFAULT_FONT_SIZE).clamp(1.0, 1638.0), + bold_cs: c.bold_cs.or(c.bold).unwrap_or(false), + italic_cs: c.italic_cs.or(c.italic).unwrap_or(false), + } + } + /// The formatting complex-script text in this run is drawn with: the complex-script font, + /// size, bold and italic in place of the others (Word formats Arabic, Persian and Hebrew + /// characters, and every character of an `rtl`/`cs` run, this way). + pub fn complex(&self) -> ResolvedChar { + ResolvedChar { font: self.font_cs.clone(), size: self.size_cs, bold: self.bold_cs, italic: self.italic_cs, ..self.clone() } + } + /// Whether the text is kerned: Word kerns only a run with Kerning for fonts set (`w:kern`) at + /// a size of at least that many points, never by default (measured in Word for Windows). + pub fn kerned(&self) -> bool { + self.kern > 0.0 && self.size >= self.kern + } + /// Does a character of this run use the complex-script properties? + pub fn uses_complex(&self, c: char) -> bool { + self.rtl || self.cs || crate::bidi::is_complex_script(c) + } + /// The size glyphs are drawn and measured at. Word sets superscript and subscript text at + /// 0.65 of the run's size rounded to the half point, a font size the format can store + /// (measured in Word for Mac from digit advances: 8 pt → 5, 9 → 6, 11 → 7, 12 → 8, + /// 16 → 10.5, 24 → 15.5, 36 → 23.5). + pub fn draw_size(&self) -> f32 { + if self.vert_align == VertAlign::Baseline { self.size } else { (self.size * 0.65 * 2.0).round() / 2.0 } + } + /// Baseline shift, points (positive = up). + pub fn baseline_shift(&self) -> f32 { + self.position + + match self.vert_align { + VertAlign::Baseline => 0.0, + VertAlign::Superscript => self.size * 0.33, + VertAlign::Subscript => -self.size * 0.14, + } + } +} + +/// Concrete paragraph formatting. +fn default_adjust_right_indent() -> bool { + true +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ResolvedPara { + pub style: String, + pub align: Align, + pub indent_left: f32, + pub indent_right: f32, + pub indent_first: f32, + pub space_before: f32, + pub space_after: f32, + /// The space before or after is automatic (HTML-style, [`AUTO_SPACING`]): see + /// `ParaProps::space_before_auto`. + #[serde(default)] + pub before_auto: bool, + #[serde(default)] + pub after_auto: bool, + pub line_spacing: LineSpacing, + pub contextual_spacing: bool, + /// Left/right indents are inside/outside: they swap on even pages. + pub mirror_indents: bool, + /// Automatically adjust the right indent for the document grid when enabled; keep the specified right indent when disabled. + #[serde(default = "default_adjust_right_indent")] + pub adjust_right_indent: bool, + pub keep_next: bool, + pub keep_lines: bool, + pub page_break_before: bool, + pub widow_control: bool, + pub outline_level: Option, + pub numbering: Option, + pub tabs: Vec, + pub shading: Option, + pub borders: Option, + /// Padding on the sides without a line (top, left, bottom, right), points. ODF's box model + /// (XSL's) keeps it for documents opened from ODF (`Settings::odf`); Word drops it. + #[serde(default)] + pub padding: [f32; 4], + pub bidi: bool, + pub drop_cap: u8, + /// The drop cap hangs in the margin (see `ParaProps::drop_cap_margin`). + #[serde(default)] + pub drop_cap_margin: bool, + /// The drop cap's distance from the text, in points. + #[serde(default)] + pub drop_cap_distance: f32, + /// The drop cap letter's font family; `None` = the text's. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub drop_cap_font: Option, + /// A text frame the paragraph sits in (see `ParaProps::frame`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub frame: Option, + pub suppress_hyphens: bool, + pub suppress_line_numbers: bool, + /// Asian typography (see `ParaProps`): East Asian line-breaking rules. + pub kinsoku: bool, + /// Latin words wrap whole (`false`: they may break at any character). + pub word_wrap: bool, + pub overflow_punct: bool, + pub top_line_punct: bool, + pub auto_space_de: bool, + pub auto_space_dn: bool, + /// Lines snap to the document grid, when the section has one. + pub snap_to_grid: bool, + /// Indents in characters (left, right, first line; 0 = in points), which win over the + /// point values; see [`ResolvedPara::set_char_unit`]. + #[serde(default)] + pub indent_chars: [f32; 3], +} + +const MAX_INDENT: f32 = 1584.0; // 22" + +impl ResolvedPara { + /// Visual (left, right) indents. Indents, like alignment, are logical: `indent_left` is the + /// start (leading) edge, which is the right edge of a right-to-left paragraph (ISO 29500 + /// `w:ind/@w:start`, read from `w:left` too). + pub fn visual_indents(&self) -> (f32, f32) { + if self.bidi { (self.indent_right, self.indent_left) } else { (self.indent_left, self.indent_right) } + } + + /// Turn the indents given in characters into points, one character being `size` points: the + /// size of the paragraph mark's font (as Word measures `w:firstLineChars` and friends). + pub fn set_char_unit(&mut self, size: f32) { + let size = if size.is_finite() { size.clamp(1.0, 1638.0) } else { 12.0 }; + let [l, r, f] = self.indent_chars; + for (chars, pts) in [(l, &mut self.indent_left), (r, &mut self.indent_right), (f, &mut self.indent_first)] { + if chars != 0.0 { + *pts = (chars * size).clamp(-MAX_INDENT, MAX_INDENT); + } + } + } +} + +impl ResolvedPara { + fn from(p: &ParaProps, style: &str) -> ResolvedPara { + let clampi = |v: Option| v.unwrap_or(0.0).clamp(-MAX_INDENT, MAX_INDENT); + let mut tabs: Vec = + p.tabs.clone().unwrap_or_default().into_iter().filter(|t| t.align != TabAlign::Clear && t.pos.is_finite()).collect(); + tabs.sort_by(|a, b| a.pos.total_cmp(&b.pos)); + ResolvedPara { + style: style.to_string(), + align: p.align.unwrap_or_default(), + indent_left: clampi(p.indent_left), + indent_right: clampi(p.indent_right), + indent_first: clampi(p.indent_first), + space_before: if p.space_before_auto == Some(true) { AUTO_SPACING } else { p.space_before.unwrap_or(0.0).clamp(0.0, 1584.0) }, + space_after: if p.space_after_auto == Some(true) { AUTO_SPACING } else { p.space_after.unwrap_or(0.0).clamp(0.0, 1584.0) }, + before_auto: p.space_before_auto == Some(true), + after_auto: p.space_after_auto == Some(true), + line_spacing: match p.line_spacing.unwrap_or_default() { + LineSpacing::Multiple(m) => LineSpacing::Multiple(if m.is_finite() { m.clamp(0.06, 132.0) } else { 1.0 }), + LineSpacing::AtLeast(v) => LineSpacing::AtLeast(if v.is_finite() { v.clamp(0.0, 1584.0) } else { 0.0 }), + LineSpacing::Exactly(v) => LineSpacing::Exactly(if v.is_finite() { v.clamp(0.7, 1584.0) } else { 12.0 }), + }, + contextual_spacing: p.contextual_spacing.unwrap_or(false), + mirror_indents: p.mirror_indents.unwrap_or(false), + adjust_right_indent: p.adjust_right_indent.unwrap_or(true), + keep_next: p.keep_next.unwrap_or(false), + keep_lines: p.keep_lines.unwrap_or(false), + page_break_before: p.page_break_before.unwrap_or(false), + widow_control: p.widow_control.unwrap_or(true), + outline_level: p.outline_level.filter(|l| *l < 9), + numbering: p.numbering.filter(|n| n.num != 0), + tabs, + shading: p.shading, + borders: p.borders.filter(Borders::any_visible), + padding: p.borders.map_or([0.0; 4], |b| { + [b.top, b.left, b.bottom, b.right] + .map(|s| s.filter(|l| !l.is_visible() && l.space.is_finite() && l.space > 0.0).map_or(0.0, |l| l.space.min(1584.0))) + }), + bidi: p.bidi.unwrap_or(false), + drop_cap: p.drop_cap.unwrap_or(0).min(10), + drop_cap_margin: p.drop_cap_margin.unwrap_or(false), + drop_cap_distance: p.drop_cap_distance.filter(|d| d.is_finite()).unwrap_or(0.0).clamp(0.0, MAX_DROP_CAP_DISTANCE), + drop_cap_font: p.drop_cap_font.clone().filter(|f| !f.trim().is_empty()), + // Explicit zero dimensions cancel a frame; omitted dimensions still fit its text. + frame: p.frame.clone().filter(|f| p.drop_cap.unwrap_or(0) == 0 && (f.w != Some(0.0) || f.h != Some(0.0))).map(|mut f| { + f.w = f.w.filter(|v| *v != 0.0); + f.h = f.h.filter(|v| *v != 0.0); + f + }), + suppress_hyphens: p.suppress_hyphens.unwrap_or(false), + suppress_line_numbers: p.suppress_line_numbers.unwrap_or(false), + kinsoku: p.kinsoku.unwrap_or(true), + word_wrap: p.word_wrap.unwrap_or(true), + overflow_punct: p.overflow_punct.unwrap_or(true), + top_line_punct: p.top_line_punct.unwrap_or(false), + auto_space_de: p.auto_space_de.unwrap_or(true), + auto_space_dn: p.auto_space_dn.unwrap_or(true), + snap_to_grid: p.snap_to_grid.unwrap_or(true), + indent_chars: [p.indent_left_chars, p.indent_right_chars, p.indent_first_chars] + .map(|c| c.filter(|c| c.is_finite()).unwrap_or(0.0).clamp(-MAX_CHARS, MAX_CHARS)), + } + } +} + +/// The largest indent in characters (Word allows indents up to 22"). +const MAX_CHARS: f32 = 1584.0; + +impl StyleSheet { + /// Paragraph properties after defaults and the style chain (without direct formatting). + pub fn para_style_props(&self, style: &str) -> (ParaProps, CharProps) { + self.para_style_props_over(style, self.default_para.clone(), self.default_chr.clone()) + } + + /// The paragraph style chain of `style` applied over `pp` / `cp`. A style the document doesn't + /// define is the default paragraph style, Normal, as in Word: a footer naming a `Footer` style + /// that `styles.xml` leaves out is formatted as Normal, not with the document defaults alone + /// (#5077). + fn para_style_props_over(&self, style: &str, mut pp: ParaProps, mut cp: CharProps) -> (ParaProps, CharProps) { + let style = if self.get(style).is_some() { style } else { "Normal" }; + for s in self.chain(style) { + if s.kind == StyleKind::Paragraph { + let mut sp = s.para.clone(); + // Tabs accumulate along the chain (with Clear removing inherited stops). + if let (Some(base), Some(add)) = (pp.tabs.clone(), sp.tabs.clone()) { + let mut merged: Vec = base.into_iter().filter(|t| !add.iter().any(|a| (a.pos - t.pos).abs() < 0.5)).collect(); + merged.extend(add); + sp.tabs = Some(merged); + } + pp.overlay(&sp); + cp.overlay(&s.chr); + } + } + pp.style = None; + (pp, cp) + } + + /// What a table style's paragraph formatting `table` changes for a paragraph with `p`, once + /// the paragraph style chain is applied over it (ECMA-376 §17.7.2): each property it gives + /// that the chain doesn't set. Formats without table styles state these directly. + pub fn table_para_patch(&self, p: &ParaProps, table: &ParaProps) -> ParaProps { + let style = p.style.clone().unwrap_or_else(|| "Normal".into()); + let mut with = self.default_para.clone(); + with.overlay(&ParaProps { style: None, ..table.clone() }); + let (with, _) = self.para_style_props_over(&style, with, CharProps::default()); + let (without, _) = self.para_style_props_over(&style, self.default_para.clone(), CharProps::default()); + diff_patch(&with, &without) + } + + /// What a table style's run formatting `table` changes for a run of character style + /// `run_style` in a paragraph of style `para_style`, once the paragraph and character style + /// chains are applied over it (ECMA-376 §17.7.2). + pub fn table_char_patch(&self, para_style: Option<&str>, run_style: Option<&str>, table: &CharProps) -> CharProps { + let chain = |defaults: CharProps| { + let (_, mut cp) = self.para_style_props_over(para_style.unwrap_or("Normal"), ParaProps::default(), defaults); + if let Some(cs) = run_style { + for s in self.chain(cs) { + if s.kind == StyleKind::Character || s.kind == StyleKind::Paragraph { + cp.overlay(&s.chr); + } + } + } + cp + }; + let mut with = self.default_chr.clone(); + with.overlay(&CharProps { style: None, ..table.clone() }); + diff_patch(&chain(with), &chain(self.default_chr.clone())) + } + + /// Resolve a paragraph's properties. + pub fn resolve_para(&self, p: &ParaProps) -> ResolvedPara { + self.resolve_para_in(p, None) + } + + /// Resolve a paragraph's properties; `table` is the paragraph formatting of the table style of + /// the table the paragraph is in. The order is that of ECMA-376 §17.7.2: document defaults, the + /// table style, the paragraph style chain, direct formatting. So the table style replaces + /// document defaults (Table Grid's single spacing beats the defaults' 8 pt after), but any + /// property the paragraph style chain sets, Normal included, beats the table style. + pub fn resolve_para_in(&self, p: &ParaProps, table: Option<&ParaProps>) -> ResolvedPara { + let style = p.style.clone().unwrap_or_else(|| "Normal".into()); + let mut defaults = self.default_para.clone(); + if let Some(t) = table { + defaults.overlay(&ParaProps { style: None, ..t.clone() }); + } + let (mut base, _) = self.para_style_props_over(&style, defaults, CharProps::default()); + let mut direct = p.clone(); + if let (Some(b), Some(d)) = (base.tabs.clone(), direct.tabs.clone()) { + let mut merged: Vec = b.into_iter().filter(|t| !d.iter().any(|a| (a.pos - t.pos).abs() < 0.5)).collect(); + merged.extend(d); + direct.tabs = Some(merged); + } + base.overlay(&direct); + let mut rp = ResolvedPara::from(&base, &style); + // Indents in characters: measured in the style's font size here; layout measures them in + // the paragraph mark's (`set_char_unit`), which direct formatting may change. + if rp.indent_chars.iter().any(|c| *c != 0.0) { + rp.set_char_unit(self.resolve_char(Some(&style), &CharProps::default()).size); + } + rp + } + + /// Resolve character formatting for a run in a paragraph of style `para_style`. + pub fn resolve_char(&self, para_style: Option<&str>, c: &CharProps) -> ResolvedChar { + self.resolve_char_in(para_style, None, c) + } + + /// Resolve character formatting for a run in a paragraph of style `para_style`; `table` is the + /// run formatting of the table style (with the cell's conditional formatting) of the table the + /// paragraph is in. Order: document defaults, the table style, the paragraph style chain, the + /// character style chain, direct formatting (ECMA-376 §17.7.2). + pub fn resolve_char_in(&self, para_style: Option<&str>, table: Option<&CharProps>, c: &CharProps) -> ResolvedChar { + let mut defaults = self.default_chr.clone(); + if let Some(t) = table { + defaults.overlay(&CharProps { style: None, ..t.clone() }); + } + let (_, mut cp) = self.para_style_props_over(para_style.unwrap_or("Normal"), ParaProps::default(), defaults); + if let Some(cs) = c.style.as_deref() { + for s in self.chain(cs) { + if s.kind == StyleKind::Character || s.kind == StyleKind::Paragraph { + cp.overlay(&s.chr); + } + } + } + // Only a character itself has a symbol font of its own (`w:sym`); a style never gives one. + cp.sym_font = None; + let mut direct = c.clone(); + direct.style = None; + cp.overlay(&direct); + ResolvedChar::from(&cp) + } +} + +/// The properties of `with` that differ from `without` (both fully resolved property sets of one +/// kind), as a set of only those; empty when serialisation fails. +fn diff_patch(with: &T, without: &T) -> T { + let (Ok(serde_json::Value::Object(w)), Ok(serde_json::Value::Object(o))) = (serde_json::to_value(with), serde_json::to_value(without)) else { + return T::default(); + }; + let patch: serde_json::Map = w.into_iter().filter(|(k, v)| o.get(k) != Some(v)).collect(); + serde_json::from_value(serde_json::Value::Object(patch)).unwrap_or_default() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Superscript and subscript sizes as Word for Mac sets them (measured from digit advances in + /// its PDF export): 0.65 of the size, rounded to the half point. + #[test] + fn raised_and_lowered_text_is_sized_to_the_half_point() { + for (size, want) in [ + (8.0, 5.0), + (9.0, 6.0), + (10.0, 6.5), + (11.0, 7.0), + (12.0, 8.0), + (13.0, 8.5), + (14.0, 9.0), + (16.0, 10.5), + (20.0, 13.0), + (24.0, 15.5), + (36.0, 23.5), + ] { + for vert_align in [VertAlign::Superscript, VertAlign::Subscript] { + let rc = ResolvedChar { size, vert_align, ..StyleSheet::builtin().resolve_char(None, &CharProps::default()) }; + assert_eq!(rc.draw_size(), want, "{size} pt {vert_align:?}"); + } + } + let rc = StyleSheet::builtin().resolve_char(None, &CharProps { size: Some(11.0), ..Default::default() }); + assert_eq!(rc.draw_size(), 11.0, "baseline text keeps its size"); + } + + /// #2036: only a character itself (direct formatting) has a symbol font of its own; a + /// style never gives one to its text. + #[test] + fn a_style_never_gives_text_a_symbol_font() { + let mut s = StyleSheet::builtin(); + if let Some(n) = s.get_mut("Normal") { + n.chr.sym_font = Some("Wingdings".into()); + } + assert_ne!(s.resolve_char(None, &CharProps::default()).font, "Wingdings"); + let direct = CharProps { sym_font: Some("Wingdings".into()), ..Default::default() }; + assert_eq!(s.resolve_char(None, &direct).font, "Wingdings"); + } + + /// #5077: a paragraph naming a style the document doesn't define is formatted as Normal, + /// paragraph and run alike, not with the document defaults alone. + #[test] + fn an_undefined_paragraph_style_is_normal() { + let mut s = StyleSheet::builtin(); + s.default_para.space_after = Some(10.0); + if let Some(n) = s.get_mut("Normal") { + n.para.space_after = Some(2.0); + n.chr.size = Some(13.0); + } + let undefined = ParaProps { style: Some("NoSuchStyle".into()), ..Default::default() }; + let normal = s.resolve_para(&ParaProps::default()); + let p = s.resolve_para(&undefined); + assert_eq!(p.space_after, 2.0); + assert_eq!(p.space_after, normal.space_after); + assert_eq!(p.line_spacing, normal.line_spacing); + assert_eq!(s.resolve_char(Some("NoSuchStyle"), &CharProps::default()).size, 13.0); + // The paragraph keeps the style it names; direct formatting still wins. + assert_eq!(p.style, "NoSuchStyle"); + let direct = ParaProps { space_after: Some(7.0), ..undefined }; + assert_eq!(s.resolve_para(&direct).space_after, 7.0); + // A defined style is unchanged. + assert_eq!(s.resolve_para(&ParaProps { style: Some("Heading1".into()), ..Default::default() }).space_after, 4.0); + } + + #[test] + fn heading_resolves() { + let s = StyleSheet::builtin(); + let p = s.resolve_para(&ParaProps { style: Some("Heading1".into()), ..Default::default() }); + assert!(p.keep_next); + assert_eq!(p.outline_level, Some(0)); + assert_eq!(p.space_after, 4.0); + let c = s.resolve_char(Some("Heading1"), &CharProps::default()); + assert_eq!(c.size, 20.0); + assert_eq!(c.font, crate::styles::HEADING_FONT); + let c2 = s.resolve_char(Some("Heading1"), &CharProps { size: Some(30.0), ..Default::default() }); + assert_eq!(c2.size, 30.0); + } + + #[test] + fn char_style_then_direct() { + let s = StyleSheet::builtin(); + let c = s.resolve_char(None, &CharProps { style: Some("Strong".into()), italic: Some(true), ..Default::default() }); + assert!(c.bold && c.italic); + assert_eq!(c.size, 12.0); + let unknown = s.resolve_char(Some("NoSuchStyle"), &CharProps::default()); + assert_eq!(unknown.size, 12.0); + } + + #[test] + fn hostile_values_clamped() { + let s = StyleSheet::builtin(); + let p = s.resolve_para(&ParaProps { + indent_left: Some(f32::INFINITY), + line_spacing: Some(LineSpacing::Multiple(f32::NAN)), + ..Default::default() + }); + assert_eq!(p.indent_left, MAX_INDENT); + assert_eq!(p.line_spacing, LineSpacing::Multiple(1.0)); + let c = s.resolve_char(None, &CharProps { size: Some(-4.0), ..Default::default() }); + assert_eq!(c.size, 1.0); + } + + /// ECMA-376 §17.7.2 for text in a table: document defaults, then the table style, then the + /// paragraph style chain (Normal included), then direct formatting. + #[test] + fn table_style_sits_between_defaults_and_paragraph_style() { + let mut s = StyleSheet::builtin(); + s.styles.push(crate::styles::Style { + id: "Spaced".into(), + based_on: Some("Normal".into()), + para: ParaProps { space_after: Some(12.0), ..Default::default() }, + chr: CharProps { color: Some(TextColor::Rgb(Rgb(0, 0, 0xFF))), ..Default::default() }, + ..Default::default() + }); + let table_p = ParaProps { space_after: Some(0.0), line_spacing: Some(LineSpacing::Multiple(1.0)), ..Default::default() }; + let red = TextColor::Rgb(Rgb(0xC0, 0, 0)); + let table_c = CharProps { bold: Some(true), color: Some(red), size: Some(9.0), ..Default::default() }; + // Document defaults alone (outside a table). + let out = s.resolve_para(&ParaProps::default()); + assert_eq!((out.space_after, out.line_spacing), (8.0, LineSpacing::Multiple(1.15))); + // The table style replaces the defaults. + let p = s.resolve_para_in(&ParaProps::default(), Some(&table_p)); + assert_eq!((p.space_after, p.line_spacing), (0.0, LineSpacing::Multiple(1.0))); + let c = s.resolve_char_in(None, Some(&table_c), &CharProps::default()); + assert!(c.bold); + assert_eq!((c.color, c.size), (red, 9.0)); + // A paragraph style beats it, for what the style sets only. + let spaced = ParaProps { style: Some("Spaced".into()), ..Default::default() }; + let p = s.resolve_para_in(&spaced, Some(&table_p)); + assert_eq!((p.space_after, p.line_spacing), (12.0, LineSpacing::Multiple(1.0))); + let c = s.resolve_char_in(Some("Spaced"), Some(&table_c), &CharProps::default()); + assert!(c.bold); + assert_eq!((c.color, c.size), (TextColor::Rgb(Rgb(0, 0, 0xFF)), 9.0)); + // A character style and direct formatting beat both. + let c = + s.resolve_char_in(Some("Spaced"), Some(&table_c), &CharProps { style: Some("Emphasis".into()), bold: Some(false), ..Default::default() }); + assert!(!c.bold && c.italic); + let p = s.resolve_para_in(&ParaProps { space_after: Some(3.0), ..spaced }, Some(&table_p)); + assert_eq!(p.space_after, 3.0); + // Normal setting a property itself beats the table style too (Word draws such rows tall). + if let Some(n) = s.get_mut("Normal") { + n.para.space_after = Some(10.0); + n.chr.size = Some(11.0); + } + assert_eq!(s.resolve_para_in(&ParaProps::default(), Some(&table_p)).space_after, 10.0); + assert_eq!(s.resolve_char_in(None, Some(&table_c), &CharProps::default()).size, 11.0); + } + + #[test] + fn header_tabs_inherit() { + let s = StyleSheet::builtin(); + let p = s.resolve_para(&ParaProps { + style: Some("Header".into()), + tabs: Some(vec![TabStop { pos: 100.0, align: TabAlign::Left, leader: Default::default() }]), + ..Default::default() + }); + assert_eq!(p.tabs.len(), 3); + assert_eq!(p.tabs[0].pos, 100.0); + } + + /// Resolved formatting saved before paragraph padding existed still reads: no padding. + #[test] + fn resolved_para_without_padding_reads() { + let p = StyleSheet::builtin().resolve_para(&ParaProps::default()); + let mut v = serde_json::to_value(&p).unwrap(); + v.as_object_mut().unwrap().remove("padding"); + let back: ResolvedPara = serde_json::from_value(v).unwrap(); + assert_eq!(back.padding, [0.0; 4]); + assert_eq!(back, ResolvedPara { padding: [0.0; 4], ..p }); + } +} diff --git a/crates/wordmodel/src/review.rs b/crates/wordmodel/src/review.rs new file mode 100644 index 000000000..de960b117 --- /dev/null +++ b/crates/wordmodel/src/review.rs @@ -0,0 +1,93 @@ +use super::{Blocks, Deserialize, Document, Serialize}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Comment { + pub author: String, + pub initials: String, + /// ISO 8601. + pub date: String, + /// Reply to this comment id. + pub parent: Option, + pub resolved: bool, + /// `Document::parts` id of the comment text. + pub part: u32, + /// Word's durable id for the comment (`w16cid:durableId`, hex), kept across saves so Word + /// keeps tracking the same comment. `None` for comments made here: the writer assigns one. + #[serde(skip_serializing_if = "Option::is_none")] + pub durable_id: Option, + /// The comment's UTC timestamp: from the file (`w16cex:dateUtc`, where `date` is local time), + /// or the time the app stamped the comment with. + #[serde(skip_serializing_if = "Option::is_none")] + pub date_utc: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum RevisionKind { + #[default] + Insert, + Delete, + Format, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default)] +pub struct Revision { + pub kind: RevisionKind, + pub author: String, + pub date: String, + /// A move (ECMA-376 §17.13.5.21–28): the name linking the source (an `Delete` revision, + /// `w:moveFrom`) with the destination (an `Insert` revision, `w:moveTo`). `None` = a plain + /// insertion or deletion. + #[serde(skip_serializing_if = "Option::is_none")] + pub move_name: Option, +} + +impl Revision { + pub fn new(kind: RevisionKind, author: impl Into, date: impl Into) -> Revision { + Revision { kind, author: author.into(), date: date.into(), move_name: None } + } +} + +impl Document { + /// The move name of revision `idx`, when it is one end of a move. + pub fn move_name(&self, idx: Option) -> Option<&str> { + idx.and_then(|i| self.revisions.get(i as usize)).and_then(|r| r.move_name.as_deref()) + } +} + +/// A building block (Insert › Quick Parts, AutoText…) kept in the document's glossary +/// (ECMA-376 §17.12: `w:docPart`). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct BuildingBlock { + /// The entry's name (`w:docPartPr/w:name`). + pub name: String, + /// Its gallery (`w:category/w:gallery`: `docParts` for Quick Parts, `autoTxt` for AutoText…). + pub gallery: String, + /// Its category within the gallery (`w:category/w:name`, e.g. "General"). + pub category: String, + #[serde(skip_serializing_if = "String::is_empty")] + pub description: String, + /// The entry's identity (`w:guid`), kept as read. + #[serde(skip_serializing_if = "String::is_empty")] + pub guid: String, + /// The paragraph style it's associated with (`w:style`), kept as read. + #[serde(skip_serializing_if = "String::is_empty")] + pub style: String, + /// Its types (`w:types/w:type`: `normal`, `bbPlcHdr` for a content control's placeholder…). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub types: Vec, + /// How it's inserted (`w:behaviors/w:behavior`: `content`, `p`, `pg`). + #[serde(skip_serializing_if = "Vec::is_empty")] + pub behaviors: Vec, + /// The content (`w:docPartBody`). + pub blocks: Blocks, +} + +/// The gallery Quick Parts saves to (`w:gallery w:val="docParts"`). +pub const QUICK_PARTS_GALLERY: &str = "docParts"; + +/// The AutoText gallery (`w:gallery w:val="autoTxt"`). +pub const AUTO_TEXT_GALLERY: &str = "autoTxt"; diff --git a/crates/wordmodel/src/section.rs b/crates/wordmodel/src/section.rs new file mode 100644 index 000000000..538dd27a8 --- /dev/null +++ b/crates/wordmodel/src/section.rs @@ -0,0 +1,1091 @@ +//! Section properties: page setup, columns, headers/footers, page numbering. + +use serde::{Deserialize, Serialize}; + +use crate::props::Borders; + +/// How a section's text sits on its pages, top to bottom (`w:vAlign`, ECMA-376 §17.6.23, +/// ST_VerticalJc). Table cells and text boxes use [`crate::props::VAlign`], which has no +/// Justified. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum PageVAlign { + #[default] + Top, + Center, + /// Justified (`both`): the room left on each page is shared out equally between the page's + /// paragraphs and tables, so the text runs from the top margin to the bottom one. + #[serde(alias = "both", alias = "justified")] + Justify, + Bottom, +} + +impl PageVAlign { + /// From a command or file value: `top|center|justify|bottom` (also `both`, `justified`, + /// `middle`); anything else is Top. + pub fn parse(v: &str) -> PageVAlign { + match v { + "center" | "middle" => PageVAlign::Center, + "justify" | "justified" | "both" => PageVAlign::Justify, + "bottom" => PageVAlign::Bottom, + _ => PageVAlign::Top, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum SectionStart { + #[default] + NextPage, + Continuous, + EvenPage, + OddPage, + NextColumn, +} + +/// A number format (ECMA-376 §17.18.59 ST_NumberFormat). Every value the format defines is +/// kept, so a document saves back the format it was opened with even where the app draws it as +/// plain decimal (see [`NumFormat::format`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum NumFormat { + #[default] + Decimal, + UpperRoman, + LowerRoman, + UpperLetter, + LowerLetter, + Ordinal, + CardinalText, + OrdinalText, + DecimalZero, + Bullet, + None, + Hex, + Chicago, + IdeographDigital, + JapaneseCounting, + Aiueo, + Iroha, + DecimalFullWidth, + DecimalHalfWidth, + JapaneseLegal, + JapaneseDigitalTenThousand, + DecimalEnclosedCircle, + DecimalFullWidth2, + AiueoFullWidth, + IrohaFullWidth, + Ganada, + Chosung, + DecimalEnclosedFullstop, + DecimalEnclosedParen, + DecimalEnclosedCircleChinese, + IdeographEnclosedCircle, + IdeographTraditional, + IdeographZodiac, + IdeographZodiacTraditional, + TaiwaneseCounting, + IdeographLegalTraditional, + TaiwaneseCountingThousand, + TaiwaneseDigital, + ChineseCounting, + ChineseLegalSimplified, + ChineseCountingThousand, + KoreanDigital, + KoreanCounting, + KoreanLegal, + KoreanDigital2, + VietnameseCounting, + RussianLower, + RussianUpper, + NumberInDash, + Hebrew1, + Hebrew2, + ArabicAlpha, + ArabicAbjad, + HindiVowels, + HindiConsonants, + HindiNumbers, + HindiCounting, + ThaiLetters, + ThaiNumbers, + ThaiCounting, + BahtText, + DollarText, +} + +/// Chapter separator for page numbers (`w:chapSep`, ECMA-376 §17.18.12). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ChapterSep { + Hyphen, + Period, + Colon, + EmDash, + EnDash, +} + +impl ChapterSep { + pub fn from_ooxml(value: &str) -> Option { + match value { + "hyphen" => Some(Self::Hyphen), + "period" => Some(Self::Period), + "colon" => Some(Self::Colon), + "emDash" => Some(Self::EmDash), + "enDash" => Some(Self::EnDash), + _ => None, + } + } + + pub const fn ooxml(self) -> &'static str { + match self { + Self::Hyphen => "hyphen", + Self::Period => "period", + Self::Colon => "colon", + Self::EmDash => "emDash", + Self::EnDash => "enDash", + } + } +} + +impl NumFormat { + /// Every format, in the order ST_NumberFormat lists them (which is also the order of the + /// binary .doc number format codes up to 0x27). + pub const ALL: [NumFormat; 62] = [ + NumFormat::Decimal, + NumFormat::UpperRoman, + NumFormat::LowerRoman, + NumFormat::UpperLetter, + NumFormat::LowerLetter, + NumFormat::Ordinal, + NumFormat::CardinalText, + NumFormat::OrdinalText, + NumFormat::Hex, + NumFormat::Chicago, + NumFormat::IdeographDigital, + NumFormat::JapaneseCounting, + NumFormat::Aiueo, + NumFormat::Iroha, + NumFormat::DecimalFullWidth, + NumFormat::DecimalHalfWidth, + NumFormat::JapaneseLegal, + NumFormat::JapaneseDigitalTenThousand, + NumFormat::DecimalEnclosedCircle, + NumFormat::DecimalFullWidth2, + NumFormat::AiueoFullWidth, + NumFormat::IrohaFullWidth, + NumFormat::DecimalZero, + NumFormat::Bullet, + NumFormat::Ganada, + NumFormat::Chosung, + NumFormat::DecimalEnclosedFullstop, + NumFormat::DecimalEnclosedParen, + NumFormat::DecimalEnclosedCircleChinese, + NumFormat::IdeographEnclosedCircle, + NumFormat::IdeographTraditional, + NumFormat::IdeographZodiac, + NumFormat::IdeographZodiacTraditional, + NumFormat::TaiwaneseCounting, + NumFormat::IdeographLegalTraditional, + NumFormat::TaiwaneseCountingThousand, + NumFormat::TaiwaneseDigital, + NumFormat::ChineseCounting, + NumFormat::ChineseLegalSimplified, + NumFormat::ChineseCountingThousand, + NumFormat::KoreanDigital, + NumFormat::KoreanCounting, + NumFormat::KoreanLegal, + NumFormat::KoreanDigital2, + NumFormat::VietnameseCounting, + NumFormat::RussianLower, + NumFormat::RussianUpper, + NumFormat::None, + NumFormat::NumberInDash, + NumFormat::Hebrew1, + NumFormat::Hebrew2, + NumFormat::ArabicAlpha, + NumFormat::ArabicAbjad, + NumFormat::HindiVowels, + NumFormat::HindiConsonants, + NumFormat::HindiNumbers, + NumFormat::HindiCounting, + NumFormat::ThaiLetters, + NumFormat::ThaiNumbers, + NumFormat::ThaiCounting, + NumFormat::BahtText, + NumFormat::DollarText, + ]; + + pub fn ooxml(self) -> &'static str { + match self { + NumFormat::Decimal => "decimal", + NumFormat::UpperRoman => "upperRoman", + NumFormat::LowerRoman => "lowerRoman", + NumFormat::UpperLetter => "upperLetter", + NumFormat::LowerLetter => "lowerLetter", + NumFormat::Ordinal => "ordinal", + NumFormat::CardinalText => "cardinalText", + NumFormat::OrdinalText => "ordinalText", + NumFormat::DecimalZero => "decimalZero", + NumFormat::Bullet => "bullet", + NumFormat::None => "none", + NumFormat::Hex => "hex", + NumFormat::Chicago => "chicago", + NumFormat::IdeographDigital => "ideographDigital", + NumFormat::JapaneseCounting => "japaneseCounting", + NumFormat::Aiueo => "aiueo", + NumFormat::Iroha => "iroha", + NumFormat::DecimalFullWidth => "decimalFullWidth", + NumFormat::DecimalHalfWidth => "decimalHalfWidth", + NumFormat::JapaneseLegal => "japaneseLegal", + NumFormat::JapaneseDigitalTenThousand => "japaneseDigitalTenThousand", + NumFormat::DecimalEnclosedCircle => "decimalEnclosedCircle", + NumFormat::DecimalFullWidth2 => "decimalFullWidth2", + NumFormat::AiueoFullWidth => "aiueoFullWidth", + NumFormat::IrohaFullWidth => "irohaFullWidth", + NumFormat::Ganada => "ganada", + NumFormat::Chosung => "chosung", + NumFormat::DecimalEnclosedFullstop => "decimalEnclosedFullstop", + NumFormat::DecimalEnclosedParen => "decimalEnclosedParen", + NumFormat::DecimalEnclosedCircleChinese => "decimalEnclosedCircleChinese", + NumFormat::IdeographEnclosedCircle => "ideographEnclosedCircle", + NumFormat::IdeographTraditional => "ideographTraditional", + NumFormat::IdeographZodiac => "ideographZodiac", + NumFormat::IdeographZodiacTraditional => "ideographZodiacTraditional", + NumFormat::TaiwaneseCounting => "taiwaneseCounting", + NumFormat::IdeographLegalTraditional => "ideographLegalTraditional", + NumFormat::TaiwaneseCountingThousand => "taiwaneseCountingThousand", + NumFormat::TaiwaneseDigital => "taiwaneseDigital", + NumFormat::ChineseCounting => "chineseCounting", + NumFormat::ChineseLegalSimplified => "chineseLegalSimplified", + NumFormat::ChineseCountingThousand => "chineseCountingThousand", + NumFormat::KoreanDigital => "koreanDigital", + NumFormat::KoreanCounting => "koreanCounting", + NumFormat::KoreanLegal => "koreanLegal", + NumFormat::KoreanDigital2 => "koreanDigital2", + NumFormat::VietnameseCounting => "vietnameseCounting", + NumFormat::RussianLower => "russianLower", + NumFormat::RussianUpper => "russianUpper", + NumFormat::NumberInDash => "numberInDash", + NumFormat::Hebrew1 => "hebrew1", + NumFormat::Hebrew2 => "hebrew2", + NumFormat::ArabicAlpha => "arabicAlpha", + NumFormat::ArabicAbjad => "arabicAbjad", + NumFormat::HindiVowels => "hindiVowels", + NumFormat::HindiConsonants => "hindiConsonants", + NumFormat::HindiNumbers => "hindiNumbers", + NumFormat::HindiCounting => "hindiCounting", + NumFormat::ThaiLetters => "thaiLetters", + NumFormat::ThaiNumbers => "thaiNumbers", + NumFormat::ThaiCounting => "thaiCounting", + NumFormat::BahtText => "bahtText", + NumFormat::DollarText => "dollarText", + } + } + /// The format an ST_NumberFormat value names; a value the format doesn't define is decimal + /// (as Word shows it). So is `custom` (a Word 2010 format string, always written with a + /// decimal fallback). + pub fn from_ooxml(s: &str) -> NumFormat { + NumFormat::ALL.into_iter().find(|f| f.ooxml() == s).unwrap_or(NumFormat::Decimal) + } + /// Format `n` (1-based). Formats the app doesn't draw yet (Hebrew, Hindi, Thai letters + /// and counting, Vietnamese, text amounts) show decimal, as do values past a sequence's end + /// (the 21st item of ①…⑳). Thai numbers are the decimal number in Thai digits (๑, ๒ … ๑๐). + pub fn format(self, n: u32) -> String { + use crate::cjk_numbers as cjk; + match self { + NumFormat::DecimalZero => format!("{n:02}"), + NumFormat::ThaiNumbers => thai_digits(n), + NumFormat::UpperRoman => roman(n).to_uppercase(), + NumFormat::LowerRoman => roman(n), + NumFormat::UpperLetter => letters(n).to_uppercase(), + NumFormat::LowerLetter => letters(n), + NumFormat::RussianUpper => russian_letters(n).to_uppercase(), + NumFormat::RussianLower => russian_letters(n), + NumFormat::ArabicAlpha => repeated_letter(&ARABIC_ALPHA, n), + NumFormat::ArabicAbjad => repeated_letter(&ARABIC_ABJAD, n), + NumFormat::Ordinal => format!("{n}{}", ordinal_suffix(n)), + NumFormat::CardinalText => cardinal_text(n), + NumFormat::OrdinalText => ordinal_text(n), + NumFormat::Bullet | NumFormat::None => String::new(), + NumFormat::Chicago => chicago(n), + NumFormat::NumberInDash => format!("- {n} -"), + NumFormat::IdeographDigital | NumFormat::KoreanDigital2 | NumFormat::TaiwaneseDigital => cjk::ideograph_digital(n), + NumFormat::JapaneseCounting => cjk::japanese_counting(n), + NumFormat::JapaneseLegal => cjk::japanese_legal(n), + NumFormat::JapaneseDigitalTenThousand => cjk::japanese_digital_ten_thousand(n), + NumFormat::Aiueo => cjk::aiueo(n), + NumFormat::AiueoFullWidth => cjk::aiueo_full_width(n), + NumFormat::Iroha => cjk::iroha(n), + NumFormat::IrohaFullWidth => cjk::iroha_full_width(n), + NumFormat::DecimalFullWidth | NumFormat::DecimalFullWidth2 => cjk::decimal_full_width(n), + NumFormat::DecimalEnclosedCircle => cjk::decimal_enclosed_circle(n), + NumFormat::DecimalEnclosedCircleChinese => cjk::decimal_enclosed_circle_chinese(n), + NumFormat::DecimalEnclosedFullstop => cjk::decimal_enclosed_fullstop(n), + NumFormat::DecimalEnclosedParen => cjk::decimal_enclosed_paren(n), + NumFormat::IdeographEnclosedCircle => cjk::ideograph_enclosed_circle(n), + NumFormat::IdeographTraditional => cjk::ideograph_traditional(n), + NumFormat::IdeographZodiac => cjk::ideograph_zodiac(n), + NumFormat::IdeographZodiacTraditional => cjk::ideograph_zodiac_traditional(n), + NumFormat::IdeographLegalTraditional => cjk::ideograph_legal_traditional(n), + NumFormat::TaiwaneseCounting => cjk::taiwanese_counting(n), + NumFormat::TaiwaneseCountingThousand => cjk::taiwanese_counting_thousand(n), + NumFormat::ChineseCounting => cjk::chinese_counting(n), + NumFormat::ChineseCountingThousand => cjk::chinese_counting_thousand(n), + NumFormat::ChineseLegalSimplified => cjk::chinese_legal_simplified(n), + NumFormat::KoreanCounting => cjk::korean_counting(n), + NumFormat::KoreanDigital => cjk::korean_digital(n), + NumFormat::KoreanLegal => cjk::korean_legal(n), + NumFormat::Ganada => cjk::ganada(n), + NumFormat::Chosung => cjk::chosung(n), + NumFormat::Decimal + | NumFormat::DecimalHalfWidth + | NumFormat::Hex + | NumFormat::VietnameseCounting + | NumFormat::Hebrew1 + | NumFormat::Hebrew2 + | NumFormat::HindiVowels + | NumFormat::HindiConsonants + | NumFormat::HindiNumbers + | NumFormat::HindiCounting + | NumFormat::ThaiLetters + | NumFormat::ThaiCounting + | NumFormat::BahtText + | NumFormat::DollarText => n.to_string(), + } + } +} + +/// `n` in Thai digits, ๐ (U+0E50) to ๙ (#5757). +fn thai_digits(n: u32) -> String { + n.to_string().chars().filter_map(|c| c.to_digit(10).and_then(|d| char::from_u32(0x0E50 + d))).collect() +} + +/// Chicago Manual of Style note marks: *, †, ‡, §, then doubled (**, ††, …) and so on. +fn chicago(n: u32) -> String { + const MARKS: [char; 4] = ['*', '\u{2020}', '\u{2021}', '\u{a7}']; + if n == 0 { + return String::new(); + } + let i = (n - 1) as usize; + let k = (i / 4 + 1).min(10); + std::iter::repeat_n(MARKS.get(i % 4).copied().unwrap_or('*'), k).collect() +} + +fn ordinal_suffix(n: u32) -> &'static str { + match (n % 10, n % 100) { + (_, 11..=13) => "th", + (1, _) => "st", + (2, _) => "nd", + (3, _) => "rd", + _ => "th", + } +} + +/// Lowercase roman numerals (0 → empty; values above 3999 use repeated `m`). +pub fn roman(mut n: u32) -> String { + const T: [(u32, &str); 13] = [ + (1000, "m"), + (900, "cm"), + (500, "d"), + (400, "cd"), + (100, "c"), + (90, "xc"), + (50, "l"), + (40, "xl"), + (10, "x"), + (9, "ix"), + (5, "v"), + (4, "iv"), + (1, "i"), + ]; + let mut s = String::new(); + n = n.min(50_000); + for (v, r) in T { + while n >= v { + s.push_str(r); + n -= v; + } + } + s +} + +/// Word-style letters: a..z, aa..zz, aaa.. (repeated letter, not base-26). +pub fn letters(n: u32) -> String { + if n == 0 { + return String::new(); + } + let n = n.min(780); + let k = (n - 1) / 26 + 1; + let c = (b'a' + ((n - 1) % 26) as u8) as char; + std::iter::repeat_n(c, k as usize).collect() +} + +/// The 29 letters Word counts Russian lists with: а..я without ё, й, ъ and ь (ы stays). +const RUSSIAN_LETTERS: [char; 29] = + ['а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'к', 'л', 'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш', 'щ', 'ы', 'э', 'ю', 'я']; + +/// Word-style Russian letters: а..я, аа..яя, ааа.. (repeated letter, as [`letters`]). +pub fn russian_letters(n: u32) -> String { + if n == 0 { + return String::new(); + } + let len = RUSSIAN_LETTERS.len() as u32; + let n = n.min(len * 30); + let k = (n - 1) / len + 1; + let c = RUSSIAN_LETTERS.get(((n - 1) % len) as usize).copied().unwrap_or('а'); + std::iter::repeat_n(c, k as usize).collect() +} + +/// `arabicAlpha`: the 28 letters in alphabetical (hijā'ī) order. +const ARABIC_ALPHA: [char; 28] = + ['أ', 'ب', 'ت', 'ث', 'ج', 'ح', 'خ', 'د', 'ذ', 'ر', 'ز', 'س', 'ش', 'ص', 'ض', 'ط', 'ظ', 'ع', 'غ', 'ف', 'ق', 'ك', 'ل', 'م', 'ن', 'ه', 'و', 'ي']; + +/// `arabicAbjad`: the 28 letters in abjad order (أبجد هوز حطي كلمن سعفص قرشت ثخذ ضظغ). +const ARABIC_ABJAD: [char; 28] = + ['أ', 'ب', 'ج', 'د', 'ه', 'و', 'ز', 'ح', 'ط', 'ي', 'ك', 'ل', 'م', 'ن', 'س', 'ع', 'ف', 'ص', 'ق', 'ر', 'ش', 'ت', 'ث', 'خ', 'ذ', 'ض', 'ظ', 'غ']; + +/// One letter of `alphabet`, repeated once per pass through it (as [`letters`]: a..z, aa..zz). +fn repeated_letter(alphabet: &[char], n: u32) -> String { + let Some(&first) = alphabet.first() else { return n.to_string() }; + if n == 0 { + return String::new(); + } + let len = alphabet.len() as u32; + let n = n.min(len * 30); + let k = (n - 1) / len + 1; + let c = alphabet.get(((n - 1) % len) as usize).copied().unwrap_or(first); + std::iter::repeat_n(c, k as usize).collect() +} + +const ONES: [&str; 20] = [ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", +]; +const TENS: [&str; 10] = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]; + +/// English cardinal words, capitalised first letter (`Twenty-One`). +pub fn cardinal_text(n: u32) -> String { + fn words(n: u32) -> String { + match n { + 0..=19 => ONES.get(n as usize).copied().unwrap_or("").to_string(), + 20..=99 => { + let t = TENS.get((n / 10) as usize).copied().unwrap_or(""); + if n.is_multiple_of(10) { t.to_string() } else { format!("{t}-{}", ONES.get((n % 10) as usize).copied().unwrap_or("")) } + } + 100..=999 => { + let h = format!("{} hundred", words(n / 100)); + if n.is_multiple_of(100) { h } else { format!("{h} {}", words(n % 100)) } + } + _ => { + let th = format!("{} thousand", words(n / 1000)); + if n.is_multiple_of(1000) { th } else { format!("{th} {}", words(n % 1000)) } + } + } + } + capitalize(&words(n.min(999_999))) +} + +/// English ordinal words (`Twenty-First`). +pub fn ordinal_text(n: u32) -> String { + let c = cardinal_text(n).to_lowercase(); + let (head, last) = match c.rfind(['-', ' ']) { + Some(i) => (c.get(..=i).unwrap_or(""), c.get(i + 1..).unwrap_or("")), + None => ("", c.as_str()), + }; + let last = match last { + "one" => "first".to_string(), + "two" => "second".to_string(), + "three" => "third".to_string(), + "five" => "fifth".to_string(), + "eight" => "eighth".to_string(), + "nine" => "ninth".to_string(), + "twelve" => "twelfth".to_string(), + l if l.ends_with('y') => format!("{}ieth", l.trim_end_matches('y')), + l => format!("{l}th"), + }; + capitalize(&format!("{head}{last}")) +} + +fn capitalize(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut up = true; + for c in s.chars() { + if up { + out.extend(c.to_uppercase()); + } else { + out.push(c); + } + up = c == '-' || c == ' '; + } + out +} + +/// Where footnotes or endnotes are placed (ECMA-376 §17.18.33/§17.18.20, `ST_FtnPos`/`ST_EdnPos`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum NotePos { + /// Footnotes: at the bottom of the page. + #[default] + PageBottom, + /// Footnotes: right below the page's last line of text. + BeneathText, + /// Endnotes: at the end of each section. + SectEnd, + /// Endnotes: at the end of the document. + DocEnd, +} + +impl NotePos { + pub fn ooxml(self) -> &'static str { + match self { + NotePos::PageBottom => "pageBottom", + NotePos::BeneathText => "beneathText", + NotePos::SectEnd => "sectEnd", + NotePos::DocEnd => "docEnd", + } + } + pub fn from_ooxml(s: &str) -> Option { + [NotePos::PageBottom, NotePos::BeneathText, NotePos::SectEnd, NotePos::DocEnd].into_iter().find(|p| p.ooxml() == s) + } +} + +/// When note numbering starts again (ECMA-376 §17.18.66, `ST_RestartNumber`). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum NoteRestart { + #[default] + Continuous, + EachSect, + EachPage, +} + +impl NoteRestart { + pub fn ooxml(self) -> &'static str { + match self { + NoteRestart::Continuous => "continuous", + NoteRestart::EachSect => "eachSect", + NoteRestart::EachPage => "eachPage", + } + } + pub fn from_ooxml(s: &str) -> Option { + [NoteRestart::Continuous, NoteRestart::EachSect, NoteRestart::EachPage].into_iter().find(|r| r.ooxml() == s) + } +} + +/// The highest note number "start at" takes (Word's own limit is lower; this keeps hostile files +/// from overflowing the counters). +pub const MAX_NOTE_START: u32 = 32_767; + +/// Footnote or endnote options as a document or a section states them (ECMA-376 §17.11.11 and +/// §17.11.4, `w:footnotePr` / `w:endnotePr`): unset fields come from the level above (a +/// section's from the document's, the document's from Word's defaults). The document's number +/// format is `Settings::footnote_format` / `endnote_format`, so its `num_fmt` stays unset. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct NoteProps { + #[serde(skip_serializing_if = "Option::is_none")] + pub pos: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub num_fmt: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub num_start: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub num_restart: Option, +} + +impl NoteProps { + pub fn is_empty(&self) -> bool { + *self == NoteProps::default() + } +} + +/// Footnote or endnote options in effect for a section (see [`crate::Document::note_options`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NoteOptions { + pub pos: NotePos, + pub num_fmt: NumFormat, + /// 1..=[`MAX_NOTE_START`]. + pub num_start: u32, + pub num_restart: NoteRestart, +} + +/// Header/footer story ids (in `Document::parts`) for a section. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(default)] +pub struct HeaderSet { + pub default: Option, + pub first: Option, + pub even: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Columns { + pub count: u32, + /// Space between columns, points. + pub space: f32, + /// Line between columns. + pub separator: bool, + /// Unequal widths (width, space after) per column; empty = equal. + pub widths: Vec<(f32, f32)>, +} + +impl Default for Columns { + fn default() -> Self { + Columns { count: 1, space: 36.0, separator: false, widths: Vec::new() } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct LineNumbering { + pub count_by: u32, + pub start: u32, + /// Distance from text, points (0 = auto). + pub distance: f32, + /// Restart each page / section / continuous. + pub restart: LineNumberRestart, +} + +impl Default for LineNumbering { + fn default() -> Self { + LineNumbering { count_by: 1, start: 1, distance: 0.0, restart: LineNumberRestart::Page } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum LineNumberRestart { + #[default] + Page, + Section, + Continuous, +} + +/// Document grid kind (`w:docGrid/@w:type`, ECMA-376 §17.6.5, §17.18.14). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum DocGridType { + /// No grid. + #[default] + Default, + /// Line grid only: line heights snap to the line pitch. + Lines, + /// Line and character grid: East Asian characters advance by the character pitch. + LinesAndChars, + /// Line grid, and all text snaps to the character grid. + SnapToChars, +} + +impl DocGridType { + pub fn ooxml(self) -> &'static str { + match self { + DocGridType::Default => "default", + DocGridType::Lines => "lines", + DocGridType::LinesAndChars => "linesAndChars", + DocGridType::SnapToChars => "snapToChars", + } + } + pub fn from_ooxml(s: &str) -> Option { + [DocGridType::Default, DocGridType::Lines, DocGridType::LinesAndChars, DocGridType::SnapToChars].into_iter().find(|t| t.ooxml() == s) + } +} + +/// Smallest and largest line or character pitch the layout uses, points. +pub const MIN_GRID_PITCH: f32 = 1.0; +pub const MAX_GRID_PITCH: f32 = 1584.0; +/// `w:charSpace` bound (4096ths of a point): ±1584 pt. +pub const MAX_CHAR_SPACE: i32 = 1584 * 4096; + +/// The document grid (`w:docGrid`): lines per page and characters per line, as pitches. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct DocGrid { + #[serde(rename = "type")] + pub kind: DocGridType, + /// Distance between grid lines, points (`w:linePitch`, twips in the file). + pub line_pitch: f32, + /// Added to the default font size to give the character pitch, 4096ths of a point + /// (`w:charSpace`). + pub char_space: i32, +} + +impl Default for DocGrid { + /// Word's grid for a new document: no grid, an 18 pt pitch kept for when one is turned on. + fn default() -> Self { + DocGrid { kind: DocGridType::Default, line_pitch: 18.0, char_space: 0 } + } +} + +impl DocGrid { + /// The line pitch, clamped to something layout can use. + pub fn line_pitch(&self) -> f32 { + if self.line_pitch.is_finite() { self.line_pitch.clamp(MIN_GRID_PITCH, MAX_GRID_PITCH) } else { 18.0 } + } + /// The character pitch for a document whose default font is `font_size` points. + pub fn char_pitch(&self, font_size: f32) -> f32 { + let size = if font_size.is_finite() { font_size.clamp(MIN_GRID_PITCH, MAX_GRID_PITCH) } else { 10.5 }; + (size + self.char_space.clamp(-MAX_CHAR_SPACE, MAX_CHAR_SPACE) as f32 / 4096.0).clamp(MIN_GRID_PITCH, MAX_GRID_PITCH) + } + /// The `w:charSpace` that gives `pitch` with the default font at `font_size` points. + pub fn char_space_for(pitch: f32, font_size: f32) -> i32 { + let pitch = if pitch.is_finite() { pitch.clamp(MIN_GRID_PITCH, MAX_GRID_PITCH) } else { font_size }; + let size = if font_size.is_finite() { font_size.clamp(MIN_GRID_PITCH, MAX_GRID_PITCH) } else { 10.5 }; + (((pitch - size) * 4096.0).round() as i32).clamp(-MAX_CHAR_SPACE, MAX_CHAR_SPACE) + } +} + +/// Page setup and section-level settings. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct SectionProps { + pub start: SectionStart, + /// Page width/height, points (already swapped for landscape). + pub page_w: f32, + pub page_h: f32, + pub landscape: bool, + pub margin_top: f32, + pub margin_bottom: f32, + pub margin_left: f32, + pub margin_right: f32, + /// Distance from the page edge to the header / footer. + pub header: f32, + pub footer: f32, + /// The space kept between a header and the body, and between the body and a footer, points + /// (ODF's header and footer spacing; Word keeps none). The margin already holds the band and + /// its spacing, so it only moves the body once content and spacing reach past the margin. + #[serde(skip_serializing_if = "is_zero")] + pub header_gap: f32, + #[serde(skip_serializing_if = "is_zero")] + pub footer_gap: f32, + pub gutter: f32, + /// Mirror margins (inside/outside) — set from the document settings. + pub columns: Columns, + pub headers: HeaderSet, + pub footers: HeaderSet, + /// Different first page header/footer. + pub title_page: bool, + pub page_num_start: Option, + pub page_num_format: NumFormat, + /// Chapter heading level and separator in `w:pgNumType`. + pub chapter_style: Option, + pub chapter_sep: Option, + /// The file carries a `w:pgNumType` element that sets nothing else (no start, decimal + /// format), so saving writes it back rather than dropping it. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub page_num_set: bool, + pub line_numbers: Option, + pub valign: PageVAlign, + /// Form protection of the section (`w:formProt`, ECMA-376 §17.6.6): `Some` when the file + /// states it. Kept so saving writes it back; editing does not act on it. + #[serde(skip_serializing_if = "Option::is_none")] + pub form_prot: Option, + pub page_borders: Option, + /// Text direction / bidi section. + pub rtl: bool, + /// Which way the section's text runs (`w:textDirection`, ECMA-376 §17.6.20): vertical + /// sections turn the page's text area (headers and footers stay horizontal). + #[serde(skip_serializing_if = "is_horizontal")] + pub text_direction: crate::props::TextDirection, + /// The file states the horizontal direction (`lrTb`) outright; saving writes it back. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub text_direction_set: bool, + /// Tracked change of the section's properties (`w:sectPrChange`). + #[serde(skip_serializing_if = "Option::is_none")] + pub fmt_change: Option>>, + /// Document grid (`w:docGrid`); `None` when the file has none (no grid). + pub doc_grid: Option, + /// The section's own footnote and endnote options (unset: the document's). + #[serde(skip_serializing_if = "NoteProps::is_empty")] + pub footnote_pr: NoteProps, + #[serde(skip_serializing_if = "NoteProps::is_empty")] + pub endnote_pr: NoteProps, + /// First-page and other-page printer trays (`w:paperSrc`). + pub paper_src_first: Option, + pub paper_src_other: Option, + /// Section-specific suppression of endnotes (`w:noEndnote`). + pub no_endnote: Option, + /// Independent footnote column count (`w15:footnoteColumns`). + pub footnote_columns: Option, + /// Whether its columns end balanced (an ODF section's `text:dont-balance-text-columns`): + /// `None` is Word's rule, balanced when a continuous section follows; `Some(true)` also at + /// the end of the document, `Some(false)` never. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub balance_columns: Option, +} + +impl Default for SectionProps { + /// Word's default: US Letter, 1" margins, 0.5" header/footer distance. + fn default() -> Self { + SectionProps { + start: SectionStart::NextPage, + page_w: 612.0, + page_h: 792.0, + landscape: false, + margin_top: 72.0, + margin_bottom: 72.0, + margin_left: 72.0, + margin_right: 72.0, + header: 36.0, + footer: 36.0, + header_gap: 0.0, + footer_gap: 0.0, + gutter: 0.0, + columns: Columns::default(), + headers: HeaderSet::default(), + footers: HeaderSet::default(), + title_page: false, + page_num_start: None, + page_num_format: NumFormat::Decimal, + chapter_style: None, + chapter_sep: None, + page_num_set: false, + line_numbers: None, + valign: PageVAlign::Top, + form_prot: None, + page_borders: None, + rtl: false, + text_direction: crate::props::TextDirection::Horizontal, + text_direction_set: false, + fmt_change: None, + doc_grid: None, + footnote_pr: NoteProps::default(), + endnote_pr: NoteProps::default(), + paper_src_first: None, + paper_src_other: None, + no_endnote: None, + footnote_columns: None, + balance_columns: None, + } + } +} + +fn is_zero(v: &f32) -> bool { + *v == 0.0 +} + +fn is_horizontal(d: &crate::props::TextDirection) -> bool { + !d.is_turned() +} + +impl SectionProps { + /// Join the section before this one into it, as when the break between them is deleted: + /// this section's setup stays, as in Word, and the headers and footers it leaves out (linked + /// to the previous section) are taken from `before`, so its text keeps showing them. + pub fn absorb(&mut self, before: &SectionProps) { + for (own, prev) in [(&mut self.headers, &before.headers), (&mut self.footers, &before.footers)] { + own.default = own.default.or(prev.default); + own.first = own.first.or(prev.first); + own.even = own.even.or(prev.even); + } + } + /// Width of the text area, points. + pub fn text_width(&self) -> f32 { + (self.page_w - self.margin_left - self.margin_right - self.gutter).max(18.0) + } + pub fn text_height(&self) -> f32 { + (self.page_h - self.margin_top - self.margin_bottom).max(18.0) + } + /// Swap to landscape/portrait (keeps margins relative to the paper as Word does: rotates them). + pub fn set_landscape(&mut self, on: bool) { + if on == self.landscape { + return; + } + self.landscape = on; + std::mem::swap(&mut self.page_w, &mut self.page_h); + let (t, b, l, r) = (self.margin_top, self.margin_bottom, self.margin_left, self.margin_right); + if on { + self.margin_top = l; + self.margin_bottom = r; + self.margin_left = b; + self.margin_right = t; + } else { + self.margin_left = t; + self.margin_right = b; + self.margin_bottom = l; + self.margin_top = r; + } + } + /// The line grid's pitch, points, when the section has a line grid (any grid but "no grid"). + pub fn grid_line_pitch(&self) -> Option { + self.doc_grid.filter(|g| g.kind != DocGridType::Default).map(|g| g.line_pitch()) + } + /// The character grid's pitch, points, when the section has one (`linesAndChars`, + /// `snapToChars`); `font_size` is the default font size (Normal style). + pub fn grid_char_pitch(&self, font_size: f32) -> Option { + self.doc_grid.filter(|g| matches!(g.kind, DocGridType::LinesAndChars | DocGridType::SnapToChars)).map(|g| g.char_pitch(font_size)) + } + /// Lines per page the line pitch gives (at least 1). + pub fn grid_lines_per_page(&self) -> u32 { + let pitch = self.doc_grid.unwrap_or_default().line_pitch(); + ((self.text_height() / pitch + 0.001).floor() as u32).max(1) + } + /// Characters per line the character pitch gives (at least 1). + pub fn grid_chars_per_line(&self, font_size: f32) -> u32 { + let pitch = self.doc_grid.unwrap_or_default().char_pitch(font_size); + ((self.text_width() / pitch + 0.001).floor() as u32).max(1) + } + /// Column (x offset from the text area's left, width) for each column, in reading order: + /// in a right-to-left section (`rtl`, `w:bidi`) the first column stands at the right edge + /// and the columns (with the space after each) run leftwards. + pub fn column_boxes(&self) -> Vec<(f32, f32)> { + let tw = self.text_width(); + let n = self.columns.count.clamp(1, 45) as usize; + let boxes: Vec<(f32, f32)> = if !self.columns.widths.is_empty() && self.columns.widths.len() == n { + let mut x = 0.0; + self.columns + .widths + .iter() + .map(|(w, sp)| { + let b = (x, w.max(18.0)); + x += w + sp; + b + }) + .collect() + } else { + let space = self.columns.space.max(0.0); + let w = ((tw - space * (n as f32 - 1.0)) / n as f32).max(18.0); + (0..n).map(|i| (i as f32 * (w + space), w)).collect() + }; + if self.rtl { boxes.into_iter().map(|(x, w)| (tw - x - w, w)).collect() } else { boxes } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn number_formats() { + assert_eq!(NumFormat::UpperRoman.format(1994), "MCMXCIV"); + assert_eq!(NumFormat::LowerLetter.format(27), "aa"); + assert_eq!(NumFormat::LowerLetter.format(0), ""); + assert_eq!(NumFormat::Ordinal.format(12), "12th"); + assert_eq!(NumFormat::Ordinal.format(22), "22nd"); + assert_eq!(NumFormat::CardinalText.format(21), "Twenty-One"); + assert_eq!(NumFormat::OrdinalText.format(21), "Twenty-First"); + assert_eq!(NumFormat::OrdinalText.format(20), "Twentieth"); + assert_eq!(NumFormat::OrdinalText.format(112), "One Hundred Twelfth"); + assert_eq!(NumFormat::DecimalZero.format(3), "03"); + assert!(!NumFormat::UpperRoman.format(u32::MAX).is_empty()); + assert_eq!(NumFormat::LowerLetter.format(u32::MAX).len(), 30); + assert_eq!([NumFormat::Chicago.format(1), NumFormat::Chicago.format(6)], ["*", "\u{2020}\u{2020}"]); + assert_eq!(NumFormat::ChineseCounting.format(11), "十一"); + assert_eq!(NumFormat::ChineseCountingThousand.format(101), "一百零一"); + assert_eq!(NumFormat::DecimalEnclosedCircle.format(3), "③"); + assert_eq!(NumFormat::Hebrew1.format(3), "3", "formats not drawn yet show decimal"); + assert_eq!(NumFormat::RussianLower.format(0), ""); + assert_eq!(NumFormat::RussianUpper.format(u32::MAX).chars().count(), 30); + for f in NumFormat::ALL { + assert_eq!(NumFormat::from_ooxml(f.ooxml()), f, "{f:?} round-trips"); + assert!(f == NumFormat::Bullet || f == NumFormat::None || !f.format(u32::MAX).is_empty(), "{f:?}"); + } + assert_eq!(NumFormat::from_ooxml("klingon"), NumFormat::Decimal, "unknown formats are decimal"); + } + + /// #5757: Thai numbers are the decimal number in Thai digits, every digit, at any length. + #[test] + fn thai_numbers_use_thai_digits() { + let labels: Vec = [1, 2, 3, 9, 10, 11, 20, 105, 2569].iter().map(|n| NumFormat::ThaiNumbers.format(*n)).collect(); + assert_eq!(labels.join(" "), "๑ ๒ ๓ ๙ ๑๐ ๑๑ ๒๐ ๑๐๕ ๒๕๖๙"); + assert_eq!(NumFormat::ThaiNumbers.format(0), "๐"); + assert_eq!(NumFormat::ThaiNumbers.format(u32::MAX), "๔๒๙๔๙๖๗๒๙๕"); + assert_eq!(NumFormat::from_ooxml("thaiNumbers"), NumFormat::ThaiNumbers); + // Formats not drawn yet still show decimal. + assert_eq!(NumFormat::ThaiLetters.format(3), "3"); + } + + #[test] + fn russian_letters_follow_word() { + // Word's own labels for a russianLower list: no ё, й, ъ or ь, then doubled letters. + let labels: Vec = (1..=31).map(|n| NumFormat::RussianLower.format(n)).collect(); + assert_eq!(labels.join(" "), "а б в г д е ж з и к л м н о п р с т у ф х ц ч ш щ ы э ю я аа бб"); + assert_eq!(NumFormat::RussianLower.format(59), "ааа"); + assert_eq!(NumFormat::RussianLower.format(100), "нннн"); + assert_eq!(NumFormat::RussianUpper.format(26), "Ы"); + assert_eq!(NumFormat::RussianUpper.format(36), "ЖЖ"); + } + + #[test] + fn arabic_letters_count_in_both_orders() { + let alpha: Vec = (1..=5).map(|n| NumFormat::ArabicAlpha.format(n)).collect(); + assert_eq!(alpha.join(" "), "أ ب ت ث ج"); + let abjad: Vec = (1..=5).map(|n| NumFormat::ArabicAbjad.format(n)).collect(); + assert_eq!(abjad.join(" "), "أ ب ج د ه"); + assert_eq!(NumFormat::ArabicAlpha.format(28), "ي"); + assert_eq!(NumFormat::ArabicAbjad.format(28), "غ"); + // Past the 28th letter they double, as Latin and Russian letters do. + assert_eq!(NumFormat::ArabicAlpha.format(29), "أأ"); + assert_eq!(NumFormat::ArabicAbjad.format(30), "بب"); + } + + #[test] + fn landscape_swaps() { + let mut s = SectionProps { margin_left: 50.0, ..Default::default() }; + s.set_landscape(true); + assert_eq!((s.page_w, s.page_h), (792.0, 612.0)); + assert_eq!(s.margin_top, 50.0); + s.set_landscape(false); + assert_eq!(s.margin_left, 50.0); + assert_eq!((s.page_w, s.page_h), (612.0, 792.0)); + } + + #[test] + fn grid_pitches_are_clamped() { + let mut s = SectionProps::default(); + assert_eq!(s.grid_line_pitch(), None); + s.doc_grid = Some(DocGrid { kind: DocGridType::Lines, line_pitch: 0.0, char_space: i32::MIN }); + assert_eq!(s.grid_line_pitch(), Some(MIN_GRID_PITCH)); + assert_eq!(s.grid_char_pitch(10.5), None); + s.doc_grid = Some(DocGrid { kind: DocGridType::LinesAndChars, line_pitch: f32::NAN, char_space: i32::MIN }); + assert_eq!(s.grid_line_pitch(), Some(18.0)); + assert_eq!(s.grid_char_pitch(10.5), Some(MIN_GRID_PITCH)); + // 648 pt of text height at an 18 pt pitch: 36 lines; 468 pt / 13 pt = 36 characters. + s.doc_grid = Some(DocGrid { kind: DocGridType::LinesAndChars, line_pitch: 18.0, char_space: DocGrid::char_space_for(13.0, 10.5) }); + assert_eq!(s.grid_lines_per_page(), 36); + assert_eq!(s.grid_chars_per_line(10.5), 36); + } + + #[test] + fn columns_split() { + let s = SectionProps { columns: Columns { count: 2, space: 36.0, ..Default::default() }, ..Default::default() }; + let b = s.column_boxes(); + assert_eq!(b.len(), 2); + assert_eq!(b[0], (0.0, 216.0)); + assert_eq!(b[1], (252.0, 216.0)); + } + + #[test] + fn rtl_columns_run_right_to_left() { + let s = SectionProps { rtl: true, columns: Columns { count: 2, space: 36.0, ..Default::default() }, ..Default::default() }; + assert_eq!(s.column_boxes(), vec![(252.0, 216.0), (0.0, 216.0)]); + let s = SectionProps { + rtl: true, + columns: Columns { count: 2, widths: vec![(100.0, 20.0), (200.0, 0.0)], ..Default::default() }, + ..Default::default() + }; + // Text width 468: the first column hugs the right edge, the second sits 20pt to its left. + assert_eq!(s.column_boxes(), vec![(368.0, 100.0), (148.0, 200.0)]); + } +} diff --git a/crates/wordmodel/src/settings.rs b/crates/wordmodel/src/settings.rs new file mode 100644 index 000000000..67be5590a --- /dev/null +++ b/crates/wordmodel/src/settings.rs @@ -0,0 +1,334 @@ +use super::{Deserialize, Rgb, Serialize, math, section, styles}; + +/// A bibliography source. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Source { + /// Unique tag cited by CITATION fields. + pub tag: String, + /// book, bookSection, article (journal), periodical, conference, report, website, film, other + pub kind: String, + /// "Last, First; Last, First" + pub author: String, + /// A corporate author ("World Health Organization"): written as it is, never split into + /// names, and in place of `author` when set (Create Source › Corporate Author, `b:Corporate`, + /// #4848). + pub corporate: String, + pub title: String, + pub year: String, + pub publisher: String, + pub city: String, + pub journal: String, + pub volume: String, + pub pages: String, + pub url: String, + /// More bibliography fields (Create Source › Show All Bibliography Fields). + pub editor: String, + pub edition: String, + pub issue: String, + pub month: String, + pub day: String, + /// Date accessed (websites). + pub accessed: String, + pub doi: String, +} + +/// A custom document property (File › Info › Properties › Custom). Citation managers keep +/// their per-document preferences here (Zotero: `ZOTERO_PREF_1`, `ZOTERO_PREF_2`…). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct CustomProp { + pub name: String, + /// OOXML variant type: `lpwstr`, `i4`, `r8`, `bool`, `filetime`…, or `raw` when `value` is + /// the property's XML content kept verbatim (vectors, blobs). + pub kind: String, + pub value: String, + /// Bookmark associated with this property (`linkTarget`), retained without evaluating it. + #[serde(skip_serializing_if = "Option::is_none")] + pub link_target: Option, + /// Declared in an ODF document as a user field (§7.4.7 `text:user-field-decl`) rather than + /// a document property: written back as one. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub odf_user_field: bool, +} + +/// Document properties (File › Info). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct CoreProps { + pub title: String, + pub subject: String, + pub creator: String, + pub keywords: String, + pub description: String, + pub category: String, + pub last_modified_by: String, + pub created: String, + pub modified: String, + pub revision: u32, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Settings { + pub track_changes: bool, + /// Record a tracked cut and paste of a sentence or paragraph as a move (Track moves); off + /// when the file has `w:doNotTrackMoves` (ECMA-376 §17.15.1.29), and then it is an + /// ordinary deletion and insertion. + pub track_moves: bool, + /// Default tab stop interval, points. + pub default_tab: f32, + pub even_odd_headers: bool, + pub mirror_margins: bool, + pub auto_hyphenation: bool, + /// Hyphenation zone (`w:hyphenationZone`), points: a line is hyphenated only when breaking + /// it at a word boundary would leave more than this empty at its end. + pub hyphenation_zone: f32, + /// Words in capitals may be hyphenated (`w:doNotHyphenateCaps` turns it off). + pub hyphenate_caps: bool, + /// Most consecutive lines that may end with a hyphen (`w:consecutiveHyphenLimit`); 0 = no limit. + pub consecutive_hyphen_limit: u32, + pub page_color: Option, + /// Picture or texture page background (Design › Page Color › Fill Effects), drawn over the colour. + pub page_picture: Option, + pub watermark: Option, + /// Theme fonts (major = headings, minor = body). + pub major_font: String, + pub minor_font: String, + /// Theme colours: dk1 lt1 dk2 lt2 accent1..6 hlink folHlink. + pub theme_colors: Vec, + pub theme_name: String, + pub footnote_format: section::NumFormat, + pub endnote_format: section::NumFormat, + /// The document's footnote and endnote placement, start and restart (`w:footnotePr` / + /// `w:endnotePr` in the settings); sections may override them. + #[serde(skip_serializing_if = "section::NoteProps::is_empty")] + pub footnote_pr: section::NoteProps, + #[serde(skip_serializing_if = "section::NoteProps::is_empty")] + pub endnote_pr: section::NoteProps, + /// Restrict Editing, when enforced: `readOnly`, `comments`, `trackedChanges`, `forms`, or + /// `none` when only formatting is restricted (`w:documentProtection w:edit`). + pub protection: Option, + /// Restrict Editing limits formatting to the styles that aren't locked, and forbids direct + /// formatting (`w:formatting="1"`). + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub protect_formatting: bool, + /// The protection password's hash, as the file's `w:documentProtection` attributes + /// (`w:algorithmName`, `w:hashValue`, `w:saltValue`, `w:spinCount`, or the older + /// `w:cryptAlgorithmSid`, `w:hash`, `w:salt`…). Empty: no password. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub protect_hash: Vec<(String, String)>, + /// Word compatibility mode the document is laid out in (`compatibilityMode`): 15 for Word + /// 2013 and later, which places a table's border at the margin rather than its text and lets + /// justified lines shrink their spaces to fit more text. + pub compat_mode: u32, + /// Fonts the document's font table classes as roman (serif, `w:family w:val="roman"`): Word + /// draws one that isn't installed in a serif face. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub roman_fonts: Vec, + /// `w:doNotUseHTMLParagraphAutoSpacing`: the space between two paragraphs is the first's space + /// after plus the second's space before, not the larger of the two. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub sum_paragraph_spacing: bool, + /// `w:doNotExpandShiftReturn`: a justified line that ends in a manual line break keeps its + /// natural spacing; by default Word stretches it to the margin like a wrapped line. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub do_not_expand_shift_return: bool, + /// Keep contextual paragraph spacing inside legacy table cells (`allowSpaceOfSameStyleInTable`). + /// Word ignores the layout override in compatibility mode 15, but the flag still round-trips. + pub allow_space_of_same_style_in_table: bool, + /// Drawing grid spacing in points, across and down (View › Gridlines); ECMA-376's default is 1/8 inch. + pub grid_h: f32, + pub grid_v: f32, + /// Equation options (`m:mathPr`), when the document has them. + #[serde(skip_serializing_if = "Option::is_none")] + pub math: Option, + /// Citation and bibliography style (References › Style) by name, when the document + /// records one (OOXML's `b:Sources/@StyleName`). + #[serde(skip_serializing_if = "Option::is_none")] + pub bib_style: Option, + /// The template the document is attached to (Tools › Templates and Add-ins): the target of + /// `w:attachedTemplate`'s relationship, a path or `file:` URL as the file gives it. + #[serde(skip_serializing_if = "Option::is_none")] + pub attached_template: Option, + /// Update the document's styles from the attached template when it is opened (`w:linkStyles`). + pub link_styles: bool, + /// The zoom the document was saved at (`w:zoom`), restored when it is opened; `None` for 100% + /// without a fit mode. + #[serde(skip_serializing_if = "Option::is_none")] + pub zoom: Option, + /// Remove personal information when saving (`w:removePersonalInformation`, ECMA-376 + /// §17.15.1.70): the saved file names comment and revision authors "Author" and leaves out + /// the document's creator and last-modified-by. The open document keeps its real values. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub remove_personal_info: bool, + /// Remove date and time from annotations when saving (`w:removeDateAndTime`, §17.15.1.69): + /// comments and tracked changes are saved without `w:date`. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub remove_date_time: bool, + /// The document was opened from ODF: the standard's rules that differ from Word's apply as + /// it is laid out. + #[serde(skip_serializing_if = "Option::is_none")] + pub odf: Option, +} + +/// A document opened from ODF, for the layout: rules of the standard that differ from Word's (a +/// paragraph's padding takes room on a side without a border, XSL's box model) apply, and what +/// the file says about laying it out. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct OdfLayout { + /// Tab stops are measured from the paragraph's left indent, default ones too (LibreOffice's + /// `TabsRelativeToIndent` in the file; from the margin when it doesn't say). The document's + /// explicit stops are held from the margin either way; the default ones are counted from the + /// indent. + pub tabs_from_indent: bool, +} + +/// A saved zoom: a percentage, or a fit mode (the percentage it last came to). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Zoom { + /// 10–500. + pub percent: u32, + pub fit: Option, +} + +/// View › Zoom's fit modes a document can be saved in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ZoomFit { + OnePage, + PageWidth, + TextWidth, +} + +/// Word's compatibility mode for documents that don't state one. +pub const LEGACY_COMPAT_MODE: u32 = 12; + +/// The compatibility mode of documents created by Word 2013 and later, and by the app. +pub const COMPAT_MODE_CURRENT: u32 = 15; + +/// Word's hyphenation zone when a document doesn't set one (0.25"), points. +pub const DEFAULT_HYPHENATION_ZONE: f32 = 18.0; + +/// ECMA-376 §17.15.1.44/45: 180 twentieths of a point when a document doesn't say. +pub const DEFAULT_GRID: f32 = 9.0; + +/// A picture fill for the page background (`v:background/v:fill` with an image). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct PagePicture { + /// Key into [`Document::media`]. + pub media: String, + /// `v:fill/@type` (`tile`, `frame`, `tile`...); tiled when `tile`, stretched to the page otherwise. + pub fill_type: String, + /// `o:title` of the fill (texture name), kept for round trip. + pub title: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Watermark { + pub text: String, + pub font: String, + pub color: Rgb, + pub diagonal: bool, + pub semitransparent: bool, + /// The shape's box (its `style` width and height, points), which the text is stretched to + /// fill (`v:textpath fitshape`); `None`: the box Word's Design › Watermark gives + /// ([`Watermark::frame`]). + #[serde(skip_serializing_if = "Option::is_none")] + pub size: Option<[f32; 2]>, +} + +impl Default for Watermark { + fn default() -> Self { + Watermark { + text: "DRAFT".into(), + font: styles::BODY_FONT.into(), + color: Rgb(0xC0, 0xC0, 0xC0), + diagonal: true, + semitransparent: true, + size: None, + } + } +} + +impl Watermark { + /// The box the text fills on a page `page_w` × `page_h` (points): its own size, else half the + /// page diagonal across (half the page width when not diagonal), as high as the text's length + /// makes it. Centred on the margins and, when diagonal, turned 45° counterclockwise (Word's + /// `rotation:315`). + pub fn frame(&self, page_w: f32, page_h: f32) -> [f32; 2] { + if let Some([w, h]) = self.size.filter(|[w, h]| w.is_finite() && h.is_finite() && *w > 0.0 && *h > 0.0) { + return [w.min(4000.0), h.min(4000.0)]; + } + let (pw, ph) = (crate::geom::finite(page_w).clamp(72.0, 4000.0), crate::geom::finite(page_h).clamp(72.0, 4000.0)); + let across = if self.diagonal { (pw * pw + ph * ph).sqrt() } else { pw }; + let width = (across * 0.5).clamp(36.0, 2000.0); + let chars = self.text.chars().take(200).count().max(1) as f32; + [width, (width / (0.55 * chars)).clamp(12.0, 400.0)] + } +} + +/// Our default theme colours. +pub const THEME_COLORS: [Rgb; 12] = [ + Rgb(0x00, 0x00, 0x00), + Rgb(0xFF, 0xFF, 0xFF), + Rgb(0x0E, 0x28, 0x41), + Rgb(0xE8, 0xE8, 0xE8), + Rgb(0x15, 0x60, 0x82), + Rgb(0xE9, 0x71, 0x32), + Rgb(0x19, 0x6B, 0x24), + Rgb(0x0F, 0x9E, 0xD5), + Rgb(0xA0, 0x2B, 0x93), + Rgb(0x4E, 0xA7, 0x2E), + Rgb(0x46, 0x78, 0x86), + Rgb(0x96, 0x60, 0x7D), +]; + +impl Default for Settings { + fn default() -> Self { + Settings { + track_changes: false, + track_moves: true, + default_tab: 36.0, + even_odd_headers: false, + mirror_margins: false, + auto_hyphenation: false, + hyphenation_zone: DEFAULT_HYPHENATION_ZONE, + hyphenate_caps: true, + consecutive_hyphen_limit: 0, + page_color: None, + page_picture: None, + watermark: None, + major_font: styles::HEADING_FONT.into(), + minor_font: styles::BODY_FONT.into(), + theme_colors: THEME_COLORS.to_vec(), + theme_name: "Craft".into(), + footnote_format: section::NumFormat::Decimal, + endnote_format: section::NumFormat::LowerRoman, + footnote_pr: section::NoteProps::default(), + endnote_pr: section::NoteProps::default(), + protection: None, + protect_formatting: false, + protect_hash: Vec::new(), + compat_mode: COMPAT_MODE_CURRENT, + roman_fonts: Vec::new(), + sum_paragraph_spacing: false, + do_not_expand_shift_return: false, + allow_space_of_same_style_in_table: false, + grid_h: DEFAULT_GRID, + grid_v: DEFAULT_GRID, + math: None, + bib_style: None, + attached_template: None, + link_styles: false, + zoom: None, + remove_personal_info: false, + remove_date_time: false, + odf: None, + } + } +} diff --git a/crates/wordmodel/src/smart_art.rs b/crates/wordmodel/src/smart_art.rs new file mode 100644 index 000000000..2d4371f6f --- /dev/null +++ b/crates/wordmodel/src/smart_art.rs @@ -0,0 +1,368 @@ +//! SmartArt graphics the app makes and edits (Insert › SmartArt): a layout, an outline of items +//! (text and level) and a colour variant. +//! +//! A SmartArt object ([`crate::graphic::Graphic`] with `smart_art` set) keeps this model next to +//! the items it is drawn with. the app lays it out with its own algorithms; the DOCX writer +//! saves it as diagram parts (data, layout, style, colours and the drawing of the shapes) and the +//! reader turns a diagram back into it only when its data part is exactly what the app writes. +//! SmartArt from other programs stays as it was read. +//! +//! Every value from outside (commands, files) goes through [`SmartArtSpec::sanitize`]: item +//! count, text length and levels are capped. + +use serde::{Deserialize, Serialize}; + +use crate::props::Rgb; + +/// Most items in a SmartArt graphic. +pub const MAX_ITEMS: usize = 100; +/// Longest item text, characters. +pub const MAX_TEXT: usize = 255; +/// Deepest item level (0 is the top). +pub const MAX_LEVEL: u8 = 7; + +/// How a SmartArt graphic arranges its items. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SmartArtLayout { + /// Blocks in rows. + #[default] + BasicList, + /// Steps left to right with arrows between. + Process, + /// Steps around a circle with arrows between. + Cycle, + /// A tree: each item under the item one level up before it. + Hierarchy, + /// Stacked bands narrowing to the top. + Pyramid, + /// The first item in the middle, the items under it around it. + Radial, + /// Quadrants of a square. + Matrix, + /// Overlapping circles. + Venn, +} + +impl SmartArtLayout { + pub const ALL: [SmartArtLayout; 8] = [ + SmartArtLayout::BasicList, + SmartArtLayout::Process, + SmartArtLayout::Cycle, + SmartArtLayout::Hierarchy, + SmartArtLayout::Pyramid, + SmartArtLayout::Radial, + SmartArtLayout::Matrix, + SmartArtLayout::Venn, + ]; + + /// The id commands use (`basicList`, `process`…). + pub fn id(self) -> &'static str { + match self { + SmartArtLayout::BasicList => "basicList", + SmartArtLayout::Process => "process", + SmartArtLayout::Cycle => "cycle", + SmartArtLayout::Hierarchy => "hierarchy", + SmartArtLayout::Pyramid => "pyramid", + SmartArtLayout::Radial => "radial", + SmartArtLayout::Matrix => "matrix", + SmartArtLayout::Venn => "venn", + } + } + + /// The layout an id names (any ASCII case, ignoring hyphens, underscores and spaces; + /// `list` is the basic list; `basic process` or `process` is process, etc.). + pub fn from_id(id: &str) -> Option { + let norm = id.chars().filter(|c| !c.is_whitespace() && *c != '-' && *c != '_').collect::(); + if norm.eq_ignore_ascii_case("list") { + return Some(SmartArtLayout::BasicList); + } + SmartArtLayout::ALL.into_iter().find(|l| l.id().eq_ignore_ascii_case(&norm) || l.label().replace(' ', "").eq_ignore_ascii_case(&norm)) + } + + /// The name shown for it. + pub fn label(self) -> &'static str { + match self { + SmartArtLayout::BasicList => "Basic List", + SmartArtLayout::Process => "Basic Process", + SmartArtLayout::Cycle => "Basic Cycle", + SmartArtLayout::Hierarchy => "Hierarchy", + SmartArtLayout::Pyramid => "Basic Pyramid", + SmartArtLayout::Radial => "Basic Radial", + SmartArtLayout::Matrix => "Basic Matrix", + SmartArtLayout::Venn => "Basic Venn", + } + } + + /// The gallery category it's shown under. + pub fn category(self) -> &'static str { + match self { + SmartArtLayout::BasicList => "List", + SmartArtLayout::Process => "Process", + SmartArtLayout::Cycle => "Cycle", + SmartArtLayout::Hierarchy => "Hierarchy", + SmartArtLayout::Pyramid => "Pyramid", + SmartArtLayout::Radial | SmartArtLayout::Venn => "Relationship", + SmartArtLayout::Matrix => "Matrix", + } + } + + /// What it's for, in a sentence (the picker shows it). + pub fn description(self) -> &'static str { + match self { + SmartArtLayout::BasicList => "Shows items of equal weight as blocks in rows.", + SmartArtLayout::Process => "Shows steps in order, left to right, with arrows between them.", + SmartArtLayout::Cycle => "Shows steps that repeat, around a circle.", + SmartArtLayout::Hierarchy => "Shows a tree: each item below the item it belongs to.", + SmartArtLayout::Pyramid => "Shows parts that build on each other, largest at the bottom.", + SmartArtLayout::Radial => "Shows items around a central idea: the first item is the centre.", + SmartArtLayout::Matrix => "Shows parts of a whole as quadrants.", + SmartArtLayout::Venn => "Shows overlapping ideas as overlapping circles.", + } + } +} + +/// The colours a SmartArt graphic takes from the document's theme. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum SmartArtColors { + /// Every shape in accent `n` (1–6). + #[default] + Accent1, + Accent2, + Accent3, + Accent4, + Accent5, + Accent6, + /// Shapes take the accents in turn. + Colorful, +} + +impl SmartArtColors { + pub const ALL: [SmartArtColors; 7] = [ + SmartArtColors::Accent1, + SmartArtColors::Accent2, + SmartArtColors::Accent3, + SmartArtColors::Accent4, + SmartArtColors::Accent5, + SmartArtColors::Accent6, + SmartArtColors::Colorful, + ]; + + pub fn id(self) -> &'static str { + match self { + SmartArtColors::Accent1 => "accent1", + SmartArtColors::Accent2 => "accent2", + SmartArtColors::Accent3 => "accent3", + SmartArtColors::Accent4 => "accent4", + SmartArtColors::Accent5 => "accent5", + SmartArtColors::Accent6 => "accent6", + SmartArtColors::Colorful => "colorful", + } + } + + pub fn from_id(id: &str) -> Option { + let norm = id.chars().filter(|c| !c.is_whitespace() && *c != '-' && *c != '_').collect::(); + SmartArtColors::ALL.into_iter().find(|c| c.id().eq_ignore_ascii_case(&norm) || c.label().replace(' ', "").eq_ignore_ascii_case(&norm)) + } + + /// The name shown for it. + pub fn label(self) -> &'static str { + match self { + SmartArtColors::Accent1 => "Accent 1", + SmartArtColors::Accent2 => "Accent 2", + SmartArtColors::Accent3 => "Accent 3", + SmartArtColors::Accent4 => "Accent 4", + SmartArtColors::Accent5 => "Accent 5", + SmartArtColors::Accent6 => "Accent 6", + SmartArtColors::Colorful => "Colorful", + } + } + + /// The theme accent (1–6) of shape `i`. + pub fn accent(self, i: usize) -> usize { + match self { + SmartArtColors::Accent1 => 1, + SmartArtColors::Accent2 => 2, + SmartArtColors::Accent3 => 3, + SmartArtColors::Accent4 => 4, + SmartArtColors::Accent5 => 5, + SmartArtColors::Accent6 => 6, + SmartArtColors::Colorful => i % 6 + 1, + } + } +} + +/// One item of the outline: its text and level (0 is the top). +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SmartArtItem { + pub text: String, + #[serde(default)] + pub level: u8, + /// Fill of the item's shape in place of the style's colour. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color: Option, +} + +impl SmartArtItem { + pub fn new(text: impl Into, level: u8) -> SmartArtItem { + SmartArtItem { text: text.into(), level, color: None } + } +} + +/// A SmartArt graphic the app can edit. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SmartArtSpec { + pub layout: SmartArtLayout, + #[serde(default)] + pub items: Vec, + #[serde(default)] + pub colors: SmartArtColors, +} + +impl SmartArtSpec { + /// A new graphic of `layout` with the app's sample items. + pub fn sample(layout: SmartArtLayout) -> SmartArtSpec { + let items: Vec = match layout { + SmartArtLayout::Hierarchy => vec![ + SmartArtItem::new("Lead", 0), + SmartArtItem::new("Design", 1), + SmartArtItem::new("Sketches", 2), + SmartArtItem::new("Build", 1), + SmartArtItem::new("Code", 2), + SmartArtItem::new("Tests", 2), + ], + SmartArtLayout::Radial => { + let mut v = vec![SmartArtItem::new("Goal", 0)]; + v.extend(["Plan", "Build", "Test", "Ship"].map(|t| SmartArtItem::new(t, 1))); + v + } + SmartArtLayout::Process | SmartArtLayout::Cycle => ["Plan", "Build", "Test", "Ship"].map(|t| SmartArtItem::new(t, 0)).to_vec(), + SmartArtLayout::Pyramid | SmartArtLayout::Venn => ["People", "Process", "Tools"].map(|t| SmartArtItem::new(t, 0)).to_vec(), + SmartArtLayout::Matrix => ["Strengths", "Weaknesses", "Chances", "Risks"].map(|t| SmartArtItem::new(t, 0)).to_vec(), + SmartArtLayout::BasicList => ["Ideas", "Notes", "Drafts", "Edits", "Final"].map(|t| SmartArtItem::new(t, 0)).to_vec(), + }; + SmartArtSpec { layout, items, colors: SmartArtColors::default() } + } + + /// Make the graphic safe to store and draw: at most [`MAX_ITEMS`] items, text cut to + /// [`MAX_TEXT`] characters with control characters as spaces and the ends trimmed, and levels + /// that follow on (the first item at level 0, each item at most one level below the one + /// before it, never below [`MAX_LEVEL`]). + pub fn sanitize(&mut self) { + self.items.truncate(MAX_ITEMS); + let mut prev: Option = None; + for it in &mut self.items { + it.text = clean_text(&it.text); + let most = prev.map_or(0, |p| p.saturating_add(1)).min(MAX_LEVEL); + it.level = it.level.min(most); + prev = Some(it.level); + } + } + + /// Whether any item has its own colour. + pub fn has_custom_colors(&self) -> bool { + self.items.iter().any(|it| it.color.is_some()) + } + + /// Give every item the style's colour again. + pub fn clear_custom_colors(&mut self) { + for it in &mut self.items { + it.color = None; + } + } + + /// The theme accent (1–6) the style gives item `i`; `None` when the item has no shape of + /// its own (it is text inside another item's shape). + pub fn style_accent(&self, i: usize) -> Option { + let it = self.items.get(i)?; + let tops = || self.items.iter().take(i).filter(|o| o.level == 0).count(); + match self.layout { + SmartArtLayout::Hierarchy => Some(self.colors.accent(i)), + SmartArtLayout::Radial if i == 0 => Some(self.colors.accent(0)), + SmartArtLayout::Radial => { + let parents = self.parents(); + let spoke = |j: usize| j > 0 && matches!(parents.get(j), Some(Some(0)) | Some(None)); + spoke(i).then(|| self.colors.accent((1..i).filter(|j| spoke(*j)).count() + 1)) + } + _ => (it.level == 0).then(|| self.colors.accent(tops())), + } + } + + /// The graphic after `sanitize`. + pub fn sanitized(mut self) -> SmartArtSpec { + self.sanitize(); + self + } + + /// The index of the item each item belongs to (the nearest item before it one level up); + /// `None` for the top-level items. Expects a sanitized graphic. + pub fn parents(&self) -> Vec> { + let mut out = Vec::with_capacity(self.items.len()); + // The last item seen at each level. + let mut last: Vec = Vec::new(); + for (i, it) in self.items.iter().enumerate() { + let l = usize::from(it.level); + last.truncate(l); + out.push(if l == 0 { None } else { last.last().copied() }); + last.push(i); + } + out + } + + /// The items that belong to item `i` (directly or deeper): the run after it at deeper levels. + pub fn descendants(&self, i: usize) -> std::ops::Range { + let Some(level) = self.items.get(i).map(|it| it.level) else { return 0..0 }; + let start = i + 1; + let end = self.items.iter().skip(start).position(|it| it.level <= level).map_or(self.items.len(), |p| start + p); + start..end + } +} + +/// `s` as item text: control characters as spaces, ends trimmed, at most [`MAX_TEXT`] characters. +pub fn clean_text(s: &str) -> String { + let trimmed = s.trim_start_matches(|c: char| c.is_whitespace() || c.is_control()); + let s: String = trimmed.chars().take(MAX_TEXT).map(|c| if c.is_control() { ' ' } else { c }).collect(); + s.trim_end().to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_caps_items_text_and_levels() { + let mut s = SmartArtSpec { + items: (0..MAX_ITEMS + 20).map(|i| SmartArtItem::new(format!("{}\n{i}", "x".repeat(400)), if i == 0 { 5 } else { 200 })).collect(), + ..Default::default() + }; + s.sanitize(); + assert_eq!(s.items.len(), MAX_ITEMS); + assert_eq!(s.items[0].level, 0, "the first item is at the top"); + assert_eq!(s.items[1].level, 1, "one level below the item before"); + assert!(s.items.iter().all(|it| it.level <= MAX_LEVEL)); + assert_eq!(s.items[0].text.chars().count(), MAX_TEXT); + assert!(!s.items[0].text.contains('\n')); + } + + #[test] + fn parents_and_descendants_follow_levels() { + let s = SmartArtSpec::sample(SmartArtLayout::Hierarchy).sanitized(); + assert_eq!(s.parents(), vec![None, Some(0), Some(1), Some(0), Some(3), Some(3)]); + assert_eq!(s.descendants(0), 1..6); + assert_eq!(s.descendants(1), 2..3); + assert_eq!(s.descendants(5), 6..6); + assert_eq!(s.descendants(99), 0..0); + assert_eq!(SmartArtLayout::from_id("VENN"), Some(SmartArtLayout::Venn)); + assert_eq!(SmartArtLayout::from_id("basic-list"), Some(SmartArtLayout::BasicList)); + assert_eq!(SmartArtLayout::from_id("Basic Process"), Some(SmartArtLayout::Process)); + assert_eq!(SmartArtLayout::from_id("basic_cycle"), Some(SmartArtLayout::Cycle)); + assert_eq!(SmartArtLayout::from_id("list"), Some(SmartArtLayout::BasicList)); + assert_eq!(SmartArtColors::from_id("colorful"), Some(SmartArtColors::Colorful)); + assert_eq!(SmartArtColors::from_id("Accent 2"), Some(SmartArtColors::Accent2)); + assert_eq!(SmartArtColors::from_id("accent-4"), Some(SmartArtColors::Accent4)); + assert_eq!(clean_text(&format!("{}Valid Text", " ".repeat(1500))), "Valid Text"); + } +} diff --git a/crates/wordmodel/src/styles.rs b/crates/wordmodel/src/styles.rs new file mode 100644 index 000000000..090df9a05 --- /dev/null +++ b/crates/wordmodel/src/styles.rs @@ -0,0 +1,1116 @@ +//! Styles: paragraph, character, table and linked styles with `based on` chains, plus the +//! built-in style set (our own values in the spirit of a modern word processor's defaults). + +use serde::{Deserialize, Serialize}; + +use std::collections::BTreeMap; + +use crate::props::{ + Align, Border, BorderStyle, Borders, CharProps, LineSpacing, ParaProps, Rgb, TabAlign, TabLeader, TabStop, TableLook, TextColor, Underline, + VAlign, +}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum StyleKind { + #[default] + Paragraph, + Character, + Table, + Numbering, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct Style { + /// Stable id (`Heading1`), referenced by `ParaProps::style` / `CharProps::style`. + pub id: String, + /// Display name (`Heading 1`). + pub name: String, + pub kind: StyleKind, + pub based_on: Option, + /// Style for the paragraph after this one (Enter at the end). + pub next: Option, + /// Linked character/paragraph style id. + pub linked: Option, + pub para: ParaProps, + pub chr: CharProps, + /// Gallery order (lower first). + pub priority: Option, + /// Shown in the Styles gallery on the Home tab. + pub quick: bool, + pub hidden: bool, + /// Built in (cannot be deleted). + pub builtin: bool, + /// Can't be applied while Restrict Editing limits formatting to a selection of styles + /// (`w:locked`). + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub locked: bool, + /// The default style of its kind (`w:default`), whatever its id: localized documents call + /// the default character, table and list styles something other than the English ids. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub default: bool, + /// Table style: conditional formatting for header row, banding etc. + pub table: Option, +} + +/// Table style conditional formats (shading + character formatting per region). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct TableStyleParts { + pub borders: Option, + /// Shading of every cell (`w:tblPr/w:shd`, or the `wholeTable` region's cell shading). + pub fill: Option, + /// Cell margins, points: [top, left, bottom, right] (`w:tblCellMar`). + pub cell_margins: Option<[f32; 4]>, + pub header_fill: Option, + pub header_chr: CharProps, + /// Cell borders of the header row (`firstRow` region's `w:tcBorders`): `between` is the + /// border between header rows, `inside_v` between its cells. + pub header_borders: Option, + pub band_fill: Option, + /// Character formatting of the odd row bands (`band1Horz` region). + pub band_chr: CharProps, + /// Cell borders of the odd row bands (`band1Horz` region's `w:tcBorders`). + pub band_borders: Option, + /// Rows per band (`w:tblStyleRowBandSize`). Unset (in the whole based-on chain) or zero, the + /// row bands are off: Word reads a missing size as zero, where ECMA-376 says one (MS-OE376 + /// §2.7.5.7, #3415). Our own banded styles set one. + pub band_size: Option, + /// Columns per band (`w:tblStyleColBandSize`); unset or zero, the column bands are off, as + /// in Word (MS-OI29500 §17.7.6.5). + pub col_band_size: Option, + pub first_col_chr: CharProps, + pub total_chr: CharProps, + pub total_border_top: Option, + /// Shading and cell borders of the first column (`firstCol` region). + pub first_col_fill: Option, + pub first_col_borders: Option, + /// The last column (`lastCol` region). + pub last_col_fill: Option, + pub last_col_chr: CharProps, + pub last_col_borders: Option, + /// Shading and cell borders of the total (last) row (`lastRow` region); a top edge left + /// unset falls back to `total_border_top`. + pub total_fill: Option, + pub total_borders: Option, + /// The odd column bands (`band1Vert` region). + pub col_band_fill: Option, + pub col_band_chr: CharProps, + pub col_band_borders: Option, + /// Per-part formatting the named fields above don't carry: every field of the parts without + /// one (even bands, corner cells), and the cell text alignment of every part. Read and write + /// parts through [`TableStyleParts::part`] / [`TableStyleParts::set_part`]. + #[serde(skip_serializing_if = "BTreeMap::is_empty")] + pub regions: BTreeMap, +} + +/// A conditional formatting part of a table style (`w:tblStylePr/@w:type`, ECMA-376 §17.7.6). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TablePart { + WholeTable, + FirstRow, + LastRow, + FirstCol, + LastCol, + Band1Vert, + Band2Vert, + Band1Horz, + Band2Horz, + NeCell, + NwCell, + SeCell, + SwCell, +} + +impl TablePart { + /// Every part, lowest precedence first (ECMA-376 §17.7.6: whole table, bands, first/last + /// column, first/last row, corner cells). + pub const ALL: [TablePart; 13] = [ + TablePart::WholeTable, + TablePart::Band1Vert, + TablePart::Band2Vert, + TablePart::Band1Horz, + TablePart::Band2Horz, + TablePart::FirstCol, + TablePart::LastCol, + TablePart::FirstRow, + TablePart::LastRow, + TablePart::NeCell, + TablePart::NwCell, + TablePart::SeCell, + TablePart::SwCell, + ]; + /// The `w:type` value. + pub fn ooxml(self) -> &'static str { + match self { + TablePart::WholeTable => "wholeTable", + TablePart::FirstRow => "firstRow", + TablePart::LastRow => "lastRow", + TablePart::FirstCol => "firstCol", + TablePart::LastCol => "lastCol", + TablePart::Band1Vert => "band1Vert", + TablePart::Band2Vert => "band2Vert", + TablePart::Band1Horz => "band1Horz", + TablePart::Band2Horz => "band2Horz", + TablePart::NeCell => "neCell", + TablePart::NwCell => "nwCell", + TablePart::SeCell => "seCell", + TablePart::SwCell => "swCell", + } + } + pub fn from_ooxml(s: &str) -> Option { + TablePart::ALL.into_iter().find(|p| p.ooxml() == s) + } + /// A part spanning whole rows (its left/right edges are the table's sides, `inside_v` the + /// edges between its cells). + pub fn is_row(self) -> bool { + matches!(self, TablePart::FirstRow | TablePart::LastRow | TablePart::Band1Horz | TablePart::Band2Horz) + } + /// A part spanning whole columns (its top/bottom edges are the table's, `between` the edges + /// between its cells). + pub fn is_column(self) -> bool { + matches!(self, TablePart::FirstCol | TablePart::LastCol | TablePart::Band1Vert | TablePart::Band2Vert) + } + fn bit(self) -> u16 { + 1 << (TablePart::ALL.iter().position(|p| *p == self).unwrap_or(0) as u16) + } +} + +/// The table style parts one cell is in (always the whole table). +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct PartSet(u16); + +impl PartSet { + /// The parts of the cell at row `ri` of `nrows`, grid column `g` spanning `span` of `ncols`, + /// in a table with options `look`, row bands of `row_size` rows and column bands of + /// `col_size` columns (a size of zero turns those bands off, as in Word). + pub fn of_cell( + look: &TableLook, + (row_size, col_size): (u32, u32), + (ri, nrows): (usize, usize), + (g, span, ncols): (usize, usize, usize), + ) -> PartSet { + let header = look.header_row && ri == 0; + let total = look.total_row && ri + 1 == nrows && nrows > 1; + let first_col = look.first_column && g == 0; + let last_col = look.last_column && g.saturating_add(span) >= ncols && ncols > 1; + let rows = row_size.min(1000) as usize; + let cols = col_size.min(1000) as usize; + let row_band = look.banded_rows && !header && rows > 0; + let odd_row = (ri.saturating_sub(usize::from(look.header_row)) / rows.max(1)).is_multiple_of(2); + let col_band = look.banded_columns && !first_col && cols > 0; + let odd_col = (g.saturating_sub(usize::from(look.first_column)) / cols.max(1)).is_multiple_of(2); + let mut bits = TablePart::WholeTable.bit(); + for (on, p) in [ + (col_band && odd_col, TablePart::Band1Vert), + (col_band && !odd_col, TablePart::Band2Vert), + (row_band && odd_row, TablePart::Band1Horz), + (row_band && !odd_row, TablePart::Band2Horz), + (first_col, TablePart::FirstCol), + (last_col, TablePart::LastCol), + (header, TablePart::FirstRow), + (total, TablePart::LastRow), + (header && last_col, TablePart::NeCell), + (header && first_col, TablePart::NwCell), + (total && last_col, TablePart::SeCell), + (total && first_col, TablePart::SwCell), + ] { + if on { + bits |= p.bit(); + } + } + PartSet(bits) + } + pub fn contains(self, p: TablePart) -> bool { + self.0 & p.bit() != 0 + } + /// The parts, lowest precedence first. + pub fn iter(self) -> impl Iterator { + TablePart::ALL.into_iter().filter(move |p| self.contains(*p)) + } +} + +/// A cell's borders from the table style's parts it is in, later parts winning. Row parts give +/// their left/right edges at the table's sides and `inside_v` between cells; column parts their +/// top/bottom edges at the table's top and bottom and `between` between rows; corner cells their +/// four edges. +pub fn part_borders(parts: &[(TablePart, Borders)], (ri, nrows): (usize, usize), (first_g, last_g): (bool, bool)) -> Borders { + let mut out = Borders::default(); + for (part, b) in parts { + let b = *b; + let mapped = if part.is_row() { + Borders { + top: b.top, + bottom: b.bottom, + left: if first_g { b.left } else { b.inside_v }, + right: if last_g { b.right } else { b.inside_v }, + between: None, + inside_v: None, + } + } else if part.is_column() { + Borders { + top: if ri == 0 { b.top } else { b.between }, + bottom: if ri + 1 >= nrows { b.bottom } else { b.between }, + left: b.left, + right: b.right, + between: None, + inside_v: None, + } + } else { + Borders { between: None, inside_v: None, ..b } + }; + out.overlay(&mapped); + } + out +} + +/// What one table style part gives its cells. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize, Default)] +#[serde(default, rename_all = "camelCase")] +pub struct PartFormat { + pub chr: CharProps, + /// Cell shading. + pub fill: Option, + /// Cell borders (the whole table's are the table borders). + pub borders: Option, + /// Horizontal alignment of the cells' paragraphs (`w:pPr/w:jc`). + pub align: Option, + /// Vertical alignment in the cells (`w:tcPr/w:vAlign`). + pub v_align: Option, +} + +impl PartFormat { + pub fn is_empty(&self) -> bool { + *self == PartFormat::default() + } + /// What `patch` sets wins; borders merge edge by edge. + pub fn overlay(&mut self, patch: &PartFormat) { + self.chr.overlay(&patch.chr); + self.fill = patch.fill.or(self.fill); + self.borders = match (self.borders, patch.borders) { + (Some(mut b), Some(p)) => { + b.overlay(&p); + Some(b) + } + (b, p) => p.or(b), + }; + self.align = patch.align.or(self.align); + self.v_align = patch.v_align.or(self.v_align); + } +} + +impl TableStyleParts { + /// Apply `patch` (a style further down a based-on chain) over these parts: what it sets wins, + /// the rest is inherited; borders merge edge by edge. + pub fn overlay(&mut self, patch: &TableStyleParts) { + let merge = |b: Option, p: Option| match (b, p) { + (Some(mut b), Some(p)) => { + b.overlay(&p); + Some(b) + } + (b, p) => p.or(b), + }; + self.borders = merge(self.borders, patch.borders); + self.header_borders = merge(self.header_borders, patch.header_borders); + self.band_borders = merge(self.band_borders, patch.band_borders); + self.band_size = patch.band_size.or(self.band_size); + self.col_band_size = patch.col_band_size.or(self.col_band_size); + self.band_chr.overlay(&patch.band_chr); + self.fill = patch.fill.or(self.fill); + self.cell_margins = patch.cell_margins.or(self.cell_margins); + self.header_fill = patch.header_fill.or(self.header_fill); + self.header_chr.overlay(&patch.header_chr); + self.band_fill = patch.band_fill.or(self.band_fill); + self.first_col_chr.overlay(&patch.first_col_chr); + self.total_chr.overlay(&patch.total_chr); + self.total_border_top = patch.total_border_top.or(self.total_border_top); + self.first_col_fill = patch.first_col_fill.or(self.first_col_fill); + self.first_col_borders = merge(self.first_col_borders, patch.first_col_borders); + self.last_col_fill = patch.last_col_fill.or(self.last_col_fill); + self.last_col_chr.overlay(&patch.last_col_chr); + self.last_col_borders = merge(self.last_col_borders, patch.last_col_borders); + self.total_fill = patch.total_fill.or(self.total_fill); + self.total_borders = merge(self.total_borders, patch.total_borders); + self.col_band_fill = patch.col_band_fill.or(self.col_band_fill); + self.col_band_chr.overlay(&patch.col_band_chr); + self.col_band_borders = merge(self.col_band_borders, patch.col_band_borders); + for (k, f) in &patch.regions { + self.regions.entry(*k).or_default().overlay(f); + } + } + + /// Part `p`'s formatting. The whole table's text formatting is the style's own + /// ([`Style::table_part`] adds it); its borders are the table borders. + pub fn part(&self, p: TablePart) -> PartFormat { + let extra = self.regions.get(&p).cloned().unwrap_or_default(); + let named = |chr: &CharProps, fill: Option, borders: Option| PartFormat { + chr: chr.clone(), + fill, + borders, + align: extra.align, + v_align: extra.v_align, + }; + match p { + TablePart::WholeTable => { + PartFormat { chr: CharProps::default(), fill: self.fill, borders: self.borders, align: None, v_align: extra.v_align } + } + TablePart::FirstRow => named(&self.header_chr, self.header_fill, self.header_borders), + TablePart::LastRow => { + // The built-in top rule is the region's top edge unless its borders set one. + let borders = match (self.total_borders, self.total_border_top) { + (Some(mut b), top) => { + b.top = b.top.or(top); + Some(b) + } + (None, top) => top.map(|t| Borders { top: Some(t), ..Borders::default() }), + }; + named(&self.total_chr, self.total_fill, borders) + } + TablePart::FirstCol => named(&self.first_col_chr, self.first_col_fill, self.first_col_borders), + TablePart::LastCol => named(&self.last_col_chr, self.last_col_fill, self.last_col_borders), + TablePart::Band1Vert => named(&self.col_band_chr, self.col_band_fill, self.col_band_borders), + TablePart::Band1Horz => named(&self.band_chr, self.band_fill, self.band_borders), + _ => extra, + } + } + + /// Replace part `p`'s formatting (for the whole table only its fill, borders and vertical + /// alignment: [`Style::set_table_part`] stores the rest). + pub fn set_part(&mut self, p: TablePart, f: PartFormat) { + let align_only = PartFormat { align: f.align, v_align: f.v_align, ..PartFormat::default() }; + let PartFormat { chr, fill, borders, .. } = f.clone(); + let rest = match p { + TablePart::WholeTable => { + self.fill = fill; + self.borders = borders; + PartFormat { v_align: f.v_align, ..PartFormat::default() } + } + TablePart::FirstRow => { + (self.header_chr, self.header_fill, self.header_borders) = (chr, fill, borders); + align_only + } + TablePart::LastRow => { + (self.total_chr, self.total_fill) = (chr, fill); + self.total_border_top = borders.and_then(|b| b.top); + self.total_borders = borders.map(|b| Borders { top: None, ..b }).filter(|b| *b != Borders::default()); + align_only + } + TablePart::FirstCol => { + (self.first_col_chr, self.first_col_fill, self.first_col_borders) = (chr, fill, borders); + align_only + } + TablePart::LastCol => { + (self.last_col_chr, self.last_col_fill, self.last_col_borders) = (chr, fill, borders); + align_only + } + TablePart::Band1Vert => { + (self.col_band_chr, self.col_band_fill, self.col_band_borders) = (chr, fill, borders); + align_only + } + TablePart::Band1Horz => { + (self.band_chr, self.band_fill, self.band_borders) = (chr, fill, borders); + align_only + } + _ => f, + }; + if rest.is_empty() { + self.regions.remove(&p); + } else { + self.regions.insert(p, rest); + } + } +} + +impl Style { + /// Table style part `p` as this style alone sets it (the whole table's text formatting and + /// paragraph alignment are the style's own). + pub fn table_part(&self, p: TablePart) -> PartFormat { + let mut f = self.table.as_ref().map(|t| t.part(p)).unwrap_or_default(); + if p == TablePart::WholeTable { + f.chr = self.chr.clone(); + f.align = self.para.align; + } + f + } + /// Replace part `p` of this table style. + pub fn set_table_part(&mut self, p: TablePart, f: PartFormat) { + if p == TablePart::WholeTable { + self.chr = f.chr.clone(); + self.para.align = f.align; + } + let mut t = self.table.take().unwrap_or_default(); + t.set_part(p, f); + self.table = (t != TableStyleParts::default()).then_some(t); + } +} + +impl TableStyleProps { + /// Part `p` with the based-on chain merged (see [`Style::table_part`]). + pub fn part(&self, p: TablePart) -> PartFormat { + let mut f = self.parts.part(p); + if p == TablePart::WholeTable { + f.chr = self.chr.clone(); + f.align = self.para.align; + } + f + } +} + +/// A table style with its based-on chain merged: what it gives the text in the table's cells +/// (`para`, `chr`) and the table itself (`parts`). +#[derive(Clone, Debug, PartialEq, Default)] +pub struct TableStyleProps { + pub para: ParaProps, + pub chr: CharProps, + pub parts: TableStyleParts, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct StyleSheet { + /// Document defaults (`w:docDefaults`). + pub default_chr: CharProps, + pub default_para: ParaProps, + /// Preserve an explicit `w:pPrDefault`, even when empty. Omission leaves defaults to + /// the application; nonempty `default_para` is always written after spacing edits. + pub default_para_present: bool, + pub styles: Vec