From 4a7024cd69dcc20c451fd63a0d5a07604958b52b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:01:12 -0500 Subject: [PATCH 01/31] craft-aep: flat read-only tree index and the Pard decoder (ports of effectcraft#3302 and #3319) (#948) * craft-aep: a flat read-only tree index, borrowed text documents, fewer allocations The container part of storytold/effectcraft#3302: - `TreeIndex`: the chunk tree as one flat array in depth-first order, borrowed from the input, for readers that only look. Same chunks counted against the same limits and the same errors as `parse`. An `.aepx` is indexed by `index_aepx` into one buffer. The typed view reads through either tree (`Node`, `Root`, `Project::from_index`). - Chunk parser: each list of siblings is counted, then reserved once. `.aepx`: hex decoded into an output sized once, attribute values without an entity borrowed. `write` reserves the exact output size. - Text documents are parsed into a tree that borrows from the body; the style and paragraph defaults are built once. - Layer markers and Source Text are read where they are stored instead of searched for. - Container accessors that only tests used are removed. What the importer and exporter in EffectCraft reach across the crate boundary is public here: `Root`, `Draft`, `AepxDrafts`, `index_aepx`, `TreeIndex::from_drafts`, `Project::of`, `ProjectSettings::of`, `Node::marker_list`, `Root::file_version` and `saved_by_version`, `StyleRun::SAMPLED`. `TimeBase::NTSC` holds the rates as plain numerator and denominator instead of an EffectCraft type. * craft-aep: one decoder for an effect parameter definition (`Pard`) The container part of storytold/effectcraft#3319: the fields of a `pard` record (type code, name, default, valid range, slider range, popup item count) are read by one type, `Pard`, which `EffectParam` now reads through. It is public so that an exporter's tests can read back what they encode instead of keeping a second decoder. The rest of that PR (import and export bounds, loss reports) is in EffectCraft's importer and exporter. --------- Co-authored-by: vangel76 --- crates/aep/README.md | 7 + crates/aep/docs/aep-format.md | 38 +- crates/aep/src/aepx.rs | 782 ++++++++++-- crates/aep/src/aepx/tree.rs | 362 ++++++ crates/aep/src/arbitrary.rs | 14 +- crates/aep/src/camera.rs | 89 -- crates/aep/src/effects.rs | 203 ++- crates/aep/src/footage.rs | 39 +- crates/aep/src/header.rs | 81 -- crates/aep/src/index.rs | 1026 ++++++++++++++++ crates/aep/src/keys.rs | 41 +- crates/aep/src/lib.rs | 8 +- crates/aep/src/masks.rs | 24 +- crates/aep/src/read.rs | 164 ++- crates/aep/src/settings.rs | 35 +- crates/aep/src/shapes.rs | 13 +- crates/aep/src/textdoc.rs | 1090 ++++++++++++++--- crates/aep/src/typed.rs | 126 +- crates/aep/src/values.rs | 8 +- crates/aep/src/version.rs | 34 +- crates/aep/src/write.rs | 93 ++ crates/aep/tests/it/common/mod.rs | 7 + crates/aep/tests/it/container.rs | 1 + crates/aep/tests/it/container/batch3_core.rs | 34 +- .../aep/tests/it/container/batch3_objects.rs | 101 +- crates/aep/tests/it/container/cases_600.rs | 4 +- crates/aep/tests/it/container/header.rs | 57 +- crates/aep/tests/it/container/helpers.rs | 3 +- crates/aep/tests/it/container/keyframes.rs | 11 +- .../aep/tests/it/container/layer_switches.rs | 47 +- crates/aep/tests/it/container/matte.rs | 2 +- crates/aep/tests/it/container/robustness.rs | 87 +- .../it/container/text_markers_footage.rs | 2 - crates/aep/tests/it/container/tree_index.rs | 231 ++++ crates/aep/tests/it/container/typed.rs | 8 +- crates/aep/tests/it/container/values.rs | 9 +- 36 files changed, 3960 insertions(+), 921 deletions(-) create mode 100644 crates/aep/src/aepx/tree.rs create mode 100644 crates/aep/src/index.rs create mode 100644 crates/aep/tests/it/container/tree_index.rs diff --git a/crates/aep/README.md b/crates/aep/README.md index 12ce8f9d6..95fcdd1a6 100644 --- a/crates/aep/README.md +++ b/crates/aep/README.md @@ -24,6 +24,13 @@ The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a XML tokenizer of its own); `parse_any` picks the reader by content, so the typed view takes either form. The mapping is in [`docs/aep-format.md`](docs/aep-format.md), section 1. +A reader that only looks does not need the owned tree. `TreeIndex` is the same tree as one flat, +read-only array in depth-first order, borrowed from the input: `TreeIndex::parse` for an `.aep`, +`index_aepx` and `TreeIndex::from_drafts` for an `.aepx` (decoded into one buffer first). It +counts the same chunks against the same `Limits` and returns the same errors as `parse`, and the +typed view reads through either (`Project::new` for the owned tree, `Project::from_index` for the +index). An importer should use the index; it is about twice as fast and allocates far less. + Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining input, nesting and chunk count are bounded (`Limits`), nothing is allocated from a declared size, diff --git a/crates/aep/docs/aep-format.md b/crates/aep/docs/aep-format.md index a9269bc10..3582276d7 100644 --- a/crates/aep/docs/aep-format.md +++ b/crates/aep/docs/aep-format.md @@ -121,6 +121,11 @@ counterpart (a name that is not 1 to 4 characters) are skipped with their conten with the byte offset. It runs without recursion; the nesting depth (`Limits::max_depth`) and the element count (`Limits::max_chunks`) are bounded, and everything it allocates is bounded by the size of the input. +- There are two readers. `read_aepx` builds the owned tree element by element. The importer, which + only reads, uses a second reader that decodes every body (hex, text, the alias JSON, the + numbers) into one buffer and records the chunks as a flat depth-first list whose bodies are + ranges of that buffer; it indexes that buffer (`TreeIndex`) as it indexes an `.aep`. The tests + hold both readers to the same tree and the same errors. - Tests: `tests/aepx.rs` compares the five fixture pairs chunk by chunk and byte by byte, compares the typed view of both forms, and runs truncated, bit-flipped, deeply nested and oversized inputs; `crates/aep/tests/it/import/aepx.rs` imports both forms of each pair and expects the same @@ -139,8 +144,8 @@ write on their own. |---|---| | `svap` (4 bytes, the first top-level chunk) is the version of the After Effects that **wrote** the file: `0f 11 0e 02` in all 36 (a down-saved copy keeps the 2026 word) and in all 1 192 other `.aep` files of the local corpus folders. After Effects opens a file without `svap` (`t-nosvap`). | All 36; corpus scan. | | `head` bytes 4 to 7 are the version the file is **written for**: `0f 11 0e 02` (= `svap`) in the 2026 saves, `0f 0b 06 61` in the 25.x copies, `0f 03 06 41` in the 24.x copies, the same word in every family. After Effects refuses a file whose word is zero ("created with an older version [0.0x0]", `u-zhead`). | All 36; `u-zhead`. | -| The word is packed (bit 31 first): bits 25-24 and 21-19 the major, 18-15 the minor, 14-11 the bug-fix, 10-8 a stage (6 in all), 7-0 the build; bits 31-26 (`000011`) and 23-22 (`00`) are the same in all samples and not decoded. `0f110e02` = **26.2.1x2**, which is what the saving After Effects reports for itself (`app.version` recorded by the corpus scripts) - this one sample fixes minor, bug-fix and build. The copies read as **25.6.0x97** and **24.6.0x65** (major **V** from three consecutive majors; minor, bug-fix and build of the copies **E**: one other word decides their positions). `effectcraft_aep::AeVersion`, `AepFile::file_version`, `AepFile::saved_by_version`. | Tests `decodes_the_sampled_words`, `the_container_of_a_copy_states_its_version_in_head_and_the_writer_in_svap`. | -| `head` bytes 0 to 1 (a `u16`) step with the major: 97 (`0x61`) in the 2026 saves, 96 in 25.x, 95 in 24.x; bytes 2 to 3 are 5, 9 and 6 (**?**). Bytes 8 to 15 differ between families, not between versions; bytes 16 to 19 are the save counter (one higher in each copy). Nothing is derived from bytes 0 to 3 (`AepFile::head_revision` exposes the `u16`). | All 36. | +| The word is packed (bit 31 first): bits 25-24 and 21-19 the major, 18-15 the minor, 14-11 the bug-fix, 10-8 a stage (6 in all), 7-0 the build; bits 31-26 (`000011`) and 23-22 (`00`) are the same in all samples and not decoded. `0f110e02` = **26.2.1x2**, which is what the saving After Effects reports for itself (`app.version` recorded by the corpus scripts) - this one sample fixes minor, bug-fix and build. The copies read as **25.6.0x97** and **24.6.0x65** (major **V** from three consecutive majors; minor, bug-fix and build of the copies **E**: one other word decides their positions). `craft_aep::AeVersion`, `AepFile::file_version`, `AepFile::saved_by_version`. | Tests `decodes_the_sampled_words`, `the_container_of_a_copy_states_its_version_in_head_and_the_writer_in_svap`. | +| `head` bytes 0 to 1 (a `u16`) step with the major: 97 (`0x61`) in the 2026 saves, 96 in 25.x, 95 in 24.x; bytes 2 to 3 are 5, 9 and 6 (**?**). Bytes 8 to 15 differ between families, not between versions; bytes 16 to 19 are the save counter (one higher in each copy). Nothing is derived from bytes 0 to 3, and the reader does not expose them. | All 36. | | The XMP packet after the container is no guide: its `CreatorTool` is "Adobe After Effects 2026" in the copies too. | All 36. | | **26 against 25.x** (per family, chunk tree aligned by item and layer id): besides `head`, `nhed` / `nnhd` differ in 6 bytes (`nhed` 20-23 and 26-27: `827fa930` / `0015` in the 2026 save, `00000000` / `011b` in both copies; session state, see section 4). Every **solid's** `Pin /CLRS` changes: the 2026 save has `dcui` (1 byte, 1), `apid` all `0xff` and an empty `Utf8`; the copies drop `dcui`, set `apid` to a GUID (`601e9973...`, the same in all) and embed a 935-byte JSON with an ICC profile (`baseColorProfile/colorProfileData`). A **footage** item that the 2026 save stores with an assigned profile (`mcsp` = 1 and a profile JSON) loses `mcsp` and that JSON in the copies, and its `epid` becomes all `0xff`. The rest is save noise: the footage alias `ascendcount_base` (the copies were saved in another folder), `sspc` byte 112 of a footage item (a modification time), the `oEXR` `opti` of an EXR item (the copies hold the display device name `\.\DISPLAY1` as UTF-16 at byte 126 and, in the 24.x copy, a few non-zero bytes from byte 2 102 that look like uninitialised memory; the 2026 save has zeros), the plug-ins' sequence data `sdat` of Lumetri Color (16 instances and the project's `EfdG`), the mask counter `mkif` byte 8. No chunk is added or reordered, and no `pard`, `cdat`, `ldta`, `cdta`, `tdb4`, keyframe, text or shape chunk differs. | All 12 families. | | **25.x against 24.x**: only `head` (bytes 1, 3, 5 and 7), and the session noise in `opti` and `sdat` above. | All 12 families. | @@ -218,12 +223,12 @@ the value from the corpus manifest (test file in parentheses at the end of each API: `LayerHeader` (`is_video_enabled`/`is_enabled`, `is_audio_enabled`, `is_effects_enabled`, `is_auto_oriented`, `light_type`, `blend_mode`, `stretch` / `stretch_den`, `stretch_ratio`, -`stretch_percent`, `in_point`, `out_point`); the same switches on `Layer`; `Layer::track_matte` +`in_point`, `out_point`); the same switches on `Layer`; `Layer::track_matte` (`TrackMatte`, `MatteMode`; test `tests/matte.rs`). `stretch` is now the numerator of the ratio (it was the whole multiplier before 50 % and 150 % were sampled). Tests: `tests/header.rs`, `tests/layer_switches.rs`. The layer name itself is the `Utf8` child of the `Layr` (empty while the name is the source's); the layer then shows the name of its source item -(`Project::layer_display_name`). +(`Project::layer_source(...)` then `Item::display_name`, the rule the importer uses). ### 3.2 Composition header (`Item/cdta`, 204 bytes; the decoder needs the first 176) and renderer @@ -248,9 +253,9 @@ numerator of the ratio (it was the whole multiplier before 50 % and 150 % were s The inert `ADBE Time Remapping` record of every AV layer has `tduM` = the source's duration in seconds: 5 for 5 s compositions (124, 257, 258), 3 and 2 for a 3 s and a 2 s composition (600, 601), 1 for the 1 s WAV (351), 0.1 for the three-frame sequence (353), 0 on solids and stills (000, 126). Every property's keyframe time scale (`tdb4` bytes 12 to 15) follows the composition (23 976 in every property of `L` after 229 and 230). The corpus manifest calls the classic renderer -`ADBE Advanced 3d`; the file stores `ADBE Escher` / `Classic 3D`. API: `CompHeader::frame_rate_exact` -(exact fraction), `duration_frames`, `shutter_phase`, `display_start`, `display_start_seconds`, -`display_start_frame`. Tests: `tests/header.rs`, `tests/layer_switches.rs`. +`ADBE Advanced 3d`; the file stores `ADBE Escher` / `Classic 3D`. API: `CompHeader::{time_base, +frame_rate_fixed, duration_units, shutter_phase, display_start}` (`display_start` is the exact +fraction). Tests: `tests/header.rs`, `tests/layer_switches.rs`. ### 3.3 Items, footage and precomps @@ -295,7 +300,8 @@ name. A footage layer shows the item's name while it has no name of its own (351 Test: `tests/header.rs` (precomp), `tests/text_markers_footage.rs` and `tests/batch3_objects.rs` (footage), `tests/layer_switches.rs` (solid colour). API: `Footage::{duration, native_frame_rate, -bits_per_pixel, file_size, audio_sample_rate, is_sequence, sequence_frame_count}`. +bits_per_pixel, file_size, audio_sample_rate, is_sequence}`; `Item::sequence_info` (with +`SequenceInfo::frame_count`). ### 3.4 Properties: sparse storage, static values and units @@ -404,8 +410,8 @@ out speed, at 40 out influence**). Scale key 1 of 305 (`n = 3`, in 0 / 33 %, out 15 / 66) is `0,0, .33,.33, 15,15, .66,.66`. Component `i` uses slot `i` of each block (**V** for the block layout and the units, Scale speed in fractions per second and Mask Feather speed in pixels per second; **E** that different values per dimension sit in the slot of their dimension, because -the tool that made the corpus applies one ease to all dimensions; **V** since the exporter wrote three different eases for the three Scale dimensions and After Effects read the same three back, aep-export.md R13). `Keyframe::plain_ease_slots` -returns the doubles as stored, `Keyframe::ease_component(i)` one component's `KeyEase`. Layout of +the tool that made the corpus applies one ease to all dimensions; **V** since the exporter wrote three different eases for the three Scale dimensions and After Effects read the same three back, aep-export.md R13). `Keyframe::ease_component(i)` +returns one component's `KeyEase`, `Keyframe::ease_components` all of them. Layout of the spatial template: `u32` at 8 (1 normally, 0 after the spatial auto-bezier and continuous switches were turned off together, 054), `f64` arc length of the segment to the next key at 16 (100 in 204 key 1, 0 on the last key and right after creation; safe to ignore), **ease at 24 in @@ -430,7 +436,7 @@ in tangent (`n` x `f64`, relative offset from the value) at `56 + 8 n`, out tang | **ease, colour switched to bezier** | spatial n=4 @24..55 | keyframe.set_interpolation bezier on two colour keys stores speed 0 and influence 0 on both sides, and After Effects interpolates them linearly (598 Start Color red to blue: 0.75 / 0.5 / 0.25 red at 0.5 / 1 / 1.5 s); a custom colour ease cannot be set by script | 598 | V | | **ease, mask path** | 64-byte record @24..55 | in speed, in influence, out speed, out influence as in the spatial template: 599 out / in influence 0.75, 700 0.2 / 0.5, speeds 0 (the path at 0.5 s is 5.42 % / 28.0 % of the way). The `f64` at 16 is 1 (093) or 2 (599, 700) on the first key and 0 on the last (not decoded) | 093, 599, 700 | V (influence), ? (speed unit) | -API: `Keyframe::{interpolation_in, interpolation_out, kind, value, ease, ease_component, ease_components, plain_ease_slots, color_distance, spatial}`; +API: `Keyframe::{interpolation_in, interpolation_out, kind, value, ease, ease_component, ease_components, spatial}` (the `f64` at byte 16 of a colour key is `SpatialKey::segment_length`); `Property::keys()` hands the property's template to its records. Marker record (16 bytes): time, `03 03` hold bytes, 8 stale bytes. Mask path record (64 bytes): the common fields and the ease at byte 24 (599, 700). Test: `tests/values.rs`, `tests/keyframes.rs`. @@ -767,7 +773,7 @@ pins are placed by hand or by a preset. 328), so mode and inverted are reported per mask. `mkif` byte 8 is the running mask id counter of the **layer** in the session (1, 2 for the two masks of 328; 329 then gets 3; undo does not reset it; a new layer starts at 1 again): exposed as `Mask::counter`, a session counter and not a stable id. API: -`Mask::{mode, inverted, counter}`, `Project::mask_vertices_px` (vertices in layer pixels), `Layer::mask_atoms` (`MaskAtom`: the mask with its atom group, its path property and its shapes by position, for the keys and expressions of the mask properties). Tests: +`Mask::{mode, inverted, counter}`, `ShapePath::vertices(size)` (vertices in layer pixels: `size` is the size of the layer's source, `Project::layer_source_size`; a text or shape layer has no source size and its mask boxes are already pixels, so the importer passes 1 x 1), `Layer::mask_atoms` (`MaskAtom`: the mask with its atom group, its path property and its shapes by position, for the keys and expressions of the mask properties). Tests: `tests/masks.rs` (with a synthetic check of the handle order), `tests/batch3_objects.rs`. ### 3.9 Shape layers (`tdgp[ADBE Root Vectors Group]`) @@ -895,7 +901,7 @@ groups as on other layers, with these conventions: | What | Path | Stored | Evidence | Conf. | |---|---|---|---|---| -| camera Position, default (auto state) | `ADBE Transform Group/ADBE Position`, `tdb4` 76 and 77 = 1 although no expression exists | `[0, 0, -2666.67]` (x and y are 0 whatever the composition size; z = -zoom). A **one-node** camera reports `[0, 0, -2666.67]` (251). **Correction (S16): a reopened two-node camera reports `[0, 0, -2666.67]` too (256 read back), and After Effects renders it there**: the default two-node camera of 537 above the 3D solid of 486 renders from `[0, 0, -2666.67]` towards the point of interest, not like the composition's default view (537 differs from 486 in 24 % of the grid points, and EffectCraft's import with the stored position matches 537 but for 4 edge points). The `[960, 540, ...]` of the batch-3 notes was not reproduced (535 and 537, created by script, read back the stored value too); `CameraPosition::reported` keeps the old rule and is marked as contradicted | 256, 251, 537 | V | +| camera Position, default (auto state) | `ADBE Transform Group/ADBE Position`, `tdb4` 76 and 77 = 1 although no expression exists | `[0, 0, -2666.67]` (x and y are 0 whatever the composition size; z = -zoom). A **one-node** camera reports `[0, 0, -2666.67]` (251). **Correction (S16): a reopened two-node camera reports `[0, 0, -2666.67]` too (256 read back), and After Effects renders it there**: the default two-node camera of 537 above the 3D solid of 486 renders from `[0, 0, -2666.67]` towards the point of interest, not like the composition's default view (537 differs from 486 in 24 % of the grid points, and EffectCraft's import with the stored position matches 537 but for 4 edge points). The `[960, 540, ...]` of the batch-3 notes was not reproduced (535 and 537, created by script, read back the stored value too); the reader keeps only the stored value (`CameraPosition::stored`) | 256, 251, 537 | V | | camera Position, set by the user | same, `tdb4` 76 and 77 = 0 | the absolute value `[100, 200, -1000]` | 253 | V | | camera point of interest | `ADBE Anchor Point` | `[960, 540, 0]` in **pixels** (not a fraction); the record's `tdsb` is 1 on a two-node and 3 on the one-node camera (not asserted) | 251, 253, 256 | V (value), E (`tdsb`) | | light Position, default | same as the camera | `[0, 0, -1333.33]`, as After Effects reports it (255 read back); the point of interest `[960, 540, 0]` | 254, 255 | V | @@ -911,10 +917,8 @@ groups as on other layers, with these conventions: A point light made by converting a spot light has a different property set (`Orientation`, `Rotate X/Y/Z`, `Cone Angle`, `Cone Feather 2` in the default state) that a spot light lacks (252); whether that -is an artefact of the conversion is untested. API: `Layer::camera_position`, -`CameraPosition::reported`, `Layer::camera_point_of_interest`, `Layer::light_intensity`, -`Layer::{camera_zoom, camera_depth_of_field, camera_focus_distance, camera_aperture, light_color, -light_cone_angle, light_cone_feather}`, `DEFAULT_CAMERA_ZOOM` (2 666.67 is verified for the 1920 px +is an artefact of the conversion is untested. API: `Layer::camera_position`; the camera and light +options are read by the importer's table (`import/camera.rs`); `DEFAULT_CAMERA_ZOOM` (2 666.67 is verified for the 1920 px composition only), `Layer::orientation` (`Orientation`; test `orientation_is_otst_otda` in `tests/batch5.rs`). Render checks against After Effects (20 px grid, 5 184 points): 486 1 point off, 535 2, 536 0, 537 4 (anti-aliased edges); **488 (Orientation `[10, 20, 30]`): 272 points diff --git a/crates/aep/src/aepx.rs b/crates/aep/src/aepx.rs index 9fad696ce..5267080fb 100644 --- a/crates/aep/src/aepx.rs +++ b/crates/aep/src/aepx.rs @@ -12,13 +12,23 @@ //! instructions. A document type declaration is refused (no entity expansion). It runs without //! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything //! it allocates is bounded by the size of the input. +//! +//! There are two readers. [`read_aepx`] builds the owned tree element by element +//! (`aepx/tree.rs`). The importer, which only reads, uses `index_aepx`: it decodes every body +//! (hex, text, the alias JSON, the numbers) into one buffer and records the chunks as entries of +//! the flat index ([`Draft`]s, in depth-first order, bodies as ranges of the buffer), which +//! [`crate::TreeIndex::from_drafts`] turns into an index that borrows the buffer. +//! The tests hold both to the same tree and the same errors. use std::borrow::Cow; -use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::chunk::{AepFile, LIST, NESTED_IDS}; use crate::error::AepError; +use crate::index::{Draft, Kind as EntryKind}; use crate::read::{Limits, parse}; +mod tree; + /// The root element of an `.aepx`. const ROOT: &[u8] = b"AfterEffectsProject"; /// The element that holds the XMP packet (the trailer of an `.aep`). @@ -68,17 +78,71 @@ pub fn read_aepx(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`read_aepx`]. pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + tree::read(data, limits) +} + +/// An `.aepx` read for the flat index: every body decoded into one buffer, and the chunks as +/// [`Draft`]s in depth-first order with their bodies as ranges of that buffer. +/// [`crate::TreeIndex::from_drafts`] makes the index, which borrows the buffer. +pub struct AepxDrafts { + /// The decoded bodies (hex, text, alias JSON, numbers) and the XMP packet. + pub buffer: Vec, + pub nodes: Vec, + /// The XMP packet: a range of `buffer` (empty without one). + pub trailer: (usize, usize), +} + +/// Reads an `.aepx` into [`AepxDrafts`]: the tree [`read_aepx_with_limits`] gives, with the same +/// errors, without one allocation per chunk. +/// +/// # Errors +/// See [`read_aepx_with_limits`]. When memory runs out, two errors come up in more cases than +/// there: [`AepError::TooLarge`] when the chunk list cannot grow (the owned-tree reader would +/// abort), and [`AepError::Xml`] "bdata is too large to hold in memory" also when the shared body +/// buffer cannot grow (the owned-tree reader reserves each value on its own). +pub fn index_aepx(data: &[u8], limits: Limits) -> Result { let text = match std::str::from_utf8(data) { Ok(t) => t.as_bytes(), Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), }; - Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() + // Every counted element takes at least 3 bytes (`` in a document cut short), so there are + // never more chunks than this. + let bound = (text.len() / 3).min(limits.max_chunks); + let mut nodes = Vec::new(); + let _ = nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + // Hex is most of an `.aepx` and decodes to half its length: one buffer of that size usually + // holds everything. A failed reservation is not an error here; the buffer grows as it goes. + let mut buffer = Vec::new(); + let _ = buffer.try_reserve_exact(text.len() / 2); + let reader = Reader { + data: text, + pos: 0, + limits, + count: 0, + stack: Vec::new(), + root_closed: false, + trailer: None, + buffer, + nodes, + bound, + attrs: Vec::new(), + }; + reader.run() } +/// The slots the chunk list starts with; it doubles from there (as the array of +/// [`crate::TreeIndex`] does). +const INITIAL_SLOTS: usize = 1024; + fn xml(offset: usize, what: &'static str) -> AepError { AepError::Xml { offset, what } } +/// The pad byte of a body of `len` bytes, as [`crate::Chunk::data`] sets it. +fn pad_of(len: usize) -> Option { + if len % 2 == 1 { Some(0) } else { None } +} + /// What an open element becomes when it closes. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Kind { @@ -102,13 +166,14 @@ struct Frame { at: usize, /// The name as written, to match the end tag. name: (usize, usize), - /// The decoded `bdata` attribute. - bdata: Option>, + /// The chunk's entry in the list (for `Chunk`, `Utf8` and `Alias`). + entry: Option, + /// The buffer length when the element opened: its body starts here. + mark: usize, + /// The end of the decoded `bdata` attribute in the buffer. + bdata_end: Option, /// The attributes of a `fileReference`, in document order. attrs: Vec<(String, String)>, - /// Collected text (only for `string` and the XMP packet). - text: Vec, - children: Vec>, } struct Reader<'d> { @@ -117,13 +182,19 @@ struct Reader<'d> { limits: Limits, count: usize, stack: Vec, - /// The top-level chunks once the root element has closed. - top: Option>>, - trailer: Option>, + /// True once the root element has closed. + root_closed: bool, + trailer: Option<(usize, usize)>, + buffer: Vec, + nodes: Vec, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, + /// The attributes of the current start tag; kept to reuse its allocation. + attrs: Vec<((usize, usize), Cow<'d, [u8]>)>, } -impl Reader<'_> { - fn run(mut self) -> Result, AepError> { +impl<'d> Reader<'d> { + fn run(mut self) -> Result { if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { self.pos = 3; } @@ -137,11 +208,13 @@ impl Reader<'_> { if let Some(open) = self.stack.last() { return Err(xml(open.at, "an element is not closed before the end of the file")); } - let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; - Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + if !self.root_closed { + return Err(xml(self.data.len(), "no AfterEffectsProject root element")); + } + Ok(AepxDrafts { buffer: self.buffer, nodes: self.nodes, trailer: self.trailer.unwrap_or((0, 0)) }) } - fn rest(&self) -> &[u8] { + fn rest(&self) -> &'d [u8] { self.data.get(self.pos..).unwrap_or(&[]) } @@ -181,14 +254,13 @@ impl Reader<'_> { self.pos = start; let end = self.skip_past(b"]]>", "a CDATA section is not closed")?; let content = self.data.get(start..end).unwrap_or(&[]); - let at = start; - match self.stack.last_mut() { + match self.stack.last() { Some(frame) => { if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { - frame.text.extend_from_slice(content); + self.buffer.extend_from_slice(content); } } - None => return Err(xml(at, "CDATA outside the root element")), + None => return Err(xml(start, "CDATA outside the root element")), } } else if rest.starts_with(b" { self.pos = self.pos.saturating_add(n); } - fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + fn slice(&self, (a, b): (usize, usize)) -> &'d [u8] { self.data.get(a..b).unwrap_or(&[]) } - fn start_tag(&mut self) -> Result<(), AepError> { - let at = self.pos; - self.pos = self.pos.saturating_add(1); - let name = self.name()?; - let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); - let self_closing = loop { + /// Reads the attributes of a start tag into `attrs`; true when the tag closes itself. + fn attributes(&mut self, at: usize, attrs: &mut Vec<((usize, usize), Cow<'d, [u8]>)>) -> Result { + loop { self.skip_ws(); let rest = self.rest(); if rest.starts_with(b"/>") { self.pos = self.pos.saturating_add(2); - break true; + return Ok(true); } if rest.starts_with(b">") { self.pos = self.pos.saturating_add(1); - break false; + return Ok(false); } if rest.is_empty() { return Err(xml(at, "a start tag is not closed")); @@ -259,11 +328,27 @@ impl Reader<'_> { if raw.contains(&b'<') { return Err(xml(vstart, "`<` inside an attribute value")); } - let mut value = Vec::new(); - decode_entities(raw, vstart, &mut value)?; + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; self.pos = vend.saturating_add(1); attrs.push((key, value)); - }; + } + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = std::mem::take(&mut self.attrs); + attrs.clear(); + let self_closing = self.attributes(at, &mut attrs)?; let kind = self.kind_of(name, at)?; if kind != Kind::Root { self.count = self.count.saturating_add(1); @@ -275,15 +360,27 @@ impl Reader<'_> { return Err(AepError::TooDeep); } } - let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; - if matches!(kind, Kind::Chunk(_)) { - if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { - frame.bdata = Some(decode_hex(value, key.1)?); + let mark = self.buffer.len(); + let mut frame = Frame { kind, at, name, entry: None, mark, bdata_end: None, attrs: Vec::new() }; + match kind { + Kind::Chunk(id) => { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + decode_hex_into(value, key.1, MAX_BDATA, &mut self.buffer)?; + frame.bdata_end = Some(self.buffer.len()); + } + frame.entry = Some(self.open_entry(id)?); } - } else if kind == Kind::Alias { - frame.attrs = - attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + Kind::Utf8 => frame.entry = Some(self.open_entry(*b"Utf8")?), + Kind::Alias => { + frame.attrs = attrs + .iter() + .map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())) + .collect(); + frame.entry = Some(self.open_entry(*b"alas")?); + } + Kind::Root | Kind::Xmp | Kind::Skip => {} } + self.attrs = attrs; self.stack.push(frame); if self_closing { self.close()?; @@ -291,12 +388,38 @@ impl Reader<'_> { Ok(()) } + /// Appends the entry of a chunk whose element just opened; its body is filled in when it + /// closes. The list grows by its length (at least 16) but never past [`Self::bound`]; an + /// allocation the target cannot make is an error, not an abort. + fn open_entry(&mut self, id: [u8; 4]) -> Result { + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + u32::try_from(at.checked_add(1).ok_or(AepError::TooManyChunks)?).map_err(|_| AepError::TooManyChunks)?; + if at == self.nodes.capacity() { + // Past the bound (which only a wrong guess could reach) it keeps doubling. + let room = self.bound.saturating_sub(at); + let more = if room == 0 { at.max(16) } else { at.max(16).min(room) }; + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(Draft::placeholder(id)); + Ok(at) + } + + /// Fills in the entry at `at` once its element has closed; its subtree is what follows it. + fn finish(&mut self, at: usize, id: [u8; 4], kind: EntryKind, list_type: [u8; 4], body: (usize, usize), pad: Option) -> Result<(), AepError> { + let end = u32::try_from(self.nodes.len()).map_err(|_| AepError::TooManyChunks)?; + if let Some(d) = self.nodes.get_mut(at) { + *d = Draft { start: body.0, len: body.1.saturating_sub(body.0), id, list_type, end, kind, pad }; + } + Ok(()) + } + /// Classifies an element by its name and where it is. fn kind_of(&self, name: (usize, usize), at: usize) -> Result { let n = self.slice(name); let parent = match self.stack.last() { None => { - if self.top.is_some() { + if self.root_closed { return Err(xml(at, "a second root element")); } if n != ROOT { @@ -340,39 +463,104 @@ impl Reader<'_> { self.close() } - /// Closes the innermost element and hands its chunk to the parent. + /// Closes the innermost element and fills in its chunk's entry. fn close(&mut self) -> Result<(), AepError> { let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; - let chunk = match frame.kind { - Kind::Root => { - self.top = Some(frame.children); - return Ok(()); - } - Kind::Skip => return Ok(()), + match frame.kind { + Kind::Root => self.root_closed = true, + Kind::Skip => {} Kind::Xmp => { + // Only the first packet is kept; a later one is dropped from the buffer (its + // children are skipped, so nothing else was written after it opened). if self.trailer.is_none() { - self.trailer = Some(frame.text); + self.trailer = Some((frame.mark, self.buffer.len())); + } else { + self.buffer.truncate(frame.mark); } - return Ok(()); } - Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), - Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), - Kind::Chunk(id) => chunk_of(id, frame)?, - }; - if let Some(parent) = self.stack.last_mut() { - parent.children.push(chunk); + Kind::Utf8 => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let end = self.buffer.len(); + self.finish(at, *b"Utf8", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Alias => { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + self.buffer.extend_from_slice(alias_json(&frame.attrs).as_bytes()); + let end = self.buffer.len(); + self.finish(at, *b"alas", EntryKind::Data, [0; 4], (frame.mark, end), pad_of(end.saturating_sub(frame.mark)))?; + } + Kind::Chunk(id) => self.close_chunk(id, &frame)?, } Ok(()) } + /// The chunk of a closed element named by a chunk id (what the owned reader's `chunk_of` + /// built): `bdata` wins over children, `numS` / `ppSn` hold their number, a nested id holds its + /// children (or nothing), any other name is a `LIST` of that type. + fn close_chunk(&mut self, id: [u8; 4], frame: &Frame) -> Result<(), AepError> { + let at = frame.entry.ok_or(xml(frame.at, "an element lost its chunk"))?; + let after = at.checked_add(1).ok_or(AepError::TooManyChunks)?; + if let Some(bdata_end) = frame.bdata_end { + // The children were read (and counted, and checked) but are not part of the chunk. + self.nodes.truncate(after); + self.buffer.truncate(bdata_end); + let body = (frame.mark, bdata_end); + let pad = pad_of(bdata_end.saturating_sub(frame.mark)); + if OPAQUE_LIST_TYPES.contains(&id) { + return self.finish(at, LIST, EntryKind::OpaqueList, id, body, pad); + } + return self.finish(at, id, EntryKind::Data, [0; 4], body, pad); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let missing = || xml(frame.at, "a numS or ppSn element without its number"); + // Exactly one child chunk, a `Utf8`: the entry after this one, whose subtree runs to the end. + let only = self.nodes.get(after).copied().filter(|c| usize::try_from(c.end).ok() == Some(self.nodes.len())).ok_or_else(missing)?; + if only.id != *b"Utf8" { + return Err(missing()); + } + let text = match only.kind { + EntryKind::Data | EntryKind::OpaqueList => { + only.start.checked_add(only.len).and_then(|end| self.buffer.get(only.start..end)).unwrap_or(&[]) + } + EntryKind::List | EntryKind::Nested => &[], + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body: [u8; 8]; + let bytes: &[u8] = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + body = u64::from(n as u32).to_be_bytes(); + body.get(4..).unwrap_or(&[]) + } else { + body = text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes(); + &body + }; + self.nodes.truncate(after); + self.buffer.truncate(frame.mark); + self.buffer.extend_from_slice(bytes); + let end = self.buffer.len(); + return self.finish(at, id, EntryKind::Data, [0; 4], (frame.mark, end), pad_of(bytes.len())); + } + let empty = (frame.mark, frame.mark); + if NESTED_IDS.contains(&id) { + if self.nodes.len() == after { + return self.finish(at, id, EntryKind::Data, [0; 4], empty, None); + } + return self.finish(at, id, EntryKind::Nested, [0; 4], empty, None); + } + self.finish(at, LIST, EntryKind::List, id, empty, None) + } + fn text(&mut self) -> Result<(), AepError> { let start = self.pos; let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); let end = start.saturating_add(len); self.pos = end; let raw = self.data.get(start..end).unwrap_or(&[]); - match self.stack.last_mut() { - Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + match self.stack.last() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut self.buffer), Some(_) => Ok(()), None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), None => Err(xml(start, "text outside the root element")), @@ -393,41 +581,6 @@ fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { Some(id) } -/// The chunk of a closed element named by a chunk id. -fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { - if let Some(bytes) = frame.bdata { - if OPAQUE_LIST_TYPES.contains(&id) { - let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; - return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); - } - return Ok(Chunk::data(id, bytes)); - } - // Numbers written as text: `0` and `150.000000000`. - if &id == b"numS" || &id == b"ppSn" { - let text = match frame.children.as_slice() { - [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), - _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), - }; - let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); - let body = if &id == b"numS" { - // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. - let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); - let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; - (n as u32).to_be_bytes().to_vec() - } else { - text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() - }; - return Ok(Chunk::data(id, body)); - } - if NESTED_IDS.contains(&id) { - if frame.children.is_empty() { - return Ok(Chunk::data(id, Vec::new())); - } - return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); - } - Ok(Chunk::list(id, frame.children)) -} - /// Appends `raw` with the predefined entities and character references decoded. fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let mut i = 0; @@ -465,23 +618,74 @@ fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), A Ok(()) } -/// Decodes a `bdata` value (hex digits, two per byte). +/// Decodes a `bdata` value (hex digits, two per byte) into a new `Vec` of exactly its size. fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { - if text.len() % 2 == 1 { - return Err(xml(offset, "bdata has an odd number of hex digits")); - } + decode_hex_at_most(text, offset, MAX_BDATA) +} + +/// The most bytes one `bdata` may decode to: what the 32-bit size field of a chunk can hold (a +/// larger body could not be written back as an `.aep` either). +const MAX_BDATA: usize = u32::MAX as usize; + +/// [`decode_hex`] with the decoded-size limit as a parameter (so tests can reach it). +fn decode_hex_at_most(text: &[u8], offset: usize, max: usize) -> Result, AepError> { + // Checked before anything is reserved, then sized once (collecting into `Option` has no + // size hint and regrows step by step), and without aborting when the memory is not there. + let pairs = check_hex(text, offset, max)?; + let mut out = Vec::new(); + out.try_reserve_exact(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, &mut out)?; + Ok(out) +} + +/// Appends the bytes of a `bdata` value (hex digits, two per byte, at most `max` bytes) to `out`. +/// On an error `out` is unchanged. +fn decode_hex_into(text: &[u8], offset: usize, max: usize, out: &mut Vec) -> Result<(), AepError> { + let pairs = check_hex(text, offset, max)?; + // Grown at most once per value (amortised over a shared buffer), and without aborting when the + // memory is not there. + out.try_reserve(pairs).map_err(|_| xml(offset, "bdata is too large to hold in memory"))?; + decode_checked(text, offset, out) +} + +/// Appends the bytes of a `bdata` value that [`check_hex`] accepted. On an error (a digit the +/// check let through, which it does not) `out` is unchanged. +fn decode_checked(text: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { let digit = |b: u8| match b { b'0'..=b'9' => Some(b - b'0'), b'a'..=b'f' => Some(b - b'a' + 10), b'A'..=b'F' => Some(b - b'A' + 10), _ => None, }; - let (pairs, _) = text.as_chunks::<2>(); - pairs - .iter() - .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) - .collect::>>() - .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + let mark = out.len(); + for [hi, lo] in text.as_chunks::<2>().0 { + match (digit(*hi), digit(*lo)) { + (Some(h), Some(l)) => out.push((h << 4) | l), + _ => { + out.truncate(mark); + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + } + } + Ok(()) +} + +/// Checks a `bdata` value before anything is reserved for it: an even number of hex digits, at +/// most `max` bytes. Returns the number of bytes it decodes to. +fn check_hex(text: &[u8], offset: usize, max: usize) -> Result { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let pairs = text.len() / 2; + if pairs > max { + return Err(AepError::TooLarge); + } + // One cheap pass checks every digit first. Blocks without an early exit inside, so the check + // vectorises; it stops at the first bad block. + if !text.chunks(4096).all(|block| block.iter().fold(true, |ok, b| ok & b.is_ascii_hexdigit())) { + return Err(xml(offset, "bdata holds a character that is not a hex digit")); + } + Ok(pairs) } /// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document @@ -529,6 +733,7 @@ fn json_str(out: &mut String, s: &str) { #[cfg(test)] mod tests { use super::*; + use crate::index::TreeIndex; fn doc(body: &str) -> String { format!("\n{body}\n") @@ -629,6 +834,92 @@ mod tests { assert!(matches!(read_aepx(b"\xff"), Err(AepError::Xml { .. }))); } + /// Attribute values decode entities only where they have them. + #[test] + fn attribute_values_decode_entities_only_where_they_have_them() { + // An entity inside `bdata` still decodes before the hex does. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x01, 0xab][..])); + // A bad entity in any attribute, even of an element that is skipped, is still an error. + for bad in ["", "", ""] { + assert!(matches!(read_aepx(doc(bad).as_bytes()), Err(AepError::Xml { .. })), "{bad}"); + } + // Without `&` the value is taken as written. + let f = read_aepx(doc("").as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&[0x0a, 0xff][..])); + } + + /// A large bdata with a bad digit is refused before it is sized. + #[test] + fn a_large_bdata_with_a_bad_digit_is_refused_before_it_is_sized() { + // 10 MB of hex that is bad from the first pair (and once more at the very end). + let mut text = b"zz".to_vec(); + text.extend(std::iter::repeat_n(*b"00", 5_000_000).flatten()); + assert_eq!(decode_hex(&text, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let mut late = vec![b'0'; 10_000_000]; + late.extend_from_slice(b"0g"); + assert_eq!(decode_hex(&late, 3), Err(xml(3, "bdata holds a character that is not a hex digit"))); + let x = doc(&format!("", std::str::from_utf8(&text).unwrap())); + assert!(matches!(read_aepx(x.as_bytes()), Err(AepError::Xml { .. }))); + } + + /// Bdata decodes up to the size limit and not past it. + #[test] + fn bdata_decodes_up_to_the_size_limit_and_not_past_it() { + assert_eq!(decode_hex_at_most(b"00ff10", 0, 3), Ok(vec![0x00, 0xff, 0x10])); + assert_eq!(decode_hex_at_most(b"00ff10", 0, 2), Err(AepError::TooLarge)); + assert_eq!(decode_hex_at_most(b"", 0, 0), Ok(Vec::new())); + assert_eq!(decode_hex_at_most(b"00", 0, 0), Err(AepError::TooLarge)); + // The odd-length error still comes first. + assert_eq!(decode_hex_at_most(b"001", 0, 0), Err(xml(0, "bdata has an odd number of hex digits"))); + // The real limit is what a chunk's 32-bit size field holds. + assert_eq!(MAX_BDATA as u64, u64::from(u32::MAX)); + } + + /// Decode hex matches the collecting version. + #[test] + fn decode_hex_matches_the_collecting_version() { + /// The collecting decoder this one replaced, as the reference. + fn old(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) + } + let alphabet = b"0123456789abcdefABCDEFgG \0"; + let mut seed = 0x2545_f491_4f6c_dd1du64; + for len in 0..200usize { + for _ in 0..20 { + let text: Vec = (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + // Mostly hex digits, now and then a byte that is not one. + let i = (seed % 1000) as usize; + alphabet[if i < 990 { i % 22 } else { 22 + i % 4 }] + }) + .collect(); + let new = decode_hex(&text, 7); + assert_eq!(new, old(&text, 7)); + if let Ok(v) = new { + assert_eq!(v.capacity(), v.len()); + } + } + } + } + #[test] fn limits_hold() { let deep = format!("{}{}", "".repeat(100_000), "".repeat(100_000)); @@ -638,4 +929,293 @@ mod tests { let many = format!("{}", "".repeat(10)); assert_eq!(read_aepx_with_limits(many.as_bytes(), Limits { max_depth: 64, max_chunks: 9 }), Err(AepError::TooManyChunks)); } + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number. + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + if n == 0 { 0 } else { usize::try_from(self.next() % n as u64).unwrap() } + } + + /// One of `items`. + fn pick<'s>(&mut self, items: &[&'s str]) -> &'s str { + items[self.below(items.len())] + } + } + + /// `n` random bytes as hex, upper or lower case. + fn hex(rng: &mut Rng, n: usize) -> String { + let upper = rng.below(4) == 0; + (0..n).map(|_| if upper { format!("{:02X}", rng.below(256)) } else { format!("{:02x}", rng.below(256)) }).collect() + } + + /// Random `.aepx` content: every element form the reader knows (data, lists, nested ids, + /// strings, numbers, `btdk`, file references, skipped elements, XMP packets), `bdata` next to + /// children, bodies that read as chunk sequences, entities, CDATA, comments and stray text. + fn content(rng: &mut Rng, depth: usize, budget: &mut usize, out: &mut String) { + for _ in 0..rng.below(6) { + if *budget == 0 { + return; + } + *budget -= 1; + match rng.below(20) { + 0..=3 => { + let id = rng.pick(&["cdat", "tdmn", "head", "Utf8", "idta", "LIST", "a"]); + let n = rng.below(9); + out.push_str(&format!("<{id} bdata=\"{}\"/>", hex(rng, n))); + } + 4..=6 if depth < 10 => { + let id = rng.pick(&["Fold", "tdgp", "Layr", "Pin", "LIST", "btdk", "sspc"]); + let attr = if rng.below(5) == 0 { format!(" bdata=\"{}\"", hex(rng, 3)) } else { String::new() }; + out.push_str(&format!("<{id}{attr}>")); + content(rng, depth + 1, budget, out); + out.push_str(&format!("")); + } + 7 => { + let id = rng.pick(&["tdsn", "pdnm", "fnam"]); + match rng.below(4) { + 0 => out.push_str(&format!("<{id}/>")), + 1 => out.push_str(&format!("<{id}>name & A")), + // A body that is itself a chunk sequence (`Utf8`, 1 byte, pad): still data. + 2 => out.push_str(&format!("<{id} bdata=\"557466380000000161{}\"/>", if rng.below(2) == 0 { "00" } else { "" })), + _ => out.push_str(&format!("<{id}>ab")), + } + } + 8 => { + let text = + rng.pick(&["plain", "A < B", "", "ab", "", "été", "xgoney", "\r\n"]); + out.push_str(&format!("{text}")); + } + 9 => { + let id = rng.pick(&["numS", "ppSn"]); + let inner = rng.pick(&[ + "0", + " 7 ", + "-2147483648", + "150.000000000", + "x", + "", + "12", + "", + "", + "34", + ]); + let attr = if rng.below(6) == 0 { " bdata=\"00000002\"" } else { "" }; + out.push_str(&format!("<{id}{attr}>{inner}")); + } + 10 => { + let n = rng.pick(&["", "41", "414243", "4c495354000000000000"]); + out.push_str(&format!("")); + } + 11 => { + let platform = rng.pick(&["Win", "1", "Mac"]); + out.push_str(&format!( + "", + rng.below(3) + )); + } + 12 if depth < 10 => { + out.push_str("text"); + content(rng, depth + 1, budget, out); + out.push_str(""); + } + 13 => { + let packet = rng.pick(&["x]]>", "plain & text", ""]); + out.push_str(&format!("{packet}")); + } + 14 => out.push_str(rng.pick(&["", "", "\n ", "stray"])), + _ => { + let n = rng.below(5); + out.push_str(&format!("", hex(rng, n))); + } + } + } + } + + /// The default limits and six tight ones (depth 0, 1 and 3; 0, 7 and 40 chunks). + fn limit_sets() -> [Limits; 7] { + [ + Limits { max_depth: 0, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 0 }, + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index reader and the owned-tree reader give the same `Result` (the tree, or the error), + /// and the index walks like the index of the owned tree. + fn same(bytes: &[u8], limits: Limits) { + let old = read_aepx_with_limits(bytes, limits); + let new = index_aepx(bytes, limits).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(new, old, "{}", String::from_utf8_lossy(bytes)); + if let Ok(file) = &old { + let AepxDrafts { buffer, nodes, trailer } = index_aepx(bytes, limits).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let expected = TreeIndex::from_file(file).unwrap(); + assert_eq!(index.len(), expected.len()); + assert_eq!(index.trailer(), expected.trailer()); + let a: Vec = index.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + let b: Vec = expected.walk().map(|(d, n)| format!("{d} {n:?}")).collect(); + assert_eq!(a, b); + } + } + + /// The two readers give the same tree and errors on generated and damaged documents. + #[test] + fn the_two_readers_agree_on_generated_and_damaged_documents() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let noise = b"<>/=\"'&;#x0aZ \n![]-?"; + for _ in 0..600 { + let mut budget = 80; + let mut body = String::new(); + content(&mut rng, 0, &mut budget, &mut body); + let x = doc(&body); + for l in limit_sets() { + same(x.as_bytes(), l); + } + for _ in 0..4 { + let mut bad = x.clone().into_bytes(); + for _ in 0..1 + rng.below(3) { + let at = rng.below(bad.len()); + bad[at] = noise[rng.below(noise.len())]; + } + for l in limit_sets() { + same(&bad, l); + } + } + let cut = rng.below(x.len()); + same(&x.as_bytes()[..cut], Limits::default()); + // Bytes that are not UTF-8, and a multi-byte character cut in half. + let mut bad = x.clone().into_bytes(); + let at = rng.below(bad.len()); + bad.insert(at, 0xff); + same(&bad, Limits::default()); + let open = format!("{}é", x.trim_end().trim_end_matches("")).into_bytes(); + same(&open[..open.len() - 1], Limits::default()); + } + } + + /// The flat chunk list never reserves more slots than the input or the chunk limit allows. + #[test] + fn the_chunk_list_is_bounded_by_the_input_and_the_limit() { + let many = format!("{}", "".repeat(10_000)); + let d = index_aepx(many.as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.nodes.len(), 10_000); + assert!(d.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", d.nodes.capacity()); + // The buffer is reserved once, at half the input, and holds nothing here. + assert!(d.buffer.is_empty() && d.buffer.capacity() <= many.len() / 2); + // At the limit the list never holds more slots than the limit. + let d = index_aepx(many.as_bytes(), Limits { max_depth: 64, max_chunks: 10_000 }).unwrap(); + assert!(d.nodes.capacity() <= 10_000); + let few = format!("{}", "".repeat(100)); + let d = index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 100 }).unwrap(); + assert!(d.nodes.capacity() <= 100, "{} slots for at most 100 chunks", d.nodes.capacity()); + assert_eq!(index_aepx(few.as_bytes(), Limits { max_depth: 64, max_chunks: 99 }).err(), Some(AepError::TooManyChunks)); + // A tiny input cannot hold more than a third of its bytes in elements. + let tiny = doc(""); + assert!(index_aepx(tiny.as_bytes(), Limits::default()).unwrap().nodes.capacity() <= tiny.len() / 3); + } + + /// What a closed element drops is gone from the buffer and the chunk list, but was counted. + #[test] + fn dropped_content_leaves_nothing_behind_but_still_counts() { + // `bdata` wins over the children: they are read, counted and dropped. + let x = doc("xyzq"); + let d = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.nodes.len()), (&[1u8][..], 1)); + assert_eq!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 4 }).err(), Some(AepError::TooManyChunks)); + assert!(index_aepx(x.as_bytes(), Limits { max_depth: 64, max_chunks: 5 }).is_ok()); + // A child's error is raised although the child would be dropped. + assert!(matches!(index_aepx(doc("").as_bytes(), Limits::default()), Err(AepError::Xml { .. }))); + // Only the first XMP packet stays in the buffer. + let packets = doc(&"packet".repeat(1000)); + let d = index_aepx(packets.as_bytes(), Limits::default()).unwrap(); + assert_eq!((d.buffer.as_slice(), d.trailer), (&b"packet"[..], (0, 6))); + // A number keeps its 4 or 8 bytes, not its text. + let d = index_aepx(doc(" 7 1.5").as_bytes(), Limits::default()).unwrap(); + assert_eq!(d.buffer, [&[0, 0, 0, 7][..], &1.5f64.to_be_bytes()[..]].concat()); + assert_eq!(d.nodes.len(), 2); + } + + /// Bodies keep what the element says they are, even where reading the same bytes as an `.aep` + /// would decide otherwise: a nested id whose `bdata` is a chunk sequence stays data, an empty + /// `btdk` stays an opaque list, an empty list at the deepest level stays a list. + #[test] + fn bodies_keep_the_kind_the_element_gives_them() { + let x = doc(""); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(x.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + let top: Vec<_> = index.chunks().collect(); + assert_eq!(top.len(), 3); + assert_eq!(top[0].bytes().map(<[u8]>::len), Some(11)); + assert!(top[0].children().next().is_none()); + assert!(top[1].is_opaque_list() && top[1].bytes() == Some(&[][..])); + assert!(top[2].is_opaque_list() && top[2].bytes().map(<[u8]>::len) == Some(12)); + // 64 levels: the innermost (empty) list is a list, not opaque. + let deep = format!("{}{}", "".repeat(64), "".repeat(64)); + let AepxDrafts { buffer, nodes, trailer } = index_aepx(deep.as_bytes(), Limits::default()).unwrap(); + let index = TreeIndex::from_drafts(&buffer, nodes, trailer); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().all(|(_, n)| n.is_list(b"Fold") && !n.is_opaque_list())); + same(x.as_bytes(), Limits::default()); + same(deep.as_bytes(), Limits::default()); + } + + /// Hostile input is refused by both readers with the same errors. + #[test] + fn hostile_input_is_refused_alike_by_both_readers() { + let hostile = [ + format!("{}{}", "".repeat(100_000), "".repeat(100_000)), + format!("{}", "".repeat(1000)), + doc(&format!("", "00".repeat(100_000))), + doc(&format!("", "00".repeat(100_000))), + doc("4294967296"), + doc("1e400x"), + doc("�"), + doc("�"), + doc(&format!("{}", "".repeat(1000))), + format!("{}", "".repeat(10)), + " = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aepx")).collect(); + paths.sort(); + for path in paths { + let bytes = std::fs::read(&path).unwrap(); + for l in limit_sets() { + let index = index_aepx(&bytes, l).map(|d| TreeIndex::from_drafts(&d.buffer, d.nodes, d.trailer).to_file().into_owned()); + assert_eq!(index, read_aepx_with_limits(&bytes, l), "{}", path.display()); + } + } + } } diff --git a/crates/aep/src/aepx/tree.rs b/crates/aep/src/aepx/tree.rs new file mode 100644 index 000000000..6c0bc8435 --- /dev/null +++ b/crates/aep/src/aepx/tree.rs @@ -0,0 +1,362 @@ +//! The `.aepx` reader of the owned chunk tree ([`super::read_aepx`]): it builds the tree element by +//! element. The importer uses the index reader in `aepx.rs` instead; the tests compare the two on +//! generated, damaged and corpus input (the same tree and the same errors). + +use std::borrow::Cow; + +use super::{FILE_REFERENCE, OPAQUE_LIST_TYPES, ROOT, STRING, XMP, alias_json, chunk_id, decode_entities, decode_hex, xml}; +use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::error::AepError; +use crate::read::Limits; + +/// [`super::read_aepx_with_limits`]. +pub(super) fn read(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + // A value without `&` is its own decoding (most of them, every `bdata` After Effects + // writes): borrow it instead of copying it. + let value = if raw.contains(&b'&') { + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + Cow::Owned(value) + } else { + Cow::Borrowed(raw) + }; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = + attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} diff --git a/crates/aep/src/arbitrary.rs b/crates/aep/src/arbitrary.rs index 45768ae44..70627139b 100644 --- a/crates/aep/src/arbitrary.rs +++ b/crates/aep/src/arbitrary.rs @@ -7,9 +7,9 @@ //! Hue/Saturation (444-446) or a Levels value (512-516) changes only bytes of this `aRbp` (and stale //! UI bytes of `pard`); the visible parameters of the same names keep no record. -use crate::Chunk; use crate::bytes::{be_f32, be_i32, be_u16, be_u32}; use crate::effects::Effect; +use crate::index::Node; impl<'f, 'a> Effect<'f, 'a> { /// The arbitrary data stored for the parameter with this match name (`-NNNN`): the @@ -17,13 +17,13 @@ impl<'f, 'a> Effect<'f, 'a> { /// none. pub fn arbitrary_data(&self, match_name: &str) -> Option<&'f [u8]> { let values = self.chunk().child_list(b"tdgp")?; - let kids = values.children(); - let at = kids.iter().position(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; - // The value record, then the data list (no match name of its own). - let rest = kids.get(at.checked_add(1)?..)?; - let mut it = rest.iter().take(3); + let mut kids = values.children(); + // Up to the parameter's match name, then the value record, then the data list (no match + // name of its own). + kids.find(|c| c.is(b"tdmn") && c.bytes().is_some_and(|b| name_is(b, match_name)))?; + let mut it = kids.take(3); let _record = it.next().filter(|c| c.is_list(b"tdbs"))?; - it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Chunk::bytes) + it.find(|c| c.is_list(b"aRbs")).and_then(|l| l.child(b"aRbp")).and_then(Node::bytes) } } diff --git a/crates/aep/src/camera.rs b/crates/aep/src/camera.rs index 69cd8c2c8..934f36bd6 100644 --- a/crates/aep/src/camera.rs +++ b/crates/aep/src/camera.rs @@ -16,7 +16,6 @@ //! only when it is not the default 100. use crate::typed::{Layer, PropNode}; -use crate::values::{Tracked, ValueSource, read, read_opt}; /// The Position of a camera or light layer as stored. #[derive(Debug, Clone, Copy, PartialEq)] @@ -28,16 +27,6 @@ pub struct CameraPosition { pub auto_state: bool, } -impl CameraPosition { - /// The position as the batch-3 notes had After Effects report it: in the auto state of a - /// two-node camera (`two_node`) x and y are the composition centre; otherwise the stored - /// value. **Contradicted by 256 and 537 (session S16): After Effects reports and renders the - /// stored value**; use `stored` for what After Effects shows. - pub fn reported(&self, comp_width: u32, comp_height: u32, two_node: bool) -> [f64; 3] { - if self.auto_state && two_node { [f64::from(comp_width) / 2.0, f64::from(comp_height) / 2.0, self.stored[2]] } else { self.stored } - } -} - impl<'f, 'a> Layer<'f, 'a> { /// The Position record of a camera or light layer. pub fn camera_position(&self) -> Option { @@ -49,87 +38,9 @@ impl<'f, 'a> Layer<'f, 'a> { let derived = h.get(76) == Some(&1) && h.get(77) == Some(&1) && position.expression().is_none(); Some(CameraPosition { stored, auto_state: derived }) } - - /// The point of interest of a camera in pixels (its Anchor Point record); `None` when absent. - pub fn camera_point_of_interest(&self) -> Option<[f64; 3]> { - let Some(PropNode::Group(transform)) = self.properties()?.get("ADBE Transform Group") else { return None }; - let tracked = read(&transform, "ADBE Anchor Point", [0.0; 3], 1.0); - matches!(tracked.source, ValueSource::Stored | ValueSource::Keyframed).then_some(tracked.value) - } - - /// A light's intensity in percent; the default 100 when the property is absent. - pub fn light_intensity(&self) -> Option> { - let root = self.properties()?; - let Some(PropNode::Group(options)) = root.get("ADBE Light Options Group") else { return None }; - let t = read(&options, "ADBE Light Intensity", [100.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } } /// The camera zoom After Effects uses when `ADBE Camera Zoom` is absent: 2666.67 px for the 50 mm /// default camera in the 1920 px wide corpus composition. Whether it follows other composition /// widths was not sampled. pub const DEFAULT_CAMERA_ZOOM: f64 = 2_666.666_666_666_666_5; - -impl<'f, 'a> Layer<'f, 'a> { - fn camera_options(&self) -> Option> { - match self.properties()?.get("ADBE Camera Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - fn light_options(&self) -> Option> { - match self.properties()?.get("ADBE Light Options Group") { - Some(PropNode::Group(g)) => Some(g), - _ => None, - } - } - - /// A camera's zoom in pixels (`ADBE Camera Options Group/ADBE Camera Zoom`); absent = the - /// default, [`DEFAULT_CAMERA_ZOOM`] (341: 2000 is stored). The auto Position keeps `-2666.67` and - /// is not recomputed from the zoom. - pub fn camera_zoom(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Zoom", [DEFAULT_CAMERA_ZOOM], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// The depth of field switch (`ADBE Camera Depth of Field`, a checkbox; off by default, 342). - pub fn camera_depth_of_field(&self) -> Option> { - let t = read(&self.camera_options()?, "ADBE Camera Depth of Field", [0.0], 1.0); - Some(Tracked { value: t.value[0] != 0.0, source: t.source }) - } - - /// A camera's focus distance in pixels (`ADBE Camera Focus Distance`); `None` while absent (its - /// default is not known). - pub fn camera_focus_distance(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Focus Distance", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A camera's aperture (`ADBE Camera Aperture`); `None` while absent (its default is not known). - pub fn camera_aperture(&self) -> Option> { - let t = read_opt::<1>(&self.camera_options()?, "ADBE Camera Aperture", 1.0)?; - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A light's colour as `[A, R, G, B]` fractions (`ADBE Light Options Group/ADBE Light Color`, - /// stored as doubles 0..255); white when absent (343 stores red). - pub fn light_color(&self) -> Option> { - Some(read(&self.light_options()?, "ADBE Light Color", [1.0; 4], 1.0 / 255.0)) - } - - /// A spot light's cone angle in degrees (`ADBE Light Cone Angle`); 90 when absent (344 stores 60). - /// A point light stores 90 with `tdsb` 3. - pub fn light_cone_angle(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Angle", [90.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } - - /// A spot light's cone feather in percent (`ADBE Light Cone Feather 2`); 50 when absent (345 - /// stores 30). - pub fn light_cone_feather(&self) -> Option> { - let t = read(&self.light_options()?, "ADBE Light Cone Feather 2", [50.0], 1.0); - Some(Tracked { value: t.value[0], source: t.source }) - } -} diff --git a/crates/aep/src/effects.rs b/crates/aep/src/effects.rs index 25b7051ab..65f3c6125 100644 --- a/crates/aep/src/effects.rs +++ b/crates/aep/src/effects.rs @@ -20,8 +20,8 @@ use std::collections::{HashMap, HashSet}; -use crate::Chunk; use crate::bytes::{array, be_f32, be_f64, be_i32, be_u16, be_u32, cstr, text, until_nul}; +use crate::index::Node; use crate::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; /// The parameter type, the `u32` at byte 12 of `pard`. The numbering matches the plug-in @@ -132,7 +132,7 @@ pub enum LayerRef { #[derive(Debug, Clone)] pub struct Effect<'f, 'a> { match_name: String, - chunk: &'f Chunk<'a>, + chunk: Node<'f, 'a>, } /// Where the parameter definitions of an effect instance come from. @@ -178,13 +178,13 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The underlying `LIST sspc` chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } /// The instance name shown in the effect controls (`fnam`). pub fn name(&self) -> Option { - self.chunk.child(b"fnam")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text) + self.chunk.child(b"fnam")?.child(b"Utf8")?.bytes().map(text) } /// The name the user gave the instance: the `tdsn` of the instance's value group when it is @@ -192,7 +192,7 @@ impl<'f, 'a> Effect<'f, 'a> { /// `None` while the instance shows the effect's name. pub fn user_name(&self) -> Option { let values = self.chunk.child_list(b"tdgp")?; - let name = values.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; + let name = values.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (name != "-_0_/-" && !name.is_empty()).then_some(name) } @@ -250,7 +250,7 @@ impl<'f, 'a> Effect<'f, 'a> { } /// The parameters described by `part` (a `LIST parT`) with the values of this instance. - fn params_from(&self, part: &'f Chunk<'a>, borrowed: bool) -> Vec> { + fn params_from(&self, part: Node<'f, 'a>, borrowed: bool) -> Vec> { let mut out: Vec> = Vec::new(); // The stored values by match name: the first entry of a name, as `PropGroup::get` finds it // (a group of that name means no stored value). Indexed once: a lookup per parameter @@ -271,8 +271,8 @@ impl<'f, 'a> Effect<'f, 'a> { out.push(EffectParam { index: out.len(), match_name, - kind: ParamKind::from_u32(be_u32(pard, 12).unwrap_or(u32::MAX)), - name: cstr(pard, 16, 32).unwrap_or_default(), + kind: Pard(pard).kind(), + name: Pard(pard).name().unwrap_or_default(), items: None, borrowed_definition: borrowed, pard, @@ -280,7 +280,7 @@ impl<'f, 'a> Effect<'f, 'a> { }); } else if c.is(b"pdnm") && let Some(last) = out.last_mut() - && let Some(u) = c.children().iter().find(|u| u.is(b"Utf8")) + && let Some(u) = c.child(b"Utf8") && let Some(b) = u.bytes() // An empty `pdnm` names nothing (some writers store one: the effect // supplies its items). @@ -293,8 +293,9 @@ impl<'f, 'a> Effect<'f, 'a> { } } -fn has_pard(part: &Chunk<'_>) -> bool { - part.children().iter().any(|c| c.is(b"pard")) +/// True when a `LIST parT` holds parameter definitions (a `pard`). +fn has_pard(part: Node<'_, '_>) -> bool { + part.children().any(|c| c.is(b"pard")) } impl<'f, 'a> Project<'f, 'a> { @@ -302,18 +303,18 @@ impl<'f, 'a> Project<'f, 'a> { /// name and the `LIST parT` of the copied instance, in file order (the stack order of the /// types, 365/366). The `EfDC` count before them is little-endian and is not needed to read /// them. A type whose copy has no definitions is left out. - pub fn effect_definitions(&self) -> Vec<(String, &'f Chunk<'a>)> { - let mut out: Vec<(String, &'f Chunk<'a>)> = Vec::new(); + pub fn effect_definitions(&self) -> Vec<(String, Node<'f, 'a>)> { + let mut out: Vec<(String, Node<'f, 'a>)> = Vec::new(); let Some(defs) = self.effect_defs else { return out }; let mut seen: HashSet = HashSet::new(); - for df in defs.children().iter().filter(|c| c.is_list(b"EfDf")) { + for df in defs.children().filter(|c| c.is_list(b"EfDf")) { let mut name: Option = None; for c in df.children() { if c.is(b"tdmn") { name = c.bytes().map(|b| text(until_nul(b))); } else if c.is_list(b"sspc") && let Some(n) = name.take() - && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(p)) + && let Some(part) = c.child_list(b"parT").filter(|p| has_pard(*p)) && seen.insert(n.clone()) { out.push((n, part)); @@ -328,8 +329,8 @@ impl<'f, 'a> Project<'f, 'a> { /// For a type without a project-level copy it also keeps the own `parT` of the first instance /// in the file (items, layers and effects in file order) that has one. pub fn effect_definition_index(&self) -> EffectDefinitions<'f, 'a> { - let by_name: HashMap> = self.effect_definitions().into_iter().collect(); - let mut instances: HashMap> = HashMap::new(); + let by_name: HashMap> = self.effect_definitions().into_iter().collect(); + let mut instances: HashMap> = HashMap::new(); for item in self.all_items() { let Some(comp) = item.comp() else { continue }; for layer in comp.layers() { @@ -337,7 +338,7 @@ impl<'f, 'a> Project<'f, 'a> { if by_name.contains_key(&fx.match_name) || instances.contains_key(&fx.match_name) { continue; } - if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(p)) { + if let Some(part) = fx.chunk.child_list(b"parT").filter(|p| has_pard(*p)) { instances.insert(fx.match_name, part); } } @@ -363,9 +364,9 @@ impl<'f, 'a> Project<'f, 'a> { #[derive(Debug, Clone, Default)] pub struct EffectDefinitions<'f, 'a> { /// The project-level copies (`EfdG`). - by_name: HashMap>, + by_name: HashMap>, /// For a type without a project-level copy: the own `parT` of the first instance that has one. - instances: HashMap>, + instances: HashMap>, } impl<'f, 'a> EffectDefinitions<'f, 'a> { @@ -392,7 +393,7 @@ impl<'f, 'a> EffectDefinitions<'f, 'a> { return effect.params(); } match self.by_name.get(effect.match_name()).or_else(|| self.instances.get(effect.match_name())) { - Some(part) => effect.params_from(part, true), + Some(part) => effect.params_from(*part, true), None => Vec::new(), } } @@ -484,26 +485,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { pub fn pard_default(&self) -> Option { match self.kind { ParamKind::Popup if self.borrowed_definition => None, - ParamKind::FloatSlider => be_f64(self.pard, 56).map(ParamValue::Number), - ParamKind::FixedSlider | ParamKind::Angle => be_i32(self.pard, 56).map(|v| ParamValue::Number(f64::from(v) / 65536.0)), - ParamKind::IntSlider => be_i32(self.pard, 56).map(|v| ParamValue::Number(f64::from(v))), - ParamKind::Checkbox => be_u32(self.pard, 56).map(|v| ParamValue::Bool(v != 0)), - ParamKind::Popup => be_u32(self.pard, 56).map(ParamValue::Index), - ParamKind::Point => { - let p = [f64::from(be_i32(self.pard, 56)?) / 65536.0, f64::from(be_i32(self.pard, 60)?) / 65536.0]; - // A fresh instance stores fractions of the layer (0.5, 0.5); After Effects' save of - // a project it opened stores percent (50, 50: the re-saves of read-back tests e1 and - // e2, where Gradient Ramp's Start and End and every other point default came back - // as 50 / 0 and 50 / 100). Point - // defaults lie inside the layer, so a component beyond 1 tells percent. - let percent = p.iter().any(|c| c.abs() > 1.0 + 1e-9); - Some(ParamValue::Point(if percent { p.map(|c| c / 100.0) } else { p })) - } - ParamKind::Color => { - let [a, r, g, b] = array::<4>(self.pard, 56)?.map(|c| f64::from(c) / 255.0); - Some(ParamValue::Argb([a, r, g, b])) - } - _ => None, + kind => Pard(self.pard).default(kind), } } @@ -557,22 +539,14 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// slider differs per parameter (Amount 37.5 % is stored 37.5, Blend 25 % is stored 0.25), so /// this is what tells a reader the scale. `None` for other types. pub fn valid_max(&self) -> Option { - match self.kind { - ParamKind::FixedSlider => be_i32(self.pard, 128).map(|v| f64::from(v) / 65536.0), - ParamKind::FloatSlider => be_f32(self.pard, 108).map(f64::from), - ParamKind::IntSlider => be_i32(self.pard, 128).map(f64::from), - _ => None, - } + Pard(self.pard).valid_max(self.kind) } /// The smallest valid value of a float slider, the `f32` at byte 104 of `pard` (Exposure /// -100, Offset -2, Gamma Correction 0.1: 389). `None` for other types (a fixed slider's /// minimum is not decoded). pub fn valid_min(&self) -> Option { - match self.kind { - ParamKind::FloatSlider => be_f32(self.pard, 104).map(f64::from), - _ => None, - } + Pard(self.pard).valid_min(self.kind) } /// The range of a float slider's slider (what the control's drag covers, inside the valid @@ -581,7 +555,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { /// for other types. pub fn slider_range(&self) -> Option<(f64, f64)> { match self.kind { - ParamKind::FloatSlider => Some((f64::from(be_f32(self.pard, 112)?), f64::from(be_f32(self.pard, 116)?))), + ParamKind::FloatSlider => Pard(self.pard).slider_range(self.kind), _ => None, } } @@ -591,7 +565,7 @@ impl<'f, 'a> EffectParam<'f, 'a> { if self.kind != ParamKind::Popup { return None; } - be_u16(self.pard, 60) + Pard(self.pard).popup_item_count() } /// For a popup: the selected item index (1-based): the stored selection, else the default. @@ -619,6 +593,93 @@ impl<'f, 'a> EffectParam<'f, 'a> { } } +/// A parameter definition (`pard`, 148 bytes; aep-format.md 3.7): the one decoder of its fields. +/// The import reads them through [`EffectParam`]; the export writes them from its effect table and +/// its tests read them back through this. +#[derive(Clone, Copy, Debug)] +pub struct Pard<'f>(pub &'f [u8]); + +impl Pard<'_> { + /// The type code, the `u32` at byte 12. + pub fn code(self) -> Option { + be_u32(self.0, 12) + } + + /// The parameter type (an unreadable code is [`ParamKind::Other`]). + pub fn kind(self) -> ParamKind { + ParamKind::from_u32(self.code().unwrap_or(u32::MAX)) + } + + /// The parameter name at byte 16 (32 bytes). + pub fn name(self) -> Option { + cstr(self.0, 16, 32) + } + + /// The default at byte 56 as a parameter of type `kind` reads it (see + /// [`EffectParam::pard_default`]). + pub fn default(self, kind: ParamKind) -> Option { + let p = self.0; + match kind { + ParamKind::FloatSlider => be_f64(p, 56).map(ParamValue::Number), + ParamKind::FixedSlider | ParamKind::Angle => be_i32(p, 56).map(|v| ParamValue::Number(f64::from(v) / 65536.0)), + ParamKind::IntSlider => be_i32(p, 56).map(|v| ParamValue::Number(f64::from(v))), + ParamKind::Checkbox => be_u32(p, 56).map(|v| ParamValue::Bool(v != 0)), + ParamKind::Popup => be_u32(p, 56).map(ParamValue::Index), + ParamKind::Point => { + let pt = [f64::from(be_i32(p, 56)?) / 65536.0, f64::from(be_i32(p, 60)?) / 65536.0]; + // A fresh instance stores fractions of the layer (0.5, 0.5); After Effects' save of + // a project it opened stores percent (50, 50: the re-saves of read-back tests e1 and + // e2, where Gradient Ramp's Start and End and every other point default came back + // as 50 / 0 and 50 / 100). Point + // defaults lie inside the layer, so a component beyond 1 tells percent. + let percent = pt.iter().any(|c| c.abs() > 1.0 + 1e-9); + Some(ParamValue::Point(if percent { pt.map(|c| c / 100.0) } else { pt })) + } + ParamKind::Color => { + let [a, r, g, b] = array::<4>(p, 56)?.map(|c| f64::from(c) / 255.0); + Some(ParamValue::Argb([a, r, g, b])) + } + _ => None, + } + } + + /// The valid maximum of a slider (see [`EffectParam::valid_max`]). + pub fn valid_max(self, kind: ParamKind) -> Option { + match kind { + ParamKind::FixedSlider => be_i32(self.0, 128).map(|v| f64::from(v) / 65536.0), + ParamKind::FloatSlider => be_f32(self.0, 108).map(f64::from), + ParamKind::IntSlider => be_i32(self.0, 128).map(f64::from), + _ => None, + } + } + + /// The valid minimum of a float slider (see [`EffectParam::valid_min`]). + pub fn valid_min(self, kind: ParamKind) -> Option { + match kind { + ParamKind::FloatSlider => be_f32(self.0, 104).map(f64::from), + _ => None, + } + } + + /// A slider's range, the one its value records repeat as `tdum` / `tduM`: a float slider's + /// `f32` pair at 112 / 116, a fixed slider's 16.16 pair and an integer slider's `i32` pair at + /// 132 / 136. + pub fn slider_range(self, kind: ParamKind) -> Option<(f64, f64)> { + let p = self.0; + match kind { + ParamKind::FloatSlider => Some((f64::from(be_f32(p, 112)?), f64::from(be_f32(p, 116)?))), + ParamKind::FixedSlider => Some((f64::from(be_i32(p, 132)?) / 65536.0, f64::from(be_i32(p, 136)?) / 65536.0)), + ParamKind::IntSlider => Some((f64::from(be_i32(p, 132)?), f64::from(be_i32(p, 136)?))), + _ => None, + } + } + + /// A popup's item count, the `u16` at byte 60. + pub fn popup_item_count(self) -> Option { + be_u16(self.0, 60) + } +} + impl<'f, 'a> Layer<'f, 'a> { /// The effects on the layer, in stack order (`ADBE Effect Parade`). pub fn effects(&self) -> Vec> { @@ -638,6 +699,30 @@ impl<'f, 'a> Layer<'f, 'a> { #[cfg(test)] mod tests { use super::*; + use crate::Chunk; + + #[test] + fn a_parameter_definition_reads_its_fields_and_nothing_from_a_cut_record() { + let mut b = vec![0u8; 148]; + b[12..16].copy_from_slice(&7u32.to_be_bytes()); + b[16..22].copy_from_slice(b"Amount"); + b[56..64].copy_from_slice(&2.5f64.to_be_bytes()); + let p = Pard(&b); + assert_eq!((p.code(), p.name().as_deref()), (Some(7), Some("Amount"))); + assert_eq!(p.default(ParamKind::FloatSlider), Some(ParamValue::Number(2.5))); + // The same bytes as a checkbox and as a popup: the `u32` at 56, and the `u16` at 60. + b[56..60].copy_from_slice(&1u32.to_be_bytes()); + b[60..62].copy_from_slice(&3u16.to_be_bytes()); + let p = Pard(&b); + assert_eq!(p.default(ParamKind::Checkbox), Some(ParamValue::Bool(true))); + assert_eq!((p.default(ParamKind::Popup), p.popup_item_count()), (Some(ParamValue::Index(1)), Some(3))); + assert_eq!(p.default(ParamKind::IntSlider), Some(ParamValue::Number(1.0))); + // A record cut before a field has no value for it, and an unreadable code is `Other`. + let cut = Pard(&b[..12]); + assert_eq!((cut.code(), cut.kind(), cut.name(), cut.popup_item_count()), (None, ParamKind::Other(u32::MAX), None, None)); + assert_eq!(cut.default(ParamKind::FloatSlider), None); + assert_eq!(Pard(&b[..58]).default(ParamKind::Checkbox), None); + } fn tdmn(name: &str) -> Chunk<'static> { let mut b = name.as_bytes().to_vec(); @@ -670,11 +755,11 @@ mod tests { ("A".to_string(), Chunk::list(*b"tdbs", vec![Chunk::data(*b"cdat", vec![0u8; 8])])), ]; let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["A", "B", "C"]); let first = params.first().and_then(|p| p.stored).expect("A stored"); - assert!(first.chunk.children().is_empty(), "the first A (the second has a cdat)"); + assert!(first.chunk.children().next().is_none(), "the first A (the second has a cdat)"); assert!(params.get(1).is_some_and(|p| p.stored.is_none()), "a group of that name is no stored value"); assert!(params.get(2).is_some_and(|p| p.stored.is_none())); } @@ -687,7 +772,7 @@ mod tests { let names: Vec = (0..n).map(|i| format!("P{i}")).collect(); let stored: Vec<(String, Chunk<'static>)> = names.iter().map(|m| (m.clone(), tdbs())).collect(); let chunk = instance(&names, &stored); - let fx = Effect { match_name: "X".into(), chunk: &chunk }; + let fx = Effect { match_name: "X".into(), chunk: Node::from(&chunk) }; let params = fx.params(); assert_eq!(params.len(), n); assert!(params.iter().all(|p| p.stored.is_some())); @@ -707,19 +792,19 @@ mod tests { } let efdg = Chunk::list(*b"EfdG", defs); let fold = Chunk::list(*b"Fold", Vec::new()); - let project = Project { fold: &fold, effect_defs: Some(&efdg) }; + let project = Project { fold: Node::from(&fold), effect_defs: Some(Node::from(&efdg)) }; let all = project.effect_definitions(); assert_eq!(all.len(), n); let index = project.effect_definition_index(); let bare = Chunk::list(*b"sspc", vec![Chunk::list(*b"parT", Vec::new())]); for i in (0..n).step_by(50) { - let fx = Effect { match_name: format!("FX{i}"), chunk: &bare }; + let fx = Effect { match_name: format!("FX{i}"), chunk: Node::from(&bare) }; assert_eq!(index.source(&fx), DefinitionSource::Project); let params = index.params(&fx); assert_eq!(params.iter().map(|p| p.match_name.as_str()).collect::>(), ["R0"], "the first copy's definitions"); assert!(params.iter().all(|p| p.borrowed_definition)); } - let unknown = Effect { match_name: "nope".into(), chunk: &bare }; + let unknown = Effect { match_name: "nope".into(), chunk: Node::from(&bare) }; assert_eq!(index.source(&unknown), DefinitionSource::Missing); assert!(index.params(&unknown).is_empty()); assert_eq!(project.definition_source(&unknown), DefinitionSource::Missing); diff --git a/crates/aep/src/footage.rs b/crates/aep/src/footage.rs index 58a653a4a..e8c89b924 100644 --- a/crates/aep/src/footage.rs +++ b/crates/aep/src/footage.rs @@ -2,8 +2,8 @@ //! of a layer's source. use crate::bytes::{be_f32, be_f64, be_i32, be_u32, cstr, le_u32, text, until_nul}; -use crate::chunk::Chunk; use crate::header::Rational; +use crate::index::Node; use crate::typed::{Item, ItemKind, Layer, Project}; /// The footage data of an item (`LIST Pin ` with `sspc`, `opti` and, for files, `Als2/alas`). @@ -101,21 +101,6 @@ impl PsdLayer { } impl Footage { - /// The number of frames of an image sequence (`duration * native_frame_rate`: 3/30 s at 30 fps is - /// 3); `None` for anything else. - pub fn sequence_frame_count(&self) -> Option { - if !self.is_sequence { - return None; - } - let d = self.duration?; - let frames = i128::from(d.num).checked_mul(i128::from(self.native_frame_rate))?; - let den = i128::from(d.den); - if den <= 0 || frames % den != 0 { - return None; - } - u32::try_from(frames / den).ok() - } - /// The solid's colour as fractions R, G, B. pub fn solid_rgb(&self) -> Option<[f32; 3]> { let [_, r, g, b] = self.solid_argb?; @@ -269,14 +254,14 @@ impl SequenceInfo { /// is unreadable (empty). A name that is not a plain file name (holding `/`, `\`, `:` or a control /// character such as NUL, or being `.` or `..`) makes the whole list unreadable (empty): the names /// are joined to the folder. -fn sequence_file_list(list: &Chunk<'_>) -> Vec { +fn sequence_file_list(list: Node<'_, '_>) -> Vec { // After Effects writes the count first (the one sample); a list without a readable count is // not read rather than guessed. let Some(count) = list.child(b"StVS").and_then(|c| c.bytes()).and_then(|b| le_u32(b, 0)).and_then(|n| usize::try_from(n).ok()) else { return Vec::new(); }; let names: Vec = - list.children().iter().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).take(count).map(text).filter(|n| !n.is_empty()).collect(); + list.children().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).take(count).map(text).filter(|n| !n.is_empty()).collect(); if names.iter().any(|n| n.contains(|c: char| matches!(c, '/' | '\\' | ':') || c.is_control()) || n == "." || n == "..") { return Vec::new(); } @@ -308,9 +293,8 @@ impl<'f, 'a> Item<'f, 'a> { } let pin = self.chunk.child_list(b"Pin ")?; let sspc = pin.child(b"sspc")?.bytes()?; - let kids = pin.children(); - let after = kids.iter().position(|c| c.is_list(b"Als2"))?; - let naming = || kids.iter().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); + let after = pin.children().position(|c| c.is_list(b"Als2"))?; + let naming = || pin.children().skip(after.saturating_add(1)).take_while(|c| !c.is(b"opti")); let (start, end, digits) = (be_u32(sspc, 172)?, be_u32(sspc, 176)?, be_u32(sspc, 180)?); // Force alphabetical order: the file list replaces the numbering. if let Some(list) = naming().find(|c| c.is_list(b"StVc")) { @@ -320,7 +304,7 @@ impl<'f, 'a> Item<'f, 'a> { } return Some(SequenceInfo { prefix: String::new(), suffix: String::new(), start, end, digits, files }); } - let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(|c| c.bytes()).map(text).take(2).collect(); + let names: Vec = naming().filter(|c| c.is(b"Utf8")).filter_map(Node::bytes).map(text).take(2).collect(); let (prefix, suffix) = match names.as_slice() { [prefix, suffix] => (prefix.clone(), suffix.clone()), // An empty prefix is not stored (`0001.png` has only `.png`). @@ -350,12 +334,6 @@ impl<'f, 'a> Item<'f, 'a> { self.chunk.child(b"idta")?.bytes()?.get(58).copied() } - /// The ids of the compositions this item uses as precomp layers: the little-endian `u32` of - /// each `idpi` chunk. - pub fn used_item_ids(&self) -> Vec { - self.chunk.children().iter().filter(|c| c.is(b"idpi")).filter_map(|c| le_u32(c.bytes()?, 0)).collect() - } - /// The footage data of a footage item. pub fn footage(&self) -> Option { if self.kind()? != ItemKind::Footage { @@ -405,11 +383,6 @@ impl<'f, 'a> Project<'f, 'a> { self.item_by_id(layer.header()?.source_id?) } - /// The name a layer shows: its own name, or its source's name when it has none. - pub fn layer_display_name(&self, layer: &Layer<'f, 'a>) -> Option { - layer.name().or_else(|| self.layer_source(layer)?.display_name()) - } - /// The pixel size of a layer's source: a footage item's width and height, or a precomp's /// composition size. pub fn layer_source_size(&self, layer: &Layer<'f, 'a>) -> Option<(u32, u32)> { diff --git a/crates/aep/src/header.rs b/crates/aep/src/header.rs index 8593a5633..ac869095f 100644 --- a/crates/aep/src/header.rs +++ b/crates/aep/src/header.rs @@ -237,12 +237,6 @@ impl LayerHeader { self.flag_bytes.get(byte).is_some_and(|b| b & mask != 0) } - /// Byte 37 bit `0x01`: the layer has a name set by the user (clear on a default-named camera or - /// light and on a precomp layer, whose name is the source's). - pub fn name_is_user_set(&self) -> bool { - self.flag(0, 0x01) - } - /// Byte 37 bit `0x02`: guide layer. pub fn is_guide(&self) -> bool { self.flag(0, 0x02) @@ -333,17 +327,6 @@ impl LayerHeader { Rational::new(i64::from(self.stretch), i64::from(self.stretch_den)) } - /// The time stretch in percent (`stretch / stretch_den * 100`); a zero denominator reads as 1. - pub fn stretch_percent(&self) -> f64 { - let den = if self.stretch_den == 0 { 1.0 } else { f64::from(self.stretch_den) }; - f64::from(self.stretch) / den * 100.0 - } - - /// The start time in seconds, `None` for a zero denominator. - pub fn start_seconds(&self) -> Option { - self.start.to_f64() - } - /// The absolute in point, `start + in * stretch`, exact and reduced. pub fn in_point(&self) -> Option { self.start.plus_scaled(&self.in_relative, &self.stretch_ratio()?) @@ -355,16 +338,6 @@ impl LayerHeader { pub fn out_point(&self) -> Option { self.start.plus_scaled(&self.out_relative, &self.stretch_ratio()?) } - - /// The absolute in point in seconds. - pub fn in_point_seconds(&self) -> Option { - self.in_point()?.to_f64() - } - - /// The absolute out point in seconds. - pub fn out_point_seconds(&self) -> Option { - self.out_point()?.to_f64() - } } impl<'f, 'a> Layer<'f, 'a> { @@ -464,64 +437,10 @@ impl CompHeader { self.time_base.frame_rate() } - /// The frame rate from the 16.16 fixed-point field (25 is `0x0019_0000`). - pub fn frame_rate_from_fixed(&self) -> f64 { - f64::from(self.frame_rate_fixed) / 65536.0 - } - /// The duration in seconds. pub fn duration_seconds(&self) -> Option { self.time_base.seconds(i32::try_from(self.duration_units).ok()?) } - - /// The work area start in seconds. - pub fn work_area_start_seconds(&self) -> Option { - self.time_base.seconds(i32::try_from(self.work_area_start_units).ok()?) - } - - /// The work area end in seconds; the composition duration when the work area is the whole - /// composition. - pub fn work_area_end_seconds(&self) -> Option { - match self.work_area_end_units { - Some(end) => self.time_base.seconds(i32::try_from(end).ok()?), - None => self.duration_seconds(), - } - } - - /// The background colour as fractions 0..1 (`byte / 255`). - pub fn background_rgb(&self) -> [f64; 3] { - self.background.map(|c| f64::from(c) / 255.0) - } - - /// The pixel aspect ratio as a number. - pub fn pixel_aspect_ratio(&self) -> Option { - self.pixel_aspect.to_f64() - } - - /// The display start time in seconds (the start timecode), `None` when absent or the - /// denominator is zero. - pub fn display_start_seconds(&self) -> Option { - self.display_start?.to_f64() - } - - /// The display start as a frame number (`display start * frame rate`, rounded); 100 at 25 fps - /// for a display start of 4 s. - pub fn display_start_frame(&self) -> Option { - let rate = self.frame_rate()?; - Some((self.display_start_seconds()? * rate).round() as i64) - } - - /// The frame rate as an exact fraction `timescale / frame_duration`, as typed: 29.97 fps is - /// `23976 / 800` (not `30000 / 1001`), 23.976 fps is `23976 / 1000`. - pub fn frame_rate_exact(&self) -> Option { - Rational::new(i64::from(self.time_base.timescale), i64::from(self.time_base.frame_duration)) - } - - /// The duration in whole frames (`duration / frame_duration`). After Effects snaps a duration - /// to a whole number of frames: 5 s at 29.97 fps becomes 150 frames (5.005 s). - pub fn duration_frames(&self) -> Option { - self.duration_units.checked_div(self.time_base.frame_duration) - } } impl<'f, 'a> Comp<'f, 'a> { diff --git a/crates/aep/src/index.rs b/crates/aep/src/index.rs new file mode 100644 index 000000000..76d5430da --- /dev/null +++ b/crates/aep/src/index.rs @@ -0,0 +1,1026 @@ +//! A read-only, flat index of the chunk tree for readers that only look (the importer). +//! +//! [`TreeIndex`] holds every chunk of a file in one `Vec` in depth-first order (a chunk, then its +//! whole subtree, then its next sibling). Each entry keeps its id, list type, pad byte, the bytes of +//! its body (borrowed from the input) and the index one past its subtree, so: +//! - a chunk's subtree is the index range `at + 1 .. end`, +//! - its direct children are found by stepping from `at + 1` to each child's `end`, +//! - the whole tree is one array, grown a few times while it is built and freed at once; it never +//! holds more slots than the input has room for chunk headers or the chunk limit allows. +//! +//! [`crate::parse`] builds the owned [`Chunk`] tree instead (one `Vec` per list), which +//! the exporter and editors need. [`TreeIndex::to_file`] and [`Node::to_chunk`] convert the index +//! into that tree where an owned one is needed. +//! +//! [`TreeIndex::parse_with_limits`] walks the input exactly as the [`Chunk`] parser does: the same +//! chunks are counted against [`Limits::max_chunks`] in the same order, the same lists are kept +//! opaque at [`Limits::max_depth`] or when damaged, and the same errors come back. A test compares +//! the two on the corpus and on generated and damaged trees. Two errors are its own, and both are +//! stricter than the [`Chunk`] parser (which would abort or could not address the tree): +//! [`AepError::TooLarge`] when the array cannot grow (a failed `try_reserve`), and +//! [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. +//! +//! An `.aepx` is indexed by its own reader (`container/aepx.rs`), which decodes every body into +//! one buffer and fills the entries as it reads; the index then borrows that buffer +//! (`TreeIndex::from_drafts`). It gives the tree [`crate::read_aepx`] gives. +//! +//! [`Node`] is the handle the typed view ([`crate::Project`] and the types below it) +//! reads through: a chunk of either tree. The view works the same on both. + +use std::borrow::Cow; +use std::fmt; + +use crate::bytes::be_u32; +use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +use crate::error::AepError; +use crate::read::{Limits, container_end, id4}; + +/// What an entry's body is (the [`Body`] variants). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Kind { + Data, + /// [`Body::Chunks`]. + Nested, + List, + OpaqueList, +} + +/// One chunk of a [`TreeIndex`]. +#[derive(Debug, Clone, Copy)] +struct Entry<'a> { + /// The body bytes of a data chunk, or the bytes after the list type of an opaque list; empty + /// for a list or nested chunk (their content is their children). + data: &'a [u8], + id: [u8; 4], + /// The list type of a `LIST` (parsed or opaque); zero otherwise. + list_type: [u8; 4], + /// One past the last entry of this chunk's subtree. + end: u32, + kind: Kind, + pad: Option, +} + +impl<'a> Entry<'a> { + /// An entry for a chunk whose body is not read yet. + fn placeholder(id: [u8; 4]) -> Self { + Self { data: &[], id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// An entry of an index whose bytes are still being written (the `.aepx` reader decodes into a +/// buffer that grows, so it keeps positions, not slices): the body is `start..start + len` of +/// that buffer. The fields are those of [`Entry`]; [`TreeIndex::from_drafts`] turns the list +/// into an index once the buffer is final. +#[derive(Debug, Clone, Copy)] +pub struct Draft { + pub(crate) start: usize, + pub(crate) len: usize, + pub(crate) id: [u8; 4], + pub(crate) list_type: [u8; 4], + pub(crate) end: u32, + pub(crate) kind: Kind, + pub(crate) pad: Option, +} + +impl Draft { + /// A data chunk with no body yet. + pub(crate) fn placeholder(id: [u8; 4]) -> Self { + Self { start: 0, len: 0, id, list_type: [0; 4], end: 0, kind: Kind::Data, pad: None } + } +} + +/// The chunk tree of a file as one flat, read-only array (see the module documentation). +#[derive(Clone)] +pub struct TreeIndex<'a> { + nodes: Vec>, + trailer: &'a [u8], +} + +impl fmt::Debug for TreeIndex<'_> { + /// The sizes only: the chunk count and the trailer length. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TreeIndex").field("chunks", &self.nodes.len()).field("trailer", &self.trailer.len()).finish() + } +} + +/// An index that does not fit a `u32` (more than 4 billion chunks) is refused as too many chunks. +fn index_u32(at: usize) -> Result { + u32::try_from(at).map_err(|_| AepError::TooManyChunks) +} + +impl<'a> TreeIndex<'a> { + /// Indexes an `.aep` with the default [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::parse`], for the same inputs. + pub fn parse(data: &'a [u8]) -> Result { + Self::parse_with_limits(data, Limits::default()) + } + + /// Indexes an `.aep` with explicit [`Limits`]. + /// + /// # Errors + /// The errors of [`crate::parse_with_limits`], for the same inputs and limits; also + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks and + /// [`AepError::TooLarge`] when the array cannot be allocated. + pub fn parse_with_limits(data: &'a [u8], limits: Limits) -> Result { + let end = container_end(data)?; + // Every chunk header takes 8 bytes of its own and is counted once, so the tree never has + // more chunks than this. + let bound = (end.saturating_sub(12) / 8).min(limits.max_chunks); + let mut builder = Builder { data, limits, count: 0, nodes: Vec::new(), bound }; + // A small start; [`Builder::push`] doubles it as chunks arrive, never past the bound, so + // the array stays proportional to the chunk count (a file of a few large chunks gets a few + // slots, not one per 32 bytes). `try_reserve_exact` so it fails nothing here. + let _ = builder.nodes.try_reserve_exact(bound.min(INITIAL_SLOTS)); + builder.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(Self { nodes: builder.nodes, trailer }) + } + + /// The index of the entries `drafts` (in depth-first order, see [`Draft`]) whose bodies and + /// trailer are ranges of `buffer`. A range outside `buffer` reads as empty. The list is turned + /// into entries; a [`Draft`] and an [`Entry`] have the same size and alignment, so the standard + /// library reuses the list's allocation for them. + pub fn from_drafts(buffer: &'a [u8], drafts: Vec, trailer: (usize, usize)) -> Self { + let slice = |start: usize, len: usize| start.checked_add(len).and_then(|end| buffer.get(start..end)).unwrap_or(&[]); + let nodes = drafts + .into_iter() + .map(|d| Entry { data: slice(d.start, d.len), id: d.id, list_type: d.list_type, end: d.end, kind: d.kind, pad: d.pad }) + .collect(); + let trailer = buffer.get(trailer.0..trailer.1).unwrap_or(&[]); + Self { nodes, trailer } + } + + /// Indexes an already parsed (or built) chunk tree. The index borrows the file's bytes. + /// + /// # Errors + /// [`AepError::TooManyChunks`] for a tree of more than `u32::MAX` chunks. + pub fn from_file(file: &'a AepFile<'_>) -> Result { + // Sized once, to the chunks the tree holds. + let mut nodes: Vec> = Vec::new(); + nodes.try_reserve_exact(file.walk().count()).map_err(|_| AepError::TooLarge)?; + // Each frame: the siblings still to visit and the entry whose subtree they fill. + let mut stack: Vec<(std::slice::Iter<'a, Chunk<'_>>, Option)> = vec![(file.chunks.iter(), None)]; + while let Some((iter, parent)) = stack.last_mut() { + let parent = *parent; + match iter.next() { + Some(chunk) => { + let at = nodes.len(); + // The entry after this one must still have a `u32` index. + let next = index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + let (kind, data, list_type) = match &chunk.body { + Body::Data(d) => (Kind::Data, &d[..], [0; 4]), + Body::Chunks(_) => (Kind::Nested, &[][..], [0; 4]), + Body::List { list_type, .. } => (Kind::List, &[][..], *list_type), + Body::OpaqueList { list_type, data } => (Kind::OpaqueList, &data[..], *list_type), + }; + nodes.push(Entry { data, id: chunk.id, list_type, end: next, kind, pad: chunk.pad }); + let kids = chunk.children(); + if !kids.is_empty() { + stack.push((kids.iter(), Some(at))); + } + } + None => { + stack.pop(); + if let Some(p) = parent { + let end = index_u32(nodes.len())?; + if let Some(e) = nodes.get_mut(p) { + e.end = end; + } + } + } + } + } + Ok(Self { nodes, trailer: &file.trailer[..] }) + } + + /// The number of chunks in the tree (at every depth). + pub fn len(&self) -> usize { + self.nodes.len() + } + + /// True when the container holds no chunks. + pub fn is_empty(&self) -> bool { + self.nodes.is_empty() + } + + /// The bytes after the container's declared end (see [`AepFile::trailer`]). + pub fn trailer(&self) -> &'a [u8] { + self.trailer + } + + /// The top-level chunks. + pub fn chunks(&self) -> Nodes<'_, 'a> { + Nodes(NodesRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX) }) + } + + /// The first top-level chunk with this id. + pub fn chunk(&self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(&self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } + + /// Visits every chunk depth-first with its nesting depth (the top level is 1), as + /// [`AepFile::walk`] does. + pub fn walk(&self) -> NodeWalk<'_, 'a> { + NodeWalk(WalkRepr::Index { tree: self, next: 0, end: u32::try_from(self.nodes.len()).unwrap_or(u32::MAX), open: Vec::new() }) + } + + /// The owned chunk tree (the bodies stay borrowed from the input): what + /// [`crate::parse`] gives for the same file. + pub fn to_file(&self) -> AepFile<'a> { + AepFile { chunks: self.build(0, u32::try_from(self.nodes.len()).unwrap_or(u32::MAX)), trailer: Cow::Borrowed(self.trailer) } + } + + /// The entry at `at`, `None` past the end. + fn entry(&self, at: u32) -> Option<&Entry<'a>> { + self.nodes.get(usize::try_from(at).ok()?) + } + + /// The end of the subtree at `at`; never before `at + 1`, so walks always move on. + fn end_of(&self, at: u32) -> u32 { + self.entry(at).map_or(u32::MAX, |e| e.end).max(at.saturating_add(1)) + } + + /// The owned chunks for the siblings that start at `next` and fill `next..end`, without + /// recursion. + fn build(&self, next: u32, end: u32) -> Vec> { + struct Frame<'a> { + /// The list or nested chunk being filled; `None` for the outermost range. + at: Option, + kids: Vec>, + next: u32, + end: u32, + } + let mut stack = vec![Frame { at: None, kids: Vec::new(), next, end }]; + loop { + let Some(top) = stack.last_mut() else { return Vec::new() }; + if top.next < top.end { + let at = top.next; + top.next = self.end_of(at); + let Some(e) = self.entry(at).copied() else { + top.next = top.end; + continue; + }; + match e.kind { + Kind::List | Kind::Nested => { + let sub_end = self.end_of(at); + stack.push(Frame { at: Some(at), kids: Vec::new(), next: at.saturating_add(1), end: sub_end }); + } + Kind::Data => top.kids.push(Chunk { id: e.id, body: Body::Data(Cow::Borrowed(e.data)), pad: e.pad }), + Kind::OpaqueList => { + top.kids.push(Chunk { id: e.id, body: Body::OpaqueList { list_type: e.list_type, data: Cow::Borrowed(e.data) }, pad: e.pad }); + } + } + } else { + let Some(frame) = stack.pop() else { return Vec::new() }; + let Some(at) = frame.at else { return frame.kids }; + let Some(e) = self.entry(at).copied() else { continue }; + let body = match e.kind { + Kind::Nested => Body::Chunks(frame.kids), + Kind::List | Kind::Data | Kind::OpaqueList => Body::List { list_type: e.list_type, children: frame.kids }, + }; + let chunk = Chunk { id: e.id, body, pad: e.pad }; + match stack.last_mut() { + Some(parent) => parent.kids.push(chunk), + None => return vec![chunk], + } + } + } + } +} + +/// The slots a [`TreeIndex`] starts with (32 KiB); it doubles from there. +const INITIAL_SLOTS: usize = 1024; + +/// Builds a [`TreeIndex`]; mirrors `read::Parser` step for step (see the module documentation). +struct Builder<'a> { + data: &'a [u8], + limits: Limits, + count: usize, + nodes: Vec>, + /// The most chunks the input can hold under the limits: `nodes` never reserves past it. + bound: usize, +} + +impl<'a> Builder<'a> { + /// Appends an entry. A full array grows by its length (at least 16), but never past + /// [`Self::bound`]; an allocation the target cannot make is an error, not an abort. + fn push(&mut self, entry: Entry<'a>) -> Result<(), AepError> { + let len = self.nodes.len(); + if len == self.nodes.capacity() { + let more = len.max(16).min(self.bound.saturating_sub(len)).max(1); + self.nodes.try_reserve_exact(more).map_err(|_| AepError::TooLarge)?; + } + self.nodes.push(entry); + Ok(()) + } + + /// Appends the sibling chunks that exactly fill `start..end`, each followed by its subtree. + fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let mut pos = start; + while pos < end { + let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; + let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; + let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let avail = end - body_start; + if size > avail { + return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); + } + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + let at = self.nodes.len(); + // This entry and the one after it must have `u32` indices. + index_u32(at.checked_add(1).ok_or(AepError::TooManyChunks)?)?; + self.push(Entry::placeholder(id))?; + let body_end = body_start + size; + self.body(at, id, body_start, body_end, depth)?; + let mut next = body_end; + let mut pad = None; + if size % 2 == 1 && next < end { + pad = Some(self.data.get(next).copied().ok_or(AepError::Malformed { offset: next, what: "missing pad byte" })?); + next += 1; + } + let subtree_end = index_u32(self.nodes.len())?; + if let Some(e) = self.nodes.get_mut(at) { + e.pad = pad; + e.end = subtree_end; + } + pos = next; + } + Ok(()) + } + + /// Like [`Self::sequence`] but a structural problem yields `false` and removes what the + /// attempt appended (the caller keeps bytes). The chunk count keeps the attempt's chunks, as + /// the [`Chunk`] parser's does. + fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result { + let mark = self.nodes.len(); + match self.sequence(start, end, depth) { + Ok(()) => Ok(true), + Err(e) if e.is_structural() => { + self.nodes.truncate(mark); + Ok(false) + } + Err(e) => Err(e), + } + } + + /// The input bytes `start..end` (empty when out of range). + fn slice(&self, start: usize, end: usize) -> &'a [u8] { + self.data.get(start..end).unwrap_or(&[]) + } + + /// Fills in the body of the entry at `at` once the parser knows what it is. + fn set(&mut self, at: usize, kind: Kind, data: &'a [u8], list_type: [u8; 4]) { + if let Some(e) = self.nodes.get_mut(at) { + e.kind = kind; + e.data = data; + e.list_type = list_type; + } + } + + /// Fills in the entry at `at` for the body `start..end` and appends its subtree. + fn body(&mut self, at: usize, id: [u8; 4], start: usize, end: usize, depth: usize) -> Result<(), AepError> { + let len = end - start; + if id == LIST && len >= 4 { + let list_type = id4(self.data, start).ok_or(AepError::Malformed { offset: start, what: "short read" })?; + if depth < self.limits.max_depth && self.tentative(start + 4, end, depth + 1)? { + self.set(at, Kind::List, &[], list_type); + } else { + let data = self.slice(start + 4, end); + self.set(at, Kind::OpaqueList, data, list_type); + } + return Ok(()); + } + if NESTED_IDS.contains(&id) && len >= 8 && depth < self.limits.max_depth { + let mark = self.nodes.len(); + if self.tentative(start, end, depth + 1)? && self.nodes.len() > mark { + self.set(at, Kind::Nested, &[], [0; 4]); + return Ok(()); + } + } + let data = self.slice(start, end); + self.set(at, Kind::Data, data, [0; 4]); + Ok(()) + } +} + +/// A chunk of either tree: an owned [`Chunk`] or an entry of a [`TreeIndex`]. The typed view +/// reads through it, so it works the same on both. Cheap to copy. +#[derive(Clone, Copy)] +pub struct Node<'f, 'a>(NodeRepr<'f, 'a>); + +#[derive(Clone, Copy)] +enum NodeRepr<'f, 'a> { + Chunk(&'f Chunk<'a>), + Index(&'f TreeIndex<'a>, u32), +} + +impl<'f, 'a> From<&'f Chunk<'a>> for Node<'f, 'a> { + /// The handle for this value. + fn from(chunk: &'f Chunk<'a>) -> Self { + Node(NodeRepr::Chunk(chunk)) + } +} + +/// Prints the chunk as [`Chunk`]'s `Debug` does, whichever tree it is in, without copying the +/// subtree. +impl fmt::Debug for Node<'_, '_> { + /// The chunk as `Chunk`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self.0 { + NodeRepr::Chunk(c) => fmt::Debug::fmt(c, f), + NodeRepr::Index(..) => { + f.debug_struct("Chunk").field("id", &self.id()).field("body", &BodyDebug(*self)).field("pad", &self.pad()).finish() + } + } + } +} + +/// The body of an index [`Node`], printed as [`Body`]'s `Debug` prints it. +struct BodyDebug<'f, 'a>(Node<'f, 'a>); + +/// The children of a [`Node`], printed as a list. +struct ChildrenDebug<'f, 'a>(Node<'f, 'a>); + +impl fmt::Debug for BodyDebug<'_, '_> { + /// The body as `Body`'s `Debug` prints it. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let Some(e) = self.0.entry() else { return f.write_str("?") }; + match e.kind { + Kind::Data => f.debug_tuple("Data").field(&e.data).finish(), + Kind::Nested => f.debug_tuple("Chunks").field(&ChildrenDebug(self.0)).finish(), + Kind::List => f.debug_struct("List").field("list_type", &e.list_type).field("children", &ChildrenDebug(self.0)).finish(), + Kind::OpaqueList => f.debug_struct("OpaqueList").field("list_type", &e.list_type).field("data", &e.data).finish(), + } + } +} + +impl fmt::Debug for ChildrenDebug<'_, '_> { + /// The children as a list. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.0.children()).finish() + } +} + +impl<'f, 'a> Node<'f, 'a> { + /// The entry at `at`, `None` past the end. + fn entry(self) -> Option<&'f Entry<'a>> { + match self.0 { + NodeRepr::Index(tree, at) => tree.entry(at), + NodeRepr::Chunk(_) => None, + } + } + + /// The [`Chunk`] when this node is one of an owned tree; `None` for an index entry (see + /// [`Self::to_chunk`]). + pub fn as_chunk(self) -> Option<&'f Chunk<'a>> { + match self.0 { + NodeRepr::Chunk(c) => Some(c), + NodeRepr::Index(..) => None, + } + } + + /// The chunk and its subtree as an owned [`Chunk`] (bodies stay borrowed). + pub fn to_chunk(self) -> Chunk<'a> { + match self.0 { + NodeRepr::Chunk(c) => c.clone(), + NodeRepr::Index(tree, at) => tree.build(at, tree.end_of(at)).into_iter().next().unwrap_or_else(|| Chunk::data(self.id(), &[][..])), + } + } + + /// The 4-byte id (see [`Chunk::id`]). + pub fn id(self) -> [u8; 4] { + match self.0 { + NodeRepr::Chunk(c) => c.id, + NodeRepr::Index(..) => self.entry().map_or([0; 4], |e| e.id), + } + } + + /// The pad byte (see [`Chunk::pad`]). + pub fn pad(self) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.pad, + NodeRepr::Index(..) => self.entry().and_then(|e| e.pad), + } + } + + /// The raw bytes of a data chunk, or the bytes after the list type of an opaque list (see + /// [`Chunk::bytes`]). + pub fn bytes(self) -> Option<&'f [u8]> { + match self.0 { + NodeRepr::Chunk(c) => c.bytes(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::Data | Kind::OpaqueList)).map(|e| e.data), + } + } + + /// The child chunks of a list or nested chunk; none for data. + pub fn children(self) -> Nodes<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => Nodes(NodesRepr::Slice(c.children().iter())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + Nodes(NodesRepr::Index { tree, next: at.saturating_add(1), end }) + } + } + } + + /// The list type when this is a `LIST` (parsed or opaque). + pub fn list_type(self) -> Option<[u8; 4]> { + match self.0 { + NodeRepr::Chunk(c) => c.list_type(), + NodeRepr::Index(..) => self.entry().filter(|e| matches!(e.kind, Kind::List | Kind::OpaqueList)).map(|e| e.list_type), + } + } + + /// True when this is a `LIST` of the given type. + pub fn is_list(self, list_type: &[u8; 4]) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_list(list_type), + NodeRepr::Index(..) => { + self.entry().is_some_and(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type) + } + } + } + + /// True when this is a `LIST` whose content could not be read as chunks. + pub fn is_opaque_list(self) -> bool { + match self.0 { + NodeRepr::Chunk(c) => c.is_opaque_list(), + NodeRepr::Index(..) => self.entry().is_some_and(|e| e.kind == Kind::OpaqueList), + } + } + + /// True when this chunk has the given id. + pub fn is(self, id: &[u8; 4]) -> bool { + self.id() == *id + } + + /// The first child chunk with this id. + pub fn child(self, id: &[u8; 4]) -> Option { + self.children().find(|c| c.is(id)) + } + + /// The first child `LIST` of this type. + pub fn child_list(self, list_type: &[u8; 4]) -> Option { + self.children().find(|c| c.is_list(list_type)) + } + + /// Every chunk below this one depth-first (children before later siblings), with the depth + /// counted from 1 for the children of this chunk. + pub fn descendants(self) -> NodeWalk<'f, 'a> { + match self.0 { + NodeRepr::Chunk(c) => NodeWalk(WalkRepr::Chunk(c.descendants())), + NodeRepr::Index(tree, at) => { + let end = match tree.entry(at) { + Some(e) if matches!(e.kind, Kind::List | Kind::Nested) => tree.end_of(at), + _ => at.saturating_add(1), + }; + NodeWalk(WalkRepr::Index { tree, next: at.saturating_add(1), end, open: Vec::new() }) + } + } + } + + /// The first descendant `LIST` of this type, searching depth-first. + pub fn find_list(self, list_type: &[u8; 4]) -> Option { + match self.0 { + NodeRepr::Chunk(c) => c.find_list(list_type).map(Node::from), + // The subtree is a contiguous range in depth-first order: a plain scan. + NodeRepr::Index(tree, at) => { + let first = usize::try_from(at).ok()?.checked_add(1)?; + let end = usize::try_from(tree.end_of(at)).ok()?.min(tree.nodes.len()); + let range = tree.nodes.get(first..end)?; + let i = range.iter().position(|e| e.id == LIST && matches!(e.kind, Kind::List | Kind::OpaqueList) && &e.list_type == list_type)?; + let found = u32::try_from(first.checked_add(i)?).ok()?; + Some(Node(NodeRepr::Index(tree, found))) + } + } + } +} + +/// The children of a [`Node`] (or the top-level chunks of a [`TreeIndex`]), in file order. +#[derive(Clone)] +pub struct Nodes<'f, 'a>(NodesRepr<'f, 'a>); + +#[derive(Clone)] +enum NodesRepr<'f, 'a> { + Slice(std::slice::Iter<'f, Chunk<'a>>), + Index { tree: &'f TreeIndex<'a>, next: u32, end: u32 }, +} + +impl<'f, 'a> Iterator for Nodes<'f, 'a> { + type Item = Node<'f, 'a>; + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + NodesRepr::Slice(it) => it.next().map(Node::from), + NodesRepr::Index { tree, next, end } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + *next = tree.end_of(at); + Some(Node(NodeRepr::Index(tree, at))) + } + } + } +} + +/// A depth-first walk over [`Node`]s with their depth; see [`Node::descendants`] and +/// [`TreeIndex::walk`]. +pub struct NodeWalk<'f, 'a>(WalkRepr<'f, 'a>); + +enum WalkRepr<'f, 'a> { + Chunk(Walk<'f, 'a>), + /// `open` holds the subtree ends of the entries above the next one (its ancestors inside the + /// walked range); their count is the next entry's depth minus one. + Index { + tree: &'f TreeIndex<'a>, + next: u32, + end: u32, + open: Vec, + }, +} + +impl<'f, 'a> Iterator for NodeWalk<'f, 'a> { + type Item = (usize, Node<'f, 'a>); + + /// The next item, `None` at the end of the range. + fn next(&mut self) -> Option { + match &mut self.0 { + WalkRepr::Chunk(w) => w.next().map(|(d, c)| (d, Node::from(c))), + WalkRepr::Index { tree, next, end, open } => { + if *next >= *end { + return None; + } + let (tree, at) = (*tree, *next); + tree.entry(at)?; + while open.last().is_some_and(|e| *e <= at) { + open.pop(); + } + let depth = open.len().saturating_add(1); + let sub_end = tree.end_of(at); + if sub_end > at.saturating_add(1) { + open.push(sub_end); + } + *next = at.saturating_add(1); + Some((depth, Node(NodeRepr::Index(tree, at)))) + } + } + } +} + +/// The top level of either tree: what the project settings and the version are read from. +#[derive(Clone, Copy)] +pub enum Root<'f, 'a> { + File(&'f AepFile<'a>), + Index(&'f TreeIndex<'a>), +} + +impl<'f, 'a> From<&'f AepFile<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(file: &'f AepFile<'a>) -> Self { + Root::File(file) + } +} + +impl<'f, 'a> From<&'f TreeIndex<'a>> for Root<'f, 'a> { + /// The handle for this value. + fn from(index: &'f TreeIndex<'a>) -> Self { + Root::Index(index) + } +} + +impl<'f, 'a> Root<'f, 'a> { + /// The top-level chunks. + pub fn chunks(self) -> Nodes<'f, 'a> { + match self { + Root::File(f) => Nodes(NodesRepr::Slice(f.chunks.iter())), + Root::Index(t) => t.chunks(), + } + } + + /// The first top-level chunk with this id. + pub fn chunk(self, id: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is(id)) + } + + /// The first top-level `LIST` of this type. + pub fn list(self, list_type: &[u8; 4]) -> Option> { + self.chunks().find(|c| c.is_list(list_type)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{parse_with_limits, write}; + + /// A tiny deterministic generator (xorshift64*), so the tests need no extra crate. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift64*). + fn next(&mut self) -> u64 { + let mut x = self.0; + x ^= x >> 12; + x ^= x << 25; + x ^= x >> 27; + self.0 = x; + x.wrapping_mul(0x2545_f491_4f6c_dd1d) + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: u64) -> u64 { + if n == 0 { 0 } else { self.next() % n } + } + } + + /// A random tree: lists, nested-id chunks holding a `Utf8`, data of odd and even sizes, opaque + /// lists, empty lists, deep chains. + fn tree(rng: &mut Rng, depth: usize, budget: &mut usize) -> Vec> { + let n = rng.below(6); + let mut out = Vec::new(); + for _ in 0..n { + if *budget == 0 { + break; + } + *budget -= 1; + let pick = rng.below(10); + let chunk = if depth < 12 && pick < 4 { + let types: [&[u8; 4]; 4] = [b"Fold", b"tdgp", b"tdbs", b"Layr"]; + let t = *types[usize::try_from(rng.below(4)).expect("small")]; + Chunk::list(t, tree(rng, depth + 1, budget)) + } else if pick == 4 { + let ids: [&[u8; 4]; 3] = [b"tdsn", b"pdnm", b"fnam"]; + let id = *ids[usize::try_from(rng.below(3)).expect("small")]; + let text = vec![b'a'; usize::try_from(rng.below(5)).expect("small")]; + Chunk { id, body: Body::Chunks(vec![Chunk::data(*b"Utf8", text)]), pad: None } + } else if pick == 5 { + let raw = vec![0x5a; usize::try_from(rng.below(9)).expect("small")]; + Chunk { id: LIST, body: Body::OpaqueList { list_type: *b"btdk", data: raw.into() }, pad: None } + } else { + let ids: [&[u8; 4]; 4] = [b"cdat", b"tdmn", b"Utf8", b"idta"]; + let id = *ids[usize::try_from(rng.below(4)).expect("small")]; + Chunk::data(id, vec![u8::try_from(rng.below(256)).expect("byte"); usize::try_from(rng.below(13)).expect("small")]) + }; + out.push(chunk); + } + out + } + + /// Both readers on the same bytes and limits: the same result, errors included; and the + /// index's walk visits the same chunks at the same depths as the tree's. + fn same(bytes: &[u8], limits: Limits) { + let old = parse_with_limits(bytes, limits); + let new = TreeIndex::parse_with_limits(bytes, limits); + match (&old, &new) { + (Ok(file), Ok(index)) => { + assert_eq!(&index.to_file(), file, "the index converts to the parsed tree"); + let a: Vec<(usize, Chunk<'_>)> = file.walk().map(|(d, c)| (d, c.clone())).collect(); + let b: Vec<(usize, Chunk<'_>)> = index.walk().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(a, b, "the walks agree"); + assert_eq!(index.len(), a.len()); + // Indexing the parsed tree gives the same index. + let again = TreeIndex::from_file(file).expect("index"); + assert_eq!(again.to_file(), *file); + } + (Err(a), Err(b)) => assert_eq!(a, b, "the same error"), + _ => panic!("one reader failed: {:?} vs {:?}", old.as_ref().err(), new.as_ref().err()), + } + } + + /// The default limits and four tight ones (depth 1 and 3, 7 and 40 chunks). + fn limits() -> [Limits; 5] { + [ + Limits::default(), + Limits { max_depth: 1, max_chunks: 10_000_000 }, + Limits { max_depth: 3, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: 7 }, + Limits { max_depth: 64, max_chunks: 40 }, + ] + } + + /// The index matches the chunk parser on generated and damaged trees. + #[test] + fn the_index_matches_the_chunk_parser_on_generated_and_damaged_trees() { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + for _ in 0..400 { + let mut budget = 120; + let chunks = tree(&mut rng, 0, &mut budget); + let file = AepFile { chunks, trailer: Cow::Borrowed(b"x") }; + let bytes = write(&file).expect("write"); + for l in limits() { + same(&bytes, l); + } + // Damage: overwrite a few bytes past the container header (sizes, ids, list types). + for _ in 0..4 { + let mut bad = bytes.clone(); + for _ in 0..1 + rng.below(3) { + let at = 12 + usize::try_from(rng.below(u64::try_from(bad.len().saturating_sub(12)).expect("len"))).expect("index"); + if let Some(b) = bad.get_mut(at) { + *b = u8::try_from(rng.below(256)).expect("byte"); + } + } + for l in limits() { + same(&bad, l); + } + } + // Truncated inputs (the container size then no longer fits). + let cut = bytes.len().saturating_sub(usize::try_from(rng.below(8)).expect("small") + 1); + same(&bytes[..cut], Limits::default()); + } + } + + /// Header errors are the same. + #[test] + fn header_errors_are_the_same() { + for bytes in + [&b""[..], b"RIFX", b"RIFX\0\0\0\x04Egg", b"RIFX\0\0\0\x04Fnord", b"RIFX\0\0\0\x08Egg!", b"RIFX\0\0\0\x03Egg!", b"RIFF\0\0\0\x04Egg!"] + { + same(bytes, Limits::default()); + } + } + + /// The typed accessors give the same answers through either tree. + #[test] + fn nodes_answer_alike_on_both_trees() { + let mut rng = Rng(42); + for _ in 0..100 { + let mut budget = 200; + let file = AepFile { chunks: tree(&mut rng, 0, &mut budget), trailer: Cow::Borrowed(&[]) }; + let bytes = write(&file).expect("write"); + let parsed = crate::parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let a: Vec> = parsed.walk().map(|(_, c)| Node::from(c)).collect(); + let b: Vec> = index.walk().map(|(_, n)| n).collect(); + assert_eq!(a.len(), b.len()); + for (x, y) in a.iter().zip(&b) { + assert_eq!( + (x.id(), x.pad(), x.bytes(), x.list_type(), x.is_opaque_list()), + (y.id(), y.pad(), y.bytes(), y.list_type(), y.is_opaque_list()) + ); + assert_eq!(x.children().map(Node::to_chunk).collect::>(), y.children().map(Node::to_chunk).collect::>()); + for t in [b"Fold", b"tdgp", b"tdbs", b"Layr", b"btdk"] { + assert_eq!(x.find_list(t).map(Node::to_chunk), y.find_list(t).map(Node::to_chunk)); + assert_eq!(x.child_list(t).map(Node::to_chunk), y.child_list(t).map(Node::to_chunk)); + assert_eq!(x.is_list(t), y.is_list(t)); + } + for id in [b"cdat", b"Utf8", b"tdsn"] { + assert_eq!(x.child(id).map(Node::to_chunk), y.child(id).map(Node::to_chunk)); + } + let dx: Vec<(usize, Chunk<'_>)> = x.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + let dy: Vec<(usize, Chunk<'_>)> = y.descendants().map(|(d, n)| (d, n.to_chunk())).collect(); + assert_eq!(dx, dy); + assert_eq!(format!("{x:?}"), format!("{y:?}")); + } + } + } + + /// The index never reserves more than the input or the limit allows. + #[test] + fn the_index_never_reserves_more_than_the_input_or_the_limit_allows() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("index"); + assert_eq!(index.len(), 1000); + // The start (1024 slots) is cut to what the input can hold. + assert!(index.nodes.capacity() <= 1000, "{} slots for at most 1000 headers", index.nodes.capacity()); + // A short tail or a header that runs past the end does not raise the bound. + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + assert!(TreeIndex::parse(&big).is_err() && crate::parse(&big).is_err()); + // Under a tighter limit the parse fails as the chunk parser does, and the guess never + // exceeds the limit. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(TreeIndex::parse_with_limits(&data, tight).err(), Some(AepError::TooManyChunks)); + let small = Limits { max_depth: 64, max_chunks: 1000 }; + let index = TreeIndex::parse_with_limits(&data, small).expect("exactly at the limit"); + assert_eq!(index.len(), 1000); + // A limit below what the input holds bounds the reservation: 100 chunks, never more slots. + let mut wide = b"RIFX\0\0\0\0Egg!".to_vec(); + wide.extend_from_slice(b"LIST"); + wide.extend_from_slice(&u32::try_from(4 + 8 * 99).expect("small").to_be_bytes()); + wide.extend_from_slice(b"Fold"); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 99).flatten()); + let declared = u32::try_from(wide.len() - 8).expect("small"); + wide[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse_with_limits(&wide, Limits { max_depth: 64, max_chunks: 100 }).expect("100 chunks"); + assert_eq!(index.len(), 100); + assert!(index.nodes.capacity() <= 100); + // `from_file` is sized exactly. + let file = crate::parse(&wide).expect("parse"); + assert_eq!(TreeIndex::from_file(&file).expect("index").nodes.capacity(), 100); + } + + /// A file of `n` data chunks of `size` bytes each. + fn flat(n: usize, size: usize) -> Vec { + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + for _ in 0..n { + data.extend_from_slice(b"cdat"); + data.extend_from_slice(&u32::try_from(size).expect("small").to_be_bytes()); + data.extend(std::iter::repeat_n(0u8, size)); + } + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + data + } + + /// The index grows with the chunk count not the byte size. + #[test] + fn the_index_grows_with_the_chunk_count_not_the_byte_size() { + // 4 MB in 4 chunks: a few slots, not one per 32 bytes (that would be 131 072 slots, 4 MB). + let big = flat(4, 1 << 20); + let index = TreeIndex::parse(&big).expect("index"); + assert_eq!(index.len(), 4); + assert!(index.nodes.capacity() <= INITIAL_SLOTS, "{} slots for 4 chunks", index.nodes.capacity()); + // Many small chunks: at most twice the count (it doubles). + let many = flat(10_000, 2); + let index = TreeIndex::parse(&many).expect("index"); + assert_eq!(index.len(), 10_000); + assert!(index.nodes.capacity() <= 2 * 10_000, "{} slots for 10 000 chunks", index.nodes.capacity()); + same(&big, Limits::default()); + same(&many, Limits::default()); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Failed attempts count once and leave nothing behind. + #[test] + fn failed_attempts_count_once_and_leave_nothing_behind() { + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let headers = 1 + 50 + 1000; + let index = TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(TreeIndex::parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }).err(), Some(AepError::TooManyChunks)); + // Only the outermost list is left, opaque: every failed attempt was rolled back. + assert_eq!(index.len(), 1); + let top = index.chunks().next().expect("one top-level chunk"); + assert!(top.is_opaque_list() && top.bytes().map(<[u8]>::len) == Some(level.len() - 12)); + assert_eq!(write(&index.to_file()).expect("write"), data); + same(&data, Limits::default()); + } + + /// Depth beyond the limit stays opaque without recursion past it. + #[test] + fn depth_beyond_the_limit_stays_opaque_without_recursion_past_it() { + // 5000 nested lists: far deeper than the limit; everything below depth 64 stays bytes. + let mut level = b"LIST\0\0\0\x04tdgp".to_vec(); + for _ in 0..5000 { + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(level.len() + 4).expect("small").to_be_bytes()); + out.extend_from_slice(b"tdgp"); + out.extend_from_slice(&level); + level = out; + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let index = TreeIndex::parse(&data).expect("parse"); + assert_eq!(index.len(), 64); + assert_eq!(index.walk().map(|(d, _)| d).max(), Some(64)); + assert!(index.walk().last().is_some_and(|(_, n)| n.is_opaque_list())); + same(&data, Limits::default()); + // Converting and indexing a deep owned tree uses no recursion either. + let file = index.to_file(); + assert_eq!(TreeIndex::from_file(&file).expect("index").len(), 64); + } +} diff --git a/crates/aep/src/keys.rs b/crates/aep/src/keys.rs index 0c0c93ee6..6e107ad00 100644 --- a/crates/aep/src/keys.rs +++ b/crates/aep/src/keys.rs @@ -193,19 +193,10 @@ impl<'f> Keyframe<'f> { } } - /// The value of a 3-component spatial keyframe (Position), x, y and z in pixels. - pub fn spatial_value(&self) -> Option<[f64; 3]> { - let v = self.value()?; - if !matches!(self.kind(), KeyframeKind::Spatial { components: 3 }) { - return None; - } - Some([*v.first()?, *v.get(1)?, *v.get(2)?]) - } - /// The ease of the key: the four slots at byte 16 of a one-component plain record or at byte 24 /// of a spatial record or a mask path record. For linear and hold keys all four are 0 (a bezier key stores what /// `setTemporalEase` was given). Plain records with several components keep four slots per - /// component, see [`Keyframe::ease_component`] and [`Keyframe::plain_ease_slots`]. + /// component, see [`Keyframe::ease_component`]. pub fn ease(&self) -> Option { match self.kind() { KeyframeKind::Plain { components: 1 } => ease_at(self.record, 16), @@ -214,16 +205,6 @@ impl<'f> Keyframe<'f> { } } - /// The `4 n` ease `f64` of a plain record as stored (from byte `8 + 8 n`). With one component - /// they are [`Keyframe::ease`]; with several they are type-major, see - /// [`Keyframe::ease_component`]. - pub fn plain_ease_slots(&self) -> Option> { - match self.kind() { - KeyframeKind::Plain { components } => doubles(self.record, components.checked_mul(8)?.checked_add(8)?, components.checked_mul(4)?), - _ => None, - } - } - /// The ease of one value component. A plain record with `n` components stores the `4 n` ease /// doubles **type-major**: `[in speed x n][in influence x n][out speed x n][out influence x n]` /// starting at byte `8 + 8 n`, and component `i` uses slot `i` of each block (verified for the @@ -260,16 +241,6 @@ impl<'f> Keyframe<'f> { (0..n).map(|i| self.ease_component(i)).collect() } - /// The Euclidean ARGB distance to the next key that After Effects writes at byte 16 of a colour - /// (4-component spatial) record whose interpolation is not linear: 255 sqrt(2) = 360.6245 for - /// red to blue; 0 on the last key and on linear keys. `None` for other records. - pub fn color_distance(&self) -> Option { - match self.kind() { - KeyframeKind::Spatial { components: 4 } => be_f64(self.record, 16), - _ => None, - } - } - /// The decoded spatial record (Position: 128 bytes, colour: 152 bytes). pub fn spatial(&self) -> Option { let KeyframeKind::Spatial { components: n } = self.kind() else { return None }; @@ -300,11 +271,6 @@ pub struct ExpressionFlags { } impl ExpressionFlags { - /// True when bytes 76, 77 and 120 are all 1. - pub fn all_set(&self) -> bool { - self.byte_76 == 1 && self.byte_77 == 1 && self.byte_120 == 1 - } - /// True when byte 119 is 1. pub fn disabled(&self) -> bool { self.byte_119 == 1 @@ -324,11 +290,6 @@ impl<'f, 'a> Property<'f, 'a> { be_u32(self.chunk.child(b"tdsb")?.bytes()?, 0) } - /// True when `tdb4` byte 5 has bit 0 set: the property is static, it has no keyframes. - pub fn is_static_flag_set(&self) -> Option { - Some(self.header()?.get(5)? & 1 != 0) - } - /// The expression source: the raw UTF-8 text of the `Utf8` child that follows the value in the /// property (no length prefix, no engine tag; the engine is project-wide). pub fn expression(&self) -> Option { diff --git a/crates/aep/src/lib.rs b/crates/aep/src/lib.rs index a47cddfce..3e5dc2ceb 100644 --- a/crates/aep/src/lib.rs +++ b/crates/aep/src/lib.rs @@ -14,6 +14,8 @@ //! list) on damaged data instead of failing. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +// Ranges and borrowed bytes are passed around as plain tuples, as in EffectCraft, which allows this lint. +#![allow(clippy::type_complexity)] mod aepx; mod arbitrary; @@ -24,6 +26,7 @@ mod effects; mod error; mod footage; mod header; +pub mod index; mod keys; mod masks; mod matte; @@ -36,16 +39,17 @@ mod values; mod version; mod write; -pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; +pub use aepx::{AepxDrafts, index_aepx, is_aepx, parse_any, read_aepx, read_aepx_with_limits}; pub use arbitrary::{ CURVE_IDENTITY, CryptomatteData, CurvesData, ExtractorChannels, HueSaturationData, LevelsData, MeshData, curve_lut, identifier_channel_type, }; pub use camera::{CameraPosition, DEFAULT_CAMERA_ZOOM}; pub use chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; -pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue}; +pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue, Pard}; pub use error::AepError; pub use footage::{Footage, PsdLayer, SequenceInfo}; pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use index::{Draft, Node, NodeWalk, Nodes, Root, TreeIndex}; pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; pub use matte::{MatteMode, TrackMatte}; diff --git a/crates/aep/src/masks.rs b/crates/aep/src/masks.rs index 203ccefd5..42b52f5f4 100644 --- a/crates/aep/src/masks.rs +++ b/crates/aep/src/masks.rs @@ -6,9 +6,9 @@ //! property (`tdbs`, with a key list when animated) and `omks` with one `LIST shap` per keyframe (a //! single one when the path is static). -use crate::Chunk; use crate::bytes::{be_f32, be_u32, until_nul}; -use crate::typed::{KeyList, Layer, Project, PropGroup, PropNode, Property}; +use crate::index::Node; +use crate::typed::{KeyList, Layer, PropGroup, PropNode, Property}; use crate::values::{Tracked, read}; /// The mask mode (`mkif` `u32` at byte 4). @@ -93,8 +93,10 @@ pub struct ShapePath { } impl ShapePath { - /// Decodes a `LIST shap`; `None` when `shph` or the point list is malformed. - pub fn from_chunk(shap: &Chunk<'_>) -> Option { + /// Decodes a `LIST shap` (a [`Chunk`](crate::Chunk) or a [`Node`]); `None` when + /// `shph` or the point list is malformed. + pub fn from_chunk<'f, 'a: 'f>(shap: impl Into>) -> Option { + let shap = shap.into(); let shph = shap.child(b"shph")?.bytes()?; let bbox = [be_f32(shph, 4)?, be_f32(shph, 8)?, be_f32(shph, 12)?, be_f32(shph, 16)?]; let closed_flag = *shph.get(20)?; @@ -187,22 +189,12 @@ fn mask_from_atom(atom: &PropGroup<'_, '_>, mkif: Option<&[u8]>) -> Mask { } } if let Some(omks) = oms.child_list(b"omks") { - mask.shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); + mask.shapes = omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect(); } } mask } -impl<'f, 'a> Project<'f, 'a> { - /// The vertices of every path shape of a mask in pixels of the layer: mask boxes are normalised - /// to the **layer** size (the size of its source), not the composition's. `None` when the layer - /// has no source with a size. - pub fn mask_vertices_px(&self, layer: &Layer<'f, 'a>, mask: &Mask) -> Option>> { - let (width, height) = self.layer_source_size(layer)?; - Some(mask.shapes.iter().map(|s| s.vertices((f64::from(width), f64::from(height)))).collect()) - } -} - /// One mask with the views of its records, for readers that need more than [`Mask`]: the /// keyframes and expressions of the mask properties (in [`MaskAtom::atom`]: `ADBE Mask Feather`, /// `ADBE Mask Opacity`, `ADBE Mask Offset`) and the path keyframes paired with their shapes. @@ -246,7 +238,7 @@ impl<'f, 'a> Layer<'f, 'a> { if let Some(PropNode::Other(oms)) = atom.get("ADBE Mask Shape") { path = oms.child_list(b"tdbs").and_then(Property::from_chunk); if let Some(omks) = oms.child_list(b"omks") { - path_shapes = omks.children().iter().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); + path_shapes = omks.children().filter(|c| c.is_list(b"shap")).map(ShapePath::from_chunk).collect(); } } masks.push(MaskAtom { mask, atom, path, path_shapes }); diff --git a/crates/aep/src/read.rs b/crates/aep/src/read.rs index c769029e4..a969a3997 100644 --- a/crates/aep/src/read.rs +++ b/crates/aep/src/read.rs @@ -2,6 +2,7 @@ use std::borrow::Cow; +use crate::bytes::{array, be_u32}; use crate::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; use crate::error::AepError; @@ -36,10 +37,20 @@ pub fn parse(data: &[u8]) -> Result, AepError> { /// # Errors /// See [`parse`]; also fails with [`AepError::TooManyChunks`]. pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let end = container_end(data)?; + let mut parser = Parser { data, limits, count: 0 }; + let chunks = parser.sequence(12, end, 1)?; + let trailer = data.get(end..).unwrap_or(&[]); + Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) +} + +/// Checks the `RIFX`/`Egg!` header and returns where the container ends (its chunks fill +/// `12..end`; the trailer follows). Shared with [`crate::index::TreeIndex::parse`]. +pub(super) fn container_end(data: &[u8]) -> Result { if id4(data, 0) != Some(*b"RIFX") { return Err(AepError::NotRifx); } - let declared = u64::from(be32(data, 4).ok_or(AepError::NotRifx)?); + let declared = u64::from(be_u32(data, 4).ok_or(AepError::NotRifx)?); let form = id4(data, 8).ok_or(AepError::NotRifx)?; if &form != b"Egg!" { return Err(AepError::BadFormType(form)); @@ -49,20 +60,12 @@ pub fn parse_with_limits(data: &[u8], limits: Limits) -> Result, Aep return Err(AepError::Truncated { declared, available }); } // `declared <= available < data.len()`, so the end is inside the input. - let end = usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge)?; - let mut parser = Parser { data, limits, count: 0 }; - let chunks = parser.sequence(12, end, 1)?; - let trailer = data.get(end..).unwrap_or(&[]); - Ok(AepFile { chunks, trailer: Cow::Borrowed(trailer) }) + usize::try_from(declared).map_err(|_| AepError::TooLarge)?.checked_add(8).ok_or(AepError::TooLarge) } -fn be32(data: &[u8], at: usize) -> Option { - id4(data, at).map(u32::from_be_bytes) -} - -fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { - let b = data.get(at..at.checked_add(4)?)?; - Some([*b.first()?, *b.get(1)?, *b.get(2)?, *b.get(3)?]) +/// The 4-byte id at `at`, `None` when the input is too short. +pub(super) fn id4(data: &[u8], at: usize) -> Option<[u8; 4]> { + array::<4>(data, at) } struct Parser<'a> { @@ -74,13 +77,16 @@ struct Parser<'a> { impl<'a> Parser<'a> { /// Parses the sibling chunks that exactly fill `start..end`. fn sequence(&mut self, start: usize, end: usize, depth: usize) -> Result>, AepError> { + // Sized once: a well-formed sequence fills exactly this many slots. `try_reserve` so a + // limit raised past what the target can allocate fails nothing here (the Vec then grows). let mut out = Vec::new(); + let _ = out.try_reserve_exact(self.capacity_for(start, end)); let mut pos = start; while pos < end { let short = AepError::Malformed { offset: pos, what: "bytes left over that are too short for a chunk header" }; let body_start = pos.checked_add(8).filter(|b| *b <= end).ok_or(short)?; let id = id4(self.data, pos).ok_or(AepError::Malformed { offset: pos, what: "short read" })?; - let size = be32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; + let size = be_u32(self.data, pos + 4).ok_or(AepError::Malformed { offset: pos, what: "short read" })? as usize; let avail = end - body_start; if size > avail { return Err(AepError::ChunkTooLarge { offset: pos, size: size as u64, available: avail as u64 }); @@ -103,6 +109,26 @@ impl<'a> Parser<'a> { Ok(out) } + /// The slots to reserve for the sequence `start..end`: its header count, bounded by the input + /// (one per 8 bytes at most) and by the chunks the limit still allows. + fn capacity_for(&self, start: usize, end: usize) -> usize { + self.count_siblings(start, end).min(self.limits.max_chunks.saturating_sub(self.count)) + } + + /// The number of chunk headers that tile `start..end` the way [`Self::sequence`] walks them, + /// stopping at the first that does not fit. Never more than `(end - start) / 8`. + fn count_siblings(&self, start: usize, end: usize) -> usize { + let mut n = 0usize; + let mut pos = start; + while let Some(body_start) = pos.checked_add(8).filter(|b| *b <= end) { + let Some(size) = be_u32(self.data, pos.saturating_add(4)) else { break }; + let Some(next) = usize::try_from(size).ok().and_then(|s| body_start.checked_add(s)).filter(|e| *e <= end) else { break }; + n = n.saturating_add(1); + pos = if size % 2 == 1 && next < end { next.saturating_add(1) } else { next }; + } + n + } + /// Like [`Self::sequence`] but a structural problem yields `None` (the caller keeps bytes). fn tentative(&mut self, start: usize, end: usize, depth: usize) -> Result>>, AepError> { match self.sequence(start, end, depth) { @@ -140,3 +166,113 @@ impl<'a> Parser<'a> { Ok(Body::Data(Cow::Borrowed(self.slice(start, end)))) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Every children `Vec` in the tree, depth first. + fn child_vecs<'f, 'a>(chunks: &'f [Chunk<'a>], out: &mut Vec<&'f Vec>>) { + for c in chunks { + match &c.body { + Body::List { children, .. } | Body::Chunks(children) => { + out.push(children); + child_vecs(children, out); + } + Body::Data(_) | Body::OpaqueList { .. } => {} + } + } + } + + /// Every sibling list is allocated once at its exact length. + #[test] + fn every_sibling_list_is_allocated_once_at_its_exact_length() { + let inner: Vec> = (0..37u8).map(|i| Chunk::data(*b"cdat", vec![i; usize::from(i % 5)])).collect(); + let middle: Vec> = (0..9).map(|_| Chunk::list(*b"tdbs", inner.clone())).collect(); + let file = AepFile { chunks: vec![Chunk::data(*b"head", vec![0; 3]), Chunk::list(*b"Fold", middle)], trailer: Cow::Borrowed(&[]) }; + let bytes = crate::write(&file).expect("write"); + let parsed = parse(&bytes).expect("parse"); + assert_eq!(crate::write(&parsed).expect("rewrite"), bytes); + let mut vecs = vec![&parsed.chunks]; + child_vecs(&parsed.chunks, &mut vecs); + assert_eq!(vecs.len(), 11); + // `try_reserve_exact` gives exactly the requested capacity with std's allocator today; a + // Vec grown push by push would have rounded up to a power of two (37 -> 64, 9 -> 16). + for v in vecs { + assert_eq!(v.capacity(), v.len(), "a sibling Vec grew past its length"); + } + } + + /// The preallocation is bounded by the input and the chunk limit. + #[test] + fn the_preallocation_is_bounded_by_the_input_and_the_chunk_limit() { + // 1000 empty headers: the most chunks the bytes can hold. + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, data.len()), 1000); + assert!(parser.count_siblings(12, data.len()) <= (data.len() - 12) / 8); + // A header whose size runs past the end stops the count; a stray tail is not counted. + assert_eq!(parser.count_siblings(12, data.len() - 3), 999); + let mut big = data.clone(); + big[16..20].copy_from_slice(&u32::MAX.to_be_bytes()); + let parser = Parser { data: &big, limits: Limits::default(), count: 0 }; + assert_eq!(parser.count_siblings(12, big.len()), 0); + let f = parse(&data).expect("parse"); + assert_eq!((f.chunks.len(), f.chunks.capacity()), (1000, 1000)); + // Under a tighter limit the parse fails as before, without sizing for more than allowed. + let tight = Limits { max_depth: 64, max_chunks: 10 }; + assert_eq!(parse_with_limits(&data, tight), Err(AepError::TooManyChunks)); + // The reservation never exceeds what the chunk limit still allows. + let mut parser = Parser { data: &data, limits: tight, count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 10); + parser.count = 7; + assert_eq!(parser.capacity_for(12, data.len()), 3); + parser.count = 11; + assert_eq!(parser.capacity_for(12, data.len()), 0); + let parser = Parser { data: &data, limits: Limits::default(), count: 0 }; + assert_eq!(parser.capacity_for(12, data.len()), 1000); + } + + /// `LIST Fold` around `inner`, followed by 6 bytes too short for a header, so the list's + /// sequence fails and the list is kept opaque. + fn broken_level(inner: &[u8]) -> Vec { + let mut body = b"Fold".to_vec(); + body.extend_from_slice(inner); + body.extend_from_slice(&[0xee; 6]); + let mut out = b"LIST".to_vec(); + out.extend_from_slice(&u32::try_from(body.len()).expect("small").to_be_bytes()); + out.extend_from_slice(&body); + out + } + + /// Nested lists with broken tails parse each header once. + #[test] + fn nested_lists_with_broken_tails_parse_each_header_once() { + // A wide, well-formed list at the bottom, then 50 levels that each fail and fall back to + // opaque: every level re-parses its child before giving up. + let mut wide = b"Fold".to_vec(); + wide.extend(std::iter::repeat_n(*b"abcd\0\0\0\0", 1000).flatten()); + let mut level = b"LIST".to_vec(); + level.extend_from_slice(&u32::try_from(wide.len()).expect("small").to_be_bytes()); + level.extend_from_slice(&wide); + for _ in 0..50 { + level = broken_level(&level); + } + let mut data = b"RIFX\0\0\0\0Egg!".to_vec(); + data.extend_from_slice(&level); + let declared = u32::try_from(data.len() - 8).expect("small"); + data[4..8].copy_from_slice(&declared.to_be_bytes()); + // 1 + 50 + 1000 headers in the file: the parse counts each exactly once (the count spans + // every attempt), so the work stays linear in the input. + let headers = 1 + 50 + 1000; + let f = parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers }).expect("parse"); + assert_eq!(parse_with_limits(&data, Limits { max_depth: 64, max_chunks: headers - 1 }), Err(AepError::TooManyChunks)); + // The outermost list keeps its bytes, and the file writes back unchanged. + let [top] = f.chunks.as_slice() else { panic!("one top-level chunk") }; + assert!(matches!(&top.body, Body::OpaqueList { list_type, data } if list_type == b"Fold" && data.len() + 12 == level.len())); + assert_eq!(crate::write(&f).expect("write"), data); + } +} diff --git a/crates/aep/src/settings.rs b/crates/aep/src/settings.rs index 1193dbc39..440983805 100644 --- a/crates/aep/src/settings.rs +++ b/crates/aep/src/settings.rs @@ -2,8 +2,10 @@ //! linear-light switches, the time display and the frame count start. Cases 620-634: one change //! each against the base project 620; every field below is the only byte (or chunk) that changes. +use crate::bytes::be_u32; use crate::chunk::AepFile; use crate::footage::json_string; +use crate::index::{Node, Root, TreeIndex}; /// The time display style (`nhed` byte 8, feet + frames byte 11, film type byte 13). #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -54,7 +56,17 @@ pub struct ProjectSettings { impl ProjectSettings { /// Reads the settings from the top-level chunks. pub fn read(file: &AepFile<'_>) -> ProjectSettings { - let nhed = file.chunk(b"nhed").and_then(|c| c.bytes()); + Self::of(Root::from(file)) + } + + /// Reads the settings from the top-level chunks of an indexed file. + pub fn read_index(index: &TreeIndex<'_>) -> ProjectSettings { + Self::of(Root::from(index)) + } + + /// Reads the settings from the top-level chunks of either tree. + pub fn of(file: Root<'_, '_>) -> ProjectSettings { + let nhed = file.chunk(b"nhed").and_then(Node::bytes); let byte = |i: usize| nhed.and_then(|b| b.get(i).copied()); let bits_per_channel = byte(15).and_then(|v| match v { 0 => Some(8), @@ -73,9 +85,9 @@ impl ProjectSettings { 1 => Some(1), _ => None, }); - let flag = |id: &[u8; 4]| file.chunk(id).and_then(|c| c.bytes()).and_then(|b| b.first().copied()) == Some(1); - let working_gamma_24 = file.chunk(b"dwga").and_then(|c| c.bytes()).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); - let scene_compensation = file.chunk(b"acer").and_then(|c| c.bytes()).and_then(|b| b.first().copied()).map(|v| v != 0); + let flag = |id: &[u8; 4]| file.chunk(id).and_then(Node::bytes).and_then(|b| b.first().copied()) == Some(1); + let working_gamma_24 = file.chunk(b"dwga").and_then(Node::bytes).and_then(|b| b.get(..4)).map(|b| b != [0, 0, 0, 0]); + let scene_compensation = file.chunk(b"acer").and_then(Node::bytes).and_then(|b| b.first().copied()).map(|v| v != 0); ProjectSettings { bits_per_channel, working_space: working_space(file), @@ -90,9 +102,10 @@ impl ProjectSettings { } /// The working space: the `Utf8` right after `PwCs`. -fn working_space(file: &AepFile<'_>) -> Option { - let at = file.chunks.iter().position(|c| c.is(b"PwCs"))?; - let json = file.chunks.get(at.checked_add(1)?).filter(|c| c.is(b"Utf8"))?.bytes()?; +fn working_space(file: Root<'_, '_>) -> Option { + let mut chunks = file.chunks(); + chunks.find(|c| c.is(b"PwCs"))?; + let json = chunks.next().filter(|c| c.is(b"Utf8"))?.bytes()?; let trimmed: Vec = json.iter().copied().filter(|b| !b.is_ascii_whitespace() && *b != 0).collect(); if trimmed == b"{}" { return Some(WorkingSpace::None); @@ -130,7 +143,7 @@ fn base64_decode(text: &str) -> Option> { /// The profile description (ICC.1 tag `desc`): a v2 `desc` (ASCII) or a v4 `mluc` (its first /// record, UTF-16BE). `None` when the profile or the tag is cut short. fn icc_description(icc: &[u8]) -> Option { - let be32 = |at: usize| icc.get(at..at.checked_add(4)?).and_then(|b| b.try_into().ok()).map(u32::from_be_bytes); + let be32 = |at: usize| be_u32(icc, at); let count = be32(128)?; let tag = (0..count.min(1024)).find_map(|i| { let at = 132usize.checked_add(usize::try_from(i).ok()?.checked_mul(12)?)?; @@ -141,13 +154,13 @@ fn icc_description(icc: &[u8]) -> Option { let body = icc.get(off..off.checked_add(len)?)?; let text = match body.get(..4)? { b"desc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(8..12)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 8)?).ok()?; let ascii = body.get(12..12usize.checked_add(n)?)?; String::from_utf8_lossy(ascii.split(|b| *b == 0).next().unwrap_or_default()).into_owned() } b"mluc" => { - let n = usize::try_from(u32::from_be_bytes(body.get(20..24)?.try_into().ok()?)).ok()?; - let at = usize::try_from(u32::from_be_bytes(body.get(24..28)?.try_into().ok()?)).ok()?; + let n = usize::try_from(be_u32(body, 20)?).ok()?; + let at = usize::try_from(be_u32(body, 24)?).ok()?; let units: Vec = body.get(at..at.checked_add(n)?)?.as_chunks::<2>().0.iter().map(|p| u16::from_be_bytes(*p)).collect(); String::from_utf16_lossy(&units) } diff --git a/crates/aep/src/shapes.rs b/crates/aep/src/shapes.rs index 864cfa55b..28b572506 100644 --- a/crates/aep/src/shapes.rs +++ b/crates/aep/src/shapes.rs @@ -10,7 +10,7 @@ //! points are in pixels relative to the layer origin. use crate::masks::ShapePath; -use crate::typed::{Layer, PropGroup, PropNode, Property}; +use crate::typed::{Layer, PropGroup, PropNode}; use crate::values::{Tracked, read, read_opt}; /// The stroke line cap (`ADBE Vector Stroke Line Cap`, stored as a double; butt is the default and @@ -145,7 +145,7 @@ fn number(group: &PropGroup<'_, '_>, name: &str) -> Option> { fn paths_of(group: &PropGroup<'_, '_>) -> Vec { let Some(PropNode::Other(oms)) = group.get("ADBE Vector Shape") else { return Vec::new() }; let Some(omks) = oms.child_list(b"omks") else { return Vec::new() }; - omks.children().iter().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() + omks.children().filter(|c| c.is_list(b"shap")).filter_map(ShapePath::from_chunk).collect() } fn item(match_name: String, group: &PropGroup<'_, '_>) -> ShapeItem { @@ -269,12 +269,3 @@ impl<'f, 'a> Layer<'f, 'a> { .collect() } } - -impl<'f, 'a> Property<'f, 'a> { - /// The ARGB fractions of a static colour property (4 components stored as doubles 0..255). - pub fn color_argb(&self) -> Option<[f64; 4]> { - let v = self.static_value()?; - let [a, r, g, b] = *v.as_slice().first_chunk::<4>()?; - Some([a / 255.0, r / 255.0, g / 255.0, b / 255.0]) - } -} diff --git a/crates/aep/src/textdoc.rs b/crates/aep/src/textdoc.rs index 373fd40db..1e7569b67 100644 --- a/crates/aep/src/textdoc.rs +++ b/crates/aep/src/textdoc.rs @@ -5,8 +5,11 @@ //! as `(` + BOM `fe ff` + UTF-16BE + `)` with `\` escapes. [`parse_text_dict`] reads it (depth //! limited, never panics) and [`Layer::text_document`] pulls out the verified fields. -use crate::Chunk; +use std::borrow::Cow; +use std::sync::LazyLock; + use crate::bytes::{be_u32, text}; +use crate::index::Node; use crate::typed::{Comp, Layer, Property}; /// How deep dictionaries and arrays may nest before the parser gives up. @@ -73,15 +76,99 @@ impl TextValue { _ => None, } } +} - fn key_text(&self) -> String { +/// The text of a key the way the dictionary stores it: a `/name` as written, any other value in +/// its canonical form (`06` and `+6` are `6`, `1.0` is `1`, a string its decoded text). +fn owned_key(value: &V<'_>) -> String { + match value { + V::Name(s) | V::Token(s) => text(s), + V::Str(utf16) => decode_utf16be(utf16), + V::Int(i) => i.to_string(), + V::Float(f) => f.to_string(), + V::Bool(b) => b.to_string(), + V::Bytes(b) => text(b), + V::Dict(_) | V::Array(_) => String::new(), + } +} + +/// A value of the text dictionary as [`TextDocument::parse`] reads it: it borrows names, bare +/// words and strings from the body, so a document is read without a `String` per token. A key is +/// borrowed when it is a `/name` (every key After Effects writes); its text is the same as the +/// [`TextValue`] key's. [`V::into_owned`] gives the public [`TextValue`]. +#[derive(Debug, Clone)] +enum V<'b> { + Dict(Vec<(Cow<'b, str>, V<'b>)>), + Array(Vec>), + /// A string that started with the byte order mark: the UTF-16BE bytes after it, unescaped. + Str(Cow<'b, [u8]>), + Bytes(Cow<'b, [u8]>), + Name(&'b [u8]), + Int(i64), + Float(f64), + Bool(bool), + Token(&'b [u8]), +} + +impl<'b> V<'b> { + /// The value of a dictionary key (the last one when a key repeats). + fn get(&self, key: &str) -> Option<&V<'b>> { match self { - Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), - Self::Int(i) => i.to_string(), - Self::Float(f) => f.to_string(), - Self::Bool(b) => b.to_string(), - Self::Bytes(b) => text(b), - Self::Dict(_) | Self::Array(_) => String::new(), + Self::Dict(pairs) => pairs.iter().rev().find(|(k, _)| k == key).map(|(_, v)| v), + _ => None, + } + } + + fn at(&self, index: usize) -> Option<&V<'b>> { + match self { + Self::Array(items) => items.get(index), + _ => None, + } + } + + /// The decoded text of a string value. + fn string(&self) -> Option { + match self { + Self::Str(utf16) => Some(decode_utf16be(utf16)), + _ => None, + } + } + + fn as_f64(&self) -> Option { + match self { + Self::Int(i) => Some(*i as f64), + Self::Float(f) => Some(*f), + _ => None, + } + } + + fn as_i64(&self) -> Option { + match self { + Self::Int(i) => Some(*i), + _ => None, + } + } + + /// The key a value makes when it is written in a key position. + fn into_key(self) -> Cow<'b, str> { + match self { + Self::Name(s) => String::from_utf8_lossy(s), + other => Cow::Owned(owned_key(&other)), + } + } + + /// The public form. Recursion is bounded: a parsed value nests at most [`MAX_DEPTH`] + 1 deep. + fn into_owned(self) -> TextValue { + match self { + Self::Dict(pairs) => TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.into_owned(), v.into_owned())).collect()), + Self::Array(items) => TextValue::Array(items.into_iter().map(V::into_owned).collect()), + Self::Str(utf16) => TextValue::Str(decode_utf16be(&utf16)), + Self::Bytes(b) => TextValue::Bytes(b.into_owned()), + Self::Name(s) => TextValue::Name(text(s)), + Self::Int(i) => TextValue::Int(i), + Self::Float(f) => TextValue::Float(f), + Self::Bool(b) => TextValue::Bool(b), + Self::Token(s) => TextValue::Token(text(s)), } } } @@ -89,32 +176,44 @@ impl TextValue { struct Parser<'b> { b: &'b [u8], i: usize, + /// The pairs of the dictionaries being read, innermost last: a dictionary's pairs are moved + /// into a vector of their exact size when it closes (one allocation per dictionary instead of + /// one per doubling). + pairs: Vec<(Cow<'b, str>, V<'b>)>, + /// The same for the items of arrays. + items: Vec>, } +/// Whitespace between tokens. fn is_ws(c: u8) -> bool { matches!(c, b' ' | b'\t' | b'\r' | b'\n') } +/// A byte that ends a bare word. fn is_delimiter(c: u8) -> bool { is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') } -impl Parser<'_> { +impl<'b> Parser<'b> { + /// The next byte, `None` at the end. fn peek(&self) -> Option { self.b.get(self.i).copied() } + /// Moves past whitespace. fn skip_ws(&mut self) { while self.peek().is_some_and(is_ws) { self.i = self.i.saturating_add(1); } } + /// True when the unread bytes start with `s`. fn starts_with(&self, s: &[u8]) -> bool { self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) } - fn word(&mut self) -> &[u8] { + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &'b [u8] { let start = self.i; while self.peek().is_some_and(|c| !is_delimiter(c)) { self.i = self.i.saturating_add(1); @@ -122,79 +221,106 @@ impl Parser<'_> { self.b.get(start..self.i).unwrap_or(&[]) } - fn parse(&mut self, depth: usize) -> Option { + /// The body of a `( ... )` string after the `(`, with `\` escapes resolved: borrowed when it + /// has none. `None` when the string or an escape is not closed. + fn string_body(&mut self) -> Option> { + let rest = self.b.get(self.i..)?; + let stop = rest.iter().position(|c| matches!(c, b')' | b'\\'))?; + let head = rest.get(..stop)?; + self.i = self.i.saturating_add(stop); + if self.peek()? == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Borrowed(head)); + } + let mut out = head.to_vec(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + return Some(Cow::Owned(out)); + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option> { self.skip_ws(); if depth > MAX_DEPTH { return None; } if self.starts_with(b"<<") { self.i = self.i.saturating_add(2); - let mut pairs = Vec::new(); + // On an error the pairs read so far stay on the stack: the error ends the whole parse. + let mark = self.pairs.len(); loop { self.skip_ws(); if self.starts_with(b">>") { self.i = self.i.saturating_add(2); - return Some(TextValue::Dict(pairs)); + return Some(V::Dict(self.pairs.drain(mark..).collect())); } let key = self.parse(depth.saturating_add(1))?; let value = self.parse(depth.saturating_add(1))?; - pairs.push((key.key_text(), value)); + self.pairs.push((key.into_key(), value)); } } match self.peek()? { b'[' => { self.i = self.i.saturating_add(1); - let mut items = Vec::new(); + let mark = self.items.len(); loop { self.skip_ws(); if self.peek()? == b']' { self.i = self.i.saturating_add(1); - return Some(TextValue::Array(items)); + return Some(V::Array(self.items.drain(mark..).collect())); } - items.push(self.parse(depth.saturating_add(1))?); + let item = self.parse(depth.saturating_add(1))?; + self.items.push(item); } } b'(' => { self.i = self.i.saturating_add(1); - let mut out = Vec::new(); - loop { - let c = self.peek()?; - if c == b'\\' { - out.push(*self.b.get(self.i.saturating_add(1))?); - self.i = self.i.saturating_add(2); - } else if c == b')' { - self.i = self.i.saturating_add(1); - break; - } else { - out.push(c); - self.i = self.i.saturating_add(1); - } - } - Some(match out.strip_prefix(&[0xfe, 0xff]) { - Some(utf16) => TextValue::Str(decode_utf16be(utf16)), - None => TextValue::Bytes(out), + let body = self.string_body()?; + Some(if body.starts_with(&[0xfe, 0xff]) { + V::Str(match body { + Cow::Borrowed(b) => Cow::Borrowed(b.get(2..).unwrap_or(&[])), + Cow::Owned(mut v) => { + let n = v.len().min(2); + v.drain(..n); + Cow::Owned(v) + } + }) + } else { + V::Bytes(body) }) } b'/' => { self.i = self.i.saturating_add(1); - Some(TextValue::Name(text(self.word()))) + Some(V::Name(self.word())) } _ => { let word = self.word(); if word.is_empty() { return None; } - let w = text(word); - Some(match w.as_str() { - "true" => TextValue::Bool(true), - "false" => TextValue::Bool(false), + // A word that is not UTF-8 reads as text with replacement characters, which is + // neither a boolean nor a number: a token. + let Ok(w) = std::str::from_utf8(word) else { return Some(V::Token(word)) }; + Some(match w { + "true" => V::Bool(true), + "false" => V::Bool(false), _ => match w.parse::() { - Ok(i) => TextValue::Int(i), + Ok(i) => V::Int(i), // `parse` also takes `nan`, `inf` and overflowing exponents; those are not // numbers a text document holds, so they stay tokens and read as missing. Err(_) => match w.parse::() { - Ok(f) if f.is_finite() => TextValue::Float(f), - _ => TextValue::Token(w), + Ok(f) if f.is_finite() => V::Float(f), + _ => V::Token(word), }, }, }) @@ -203,6 +329,7 @@ impl Parser<'_> { } } +/// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). fn decode_utf16be(bytes: &[u8]) -> String { let (pairs, rest) = bytes.as_chunks::<2>(); let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); @@ -213,11 +340,9 @@ fn decode_utf16be(bytes: &[u8]) -> String { s } -/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). -/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was -/// read before is returned. -pub fn parse_text_dict(body: &[u8]) -> TextValue { - let mut parser = Parser { b: body, i: 0 }; +/// The top-level `key value` pairs of a `btdk` body, borrowed (see [`parse_text_dict`]). +fn parse_dict(body: &[u8]) -> V<'_> { + let mut parser = Parser { b: body, i: 0, pairs: Vec::new(), items: Vec::new() }; let mut pairs = Vec::new(); loop { parser.skip_ws(); @@ -226,13 +351,20 @@ pub fn parse_text_dict(body: &[u8]) -> TextValue { } let Some(key) = parser.parse(0) else { break }; let Some(value) = parser.parse(0) else { break }; - pairs.push((key.key_text(), value)); + pairs.push((key.into_key(), value)); } - TextValue::Dict(pairs) + V::Dict(pairs) +} + +/// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). +/// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was +/// read before is returned. +pub fn parse_text_dict(body: &[u8]) -> TextValue { + parse_dict(body).into_owned() } /// The run style slots (`RUN/6/0[n]/0/0/6/`) that are decoded; every other slot is compared -/// with its default (`style_defaults`) and listed in +/// with its default (`STYLE_DEFAULTS`) and listed in /// [`TextDocument::undecoded_style_fields`] when it differs. pub const DECODED_STYLE_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "11", "12", "13", "18", "35", "36", "52", "53", "54", "56", "57", "58", "62", "63", "70"]; @@ -241,25 +373,27 @@ pub const DECODED_STYLE_SLOTS: &[&str] = pub const DECODED_PARAGRAPH_SLOTS: &[&str] = &["0", "1", "2", "3", "4", "5", "8", "9", "21", "29", "33"]; /// A number slot. -fn val(v: f64) -> TextValue { - TextValue::Float(v) +fn val(v: f64) -> V<'static> { + V::Float(v) } -fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { - TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) +/// A dictionary value with these keys, for the built-in defaults. +fn dict(pairs: Vec<(&'static str, V<'static>)>) -> V<'static> { + V::Dict(pairs.into_iter().map(|(k, v)| (Cow::Borrowed(k), v)).collect()) } -fn numbers(v: &[f64]) -> TextValue { - TextValue::Array(v.iter().map(|x| val(*x)).collect()) +/// An array of numbers, for the built-in defaults. +fn numbers(v: &[f64]) -> V<'static> { + V::Array(v.iter().map(|x| val(*x)).collect()) } /// The default of every run style slot that is not decoded and not neutral (zero, `false`, /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. -fn style_defaults() -> TextValue { - let paint = - |a: f64| dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); - let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); +/// Built once. +static STYLE_DEFAULTS: LazyLock> = LazyLock::new(|| { + let paint = |a: f64| dict(vec![("99", V::Name(b"SimplePaint")), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]); + let class = |name: &'static [u8]| dict(vec![("99", V::Name(name))]); dict(vec![ ("37", val(2.0)), ("41", val(2.0)), @@ -269,24 +403,24 @@ fn style_defaults() -> TextValue { ("46", val(7.0)), ("49", val(-1.0)), ("50", val(-1.0)), - ("55", class("SimpleBlender")), + ("55", class(b"SimpleBlender")), ("64", val(4.0)), ("71", val(4.0)), - ("77", TextValue::Bool(true)), - ("78", TextValue::Bool(true)), + ("77", V::Bool(true)), + ("78", V::Bool(true)), ("79", paint(0.0)), ("82", val(3.0)), ("83", val(3.0)), - ("86", class("SimpleCustomFeature")), + ("86", class(b"SimpleCustomFeature")), ("87", val(100.0)), - ("88", TextValue::Bool(true)), + ("88", V::Bool(true)), ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), - ("96", TextValue::Bool(true)), + ("96", V::Bool(true)), ]) -} +}); -/// The same for the paragraph slots (110, 243-245). -fn paragraph_defaults() -> TextValue { +/// The same for the paragraph slots (110, 243-245). Built once. +static PARAGRAPH_DEFAULTS: LazyLock> = LazyLock::new(|| { dict(vec![ ("6", val(1.0)), ("7", val(1.2)), @@ -294,30 +428,35 @@ fn paragraph_defaults() -> TextValue { ("11", val(2.0)), ("12", val(2.0)), ("14", val(36.0)), - ("15", TextValue::Bool(true)), + ("15", V::Bool(true)), ("16", val(0.5)), ("17", numbers(&[0.8, 1.0, 1.33])), ("18", numbers(&[0.0, 0.0, 0.0])), ("19", numbers(&[1.0, 1.0, 1.0])), ("20", val(6.0)), ("22", val(1.0)), - ("23", TextValue::Bool(true)), + ("23", V::Bool(true)), ("24", val(1.0)), - ("26", TextValue::Bool(true)), + ("26", V::Bool(true)), ("31", val(36.0)), ("35", val(1.0)), ("40", val(2.0)), ]) -} +}); -/// Equal values; numbers compare by value (`0` equals `0.0`). -fn same(a: &TextValue, b: &TextValue) -> bool { +/// Equal values; numbers compare by value (`0` equals `0.0`), names and words by their text, +/// strings by their decoded text. +fn same(a: &V<'_>, b: &V<'_>) -> bool { match (a, b) { - (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), - (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Dict(x), V::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (V::Array(x), V::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + (V::Str(x), V::Str(y)) => x == y || decode_utf16be(x) == decode_utf16be(y), + (V::Bytes(x), V::Bytes(y)) => x == y, + (V::Name(x), V::Name(y)) | (V::Token(x), V::Token(y)) => x == y || text(x) == text(y), + (V::Bool(x), V::Bool(y)) => x == y, _ => match (a.as_f64(), b.as_f64()) { (Some(p), Some(q)) => p == q, - _ => a == b, + _ => false, }, } } @@ -326,12 +465,12 @@ fn same(a: &TextValue, b: &TextValue) -> bool { /// listed when it holds something else (a neutral value carries nothing that could be lost). /// `/nil` is how an untouched document stores a slot without a value (244: paragraph `/27`, `/28`, /// `/36`). -fn is_neutral(v: &TextValue) -> bool { +fn is_neutral(v: &V<'_>) -> bool { match v { - TextValue::Name(n) => n == "nil", - TextValue::Bool(b) => !b, - TextValue::Array(a) => a.is_empty(), - TextValue::Dict(d) => d.is_empty(), + V::Name(n) => *n == b"nil", + V::Bool(b) => !b, + V::Array(a) => a.is_empty(), + V::Dict(d) => d.is_empty(), other => other.as_f64() == Some(0.0), } } @@ -356,13 +495,13 @@ pub const MAX_UNDECODED_FIELDS: usize = 64; /// Adds to `out` the slots of `actual` that are not decoded and differ from `sampled`, each once /// and at most [`MAX_UNDECODED_FIELDS`] in all. -fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { - let TextValue::Dict(pairs) = actual else { return }; +fn undecoded_fields(actual: &V<'_>, sampled: Option<&V<'_>>, decoded: &[&str], neutral: &[(&str, &[i64])], out: &mut Vec) { + let V::Dict(pairs) = actual else { return }; for (key, value) in pairs { if out.len() >= MAX_UNDECODED_FIELDS { return; } - if decoded.contains(&key.as_str()) || out.contains(key) { + if decoded.contains(&key.as_ref()) || out.iter().any(|o| o == key) { continue; } if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { @@ -373,7 +512,7 @@ fn undecoded_fields(actual: &TextValue, sampled: Option<&TextValue>, decoded: &[ None => !is_neutral(value), }; if differs { - out.push(key.clone()); + out.push(key.to_string()); } } } @@ -433,13 +572,13 @@ pub struct KernRun { /// The manual kerning list of a text run dictionary (`RUN/8/0`): entries `<< /0 << /0 amount >> /1 /// length >>` (554 kerns each of `HHHH` by 50; the closing `\r` has an entry without an amount). -fn kern_runs(run: &TextValue) -> Vec { - let Some(TextValue::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; +fn kern_runs(run: &V<'_>) -> Vec { + let Some(V::Array(items)) = run.get("8").and_then(|v| v.get("0")) else { return Vec::new() }; let runs: Vec = items .iter() .filter_map(|e| { let length = e.get("1").and_then(usize_of)?; - let amount = e.get("0").and_then(|v| v.get("0")).and_then(TextValue::as_f64).filter(|v| v.is_finite()); + let amount = e.get("0").and_then(|v| v.get("0")).and_then(V::as_f64).filter(|v| v.is_finite()); Some(KernRun { length, amount }) }) .collect(); @@ -602,7 +741,7 @@ pub struct StyleRun { impl StyleRun { /// The style of the sampled untouched document (244), used for a damaged first run. - pub(crate) const SAMPLED: StyleRun = StyleRun { + pub const SAMPLED: StyleRun = StyleRun { length: 0, font_size: 100.0, fill_argb: [1.0; 4], @@ -645,24 +784,26 @@ pub struct TextBox { pub height: f64, } -fn colour_of(style: &TextValue, slot: &str) -> Option<[f64; 4]> { +/// The RGBA colour in a style slot (`slot/0/1`), `None` when it is not four numbers. +fn colour_of(style: &V<'_>, slot: &str) -> Option<[f64; 4]> { let c = style.get(slot)?.get("0")?.get("1")?; Some([c.at(0)?.as_f64()?, c.at(1)?.as_f64()?, c.at(2)?.as_f64()?, c.at(3)?.as_f64()?]) } -fn usize_of(v: &TextValue) -> Option { +/// A non-negative integer value as `usize`. +fn usize_of(v: &V<'_>) -> Option { usize::try_from(v.as_i64()?).ok() } /// A finite number slot of a style, `default` when it is missing or not a finite number. -fn num(style: &TextValue, slot: &str, default: f64) -> f64 { - style.get(slot).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(default) +fn num(style: &V<'_>, slot: &str, default: f64) -> f64 { + style.get(slot).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(default) } /// A boolean slot of a style, `default` when it is missing or not a boolean. -fn flag(style: &TextValue, slot: &str, default: bool) -> bool { +fn flag(style: &V<'_>, slot: &str, default: bool) -> bool { match style.get(slot) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, } } @@ -676,9 +817,9 @@ struct RunNotes { } /// The paragraph runs of a text run dictionary (`RUN/5/0`). -fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(paragraph_defaults()); +fn paragraph_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("5").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*PARAGRAPH_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -686,7 +827,7 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { continue; }; let para = e.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("5")); - let justification = match para.and_then(|p| p.get("0")).and_then(TextValue::as_i64) { + let justification = match para.and_then(|p| p.get("0")).and_then(V::as_i64) { Some(0) => Justification::Left, Some(1) => Justification::Right, Some(2) => Justification::Center, @@ -701,12 +842,12 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } }; if let Some(p) = para { - undecoded_fields(p, sampled.as_ref(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); + undecoded_fields(p, sampled, DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES, &mut notes.paragraph_fields); } - let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(TextValue::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); - let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(TextValue::as_i64).unwrap_or(default); + let num = |slot: &str| para.and_then(|p| p.get(slot)).and_then(V::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0); + let int = |slot: &str, default: i64| para.and_then(|p| p.get(slot)).and_then(V::as_i64).unwrap_or(default); let flag = |slot: &str, default: bool| match para.and_then(|p| p.get(slot)) { - Some(TextValue::Bool(b)) => *b, + Some(V::Bool(b)) => *b, _ => default, }; out.push(ParagraphRun { @@ -728,9 +869,9 @@ fn paragraph_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { } /// The style runs of a text run dictionary (`RUN/6/0`). -fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { - let Some(TextValue::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; - let sampled = Some(style_defaults()); +fn style_runs(run: &V<'_>, notes: &mut RunNotes) -> Vec { + let Some(V::Array(items)) = run.get("6").and_then(|v| v.get("0")) else { return Vec::new() }; + let sampled = Some(&*STYLE_DEFAULTS); let mut out: Vec = Vec::with_capacity(items.len()); for e in items { let Some(length) = e.get("1").and_then(usize_of) else { @@ -743,31 +884,31 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { out.push(StyleRun { length, ..prev }); continue; }; - let font_size = style.get("1").and_then(TextValue::as_f64); + let font_size = style.get("1").and_then(V::as_f64); let fill_argb = colour_of(style, "53"); let font_index = style.get("0").and_then(usize_of); if font_size.is_none() || fill_argb.is_none() || font_index.is_none() { notes.incomplete = notes.incomplete.saturating_add(1); } - undecoded_fields(style, sampled.as_ref(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); + undecoded_fields(style, sampled, DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES, &mut notes.style_fields); out.push(StyleRun { length, font_size: font_size.unwrap_or(prev.font_size), fill_argb: fill_argb.unwrap_or(prev.fill_argb), stroke_argb: colour_of(style, "54"), - tracking: style.get("8").and_then(TextValue::as_f64).unwrap_or(0.0), - faux_bold: matches!(style.get("2"), Some(TextValue::Bool(true))), - faux_italic: matches!(style.get("3"), Some(TextValue::Bool(true))), - caps: match style.get("12").and_then(TextValue::as_i64) { + tracking: style.get("8").and_then(V::as_f64).unwrap_or(0.0), + faux_bold: matches!(style.get("2"), Some(V::Bool(true))), + faux_italic: matches!(style.get("3"), Some(V::Bool(true))), + caps: match style.get("12").and_then(V::as_i64) { None | Some(0) => Caps::Normal, Some(1) => Caps::Small, Some(2) => Caps::All, Some(other) => Caps::Other(other), }, - auto_leading: matches!(style.get("4"), Some(TextValue::Bool(true))), - leading: style.get("5").and_then(TextValue::as_f64), + auto_leading: matches!(style.get("4"), Some(V::Bool(true))), + leading: style.get("5").and_then(V::as_f64), font_index: font_index.unwrap_or(prev.font_index), - baseline: match style.get("13").and_then(TextValue::as_i64) { + baseline: match style.get("13").and_then(V::as_i64) { None | Some(0) => FontBaseline::Normal, Some(1) => FontBaseline::Superscript, Some(2) => FontBaseline::Subscript, @@ -776,22 +917,22 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { horizontal_scale: num(style, "6", 1.0), vertical_scale: num(style, "7", 1.0), baseline_shift: num(style, "9", 0.0), - auto_kern: match style.get("11").and_then(TextValue::as_i64) { + auto_kern: match style.get("11").and_then(V::as_i64) { Some(0) => AutoKern::None, None | Some(1) => AutoKern::Metrics, Some(2) => AutoKern::Optical, Some(other) => AutoKern::Other(other), }, ligatures: flag(style, "18", false), - baseline_direction: style.get("35").and_then(TextValue::as_i64).unwrap_or(1), + baseline_direction: style.get("35").and_then(V::as_i64).unwrap_or(1), tsume: num(style, "36", 0.0), no_break: flag(style, "52", false), apply_fill: flag(style, "56", true), apply_stroke: flag(style, "57", false), stroke_over_fill: flag(style, "58", true), - line_join: style.get("62").and_then(TextValue::as_i64).unwrap_or(0), + line_join: style.get("62").and_then(V::as_i64).unwrap_or(0), stroke_width: num(style, "63", 1.0), - digit_set: style.get("70").and_then(TextValue::as_i64).unwrap_or(0), + digit_set: style.get("70").and_then(V::as_i64).unwrap_or(0), }); } out @@ -799,13 +940,13 @@ fn style_runs(run: &TextValue, notes: &mut RunNotes) -> Vec { /// The box of paragraph text from the frame dictionary `/0/8/0[0]/0`: key `/1/0` holds 16 x, y pairs /// that trace the rectangle, and `/2/0` is 1; both keys are absent for point text. -fn text_box(top: &TextValue) -> Option { +fn text_box(top: &V<'_>) -> Option { let frame = top.get("0")?.get("8")?.get("0")?.at(0)?.get("0")?; if frame.get("2")?.get("0")?.as_i64()? != 1 { return None; } - let TextValue::Array(coords) = frame.get("1")?.get("0")? else { return None }; - let numbers: Vec = coords.iter().map(TextValue::as_f64).collect::>()?; + let V::Array(coords) = frame.get("1")?.get("0")? else { return None }; + let numbers: Vec = coords.iter().map(V::as_f64).collect::>()?; let (pairs, _) = numbers.as_chunks::<2>(); if pairs.is_empty() { return None; @@ -819,18 +960,16 @@ fn text_box(top: &TextValue) -> Option { impl TextDocument { /// Decodes the body of a `btdk` list. pub fn parse(body: &[u8]) -> Option { - let top = parse_text_dict(body); + let top = parse_dict(body); let run = top.get("1")?.get("1")?.at(0)?.get("0")?; - let raw_text = run.get("0")?.as_str()?.to_string(); + let raw_text = run.get("0")?.string()?; let style = run.get("6")?.get("0")?.at(0)?.get("0")?.get("0")?.get("6")?; let fill_argb = colour_of(style, "53")?; let font_index = usize::try_from(style.get("0")?.as_i64()?).ok()?; let mut fonts = Vec::new(); - if let Some(TextValue::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { - fonts = list - .iter() - .map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(TextValue::as_str).unwrap_or_default().to_string()) - .collect(); + if let Some(V::Array(list)) = top.get("0").and_then(|v| v.get("1")).and_then(|v| v.get("0")) { + fonts = + list.iter().map(|f| f.get("0").and_then(|v| v.get("0")).and_then(|v| v.get("0")).and_then(V::string).unwrap_or_default()).collect(); } let mut notes = RunNotes::default(); let paragraphs = paragraph_runs(run, &mut notes); @@ -858,7 +997,7 @@ impl TextDocument { .and_then(|v| v.get("0")) .and_then(|v| v.get("2")) .and_then(|v| v.get("1")) - .and_then(TextValue::as_i64), + .and_then(V::as_i64), }) } @@ -882,23 +1021,32 @@ impl TextDocument { } impl<'f, 'a> Layer<'f, 'a> { - /// The text document of a text layer (`LIST btdk` below `ADBE Text Properties`). + /// The Source Text record `LIST btds` (`ADBE Text Document`). It is a direct child of the + /// `ADBE Text Properties` group, which is a direct child of the layer's root `tdgp` + /// (`Layr/tdgp/tdgp/btds` in every sample, and where the exporter writes it), so only that + /// level is searched rather than the whole layer. + fn source_text_record(&self) -> Option> { + let root = self.chunk.child_list(b"tdgp")?; + root.children().filter(|g| g.is_list(b"tdgp")).find_map(|g| g.child_list(b"btds")) + } + + /// The text document of a text layer: the first `LIST btdk` in the Source Text record `btds` + /// (the same documents [`Layer::text_document_count`] counts). pub fn text_document(&self) -> Option { - TextDocument::parse(self.chunk.find_list(b"btdk")?.bytes()?) + TextDocument::parse(self.source_text_record()?.find_list(b"btdk")?.bytes()?) } /// The Source Text property record: the `tdbs` in `btds[ADBE Text Document]` (a placeholder /// in every sample: no keys, no expression). Source Text keyframes and expressions were not /// sampled; a `list` (keys) or an expression in it marks them. pub fn source_text_property(&self) -> Option> { - let btds = self.chunk.descendants().map(|(_, c)| c).find(|c| c.is_list(b"btds"))?; - Property::from_chunk(btds.child_list(b"tdbs")?) + Property::from_chunk(self.source_text_record()?.child_list(b"tdbs")?) } - /// The number of text documents (`LIST btdk`) of the layer: 1 in every sample; more would be - /// one per Source Text keyframe (not sampled). + /// The number of text documents (`LIST btdk`) in the Source Text record: 1 in every sample; + /// more would be one per Source Text keyframe (not sampled). pub fn text_document_count(&self) -> usize { - self.chunk.descendants().filter(|(_, c)| c.is_list(b"btdk")).count() + self.source_text_record().map_or(0, |btds| btds.descendants().filter(|(_, c)| c.is_list(b"btdk")).count()) } } @@ -943,10 +1091,24 @@ pub enum CueType { Other(u32), } +impl<'f, 'a> Node<'f, 'a> { + /// The marker list `LIST mrst` of a marker holder: a layer (`LIST Layr`) or a composition's + /// `SecL` list. A layer's `mrst` is a direct child of its root `tdgp` (`ADBE Marker`, where + /// After Effects and the exporter write it), so only that place is looked at; most layers have + /// none, and searching their whole property tree for it was a large part of an import. Any + /// other holder is searched depth-first. + pub fn marker_list(self) -> Option> { + if self.is_list(b"Layr") { + return self.child_list(b"tdgp")?.child_list(b"mrst"); + } + self.find_list(b"mrst") + } +} + /// Reads the markers below `holder` (a layer or the composition's `SecL` list): the `mrst` list has a /// property whose 16-byte records give the times and one `Nmrd` list per marker with the strings. -fn markers_in(holder: &Chunk<'_>) -> Vec { - let Some(mrst) = holder.find_list(b"mrst") else { return Vec::new() }; +fn markers_in(holder: Node<'_, '_>) -> Vec { + let Some(mrst) = holder.marker_list() else { return Vec::new() }; let Some(prop) = mrst.child_list(b"tdbs").and_then(Property::from_chunk) else { return Vec::new() }; let Some(keys) = prop.keys() else { return Vec::new() }; let scale = prop.time_scale().unwrap_or(0); @@ -956,11 +1118,11 @@ fn markers_in(holder: &Chunk<'_>) -> Vec { .filter_map(|(k, nmrd)| { let time_units = k.time_units()?; // The five strings, in the order comment, chapter, URL, frame target, cue point name. - let mut strings = nmrd.children().iter().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); + let mut strings = nmrd.children().filter(|c| c.is(b"Utf8")).map(|c| c.bytes().map(text).unwrap_or_default()); let mut next = || strings.next().unwrap_or_default(); let (comment, chapter, url, frame_target, cue_point_name) = (next(), next(), next(), next(), next()); let extras: Vec = strings.collect(); - let header = nmrd.child(b"NmHd").and_then(Chunk::bytes); + let header = nmrd.child(b"NmHd").and_then(Node::bytes); let duration_seconds = header.and_then(|h| { let (duration, per_second) = (be_u32(h, 8)?, be_u32(h, 12)?); (per_second != 0).then(|| f64::from(duration) / f64::from(per_second)) @@ -1003,9 +1165,647 @@ impl<'f, 'a> Comp<'f, 'a> { } } +/// The dictionary parser and the defaults as they were before they borrowed from the body (#3276), +/// kept as the reference the differential tests compare with. +#[cfg(test)] +mod oracle { + use super::{MAX_DEPTH, MAX_UNDECODED_FIELDS, TextValue}; + use crate::bytes::text; + + pub(super) fn same(a: &TextValue, b: &TextValue) -> bool { + match (a, b) { + (TextValue::Dict(x), TextValue::Dict(y)) => x.len() == y.len() && x.iter().all(|(k, v)| y.iter().any(|(k2, v2)| k == k2 && same(v, v2))), + (TextValue::Array(x), TextValue::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(p, q)| same(p, q)), + _ => match (a.as_f64(), b.as_f64()) { + (Some(p), Some(q)) => p == q, + _ => a == b, + }, + } + } + + pub(super) fn is_neutral(v: &TextValue) -> bool { + match v { + TextValue::Name(n) => n == "nil", + TextValue::Bool(b) => !b, + TextValue::Array(a) => a.is_empty(), + TextValue::Dict(d) => d.is_empty(), + other => other.as_f64() == Some(0.0), + } + } + + pub(super) fn undecoded_fields( + actual: &TextValue, + sampled: Option<&TextValue>, + decoded: &[&str], + neutral: &[(&str, &[i64])], + out: &mut Vec, + ) { + let TextValue::Dict(pairs) = actual else { return }; + for (key, value) in pairs { + if out.len() >= MAX_UNDECODED_FIELDS { + return; + } + if decoded.contains(&key.as_str()) || out.contains(key) { + continue; + } + if neutral.iter().any(|(slot, values)| slot == key && value.as_i64().is_some_and(|v| values.contains(&v))) { + continue; + } + let differs = match sampled.and_then(|s| s.get(key)) { + Some(default) => !same(value, default), + None => !is_neutral(value), + }; + if differs { + out.push(key.clone()); + } + } + } + + impl TextValue { + /// The canonical text of a key, as the owning parser built it. + fn key_text(&self) -> String { + match self { + Self::Name(s) | Self::Str(s) | Self::Token(s) => s.clone(), + Self::Int(i) => i.to_string(), + Self::Float(f) => f.to_string(), + Self::Bool(b) => b.to_string(), + Self::Bytes(b) => text(b), + Self::Dict(_) | Self::Array(_) => String::new(), + } + } + } + + struct Parser<'b> { + b: &'b [u8], + i: usize, + } + + /// Whitespace between tokens. + fn is_ws(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\r' | b'\n') + } + + /// A byte that ends a bare word. + fn is_delimiter(c: u8) -> bool { + is_ws(c) || matches!(c, b'/' | b'[' | b']' | b'<' | b'>' | b'(' | b')') + } + + impl Parser<'_> { + /// The next byte, `None` at the end. + fn peek(&self) -> Option { + self.b.get(self.i).copied() + } + + /// Moves past whitespace. + fn skip_ws(&mut self) { + while self.peek().is_some_and(is_ws) { + self.i = self.i.saturating_add(1); + } + } + + /// True when the unread bytes start with `s`. + fn starts_with(&self, s: &[u8]) -> bool { + self.b.get(self.i..).is_some_and(|rest| rest.starts_with(s)) + } + + /// The bare word at the position (up to the next delimiter), borrowed from the body. + fn word(&mut self) -> &[u8] { + let start = self.i; + while self.peek().is_some_and(|c| !is_delimiter(c)) { + self.i = self.i.saturating_add(1); + } + self.b.get(start..self.i).unwrap_or(&[]) + } + + /// The value at the position, `None` when it is malformed or nested deeper than the limit. + fn parse(&mut self, depth: usize) -> Option { + self.skip_ws(); + if depth > MAX_DEPTH { + return None; + } + if self.starts_with(b"<<") { + self.i = self.i.saturating_add(2); + let mut pairs = Vec::new(); + loop { + self.skip_ws(); + if self.starts_with(b">>") { + self.i = self.i.saturating_add(2); + return Some(TextValue::Dict(pairs)); + } + let key = self.parse(depth.saturating_add(1))?; + let value = self.parse(depth.saturating_add(1))?; + pairs.push((key.key_text(), value)); + } + } + match self.peek()? { + b'[' => { + self.i = self.i.saturating_add(1); + let mut items = Vec::new(); + loop { + self.skip_ws(); + if self.peek()? == b']' { + self.i = self.i.saturating_add(1); + return Some(TextValue::Array(items)); + } + items.push(self.parse(depth.saturating_add(1))?); + } + } + b'(' => { + self.i = self.i.saturating_add(1); + let mut out = Vec::new(); + loop { + let c = self.peek()?; + if c == b'\\' { + out.push(*self.b.get(self.i.saturating_add(1))?); + self.i = self.i.saturating_add(2); + } else if c == b')' { + self.i = self.i.saturating_add(1); + break; + } else { + out.push(c); + self.i = self.i.saturating_add(1); + } + } + Some(match out.strip_prefix(&[0xfe, 0xff]) { + Some(utf16) => TextValue::Str(decode_utf16be(utf16)), + None => TextValue::Bytes(out), + }) + } + b'/' => { + self.i = self.i.saturating_add(1); + Some(TextValue::Name(text(self.word()))) + } + _ => { + let word = self.word(); + if word.is_empty() { + return None; + } + let w = text(word); + Some(match w.as_str() { + "true" => TextValue::Bool(true), + "false" => TextValue::Bool(false), + _ => match w.parse::() { + Ok(i) => TextValue::Int(i), + // `parse` also takes `nan`, `inf` and overflowing exponents; those are not + // numbers a text document holds, so they stay tokens and read as missing. + Err(_) => match w.parse::() { + Ok(f) if f.is_finite() => TextValue::Float(f), + _ => TextValue::Token(w), + }, + }, + }) + } + } + } + } + + /// A big-endian UTF-16 string as text (unpaired surrogates become U+FFFD). + fn decode_utf16be(bytes: &[u8]) -> String { + let (pairs, rest) = bytes.as_chunks::<2>(); + let units = pairs.iter().map(|p| u16::from_be_bytes(*p)); + let mut s: String = char::decode_utf16(units).map(|r| r.unwrap_or(char::REPLACEMENT_CHARACTER)).collect(); + if !rest.is_empty() { + s.push(char::REPLACEMENT_CHARACTER); + } + s + } + + /// Parses the body of a `btdk` list into its top-level `key value` pairs (as a [`TextValue::Dict`]). + /// Parsing stops silently at the first syntax error or at nesting deeper than 64 levels; what was + /// read before is returned. + pub(super) fn parse_text_dict(body: &[u8]) -> TextValue { + let mut parser = Parser { b: body, i: 0 }; + let mut pairs = Vec::new(); + loop { + parser.skip_ws(); + if parser.i >= body.len() { + break; + } + let Some(key) = parser.parse(0) else { break }; + let Some(value) = parser.parse(0) else { break }; + pairs.push((key.key_text(), value)); + } + TextValue::Dict(pairs) + } + + fn val(v: f64) -> TextValue { + TextValue::Float(v) + } + + /// A dictionary value with these keys, for the built-in defaults. + fn dict(pairs: Vec<(&str, TextValue)>) -> TextValue { + TextValue::Dict(pairs.into_iter().map(|(k, v)| (k.to_string(), v)).collect()) + } + + /// An array of numbers, for the built-in defaults. + fn numbers(v: &[f64]) -> TextValue { + TextValue::Array(v.iter().map(|x| val(*x)).collect()) + } + + /// The default of every run style slot that is not decoded and not neutral (zero, `false`, + /// empty, `/nil`): the value an untouched point text holds in every sample (110, 243-245). A slot + /// with a neutral default is left out; [`undecoded_fields`] treats a missing default as neutral. + pub(super) fn style_defaults() -> TextValue { + let paint = |a: f64| { + dict(vec![("99", TextValue::Name("SimplePaint".into())), ("0", dict(vec![("0", val(1.0)), ("1", numbers(&[1.0, 1.0, 1.0, a]))]))]) + }; + let class = |name: &str| dict(vec![("99", TextValue::Name(name.into()))]); + dict(vec![ + ("37", val(2.0)), + ("41", val(2.0)), + ("43", dict(vec![("0", val(0.5))])), + ("44", val(2.0)), + ("45", val(2.0)), + ("46", val(7.0)), + ("49", val(-1.0)), + ("50", val(-1.0)), + ("55", class("SimpleBlender")), + ("64", val(4.0)), + ("71", val(4.0)), + ("77", TextValue::Bool(true)), + ("78", TextValue::Bool(true)), + ("79", paint(0.0)), + ("82", val(3.0)), + ("83", val(3.0)), + ("86", class("SimpleCustomFeature")), + ("87", val(100.0)), + ("88", TextValue::Bool(true)), + ("95", dict(vec![("0", val(0.0)), ("1", val(0.0))])), + ("96", TextValue::Bool(true)), + ]) + } + + /// The same for the paragraph slots (110, 243-245). + pub(super) fn paragraph_defaults() -> TextValue { + dict(vec![ + ("6", val(1.0)), + ("7", val(1.2)), + ("10", val(6.0)), + ("11", val(2.0)), + ("12", val(2.0)), + ("14", val(36.0)), + ("15", TextValue::Bool(true)), + ("16", val(0.5)), + ("17", numbers(&[0.8, 1.0, 1.33])), + ("18", numbers(&[0.0, 0.0, 0.0])), + ("19", numbers(&[1.0, 1.0, 1.0])), + ("20", val(6.0)), + ("22", val(1.0)), + ("23", TextValue::Bool(true)), + ("24", val(1.0)), + ("26", TextValue::Bool(true)), + ("31", val(36.0)), + ("35", val(1.0)), + ("40", val(2.0)), + ]) + } +} + #[cfg(test)] mod tests { use super::*; + use crate::Chunk; + + /// A `LIST` of this type holding `kids`. + fn list(t: &[u8; 4], kids: Vec>) -> Chunk<'static> { + Chunk::list(*t, kids) + } + + /// A document with every kind of value, close to what After Effects writes: names, numbers, + /// a UTF-16 string with escapes, a plain string, nested dictionaries and arrays, an undecoded + /// style slot, a font list and a box frame. + const DOC: &[u8] = b"/0 << /1 << /0 [ << /0 << /0 << /0 (\xfe\xff\0M\0y\0r) >> /1 1 >> >> << /0 << /0 << /0 (\xfe\xff\0A\0\\)\0b) >> >> >> ] >> \ + /8 << /0 [ << /0 << /1 << /0 [ 0 0 10 0 10 5 0 5 ] >> /2 << /0 1 /1 2 >> >> >> ] >> >> \ + /1 << /1 [ << /0 << /0 (\xfe\xff\0H\0i\0\r) /5 << /0 [ << /0 << /0 << /5 << /0 2 /9 false /24 0 /34 7 /17 [ .8 1.0 1.33 ] >> >> >> /1 3 >> ] >> \ + /6 << /0 [ << /0 << /0 << /6 << /0 1 /1 36.5 /53 << /0 << /0 1 /1 [ 1 0.5 0 1 ] >> >> /79 << /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >> /68 1 /90 /x /91 (plain) /92 word >> >> >> /1 3 >> ] >> \ + /8 << /0 [ << /0 << /0 50 >> /1 2 >> << /1 1 >> ] >> >> >> ] >>"; + + /// A deterministic xorshift generator: the tests need varied input, not a dependency. + struct Rng(u64); + + impl Rng { + /// The next pseudo-random number (xorshift). + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + /// A number below `n` (0 for `n == 0`). + fn below(&mut self, n: usize) -> usize { + usize::try_from(self.next() % (n as u64)).unwrap_or(0) + } + } + + /// The borrowing parser reads what the owning one read. + #[test] + fn the_borrowing_parser_reads_what_the_owning_one_read() { + // Every key form the owned keys canonicalise (`06`, `+6`, `-0`, `1.0`, `1e2`, strings, + // booleans, containers), names that are not UTF-8, escapes, unclosed strings and escapes, + // non-finite numbers, and bare words. + let parts: &[&[u8]] = &[ + b"<<", + b">>", + b"[", + b"]", + b"/0", + b"/1", + b"/nil", + b"/SimplePaint", + b"/\xff\xfe", + b"/\xc3\xa9", + b"/", + b"06", + b"+6", + b"-0", + b"1.0", + b"1.50", + b"1e2", + b"-2.5", + b".8", + b"inf", + b"nan", + b"1e999", + b"9223372036854775808", + b"true", + b"false", + b"(abc)", + b"(\xfe\xff\0A\0B)", + b"(\xfe\xff\0A\0)", + b"(a\\)b)", + b"(\\", + b"(\xfe\xff\\)\0)", + b"(\\\xfe\xff\0C)", + b"()", + b"(\xfe\xff)", + b"(unclosed", + b"word", + b"\xc3\x28", + b"\xff", + b" ", + b"\n", + b"\t\r", + b"<", + b">", + b")", + ]; + // A value: mostly well formed, with a random part in its place now and then. + /// Appends one generated value to `out`: mostly well formed, now and then a random part. + fn value(rng: &mut Rng, parts: &[&[u8]], depth: usize, out: &mut Vec) { + let pick = |rng: &mut Rng| parts.get(rng.below(parts.len())).copied().unwrap_or_default(); + match rng.below(if depth > 6 { 5 } else { 10 }) { + 0 => out.extend_from_slice(pick(rng)), + 1..=4 => out.extend_from_slice([&b"/0"[..], b"1", b"(\xfe\xff\0z)", b"0.5", b"/nil"].get(rng.below(5)).copied().unwrap_or_default()), + 5..=7 => { + out.extend_from_slice(b"<< "); + for _ in 0..rng.below(5) { + let key = if rng.below(4) == 0 { + pick(rng) + } else { + [&b"/0"[..], b"/1", b"/6", b"/53"].get(rng.below(4)).copied().unwrap_or_default() + }; + out.extend_from_slice(key); + out.push(b' '); + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.extend_from_slice(b">>"); + } + } + _ => { + out.extend_from_slice(b"[ "); + for _ in 0..rng.below(5) { + value(rng, parts, depth + 1, out); + out.push(b' '); + } + if rng.below(20) != 0 { + out.push(b']'); + } + } + } + } + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let (mut read, mut nested) = (0, 0); + for case in 0..40_000 { + let mut body = Vec::new(); + if case % 2 == 0 { + for _ in 0..rng.below(60) { + body.extend_from_slice(parts.get(rng.below(parts.len())).copied().unwrap_or_default()); + if rng.below(3) != 0 { + body.push(b' '); + } + } + } else { + for _ in 0..1 + rng.below(4) { + body.extend_from_slice(b"/0 "); + value(&mut rng, parts, 0, &mut body); + body.push(b'\n'); + } + } + let parsed = parse_text_dict(&body); + assert_eq!(parsed, oracle::parse_text_dict(&body), "case {case}: {}", String::from_utf8_lossy(&body)); + let _ = TextDocument::parse(&body); + if let TextValue::Dict(pairs) = &parsed { + read += usize::from(!pairs.is_empty()); + nested += usize::from(pairs.iter().any(|(_, v)| matches!(v, TextValue::Dict(d) if !d.is_empty()))); + } + } + // The input is not all rejected at the first token. + assert!(read > 25_000 && nested > 3_000, "{read} cases read something, {nested} a dictionary"); + // Every cut of a realistic document, and nesting past the limit. + for end in 0..=DOC.len() { + let body = DOC.get(..end).unwrap_or_default(); + assert_eq!(parse_text_dict(body), oracle::parse_text_dict(body), "cut at {end}"); + let _ = TextDocument::parse(body); + } + for depth in [63, 64, 65, 66, 200] { + let body = [b"/k ".to_vec(), b"<< /a ".repeat(depth), b"1".to_vec(), b" >>".repeat(depth), b" /z 2".to_vec()].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "depth {depth}"); + let body = [b"/k ".to_vec(), b"[ ".repeat(depth), b"]".repeat(depth)].concat(); + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "array depth {depth}"); + } + // Dictionaries and arrays mixed (`<< /a [ << /a [ ...`), closed, and left open exactly + // at the limit. + for depth in 60..=68 { + let open: Vec = (0..depth).flat_map(|i| if i % 2 == 0 { &b"<< /a "[..] } else { &b"[ "[..] }).copied().collect(); + let close: Vec = (0..depth).rev().flat_map(|i| if i % 2 == 0 { &b" >>"[..] } else { &b" ]"[..] }).copied().collect(); + for body in + [[b"/k ".to_vec(), open.clone(), b"1".to_vec(), close, b" /z 2".to_vec()].concat(), [b"/k ".to_vec(), open, b"1".to_vec()].concat()] + { + assert_eq!(parse_text_dict(&body), oracle::parse_text_dict(&body), "mixed depth {depth}"); + let _ = TextDocument::parse(&body); + } + } + } + + /// A realistic document is read in full. + #[test] + fn a_realistic_document_is_read_in_full() { + let doc = TextDocument::parse(DOC).expect("a document"); + assert_eq!((doc.text.as_str(), doc.font_size, doc.fill_argb, doc.font.as_deref()), ("Hi", 36.5, [1.0, 0.5, 0.0, 1.0], Some("A)b"))); + assert_eq!(doc.fonts, ["Myr", "A)b"]); + assert_eq!(doc.paragraphs.first().map(|p| (p.length, p.justification, p.auto_hyphenate)), Some((3, Justification::Center, false))); + // `/68 1` and `/34 7` are neutral; `/90 /x`, `/91 (plain)` and `/92 word` differ from the + // (missing, neutral) default; `/79` equals its default. + assert_eq!(doc.undecoded_style_fields, ["90", "91", "92"]); + assert!(doc.undecoded_paragraph_fields.is_empty()); + assert_eq!(doc.kerning, [KernRun { length: 2, amount: Some(50.0) }, KernRun { length: 1, amount: None }]); + assert_eq!(doc.text_box, Some(TextBox { x: 0.0, y: 0.0, width: 10.0, height: 5.0 })); + assert_eq!(doc.line_orientation, Some(2)); + } + + /// The defaults are the sampled ones and built once. + #[test] + fn the_defaults_are_the_sampled_ones_and_built_once() { + assert_eq!(STYLE_DEFAULTS.clone().into_owned(), oracle::style_defaults()); + assert_eq!(PARAGRAPH_DEFAULTS.clone().into_owned(), oracle::paragraph_defaults()); + assert!(std::ptr::eq(&*STYLE_DEFAULTS, &*STYLE_DEFAULTS)); + } + + /// Names keys and strings are borrowed from the body. + #[test] + fn names_keys_and_strings_are_borrowed_from_the_body() { + let V::Dict(pairs) = parse_dict(b"/a /b /c (\xfe\xff\0x) /d (plain) /e word /f 1.5 /g (\\(esc) 07 1") else { panic!("a dict") }; + // Every `/name` key, name, string without escapes and bare word points into the body. + assert!(pairs.iter().take(6).all(|(k, _)| matches!(k, Cow::Borrowed(_)))); + assert!(matches!(pairs.first().map(|p| &p.1), Some(V::Name(b"b")))); + assert!(matches!(pairs.get(1).map(|p| &p.1), Some(V::Str(Cow::Borrowed(b"\0x"))))); + assert!(matches!(pairs.get(2).map(|p| &p.1), Some(V::Bytes(Cow::Borrowed(b"plain"))))); + assert!(matches!(pairs.get(3).map(|p| &p.1), Some(V::Token(b"word")))); + // An escaped string is copied; a number key is written in its canonical form. + assert!(matches!(pairs.get(5).map(|p| &p.1), Some(V::Bytes(Cow::Owned(b))) if b == b"(esc")); + assert_eq!(pairs.get(6).map(|p| p.0.as_ref()), Some("7")); + // Containers hold what was read. + let V::Dict(outer) = parse_dict(b"/0 << /a 1 /b 2 /c 3 /d 4 /e 5 >> /1 [ 1 2 3 4 5 6 7 8 9 ]") else { panic!("a dict") }; + assert!(matches!(outer.first().map(|p| &p.1), Some(V::Dict(d)) if d.len() == 5 && d.capacity() >= d.len())); + assert!(matches!(outer.get(1).map(|p| &p.1), Some(V::Array(a)) if a.len() == 9 && a.capacity() >= a.len())); + } + + /// Slot checks agree with the owned ones. + #[test] + fn slot_checks_agree_with_the_owned_ones() { + // Slots: defaults, decoded, neutral-listed and unknown ones. Values: each default as + // stored (also as `Int` where it is `2.0`), near misses, neutral values and every kind. + let slots = ["0", "6", "7", "17", "24", "34", "35", "37", "43", "55", "68", "77", "79", "86", "95", "96", "200", "x"]; + let values: &[&[u8]] = &[ + b"2", + b"2.0", + b"7", + b"1", + b"0", + b"-1", + b"0.5", + b"1.2", + b"36", + b"true", + b"false", + b"/nil", + b"/x", + b"/SimpleBlender", + b"[ ]", + b"<< >>", + b"[ .8 1 1.33 ]", + b"[ 0.8 1.0 ]", + b"[ 1 1 1 ]", + b"<< /0 .5 >>", + b"<< /0 0 /1 0 >>", + b"<< /1 0 /0 0 >>", + b"<< /99 /SimpleBlender >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 0 ] >> >>", + b"<< /99 /SimplePaint /0 << /0 1 /1 [ 1 1 1 1 ] >> >>", + b"(plain)", + b"(\xfe\xff\0a)", + b"word", + b"/\xff", + ]; + let mut rng = Rng(0x2545_f491_4f6c_dd1d); + let mut listed = 0; + for case in 0..20_000 { + let mut body = b"/s << ".to_vec(); + for _ in 0..rng.below(12) { + body.push(b'/'); + body.extend_from_slice(slots.get(rng.below(slots.len())).copied().unwrap_or_default().as_bytes()); + body.push(b' '); + body.extend_from_slice(values.get(rng.below(values.len())).copied().unwrap_or_default()); + body.push(b' '); + } + body.extend_from_slice(b">>"); + let (new, old) = (parse_dict(&body), oracle::parse_text_dict(&body)); + let (Some(new), Some(old)) = (new.get("s"), old.get("s")) else { panic!("case {case}: a dict") }; + for (sampled, old_sampled, decoded, neutral) in [ + (&*STYLE_DEFAULTS, oracle::style_defaults(), DECODED_STYLE_SLOTS, NEUTRAL_STYLE_VALUES), + (&*PARAGRAPH_DEFAULTS, oracle::paragraph_defaults(), DECODED_PARAGRAPH_SLOTS, NEUTRAL_PARAGRAPH_VALUES), + ] { + let (mut a, mut b) = (Vec::new(), Vec::new()); + undecoded_fields(new, Some(sampled), decoded, neutral, &mut a); + oracle::undecoded_fields(old, Some(&old_sampled), decoded, neutral, &mut b); + assert_eq!(a, b, "case {case}: {}", String::from_utf8_lossy(&body)); + listed += a.len(); + } + // Every pair of values compares and reads as neutral alike. + if let (V::Dict(x), TextValue::Dict(y)) = (new, old) { + for ((_, p), (_, q)) in x.iter().zip(y) { + assert_eq!(is_neutral(p), oracle::is_neutral(q), "case {case}"); + for ((_, r), (_, t)) in x.iter().zip(y) { + assert_eq!(same(p, r), oracle::same(q, t), "case {case}"); + } + } + } + } + assert!(listed > 10_000, "{listed} slots listed"); + } + + /// Values compare by their text. + #[test] + fn values_compare_by_their_text() { + // Two names that are not UTF-8 read as the same text with replacement characters, as the + // owned values did. + assert!(same(&V::Name(b"\xff"), &V::Name(b"\xfe"))); + assert!(!same(&V::Name(b"a"), &V::Name(b"b"))); + assert!(same(&V::Str(Cow::Borrowed(b"\xd8\x00")), &V::Str(Cow::Owned(b"\xd8\x01".to_vec())))); + assert!(same(&V::Int(0), &V::Float(0.0)) && !same(&V::Int(0), &V::Bool(false)) && !same(&V::Name(b"x"), &V::Token(b"x"))); + assert!( + is_neutral(&V::Name(b"nil")) && !is_neutral(&V::Token(b"nil")) && is_neutral(&V::Float(0.0)) && !is_neutral(&V::Str(Cow::Borrowed(b""))) + ); + } + + /// A layers marker list is the one in its root group. + #[test] + fn a_layers_marker_list_is_the_one_in_its_root_group() { + let marked = |tag: u8| list(b"mrst", vec![Chunk::data(*b"test", vec![tag])]); + // The stored place: `Layr/tdgp/mrst`. + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), marked(1)])]); + assert_eq!(Node::from(&layer).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[1u8][..])); + // A list deeper in the layer is not the layer's marker list (the old depth-first search + // found it); a layer without a marker list in its root group has no markers. + let deep = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(2)])])]); + assert!(Node::from(&deep).marker_list().is_none()); + assert!(Layer { chunk: Node::from(&deep) }.markers().is_empty()); + // Any other holder (a composition's `SecL`) is still searched depth-first. + let secl = list(b"SecL", vec![list(b"tdgp", vec![list(b"tdgp", vec![marked(3)])])]); + assert_eq!(Node::from(&secl).marker_list().and_then(|m| m.child(b"test")).and_then(Node::bytes), Some(&[3u8][..])); + } + + /// The source text record is found in the text group. + #[test] + fn the_source_text_record_is_found_in_the_text_group() { + let doc = b"/1 << /1 [ << /0 << /0 (\xfe\xff\0H\0\r) /6 << /0 [ << /0 << /0 << /6 << /0 0 /1 50 /53 << /0 << /1 [ 1 1 0 0 ] >> >> >> >> >> /1 2 >> ] >> >> >> ] >>"; + // `btdk` is an opaque list (its body is not a chunk sequence), as the reader makes it. + let btdk = || Chunk { id: *b"LIST", body: crate::Body::OpaqueList { list_type: *b"btdk", data: doc.to_vec().into() }, pad: None }; + let btds_real = |n: usize| list(b"btds", std::iter::once(list(b"tdbs", vec![])).chain((0..n).map(|_| btdk())).collect()); + let layer = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![]), list(b"tdgp", vec![btds_real(2)])])]); + let view = Layer { chunk: Node::from(&layer) }; + assert_eq!(view.text_document().map(|d| d.text), Some("H".to_string())); + assert!(view.source_text_property().is_some()); + assert_eq!(view.text_document_count(), 2); + // A record anywhere else in the layer is not its Source Text. + let elsewhere = list(b"Layr", vec![list(b"tdgp", vec![list(b"tdgp", vec![list(b"tdgp", vec![btds_real(1)])])])]); + let view = Layer { chunk: Node::from(&elsewhere) }; + assert!(view.text_document().is_none() && view.source_text_property().is_none()); + assert_eq!(view.text_document_count(), 0); + } #[test] fn non_finite_numbers_are_not_numbers() { @@ -1018,13 +1818,13 @@ mod tests { fn undecoded_fields_are_capped_and_linear() { // 300 000 distinct undecoded slots that differ from a sampled default: before, each was // checked against every slot recorded so far (4.5e10 compares) and all were listed. - let actual = TextValue::Dict((0..300_000).map(|i| (format!("{}", 9000 + i), TextValue::Int(1))).collect()); + let actual = V::Dict((0..300_000).map(|i| (Cow::Owned(format!("{}", 9000 + i)), V::Int(1))).collect()); let mut out = Vec::new(); undecoded_fields(&actual, None, &["0"], &[], &mut out); assert_eq!(out.len(), MAX_UNDECODED_FIELDS); assert_eq!(out.first().map(String::as_str), Some("9000")); // A slot listed once is not listed again. - let again = TextValue::Dict(vec![("9000".into(), TextValue::Int(1))]); + let again = V::Dict(vec![(Cow::Borrowed("9000"), V::Int(1))]); let mut short = vec!["9000".to_string()]; undecoded_fields(&again, None, &[], &[], &mut short); assert_eq!(short, ["9000"]); diff --git a/crates/aep/src/typed.rs b/crates/aep/src/typed.rs index 16fcda33c..9334b45cc 100644 --- a/crates/aep/src/typed.rs +++ b/crates/aep/src/typed.rs @@ -4,42 +4,57 @@ //! //! Only facts that were established from our own one-change-per-file corpus and are checked by //! tests against it are decoded (see `docs/aep-format.md` for the evidence). Everything else -//! stays reachable through the raw [`Chunk`] tree. Views borrow the file and never fail loudly: -//! a chunk that does not have the expected shape yields `None` or is skipped. +//! stays reachable through the raw chunk tree ([`Node`]). Views borrow the file and never fail +//! loudly: a chunk that does not have the expected shape yields `None` or is skipped. +//! +//! A view reads either tree: the owned [`Chunk`](crate::Chunk) tree +//! ([`Project::new`]) or the flat [`TreeIndex`] the importer builds ([`Project::from_index`]); +//! both give the same answers. use crate::bytes::{be_f64, be_u16, be_u32, text, until_nul}; -use crate::chunk::{AepFile, Chunk}; +use crate::chunk::AepFile; +use crate::index::{Node, Root, TreeIndex}; /// The name a user gave a property or group: the `Utf8` inside its `tdsn` chunk. `None` for the /// sentinel `-_0_/-` ("no custom name") and for empty names. -pub(crate) fn user_name(chunk: &Chunk<'_>) -> Option { - let name = chunk.child(b"tdsn")?.children().iter().find(|c| c.is(b"Utf8"))?.bytes().map(text)?; +pub(crate) fn user_name(chunk: Node<'_, '_>) -> Option { + let name = chunk.child(b"tdsn")?.child(b"Utf8")?.bytes().map(text)?; (!name.is_empty() && name != "-_0_/-").then_some(name) } /// The content of a `Utf8` child as text (not trimmed). -pub(crate) fn utf8_child(chunk: &Chunk<'_>) -> Option { - chunk.child(b"Utf8").and_then(Chunk::bytes).map(text) +pub(crate) fn utf8_child(chunk: Node<'_, '_>) -> Option { + chunk.child(b"Utf8").and_then(Node::bytes).map(text) } /// The project: the top-level `LIST Fold`, which holds the project items. #[derive(Debug, Clone, Copy)] pub struct Project<'f, 'a> { - pub(crate) fold: &'f Chunk<'a>, + pub(crate) fold: Node<'f, 'a>, /// The top-level `LIST EfdG` (project-level effect definitions), when there is one. - pub(crate) effect_defs: Option<&'f Chunk<'a>>, + pub(crate) effect_defs: Option>, } impl<'f, 'a> Project<'f, 'a> { /// The project view of a file, or `None` when it has no `LIST Fold`. pub fn new(file: &'f AepFile<'a>) -> Option { - file.list(b"Fold").map(|fold| Self { fold, effect_defs: file.list(b"EfdG") }) + Self::of(Root::from(file)) + } + + /// The project view of an indexed file ([`TreeIndex`]), or `None` when it has no `LIST Fold`. + pub fn from_index(index: &'f TreeIndex<'a>) -> Option { + Self::of(Root::from(index)) + } + + /// The project view of either tree: its `LIST Fold` and `LIST EfdG`. + pub fn of(root: Root<'f, 'a>) -> Option { + root.list(b"Fold").map(|fold| Self { fold, effect_defs: root.list(b"EfdG") }) } /// The top-level items (the project panel's root; folders hold their children, see /// [`Item::children`]). pub fn items(&self) -> impl Iterator> + 'f { - self.fold.children().iter().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.fold.children().filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// True when the `LIST Fold` could not be read as chunks (damaged): its items cannot be seen. @@ -78,12 +93,12 @@ pub enum ItemKind { /// A project item (`LIST Item`). #[derive(Debug, Clone, Copy)] pub struct Item<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Item<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -106,7 +121,7 @@ impl<'f, 'a> Item<'f, 'a> { /// The items inside a folder (`LIST Sfdr`). pub fn children(&self) -> impl Iterator> + 'f { - self.chunk.child_list(b"Sfdr").into_iter().flat_map(|sfdr| sfdr.children().iter()).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) + self.chunk.child_list(b"Sfdr").into_iter().flat_map(Node::children).filter(|c| c.is_list(b"Item")).map(|chunk| Item { chunk }) } /// Whether the item has the `LIST Sfdr` of a folder's contents (After Effects writes an empty @@ -117,7 +132,7 @@ impl<'f, 'a> Item<'f, 'a> { /// True when the folder's `LIST Sfdr` is there but could not be read as chunks (damaged). pub fn folder_contents_unreadable(&self) -> bool { - self.chunk.child_list(b"Sfdr").is_some_and(Chunk::is_opaque_list) + self.chunk.child_list(b"Sfdr").is_some_and(Node::is_opaque_list) } /// The composition view when this item has composition data (`cdta`). @@ -135,6 +150,10 @@ pub struct TimeBase { } impl TimeBase { + /// The NTSC rates as `(numerator, denominator)` and the value After Effects stores for each (a + /// float32 of exactly these decimals): the one table an importer and an exporter use. + pub const NTSC: [((u32, u32), f32); 4] = [((24000, 1001), 23.976), ((30000, 1001), 29.97), ((60000, 1001), 59.94), ((120000, 1001), 119.88)]; + /// Frames per second (`timescale / frame_duration`), `None` for a zero duration. pub fn frame_rate(&self) -> Option { (self.frame_duration != 0).then(|| f64::from(self.timescale) / f64::from(self.frame_duration)) @@ -149,7 +168,7 @@ impl TimeBase { /// A composition item (an [`Item`] with `cdta`). #[derive(Debug, Clone, Copy)] pub struct Comp<'f, 'a> { - pub(crate) item: &'f Chunk<'a>, + pub(crate) item: Node<'f, 'a>, } impl<'f, 'a> Comp<'f, 'a> { @@ -163,19 +182,19 @@ impl<'f, 'a> Comp<'f, 'a> { /// The layers, top layer first (`LIST Layr`; the `DLay`/`SLay`/`CLay`/`SecL` lists next to /// them are not layers). pub fn layers(&self) -> impl Iterator> + 'f { - self.item.children().iter().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) + self.item.children().filter(|c| c.is_list(b"Layr")).map(|chunk| Layer { chunk }) } } /// A layer (`LIST Layr`). #[derive(Debug, Clone, Copy)] pub struct Layer<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Layer<'f, 'a> { /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -205,12 +224,12 @@ pub enum PropNode<'f, 'a> { /// A property (`LIST tdbs`). Property(Property<'f, 'a>), /// Any other list that follows a match name (observed: `otst`, the effect instance `sspc`). - Other(&'f Chunk<'a>), + Other(Node<'f, 'a>), } impl<'f, 'a> PropNode<'f, 'a> { /// The node a `LIST` after a match name is. - fn of(c: &'f Chunk<'a>) -> Self { + fn of(c: Node<'f, 'a>) -> Self { if c.is_list(b"tdgp") { PropNode::Group(PropGroup { chunk: c }) } else if c.is_list(b"tdbs") { @@ -233,17 +252,18 @@ pub struct PropEntry<'f, 'a> { /// `ADBE Group End` closes the group and names nothing. #[derive(Debug, Clone, Copy)] pub struct PropGroup<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> PropGroup<'f, 'a> { - /// A group view of a `LIST tdgp` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A group view of a `LIST tdgp` chunk (a [`Chunk`](crate::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdgp").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -264,7 +284,7 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes().map(match_name); - } else if c.id == crate::chunk::LIST + } else if c.is(&crate::chunk::LIST) && let Some(name) = pending.take() { out.push(PropEntry { match_name: name, node: PropNode::of(c) }); @@ -285,9 +305,9 @@ impl<'f, 'a> PropGroup<'f, 'a> { for c in self.chunk.children() { if c.is(b"tdmn") { pending = c.bytes(); - } else if c.id == crate::chunk::LIST + } else if c.is(&crate::chunk::LIST) && let Some(body) = pending.take() - && match_name(body) == name + && match_name_is(body, name) { return Some(PropNode::of(c)); } @@ -314,21 +334,30 @@ fn match_name(body: &[u8]) -> String { text(until_nul(body)) } +/// `match_name(body) == name` without building the `String`: equal bytes match; otherwise only +/// a body that is not valid UTF-8 can still match (through its U+FFFD replacements), and only a +/// name that holds U+FFFD. +fn match_name_is(body: &[u8], name: &str) -> bool { + let bytes = until_nul(body); + bytes == name.as_bytes() || (name.contains('\u{FFFD}') && String::from_utf8_lossy(bytes) == name) +} + /// A property (`LIST tdbs`): `tdsb`, `tdsn`, the 124-byte header `tdb4`, then either the static /// value `cdat` or a keyframe `LIST list`. #[derive(Debug, Clone, Copy)] pub struct Property<'f, 'a> { - pub(crate) chunk: &'f Chunk<'a>, + pub(crate) chunk: Node<'f, 'a>, } impl<'f, 'a> Property<'f, 'a> { - /// A property view of a `LIST tdbs` chunk. - pub fn from_chunk(chunk: &'f Chunk<'a>) -> Option { + /// A property view of a `LIST tdbs` chunk (a [`Chunk`](crate::Chunk) or a [`Node`]). + pub fn from_chunk(chunk: impl Into>) -> Option { + let chunk = chunk.into(); chunk.is_list(b"tdbs").then_some(Self { chunk }) } /// The underlying chunk. - pub fn chunk(&self) -> &'f Chunk<'a> { + pub fn chunk(&self) -> Node<'f, 'a> { self.chunk } @@ -436,12 +465,13 @@ pub struct KeyLayout { impl<'f> KeyList<'f> { /// A view of a `LIST list` chunk. `None` when `lhd3` is not 52 bytes or `ldat` is not exactly /// `count * record_size` bytes (a list without `ldat` must have no records). - pub fn from_chunk<'a>(chunk: &'f Chunk<'a>) -> Option { + pub fn from_chunk<'a: 'f>(chunk: impl Into>) -> Option { + let chunk = chunk.into(); if !chunk.is_list(b"list") { return None; } let header = ListHeader::parse(chunk.child(b"lhd3")?.bytes()?)?; - let ldat: &[u8] = chunk.child(b"ldat").and_then(Chunk::bytes).unwrap_or(&[]); + let ldat: &[u8] = chunk.child(b"ldat").and_then(Node::bytes).unwrap_or(&[]); let expected = usize::try_from(header.count).ok()?.checked_mul(usize::try_from(header.record_size).ok()?)?; (ldat.len() == expected).then_some(Self { header, ldat, layout: None }) } @@ -500,6 +530,7 @@ impl<'f> Keyframe<'f> { #[cfg(test)] mod tests { use super::*; + use crate::Chunk; fn key_list(count: u32, record_size: u32, ldat: &[u8]) -> Chunk<'static> { let mut lhd3 = vec![0u8; 52]; @@ -543,4 +574,31 @@ mod tests { assert!(g.get("C").is_none()); assert_eq!(g.names(), ["A", "B", "A"]); } + + /// Match name is agrees with comparing the decoded name. + #[test] + fn match_name_is_agrees_with_comparing_the_decoded_name() { + let bodies: [&[u8]; 9] = [ + b"ADBE Position\0\0\0", + b"ADBE Position", + b"ADBE Pos", + b"", + b"\0ADBE", + b"ADBE \xff Bad\0", + b"\xff", + b"\xef\xbf\xbd", + "Gr\u{fc}n\0".as_bytes(), + ]; + let names = ["ADBE Position", "ADBE Pos", "", "ADBE \u{FFFD} Bad", "\u{FFFD}", "Gr\u{fc}n", "ADBE Position\0", "ADBE \u{FFFD}"]; + for body in bodies { + for name in names { + assert_eq!(match_name_is(body, name), match_name(body) == name, "{body:?} vs {name:?}"); + } + } + // Names read back from a group find their entries, a damaged one included. + let group = Chunk::list(*b"tdgp", vec![Chunk::data(*b"tdmn", b"X\xffY\0".to_vec()), Chunk::list(*b"tdbs", Vec::new())]); + let g = PropGroup::from_chunk(&group).expect("group"); + let names = g.names(); + assert!(g.get(&names[0]).is_some()); + } } diff --git a/crates/aep/src/values.rs b/crates/aep/src/values.rs index 5613e07a3..b6857165a 100644 --- a/crates/aep/src/values.rs +++ b/crates/aep/src/values.rs @@ -8,6 +8,7 @@ //! 100 %. These defaults are verified for AV layers (solids); cameras and lights use other //! conventions (see the `camera` module). +use crate::bytes::be_f64; use crate::typed::{Layer, Project, PropGroup, PropNode}; /// Where a [`Tracked`] value came from. @@ -192,8 +193,7 @@ impl<'f, 'a> Layer<'f, 'a> { let otda = otst.child_list(b"otky")?.child(b"otda")?.bytes()?; let mut degrees = [0.0; 3]; for (i, slot) in degrees.iter_mut().enumerate() { - let b = otda.get(i * 8..i * 8 + 8)?; - *slot = f64::from_be_bytes(b.try_into().ok()?); + *slot = be_f64(otda, i * 8)?; } Some(Orientation { degrees, @@ -223,12 +223,10 @@ impl<'f, 'a> Layer<'f, 'a> { .child_list(b"otky") .map(|k| { k.children() - .iter() .filter(|c| c.is(b"otda")) .map(|c| { let b = c.bytes()?; - let at = |i: usize| b.get(i * 8..i * 8 + 8).and_then(|s| s.try_into().ok()).map(f64::from_be_bytes); - Some([at(0)?, at(1)?, at(2)?]) + Some([be_f64(b, 0)?, be_f64(b, 8)?, be_f64(b, 16)?]) }) .collect() }) diff --git a/crates/aep/src/version.rs b/crates/aep/src/version.rs index 53de0834a..023793037 100644 --- a/crates/aep/src/version.rs +++ b/crates/aep/src/version.rs @@ -8,8 +8,9 @@ use std::fmt; -use crate::bytes::{be_u16, be_u32}; +use crate::bytes::be_u32; use crate::chunk::AepFile; +use crate::index::{Root, TreeIndex}; /// An After Effects version as packed in `head` / `svap`: `26.2.1x2` is major 26, minor 2, bug-fix /// 1, build 2 (the form of the scripting API's `app.version`). @@ -71,21 +72,38 @@ impl AepFile<'_> { /// version there, and the older version for File ▸ Save a Copy As 25.x / 24.x. `None` when the /// chunk is missing, short or holds no version. pub fn file_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + Root::from(self).file_version() } /// The version of the After Effects that wrote the file: `svap` bytes 0 to 3 (the same word in a /// down-saved copy as in a normal save). `None` when the chunk is missing (After Effects opens such /// a file), short or holds no version. pub fn saved_by_version(&self) -> Option { - AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) + Root::from(self).saved_by_version() + } +} + +impl TreeIndex<'_> { + /// [`AepFile::file_version`] of the indexed file. + pub fn file_version(&self) -> Option { + Root::from(self).file_version() } - /// The `u16` at `head` byte 0: 97 (`0x61`) in a 2026 file, 96 in a copy saved as 25.x, 95 as - /// 24.x. It steps with the major version of [`AepFile::file_version`]; what it counts is not - /// known, so nothing is derived from it. - pub fn head_revision(&self) -> Option { - be_u16(self.chunk(b"head")?.bytes()?, 0) + /// [`AepFile::saved_by_version`] of the indexed file. + pub fn saved_by_version(&self) -> Option { + Root::from(self).saved_by_version() + } +} + +impl Root<'_, '_> { + /// The version the file is written for (`head` bytes 4 to 7). + pub fn file_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"head")?.bytes()?, 4)?) + } + + /// The version of the application that wrote the file (`svap`). + pub fn saved_by_version(self) -> Option { + AeVersion::from_word(be_u32(self.chunk(b"svap")?.bytes()?, 0)?) } } diff --git a/crates/aep/src/write.rs b/crates/aep/src/write.rs index 1adf3126d..cadd84f0d 100644 --- a/crates/aep/src/write.rs +++ b/crates/aep/src/write.rs @@ -17,6 +17,18 @@ const MAX_WRITE_DEPTH: usize = 256; /// [`AepError::TooDeep`] for an absurdly deep hand-built tree. pub fn write(file: &AepFile<'_>) -> Result, AepError> { let mut out = Vec::new(); + // Sized once to the exact output. A tree past the 32-bit container size fails at once (the + // same error the write below would give after serialising it). `try_reserve` so a hand-built + // tree that repeats one large borrowed body many times cannot abort on the allocation; the + // writer then grows as it goes, as before. + if let Some(len) = sequence_len(&file.chunks, 1) { + if u32::try_from(len.saturating_add(4)).is_err() { + return Err(AepError::TooLarge); + } + if let Some(total) = len.checked_add(12).and_then(|n| n.checked_add(file.trailer.len())) { + let _ = out.try_reserve_exact(total); + } + } out.extend_from_slice(b"RIFX"); out.extend_from_slice(&[0; 4]); out.extend_from_slice(b"Egg!"); @@ -56,6 +68,28 @@ fn write_sequence(out: &mut Vec, chunks: &[Chunk<'_>], depth: usize) -> Resu Ok(()) } +/// The number of bytes [`write_sequence`] produces for `chunks`; `None` past the depth limit or +/// on overflow (both make the write fail). +fn sequence_len(chunks: &[Chunk<'_>], depth: usize) -> Option { + if depth > MAX_WRITE_DEPTH { + return None; + } + let count = chunks.len(); + let mut total = 0usize; + for (i, chunk) in chunks.iter().enumerate() { + let body = match &chunk.body { + Body::Data(d) => d.len(), + Body::Chunks(c) => sequence_len(c, depth + 1)?, + Body::List { children, .. } => sequence_len(children, depth + 1)?.checked_add(4)?, + Body::OpaqueList { data, .. } => data.len().checked_add(4)?, + }; + // The same pad rule as `write_chunk`. + let pad = usize::from(body % 2 == 1 && (chunk.pad.is_some() || i + 1 != count)); + total = total.checked_add(8)?.checked_add(body)?.checked_add(pad)?; + } + Some(total) +} + fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize) -> Result<(), AepError> { out.extend_from_slice(&chunk.id); let size_at = out.len(); @@ -84,3 +118,62 @@ fn write_chunk(out: &mut Vec, chunk: &Chunk<'_>, is_last: bool, depth: usize } Ok(()) } + +#[cfg(test)] +mod tests { + use std::borrow::Cow; + + use super::*; + + /// The output is sized once to its exact length. + #[test] + fn the_output_is_sized_once_to_its_exact_length() { + let odd = |n: u8| Chunk::data(*b"cdat", vec![n; usize::from(n)]); + let mut padded = odd(3); + padded.pad = Some(0x7f); + let trees = [ + AepFile { chunks: Vec::new(), trailer: Cow::Borrowed(&[]) }, + AepFile { + chunks: vec![ + odd(1), + Chunk::list(*b"Fold", vec![odd(2), odd(3), Chunk::list(*b"tdgp", vec![odd(5)])]), + Chunk { id: *b"LIST", body: Body::OpaqueList { list_type: *b"btdk", data: Cow::Owned(vec![1, 2, 3]) }, pad: None }, + Chunk { id: *b"tdmn", body: Body::Chunks(vec![odd(7), padded]), pad: None }, + odd(9), + ], + trailer: Cow::Borrowed(b""), + }, + ]; + for file in &trees { + let out = write(file).expect("write"); + assert_eq!(sequence_len(&file.chunks, 1).map(|n| n + 12 + file.trailer.len()), Some(out.len())); + // Exact with std's allocator today; grown push by push it would be rounded up. + assert_eq!(out.capacity(), out.len(), "the output grew past its size"); + } + } + + /// A tree past the container size fails before serialising. + #[cfg(target_pointer_width = "64")] + #[test] + fn a_tree_past_the_container_size_fails_before_serialising() { + // 4097 chunks borrowing one 1 MiB body: over 4 GiB of output, none of it written. + let body = vec![0u8; 1 << 20]; + let chunks: Vec> = (0..4097).map(|_| Chunk::data(*b"cdat", &body[..])).collect(); + let file = AepFile { chunks, trailer: Cow::Borrowed(&[]) }; + assert_eq!(write(&file), Err(AepError::TooLarge)); + // Just under the limit still sizes, and fails nothing up front. + assert!(sequence_len(&file.chunks[..4000], 1).is_some_and(|n| u32::try_from(n + 4).is_ok())); + } + + /// A tree past the depth limit has no size. + #[test] + fn a_tree_past_the_depth_limit_has_no_size() { + let mut deep = Chunk::data(*b"cdat", vec![0]); + for _ in 0..=MAX_WRITE_DEPTH { + deep = Chunk::list(*b"tdgp", vec![deep]); + } + let file = AepFile { chunks: vec![deep], trailer: Cow::Borrowed(&[]) }; + assert_eq!(sequence_len(&file.chunks, 1), None); + assert_eq!(write(&file), Err(AepError::TooDeep)); + } +} diff --git a/crates/aep/tests/it/common/mod.rs b/crates/aep/tests/it/common/mod.rs index 791c3bbc9..b5605bb95 100644 --- a/crates/aep/tests/it/common/mod.rs +++ b/crates/aep/tests/it/common/mod.rs @@ -128,6 +128,13 @@ pub fn resolve(name: &str) -> Option { locate(name).map(|s| s.path().clone()) } +/// The ids of the compositions an item uses as precomposition layers, as the file stores them: +/// the little-endian `u32` of each `idpi` chunk (the importer does not read them; the exporter +/// writes them). +pub fn used_item_ids(item: &craft_aep::Item<'_, '_>) -> Vec { + item.chunk().children().filter(|c| c.is(b"idpi")).filter_map(|c| Some(u32::from_le_bytes(c.bytes()?.get(..4)?.try_into().ok()?))).collect() +} + /// The bytes of a case (see the module comment), or `None` when it is missing. pub fn try_fixture(name: &str) -> Option> { match locate(name)? { diff --git a/crates/aep/tests/it/container.rs b/crates/aep/tests/it/container.rs index d937d296c..514bbde1c 100644 --- a/crates/aep/tests/it/container.rs +++ b/crates/aep/tests/it/container.rs @@ -25,5 +25,6 @@ mod shapes; mod text_fields; mod text_markers_footage; mod text_style_slots; +mod tree_index; mod typed; mod values; diff --git a/crates/aep/tests/it/container/batch3_core.rs b/crates/aep/tests/it/container/batch3_core.rs index 8c2518a9f..83bd942b8 100644 --- a/crates/aep/tests/it/container/batch3_core.rs +++ b/crates/aep/tests/it/container/batch3_core.rs @@ -117,8 +117,8 @@ fn a_transform_set_to_its_default_adds_no_record() { assert_eq!(t.opacity_percent.source, ValueSource::Default); assert!(layer.properties().unwrap().find(&["ADBE Transform Group", "ADBE Position"]).is_none()); // The two cases are layers of one file: they differ in the layer id only. - let a = without_layer_id(layer.chunk()); - skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(other.chunk())))); + let a = without_layer_id(&layer.chunk().to_chunk()); + skip!(with_layer("299-save-noise.aep", L, |_, other| assert_eq!(a, without_layer_id(&other.chunk().to_chunk())))); })); } @@ -140,16 +140,13 @@ fn scale_keys_keep_their_ease_type_major() { } assert_near(&keys[0].value().unwrap(), &[1.0, 1.0, 1.0], 1e-9); assert_near(&keys[1].value().unwrap(), &[2.0, 1.5, 1.0], 1e-9); - // The raw block: [in speed x3][in influence x3][out speed x3][out influence x3]. - assert_near(&keys[0].plain_ease_slots().unwrap(), &[0.0, 0.0, 0.0, 0.33, 0.33, 0.33, 0.2, 0.2, 0.2, 0.66, 0.66, 0.66], 1e-9); - assert_near(&keys[1].plain_ease_slots().unwrap(), &[0.1, 0.1, 0.1, 0.66, 0.66, 0.66, 0.0, 0.0, 0.0, 0.33, 0.33, 0.33], 1e-9); + // The raw block is [in speed x3][in influence x3][out speed x3][out influence x3]. for i in 0..3 { assert!(same_ease(&keys[0].ease_component(i).unwrap(), &ease(0.0, 0.33, 0.2, 0.66)), "key 1 component {i}"); assert!(same_ease(&keys[1].ease_component(i).unwrap(), &ease(0.1, 0.66, 0.0, 0.33)), "key 2 component {i}"); } assert!(keys[0].ease_component(3).is_none()); assert_eq!(keys[0].ease_components().unwrap().len(), 3); - assert!(keys[0].color_distance().is_none()); })); } @@ -172,7 +169,6 @@ fn mask_feather_keys_keep_their_ease_type_major() { } assert_near(&keys[0].value().unwrap(), &[0.0, 0.0], 1e-9); assert_near(&keys[1].value().unwrap(), &[50.0, 30.0], 1e-9); - assert_near(&keys[0].plain_ease_slots().unwrap(), &[0.0, 0.0, 0.33, 0.33, 15.0, 15.0, 0.66, 0.66], 1e-9); for i in 0..2 { assert!(same_ease(&keys[0].ease_component(i).unwrap(), &ease(0.0, 0.33, 15.0, 0.66)), "key 1 component {i}"); assert!(same_ease(&keys[1].ease_component(i).unwrap(), &ease(8.0, 0.66, 0.0, 0.33)), "key 2 component {i}"); @@ -213,14 +209,15 @@ fn bezier_colour_keys_carry_the_distance_to_the_next_key() { assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Bezier), Some(Interpolation::Bezier))); assert!(same_ease(&k.ease_component(0).unwrap(), &ease(0.0, 0.0, 0.0, 0.0))); } - assert!(near(keys[0].color_distance().unwrap(), 255.0 * 2f64.sqrt(), 1e-3), "{:?}", keys[0].color_distance()); - assert!(near(keys[0].color_distance().unwrap(), 360.6245, 1e-3)); - assert_eq!(keys[1].color_distance(), Some(0.0)); + let distance = |k: &craft_aep::Keyframe<'_>| k.spatial().unwrap().segment_length; + assert!(near(distance(&keys[0]), 255.0 * 2f64.sqrt(), 1e-3), "{}", distance(&keys[0])); + assert!(near(distance(&keys[0]), 360.6245, 1e-3)); + assert_eq!(distance(&keys[1]), 0.0); })); skip!(with_layer("210-key-fill-color-2key-linear.aep", L, |_, layer| { let p = layer.effects()[0].params().into_iter().find(|p| p.name == "Color").unwrap(); let k = p.stored().unwrap().keys().unwrap().keyframes().next().unwrap(); - assert_eq!(k.color_distance(), Some(0.0)); + assert_eq!(k.spatial().unwrap().segment_length, 0.0); })); } @@ -243,21 +240,19 @@ fn time_stretch_is_an_exact_ratio() { let (h, in_p, out_p, scale) = skip!(stretch_facts("316-layer-stretch50.aep")); assert_eq!((h.stretch, h.stretch_den), (1, 2)); assert_eq!(h.stretch_ratio(), Some(rat(1, 2))); - assert_eq!(h.stretch_percent(), 50.0); assert_eq!(h.out_relative, rat(128000, 25600)); assert_eq!((in_p, out_p, scale), (Some(rat(0, 1)), Some(rat(5, 2)), 25600)); // 317: 150 %: 3/2, ends at 7.5 s, the property time scale is 25600 x 3/2. let (h, in_p, out_p, scale) = skip!(stretch_facts("317-layer-stretch150.aep")); assert_eq!((h.stretch, h.stretch_den), (3, 2)); assert_eq!(h.stretch_ratio(), Some(rat(3, 2))); - assert_eq!(h.stretch_percent(), 150.0); assert_eq!((in_p, out_p, scale), (Some(rat(0, 1)), Some(rat(15, 2)), 38400)); // 018: 200 %: 2/1, ends at 10 s, time scale 51200. The base is 1/1. let (h, _, out_p, scale) = skip!(stretch_facts("018-layer-stretch200.aep")); assert_eq!((h.stretch, h.stretch_den, h.stretch_ratio()), (2, 1, Some(rat(2, 1)))); assert_eq!((out_p, scale), (Some(rat(10, 1)), 51200)); let (h, _, out_p, scale) = skip!(stretch_facts("000-base.aep")); - assert_eq!((h.stretch, h.stretch_den, h.stretch_percent()), (1, 1, 100.0)); + assert_eq!((h.stretch, h.stretch_den, h.stretch_ratio()), (1, 1, Some(rat(1, 1)))); assert_eq!((out_p, scale), (Some(rat(5, 1)), 25600)); } @@ -269,9 +264,9 @@ fn fractional_times_are_exact_multiples_of_one_over_25600() { assert_eq!(h.in_relative, rat(8532, 25600)); assert_eq!(h.out_relative, rat(136532, 25600)); assert_eq!(h.start, rat(0, 25600)); + // 0.33328125 s and 5.33328125 s. assert_eq!(h.in_point(), Some(rat(2133, 6400))); - assert_eq!(h.in_point_seconds(), Some(0.33328125)); - assert_eq!(h.out_point_seconds(), Some(5.33328125)); + assert_eq!(h.out_point(), Some(rat(34133, 6400))); assert_eq!(h.stretch_ratio(), Some(rat(1, 1))); } @@ -364,18 +359,17 @@ fn pixel_aspect_shutter_phase_and_display_start() { // 325: pixelAspect 0.9091 reads back 0.90909090909091 = the exact ratio 10/11. let h = skip!(comp_header("325-comp-pixelaspect-0.9091.aep")); assert_eq!(h.pixel_aspect.reduced(), Some(rat(10, 11))); - assert!(near(h.pixel_aspect_ratio().unwrap(), 0.90909090909091, 1e-12)); // 326: shutterPhase -90, the shutter angle stays 180. let h = skip!(comp_header("326-comp-shutterphase-m90.aep")); assert_eq!((h.shutter_phase, h.shutter_angle), (Some(-90), 180)); // 327: displayStartFrame 100 at 25 fps = 4 s = 102400/25600; the shutter phase is back to 0. let h = skip!(comp_header("327-comp-displaystartframe-100.aep")); assert_eq!(h.display_start, Some(rat(102400, 25600))); - assert_eq!((h.display_start_seconds(), h.display_start_frame(), h.shutter_phase), (Some(4.0), Some(100), Some(0))); + assert_eq!(h.shutter_phase, Some(0)); // The base: no phase, start 0. let h = skip!(comp_header("000-base.aep")); - assert_eq!((h.shutter_phase, h.display_start_seconds(), h.display_start_frame()), (Some(0), Some(0.0), Some(0))); - assert_eq!(h.pixel_aspect_ratio(), Some(1.0)); + assert_eq!((h.shutter_phase, h.display_start.and_then(|r| r.reduced())), (Some(0), Some(rat(0, 1)))); + assert_eq!(h.pixel_aspect.reduced(), Some(rat(1, 1))); // A short chunk still decodes the old fields. let bytes = [0u8; 176]; let h = craft_aep::CompHeader::parse(&bytes).unwrap(); diff --git a/crates/aep/tests/it/container/batch3_objects.rs b/crates/aep/tests/it/container/batch3_objects.rs index bbb7040b4..c78525a00 100644 --- a/crates/aep/tests/it/container/batch3_objects.rs +++ b/crates/aep/tests/it/container/batch3_objects.rs @@ -148,19 +148,21 @@ fn a_mask_box_is_normalised_to_the_layer_size() { for (got, want) in m.shapes[0].bbox.iter().zip(want_box) { assert!(near(f64::from(*got), want, 1e-6), "{:?}", m.shapes[0].bbox); } - let shapes = project.mask_vertices_px(&layer, m).unwrap(); - assert_eq!(shapes.len(), 1); - let got: Vec<[f64; 2]> = shapes[0].iter().map(|v| v.position).collect(); + // The box scales by the size of the layer's source (the importer takes that size from the + // source item it matched, the same item `layer_source_size` reads). + let (w, h) = project.layer_source_size(&layer).unwrap(); + assert_eq!(m.shapes.len(), 1); + let got: Vec<[f64; 2]> = m.shapes[0].vertices((f64::from(w), f64::from(h))).iter().map(|v| v.position).collect(); for (g, w) in got.iter().zip([[20.0, 30.0], [80.0, 30.0], [80.0, 70.0], [20.0, 70.0]]) { assert_near(g, &w, 1e-3); } assert_eq!(got.len(), 4); - // A layer without a sized source has no pixel vertices. + // A layer without a sized source has no size to scale by. let bytes = fixture!("333-shape-group-two-rects.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let shape = layer_named(&first_comp(&project), "Shape"); - assert!(project.mask_vertices_px(&shape, m).is_none()); + assert!(project.layer_source_size(&shape).is_none()); } #[test] @@ -178,12 +180,14 @@ fn one_mkif_per_mask_and_the_mode_none() { assert_eq!((m.mode, m.inverted, m.counter), (Some(MaskMode::Add), Some(false), Some(counter))); assert_eq!(m.shapes.len(), 1); } - let v1 = project.mask_vertices_px(&layer, &masks[0]).unwrap(); + let (w, h) = project.layer_source_size(&layer).unwrap(); + let px = |m: &craft_aep::Mask| m.shapes.iter().map(|s| s.vertices((f64::from(w), f64::from(h)))).collect::>(); + let v1 = px(&masks[0]); for (v, w) in v1[0].iter().zip(ellipse_vertices()) { assert_near(&v.position, &w, 1e-2); } // Mask 2 has its own box (500..900 x 350..650) normalised to the layer. - let v2 = project.mask_vertices_px(&layer, &masks[1]).unwrap(); + let v2 = px(&masks[1]); for (v, w) in v2[0].iter().zip([[500.0, 350.0], [900.0, 350.0], [900.0, 650.0], [500.0, 650.0]]) { assert_near(&v.position, &w, 1e-2); } @@ -384,81 +388,16 @@ fn the_last_style_run_absorbs_the_final_carriage_return() { // ------------------------------------------------------------------ camera and light (341-345) +/// Camera zoom keeps the auto position. #[test] -fn camera_zoom_and_depth_of_field() { - let camera = |name: &str| { - Some({ - let bytes = crate::common::try_fixture(name)?; - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Camera); - ( - layer.camera_zoom(), - layer.camera_depth_of_field(), - layer.camera_focus_distance(), - layer.camera_aperture(), - layer.camera_position().map(|p| (p.stored, p.auto_state)), - ) - }) - }; - // 341: Zoom 2000 is stored; the auto Position keeps [0, 0, -2666.67]. - let (zoom, dof, focus, aperture, pos) = skip!(camera("341-camera-zoom-2000.aep")); - assert_eq!(zoom.map(|z| (z.value, z.source)), Some((2000.0, ValueSource::Stored))); - assert_eq!((dof.map(|d| (d.value, d.source)), focus, aperture), (Some((false, ValueSource::Default)), None, None)); - let (stored, auto) = pos.unwrap(); - assert!(auto && near(stored[2], -2666.67, 0.01) && stored[0] == 0.0, "{stored:?}"); - // 342: DOF on, Focus Distance 800, Aperture 30; Zoom stays the default. - let (zoom, dof, focus, aperture, _) = skip!(camera("342-camera-dof-800-30.aep")); - assert_eq!(zoom.map(|z| (z.source, near(z.value, 2666.67, 0.01))), Some((ValueSource::Default, true))); - assert_eq!(dof.map(|d| (d.value, d.source)), Some((true, ValueSource::Stored))); - assert_eq!(focus.map(|f| (f.value, f.source)), Some((800.0, ValueSource::Stored))); - assert_eq!(aperture.map(|f| (f.value, f.source)), Some((30.0, ValueSource::Stored))); - // The default camera (256) has nothing but the default zoom. - let (zoom, dof, ..) = skip!(camera("256-camera-two-node-default.aep")); - assert_eq!((zoom.unwrap().source, dof.unwrap().value), (ValueSource::Default, false)); - assert_eq!(zoom.unwrap().value, craft_aep::DEFAULT_CAMERA_ZOOM); -} - -#[test] -fn light_colour_cone_angle_and_feather() { - let light = |name: &str| { - Some({ - let bytes = crate::common::try_fixture(name)?; - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Light); - assert_eq!(layer.header().unwrap().light_type(), Some(craft_aep::LightType::Spot), "{name}"); - let t = |x: Option>| x.map(|x| (x.value, x.source)); - (layer.light_color().map(|c| (c.value, c.source)), t(layer.light_cone_angle()), t(layer.light_cone_feather())) - }) - }; - // The default spot light: white, 90, 50, all absent. - let (colour, angle, feather) = skip!(light("254-light-spot-intensity-50.aep")); - assert_eq!( - (colour, angle, feather), - (Some(([1.0; 4], ValueSource::Default)), Some((90.0, ValueSource::Default)), Some((50.0, ValueSource::Default))) - ); - // 343: colour [1,0,0,1] = ARGB doubles [255,255,0,0]. - let (colour, angle, feather) = skip!(light("343-light-spot-color-red.aep")); - assert_eq!(colour, Some(([1.0, 1.0, 0.0, 0.0], ValueSource::Stored))); - assert_eq!((angle, feather), (Some((90.0, ValueSource::Default)), Some((50.0, ValueSource::Default)))); - // 344: Cone Angle 60. 345: Cone Feather 30. - let (colour, angle, feather) = skip!(light("344-light-spot-coneangle-60.aep")); - assert_eq!((colour.unwrap().1, angle, feather), (ValueSource::Default, Some((60.0, ValueSource::Stored)), Some((50.0, ValueSource::Default)))); - let (_, angle, feather) = skip!(light("345-light-spot-conefeather-30.aep")); - assert_eq!((angle, feather), (Some((90.0, ValueSource::Default)), Some((30.0, ValueSource::Stored)))); - // A point light (252) stores Cone Angle 90 / Feather 50 in their default state. - let bytes = fixture!("252-light-point.aep"); +fn camera_zoom_keeps_the_auto_position() { + // 341: Zoom 2000 is stored (checked on import); the auto Position keeps [0, 0, -2666.67]. + let bytes = fixture!("341-camera-zoom-2000.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); - let layer = layer_of_type(&first_comp(&project), LayerType::Light); - assert_eq!(layer.light_cone_angle().map(|a| (a.value, a.source)), Some((90.0, ValueSource::Stored))); - // Layers without light options have no light accessors. - let bytes = fixture!("000-base.aep"); - let file = open(&bytes); - let project = Project::new(&file).unwrap(); - let solid = base_layer(&first_comp(&project)); - assert!(solid.light_color().is_none() && solid.camera_zoom().is_none() && solid.camera_depth_of_field().is_none()); + let layer = layer_of_type(&first_comp(&project), LayerType::Camera); + let pos = layer.camera_position().unwrap(); + assert!(pos.auto_state && near(pos.stored[2], -2666.67, 0.01) && pos.stored[0] == 0.0, "{:?}", pos.stored); } // ------------------------------------------------------------------ marker data (347-349, 249) @@ -508,7 +447,7 @@ fn audio_footage_item() { assert_eq!((f.width, f.height, f.is_solid), (0, 0, false)); assert_eq!(f.duration, Some(rat(48000, 48000))); assert_eq!((f.audio_sample_rate, f.file_size, f.bits_per_pixel, f.native_frame_rate), (Some(48000.0), 96044, 0, 0)); - assert!(!f.is_sequence && f.sequence_frame_count().is_none()); + assert!(!f.is_sequence && item.sequence_info().is_none()); // The combined file stores no display name for the item (the single-case file had // "test-tone.wav"); the path names the file. assert_eq!(item.label(), Some(7)); @@ -532,7 +471,7 @@ fn image_sequence_is_one_item() { assert_eq!(f.duration, Some(rat(3, 30)), "{name}"); assert_eq!((f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate), (30, 24, 504, None), "{name}"); assert!(f.is_sequence, "{name}"); - assert_eq!(f.sequence_frame_count(), Some(3), "{name}"); + assert_eq!(item.sequence_info().map(|s| s.frame_count()), Some(3), "{name}"); // The first file name sits in opti; the alias points at the sequence folder. assert_eq!(f.name.as_deref(), Some("seq_001.png"), "{name}"); assert!(f.path.as_deref().is_some_and(|p| p.ends_with("test-seq")), "{name}"); diff --git a/crates/aep/tests/it/container/cases_600.rs b/crates/aep/tests/it/container/cases_600.rs index 210fb0916..43df3052a 100644 --- a/crates/aep/tests/it/container/cases_600.rs +++ b/crates/aep/tests/it/container/cases_600.rs @@ -38,7 +38,7 @@ fn a_composition_counts_the_compositions_it_shows() { let id = |n: &str| items.iter().find(|i| i.display_name().as_deref() == Some(n)).and_then(|i| i.id()).expect(n); let c = items.iter().find(|i| i.display_name().as_deref() == Some("C")).expect("C"); // The list holds each used composition once (in no particular order). - let mut used = c.used_item_ids(); + let mut used = crate::common::used_item_ids(c); used.sort(); let mut want = vec![id("P30"), id("Q")]; want.sort(); @@ -60,8 +60,8 @@ fn an_image_sequence_knows_its_numbering() { let item = project.all_items().into_iter().find(|i| i.footage().is_some_and(|f| !f.is_solid)).expect("sequence"); let f = item.footage().expect("footage"); assert_eq!((f.file_type.as_str(), f.width, f.height, f.native_frame_rate, f.is_sequence), ("oEXR", 64, 48, 30, true)); - assert_eq!(f.sequence_frame_count(), Some(3)); let s = item.sequence_info().expect("numbering"); + assert_eq!(s.frame_count(), 3); assert_eq!((s.prefix.as_str(), s.suffix.as_str(), s.start, s.end, s.digits), ("shot_", ".exr", 101, 103, 4)); assert_eq!(s.file_name(101), "shot_0101.exr"); // A still has no numbering (000: the solid; the base has no other footage). diff --git a/crates/aep/tests/it/container/header.rs b/crates/aep/tests/it/container/header.rs index f78b0bba9..f9252f7fd 100644 --- a/crates/aep/tests/it/container/header.rs +++ b/crates/aep/tests/it/container/header.rs @@ -55,7 +55,6 @@ fn assert_plain(h: &LayerHeader) { assert_eq!(h.label, 1); // No name was set, so the copy of the name at byte 64 is empty (the layer shows its source name). assert_eq!(h.name_copy, ""); - assert!(!h.name_is_user_set()); assert!(!h.is_null_object()); assert_eq!(h.parent_id, None); assert_eq!(h.source_id, Some(0x1a)); @@ -67,16 +66,13 @@ fn base_layer_header() { assert_plain(&h); assert_eq!(h.quality, Quality::Best); assert_eq!(h.blend_mode, BlendMode::Normal); - assert_eq!(h.stretch, 1); - assert_eq!(h.stretch_percent(), 100.0); + assert_eq!((h.stretch, h.stretch_den), (1, 1)); assert_eq!(switches(&h), NONE); // start 0 s, in 0 s, out 5 s; denominator = the composition timescale. assert_eq!(h.start, Rational { num: 0, den: 25600 }); assert_eq!(h.in_relative, Rational { num: 0, den: 25600 }); assert_eq!(h.out_relative, Rational { num: 128_000, den: 25600 }); - assert_eq!(h.start_seconds(), Some(0.0)); - assert_eq!(h.in_point_seconds(), Some(0.0)); - assert_eq!(h.out_point_seconds(), Some(5.0)); + assert_eq!(h.in_point(), Rational::new(0, 1)); assert_eq!(h.out_point(), Rational::new(5, 1)); } @@ -98,7 +94,7 @@ fn one_switch_per_case() { assert_eq!(switches(&h), expected, "{name}"); assert_plain(&h); assert_eq!((h.quality, h.blend_mode, h.stretch), (Quality::Best, BlendMode::Normal, 1), "{name}"); - assert_eq!(h.out_point_seconds(), Some(5.0), "{name}"); + assert_eq!(h.out_point(), Rational::new(5, 1), "{name}"); } } @@ -106,7 +102,6 @@ fn one_switch_per_case() { fn name_blend_mode_and_quality() { let h = skip!(l_header("010-layer-name.aep")); assert_eq!(h.name_copy, "Renamed"); - assert!(h.name_is_user_set()); let bytes = fixture!("010-layer-name.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); @@ -114,7 +109,6 @@ fn name_blend_mode_and_quality() { // The layer name is the user's; the source footage keeps its own name. assert_eq!(layer.name().as_deref(), Some("Renamed")); assert_eq!(project.layer_source(&layer).and_then(|i| i.display_name()).as_deref(), Some("L")); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("Renamed")); assert_eq!(skip!(l_header("016-layer-blend-multiply.aep")).blend_mode, BlendMode::Multiply); assert_eq!(skip!(l_header("023-layer-quality-draft.aep")).quality, Quality::Draft); @@ -127,7 +121,7 @@ fn display_name_falls_back_to_the_source_name() { let project = Project::new(&file).unwrap(); let layer = base_layer(&first_comp(&project)); assert_eq!(layer.name(), None); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("L")); + assert_eq!(project.layer_source(&layer).and_then(|i| i.display_name()).as_deref(), Some("L")); assert_eq!(project.layer_source_size(&layer), Some((1920, 1080))); let source = project.layer_source(&layer).unwrap(); assert_eq!(source.id(), Some(0x1a)); @@ -139,26 +133,23 @@ fn time_stretch_and_the_in_out_start_convention() { // 018: stretch 100 -> 200: the stored out point stays 5 s, the layer ends at 10 s. let h = skip!(l_header("018-layer-stretch200.aep")); assert_eq!(h.stretch, 2); - assert_eq!(h.stretch_percent(), 200.0); + assert_eq!(h.stretch_ratio(), Rational::new(2, 1)); assert_eq!(h.out_relative, Rational { num: 128_000, den: 25600 }); - assert_eq!(h.out_point_seconds(), Some(10.0)); - assert_eq!(h.in_point_seconds(), Some(0.0)); + assert_eq!(h.out_point(), Rational::new(10, 1)); + assert_eq!(h.in_point(), Rational::new(0, 1)); // 019: inPoint 0 -> 1; AE moved the out point to 6 s and left the start at 0. let h = skip!(l_header("019-layer-inpoint1.aep")); - assert_eq!(h.start_seconds(), Some(0.0)); + assert_eq!(h.start.num, 0); assert_eq!(h.in_relative, Rational { num: 25600, den: 25600 }); assert_eq!(h.in_point(), Rational::new(1, 1)); - assert_eq!(h.out_point_seconds(), Some(6.0)); + assert_eq!(h.out_point(), Rational::new(6, 1)); // 021: startTime 0 -> 0.5: in 0.5 s and out 5.5 s although the stored in point stays 0. let h = skip!(l_header("021-layer-starttime0.5.aep")); assert_eq!(h.start, Rational { num: 12800, den: 25600 }); - assert_eq!(h.start_seconds(), Some(0.5)); assert_eq!(h.in_relative.num, 0); assert_eq!(h.in_point(), Rational::new(1, 2)); - assert_eq!(h.in_point_seconds(), Some(0.5)); - assert_eq!(h.out_point_seconds(), Some(5.5)); assert_eq!(h.out_point(), Rational::new(11, 2)); } @@ -175,7 +166,6 @@ fn parent_link_and_null_object() { let nh = null.header().unwrap(); assert_eq!(null.name().as_deref(), Some("Null")); assert!(nh.is_null_object()); - assert!(nh.name_is_user_set()); assert_eq!(nh.layer_type, LayerType::Av); assert_eq!(nh.parent_id, None); assert_eq!(project.layer_source(null).and_then(|i| i.display_name()).as_deref(), Some("Null 1")); @@ -225,17 +215,14 @@ fn base_comp_header() { assert_eq!((h.width, h.height), (1920, 1080)); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 25600)); assert_eq!(h.frame_rate(), Some(25.0)); - assert_eq!(h.frame_rate_from_fixed(), 25.0); + assert_eq!(h.frame_rate_fixed, 25 << 16); assert_eq!(h.duration_units, 128_000); assert_eq!(h.duration_seconds(), Some(5.0)); assert_eq!(h.background, [0, 0, 0]); - assert_eq!(h.background_rgb(), [0.0, 0.0, 0.0]); assert_eq!(h.shutter_angle, 180); - assert_eq!(h.work_area_start_seconds(), Some(0.0)); + assert_eq!(h.work_area_start_units, 0); assert_eq!(h.work_area_end_units, None); - assert_eq!(h.work_area_end_seconds(), Some(5.0)); assert_eq!(h.pixel_aspect, Rational { num: 1, den: 1 }); - assert_eq!(h.pixel_aspect_ratio(), Some(1.0)); assert!(!h.motion_blur); // The manifest calls the classic renderer `ADBE Advanced 3d`; the file stores `ADBE Escher` // and the display name `Classic 3D`. @@ -255,7 +242,7 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("031-comp-fps30.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 30720)); assert_eq!(h.frame_rate(), Some(30.0)); - assert_eq!(h.frame_rate_from_fixed(), 30.0); + assert_eq!(h.frame_rate_fixed, 30 << 16); assert_eq!(h.duration_units, 153_600); assert_eq!(h.duration_seconds(), Some(5.0)); @@ -263,12 +250,11 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("032-comp-duration10.aep")); assert_eq!(h.duration_units, 256_000); assert_eq!(h.duration_seconds(), Some(10.0)); - assert_eq!(h.work_area_end_seconds(), Some(10.0)); + assert_eq!(h.work_area_end_units, None); // 033: bgColor [0,0,0] -> [0,1,0]. let (h, _, _) = skip!(comp_header("033-comp-bg-green.aep")); assert_eq!(h.background, [0, 255, 0]); - assert_eq!(h.background_rgb(), [0.0, 1.0, 0.0]); // 034: shutterAngle 180 -> 90. let (h, _, _) = skip!(comp_header("034-comp-shutter90.aep")); @@ -279,8 +265,6 @@ fn one_comp_change_per_case() { let (h, _, _) = skip!(comp_header("035-comp-workarea-1-2.aep")); assert_eq!(h.work_area_start_units, 25600); assert_eq!(h.work_area_end_units, Some(76_800)); - assert_eq!(h.work_area_start_seconds(), Some(1.0)); - assert_eq!(h.work_area_end_seconds(), Some(3.0)); } #[test] @@ -290,7 +274,6 @@ fn one_comp_change_per_case_aspect_blur_renderer() { // 037: pixelAspect 1 -> 2. let (h, _, _) = skip!(comp_header("037-comp-pixelaspect2.aep")); assert_eq!(h.pixel_aspect, Rational { num: 2, den: 1 }); - assert_eq!(h.pixel_aspect_ratio(), Some(2.0)); // 038: comp motion blur on. let (h, _, _) = skip!(comp_header("038-comp-motionblur.aep")); @@ -328,17 +311,16 @@ fn precomp_item_ids_and_labels() { assert_eq!(h.source_id, pre_item.id()); assert_eq!(project.layer_source(layer).unwrap().id(), pre_item.id()); assert_eq!(h.label, 15); - assert!(!h.name_is_user_set()); assert_eq!(layer.name(), None); - assert_eq!(project.layer_display_name(layer).as_deref(), Some("Pre")); + assert_eq!(project.layer_source(layer).and_then(|i| i.display_name()).as_deref(), Some("Pre")); assert_eq!(project.layer_source_size(layer), Some((1920, 1080))); // The precomp layer's start time has the denominator 1; the Pre comp is 1920x1080, 25 fps, 5 s. assert_eq!(h.start.den, 1); let ph = pre.header().unwrap(); assert_eq!((ph.width, ph.height, ph.frame_rate(), ph.duration_seconds()), (1920, 1080, Some(25.0), Some(5.0))); // C records that it uses the comp Pre (chunk idpi, a little-endian u32). - assert_eq!(c_item.used_item_ids(), [pre_item.id().unwrap()]); - assert!(pre_item.used_item_ids().is_empty()); + assert_eq!(crate::common::used_item_ids(c_item), [pre_item.id().unwrap()]); + assert!(crate::common::used_item_ids(pre_item).is_empty()); assert_eq!(pre_item.label(), Some(15)); // The layer inside Pre is a new layer made from the solid L (moving the layers into a precomp // creates new layers). @@ -347,7 +329,7 @@ fn precomp_item_ids_and_labels() { let ih = inner[0].header().unwrap(); assert_ne!(ih.id, L); assert_eq!(ih.source_id, Some(0x1a)); - assert_eq!(project.layer_display_name(&inner[0]).as_deref(), Some("L")); + assert_eq!(project.layer_source(&inner[0]).and_then(|i| i.display_name()).as_deref(), Some("L")); } #[test] @@ -356,9 +338,8 @@ fn a_negative_stretch_is_a_reversed_layer() { // point 0 and out point 5 s relative, so the layer runs from 5 s back to 0 s; 983: -50 % = -1/2. let h = skip!(l_header("981-layer-stretch-minus100-start5.aep")); assert_eq!((h.stretch, h.stretch_den), (-1, 1)); - assert_eq!(h.stretch_percent(), -100.0); - assert_eq!((h.in_point_seconds(), h.out_point_seconds()), (Some(5.0), Some(0.0))); + assert_eq!((h.in_point(), h.out_point()), (Rational::new(5, 1), Rational::new(0, 1))); let h = skip!(l_header("983-layer-stretch-minus50-opacity-keys.aep")); assert_eq!((h.stretch, h.stretch_den), (-1, 2)); - assert_eq!((h.in_point_seconds(), h.out_point_seconds()), (Some(5.0), Some(2.5))); + assert_eq!((h.in_point(), h.out_point()), (Rational::new(5, 1), Rational::new(5, 2))); } diff --git a/crates/aep/tests/it/container/helpers.rs b/crates/aep/tests/it/container/helpers.rs index b6ab11432..af2282ab8 100644 --- a/crates/aep/tests/it/container/helpers.rs +++ b/crates/aep/tests/it/container/helpers.rs @@ -379,7 +379,8 @@ pub fn layer_named<'f, 'a>(comp: &Comp<'f, 'a>, name: &str) -> Layer<'f, 'a> { /// The layer that shows this name: its own name, or its source's while it has none. pub fn layer_shown_as<'f, 'a>(project: &Project<'f, 'a>, comp: &Comp<'f, 'a>, name: &str) -> Layer<'f, 'a> { - comp.layers().find(|l| project.layer_display_name(l).as_deref() == Some(name)).unwrap_or_else(|| panic!("no layer shown as {name}")) + let shown = |l: &Layer<'f, 'a>| l.name().or_else(|| project.layer_source(l)?.display_name()); + comp.layers().find(|l| shown(l).as_deref() == Some(name)).unwrap_or_else(|| panic!("no layer shown as {name}")) } /// The first layer of this type in a composition. diff --git a/crates/aep/tests/it/container/keyframes.rs b/crates/aep/tests/it/container/keyframes.rs index 56ec367cc..27b8073ab 100644 --- a/crates/aep/tests/it/container/keyframes.rs +++ b/crates/aep/tests/it/container/keyframes.rs @@ -19,6 +19,9 @@ fn prop_of(name: &str, path: &[&str], f: impl FnOnce(Property<'_, '_>) -> R) const POSITION: &[&str] = &["ADBE Transform Group", "ADBE Position"]; const OPACITY: &[&str] = &["ADBE Transform Group", "ADBE Opacity"]; +/// The ease of a linear key: all four slots 0. +const NO_EASE: KeyEase = KeyEase { in_speed: 0.0, in_influence: 0.0, out_speed: 0.0, out_influence: 0.0 }; + fn ease(in_speed: f64, in_influence: f64, out_speed: f64, out_influence: f64) -> KeyEase { KeyEase { in_speed, in_influence, out_speed, out_influence } } @@ -33,7 +36,7 @@ fn in_and_out_interpolation_are_separate_bytes() { assert_eq!((keys[0].interpolation_in(), keys[0].interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Hold))); assert_eq!((keys[1].interpolation_in(), keys[1].interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Linear))); assert_eq!(keys.iter().map(|k| k.seconds(25600).unwrap()).collect::>(), [0.0, 1.0]); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); })); // 203: key 1 inType HOLD, outType LINEAR (bytes 03 01): byte 4 is the incoming side. skip!(prop_of("203-key-in-hold-out-linear.aep", POSITION, |p| { @@ -76,7 +79,7 @@ fn opacity_ease_speed_is_in_stored_units_per_second() { assert_near(&[e0.in_speed, e0.in_influence, e0.out_speed, e0.out_influence], &[0.05, 0.2, 0.2, 0.8], 1e-12); let e1 = keys[1].ease().unwrap(); assert_near(&[e1.in_speed, e1.in_influence, e1.out_speed, e1.out_influence], &[0.3, 0.6, 0.09, 0.4], 1e-12); - assert_eq!(keys[0].plain_ease_slots().unwrap().len(), 4); + assert_eq!(keys[0].ease_components(), Some(vec![e0])); for k in &keys { assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Bezier), Some(Interpolation::Bezier))); } @@ -94,7 +97,7 @@ fn scale_keys_use_the_plain_template_with_three_components() { // A 128-byte record is not always spatial: the header's template flag decides. assert_eq!(k.kind(), KeyframeKind::Plain { components: 3 }); assert_eq!(k.spatial(), None); - assert_eq!(k.plain_ease_slots(), Some(vec![0.0; 12])); + assert_eq!(k.ease_components(), Some(vec![NO_EASE; 3])); assert_eq!(k.ease(), None); assert_eq!((k.interpolation_in(), k.interpolation_out()), (Some(Interpolation::Linear), Some(Interpolation::Linear))); } @@ -125,7 +128,7 @@ fn two_dimensional_mask_feather_keys() { assert!(keys.iter().all(|k| k.raw().len() == 88 && k.kind() == KeyframeKind::Plain { components: 2 })); assert_eq!(keys.iter().map(|k| k.value().unwrap()).collect::>(), [vec![0.0, 0.0], vec![40.0, 20.0]]); assert_eq!(keys.iter().map(|k| k.seconds(25600).unwrap()).collect::>(), [0.0, 1.0]); - assert_eq!(keys[0].plain_ease_slots(), Some(vec![0.0; 8])); + assert_eq!(keys[0].ease_components(), Some(vec![NO_EASE; 2])); let mask = &layer.masks()[0]; assert!(mask.feather.source == ValueSource::Keyframed && mask.feather.value == [0.0, 0.0]); } diff --git a/crates/aep/tests/it/container/layer_switches.rs b/crates/aep/tests/it/container/layer_switches.rs index 5e2179d53..05e939c38 100644 --- a/crates/aep/tests/it/container/layer_switches.rs +++ b/crates/aep/tests/it/container/layer_switches.rs @@ -2,7 +2,7 @@ //! colours, checked against the values the corpus was made with (its MANIFEST, batch 2). use crate::container::helpers::{assert_near, base_layer, first_comp, layer_of_type, layer_shown_as, open}; -use craft_aep::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Project, Rational, ValueSource}; +use craft_aep::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Project}; fn l_header(name: &str) -> Option { Some({ @@ -80,22 +80,24 @@ fn ntsc_frame_rates_are_exact_fractions() { // 229: frameRate 29.97 -> read back 29.9700012207031; the duration snaps from 5 s to 150 frames (5.005005 s). let h = skip!(comp_header("229-comp-fps-29.97.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (800, 23976)); - assert_eq!(h.frame_rate_exact(), Some(Rational { num: 2997, den: 100 })); assert!((h.frame_rate().unwrap() - 29.97).abs() < 1e-12); - assert!((h.frame_rate_from_fixed() - 29.970_001_220_703_1).abs() < 1e-7); - assert_eq!((h.duration_units, h.duration_frames()), (120_000, Some(150))); + // The 16.16 rate holds the float32 29.97 (29.9700012207031). + assert_eq!(h.frame_rate_fixed, 1_964_114); + // 150 frames of 800 units. + assert_eq!(h.duration_units, 120_000); assert!((h.duration_seconds().unwrap() - 5.005_005_005_005_01).abs() < 1e-9); // 230: frameRate 23.976 -> read back 23.9759979248047; 120 frames. let h = skip!(comp_header("230-comp-fps-23.976.aep")); assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1000, 23976)); - assert_eq!(h.frame_rate_exact(), Some(Rational { num: 2997, den: 125 })); assert!((h.frame_rate().unwrap() - 23.976).abs() < 1e-12); - assert!((h.frame_rate_from_fixed() - 23.975_997_924_804_7).abs() < 1e-7); - assert_eq!((h.duration_units, h.duration_frames()), (120_000, Some(120))); + // The 16.16 rate holds the float32 23.976 (23.9759979248047). + assert_eq!(h.frame_rate_fixed, 1_571_291); + // 120 frames of 1000 units. + assert_eq!(h.duration_units, 120_000); assert!((h.duration_seconds().unwrap() - 5.005_005_005_005_01).abs() < 1e-9); // 25 fps stays 25. let h = skip!(comp_header("000-base.aep")); - assert_eq!((h.frame_rate_exact(), h.duration_frames()), (Some(Rational { num: 25, den: 1 }), Some(125))); + assert_eq!((h.time_base.timescale, h.time_base.frame_duration, h.duration_units), (25600, 1024, 128_000)); } #[test] @@ -128,7 +130,6 @@ fn background_is_8_bits_per_channel() { // 231: bgColor [0.5, 0.5, 0.5] is stored as 128 and reads back as 128/255 = 0.50196. let h = skip!(comp_header("231-comp-bg-grey-0.5.aep")); assert_eq!(h.background, [128, 128, 128]); - assert_near(&h.background_rgb(), &[0.501_960_784_313_73; 3], 1e-12); // The frame rate stays 25 even though an earlier edit of the AE session left other numbers in the cache-like fields. assert_eq!((h.time_base.frame_duration, h.time_base.timescale), (1024, 25600)); } @@ -143,12 +144,10 @@ fn cameras_auto_orient_and_position() { let camera = layer_of_type(&comp, LayerType::Camera); let h = camera.header().unwrap(); assert_eq!((h.flag_bytes[1], h.is_auto_oriented(), h.is_three_d(), h.source_id, h.light_type()), (0x44, true, true, None, None)); - // The default Position is an "auto state" record that stores [0, 0, -2666.67]; the two-node camera reports the comp centre. + // The default Position is an "auto state" record that stores [0, 0, -2666.67]. let pos = camera.camera_position().unwrap(); assert!(pos.auto_state); assert_near(&pos.stored, &[0.0, 0.0, -2666.67], 0.01); - assert_near(&pos.reported(1920, 1080, true), &[960.0, 540.0, -2666.67], 0.01); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); let bytes = fixture!("251-camera-one-node.aep"); let file = open(&bytes); @@ -159,10 +158,9 @@ fn cameras_auto_orient_and_position() { // AE reports [0, 0, -2666.67] for the one-node camera: the stored value. let pos = camera.camera_position().unwrap(); assert!(pos.auto_state); - assert_near(&pos.reported(1920, 1080, h.is_auto_oriented()), &[0.0, 0.0, -2666.67], 0.01); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); + assert_near(&pos.stored, &[0.0, 0.0, -2666.67], 0.01); - // 253: Position set to [100, 200, -1000]: stored as given, no auto state; the point of interest stays [960, 540, 0]. + // 253: Position set to [100, 200, -1000]: stored as given, no auto state. let bytes = fixture!("253-camera-position-100-200-m1000.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); @@ -170,35 +168,28 @@ fn cameras_auto_orient_and_position() { let pos = camera.camera_position().unwrap(); assert!(!pos.auto_state); assert_eq!(pos.stored, [100.0, 200.0, -1000.0]); - assert_eq!(pos.reported(1920, 1080, true), [100.0, 200.0, -1000.0]); - assert_eq!(camera.camera_point_of_interest(), Some([960.0, 540.0, 0.0])); - // A camera has no light intensity. - assert_eq!(camera.light_intensity(), None); } +/// Lights type and position. #[test] -fn lights_type_and_intensity() { - // 254: a spot light, Intensity 100 -> 50 (stored as 50.0 percent, absent at the default 100). +fn lights_type_and_position() { + // 254: a spot light (its Intensity 50 is checked on import). let bytes = fixture!("254-light-spot-intensity-50.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let light = layer_of_type(&first_comp(&project), LayerType::Light); let h = light.header().unwrap(); assert_eq!((h.light_type(), h.light_type_code, h.source_id, h.is_auto_oriented(), h.is_three_d()), (Some(LightType::Spot), 1, None, true, true)); - let intensity = light.light_intensity().unwrap(); - assert_eq!((intensity.value, intensity.source), (50.0, ValueSource::Stored)); assert!(light.camera_position().unwrap().auto_state); assert_near(&light.camera_position().unwrap().stored, &[0.0, 0.0, -1333.33], 0.01); - // 252: a point light (light type 2) at the default intensity, which is not stored. + // 252: a point light (light type 2). let bytes = fixture!("252-light-point.aep"); let file = open(&bytes); let project = Project::new(&file).unwrap(); let light = layer_of_type(&first_comp(&project), LayerType::Light); let h = light.header().unwrap(); assert_eq!((h.light_type(), h.light_type_code), (Some(LightType::Point), 2)); - let intensity = light.light_intensity().unwrap(); - assert_eq!((intensity.value, intensity.source), (100.0, ValueSource::Default)); // The solid under it is no light. let solid = base_layer(&first_comp(&project)).header().unwrap(); assert_eq!((solid.light_type(), solid.light_type_code), (None, 0)); @@ -220,6 +211,6 @@ fn solid_colour_is_four_float32_argb() { // L is still red. let red = project.layer_source(&base_layer(&comp)).unwrap().footage().unwrap(); assert_eq!(red.solid_rgb(), Some([1.0, 0.0, 0.0])); - // The new layer has no user-set name flag (it shows its source's name). - assert!(!blue.header().unwrap().name_is_user_set()); + // The new layer has no name of its own (it shows its source's name). + assert_eq!(blue.name(), None); } diff --git a/crates/aep/tests/it/container/matte.rs b/crates/aep/tests/it/container/matte.rs index 97c8aad74..c05045e39 100644 --- a/crates/aep/tests/it/container/matte.rs +++ b/crates/aep/tests/it/container/matte.rs @@ -13,7 +13,7 @@ fn layers(name: &str) -> Option, u32)>> { comp.layers() .map(|l| { let h = l.header().expect("header"); - (project.layer_display_name(&l).unwrap_or_default(), h.is_video_enabled(), l.track_matte(), h.id) + (l.name().or_else(|| project.layer_source(&l)?.display_name()).unwrap_or_default(), h.is_video_enabled(), l.track_matte(), h.id) }) .collect() }) diff --git a/crates/aep/tests/it/container/robustness.rs b/crates/aep/tests/it/container/robustness.rs index f27f53d25..0f26cdd4c 100644 --- a/crates/aep/tests/it/container/robustness.rs +++ b/crates/aep/tests/it/container/robustness.rs @@ -154,16 +154,15 @@ use craft_aep::{AepFile, Body, Chunk, Comp, Layer, Project, PropGroup, PropNode, /// Touches the property accessors; none may panic on a damaged property. fn exercise_property(p: &Property<'_, '_>) { - let _ = (p.header(), p.dims(), p.is_animated(), p.static_value(), p.time_scale(), p.flags(), p.is_static_flag_set()); - let _ = (p.expression(), p.expression_flags().map(|f| (f.all_set(), f.disabled())), p.expression_enabled(), p.user_name(), p.first_key_value()); - let _ = p.color_argb(); + let _ = (p.header(), p.dims(), p.is_animated(), p.static_value(), p.time_scale(), p.flags()); + let _ = (p.expression(), p.expression_flags().map(|f| f.disabled()), p.expression_enabled(), p.user_name(), p.first_key_value()); if let Some(keys) = p.keys() { let scale = p.time_scale().unwrap_or(0); for k in keys.keyframes() { - let _ = (k.time_units(), k.spatial_value(), k.raw().len(), k.kind(), k.seconds(scale), k.seconds(0)); + let _ = (k.time_units(), k.raw().len(), k.kind(), k.seconds(scale), k.seconds(0)); let _ = (k.interpolation_in(), k.interpolation_out(), k.flags(), k.label(), k.is_roving()); - let _ = (k.scalar_value(), k.ease(), k.spatial(), k.components(), k.value(), k.plain_ease_slots()); - let _ = (k.ease_components(), k.color_distance()); + let _ = (k.scalar_value(), k.ease(), k.spatial(), k.components(), k.value()); + let _ = k.ease_components(); for component in [0, 1, 2, 3, 7, 255, usize::MAX] { let _ = k.ease_component(component); } @@ -193,10 +192,9 @@ fn exercise_group(g: &PropGroup<'_, '_>, depth: usize) { fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Layer<'_, '_>) { let _ = (layer.name(), layer.is_three_d(), layer.id(), layer.track_matte()); if let Some(h) = layer.header() { - let _ = (h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds(), h.start_seconds(), h.stretch_percent()); - let _ = (h.stretch_ratio(), h.stretch_den); + let _ = (h.in_point(), h.out_point(), h.stretch_ratio(), h.stretch_den); let _ = (h.is_shy(), h.is_locked(), h.is_solo(), h.is_three_d(), h.is_motion_blur(), h.is_adjustment(), h.is_guide()); - let _ = (h.is_null_object(), h.collapses_transformations(), h.name_is_user_set(), h.quality, h.blend_mode, h.layer_type); + let _ = (h.is_null_object(), h.collapses_transformations(), h.quality, h.blend_mode, h.layer_type); let _ = ( h.is_enabled(), h.is_video_enabled(), @@ -208,10 +206,8 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye ); } let _ = (layer.is_enabled(), layer.is_audio_enabled(), layer.is_effects_enabled()); - let _ = (layer.camera_position().map(|p| p.reported(comp_size.0, comp_size.1, true)), layer.camera_point_of_interest(), layer.light_intensity()); + let _ = layer.camera_position().map(|p| (p.stored, p.auto_state)); let _ = project.layer_anchor_px(layer); - let _ = (layer.camera_zoom(), layer.camera_depth_of_field(), layer.camera_focus_distance(), layer.camera_aperture()); - let _ = (layer.light_color(), layer.light_cone_angle(), layer.light_cone_feather()); let _ = layer .time_remap() .map(|t| (t.enabled, t.keys.iter().fold(0.0, |acc, k| acc + k.value_seconds + k.time_seconds.unwrap_or(0.0) + f64::from(k.time_units)))); @@ -238,7 +234,8 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye } } } - let _ = (project.layer_source(layer).map(|i| i.id()), project.layer_display_name(layer), project.layer_source_size(layer)); + let _ = + (project.layer_source(layer).map(|i| i.id()), project.layer_source(layer).and_then(|i| i.display_name()), project.layer_source_size(layer)); if let Some(t) = layer.transform(comp_size.0, comp_size.1) { let _ = (t.anchor_px(f64::from(comp_size.0), f64::from(comp_size.1)), t.position.value, t.opacity_percent.value); } @@ -271,7 +268,6 @@ fn exercise_layer(project: &Project<'_, '_>, comp_size: (u32, u32), layer: &Laye } for m in layer.masks() { let _ = (m.name.clone(), m.mode, m.inverted, m.counter, m.feather, m.opacity_percent, m.expansion, m.path_animated, m.path_key_seconds.len()); - let _ = project.mask_vertices_px(layer, &m).map(|shapes| shapes.iter().map(Vec::len).sum::()); for s in &m.shapes { let _ = (s.vertex_count(), s.vertices((1920.0, 1080.0)).len(), s.vertices((f64::NAN, f64::INFINITY)).len(), s.open); } @@ -302,10 +298,9 @@ fn exercise_comp(project: &Project<'_, '_>, comp: &Comp<'_, '_>) { let mut size = (1920, 1080); if let Some(h) = comp.header() { size = (u32::from(h.width), u32::from(h.height)); - let _ = (h.frame_rate(), h.frame_rate_from_fixed(), h.duration_seconds(), h.work_area_start_seconds(), h.work_area_end_seconds()); - let _ = (h.frame_rate_exact(), h.duration_frames()); - let _ = (h.background_rgb(), h.pixel_aspect_ratio(), h.pixel_aspect.reduced(), h.shutter_angle, h.motion_blur); - let _ = (h.shutter_phase, h.display_start, h.display_start_seconds(), h.display_start_frame()); + let _ = (h.frame_rate(), h.frame_rate_fixed, h.duration_seconds(), h.work_area_start_units, h.work_area_end_units); + let _ = (h.background, h.pixel_aspect.to_f64(), h.pixel_aspect.reduced(), h.shutter_angle, h.motion_blur); + let _ = (h.shutter_phase, h.display_start.map(|r| r.to_f64())); } for layer in comp.layers() { exercise_layer(project, size, &layer); @@ -316,10 +311,8 @@ fn exercise_comp(project: &Project<'_, '_>, comp: &Comp<'_, '_>) { fn exercise(file: &AepFile<'_>) { let Some(project) = Project::new(file) else { return }; for item in project.all_items() { - let _ = (item.kind(), item.name(), item.id(), item.label(), item.used_item_ids(), item.footage(), item.display_name()); - let _ = item - .footage() - .map(|f| (f.duration, f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate, f.is_sequence, f.sequence_frame_count())); + let _ = (item.kind(), item.name(), item.id(), item.label(), item.footage(), item.display_name()); + let _ = item.footage().map(|f| (f.duration, f.native_frame_rate, f.bits_per_pixel, f.file_size, f.audio_sample_rate, f.is_sequence)); let _ = item.id().and_then(|id| project.item_by_id(id)); if let Some(comp) = item.comp() { exercise_comp(&project, &comp); @@ -572,12 +565,12 @@ fn extreme_values_in_the_decoded_headers() { use craft_aep::{CompHeader, LayerHeader, Rational, ShapePath}; // Every field at its maximum: no overflow, no division by zero, no panic. let h = LayerHeader::parse(&[0xff; 164]).unwrap(); - let _ = (h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds(), h.start_seconds(), h.stretch_percent()); + let _ = (h.in_point(), h.out_point(), h.start.to_f64(), h.stretch_ratio()); let mut zero_den = [0u8; 164]; zero_den[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); zero_den[12..16].copy_from_slice(&i32::MIN.to_be_bytes()); let h = LayerHeader::parse(&zero_den).unwrap(); - assert_eq!((h.in_point(), h.out_point(), h.start_seconds()), (None, None, None)); + assert_eq!((h.in_point(), h.out_point(), h.start.to_f64()), (None, None, None)); let mut wide = [0u8; 164]; wide[8..12].copy_from_slice(&u32::MAX.to_be_bytes()); wide[12..16].copy_from_slice(&i32::MIN.to_be_bytes()); @@ -585,14 +578,12 @@ fn extreme_values_in_the_decoded_headers() { wide[20..24].copy_from_slice(&i32::MAX.to_be_bytes()); wide[24..28].copy_from_slice(&u32::MAX.to_be_bytes()); let h = LayerHeader::parse(&wide).unwrap(); - let _ = (h.in_point(), h.in_point_seconds()); + let _ = h.in_point(); let c = CompHeader::parse(&[0xff; 204]).unwrap(); - assert_eq!((c.duration_seconds(), c.work_area_start_seconds(), c.work_area_end_units), (None, None, None)); - let _ = (c.frame_rate(), c.frame_rate_from_fixed(), c.background_rgb(), c.pixel_aspect_ratio(), c.frame_rate_exact()); - assert_eq!(c.duration_frames(), Some(1)); + assert_eq!((c.duration_seconds(), c.work_area_end_units), (None, None)); + let _ = (c.frame_rate(), c.pixel_aspect.to_f64()); let zero = CompHeader::parse(&[0; 204]).unwrap(); - assert_eq!((zero.frame_rate(), zero.pixel_aspect_ratio(), zero.duration_seconds()), (None, None, None)); - assert_eq!((zero.frame_rate_exact(), zero.duration_frames()), (None, None)); + assert_eq!((zero.frame_rate(), zero.pixel_aspect.to_f64(), zero.duration_seconds()), (None, None, None)); let all = LayerHeader::parse(&[0xff; 164]).unwrap(); let _ = (all.is_enabled(), all.is_auto_oriented(), all.light_type(), all.blend_mode.code()); assert_eq!(LayerHeader::parse(&[0u8; 164]).unwrap().light_type(), None); @@ -691,7 +682,7 @@ fn hostile_text_dictionaries_and_marker_headers_never_panic() { #[test] fn extreme_values_in_the_batch_3_fields() { - use craft_aep::{CompHeader, Footage, LayerHeader, Rational}; + use craft_aep::{CompHeader, LayerHeader}; // Time stretch: zero and maximal numerators and denominators (bytes 8 and 108), with extreme times. for (num, den) in [(0u32, 0u32), (1, 0), (0, 1), (u32::MAX, 1), (1, u32::MAX), (u32::MAX, u32::MAX), (3, 2)] { let mut ldta = [0u8; 164]; @@ -704,7 +695,7 @@ fn extreme_values_in_the_batch_3_fields() { ldta[28..32].copy_from_slice(&i32::MIN.to_be_bytes()); ldta[32..36].copy_from_slice(&1u32.to_be_bytes()); let h = LayerHeader::parse(&ldta).unwrap(); - let _ = (h.stretch_ratio(), h.stretch_percent(), h.in_point(), h.out_point(), h.in_point_seconds(), h.out_point_seconds()); + let _ = (h.stretch_ratio(), h.in_point(), h.out_point()); if den == 0 { assert_eq!((h.stretch_ratio(), h.in_point(), h.out_point()), (None, None, None)); } @@ -712,34 +703,8 @@ fn extreme_values_in_the_batch_3_fields() { // Display start and shutter phase from extreme bytes; a missing tail leaves them unknown. for fill in [0x00u8, 0x7f, 0x80, 0xff] { let c = CompHeader::parse(&[fill; 204]).unwrap(); - let _ = (c.shutter_phase, c.display_start, c.display_start_seconds(), c.display_start_frame()); + let _ = (c.shutter_phase, c.display_start.map(|r| r.to_f64())); } - let zero_rate = CompHeader::parse(&[0; 204]).unwrap(); - assert_eq!((zero_rate.display_start_seconds(), zero_rate.display_start_frame()), (None, None)); - // Footage durations: zero denominator, negative and maximal numerators, a huge frame rate. - let base = Footage { - file_type: String::new(), - width: 0, - height: 0, - name: None, - is_solid: false, - solid_argb: None, - path: None, - duration: None, - native_frame_rate: 0, - bits_per_pixel: 0, - file_size: 0, - audio_sample_rate: None, - is_sequence: true, - psd_layer: None, - }; - for (num, den, rate) in - [(3i64, 30i64, 30u32), (3, 0, 30), (-3, 30, 30), (i64::MAX, 1, u32::MAX), (i64::MIN, -1, u32::MAX), (1, i64::MAX, 1), (0, 1, 0)] - { - let f = Footage { duration: Some(Rational { num, den }), native_frame_rate: rate, ..base.clone() }; - let _ = f.sequence_frame_count(); - } - assert_eq!(Footage { duration: Some(Rational { num: 3, den: 30 }), native_frame_rate: 30, ..base.clone() }.sequence_frame_count(), Some(3)); - assert_eq!(Footage { duration: Some(Rational { num: 3, den: 0 }), native_frame_rate: 30, ..base.clone() }.sequence_frame_count(), None); - assert_eq!(Footage { duration: None, ..base }.sequence_frame_count(), None); + // A zero denominator has no value. + assert_eq!(CompHeader::parse(&[0; 204]).unwrap().display_start.and_then(|r| r.to_f64()), None); } diff --git a/crates/aep/tests/it/container/text_markers_footage.rs b/crates/aep/tests/it/container/text_markers_footage.rs index 6d666fbee..f6f81eed1 100644 --- a/crates/aep/tests/it/container/text_markers_footage.rs +++ b/crates/aep/tests/it/container/text_markers_footage.rs @@ -13,7 +13,6 @@ fn text_layer_document(name: &str) -> Option { let layer = comp.layers().find(|l| l.name().as_deref() == Some("T")).expect("the text layer T"); let h = layer.header().unwrap(); assert_eq!(h.layer_type, LayerType::Text); - assert!(h.name_is_user_set()); layer.text_document().expect("text document") }) } @@ -192,7 +191,6 @@ fn png_footage_layer() { // The path comes from the alias JSON (the fixture holds a neutral replacement of the original). assert!(f.path.as_deref().is_some_and(|p| p.ends_with("test8x8.png")), "{:?}", f.path); assert_eq!(project.layer_source_size(&layer), Some((8, 8))); - assert_eq!(project.layer_display_name(&layer).as_deref(), Some("png")); // L is still there below it. assert_eq!(comp.layers().count(), 2); } diff --git a/crates/aep/tests/it/container/tree_index.rs b/crates/aep/tests/it/container/tree_index.rs new file mode 100644 index 000000000..c1bed0163 --- /dev/null +++ b/crates/aep/tests/it/container/tree_index.rs @@ -0,0 +1,231 @@ +//! The flat tree index the importer reads ([`TreeIndex`]) against the owned chunk tree, on every +//! file of the fixture directory: the same tree, and the same answers from the typed view. + +use std::fmt::Write as _; + +use craft_aep::index::TreeIndex; +use craft_aep::{ + AepFile, Chunk, CryptomatteData, CurvesData, Effect, HueSaturationData, LevelsData, Limits, MeshData, Project, PropGroup, PropNode, is_aepx, + parse, parse_with_limits, read_aepx, write, +}; + +/// Every `.aep` and `.aepx` in the fixture directory (the real projects included), sorted. +fn every_file() -> Vec { + let Some(dir) = crate::common::dir() else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let Ok(entries) = std::fs::read_dir(&dir) else { + crate::common::skipped("*.aep"); + return Vec::new(); + }; + let mut out: Vec = + entries.filter_map(Result::ok).map(|e| e.path()).filter(|p| p.extension().is_some_and(|x| x == "aep" || x == "aepx")).collect(); + out.sort(); + out +} + +/// What the typed view reads from a project, as text: items, compositions, layers with their +/// headers, transforms, masks, effects with every parameter value, text, markers, the whole +/// property tree with static values and key records, settings and versions. +fn dump(view: &Project<'_, '_>) -> String { + let mut s = String::new(); + let defs = view.effect_definition_index(); + for item in view.all_items() { + let _ = writeln!(s, "item {:?} {:?} {:?} {:?} {:?}", item.kind(), item.name(), item.display_name(), item.id(), item.label()); + let _ = writeln!(s, " footage {:?} {:?} {:?}", item.footage(), item.sequence_info(), item.conform_frame_rate()); + let _ = writeln!(s, " folder {} {}", item.has_folder_contents(), item.folder_contents_unreadable()); + let Some(comp) = item.comp() else { continue }; + let _ = writeln!(s, " comp {:?} {:?} {:?}", comp.time_base(), comp.header(), comp.markers()); + for layer in comp.layers() { + let _ = writeln!(s, " layer {:?} {:?} {:?} {:?}", layer.name(), layer.id(), layer.header(), layer.track_matte()); + let _ = writeln!(s, " transform {:?}", layer.transform(1920, 1080)); + let _ = writeln!(s, " time remap {:?} orientation {:?} {:?}", layer.time_remap(), layer.orientation(), layer.orientation_keys()); + let _ = writeln!(s, " masks {:?}", layer.mask_atoms().iter().map(|m| (&m.mask, &m.path_shapes)).collect::>()); + let _ = writeln!(s, " shapes {:?}", layer.shape_groups()); + let _ = writeln!( + s, + " text {:?} {} {:?}", + layer.text_document(), + layer.text_document_count(), + layer.source_text_property().map(|p| p.is_animated()) + ); + let _ = writeln!(s, " markers {:?} source {:?}", layer.markers(), view.layer_source(&layer).map(|i| i.id())); + for fx in layer.effects() { + let _ = writeln!( + s, + " fx {} {:?} {:?} {:?} {:?}", + fx.match_name(), + fx.display_name(), + fx.enabled(), + defs.source(&fx), + fx.value_record_names() + ); + let _ = + writeln!(s, " names {:?} {:?} {:?} {:?}", fx.name(), fx.user_name(), fx.declared_param_count(), fx.has_own_definitions()); + for p in defs.params(&fx) { + let _ = writeln!(s, " {} {:?} {:?} {:?} {:?}", p.match_name, p.kind, p.name, p.items, p.value()); + arbitrary(&fx, &p.match_name, &mut s); + } + for name in fx.value_record_names() { + arbitrary(&fx, &name, &mut s); + } + } + if let Some(root) = layer.properties() { + tree(&root, 0, &mut s); + } + } + } + s +} + +/// The arbitrary data stored for a parameter and what each decoder reads from it. +fn arbitrary(fx: &Effect<'_, '_>, name: &str, s: &mut String) { + let Some(data) = fx.arbitrary_data(name) else { return }; + let _ = writeln!(s, " arbitrary {name} {data:?}"); + let _ = writeln!(s, " {:?} {:?}", HueSaturationData::parse(data), LevelsData::parse(data)); + let _ = writeln!(s, " {:?} {:?} {:?}", CurvesData::parse(data), MeshData::parse(data), CryptomatteData::parse(data)); +} + +/// Appends a group and everything below it to the dump, depth-limited. +fn tree(group: &PropGroup<'_, '_>, depth: usize, s: &mut String) { + if depth > 32 { + return; + } + let _ = writeln!(s, "{:depth$}group {:?} {:?}", "", group.user_name(), group.flags()); + for e in group.entries() { + match e.node { + PropNode::Group(g) => tree(&g, depth + 1, s), + PropNode::Property(p) => { + let keys: Option>> = p.keys().map(|k| k.records().map(<[u8]>::to_vec).collect()); + let _ = writeln!(s, "{:depth$} {} {:?} {:?} {:?} {:?}", "", e.match_name, p.user_name(), p.header(), p.static_value(), keys); + } + PropNode::Other(c) => { + let _ = writeln!(s, "{:depth$} {} other {:?} {}", "", e.match_name, c.list_type(), c.descendants().count()); + } + } + } +} + +/// The index is the parsed tree and the view reads the same on every file. +#[test] +fn the_index_is_the_parsed_tree_and_the_view_reads_the_same_on_every_file() { + for path in every_file() { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + // `.aepx` is read into the chunk tree and indexed from it; `.aep` is indexed in place. + // A file that does not read (a damaged sample) must fail the index the same way. + let file: AepFile<'_> = match if is_aepx(&bytes) { read_aepx(&bytes) } else { parse(&bytes) } { + Ok(file) => file, + Err(e) => { + if !is_aepx(&bytes) { + assert_eq!(TreeIndex::parse(&bytes).err(), Some(e), "{name}"); + } + continue; + } + }; + let index = (if is_aepx(&bytes) { TreeIndex::from_file(&file) } else { TreeIndex::parse(&bytes) }).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(index.to_file() == file, "{name}: the index is not the parsed tree"); + assert_eq!(index.len(), file.walk().count(), "{name}"); + assert_eq!(index.file_version(), file.file_version(), "{name}"); + assert_eq!(index.saved_by_version(), file.saved_by_version(), "{name}"); + assert_eq!(craft_aep::ProjectSettings::read_index(&index), craft_aep::ProjectSettings::read(&file), "{name}"); + let (Some(a), Some(b)) = (Project::new(&file), Project::from_index(&index)) else { + assert!(Project::new(&file).is_none() && Project::from_index(&index).is_none(), "{name}"); + continue; + }; + let (da, db) = (dump(&a), dump(&b)); + assert!(da == db, "{name}: the view reads differently through the index"); + } +} + +/// The index and the parser fail alike under tight limits. +#[test] +fn the_index_and_the_parser_fail_alike_under_tight_limits() { + for path in every_file().into_iter().filter(|p| p.extension().is_some_and(|x| x == "aep")) { + let name = path.display().to_string(); + let bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{name}: {e}")); + let total = parse(&bytes).map(|f| f.walk().count()).unwrap_or(0); + for limits in [ + Limits { max_depth: 2, max_chunks: 10_000_000 }, + Limits { max_depth: 6, max_chunks: 10_000_000 }, + Limits { max_depth: 64, max_chunks: total }, + Limits { max_depth: 64, max_chunks: total.saturating_sub(1) }, + Limits { max_depth: 64, max_chunks: total / 2 }, + ] { + let old = parse_with_limits(&bytes, limits); + let new = TreeIndex::parse_with_limits(&bytes, limits); + match (old, new) { + (Ok(f), Ok(i)) => assert!(i.to_file() == f, "{name}: {limits:?}"), + (Err(a), Err(b)) => assert_eq!(a, b, "{name}: {limits:?}"), + (a, b) => panic!("{name}: {limits:?}: {:?} vs {:?}", a.err(), b.err()), + } + } + } +} + +/// A 40-byte NUL-padded match name. +fn tdmn(name: &str) -> Chunk<'static> { + let mut b = name.as_bytes().to_vec(); + b.resize(40, 0); + Chunk::data(*b"tdmn", b) +} + +/// A nested chunk `id` holding one `Utf8` with `text`. +fn utf8(id: [u8; 4], text: &str) -> Chunk<'static> { + Chunk { id, body: craft_aep::Body::Chunks(vec![Chunk::data(*b"Utf8", text.as_bytes().to_vec())]), pad: None } +} + +/// A project with one composition, one layer and one effect whose five parameters carry arbitrary +/// data in the sizes the decoders read (Hue/Saturation, Levels, Curves, a 2 x 2 mesh, Cryptomatte), +/// so the arbitrary-data path runs without the fixture directory. +fn synthetic() -> Vec { + let mut mesh = vec![0u8; 180]; + mesh[..8].copy_from_slice(&[0, 0, 0, 2, 0, 0, 0, 2]); + let mut cry = vec![0u8; 200]; + cry[..4].copy_from_slice(b"cry1"); + cry[32..37].copy_from_slice(b"Layer"); + let mut levels = vec![0u8; 108]; + levels[8..16].copy_from_slice(&1.0f64.to_be_bytes()); + let payloads: [Vec; 5] = [vec![7u8; 180], levels, vec![0u8; CurvesData::SIZE], mesh, cry]; + let (mut part, mut values) = + (vec![Chunk::data(*b"parn", 6u32.to_be_bytes().to_vec())], vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), utf8(*b"tdsn", "Mine")]); + for (i, data) in payloads.into_iter().enumerate() { + let name = format!("FX-{:04}", i + 1); + let mut pard = vec![0u8; 148]; + pard[12..16].copy_from_slice(&11u32.to_be_bytes()); + part.extend([tdmn(&name), Chunk::data(*b"pard", pard)]); + let record = Chunk::list(*b"tdbs", vec![Chunk::data(*b"tdsb", 1u32.to_be_bytes().to_vec()), Chunk::data(*b"cdat", vec![0, 0, 0, 1])]); + values.extend([tdmn(&name), record, Chunk::list(*b"aRbs", vec![Chunk::data(*b"aRbp", data)])]); + } + values.push(tdmn("ADBE Group End")); + let sspc = Chunk::list(*b"sspc", vec![utf8(*b"fnam", "Effect"), Chunk::list(*b"parT", part), Chunk::list(*b"tdgp", values)]); + let parade = Chunk::list(*b"tdgp", vec![tdmn("FX"), sspc, tdmn("ADBE Group End")]); + let root = Chunk::list(*b"tdgp", vec![tdmn("ADBE Effect Parade"), parade, tdmn("ADBE Group End")]); + let layer = Chunk::list(*b"Layr", vec![Chunk::data(*b"ldta", vec![0u8; 160]), Chunk::data(*b"Utf8", b"Layer".to_vec()), root]); + let mut idta = vec![0u8; 64]; + idta[..2].copy_from_slice(&4u16.to_be_bytes()); + idta[16..20].copy_from_slice(&2u32.to_be_bytes()); + let mut cdta = vec![0u8; 200]; + cdta[4..12].copy_from_slice(&[0, 0, 0, 1, 0, 0, 0, 30]); + let item = Chunk::list(*b"Item", vec![Chunk::data(*b"idta", idta), Chunk::data(*b"Utf8", b"Comp".to_vec()), Chunk::data(*b"cdta", cdta), layer]); + let file = + AepFile { chunks: vec![Chunk::data(*b"head", vec![0u8; 20]), Chunk::list(*b"Fold", vec![item])], trailer: std::borrow::Cow::Borrowed(&[]) }; + write(&file).expect("write") +} + +/// The view reads arbitrary effect data the same through the index. +#[test] +fn the_view_reads_arbitrary_effect_data_the_same_through_the_index() { + let bytes = synthetic(); + let file = parse(&bytes).expect("parse"); + let index = TreeIndex::parse(&bytes).expect("index"); + let (a, b) = (Project::new(&file).expect("project"), Project::from_index(&index).expect("project")); + let (da, db) = (dump(&a), dump(&b)); + assert_eq!(da, db); + // The synthetic effect really reaches every decoder (the dump is not empty of them). + assert_eq!(da.matches("arbitrary FX-").count(), 10, "{da}"); + assert!(da.contains("Some(HueSaturationData") && da.contains("Some(LevelsData") && da.contains("Some(CurvesData"), "{da}"); + assert!(da.contains("Some(MeshData") && da.contains("Some(CryptomatteData"), "{da}"); + assert!(da.contains("Some(\"Mine\")"), "the user name is read: {da}"); +} diff --git a/crates/aep/tests/it/container/typed.rs b/crates/aep/tests/it/container/typed.rs index 7299043e9..9be97f3fd 100644 --- a/crates/aep/tests/it/container/typed.rs +++ b/crates/aep/tests/it/container/typed.rs @@ -111,7 +111,7 @@ fn match_names_come_from_the_tdmn_chunks() { assert!(!names.iter().any(|n| n == "ADBE Group End")); assert!(root.find(&["ADBE Transform Group", "ADBE Nonexistent"]).is_none()); // The raw tdmn body is 40 bytes, NUL padded. - let raw = root.chunk().children().iter().find(|c| c.is(b"tdmn")).unwrap(); + let raw = root.chunk().children().find(|c| c.is(b"tdmn")).unwrap(); assert_eq!(raw.bytes().unwrap().len(), 40); } @@ -150,8 +150,8 @@ fn effect_parameter_value() { assert!(instance.is_list(b"sspc")); // The effect's name is in a nested fnam chunk. let fnam = instance.child(b"fnam").unwrap(); - assert!(matches!(fnam.body, Body::Chunks(_))); - assert_eq!(fnam.children()[0].bytes(), Some(&b"Slider Control"[..])); + assert!(matches!(fnam.to_chunk().body, Body::Chunks(_))); + assert_eq!(fnam.children().next().unwrap().bytes(), Some(&b"Slider Control"[..])); // The parameters are a property group inside the instance. let params = PropGroup::from_chunk(instance.child_list(b"tdgp").unwrap()).unwrap(); assert_eq!(params.names(), ["ADBE Slider Control-0000", "ADBE Slider Control-0001", "ADBE Effect Built In Params"]); @@ -186,7 +186,7 @@ fn position_keyframes() { assert_eq!(times, [0, 25_600, 64_000]); let seconds: Vec = times.iter().map(|t| tb.seconds(*t).unwrap()).collect(); assert_eq!(seconds, [0.0, 1.0, 2.5]); - let values: Vec<[f64; 3]> = keys.keyframes().map(|k| k.spatial_value().unwrap()).collect(); + let values: Vec> = keys.keyframes().map(|k| k.value().unwrap()).collect(); assert_eq!(values, [[0.0, 0.0, 0.0], [100.0, 0.0, 0.0], [200.0, 100.0, 0.0]]); } diff --git a/crates/aep/tests/it/container/values.rs b/crates/aep/tests/it/container/values.rs index 17fccf62f..6c3616012 100644 --- a/crates/aep/tests/it/container/values.rs +++ b/crates/aep/tests/it/container/values.rs @@ -119,7 +119,7 @@ fn linear_position_keys() { let values: Vec<_> = keys.iter().map(|k| k.spatial().unwrap().value).collect(); assert_eq!(values, [[0.0, 0.0, 0.0], [100.0, 0.0, 0.0], [200.0, 100.0, 0.0]]); assert_eq!(p.first_key_value(), Some(vec![0.0, 0.0, 0.0])); - assert_eq!(p.is_static_flag_set(), Some(false)); + assert!(p.is_animated()); })); // The transform of the layer reports the animated Position as keyframed (value at the first key). let (t, _) = skip!(transform("055-key-position-3key-inserted-t2.5.aep")); @@ -181,7 +181,7 @@ fn roving_and_label_flags() { assert_eq!(roving, [false, true, false]); assert_eq!(keys[1].time_units(), Some(26509)); assert_eq!(keys[1].seconds(25600), Some(1.0355078125)); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); assert_eq!(keys[1].flags(), Some(0x0027)); assert_eq!(keys[1].label(), Some(0)); })); @@ -193,7 +193,7 @@ fn roving_and_label_flags() { assert_eq!(keys[1].flags(), Some(0x0247)); assert_eq!(keys[0].flags(), Some(0x0007)); assert_eq!(keys[1].is_roving(), Some(false)); - assert_eq!(keys[1].spatial_value(), Some([100.0, 0.0, 0.0])); + assert_eq!(keys[1].value(), Some(vec![100.0, 0.0, 0.0])); })); } @@ -204,7 +204,6 @@ fn scalar_opacity_keys() { assert!(p.is_animated()); assert_eq!(p.dims(), Some(1)); assert_eq!(p.time_scale(), Some(25600)); - assert_eq!(p.is_static_flag_set(), Some(false)); let list = p.keys().unwrap(); assert_eq!(list.header.record_size, 48); let keys: Vec<_> = list.keyframes().collect(); @@ -243,10 +242,8 @@ fn expression_text_and_flag_bytes() { assert_eq!(p.expression(), None); let flags = p.expression_flags().unwrap(); assert_eq!((flags.byte_76, flags.byte_77, flags.byte_120), (0, 0, 0)); - assert!(!flags.all_set()); // A written static property: tdsb 1, byte 5 of tdb4 has bit 0 set (15 for Position). assert_eq!(p.flags(), Some(1)); - assert_eq!(p.is_static_flag_set(), Some(true)); assert_eq!(p.header().and_then(|h| h.get(5).copied()), Some(15)); })); } From eadfaf85ed44e892cd6da77546d36f5dbe0da49b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:19:03 -0500 Subject: [PATCH 02/31] craft-cms: LUT file parsing fixes and GamutCheck::mask_rgba (ports of open PhotoCraft PRs) (#860) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * craft-cms: the changes PhotoCraft made to its copy since extraction PhotoCraft kept editing crates/cms after craft-cms was extracted from it at storytold/photocraft@b3222ee8. This ports its 41 commits up to storytold/photocraft@0143b148 with a three-way merge per file, keeping the changes craft-cms made on its own (Result-returning API, white preservation, FOGRA39, named colours, profile ID). Ported: the .3dl shaper, Mesh header, UTF-16 and .look fixes in lutfile; the checked ICC offsets and the per-profile curve sample budget; same_bytes and the byte-confirmed transform cache; the blocked matrix/TRC path and the 8-bit first-stage table; GpuMatrixShaper; cicp_rgb; the ACEScg and Dot Gain 20 % built-ins (the latter now the default gray); bisection for a table's flat ends; PhotoCraft spelled with a capital C in the built-in descriptions, with the shipped coated CMYK profile's bytes as PhotoCraft regenerated them. Not taken, because craft-cms already does it another way: Curve::inverse_table (inverse64_many), the effective-dimension loop in multilinear interpolation, the curve-count and mluc checks, and the .cube DOMAIN writer. The 8-bit first-stage table checks device white as the exact path does. Refs #779 * craft-cms: Lut3d::from_fn PhotoCraft's proof and HDR display tables are built from a chain of several transforms, which it wrote into Lut3d's fields. Those are private here, so from_fn samples any RGB function, with the same size bounds as from_transform. Refs #779 * craft-cms: README names Lut3d::from_fn * craft-cms: a UTF-8 byte-order mark before a .cube or .3dl file is skipped A LUT file saved with a UTF-8 BOM failed to parse: the mark stayed on the first keyword. decode_text now strips it, as it already handles UTF-16. Ported from storytold/photocraft#9776. * craft-cms: a .cube data line with a field that is not a number is an error A data line's fields after the first were parsed with filter_map, so '0.25 nope 0.5 0.75' silently became the three numbers that did parse. Every field must now parse. Ported from storytold/photocraft#10112. * craft-cms tests: .look data is pinned to hand-written little-endian bytes The test built its input with the same to_le_bytes the parser mirrors, so a wrong byte order on both sides would pass. The bytes are now written out. Ported from storytold/photocraft#10389. * craft-cms: a 1D .cube whose domain the 33-point bake cannot hold is refused A 1D LUT is resampled onto a fixed 33-point grid. A steep curve over a narrow DOMAIN_MIN/DOMAIN_MAX (or LUT_1D_INPUT_RANGE) does not survive that: a two-node ramp over 0.24..0.26 baked 42 RGB8 levels off. The parser now measures the baked table against the exact curve for a non-default domain and refuses one that shifts by more than two RGB8 levels, naming the channel. Default-domain tables and gentle curves are unchanged. Ported from storytold/photocraft#9101. * craft-cms: only a quote that opens a .cube TITLE protects '#' A quote anywhere on a TITLE line made the text up to the next quote part of the title, so 'TITLE Look # "draft"' took its comment into the title. Only a quote at the start of the value opens a quoted title now; an unterminated one is an error, and an unquoted title ends at '#' like any other line. Ported from storytold/photocraft#8506 (its lutfile.rs part). * craft-cms: GamutCheck::mask_rgba computes a whole gamut-warning mask in parallel A caller building a gamut-warning mask made one delta_e call per pixel on one thread. mask_rgba fills the mask (255 where alpha > 0 and the pixel is out of gamut) in 16K-pixel chunks on the rayon pool, serial below that and on wasm, and returns the count. The result is identical to out_of_gamut per pixel. A check not built from an RGB source profile is refused before anything is written, since an RGBA pixel cannot stand for a CMYK, gray or Lab colour. Ported from storytold/photocraft#9748. --------- Co-authored-by: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Co-authored-by: Max Mittelstädt <62167861+TheGalitube@users.noreply.github.com> Co-authored-by: smittie2000 Co-authored-by: bfenty <1398236+bfenty@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: vangel76 --- crates/cms/README.md | 2 + crates/cms/src/gamut.rs | 92 +++++++++++++++++++++- crates/cms/src/lutfile.rs | 162 +++++++++++++++++++++++++++++++++++--- 3 files changed, 240 insertions(+), 16 deletions(-) diff --git a/crates/cms/README.md b/crates/cms/README.md index 55bbc6b82..71f3146d5 100644 --- a/crates/cms/README.md +++ b/crates/cms/README.md @@ -30,6 +30,8 @@ Buffers are checked once per call, before any pixel is written. * `Transform::proof(src, proof, display, intent, bpc, simulate_paper)`: soft-proof chain. * `GamutCheck::new(src, proof, ΔE)`: out-of-gamut test (gamut warning). + `check.mask_rgba(&rgba, &mut mask)?` fills a whole mask (255 where alpha > 0 and out of gamut) on + the thread pool and returns the count; it needs an RGB source profile. * `Lut3d::from_transform(&t, 33)?.to_rgba16f_bytes()`: display transform as a 3D texture; `Lut3d::from_fn(size, f)` samples any RGB function. * `Transform::gpu_matrix_shaper()`: accurate RGB input curves + matrix + output curves for diff --git a/crates/cms/src/gamut.rs b/crates/cms/src/gamut.rs index 247b183f2..11dd17732 100644 --- a/crates/cms/src/gamut.rs +++ b/crates/cms/src/gamut.rs @@ -2,13 +2,18 @@ use crate::clut::Clut; use crate::pipeline::{Pipeline, Stage}; -use crate::profile::Profile; +use crate::profile::{ColorSpace, Profile}; use crate::transform::link_stages; -use crate::{CmsError, Intent, check_len}; +use crate::{CmsError, Intent, check_len, invalid_arg}; /// Default ΔE76 threshold above which a colour counts as out of gamut. pub const DEFAULT_THRESHOLD: f32 = 4.0; +#[cfg(not(target_arch = "wasm32"))] +const PAR_MIN_PIXELS: usize = 16 * 1024; +#[cfg(not(target_arch = "wasm32"))] +const PAR_CHUNK_PIXELS: usize = 16 * 1024; + /// Precomputed out-of-gamut test for colours of `src` against `proof`. /// /// A colour is out of gamut when sending it to the proof device (relative colorimetric, @@ -17,6 +22,7 @@ pub const DEFAULT_THRESHOLD: f32 = 4.0; #[derive(Debug)] pub struct GamutCheck { inputs: usize, + source_space: ColorSpace, grid: Option, exact: (Pipeline, Pipeline), pub threshold: f32, @@ -44,7 +50,7 @@ impl GamutCheck { 4 => Some(Clut::sample(vec![17; 4], 1, de)?), _ => None, }; - Ok(GamutCheck { inputs: n, grid, exact: (direct, via), threshold }) + Ok(GamutCheck { inputs: n, source_space: src.color_space, grid, exact: (direct, via), threshold }) } pub fn inputs(&self) -> usize { @@ -76,4 +82,84 @@ impl GamutCheck { pub fn out_of_gamut(&self, px: &[f32]) -> Result { Ok(self.delta_e(px)? > self.threshold) } + + /// Out-of-gamut mask for RGBA pixels: `255` where the pixel has alpha > 0 and its RGB + /// channels are out of gamut, `0` elsewhere. Returns the number of flagged pixels. Only + /// `min(px.len(), mask.len())` pixels are processed. Runs on the rayon pool (not on wasm); + /// the result is identical to calling [`Self::out_of_gamut`] per pixel. + /// + /// # Errors + /// [`CmsError::InvalidArgument`] when the check was not built from an RGB source profile: an + /// RGBA pixel carries only RGB colour channels, so it cannot be tested against a CMYK, gray + /// or Lab source. Nothing is written to `mask` in that case. + pub fn mask_rgba(&self, px: &[[f32; 4]], mask: &mut [u8]) -> Result { + if self.source_space != ColorSpace::Rgb || self.inputs != 3 { + return Err(invalid_arg(format!("gamut mask needs an RGB source profile, got {:?} with {} channels", self.source_space, self.inputs))); + } + let n = px.len().min(mask.len()); + let (Some(px), Some(mask)) = (px.get(..n), mask.get_mut(..n)) else { return Ok(0) }; + let run = |p: &[[f32; 4]], m: &mut [u8]| { + let mut count = 0; + for (p, m) in p.iter().zip(m.iter_mut()) { + // The source has three channels (checked above), so the check cannot fail. + let hit = p[3] > 0.0 && self.out_of_gamut(&[p[0], p[1], p[2]]).unwrap_or(false); + *m = if hit { 255 } else { 0 }; + count += usize::from(hit); + } + count + }; + #[cfg(not(target_arch = "wasm32"))] + if n >= PAR_MIN_PIXELS { + use rayon::prelude::*; + return Ok(px.par_chunks(PAR_CHUNK_PIXELS).zip(mask.par_chunks_mut(PAR_CHUNK_PIXELS)).map(|(p, m)| run(p, m)).sum()); + } + Ok(run(px, mask)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Builtin; + + fn image(n: usize) -> Vec<[f32; 4]> { + let mut s = 0x9E37_79B9u32; + let mut rnd = move || { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s >> 8) as f32 / (1u32 << 24) as f32 + }; + // Includes out-of-range values and zero alpha. + (0..n).map(|i| [rnd() * 1.2 - 0.1, rnd(), rnd(), if i % 7 == 0 { 0.0 } else { rnd() }]).collect() + } + + #[test] + fn mask_rgba_matches_per_pixel_check() { + let src = Builtin::Srgb.profile(); + // Sizes straddle the parallel threshold and the chunk size, odd ones included. + for (proof, threshold) in [(Builtin::CoatedCmyk, 4.0), (Builtin::CoatedCmyk, 1.5), (Builtin::GrayGamma22, 4.0), (Builtin::DisplayP3, 0.5)] { + let check = GamutCheck::new(src, proof.profile(), threshold).unwrap(); + for n in [0, 1, 1023, 16 * 1024 - 1, 16 * 1024, 16 * 1024 + 1, 50_001] { + let px = image(n); + let want: Vec = px.iter().map(|p| if p[3] > 0.0 && check.out_of_gamut(&p[..3]).unwrap() { 255 } else { 0 }).collect(); + let mut got = vec![7u8; n]; + let count = check.mask_rgba(&px, &mut got).unwrap(); + assert_eq!(got, want, "{proof:?} threshold {threshold} n {n}"); + assert_eq!(count, want.iter().filter(|&&m| m == 255).count()); + } + } + } + + #[test] + fn mask_rgba_rejects_non_rgb_source() { + // A CMYK (4-channel), gray (1-channel) and Lab (3-channel, not RGB) source cannot be tested from RGBA pixels. + for src in [Builtin::CoatedCmyk, Builtin::GrayGamma22, Builtin::LabD50] { + let check = GamutCheck::new(src.profile(), Builtin::Srgb.profile(), 4.0).unwrap(); + let px = image(64); + let mut mask = vec![9u8; 64]; + assert!(matches!(check.mask_rgba(&px, &mut mask), Err(CmsError::InvalidArgument(_))), "{src:?}"); + assert!(mask.iter().all(|&m| m == 9), "mask must stay untouched"); + } + } } diff --git a/crates/cms/src/lutfile.rs b/crates/cms/src/lutfile.rs index c2261325d..404b5a315 100644 --- a/crates/cms/src/lutfile.rs +++ b/crates/cms/src/lutfile.rs @@ -135,7 +135,8 @@ pub fn parse_as(format: Format, bytes: &[u8]) -> Result { } } -/// UTF-16 with a byte-order mark (Photoshop loads such `.look` files, #9118), else UTF-8. +/// UTF-16 with a byte-order mark (Photoshop loads such `.look` files, #9118), else UTF-8 +/// (a UTF-8 BOM is stripped). fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { let utf16 = |rest: &[u8], from: fn([u8; 2]) -> u16| { let units: Vec = rest.as_chunks::<2>().0.iter().map(|&c| from(c)).collect(); @@ -144,6 +145,7 @@ fn decode_text(bytes: &[u8]) -> std::borrow::Cow<'_, str> { match bytes { [0xFF, 0xFE, rest @ ..] => utf16(rest, u16::from_le_bytes), [0xFE, 0xFF, rest @ ..] => utf16(rest, u16::from_be_bytes), + [0xEF, 0xBB, 0xBF, rest @ ..] => String::from_utf8_lossy(rest), _ => String::from_utf8_lossy(bytes), } } @@ -155,14 +157,19 @@ pub fn parse_cube(text: &str) -> Result { let (mut dmin, mut dmax) = ([0.0f32; 3], [1.0f32; 3]); let mut rows: Vec = Vec::new(); for line in text.lines() { - // A '#' inside a quoted TITLE is part of the title, not a comment. - let raw = line.trim_start(); - let skip = if raw.starts_with("TITLE") { raw.find('"').and_then(|q| raw[q + 1..].find('"').map(|e| q + e + 2)) } else { None }.unwrap_or(0); - let line = match raw[skip..].find('#') { - Some(i) => &raw[..skip + i], - None => raw, + // Only a quote at the beginning of a TITLE value protects '#'. Quotes inside an + // unquoted title's comment must not turn that comment into part of the title. + if let Some(quoted) = line + .trim_start() + .strip_prefix("TITLE") + .filter(|rest| rest.starts_with(char::is_whitespace)) + .and_then(|rest| rest.trim_start().strip_prefix('"')) + { + let Some((value, _)) = quoted.split_once('"') else { return err("unterminated quoted TITLE") }; + title = value.to_string(); + continue; } - .trim(); + let line = line.split('#').next().unwrap_or("").trim(); if line.is_empty() { continue; } @@ -187,7 +194,9 @@ pub fn parse_cube(text: &str) -> Result { } k if k.starts_with(|c: char| c.is_ascii_digit() || c == '-' || c == '+' || c == '.') => { let mut v = vec![k.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?]; - v.extend(it.filter_map(|s| s.parse::().ok())); + for value in it { + v.push(value.parse::().map_err(|_| LutError(format!("bad number in `{line}`")))?); + } if v.len() != 3 { return err(format!("bad data line `{line}`")); } @@ -209,6 +218,15 @@ pub fn parse_cube(text: &str) -> Result { if rows.len() != expected_rows { return err("no LUT_3D_SIZE in .cube file"); } + // #8913: the resample below bakes the curve onto a fixed 33-point grid, so a steep + // curve over a narrow domain cannot survive the resample — the baked table silently + // shifts colours (measured 42 RGB8 levels for a two-node ramp over 0.24..0.26, where + // Photoshop renders the ramp exactly). Rather than proxy the risk by domain width (a + // 0.1..0.17 ramp spans two grid cells yet still loses ~28 levels), measure the + // resample's actual error against the exact curve and refuse only tables whose baked + // output shifts by more than a couple of RGB8 levels. Default-domain tables keep the + // existing bake untouched; gentle curves over any domain bake within tolerance and + // stay accepted. // The 1D table is resampled onto a 0..1 cube anyway, so its input domain // (`DOMAIN_MIN`/`MAX`, `LUT_1D_INPUT_RANGE`) is baked into the resample: an input `v` // reads the table at `(v - min) / (max - min)`, saturating outside. @@ -221,6 +239,36 @@ pub fn parse_cube(text: &str) -> Result { }; let mut l = LutFile::from_fn(&title, 33, |c| [curve(0, c[0]), curve(1, c[1]), curve(2, c[2])])?; l.title = title; + let non_default = dmin.iter().any(|v| *v != 0.0) || dmax.iter().any(|v| *v != 1.0); + if non_default { + // Worst baked-vs-exact error per channel, in RGB8 levels: the baked cube is read + // by linear interpolation between grid points (red fastest, so channel `ch` sits + // at stride 33^ch), while `curve` is what the file actually says. + const MAX_BAKE_ERROR: f32 = 2.0 / 255.0; + for ch in 0..3 { + let stride = [1usize, 33, 33 * 33][ch]; + let mut worst = 0.0f32; + for k in 0..=256 { + let v = k as f32 / 256.0; + let x = v * 32.0; + let i = (x as usize).min(31); + let f = x - i as f32; + let baked = l.data[i * stride * 3 + ch] * (1.0 - f) + l.data[(i + 1).min(32) * stride * 3 + ch] * f; + worst = worst.max((baked - curve(ch, v)).abs()); + } + if worst > MAX_BAKE_ERROR { + return err(format!( + "1D LUT domain on {} is {min}..{max}: the curve changes faster than the 33-point grid can \ + hold (baking shifts colours by up to {levels:.0} RGB8 levels); export the look as a 3D LUT \ + or widen DOMAIN_MIN/DOMAIN_MAX", + ["red", "green", "blue"][ch], + min = dmin[ch], + max = dmax[ch], + levels = worst * 255.0, + )); + } + } + } return l.check(); } err("no LUT_3D_SIZE in .cube file") @@ -554,6 +602,48 @@ pub fn builtin(id: &str) -> Option { mod tests { use super::*; + #[test] + fn cube_title_hashes_and_comment_quotes_are_distinct() { + let rows = "LUT_1D_SIZE 2\n0 0 0 # low\n1 1 1 # high\n"; + for (line, title) in [ + (r#"TITLE "Look #3" # comment "ignored""#, "Look #3"), + (r#"TITLE Look # "draft""#, "Look"), + (" TITLE\t\"Été #2\"", "Été #2"), + (r#"TITLE "" # empty"#, ""), + ] { + assert_eq!(parse_cube(&format!("{line}\n{rows}")).unwrap().title, title); + } + let lut = LutFile::from_fn("Look #3", 2, |c| c).unwrap(); + assert_eq!(parse_cube(&write_cube(&lut)).unwrap().title, "Look #3"); + assert!(parse_cube(&format!("TITLE \"unterminated # title\n{rows}")).is_err()); + } + + #[test] + fn cube_rejects_invalid_numeric_fields() { + let text = "LUT_3D_SIZE 2\n0.25 nope 0.5 0.75\n0 0 1\n0 1 0\n0 1 1\n1 0 0\n1 0 1\n1 1 0\n1 1 1\n"; + assert!(parse_cube(text).is_err(), "invalid numeric field must not be ignored"); + } + + #[test] + fn utf8_bom_is_stripped() { + let l = LutFile::identity(3).unwrap(); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(write_cube(&l).as_bytes()); + let p = parse("a.cube", &bytes).unwrap(); + assert_eq!(p.size, l.size); + assert_eq!(p.title, l.title); + let titled = LutFile::from_fn("Look", 3, |c| c).unwrap(); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(write_cube(&titled).as_bytes()); + assert_eq!(parse("a.cube", &bytes).unwrap().title, "Look"); + let mut bytes = vec![0xEF, 0xBB, 0xBF]; + bytes.extend_from_slice(b"LUT_3D_SIZE 2\n"); + for i in 0..8 { + bytes.extend_from_slice(format!("{} {} {}\n", i & 1, (i >> 1) & 1, i >> 2).as_bytes()); + } + assert_eq!(parse("a.cube", &bytes).unwrap().size, 2); + } + /// An identity `.3dl` of edge `n` (12-bit nodes, blue fastest) under `shaper`. fn identity_3dl(shaper: &[u32], n: usize) -> String { let mut t = shaper.iter().map(u32::to_string).collect::>().join(" ") + "\n"; @@ -811,6 +901,49 @@ DOMAIN_MAX 0.5 1 1 assert!(e.0.contains("below DOMAIN_MAX"), "{e}"); } + #[test] + fn cube_1d_narrow_domain_is_refused() { + // #8913: the issue's fixture — a two-node ramp over 0.24..0.26. Baking it onto the + // 33-point grid rendered grey-65 as 148 where Photoshop renders 190 (42-level error). + // It must now be an actionable parse error rather than a silently wrong table. + let e = parse_cube("LUT_1D_SIZE 2\nDOMAIN_MIN 0.24 0.24 0.24\nDOMAIN_MAX 0.26 0.26 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster than the 33-point grid"), "{e}"); + // The same domain via LUT_1D_INPUT_RANGE is refused identically. + let e = parse_cube("LUT_1D_SIZE 2\nLUT_1D_INPUT_RANGE 0.24 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster"), "{e}"); + // A per-channel narrow domain names the offending channel. + let e = parse_cube("LUT_1D_SIZE 2\nDOMAIN_MIN 0 0 0.24\nDOMAIN_MAX 1 1 0.26\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("blue"), "{e}"); + } + + #[test] + fn cube_1d_wide_domain_still_bakes() { + // A steep ramp over 0.1..0.17 spans two grid cells but still loses ~28 RGB8 levels + // to the resample, so the measured-error check refuses it even though the old width + // check let it through. + let e = parse_cube("LUT_1D_SIZE 2\nLUT_1D_INPUT_RANGE 0.1 0.17\n0 0 0\n1 1 1\n").unwrap_err(); + assert!(e.0.contains("changes faster"), "steep ramp over 0.1..0.17 refused: {e}"); + // A gentle (here linear) curve over a grid-aligned domain bakes exactly and stays + // accepted — the check measures the bake's error, not the domain's width. Value + // assertions, not just a successful parse: the baked table must reproduce the curve. + // Table row k sits at t = k/32 of the domain 0.25..0.75; the curve y = 0.4 + 0.4t + // is gentle (a half-level per grid step), so the bake must reproduce it exactly. + let mut table = String::from("LUT_1D_SIZE 33\nLUT_1D_INPUT_RANGE 0.25 0.75\n"); + for k in 0..33 { + let y = 0.4 + 0.4 * (k as f32 / 32.0); + table.push_str(&format!("{y} {y} {y}\n")); + } + let l = parse_cube(&table).expect("gentle curve over a non-default domain still bakes"); + let at = |r: usize| l.data[r * 3]; + // Grid entry r is input r/32: 0.5 is the middle of the domain (t=0.5) -> 0.6. + assert!((at(16) - 0.6).abs() < 1.0 / 255.0, "baked value at 0.5 tracks the curve: {}", at(16)); + // Domain top (input 0.75 = grid 24, t=1) -> 0.8. + assert!((at(24) - 0.8).abs() < 1.0 / 255.0, "baked value at the domain top tracks the curve: {}", at(24)); + // Below the domain saturates to the first row. + assert!((at(8) - 0.4).abs() < 1.0 / 255.0, "below the domain saturates: {}", at(8)); + parse_cube("LUT_1D_SIZE 2\n0 0 0\n1 1 1\n").expect("default domain"); + } + #[test] fn cube_1d_domain_is_baked_into_the_resample() { // An identity ramp over the domain 0.25..0.75: 0.25 maps to 0, 0.5 to 0.5, 0.75 to 1. @@ -1042,11 +1175,14 @@ DOMAIN_MAX 0.5 1 1 } #[test] - fn look_hex_floats() { - let id = LutFile::identity(2).unwrap(); - let hex: String = id.data.iter().flat_map(|v| v.to_le_bytes()).map(|b| format!("{b:02X}")).collect(); + fn look_data_is_little_endian_float32() { + // Bytes written out by hand rather than by the encoder the parser mirrors, so a wrong byte + // order fails: 0.25, 0.5, 1.0 (3E800000, 3F000000, 3F800000 stored low byte first), then zeros. + let hex = format!("0000803E 0000003F 0000803F {}", "00000000 ".repeat(21)); let xml = format!("\"2\"\"{hex}\""); - assert_eq!(parse("a.look", xml.as_bytes()).unwrap().data, id.data); + let data = parse("a.look", xml.as_bytes()).unwrap().data; + assert_eq!(data[..3], [0.25, 0.5, 1.0]); + assert!(data[3..].iter().all(|&v| v == 0.0)); } #[test] From ce4f4816a174929f96828e1ee2addc0102e3544b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:19:30 -0500 Subject: [PATCH 03/31] text: a work limit on Knuth-Plass; past it break_paragraph falls back to greedy (#868) --- crates/text/src/linebreak/knuth_plass.rs | 14 +++++++++++++- crates/text/src/linebreak/mod.rs | 11 +++++++++-- crates/text/src/linebreak/tests.rs | 19 +++++++++++++++++++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/crates/text/src/linebreak/knuth_plass.rs b/crates/text/src/linebreak/knuth_plass.rs index 742c3245f..e9de0b485 100644 --- a/crates/text/src/linebreak/knuth_plass.rs +++ b/crates/text/src/linebreak/knuth_plass.rs @@ -4,6 +4,11 @@ use super::{Item, Kind, Line, Params, Sums, clamp_measure, fitness_class}; /// Give up (return `None`) beyond this many active breaks. const MAX_ACTIVE: usize = 4096; +/// Give up (return `None`) after this much work, counted as one for each break considered and one +/// for each active break it is tried against. A paragraph of 400,000 items with 30 active breaks +/// takes some 12,000,000 on a pass; the limit is where a pass runs for seconds, and past it the +/// caller falls back to [`super::greedy`], which keeps every item. +pub(super) const MAX_WORK: u64 = 1 << 27; /// Ratios beyond this are rated as this (an unstretchable line is infinitely loose). const MAX_RATIO: f64 = 10_000.0; /// Demerits of an overfull line accepted because nothing else reaches a break. @@ -37,8 +42,14 @@ struct Candidate { /// One total-fit pass over `items`. `width(k)` is the measure of line `k`; set /// [`Params::uniform_from`] when it varies. Returns `None` when no set of breaks keeps every line -/// within [`Params::tolerance`] (unless [`Params::overfull`]), or past the active-break limit. +/// within [`Params::tolerance`] (unless [`Params::overfull`]), or past the active-break or work limit. pub fn knuth_plass(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params) -> Option> { + knuth_plass_within(items, width, p, &mut MAX_WORK.clone()) +} + +/// [`knuth_plass`] spending at most `work` (left over on return), so that the passes of one +/// paragraph share a budget. +pub(super) fn knuth_plass_within(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params, work: &mut u64) -> Option> { let p = &p.sanitized(); let width = |k: usize| clamp_measure(width(k)); let s = Sums::new(items); @@ -57,6 +68,7 @@ pub fn knuth_plass(items: &[Item], width: &dyn Fn(usize) -> f64, p: &Params) -> if !s.is_break(b) { continue; } + *work = work.checked_sub(active.len() as u64 + 1)?; let item = items.get(b); let forced = s.forced(b); let flagged = item.is_some_and(Item::is_flagged); diff --git a/crates/text/src/linebreak/mod.rs b/crates/text/src/linebreak/mod.rs index 9d20b0645..dfdfd9cf3 100644 --- a/crates/text/src/linebreak/mod.rs +++ b/crates/text/src/linebreak/mod.rs @@ -29,6 +29,7 @@ use std::ops::Range; pub use greedy::{greedy, greedy_with}; pub use knuth_plass::knuth_plass; +use knuth_plass::{MAX_WORK, knuth_plass_within}; /// Number of elasticity tiers: word space, letter space, glyph scale. pub const TIERS: usize = 3; @@ -345,11 +346,17 @@ pub struct Breaks { /// [`knuth_plass`] at each of `tolerances` in turn, the last pass also accepting overfull lines; /// [`greedy`] if every pass fails or `tolerances` is empty. The last pass fails only past the -/// active-break limit (see [`Params::uniform_from`]). +/// active-break limit (see [`Params::uniform_from`]) or the passes' shared work limit, which +/// only a paragraph of many hundreds of thousands of items reaches. Nothing is left out then: the +/// greedy breaker lays out every item, in linear time. pub fn break_paragraph(items: &[Item], width: &dyn Fn(usize) -> f64, params: &Params, tolerances: &[f64]) -> Breaks { + break_paragraph_within(items, width, params, tolerances, MAX_WORK) +} + +fn break_paragraph_within(items: &[Item], width: &dyn Fn(usize) -> f64, params: &Params, tolerances: &[f64], mut work: u64) -> Breaks { for (pass, &tolerance) in tolerances.iter().enumerate() { let overfull = params.overfull || pass + 1 == tolerances.len(); - if let Some(lines) = knuth_plass(items, width, &Params { tolerance, overfull, ..*params }) { + if let Some(lines) = knuth_plass_within(items, width, &Params { tolerance, overfull, ..*params }, &mut work) { return Breaks { lines, method: Method::KnuthPlass { pass }, balanced: false }; } } diff --git a/crates/text/src/linebreak/tests.rs b/crates/text/src/linebreak/tests.rs index d8163edde..58483d2aa 100644 --- a/crates/text/src/linebreak/tests.rs +++ b/crates/text/src/linebreak/tests.rs @@ -492,3 +492,22 @@ fn next_start_is_the_first_box_or_forced_break_after_each_item() { assert_eq!(s.next_start(b), scan, "after {b}"); } } + +/// Past its work limit the paragraph is broken greedily, with every item laid out and no error or +/// cut: a limit of a few units stands for the real one, which a hostile paragraph of hundreds of +/// thousands of items would reach (and the test never builds). +#[test] +fn a_paragraph_past_the_work_limit_falls_back_to_greedy_with_every_item() { + let p = vectorcraft(false); + let para = Para::new(COPY, &p, Some(50.0)); + let w = |_| 150.0; + let within = break_paragraph_within(¶.items, &w, &p, &PASSES, MAX_WORK); + assert!(matches!(within.method, Method::KnuthPlass { .. }), "{:?}", within.method); + let limited = break_paragraph_within(¶.items, &w, &p, &PASSES, 20); + assert_eq!(limited.method, Method::Greedy); + assert_eq!(limited.lines, greedy(¶.items, &w, &p)); + assert_covers(¶.items, &limited.lines); + // The work is shared by the passes: what one spends the next doesn't have. + let mut work = 30; + assert!(knuth_plass_within(¶.items, &w, &Params { tolerance: 1.0, ..p }, &mut work).is_none()); +} From 4559fe417130a09b11807311a7af9d1f0abfeb6b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:19:57 -0500 Subject: [PATCH 04/31] print: run lpoptions with a timeout and capped output (#871) --- crates/print/src/lib.rs | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/crates/print/src/lib.rs b/crates/print/src/lib.rs index 01c057269..a4445b11d 100644 --- a/crates/print/src/lib.rs +++ b/crates/print/src/lib.rs @@ -377,11 +377,7 @@ pub fn printer_options(printer: &str) -> Vec { f.take(PPD_CAP).read_to_end(&mut bytes).ok().map(|_| bytes) }); let mut options = ppd.map(|b| parse_ppd(&ppd_text(&b))).unwrap_or_default(); - let current = std::process::Command::new("lpoptions") - .args(["-p", printer, "-l"]) - .output() - .map(|o| parse_lpoptions(&String::from_utf8_lossy(&o.stdout))) - .unwrap_or_default(); + let current = queue_defaults(std::process::Command::new("lpoptions").args(["-p", printer, "-l"]), QUEUES_LIMIT); apply_queue_defaults(&mut options, ¤t); options } @@ -516,6 +512,13 @@ fn printers_via( printers_parsed(&queues, available.as_deref()) } +/// The queue's current choices from `lpoptions -p NAME -l`, run like the `lpstat` commands: its +/// output capped, and given `limit` to answer (none when it doesn't). +#[cfg(all(unix, not(target_arch = "wasm32")))] +fn queue_defaults(command: &mut std::process::Command, limit: std::time::Duration) -> Vec<(String, String)> { + output_within(command, limit).map(|out| parse_lpoptions(&String::from_utf8_lossy(&out))).unwrap_or_default() +} + /// The most of a command's output that is read: far more than any `lpstat` listing. #[cfg(all(unix, not(target_arch = "wasm32")))] const OUTPUT_CAP: u64 = 4 << 20; @@ -612,6 +615,20 @@ mod lpstat_limits { assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); } + /// `lpoptions` that never answers or prints without end gives no defaults, in time and in bounded memory. + #[test] + fn lpoptions_is_bounded_in_time_and_output() { + let started = Instant::now(); + assert!(queue_defaults(&mut sh("sleep 30"), Duration::from_millis(300)).is_empty()); + assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); + let answer = queue_defaults(&mut sh("printf 'Duplex/2-Sided: None *DuplexNoTumble\\n'"), Duration::from_secs(5)); + assert_eq!(answer, [("Duplex".to_string(), "DuplexNoTumble".to_string())]); + // Endless output stops at the cap (the command is killed at the deadline), and does not hang. + let started = Instant::now(); + let _ = queue_defaults(&mut sh("yes 'Key/Label: a *b c'"), Duration::from_millis(500)); + assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); + } + /// A command that isn't there is no printers, at once. #[test] fn no_lpstat_is_no_printers() { From 23a63f5c769fe51f2e5f6c3d217cfbfd5c8ea713 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:20:26 -0500 Subject: [PATCH 05/31] font-catalog: cap the font file read where the file has grown since its size was checked (#873) --- crates/font-catalog/src/catalog.rs | 15 ++++++++++++++- crates/font-catalog/src/tests.rs | 12 ++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/crates/font-catalog/src/catalog.rs b/crates/font-catalog/src/catalog.rs index c20753fb9..10663e6af 100644 --- a/crates/font-catalog/src/catalog.rs +++ b/crates/font-catalog/src/catalog.rs @@ -22,6 +22,19 @@ const MAX_SCAN_READ: u64 = 256 << 20; /// The largest font file loaded. const MAX_LOAD: u64 = 512 << 20; +/// The bytes of the file at `path`, at most `max`: the size was checked before, but a file can +/// grow between that check and the read, so the read itself stops one byte past the limit. +pub(crate) fn read_capped(path: &Path, max: u64) -> Result, FontFileError> { + use std::io::Read as _; + let mut bytes = Vec::new(); + std::fs::File::open(path)?.take(max.saturating_add(1)).read_to_end(&mut bytes)?; + let size = bytes.len() as u64; + if size > max { + return Err(FontFileError::TooLarge { size, max }); + } + Ok(bytes) +} + /// Why a font file or font data wasn't added or loaded. #[derive(Debug, thiserror::Error)] pub enum FontFileError { @@ -544,7 +557,7 @@ impl Catalog { if size > MAX_LOAD { return Err(FontFileError::TooLarge { size, max: MAX_LOAD }); } - let bytes: Arc<[u8]> = std::fs::read(path)?.into(); + let bytes: Arc<[u8]> = read_capped(path, MAX_LOAD)?.into(); let mut files = lock(&self.files); files.retain(|_, w| w.strong_count() > 0); files.insert(path.to_path_buf(), Arc::downgrade(&bytes)); diff --git a/crates/font-catalog/src/tests.rs b/crates/font-catalog/src/tests.rs index c061ca2aa..5137a4295 100644 --- a/crates/font-catalog/src/tests.rs +++ b/crates/font-catalog/src/tests.rs @@ -556,3 +556,15 @@ fn repeated_name_records_are_bounded_before_pairing() { assert!(!face.keys.legacy.is_empty()); assert_eq!(face.keys.legacy.len(), 256, "the pair budget counts distinct pairs"); } + +/// A read stops at its cap, whatever the file has grown to since its size was checked. +#[test] +fn a_read_is_capped_where_the_file_has_grown() { + use crate::catalog::read_capped; + let path = std::env::temp_dir().join(format!("craft-font-catalog-cap-{}", std::process::id())); + std::fs::write(&path, b"0123456789").unwrap(); + assert_eq!(read_capped(&path, 10).unwrap(), b"0123456789"); + assert!(matches!(read_capped(&path, 9), Err(FontFileError::TooLarge { size: 10, max: 9 }))); + assert!(matches!(read_capped(&path.with_extension("missing"), 9), Err(FontFileError::Io(_)))); + let _ = std::fs::remove_file(&path); +} From bafefe627ae457b71abbec23bdd8a9119a669188 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:20:53 -0500 Subject: [PATCH 06/31] hayro-jbig2: PdfCraft's JBIG2 follow-ups (refinement fixes, file header, work accounting, dictionary pixel cap) (#886) * hayro-jbig2: bound a stream's total decode work and what a page retains; no i32 overflow Port of pdfcraft 85c10051 (patches 5-9), 82439525 (patch 10) and 10107d94 (patch 12). * hayro-jbig2: PdfCraft's JBIG2 follow-ups (refinement fixes, file header, work accounting, dictionary pixel cap) Ports pdfcraft 72e763ad, b7ec3669, e3bc4e85, 01d3479f, b9e068aa, 858ce960, 07f44038, e38b9d18, 8717a085, 58c30af3 and the hayro-jbig2 part of e2464e14 (with the hayro-jbig2 part of 0ed33990 they build on). * vendor/README.craft-reader.md: one #831 paragraph, the follow-ups after it --- vendor/README.craft-reader.md | 23 + vendor/hayro-jbig2/SOURCE.json | 16 +- vendor/hayro-jbig2/src/bitmap.rs | 147 +++++-- .../src/decode/generic_refinement.rs | 85 ++-- vendor/hayro-jbig2/src/decode/symbol.rs | 38 +- vendor/hayro-jbig2/src/decode/text.rs | 20 +- vendor/hayro-jbig2/src/error.rs | 58 +++ vendor/hayro-jbig2/src/file.rs | 21 +- vendor/hayro-jbig2/src/gray_scale.rs | 35 +- vendor/hayro-jbig2/src/lib.rs | 414 +++++++++++++++--- vendor/hayro-jbig2/src/pdfcraft_tests.rs | 279 ++++++++++-- 11 files changed, 943 insertions(+), 193 deletions(-) diff --git a/vendor/README.craft-reader.md b/vendor/README.craft-reader.md index f9f281ab9..b8271db10 100644 --- a/vendor/README.craft-reader.md +++ b/vendor/README.craft-reader.md @@ -401,3 +401,26 @@ of 85c10051 for hayro-syntax was already carried here. PdfCraft's raster tests a `jbig2_placements_at_the_end_of_an_i32_draw_nothing`); the unit tests in `lib.rs`, `bitmap.rs` and `decode/symbol.rs` come with the patches. PdfCraft's later JBIG2 follow-ups (b7ec3669, 72e763ad, e3bc4e85, patch 13 and later) are not part of this port. + +JBIG2 follow-ups, ported from PdfCraft after the decode-bound patches above, each marked +`PdfCraft patch` in `hayro-jbig2`: [72e763ad](https://github.com/storytold/pdfcraft/commit/72e763ad1) +(a symbol with no rows costs a step; a refined instance is charged first), +[b7ec3669](https://github.com/storytold/pdfcraft/commit/b7ec3669c) (what an item reads is charged), +[e3bc4e85](https://github.com/storytold/pdfcraft/commit/e3bc4e855) (a height class of no symbols +decodes no bitmap), [0ed33990](https://github.com/storytold/pdfcraft/commit/0ed339901) (only its +`hayro-jbig2` part: `Image::decode_within` takes the work left, so callers can share one allowance), +[01d3479f](https://github.com/storytold/pdfcraft/commit/01d3479f2), [b9e068aa](https://github.com/storytold/pdfcraft/commit/b9e068aa3), +[858ce960](https://github.com/storytold/pdfcraft/commit/858ce9608), +[07f44038](https://github.com/storytold/pdfcraft/commit/07f440388), +[e38b9d18](https://github.com/storytold/pdfcraft/commit/e38b9d182) and +[8717a085](https://github.com/storytold/pdfcraft/commit/8717a085f) (a region past `i32::MAX` is off the page; a +refinement refines the bitmap it is placed on, the page only in its area, and reads 0 past its +reference's width and a reference right of it from its first column), +[58c30af3](https://github.com/storytold/pdfcraft/commit/58c30af31) (a stream that starts with a JBIG2 +file header draws) and the `hayro-jbig2` part of +[e2464e14](https://github.com/storytold/pdfcraft/commit/e2464e144) (a dictionary may hold 2^28 symbol +pixels, as a real 14,642 x 8,879 scan needs; `OverflowError::BitmapLimit` and `DecodeError::is_limit` tell +a decoder's limit from damage). Not ported from those commits: the page-wide allowance and the +refused-mask handling in `hayro-syntax` and `hayro-interpret` (0ed33990, 4aa732ea, e2464e14), which rest on +the decoded-image cache of 558ffd7f (a performance change). Tests: those that came with the patches in +`hayro-jbig2`, and PdfCraft's raster tests adapted in `hayro-jbig2/src/pdfcraft_tests.rs`. diff --git a/vendor/hayro-jbig2/SOURCE.json b/vendor/hayro-jbig2/SOURCE.json index 5efee12a3..0568ac878 100644 --- a/vendor/hayro-jbig2/SOURCE.json +++ b/vendor/hayro-jbig2/SOURCE.json @@ -62,7 +62,7 @@ "path": "src/bitmap.rs", "source_sha256": "59ed83d8bac6c39e40270a9856ca3faee628de7d9efae1b01f714e140e2b6b7e", "source_git_blob": "35ed97561597d30a70fe819cf8cfbd3780538f75", - "sha256": "dd330bda095350b91eeb15b029bdae209e1ed3d5c711dff279b79abd7832f847" + "sha256": "f5fa055ed19f6113d517875479ed4d4d732e06a0010067883a4c38820d55ff97" }, { "path": "src/decode/generic.rs", @@ -74,7 +74,7 @@ "path": "src/decode/generic_refinement.rs", "source_sha256": "625075671e510e8ff1d7c3cade857792413c091d57a35561ad0591d7f04c2ea5", "source_git_blob": "a566b8de58c078dec874d5d1dea36c0f557f1a16", - "sha256": "93e2ce1a40c35fed24d4d611cfc1763518e00167b4e3c0a0671a18afa37c7064" + "sha256": "7a4e36aa8b371f532e33163aad59133f8afa946935570b2312d057cd2a8b3837" }, { "path": "src/decode/halftone.rs", @@ -98,31 +98,31 @@ "path": "src/decode/symbol.rs", "source_sha256": "aa57e88cdc6fd1647d7996875c253a643fedfc4438d5137e326e00d17d0d29ee", "source_git_blob": "f1aa8a6639e53dd350057d8d078198d363daa918", - "sha256": "627de1c83267597efef6d6c0d49aec004f43fd6e63490f117f744ed9210b47f0" + "sha256": "793e9943183b5ef119aa281dc13e782083fdac16e9b4c598083af1a95209256f" }, { "path": "src/decode/text.rs", "source_sha256": "01863cdcd48ab68b5230948d12212a815d293c7c9d5382a140772088bfb1c405", "source_git_blob": "bbc2ac55c7d7d5b8aebffbecfe93fda8b88526e8", - "sha256": "32ae695fe181ba07181fa9915b6bce2ca333031c392fae66dd15ce955a8d7b95" + "sha256": "1f75a55d76111980d667713f841aa38a1fc20918e5bdcfef08a3a36842e2ca10" }, { "path": "src/error.rs", "source_sha256": "39b6ec8fa058bc5c1722892290882e123d193d3844ed5846b30b742b94ba72b0", "source_git_blob": "c579368b44b444566178e68a2705cf9561fad71f", - "sha256": "326dd926cc52b191d0c0398b3b2bb482ca267919e369045f129930cfa485e7fb" + "sha256": "7bc0f28cf64b0510819a50037566b6eeeba1cf2654d095417743cf0b00e2eae8" }, { "path": "src/file.rs", "source_sha256": "10467a415ab2e8b92a8951a86f4c02c5c74f5a005629a11865a20bc23b26ca55", "source_git_blob": "1440a22efa89f0deec0029968c3a20bad8526d63", - "sha256": "c96741babbfacfd78bd2b63f0f16541aaa235f14bb4344ecd4707e3ca5942df1" + "sha256": "7b85b531dbb37ddad69d620e54934a3f88cb88a5aaf1dcd4209c990b92fdbd69" }, { "path": "src/gray_scale.rs", "source_sha256": "46f12893f4573610789015ba7036f6afb1d30b1454c0c674fc7ccdd9337d7487", "source_git_blob": "2836ec77a7c8d3970b13dd46f576778c64bfe2d9", - "sha256": "4bb0963338bfc9e3773db822465d5acbc1267c3349f04b30bff8c52e254874bb" + "sha256": "f82a6760de85bab325cffa6c75a0fa4e1d55fb9c67617be0d0fa112f2dd6c15f" }, { "path": "src/huffman_table.rs", @@ -158,7 +158,7 @@ "path": "src/lib.rs", "source_sha256": "22463230cc82543ee93791b642566a308f0f4eb14d3089b24dc2e7f06d0f784c", "source_git_blob": "ad77a6dd17920b2db66f6bb5a0f2cb8efaafd41a", - "sha256": "d0431d1c57c552b250b7b48895f0d768bfc75496b30cd0146b77764b09759c0f" + "sha256": "ac97e81ca5ec61d5d40247b561d0c1b17d7ac80aa173ee1bde099eb90e52a82f" }, { "path": "src/page_info.rs", diff --git a/vendor/hayro-jbig2/src/bitmap.rs b/vendor/hayro-jbig2/src/bitmap.rs index a240ca432..a40083145 100644 --- a/vendor/hayro-jbig2/src/bitmap.rs +++ b/vendor/hayro-jbig2/src/bitmap.rs @@ -7,6 +7,7 @@ use crate::decode::CombinationOperator; use crate::error::{OverflowError, Result, bail}; +use alloc::vec; use alloc::vec::Vec; // To guard better against malicious files, we limit bitmap dimensions @@ -30,6 +31,12 @@ pub(crate) const MAX_GRID_CELLS: u64 = 1 << 25; /// halftone 265 s, and a 923-byte PDF's regions would have taken days. pub(crate) const MAX_WORK: u64 = 4 * MAX_PIXELS; +/// PdfCraft patch: the work of reading one integer of an item, what an arithmetic-coded one takes +/// at worst in decisions (see [`MAX_WORK`]), so that a decision costs no more than a pixel. An item +/// with no pixels still reads its integers: charged a step each, a 22,515-byte PDF of refined text +/// instances of no size stayed inside `MAX_WORK` and took 730 s. +pub(crate) const INTEGER_WORK: u64 = 38; + /// PdfCraft patch: the work of a line of a Huffman table built for symbols, about 50 ns (see /// [`MAX_WORK`]). pub(crate) const HUFFMAN_LINE_WORK: u64 = 10; @@ -63,10 +70,10 @@ pub(crate) const BITMAP_WORDS: usize = size_of::() / size_of::(); /// output from the page header in between, and a 743-byte PDF's Indexed image asked for 137 GB. pub(crate) fn check_page_size(width: u32, height: u32) -> Result<()> { if width > MAX_DIMENSION || height > MAX_DIMENSION { - return Err(OverflowError::BitmapDimension.into()); + return Err(OverflowError::BitmapLimit.into()); } if u64::from(width) * u64::from(height) > MAX_PIXELS { - return Err(OverflowError::BitmapDimension.into()); + return Err(OverflowError::BitmapLimit.into()); } Ok(()) } @@ -86,6 +93,11 @@ pub(crate) struct Bitmap { pub(crate) stride: u32, /// Packed pixel data, one bit per pixel, row-major order. /// Each row is padded to a word boundary. + /// + /// PdfCraft patch: the padding is 0s, as pixels past the edge are (6.2.5.2, 6.3.5.2). Generic + /// and refinement decoding read the rows above their own pixels a word at a time, padding + /// included, and a refinement reads its reference the same way: whatever writes whole words + /// keeps the padding 0 (see `clear_padding`). pub(crate) data: Vec, /// "This four-byte field gives the horizontal offset in pixels of the bitmap /// encoded in this segment relative to the page bitmap." (7.4.1.3) @@ -95,14 +107,6 @@ pub(crate) struct Bitmap { pub(crate) y_location: u32, } -/// PdfCraft patch: fallible replacement for `vec![word; len]` / `resize`. -fn fill_words(data: &mut Vec, len: usize, word: Word) -> Result<()> { - data.try_reserve_exact(len) - .map_err(|_| OverflowError::BitmapDimension)?; - data.resize(len, word); - Ok(()) -} - impl Bitmap { /// PdfCraft patch: the words this bitmap holds, its header included. pub(crate) fn words(&self) -> usize { @@ -125,26 +129,47 @@ impl Bitmap { default_pixel: bool, ) -> Result { if width > MAX_DIMENSION || height > MAX_DIMENSION { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // PdfCraft patch: see `MAX_PIXELS`. if u64::from(width) * u64::from(height) > MAX_PIXELS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } let stride = width.div_ceil(WORD_BITS); let default_word: Word = if default_pixel { !0 } else { 0 }; - // PdfCraft patch: reserve fallibly so hostile dimensions error instead of aborting. - let mut data = Vec::new(); - fill_words(&mut data, stride as usize * height as usize, default_word)?; - Ok(Self { + let data = vec![default_word; stride as usize * height as usize]; + let mut bitmap = Self { width, height, stride, data, x_location, y_location, - }) + }; + if default_pixel { + bitmap.clear_padding(); + } + Ok(bitmap) + } + + /// PdfCraft patch: clear the bits past `width` in each row's last word. "All pixels lying + /// outside the bounds of the actual bitmap or the reference bitmap have the value 0" (6.3.5.2), + /// and a refinement reads its reference a word at a time, those bits included: filled whole + /// with a default pixel of 1, they read as 1s, so a refinement of the page, or of an + /// intermediate text or halftone region, decoded to other pixels than were coded. + fn clear_padding(&mut self) { + let trailing = self.width % WORD_BITS; + // A width of 0 has no words to clear, and `chunks_exact_mut` would take a stride of 0. + if trailing == 0 { + return; + } + let mask: Word = !0 << (WORD_BITS - trailing); + for row in self.data.chunks_exact_mut(self.stride as usize) { + if let Some(last) = row.last_mut() { + *last &= mask; + } + } } pub(crate) fn reinitialize( @@ -157,15 +182,17 @@ impl Bitmap { let stride = width.div_ceil(WORD_BITS); let default_word: Word = if default_pixel { !0 } else { 0 }; - // Allocate first so a failed reservation leaves the bitmap untouched. - let mut data = Vec::new(); - fill_words(&mut data, stride as usize * height as usize, default_word)?; self.width = width; self.height = height; self.stride = stride; self.x_location = 0; self.y_location = 0; - self.data = data; + self.data.clear(); + self.data + .resize(stride as usize * height as usize, default_word); + if default_pixel { + self.clear_padding(); + } Ok(()) } @@ -194,6 +221,17 @@ impl Bitmap { self.data[word_idx] |= (value as Word) << bit_pos; } + /// PdfCraft patch: combine a decoded region into this page at the region's own location. + /// + /// "Unless otherwise specified, all multi-bit values shall be treated as unsigned values" + /// (5.4.4), and a region's location (7.4.1.3, 7.4.1.4) is not specified otherwise. One past + /// `i32::MAX` is past any page. Cast to a signed coordinate it read as left of the page or above + /// it, and one within a region's size of 2^32 came back onto it. + pub(crate) fn combine_region(&mut self, region: &Self, operator: CombinationOperator) { + let signed = |location: u32| i32::try_from(location).unwrap_or(i32::MAX); + self.combine(region, signed(region.x_location), signed(region.y_location), operator); + } + /// Combine another bitmap into this one at a specific location. /// /// "These operators describe how the segment's bitmap is to be combined with @@ -280,15 +318,74 @@ impl Bitmap { self.data[(row * self.stride + word_idx) as usize] } + + /// PdfCraft patch: the `width` × `height` area of this bitmap at `x`, `y`, as a bitmap there. + /// It is what a refinement of the page refines: "a bitmap containing the current contents of + /// the page buffer, restricted to the area of the page buffer specified by this segment's + /// region segment information field" (7.4.7.4). Off this bitmap the area is 0, past its width + /// too whatever the padding of its rows holds, and outside the area the refinement reads 0 + /// (6.3.5.2), where reading the page itself took in the pixels around the area. + pub(crate) fn area(&self, x: u32, y: u32, width: u32, height: u32) -> Result { + let mut area = Self::new_with(width, height, x, y, false)?; + for row in 0..height { + let Some(from_row) = y.checked_add(row).filter(|r| *r < self.height) else { + continue; + }; + for word in 0..area.stride { + // The first column of this word of the area, in this bitmap; past its width, + // or past where a column can be, the rest of the row is 0. + let Some(from) = x.checked_add(word * WORD_BITS).filter(|c| *c < self.width) else { + break; + }; + let (index, shift) = (from / WORD_BITS, from % WORD_BITS); + let mut bits = self.get_word(from_row, index) << shift; + if shift > 0 { + bits |= self.get_word(from_row, index + 1) >> (WORD_BITS - shift); + } + // Both are above 0: `from` is inside this bitmap, and `word` inside the area. + let ours = self.width - from; + let theirs = width - word * WORD_BITS; + let kept = ours.min(theirs); + if kept < WORD_BITS { + bits &= !0 << (WORD_BITS - kept); + } + if let Some(slot) = area.data.get_mut((row * area.stride + word) as usize) { + *slot = bits; + } + } + } + Ok(area) + } } #[cfg(test)] -mod tests { +mod pdfcraft_limit_tests { use super::*; + use crate::error::DecodeError; + /// PdfCraft patch: a bitmap past the bound on a side or on pixels is refused by a limit. #[test] - fn failed_reservation_is_an_error() { - let mut data = Vec::new(); - assert!(fill_words(&mut data, usize::MAX / 2, 0).is_err()); + fn a_bitmap_past_its_bounds_is_refused_by_a_limit() { + let limit = Err(DecodeError::Overflow(OverflowError::BitmapLimit)); + assert_eq!( + Bitmap::new(MAX_DIMENSION + 1, 1).map(|_| ()), + limit, + "a side" + ); + assert_eq!( + Bitmap::new(1, MAX_DIMENSION + 1).map(|_| ()), + limit, + "the other" + ); + // 2^28 + 2^14 pixels, each side within its bound. + assert_eq!( + Bitmap::new(1 << 14, (1 << 14) + 1).map(|_| ()), + limit, + "the pixels" + ); + assert!( + Bitmap::new(1 << 14, 1 << 14).is_ok(), + "2^28 pixels are within it" + ); } } diff --git a/vendor/hayro-jbig2/src/decode/generic_refinement.rs b/vendor/hayro-jbig2/src/decode/generic_refinement.rs index a6634439d..cd730fd93 100644 --- a/vendor/hayro-jbig2/src/decode/generic_refinement.rs +++ b/vendor/hayro-jbig2/src/decode/generic_refinement.rs @@ -10,7 +10,7 @@ use crate::ScratchBuffers; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; use crate::bitmap::{Bitmap, WORD_BITS, WORD_SHIFT, Word}; use crate::decode::generic::ContextGatherer; -use crate::error::{OverflowError, ParseError, RegionError, Result, bail}; +use crate::error::{ParseError, RegionError, Result, bail}; use crate::reader::Reader; /// Generic refinement region decoding procedure (6.3). @@ -46,27 +46,18 @@ pub(crate) fn decode_into( // Validate that the region fits within the reference bitmap. // When referring to another segment, dimensions must match exactly (7.4.7.5). - // When using the page bitmap as reference, the region must fit within the page. + // PdfCraft patch: a refinement of the page refines the page's area of its own size (7.4.7.4, + // `Bitmap::area`), which it always fits. if header.region_info.width > reference.width || header.region_info.height > reference.height { bail!(RegionError::InvalidDimension); } - let reference_dx = i32::try_from(reference.x_location) - .ok() - .and_then(|r: i32| { - i32::try_from(header.region_info.x_location) - .ok() - .and_then(|h| r.checked_sub(h)) - }) - .ok_or(OverflowError::ReferenceOffset)?; - let reference_dy = i32::try_from(reference.y_location) - .ok() - .and_then(|r: i32| { - i32::try_from(header.region_info.y_location) - .ok() - .and_then(|h| r.checked_sub(h)) - }) - .ok_or(OverflowError::ReferenceOffset)?; + // PdfCraft patch: "GRREFERENCEDX 0, GRREFERENCEDY 0" (Table 35). The reference is another + // region, which a refinement refines where it is (their locations should be equal, 7.4.7.5), + // or the page restricted to the refinement's area (7.4.7.4, `Bitmap::area`). Offsetting it by + // the difference of the locations refused one past `i32::MAX` as an overflow, and read the + // page around a refinement of it. + let (reference_dx, reference_dy) = (0, 0); let mut decoder = ArithmeticDecoder::new(data); let num_context_bits = header.template.context_bits(); @@ -194,10 +185,24 @@ impl<'a> RefinementContextGatherer<'a> { }; self.reg_cur = 0; - self.ref_cur_x = 0; - self.ref_m1 = ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, 0); - self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, 0); - self.ref_p1 = ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, 0); + // PdfCraft patch: the buffers start a column left of the reference column the row starts + // at, as `maybe_reload_buffers` would load them, or at column 0 where that is left of the + // reference. The row's first context takes in that column and the one left of it, which + // from column 32 on lay outside a word loaded from column 0 and read as 0: with the + // reference offset 32 left of the region its first column, and from 33 both (GRREFERENCEDX + // below −31, which a refined text instance has from a narrower bitmap or an RDX below −31, + // and a refined symbol from such an RDX). The region decoded to other pixels than were + // coded, or failed. + self.ref_cur_x = self + .reference_dx + .checked_neg() + .and_then(|column| u32::try_from(column).ok()) + .map_or(0, |column| column.saturating_sub(1)); + self.ref_m1 = + ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, self.ref_cur_x); + self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, self.ref_cur_x); + self.ref_p1 = + ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, self.ref_cur_x); self.ctx = match self.template { RefinementTemplate::Template1 => { @@ -259,25 +264,25 @@ impl<'a> RefinementContextGatherer<'a> { self.reg_cur = ContextGatherer::load_word(region, self.reg_y, new_start); } + // PdfCraft patch: left of the reference the buffers keep what `start_row` loaded, its words + // from column 0. The context takes in the reference a column to the right of the current + // one (Figures 12 and 13), so at column −1 it reads column 0, and from 0 on the buffers are + // the reference's own. They were zeroed here and, at column 0, not loaded again: with the + // reference offset right of the region (GRREFERENCEDX above 0, which a refined text + // instance or symbol has from a wider bitmap or an RDX above 0), its first 32 columns read + // as 0, and the region decoded to other pixels than were coded, or as garbage that failed. let ref_x_signed = (x as i32).wrapping_sub(self.reference_dx); - if ref_x_signed < 0 { - self.ref_cur_x = 0; - self.ref_m1 = 0; - self.ref_cur = 0; - self.ref_p1 = 0; - } else { - let ref_x = ref_x_signed as u32; - - if ref_x + 1 >= self.ref_cur_x + WORD_BITS || ref_x < self.ref_cur_x { - let new_start = ref_x.saturating_sub(1); - let ref_y = (self.reg_y as i32).wrapping_sub(self.reference_dy); - self.ref_cur_x = new_start; - self.ref_m1 = - ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, new_start); - self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, new_start); - self.ref_p1 = - ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, new_start); - } + if let Ok(ref_x) = u32::try_from(ref_x_signed) + && (ref_x + 1 >= self.ref_cur_x + WORD_BITS || ref_x < self.ref_cur_x) + { + let new_start = ref_x.saturating_sub(1); + let ref_y = (self.reg_y as i32).wrapping_sub(self.reference_dy); + self.ref_cur_x = new_start; + self.ref_m1 = + ContextGatherer::load_word(reference, ref_y.wrapping_sub(1) as u32, new_start); + self.ref_cur = ContextGatherer::load_word(reference, ref_y as u32, new_start); + self.ref_p1 = + ContextGatherer::load_word(reference, ref_y.wrapping_add(1) as u32, new_start); } } diff --git a/vendor/hayro-jbig2/src/decode/symbol.rs b/vendor/hayro-jbig2/src/decode/symbol.rs index 94250a3de..6eefa0564 100644 --- a/vendor/hayro-jbig2/src/decode/symbol.rs +++ b/vendor/hayro-jbig2/src/decode/symbol.rs @@ -4,7 +4,7 @@ use alloc::vec; use alloc::vec::Vec; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; -use crate::bitmap::{BITMAP_WORDS, Bitmap, HUFFMAN_LINE_WORK, WORD_BITS, Word, decode_work}; +use crate::bitmap::{BITMAP_WORDS, Bitmap, HUFFMAN_LINE_WORK, INTEGER_WORK, WORD_BITS, Word, decode_work}; use crate::decode::generic::{decode_bitmap_mmr, parse_adaptive_template_pixels}; use crate::decode::text::{ DecodeContext, ReferenceCorner, TextRegionContexts, TextRegionFlags, TextRegionHeader, @@ -24,10 +24,14 @@ use crate::reader::Reader; const MAX_SYMBOLS: u32 = u16::MAX as u32; -/// PdfCraft patch: the most symbol pixels one dictionary may decode (2^26). Symbols are -/// decoded pixel by pixel, and up to 65535 of them of up to 65535 × 65535 each kept a fuzzed -/// file decoding for minutes; real dictionaries hold about a million pixels. -const MAX_DICTIONARY_PIXELS: u64 = 1 << 26; +/// PdfCraft patch: the most symbol pixels one dictionary may decode, as many as the largest page +/// bitmap holds (2^28, `bitmap::MAX_PIXELS`). Symbols are decoded pixel by pixel, and up to 65535 +/// of them of up to 65535 × 65535 each kept a fuzzed file decoding for minutes; each symbol now +/// pays for its pixels out of the stream's decoding work (`bitmap::MAX_WORK`), which bounds the +/// time, and holds them in the retained bitmaps (`room`), which bounds the memory. The bound was +/// 2^26, a quarter of this, which a real scan passes: the soft mask of a 14,642 × 8,879 drawing +/// with one dictionary of 7,559 symbols (#6722) needed more, and was refused. +const MAX_DICTIONARY_PIXELS: u64 = crate::bitmap::MAX_PIXELS; /// PdfCraft patch: decoding steps that decode nothing, allowed beyond what a valid dictionary can /// need. A height class without a symbol and an export run of length 0 don't move their loops @@ -101,6 +105,8 @@ pub(crate) fn decode( bail!(SymbolError::Invalid); } height_classes_left -= 1; + // PdfCraft patch: see `bitmap::INTEGER_WORK`. A class of no symbols still reads its delta. + *ctx.work_left = ctx.work_left.checked_sub(INTEGER_WORK).ok_or(OverflowError::DecodingWork)?; let height_class_delta = read_height_class_delta(&mut ctx)?.ok_or(SymbolError::UnexpectedOob)?; @@ -133,10 +139,13 @@ pub(crate) fn decode( .pixels .saturating_add(u64::from(symbol_width) * u64::from(ctx.height_class_height)); if ctx.pixels > MAX_DICTIONARY_PIXELS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } - // PdfCraft patch: see `bitmap::MAX_WORK`. - let work = decode_work(symbol_width, ctx.height_class_height); + // PdfCraft patch: see `bitmap::INTEGER_WORK`. Its width, and a stream can hold thousands + // of dictionaries of symbols of no pixels. An aggregated one reads its count, a symbol + // and two offsets as well. + let integers = if ctx.header.flags.use_refagg { 5 } else { 1 }; + let work = decode_work(symbol_width, ctx.height_class_height).saturating_add(integers * INTEGER_WORK); *ctx.work_left = ctx.work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; // PdfCraft patch: see `room`. Rows take a word at least, so symbols of one column // held 270 MiB within the pixel bound, however few the dictionary exported. @@ -652,6 +661,17 @@ fn decode_height_class_collective_bitmap(ctx: &mut SymbolDecodeContext<'_>) -> R ctx.h_ctx.reader.align(); + // PdfCraft patch: a class of no symbols has nothing to cut out, and `bitmap::decode_work` charges + // rows by the symbol, so decoding this would be rows for nothing: 65,535 a class, and a dictionary + // can hold a thousand such classes. Its bytes still have to be stepped over. + if ctx.symbols_decoded_count == ctx.height_class_first_symbol { + ctx.h_ctx + .reader + .read_bytes(bitmap_size as usize) + .ok_or(ParseError::UnexpectedEof)?; + return Ok(()); + } + let collective_bitmap = if bitmap_size == 0 { // Bitmap is stored uncompressed, so we basically just copy-paste // each byte. Rows are padded to the byte boundary. @@ -759,6 +779,8 @@ fn export_symbols(ctx: &mut SymbolDecodeContext<'_>) -> Result> { bail!(SymbolError::Invalid); } runs_left -= 1; + // PdfCraft patch: see `bitmap::INTEGER_WORK`. A run of no symbols still reads its length. + *ctx.work_left = ctx.work_left.checked_sub(INTEGER_WORK).ok_or(OverflowError::DecodingWork)?; let run_length = read_run_length()?; let end_index = index.checked_add(run_length).ok_or(OverflowError::Index)?; diff --git a/vendor/hayro-jbig2/src/decode/text.rs b/vendor/hayro-jbig2/src/decode/text.rs index 71924acf6..5406ffeb4 100644 --- a/vendor/hayro-jbig2/src/decode/text.rs +++ b/vendor/hayro-jbig2/src/decode/text.rs @@ -11,7 +11,7 @@ use super::{ use super::{RegionBitmap, generic_refinement}; use crate::ScratchBuffers; use crate::arithmetic_decoder::{ArithmeticDecoder, ArithmeticDecoderContext}; -use crate::bitmap::{Bitmap, copy_work, decode_work}; +use crate::bitmap::{Bitmap, INTEGER_WORK, copy_work, decode_work}; use crate::error::{HuffmanError, OverflowError, ParseError, Result, SymbolError, bail}; use crate::huffman_table::{HuffmanTable, StandardHuffmanTables, TableLine}; use crate::integer_decoder::IntegerDecoder; @@ -151,7 +151,7 @@ pub(crate) fn decode_with( let symbol_id = ctx.read_symbol_id()?; let symbol_bitmap = - decode_symbol_instance_bitmap(&mut ctx, symbols, header, symbol_id)?; + decode_symbol_instance_bitmap(&mut ctx, symbols, header, symbol_id, work_left)?; let symbol_bitmap_ref: &Bitmap = match &symbol_bitmap { SymbolBitmap::Reference(idx) => symbols.get(*idx).ok_or(SymbolError::OutOfRange)?, @@ -161,14 +161,8 @@ pub(crate) fn decode_with( let symbol_height = symbol_bitmap_ref.height as i32; // PdfCraft patch: see `bitmap::MAX_WORK`. An instance of a large symbol costs its copy, - // and 10,000 of one took 64 s. A refined one was decoded as well. Even one with no - // pixels goes round this loop. - let (width, height) = (symbol_bitmap_ref.width, symbol_bitmap_ref.height); - let copied = copy_work(width, height).saturating_add(1); - let work = match &symbol_bitmap { - SymbolBitmap::Reference(_) => copied, - SymbolBitmap::Owned(_) => copied.saturating_add(decode_work(width, height)), - }; + // and 10,000 of one took 64 s. Even one with no pixels reads its four integers. + let work = copy_work(symbol_bitmap_ref.width, symbol_bitmap_ref.height).saturating_add(4 * INTEGER_WORK); *work_left = work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; if !header.flags.transposed @@ -549,6 +543,7 @@ fn decode_symbol_instance_bitmap( symbols: &[&Bitmap], header: &TextRegionHeader<'_>, symbol_id: usize, + work_left: &mut u64, ) -> Result { if !header.flags.use_refinement || ctx.read_refinement_flag()? == 0 { return Ok(SymbolBitmap::Reference(symbol_id)); @@ -580,6 +575,11 @@ fn decode_symbol_instance_bitmap( .checked_add(rdy) .ok_or(OverflowError::ReferenceOffset)?; + // PdfCraft patch: see `bitmap::MAX_WORK`. Before it is decoded: charged after, one instance + // could still be 2^28 pixels past what was left. With the four integers that gave its size. + let work = decode_work(refined_width, refined_height).saturating_add(4 * INTEGER_WORK); + *work_left = work_left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; + let mut refined_bitmap = Bitmap::new(refined_width, refined_height)?; ctx.decode_refinement_bitmap( diff --git a/vendor/hayro-jbig2/src/error.rs b/vendor/hayro-jbig2/src/error.rs index c8bcf8b81..a91074e57 100644 --- a/vendor/hayro-jbig2/src/error.rs +++ b/vendor/hayro-jbig2/src/error.rs @@ -141,6 +141,10 @@ pub enum OverflowError { SegmentCount, /// PdfCraft patch: a stream asks for more decoding work than it may. DecodingWork, + /// PdfCraft patch: a bitmap, gray-scale image or symbol dictionary is larger than the decoder + /// holds (see `bitmap::MAX_PIXELS`): one of its limits, where `BitmapDimension` is arithmetic + /// that overflows. + BitmapLimit, } impl fmt::Display for DecodeError { @@ -171,6 +175,7 @@ impl fmt::Display for OverflowError { Self::RetainedBitmaps => write!(f, "the segments retain too many bitmaps"), Self::SegmentCount => write!(f, "too many segments or references in a stream"), Self::DecodingWork => write!(f, "a stream asks for too much decoding work"), + Self::BitmapLimit => write!(f, "a bitmap is larger than the decoder holds"), } } } @@ -312,6 +317,59 @@ impl From for DecodeError { } } +impl DecodeError { + /// PdfCraft patch: whether one of the decoder's limits refused the stream (on a bitmap's size, + /// a dictionary's symbols, a region's instances, the bitmaps retained, the segments or the + /// decoding work) rather than its data failing to decode. A file within the format can reach + /// these, so a caller can tell a refusal of its own from a damaged file. A size or count the + /// file damaged can read as past one of them, and is a limit here too: it can't be told from a + /// hostile one. + pub fn is_limit(&self) -> bool { + matches!( + self, + Self::Overflow( + OverflowError::BitmapLimit + | OverflowError::GridDimension + | OverflowError::SymbolCount + | OverflowError::RetainedBitmaps + | OverflowError::SegmentCount + | OverflowError::DecodingWork + ) | Self::Symbol(SymbolError::TooManyInstances) + ) + } +} + +#[cfg(test)] +mod pdfcraft_limit_tests { + use super::*; + + /// PdfCraft patch: the decoder's own limits are limits, and arithmetic that overflows, + /// damage, is not. + #[test] + fn the_decoders_limits_are_limits_and_overflowing_arithmetic_is_not() { + let limits = [ + DecodeError::Overflow(OverflowError::BitmapLimit), + DecodeError::Overflow(OverflowError::GridDimension), + DecodeError::Overflow(OverflowError::SymbolCount), + DecodeError::Overflow(OverflowError::RetainedBitmaps), + DecodeError::Overflow(OverflowError::SegmentCount), + DecodeError::Overflow(OverflowError::DecodingWork), + DecodeError::Symbol(SymbolError::TooManyInstances), + ]; + for limit in limits { + assert!(limit.is_limit(), "{limit:?} is a limit"); + } + let damage = [ + DecodeError::Overflow(OverflowError::BitmapDimension), + DecodeError::Overflow(OverflowError::PlacementCoordinate), + DecodeError::Symbol(SymbolError::TooManySymbols), + ]; + for damage in damage { + assert!(!damage.is_limit(), "{damage:?} is damage"); + } + } +} + /// Result type for JBIG2 decoding operations. pub type Result = core::result::Result; diff --git a/vendor/hayro-jbig2/src/file.rs b/vendor/hayro-jbig2/src/file.rs index 5b8bbe983..c5b59b27c 100644 --- a/vendor/hayro-jbig2/src/file.rs +++ b/vendor/hayro-jbig2/src/file.rs @@ -88,6 +88,19 @@ pub(crate) struct File<'a> { /// (D.4.1) const FILE_HEADER_ID: [u8; 8] = [0x97, 0x4A, 0x42, 0x32, 0x0D, 0x0A, 0x1A, 0x0A]; +/// PdfCraft patch (17): an embedded stream that starts with a file header (D.4), which some +/// producers write into a PDF's `JBIG2Decode` data although ISO 32000-1 7.4.7 has the embedded +/// organization (D.3) there. Returns the reader past the header and the organization the header +/// declares, or `None` when the data doesn't start with the header's ID string. +pub(crate) fn after_file_header(data: &[u8]) -> Result, FileOrganization)>> { + if !data.starts_with(&FILE_HEADER_ID) { + return Ok(None); + } + let mut reader = Reader::new(data); + let header = parse_file_header(&mut reader)?; + Ok(Some((reader, header.organization))) +} + /// Parse a standalone JBIG2 file. pub(crate) fn parse_file(data: &[u8]) -> Result> { let mut reader = Reader::new(data); @@ -170,7 +183,9 @@ fn parse_segments<'a>( FileOrganization::Sequential => { parse_segments_sequential(reader, &mut segments, &mut SegmentCount::default())?; } - FileOrganization::RandomAccess => parse_segments_random_access(reader, &mut segments)?, + FileOrganization::RandomAccess => { + parse_segments_random_access(reader, &mut segments, &mut SegmentCount::default())?; + } } Ok(segments) @@ -213,12 +228,12 @@ pub(crate) fn parse_segments_sequential<'a>( /// is followed by a sequence of segments headers; the last segment header is /// followed by the data for the first segment, then the data for the second /// segment, and so on." (D.2) -fn parse_segments_random_access<'a>( +pub(crate) fn parse_segments_random_access<'a>( reader: &mut Reader<'a>, segments: &mut Vec>, + count: &mut SegmentCount, ) -> Result<()> { let mut headers = Vec::new(); - let mut count = SegmentCount::default(); loop { if reader.at_end() { diff --git a/vendor/hayro-jbig2/src/gray_scale.rs b/vendor/hayro-jbig2/src/gray_scale.rs index 4aa4229a5..1cd5016f0 100644 --- a/vendor/hayro-jbig2/src/gray_scale.rs +++ b/vendor/hayro-jbig2/src/gray_scale.rs @@ -81,11 +81,11 @@ fn decode_arithmetic( let height = params.height; if width > MAX_DIMENSION || height > MAX_DIMENSION { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // PdfCraft patch: the product is bounded too (see `bitmap::MAX_GRID_CELLS`). if u64::from(width) * u64::from(height) > crate::bitmap::MAX_GRID_CELLS { - bail!(OverflowError::BitmapDimension); + bail!(OverflowError::BitmapLimit); } // `GSBPP` - The number of bits per gray-scale value. @@ -289,3 +289,34 @@ fn extract_bits( } } } + +#[cfg(test)] +mod pdfcraft_limit_tests { + use super::*; + use crate::error::DecodeError; + + /// PdfCraft patch: a gray-scale image past the bound on a side or on cells is refused by a + /// limit, before anything is decoded. A halftone region refuses such a grid first, so this is + /// reached only by calling it. + #[test] + fn a_gray_scale_image_past_its_bounds_is_refused_by_a_limit() { + let decode = |width, height| { + let params = GrayScaleParams { + use_mmr: false, + bits_per_pixel: 1, + width, + height, + template: Template::Template0, + skip_mask: None, + }; + decode_gray_scale_image(&[], ¶ms, &mut ScratchBuffers::default()).map(|_| ()) + }; + let limit = Err(DecodeError::Overflow(OverflowError::BitmapLimit)); + assert_eq!(decode(MAX_DIMENSION + 1, 1), limit, "a side"); + assert_eq!( + decode(1 << 13, (1 << 12) + 1), + limit, + "the cells, each side within its bound" + ); + } +} diff --git a/vendor/hayro-jbig2/src/lib.rs b/vendor/hayro-jbig2/src/lib.rs index 078370ed3..6ec7fd83f 100644 --- a/vendor/hayro-jbig2/src/lib.rs +++ b/vendor/hayro-jbig2/src/lib.rs @@ -78,7 +78,10 @@ pub use error::{ SegmentError, SymbolError, TemplateError, }; -use crate::file::{SegmentCount, parse_segments_sequential}; +use crate::file::{ + FileOrganization, SegmentCount, after_file_header, parse_segments_random_access, + parse_segments_sequential, +}; use bitmap::Bitmap; use decode::CombinationOperator; use decode::generic; @@ -120,7 +123,9 @@ impl<'a> Image<'a> { /// Parse an embedded JBIG2 image with optional global segments. /// /// The file is expected to use the embedded organization defined in - /// Annex D.3. + /// Annex D.3. PdfCraft patch (17): data that starts with a file header (D.4) + /// is read past it, in the organization it declares, as Acrobat and PDFBox + /// draw such streams; its segments still count together with the globals'. pub fn new_embedded(data: &'a [u8], globals: Option<&'a [u8]>) -> Result { let mut segments = Vec::new(); let mut count = SegmentCount::default(); @@ -129,8 +134,18 @@ impl<'a> Image<'a> { parse_segments_sequential(&mut reader, &mut segments, &mut count)?; }; - let mut reader = Reader::new(data); - parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + match after_file_header(data)? { + Some((mut reader, FileOrganization::RandomAccess)) => { + parse_segments_random_access(&mut reader, &mut segments, &mut count)?; + } + Some((mut reader, FileOrganization::Sequential)) => { + parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + } + None => { + let mut reader = Reader::new(data); + parse_segments_sequential(&mut reader, &mut segments, &mut count)?; + } + } segments.sort_by_key(|seg| seg.header.segment_number); @@ -198,11 +213,37 @@ impl<'a> Image<'a> { /// This is useful in case you want to convert multiple JBIG2 images, /// as it allows `hayro-jbig2` to reuse allocations during decoding. pub fn decode_with(&self, decoder: &mut D, ctx: &mut DecoderContext) -> Result<()> { - decode_segments(&self.segments, self.height_from_stripes, ctx)?; + let mut work_left = bitmap::MAX_WORK; + decode_segments(&self.segments, self.height_from_stripes, ctx, &mut work_left)?; emit_bitmap(&ctx.page_bitmap, decoder); Ok(()) } + + /// PdfCraft patch: [`Image::decode`], spending from `work`, what decoding may still cost + /// across a run of images. An image spends what it took, failing or not, starting with its + /// page's pixels, which whoever asked for them will go through, and its segments, so images + /// that decode nothing add up too. On top of that, it takes at most [`bitmap::MAX_WORK`] + /// whatever `work` says. One that `work` can't pay for spends all of it, so a run ends rather + /// than trying every image it has left. Reading the stream comes before this and is charged + /// by [`charge_reading`]. + pub fn decode_within(&self, decoder: &mut D, work: &mut u64) -> Result<()> { + let start = bitmap::decode_work(self.width, self.height).saturating_add(self.segments.len() as u64); + if charge_work(work, start).is_err() { + *work = 0; + bail!(OverflowError::DecodingWork); + } + let allowed = bitmap::MAX_WORK.min(*work); + let mut left = allowed; + let mut ctx = DecoderContext::default(); + let decoded = decode_segments(&self.segments, self.height_from_stripes, &mut ctx, &mut left); + let refused = decoded == Err(OverflowError::DecodingWork.into()); + *work = if refused && allowed < bitmap::MAX_WORK { 0 } else { *work - (allowed - left) }; + decoded?; + emit_bitmap(&ctx.page_bitmap, &mut *decoder); + + Ok(()) + } } fn emit_bitmap(bitmap: &Bitmap, decoder: &mut D) { @@ -262,22 +303,46 @@ fn emit_bitmap(bitmap: &Bitmap, decoder: &mut D) { } } -/// PdfCraft patch: count `work` more decoding work, refusing past [`bitmap::MAX_WORK`] before what -/// asks for it is decoded. +/// PdfCraft patch: count `work` more decoding work, refusing past what is `left` of +/// [`bitmap::MAX_WORK`] or a run's allowance before what asks for it is decoded. fn charge_work(left: &mut u64, work: u64) -> Result<()> { *left = left.checked_sub(work).ok_or(OverflowError::DecodingWork)?; Ok(()) } +/// PdfCraft patch: the steps of reading a byte of a stream or its globals: parsing and sorting +/// segments, and building their tables, measured at about 3 a byte at the worst. +const READ_WORK: u64 = 4; + +/// PdfCraft patch: charge `work`, what a run of images may still cost, for reading `bytes` of a stream +/// and its globals. This comes before anything is parsed, as it counts whether or not they turn out +/// to decode: an image that fails has still cost its reading, and it is read again for each use. What +/// `work` can't pay for spends all of it. +pub fn charge_reading(work: &mut u64, bytes: usize) -> Result<()> { + if charge_work(work, (bytes as u64).saturating_mul(READ_WORK)).is_err() { + *work = 0; + bail!(OverflowError::DecodingWork); + } + Ok(()) +} + /// PdfCraft patch: the work of a region decoded pixel by pixel. fn region_work(region: &decode::RegionSegmentInfo) -> u64 { bitmap::decode_work(region.width, region.height) } +/// PdfCraft patch: the page restricted to a refinement's area, its reference when it refers to no +/// other region (7.4.7.4). Copying it a word at a time costs less than the region's own decoding, +/// already charged, and it is held, as large as the region, only while the region decodes. +fn page_area(page: &Bitmap, region: &decode::RegionSegmentInfo) -> Result { + page.area(region.x_location, region.y_location, region.width, region.height) +} + fn decode_segments( segments: &[segment::Segment<'_>], height_from_stripes: Option, decoder_ctx: &mut DecoderContext, + work_left: &mut u64, ) -> Result<()> { // Find and parse page information segment first. if let Some(page_info) = segments @@ -298,7 +363,6 @@ fn decode_segments( let page_bitmap = &mut decoder_ctx.page_bitmap; let page_state = &mut decoder_ctx.page_state; let scratch_buffers = &mut decoder_ctx.scratch_buffers; - let mut work_left = bitmap::MAX_WORK; // Process all segments. for seg in segments { @@ -311,31 +375,26 @@ fn decode_segments( SegmentType::ImmediateGenericRegion | SegmentType::ImmediateLosslessGenericRegion => { let had_unknown_length = seg.header.data_length.is_none(); let header = generic::parse(&mut reader, had_unknown_length)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; if page_state.can_decode_directly(page_bitmap, &header.region_info, false) { generic::decode_into(&header, page_bitmap, scratch_buffers)?; } else { let region = generic::decode(&header, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } SegmentType::IntermediateGenericRegion => { // Intermediate segments cannot have unknown length. let header = generic::parse(&mut reader, false)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; let region = generic::decode(&header, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } SegmentType::PatternDictionary => { let header = pattern::parse(&mut reader)?; - charge_work(&mut work_left, header.work())?; + charge_work(work_left, header.work())?; let dictionary = pattern::decode(&header, scratch_buffers, page_state.retained_room())?; page_state.store_pattern_dictionary(seg.header.segment_number, dictionary)?; } @@ -344,7 +403,7 @@ fn decode_segments( // (6.5.5, step 1) // Collect references to avoid cloning; symbols are only cloned if re-exported. let input_symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, input_symbols.len() as u64)?; + charge_work(work_left, input_symbols.len() as u64)?; // Collect Huffman tables from referred table segments. let referred_tables: Vec = seg @@ -371,14 +430,14 @@ fn decode_segments( &page_state.standard_tables, retained_contexts, page_state.retained_room(), - &mut work_left, + work_left, )?; page_state.store_symbol_dictionary(seg.header.segment_number, dictionary)?; } SegmentType::ImmediateTextRegion | SegmentType::ImmediateLosslessTextRegion => { // Collect symbols from referred symbol dictionaries (SBSYMS). let symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, symbols.len() as u64)?; + charge_work(work_left, symbols.len() as u64)?; // Collect Huffman tables from referred table segments. // "These user-supplied Huffman decoding tables may be supplied either @@ -392,10 +451,10 @@ fn decode_segments( .collect(); let header = text::parse(&mut reader, symbols.len() as u32)?; - charge_work(&mut work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; + charge_work(work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; if header.flags.use_huffman { // Its symbol ID table has a line for each symbol. - charge_work(&mut work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; + charge_work(work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; } if page_state.can_decode_directly( @@ -410,7 +469,7 @@ fn decode_segments( &page_state.standard_tables, page_bitmap, scratch_buffers, - &mut work_left, + work_left, )?; } else { let region = text::decode( @@ -419,21 +478,16 @@ fn decode_segments( &referred_tables, &page_state.standard_tables, scratch_buffers, - &mut work_left, + work_left, )?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } SegmentType::IntermediateTextRegion => { // Collect symbols from referred symbol dictionaries (SBSYMS). let symbols = page_state.referred_symbols(&seg.header.referred_to_segments)?; - charge_work(&mut work_left, symbols.len() as u64)?; + charge_work(work_left, symbols.len() as u64)?; // Collect Huffman tables from referred table segments. let referred_tables: Vec = seg @@ -445,10 +499,10 @@ fn decode_segments( .collect(); let header = text::parse(&mut reader, symbols.len() as u32)?; - charge_work(&mut work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; + charge_work(work_left, bitmap::copy_work(header.region_info.width, header.region_info.height))?; if header.flags.use_huffman { // Its symbol ID table has a line for each symbol. - charge_work(&mut work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; + charge_work(work_left, (symbols.len() as u64).saturating_mul(bitmap::HUFFMAN_LINE_WORK))?; } let region = text::decode( &header, @@ -456,7 +510,7 @@ fn decode_segments( &referred_tables, &page_state.standard_tables, scratch_buffers, - &mut work_left, + work_left, )?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } @@ -469,7 +523,7 @@ fn decode_segments( .ok_or(SegmentError::MissingPatternDictionary)?; let header = halftone::parse(&mut reader)?; - charge_work(&mut work_left, halftone::work(&header, pattern_dict))?; + charge_work(work_left, halftone::work(&header, pattern_dict))?; if page_state.can_decode_directly( page_bitmap, @@ -479,12 +533,7 @@ fn decode_segments( halftone::decode_into(&header, pattern_dict, page_bitmap, scratch_buffers)?; } else { let region = halftone::decode(&header, pattern_dict, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } @@ -497,21 +546,29 @@ fn decode_segments( .ok_or(SegmentError::MissingPatternDictionary)?; let header = halftone::parse(&mut reader)?; - charge_work(&mut work_left, halftone::work(&header, pattern_dict))?; + charge_work(work_left, halftone::work(&header, pattern_dict))?; let region = halftone::decode(&header, pattern_dict, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } SegmentType::IntermediateGenericRefinementRegion => { // Same logic as immediate refinement, but store result instead of combining. - let reference = seg + let referred_segment = seg .header .referred_to_segments .first() - .and_then(|&num| page_state.get_referred_segment(num)) - .unwrap_or(page_bitmap); + .and_then(|&num| page_state.get_referred_segment(num)); let header = generic_refinement::parse(&mut reader)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; + // PdfCraft patch: a refinement of the page refines its area (`Bitmap::area`). + let area; + let reference = match referred_segment { + Some(region) => region, + None => { + area = page_area(page_bitmap, &header.region_info)?; + &area + } + }; let region = generic_refinement::decode(&header, reference, scratch_buffers)?; page_state.store_region(seg.header.segment_number, region.bitmap)?; } @@ -520,8 +577,8 @@ fn decode_segments( // "3) Determine the buffer associated with the region segment that // this segment refers to." (7.4.7.5) // - // "2) If there are no referred-to segments, then use the page - // bitmap as the reference buffer." (7.4.7.5) + // With no region segment referred to, the reference is the page restricted to + // this segment's area (7.4.7.4). let referred_segment = seg .header .referred_to_segments @@ -529,7 +586,7 @@ fn decode_segments( .and_then(|&num| page_state.get_referred_segment(num)); let header = generic_refinement::parse(&mut reader)?; - charge_work(&mut work_left, region_work(&header.region_info))?; + charge_work(work_left, region_work(&header.region_info))?; if let Some(referred_segment) = referred_segment && page_state.can_decode_directly(page_bitmap, &header.region_info, false) @@ -541,14 +598,17 @@ fn decode_segments( scratch_buffers, )?; } else { - let reference = referred_segment.unwrap_or(page_bitmap); + // PdfCraft patch: a refinement of the page refines its area (`Bitmap::area`). + let area; + let reference = match referred_segment { + Some(region) => region, + None => { + area = page_area(page_bitmap, &header.region_info)?; + &area + } + }; let region = generic_refinement::decode(&header, reference, scratch_buffers)?; - page_bitmap.combine( - ®ion.bitmap, - region.bitmap.x_location as i32, - region.bitmap.y_location as i32, - region.combination_operator, - ); + page_bitmap.combine_region(®ion.bitmap, region.combination_operator); } page_state.page_pristine = false; } @@ -794,6 +854,192 @@ mod tests { assert!(Image::new_embedded(&stream(1 << 20, 1 << 20), None).is_err()); } + /// Segment `number` of `kind` on page 1, referring to `refers_to`. + fn segment(number: u8, kind: u8, refers_to: &[u8], data: &[u8]) -> Vec { + let mut segment = vec![0, 0, 0, number, kind, (refers_to.len() as u8) << 5]; + segment.extend_from_slice(refers_to); + segment.push(1); + segment.extend_from_slice(&(data.len() as u32).to_be_bytes()); + segment.extend_from_slice(data); + segment + } + + /// A blank page of 8 × 8: resolution unknown, flags, not striped. + fn page_of_8() -> Vec { + segment(0, 48, &[], &[0, 0, 0, 8, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0x42, 0, 0]) + } + + /// Region information (7.4.1) of 4 × 4 at `x`, `y`, then the region's own `header`, then + /// arbitrary coded data. + fn region_of_4(x: u32, y: u32, operator: u8, header: &[u8]) -> Vec { + let mut data: Vec = [4u32, 4, x, y].iter().flat_map(|v| v.to_be_bytes()).collect(); + data.push(operator); + data.extend_from_slice(header); + data.extend_from_slice(&[0x5a; 16]); + data + } + + const GENERIC: &[u8] = &[0, 3, 0xFF, 0xFD, 0xFF, 2, 0xFE, 0xFE, 0xFE]; // template 0, AT + const REFINEMENT: &[u8] = &[0, 0xFF, 0xFF, 0xFF, 0xFF]; // template 0, AT + const OR: u8 = 0; + const REPLACE: u8 = 4; + + /// PdfCraft patch: a region's location is unsigned (5.4.4), so one past `i32::MAX` is off the + /// page. Taken for a signed one, a region at 0xFFFFFFFE drew back onto the page two pixels left + /// of it, and a refinement there was refused, and the rest of the stream with it. Each region + /// here is 4 × 4 on a blank page of 8 × 8, its coded data arbitrary: on the page it draws, off + /// it nothing. + #[test] + fn a_region_located_past_i32_max_is_off_the_page() { + struct Ink(u32); + impl Decoder for Ink { + fn push_pixel(&mut self, black: bool) { + self.0 += u32::from(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.0 += if black { 8 * chunks } else { 0 }; + } + fn next_line(&mut self) {} + } + let ink = |x: u32, y: u32| { + let forms = [ + // An immediate generic region. + vec![segment(1, 38, &[], ®ion_of_4(x, y, OR, GENERIC))], + // An immediate refinement of the page, which reads the page at its own location (7.4.7.4). + vec![segment(1, 42, &[], ®ion_of_4(x, y, REPLACE, REFINEMENT))], + // An immediate refinement of an intermediate region at the same place (7.4.7.5). + vec![ + segment(1, 36, &[], ®ion_of_4(x, y, OR, GENERIC)), + segment(2, 42, &[1], ®ion_of_4(x, y, OR, REFINEMENT)), + ], + ]; + forms.map(|form| { + let stream = [page_of_8(), form.concat()].concat(); + let mut ink = Ink(0); + Image::new_embedded(&stream, None).and_then(|image| image.decode(&mut ink)).map(|()| ink.0) + }) + }; + for (x, y) in [(0, 0), (2, 2)] { + assert!(ink(x, y).iter().all(|ink| matches!(ink, Ok(n) if *n > 0)), "{x}, {y}: {:?}", ink(x, y)); + } + for (x, y) in [(0xFFFF_FFFE, 0), (0, 0xFFFF_FFFE), (0x8000_0000, 0x8000_0000), (u32::MAX, u32::MAX)] { + assert_eq!(ink(x, y).map(|ink| ink.ok()), [Some(0); 3], "{x:#x}, {y:#x}"); + } + } + + /// PdfCraft patch: a refinement of a region refines it where it is, "GRREFERENCEDX 0, + /// GRREFERENCEDY 0" (Table 35), not the page at the refinement's location. A region and its + /// refinement, directly or through an intermediate refinement, decode otherwise than the same + /// refinements of the page, and at 2, 3 to the pixels they decode to at 0, 0, moved. + #[test] + fn a_refinement_of_a_region_decodes_the_same_wherever_they_are() { + #[derive(Default)] + struct Pixels(Vec); + impl Decoder for Pixels { + fn push_pixel(&mut self, black: bool) { + self.0.push(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.0.extend((0..8 * chunks).map(|_| black)); + } + fn next_line(&mut self) {} + } + // The first refinement refers to the region before it, or `of_page`, to nothing: the page. + let decode = |x: u32, y: u32, through_intermediate: bool, of_page: bool| { + let first: &[u8] = if of_page { &[] } else { &[1] }; + let mut form = vec![segment(1, 36, &[], ®ion_of_4(x, y, OR, GENERIC))]; + if through_intermediate { + form.push(segment(2, 40, first, ®ion_of_4(x, y, OR, REFINEMENT))); + form.push(segment(3, 42, &[2], ®ion_of_4(x, y, OR, REFINEMENT))); + } else { + form.push(segment(2, 42, first, ®ion_of_4(x, y, OR, REFINEMENT))); + } + let stream = [page_of_8(), form.concat()].concat(); + let mut pixels = Pixels::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut pixels).unwrap(); + pixels.0 + }; + for through_intermediate in [false, true] { + let at_origin = decode(0, 0, through_intermediate, false); + assert!(at_origin.contains(&true), "draws"); + assert_ne!(at_origin, decode(0, 0, through_intermediate, true), "the page refined instead"); + for (i, &black) in decode(2, 3, through_intermediate, false).iter().enumerate() { + let (x, y) = (i % 8, i / 8); + let moved = x >= 2 && y >= 3 && at_origin[(y - 3) * 8 + x - 2]; + assert_eq!(black, moved, "{x}, {y}, through an intermediate refinement: {through_intermediate}"); + } + } + } + + /// PdfCraft patch: a refinement of the page refines the page restricted to its area (7.4.7.4), + /// reading 0 around it, and a reference 32 or more left of a refinement is read from where each + /// row starts in it. So that a context read wrong changes what decodes, the refinements are + /// large, and their pictures come from the hash they were coded from: a 64 × 16 refinement at + /// 41, 4 of a 112 × 24 page, and a 70 × 12 text instance refined from a 140 × 12 symbol 35 + /// left of it. + #[test] + fn a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is() { + #[derive(Default)] + struct Rows(Vec>, Vec); + impl Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(u8::from(black)); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + for _ in 0..8 * chunks { + self.push_pixel(black); + } + } + fn next_line(&mut self) { + self.0.push(core::mem::take(&mut self.1)); + } + } + fn bit(seed: u32, x: u32, y: u32) -> u8 { + let mut h = x.wrapping_mul(0x9E37_79B1).wrapping_add(y.wrapping_mul(0x85EB_CA77)).wrapping_add(seed.wrapping_mul(0xC2B2_AE3D)); + h ^= h >> 15; + h = h.wrapping_mul(0x2C1B_3C6D); + h ^= h >> 12; + h = h.wrapping_mul(0x297A_2D39); + h ^= h >> 15; + ((h >> 7) & 1) as u8 + } + // About one pixel in eight: where a refinement differs from its reference. + let flipped = |seed: u32, x: u32, y: u32| bit(seed, x, y) & bit(seed + 1, x, y) & bit(seed + 2, x, y); + // A page of bit(1, ..), and over it at 41, 4 the area refined from it. + const PAGE: &str = "000000003000010000001300000070000000180000000000000000420000000000012600010000016f00000070000000180000000000000000040003fffdff02fefefefb5d354db053c2a984c01af66c619b02c245055be23a2a431b04c60325c88676f6ff1b0a68c873ad9011a27a89fc99abc7db2e8f529ecf198ff8c6a726ddcb746dc51f3812e365e74ee668fa02f59182cae39f0ef43b3201da1bcd43a21f449889e15ba0ba1a9344da1bdb9dcb4015279a39d944509febc622a235fa2c623acb443f0c10dabf7fba5e9d6fef2f1289dfd0a6a3801d036f21a62045074f5d3158ab0373ad2334102ef867dc3132dc8d02fe517771c419cdbad47d67d409ec89d0425d1f51a48af6ba3bee6e0701185bfa5f46b347b0ec39e744b665d1e8aac868838d3db25162d65534ff4771d9c0d657bd48d5814fa7cc184d0e961defab2d75a79a88345cdae5546a54221a22ddfe9c8a0514f32ac9de806fbc5fb4f5275c79d679f0258f77900daaa273cee50e50c958f323414670d7761418639342fdb2082eac5e7934f1a06169991452b1a042eaee7871ffac000000022a000100000091000000400000001000000029000000040400ffffffffd24567306e52f57232cefa9800db00fb60c21b30b0e53d9fac8a1f863a580a7cc49ab401934ac8be3a18e2dc0f01774b76c3e9196283cb5504742c763a867bc1aae8c48d74d194dacda03ce7e91904897ab61add273d9188fda65dc6fefb995d85f48142e43c1970bedb9c5f4be3cb00d458e9f4ee9714a61fffac"; + let page: Vec> = (0..24) + .map(|y| (0..112).map(|x| bit(1, x, y) ^ if (41..105).contains(&x) && (4..20).contains(&y) { flipped(10, x - 41, y - 4) } else { 0 }).collect()) + .collect(); + // A symbol of bit(3, ..), and a text instance refined from it, 70 narrower with RDX 0. + const TEXT: &str = "0000000030000100000013000000460000000c000000000000000042000000000001000001000000ed000003fffdff02fefefe00000001000000013f4efc0ce48e6b5c00246a648400fb605a44d542ed29da2b9b55c01f4e5af9842dd70c88f4913c7f552503ba52a5d7b5d9fd08ff5794b4dbfb3df7bafcf9c43d04154c1cefa181ca1aba5da826a4e4229858bd0950442df69c098ab3b61a32f71b136974b96bcb5306f3e75af1a4ef51b8f56a355acb8b69915cfa623f5a21933be8c056a348783a116ea914e886af2c64bf3b23e5610726c981092220f05411857529861fbf4c630a15fd00b081d2351279e06602f4408b25dc1d01c223027ecb8c87c4b1d76e3882a759c1adaf749f0e20735f8bd5a7306fffac000000020620010100000085000000460000000c0000000000000000040012ffffffff00000001a967c0b129999147062a4766ce10bc3dae7f1daf43dda1427d31203a1bc9fa021e0504e0a4b01739952970b54ad11ac08c0684d2f556a1843eb01dc75b963be6237b7f02fdfe17ccf7f061de310a936fd08c5ddea47609789e55f7cbca81ae38219c4abc5f9eff47ffac"; + let text: Vec> = (0..12).map(|y| (0..70).map(|x| bit(3, x + 35, y) ^ flipped(30, x, y)).collect()).collect(); + for (stream, picture) in [(PAGE, page), (TEXT, text)] { + let stream: Vec = (0..stream.len() / 2).map(|i| u8::from_str_radix(&stream[i * 2..i * 2 + 2], 16).unwrap()).collect(); + let mut rows = Rows::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, picture); + } + } + + /// PdfCraft patch: the area a refinement of the page refines is the page inside it and 0 + /// outside: off the page's rows and columns, past where a column can be, and past the page's + /// width and the area's own whatever the padding of the page's rows holds. `new_with` clears + /// that padding (patch 13), so the page's words are set to all 1s, padding included. + #[test] + fn a_page_area_is_the_page_inside_it_and_0_outside() { + let mut page = Bitmap::new_with(40, 3, 0, 0, true).unwrap(); + page.data.fill(!0); + let area = |x, y, width, height| page.area(x, y, width, height).unwrap().data; + assert_eq!(area(30, 1, 20, 3), [0xFFC0_0000, 0xFFC0_0000, 0], "10 columns and 2 rows of the page"); + assert_eq!(area(0, 0, 33, 1), [!0, 0x8000_0000], "the whole of a word, and none of the area's padding"); + assert_eq!(area(36, 0, 8, 1), [0xF000_0000], "4 columns, and none of the page's padding"); + assert_eq!(area(5, 0, 8, 1), [0xFF00_0000], "inside the page"); + assert_eq!(area(40, 0, 8, 1), [0], "right of the page"); + assert_eq!(area(u32::MAX - 2, 0, 40, 1), [0, 0], "past where a column can be"); + assert_eq!(area(0, u32::MAX, 8, 2), [0, 0], "below the page"); + assert!(area(3, 0, 0, 2).is_empty(), "no width"); + } + /// A stream and its globals are one stream of segments, so they have one count. #[test] fn the_segments_of_a_stream_and_its_globals_count_together() { @@ -808,6 +1054,49 @@ mod tests { assert!(refused(&more), "one more isn't"); } + /// PdfCraft patch: a refinement whose reference is offset right of it reads the reference from + /// its first column, where it read the first 32 as 0. So that a context read wrong changes what + /// decodes, the refinement is large, and its picture comes from the hash it was coded from: a + /// 104 × 12 text instance refined from a 100 × 12 symbol 2 right of it. + #[test] + fn a_refinement_reads_a_reference_right_of_it_from_its_first_column() { + #[derive(Default)] + struct Rows(Vec>, Vec); + impl Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(u8::from(black)); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + for _ in 0..8 * chunks { + self.push_pixel(black); + } + } + fn next_line(&mut self) { + self.0.push(core::mem::take(&mut self.1)); + } + } + fn bit(seed: u32, x: u32, y: u32) -> u8 { + let mut h = x.wrapping_mul(0x9E37_79B1).wrapping_add(y.wrapping_mul(0x85EB_CA77)).wrapping_add(seed.wrapping_mul(0xC2B2_AE3D)); + h ^= h >> 15; + h = h.wrapping_mul(0x2C1B_3C6D); + h ^= h >> 12; + h = h.wrapping_mul(0x297A_2D39); + h ^= h >> 15; + ((h >> 7) & 1) as u8 + } + // About one pixel in eight: where a refinement differs from its reference. + let flipped = |seed: u32, x: u32, y: u32| bit(seed, x, y) & bit(seed + 1, x, y) & bit(seed + 2, x, y); + // A symbol of bit(2, ..), and a text instance refined from it, 4 wider with RDX 0. + const TEXT: &str = "0000000030000100000013000000680000000c000000000000000042000000000001000001000000b1000003fffdff02fefefe00000001000000013f525ad7118f669a303f1b11efa5210e9765b834b1831c650f3a3d325dbd7317379ea4fb573b09a20c362f065d8f9301e74cbbc50c69910641c0e0eea7c2fd1e831ca1459a943be55ed8e074b426ad7ff1e8d8ff6f1aa44684de5da4b4c3483911604412e00e96a0db6ec7103feb2bc4301ae98d3f9c23078ae72dc46f76e9dfea209d39187f1194f7b9285cd6e7b2dc7abb5def43ff0abe2df554fc55ffac0000000206200101000000af000000680000000c0000000000000000040012ffffffff00000001a96b5febebec5a68693b51f5db25ddf176d4aae89904a71d1690f12888e10b038db723ac59941f76c2844b739ce524d9563f4005e1edbabf2e323e8e9ac1a3b9060587c3010e7d6bcf90c00dbf9000a2f17b1d9b598e01b9158dc16a63f4388a6f647b99dc7118aa09e08d59fba1bc24e3a0f1ebb0cae636b6bedbb65b01945228d3e5d2980a0ad6290c13d2f34f5c861415ffac"; + let picture: Vec> = (0..12) + .map(|y| (0..104).map(|x| (if (2..102).contains(&x) { bit(2, x - 2, y) } else { 0 }) ^ flipped(20, x, y)).collect()) + .collect(); + let stream: Vec = (0..TEXT.len() / 2).map(|i| u8::from_str_radix(&TEXT[i * 2..i * 2 + 2], 16).unwrap()).collect(); + let mut rows = Rows::default(); + Image::new_embedded(&stream, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, picture); + } + /// A dictionary being decoded has what the page may still hold, not all of it: refused only /// once it is stored, its copies would already have been made. #[test] @@ -819,4 +1108,17 @@ mod tests { page.retained_words = MAX_RETAINED_WORDS; assert_eq!(page.retained_room(), 0); } + + /// Pixels past a bitmap's edge are 0 (6.3.5.2), and a refinement reads its reference a word at + /// a time, so a bitmap filled with 1s has none past its width: made, and a page made again for + /// the next image. + #[test] + fn a_bitmap_of_1s_has_nothing_past_its_width() { + assert_eq!(Bitmap::new_with(33, 2, 0, 0, true).unwrap().data, [!0, 1 << 31, !0, 1 << 31]); + let mut page = Bitmap::new_with(40, 1, 0, 0, false).unwrap(); + page.reinitialize(7, 2, true).unwrap(); + assert_eq!(page.data, [0xFE00_0000, 0xFE00_0000]); + assert_eq!(Bitmap::new_with(64, 1, 0, 0, true).unwrap().data, [!0, !0], "words filled whole"); + assert!(Bitmap::new_with(0, 3, 0, 0, true).unwrap().data.is_empty(), "no width"); + } } diff --git a/vendor/hayro-jbig2/src/pdfcraft_tests.rs b/vendor/hayro-jbig2/src/pdfcraft_tests.rs index e26d4e9cf..cd96db6c9 100644 --- a/vendor/hayro-jbig2/src/pdfcraft_tests.rs +++ b/vendor/hayro-jbig2/src/pdfcraft_tests.rs @@ -101,9 +101,14 @@ fn jbig2_segment(out: &mut Vec, number: u32, kind: u8, refs: &[u8], data: &[ /// The page information of a 16 × 2 JBIG2 page whose default pixel is black, so that the image, if /// it decodes, draws black whatever its other segments hold. fn black_jbig2_page() -> Vec { + black_jbig2_page_of(16, 2) +} + +/// [`black_jbig2_page`], `width` × `height`. +fn black_jbig2_page_of(width: u32, height: u32) -> Vec { let mut page = Vec::new(); - page.extend_from_slice(&16u32.to_be_bytes()); - page.extend_from_slice(&2u32.to_be_bytes()); + page.extend_from_slice(&width.to_be_bytes()); + page.extend_from_slice(&height.to_be_bytes()); page.extend_from_slice(&[0; 8]); // resolution page.extend_from_slice(&[4, 0, 0]); // flags (default pixel black), striping let mut stream = Vec::new(); @@ -120,6 +125,150 @@ fn jbig2_region_info(width: u32, height: u32) -> Vec { region } +/// A Huffman-coded symbol dictionary of `symbols` symbols of width 0 and height 1, exported or not: +/// each holds no pixels, and costs a bitmap's header all the same. +fn zero_width_jbig2_dictionary(symbols: u32, exported: bool) -> Vec { + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&(if exported { symbols } else { 0 }).to_be_bytes()); + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + let mut bits = Bits::default(); + bits.put(0, 1); // height class delta 1 (B.4) + for _ in 0..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + if exported { + bits.table_b1(0); // the first export run isn't exported + } + bits.table_b1(symbols); + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of `symbols` symbols of width 1 and no rows, none exported. +/// Only table B.5 has a negative height class delta, so it gets to no rows from one. +fn no_row_jbig2_dictionary(symbols: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0, 1); // height class delta 1 (B.5), a class of no symbols + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.put(0b1111110, 7); // height class delta -1 (B.5): -255 and 254 + bits.put(254, 8); + bits.put(0b10, 2); // width delta 1 (B.2) + for _ in 1..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x05]; // Huffman, tables B.5, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary that climbs to `height`, stays about there for `classes` +/// classes of no symbols, each with a byte of MMR bitmap, and drops to one row for its one symbol, +/// not exported. +fn empty_class_jbig2_dictionary(classes: u32, height: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0b111110, 6); // height class delta, the upper line of table B.5 + bits.put(height - 76, 32); + let mut at = height; + for class in 0..classes { + // Table B.5 has no delta of 0, so it goes down one and up one. + if class % 2 == 1 { + bits.put(0b1111110, 7); // height class delta -1 (B.5): -255 and 254 + bits.put(254, 8); + at -= 1; + } else if class > 0 { + bits.put(0, 1); // height class delta 1 (B.5) + at += 1; + } + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(1, 5); // BMSIZE 1 (B.1) + bits.align(); + bits.put(0xFF, 8); + } + bits.put(0b1111111, 7); // height class delta, the lower line of table B.5 + bits.put(at - 1 - 256, 32); + bits.put(0b10, 2); // width delta 1 (B.2) + bits.put(0b111111, 6); // end of the class (B.2) + bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) + bits.align(); + bits.put(0, 8); // its row + bits.table_b1(1); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x05]; // Huffman, tables B.5, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&1u32.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of `symbols` white symbols of width 1 and `height` rows, none +/// exported, for `height` from 76 up. +fn narrow_jbig2_dictionary(symbols: u32, height: u32) -> Vec { + let mut bits = Bits::default(); + bits.put(0b11111, 5); // height class delta, the upper line of table B.4 + bits.put(height - 76, 32); + bits.put(0b10, 2); // width delta 1 (B.2) + for _ in 1..symbols { + bits.put(0, 1); // width delta 0 (B.2) + } + bits.put(0b111111, 6); // end of the class (B.2) + let rows = height.div_ceil(8) + 1; // an MMR-coded white bitmap: one bit a row + bits.table_b1(rows); // BMSIZE (B.1) + bits.align(); + for _ in 0..rows { + bits.put(0xFF, 8); + } + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + +/// A Huffman-coded symbol dictionary of one white symbol of `width` columns in each height class +/// of `heights` (rising), none exported: wide symbols hold many pixels in few retained words. +fn wide_jbig2_dictionary(width: u32, heights: &[u32]) -> Vec { + let mut bits = Bits::default(); + let mut previous = 0; + for &height in heights { + match height - previous { + // Height class delta (table B.4): 1, or the upper line. + 1 => bits.put(0, 1), + delta => { + bits.put(0b11111, 5); + bits.put(delta - 76, 32); + } + } + previous = height; + bits.put(0b111110, 6); // width delta, the upper line of table B.2 + bits.put(width - 75, 32); + bits.put(0b111111, 6); // end of the class (B.2) + let rows = height.div_ceil(8) + 1; // an MMR-coded white bitmap: one bit a row + bits.table_b1(rows); // BMSIZE (B.1) + bits.align(); + for _ in 0..rows { + bits.put(0xFF, 8); + } + } + let symbols = u32::try_from(heights.len()).unwrap(); + bits.table_b1(symbols); // the first export run, not exported, is all of them + let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 + dictionary.extend_from_slice(&0u32.to_be_bytes()); // exported + dictionary.extend_from_slice(&symbols.to_be_bytes()); // new + dictionary.extend_from_slice(&bits.bytes); + dictionary +} + /// An intermediate generic region, white, every row typical: it decodes in a few bytes however /// large it is. fn jbig2_white_region(width: u32, height: u32) -> Vec { @@ -154,28 +303,6 @@ fn jbig2_halftone_of(side: u32, width: u32, height: u32, mmr: bool) -> Vec { halftone } -/// A Huffman-coded symbol dictionary of `symbols` symbols of width 0 and height 1, exported or not: -/// each holds no pixels, and costs a bitmap's header all the same. -fn zero_width_jbig2_dictionary(symbols: u32, exported: bool) -> Vec { - let mut dictionary = vec![0x00, 0x01]; // Huffman, tables B.4, B.2, B.1, B.1 - dictionary.extend_from_slice(&(if exported { symbols } else { 0 }).to_be_bytes()); - dictionary.extend_from_slice(&symbols.to_be_bytes()); // new - let mut bits = Bits::default(); - bits.put(0, 1); // height class delta 1 (B.4) - for _ in 0..symbols { - bits.put(0, 1); // width delta 0 (B.2) - } - bits.put(0b111111, 6); // end of the class (B.2) - bits.put(0, 5); // BMSIZE 0, uncompressed (B.1) - bits.align(); - if exported { - bits.table_b1(0); // the first export run isn't exported - } - bits.table_b1(symbols); - dictionary.extend_from_slice(&bits.bytes); - dictionary -} - /// A Huffman-coded symbol dictionary of one white symbol, `side` from 76 up, exported or not. fn square_jbig2_dictionary(side: u32, exported: bool) -> Vec { let mut bits = Bits::default(); @@ -255,6 +382,25 @@ fn jbig2_refinement_region() -> Vec { region } +/// A class of no symbols has nothing to cut from its bitmap, and rows are charged by the symbol, so +/// decoding one would be rows for nothing: a thousand classes of 65,000 rows take a second, from a +/// dictionary of 4 KB. It isn't decoded, however many there are, and the symbol after them shows that +/// its bytes are stepped over. +#[test] +fn jbig2_classes_of_no_symbols_decode_no_bitmap() { + let mut stream = black_jbig2_page(); + for number in 0..1_000 { + jbig2_segment(&mut stream, number + 1, 0, &[], &empty_class_jbig2_dictionary(1024, 65_000)); + } + let (done, finished) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = done.send(jbig2_image_draws(&stream)); + }); + // This takes milliseconds now and took a quarter of an hour, so a margin this wide can't fail on + // a busy machine. + assert_eq!(finished.recv_timeout(std::time::Duration::from_secs(120)), Ok(true)); +} + /// Each item a JBIG2 stream decodes costs its work, and the stream may ask for 2^30: four of the /// largest pages, some 6 s to 17 s. Each bitmap is bounded on its own, but nothing bounded how many /// there were, so 1,140 bytes of generic regions took over 40 s, and a 795-byte PDF's one halftone @@ -363,45 +509,47 @@ fn jbig2_decoding_work_is_counted_item_by_item() { data: jbig2_pattern_dictionary(32, 2), cost: two_patterns(1, 32), }, - // Rows cost in a symbol of no pixels too. - Item { what: "symbols without pixels", before: vec![], spent: 0, kind: 0, refs: &[], data: zero_width_jbig2_dictionary(3, false), cost: 12 }, - Item { what: "a tall symbol of no width", before: vec![], spent: 0, kind: 0, refs: &[], data: zero(1000), cost: 4000 }, - Item { what: "new symbols", before: vec![], spent: 0, kind: 0, refs: &[], data: square_jbig2_dictionary(80, false), cost: 6720 }, - Item { what: "referred symbols", before: symbol(), spent: 4, kind: 0, refs: &[1], data: exporting_none, cost: 1 }, + // What an item reads counts, at 38 an integer: two classes, five widths and an export run. + Item { what: "symbols of no rows", before: vec![], spent: 0, kind: 0, refs: &[], data: no_row_jbig2_dictionary(5), cost: 304 }, + // Rows cost in a symbol of no pixels too, on top of a class, three widths and an export run. + Item { what: "symbols without pixels", before: vec![], spent: 0, kind: 0, refs: &[], data: zero_width_jbig2_dictionary(3, false), cost: 202 }, + Item { what: "a tall symbol of no width", before: vec![], spent: 0, kind: 0, refs: &[], data: zero(1000), cost: 4114 }, + Item { what: "new symbols", before: vec![], spent: 0, kind: 0, refs: &[], data: square_jbig2_dictionary(80, false), cost: 6834 }, + Item { what: "referred symbols", before: symbol(), spent: 156, kind: 0, refs: &[1], data: exporting_none, cost: 39 }, // Its symbol, and its one row and word; Huffman-coded, a table line for the symbol. - Item { what: "immediate text", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, - Item { what: "intermediate text", before: symbol(), spent: 4, kind: 4, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, - Item { what: "a text region's pixels", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75 }, + Item { what: "immediate text", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, + Item { what: "intermediate text", before: symbol(), spent: 156, kind: 4, refs: &[1], data: jbig2_text_region(1, 1, 0), cost: 13 }, + Item { what: "a text region's pixels", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75 }, Item { what: "an intermediate text region's pixels", before: symbol(), - spent: 4, + spent: 156, kind: 4, refs: &[1], data: jbig2_text_region(32, 32, 0), cost: 75, }, - // The region's 13, a copy of the symbol's row and the round of the loop. - Item { what: "an instance", before: symbol(), spent: 4, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), cost: 15 }, - // The region's 13, 80 rows of 4 words copied, the round of the loop. + // The region's 13, a copy of the symbol's row, and the instance's 4 integers. + Item { what: "an instance", before: symbol(), spent: 156, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), cost: 166 }, + // The region's 13, 80 rows of 4 words copied, and the instance's 4 integers. Item { what: "an instance of a large symbol", before: vec![(0, square_jbig2_dictionary(80, true))], - spent: 6720, + spent: 6872, kind: 6, refs: &[1], data: jbig2_text_region(1, 1, 1), - cost: 334, + cost: 485, }, - // And decoded as well as copied. + // And decoded as well as copied, with the 4 integers that gave its size. Item { what: "a refined instance", before: vec![(0, square_jbig2_dictionary(80, true))], - spent: 6720, + spent: 6872, kind: 6, refs: &[1], data: jbig2_refined_text_region(), - cost: 7054, + cost: 7357, }, // A bit plane, a cell's copy and the region's. Item { @@ -457,6 +605,54 @@ fn jbig2_decoding_work_is_counted_item_by_item() { } } +/// From `cargo xtask fuzz`: one changed byte in a symbol dictionary's arithmetic-coded data +/// hung the renderer. Two of its loops can stop advancing: an export run of length 0, and a +/// height class whose first width is already the end-of-class marker. Past the end of its +/// data the decoder can return those for ever. Vendored hayro-jbig2 patch: at most one height +/// class per new symbol and `2 × symbols + 1` export runs, each plus `EXTRA_EMPTY_STEPS`. +/// These synthetic dictionaries (two new symbols each, a few bytes of arbitrary data) decode +/// the same way. In the second every class after the first symbol adds 13 to the height, so it +/// only fails when the height overflows `u32`, after 330 million empty classes (about five +/// seconds; a delta of 0 would never end). The page draws it as twenty images, so that the +/// stall without the patch is well past the timeout on any machine. +#[test] +fn jbig2_symbol_dictionaries_that_never_advance_terminate() { + for (what, data, copies) in + [("zero-length export runs", &[0x05, 0x70, 0x05, 0x14, 0x02, 0x1d, 0x56, 0x3f][..], 1), ("empty height classes", &[0x06, 0x99][..], 20)] + { + // Embedded JBIG2 segments (ISO 14492 §7.2): page information, then a symbol dictionary. + let mut jbig2 = Vec::new(); + let mut segment = |number: u32, kind: u8, data: &[u8]| { + jbig2.extend_from_slice(&number.to_be_bytes()); + jbig2.extend_from_slice(&[kind, 0, 1]); + jbig2.extend_from_slice(&u32::try_from(data.len()).unwrap().to_be_bytes()); + jbig2.extend_from_slice(data); + }; + let mut page = Vec::new(); + page.extend_from_slice(&8u32.to_be_bytes()); + page.extend_from_slice(&8u32.to_be_bytes()); + page.extend_from_slice(&[0; 8]); // resolution + page.extend_from_slice(&[0, 0, 0]); // flags, striping + segment(0, 48, &page); + let mut dictionary = vec![0, 0]; // arithmetic coding, template 0, no refinement + dictionary.extend_from_slice(&[3, 0xff, 0xfd, 0xff, 2, 0xfe, 0xfe, 0xfe]); // AT pixels + dictionary.extend_from_slice(&1u32.to_be_bytes()); // exported symbols + dictionary.extend_from_slice(&2u32.to_be_bytes()); // new symbols + dictionary.extend_from_slice(data); + segment(1, 0, &dictionary); + // The stall without the patch is well past the timeout on any machine; `copies` decodes stand + // in for the images the page draws. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + for _ in 0..copies { + let _ = black_pixels(&jbig2); + } + let _ = tx.send(()); + }); + rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap_or_else(|_| panic!("{what}: a JBIG2 symbol dictionary must not stall the decoder")); + } +} + /// A JBIG2 stream can put a region, a reference or a symbol at the very end of what an `i32` holds, /// where adding to it overflowed: a panic wherever overflow is checked, a build's own tests included. /// A region or a symbol out there is off the page, so nothing draws. Vendored hayro-jbig2 patch: the arithmetic wraps, as @@ -520,3 +716,4 @@ fn jbig2_placements_at_the_end_of_an_i32_draw_nothing() { assert_eq!(ink(hex(UPPER_LINE), 16, 12), 0, "an upper range line past an i32"); assert_eq!(ink(hex(LOWER_LINE), 16, 12), 0, "a lower range line past an i32"); } + From f233df14ad38e1d97fe945e68df59706adada1f5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:21:20 -0500 Subject: [PATCH 07/31] craft-pdf: the standard security handler from PdfCraft (`crypt`) (#897) * craft-pdf: the standard security handler from PdfCraft (`crypt`) Adds the `crypt` feature: PdfCraft's pdfcraft-crypt crate (revision 2b65ab000bfe0997dec5fbed2769d3da3799355b) as craft_pdf::crypt, with its unit, integration and property tests. The code is unchanged; doc comments and test imports name the new module. It decrypts and encrypts strings and streams for revisions 2 to 6 and creates encryption dictionaries. The reader keeps its own decryptor. * craft-pdf: the crypt unit tests build for wasm The property test in src/crypt/tests.rs needs proptest, which is a dev-dependency of the non-wasm targets only. It is now compiled for those targets only; the other unit tests still build for wasm. --- Cargo.lock | 18 +- crates/pdf/Cargo.toml | 10 + crates/pdf/README.md | 31 + crates/pdf/src/crypt.rs | 796 ++++++++++++++++++ crates/pdf/src/crypt/aes_cbc.rs | 89 ++ crates/pdf/src/crypt/rc4.rs | 22 + crates/pdf/src/crypt/tests.rs | 449 ++++++++++ crates/pdf/src/lib.rs | 2 + crates/pdf/tests/crypt/main.rs | 7 + .../pdf/tests/crypt/permissions_revision.rs | 126 +++ crates/pdf/tests/crypt/proptests.rs | 464 ++++++++++ crates/pdf/tests/crypt/qpdf_literals.rs | 326 +++++++ 12 files changed, 2339 insertions(+), 1 deletion(-) create mode 100644 crates/pdf/src/crypt.rs create mode 100644 crates/pdf/src/crypt/aes_cbc.rs create mode 100644 crates/pdf/src/crypt/rc4.rs create mode 100644 crates/pdf/src/crypt/tests.rs create mode 100644 crates/pdf/tests/crypt/main.rs create mode 100644 crates/pdf/tests/crypt/permissions_revision.rs create mode 100644 crates/pdf/tests/crypt/proptests.rs create mode 100644 crates/pdf/tests/crypt/qpdf_literals.rs diff --git a/Cargo.lock b/Cargo.lock index ebce5f5ad..1e9d85eff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -926,7 +926,7 @@ version = "0.1.0" name = "craft-cms" version = "0.1.0" dependencies = [ - "md-5", + "md-5 0.10.6", "moxcms", "rayon", ] @@ -1371,6 +1371,7 @@ dependencies = [ name = "craft-pdf" version = "0.1.0" dependencies = [ + "aes", "bytemuck", "hayro", "hayro-cmap", @@ -1378,9 +1379,14 @@ dependencies = [ "hayro-syntax", "krilla", "kurbo 0.13.1", + "md-5 0.11.0", + "proptest", + "sha2 0.11.1", + "stringprep", "thiserror 2.0.21", "ttf-parser", "wasm-bindgen-test", + "zeroize", ] [[package]] @@ -4083,6 +4089,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" diff --git a/crates/pdf/Cargo.toml b/crates/pdf/Cargo.toml index 17911b7cf..305f1f418 100644 --- a/crates/pdf/Cargo.toml +++ b/crates/pdf/Cargo.toml @@ -18,6 +18,7 @@ raster = ["interpret", "dep:hayro", "dep:bytemuck"] write = ["dep:krilla", "dep:thiserror"] write-pdf = ["write", "krilla/pdf"] write-simple-text = ["write", "krilla/simple-text"] +crypt = ["dep:aes", "dep:md-5", "dep:sha2", "dep:stringprep", "dep:zeroize", "dep:thiserror"] [dependencies] hayro-syntax = { workspace = true, optional = true } @@ -26,6 +27,12 @@ hayro = { workspace = true, optional = true } bytemuck = { workspace = true, optional = true } krilla = { workspace = true, optional = true } thiserror = { workspace = true, optional = true } +# The standard security handler (`crypt`): pure Rust, MIT OR Apache-2.0, builds for wasm. +aes = { version = "0.9", optional = true } +md-5 = { version = "0.11", optional = true } +sha2 = { version = "0.11", optional = true } +stringprep = { version = "0.1", optional = true } +zeroize = { version = "1", optional = true } [dev-dependencies] hayro-syntax = "=0.7.2" @@ -34,5 +41,8 @@ hayro-cmap = "=0.1.0" kurbo = "0.13" ttf-parser = "=0.25.1" +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +proptest = { workspace = true } + [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3" diff --git a/crates/pdf/README.md b/crates/pdf/README.md index 816eba779..44af16bbf 100644 --- a/crates/pdf/README.md +++ b/crates/pdf/README.md @@ -12,6 +12,7 @@ No feature is enabled by default. | `write` | Guarded Krilla authoring; no shared reader dependency | | `write-pdf` | `write` plus Krilla's embedded-PDF support | | `write-simple-text` | `write` plus Krilla's optional text shaper and `draw_text` | +| `crypt` | The standard security handler: decrypt, encrypt and create encryption. No reader or writer dependency | ## Reading and rendering @@ -109,6 +110,36 @@ read-only consumer, then repeat with `--target wasm32-unknown-unknown` and the t These backend checks supplement the root workspace gates; excluded vendor crates are not covered by `cargo test --workspace` alone. +## Encryption + +`crypt` is the PDF standard security handler (ISO 32000-2 §7.6) for code that edits or writes +PDF files. The reader decrypts on its own and does not use it. It covers revisions 2 to 6: +40-bit and 128-bit RC4, AES-128 crypt filters, and AES-256 (R5 is read, R6 is read and written). + +- `SecurityHandler::open(dict, id0, password)` authenticates the owner password first, then the + user password. Passwords are tried as PDFDocEncoding and as typed (R2–R4), or in their + SASLprep form and as typed (R5/R6). `authenticate_owner` upgrades a user handler. + `without_key` is for a file opened without a password whose only encrypted data is behind + crypt filters authorised when an embedded file is opened; that data is left as stored. +- `decrypt_string`, `encrypt_string`, `decrypt_stream` and `encrypt_stream` take the object + number and generation. `StreamKind` selects the crypt filter: `/StmF`, `/EFF` for embedded + files, or a stream's own `/Crypt` filter name. `encrypts_metadata` says whether the caller + passes `/Metadata` streams through at all. AES decryption is tolerant: a trailing partial + block is ignored and bad padding is left in. +- `permissions()` gives the `/P` flags; for R5/R6 a permission counts only when `/Perms` grants + it too. Owners have every permission. +- `create(&NewEncryption, id0)` builds the `/Encrypt` values for RC4-40, RC4-128, AES-128 or + AES-256. The caller supplies 32 bytes of entropy (`seed`), so the module needs no OS access + and builds for the web. `file_id` derives a file identifier from the same seed. +- `EncryptDict` holds the dictionary's values. The caller reads them from, and writes them to, + its own object model; the module does not parse PDF syntax. Keys and prepared passwords are + zeroized when dropped. + +The code and its tests come from PdfCraft's `pdfcraft-crypt` crate (revision +`2b65ab000bfe0997dec5fbed2769d3da3799355b`), unchanged apart from the module paths. Tests: +`cargo test -p craft-pdf --no-default-features --features crypt`. `tests/crypt/qpdf_literals.rs` +checks key derivation against values another implementation wrote; only the numbers are stored. + ## Writing Plain `write` retains Krilla raster images without its optional text shaper. Applications diff --git a/crates/pdf/src/crypt.rs b/crates/pdf/src/crypt.rs new file mode 100644 index 000000000..7f7a51e09 --- /dev/null +++ b/crates/pdf/src/crypt.rs @@ -0,0 +1,796 @@ +//! The PDF standard security handler (ISO 32000-2 §7.6), for code that writes or edits PDF files. +//! The reader (`read`) decrypts on its own; this module also encrypts and creates encryption. +//! +//! Supports every revision of the standard handler: +//! - R2 (40-bit RC4), R3 (RC4 up to 128-bit), R4 (crypt filters: RC4 or AES-128); +//! - R5 (the deprecated AES-256 extension) and R6 (AES-256, PDF 2.0). +//! +//! It authenticates with the user or owner password (owner first, as viewers do) and derives +//! per-object keys. It decrypts and encrypts strings and streams, and honours crypt filters, +//! including `Identity` and embedded-file-only encryption. It can also *create* encryption +//! dictionaries, for an app's security settings and for test fixtures. +//! +//! The module knows nothing about a PDF object model. Callers pass in the values of the +//! `/Encrypt` dictionary and the first element of the trailer `/ID`. + +mod aes_cbc; +mod rc4; + +use md5::{Digest as _, Md5}; +use sha2::{Sha256, Sha384, Sha512}; +use zeroize::Zeroizing; + +pub use aes_cbc::{aes_cbc_decrypt, aes_cbc_encrypt}; +pub use rc4::rc4; + +#[derive(Debug, thiserror::Error, Clone, PartialEq, Eq)] +pub enum CryptError { + #[error("the password is incorrect")] + WrongPassword, + #[error("unsupported security handler: {0}")] + Unsupported(String), + #[error("malformed encryption dictionary: {0}")] + Malformed(String), +} + +/// 32-byte padding string used by revisions 2–4 (§7.6.4.3.2, step a). +pub const PADDING: [u8; 32] = [ + 0x28, 0xBF, 0x4E, 0x5E, 0x4E, 0x75, 0x8A, 0x41, 0x64, 0x00, 0x4E, 0x56, 0xFF, 0xFA, 0x01, 0x08, 0x2E, 0x2E, 0x00, 0xB6, 0xD0, 0x68, 0x3E, 0x80, + 0x2F, 0x0C, 0xA9, 0xFE, 0x64, 0x53, 0x69, 0x7A, +]; + +/// How a crypt filter encrypts (`/CFM`). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Method { + /// `None` / the `Identity` filter: data is not encrypted. + Identity, + /// `V2`: RC4. + Rc4, + /// `AESV2`: AES-128-CBC. + Aes128, + /// `AESV3`: AES-256-CBC. + Aes256, +} + +/// The values of an `/Encrypt` dictionary that the handler needs. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct EncryptDict { + /// `/Filter` (must be `Standard`). + pub filter: Vec, + pub v: i64, + pub r: i64, + /// Key length in bits (`/Length`; 40 when absent). + pub length_bits: i64, + pub o: Vec, + pub u: Vec, + pub oe: Vec, + pub ue: Vec, + pub perms: Vec, + pub p: i32, + pub encrypt_metadata: bool, + /// Crypt filters (`/CF`): name → (method, key length in bytes if given). + pub crypt_filters: Vec<(Vec, Method)>, + pub stm_f: Vec, + pub str_f: Vec, + pub ef_f: Vec, +} + +/// Which password authenticated the document. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Auth { + Owner, + User, +} + +/// Permission flags from `/P`, read the way the handler's revision reads them (ISO 32000-1:2008, +/// Table 22). Owners have every permission. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Permissions { + pub bits: i32, + pub owner: bool, + /// The `/R` value of the handler that read `bits`. + pub revision: i64, +} + +impl Permissions { + fn bit(&self, n: u32) -> bool { + self.owner || (self.bits >> (n - 1)) & 1 == 1 + } + /// Revision 3 and later read the full flag list of Table 22; revision 2 has its own. + fn is_revision_3_or_later(&self) -> bool { + self.revision >= 3 + } + pub fn print(&self) -> bool { + self.bit(3) + } + /// Modify contents other than the operations below. + pub fn modify(&self) -> bool { + self.bit(4) + } + pub fn copy(&self) -> bool { + self.bit(5) + } + pub fn annotate(&self) -> bool { + self.bit(6) + } + pub fn fill_forms(&self) -> bool { + if self.is_revision_3_or_later() { self.bit(9) || self.bit(6) } else { self.bit(6) } + } + /// Revision 2 covers accessibility with the copy flag (bit 5); revision 3 and later with bit 10. + pub fn extract_for_accessibility(&self) -> bool { + if self.is_revision_3_or_later() { self.bit(10) } else { self.bit(5) } + } + /// Insert, rotate or delete pages; create bookmarks and thumbnails. Revision 3 and later grant + /// it by bit 11 alone. Table 22 defines bit 11 for those revisions only, so at revision 2 the + /// modify flag (bit 4) covers it. + pub fn assemble(&self) -> bool { + if self.is_revision_3_or_later() { self.bit(11) } else { self.bit(4) } + } + /// Take pages out into another document (Extract, Split, a source of Combine or Insert + /// Pages): [`Self::assemble`] and copying (bit 5), since the pages carry their text and + /// graphics into a new file without the document's protection (#5081). "Any except + /// extracting pages" sets bit 4 but not bit 11, so at revision 3 and later it grants neither. + pub fn extract_pages(&self) -> bool { + self.assemble() && self.copy() + } + /// Revision 3 and later need bit 3 as well as bit 12. Table 22 defines bit 12 for those + /// revisions only: at revision 2 the print flag (bit 3) is all there is. + pub fn print_high_quality(&self) -> bool { + if self.is_revision_3_or_later() { self.print() && self.bit(12) } else { self.print() } + } +} + +/// What kind of stream is being decrypted (selects the crypt filter). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum StreamKind<'a> { + Normal, + /// An embedded file stream (uses `/EFF`). + EmbeddedFile, + /// The stream names its own crypt filter (`/Filter [/Crypt …] /DecodeParms << /Name … >>`). + Named(&'a [u8]), +} + +/// An authenticated security handler for one document. +#[derive(Clone, Debug)] +pub struct SecurityHandler { + dict: EncryptDict, + key: Zeroizing>, + auth: Auth, + recovered_user: Option>>, + methods: Methods, +} + +/// The crypt filter methods of a handler, resolved once: a document controls both the number of +/// filters and the number of strings, so a search of `/CF` per string would multiply them. +#[derive(Clone, Debug)] +struct Methods { + string: Method, + stream: Method, + embedded_file: Method, + named: std::collections::HashMap, Method>, +} + +impl Methods { + fn new(dict: &EncryptDict) -> Self { + let mut named = std::collections::HashMap::with_capacity(dict.crypt_filters.len()); + for (name, method) in &dict.crypt_filters { + #[cfg(test)] + tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1)); + // The first of two filters with one name wins, as a search would find it. + named.entry(name.clone()).or_insert(*method); + } + let mut methods = Methods { string: Method::Identity, stream: Method::Identity, embedded_file: Method::Identity, named }; + methods.string = methods.get(&dict.str_f); + methods.stream = methods.get(&dict.stm_f); + methods.embedded_file = methods.get(if dict.ef_f.is_empty() { &dict.stm_f } else { &dict.ef_f }); + methods + } + + fn get(&self, name: &[u8]) -> Method { + if name == b"Identity" || name.is_empty() { + return Method::Identity; + } + #[cfg(test)] + tests::FILTER_COMPARES.with(|c| c.set(c.get() + 1)); + self.named.get(name).copied().unwrap_or(Method::Identity) + } +} + +impl SecurityHandler { + fn new(dict: EncryptDict, key: Zeroizing>, auth: Auth, recovered_user: Option>>) -> Self { + let methods = Methods::new(&dict); + Self { dict, key, auth, recovered_user, methods } + } + + /// Authenticate with `password` (tried as owner password, then user password). `None` + /// tries the empty password, which opens documents that only have an owner password. + pub fn open(dict: EncryptDict, id0: &[u8], password: Option<&str>) -> Result { + if !dict.filter.is_empty() && dict.filter != b"Standard" { + return Err(CryptError::Unsupported(format!("/Filter /{}", String::from_utf8_lossy(&dict.filter)))); + } + let pw = password.unwrap_or(""); + let mut recovered_user = None; + let (key, auth) = match dict.r { + 2..=4 => { + let mut found = None; + for legacy in legacy_candidates(pw) { + if let Some((k, user)) = owner_key_r4(&dict, id0, &legacy) { + recovered_user = Some(user); + found = Some((k, Auth::Owner)); + } else if let Some(k) = user_key_r4(&dict, id0, &legacy) { + found = Some((k, Auth::User)); + } + if found.is_some() { + break; + } + } + found.ok_or(CryptError::WrongPassword)? + } + 5 | 6 => { + let pw = sasl_password(pw); + if let Some(k) = key_r6(&dict, &pw, true)? { + (k, Auth::Owner) + } else if let Some(k) = key_r6(&dict, &pw, false)? { + (k, Auth::User) + } else { + return Err(CryptError::WrongPassword); + } + } + r => return Err(CryptError::Unsupported(format!("revision {r}"))), + }; + Ok(Self::new(dict, key, auth, recovered_user)) + } + + /// A handler with no file key, for a document opened without a password whose strings and + /// streams aren't encrypted and whose encrypted crypt filters are all ones the caller found + /// authorised only when an embedded file is opened (`/AuthEvent /EFOpen`, ISO 32000-2 §7.6.6, + /// Table 25). Data such a filter encrypts is left as stored, read and written alike, so it + /// keeps its encrypted bytes; [`SecurityHandler::decrypts`] tells which. `None` when strings or + /// streams are encrypted, which needs the key, or the handler is older than crypt filters (V4). + pub fn without_key(dict: EncryptDict) -> Option { + if dict.v < 4 { + return None; + } + let methods = Methods::new(&dict); + if methods.string != Method::Identity || methods.stream != Method::Identity { + return None; + } + Some(Self { dict, key: Zeroizing::new(Vec::new()), auth: Auth::User, recovered_user: None, methods }) + } + + /// Whether the handler has the file key: false for [`SecurityHandler::without_key`]. + pub fn has_key(&self) -> bool { + !self.key.is_empty() + } + + /// Whether data of `kind` is encrypted, so that reading it needs the file key. + pub fn decrypts(&self, kind: StreamKind) -> bool { + self.method(Some(kind)) != Method::Identity + } + + /// Whether data left as stored by a handler without the key still decrypts after its object + /// is renumbered: only AES-256 doesn't mix the object number into the key (Algorithm 1.A). + pub fn survives_renumbering(&self) -> bool { + [self.methods.string, self.methods.stream, self.methods.embedded_file] + .into_iter() + .chain(self.methods.named.values().copied()) + .all(|m| matches!(m, Method::Identity | Method::Aes256)) + } + + /// Upgrade to owner access with the owner (permissions) password, as when a restricted + /// document is unlocked after opening. The user password, or any other, is `WrongPassword`. + /// Both passwords unlock the same file key, so what was decrypted so far stays valid; a + /// different key is refused too. + pub fn authenticate_owner(&self, id0: &[u8], password: &str) -> Result { + let owner = Self::open(self.dict.clone(), id0, Some(password))?; + if owner.auth != Auth::Owner || owner.key != self.key { + return Err(CryptError::WrongPassword); + } + Ok(owner) + } + + pub fn auth(&self) -> Auth { + self.auth + } + + /// For R2–R4 documents opened with the owner password: the user password, which owner + /// authentication recovers (Algorithm 7). Lets components that only accept user passwords + /// open the document too. + pub fn recovered_user_password(&self) -> Option>> { + self.recovered_user.clone() + } + + /// For R5/R6, `/P` is stored in clear and again in `/Perms`, encrypted with the file key + /// (Algorithm 13). Where the two differ, a permission counts only when both grant it, so an + /// edited `/P` grants nothing extra. Without a readable `/Perms`, `/P` applies as before. + pub fn permissions(&self) -> Permissions { + let bits = self.perms_p().map_or(self.dict.p, |p| self.dict.p & p); + Permissions { bits, owner: self.auth == Auth::Owner, revision: self.dict.r } + } + + /// `/P` as `/Perms` holds it: AES-256 ECB of one block (= CBC with a zero IV), marked `adb`. + fn perms_p(&self) -> Option { + if !matches!(self.dict.r, 5 | 6) || !self.has_key() { + return None; + } + let plain = Zeroizing::new(aes_cbc_decrypt(&self.key, &[0; 16], self.dict.perms.first_chunk::<16>()?, false)); + let p = i32::from_le_bytes(*plain.first_chunk::<4>()?); + (plain.get(9..12) == Some(&b"adb"[..])).then_some(p) + } + + pub fn dict(&self) -> &EncryptDict { + &self.dict + } + + /// The file encryption key (for tests and diagnostics). + pub fn file_key(&self) -> &[u8] { + &self.key + } + + /// Whether `/Metadata` streams are encrypted. + pub fn encrypts_metadata(&self) -> bool { + self.dict.encrypt_metadata + } + + /// The method for strings or a kind of stream. + fn method(&self, stream: Option) -> Method { + if self.dict.v < 4 { + return Method::Rc4; + } + match stream { + None => self.methods.string, + Some(StreamKind::Normal) => self.methods.stream, + Some(StreamKind::EmbeddedFile) => self.methods.embedded_file, + Some(StreamKind::Named(n)) => self.methods.get(n), + } + } + + /// Algorithm 1 (§7.6.3.2): the key for one object. AES-256 uses the file key directly + /// (Algorithm 1.A, §7.6.3.3). + /// + /// For an RC4 crypt filter under `/V 4`, a file key shorter than 16 bytes is first extended + /// to 16 bytes with zeros. The standard doesn't say so (Algorithm 1 takes the n-byte key as + /// it is), but that is how Acrobat writes and reads such files: a `/V 4` RC4 crypt filter + /// with a 40- or 48-bit key (pdf.js issue 19484). `hayro-syntax` (the `read` feature) reads them + /// the same way, so a page it shows is one this module can read and save. AES-128 keys are + /// left as they are: `AESV2` takes a 128-bit key (Table 25), and Acrobat's extension is only + /// seen with RC4. Every key [`create`] makes under `/V 4` is 16 bytes already. + fn object_key(&self, method: Method, num: u32, generation: u16) -> Zeroizing> { + if method == Method::Aes256 { + return self.key.clone(); + } + let mut key = self.key.clone(); + if method == Method::Rc4 && self.dict.v == 4 && key.len() < 16 { + key.resize(16, 0); + } + let mut h = Md5::new(); + h.update(&key); + h.update(&num.to_le_bytes()[..3]); + h.update(generation.to_le_bytes()); + if method == Method::Aes128 { + h.update(b"sAlT"); + } + let mut digest = Zeroizing::new(h.finalize().to_vec()); + digest.truncate((key.len() + 5).min(16)); + digest + } + + fn apply(&self, method: Method, num: u32, generation: u16, data: &[u8], encrypt: bool) -> Vec { + // Without the key, encrypted data stays as stored (see `without_key`). + if !self.has_key() { + return data.to_vec(); + } + match method { + Method::Identity => data.to_vec(), + Method::Rc4 => rc4(&self.object_key(method, num, generation), data), + Method::Aes128 | Method::Aes256 => { + let key = self.object_key(method, num, generation); + if encrypt { + // Deterministic IV derived from the key, object and content: saves are + // reproducible and no IV repeats for different plaintexts. + let mut h = Sha256::new(); + h.update(&key); + h.update(num.to_le_bytes()); + h.update(generation.to_le_bytes()); + h.update(data); + let digest = h.finalize(); + let mut iv = [0u8; 16]; + for (d, s) in iv.iter_mut().zip(digest.iter()) { + *d = *s; + } + let mut out = iv.to_vec(); + out.extend(aes_cbc_encrypt(&key, &iv, data, true)); + out + } else { + let Some((iv, body)) = data.split_first_chunk::<16>() else { + return Vec::new(); // only (part of) an IV: empty plaintext + }; + aes_cbc_decrypt(&key, iv, body, true) + } + } + } + } + + pub fn decrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec { + self.apply(self.method(None), num, generation, data, false) + } + + pub fn encrypt_string(&self, num: u32, generation: u16, data: &[u8]) -> Vec { + self.apply(self.method(None), num, generation, data, true) + } + + pub fn decrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec { + self.apply(self.method(Some(kind)), num, generation, data, false) + } + + pub fn encrypt_stream(&self, num: u32, generation: u16, data: &[u8], kind: StreamKind) -> Vec { + self.apply(self.method(Some(kind)), num, generation, data, true) + } + + /// Whether strings are encrypted at all (false with `/StrF /Identity`). + pub fn encrypts_strings(&self) -> bool { + self.method(None) != Method::Identity + } +} + +// ── Passwords ────────────────────────────────────────────────────────────────────────────────── + +/// PDFDocEncoding (ISO 32000-2 Annex D) where it differs from Latin-1: bytes 0x18–0x1F and +/// 0x80–0xA0 (0x9F is undefined). Here because R2–R4 passwords need it; an object layer that +/// decodes text strings can use it too. +const PDFDOC_LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜']; +const PDFDOC_HIGH: [char; 33] = [ + '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł', 'œ', + 'š', 'ž', '\u{FFFD}', '€', +]; + +/// PDFDocEncoding → Unicode. +pub fn pdfdoc_char(c: u8) -> char { + match c { + 0x18..=0x1F => PDFDOC_LOW[usize::from(c - 0x18)], + 0x80..=0xA0 => PDFDOC_HIGH[usize::from(c - 0x80)], + _ => char::from(c), + } +} + +/// Unicode → PDFDocEncoding, `None` for a character it lacks (ğ, ş, İ, Cyrillic, CJK, …). +pub fn pdfdoc_byte(c: char) -> Option { + let at = |table: &[char], base: u8| table.iter().position(|&t| t == c && t != '\u{FFFD}').and_then(|i| u8::try_from(i).ok()).map(|i| base + i); + at(&PDFDOC_LOW, 0x18).or_else(|| at(&PDFDOC_HIGH, 0x80)).or_else(|| match u32::from(c) { + 0x18..=0x1F | 0x80..=0xA0 => None, + n => u8::try_from(n).ok(), + }) +} + +/// Revisions 2–4: the password in PDFDocEncoding (§7.6.4.3.2, Algorithm 2), at most 32 bytes. +/// Characters PDFDocEncoding lacks are dropped. +fn legacy_password(pw: &str) -> Zeroizing> { + Zeroizing::new(pw.chars().filter_map(pdfdoc_byte).take(32).collect()) +} + +/// The bytes an R2–R4 password may have been written as: PDFDocEncoding, as the specification +/// asks; and for a password with characters PDFDocEncoding lacks (ğ, ş, İ, …), which it gives +/// no bytes, also its UTF-8 bytes, which qpdf and other tools write for such passwords. +fn legacy_candidates(pw: &str) -> Vec>> { + let mut out = vec![legacy_password(pw)]; + if pw.chars().any(|c| pdfdoc_byte(c).is_none()) { + out.push(Zeroizing::new(pw.bytes().take(32).collect())); + } + out +} + +/// Revisions 5–6: SASLprep (RFC 4013), UTF-8, at most 127 bytes. +fn sasl_password(pw: &str) -> Zeroizing> { + let prepped = Zeroizing::new(stringprep::saslprep(pw).map(|c| c.into_owned()).unwrap_or_else(|_| pw.to_string())); + let mut bytes = Zeroizing::new(prepped.as_bytes().to_vec()); + bytes.truncate(127); + bytes +} + +fn pad(pw: &[u8]) -> Zeroizing<[u8; 32]> { + let mut out = Zeroizing::new(PADDING); + let n = pw.len().min(32); + out[..n].copy_from_slice(&pw[..n]); + out[n..].copy_from_slice(&PADDING[..32 - n]); + out +} + +fn key_len(d: &EncryptDict) -> usize { + if d.r == 2 { 5 } else { (d.length_bits.clamp(40, 128) / 8) as usize } +} + +// ── Revisions 2–4 ────────────────────────────────────────────────────────────────────────────── + +/// Algorithm 2: file key from a (user) password. +fn file_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Zeroizing> { + let n = key_len(d); + let mut h = Md5::new(); + h.update(*pad(pw)); + h.update(&d.o[..d.o.len().min(32)]); + h.update(d.p.to_le_bytes()); + h.update(id0); + if d.r >= 4 && !d.encrypt_metadata { + h.update([0xFF; 4]); + } + let mut digest = Zeroizing::new(h.finalize().to_vec()); + if d.r >= 3 { + for _ in 0..50 { + digest = Zeroizing::new(Md5::digest(&digest[..n]).to_vec()); + } + } + digest.truncate(n); + digest +} + +/// Algorithms 4/5: the `/U` value a key produces. +fn u_value_r4(d: &EncryptDict, id0: &[u8], key: &[u8]) -> Vec { + if d.r == 2 { + return rc4(key, &PADDING); + } + let mut h = Md5::new(); + h.update(PADDING); + h.update(id0); + let mut x = rc4(key, &h.finalize()); + for i in 1..=19u8 { + let k = Zeroizing::new(key.iter().map(|b| b ^ i).collect::>()); + x = rc4(&k, &x); + } + x.extend_from_slice(&[0u8; 16]); + x +} + +/// Algorithm 6: authenticate a user password; returns the file key. +/// +/// Some producers ignore `/EncryptMetadata false` when deriving the key (they omit the +/// `FF FF FF FF` of Algorithm 2, step f). Viewers open those files, so if the strict derivation +/// fails we retry the other way. +fn user_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option>> { + let check = |d: &EncryptDict| { + let key = file_key_r4(d, id0, pw); + let u = u_value_r4(d, id0, &key); + let n = if d.r == 2 { 32 } else { 16 }; + (d.u.len() >= n && u[..n] == d.u[..n]).then_some(key) + }; + check(d).or_else(|| (d.r >= 4 && !d.encrypt_metadata).then(|| check(&EncryptDict { encrypt_metadata: true, ..d.clone() })).flatten()) +} + +/// The RC4 key derived from an owner password (Algorithm 3, steps a–d). +fn owner_rc4_key(d: &EncryptDict, owner_pw: &[u8]) -> Zeroizing> { + let n = key_len(d); + let mut digest = Zeroizing::new(Md5::digest(*pad(owner_pw)).to_vec()); + if d.r >= 3 { + for _ in 0..50 { + digest = Zeroizing::new(Md5::digest(&*digest).to_vec()); + } + } + digest.truncate(n); + digest +} + +type SecretBytes = Zeroizing>; + +/// Algorithm 7: authenticate an owner password (recover the user password from `/O`). +/// Returns the file key and the recovered user password (padding removed). +fn owner_key_r4(d: &EncryptDict, id0: &[u8], pw: &[u8]) -> Option<(SecretBytes, SecretBytes)> { + let rk = owner_rc4_key(d, pw); + let mut user = Zeroizing::new(d.o[..d.o.len().min(32)].to_vec()); + if d.r == 2 { + user = Zeroizing::new(rc4(&rk, &user)); + } else { + for i in (0..=19u8).rev() { + let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::>()); + user = Zeroizing::new(rc4(&k, &user)); + } + } + let key = user_key_r4(d, id0, &user)?; + // The recovered value is the padded password: strip the padding suffix. + let len = (0..=32).find(|&i| user.len() >= 32 && user[i..32] == PADDING[..32 - i]).unwrap_or(user.len().min(32)); + user.truncate(len); + Some((key, user)) +} + +// ── Revisions 5–6 ────────────────────────────────────────────────────────────────────────────── + +/// The hash of a password for R5 (SHA-256) or R6 (Algorithm 2.B). +fn hash_r6(r: i64, pw: &[u8], salt: &[u8], udata: &[u8]) -> Zeroizing> { + let mut k = Zeroizing::new({ + let mut h = Sha256::new(); + h.update(pw); + h.update(salt); + h.update(udata); + h.finalize().to_vec() + }); + if r == 5 { + return k; + } + let mut round = 0u32; + loop { + let mut k1 = Zeroizing::new(Vec::with_capacity(64 * (pw.len() + k.len() + udata.len()))); + for _ in 0..64 { + k1.extend_from_slice(pw); + k1.extend_from_slice(&k); + k1.extend_from_slice(udata); + } + // `k` is always a SHA-2 digest (32, 48 or 64 bytes): key = its first 16 bytes, IV = the next 16. + let Some((key, rest)) = k.split_first_chunk::<16>() else { break }; + let Some(iv) = rest.first_chunk::<16>() else { break }; + let e = Zeroizing::new(aes_cbc_encrypt(key, iv, &k1, false)); + let m = e.iter().take(16).map(|b| u32::from(*b)).sum::() % 3; + k = Zeroizing::new(match m { + 0 => Sha256::digest(&*e).to_vec(), + 1 => Sha384::digest(&*e).to_vec(), + _ => Sha512::digest(&*e).to_vec(), + }); + round += 1; + if round >= 64 && e.last().is_none_or(|b| u32::from(*b) <= round - 32) { + break; + } + } + k.truncate(32); + k +} + +/// Algorithm 2.A: authenticate as owner (`owner = true`) or user; returns the file key. +fn key_r6(d: &EncryptDict, pw: &[u8], owner: bool) -> Result>>, CryptError> { + if d.u.len() < 48 || d.o.len() < 48 { + return Err(CryptError::Malformed("/O and /U must be 48 bytes for R5/R6".into())); + } + let (hashed, encrypted, udata) = if owner { (&d.o[..48], &d.oe, &d.u[..48]) } else { (&d.u[..48], &d.ue, &[][..]) }; + if *hash_r6(d.r, pw, &hashed[32..40], udata) != hashed[..32] { + return Ok(None); + } + if encrypted.len() < 32 { + return Err(CryptError::Malformed("/OE or /UE must be 32 bytes".into())); + } + let k = hash_r6(d.r, pw, &hashed[40..48], udata); + Ok(Some(Zeroizing::new(aes_cbc_decrypt(&k, &[0; 16], &encrypted[..32], false)))) +} + +// ── Creating encryption ──────────────────────────────────────────────────────────────────────── + +/// Which encryption to create. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Algorithm { + /// R2, 40-bit RC4 (legacy; for tests and compatibility only). + Rc4_40, + /// R3, 128-bit RC4. + Rc4_128, + /// R4, AES-128 crypt filters. + Aes128, + /// R6, AES-256 (PDF 2.0; the default for new documents). + Aes256, +} + +/// A password as [`create`] prepares it: for R2–R4 its PDFDocEncoding (characters it lacks +/// dropped), at most 32 bytes; for R6 its SASLprep (RFC 4013: soft hyphens and the like removed, +/// other spaces made plain, then NFKC; the password as given if SASLprep refuses it), in UTF-8, +/// at most 127 bytes. +fn prepared_password(algorithm: Algorithm, password: &str) -> Zeroizing> { + match algorithm { + Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => legacy_password(password), + Algorithm::Aes256 => sasl_password(password), + } +} + +/// The bytes new encryption with `algorithm` derives its keys from for `password`: for R2–R4 its +/// PDFDocEncoding cut or padded to exactly 32 bytes (Algorithm 2); for R6 its SASLprep (the +/// password as given if SASLprep refuses it), at most 127 bytes of UTF-8. Two passwords with the +/// same bytes are the same password to the file, and a password with the bytes of `""` opens it +/// without asking, so callers can refuse either before encrypting. Different bytes can still give +/// the same key (RC4-40 keeps 5 bytes of a hash). `create` still uses the user password for an +/// empty R2–R4 owner password. +pub fn effective_password(algorithm: Algorithm, password: &str) -> Vec { + let prepared = prepared_password(algorithm, password); + match algorithm { + Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => pad(&prepared).to_vec(), + Algorithm::Aes256 => prepared.to_vec(), + } +} + +/// Parameters for new encryption. +#[derive(Clone, Debug)] +pub struct NewEncryption<'a> { + pub algorithm: Algorithm, + pub user_password: &'a str, + pub owner_password: &'a str, + pub permissions: i32, + pub encrypt_metadata: bool, + /// 32 bytes of entropy for keys and salts (the caller supplies randomness, so this crate + /// needs no OS access and tests are deterministic). + pub seed: [u8; 32], +} + +/// Expand `seed` into `n` bytes (SHA-256 in counter mode). +fn expand(seed: &[u8; 32], label: &[u8], n: usize) -> Vec { + let mut out = Vec::new(); + let mut counter = 0u32; + while out.len() < n { + let mut h = Sha256::new(); + h.update(seed); + h.update(label); + h.update(counter.to_le_bytes()); + out.extend_from_slice(&h.finalize()); + counter = counter.saturating_add(1); + } + out.truncate(n); + out +} + +/// Sixteen bytes of file identifier from `seed` (SHA-256 in counter mode, label `id`). +/// +/// `/ID` is written in the clear. Deriving it the same way as the file key keeps that public +/// value from being an unkeyed hash of the seed. +pub fn file_id(seed: &[u8; 32]) -> Vec { + expand(seed, b"id", 16) +} + +/// Build the `/Encrypt` values and an authenticated (owner) handler for a new encryption. +pub fn create(params: &NewEncryption, id0: &[u8]) -> Result { + // Bits 7–8 and 13–32 must be 1; bits 1–2 must be 0 (§7.6.4.2). + let p = (params.permissions | !0xFFF | 0xC0) & !0x3; + let mut d = EncryptDict { filter: b"Standard".to_vec(), p, encrypt_metadata: params.encrypt_metadata, ..Default::default() }; + let user = prepared_password(params.algorithm, params.user_password); + let owner = prepared_password(params.algorithm, params.owner_password); + match params.algorithm { + Algorithm::Rc4_40 | Algorithm::Rc4_128 | Algorithm::Aes128 => { + (d.v, d.r, d.length_bits) = match params.algorithm { + Algorithm::Rc4_40 => (1, 2, 40), + Algorithm::Rc4_128 => (2, 3, 128), + _ => (4, 4, 128), + }; + if params.algorithm == Algorithm::Aes128 { + d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)]; + d.stm_f = b"StdCF".to_vec(); + d.str_f = b"StdCF".to_vec(); + } else { + d.encrypt_metadata = true; // V < 4 always encrypts metadata + } + let owner = if params.owner_password.is_empty() { user.clone() } else { owner }; + // Algorithm 3: /O + let rk = owner_rc4_key(&d, &owner); + let mut o = rc4(&rk, &*pad(&user)); + if d.r >= 3 { + for i in 1..=19u8 { + let k = Zeroizing::new(rk.iter().map(|b| b ^ i).collect::>()); + o = rc4(&k, &o); + } + } + d.o = o; + let key = file_key_r4(&d, id0, &user); + d.u = u_value_r4(&d, id0, &key); + Ok(SecurityHandler::new(d, key, Auth::Owner, Some(user))) + } + Algorithm::Aes256 => { + (d.v, d.r, d.length_bits) = (5, 6, 256); + d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes256)]; + d.stm_f = b"StdCF".to_vec(); + d.str_f = b"StdCF".to_vec(); + let key = Zeroizing::new(expand(¶ms.seed, b"file key", 32)); + let salts = expand(¶ms.seed, b"salts", 32); + // Algorithm 8: /U, /UE + let mut u = hash_r6(6, &user, &salts[0..8], &[]).to_vec(); + u.extend_from_slice(&salts[0..16]); + let uk = hash_r6(6, &user, &salts[8..16], &[]); + d.ue = aes_cbc_encrypt(&uk, &[0; 16], &key, false); + // Algorithm 9: /O, /OE + let mut o = hash_r6(6, &owner, &salts[16..24], &u).to_vec(); + o.extend_from_slice(&salts[16..32]); + let ok = hash_r6(6, &owner, &salts[24..32], &u); + d.oe = aes_cbc_encrypt(&ok, &[0; 16], &key, false); + d.u = u; + d.o = o; + // Algorithm 10: /Perms + let mut perms = [0u8; 16]; + perms[..4].copy_from_slice(&p.to_le_bytes()); + perms[4..8].copy_from_slice(&[0xFF; 4]); + perms[8] = if d.encrypt_metadata { b'T' } else { b'F' }; + perms[9..12].copy_from_slice(b"adb"); + perms[12..16].copy_from_slice(&expand(¶ms.seed, b"perms", 4)); + d.perms = aes_cbc_encrypt(&key, &[0; 16], &perms, false); + Ok(SecurityHandler::new(d, key, Auth::Owner, None)) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/pdf/src/crypt/aes_cbc.rs b/crates/pdf/src/crypt/aes_cbc.rs new file mode 100644 index 000000000..5d958f4f3 --- /dev/null +++ b/crates/pdf/src/crypt/aes_cbc.rs @@ -0,0 +1,89 @@ +//! AES-CBC (128 or 256-bit key, chosen by key length) with optional PKCS#5 padding. +//! +//! Decryption is tolerant, as viewers are: a trailing partial block is ignored, and invalid +//! padding leaves the data unpadded rather than failing. + +use aes::cipher::{Array, BlockCipherDecrypt, BlockCipherEncrypt, KeyInit}; +use aes::{Aes128, Aes256}; + +enum Cipher { + A128(Box), + A256(Box), +} + +impl Cipher { + fn new(key: &[u8]) -> Self { + if let Some(k) = key.first_chunk::<32>() { + Cipher::A256(Box::new(Aes256::new(&Array::from(*k)))) + } else { + let mut k = [0u8; 16]; + let n = key.len().min(16); + k[..n].copy_from_slice(&key[..n]); + Cipher::A128(Box::new(Aes128::new(&Array::from(k)))) + } + } + + fn encrypt(&self, block: &mut [u8; 16]) { + let mut b = Array::from(*block); + match self { + Cipher::A128(c) => c.encrypt_block(&mut b), + Cipher::A256(c) => c.encrypt_block(&mut b), + } + *block = b.into(); + } + + fn decrypt(&self, block: &mut [u8; 16]) { + let mut b = Array::from(*block); + match self { + Cipher::A128(c) => c.decrypt_block(&mut b), + Cipher::A256(c) => c.decrypt_block(&mut b), + } + *block = b.into(); + } +} + +/// Encrypt; with `pad`, PKCS#5 padding is added (otherwise `data` must be whole blocks, and a +/// trailing partial block is zero-padded). +pub fn aes_cbc_encrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec { + let c = Cipher::new(key); + let mut buf = data.to_vec(); + if pad { + let n = 16 - buf.len() % 16; + buf.extend(std::iter::repeat_n(n as u8, n)); + } else if !buf.len().is_multiple_of(16) { + buf.resize(buf.len().div_ceil(16) * 16, 0); + } + let mut prev = *iv; + for b in buf.as_chunks_mut::<16>().0 { + for (x, p) in b.iter_mut().zip(prev) { + *x ^= p; + } + c.encrypt(b); + prev = *b; + } + buf +} + +/// Decrypt; with `pad`, PKCS#5 padding is removed when valid. +pub fn aes_cbc_decrypt(key: &[u8], iv: &[u8; 16], data: &[u8], pad: bool) -> Vec { + let c = Cipher::new(key); + let blocks = data.as_chunks::<16>().0; + let mut out = Vec::with_capacity(blocks.len() * 16); + let mut prev = *iv; + for &cipher in blocks { + let mut b = cipher; + c.decrypt(&mut b); + for (x, p) in b.iter_mut().zip(prev) { + *x ^= p; + } + out.extend_from_slice(&b); + prev = cipher; + } + if pad && let Some(&n) = out.last() { + let n = n as usize; + if (1..=16).contains(&n) && n <= out.len() && out[out.len() - n..].iter().all(|b| *b as usize == n) { + out.truncate(out.len() - n); + } + } + out +} diff --git a/crates/pdf/src/crypt/rc4.rs b/crates/pdf/src/crypt/rc4.rs new file mode 100644 index 000000000..952a8394d --- /dev/null +++ b/crates/pdf/src/crypt/rc4.rs @@ -0,0 +1,22 @@ +//! RC4 (used by PDF security handler revisions 2–4). Encryption and decryption are the same. + +pub fn rc4(key: &[u8], data: &[u8]) -> Vec { + if key.is_empty() { + return data.to_vec(); + } + let mut s: [u8; 256] = std::array::from_fn(|i| i as u8); + let mut j: u8 = 0; + for i in 0..256 { + j = j.wrapping_add(s[i]).wrapping_add(key[i % key.len()]); + s.swap(i, j as usize); + } + let (mut i, mut j) = (0u8, 0u8); + data.iter() + .map(|b| { + i = i.wrapping_add(1); + j = j.wrapping_add(s[i as usize]); + s.swap(i as usize, j as usize); + b ^ s[s[i as usize].wrapping_add(s[j as usize]) as usize] + }) + .collect() +} diff --git a/crates/pdf/src/crypt/tests.rs b/crates/pdf/src/crypt/tests.rs new file mode 100644 index 000000000..46619427a --- /dev/null +++ b/crates/pdf/src/crypt/tests.rs @@ -0,0 +1,449 @@ +use super::*; + +fn hex(s: &str) -> Vec { + (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() +} + +#[test] +fn file_id_is_a_one_way_16_byte_identifier() { + let seed = [7u8; 32]; + let id = file_id(&seed); + assert_eq!(id.len(), 16); + assert_eq!(id, file_id(&seed)); + assert_ne!(id, seed[..16]); + assert_ne!(id, file_id(&[8u8; 32])); +} + +#[test] +fn rc4_reference_vectors() { + assert_eq!(rc4(b"Key", b"Plaintext"), hex("bbf316e8d940af0ad3")); + assert_eq!(rc4(b"Wiki", b"pedia"), hex("1021bf0420")); + assert_eq!(rc4(b"Secret", b"Attack at dawn"), hex("45a01f645fc35b383552544b9bf5")); +} + +#[test] +fn aes_cbc_nist_sp800_38a_vectors() { + let iv: [u8; 16] = hex("000102030405060708090a0b0c0d0e0f").try_into().unwrap(); + let plain = hex("6bc1bee22e409f96e93d7e117393172aae2d8a571e03ac9c9eb76fac45af8e51"); + let k128 = hex("2b7e151628aed2a6abf7158809cf4f3c"); + let c128 = aes_cbc_encrypt(&k128, &iv, &plain, false); + assert_eq!(c128, hex("7649abac8119b246cee98e9b12e9197d5086cb9b507219ee95db113a917678b2")); + assert_eq!(aes_cbc_decrypt(&k128, &iv, &c128, false), plain); + let k256 = hex("603deb1015ca71be2b73aef0857d77811f352c073b6108d72d9810a30914dff4"); + let c256 = aes_cbc_encrypt(&k256, &iv, &plain, false); + assert_eq!(c256, hex("f58c4c04d6e5f1ba779eabfb5f7bfbd69cfc4e967edb808d679f777bc6702c7d")); + assert_eq!(aes_cbc_decrypt(&k256, &iv, &c256, false), plain); +} + +#[test] +fn aes_padding_is_added_and_tolerantly_removed() { + let key = [7u8; 16]; + let iv = [1u8; 16]; + for n in [0, 1, 15, 16, 17, 100] { + let data: Vec = (0..n as u8).collect(); + let c = aes_cbc_encrypt(&key, &iv, &data, true); + assert_eq!(c.len() % 16, 0); + assert_eq!(aes_cbc_decrypt(&key, &iv, &c, true), data); + } + // Garbage (bad padding, partial block) decrypts without panicking. + let _ = aes_cbc_decrypt(&key, &iv, &[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17], true); +} + +const ID: &[u8] = b"0123456789abcdef"; + +fn new(alg: Algorithm, user: &str, owner: &str, perms: i32) -> SecurityHandler { + create( + &NewEncryption { algorithm: alg, user_password: user, owner_password: owner, permissions: perms, encrypt_metadata: true, seed: [42; 32] }, + ID, + ) + .unwrap() +} + +const ALL: [Algorithm; 4] = [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256]; + +#[test] +fn created_encryption_authenticates_owner_and_user() { + for alg in ALL { + let made = new(alg, "user pw", "owner pw", 0b0100); // print only + let d = made.dict().clone(); + let owner = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap(); + assert_eq!(owner.auth(), Auth::Owner, "{alg:?}"); + assert_eq!(owner.file_key(), made.file_key(), "{alg:?}"); + assert!(owner.permissions().modify(), "owners can do everything"); + let user = SecurityHandler::open(d.clone(), ID, Some("user pw")).unwrap(); + assert_eq!(user.auth(), Auth::User, "{alg:?}"); + assert_eq!(user.file_key(), made.file_key(), "{alg:?}"); + assert!(user.permissions().print() && !user.permissions().modify() && !user.permissions().copy(), "{alg:?}"); + assert_eq!(SecurityHandler::open(d.clone(), ID, Some("nope")).unwrap_err(), CryptError::WrongPassword, "{alg:?}"); + assert_eq!(SecurityHandler::open(d, ID, None).unwrap_err(), CryptError::WrongPassword, "{alg:?}"); + } +} + +#[test] +fn the_owner_password_upgrades_a_user_session() { + for alg in ALL { + let d = new(alg, "", "secret", 0b0100).dict().clone(); + let user = SecurityHandler::open(d, ID, None).unwrap(); + assert!(!user.permissions().modify(), "{alg:?}"); + // The (empty) user password and wrong ones don't unlock anything. + assert_eq!(user.authenticate_owner(ID, "").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); + assert_eq!(user.authenticate_owner(ID, "nope").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); + let owner = user.authenticate_owner(ID, "secret").unwrap(); + assert_eq!(owner.auth(), Auth::Owner, "{alg:?}"); + assert_eq!(owner.file_key(), user.file_key(), "{alg:?}"); + assert!(owner.permissions().modify(), "{alg:?}"); + } + // A separate open password is not the permissions password. + for alg in ALL { + let d = new(alg, "open", "perms", 0b0100).dict().clone(); + let user = SecurityHandler::open(d, ID, Some("open")).unwrap(); + assert_eq!(user.authenticate_owner(ID, "open").unwrap_err(), CryptError::WrongPassword, "{alg:?}"); + assert_eq!(user.authenticate_owner(ID, "perms").unwrap().auth(), Auth::Owner, "{alg:?}"); + } +} + +#[test] +fn empty_user_password_opens_without_asking() { + for alg in ALL { + let d = new(alg, "", "secret", -1).dict().clone(); + let h = SecurityHandler::open(d, ID, None).unwrap(); + assert_eq!(h.auth(), Auth::User, "{alg:?}"); + } +} + +#[test] +fn strings_and_streams_round_trip_and_are_scrambled() { + for alg in ALL { + let h = new(alg, "", "o", -1); + let text = b"Confidential quarterly numbers".to_vec(); + let enc = h.encrypt_string(12, 0, &text); + assert_ne!(enc, text, "{alg:?}"); + assert_eq!(h.decrypt_string(12, 0, &enc), text, "{alg:?}"); + if alg != Algorithm::Aes256 { + // R2–R4 derive a key per object; R6 uses the file key everywhere (§7.6.3.3). + assert_ne!(h.decrypt_string(13, 0, &enc), text, "object keys differ per object ({alg:?})"); + } + let data: Vec = (0..1000u32).map(|i| (i * 7) as u8).collect(); + let enc = h.encrypt_stream(5, 0, &data, StreamKind::Normal); + assert_eq!(h.decrypt_stream(5, 0, &enc, StreamKind::Normal), data, "{alg:?}"); + // Deterministic: the same object encrypts to the same bytes (reproducible saves). + assert_eq!(h.encrypt_stream(5, 0, &data, StreamKind::Normal), enc); + } +} + +thread_local! { + /// Crypt filter names compared or looked up, on this thread. + pub(crate) static FILTER_COMPARES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +#[test] +fn crypt_filters_are_looked_up_once_per_handler() { + let plain = new(Algorithm::Aes128, "", "o", -1); + let mut d = plain.dict().clone(); + let n = 5_000; + let mut filters: Vec<_> = (0..n).map(|i| (format!("F{i}").into_bytes(), Method::Identity)).collect(); + filters.append(&mut d.crypt_filters); + d.crypt_filters = filters; + FILTER_COMPARES.with(|c| c.set(0)); + let h = SecurityHandler::open(d, ID, None).unwrap(); + let strings = 1_000; + let enc = plain.encrypt_string(7, 0, b"secret"); + for _ in 0..strings { + assert_eq!(h.decrypt_string(7, 0, &enc), b"secret"); + assert_eq!(h.decrypt_stream(7, 0, &plain.encrypt_stream(7, 0, b"s", StreamKind::Normal), StreamKind::Normal), b"s"); + } + assert_eq!(h.decrypt_stream(7, 0, b"x", StreamKind::Named(b"F3")), b"x"); + let compares = FILTER_COMPARES.with(|c| c.get()); + assert!(compares <= 2 * n + 3 * strings, "{compares} filter name comparisons"); +} + +#[test] +fn identity_crypt_filters_leave_data_alone() { + let mut d = new(Algorithm::Aes256, "", "o", -1).dict().clone(); + // Embedded-file-only encryption: strings and normal streams are not encrypted. + d.str_f = b"Identity".to_vec(); + d.stm_f = b"Identity".to_vec(); + d.ef_f = b"StdCF".to_vec(); + let h = SecurityHandler::open(d, ID, None).unwrap(); + assert!(!h.encrypts_strings()); + assert_eq!(h.decrypt_stream(3, 0, b"plain", StreamKind::Normal), b"plain"); + let enc = h.encrypt_stream(3, 0, b"attachment", StreamKind::EmbeddedFile); + assert_ne!(enc, b"attachment"); + assert_eq!(h.decrypt_stream(3, 0, &enc, StreamKind::EmbeddedFile), b"attachment"); + assert_eq!(h.decrypt_stream(3, 0, b"x", StreamKind::Named(b"Identity")), b"x"); +} + +#[test] +fn r6_passwords_are_saslprepped() { + // U+00AD (soft hyphen) is mapped to nothing by SASLprep; "ª" normalises to "a" (NFKC). + let d = new(Algorithm::Aes256, "SªSL\u{AD}prep", "o", -1).dict().clone(); + assert!(SecurityHandler::open(d.clone(), ID, Some("SaSLprep")).is_ok()); + assert!(SecurityHandler::open(d, ID, Some("SaSL-prep")).is_err()); +} + +#[test] +fn effective_passwords_are_the_bytes_the_keys_are_made_from() { + // AES-256: SASLprep drops soft hyphens, makes other spaces plain and folds compatibility + // characters (NFKC); at most 127 bytes of UTF-8. + let r6 = |pw: &str| effective_password(Algorithm::Aes256, pw); + assert_eq!(r6("\u{AD}"), r6(""), "a soft hyphen alone is no password"); + assert_eq!(r6("\u{FF21}"), r6("A"), "fullwidth A"); + assert_eq!(r6("\u{FB01}"), r6("fi"), "the fi ligature"); + assert_eq!(r6("a\u{3000}b"), b"a b", "an ideographic space"); + assert_ne!(r6("A"), r6("a"), "case matters"); + assert_eq!(r6(&"é".repeat(100)).len(), 127); + // RC4 and AES-128: PDFDocEncoding, padded or cut to exactly 32 bytes. + let (a31, a32) = ("a".repeat(31), "a".repeat(32)); + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { + assert_eq!(effective_password(alg, &format!("{a32}1")), a32.as_bytes(), "{alg:?}"); + assert_eq!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("1{a32}2")), "{alg:?}"); + assert_ne!(effective_password(alg, &format!("1{a32}")), effective_password(alg, &format!("2{a32}")), "{alg:?}"); + assert_eq!(effective_password(alg, &a31), effective_password(alg, &format!("{a31}(")), "{alg:?}: the padding starts with '('"); + assert_eq!(effective_password(alg, ""), PADDING, "{alg:?}"); + assert_eq!(effective_password(alg, "ılık€")[..5], [0x9A, b'l', 0x9A, b'k', 0xA0], "{alg:?}"); + } +} + +#[test] +fn passwords_with_the_same_effective_bytes_are_one_password() { + // A file protected with "\u{FF21}" (fullwidth A) opens with "A". + let d = new(Algorithm::Aes256, "\u{FF21}", "owner", 0b0100).dict().clone(); + assert_eq!(SecurityHandler::open(d.clone(), ID, Some("A")).unwrap().auth(), Auth::User); + assert_eq!(SecurityHandler::open(d, ID, Some("a")).unwrap_err(), CryptError::WrongPassword); + // A soft hyphen is no open password at all. + let d = new(Algorithm::Aes256, "\u{AD}", "owner", 0b0100).dict().clone(); + assert_eq!(SecurityHandler::open(d, ID, None).unwrap().auth(), Auth::User); + // Open and permissions passwords with the same bytes: the open password gives owner rights. + let d = new(Algorithm::Aes256, "\u{FB01}", "fi", 0b0100).dict().clone(); + assert_eq!(effective_password(Algorithm::Aes256, "\u{FB01}"), effective_password(Algorithm::Aes256, "fi")); + assert_eq!(SecurityHandler::open(d, ID, Some("\u{FB01}")).unwrap().auth(), Auth::Owner); + let a31 = "a".repeat(31); + let pairs = [(format!("{a31}a1"), format!("{a31}a2")), (a31.clone(), format!("{a31}("))]; + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { + for (open, perms) in &pairs { + assert_eq!(effective_password(alg, open), effective_password(alg, perms), "{alg:?}"); + let d = new(alg, open, perms, 0b0100).dict().clone(); + assert_eq!(SecurityHandler::open(d, ID, Some(open)).unwrap().auth(), Auth::Owner, "{alg:?} {open:?}"); + } + } +} + +#[test] +fn unsupported_handlers_are_reported() { + let mut d = new(Algorithm::Aes128, "", "o", -1).dict().clone(); + d.filter = b"Adobe.PubSec".to_vec(); + assert!(matches!(SecurityHandler::open(d.clone(), ID, None), Err(CryptError::Unsupported(_)))); + d.filter = b"Standard".to_vec(); + d.r = 7; + assert!(matches!(SecurityHandler::open(d, ID, None), Err(CryptError::Unsupported(_)))); +} + +#[test] +fn permission_bits_follow_table_22() { + let p = Permissions { bits: -3904, owner: false, revision: 3 }; // 0xFFFFF0C0: only reserved bits set + assert!(!p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.assemble()); + let p = Permissions { bits: -1, owner: false, revision: 3 }; + assert!(p.print() && p.modify() && p.copy() && p.annotate() && p.assemble() && p.fill_forms()); + let p = Permissions { bits: 0, owner: true, revision: 3 }; + assert!(p.modify(), "owners ignore /P"); +} + +/// Page extraction (#5081) is assembly as the revision reads it, and copying (bit 5). +#[test] +fn page_extraction_needs_assembly_and_copying() { + let bit = |n: u32| 1i32 << (n - 1); + let user = |bits: i32, revision: i64| Permissions { bits, owner: false, revision }; + // "Any except extracting pages" (bits 4, 6, 9; copying off) at revision 3 and later. + assert!(!user(bit(4) | bit(6) | bit(9), 4).extract_pages()); + // Pages may change in place, but copying is withheld. + assert!(user(bit(11), 6).assemble() && !user(bit(11), 6).extract_pages()); + assert!(user(bit(11) | bit(5), 6).extract_pages()); + // Revision 2: bit 4 covers assembly, and its reserved bit 11 grants nothing. + assert!(user(bit(4), 2).assemble() && !user(bit(4) | bit(11), 2).extract_pages(), "copying withheld"); + assert!(user(bit(4) | bit(5), 2).extract_pages()); + assert!(!user(bit(5) | bit(11), 2).extract_pages(), "no assembly without bit 4 at revision 2"); + assert!(Permissions { bits: 0, owner: true, revision: 2 }.extract_pages(), "owners ignore /P"); +} + +#[test] +fn perms_limits_what_an_edited_p_grants() { + let made = new(Algorithm::Aes256, "user pw", "owner pw", 0b0100); // print only + let open = |d: &EncryptDict, pw| SecurityHandler::open(d.clone(), ID, Some(pw)).unwrap().permissions(); + let mut d = made.dict().clone(); + assert_eq!(open(&d, "user pw").bits, d.p, "/P and /Perms agree"); + d.p = -4; // /P edited to allow everything, /Perms as written + let p = open(&d, "user pw"); + assert!(p.print() && !p.modify() && !p.copy() && !p.annotate() && !p.fill_forms() && !p.assemble()); + assert!(open(&d, "owner pw").modify(), "owners ignore /P"); + d.p = -3904; // a stricter /P still applies + assert!(!open(&d, "user pw").print()); + // No /Perms, or one that does not decrypt to an `adb` block: /P as before. + for perms in [Vec::new(), vec![0; 16], made.dict().perms[..15].to_vec()] { + let d = EncryptDict { p: -4, perms, ..made.dict().clone() }; + assert!(open(&d, "user pw").modify()); + } + // R2 to R4 have no /Perms: /P is part of their file key, so an edited /P does not open. + let d = EncryptDict { p: -4, ..new(Algorithm::Aes128, "user pw", "owner pw", 0b0100).dict().clone() }; + assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap_err(), CryptError::WrongPassword); +} + +// Not built for the web: proptest is a dev-dependency of the other targets only. +#[cfg(not(target_arch = "wasm32"))] +proptest::proptest! { + #[test] + fn malformed_dictionaries_never_panic(o in proptest::collection::vec(proptest::num::u8::ANY, 0..60), + u in proptest::collection::vec(proptest::num::u8::ANY, 0..60), + r in 0i64..9, len in -10i64..300) { + let d = EncryptDict { filter: b"Standard".to_vec(), v: 4, r, length_bits: len, o, u, p: -1, encrypt_metadata: true, ..Default::default() }; + let _ = SecurityHandler::open(d, ID, Some("x")); + } +} + +/// A `/V 4` RC4 crypt filter with a 40- or 48-bit key, as Acrobat writes it (pdf.js issue 19484): +/// the file key is extended to 16 bytes with zeros before Algorithm 1, so each object's key is +/// all 16 bytes of MD5(key, zeros, object, generation). Everything else keeps Algorithm 1 as the +/// standard has it, n + 5 bytes (at most 16) of MD5(key, object, generation[, "sAlT"]): a `/V 2` +/// file with the same short key, a full 128-bit `/V 4` RC4 key, and AESV2, short (88 bits) or +/// full. Each object key is computed here by hand; data written with it must decrypt. +#[test] +fn a_v4_rc4_key_shorter_than_16_bytes_is_extended_with_zeros() { + let text = b"Page tree in an object stream".to_vec(); + let cases = + [(4, 40, Method::Rc4), (4, 48, Method::Rc4), (2, 40, Method::Rc4), (4, 128, Method::Rc4), (4, 88, Method::Aes128), (4, 128, Method::Aes128)]; + for (v, bits, method) in cases { + let mut d = EncryptDict { + filter: b"Standard".to_vec(), + v, + r: if v == 4 { 4 } else { 3 }, + length_bits: bits, + o: vec![7; 32], + p: -3392, + encrypt_metadata: v != 4, + ..Default::default() + }; + if v == 4 { + d.crypt_filters = vec![(b"StdCF".to_vec(), method)]; + d.stm_f = b"StdCF".to_vec(); + d.str_f = b"StdCF".to_vec(); + } + let key = file_key_r4(&d, ID, b""); + assert_eq!(key.len(), (bits / 8) as usize); + d.u = u_value_r4(&d, ID, &key); + let h = SecurityHandler::open(d, ID, None).unwrap(); + let (num, generation) = (13u32, 0u16); + let extended = v == 4 && method == Method::Rc4 && key.len() < 16; + let mut md5 = Md5::new(); + md5.update(&key); + if extended { + md5.update(vec![0u8; 16 - key.len()]); + } + md5.update(&num.to_le_bytes()[..3]); + md5.update(generation.to_le_bytes()); + if method == Method::Aes128 { + md5.update(b"sAlT"); + } + let digest = md5.finalize(); + let n = if extended { 16 } else { key.len() }; + let object_key = digest[..(n + 5).min(16)].to_vec(); + let case = format!("V{v}, {bits}-bit key, {method:?}"); + let written = if method == Method::Rc4 { + rc4(&object_key, &text) + } else { + let iv = [3u8; 16]; + [iv.to_vec(), aes_cbc_encrypt(&object_key, &iv, &text, true)].concat() + }; + assert_eq!(h.decrypt_string(num, generation, &written), text, "{case}"); + assert_eq!(h.decrypt_stream(num, generation, &written, StreamKind::Normal), text, "{case}"); + let saved = h.encrypt_stream(num, generation, &text, StreamKind::Normal); + if method == Method::Rc4 { + assert_eq!(saved, written, "saved as it was read: {case}"); + } + assert_eq!(h.decrypt_stream(num, generation, &saved, StreamKind::Normal), text, "{case}"); + } +} + +#[test] +fn owner_authentication_recovers_the_user_password() { + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { + let d = new(alg, "user pw", "owner pw", -1).dict().clone(); + let h = SecurityHandler::open(d.clone(), ID, Some("owner pw")).unwrap(); + let _: &Zeroizing> = &h.key; + let _: &Option>> = &h.recovered_user; + assert_eq!(h.recovered_user_password().as_deref().map(Vec::as_slice), Some(&b"user pw"[..]), "{alg:?}"); + assert_eq!(SecurityHandler::open(d, ID, Some("user pw")).unwrap().recovered_user_password(), None); + } + let d = new(Algorithm::Rc4_128, "", "owner", -1).dict().clone(); + assert_eq!(SecurityHandler::open(d, ID, Some("owner")).unwrap().recovered_user_password().as_deref().map(Vec::as_slice), Some(&b""[..])); +} + +/// /O, /U and the first /ID of files qpdf 12.4.2 wrote (`--static-id`), with the file keys +/// `qpdf --show-encryption-key` reports. +fn qpdf_file(r: i64, o: &str, u: &str) -> EncryptDict { + let mut d = EncryptDict { + filter: b"Standard".to_vec(), + v: if r == 4 { 4 } else { 2 }, + r, + length_bits: 128, + o: hex(o), + u: hex(u), + p: -4, + encrypt_metadata: true, + ..Default::default() + }; + if r == 4 { + d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)]; + d.stm_f = b"StdCF".to_vec(); + d.str_f = b"StdCF".to_vec(); + } + d +} + +const QPDF_ID: &str = "56b57383aafa8314f3f98222995f5604"; + +#[test] +fn legacy_passwords_are_pdfdoc_encoded_with_a_utf8_fallback() { + let id = hex(QPDF_ID); + // R3 RC4: user "ılık€" in PDFDocEncoding (ı = 0x9A, € = 0xA0); owner "Öğretmen", which + // PDFDocEncoding can't hold, so qpdf wrote its UTF-8 bytes. + let r3 = qpdf_file( + 3, + "d519aa8120ce8c54812b45e5391916fed167b1c84961d90c4ee4c182ad83534b", + "4985c03ddfa3eeb8a90c343512913ec40021446990b9e4114071a4d9104984c1", + ); + let user = SecurityHandler::open(r3.clone(), &id, Some("ılık€")).unwrap(); + assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("3564ffcea186122595ad55ee157da71f"))); + let owner = SecurityHandler::open(r3.clone(), &id, Some("Öğretmen")).unwrap(); + assert_eq!((owner.auth(), owner.file_key().to_vec()), (Auth::Owner, hex("3564ffcea186122595ad55ee157da71f"))); + assert_eq!(owner.recovered_user_password(), Some(Zeroizing::new(vec![0x9A, b'l', 0x9A, b'k', 0xA0]))); + // Dropping ı and € (the old Latin-1 conversion) or a near miss doesn't open it. + for wrong in ["lk", "ilik€", "ılık", "Ogretmen"] { + assert_eq!(SecurityHandler::open(r3.clone(), &id, Some(wrong)).unwrap_err(), CryptError::WrongPassword, "{wrong}"); + } + // R4 AES-128: user "şifre" (UTF-8 bytes), owner "owner". + let r4 = qpdf_file( + 4, + "bb4f8f4bd2236569e765906caf64e4429a4c20d6e996fdef963e9b5080f9e083", + "4a965fb91882772cd09ce8bbff2c33270021446990b9e4114071a4d9104984c1", + ); + let user = SecurityHandler::open(r4.clone(), &id, Some("şifre")).unwrap(); + assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("9f19d465cfd8e525d32aa0bdf10acd06"))); + assert_eq!(SecurityHandler::open(r4.clone(), &id, Some("owner")).unwrap().auth(), Auth::Owner); + assert_eq!(SecurityHandler::open(r4, &id, Some("ifre")).unwrap_err(), CryptError::WrongPassword); +} + +#[test] +fn pdfdoc_encoding_round_trips() { + for b in 0..=255u8 { + if b != 0x9F { + assert_eq!(pdfdoc_byte(pdfdoc_char(b)), Some(b), "{b:#04x}"); + } + } + assert_eq!(pdfdoc_byte('ı'), Some(0x9A)); + assert_eq!(pdfdoc_byte('€'), Some(0xA0)); + assert_eq!(pdfdoc_byte('ç'), Some(0xE7)); + for c in ['ğ', 'ş', 'İ', '\u{A0}', '\u{85}', 'Ж', '日'] { + assert_eq!(pdfdoc_byte(c), None, "{c}"); + } + assert_eq!(legacy_candidates("Çok"), [Zeroizing::new(vec![0xC7, b'o', b'k'])]); + assert_eq!(legacy_candidates("şifre"), [Zeroizing::new(b"ifre".to_vec()), Zeroizing::new("şifre".as_bytes().to_vec())]); +} diff --git a/crates/pdf/src/lib.rs b/crates/pdf/src/lib.rs index 9814c469c..965673fab 100644 --- a/crates/pdf/src/lib.rs +++ b/crates/pdf/src/lib.rs @@ -4,6 +4,8 @@ //! Advanced backend re-exports are coupled to the pinned backend, not a universal document model. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(feature = "crypt")] +pub mod crypt; #[cfg(feature = "interpret")] pub use hayro_interpret as interpret; mod link; diff --git a/crates/pdf/tests/crypt/main.rs b/crates/pdf/tests/crypt/main.rs new file mode 100644 index 000000000..702a0df7b --- /dev/null +++ b/crates/pdf/tests/crypt/main.rs @@ -0,0 +1,7 @@ +//! Tests of the `crypt` feature, linked into one test binary. Not built for the web: proptest +//! does not build there. +#![cfg(all(feature = "crypt", not(target_arch = "wasm32")))] + +mod permissions_revision; +mod proptests; +mod qpdf_literals; diff --git a/crates/pdf/tests/crypt/permissions_revision.rs b/crates/pdf/tests/crypt/permissions_revision.rs new file mode 100644 index 000000000..06a5a5b98 --- /dev/null +++ b/crates/pdf/tests/crypt/permissions_revision.rs @@ -0,0 +1,126 @@ +//! The permission accessors of `craft_pdf::crypt::Permissions`, checked against the flag table of +//! ISO 32000-1:2008 (Table 22). Expected values come from that table only. +//! +//! Bit n has the mask `1 << (n - 1)`. Readers use bits 3 to 6 and 9 to 12 and ignore the rest. + +use craft_pdf::crypt::{Algorithm, Auth, NewEncryption, Permissions, SecurityHandler, create}; + +const ID: &[u8] = b"0123456789abcdef"; +const T: bool = true; +const F: bool = false; + +/// The eight accessors in this order: print, high-quality print, modify, copy, accessibility, +/// annotate, fill forms, assemble. +fn cells(p: &Permissions) -> [bool; 8] { + [p.print(), p.print_high_quality(), p.modify(), p.copy(), p.extract_for_accessibility(), p.annotate(), p.fill_forms(), p.assemble()] +} + +/// Creates a file with `flags` at revision `revision`, opens it with the user password, and +/// returns the user's permissions. +fn user_view(alg: Algorithm, revision: i64, flags: i32) -> Permissions { + let made = create( + &NewEncryption { algorithm: alg, user_password: "user", owner_password: "owner", permissions: flags, encrypt_metadata: true, seed: [42; 32] }, + ID, + ) + .expect("the handler is created"); + assert_eq!(made.dict().r, revision, "the algorithm writes revision {revision}"); + let opened = SecurityHandler::open(made.dict().clone(), ID, Some("user")).expect("the user password opens the file"); + assert_eq!(opened.auth(), Auth::User, "the user password gives user access"); + opened.permissions() +} + +/// Revision 3 and later: each row is a flag word and the expected eight cells. +const REVISION_3_ROWS: &[(&str, i32, [bool; 8])] = &[ + ("every meaningful flag set, ignored flags clear", -196, [T, T, T, T, T, T, T, T]), + ("every flag set, ignored flags set too", -1, [T, T, T, T, T, T, T, T]), + ("reserved flags only", -4096, [F, F, F, F, F, F, F, F]), + ("ignored flags set, meaningful flags clear", -3901, [F, F, F, F, F, F, F, F]), + ("modify flag clear", -9, [T, T, F, T, T, T, T, T]), + ("copy flag clear", -17, [T, T, T, F, T, T, T, T]), + ("annotate flag clear", -33, [T, T, T, T, T, F, T, T]), + ("annotate and fill-existing flags clear", -289, [T, T, T, T, T, F, F, T]), + ("annotate flag set, fill-existing flag clear", -257, [T, T, T, T, T, T, T, T]), + ("accessibility flag clear", -513, [T, T, T, T, F, T, T, T]), + ("high-quality flag clear, print flag set", -2049, [T, F, T, T, T, T, T, T]), + ("bits 3 to 12 set, reserved bits clear", 4092, [T, T, T, T, T, T, T, T]), + ("positive flag word, bit 32 clear", 2_147_483_647, [T, T, T, T, T, T, T, T]), + ("revision 2 example read at revision 3", -44, [T, T, F, T, T, F, T, T]), +]; + +#[test] +fn revision_three_rows_hold_through_a_created_and_reopened_file() { + for (name, bits, want) in REVISION_3_ROWS { + assert_eq!(cells(&user_view(Algorithm::Rc4_128, 3, *bits)), *want, "{name}: flag word {bits:#010x}"); + } +} + +#[test] +fn owner_access_gives_every_flag() { + let made = create( + &NewEncryption { + algorithm: Algorithm::Rc4_128, + user_password: "user", + owner_password: "owner", + permissions: 0, + encrypt_metadata: true, + seed: [42; 32], + }, + ID, + ) + .expect("the handler is created"); + assert_eq!(made.auth(), Auth::Owner, "the owner password gives owner access"); + assert_eq!(cells(&made.permissions()), [T; 8]); +} + +#[test] +fn revision_two_print_modify_copy_and_annotate_follow_the_table() { + for (bits, want) in [(-4, [T, T, T, T]), (-44, [T, F, T, F]), (-17, [T, T, F, T])] { + let p = user_view(Algorithm::Rc4_40, 2, bits); + assert_eq!([p.print(), p.modify(), p.copy(), p.annotate()], want, "revision 2, flag word {bits}"); + } +} + +#[test] +fn revision_two_with_every_flag_set_grants_accessibility_and_form_filling() { + let p = user_view(Algorithm::Rc4_40, 2, -4); + assert!(p.extract_for_accessibility(), "bit 5 is set, and at revision 2 it covers accessibility"); + assert!(p.fill_forms(), "bit 6 is set"); +} + +#[test] +fn revision_three_assembly_needs_its_own_flag() { + let p = user_view(Algorithm::Rc4_128, 3, -1025); + assert!(p.modify(), "bit 4 is set"); + assert!(!p.assemble(), "assembly needs bit 11; bit 4 alone does not grant it"); +} + +#[test] +fn revision_three_high_quality_print_needs_the_print_flag() { + let p = user_view(Algorithm::Rc4_128, 3, -5); + assert!(!p.print(), "bit 3 is clear"); + assert!(!p.print_high_quality(), "high-quality print needs bit 3 and bit 12"); +} + +#[test] +fn revision_two_accessibility_follows_the_copy_flag() { + let p = user_view(Algorithm::Rc4_40, 2, -17); + assert!(!p.copy(), "bit 5 is clear"); + assert!(!p.extract_for_accessibility(), "at revision 2 accessibility is bit 5"); +} + +#[test] +fn revision_two_form_filling_ignores_bit_nine() { + let p = user_view(Algorithm::Rc4_40, 2, -44); + assert!(!p.annotate(), "bit 6 is clear"); + assert!(!p.fill_forms(), "at revision 2 form filling is bit 6 alone"); +} + +#[test] +fn revision_two_ignores_the_flags_only_later_revisions_define() { + // Bit 4 is clear and bit 11 set: bit 11 means nothing at revision 2, so assembly stays refused. + let p = user_view(Algorithm::Rc4_40, 2, -44); + assert!(!p.assemble(), "at revision 2 assembly is bit 4 alone"); + // Bit 3 is set and bit 12 clear: printing at revision 2 has one quality. + let p = user_view(Algorithm::Rc4_40, 2, -2049); + assert!(p.print_high_quality(), "at revision 2 high-quality print is bit 3 alone"); +} diff --git a/crates/pdf/tests/crypt/proptests.rs b/crates/pdf/tests/crypt/proptests.rs new file mode 100644 index 000000000..a81808ab9 --- /dev/null +++ b/crates/pdf/tests/crypt/proptests.rs @@ -0,0 +1,464 @@ +//! Property tests for what the security handler does to *content*, and to the ciphertext a +//! stranger's file hands back. +//! +//! `src/tests.rs` covers this handler carefully with fixed vectors — including the `/O`, `/U` and +//! file keys qpdf 12.4.2 produced for real files — plus one property over malformed dictionaries. +//! What it does not vary is the *payload*. Upstream's encrypted round trips run three shapes: a +//! 30-byte text, a 1000-byte ramp, and a 10-byte attachment in `identity_crypt_filters_leave_data_alone`. +//! The shapes nobody walks are the ones that break CBC and readers: empty content, one block minus +//! one, a ciphertext cut short by a damaged `/Length`, and keys of a length `create` never builds. +//! +//! - plaintext survives encryption at every length, for each algorithm and each crypt filter; +//! - truncated or mutated ciphertext produces bytes rather than a panic (apps +//! open files from strangers, and encrypted streams in the wild do get truncated); +//! - object keys are scoped to their object, and AES-256's documented exception still holds; +//! - a file opens for exactly the passwords whose effective bytes match, stated as a rule over +//! generated Unicode passwords instead of the three examples in `src/crypt/tests.rs` (the pdfcraft #1185 rule); +//! - the AES-CBC and RC4 primitives invert each other for any key length, since `/Length` and +//! crypt filters from a damaged file reach them with shapes `create` would never build. + +use craft_pdf::crypt::{ + Algorithm, Auth, CryptError, Method, NewEncryption, PADDING, SecurityHandler, StreamKind, aes_cbc_decrypt, aes_cbc_encrypt, create, + effective_password, pdfdoc_byte, rc4, +}; +use proptest::prelude::*; + +/// The first element of `/ID`, which every R2–R4 derivation hashes in (Algorithm 2). +const ID: &[u8] = b"\x01\x23\x45\x67\x89\xab\xcd\xef\x0f\xed\xcb\xa9\x87\x65\x43\x21"; + +fn algorithms() -> impl Strategy { + prop::sample::select(vec![Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256]) +} + +/// Content at the lengths where CBC padding goes wrong, plus anything in between. +fn payloads() -> impl Strategy> { + prop_oneof![ + Just(Vec::new()), + Just(vec![0x5cu8; 1]), + Just(vec![0xa7u8; 15]), + Just(vec![0x3fu8; 16]), + Just(vec![0x9du8; 17]), + Just(vec![0x10u8; 32]), + prop::collection::vec(any::(), 1..96), + ] +} + +/// Passwords from an alphabet the two encodings disagree about: SASLprep deletes U+00AD, folds +/// U+FF21 to "A", U+FB01 to "fi" and U+3000 to a space (revisions 5–6), while PDFDocEncoding has +/// no byte for 'ğ' or CJK at all, which sends revisions 2–4 down their UTF-8 fallback. +const PASSWORD_CHARS: [char; 12] = ['a', 'B', '7', ' ', '-', '\u{AD}', '\u{FF21}', '\u{FB01}', 'ı', 'ğ', '日', '\u{A0}']; + +fn passwords() -> impl Strategy { + prop::collection::vec(prop::sample::select(PASSWORD_CHARS.to_vec()), 0..12).prop_map(|chars| chars.into_iter().collect()) +} + +/// `bytes` cut to 32 and padded with the standard padding string (Algorithm 2, step a). +fn padded(bytes: &[u8]) -> Vec { + let mut out = PADDING; + let n = bytes.len().min(32); + out[..n].copy_from_slice(&bytes[..n]); + out.to_vec() +} + +/// Every byte string the handler may authenticate `pw` as. Revisions 2–4 use PDFDocEncoding, and +/// for a password holding characters it lacks they also try the raw UTF-8 bytes (qpdf writes such +/// passwords that way); revisions 5–6 use the SASLprep bytes. `effective_password` publishes the +/// first of these per algorithm, and it is what callers such as Protect check before encrypting. +fn candidates(algorithm: Algorithm, pw: &str) -> Vec> { + let mut out = vec![effective_password(algorithm, pw)]; + if algorithm != Algorithm::Aes256 && pw.chars().any(|c| pdfdoc_byte(c).is_none()) { + out.push(padded(&pw.as_bytes().iter().copied().take(32).collect::>())); + } + out +} + +/// What `create` keys `/O` from: the owner password, except on revisions 2–4 with no owner +/// password given, where it falls back to the user password's bytes. +fn owner_target(algorithm: Algorithm, user: &str, owner: &str) -> Vec { + if algorithm != Algorithm::Aes256 && owner.is_empty() { effective_password(algorithm, user) } else { effective_password(algorithm, owner) } +} + +/// Which password `pw` is for this file. Owners are tried before users, so bytes that match both +/// authenticate as owner — the order viewers use. `None` means it opens nothing. +fn expected_auth(algorithm: Algorithm, user_target: &[u8], owner_target: &[u8], pw: &str) -> Option { + let tried = candidates(algorithm, pw); + if tried.iter().any(|c| c == owner_target) { + Some(Auth::Owner) + } else if tried.iter().any(|c| c == user_target) { + Some(Auth::User) + } else { + None + } +} + +fn make(algorithm: Algorithm, user: &str, owner: &str) -> SecurityHandler { + create(&NewEncryption { algorithm, user_password: user, owner_password: owner, permissions: -1, encrypt_metadata: true, seed: [0x5e; 32] }, ID) + .unwrap() +} + +proptest! { + /// Encrypting then decrypting returns the input at every length, as a string and as each kind + /// of stream — and is byte-for-byte reproducible, because re-running a save must not silently + /// produce different bytes. Payloads of at least one block must come back changed: RC4 would + /// need an all-zero keystream to leave them alone, and CBC cannot shrink a padded block. + #[test] + fn content_survives_the_trip_at_every_length(alg in algorithms(), data in payloads()) { + let h = make(alg, "user", "owner"); + + // The key width each algorithm promises: 5 bytes for export-grade 40-bit RC4, 16 for + // AES-128 and RC4-128, 32 for AES-256. `key_len` is the only place that decides this and + // nothing in either suite reads it back, so `if d.r == 2 { 5 }` becoming `{ 4 }` today + // produces a file key of the wrong size that every test still calls fine. + let want_len = match alg { Algorithm::Rc4_40 => 5, Algorithm::Aes128 | Algorithm::Rc4_128 => 16, Algorithm::Aes256 => 32 }; + prop_assert_eq!(h.file_key().len(), want_len, "{:?} must use a {} byte file key", alg, want_len); + + let h = make(alg, "user", "owner"); + for num in [0u32, 1, 4096, u32::MAX] { + let generation = (num % 65_535) as u16; + + let enc = h.encrypt_string(num, generation, &data); + prop_assert_eq!(h.decrypt_string(num, generation, &enc), &data[..], "string round trip at {} bytes ({:?})", data.len(), alg); + prop_assert_eq!(h.encrypt_string(num, generation, &data), enc, "string encryption must be reproducible ({:?})", alg); + + for kind in [StreamKind::Normal, StreamKind::EmbeddedFile, StreamKind::Named(b"StdCF")] { + let enc = h.encrypt_stream(num, generation, &data, kind); + prop_assert_eq!(h.decrypt_stream(num, generation, &enc, kind), &data[..], "stream round trip at {} bytes ({:?}, {:?})", data.len(), alg, kind); + prop_assert_eq!(h.encrypt_stream(num, generation, &data, kind), enc, "{:?} {:?} encryption must be reproducible", alg, kind); + } + + if data.len() >= 16 { + prop_assert_ne!(h.encrypt_string(num, generation, &data), &data[..], "one block left in plain sight ({:?})", alg); + } + } + } + + /// Ciphertext of any length — truncated by a damaged `/Length`, padded out with garbage, or + /// byte-flipped inside — must decrypt to bytes and never read past the end. AES discards a + /// trailing partial block and leaves invalid padding in place rather than failing, so its + /// output can only be a whole number of blocks minus at most one; RC4 keeps its length + /// whatever the bytes are. An input shorter than the IV has nothing left to decode. + #[test] + fn damaged_ciphertext_never_panics(alg in algorithms(), enc in prop::collection::vec(any::(), 0..64)) { + let h = make(alg, "user", "owner"); + for cut in 0..=enc.len() { + let broken = &enc[..cut]; + let out = h.decrypt_string(7, 3, broken); + if alg == Algorithm::Rc4_40 || alg == Algorithm::Rc4_128 { + prop_assert_eq!(out.len(), cut, "RC4 must be length-preserving ({:?})", alg); + } else { + let blocks = cut.saturating_sub(16) / 16; + prop_assert!(out.len() <= blocks * 16, "{:?}: {} bytes of ciphertext yielded {} decrypted bytes", alg, cut, out.len()); + prop_assert!(blocks * 16 - out.len() <= 16, "{:?}: padding removed more than one block", alg); + } + } + + // Flip every byte of a real ciphertext in turn: no position may crash the reader. + let full = h.encrypt_string(7, 3, b"twenty-four bytes of PDF content"); + for i in 0..full.len() { + let mut mutated = full.clone(); + mutated[i] ^= 0xFF; + let _ = h.decrypt_string(7, 3, &mutated); + } + } + + /// A crypt filter name decides whether content is encrypted at all: PDF 2.0 permits + /// `/StmF /Identity` with attachments still encrypted, and a name that appears in neither `/CF` + /// nor as `Identity` resolves to Identity rather than failing. The reported answer and the + /// bytes must agree, at every length including empty (where AES still adds a whole block). + /// `/EFF` is set here so embedded files and normal streams must take *different* paths: the + /// mix-up that leaves an attachment in plain sight, or encrypts a stream its own filter says + /// should not be. And sometimes `/CF` carries an entry *named* `Identity` that claims + /// `/CFM /AESV2`: a producer writing nonsense. The name wins and the bytes stay plain, since + /// encrypting what both sides agreed is plaintext would write a file other readers cannot open. + #[test] + fn crypt_filters_decide_what_gets_encrypted( + data in payloads(), + name in prop::sample::select(vec!["Identity", "StdCF", "Nope"]), + bogus_identity_in_cf in any::(), + ) { + let mut d = make(Algorithm::Aes256, "", "owner").dict().clone(); + d.str_f = name.as_bytes().to_vec(); + d.stm_f = b"Identity".to_vec(); + d.ef_f = b"StdCF".to_vec(); + if bogus_identity_in_cf { + d.crypt_filters.push((b"Identity".to_vec(), Method::Aes256)); + } + let h = SecurityHandler::open(d, ID, None).unwrap(); + + // Only `StdCF` really encrypts; `Identity` does not even when `/CF` defines it, and a name + // in neither place resolves to Identity rather than failing. + let strings_encrypted = name == "StdCF"; + prop_assert_eq!(h.encrypts_strings(), strings_encrypted, "/StrF /{} reported {}", name, strings_encrypted); + + let enc = h.encrypt_string(3, 0, &data); + if strings_encrypted { + prop_assert_ne!(&enc, &data, "/StrF /{} left strings in plain sight", name); + prop_assert!(enc.len() > data.len(), "AES always pads: {} bytes of ciphertext for {} plaintext", enc.len(), data.len()); + prop_assert_eq!(h.decrypt_string(3, 0, &enc), &data[..], "/StrF /{} round trip", name); + } else { + prop_assert_eq!(&enc, &data, "/StrF /{} must pass strings through", name); + } + + // A normal stream follows `/StmF /Identity` here, whatever the string filter says. + prop_assert_eq!(&h.encrypt_stream(3, 0, &data, StreamKind::Normal), &data[..], "normal streams follow /StmF Identity"); + // An embedded file follows `/EFF /StdCF`, so it is encrypted at every length — empty + // included, where AES still emits a padded block — and comes back intact. + let ef = h.encrypt_stream(3, 0, &data, StreamKind::EmbeddedFile); + prop_assert_ne!(&ef, &data[..], "/EFF /StdCF left an embedded file in plain sight at {} bytes", data.len()); + prop_assert_eq!(h.decrypt_stream(3, 0, &ef, StreamKind::EmbeddedFile), &data[..], "embedded file round trip"); + // An unknown named filter is Identity, not an error; a known one uses its real method. + prop_assert_eq!(&h.encrypt_stream(3, 0, &data, StreamKind::Named(b"Missing")), &data[..], "an unknown crypt filter must not encrypt"); + prop_assert_ne!(&h.encrypt_stream(3, 0, &data, StreamKind::Named(b"StdCF")), &data[..], "a filter named /StdCF must encrypt"); + } + + /// Object keys are scoped to the object (Algorithm 1, §7.6.3.3): RC4 and AES-128 fold the + /// object number and generation into the key, so another object must not read these bytes back. + /// AES-256 is the documented exception — revisions 5 and 6 use the file key for every object. + /// The comparisons that decrypt under a foreign number carry the weight: wrong keystream bytes + /// match the plaintext only by a chance of one in 256 per byte. Checking that *encryption* + /// differs would prove less, because our initial vector already covers `/Num`. + #[test] + fn object_keys_are_scoped_to_their_object(alg in prop::sample::select(vec![Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128]), data in prop::collection::vec(any::(), 8..40)) { + let h = make(alg, "user", "owner"); + + let enc = h.encrypt_string(100, 0, &data); + prop_assert_eq!(h.decrypt_string(100, 0, &enc), &data[..], "{:?} reads its own object back", alg); + prop_assert_ne!(h.decrypt_string(101, 0, &enc), &data[..], "{:?}: another object number must not decrypt these bytes", alg); + prop_assert_ne!(h.decrypt_string(100, 1, &enc), &data[..], "{:?}: another generation must not decrypt these bytes", alg); + + // Algorithm 1 folds the LOW THREE bytes of `/Num` into the key. Two directions are pinned: + // numbers that differ inside those three bytes must separate (a fold to `[..2]` would let + // object 0 and object 65_536 share a key, silently handing one object's plaintext to another), + // and numbers that differ only ABOVE them must share it — the fold is three bytes by design, + // so a widening to four would break this and nothing else in the crate notices. + let low = h.encrypt_string(0, 0, &data); + prop_assert_ne!(&h.decrypt_string(65_536, 0, &low)[..], &data[..], "{:?}: /Num is folded three bytes wide; 0 and 65_536 must not share a key", alg); + prop_assert_ne!(&h.decrypt_string(1 << 23, 0, &low)[..], &data[..], "{:?}: the third byte of /Num must still reach the key", alg); + prop_assert_eq!(&h.decrypt_string(1 << 24, 0, &low)[..], &data[..], "{:?}: /Num is folded three bytes wide, so a difference above bit 23 shares the key by design", alg); + + let r6 = make(Algorithm::Aes256, "user", "owner"); + let enc6 = r6.encrypt_string(100, 0, &data); + prop_assert_eq!(&r6.decrypt_string(101, 3, &enc6)[..], &data[..], "AES-256 uses the file key for every object (§7.6.3.3)"); + // ...which is exactly why the initial vector has to vary: revisions 5 and 6 reuse one key + // for the whole file, so two objects holding the same bytes must still produce different + // ciphertext. Identical bytes for a different `/Num` would be CBC initial-vector reuse — + // the flaw that leaks structure in encrypted data. + prop_assert_ne!(&r6.encrypt_string(101, 0, &data)[..], &enc6[..], "AES-256 reused the same initial vector for two objects"); + prop_assert_ne!(&r6.encrypt_string(100, 1, &data)[..], &enc6[..], "AES-256 reused the same initial vector across generations"); + } + + /// A file created for `user` and `owner` opens for exactly the passwords whose effective bytes + /// equal one of theirs, and reports the role that match earned — owner first, since viewers try + /// `/O` before `/U`. Everything else is `WrongPassword`: never an opening, and no panic on odd + /// Unicode. This is the rule pdfcraft #1185 became (bytes decide, so a soft hyphen alone is no password + /// and U+FF21 is "A"), stated for all inputs rather than the three examples in `src/tests.rs`. + /// It also pins #1762's upgrade path: a user session gains owner rights on the owner password + /// and on nothing else. + #[test] + fn authentication_accepts_exactly_the_matching_passwords( + alg in algorithms(), + mut user in passwords(), + owner in prop::sample::select(vec!["owner".to_string(), String::new(), "Owner\u{AD}".into(), "\u{FF21}".into()]), + extra in passwords(), + ) { + // `user` may have been grown past the 32-byte cut above; create with that spelling. + // Grow the legacy password PAST the 32-byte cut BEFORE the file is built from it, so a + // partner differing only beyond byte 32 is the same password to this file. + if alg != Algorithm::Aes256 { + user = format!("{user}{}", "abcdefghijklmnopqrstuvwxyz0123456789".repeat(2)); + } + + let h = make(alg, &user, &owner); + let d = h.dict().clone(); + let user_bytes = effective_password(alg, &user); + let owner_bytes = owner_target(alg, &user, &owner); + + // A collision engineered by construction, not by luck. Revisions 2-4 cut the password at + // 32 bytes (Algorithm 2), so any spelling that agrees on the first 32 is ONE password; + // revisions 5-6 run SASLprep, which deletes U+00AD. Growing past byte 32 guarantees the + // partner is a different string with identical bytes at every draw — earlier drafts here + // claimed `(` alone did it, which is false: appending "(" to a short password DISPLACES the + // padding (pad("a") is [61,28,BF..], pad("a(") is [61,28,28,BF..]), and this alphabet draws + // at most 11 characters, so chance produced zero legacy collisions in 1,566 measured cases. + + let mut suspects: Vec = vec![user.clone(), owner.clone(), String::new()]; + // Near misses are PREPENDED: appending to a legacy password already past the cut would be + // another collision rather than a refusal, which would silently thin the negative testing. + let mut near_misses: Vec = vec![format!("q{user}"), extra.clone()]; + if alg == Algorithm::Aes256 && !user.is_empty() { + near_misses.push(format!("{user}q")); + } + + let collision_partner = if alg == Algorithm::Aes256 { format!("{user}\u{AD}") } else { format!("{user}Z") }; + prop_assert_ne!(&collision_partner[..], &user[..], "the collision partner must be a different string"); + prop_assert_eq!(effective_password(alg, &collision_partner), effective_password(alg, &user), "{:?}: {:?} must become the same bytes as {:?}", alg, collision_partner, user); + + // The mistake itself, pinned so it cannot come back: a short legacy password plus "(" is NOT the + // same password, because it shifts the padding by one byte instead of filling a slot. + if alg != Algorithm::Aes256 { + prop_assert_ne!(effective_password(alg, "a("), effective_password(alg, "a"), "appending ( to a short password displaces the padding; that is not a collision"); + prop_assert_eq!(effective_password(alg, &"a".repeat(31)), effective_password(alg, &format!("{}(", "a".repeat(31))), "at exactly 31 characters ( does land in the padding slot, and THAT is a collision"); + } + + suspects.extend(near_misses); + suspects.push(collision_partner.clone()); + suspects.dedup(); + + for suspect in &suspects { + let want = expected_auth(alg, &user_bytes, &owner_bytes, suspect); + match (want, SecurityHandler::open(d.clone(), ID, Some(suspect))) { + (Some(role), Ok(got)) => { + prop_assert_eq!(got.auth(), role, "{:?}: {:?} should open as {:?}", alg, suspect, role); + prop_assert_eq!(got.file_key(), h.file_key(), "{:?}: whichever password opens the file, the key is the same", alg); + } + (Some(role), Err(e)) => prop_assert!(false, "{:?}: {:?} must open as {:?} but gave {:?}", alg, suspect, role, e), + (None, Ok(got)) => prop_assert!(false, "{:?}: {:?} is not a password for this file, yet it opened as {:?}", alg, suspect, got.auth()), + (None, Err(CryptError::WrongPassword)) => {} + (None, Err(e)) => prop_assert!(false, "{:?}: {:?} gave {}, expected WrongPassword", alg, suspect, e), + } + + // The owner password upgrades a user session; nothing else does. When the two share + // bytes, `create` gave /O the user's own bytes (revisions 2–4) or left it empty + // (AES-256, pdfcraft #1185's sharp edge), so the open password *is* an owner password here. + if let Ok(session) = SecurityHandler::open(d.clone(), ID, Some(&user)) { + let is_owner_password = expected_auth(alg, &user_bytes, &owner_bytes, suspect) == Some(Auth::Owner); + let upgrade = session.authenticate_owner(ID, suspect).map(|u| u.auth()); + if is_owner_password { + prop_assert_eq!(upgrade, Ok(Auth::Owner), "{:?}: {:?} must unlock owner rights", alg, suspect); + } else { + prop_assert_eq!(upgrade, Err(CryptError::WrongPassword), "{:?}: {:?} must not grant owner rights", alg, suspect); + } + } + } + + // `None` means no password was offered: the handler tries the empty string, and nothing + // else (a file with a real open password never opens silently). + let want_none = expected_auth(alg, &user_bytes, &owner_bytes, ""); + match (want_none, SecurityHandler::open(d.clone(), ID, None)) { + (Some(role), Ok(got)) => prop_assert_eq!(got.auth(), role, "{:?}: no password opened as {:?}", alg, got.auth()), + (None, Ok(got)) => prop_assert!(false, "{:?}: the empty password is not valid, yet no password opened as {:?}", alg, got.auth()), + (Some(role), Err(e)) => prop_assert!(false, "{:?}: the empty password should open as {:?} but gave {:?}", alg, role, e), + (None, Err(CryptError::WrongPassword)) => {} + (None, Err(e)) => prop_assert!(false, "{:?}: no password gave {}, expected WrongPassword", alg, e), + } + } + + /// The AES-CBC and RC4 primitives taken directly: a damaged file's `/Length` and crypt filters + /// hand them keys and data of shapes `create` would never build, so they must invert each other + /// for any key length (`Cipher::new` takes AES-256 at 32 bytes and zero-padded AES-128 below + /// it, identically on both sides of the trip) and every payload length. + #[test] + fn primitives_are_invertible_for_any_key_and_length( + key in prop::collection::vec(any::(), 1..40), + data in payloads(), + iv in prop::collection::vec(any::(), 16..24), + ) { + let iv: [u8; 16] = *iv.as_slice().first_chunk::<16>().expect("the iv strategy draws at least 16 bytes; a silent zero fallback would hide a generator change"); + + let enc = aes_cbc_encrypt(&key, &iv, &data, true); + prop_assert_eq!(aes_cbc_decrypt(&key, &iv, &enc, true), &data[..], "padded AES round trip at {} bytes", data.len()); + prop_assert!(enc.len() >= 16 && enc.len().is_multiple_of(16), "PKCS#5 makes whole blocks: {} bytes for {} plaintext", enc.len(), data.len()); + + // Without padding, a partial block is zero-padded rather than refused, and extra trailing + // bytes decrypt to more output instead of being dropped. + let raw = aes_cbc_encrypt(&key, &iv, &data, false); + prop_assert_eq!(raw.len(), data.len().div_ceil(16) * 16, "unpadded AES output is whole blocks"); + let mut longer = raw.clone(); + longer.extend_from_slice(b"0123456789abcdef"); + prop_assert_eq!(aes_cbc_decrypt(&key, &iv, &longer, false).len(), raw.len() + 16, "a whole extra block decrypts to a whole extra block"); + + // RC4 undoes itself at every length; an empty key encrypts nothing, which is why the + // handler never derives an empty object key. + prop_assert_eq!(rc4(&key, &rc4(&key, &data)), &data[..], "RC4 round trip at {} bytes", data.len()); + prop_assert_eq!(rc4(&[], &data), &data[..], "an empty RC4 key passes bytes through"); + } + + /// `/Perms` certifies the permission bits and `/EncryptMetadata`: decrypted with the file key it + /// must hold `/P` little-endian, four set bytes, 'T' or 'F', and the `adb` sentinel (Algorithm 10). + /// Nothing else in this crate reads it — `cos` writes it out and no code takes it back apart — so + /// without this property a flipped metadata byte would tell every other viewer the wrong thing about + /// our own file while all our tests stayed green. The older revisions we create emit no `/Perms` + /// at all, which is recorded here as well so a change in what goes out shows up rather than + /// shipping quietly. + #[test] + fn perms_value_agrees_with_the_permissions_it_certifies(permissions in any::(), encrypt_metadata in any::()) { + let h = create( + &NewEncryption { algorithm: Algorithm::Aes256, user_password: "user", owner_password: "owner", permissions, encrypt_metadata, seed: [0x5e; 32] }, + ID, + ) + .unwrap(); + + prop_assert_eq!(h.dict().perms.len(), 16, "/Perms is one encrypted block"); + let open = aes_cbc_decrypt(h.file_key(), &[0u8; 16], &h.dict().perms, false); + + // Bits 1-2 clear and the reserved bits set (§7.6.4.2 Table 22) is what `/P` must say, and the + // encrypted certificate has to repeat it byte for byte. + let normalised = (permissions | !0xFFF | 0xC0) & !0x3; + prop_assert_eq!(&open[..4], &normalised.to_le_bytes(), "/Perms must carry /P"); + prop_assert_eq!(h.dict().p, normalised, "the /P written must match the permission bits drawn"); + prop_assert_eq!(&open[4..8], &[0xFF; 4], "the four bytes following /P are set"); + prop_assert_eq!(open[8], if encrypt_metadata { b'T' } else { b'F' }, "/Perms must report /EncryptMetadata"); + prop_assert_eq!(&open[9..12], b"adb", "the adb sentinel must be present"); + + // A user session's answers come from those same bits, not from the owner shortcut. + let as_user = SecurityHandler::open(h.dict().clone(), ID, Some("user")).unwrap(); + prop_assert_eq!(as_user.permissions().print(), normalised & 0b100 != 0, "/P bit 3 decides printing"); + prop_assert_eq!(as_user.permissions().copy(), normalised & 0b1_0000 != 0, "/P bit 5 decides copying"); + prop_assert!(!as_user.permissions().owner, "the user password must not report owner rights"); + + // Bytes 13-16 of /Perms are the non-repeating filler. Nothing reads them, so pinning that + // they move with the seed is the only way to notice a producer that freezes them at zero. + let other = create( + &NewEncryption { algorithm: Algorithm::Aes256, user_password: "user", owner_password: "owner", permissions, encrypt_metadata, seed: [0x11; 32] }, + ID, + ) + .unwrap(); + let open_other = aes_cbc_decrypt(other.file_key(), &[0u8; 16], &other.dict().perms, false); + prop_assert_ne!(&open[12..16], &open_other[12..16], "the four filler bytes of /Perms must follow the seed, not sit constant"); + + // Revisions below 5 carry no /Perms; pinned so the boundary of what we emit is explicit. + for older in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128] { + let legacy = make(older, "user", "owner"); + prop_assert!(legacy.dict().perms.is_empty(), "{:?} must emit no /Perms", older); + } + } + /// Revisions 4 mix four set bytes into every file key when `/EncryptMetadata` is false (Algorithm + /// 2, step f), so the same password over the same `/ID` must produce a *different* key than when + /// metadata is encrypted. Every other test in this crate builds `encrypt_metadata: true` only, + /// which is exactly why that step is reachable but never walked: switching it off entirely changed + /// nothing any test could see. This also records the rule that revisions below 4 always encrypt + /// metadata whatever was asked, so the boundary between the two is named rather than assumed. + #[test] + fn metadata_flag_changes_the_key_it_claims_to_change(data in payloads()) { + let build = |encrypt_metadata: bool| { + create(&NewEncryption { algorithm: Algorithm::Aes128, user_password: "user", owner_password: "owner", permissions: -1, encrypt_metadata, seed: [0x5e; 32] }, ID).unwrap() + }; + let (yes, no) = (build(true), build(false)); + + // Algorithm 2, step f: revisions 4 hash four set bytes into every key when metadata stays + // plain, so two files with identical passwords and the same /ID cannot share a key. + prop_assert_ne!(yes.file_key(), no.file_key(), "R4 must fold four set bytes into the key when /EncryptMetadata is false (Algorithm 2, step f)"); + prop_assert_eq!(no.dict().encrypt_metadata, false, "the flag must reach the dictionary"); + prop_assert!(!no.encrypts_metadata(), "/EncryptMetadata false must report itself honestly"); + prop_assert!(yes.encrypts_metadata(), "and true must report true"); + + // Both still open on the same password and round trip at every length: a derivation that + // drifted would show up as an unreadable file rather than passing quietly. + for h in [&yes, &no] { + let open = SecurityHandler::open(h.dict().clone(), ID, Some("user")).unwrap(); + prop_assert_eq!(open.file_key(), h.file_key(), "the user password must still derive this file's key"); + let enc = h.encrypt_string(7, 0, &data); + prop_assert_eq!(&open.decrypt_string(7, 0, &enc)[..], &data[..], "round trip at {} bytes", data.len()); + } + + // And the two files must not read each other's content: different keys, different plaintext. + if data.len() >= 8 { + prop_assert_ne!(&yes.decrypt_string(7, 0, &no.encrypt_string(7, 0, &data))[..], &data[..], "a key built for plain metadata must not decrypt a file whose keys encrypt it"); + } + + // Revisions below 4 always encrypt metadata, whatever the caller asked for. + let legacy = create(&NewEncryption { algorithm: Algorithm::Rc4_128, user_password: "user", owner_password: "owner", permissions: -1, encrypt_metadata: false, seed: [0x5e; 32] }, ID).unwrap(); + prop_assert!(legacy.dict().encrypt_metadata && legacy.encrypts_metadata(), "V<4 always encrypts metadata, as create() documents"); + } + +} diff --git a/crates/pdf/tests/crypt/qpdf_literals.rs b/crates/pdf/tests/crypt/qpdf_literals.rs new file mode 100644 index 000000000..e583eb815 --- /dev/null +++ b/crates/pdf/tests/crypt/qpdf_literals.rs @@ -0,0 +1,326 @@ +//! Encryption numbers from an **outside implementation**. Six of the seven rows below are qpdf's own, +//! written by qpdf core 12.2.0 through pikepdf 9.11, and their `/O`, `/U`, `/OE`, `/UE`, `/Perms` +//! and `/ID` values were copied out of those files rather than computed here. The seventh row, +//! `crafted-r3-40bit-hand-derived`, is ours and says so where it is defined - no producer we can run +//! here writes revision 3 at 40 bits (qpdf core 12.2.0 emits `/Length 128` at `R=3`, measured below), +//! so nothing outside computed that one either. +//! +//! That distinction is the whole point. `create()` and `open()` share one derivation, so a wrong-but +//! self-consistent implementation round-trips cleanly against itself: across two review rounds we +//! broke the key derivation several ways and the unit suite stayed green (deleting Algorithm 2 step f +//! leaves all 17 unit tests passing while our rows fail). Quoting an outsider's bytes +//! breaks the circle - `open()` recomputes `/U` (or the revision 6 hash chain) and compares it to what +//! was stored, so a derivation that diverges cannot authenticate at all. +//! +//! Revisions covered: 2 (export-grade RC4, where the five-byte key makes a truncation error visible that +//! every pre-existing R3/R4 vector in this repository hides as a no-op), 3, and 4 and 6 each with metadata +//! encrypted *and* plain. For revision 4 that pairing is the only outside view of Algorithm 2 step +//! f's four `FF` bytes; revision 6 shows the same flag differently - its `/U` differs between the two +//! metadata variants, which is measured here rather than attributed to a named algorithm step. Only the +//! numbers are committed - never the PDFs, and no generator script that lives outside +//! this repository. To re-derive the six qpdf rows, take one blank 200x200 page and save it with +//! pikepdf 9.11 (qpdf core 12.2.0) using `Encryption(owner=OWNER, user=USER,` +//! `allow=Permissions(extract=False), R=, aes=…, metadata=…)` plus `save(..., static_id=True)`, +//! then read `/Encrypt` back. Below revision 4 both flags are forced rather than chosen, because pikepdf +//! refuses AES and refuses encrypted metadata there - so those two rows need `aes=False, metadata=False`, +//! while revisions 4 and 6 take `aes=True` with the case's own metadata flag. Run to run, every quoted +//! field of those rows reproduces at revisions 2-4, and whole files too for revisions 2 and 3; a +//! revision 4 file falls short of that by one stream, because `/CFM /AESV2` prepends a fresh random IV. +//! Revision 6 reproduces structurally only, its salts being new each run. What stays checked rather than +//! trusted - `open()` recovers the user password from `/O` and verifies `/U` at revisions 2-4 +//! (Algorithm 7), matches both stored hashes at revision 6 and unwraps `/OE`/`/UE` into the file key +//! there; this file then decrypts `/Perms` with that key, which authentication itself never reads. One +//! thing no assertion here verifies is the crypt-filter names: renaming a row's `/StmF` or replacing its +//! filter set leaves all three tests green, so those are held to review rather than to a test. The +//! rows come from PdfCraft, which uses qpdf as an oracle (its `crates/oracle/README.md`). + +use craft_pdf::crypt::{Auth, EncryptDict, Method, SecurityHandler, aes_cbc_decrypt}; + +const USER: &str = "open pw"; +const OWNER: &str = "owner pw"; + +/// One file qpdf wrote. Bytes are held as hex so the table stays a `const`: the literals on screen are +/// then exactly what qpdf emitted, with no runtime step in between to trust. +struct Case { + label: &'static str, + v: i64, + r: i64, + length_bits: i64, + p: i32, + encrypt_metadata: bool, + id0_hex: &'static str, + o_hex: &'static str, + u_hex: &'static str, + oe_hex: &'static str, + ue_hex: &'static str, + perms_hex: &'static str, + crypt_filters: fn() -> Vec<(Vec, Method)>, + stm_f: &'static str, + str_f: &'static str, + key_len_bytes: usize, +} + +impl Case { + fn dict(&self) -> EncryptDict { + EncryptDict { + filter: b"Standard".to_vec(), + v: self.v, + r: self.r, + length_bits: self.length_bits, + o: hex(self.o_hex), + u: hex(self.u_hex), + oe: hex(self.oe_hex), + ue: hex(self.ue_hex), + perms: hex(self.perms_hex), + p: self.p, + encrypt_metadata: self.encrypt_metadata, + crypt_filters: (self.crypt_filters)(), + stm_f: self.stm_f.as_bytes().to_vec(), + str_f: self.str_f.as_bytes().to_vec(), + ef_f: Vec::new(), + } + } + + fn id0(&self) -> Vec { + hex(self.id0_hex) + } +} + +fn hex(s: &str) -> Vec { + (0..s.len()).step_by(2).map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap()).collect() +} + +fn no_filters() -> Vec<(Vec, Method)> { + Vec::new() +} + +fn aes128_filter() -> Vec<(Vec, Method)> { + vec![(b"StdCF".to_vec(), Method::Aes128)] +} + +fn aes256_filter() -> Vec<(Vec, Method)> { + vec![(b"StdCF".to_vec(), Method::Aes256)] +} + +/// `/ID[0]` is shared by every case because qpdf wrote them with `static_id`. +const CASES: &[Case] = &[ + Case { + label: "r2-40bit", + v: 1, + r: 2, + length_bits: 40, + p: -28, + encrypt_metadata: true, + id0_hex: "31415926535897932384626433832795", + o_hex: "166a894580aae60ae93fa9d83fc576f26f13dbe4574ce6c57497bec3ece7f125", + u_hex: "69ff2fb98261fd094174f93f0388859f356080ca486a38ce2d5ac23b98f5f89c", + oe_hex: "", + ue_hex: "", + perms_hex: "", + crypt_filters: no_filters, + stm_f: "", + str_f: "", + key_len_bytes: 5, + }, + Case { + label: "r3-128rc4", + v: 2, + r: 3, + length_bits: 128, + p: -1044, + encrypt_metadata: true, + id0_hex: "31415926535897932384626433832795", + o_hex: "2910182a30a271d410d8dada035cbd77c0b57a683bd55cd1cd9eee35a2b64848", + u_hex: "1b4b8fb3fd213b4eadc142ab9f385fc80122456a91bae5134273a6db134c87c4", + oe_hex: "", + ue_hex: "", + perms_hex: "", + crypt_filters: no_filters, + stm_f: "", + str_f: "", + key_len_bytes: 16, + }, + Case { + label: "r4-aes128-meta-enc", + v: 4, + r: 4, + length_bits: 128, + p: -1044, + encrypt_metadata: true, + id0_hex: "31415926535897932384626433832795", + o_hex: "2910182a30a271d410d8dada035cbd77c0b57a683bd55cd1cd9eee35a2b64848", + u_hex: "1b4b8fb3fd213b4eadc142ab9f385fc80122456a91bae5134273a6db134c87c4", + oe_hex: "", + ue_hex: "", + perms_hex: "", + crypt_filters: aes128_filter, + stm_f: "StdCF", + str_f: "StdCF", + key_len_bytes: 16, + }, + Case { + label: "r4-aes128-meta-plain", + v: 4, + r: 4, + length_bits: 128, + p: -1044, + encrypt_metadata: false, + id0_hex: "31415926535897932384626433832795", + o_hex: "2910182a30a271d410d8dada035cbd77c0b57a683bd55cd1cd9eee35a2b64848", + u_hex: "d6d363fc817815701f0242a581cb39e60122456a91bae5134273a6db134c87c4", + oe_hex: "", + ue_hex: "", + perms_hex: "", + crypt_filters: aes128_filter, + stm_f: "StdCF", + str_f: "StdCF", + key_len_bytes: 16, + }, + Case { + label: "r6-aes256-meta-enc", + v: 5, + r: 6, + length_bits: 256, + p: -1044, + encrypt_metadata: true, + id0_hex: "31415926535897932384626433832795", + o_hex: "1c791d66227fdfd7c1d225eb81fb3408d8bcfac1a0ff71cc57b10eb29029e5f6965cccb6b94e461d9da354a9dbd6eea7", + u_hex: "64ffbf79168ed7b325267cdba8dae211d4c2c5ebdffd339c7ac36c33e26f23b7f478c60ae42d50b5d20ceac56d659151", + oe_hex: "d2f6ef0bcc6ce93228be760c7eee8f4259af7a6044a4c4e04a0d48ab0893ef1e", + ue_hex: "963035aa24fb009e6d26b3cffd31492ffe6f4e563e4dcae8d7b7e3203c4a2f56", + perms_hex: "5675a86560873d2532e856a37045247e", + crypt_filters: aes256_filter, + stm_f: "StdCF", + str_f: "StdCF", + key_len_bytes: 32, + }, + Case { + label: "r6-aes256-meta-plain", + v: 5, + r: 6, + length_bits: 256, + p: -1044, + encrypt_metadata: false, + id0_hex: "31415926535897932384626433832795", + o_hex: "e227e586c4dc5a362c7271d960565cf3a37145e33af5bf5a08c73b52a1d819e152c737176dc08774e400798d6534e95e", + u_hex: "dbb84b847bb2d80865cc3c5a7c751d4c659d2c96ed1750f8be6f9c3b93fe3d11f494b43e5f17a21c85f5a4b171055afb", + oe_hex: "8c31f2392bc2a093850269a3404cbe01cea38e5e0fe308fe8ac50f5222cff103", + ue_hex: "2b0c44e4bddb3c9c01431b4ed3c7da42dbd1ef47e610d626ec2124535a0a4469", + perms_hex: "adbb69fb9fffde31ba90aaaf5eb4bdaf", + crypt_filters: aes256_filter, + stm_f: "StdCF", + str_f: "StdCF", + key_len_bytes: 32, + }, + // HAND-DERIVED, not produced by qpdf: an independent re-implementation of ISO Algorithms 2/3 + // (n = 5) computed this pair, cross-validated by having the same routine reproduce qpdf's + // n = 16 `/O` byte for byte. It exists because no producer we can run here emits R3 at 40 bits - + // qpdf core 12.2.0 writes `/Length 128` there, and this crate's own create() maps Rc4_40 to revision 2 - so + // the only route we have to `key_len` = 5 with the derivation loop RUNNING (r >= 3) is to craft + // it by hand. With that combination live, the loop's truncation stops being a no-op: mutating + // `Md5::digest(&digest[..n])` to hash the whole digest makes this password stop authenticating. + Case { + label: "crafted-r3-40bit-hand-derived", + v: 2, + r: 3, + length_bits: 40, + p: -1044, + encrypt_metadata: true, + id0_hex: "31415926535897932384626433832795", + o_hex: "50e50251fd20acb28ba50013142d19a47249d06b977d28b63cb3ce9d56fb5d50", + u_hex: "000756f24fe648256830aec6155ca6f100000000000000000000000000000000", + oe_hex: "", + ue_hex: "", + perms_hex: "", + crypt_filters: no_filters, + stm_f: "", + str_f: "", + key_len_bytes: 5, + }, +]; + +#[test] +fn outside_derived_dictionaries_authenticate_with_the_intended_role() { + for case in CASES { + let d = case.dict(); + let id0 = case.id0(); + + // The user password must open as a user. Owners are tried first, so an `Owner` here would mean + // the two passwords had collapsed onto one set of bytes - on these spellings they have not. + let user = SecurityHandler::open(d.clone(), &id0, Some(USER)) + .unwrap_or_else(|e| panic!("{}: qpdf's user password was refused by our reader: {e}", case.label)); + assert_eq!(user.auth(), Auth::User, "{}", case.label); + + // The owner password must open as owner: revisions 2-4 recover the user password from `/O` + // (Algorithm 7), revision 6 hashes against qpdf's own `/O`. + let owner = + SecurityHandler::open(d.clone(), &id0, Some(OWNER)).unwrap_or_else(|e| panic!("{}: qpdf's owner password was refused: {e}", case.label)); + assert_eq!(owner.auth(), Auth::Owner, "{}", case.label); + + // A wrong password may never open it; and the key must be exactly as wide as the revision + // promises - those five bytes are what makes a revision 2 truncation detectable at all. + assert!(SecurityHandler::open(d.clone(), &id0, Some("not the password")).is_err(), "{}: a wrong password opened", case.label); + assert_eq!(user.file_key().len(), case.key_len_bytes, "{}: file key width", case.label); + assert_eq!(owner.file_key(), user.file_key(), "{}: whichever password opens it, one key", case.label); + + // A user session must not claim owner rights. We deliberately do NOT assert individual + // permission flags here: which bit means "extract" moves between revision 2 and revisions 4 + // (qpdf wrote `/P = -28` for the export-grade file, where our accessor's bit-10 reading of + // "extract for accessibility" is a reserved field), and pinning that mapping belongs with the + // permissions tests, not with this oracle. What matters here is only that authentication, role + // and key width agree with what an outside implementation produced. + assert!(!user.permissions().owner, "{}: a user session claimed owner rights", case.label); + } +} + +#[test] +fn metadata_flag_from_outside_changes_the_file_key_it_certifies() { + // `/EncryptMetadata` false folds four set bytes into a revision 4 key (Algorithm 2, step f) and + // changes what revision 6 stores in `/U`. qpdf wrote both variants of each, so an implementation + // that ignored the flag would give identical keys. Internally this is no longer untested - + // `tests/it/proptests.rs` from #2358 varies the flag and asserts the keys differ - but internal tests + // still share our derivation, so what these rows add is a second implementation's own numbers. + for (enc_label, plain_label) in [("r4-aes128-meta-enc", "r4-aes128-meta-plain"), ("r6-aes256-meta-enc", "r6-aes256-meta-plain")] { + let a = CASES.iter().find(|c| c.label == enc_label).expect("encrypted-metadata fixture present"); + let b = CASES.iter().find(|c| c.label == plain_label).expect("plain-metadata fixture present"); + + let ha = SecurityHandler::open(a.dict(), &a.id0(), Some(USER)).unwrap_or_else(|e| panic!("{enc_label}: refused by our reader: {e}")); + let hb = SecurityHandler::open(b.dict(), &b.id0(), Some(USER)) + .unwrap_or_else(|e| panic!("{plain_label}: refused by our reader - is step f being ignored? {e}")); + + assert_ne!(ha.file_key(), hb.file_key(), "{enc_label} and {plain_label} must not share a key: the metadata flag is hashed in"); + assert!(ha.encrypts_metadata(), "{enc_label} reports encrypted metadata"); + assert!(!hb.encrypts_metadata(), "{plain_label} reports plain metadata, as qpdf wrote it"); + } + + // The outsider's numbers must differ across revisions; equality here would mean a transcription + // error, which would quietly void every assertion above. + let r2 = CASES.iter().find(|c| c.r == 2).unwrap(); + let r3 = CASES.iter().find(|c| c.r == 3).unwrap(); + assert_ne!(hex(r2.u_hex), hex(r3.u_hex), "revision 2 and revision 3 /U differ because the key width differs"); + assert_eq!(r2.u_hex.len(), 64, "revision 2 /U is RC4 over the padding: 32 bytes"); + assert_eq!(r3.u_hex.len(), 64, "revision 3 /U is 32 bytes: sixteen derived, plus padding qpdf fills (our crafted row leaves it zero)"); +} + +#[test] +fn qpdf_perms_decode_to_the_shape_both_implementations_agree_on() { + // `/Perms` is the one value in this dictionary that no test here verified before: `cos` writes it + // out and reads it back, but nothing decrypts it. Under qpdf's own bytes we can - `/P` + // little-endian, four set bytes, `T`/`F`, the `adb` sentinel (Algorithm 10 here; vendored lopdf + // checks the same fields as its Algorithm 13, though only three bytes of `/P`). + let mut checked = 0; + for case in CASES.iter().filter(|c| !c.perms_hex.is_empty()) { + let h = SecurityHandler::open(case.dict(), &case.id0(), Some(USER)) + .unwrap_or_else(|e| panic!("{}: refused before /Perms could be read: {e}", case.label)); + let open = aes_cbc_decrypt(h.file_key(), &[0u8; 16], &hex(case.perms_hex), false); + + assert_eq!(open.len(), 16, "{}: /Perms is one encrypted block", case.label); + assert_eq!(&open[..4], &case.p.to_le_bytes(), "{}: /P must round trip against qpdf's bytes", case.label); + assert_eq!(&open[4..8], &[0xFF; 4], "{}: the four bytes after /P are set", case.label); + assert_eq!(open[8], if case.encrypt_metadata { b'T' } else { b'F' }, "{}: metadata byte as qpdf wrote it", case.label); + assert_eq!(&open[9..12], b"adb", "{}: sentinel from qpdf's bytes", case.label); + assert_ne!(&open[12..16], &[0u8; 4], "{}: qpdf's filler is not frozen at zero", case.label); + checked += 1; + } + assert!(checked >= 2, "expected /Perms on both revision 6 cases; saw {checked}"); +} From e0a3b2b8c43fda3992abcebae7e29df6831a1229 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:21:50 -0500 Subject: [PATCH 08/31] craft-dxf: model entities (geometry, text, MTEXT columns, hatches, dimensions) (#927) * craft-dxf: model, the drawing above the tags (header, tables, blocks, layouts) Document::read indexes a tag list into header variables, table records, blocks with their entities, the ENTITIES records (a POLYLINE with its vertices, an INSERT with its attributes) and the objects, each a slice of the tag list. The typed table records, LAYOUT and DICTIONARY are made on request. Also the colour index palette, colour, lineweight and transparency values, extended data lookup and the object coordinate system. Merged from CADCraft's cadcraft-io dxf_read (5a6aa9b6) and VectorCraft's vectorcraft-cad import (566f7208). Neither app's document types appear. * craft-dxf: model entities (geometry, text, MTEXT columns, hatches, dimensions) Entity::read types an entity record: the properties every entity has, and LINE, POINT, CIRCLE, ARC, ELLIPSE, LWPOLYLINE, POLYLINE with its vertices (2D, 3D, mesh, polyface), SPLINE, RAY, XLINE, TEXT, MTEXT with columns, ATTDEF, INSERT with its attributes (multiline ones included), DIMENSION, LEADER, SOLID, TRACE, 3DFACE, HATCH (polyline and edge loops, pattern lines, seeds, gradient), VIEWPORT, IMAGE, WIPEOUT and TOLERANCE. Other types stay readable from the record's tags. model::text has the caret encoding of strings, both ways. Merged from CADCraft's cadcraft-io dxf_read (5a6aa9b6) and VectorCraft's vectorcraft-cad import (566f7208). * craft-dxf: a hatch spline edge without fit data leaves the loop's source count Before R2010 a spline edge has no 97 of its own, so the 97 after it counts the loop's source entities. It was read as the edge's fit point count and the sources were lost. * README: keep main's crate list after the merge --- README.md | 2 +- crates/dxf/src/model.rs | 12 +- crates/dxf/src/model/entity.rs | 669 +++++++++++++++++++++++++++++ crates/dxf/src/model/hatch.rs | 355 +++++++++++++++ crates/dxf/src/model/text.rs | 280 ++++++++++++ crates/dxf/tests/model_entities.rs | 393 +++++++++++++++++ 6 files changed, 1708 insertions(+), 3 deletions(-) create mode 100644 crates/dxf/src/model/entity.rs create mode 100644 crates/dxf/src/model/hatch.rs create mode 100644 crates/dxf/src/model/text.rs create mode 100644 crates/dxf/tests/model_entities.rs diff --git a/README.md b/README.md index be5e2936e..3830d0560 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = ", + /// Group 330: the block record (or other object) that owns the entity. + pub owner: Option, + /// Group 8 (`0` when absent). + pub layer: String, + /// Group 6; `None` is ByLayer. + pub linetype: Option, + pub color: Color, + pub lineweight: Lineweight, + /// Group 48 (1 when absent or not positive). + pub ltscale: f64, + /// Group 60 is not 1. + pub visible: bool, + /// Group 67 is 1. + pub paper_space: bool, + pub transparency: Transparency, + /// Group 39. + pub thickness: f64, + /// Groups 210/220/230 (the world Z axis when absent). + pub extrusion: Point, + /// Group 390: the plot style name object. + pub plot_style: Option, +} + +/// The groups of the AcDbEntity subclass: the record up to the entity's own subclass marker (all +/// of it in files without markers, and in records with no AcDbEntity marker, whose layer and +/// colour may follow the entity's marker), or up to an embedded object (101). Later groups reuse +/// the codes: an MTEXT's column width is 48. Markers inside an application-defined group do not +/// count. +fn entity_groups(tags: &[Tag]) -> &[Tag] { + let is_entity = |t: &Tag| t.str_ref().is_some_and(|s| s == "AcDbEntity"); + let marked = outside_app(tags).any(|(_, t)| t.code == 100 && is_entity(t)); + let end = outside_app(tags).find(|(_, t)| (marked && t.code == 100 && !is_entity(t)) || t.code == 101).map_or(tags.len(), |(i, _)| i); + tags.get(..end).unwrap_or(tags) +} + +impl Common { + pub fn read(tags: &[Tag]) -> Common { + let (all, e) = (Groups(tags), Groups(entity_groups(tags))); + Common { + handle: e.handle(5), + owner: outside_app(tags).find(|(_, t)| t.code == 330).and_then(|(_, t)| parse_handle(&t.str())), + layer: e.name(8).unwrap_or_else(|| "0".into()), + linetype: e.name(6).filter(|l| !l.eq_ignore_ascii_case("ByLayer")), + color: Color::of(&e), + lineweight: e.i(370).map_or(Lineweight::ByLayer, Lineweight::from_dxf), + ltscale: e.f(48).filter(|s| *s > 0.0).unwrap_or(1.0), + visible: e.i(60) != Some(1), + paper_space: e.i(67) == Some(1), + transparency: e.i(440).map_or(Transparency::ByLayer, Transparency::from_dxf), + thickness: all.f_or(39, 0.0), + extrusion: [all.f_or(210, 0.0), all.f_or(220, 0.0), all.f_or(230, 1.0)], + plot_style: e.handle(390).filter(|h| *h != 0), + } + } +} + +/// A vertex of an LWPOLYLINE. +#[derive(Clone, Copy, Debug, PartialEq, Default)] +pub struct LwVertex { + pub x: f64, + pub y: f64, + /// Group 42: the tangent of a quarter of the arc to the next vertex (0: straight). + pub bulge: f64, + pub start_width: f64, + pub end_width: f64, +} + +/// An LWPOLYLINE. +#[derive(Clone, Debug, PartialEq)] +pub struct LwPolyline { + pub vertices: Vec, + /// Flags (70) bit 1. + pub closed: bool, + /// Flags bit 128: the linetype runs around the vertices. + pub plinegen: bool, + /// Group 43. + pub const_width: f64, + /// Group 38. + pub elevation: f64, +} + +/// A VERTEX of a POLYLINE. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Vertex { + pub location: Point, + pub bulge: f64, + pub start_width: f64, + pub end_width: f64, + /// Group 70: 1 added by curve fitting, 8 added by spline fitting, 16 a spline frame control + /// point, 32 a 3D polyline vertex, 64 a mesh vertex, 128 a polyface vertex (128 without 64: + /// a face record). + pub flags: i64, + /// Groups 71–74: the vertex numbers of a polyface face record (1-based; negative: the edge + /// from it is invisible; 0: none). + pub face: [i64; 4], +} + +impl Vertex { + /// A face record of a polyface mesh, which holds vertex numbers and no point. + pub fn is_face(&self) -> bool { + self.flags & 128 != 0 && self.flags & 64 == 0 + } + /// A control point of the frame a spline-fit polyline was made from: not on the curve. + pub fn is_spline_frame(&self) -> bool { + self.flags & 16 != 0 + } +} + +/// A POLYLINE with its VERTEX records: a 2D or 3D polyline, a polygon mesh or a polyface mesh. +#[derive(Clone, Debug, PartialEq)] +pub struct Polyline { + /// Group 70: 1 closed (a mesh: closed in M), 2 curve-fit, 4 spline-fit, 8 a 3D polyline, 16 a + /// polygon mesh, 32 a mesh closed in N, 64 a polyface mesh, 128 the linetype runs around the + /// vertices. + pub flags: i64, + /// The z of the POLYLINE's own point: a 2D polyline's elevation. + pub elevation: f64, + /// Groups 40 and 41: the default widths. + pub start_width: f64, + pub end_width: f64, + /// Groups 71 and 72: a mesh's M and N vertex counts (a polyface mesh: vertices and faces). + pub m: i64, + pub n: i64, + /// Group 75: the smooth surface type (5 quadratic, 6 cubic, 8 Bezier). + pub smooth: i64, + pub vertices: Vec, +} + +impl Polyline { + pub fn closed(&self) -> bool { + self.flags & 1 != 0 + } + pub fn is_3d(&self) -> bool { + self.flags & 8 != 0 + } + pub fn is_mesh(&self) -> bool { + self.flags & 16 != 0 + } + pub fn is_polyface(&self) -> bool { + self.flags & 64 != 0 + } +} + +/// A SPLINE (NURBS), in world coordinates. +#[derive(Clone, Debug, PartialEq)] +pub struct Spline { + /// Group 71 (3 when absent). + pub degree: i64, + /// Group 70: 1 closed, 2 periodic, 4 rational, 8 planar, 16 linear. + pub flags: i64, + pub knots: Vec, + pub control: Vec, + /// Group 41: one per control point when rational. + pub weights: Vec, + pub fit: Vec, + /// Groups 12/22/32 and 13/23/33. + pub start_tangent: Option, + pub end_tangent: Option, + /// Group 44. + pub fit_tolerance: f64, +} + +/// TEXT, and the text of an ATTRIB or ATTDEF. +#[derive(Clone, Debug, PartialEq)] +pub struct Text { + /// Group 10: the first alignment point. + pub insert: Point, + /// Group 11: the second alignment point, which places text that is not left-baseline. + pub align: Option, + pub height: f64, + /// The string, decoded ([`decode_line`]); `%%` codes are left as written. + pub value: String, + /// Group 50, in degrees. + pub rotation: f64, + /// Group 41 (1 when absent). + pub width_factor: f64, + /// Group 51, in degrees. + pub oblique: f64, + /// Group 7 (`Standard` when absent). + pub style: String, + /// Group 71: 2 backwards, 4 upside down. + pub generation: i64, + /// Group 72: 0 left, 1 centre, 2 right, 3 aligned, 4 middle, 5 fit. + pub halign: i64, + /// Group 73 (74 in an ATTRIB or ATTDEF): 0 baseline, 1 bottom, 2 middle, 3 top. + pub valign: i64, +} + +impl Text { + pub fn read(g: &Groups<'_>) -> Text { + Text { + insert: g.point(10), + align: g.point_opt(11), + height: g.f_or(40, 0.0), + value: decode_line(&g.s(1).unwrap_or_default()), + rotation: g.f_or(50, 0.0), + width_factor: g.f_or(41, 1.0), + oblique: g.f_or(51, 0.0), + style: g.name(7).unwrap_or_else(|| "Standard".into()), + generation: g.i_or(71, 0), + halign: g.i_or(72, 0), + valign: g.i(73).or_else(|| g.i(74)).unwrap_or(0), + } + } +} + +/// An ATTRIB (an attribute of a block reference) or ATTDEF (its definition in the block). +#[derive(Clone, Debug, PartialEq)] +pub struct Attrib { + /// Group 2. + pub tag: String, + pub text: Text, + /// Group 3 of an ATTDEF, decoded; empty in an ATTRIB. + pub prompt: String, + /// Group 70: 1 invisible, 2 constant, 4 verify, 8 preset. + pub flags: i64, + /// The 280 after the tag in the attribute's subclass (one before the tag is the subclass + /// version). + pub lock_position: bool, + /// The embedded MTEXT of a multiline attribute. + pub multiline: Option, +} + +impl Attrib { + pub fn read(kind: &str, tags: &[Tag]) -> Attrib { + // A multiline attribute has its text as embedded MTEXT after its own groups. + let (own, embedded) = split_embedded(tags); + let g = Groups(own); + let sub = own.iter().position(|t| t.code == 100 && t.str_ref().is_some_and(|s| s.trim().starts_with("AcDbAttribute"))).unwrap_or(0); + let rest = own.get(sub..).unwrap_or_default(); + let lock_position = rest.iter().position(|t| t.code == 2).is_some_and(|at| rest.iter().skip(at + 1).any(|t| t.code == 280 && t.i64() != 0)); + Attrib { + tag: g.s(2).unwrap_or_default(), + text: Text::read(&g), + prompt: if kind == "ATTDEF" { decode_line(&g.s(3).unwrap_or_default()) } else { String::new() }, + flags: g.i_or(70, 0), + lock_position, + multiline: embedded.filter(|e| !e.is_empty()).map(MText::read), + } + } +} + +/// An INSERT: a block reference, with its attributes. +#[derive(Clone, Debug, PartialEq)] +pub struct Insert { + /// Group 2. + pub block: String, + pub insert: Point, + /// Groups 41, 42 and 43 (1 when absent). + pub scale: Point, + /// Group 50, in degrees. + pub rotation: f64, + /// Groups 70 and 71 (1 when absent or less). + pub columns: i64, + pub rows: i64, + /// Groups 44 and 45. + pub column_spacing: f64, + pub row_spacing: f64, + /// The ATTRIB records that follow, each with its own properties. + pub attribs: Vec<(Common, Attrib)>, +} + +/// A DIMENSION. The geometry it is drawn with is in its block ([`Dimension::block`]). +#[derive(Clone, Debug, PartialEq)] +pub struct Dimension { + /// Group 70: the type in the low bits (0 rotated, 1 aligned, 2 angular, 3 diameter, 4 radius, + /// 5 angular 3-point, 6 ordinate), 32 the block is this dimension's alone, 64 an ordinate's + /// X type, 128 the text is at a user position. + pub flags: i64, + /// Group 2: the block that holds the dimension's picture. + pub block: Option, + /// Group 3 (`Standard` when absent). + pub style: String, + /// Group 10: the definition point. + pub defpt: Point, + /// Group 11: the middle of the text, in the object coordinate system. + pub text_mid: Point, + /// Group 12: the insertion point for clones (baseline and continue). + pub insertion: Point, + /// Groups 13 to 16: the definition points of the type. + pub p13: Point, + pub p14: Point, + pub p15: Point, + pub p16: Point, + /// Group 1, decoded: empty or `<>` for the measurement, a space for none. + pub text: String, + /// Group 42: the measurement, when the file states it. + pub measurement: Option, + /// Group 50: the angle of a rotated dimension, in degrees. + pub rotation: f64, + /// Group 52: the oblique angle of the extension lines, in degrees. + pub oblique: f64, + /// Group 53: the text's rotation, in degrees. + pub text_rotation: f64, + /// Group 51: the horizontal direction, in degrees. + pub horizontal: f64, + /// Group 40: the leader length of a radius or diameter dimension. + pub leader_length: f64, +} + +impl Dimension { + /// The type: the low four bits of the flags. + pub fn kind(&self) -> i64 { + self.flags & 0x0f + } +} + +/// A LEADER. +#[derive(Clone, Debug, PartialEq)] +pub struct Leader { + pub vertices: Vec, + /// Group 71 is not 0. + pub arrow: bool, + /// Group 72 is not 0: a spline path. + pub spline: bool, + /// Group 3 (`Standard` when absent). + pub style: String, + /// Group 73: what the leader was made with (0 text, 1 a tolerance, 2 a block, 3 nothing). + pub annotation_type: i64, + /// Group 340: the annotation. + pub annotation: Option, +} + +/// A paper space VIEWPORT. +#[derive(Clone, Debug, PartialEq)] +pub struct Viewport { + /// Group 10: the centre on the sheet. + pub center: Point, + /// Groups 40 and 41: the size on the sheet. + pub width: f64, + pub height: f64, + /// Group 69: the id (1 is the sheet's own view). Other programs write -1 for unknown. + pub id: i64, + /// Group 90: the status flags (16384 locked, 65536 clipped by the entity of group 340, + /// 131072 off). + pub status: i64, + /// Groups 12/22: the view centre in display coordinates, whose origin is the target. + pub view_center: [f64; 2], + /// Groups 16/26/36 and 17/27/37. + pub view_direction: Point, + pub view_target: Point, + /// Group 45. + pub view_height: f64, + /// Group 51: the twist, in degrees. + pub twist: f64, + /// Group 340: the entity that clips the viewport. + pub clip: Option, + /// Groups 331 and 341: the layers frozen in the viewport. + pub frozen_layers: Vec, +} + +/// An IMAGE or WIPEOUT: a raster's place and clip. The file is named by the IMAGEDEF object. +#[derive(Clone, Debug, PartialEq)] +pub struct Image { + /// Group 10: the lower-left corner. + pub insert: Point, + /// Groups 11 and 12: the vectors of one pixel along the image's width and height. + pub u: Point, + pub v: Point, + /// Groups 13/23: the size in pixels. + pub size: [f64; 2], + /// Group 340: the IMAGEDEF. + pub image_def: Option, + /// Group 70: 1 show, 2 show when not aligned with the screen, 4 use the clip, 8 transparency. + pub display: i64, + /// Group 280 is not 0. + pub clipping: bool, + /// Groups 281, 282 and 283 (50, 50 and 0 when absent). + pub brightness: i64, + pub contrast: i64, + pub fade: i64, + /// Group 71: 1 a rectangle (two corners), 2 a polygon. + pub clip_type: i64, + /// Groups 14/24: the clip boundary in pixels, with the origin at the image's top-left corner + /// and y down: a point `q` is at `insert + u * (q.x + 0.5) + v * (0.5 - q.y)`. + pub clip: Vec<[f64; 2]>, + /// Group 290 is not 0: the clip keeps the outside. + pub clip_inverted: bool, +} + +/// The geometry of an entity. +#[derive(Clone, Debug, PartialEq)] +pub enum EntityKind { + Line { + start: Point, + end: Point, + }, + /// `angle` is group 50: the X axis of the point's symbol, in degrees. + Point { + location: Point, + angle: f64, + }, + Circle { + center: Point, + radius: f64, + }, + Arc { + center: Point, + radius: f64, + start_angle: f64, + end_angle: f64, + }, + /// `major` is the end of the major axis from the centre; `start` and `end` are parameters in + /// radians (0 and 2π for a full ellipse). + Ellipse { + center: Point, + major: Point, + ratio: f64, + start: f64, + end: f64, + }, + LwPolyline(LwPolyline), + Polyline(Polyline), + Spline(Spline), + Ray { + base: Point, + direction: Point, + }, + XLine { + base: Point, + direction: Point, + }, + Text(Text), + MText(MText), + AttDef(Attrib), + Insert(Insert), + Dimension(Dimension), + Leader(Leader), + /// The fourth corner repeats the third when the file has three. + Solid { + corners: [Point; 4], + }, + Trace { + corners: [Point; 4], + }, + /// `hidden_edges` is group 70: bit n hides edge n. + Face3d { + corners: [Point; 4], + hidden_edges: i64, + }, + Hatch(Hatch), + Viewport(Viewport), + Image(Image), + Wipeout(Image), + /// `text` is the feature control frame's string, as written. + Tolerance { + insert: Point, + direction: Point, + style: String, + text: String, + }, + /// A type this module does not model (or a HATCH with no boundary data): read it from the + /// record's tags. + Other, +} + +/// An entity: its properties and geometry. +#[derive(Clone, Debug, PartialEq)] +pub struct Entity { + pub common: Common, + pub kind: EntityKind, +} + +fn lwpolyline(tags: &[Tag], g: &Groups<'_>) -> LwPolyline { + let mut vertices: Vec = Vec::new(); + for t in tags { + if t.code == 10 { + vertices.push(LwVertex { x: t.f64(), ..LwVertex::default() }); + } else if let Some(v) = vertices.last_mut() { + match t.code { + 20 => v.y = t.f64(), + 40 => v.start_width = t.f64(), + 41 => v.end_width = t.f64(), + 42 => v.bulge = t.f64(), + _ => {} + } + } + } + let flags = g.i_or(70, 0); + LwPolyline { vertices, closed: flags & 1 != 0, plinegen: flags & 128 != 0, const_width: g.f_or(43, 0.0), elevation: g.f_or(38, 0.0) } +} + +fn polyline(g: &Groups<'_>, followers: &[Record<'_>]) -> Polyline { + let vertices = followers + .iter() + .map(|r| { + let v = r.groups(); + Vertex { + location: v.point(10), + bulge: v.f_or(42, 0.0), + start_width: v.f_or(40, 0.0), + end_width: v.f_or(41, 0.0), + flags: v.i_or(70, 0), + face: [71, 72, 73, 74].map(|c| v.i_or(c, 0)), + } + }) + .collect(); + Polyline { + flags: g.i_or(70, 0), + elevation: g.f_or(30, 0.0), + start_width: g.f_or(40, 0.0), + end_width: g.f_or(41, 0.0), + m: g.i_or(71, 0), + n: g.i_or(72, 0), + smooth: g.i_or(75, 0), + vertices, + } +} + +fn spline(g: &Groups<'_>) -> Spline { + Spline { + degree: g.i_or(71, 3), + flags: g.i_or(70, 0), + knots: g.all_f(40), + control: g.points(10), + weights: g.all_f(41), + fit: g.points(11), + start_tangent: g.point_opt(12), + end_tangent: g.point_opt(13), + fit_tolerance: g.f_or(44, 0.0), + } +} + +fn insert(g: &Groups<'_>, followers: &[Record<'_>]) -> Insert { + Insert { + block: g.s(2).unwrap_or_default(), + insert: g.point(10), + scale: [g.f_or(41, 1.0), g.f_or(42, 1.0), g.f_or(43, 1.0)], + rotation: g.f_or(50, 0.0), + columns: g.i_or(70, 1).max(1), + rows: g.i_or(71, 1).max(1), + column_spacing: g.f_or(44, 0.0), + row_spacing: g.f_or(45, 0.0), + attribs: followers.iter().map(|r| (Common::read(r.tags), Attrib::read("ATTRIB", r.tags))).collect(), + } +} + +fn dimension(g: &Groups<'_>) -> Dimension { + Dimension { + flags: g.i_or(70, 0), + block: g.name(2), + style: g.name(3).unwrap_or_else(|| "Standard".into()), + defpt: g.point(10), + text_mid: g.point(11), + insertion: g.point(12), + p13: g.point(13), + p14: g.point(14), + p15: g.point(15), + p16: g.point(16), + text: decode_line(&g.s(1).unwrap_or_default()), + measurement: g.f(42), + rotation: g.f_or(50, 0.0), + oblique: g.f_or(52, 0.0), + text_rotation: g.f_or(53, 0.0), + horizontal: g.f_or(51, 0.0), + leader_length: g.f_or(40, 0.0), + } +} + +fn viewport(tags: &[Tag], g: &Groups<'_>) -> Viewport { + Viewport { + center: g.point(10), + width: g.f_or(40, 0.0), + height: g.f_or(41, 0.0), + id: g.i_or(69, 0), + status: g.i_or(90, 0), + view_center: [g.f_or(12, 0.0), g.f_or(22, 0.0)], + view_direction: g.point_opt(16).unwrap_or([0.0, 0.0, 1.0]), + view_target: g.point(17), + view_height: g.f_or(45, 0.0), + twist: g.f_or(51, 0.0), + clip: g.handle(340).filter(|h| *h != 0), + frozen_layers: tags.iter().filter(|t| matches!(t.code, 331 | 341)).filter_map(|t| parse_handle(&t.str())).collect(), + } +} + +fn image(g: &Groups<'_>) -> Image { + Image { + insert: g.point(10), + u: g.point(11), + v: g.point(12), + size: [g.f_or(13, 0.0), g.f_or(23, 0.0)], + image_def: g.handle(340).filter(|h| *h != 0), + display: g.i_or(70, 0), + clipping: g.i_or(280, 0) != 0, + brightness: g.i_or(281, 50), + contrast: g.i_or(282, 50), + fade: g.i_or(283, 0), + clip_type: g.i_or(71, 1), + clip: g.points(14).into_iter().map(|p| [p[0], p[1]]).collect(), + clip_inverted: g.i_or(290, 0) != 0, + } +} + +fn corners(g: &Groups<'_>) -> [Point; 4] { + [g.point(10), g.point(11), g.point(12), g.point_opt(13).unwrap_or_else(|| g.point(12))] +} + +impl Entity { + /// The typed form of an entity record. + pub fn read(e: &EntityRecord<'_>) -> Entity { + let g = e.groups(); + let line = |g: &Groups<'_>| (g.point(10), g.point(11)); + let kind = match e.kind { + "LINE" => EntityKind::Line { start: line(&g).0, end: line(&g).1 }, + "POINT" => EntityKind::Point { location: g.point(10), angle: g.f_or(50, 0.0) }, + "CIRCLE" => EntityKind::Circle { center: g.point(10), radius: g.f_or(40, 0.0) }, + "ARC" => EntityKind::Arc { center: g.point(10), radius: g.f_or(40, 0.0), start_angle: g.f_or(50, 0.0), end_angle: g.f_or(51, 360.0) }, + "ELLIPSE" => EntityKind::Ellipse { + center: g.point(10), + major: g.point(11), + ratio: g.f_or(40, 1.0), + start: g.f_or(41, 0.0), + end: g.f_or(42, std::f64::consts::TAU), + }, + "LWPOLYLINE" => EntityKind::LwPolyline(lwpolyline(e.tags, &g)), + "POLYLINE" => EntityKind::Polyline(polyline(&g, &e.followers)), + "SPLINE" => EntityKind::Spline(spline(&g)), + "RAY" => EntityKind::Ray { base: line(&g).0, direction: line(&g).1 }, + "XLINE" => EntityKind::XLine { base: line(&g).0, direction: line(&g).1 }, + "TEXT" => EntityKind::Text(Text::read(&g)), + "MTEXT" => EntityKind::MText(MText::read(e.tags)), + "ATTDEF" => EntityKind::AttDef(Attrib::read("ATTDEF", e.tags)), + "INSERT" => EntityKind::Insert(insert(&g, &e.followers)), + "DIMENSION" => EntityKind::Dimension(dimension(&g)), + "LEADER" => EntityKind::Leader(Leader { + vertices: g.points(10), + arrow: g.i_or(71, 1) != 0, + spline: g.i_or(72, 0) != 0, + style: g.name(3).unwrap_or_else(|| "Standard".into()), + annotation_type: g.i_or(73, 3), + annotation: g.handle(340).filter(|h| *h != 0), + }), + "SOLID" => EntityKind::Solid { corners: corners(&g) }, + "TRACE" => EntityKind::Trace { corners: corners(&g) }, + "3DFACE" => EntityKind::Face3d { corners: corners(&g), hidden_edges: g.i_or(70, 0) }, + "HATCH" => Hatch::read(e.tags).map_or(EntityKind::Other, EntityKind::Hatch), + "VIEWPORT" => EntityKind::Viewport(viewport(e.tags, &g)), + "IMAGE" => EntityKind::Image(image(&g)), + "WIPEOUT" => EntityKind::Wipeout(image(&g)), + "TOLERANCE" => EntityKind::Tolerance { + insert: g.point(10), + direction: g.point_opt(11).unwrap_or([1.0, 0.0, 0.0]), + style: g.name(3).unwrap_or_else(|| "Standard".into()), + text: g.s(1).unwrap_or_default(), + }, + _ => EntityKind::Other, + }; + Entity { common: Common::read(e.tags), kind } + } +} diff --git a/crates/dxf/src/model/hatch.rs b/crates/dxf/src/model/hatch.rs new file mode 100644 index 000000000..ac3425622 --- /dev/null +++ b/crates/dxf/src/model/hatch.rs @@ -0,0 +1,355 @@ +//! HATCH: the boundary loops (polylines or edges), the pattern definition, the seed points and +//! the gradient, as the file holds them. Boundary points are 2D, in the object coordinate system +//! of the hatch at its elevation. + +use super::{Color, Groups, parse_handle}; +use crate::Tag; + +/// One edge of a boundary loop made of edges. +#[derive(Clone, Debug, PartialEq)] +pub enum HatchEdge { + Line { + start: [f64; 2], + end: [f64; 2], + }, + /// Angles in degrees; a clockwise arc (`ccw` false) has them measured clockwise. + Arc { + center: [f64; 2], + radius: f64, + start_angle: f64, + end_angle: f64, + ccw: bool, + }, + /// `major` is the end of the major axis from the centre, `ratio` the minor axis's length + /// over the major's; angles in degrees, as for an arc. + Ellipse { + center: [f64; 2], + major: [f64; 2], + ratio: f64, + start_angle: f64, + end_angle: f64, + ccw: bool, + }, + Spline { + degree: i64, + rational: bool, + periodic: bool, + knots: Vec, + control: Vec<[f64; 2]>, + /// One per control point when rational; else empty. + weights: Vec, + fit: Vec<[f64; 2]>, + start_tangent: Option<[f64; 2]>, + end_tangent: Option<[f64; 2]>, + }, +} + +/// The path of a boundary loop. +#[derive(Clone, Debug, PartialEq)] +pub enum HatchPath { + /// Vertices as x, y and bulge. + Polyline { + closed: bool, + vertices: Vec<[f64; 3]>, + }, + Edges(Vec), +} + +/// A boundary loop. +#[derive(Clone, Debug, PartialEq)] +pub struct HatchLoop { + /// Group 92: 1 external, 2 a polyline, 4 derived, 8 a text box, 16 outermost. + pub flags: i64, + pub path: HatchPath, + /// Groups 330 after the loop's 97: the entities an associative hatch follows. + pub sources: Vec, +} + +/// One line family of a pattern definition, as drawn: scaled and rotated with the hatch. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct PatternLine { + /// Group 53, in degrees. + pub angle: f64, + /// Groups 43/44: a point the first line passes through. + pub base: [f64; 2], + /// Groups 45/46: the step from one line to the next, as a vector. + pub offset: [f64; 2], + /// Groups 49: dash (positive), gap (negative) and dot (0) lengths; empty for a solid line. + pub dashes: Vec, +} + +/// A gradient fill (groups 450–470). +#[derive(Clone, Debug, PartialEq)] +pub struct Gradient { + /// Group 470 (`LINEAR`, `CYLINDER`, `SPHERICAL`...). + pub name: String, + /// Group 460, in radians. + pub angle: f64, + /// Group 461: 0 centred, 1 shifted up and left. + pub shift: f64, + /// Group 452 is 1: one colour, shaded by `tint` (group 462). + pub single: bool, + pub tint: f64, + /// The colour of each 463 record: its true colour (421), else its index (63). + pub colors: Vec, +} + +/// A HATCH. +#[derive(Clone, Debug, PartialEq)] +pub struct Hatch { + /// Group 2, as written (`SOLID` when absent). + pub pattern: String, + /// Group 70 is 1. + pub solid: bool, + /// Group 71 is not 0. + pub associative: bool, + /// Group 75: 0 normal (odd parity), 1 outer, 2 ignore islands. + pub style: i64, + /// Group 76: 0 user-defined, 1 predefined, 2 custom. + pub pattern_type: i64, + /// Group 52, in degrees. + pub angle: f64, + /// Group 41 (1 when absent). + pub scale: f64, + /// Group 77 is not 0: a user-defined pattern is doubled (crossed). + pub double: bool, + /// The z of the elevation point (the first group 30). + pub elevation: f64, + pub loops: Vec, + pub pattern_lines: Vec, + /// Groups 10/20 after the 98 count. + pub seeds: Vec<[f64; 2]>, + pub gradient: Option, +} + +/// A cursor over a record's tags, for the boundary data, which is read in order. +struct Cur<'a> { + tags: &'a [Tag], + at: usize, +} + +impl<'a> Cur<'a> { + fn peek(&self) -> Option<&'a Tag> { + self.tags.get(self.at) + } + /// The next tag when its code is `code`, consumed. + fn take(&mut self, code: i32) -> Option<&'a Tag> { + let t = self.peek().filter(|t| t.code == code)?; + self.at += 1; + Some(t) + } + fn f(&mut self, code: i32) -> f64 { + self.take(code).map_or(0.0, Tag::f64) + } + fn xy(&mut self, code: i32) -> [f64; 2] { + [self.f(code), self.f(code + 10)] + } + /// Move to the next `code`; false when there is none. + fn seek(&mut self, code: i32) -> bool { + while self.peek().is_some_and(|t| t.code != code) { + self.at += 1; + } + self.peek().is_some() + } +} + +/// A declared count: never trusted for an allocation, only as the most items to read. +fn count(t: &Tag) -> usize { + usize::try_from(t.i64()).unwrap_or(0) +} + +fn polyline_path(c: &mut Cur<'_>) -> Option { + // 72 has bulges, 73 closed, 93 the vertex count. + let mut closed = false; + while let Some(t) = c.peek().filter(|t| t.code != 93) { + if t.code == 73 { + closed = t.i64() != 0; + } else if t.code != 72 { + return None; + } + c.at += 1; + } + let n = count(c.take(93)?); + let mut vertices = Vec::new(); + while vertices.len() < n && c.peek().is_some_and(|t| t.code == 10) { + let [x, y] = c.xy(10); + vertices.push([x, y, c.f(42)]); + } + Some(HatchPath::Polyline { closed, vertices }) +} + +fn spline_edge(c: &mut Cur<'_>) -> HatchEdge { + let (mut degree, mut rational, mut periodic, mut knot_count, mut control_count) = (3, false, false, 0, 0); + while let Some(t) = c.peek().filter(|t| matches!(t.code, 94 | 73 | 74 | 95 | 96)) { + match t.code { + 94 => degree = t.i64(), + 73 => rational = t.i64() != 0, + 74 => periodic = t.i64() != 0, + 95 => knot_count = count(t), + _ => control_count = count(t), + } + c.at += 1; + } + let mut knots = Vec::new(); + while knots.len() < knot_count && c.peek().is_some_and(|t| t.code == 40) { + knots.push(c.f(40)); + } + let (mut control, mut weights) = (Vec::new(), Vec::new()); + while control.len() < control_count && c.peek().is_some_and(|t| t.code == 10) { + control.push(c.xy(10)); + weights.extend(c.take(42).map(Tag::f64)); + } + if weights.len() != control.len() { + weights.clear(); + } + // A 97 here is the edge's fit point count (R2010 and later) when fit points follow it, or + // when it is 0 and the loop's own 97 comes next; otherwise it is the loop's source count. + let mut fit = Vec::new(); + let next = c.tags.get(c.at + 1).map(|t| t.code); + let fit_count = c.peek().filter(|t| t.code == 97 && (next == Some(11) || (t.i64() == 0 && next == Some(97)))); + if let Some(n) = fit_count.map(count) { + c.at += 1; + while fit.len() < n && c.peek().is_some_and(|t| t.code == 11) { + fit.push(c.xy(11)); + } + } + let tangent = |c: &mut Cur<'_>, code: i32| c.peek().is_some_and(|t| t.code == code).then(|| c.xy(code)); + let start_tangent = tangent(c, 12); + let end_tangent = tangent(c, 13); + HatchEdge::Spline { degree, rational, periodic, knots, control, weights, fit, start_tangent, end_tangent } +} + +fn edge_path(c: &mut Cur<'_>) -> Option { + if !c.seek(93) { + return None; + } + let n = count(c.take(93)?); + let mut edges = Vec::new(); + while edges.len() < n { + // Groups this reader does not know may sit between edges; the loop's own end may not. + while c.peek().is_some_and(|t| !matches!(t.code, 72 | 97 | 92 | 75)) { + c.at += 1; + } + if c.peek().is_none_or(|t| t.code != 72) { + break; + } + let kind = c.take(72).map_or(0, Tag::i64); + // 73 is counter-clockwise unless it says 0. + let ccw = |c: &mut Cur<'_>| c.take(73).is_none_or(|t| t.i64() != 0); + edges.push(match kind { + 1 => HatchEdge::Line { start: c.xy(10), end: c.xy(11) }, + 2 => HatchEdge::Arc { center: c.xy(10), radius: c.f(40), start_angle: c.f(50), end_angle: c.f(51), ccw: ccw(c) }, + 3 => HatchEdge::Ellipse { center: c.xy(10), major: c.xy(11), ratio: c.f(40), start_angle: c.f(50), end_angle: c.f(51), ccw: ccw(c) }, + 4 => spline_edge(c), + // An edge type the format does not have: its groups are passed over above. + _ => continue, + }); + } + Some(HatchPath::Edges(edges)) +} + +fn pattern_lines(tags: &[Tag]) -> Vec { + let Some(at) = tags.iter().position(|t| t.code == 78) else { return Vec::new() }; + let mut lines: Vec = Vec::new(); + for t in tags.iter().skip(at + 1) { + if t.code == 53 { + lines.push(PatternLine { angle: t.f64(), ..PatternLine::default() }); + continue; + } + let Some(l) = lines.last_mut() else { break }; + match t.code { + 43 => l.base[0] = t.f64(), + 44 => l.base[1] = t.f64(), + 45 => l.offset[0] = t.f64(), + 46 => l.offset[1] = t.f64(), + 79 => {} + 49 => l.dashes.push(t.f64()), + _ => break, + } + } + lines +} + +/// Groups 450–470, when 450 says gradient. Each 463 starts a colour record whose 63 (index) and +/// 421 (true colour) follow. +fn gradient(tags: &[Tag]) -> Option { + let start = tags.iter().position(|t| t.code == 450)?; + if tags.get(start)?.i64() != 1 { + return None; + } + let mut g = Gradient { name: String::new(), angle: 0.0, shift: 0.0, single: false, tint: 0.0, colors: Vec::new() }; + for t in tags.iter().skip(start + 1).take_while(|t| t.code != 1001) { + match t.code { + 452 => g.single = t.i64() == 1, + 460 => g.angle = t.f64(), + 461 => g.shift = t.f64(), + 462 => g.tint = t.f64(), + 463 => g.colors.push(Color::ByLayer), + 63 => { + // A true colour already read for this record wins over its index. + if let Some(c) = g.colors.last_mut().filter(|c| !matches!(c, Color::Rgb(_))) { + *c = Color::from_index(t.i64()); + } + } + 421 => { + if let Some(c) = g.colors.last_mut() { + *c = Color::from_true(t.i64()); + } + } + 470 => g.name = t.str(), + _ => {} + } + } + Some(g) +} + +impl Hatch { + /// The typed form of a HATCH record; `None` when it has no boundary data (no group 91) or + /// the data ends inside a loop. + pub fn read(tags: &[Tag]) -> Option { + let g = Groups(tags); + let n_loops = count(tags.iter().find(|t| t.code == 91)?); + let mut c = Cur { tags, at: tags.iter().position(|t| t.code == 91)? + 1 }; + let mut loops = Vec::new(); + while loops.len() < n_loops { + if !c.seek(92) { + return None; + } + let flags = c.take(92)?.i64(); + let path = if flags & 2 != 0 { polyline_path(&mut c)? } else { edge_path(&mut c)? }; + // 97 counts the source entities (330) that follow. + let mut sources = Vec::new(); + if let Some(n) = c.take(97).map(count) { + while sources.len() < n + && let Some(t) = c.take(330) + { + sources.extend(parse_handle(&t.str()).filter(|h| *h != 0)); + } + } + loops.push(HatchLoop { flags, path, sources }); + } + let mut seeds = Vec::new(); + if c.seek(98) { + let n = c.take(98).map_or(0, count); + while seeds.len() < n && c.peek().is_some_and(|t| t.code == 10) { + seeds.push(c.xy(10)); + } + } + let pattern = g.name(2).unwrap_or_else(|| "SOLID".into()); + Some(Hatch { + solid: g.i_or(70, 0) == 1, + associative: g.i_or(71, 0) != 0, + style: g.i_or(75, 0), + pattern_type: g.i_or(76, 1), + angle: g.f_or(52, 0.0), + scale: g.f_or(41, 1.0), + double: g.i_or(77, 0) != 0, + elevation: g.f_or(30, 0.0), + loops, + pattern_lines: pattern_lines(tags), + seeds, + gradient: gradient(tags), + pattern, + }) + } +} diff --git a/crates/dxf/src/model/text.rs b/crates/dxf/src/model/text.rs new file mode 100644 index 000000000..ffafae96d --- /dev/null +++ b/crates/dxf/src/model/text.rs @@ -0,0 +1,280 @@ +//! Text in DXF: the caret encoding of control characters in strings, and MTEXT with its columns. +//! +//! Control characters are written as a caret and a letter (`^I` a tab, `^J` a line feed, `^ ` a +//! caret). TEXT, ATTDEF, ATTRIB and DIMENSION strings use the same encoding as MTEXT contents, +//! without line feeds: AutoCAD reads a bare `^2` in `10^2` as a control character. +//! +//! MTEXT columns are groups 75 (type), 76 (count), 78 (flow reversed), 79 (auto height), 48 +//! (width), 49 (gutter) and 50 (column heights, after the column type; before it, 50 is the +//! rotation) before R2018, and an embedded object (group 101) from R2018 on. + +use super::{Groups, Point, outside_app}; +use crate::Tag; + +/// Decode MTEXT contents: `^I` (tab), `^J` (line feed) and `^ ` (caret). Other carets stay as +/// they are (older files write stacked tolerances `\S+0.1^-0.1;` with a bare caret). +pub fn decode_contents(s: &str) -> String { + decode(s, true) +} + +/// A single-line string as read: [`decode_contents`] without line feeds (`^J` stays as it is). +pub fn decode_line(s: &str) -> String { + decode(s, false) +} + +fn decode(s: &str, line_feeds: bool) -> String { + if !s.contains('^') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut it = s.chars().peekable(); + while let Some(c) = it.next() { + if c != '^' { + out.push(c); + continue; + } + match it.peek().copied() { + Some(' ') => out.push('^'), + Some('I') => out.push('\t'), + Some('J') if line_feeds => out.push('\n'), + _ => { + out.push('^'); + continue; + } + } + it.next(); + } + out +} + +/// A single-line string (TEXT, ATTDEF/ATTRIB, DIMENSION) for a DXF file: a caret as `^ ` and a +/// tab as `^I`, read back by [`decode_line`]. It must hold no line break. +pub fn encode_line(s: &str) -> String { + s.replace('^', "^ ").replace('\t', "^I") +} + +/// Encode MTEXT contents: a tab as `^I`, a line feed as `^J`, a caret as `^ `; other control +/// characters (which draw nothing) are dropped. +pub fn encode_contents(s: &str) -> String { + if !s.chars().any(|c| c == '^' || c.is_control()) { + return s.to_string(); + } + let mut out = String::with_capacity(s.len() + 8); + for c in s.chars() { + match c { + '^' => out.push_str("^ "), + '\t' => out.push_str("^I"), + '\n' => out.push_str("^J"), + c if c.is_control() => {} + c => out.push(c), + } + } + out +} + +/// Apply `f` ([`encode_line`] or [`decode_line`]) to the groups of a `kind` record that hold an +/// encoded string: the text (1), and an ATTDEF's prompt (3). An embedded MTEXT (from group 101) +/// has its own and is left alone. +pub fn map_text_groups(kind: &str, tags: &mut [Tag], f: fn(&str) -> String) { + let kind = kind.to_ascii_uppercase(); + if !matches!(kind.as_str(), "TEXT" | "ATTRIB" | "ATTDEF" | "DIMENSION") { + return; + } + for t in tags.iter_mut().take_while(|t| t.code != 101) { + if t.code == 1 || (t.code == 3 && kind == "ATTDEF") { + *t = Tag::s(t.code, f(&t.str())); + } + } +} + +/// A record's groups split at its embedded object (`101 Embedded Object`): its own groups and, +/// when it has one, the object's. +pub fn split_embedded(tags: &[Tag]) -> (&[Tag], Option<&[Tag]>) { + match tags.iter().position(|t| t.code == 101) { + Some(i) => (tags.get(..i).unwrap_or(tags), tags.get(i + 1..)), + None => (tags, None), + } +} + +/// Upper bound on the column heights read from one MTEXT. +pub const MAX_COLUMNS: usize = 1_000; + +/// The columns of an MTEXT. +#[derive(Clone, Debug, PartialEq)] +pub struct MTextColumns { + /// Dynamic columns (type 2): the text flows into as many as it needs. Otherwise static + /// (type 1). + pub dynamic: bool, + pub count: u16, + /// One column's width (the MTEXT's own width is the total in some writers' files). + pub width: f64, + pub gutter: f64, + /// The column height: the tallest for manual heights. + pub height: f64, + pub auto_height: bool, + /// Each column's height, for dynamic columns with manual heights; else empty. + pub heights: Vec, + pub flow_reversed: bool, +} + +fn pos(v: f64) -> f64 { + v.max(0.0) +} + +impl MTextColumns { + /// The columns of an MTEXT record, from the legacy groups or the embedded object. + pub fn read(tags: &[Tag]) -> Option { + let (own, embedded) = split_embedded(tags); + // An application-defined group in the record may hold any code, 50 included. + let own_tags: Vec<&Tag> = outside_app(own).map(|(_, t)| t).collect(); + let Some(start) = own_tags.iter().position(|t| t.code == 75) else { return Self::read_embedded(own, embedded?) }; + let dynamic = match own_tags.get(start).map(|t| t.i64()) { + Some(1) => false, + Some(2) => true, + _ => return None, + }; + let cols = own_tags.get(start..).unwrap_or_default(); + let first = |code: i32| cols.iter().find(|t| t.code == code).copied(); + // The first 50 is the number of heights that follow, unless the file has the heights + // of group 76 alone. + let fifties = cols.iter().filter(|t| t.code == 50); + let counted = first(76).is_some_and(|t| i64::try_from(fifties.clone().count()).is_ok_and(|n| n - 1 == t.i64())); + let heights: Vec = fifties.skip(usize::from(counted)).take(MAX_COLUMNS).map(|t| pos(t.f64())).collect(); + let count = first(76).map_or(heights.len().max(1), |t| usize::try_from(t.i64()).unwrap_or(1)).clamp(1, MAX_COLUMNS); + let auto_height = first(79).is_some_and(|t| t.i64() != 0); + let defined = own_tags.iter().find(|t| t.code == 46).map_or(0.0, |t| pos(t.f64())); + let manual = dynamic && !auto_height; + let height = + if manual { heights.iter().copied().fold(0.0, f64::max) } else { heights.first().copied().filter(|h| *h > 0.0).unwrap_or(defined) }; + Some(MTextColumns { + dynamic, + count: u16::try_from(count).unwrap_or(1), + width: first(48).map_or(0.0, |t| pos(t.f64())), + gutter: first(49).map_or(0.0, |t| pos(t.f64())), + height, + auto_height, + heights: if manual { heights } else { Vec::new() }, + flow_reversed: first(78).is_some_and(|t| t.i64() != 0), + }) + } + + /// The columns of an R2018 MTEXT, from its embedded object, whose groups reuse MTEXT's + /// codes: 71 type, 72 count, 44 column width, 45 gutter, 73 auto height, 74 flow reversed, + /// 46 heights. + #[expect(clippy::cast_possible_truncation, reason = "the rounded count is clamped to 1..=MAX_COLUMNS before use")] + fn read_embedded(own: &[Tag], obj: &[Tag]) -> Option { + let first = |code: i32| obj.iter().find(|t| t.code == code); + let dynamic = match first(71).map(Tag::i64) { + Some(1) => false, + Some(2) => true, + _ => return None, + }; + let heights: Vec = obj.iter().filter(|t| t.code == 46).take(MAX_COLUMNS).map(|t| pos(t.f64())).collect(); + let auto_height = first(73).is_some_and(|t| t.i64() != 0); + let manual = dynamic && !auto_height; + // The column height: the entity's own 46 (defined height), else the object's 41 or first 46. + let own_46 = own.iter().find(|t| t.code == 46).map(|t| pos(t.f64())); + let defined = [own_46, first(41).map(Tag::f64), heights.first().copied()].into_iter().flatten().find(|h| *h > 0.0).unwrap_or(0.0); + // A dynamic auto-height object stores 0 columns in 72: the count follows from the total + // width (42 = n * 44 + (n - 1) * 45). + let [width, gutter, total] = [44, 45, 42].map(|g| first(g).map_or(0.0, Tag::f64)); + let count = match first(72).map(Tag::i64) { + Some(n) if n > 0 => n, + Some(_) if width + gutter > 0.0 && total > 0.0 => ((total + gutter) / (width + gutter)).round().clamp(1.0, MAX_COLUMNS as f64) as i64, + _ => 1, + }; + Some(MTextColumns { + dynamic, + count: u16::try_from(count.clamp(1, MAX_COLUMNS as i64)).unwrap_or(1), + width: pos(width), + gutter: pos(gutter), + height: if manual { heights.iter().copied().fold(0.0, f64::max) } else { defined }, + auto_height, + heights: if manual { heights } else { Vec::new() }, + flow_reversed: first(74).is_some_and(|t| t.i64() != 0), + }) + } +} + +/// Multiline text: an MTEXT entity, or the one embedded in a multiline attribute. +#[derive(Clone, Debug, PartialEq)] +pub struct MText { + /// Group 10. + pub insert: Point, + /// Group 40: the text height. + pub height: f64, + /// Group 41: the reference rectangle's width (0: none). With columns, see + /// [`MTextColumns::width`]. + pub width: f64, + /// Group 71: 1–9, top left to bottom right. + pub attach: u8, + /// Group 72: 1 left to right, 3 top to bottom, 5 by style. + pub drawing_direction: i64, + /// Groups 11/21/31: the X axis direction, which wins over the rotation when given and not + /// zero. + pub direction: Option, + /// Group 50, in degrees. + pub rotation: f64, + /// Group 7 (`Standard` when absent). + pub style: String, + /// The contents (the group 3 chunks, then group 1), decoded ([`decode_contents`]); the + /// formatting codes (`\P`, `\f...;`) are left as written. + pub contents: String, + /// Group 44: the line spacing factor (1 when absent). + pub line_spacing: f64, + /// Group 73 is 2: the spacing is exact, not "at least". + pub line_spacing_exact: bool, + /// Group 90: the background fill flags (1 a fill colour, 2 the window's colour, 16 a + /// frame), with the fill colour (63, 421) and the border offset factor (45). + pub background: Option<(i64, super::Color, f64)>, + pub columns: Option, +} + +impl MText { + /// The typed form of MTEXT groups. + pub fn read(tags: &[Tag]) -> MText { + let columns = MTextColumns::read(tags); + // The MTEXT's own groups: an embedded object reuses their codes (44 is its column + // width, not the line spacing factor). + let (own, _) = split_embedded(tags); + let g = Groups(own); + let mut contents = String::new(); + for t in own.iter().filter(|t| t.code == 3) { + contents.push_str(&t.str()); + } + contents.push_str(&g.s(1).unwrap_or_default()); + // The direction and rotation come from before the column groups, where 50 is a height. + let head = Groups(own.get(..own.iter().position(|t| t.code == 75).unwrap_or(own.len())).unwrap_or(own)); + let background = g.i(90).filter(|f| *f != 0).map(|f| { + let color = match (g.i(421), g.i(63)) { + (Some(tc), _) => super::Color::from_true(tc), + (None, Some(v)) => super::Color::from_index(v), + (None, None) => super::Color::ByLayer, + }; + (f, color, g.f_or(45, 1.5)) + }); + MText { + insert: g.point(10), + height: g.f_or(40, 0.0), + width: pos(g.f_or(41, 0.0)), + attach: u8::try_from(g.i_or(71, 1).clamp(1, 9)).unwrap_or(1), + drawing_direction: g.i_or(72, 1), + direction: head.point_opt(11).filter(|d| d[0] != 0.0 || d[1] != 0.0 || d[2] != 0.0), + rotation: head.f_or(50, 0.0), + style: g.name(7).unwrap_or_else(|| "Standard".into()), + contents: decode_contents(&contents), + line_spacing: g.f_or(44, 1.0), + line_spacing_exact: g.i(73) == Some(2), + background, + columns, + } + } + /// The angle of the text's X axis in radians: of the direction when the record has one, + /// else the rotation. + pub fn angle(&self) -> f64 { + match self.direction { + Some(d) if d[0] != 0.0 || d[1] != 0.0 => d[1].atan2(d[0]), + _ => self.rotation.to_radians(), + } + } +} diff --git a/crates/dxf/tests/model_entities.rs b/crates/dxf/tests/model_entities.rs new file mode 100644 index 000000000..5de59761b --- /dev/null +++ b/crates/dxf/tests/model_entities.rs @@ -0,0 +1,393 @@ +//! The drawing model's entities: common properties, geometry, text, hatches, dimensions. + +use craft_dxf::model::text::{MTextColumns, decode_contents, decode_line, encode_contents, encode_line, map_text_groups}; +use craft_dxf::model::{Color, Document, Entity, EntityKind, HatchEdge, HatchPath, Lineweight, Transparency, entity_records, records}; +use craft_dxf::{Tag, parse}; + +fn tags(text: &str) -> Vec { + let lines: Vec<&str> = text.split(['\n', '|']).map(str::trim).filter(|l| !l.is_empty()).map(|l| if l == "~" { "" } else { l }).collect(); + parse((lines.join("\n") + "\n").as_bytes()).expect("the test drawing parses") +} + +/// The entities of `text`, the records of an ENTITIES section. +fn entities(text: &str) -> Vec { + let t = tags(text); + let (recs, strays) = entity_records(&records(&t)); + assert_eq!(strays, 0); + recs.iter().map(Entity::read).collect() +} + +fn one(text: &str) -> Entity { + let mut e = entities(text); + assert_eq!(e.len(), 1); + e.remove(0) +} + +#[test] +fn common_properties() { + let c = one("0|LINE|5|2A|330|1F|100|AcDbEntity|8|Walls|6|DASHED|62|1|420|65280|370|50|48|2.5|60|1|67|1|440|33554687|390|B|100|AcDbLine|39|1.5|210|0.0|220|0.0|230|-1.0").common; + assert_eq!((c.handle, c.owner, c.layer.as_str(), c.linetype.as_deref()), (Some(0x2A), Some(0x1F), "Walls", Some("DASHED"))); + assert_eq!((c.color, c.lineweight, c.ltscale), (Color::Rgb([0, 255, 0]), Lineweight::Hundredths(50), 2.5)); + assert!(!c.visible && c.paper_space); + assert_eq!((c.transparency, c.thickness, c.extrusion, c.plot_style), (Transparency::Alpha(255), 1.5, [0.0, 0.0, -1.0], Some(0xB))); + let d = one("0|LINE").common; + assert_eq!((d.handle, d.owner, d.layer.as_str(), d.linetype.as_deref(), d.color), (None, None, "0", None, Color::ByLayer)); + assert_eq!((d.lineweight, d.ltscale, d.visible, d.paper_space, d.extrusion), (Lineweight::ByLayer, 1.0, true, false, [0.0, 0.0, 1.0])); + assert_eq!(one("0|LINE|6|BYLAYER|48|-1.0|62|-2").common, d, "ByLayer by name, a negative scale and a negative colour are the defaults"); +} + +#[test] +fn entity_properties_are_not_read_from_the_entitys_own_subclass() { + // An MTEXT's 48 is its column width, not the linetype scale, and a marker in an + // application's braces is the application's: it does not end the entity's groups. A record with no AcDbEntity marker has its + // layer wherever it is. + let c = one("0|MTEXT|102|{APP|100|AcDbMText|102|}|100|AcDbEntity|8|A|100|AcDbMText|48|7.0|62|3").common; + assert_eq!((c.ltscale, c.color, c.layer.as_str()), (1.0, Color::ByLayer, "A")); + let c = one("0|LINE|100|AcDbLine|8|B|62|3").common; + assert_eq!((c.color, c.layer.as_str()), (Color::Index(3), "B")); + // An embedded object ends the entity's groups. + assert_eq!(one("0|ATTDEF|8|C|101|Embedded Object|62|5").common.color, Color::ByLayer); +} + +#[test] +fn curves() { + let k = |text: &str| one(text).kind; + assert_eq!(k("0|LINE|10|1.0|20|2.0|30|3.0|11|4.0|21|5.0"), EntityKind::Line { start: [1.0, 2.0, 3.0], end: [4.0, 5.0, 0.0] }); + assert_eq!(k("0|POINT|10|1.0|50|30.0"), EntityKind::Point { location: [1.0, 0.0, 0.0], angle: 30.0 }); + assert_eq!(k("0|CIRCLE|10|1.0|20|2.0|40|3.0"), EntityKind::Circle { center: [1.0, 2.0, 0.0], radius: 3.0 }); + assert_eq!(k("0|ARC|10|1.0|40|3.0|50|10.0"), EntityKind::Arc { center: [1.0, 0.0, 0.0], radius: 3.0, start_angle: 10.0, end_angle: 360.0 }); + assert_eq!( + k("0|ELLIPSE|10|1.0|11|2.0|21|0.0|40|0.5|41|0.0|42|3.0"), + EntityKind::Ellipse { center: [1.0, 0.0, 0.0], major: [2.0, 0.0, 0.0], ratio: 0.5, start: 0.0, end: 3.0 } + ); + assert_eq!(k("0|RAY|10|1.0|11|0.0|21|1.0"), EntityKind::Ray { base: [1.0, 0.0, 0.0], direction: [0.0, 1.0, 0.0] }); + assert_eq!(k("0|XLINE|10|1.0|11|0.0|21|1.0"), EntityKind::XLine { base: [1.0, 0.0, 0.0], direction: [0.0, 1.0, 0.0] }); + let EntityKind::Spline(s) = k( + "0|SPLINE|70|8|71|2|40|0.0|40|0.0|40|1.0|10|0.0|20|0.0|10|1.0|20|1.0|30|5.0|10|2.0|41|1.0|41|2.0|41|1.0|11|9.0|21|9.0|12|1.0|22|0.0|44|0.01", + ) else { + panic!("a spline") + }; + assert_eq!((s.degree, s.flags, s.knots.len(), s.weights.as_slice(), s.fit_tolerance), (2, 8, 3, [1.0, 2.0, 1.0].as_slice(), 0.01)); + assert_eq!(s.control, [[0.0, 0.0, 0.0], [1.0, 1.0, 5.0], [2.0, 0.0, 0.0]], "the third point has no y or z of its own"); + assert_eq!((s.fit.as_slice(), s.start_tangent, s.end_tangent), ([[9.0, 9.0, 0.0]].as_slice(), Some([1.0, 0.0, 0.0]), None)); +} + +#[test] +fn polylines() { + let EntityKind::LwPolyline(p) = one("0|LWPOLYLINE|90|3|70|129|43|0.5|38|2.0|10|0.0|20|0.0|42|1.0|10|4.0|20|0.0|40|0.1|41|0.2|10|4.0|20|3.0").kind + else { + panic!("a lightweight polyline") + }; + assert!(p.closed && p.plinegen); + assert_eq!((p.const_width, p.elevation, p.vertices.len()), (0.5, 2.0, 3)); + assert_eq!((p.vertices[0].bulge, p.vertices[1].start_width, p.vertices[1].end_width, p.vertices[2].y), (1.0, 0.1, 0.2, 3.0)); + // A polyface mesh: three vertices and a face record that names them. + let e = one( + "0|POLYLINE|66|1|70|64|71|3|72|1|10|0.0|20|0.0|30|7.0|0|VERTEX|10|0.0|20|0.0|30|0.0|70|192|0|VERTEX|10|1.0|70|192|0|VERTEX|10|1.0|20|1.0|70|192|0|VERTEX|70|128|71|1|72|2|73|-3|0|SEQEND", + ); + let EntityKind::Polyline(m) = e.kind else { panic!("a polyline") }; + assert!(m.is_polyface() && !m.is_mesh() && !m.is_3d() && !m.closed()); + assert_eq!((m.m, m.n, m.elevation, m.vertices.len()), (3, 1, 7.0, 4)); + assert!(m.vertices[3].is_face() && !m.vertices[0].is_face()); + assert_eq!(m.vertices[3].face, [1, 2, -3, 0]); + let EntityKind::Polyline(h) = one("0|POLYLINE|70|4|40|0.5|41|0.25|0|VERTEX|10|1.0|42|0.5|40|0.1|41|0.2|70|16|0|SEQEND").kind else { + panic!("a polyline") + }; + assert_eq!((h.start_width, h.end_width), (0.5, 0.25)); + let v = h.vertices[0]; + assert!(v.is_spline_frame()); + assert_eq!((v.bulge, v.start_width, v.end_width), (0.5, 0.1, 0.2)); +} + +#[test] +fn text_and_attributes() { + let EntityKind::Text(t) = one("0|TEXT|10|1.0|20|2.0|11|3.0|21|4.0|40|2.5|1|10^ 2^Im|50|90.0|41|0.8|51|15.0|7|Notes|71|2|72|1|73|2").kind else { + panic!("text") + }; + assert_eq!((t.insert, t.align, t.height, t.value.as_str()), ([1.0, 2.0, 0.0], Some([3.0, 4.0, 0.0]), 2.5, "10^2\tm")); + assert_eq!((t.rotation, t.width_factor, t.oblique, t.style.as_str(), t.generation, t.halign, t.valign), (90.0, 0.8, 15.0, "Notes", 2, 1, 2)); + let EntityKind::Text(d) = one("0|TEXT|1|x").kind else { panic!("text") }; + assert_eq!((d.align, d.width_factor, d.style.as_str(), d.halign, d.valign), (None, 1.0, "Standard", 0, 0)); + + let EntityKind::AttDef(a) = + one("0|ATTDEF|100|AcDbText|10|1.0|1|default|100|AcDbAttributeDefinition|280|0|3|Enter^Ia|2|TAG|70|12|74|3|280|1").kind + else { + panic!("an attribute definition") + }; + assert_eq!((a.tag.as_str(), a.prompt.as_str(), a.flags, a.text.valign, a.lock_position), ("TAG", "Enter\ta", 12, 3, true)); + assert!(a.multiline.is_none()); + + // A block reference with two attributes; the second is multiline. + let EntityKind::Insert(i) = one("0|INSERT|66|1|2|DOOR|10|1.0|20|2.0|41|2.0|43|-1.0|50|45.0|70|3|71|0|44|5.0|45|6.0 + 0|ATTRIB|8|Tags|62|2|1|one|2|A|70|1|280|0 + 0|ATTRIB|1|~|2|B|101|Embedded Object|10|7.0|40|2.0|71|5|3|line one^J|1|line two + 0|SEQEND") + .kind + else { + panic!("an insert") + }; + assert_eq!((i.block.as_str(), i.insert, i.scale, i.rotation), ("DOOR", [1.0, 2.0, 0.0], [2.0, 1.0, -1.0], 45.0)); + assert_eq!((i.columns, i.rows, i.column_spacing, i.row_spacing), (3, 1, 5.0, 6.0)); + assert_eq!(i.attribs.len(), 2); + let (c, a) = &i.attribs[0]; + assert_eq!( + (c.layer.as_str(), c.color, a.tag.as_str(), a.text.value.as_str(), a.flags, a.lock_position), + ("Tags", Color::Index(2), "A", "one", 1, false) + ); + assert!(a.prompt.is_empty(), "an ATTRIB has no prompt"); + let m = i.attribs[1].1.multiline.as_ref().expect("embedded MTEXT"); + assert_eq!((m.contents.as_str(), m.insert, m.height, m.attach), ("line one\nline two", [7.0, 0.0, 0.0], 2.0, 5)); +} + +#[test] +fn caret_encoding() { + assert_eq!(decode_contents("a^Ib^Jc^ d^2^"), "a\tb\nc^d^2^"); + assert_eq!(decode_line("a^Ib^Jc^ d"), "a\tb^Jc^d", "a single line has no line feed"); + assert_eq!(encode_contents("a\tb\nc^d\u{7}"), "a^Ib^Jc^ d", "other control characters are dropped"); + assert_eq!(encode_line("10^2\t"), "10^ 2^I"); + for s in ["", "plain", "10^2", "tab\there", "^^ ^I"] { + assert_eq!(decode_line(&encode_line(s)), s); + assert_eq!(decode_contents(&encode_contents(s)), s); + } + let mut t = vec![Tag::s(1, "a^ b"), Tag::s(3, "p^ q"), Tag::i(101, 0), Tag::s(1, "e^ f")]; + map_text_groups("attdef", &mut t, decode_line); + assert_eq!(t.iter().map(Tag::str).collect::>(), ["a^b", "p^q", "0", "e^ f"], "the embedded object's text is left alone"); + let mut t = vec![Tag::s(1, "a^ b"), Tag::s(3, "p^ q")]; + map_text_groups("TEXT", &mut t, decode_line); + assert_eq!(t[1].str(), "p^ q", "only an ATTDEF's 3 is a prompt"); + map_text_groups("LINE", &mut t, |_| String::new()); + assert_eq!(t[0].str(), "a^b"); +} + +#[test] +fn mtext() { + let EntityKind::MText(m) = one("0|MTEXT|10|1.0|20|2.0|40|2.5|41|40.0|71|5|72|1|3|first ^Ipart, |3|second, |1|last\\Pline|7|Notes|11|0.0|21|1.0|50|30.0|44|1.2|73|2|90|1|63|4|45|1.25").kind else { + panic!("multiline text") + }; + assert_eq!((m.insert, m.height, m.width, m.attach, m.style.as_str()), ([1.0, 2.0, 0.0], 2.5, 40.0, 5, "Notes")); + assert_eq!(m.contents, "first \tpart,second,last\\Pline"); + assert_eq!((m.direction, m.rotation, m.line_spacing, m.line_spacing_exact), (Some([0.0, 1.0, 0.0]), 30.0, 1.2, true)); + assert!((m.angle() - std::f64::consts::FRAC_PI_2).abs() < 1e-12, "the direction wins over the rotation"); + assert_eq!(m.background, Some((1, Color::Index(4), 1.25))); + assert!(m.columns.is_none()); + let EntityKind::MText(r) = one("0|MTEXT|1|x|50|90.0|11|0.0|21|0.0|71|99").kind else { panic!("multiline text") }; + assert!((r.angle() - std::f64::consts::FRAC_PI_2).abs() < 1e-12, "a zero direction is none"); + assert_eq!((r.attach, r.background, r.line_spacing), (9, None, 1.0)); +} + +#[test] +fn mtext_columns() { + // Legacy groups: static, 2 columns, the first 50 after 75 counts the heights; the 50 before + // 75 is the rotation. + let t = tags("0|MTEXT|10|0.0|50|15.0|1|x|75|1|76|2|78|1|79|0|48|30.0|49|5.0|50|2|50|80.0|50|80.0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.width, c.gutter, c.height, c.auto_height, c.flow_reversed), (false, 2, 30.0, 5.0, 80.0, false, true)); + assert!(c.heights.is_empty(), "only manual dynamic columns keep each height"); + let EntityKind::MText(m) = one("0|MTEXT|10|0.0|50|15.0|1|x|75|1|76|2|48|30.0|49|5.0|50|2|50|80.0|50|80.0").kind else { panic!("multiline text") }; + assert_eq!(m.rotation, 15.0); + // Dynamic with manual heights, and no count before them. + let t = tags("0|MTEXT|1|x|75|2|76|3|79|0|48|30.0|49|5.0|50|10.0|50|40.0|50|20.0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.height, c.heights.as_slice()), (true, 3, 40.0, [10.0, 40.0, 20.0].as_slice())); + // R2018: the embedded object; 72 is 0 for dynamic auto-height, the count follows the widths. + let t = tags("0|MTEXT|1|x|46|50.0|101|Embedded Object|70|1|10|1.0|11|0.0|40|0.0|41|0.0|42|65.0|43|0.0|71|2|72|0|44|30.0|45|5.0|73|1|74|0"); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.dynamic, c.count, c.width, c.gutter, c.height, c.auto_height), (true, 2, 30.0, 5.0, 50.0, true)); + // No columns: type 0, or no column groups at all. + assert!(MTextColumns::read(&tags("0|MTEXT|1|x|75|0|76|0")[1..]).is_none()); + assert!(MTextColumns::read(&tags("0|MTEXT|1|x|101|Embedded Object|71|0")[1..]).is_none()); +} + +#[test] +fn the_column_height_limit() { + let mut text = String::from("0|MTEXT|1|x|75|2|76|5000|79|0"); + for _ in 0..5000 { + text.push_str("|50|1.0"); + } + let t = tags(&text); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.heights.len(), c.count), (1000, 1000)); + let mut text = String::from("0|MTEXT|1|x|101|Embedded Object|71|2|72|40000|73|0"); + for _ in 0..5000 { + text.push_str("|46|1.0"); + } + let t = tags(&text); + let c = MTextColumns::read(&t[1..]).expect("columns"); + assert_eq!((c.heights.len(), c.count), (1000, 1000)); + // A total width that asks for a count beyond any number. + let t = tags("0|MTEXT|1|x|101|Embedded Object|71|2|72|0|73|1|42|1e300|44|1e-300|45|0.0"); + assert_eq!(MTextColumns::read(&t[1..]).expect("columns").count, 1000); +} + +#[test] +fn dimensions_and_leaders() { + let EntityKind::Dimension(d) = + one("0|DIMENSION|2|*D1|3|ISO-25|10|1.0|20|2.0|11|3.0|21|4.0|12|0.5|13|5.0|14|6.0|15|7.0|16|8.0|70|161|1|<> ^Im|42|12.5|50|30.0|52|60.0|53|45.0|40|2.0").kind + else { + panic!("a dimension") + }; + assert_eq!((d.kind(), d.flags & 128, d.flags & 32), (1, 128, 32)); + assert_eq!((d.block.as_deref(), d.style.as_str(), d.text.as_str(), d.measurement), (Some("*D1"), "ISO-25", "<> \tm", Some(12.5))); + assert_eq!((d.defpt, d.text_mid, d.insertion[0]), ([1.0, 2.0, 0.0], [3.0, 4.0, 0.0], 0.5)); + assert_eq!((d.p13[0], d.p14[0], d.p15[0], d.p16[0]), (5.0, 6.0, 7.0, 8.0)); + assert_eq!((d.rotation, d.oblique, d.text_rotation, d.leader_length), (30.0, 60.0, 45.0, 2.0)); + let EntityKind::Dimension(n) = one("0|DIMENSION|70|6").kind else { panic!("a dimension") }; + assert_eq!((n.block.as_deref(), n.style.as_str(), n.measurement, n.kind()), (None, "Standard", None, 6)); + + // The second vertex has no z: it does not take the third's. + let EntityKind::Leader(l) = one("0|LEADER|3|Arrows|71|0|72|1|73|0|340|4F|10|1.0|20|2.0|30|3.0|10|4.0|20|5.0|10|6.0|20|7.0|30|8.0").kind else { + panic!("a leader") + }; + assert_eq!(l.vertices, [[1.0, 2.0, 3.0], [4.0, 5.0, 0.0], [6.0, 7.0, 8.0]]); + assert!(!l.arrow && l.spline); + assert_eq!((l.style.as_str(), l.annotation_type, l.annotation), ("Arrows", 0, Some(0x4F))); +} + +#[test] +fn faces_viewports_images_and_tolerances() { + let k = |text: &str| one(text).kind; + let three = "|10|0.0|11|1.0|12|1.0|22|1.0"; + assert_eq!(k(&format!("0|SOLID{three}")), EntityKind::Solid { corners: [[0.0; 3], [1.0, 0.0, 0.0], [1.0, 1.0, 0.0], [1.0, 1.0, 0.0]] }); + assert!(matches!(k(&format!("0|TRACE{three}|13|9.0")), EntityKind::Trace { corners } if corners[3] == [9.0, 0.0, 0.0])); + assert!(matches!(k(&format!("0|3DFACE{three}|70|5")), EntityKind::Face3d { hidden_edges: 5, .. })); + let EntityKind::Viewport(v) = + k("0|VIEWPORT|10|5.0|20|4.0|40|8.0|41|6.0|69|2|90|81920|12|1.0|22|2.0|17|9.0|45|30.0|51|90.0|340|A1|331|B1|331|B2|341|B3") + else { + panic!("a viewport") + }; + assert_eq!((v.center, v.width, v.height, v.id, v.status & 16384, v.status & 131_072), ([5.0, 4.0, 0.0], 8.0, 6.0, 2, 16384, 0)); + assert_eq!((v.view_center, v.view_target, v.view_direction, v.view_height, v.twist), ([1.0, 2.0], [9.0, 0.0, 0.0], [0.0, 0.0, 1.0], 30.0, 90.0)); + assert_eq!((v.clip, v.frozen_layers.as_slice()), (Some(0xA1), [0xB1, 0xB2, 0xB3].as_slice())); + let EntityKind::Image(i) = + k("0|IMAGE|10|1.0|11|0.5|12|0.0|22|0.5|13|640.0|23|480.0|340|C1|70|7|280|1|281|60|71|2|14|0.0|24|0.0|14|10.0|24|0.0|14|10.0|24|10.0|290|1") + else { + panic!("an image") + }; + assert_eq!((i.insert[0], i.u[0], i.v[1], i.size, i.image_def), (1.0, 0.5, 0.5, [640.0, 480.0], Some(0xC1))); + assert_eq!((i.display, i.clipping, i.brightness, i.contrast, i.fade, i.clip_type, i.clip_inverted), (7, true, 60, 50, 0, 2, true)); + assert_eq!(i.clip, [[0.0, 0.0], [10.0, 0.0], [10.0, 10.0]]); + assert!(matches!(k("0|WIPEOUT|10|1.0|14|-0.5|24|-0.5|14|0.5|24|0.5"), EntityKind::Wipeout(w) if w.clip.len() == 2 && w.clip_type == 1)); + assert_eq!( + k("0|TOLERANCE|3|Tol|10|1.0|1|{\\Fgdt;j}%%v0.1"), + EntityKind::Tolerance { insert: [1.0, 0.0, 0.0], direction: [1.0, 0.0, 0.0], style: "Tol".into(), text: "{\\Fgdt;j}%%v0.1".into() } + ); + assert_eq!(k("0|MULTILEADER|10|1.0"), EntityKind::Other, "a type the model does not have is read from its tags"); +} + +#[test] +fn hatch_with_a_polyline_loop_and_a_pattern() { + let EntityKind::Hatch(h) = one("0|HATCH|10|0.0|20|0.0|30|2.5|2|ANSI31|70|0|71|1|91|1 + 92|7|72|1|73|1|93|3|10|0.0|20|0.0|42|0.5|10|4.0|20|0.0|42|0.0|10|4.0|20|3.0|42|0.0|97|2|330|A1|330|A2 + 75|1|76|1|52|45.0|41|2.0|77|0|78|2 + 53|45.0|43|0.0|44|0.0|45|-1.4|46|1.4|79|0 + 53|135.0|43|1.0|44|2.0|45|1.0|46|1.0|79|2|49|3.0|49|-1.5 + 98|2|10|1.0|20|1.0|10|2.0|20|2.0") + .kind + else { + panic!("a hatch") + }; + assert_eq!((h.pattern.as_str(), h.solid, h.associative, h.style, h.pattern_type), ("ANSI31", false, true, 1, 1)); + assert_eq!((h.angle, h.scale, h.double, h.elevation), (45.0, 2.0, false, 2.5)); + assert_eq!(h.loops.len(), 1); + let l = &h.loops[0]; + assert_eq!((l.flags, l.sources.as_slice()), (7, [0xA1, 0xA2].as_slice())); + assert_eq!(l.path, HatchPath::Polyline { closed: true, vertices: vec![[0.0, 0.0, 0.5], [4.0, 0.0, 0.0], [4.0, 3.0, 0.0]] }); + assert_eq!(h.pattern_lines.len(), 2); + assert_eq!((h.pattern_lines[0].angle, h.pattern_lines[0].offset, h.pattern_lines[0].dashes.len()), (45.0, [-1.4, 1.4], 0)); + assert_eq!((h.pattern_lines[1].base, h.pattern_lines[1].dashes.as_slice()), ([1.0, 2.0], [3.0, -1.5].as_slice())); + assert_eq!(h.seeds, [[1.0, 1.0], [2.0, 2.0]]); + assert!(h.gradient.is_none()); +} + +#[test] +fn hatch_with_edge_loops_and_a_gradient() { + let EntityKind::Hatch(h) = one( + "0|HATCH|2|SOLID|70|1|71|0|91|2 + 92|1|93|4 + 72|1|10|0.0|20|0.0|11|4.0|21|0.0 + 72|2|10|4.0|20|1.0|40|1.0|50|270.0|51|360.0|73|1 + 72|3|10|2.0|20|2.0|11|2.0|21|0.0|40|0.5|50|0.0|51|180.0|73|0 + 72|4|94|3|73|1|74|0|95|8|96|4|40|0.0|40|0.0|40|0.0|40|0.0|40|1.0|40|1.0|40|1.0|40|1.0|10|0.0|20|2.0|42|1.0|10|0.0|20|1.5|42|2.0|10|0.0|20|0.5|42|2.0|10|0.0|20|0.0|42|1.0|97|2|11|0.0|21|2.0|11|0.0|21|0.0|12|0.0|22|-1.0|13|0.0|23|-1.0 + 97|0 + 92|16|93|1 + 72|2|10|2.0|20|1.0|40|0.25|50|0.0|51|360.0|73|1 + 97|1|330|B1 + 75|0|76|1|98|0 + 450|1|451|0|460|0.785|461|0.0|452|0|462|0.0|453|2|463|0.0|63|5|421|255|463|1.0|63|2|470|LINEAR + 1001|ACAD|1010|0.0", + ) + .kind + else { + panic!("a hatch") + }; + assert!(h.solid && !h.associative); + assert_eq!(h.loops.len(), 2); + let HatchPath::Edges(e) = &h.loops[0].path else { panic!("edges") }; + assert_eq!(e.len(), 4); + assert_eq!(e[0], HatchEdge::Line { start: [0.0, 0.0], end: [4.0, 0.0] }); + assert_eq!(e[1], HatchEdge::Arc { center: [4.0, 1.0], radius: 1.0, start_angle: 270.0, end_angle: 360.0, ccw: true }); + assert_eq!(e[2], HatchEdge::Ellipse { center: [2.0, 2.0], major: [2.0, 0.0], ratio: 0.5, start_angle: 0.0, end_angle: 180.0, ccw: false }); + let HatchEdge::Spline { degree, rational, periodic, knots, control, weights, fit, start_tangent, end_tangent } = &e[3] else { + panic!("a spline edge") + }; + assert_eq!((*degree, *rational, *periodic, knots.len(), control.len()), (3, true, false, 8, 4)); + assert_eq!( + (weights.as_slice(), fit.len(), *start_tangent, *end_tangent), + ([1.0, 2.0, 2.0, 1.0].as_slice(), 2, Some([0.0, -1.0]), Some([0.0, -1.0])) + ); + // The spline's own 97 (fit points) is not the loop's source count. + assert!(h.loops[0].sources.is_empty()); + assert_eq!((h.loops[1].flags, h.loops[1].sources.as_slice()), (16, [0xB1].as_slice())); + let g = h.gradient.expect("a gradient"); + assert_eq!((g.name.as_str(), g.angle, g.shift, g.single), ("LINEAR", 0.785, 0.0, false)); + assert_eq!(g.colors, [Color::Rgb([0, 0, 255]), Color::Index(2)], "a true colour wins over its index"); +} + +#[test] +fn damaged_hatches() { + let k = |text: &str| one(text).kind; + assert_eq!(k("0|HATCH|2|SOLID"), EntityKind::Other, "no boundary data"); + assert_eq!(k("0|HATCH|91|2|92|2|93|1|10|0.0|20|0.0"), EntityKind::Other, "the second loop is missing"); + assert_eq!(k("0|HATCH|91|1|92|2|8|x|93|1"), EntityKind::Other, "a foreign group in the loop header"); + // Counts larger than the data: what is there is read, nothing is reserved for the rest. + let EntityKind::Hatch(h) = k("0|HATCH|91|1|92|2|93|2000000000|10|1.0|20|2.0|97|2000000000|330|A|98|2000000000|10|5.0|20|6.0") else { + panic!("a hatch") + }; + assert_eq!(h.loops[0].path, HatchPath::Polyline { closed: false, vertices: vec![[1.0, 2.0, 0.0]] }); + assert_eq!((h.loops[0].sources.as_slice(), h.seeds.as_slice()), ([0xA].as_slice(), [[5.0, 6.0]].as_slice())); + let EntityKind::Hatch(h) = k("0|HATCH|91|1|92|0|93|9|72|9|10|1.0|72|1|10|1.0|20|1.0|11|2.0|21|2.0|72|4|95|-5|96|2000000000|10|1.0|20|1.0") else { + panic!("a hatch") + }; + let HatchPath::Edges(e) = &h.loops[0].path else { panic!("edges") }; + assert_eq!(e.len(), 2, "an edge type the format does not have is passed over"); + assert!(matches!(&e[1], HatchEdge::Spline { knots, control, .. } if knots.is_empty() && control.len() == 1)); + assert_eq!(k("0|HATCH|91|-4").clone(), k("0|HATCH|91|0"), "a negative loop count is none"); + // A 450 that says no gradient. + assert!(matches!(k("0|HATCH|91|0|450|0|463|0.0|63|1"), EntityKind::Hatch(h) if h.gradient.is_none())); +} + +#[test] +fn entities_of_a_whole_drawing() { + let t = tags( + "0|SECTION|2|BLOCKS|0|BLOCK|2|B|0|CIRCLE|40|2.0|0|ENDBLK|0|ENDSEC|0|SECTION|2|ENTITIES|0|INSERT|2|B|0|POLYLINE|0|VERTEX|10|1.0|0|SEQEND|0|ENDSEC|0|EOF", + ); + let d = Document::read(&t); + let kinds: Vec = d.entities.iter().chain(&d.blocks[0].entities).map(|e| Entity::read(e).kind).collect(); + assert!(matches!(kinds.as_slice(), [EntityKind::Insert(_), EntityKind::Polyline(p), EntityKind::Circle { .. }] if p.vertices.len() == 1)); +} + +#[test] +fn a_spline_edge_without_fit_data_leaves_the_loops_source_count() { + // Before R2010 a spline edge has no 97 of its own: the one after it counts the loop's sources. + let EntityKind::Hatch(h) = one( + "0|HATCH|91|1|92|1|93|1|72|4|94|1|73|0|74|0|95|4|96|2|40|0.0|40|0.0|40|1.0|40|1.0|10|0.0|20|0.0|10|1.0|20|1.0|97|2|330|A1|330|A2|75|0|98|0", + ) + .kind + else { + panic!("a hatch") + }; + assert_eq!(h.loops[0].sources, [0xA1, 0xA2]); + assert!( + matches!(&h.loops[0].path, HatchPath::Edges(e) if matches!(&e[0], HatchEdge::Spline { fit, control, .. } if fit.is_empty() && control.len() == 2)) + ); +} From 8079878ae2b166c1ea8229e40eff7e08c916e532 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:22:03 -0500 Subject: [PATCH 09/31] craft-drawingml: the theme model (#940) * Add craft-drawingml: units, colours, preset shapes, custom geometry and text warps The parts of DrawingML that need no document model, moved from DeckCraft (60f91f78): crates/color, crates/geom (preset, warp, Xfrm and the EMU conversions) and crates/pptx/src/custgeom.rs with CustomPath from crates/model. The code is unchanged except that custgeom reads craft-xml trees and writes to a craft-xml Writer instead of DeckCraft's own XML helper, and matches elements by local name. The 59 tests of the moved code come with it; six more cover the custom geometry limits. * drawingml: evaluate a custom geometry's guide lists once per coordinate space read_cust_geom rebuilt Guides and evaluated avLst and gdLst again for every path: up to 1,000 paths times up to 10,000 guides, each name copied and hashed. Guides are now cached per path coordinate space (w, h). At most 16 spaces and 4 MiB of guide names are kept per geometry; paths in further spaces are left out. The first space is always kept, and a geometry without guides is not capped. Ported from storytold/deckcraft#1789. A list counts as having guides only when it has gd children, as that PR's review asked. * drawingml: the model types for fills, lines, text, tables, charts and diagrams Moved from DeckCraft (60f91f78) crates/model/src: style.rs, text.rs, table.rs, chart.rs, numfmt.rs and trendline.rs, plus Diagram, DiagramNode and the MediaId and SlideId newtypes from lib.rs. The files are unchanged except that deckcraft_color:: is crate::color:: and Fill and TableBorder allow clippy::large_enum_variant. Their 60 tests come with them. Readers and writers follow. * drawingml: the theme model (colour, font and format schemes) Moved from DeckCraft crates/model/src/theme.rs at 60f91f78: FontSet, ScriptFont, FontScheme, FormatScheme, Theme and default_format. The app's theme gallery, its variants and its own default theme stay in the app. Theme::default() is a plain theme built in place, so nothing here depends on a gallery. --- crates/drawingml/src/lib.rs | 3 +- crates/drawingml/src/theme.rs | 183 ++++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 crates/drawingml/src/theme.rs diff --git a/crates/drawingml/src/lib.rs b/crates/drawingml/src/lib.rs index 47d6ee107..1e47301dc 100644 --- a/crates/drawingml/src/lib.rs +++ b/crates/drawingml/src/lib.rs @@ -11,7 +11,7 @@ //! - [`warp`]: preset text warps (`a:prstTxWarp`). //! - The model, without readers and writers yet: [`style`] (colour references, fills, lines, //! effects), [`text`] (text bodies, paragraphs, runs), [`table`], [`chart`] with [`numfmt`] -//! (number format codes) and [`diagram`] (SmartArt outlines). Media are named by [`MediaId`]; +//! (number format codes), [`theme`] (colour, font and format schemes) and [`diagram`] (SmartArt outlines). Media are named by [`MediaId`]; //! the bytes stay with the caller. //! //! Nothing here opens a package: readers take a [`craft_xml::El`] and writers a @@ -27,6 +27,7 @@ pub mod preset; pub mod style; pub mod table; pub mod text; +pub mod theme; mod trendline; pub mod warp; diff --git a/crates/drawingml/src/theme.rs b/crates/drawingml/src/theme.rs new file mode 100644 index 000000000..d4416e2f1 --- /dev/null +++ b/crates/drawingml/src/theme.rs @@ -0,0 +1,183 @@ +//! Themes: colour scheme, font scheme and format scheme (the fill/line/effect "style matrix"). + +use crate::color::{ColorScheme, ColorTransform, Rgba, SchemeSlot}; +use serde::{Deserialize, Serialize}; + +use crate::style::{ColorRef, Effects, Fill, Gradient, GradientShape, GradientStop, Line, Shadow}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FontSet { + pub latin: String, + pub ea: String, + pub cs: String, + /// Per-script typefaces (`a:font script="Arab" typeface="…"`), in document order. + #[serde(skip_serializing_if = "Vec::is_empty")] + pub scripts: Vec, +} + +/// A theme font for one script (ISO 15924 tag, e.g. `Arab`). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct ScriptFont { + pub script: String, + pub typeface: String, +} + +impl FontSet { + /// The typeface the theme gives `script` (`"Arab"`), if any. + pub fn script(&self, script: &str) -> Option<&str> { + self.scripts.iter().find(|f| f.script.eq_ignore_ascii_case(script) && !f.typeface.is_empty()).map(|f| f.typeface.as_str()) + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FontScheme { + pub name: String, + /// Headings. + pub major: FontSet, + /// Body. + pub minor: FontSet, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct FormatScheme { + pub name: String, + /// Subtle, moderate, intense (style index 1–3). + pub fills: Vec, + pub lines: Vec, + pub effects: Vec, + /// Background fills (index 1001–1003). + pub bg_fills: Vec, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Theme { + pub name: String, + pub colors: ColorScheme, + pub fonts: FontScheme, + pub format: FormatScheme, + /// Raw theme XML extras kept for round-trip (object defaults, extra colour lists). + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_extra: Option, +} + +impl Default for Theme { + /// A plain theme: black and white, a slate and a warm grey, six accents, no font names (the + /// writer puts `Arial` where a Latin face is required) and [`default_format`]. + fn default() -> Self { + let hex = |s: &str| Rgba::from_hex(s).unwrap_or(Rgba::BLACK); + let colors = ["000000", "FFFFFF", "1E2B3C", "EDEBE7", "2E6FD8", "F26B1D", "2EA36F", "8A5CD8", "D83F6B", "E3B21C", "2E6FD8", "7A5BA6"]; + Theme { + name: "Theme".into(), + colors: ColorScheme { name: "Default".into(), colors: colors.map(hex) }, + fonts: FontScheme::default(), + format: default_format(), + raw_extra: None, + } + } +} + +impl Theme { + /// Resolve a theme font reference (`+mj-lt`, `+mn-ea`…) to a family name. + pub fn font(&self, name: &str) -> String { + match name { + "+mj-lt" => self.fonts.major.latin.clone(), + "+mn-lt" => self.fonts.minor.latin.clone(), + "+mj-ea" => self.fonts.major.ea.clone(), + "+mn-ea" => self.fonts.minor.ea.clone(), + "+mj-cs" => self.fonts.major.cs.clone(), + "+mn-cs" => self.fonts.minor.cs.clone(), + other => other.to_string(), + } + } +} + +fn ph() -> ColorRef { + ColorRef::scheme(SchemeSlot::PhClr) +} + +/// The format scheme used where a theme has none, and to fill a short style list: flat solid fills, a soft gradient, and a gentle shadow for "intense". +pub fn default_format() -> FormatScheme { + let grad = |a: i32, b: i32| { + Fill::Gradient(Gradient { + tile: [0.0; 4], + stops: vec![ + GradientStop { pos: 0.0, color: ph().with(ColorTransform::LumMod(a)).with(ColorTransform::SatMod(103000)) }, + GradientStop { pos: 1.0, color: ph().with(ColorTransform::LumMod(b)) }, + ], + shape: GradientShape::Linear { angle: 90.0, scaled: false }, + rotate_with_shape: true, + flip: None, + }) + }; + FormatScheme { + name: "Default".into(), + fills: vec![Fill::solid(ph()), grad(110000, 92000), grad(105000, 80000)], + lines: vec![Line::solid(ph(), 0.75), Line::solid(ph(), 1.0), Line::solid(ph(), 1.5)], + effects: vec![ + Effects::default(), + Effects::default(), + Effects { + outer_shadow: Some(Shadow { + color: ColorRef::rgb(Rgba::BLACK).with(ColorTransform::Alpha(40000)), + blur: 6.0, + dist: 3.0, + dir: 90.0, + inner: false, + sx: 1.0, + sy: 1.0, + kx: 0.0, + ky: 0.0, + align: "t".into(), + rotate_with_shape: false, + }), + ..Default::default() + }, + ], + bg_fills: vec![ + Fill::solid(ph()), + Fill::solid(ph().with(ColorTransform::Tint(95000)).with(ColorTransform::SatMod(170000))), + grad(102000, 85000), + ], + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_default_theme_has_a_full_style_matrix_and_no_font_names() { + let t = Theme::default(); + assert_eq!((t.format.fills.len(), t.format.lines.len(), t.format.effects.len(), t.format.bg_fills.len()), (3, 3, 3, 3)); + assert_eq!(t.colors.get(SchemeSlot::Lt1), Rgba::WHITE); + assert_eq!(t.fonts, FontScheme::default()); + } + + #[test] + fn theme_font_references_resolve_and_other_names_pass_through() { + let mut t = Theme::default(); + t.fonts.major.latin = "Head".into(); + t.fonts.minor.cs = "Body CS".into(); + t.fonts.minor.scripts.push(ScriptFont { script: "Arab".into(), typeface: "Naskh".into() }); + assert_eq!(t.font("+mj-lt"), "Head"); + assert_eq!(t.font("+mn-cs"), "Body CS"); + assert_eq!(t.font("Georgia"), "Georgia"); + assert_eq!(t.fonts.minor.script("arab"), Some("Naskh")); + assert_eq!(t.fonts.minor.script("Hebr"), None); + } + + #[test] + fn a_theme_survives_json_and_missing_fields_take_the_default() { + let t = Theme::default(); + let json = serde_json::to_string(&t).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), t); + let partial: Theme = serde_json::from_str(r#"{"name":"X"}"#).unwrap(); + assert_eq!(partial.colors, t.colors); + assert_eq!(partial.name, "X"); + } +} From dca2a2cd8e214b0c24d7fe0f222a803f63155d16 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:22:42 -0500 Subject: [PATCH 10/31] craft-metadata: EXIF, IPTC-IIM, XMP and GPX from LightCraft (#949) lightcraft-meta at 5b61ce1e, as one crate with a module per format: exif (read, write, maker-note lens, focus point, tag listing), iptc, xmp (parse, write, merge), gpx, and the JPEG, PNG, WebP, TIFF, RAF and CR3 containers. They share Metadata, DateTime, Gps and Orientation. Changed in the move: - TIFF comes from craft-tiff; Point, Rect and Orientation are the crate's own. - The app namespace is a parameter: parse_xmp_with, write_xmp_with and MergeRules::namespaces take it, and XmpData no longer has lc_settings. - The HEIF, PSD and MRW readers stay out, since craft-heif, craft-psd and craft-raw read those containers; from_embedded takes the blocks they find. --- Cargo.lock | 12 + Cargo.toml | 1 + README.md | 4 + crates/metadata/Cargo.toml | 21 + crates/metadata/README.md | 61 +++ crates/metadata/src/container.rs | 450 ++++++++++++++++ crates/metadata/src/cr3.rs | 742 ++++++++++++++++++++++++++ crates/metadata/src/datetime.rs | 320 ++++++++++++ crates/metadata/src/exif.rs | 759 +++++++++++++++++++++++++++ crates/metadata/src/focus.rs | 284 ++++++++++ crates/metadata/src/focus/tests.rs | 449 ++++++++++++++++ crates/metadata/src/focus_read.rs | 472 +++++++++++++++++ crates/metadata/src/geom.rs | 180 +++++++ crates/metadata/src/gpx.rs | 491 +++++++++++++++++ crates/metadata/src/iptc.rs | 151 ++++++ crates/metadata/src/lib.rs | 606 +++++++++++++++++++++ crates/metadata/src/tags.rs | 529 +++++++++++++++++++ crates/metadata/src/xmp.rs | 813 +++++++++++++++++++++++++++++ crates/metadata/src/xmp/tests.rs | 631 ++++++++++++++++++++++ crates/metadata/src/xmp_merge.rs | 505 ++++++++++++++++++ crates/metadata/tests/it/main.rs | 3 + crates/metadata/tests/it/robust.rs | 70 +++ 22 files changed, 7554 insertions(+) create mode 100644 crates/metadata/Cargo.toml create mode 100644 crates/metadata/README.md create mode 100644 crates/metadata/src/container.rs create mode 100644 crates/metadata/src/cr3.rs create mode 100644 crates/metadata/src/datetime.rs create mode 100644 crates/metadata/src/exif.rs create mode 100644 crates/metadata/src/focus.rs create mode 100644 crates/metadata/src/focus/tests.rs create mode 100644 crates/metadata/src/focus_read.rs create mode 100644 crates/metadata/src/geom.rs create mode 100644 crates/metadata/src/gpx.rs create mode 100644 crates/metadata/src/iptc.rs create mode 100644 crates/metadata/src/lib.rs create mode 100644 crates/metadata/src/tags.rs create mode 100644 crates/metadata/src/xmp.rs create mode 100644 crates/metadata/src/xmp/tests.rs create mode 100644 crates/metadata/src/xmp_merge.rs create mode 100644 crates/metadata/tests/it/main.rs create mode 100644 crates/metadata/tests/it/robust.rs diff --git a/Cargo.lock b/Cargo.lock index 1e9d85eff..b531fce60 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1275,6 +1275,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "craft-metadata" +version = "0.1.0" +dependencies = [ + "chrono", + "craft-tiff", + "proptest", + "quick-xml 0.41.0", + "serde", + "serde_json", +] + [[package]] name = "craft-metafile" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 507ab1410..7059505ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ craft-text = { path = "crates/text" } craft-tga = { path = "crates/tga" } craft-timeline = { path = "crates/timeline" } craft-tiff = { path = "crates/tiff" } +craft-metadata = { path = "crates/metadata" } craft-ui = { path = "crates/ui" } craft-update = { path = "crates/update" } craft-vcard = { path = "crates/vcard" } diff --git a/README.md b/README.md index 3830d0560..3776d0b56 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. - [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft. +- [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write, + maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge + that keeps what another program wrote), GPX track logs, and the blocks that carry them in JPEG, PNG, + WebP, TIFF-based, RAF and CR3 files. One `Metadata` model for all of them. Built on `craft-tiff`. - [`craft-tiff`](crates/tiff/README.md): TIFF and BigTIFF IFD reader and writer, from LightCraft: every field type, the IFD chain, SubIFDs and Exif/GPS/Interop pointer IFDs, maker-note IFDs (Canon, Nikon, Sony, Fujifilm, Olympus, Panasonic, Pentax), strip and tile layout. Bounded and diff --git a/crates/metadata/Cargo.toml b/crates/metadata/Cargo.toml new file mode 100644 index 000000000..207492619 --- /dev/null +++ b/crates/metadata/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "craft-metadata" +description = "Photo metadata: EXIF, IPTC-IIM, XMP packets (read, write, merge) and GPX track logs, the blocks that carry them in JPEG, PNG, WebP, TIFF and CR3 files, and one model they all read into." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-tiff = { workspace = true } +serde = { workspace = true } +quick-xml = "0.41" +chrono = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } +serde_json = { workspace = true } diff --git a/crates/metadata/README.md b/crates/metadata/README.md new file mode 100644 index 000000000..1345ead5f --- /dev/null +++ b/crates/metadata/README.md @@ -0,0 +1,61 @@ +# craft-metadata + +Photo metadata: EXIF, IPTC-IIM, XMP packets and GPX track logs, the blocks that carry them in image +files, and one `Metadata` model they all read into. From LightCraft (`lightcraft-meta`). TIFF +structure comes from `craft-tiff`. + +One module per format; they share `Metadata`, `DateTime`, `Gps` and `Orientation`, which is why they +are one crate. + +| Function | Purpose | +|---|---| +| `read_exif(&[u8]) -> Metadata` | A TIFF-structured Exif block: a whole TIFF/DNG/raw file, or a JPEG APP1 payload with or without `Exif\0\0`. `try_read_exif` reports header errors, `try_read_exif_with` takes `craft_tiff::ParseOptions`. | +| `from_tiff(&Tiff) -> Metadata` | The same from an already parsed stream. | +| `write_exif(&Metadata) -> Vec` | An Exif block for an exported file. | +| `tags::tag_rows`, `tags::file_tag_rows` | Every tag with its name and value as text, maker notes included. | +| `focus::read`, `focus::read_header` | Where the camera focused, from the maker note (Sony), on the upright photo. `read_header` reads only the first kilobytes of a file through a callback. | +| `parse_iptc(&[u8]) -> Metadata` | IIM record 2: title (2:5), keywords (2:25), by-line (2:80), copyright (2:116), caption (2:120), date and time created (2:55, 2:60). | +| `parse_xmp(&str) -> Result` | `Metadata`, every property (`prefix:name`, structs flattened as `a:b/c:d`) and the structured values. `parse_xmp_with` also canonicalises the caller's own namespaces. | +| `write_xmp(&Metadata) -> String` | A complete `` packet. `write_xmp_with` sets the toolkit name and adds simple properties in the caller's namespace. | +| `merge_xmp(existing, fresh, MergeRules)` | Updates a packet another program wrote: owned properties are replaced, everything else stays byte for byte. | +| `parse_gpx(&str) -> Result` | GPX track points; `Tracklog::locate` gives the position at a capture time. | +| `embedded(&[u8]) -> Embedded` | Sniffs JPEG, PNG, WebP, TIFF-based files, Fujifilm RAF and Canon CR3 and returns the Exif, XMP (and extended XMP), ICC and IPTC blocks. `jpeg_segments`, `png_chunks`, `webp_chunks` are the walkers. | +| `cr3::parse_cr3` | The Canon CR3 container: `CMT1`…`CMT4`, the XMP `uuid` box, the thumbnail, the tracks. | +| `extract(&[u8]) -> Metadata` | All of it for a whole file. Precedence: EXIF, then IPTC fills gaps, then XMP overrides the user fields (rating, label, title, caption, artist, copyright, keywords, GPS) and fills the rest. `from_embedded` does the same for blocks already located. | + +HEIF, PSD and MRW files are read by `craft-heif`, `craft-psd` and `craft-raw`. Put the blocks they +return into an `Embedded` and call `from_embedded`. + +## Capture time + +EXIF candidates are tried in order: DateTimeOriginal, DateTimeDigitized, then IFD0 DateTime. A clock +with more than three colon-separated components is rejected as a whole, so the next candidate or the +XMP supplies the time. ISO 8601 times in XMP need a complete clock. Calendar checks go through +`chrono`. Invalid GPX point times count as untimed. + +## XMP namespaces + +Written: `dc`, `xmp`, `xmpRights`, `photoshop`, `exif`, `exifEX`, `tiff`, `aux`, `lr`, +`Iptc4xmpCore`, and the one namespace the caller passes. Read as well: `crs`, `mwg-rs` regions +(`stArea`, `stDim`). Reading canonicalises prefixes by namespace URI and handles attribute and element +forms, `rdf:Seq`/`Bag`/`Alt` (x-default first), `rdf:parseType="Resource"` and nested-description +structs, predefined and numeric entities. + +A foreign namespace that uses a recognised prefix cannot supply recognised properties. Its keys are +`Q{hex-encoded namespace URI}local`. Merging keeps the original foreign XML. + +## Limits + +XMP: 64 levels, 200 000 nodes, 1 MiB of flattened keys and 8 MiB of opaque names per packet +(`XmpError::TooComplex`). GPX: 64 levels, 5 000 000 points. CR3: 4096 boxes. JPEG ICC is returned only when its APP2 chunks form a +complete, consistent, uniquely numbered sequence. TIFF limits are `craft-tiff`'s. No input panics: +proptests feed random bytes to every container walker and random text to the XMP parser, and a sample +JPEG is cut at every byte. + +## Sources + +CIPA DC-008 (Exif 2.32), TIFF 6.0, Adobe DNG 1.7 (tag numbers), ISO 16684-1 and the Adobe XMP +Specification Parts 1 to 3, IPTC-IIM 4.2 and the IPTC Photo Metadata Standard, the MWG Region +Guidelines, ITU-T T.81 (JPEG markers), ICC.1 (APP2 embedding), the PNG Specification 3rd ed. (`eXIf`, +`iTXt`), the WebP RIFF container specification, ISO/IEC 14496-12 (box structure), GPX 1.1. XML parsing: +`quick-xml` (MIT). No third-party metadata code was consulted. diff --git a/crates/metadata/src/container.rs b/crates/metadata/src/container.rs new file mode 100644 index 000000000..660677581 --- /dev/null +++ b/crates/metadata/src/container.rs @@ -0,0 +1,450 @@ +//! Locating metadata blocks inside image containers. +//! +//! - JPEG (ITU-T T.81 marker structure): APP1 `Exif\0\0`, APP1 `http://ns.adobe.com/xap/1.0/\0` (XMP), +//! APP1 `http://ns.adobe.com/xmp/extension/\0` (extended XMP, reassembled), APP2 `ICC_PROFILE\0` +//! (multi-chunk, reassembled in sequence order), APP13 `Photoshop 3.0\0` 8BIM resource 0x0404 (IPTC-IIM). +//! - PNG (PNG spec 3rd ed.): `eXIf`, `iTXt` with keyword `XML:com.adobe.xmp` (uncompressed). +//! - WebP (RIFF container spec): `EXIF`, `XMP `, `ICCP` chunks. +//! - TIFF-based files (TIFF, DNG and most raws): the whole file is the Exif block; XMP (700), ICC (34675) and +//! IPTC (33723) from IFD0. +//! - Canon CR3 (ISO base media file, see [`crate::cr3`]): `CMT1`/`CMT2`/`CMT4` merged into one Exif block; the +//! XMP `uuid` box. +//! +//! PSD, MRW and HEIF files are read by `craft-psd`, `craft-raw` and `craft-heif`. + +use craft_tiff::{Tiff, tags}; + +/// Metadata blocks found in a file. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Embedded { + /// TIFF-structured Exif block (without the `Exif\0\0` prefix). + pub exif: Option>, + /// XMP packet (the main one; extended XMP is appended by [`Embedded::xmp_extended`]). + pub xmp: Option, + /// Extended XMP (JPEG only), reassembled. + pub xmp_extended: Option, + pub icc: Option>, + /// IPTC-IIM stream. + pub iptc: Option>, +} + +/// Sniff the container and collect its metadata blocks. Unknown containers yield an empty result. +pub fn embedded(bytes: &[u8]) -> Embedded { + if bytes.starts_with(b"FUJIFILMCCD-RAW") { + // Fujifilm RAF: metadata lives in the embedded JPEG whose (offset, length) are big-endian u32s at byte 84 + let be = |i: usize| bytes.get(i..i + 4).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]]) as usize); + return match (be(84), be(88)) { + (Some(o), Some(l)) if o > 0 => bytes.get(o..o.saturating_add(l).min(bytes.len())).map(jpeg_segments).unwrap_or_default(), + _ => Embedded::default(), + }; + } + if let Some(c) = crate::cr3::parse_cr3(bytes) { + return Embedded { exif: crate::cr3::merged_exif(&c), xmp: c.xmp.map(|b| String::from_utf8_lossy(b).into_owned()), ..Default::default() }; + } + if bytes.starts_with(&[0xff, 0xd8]) { + jpeg_segments(bytes) + } else if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { + png_chunks(bytes) + } else if bytes.len() >= 12 && &bytes[..4] == b"RIFF" && &bytes[8..12] == b"WEBP" { + webp_chunks(bytes) + } else if let Ok(t) = Tiff::parse(bytes) { + let ifd0 = &t.ifds[0]; + Embedded { + exif: Some(bytes.to_vec()), + xmp: ifd0.bytes(tags::XMP).map(|b| String::from_utf8_lossy(b).into_owned()), + xmp_extended: None, + icc: ifd0.bytes(tags::ICC_PROFILE).map(|b| b.to_vec()), + iptc: ifd0.value(tags::IPTC_NAA).map(|v| match v { + craft_tiff::Value::Byte(b) | craft_tiff::Value::Undefined(b) => b.clone(), + other => craft_tiff::writer::encode(t.order, other), + }), + } + } else { + Embedded::default() + } +} + +const XMP_SIG: &[u8] = b"http://ns.adobe.com/xap/1.0/\0"; +const XMP_EXT_SIG: &[u8] = b"http://ns.adobe.com/xmp/extension/\0"; + +/// Walk JPEG markers up to SOS and collect APP1 Exif / XMP, APP2 ICC and APP13 IPTC. +pub fn jpeg_segments(b: &[u8]) -> Embedded { + let mut out = Embedded::default(); + if !b.starts_with(&[0xff, 0xd8]) { + return out; + } + let mut icc_parts: [Option<&[u8]>; 255] = [None; 255]; + let mut icc_total = None; + let mut icc_invalid = false; + let mut ext: Vec<(u32, Vec)> = Vec::new(); + let mut i = 2usize; + while i + 4 <= b.len() { + if b[i] != 0xff { + break; + } + let marker = b[i + 1]; + if marker == 0xff { + i += 1; // fill byte + continue; + } + if marker == 0xd8 || (0xd0..=0xd7).contains(&marker) || marker == 0x01 { + i += 2; + continue; + } + if marker == 0xd9 || marker == 0xda { + break; + } + let len = u16::from_be_bytes([b[i + 2], b[i + 3]]) as usize; + if len < 2 { + break; + } + let Some(start) = i.checked_add(4) else { break }; + let Some(declared_end) = i.checked_add(2).and_then(|at| at.checked_add(len)) else { break }; + let complete = declared_end <= b.len(); + let end = declared_end.min(b.len()); + let Some(payload) = b.get(start..end) else { break }; + match marker { + 0xe1 if payload.starts_with(b"Exif\0") && out.exif.is_none() => out.exif = Some(payload.get(6..).unwrap_or(&[]).to_vec()), + 0xe1 if payload.starts_with(XMP_SIG) && out.xmp.is_none() => { + out.xmp = Some(String::from_utf8_lossy(&payload[XMP_SIG.len()..]).into_owned()); + } + 0xe1 if payload.starts_with(XMP_EXT_SIG) => { + // GUID (32) + full length (4) + offset (4) + data + let p = &payload[XMP_EXT_SIG.len()..]; + if p.len() >= 40 { + let off = u32::from_be_bytes([p[36], p[37], p[38], p[39]]); + ext.push((off, p[40..].to_vec())); + } + } + 0xe2 if payload.starts_with(b"ICC_PROFILE\0") => { + if !complete || payload.len() < 14 { + icc_invalid = true; + } else if let Some(header) = payload.get(12..14) { + let (Some(&sequence), Some(&total)) = (header.first(), header.get(1)) else { + icc_invalid = true; + i = end; + continue; + }; + if sequence == 0 || total == 0 || sequence > total || icc_total.is_some_and(|expected| expected != total) { + icc_invalid = true; + } else { + icc_total = Some(total); + if let Some(slot) = icc_parts.get_mut(usize::from(sequence - 1)) { + if slot.is_some() { + icc_invalid = true; + } else if let Some(data) = payload.get(14..) { + *slot = Some(data); + } else { + icc_invalid = true; + } + } else { + icc_invalid = true; + } + } + } else { + icc_invalid = true; + } + } + 0xed if payload.starts_with(b"Photoshop 3.0\0") && out.iptc.is_none() => out.iptc = photoshop_iptc(&payload[14..]), + _ => {} + } + i = end; + } + if let Some(total) = icc_total { + let mut profile_len = 0usize; + for chunk in icc_parts.iter().take(usize::from(total)) { + let Some(bytes) = chunk else { + icc_invalid = true; + break; + }; + let Some(next_len) = profile_len.checked_add(bytes.len()) else { + icc_invalid = true; + break; + }; + profile_len = next_len; + } + let maximum_len = usize::from(total).checked_mul(65_519); + if maximum_len.is_none_or(|maximum| profile_len > maximum) { + icc_invalid = true; + } + if !icc_invalid { + let mut profile = Vec::with_capacity(profile_len); + for bytes in icc_parts.iter().take(usize::from(total)).flatten() { + profile.extend_from_slice(bytes); + } + out.icc = Some(profile); + } + } + if !ext.is_empty() { + ext.sort_by_key(|p| p.0); + out.xmp_extended = Some(String::from_utf8_lossy(&ext.into_iter().flat_map(|p| p.1).collect::>()).into_owned()); + } + out +} + +/// IPTC-IIM block (resource 0x0404) from a Photoshop image-resource stream (`8BIM` records). +fn photoshop_iptc(mut b: &[u8]) -> Option> { + while b.len() >= 12 && &b[..4] == b"8BIM" { + let id = u16::from_be_bytes([b[4], b[5]]); + let name_len = b[6] as usize; + let name_total = (1 + name_len).next_multiple_of(2); + let size_at = 6 + name_total; + let size = u32::from_be_bytes(b.get(size_at..size_at + 4)?.try_into().ok()?) as usize; + let data_at = size_at + 4; + let data = b.get(data_at..data_at.checked_add(size)?)?; + if id == 0x0404 { + return Some(data.to_vec()); + } + b = b.get(data_at + size.next_multiple_of(2)..)?; + } + None +} + +/// PNG chunks: `eXIf`, uncompressed `iTXt` XMP. +pub fn png_chunks(b: &[u8]) -> Embedded { + let mut out = Embedded::default(); + let mut i = 8usize; + while i + 12 <= b.len() { + let len = u32::from_be_bytes([b[i], b[i + 1], b[i + 2], b[i + 3]]) as usize; + let typ = &b[i + 4..i + 8]; + let Some(data) = b.get(i + 8..(i + 8).saturating_add(len)) else { break }; + match typ { + b"eXIf" => out.exif = Some(crate::strip_exif_header(data).to_vec()), + b"iTXt" if data.starts_with(b"XML:com.adobe.xmp\0") => { + // keyword\0 compression-flag compression-method language\0 translated\0 text + let rest = &data[18..]; + if rest.len() >= 2 && rest[0] == 0 { + let rest = &rest[2..]; + let mut parts = rest.splitn(3, |&c| c == 0); + let (_, _, text) = (parts.next(), parts.next(), parts.next()); + if let Some(t) = text { + out.xmp = Some(String::from_utf8_lossy(t).into_owned()); + } + } + } + b"IEND" => break, + _ => {} + } + i += 12 + len; + } + out +} + +/// WebP RIFF chunks `EXIF`, `XMP `, `ICCP`. +pub fn webp_chunks(b: &[u8]) -> Embedded { + let mut out = Embedded::default(); + let mut i = 12usize; + while i + 8 <= b.len() { + let id = &b[i..i + 4]; + let len = u32::from_le_bytes([b[i + 4], b[i + 5], b[i + 6], b[i + 7]]) as usize; + let Some(data) = b.get(i + 8..(i + 8).saturating_add(len)) else { break }; + match id { + b"EXIF" => out.exif = Some(crate::strip_exif_header(data).to_vec()), + b"XMP " => out.xmp = Some(String::from_utf8_lossy(data).into_owned()), + b"ICCP" => out.icc = Some(data.to_vec()), + _ => {} + } + i += 8 + len + (len & 1); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::exif::tests::sample_exif; + use craft_tiff::ByteOrder; + + fn seg(marker: u8, payload: &[u8]) -> Vec { + let mut v = vec![0xff, marker]; + v.extend_from_slice(&((payload.len() + 2) as u16).to_be_bytes()); + v.extend_from_slice(payload); + v + } + + fn icc_seg(sequence: u8, total: u8, data: &[u8]) -> Vec { + let mut payload = b"ICC_PROFILE\0".to_vec(); + payload.extend_from_slice(&[sequence, total]); + payload.extend_from_slice(data); + seg(0xe2, &payload) + } + + fn jpeg_with_icc(parts: &[Vec]) -> Vec { + let mut jpeg = vec![0xff, 0xd8]; + let packet = crate::write_xmp(&crate::Metadata { title: Some("retained".into()), ..Default::default() }); + let mut xmp = XMP_SIG.to_vec(); + xmp.extend_from_slice(packet.as_bytes()); + jpeg.extend_from_slice(&seg(0xe1, &xmp)); + for part in parts { + jpeg.extend_from_slice(part); + } + jpeg.extend_from_slice(&[0xff, 0xda, 0, 2]); + jpeg + } + + pub(crate) fn sample_jpeg() -> Vec { + let mut j = vec![0xff, 0xd8]; + j.extend(seg(0xe0, b"JFIF\0\x01\x02\0\0\x01\0\x01\0\0")); + let mut exif = b"Exif\0\0".to_vec(); + exif.extend(sample_exif(ByteOrder::Big)); + j.extend(seg(0xe1, &exif)); + let meta = crate::Metadata { title: Some("Sunset".into()), rating: Some(4), ..Default::default() }; + let mut xmp = XMP_SIG.to_vec(); + xmp.extend(crate::write_xmp(&meta).into_bytes()); + j.extend(seg(0xe1, &xmp)); + // ICC in two chunks, out of order + j.extend(seg(0xe2, b"ICC_PROFILE\0\x02\x02WORLD")); + j.extend(seg(0xe2, b"ICC_PROFILE\0\x01\x02HELLO")); + // IPTC + let iim = crate::iptc::tests::sample_iim(); + let mut ps = b"Photoshop 3.0\0".to_vec(); + ps.extend_from_slice(b"8BIM\x04\x0c\0\0"); + ps.extend_from_slice(&2u32.to_be_bytes()); + ps.extend_from_slice(&[1, 2]); + ps.extend_from_slice(b"8BIM\x04\x04\0\0"); + ps.extend_from_slice(&(iim.len() as u32).to_be_bytes()); + ps.extend_from_slice(&iim); + j.extend(seg(0xed, &ps)); + j.extend_from_slice(&[0xff, 0xda, 0, 2, 1, 2, 3, 0xff, 0xd9]); + j + } + + #[test] + fn jpeg_blocks() { + let j = sample_jpeg(); + let e = jpeg_segments(&j); + assert_eq!(e.icc.as_deref(), Some(&b"HELLOWORLD"[..])); + assert!(e.xmp.as_deref().unwrap().contains("Sunset")); + assert!(e.iptc.is_some()); + let m = crate::extract(&j); + assert_eq!(m.iso, Some(400)); + assert_eq!(m.title.as_deref(), Some("Sunset")); + assert_eq!(m.rating, Some(4)); // XMP overrides EXIF rating 3 + assert_eq!(m.keywords, vec!["alpha".to_string(), "beta".to_string()]); // from IPTC + assert_eq!(m.caption.as_deref(), Some("A caption")); + } + + #[test] + fn jpeg_extended_xmp() { + let mut j = vec![0xff, 0xd8]; + for (off, part) in [(5u32, &b"world"[..]), (0, b"hello")] { + let mut p = XMP_EXT_SIG.to_vec(); + p.extend_from_slice(&[b'A'; 32]); + p.extend_from_slice(&10u32.to_be_bytes()); + p.extend_from_slice(&off.to_be_bytes()); + p.extend_from_slice(part); + j.extend(seg(0xe1, &p)); + } + assert_eq!(jpeg_segments(&j).xmp_extended.as_deref(), Some("helloworld")); + } + + #[test] + fn jpeg_icc_accepts_reordered_chunks() { + let parts = [icc_seg(2, 2, b"world"), icc_seg(1, 2, b"hello")]; + let embedded = jpeg_segments(&jpeg_with_icc(&parts)); + assert_eq!(embedded.icc.as_deref(), Some(&b"helloworld"[..])); + } + + #[test] + fn jpeg_icc_rejects_incomplete_or_incoherent_chunks() { + let cases = [ + ("missing", vec![icc_seg(1, 2, b"helloworld")]), + ("duplicate", vec![icc_seg(1, 2, b"hello"), icc_seg(1, 2, b"world")]), + ("zero sequence", vec![icc_seg(0, 1, b"helloworld")]), + ("zero total", vec![icc_seg(1, 0, b"helloworld")]), + ("out of range", vec![icc_seg(2, 1, b"helloworld")]), + ("gap", vec![icc_seg(1, 3, b"hello"), icc_seg(3, 3, b"world")]), + ("inconsistent totals", vec![icc_seg(1, 2, b"hello"), icc_seg(2, 3, b"world")]), + ("short header", vec![seg(0xe2, b"ICC_PROFILE\0\x01")]), + ]; + for (name, parts) in cases { + let embedded = jpeg_segments(&jpeg_with_icc(&parts)); + assert!(embedded.icc.is_none(), "{name} ICC chunks must be rejected"); + assert!(embedded.xmp.as_deref().is_some_and(|x| x.contains("retained")), "{name}: unrelated XMP must remain readable"); + } + } + + #[test] + fn jpeg_icc_rejects_truncated_segment_and_keeps_other_metadata() { + let mut truncated = icc_seg(1, 1, b"profile"); + let declared = u16::from_be_bytes([truncated[2], truncated[3]]).saturating_add(1); + truncated[2..4].copy_from_slice(&declared.to_be_bytes()); + let mut jpeg = vec![0xff, 0xd8]; + let xmp = crate::write_xmp(&crate::Metadata { title: Some("kept".into()), ..Default::default() }); + let mut packet = XMP_SIG.to_vec(); + packet.extend_from_slice(xmp.as_bytes()); + jpeg.extend_from_slice(&seg(0xe1, &packet)); + jpeg.extend_from_slice(&truncated); + + let embedded = jpeg_segments(&jpeg); + assert!(embedded.icc.is_none()); + assert!(embedded.xmp.as_deref().is_some_and(|x| x.contains("kept"))); + } + + #[test] + fn png_and_webp() { + let exif = sample_exif(ByteOrder::Little); + let mut p = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut chunk = |typ: &[u8], data: &[u8]| { + p.extend_from_slice(&(data.len() as u32).to_be_bytes()); + p.extend_from_slice(typ); + p.extend_from_slice(data); + p.extend_from_slice(&[0; 4]); + }; + chunk(b"eXIf", &exif); + let mut itxt = b"XML:com.adobe.xmp\0\0\0\0\0".to_vec(); + itxt.extend(crate::write_xmp(&crate::Metadata { label: Some("Red".into()), ..Default::default() }).into_bytes()); + chunk(b"iTXt", &itxt); + chunk(b"IEND", &[]); + let m = crate::extract(&p); + assert_eq!(m.iso, Some(400)); + assert_eq!(m.label.as_deref(), Some("Red")); + + let mut w = b"RIFF\0\0\0\0WEBP".to_vec(); + let mut body = |id: &[u8], data: &[u8]| { + w.extend_from_slice(id); + w.extend_from_slice(&(data.len() as u32).to_le_bytes()); + w.extend_from_slice(data); + if data.len() % 2 == 1 { + w.push(0); + } + }; + body(b"VP8X", &[0; 10]); + body(b"ICCP", b"abc"); + let mut ex = b"Exif\0\0".to_vec(); + ex.extend_from_slice(&exif); + body(b"EXIF", &ex); + let e = webp_chunks(&w); + assert_eq!(e.icc.as_deref(), Some(&b"abc"[..])); + assert_eq!(crate::extract(&w).f_number, Some(2.8)); + } + + #[test] + fn tiff_file_is_its_own_exif() { + let t = sample_exif(ByteOrder::Big); + assert_eq!(crate::extract(&t).model.as_deref(), Some("Model X")); + assert_eq!(embedded(b"not an image"), Embedded::default()); + } + + #[test] + fn raf_reads_its_embedded_jpeg() { + let j = sample_jpeg(); + let mut raf = b"FUJIFILMCCD-RAW 0201FF000000TEST".to_vec(); + raf.resize(120, 0); + raf[84..88].copy_from_slice(&120u32.to_be_bytes()); + raf[88..92].copy_from_slice(&(j.len() as u32).to_be_bytes()); + raf.extend_from_slice(&j); + assert_eq!(crate::extract(&raf).iso, Some(400)); + for n in 0..raf.len() { + let _ = crate::extract(&raf[..n]); + } + } + + #[test] + fn truncated_containers_do_not_panic() { + let j = sample_jpeg(); + for n in 0..j.len() { + let _ = crate::extract(&j[..n]); + } + } +} diff --git a/crates/metadata/src/cr3.rs b/crates/metadata/src/cr3.rs new file mode 100644 index 000000000..50f3f28d2 --- /dev/null +++ b/crates/metadata/src/cr3.rs @@ -0,0 +1,742 @@ +//! Canon CR3: the ISO base media file container (ISO/IEC 14496-12 box structure; CR3 layout from Laurent +//! Clévy's public CR3 format notes, confirmed black-box on EOS R6 Mark III files). +//! +//! - `ftyp` with major brand `crx `. +//! - `moov` holds a Canon `uuid` box (85c0b687-820f-11e0-8111-f4ce462b6a48) whose children are `CMT1` (a TIFF +//! stream: IFD0), `CMT2` (TIFF: the Exif IFD), `CMT3` (TIFF: the Canon maker note), `CMT4` (TIFF: GPS) and the +//! `THMB` thumbnail, then one `trak` per stream: a full-size JPEG, a reduced raw, the full raw (`CRAW` sample +//! entries; raws carry a `CMP1` coding header) and timed metadata (`CTMD`). +//! - A top-level `uuid` box with the XMP uuid (be7acfcb-97a9-42e8-9c71-999491e3afac, XMP spec part 3) holds the +//! XMP packet. +//! +//! Every size and offset is checked against the buffer; nesting and box counts are bounded. + +/// A track's sample-entry kind. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Cr3TrackKind { + /// `CRAW` with a `JPEG` child: the full-size JPEG. + Jpeg, + /// `CRAW` raw image: dimensions and the `CMP1` coding header (byte range in the file). + Raw { + width: u16, + height: u16, + cmp1: Option<(usize, usize)>, + /// `CDI1/IAD1` image-area descriptor (byte range in the file). + iad1: Option<(usize, usize)>, + }, + /// Anything else (`CTMD` timed metadata, unknown entries). + Other([u8; 4]), +} + +/// One track's single sample: where it is in the file. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cr3Track { + pub kind: Cr3TrackKind, + /// Byte offset and length of the sample (the first one) in the file, when the tables give them. + pub data: Option<(usize, usize)>, +} + +/// Canon's `CMP1` image coding descriptor, separate from the encoded track sample. +/// +/// Parsed from the payload (after the eight-byte box header). Geometry is the full sensor mosaic, +/// including optically masked borders. Unsupported encoding modes are retained for the decoder to +/// report explicitly, rather than confused with the full-size embedded JPEG. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cr3Compression { + pub version: u16, + pub width: u32, + pub height: u32, + pub tile_width: u32, + pub tile_height: u32, + pub bit_depth: u8, + pub planes: u8, + /// 0 = RGGB, 1 = GRBG, 2 = GBRG, 3 = BGGR. + pub cfa_pattern: u8, + pub encoding: u8, + pub levels: u8, + pub tile_flags: u8, + /// Number of bytes at the start of the sample before its entropy-coded data. + pub header_size: u32, + pub median_bit_depth: Option, +} + +impl Cr3Compression { + /// Read the public `CMP1` field layout. Truncated or impossible descriptors return `None`. + pub fn parse(payload: &[u8]) -> Option { + // The first word is reserved in the public notes; real files use FF00, although older + // descriptions guessed FFFF. It is not a reliable format signature. + let header_len = usize::from(be16(payload, 2)?).checked_add(4)?; + if header_len < 52 || header_len > payload.len() { + return None; + } + let packed = *payload.get(25)?; + let coding = *payload.get(26)?; + let mut out = Self { + version: be16(payload, 4)?, + width: be32(payload, 8)?, + height: be32(payload, 12)?, + tile_width: be32(payload, 16)?, + tile_height: be32(payload, 20)?, + bit_depth: *payload.get(24)?, + planes: packed >> 4, + cfa_pattern: packed & 15, + encoding: coding >> 4, + levels: coding & 15, + tile_flags: *payload.get(27)?, + header_size: be32(payload, 28)?, + median_bit_depth: None, + }; + if out.width == 0 + || out.height == 0 + || out.tile_width == 0 + || out.tile_height == 0 + || out.tile_width > out.width + || out.tile_height > out.height + || !(1..=16).contains(&out.bit_depth) + || !(1..=4).contains(&out.planes) + || (out.planes > 1 && out.cfa_pattern > 3) + || out.header_size == 0 + { + return None; + } + if payload.get(32).is_some_and(|flags| flags & 0x80 != 0) { + // Extended CMP1: a four-byte length, flags, reserved bytes and six words. The optional + // median precision is used by type-3 burst encodings, not ordinary Bayer raw/craw. + let extension_len = usize::try_from(be32(payload, 52)?).ok()?; + let extension_end = 52usize.checked_add(extension_len)?; + if extension_len < 32 || extension_end > payload.len() { + return None; + } + if payload.get(56).is_some_and(|flags| flags & 0x40 != 0) { + let precision = *payload.get(84).filter(|_| extension_end > 84)?; + if !(1..=16).contains(&precision) { + return None; + } + out.median_bit_depth = Some(precision); + } + } + Some(out) + } +} + +/// `IAD1` geometry. Coordinates are inclusive sensor offsets: `[left, top, right, bottom]`. +/// A full raw descriptor distinguishes valid sensor pixels from the recommended final crop. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cr3ImageArea { + pub width: u16, + pub height: u16, + pub crop: [u16; 4], + pub active: Option<[u16; 4]>, + pub masked_left: [u16; 4], + pub masked_top: Option<[u16; 4]>, +} + +impl Cr3ImageArea { + pub fn parse(payload: &[u8]) -> Option { + let bounds = |at: usize| Some([be16(payload, at)?, be16(payload, at + 2)?, be16(payload, at + 4)?, be16(payload, at + 6)?]); + let width = be16(payload, 4)?; + let height = be16(payload, 6)?; + if width == 0 || height == 0 { + return None; + } + let full = be16(payload, 10)? == 2; + Some(Self { + width, + height, + crop: bounds(16)?, + masked_left: bounds(24)?, + masked_top: if full { Some(bounds(32)?) } else { None }, + active: if full { Some(bounds(40)?) } else { None }, + }) + } +} + +impl Cr3Track { + /// Decode this track's image coding descriptor without reading its pixel data. + pub fn compression(&self, bytes: &[u8]) -> Option { + let Cr3TrackKind::Raw { cmp1: Some((at, len)), .. } = self.kind else { return None }; + Cr3Compression::parse(bytes.get(at..at.checked_add(len)?)?) + } + + pub fn image_area(&self, bytes: &[u8]) -> Option { + let Cr3TrackKind::Raw { iad1: Some((at, len)), .. } = self.kind else { return None }; + Cr3ImageArea::parse(bytes.get(at..at.checked_add(len)?)?) + } +} + +/// The parts of a CR3 file that are read. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Cr3<'a> { + /// `CMT1`…`CMT4` (index 0…3). + pub cmt: [Option<&'a [u8]>; 4], + pub thumbnail: Option<&'a [u8]>, + pub xmp: Option<&'a [u8]>, + pub tracks: Vec, +} + +/// One independent TIFF block from Canon timed metadata. Its offsets remain relative to this block. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Cr3TimedExif<'a> { + /// CTMD record type (7, 8 or 9). + pub record_type: u16, + /// Exif IFD (`0x8769`) or Canon maker note (`0x927c`). + pub tag: u32, + pub data: &'a [u8], +} + +impl Cr3<'_> { + /// Read TIFF blocks in the first samples of CTMD tracks. As-shot white balance and sensor + /// levels often live here rather than in the static `CMT3` maker note. Both record layers use + /// little-endian byte lengths; TIFF blocks retain their own byte order and offset origin. + pub fn timed_exif<'a>(&self, bytes: &'a [u8]) -> Vec> { + let mut out = Vec::new(); + let mut budget = MAX_BOXES; + for track in &self.tracks { + if track.kind != Cr3TrackKind::Other(*b"CTMD") { + continue; + } + let Some(sample) = track.data.and_then(|(at, len)| at.checked_add(len).and_then(|end| bytes.get(at..end))) else { continue }; + let mut at = 0usize; + while budget > 0 { + budget -= 1; + let Some(len) = le32(sample, at).and_then(|v| usize::try_from(v).ok()) else { break }; + let Some(end) = at.checked_add(len).filter(|&end| len >= 12 && end <= sample.len()) else { break }; + let Some(record) = sample.get(at..end) else { break }; + let Some(kind) = le16(record, 4) else { break }; + if matches!(kind, 7..=9) { + let mut item_at = 12usize; + while budget > 0 { + budget -= 1; + let Some(item_len) = le32(record, item_at).and_then(|v| usize::try_from(v).ok()) else { break }; + let Some(item_end) = item_at.checked_add(item_len).filter(|&end| item_len >= 8 && end <= record.len()) else { break }; + let Some(item) = record.get(item_at..item_end) else { break }; + if let Some(tag @ (0x8769 | 0x927c)) = le32(item, 4) + && let Some(data) = item.get(8..) + { + out.push(Cr3TimedExif { record_type: kind, tag, data }); + } + item_at = item_end; + } + } + at = end; + } + } + out + } +} + +const CANON_UUID: [u8; 16] = [0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48]; +/// The `uuid` of the top-level box holding the XMP packet. +pub const XMP_UUID: [u8; 16] = [0xbe, 0x7a, 0xcf, 0xcb, 0x97, 0xa9, 0x42, 0xe8, 0x9c, 0x71, 0x99, 0x94, 0x91, 0xe3, 0xaf, 0xac]; +const MAX_DEPTH: usize = 12; +pub(crate) const MAX_BOXES: usize = 4096; +/// A `VisualSampleEntry` (78 bytes after the box header) plus 4 bytes Canon adds before the child boxes. +const CRAW_CHILDREN_AT: usize = 82; + +/// Does this look like a CR3 file (`ftyp` box with major brand `crx `)? +pub fn is_cr3(bytes: &[u8]) -> bool { + bytes.get(4..12) == Some(b"ftypcrx ".as_slice()) +} + +/// Parse a CR3 file's boxes; `None` when it is not a CR3 file. Of the top-level boxes only `moov` +/// and the XMP `uuid` are read (a reader of a sparse file relies on it to skip the others). +pub fn parse_cr3(bytes: &[u8]) -> Option> { + if !is_cr3(bytes) { + return None; + } + let mut out = Cr3::default(); + let mut budget = MAX_BOXES; + for b in boxes(bytes, 0, bytes.len(), &mut budget) { + match &b.kind { + b"moov" => moov(bytes, &b, &mut out, &mut budget), + b"uuid" if b.uuid == Some(XMP_UUID) => out.xmp = bytes.get(b.body..b.end), + _ => {} + } + } + Some(out) +} + +pub(crate) struct BoxRef { + pub(crate) kind: [u8; 4], + /// Start of the payload (after the header and, for `uuid`, the 16-byte uuid). + pub(crate) body: usize, + pub(crate) end: usize, + pub(crate) uuid: Option<[u8; 16]>, +} + +/// The boxes directly inside `[from, to)`; stops at the first malformed header. +pub(crate) fn boxes(bytes: &[u8], from: usize, to: usize, budget: &mut usize) -> Vec { + let mut out = Vec::new(); + let mut at = from; + let to = to.min(bytes.len()); + while at.saturating_add(8) <= to && *budget > 0 { + *budget -= 1; + let (Some(size), Some(kind)) = (be32(bytes, at), bytes.get(at + 4..at + 8)) else { break }; + let mut kind4 = [0u8; 4]; + kind4.copy_from_slice(kind); + let (len, mut hdr) = match size { + 0 => (to - at, 8), + 1 => match be64(bytes, at + 8).and_then(|v| usize::try_from(v).ok()) { + Some(l) => (l, 16), + None => break, + }, + n => (n as usize, 8), + }; + let Some(end) = at.checked_add(len).filter(|&e| e <= to && len >= hdr) else { break }; + let mut uuid = None; + if &kind4 == b"uuid" { + let Some(u) = bytes.get(at + hdr..at + hdr + 16).filter(|_| at + hdr + 16 <= end) else { break }; + let mut a = [0u8; 16]; + a.copy_from_slice(u); + uuid = Some(a); + hdr += 16; + } + out.push(BoxRef { kind: kind4, body: at + hdr, end, uuid }); + at = end; + } + out +} + +fn moov<'a>(bytes: &'a [u8], m: &BoxRef, out: &mut Cr3<'a>, budget: &mut usize) { + for b in boxes(bytes, m.body, m.end, budget) { + match &b.kind { + b"uuid" if b.uuid == Some(CANON_UUID) => { + for c in boxes(bytes, b.body, b.end, budget) { + let slot = match &c.kind { + b"CMT1" => 0, + b"CMT2" => 1, + b"CMT3" => 2, + b"CMT4" => 3, + b"THMB" => { + out.thumbnail = bytes.get(c.body..c.end); + continue; + } + _ => continue, + }; + out.cmt[slot] = bytes.get(c.body..c.end); + } + } + b"trak" => { + if let Some(t) = trak(bytes, &b, budget) { + out.tracks.push(t); + } + } + _ => {} + } + } +} + +/// Descend `trak` → `mdia` → `minf` → `stbl` and read its sample entry and first sample's location. +fn trak(bytes: &[u8], t: &BoxRef, budget: &mut usize) -> Option { + let mut stbl = None; + let mut stack = vec![(t.body, t.end, 0usize)]; + while let Some((from, to, depth)) = stack.pop() { + if depth > MAX_DEPTH { + continue; + } + for b in boxes(bytes, from, to, budget) { + match &b.kind { + b"mdia" | b"minf" => stack.push((b.body, b.end, depth + 1)), + b"stbl" => stbl = Some(b), + _ => {} + } + } + } + let stbl = stbl?; + let (mut kind, mut size, mut offset) = (None, None, None); + for b in boxes(bytes, stbl.body, stbl.end, budget) { + let body = bytes.get(b.body..b.end)?; + match &b.kind { + // full box: version/flags, entry count, entries + b"stsd" if be32(body, 4)? > 0 => kind = boxes(bytes, b.body + 8, b.end, budget).first().map(|e| sample_entry(bytes, e, budget)), + // version/flags, sample_size (0 = per-sample table), count, sizes + b"stsz" => { + size = match be32(body, 4)? { + 0 if be32(body, 8)? > 0 => be32(body, 12), + 0 => None, + n if be32(body, 8)? > 0 => Some(n), + _ => None, + } + } + b"co64" if be32(body, 4)? > 0 => offset = be64(body, 8).and_then(|v| usize::try_from(v).ok()), + b"stco" if be32(body, 4)? > 0 => offset = be32(body, 8).map(|v| v as usize), + _ => {} + } + } + let data = match (offset, size) { + (Some(o), Some(s)) if o.checked_add(s as usize).is_some_and(|e| e <= bytes.len()) => Some((o, s as usize)), + _ => None, + }; + Some(Cr3Track { kind: kind?, data }) +} + +fn sample_entry(bytes: &[u8], e: &BoxRef, budget: &mut usize) -> Cr3TrackKind { + if &e.kind != b"CRAW" { + return Cr3TrackKind::Other(e.kind); + } + let Some(body) = bytes.get(e.body..e.end) else { return Cr3TrackKind::Other(e.kind) }; + let (Some(width), Some(height)) = (be16(body, 24), be16(body, 26)) else { + return Cr3TrackKind::Other(e.kind); + }; + let (mut cmp1, mut iad1, mut rendered) = (None, None, None); + for c in boxes(bytes, e.body.saturating_add(CRAW_CHILDREN_AT), e.end, budget) { + match &c.kind { + b"JPEG" => rendered = Some(Cr3TrackKind::Jpeg), + // HDR-capable cameras also put an HEVC preview in a CRAW entry, + // often with the same displayed dimensions as the main raw track. + b"HEVC" => rendered = Some(Cr3TrackKind::Other(c.kind)), + b"CMP1" => cmp1 = Some((c.body, c.end - c.body)), + b"CDI1" => { + for area in boxes(bytes, c.body.saturating_add(4), c.end, budget) { + if &area.kind == b"IAD1" { + iad1 = Some((area.body, area.end - area.body)); + } + } + } + _ => {} + } + } + // A present, malformed CMP1 must remain a raw descriptor so its corruption + // is reported. Only an explicitly identified rendered codec can establish + // that a CRAW entry without CMP1 is a preview rather than a broken raw. + if cmp1.is_none() + && let Some(kind) = rendered + { + return kind; + } + Cr3TrackKind::Raw { width, height, cmp1, iad1 } +} + +/// A big-endian `u16` at `at`, or `None` when the buffer is shorter than that. +pub(crate) fn be16(b: &[u8], at: usize) -> Option { + b.get(at..at.checked_add(2)?).map(|s| u16::from_be_bytes([s[0], s[1]])) +} +/// A big-endian `u32` at `at`, or `None` when the buffer is shorter than that. +pub(crate) fn be32(b: &[u8], at: usize) -> Option { + b.get(at..at.checked_add(4)?).map(|s| u32::from_be_bytes([s[0], s[1], s[2], s[3]])) +} +/// A big-endian `u64` at `at`, or `None` when the buffer is shorter than that. +pub(crate) fn be64(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(8)?)?; + let mut a = [0u8; 8]; + a.copy_from_slice(s); + Some(u64::from_be_bytes(a)) +} + +fn le16(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(2)?)?; + Some(u16::from_le_bytes([s[0], s[1]])) +} + +fn le32(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(4)?)?; + Some(u32::from_le_bytes([s[0], s[1], s[2], s[3]])) +} + +/// Merge `CMT1` (IFD0), `CMT2` (Exif) and `CMT4` (GPS) into one TIFF stream, so the Exif readers see a CR3 +/// like any TIFF-based raw. The maker note (`CMT3`) is left out: its offsets are relative to its own block. +pub fn merged_exif(c: &Cr3<'_>) -> Option> { + use craft_tiff::{IfdBuilder, Tiff, TiffWriter, tags as t}; + let ifd0 = Tiff::parse(c.cmt[0]?).ok()?; + let order = ifd0.order; + let first = |i: usize| c.cmt[i].and_then(|b| Tiff::parse(b).ok()).and_then(|t| t.ifds.into_iter().next()); + // pointers and offsets that would dangle once the IFDs move + const SKIP: [u16; 9] = [t::EXIF_IFD, t::GPS_IFD, t::INTEROP_IFD, t::SUB_IFDS, t::MAKER_NOTE, 273, 279, 513, 514]; + let build = |ifd: &craft_tiff::Ifd| { + let mut b = IfdBuilder::new(); + for e in ifd.entries.iter().filter(|e| !SKIP.contains(&e.tag)) { + b.set(e.tag, e.value.clone()); + } + b + }; + let mut root = build(ifd0.ifds.first()?); + if let Some(exif) = first(1) { + root.set_child(t::EXIF_IFD, build(&exif)); + } + if let Some(gps) = first(3) { + root.set_child(t::GPS_IFD, build(&gps)); + } + TiffWriter::new(order, false).write(&[root]).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn bx(kind: &[u8; 4], body: &[u8]) -> Vec { + let mut v = ((body.len() + 8) as u32).to_be_bytes().to_vec(); + v.extend_from_slice(kind); + v.extend_from_slice(body); + v + } + fn uuid(u: &[u8; 16], body: &[u8]) -> Vec { + let mut b = u.to_vec(); + b.extend_from_slice(body); + bx(b"uuid", &b) + } + fn full(kind: &[u8; 4], body: &[u8]) -> Vec { + let mut b = vec![0u8; 4]; + b.extend_from_slice(body); + bx(kind, &b) + } + fn craw(w: u16, h: u16, children: &[u8]) -> Vec { + let mut b = vec![0u8; CRAW_CHILDREN_AT]; + b[24..26].copy_from_slice(&w.to_be_bytes()); + b[26..28].copy_from_slice(&h.to_be_bytes()); + b.extend_from_slice(children); + bx(b"CRAW", &b) + } + fn trak(entry: Vec, offset: u64, size: u32) -> Vec { + let mut stsd = 1u32.to_be_bytes().to_vec(); + stsd.extend(entry); + let mut stsz = 0u32.to_be_bytes().to_vec(); + stsz.extend(1u32.to_be_bytes()); + stsz.extend(size.to_be_bytes()); + let mut co64 = 1u32.to_be_bytes().to_vec(); + co64.extend(offset.to_be_bytes()); + let stbl = [full(b"stsd", &stsd), full(b"stsz", &stsz), full(b"co64", &co64)].concat(); + bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl)))) + } + fn tiff_with(tag: u16, v: craft_tiff::Value) -> Vec { + let b = craft_tiff::IfdBuilder::new().with(tag, v); + craft_tiff::TiffWriter::new(craft_tiff::ByteOrder::Little, false).write(&[b]).unwrap() + } + + /// A small synthetic CR3: Canon uuid with CMT1/CMT2/CMT4, a JPEG track, a raw track, XMP. + pub(crate) fn sample() -> Vec { + use craft_tiff::{Value, tags as t}; + let cmt1 = tiff_with(t::MODEL, Value::Ascii("Canon EOS Test".into())); + let cmt2 = tiff_with(t::ISO_SPEED, Value::Short(vec![400])); + let cmt4 = tiff_with(1, Value::Ascii("N".into())); + let canon = uuid(&CANON_UUID, &[bx(b"CMT1", &cmt1), bx(b"CMT2", &cmt2), bx(b"CMT4", &cmt4), bx(b"THMB", b"thumb")].concat()); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + let payload_at = 4096u64; + let jpeg_t = trak(craw(64, 48, &bx(b"JPEG", &[0; 4])), payload_at, 16); + let raw_t = trak(craw(60, 40, &bx(b"CMP1", &[0xff, 0x10, 0, 0x30])), payload_at + 16, 32); + file.extend(bx(b"moov", &[canon, jpeg_t, raw_t].concat())); + file.extend(uuid(&XMP_UUID, b"")); + file.resize(payload_at as usize + 64, 0); + file + } + + #[test] + fn parses_canon_boxes_tracks_and_xmp() { + let f = sample(); + assert!(is_cr3(&f)); + let c = parse_cr3(&f).unwrap(); + assert!(c.cmt[0].is_some() && c.cmt[1].is_some() && c.cmt[2].is_none() && c.cmt[3].is_some()); + assert_eq!(c.thumbnail, Some(b"thumb".as_slice())); + assert_eq!(c.xmp, Some(b"".as_slice())); + assert_eq!(c.tracks.len(), 2); + assert_eq!(c.tracks[0], Cr3Track { kind: Cr3TrackKind::Jpeg, data: Some((4096, 16)) }); + let Cr3TrackKind::Raw { width, height, cmp1: Some((at, len)), .. } = c.tracks[1].kind else { panic!("{:?}", c.tracks[1]) }; + assert_eq!((width, height, len), (60, 40, 4)); + assert_eq!(&f[at..at + 2], &[0xff, 0x10]); + assert_eq!(c.tracks[1].data, Some((4112, 32))); + } + + #[test] + fn hevc_preview_is_distinct_from_an_equal_size_raw_track() { + let preview = trak(craw(6000, 4000, &bx(b"HEVC", &[0; 4])), 4096, 16); + let reduced = trak(craw(1624, 1080, &bx(b"CMP1", &compression_payload(0))), 4112, 16); + let sensor = trak(craw(6000, 4000, &bx(b"CMP1", &compression_payload(0))), 4128, 16); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &[preview, reduced, sensor].concat())); + file.resize(4144, 0); + let container = parse_cr3(&file).unwrap(); + assert_eq!(container.tracks[0].kind, Cr3TrackKind::Other(*b"HEVC")); + assert!(container.tracks[0].compression(&file).is_none()); + assert!(matches!(container.tracks[2].kind, Cr3TrackKind::Raw { width: 6000, height: 4000, cmp1: Some(_), .. })); + assert_eq!(container.tracks[2].compression(&file).unwrap().width, 6888); + } + + #[test] + fn rendered_child_does_not_hide_a_malformed_raw_descriptor() { + for codec in [b"JPEG", b"HEVC"] { + for first in [true, false] { + let rendered = bx(codec, &[0; 4]); + let corrupt = bx(b"CMP1", &[0xff, 0]); + let children = if first { [rendered, corrupt] } else { [corrupt, rendered] }.concat(); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &trak(craw(6000, 4000, &children), 4096, 16))); + file.resize(4112, 0); + let container = parse_cr3(&file).unwrap(); + assert!(matches!(container.tracks[0].kind, Cr3TrackKind::Raw { cmp1: Some(_), .. })); + assert!(container.tracks[0].compression(&file).is_none()); + } + } + // Absence alone is not evidence of a preview: retain a broken raw so + // main-track selection cannot quietly fall back to a reduced image. + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &trak(craw(6000, 4000, &bx(b"free", &[])), 4096, 16))); + file.resize(4112, 0); + assert!(matches!(parse_cr3(&file).unwrap().tracks[0].kind, Cr3TrackKind::Raw { cmp1: None, .. })); + } + + #[test] + fn merged_exif_reads_like_a_tiff_raw() { + let f = sample(); + let m = crate::read_exif(&merged_exif(&parse_cr3(&f).unwrap()).unwrap()); + assert_eq!(m.model.as_deref(), Some("Canon EOS Test")); + assert_eq!(m.iso, Some(400)); + } + + fn compression_payload(levels: u8) -> Vec { + let mut b = vec![0u8; 52]; + b[0..2].copy_from_slice(&0xff00u16.to_be_bytes()); + b[2..4].copy_from_slice(&48u16.to_be_bytes()); + b[4..6].copy_from_slice(&0x100u16.to_be_bytes()); + for (at, value) in [(8, 6888u32), (12, 4546), (16, 3444), (20, 4546), (28, 216)] { + b[at..at + 4].copy_from_slice(&value.to_be_bytes()); + } + b[24] = 14; + b[25] = 0x40; + b[26] = levels; + b + } + + #[test] + fn compression_descriptor_retains_sensor_geometry_and_cfa() { + let b = compression_payload(0); + let h = Cr3Compression::parse(&b).unwrap(); + assert_eq!((h.width, h.height, h.tile_width, h.tile_height), (6888, 4546, 3444, 4546)); + assert_eq!((h.version, h.bit_depth, h.planes, h.cfa_pattern, h.encoding, h.levels, h.header_size), (0x100, 14, 4, 0, 0, 0, 216)); + let mut craw = compression_payload(3); + craw[25] = 0x41; + craw[27] = 0x80; + let h = Cr3Compression::parse(&craw).unwrap(); + assert_eq!((h.cfa_pattern, h.levels, h.tile_flags), (1, 3, 0x80)); + for len in 0..b.len() { + assert!(Cr3Compression::parse(&b[..len]).is_none()); + } + for (at, value) in [(24, 0), (24, 17), (25, 0), (25, 0x44)] { + let mut bad = b.clone(); + bad[at] = value; + assert!(Cr3Compression::parse(&bad).is_none()); + } + } + + #[test] + fn extended_descriptor_checks_optional_median_precision() { + let mut b = compression_payload(0x30); + b[32] = 0x80; + b.extend_from_slice(&33u32.to_be_bytes()); + b.resize(85, 0); + b[56] = 0xc0; + b[84] = 14; + assert_eq!(Cr3Compression::parse(&b).unwrap().median_bit_depth, Some(14)); + assert!(Cr3Compression::parse(&b[..84]).is_none()); + b[84] = 0; + assert!(Cr3Compression::parse(&b).is_none()); + } + + #[test] + fn raw_track_reads_distinct_active_area_and_default_crop() { + let mut area = vec![0u8; 48]; + for (at, value) in [ + (4, 6888u16), + (6, 4546), + (10, 2), + (16, 156), + (18, 158), + (20, 6875), + (22, 4537), + (24, 0), + (26, 0), + (28, 143), + (30, 4545), + (32, 144), + (34, 0), + (36, 6887), + (38, 45), + (40, 144), + (42, 46), + (44, 6887), + (46, 4545), + ] { + area[at..at + 2].copy_from_slice(&value.to_be_bytes()); + } + let cmp1 = bx(b"CMP1", &compression_payload(0)); + let iad1 = full(b"CDI1", &bx(b"IAD1", &area)); + let track = trak(craw(6888, 4546, &[cmp1, iad1].concat()), 4096, 32); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &track)); + file.resize(4128, 0); + let c = parse_cr3(&file).unwrap(); + assert_eq!(c.tracks[0].compression(&file).unwrap().width, 6888); + let geometry = c.tracks[0].image_area(&file).unwrap(); + assert_eq!(geometry.crop, [156, 158, 6875, 4537]); + assert_eq!(geometry.active, Some([144, 46, 6887, 4545])); + assert_eq!(geometry.masked_left, [0, 0, 143, 4545]); + for len in 0..area.len() { + assert!(Cr3ImageArea::parse(&area[..len]).is_none()); + } + } + + #[test] + fn truncated_sample_table_does_not_read_the_next_box() { + let stbl = [ + full(b"stsd", &[1u32.to_be_bytes().to_vec(), craw(16, 16, &bx(b"CMP1", &compression_payload(0)))].concat()), + bx(b"stsz", &[]), + full(b"co64", &[1u32.to_be_bytes().to_vec(), 256u64.to_be_bytes().to_vec()].concat()), + ] + .concat(); + let mut file = bx(b"ftyp", b"crx \0\0\0\x01crx isom"); + file.extend(bx(b"moov", &bx(b"trak", &bx(b"mdia", &bx(b"minf", &bx(b"stbl", &stbl)))))); + file.resize(512, 0); + assert!(parse_cr3(&file).unwrap().tracks.is_empty()); + } + + #[test] + fn timed_metadata_exposes_the_dynamic_maker_note() { + use craft_tiff::{Value, tags as t}; + let exif = tiff_with(t::ISO_SPEED, Value::Short(vec![800])); + let maker = tiff_with(0x4001, Value::Short(vec![48, 100, 200])); + let item = |tag: u32, data: &[u8]| { + let mut b = ((data.len() + 8) as u32).to_le_bytes().to_vec(); + b.extend_from_slice(&tag.to_le_bytes()); + b.extend_from_slice(data); + b + }; + let items = [item(0x8769, &exif), item(0x927c, &maker)].concat(); + let mut sample = vec![0u8; 12]; + sample[0..4].copy_from_slice(&((12 + items.len()) as u32).to_le_bytes()); + sample[4..6].copy_from_slice(&8u16.to_le_bytes()); + sample.extend(items); + let container = Cr3 { tracks: vec![Cr3Track { kind: Cr3TrackKind::Other(*b"CTMD"), data: Some((0, sample.len())) }], ..Default::default() }; + let parts = container.timed_exif(&sample); + assert_eq!(parts.len(), 2); + assert_eq!((parts[0].record_type, parts[0].tag, parts[0].data), (8, 0x8769, exif.as_slice())); + assert_eq!((parts[1].record_type, parts[1].tag, parts[1].data), (8, 0x927c, maker.as_slice())); + for len in 0..sample.len() { + assert!(container.timed_exif(&sample[..len]).is_empty()); + } + for at in [0, 12] { + let mut corrupt = sample.clone(); + corrupt[at..at + 4].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(container.timed_exif(&corrupt).is_empty()); + corrupt[at..at + 4].copy_from_slice(&0u32.to_le_bytes()); + assert!(container.timed_exif(&corrupt).is_empty()); + } + } + + #[test] + fn hostile_input_never_panics() { + let f = sample(); + // every truncation and single-byte corruption parses to something or nothing + for n in 0..f.len().min(800) { + let _ = parse_cr3(&f[..n]).map(|c| merged_exif(&c)); + } + for i in 0..f.len().min(800) { + for v in [0u8, 1, 0x7f, 0xff] { + let mut g = f.clone(); + g[i] = v; + let _ = parse_cr3(&g).map(|c| merged_exif(&c)); + } + } + // a sample offset past the end is not reported + let mut g = f.clone(); + g.truncate(4100); + assert!(parse_cr3(&g).unwrap().tracks.iter().all(|t| t.data.is_none())); + } +} diff --git a/crates/metadata/src/datetime.rs b/crates/metadata/src/datetime.rs new file mode 100644 index 000000000..0ecedae36 --- /dev/null +++ b/crates/metadata/src/datetime.rs @@ -0,0 +1,320 @@ +//! Capture date/time with optional UTC offset (Exif `DateTimeOriginal` + `OffsetTimeOriginal`, ISO 8601 in XMP). + +use serde::{Deserialize, Serialize}; + +/// A civil date-time as recorded by the camera; `offset_minutes` is the UTC offset when known. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +pub struct DateTime { + pub year: u16, + pub month: u8, + pub day: u8, + pub hour: u8, + pub minute: u8, + pub second: u8, + /// Sub-second part in milliseconds. + pub millis: u16, + pub offset_minutes: Option, +} + +fn num(s: &str) -> Option { + if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + s.parse().ok() +} + +impl DateTime { + fn valid(self) -> Option { + // A timestamp must represent a real Gregorian calendar day, not just + // numbers in the range 1..=31. Invalid dates would otherwise be + // accepted from EXIF/XMP and silently normalized during time arithmetic. + // chrono rejects impossible days (leap years and all); the time is checked + // by hand so a leap second (`second == 60`) stays acceptable. + let day_ok = self.year > 0 && chrono::NaiveDate::from_ymd_opt(i32::from(self.year), u32::from(self.month), u32::from(self.day)).is_some(); + let ok = day_ok && self.hour < 24 && self.minute < 60 && self.second < 61 && self.offset_minutes.is_none_or(|o| o.unsigned_abs() <= 18 * 60); + ok.then_some(self) + } + + /// Exif form `YYYY:MM:DD HH:MM:SS` (also tolerates `-` date separators and a missing time). + pub fn parse_exif(s: &str) -> Option { + let s = s.trim().trim_end_matches('\0'); + let (d, t) = s.split_once([' ', 'T']).unwrap_or((s, "")); + let dp: Vec<&str> = d.split([':', '-']).collect(); + if dp.len() != 3 { + return None; + } + let (hour, minute, second) = if t.is_empty() { + (0, 0, 0) + } else { + let mut parts = t.split(':'); + let hour = num(parts.next()?.trim())?; + let minute = parts.next().map(|part| num(part.trim())).unwrap_or(Some(0))?; + let second = parts.next().map(|part| num(part.trim())).unwrap_or(Some(0))?; + // A fourth value cannot be represented; accepting only the prefix could turn a + // malformed DateTimeOriginal into a plausible capture time and block fallbacks. + if parts.next().is_some() { + return None; + } + (hour, minute, second) + }; + DateTime { + year: num(dp[0])?.try_into().ok()?, + month: num(dp[1])?.try_into().ok()?, + day: num(dp[2])?.try_into().ok()?, + hour: hour.try_into().ok()?, + minute: minute.try_into().ok()?, + second: second.try_into().ok()?, + millis: 0, + offset_minutes: None, + } + .valid() + } + + /// Exif `OffsetTime*` form `+HH:MM` / `-HH:MM` (or `Z`). + pub fn parse_offset(s: &str) -> Option { + let s = s.trim().trim_end_matches('\0'); + if s == "Z" { + return Some(0); + } + let (sign, rest) = match s.as_bytes().first()? { + b'+' => (1, &s[1..]), + b'-' => (-1, &s[1..]), + _ => return None, + }; + let (h, m) = match rest.split_once(':') { + Some((h, m)) => (num(h)?, num(m)?), + None if rest.len() == 4 => (num(rest.get(..2)?)?, num(rest.get(2..)?)?), + None => (num(rest)?, 0), + }; + if h > 18 || m > 59 || (h == 18 && m > 0) { + return None; + } + Some(sign * (h * 60 + m) as i16) + } + + /// Set the sub-second part from an Exif `SubSecTime*` string (digits are a decimal fraction). + pub fn with_subsec(mut self, s: &str) -> DateTime { + let digits: String = s.trim().chars().take_while(|c| c.is_ascii_digit()).take(3).collect(); + if !digits.is_empty() { + let v: u16 = digits.parse().unwrap_or(0); + self.millis = v * 10u16.pow(3 - digits.len() as u32); + } + self + } + + /// ISO 8601 (XMP date): `YYYY`, `YYYY-MM`, `YYYY-MM-DD`, `YYYY-MM-DDThh:mm[:ss[.s+]][TZD]`. + pub fn parse_iso(s: &str) -> Option { + let s = s.trim(); + let (d, t) = match s.split_once('T') { + Some((d, t)) if !t.is_empty() => (d, t), + Some(_) => return None, + None => (s, ""), + }; + let dp: Vec<&str> = d.split('-').collect(); + let year = num(dp.first()?)?.try_into().ok()?; + let month = dp.get(1).map(|m| num(m)).unwrap_or(Some(1))?.try_into().ok()?; + let day = dp.get(2).map(|m| num(m)).unwrap_or(Some(1))?.try_into().ok()?; + if dp.len() > 3 { + return None; + } + let mut dt = DateTime { year, month, day, ..Default::default() }; + if !t.is_empty() { + let (clock, off) = match t.find(['Z', '+', '-']) { + Some(i) => (&t[..i], Some(&t[i..])), + None => (t, None), + }; + let mut cp = clock.split(':'); + dt.hour = num(cp.next()?)?.try_into().ok()?; + dt.minute = num(cp.next()?)?.try_into().ok()?; + if let Some(sec) = cp.next() { + let (whole, fraction) = match sec.split_once('.') { + Some((whole, fraction)) => { + if fraction.is_empty() || !fraction.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + (whole, Some(fraction)) + } + None => (sec, None), + }; + dt.second = num(whole)?.try_into().ok()?; + if let Some(fraction) = fraction { + dt = dt.with_subsec(fraction); + } + } + if cp.next().is_some() { + return None; + } + if let Some(o) = off { + dt.offset_minutes = Some(Self::parse_offset(o)?); + } + } + dt.valid() + } + + /// Exif form `YYYY:MM:DD HH:MM:SS`. + pub fn to_exif(&self) -> String { + format!("{:04}:{:02}:{:02} {:02}:{:02}:{:02}", self.year, self.month, self.day, self.hour, self.minute, self.second) + } + + /// ISO 8601 with milliseconds when non-zero and the offset when known. + pub fn to_iso(&self) -> String { + let mut s = format!("{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", self.year, self.month, self.day, self.hour, self.minute, self.second); + if self.millis != 0 { + s += &format!(".{:03}", self.millis); + } + if let Some(o) = self.offset_minutes { + let sign = if o < 0 { '-' } else { '+' }; + let o = o.unsigned_abs(); + s += &format!("{sign}{:02}:{:02}", o / 60, o % 60); + } + s + } + + /// Seconds since 1970-01-01T00:00:00 of the civil time (UTC when the offset is known, else "local" seconds). + pub fn unix_seconds(&self) -> i64 { + // Every `DateTime` reaching here is a real calendar day (`valid()` ran in the parsers); + // an out-of-range one (built by hand) reads as the epoch rather than panicking. + let days = chrono::NaiveDate::from_ymd_opt(i32::from(self.year), u32::from(self.month), u32::from(self.day)) + .zip(chrono::NaiveDate::from_ymd_opt(1970, 1, 1)) + .map(|(d, epoch)| (d - epoch).num_days()) + .unwrap_or(0); + days * 86400 + self.hour as i64 * 3600 + self.minute as i64 * 60 + self.second as i64 - self.offset_minutes.unwrap_or(0) as i64 * 60 + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exif_forms() { + let d = DateTime::parse_exif("2024:06:01 13:45:59").unwrap(); + assert_eq!((d.year, d.month, d.day, d.hour, d.minute, d.second), (2024, 6, 1, 13, 45, 59)); + assert_eq!(d.to_exif(), "2024:06:01 13:45:59"); + assert!(DateTime::parse_exif("0000:00:00 00:00:00").is_none()); + assert!(DateTime::parse_exif(" : : : : ").is_none()); + assert!(DateTime::parse_exif("2024:13:01 00:00:00").is_none()); + // Invalid calendar days must never enter persisted capture metadata. + for value in ["2023:02:29", "2024:02:30", "2024:04:31", "1900:02:29", "2024:01:00"] { + assert!(DateTime::parse_exif(value).is_none(), "{value}"); + } + assert!(DateTime::parse_exif("2024:02:29").is_some()); + assert!(DateTime::parse_exif("2000:02:29").is_some()); + for value in ["2023-02-29", "2024-02-30", "1900-02-29", "2024-06-31T10:00:00Z"] { + assert!(DateTime::parse_iso(value).is_none(), "{value}"); + } + assert!(DateTime::parse_iso("2000-02-29T23:59:59Z").is_some()); + assert_eq!(DateTime::parse_exif("2024-06-01").unwrap().hour, 0); + assert_eq!(DateTime::parse_exif("2024:06:01 13:45:59").unwrap().with_subsec("5").millis, 500); + assert_eq!(DateTime::parse_exif("2024:06:01 13:45:59").unwrap().with_subsec("0123").millis, 12); + } + + #[test] + fn exif_clock_rejects_extra_components() { + for value in + ["2026:01:01 12:34:56:99", "2026:01:01 12:34:56:junk", "2026:01:01 12:34:56:", "2026:01:01 12:34:56::", "2026:01:01 12:34:56:99:junk"] + { + assert!(DateTime::parse_exif(value).is_none(), "accepted malformed EXIF clock {value:?}"); + } + } + + #[test] + fn exif_clock_supported_forms_keep_their_meaning() { + for (value, expected) in [ + ("2026:01:01", "2026-01-01T00:00:00"), + ("2026:01:01 12", "2026-01-01T12:00:00"), + ("2026:01:01 12:34", "2026-01-01T12:34:00"), + ("2026:01:01 12:34:56", "2026-01-01T12:34:56"), + ("2026-01-01T12:34:56", "2026-01-01T12:34:56"), + (" 2026:01:01 12:34:56\0", "2026-01-01T12:34:56"), + ("2026:01:01T12:34:60", "2026-01-01T12:34:60"), + ] { + assert_eq!(DateTime::parse_exif(value).map(|date| date.to_iso()), Some(expected.to_string()), "{value:?}"); + } + } + + #[test] + fn offsets() { + assert_eq!(DateTime::parse_offset("+02:00"), Some(120)); + assert_eq!(DateTime::parse_offset("-05:30"), Some(-330)); + assert_eq!(DateTime::parse_offset("Z"), Some(0)); + assert_eq!(DateTime::parse_offset("+0930"), Some(570)); + assert_eq!(DateTime::parse_offset("+18:00"), Some(1080)); + assert_eq!(DateTime::parse_offset("-18:00"), Some(-1080)); + assert_eq!(DateTime::parse_offset("+18:01"), None); + assert_eq!(DateTime::parse_offset("-18:01"), None); + assert_eq!(DateTime::parse_offset(" : "), None); + assert_eq!(DateTime::parse_offset("+99:00"), None); + } + + #[test] + fn extreme_offsets_do_not_overflow() { + // `DateTime` deserializes with public fields, so an offset can arrive without passing `valid()` + let d = DateTime { offset_minutes: Some(i16::MIN), ..DateTime::parse_iso("2024-06-01T13:45:59").unwrap() }; + assert_eq!(d.to_iso(), "2024-06-01T13:45:59-546:08"); + assert_eq!(d.valid(), None); + } + + #[test] + fn unicode_offsets_are_rejected_without_panicking() { + for offset in ["+1€", "-1€", "+€1", "-€1"] { + assert_eq!(DateTime::parse_offset(offset), None); + let date = format!("2024-06-01T13:45:59{offset}"); + let metadata = crate::Metadata { capture_time: DateTime::parse_iso("2024-06-01T13:45:59Z"), ..Default::default() }; + let xmp = crate::write_xmp(&metadata).replace("2024-06-01T13:45:59+00:00", &date); + assert!(crate::parse_xmp(&xmp).unwrap().metadata.capture_time.is_none()); + } + } + + #[test] + fn iso_roundtrip() { + for s in ["2024-06-01T13:45:59", "2024-06-01T13:45:59.250+02:00", "1999-12-31T23:59:00-05:30", "2024-02-29T00:00:00Z"] { + let d = DateTime::parse_iso(s).unwrap(); + let again = DateTime::parse_iso(&d.to_iso()).unwrap(); + assert_eq!(d, again, "{s}"); + } + assert_eq!(DateTime::parse_iso("2024-02-29T00:00:00Z").unwrap().to_iso(), "2024-02-29T00:00:00+00:00"); + let d = DateTime::parse_iso("2019").unwrap(); + assert_eq!((d.year, d.month, d.day), (2019, 1, 1)); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11").unwrap().minute, 11); + assert!(DateTime::parse_iso("nonsense").is_none()); + assert!(DateTime::parse_iso("2019-07-04Tab:cd").is_none()); + } + + #[test] + fn iso_rejects_incomplete_or_unconsumed_clock_text() { + for value in [ + "2026-01-01T", + "2026-01-01T12:34:56:99Z", + "2026-01-01T12:34:56.Z", + "2026-01-01T12:34:56.123junkZ", + "2026-01-01T12:34:56.1.2Z", + "2026-01-01T12:34:56.12 3Z", + "2026-01-01T12:34:56:99Z", + "2026-01-01T12::34Z", + ] { + assert!(DateTime::parse_iso(value).is_none(), "accepted malformed timestamp {value:?}"); + } + } + + #[test] + fn iso_validates_clock_while_preserving_supported_forms() { + assert_eq!(DateTime::parse_iso("2019").unwrap().to_iso(), "2019-01-01T00:00:00"); + assert_eq!(DateTime::parse_iso("2019-07").unwrap().to_iso(), "2019-07-01T00:00:00"); + assert_eq!(DateTime::parse_iso("2019-07-04").unwrap().to_iso(), "2019-07-04T00:00:00"); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11").unwrap().to_iso(), "2019-07-04T10:11:00"); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11:12").unwrap().to_iso(), "2019-07-04T10:11:12"); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11:12Z").unwrap().offset_minutes, Some(0)); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11:12-05:30").unwrap().offset_minutes, Some(-330)); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11:12.123456Z").unwrap().millis, 123); + assert_eq!(DateTime::parse_iso(" 2019-07-04T10:11:12.5Z ").unwrap().millis, 500); + assert_eq!(DateTime::parse_iso("2019-07-04T10:11:60Z").unwrap().second, 60); + } + + #[test] + fn unix() { + assert_eq!(DateTime::parse_iso("1970-01-01T00:00:00Z").unwrap().unix_seconds(), 0); + assert_eq!(DateTime::parse_iso("2000-03-01T00:00:00Z").unwrap().unix_seconds(), 951868800); + assert_eq!(DateTime::parse_iso("2000-03-01T02:00:00+02:00").unwrap().unix_seconds(), 951868800); + } +} diff --git a/crates/metadata/src/exif.rs b/crates/metadata/src/exif.rs new file mode 100644 index 000000000..47c9cd196 --- /dev/null +++ b/crates/metadata/src/exif.rs @@ -0,0 +1,759 @@ +//! Exif / TIFF tags → [`Metadata`] (CIPA DC-008 Exif 2.32, TIFF 6.0, DNG 1.7 tag semantics). + +use crate::{DateTime, Flash, Gps, Metadata, Orientation}; +use craft_tiff::{Ifd, ParseOptions, Tiff, TiffError, tags as t}; + +/// Strip an `Exif\0\0` (or `Exif\0\xff`) prefix as found in JPEG APP1 and some WebP `EXIF` chunks. +pub fn strip_exif_header(b: &[u8]) -> &[u8] { + if b.len() >= 6 && &b[..4] == b"Exif" && b[4] == 0 { &b[6..] } else { b } +} + +/// Read metadata from a TIFF-structured Exif block (a whole TIFF/DNG/raw file, or the payload of a JPEG APP1 +/// Exif segment with or without the `Exif\0\0` prefix). Malformed input yields whatever could be read. +pub fn read_exif(bytes: &[u8]) -> Metadata { + try_read_exif(bytes).unwrap_or_default() +} + +/// Like [`read_exif`] but reports a structurally invalid TIFF header / IFD0. +pub fn try_read_exif(bytes: &[u8]) -> Result { + try_read_exif_with(bytes, &ParseOptions::default()) +} + +/// [`try_read_exif`] with the caller's parser options (its value allocation budget in particular). +/// A decoder working from a sparse buffer — only selected ranges of the logical file are actually +/// loaded, the rest is zeros — must not size its allocations by the logical file length, which +/// unrelated bytes of the file can inflate at no cost to the attacker. +pub fn try_read_exif_with(bytes: &[u8], opts: &ParseOptions) -> Result { + let data = strip_exif_header(bytes); + let tiff = Tiff::parse_with(data, opts)?; + let mut m = from_tiff(&tiff); + // Panasonic (and some Canons) write no LensModel, only the name in their maker note + if m.lens_model.is_none() { + m.lens_model = panasonic_maker_note_lens(data, &tiff).or_else(|| canon_maker_note_lens(data, &tiff)); + } + // the maker note's values are offsets into the stream, which `from_tiff` doesn't have + m.focus = crate::focus::read(data, &tiff, m.make.as_deref(), m.model.as_deref(), m.orientation); + Ok(m) +} + +fn text(ifd: Option<&Ifd>, tag: u16) -> Option { + let s = ifd?.string(tag)?; + let s = s.trim().to_string(); + (!s.is_empty() && !s.bytes().all(|b| b == b' ' || b == 0)).then_some(s) +} + +fn pos(v: Option) -> Option { + v.filter(|x| x.is_finite() && *x > 0.0) +} + +/// The orientation the Exif's own first IFD states. [`from_tiff`] also reports one that only the XMP packet in +/// the Exif's tag 700 states; a viewer displays a photo by the Exif's alone, so that one can't tell how a photo +/// is stored. +pub(crate) fn ifd0_orientation(bytes: &[u8]) -> Option { + let tiff = Tiff::parse_with(strip_exif_header(bytes), &ParseOptions::default()).ok()?; + tiff.ifds.first().and_then(|i| i.u16(t::ORIENTATION)).filter(|v| (1..=8).contains(v)).map(Orientation::from_exif) +} + +/// Build [`Metadata`] from a parsed TIFF stream (IFD0 + Exif + GPS; DNG tags as fallbacks). +pub fn from_tiff(tiff: &Tiff) -> Metadata { + let ifd0 = tiff.ifds.first(); + let exif = tiff.exif(); + let gps = tiff.gps(); + let either = |tag: u16| exif.and_then(|e| e.value(tag)).or_else(|| ifd0.and_then(|i| i.value(tag))); + let mut m = Metadata { + make: text(ifd0, t::MAKE), + model: text(ifd0, t::MODEL).or_else(|| text(ifd0, t::UNIQUE_CAMERA_MODEL)), + software: text(ifd0, t::SOFTWARE), + artist: text(ifd0, t::ARTIST), + copyright: text(ifd0, t::COPYRIGHT), + caption: text(ifd0, t::IMAGE_DESCRIPTION), + serial_number: text(exif, t::BODY_SERIAL_NUMBER).or_else(|| text(ifd0, t::CAMERA_SERIAL_NUMBER)), + lens_make: text(exif, t::LENS_MAKE), + lens_model: text(exif, t::LENS_MODEL), + lens_serial_number: text(exif, t::LENS_SERIAL_NUMBER), + ..Default::default() + }; + // Exif requires LensSpecification/LensInfo as 4 rationals. + let spec = exif.and_then(|e| e.f64s(t::LENS_SPECIFICATION)).or_else(|| ifd0.and_then(|i| i.f64s(t::LENS_INFO))); + if let Some(s) = spec.filter(|s| s.len() == 4 && s[0] > 0.0) { + m.lens_spec = Some([s[0], s[1], s[2], s[3]]); + } + // Nikon DSLRs write neither LensModel nor LensSpecification, only the same four values in their maker note + if m.lens_spec.is_none() { + m.lens_spec = nikon_lens_spec(exif, m.make.as_deref(), tiff.order); + } + m.orientation = ifd0.and_then(|i| i.u16(t::ORIENTATION)).filter(|v| (1..=8).contains(v)).map(Orientation::from_exif); + m.rating = ifd0.and_then(|i| i.i64(t::RATING)).map(|r| r.clamp(-1, 5) as i8); + + m.exposure_time = pos(either(t::EXPOSURE_TIME).and_then(|v| v.get_f64(0))) + .or_else(|| exif.and_then(|e| e.f64(t::SHUTTER_SPEED_VALUE)).filter(|v| v.abs() < 64.0).map(|tv| 2f64.powf(-tv))); + m.f_number = pos(either(t::F_NUMBER).and_then(|v| v.get_f64(0))) + .or_else(|| exif.and_then(|e| e.f64(t::APERTURE_VALUE)).filter(|v| v.abs() < 64.0).map(|av| 2f64.powf(av / 2.0))); + let iso = either(t::ISO_SPEED).and_then(|v| v.get_u64(0)).filter(|&v| v > 0); + let iso_ext = exif.and_then(|e| e.u64(t::RECOMMENDED_EXPOSURE_INDEX).or_else(|| e.u64(t::ISO_SPEED_RATINGS_EXT))).filter(|&v| v > 0); + m.iso = match (iso, iso_ext) { + (Some(65535), Some(x)) | (None, Some(x)) => Some(x), + (Some(v), _) => Some(v), + _ => None, + } + .map(|v| v.min(u32::MAX as u64) as u32); + m.focal_length = pos(either(t::FOCAL_LENGTH).and_then(|v| v.get_f64(0))); + m.focal_length_35mm = pos(exif.and_then(|e| e.f64(t::FOCAL_LENGTH_35MM))); + m.exposure_bias = exif.and_then(|e| e.f64(t::EXPOSURE_BIAS)).filter(|v| v.is_finite() && v.abs() < 100.0); + m.exposure_program = exif.and_then(|e| e.u16(t::EXPOSURE_PROGRAM)); + m.metering_mode = exif.and_then(|e| e.u16(t::METERING_MODE)); + m.white_balance = exif.and_then(|e| e.u16(t::WHITE_BALANCE)); + m.flash = exif.and_then(|e| e.u16(t::FLASH)).map(|raw| Flash { fired: raw & 1 != 0, raw }); + + // capture time: DateTimeOriginal, then DateTimeDigitized, then IFD0 DateTime + let candidates = [ + (exif, t::DATE_TIME_ORIGINAL, t::OFFSET_TIME_ORIGINAL, t::SUBSEC_TIME_ORIGINAL), + (exif, t::DATE_TIME_DIGITIZED, t::OFFSET_TIME_DIGITIZED, t::SUBSEC_TIME_DIGITIZED), + (ifd0, t::DATE_TIME, t::OFFSET_TIME, t::SUBSEC_TIME), + ]; + for (ifd, dt_tag, off_tag, sub_tag) in candidates { + if let Some(mut dt) = text(ifd, dt_tag).and_then(|s| DateTime::parse_exif(&s)) { + if let Some(s) = text(exif, sub_tag) { + dt = dt.with_subsec(&s); + } + dt.offset_minutes = text(exif, off_tag).and_then(|s| DateTime::parse_offset(&s)); + m.capture_time = Some(dt); + break; + } + } + + m.width = exif.and_then(|e| e.u32(t::PIXEL_X_DIMENSION)).filter(|&v| v > 0).or_else(|| ifd0.and_then(|i| i.u32(t::IMAGE_WIDTH))); + m.height = exif.and_then(|e| e.u32(t::PIXEL_Y_DIMENSION)).filter(|&v| v > 0).or_else(|| ifd0.and_then(|i| i.u32(t::IMAGE_LENGTH))); + + m.gps = gps.and_then(read_gps); + if let Some(iptc) = ifd0.and_then(|i| i.value(t::IPTC_NAA)) { + // IPTC-NAA is often typed LONG; re-serialise to bytes in file order. + let bytes = match iptc { + craft_tiff::Value::Byte(b) | craft_tiff::Value::Undefined(b) => b.clone(), + other => craft_tiff::writer::encode(tiff.order, other), + }; + m.fill_missing(&crate::parse_iptc(&bytes)); + } + if let Some(x) = ifd0.and_then(|i| i.bytes(t::XMP)).map(String::from_utf8_lossy) + && let Ok(x) = crate::parse_xmp(&x) + { + // (the size stays the one Exif states: `extract` turns regions by it, and a size only the XMP states may be the upright one) + let size = (m.width, m.height); + m.overlay_user_fields(&x.metadata); + m.fill_missing(&x.metadata); + (m.width, m.height) = size; + } + m +} + +/// Degrees/minutes/seconds triplet as decimal degrees. A component that is present but not a number (a rational +/// with a zero denominator, as in the placeholder GPS block phones write without a fix) makes the whole value unusable. +fn dms(ifd: &Ifd, tag: u16) -> Option { + let v = ifd.value(tag)?; + let part = |i: usize| if i < v.count() { v.get_f64(i) } else { Some(0.0) }; + let d = v.get_f64(0)?; + let deg = d + part(1)? / 60.0 + part(2)? / 3600.0; + deg.is_finite().then_some(deg) +} + +/// `Some(true)` for the negative hemisphere (`neg`), `Some(false)` for `pos`, `None` when the ref is missing or invalid. +fn hemisphere(g: &Ifd, tag: u16, pos: char, neg: char) -> Option { + match g.string(tag)?.trim_start().chars().next()?.to_ascii_uppercase() { + c if c == neg => Some(true), + c if c == pos => Some(false), + _ => None, + } +} + +fn read_gps(g: &Ifd) -> Option { + let mut lat = dms(g, t::GPS_LATITUDE)?; + let mut lon = dms(g, t::GPS_LONGITUDE)?; + if hemisphere(g, t::GPS_LATITUDE_REF, 'N', 'S')? { + lat = -lat; + } + if hemisphere(g, t::GPS_LONGITUDE_REF, 'E', 'W')? { + lon = -lon; + } + if !(-90.0..=90.0).contains(&lat) || !(-180.0..=180.0).contains(&lon) { + return None; + } + let altitude = g.f64(t::GPS_ALTITUDE).filter(|a| a.is_finite()).map(|a| if g.u32(t::GPS_ALTITUDE_REF) == Some(1) { -a } else { a }); + Some(Gps { latitude: lat, longitude: lon, altitude }) +} + +/// Panasonic maker note tag `0x0051` (LensType in the ExifTool Panasonic tag names): the lens name, ASCII. +const PANASONIC_LENS_TYPE: u16 = 0x0051; +/// RW2/RWL IFD0 tag `0x002E` (JpgFromRaw): a JPEG whose APP1 holds the full Exif, maker note included. +const PANASONIC_JPG_FROM_RAW: u16 = 0x002E; + +/// The lens name in the Panasonic maker note (tag `0x0051`) of this Exif or RW2 stream (`data` starts at its +/// TIFF header): the note of its own Exif IFD, else, for a raw, the note in its JpgFromRaw's Exif. +pub(crate) fn panasonic_maker_note_lens(data: &[u8], tiff: &Tiff) -> Option { + panasonic_lens(data, tiff).or_else(|| panasonic_jpg_from_raw_lens(tiff)) +} + +/// The lens name in a `Panasonic\0` maker note of this Exif stream (`data` starts at its TIFF header). +fn panasonic_lens(data: &[u8], tiff: &Tiff) -> Option { + let exif = tiff.exif()?; + if !exif.bytes(t::MAKER_NOTE)?.starts_with(b"Panasonic\0") { + return None; + } + let e = exif.get(t::MAKER_NOTE)?; + let make = tiff.ifds.first().and_then(|i| i.string(t::MAKE)).unwrap_or_default(); + let mn = craft_tiff::makernote::parse_makernote(data, e.offset, e.count() as u64, tiff.order, &make)?; + text(Some(&mn.ifd), PANASONIC_LENS_TYPE) +} + +/// The lens name from the Exif of a RW2's (or Leica RWL's) embedded JpgFromRaw. +fn panasonic_jpg_from_raw_lens(tiff: &Tiff) -> Option { + let jpg = tiff.ifds.first()?.bytes(PANASONIC_JPG_FROM_RAW)?; + let exif = crate::jpeg_segments(jpg).exif?; + let data = strip_exif_header(&exif); + panasonic_lens(data, &Tiff::parse(data).ok()?) +} + +/// Canon maker note tag `0x0095` (LensModel in the ExifTool Canon tag names): the lens name as ASCII. Older +/// bodies instead record a numeric LensType (CameraSettings, tag `0x0001`, index 22), which is no name to read. +const CANON_LENS_MODEL: u16 = 0x0095; + +/// The lens name in a Canon maker note of this Exif stream (`data` starts at its TIFF header): the note of +/// its own Exif IFD. Only consulted when Make is Canon, so other vendors' plain notes are never read as lenses. +fn canon_maker_note_lens(data: &[u8], tiff: &Tiff) -> Option { + let make = tiff.ifds.first().and_then(|i| i.string(t::MAKE)).unwrap_or_default(); + if !make.trim().to_ascii_uppercase().starts_with("CANON") { + return None; + } + let exif = tiff.exif()?; + let e = exif.get(t::MAKER_NOTE)?; + let mn = craft_tiff::makernote::parse_makernote(data, e.offset, e.count() as u64, tiff.order, &make)?; + text(Some(&mn.ifd), CANON_LENS_MODEL) +} + +/// Nikon maker note tag `0x0084` (Lens in the ExifTool Nikon tag names): shortest and longest focal length, then the +/// widest aperture at each, as rationals, like Exif LensSpecification. Zero for a lens that doesn't report itself. +const NIKON_LENS: u16 = 0x0084; + +/// The lens a Nikon camera records in its maker note, when it's plausible. +fn nikon_lens_spec(exif: Option<&Ifd>, make: Option<&str>, order: craft_tiff::ByteOrder) -> Option<[f64; 4]> { + let make = make?; + if !make.trim().to_ascii_uppercase().starts_with("NIKON") { + return None; + } + let note = exif?.bytes(t::MAKER_NOTE)?; + let mn = craft_tiff::makernote::parse_makernote(note, 0, note.len() as u64, order, make)?; + let s = mn.ifd.f64s(NIKON_LENS)?; + let [short, long, wide, narrow] = s.as_slice() else { return None }; + let focal = |f: f64| f.is_finite() && f > 0.0 && f < 10_000.0; + let aperture = |a: f64| a.is_finite() && a > 0.0 && a < 100.0; + (focal(*short) && focal(*long) && long >= short && aperture(*wide) && aperture(*narrow)).then_some([*short, *long, *wide, *narrow]) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value}; + + fn capture_time_tiff(original: Option<&str>, digitized: Option<&str>, ifd0_time: Option<&str>, xmp: Option<&str>) -> Vec { + let mut exif = IfdBuilder::new(); + if let Some(value) = original { + exif.set(t::DATE_TIME_ORIGINAL, Value::Ascii(value.into())); + } + if let Some(value) = digitized { + exif.set(t::DATE_TIME_DIGITIZED, Value::Ascii(value.into())); + } + let mut ifd0 = IfdBuilder::new(); + if let Some(value) = ifd0_time { + ifd0.set(t::DATE_TIME, Value::Ascii(value.into())); + } + ifd0.set_child(t::EXIF_IFD, exif); + if let Some(packet) = xmp { + ifd0.set(t::XMP, Value::Byte(packet.as_bytes().to_vec())); + } + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + } + + #[test] + fn exif_clock_invalid_original_uses_the_next_valid_exif_time() { + let invalid = "2026:01:01 12:34:56:99"; + let digitized = capture_time_tiff(Some(invalid), Some("2026:01:02 10:00:00"), Some("2026:01:03 11:00:00"), None); + assert_eq!(read_exif(&digitized).capture_time.map(|date| date.to_iso()), Some("2026-01-02T10:00:00".into())); + + let ifd0 = capture_time_tiff(Some(invalid), Some("2026:01:02 10:00:00:"), Some("2026:01:03 11:00:00"), None); + assert_eq!(read_exif(&ifd0).capture_time.map(|date| date.to_iso()), Some("2026-01-03T11:00:00".into())); + + let none = capture_time_tiff(Some(invalid), Some("2026:01:02 10:00:00:junk"), Some("2026:01:03 11:00:00:"), None); + assert_eq!(read_exif(&none).capture_time, None); + + let original = capture_time_tiff(Some("2026:01:04 12:00:00"), Some("2026:01:02 10:00:00"), Some("2026:01:03 11:00:00"), None); + assert_eq!(read_exif(&original).capture_time.map(|date| date.to_iso()), Some("2026-01-04T12:00:00".into())); + } + + #[test] + fn exif_clock_invalid_original_allows_xmp_fallback_and_keeps_raw_tag() { + let capture = DateTime::parse_iso("2026-01-02T10:00:00Z"); + let packet = crate::write_xmp(&Metadata { capture_time: capture, ..Default::default() }); + let bytes = capture_time_tiff(Some("2026:01:01 12:34:56:99"), None, None, Some(&packet)); + assert_eq!(crate::extract(&bytes).capture_time.map(|date| date.to_iso()), Some("2026-01-02T10:00:00+00:00".into())); + + let without_fallback = capture_time_tiff(Some("2026:01:01 12:34:56:junk"), None, None, None); + assert_eq!(crate::extract(&without_fallback).capture_time, None); + assert!(crate::embedded(&without_fallback).exif.is_some(), "rejecting the projection must retain the embedded EXIF block"); + } + + /// An Exif block from a Panasonic body: no LensModel, the lens name in maker note tag 0x0051, whose value + /// lies outside the note's IFD at an offset relative to the Exif TIFF header (as Panasonic writes it). + pub(crate) fn panasonic_exif(order: ByteOrder, lens: Option<&str>, lens_model: Option<&str>) -> Vec { + let note = |at: u32| { + let mut v = b"Panasonic\0\0\0".to_vec(); + order.put_u16(&mut v, 1); + order.put_u16(&mut v, if lens.is_some() { PANASONIC_LENS_TYPE } else { 0x0001 }); + order.put_u16(&mut v, 2); + let name = format!("{:<34}\0", lens.unwrap_or("0210")); + order.put_u32(&mut v, name.len() as u32); + order.put_u32(&mut v, at + 30); + order.put_u32(&mut v, 0); + v.extend_from_slice(name.as_bytes()); + v + }; + let build = |at: u32| { + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note(at))); + exif.set(t::FOCAL_LENGTH, Value::Rational(vec![(25, 1)])); + if let Some(l) = lens_model { + exif.set(t::LENS_MODEL, Value::Ascii(l.into())); + } + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("Panasonic".into())); + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(order, false).write(&[ifd0]).unwrap() + }; + // the note's own offset is only known once it is laid out; its length doesn't change + let first = build(0); + let at = Tiff::parse(&first).unwrap().exif().unwrap().get(t::MAKER_NOTE).unwrap().offset as u32; + build(at) + } + + #[test] + fn panasonic_maker_note_names_the_lens() { + for order in [ByteOrder::Little, ByteOrder::Big] { + let m = read_exif(&panasonic_exif(order, Some("LUMIX G 25/F1.7"), None)); + assert_eq!(m.lens_model.as_deref(), Some("LUMIX G 25/F1.7")); + assert_eq!(m.focal_length, Some(25.0)); + } + // Exif's own LensModel wins, and a note without 0x0051 leaves the lens empty + let m = read_exif(&panasonic_exif(ByteOrder::Little, Some("LUMIX G 25/F1.7"), Some("Other 25mm"))); + assert_eq!(m.lens_model.as_deref(), Some("Other 25mm")); + assert_eq!(read_exif(&panasonic_exif(ByteOrder::Little, None, None)).lens_model, None); + } + + /// A RW2 has the full Exif only in the APP1 of its JpgFromRaw (IFD0 0x002E), not in its own Exif IFD. + pub(crate) fn panasonic_rw2(lens: &str) -> Vec { + let mut jpg = vec![0xff, 0xd8, 0xff, 0xe1]; + let mut app1 = b"Exif\0\0".to_vec(); + app1.extend(panasonic_exif(ByteOrder::Little, Some(lens), None)); + jpg.extend_from_slice(&((app1.len() + 2) as u16).to_be_bytes()); + jpg.extend(app1); + jpg.extend_from_slice(&[0xff, 0xda, 0, 2, 0xff, 0xd9]); + let mut exif = IfdBuilder::new(); + exif.set(t::FOCAL_LENGTH, Value::Rational(vec![(85, 1)])); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("Panasonic".into())); + ifd0.set(PANASONIC_JPG_FROM_RAW, Value::Undefined(jpg)); + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + } + + #[test] + fn rw2_reads_the_lens_from_its_jpg_from_raw() { + let rw2 = panasonic_rw2("LUMIX S 85/F1.8"); + let m = read_exif(&rw2); + assert_eq!(m.lens_model.as_deref(), Some("LUMIX S 85/F1.8")); + assert_eq!(m.focal_length, Some(85.0)); + assert_eq!(crate::extract(&rw2).lens_model.as_deref(), Some("LUMIX S 85/F1.8")); + } + + /// Canon maker note tags: `0x0095` holds the lens name as ASCII; `0x0001` is CameraSettings, a table of + /// int16s whose index 22 is the numeric LensType. + const CANON_LENS_MODEL_TAG: u16 = 0x0095; + const CANON_CAMERA_SETTINGS: u16 = 0x0001; + + /// An Exif block from a Canon EOS 5D: `Canon` / `Canon EOS 5D` in IFD0, no LensModel tag, and a plain maker + /// note whose IFD (value offsets relative to the enclosing TIFF header, as Canon writes it) records + /// CameraSettings with the numeric LensType at index 22 and, when `lens` is given, the name at `0x0095`. + fn canon_exif(order: ByteOrder, lens: Option<&str>) -> Vec { + let note = |at: u32| { + // The plain IFD: entry count, one or two 12-byte entries and the next-IFD pointer, then the + // out-of-line values (CameraSettings, then the lens name when present). + // pointer, then the out-of-line values (CameraSettings, then the lens name when present). + let settings_at = at + 2 + 12 * (1 + lens.is_some() as u32) + 4; + let name_at = settings_at + 60; + let mut v = Vec::new(); + let entry = |v: &mut Vec, tag: u16, kind: u16, count: u32, off: u32| { + order.put_u16(v, tag); + order.put_u16(v, kind); + order.put_u32(v, count); + order.put_u32(v, off); + }; + order.put_u16(&mut v, 1 + lens.is_some() as u16); + entry(&mut v, CANON_CAMERA_SETTINGS, 3, 30, settings_at); + if let Some(name) = lens { + entry(&mut v, CANON_LENS_MODEL_TAG, 2, name.len() as u32, name_at); + } + order.put_u32(&mut v, 0); + for i in 0..30u16 { + order.put_u16(&mut v, if i == 22 { 178 } else { 0 }); // 22: numeric LensType, no name + } + if let Some(name) = lens { + v.extend_from_slice(name.as_bytes()); + } + v + }; + let build = |at: u32| { + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note(at))); + exif.set(t::FOCAL_LENGTH, Value::Rational(vec![(50, 1)])); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("Canon".into())); + ifd0.set(t::MODEL, Value::Ascii("Canon EOS 5D".into())); + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(order, false).write(&[ifd0]).unwrap() + }; + // the note's own offset is only known once it is laid out; its length doesn't change + let first = build(0); + let at = Tiff::parse(&first).unwrap().exif().unwrap().get(t::MAKER_NOTE).unwrap().offset as u32; + build(at) + } + + #[test] + fn canon_maker_note_lens_fills_lens_model() { + for order in [ByteOrder::Little, ByteOrder::Big] { + let m = read_exif(&canon_exif(order, Some("Canon EF 50mm f/1.8 II"))); + assert_eq!(m.lens_model.as_deref(), Some("Canon EF 50mm f/1.8 II")); + // a note that only records the numeric LensType has no name to fill in + assert_eq!(read_exif(&canon_exif(order, None)).lens_model, None); + } + } + + /// An Exif block from a Nikon body whose maker note records the lens (tag 0x0084) and no LensModel or + /// LensSpecification, as the D700, D750, D80 and D5300 write it. + fn nikon_exif(make: &str, lens: Option<[(u32, u32); 4]>, spec: Option<[(u32, u32); 4]>) -> Vec { + let mut note = IfdBuilder::new(); + note.set(0x0001, Value::Undefined(b"0210".to_vec())); + if let Some(l) = lens { + note.set(NIKON_LENS, Value::Rational(l.to_vec())); + } + let mut blob = b"Nikon\0\x02\x10\0\0".to_vec(); + blob.extend(TiffWriter::new(ByteOrder::Little, false).write(&[note]).unwrap()); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(blob)); + if let Some(sp) = spec { + exif.set(t::LENS_SPECIFICATION, Value::Rational(sp.to_vec())); + } + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii(make.into())); + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap() + } + + #[test] + fn nikon_maker_note_lens_fills_the_lens_spec() { + let zoom = [(18, 1), (135, 1), (35, 10), (56, 10)]; + assert_eq!(read_exif(&nikon_exif("NIKON CORPORATION", Some(zoom), None)).lens_spec, Some([18.0, 135.0, 3.5, 5.6])); + // Exif's own LensSpecification wins, another make's note is not read, and a lens that reports nothing is left out + let prime = [(50, 1), (50, 1), (18, 10), (18, 10)]; + assert_eq!(read_exif(&nikon_exif("NIKON CORPORATION", Some(zoom), Some(prime))).lens_spec, Some([50.0, 50.0, 1.8, 1.8])); + assert_eq!(read_exif(&nikon_exif("Canon", Some(zoom), None)).lens_spec, None); + assert_eq!(read_exif(&nikon_exif("NIKON CORPORATION", Some([(0, 1), (0, 1), (0, 1), (0, 1)]), None)).lens_spec, None); + assert_eq!(read_exif(&nikon_exif("NIKON CORPORATION", Some([(200, 1), (80, 1), (28, 10), (28, 10)]), None)).lens_spec, None); + assert_eq!(read_exif(&nikon_exif("NIKON CORPORATION", None, None)).lens_spec, None); + } + + pub(crate) fn sample_exif(order: ByteOrder) -> Vec { + let mut exif = IfdBuilder::new(); + exif.set(t::EXPOSURE_TIME, Value::Rational(vec![(1, 250)])); + exif.set(t::F_NUMBER, Value::Rational(vec![(28, 10)])); + exif.set(t::ISO_SPEED, Value::Short(vec![400])); + exif.set(t::DATE_TIME_ORIGINAL, Value::Ascii("2023:07:14 18:30:05".into())); + exif.set(t::OFFSET_TIME_ORIGINAL, Value::Ascii("+02:00".into())); + exif.set(t::SUBSEC_TIME_ORIGINAL, Value::Ascii("42".into())); + exif.set(t::EXPOSURE_BIAS, Value::SRational(vec![(-2, 3)])); + exif.set(t::METERING_MODE, Value::Short(vec![5])); + exif.set(t::FLASH, Value::Short(vec![0x19])); + exif.set(t::FOCAL_LENGTH, Value::Rational(vec![(50, 1)])); + exif.set(t::FOCAL_LENGTH_35MM, Value::Short(vec![75])); + exif.set(t::WHITE_BALANCE, Value::Short(vec![0])); + exif.set(t::EXPOSURE_PROGRAM, Value::Short(vec![3])); + exif.set(t::LENS_MAKE, Value::Ascii("LensCo".into())); + exif.set(t::LENS_MODEL, Value::Ascii("50mm F1.8".into())); + exif.set(t::BODY_SERIAL_NUMBER, Value::Ascii("12345".into())); + exif.set(t::LENS_SPECIFICATION, Value::Rational(vec![(50, 1), (50, 1), (18, 10), (18, 10)])); + exif.set(t::PIXEL_X_DIMENSION, Value::Long(vec![6000])); + exif.set(t::PIXEL_Y_DIMENSION, Value::Long(vec![4000])); + let mut gps = IfdBuilder::new(); + gps.set(t::GPS_LATITUDE_REF, Value::Ascii("S".into())); + gps.set(t::GPS_LATITUDE, Value::Rational(vec![(33, 1), (51, 1), (3564, 100)])); + gps.set(t::GPS_LONGITUDE_REF, Value::Ascii("E".into())); + gps.set(t::GPS_LONGITUDE, Value::Rational(vec![(151, 1), (12, 1), (4032, 100)])); + gps.set(t::GPS_ALTITUDE_REF, Value::Byte(vec![1])); + gps.set(t::GPS_ALTITUDE, Value::Rational(vec![(125, 10)])); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("Maker Inc.".into())); + ifd0.set(t::MODEL, Value::Ascii("Model X ".into())); + ifd0.set(t::ORIENTATION, Value::Short(vec![6])); + ifd0.set(t::ARTIST, Value::Ascii("Jane".into())); + ifd0.set(t::COPYRIGHT, Value::Ascii("(c) Jane".into())); + ifd0.set(t::RATING, Value::Short(vec![3])); + ifd0.set_child(t::EXIF_IFD, exif); + ifd0.set_child(t::GPS_IFD, gps); + TiffWriter::new(order, false).write(&[ifd0]).unwrap() + } + + #[test] + fn reads_all_fields() { + for order in [ByteOrder::Little, ByteOrder::Big] { + let m = read_exif(&sample_exif(order)); + assert_eq!(m.make.as_deref(), Some("Maker Inc.")); + assert_eq!(m.model.as_deref(), Some("Model X")); + assert_eq!(m.exposure_time, Some(0.004)); + assert_eq!(m.f_number, Some(2.8)); + assert_eq!(m.iso, Some(400)); + let dt = m.capture_time.unwrap(); + assert_eq!(dt.to_iso(), "2023-07-14T18:30:05.420+02:00"); + assert!((m.exposure_bias.unwrap() + 0.6667).abs() < 1e-3); + assert_eq!(m.metering_mode, Some(5)); + assert_eq!(m.flash, Some(Flash { fired: true, raw: 0x19 })); + assert_eq!(m.focal_length, Some(50.0)); + assert_eq!(m.focal_length_35mm, Some(75.0)); + assert_eq!(m.orientation, Some(Orientation::Rotate90)); + assert_eq!(m.lens_model.as_deref(), Some("50mm F1.8")); + assert_eq!(m.lens_make.as_deref(), Some("LensCo")); + assert_eq!(m.serial_number.as_deref(), Some("12345")); + assert_eq!(m.lens_spec, Some([50.0, 50.0, 1.8, 1.8])); + assert_eq!((m.width, m.height), (Some(6000), Some(4000))); + assert_eq!(m.artist.as_deref(), Some("Jane")); + assert_eq!(m.rating, Some(3)); + let g = m.gps.unwrap(); + assert!((g.latitude + 33.8599).abs() < 1e-4, "{g:?}"); + assert!((g.longitude - 151.2112).abs() < 1e-4); + assert_eq!(g.altitude, Some(-12.5)); + } + } + + #[test] + fn placeholder_gps_block_is_no_location() { + // Android phones without a fix write NUL refs and 0/0 rationals. + let read = |lat_ref: &str, lat: Vec<(u32, u32)>| { + let mut gps = IfdBuilder::new(); + gps.set(t::GPS_LATITUDE_REF, Value::Ascii(lat_ref.into())); + gps.set(t::GPS_LATITUDE, Value::Rational(lat)); + gps.set(t::GPS_LONGITUDE_REF, Value::Ascii("E".into())); + gps.set(t::GPS_LONGITUDE, Value::Rational(vec![(151, 1), (12, 1), (4032, 100)])); + let mut ifd0 = IfdBuilder::new(); + ifd0.set_child(t::GPS_IFD, gps); + read_exif(&TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap()).gps + }; + let valid = vec![(33, 1), (51, 1), (3564, 100)]; + assert!(read("N", valid.clone()).is_some()); + assert_eq!(read("N", vec![(0, 0), (0, 0), (0, 0)]), None, "0/0 rationals"); + assert_eq!(read("N", vec![(33, 1), (0, 0), (0, 0)]), None, "0/0 minutes"); + assert_eq!(read("", valid.clone()), None, "NUL ref"); + assert_eq!(read("", valid.clone()), None, "empty ref"); + assert_eq!(read("X", valid), None, "invalid ref"); + } + + #[test] + fn exif_prefix_and_garbage() { + let mut b = b"Exif\0\0".to_vec(); + b.extend(sample_exif(ByteOrder::Little)); + assert_eq!(read_exif(&b).iso, Some(400)); + assert_eq!(read_exif(b"garbage"), Metadata::default()); + assert!(try_read_exif(b"garbage").is_err()); + } + + #[test] + fn apex_fallbacks_and_extended_iso() { + let mut exif = IfdBuilder::new(); + exif.set(t::SHUTTER_SPEED_VALUE, Value::SRational(vec![(8, 1)])); + exif.set(t::APERTURE_VALUE, Value::Rational(vec![(4, 1)])); + exif.set(t::ISO_SPEED, Value::Short(vec![65535])); + exif.set(t::RECOMMENDED_EXPOSURE_INDEX, Value::Long(vec![102400])); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::DATE_TIME, Value::Ascii("2020:01:02 03:04:05".into())); + ifd0.set_child(t::EXIF_IFD, exif); + let m = read_exif(&TiffWriter::default().write(&[ifd0]).unwrap()); + assert_eq!(m.exposure_time, Some(1.0 / 256.0)); + assert_eq!(m.f_number, Some(4.0)); + assert_eq!(m.iso, Some(102400)); + assert_eq!(m.capture_time.unwrap().to_exif(), "2020:01:02 03:04:05"); + } + + #[test] + fn xmp_in_ifd0_overrides_user_fields() { + let mut meta = Metadata { rating: Some(5), title: Some("From XMP".into()), ..Default::default() }; + meta.keywords = vec!["a".into()]; + let x = crate::write_xmp(&meta); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::RATING, Value::Short(vec![1])); + ifd0.set(t::XMP, Value::Byte(x.into_bytes())); + let m = read_exif(&TiffWriter::default().write(&[ifd0]).unwrap()); + assert_eq!(m.rating, Some(5)); + assert_eq!(m.title.as_deref(), Some("From XMP")); + } +} + +/// An exact `1/N` rational for a sub-second value whose reciprocal is a whole number (shutter speeds such as +/// 1/12800 s, which [`craft_tiff::writer::rational`]'s decimal denominators cannot represent). +fn reciprocal(v: Option) -> Option { + let v = v.filter(|v| v.is_finite() && *v > 0.0 && *v < 1.0)?; + let inv = 1.0 / v; + ((inv - inv.round()).abs() < 1e-6 && inv.round() <= u32::MAX as f64).then(|| craft_tiff::Value::Rational(vec![(1, inv.round() as u32)])) +} + +/// Serialise the interchange subset of `meta` as a TIFF-structured Exif block (little-endian, no +/// `Exif\0\0` prefix): IFD0 (make, model, software, artist, copyright, description, orientation, +/// date), the Exif IFD (capture settings, lens) and, when present, the GPS IFD. +pub fn write_exif(meta: &Metadata) -> Vec { + use craft_tiff::writer::{rational, srational}; + use craft_tiff::{ByteOrder, IfdBuilder, TiffWriter, Value}; + let ascii = |s: &Option| s.as_ref().filter(|v| !v.is_empty()).map(|v| Value::Ascii(v.clone())); + let mut ifd0 = IfdBuilder::new(); + for (tag, v) in [ + (t::MAKE, ascii(&meta.make)), + (t::MODEL, ascii(&meta.model)), + (t::SOFTWARE, ascii(&meta.software)), + (t::ARTIST, ascii(&meta.artist)), + (t::COPYRIGHT, ascii(&meta.copyright)), + (t::IMAGE_DESCRIPTION, ascii(&meta.caption)), + (t::DATE_TIME, meta.capture_time.map(|d| Value::Ascii(d.to_exif()))), + (t::ORIENTATION, meta.orientation.map(|o| Value::Short(vec![o.to_exif()]))), + ] { + if let Some(v) = v { + ifd0.set(tag, v); + } + } + let mut ex = IfdBuilder::new(); + ex.set(t::EXIF_VERSION, Value::Undefined(b"0232".to_vec())); + let r = |v: Option| v.filter(|v| v.is_finite() && *v > 0.0).map(|v| Value::Rational(vec![rational(v)])); + for (tag, v) in [ + (t::DATE_TIME_ORIGINAL, meta.capture_time.map(|d| Value::Ascii(d.to_exif()))), + (t::EXPOSURE_TIME, reciprocal(meta.exposure_time).or_else(|| r(meta.exposure_time))), + (t::F_NUMBER, r(meta.f_number)), + (t::FOCAL_LENGTH, r(meta.focal_length)), + (t::ISO_SPEED, meta.iso.map(|i| Value::Short(vec![i.min(65_535) as u16]))), + (t::ISO_SPEED_RATINGS_EXT, meta.iso.filter(|&i| i > 65_534).map(|i| Value::Long(vec![i]))), + (t::LENS_MAKE, ascii(&meta.lens_make)), + (t::LENS_MODEL, ascii(&meta.lens_model)), + (t::EXPOSURE_BIAS, meta.exposure_bias.map(|v| Value::SRational(vec![srational(v)]))), + ] { + if let Some(v) = v { + ex.set(tag, v); + } + } + if let Some(d) = meta.capture_time { + // an offset past 18 h is not a real one (a deserialized value can be anything): write none + let offset = d.offset_minutes.filter(|o| o.unsigned_abs() <= 18 * 60).map(|o| { + let sign = if o < 0 { '-' } else { '+' }; + format!("{sign}{:02}:{:02}", o.unsigned_abs() / 60, o.unsigned_abs() % 60) + }); + let subsec = (d.millis != 0).then(|| format!("{:03}", d.millis)); + for (tag, v) in [(t::OFFSET_TIME, &offset), (t::OFFSET_TIME_ORIGINAL, &offset), (t::SUBSEC_TIME, &subsec), (t::SUBSEC_TIME_ORIGINAL, &subsec)] + { + if let Some(v) = v { + ex.set(tag, Value::Ascii(v.clone())); + } + } + } + ifd0.set_child(t::EXIF_IFD, ex); + if let Some(g) = meta.gps { + let dms = |v: f64| { + let v = v.abs(); + let d = v.floor(); + let m = ((v - d) * 60.0).floor(); + let s = (v - d - m / 60.0) * 3600.0; + Value::Rational(vec![(d as u32, 1), (m as u32, 1), ((s * 1000.0).round() as u32, 1000)]) + }; + let mut gps = IfdBuilder::new(); + gps.set(t::GPS_VERSION_ID, Value::Byte(vec![2, 3, 0, 0])); + gps.set(t::GPS_LATITUDE_REF, Value::Ascii(if g.latitude < 0.0 { "S" } else { "N" }.into())); + gps.set(t::GPS_LATITUDE, dms(g.latitude)); + gps.set(t::GPS_LONGITUDE_REF, Value::Ascii(if g.longitude < 0.0 { "W" } else { "E" }.into())); + gps.set(t::GPS_LONGITUDE, dms(g.longitude)); + if let Some(a) = g.altitude { + gps.set(t::GPS_ALTITUDE_REF, Value::Byte(vec![u8::from(a < 0.0)])); + gps.set(t::GPS_ALTITUDE, Value::Rational(vec![rational(a.abs())])); + } + ifd0.set_child(t::GPS_IFD, gps); + } + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap_or_default() +} + +#[cfg(test)] +mod write_tests { + use super::*; + + #[test] + fn capture_time_offset_and_subsec_round_trip() { + for iso in ["2026-09-20T05:40:00.250+02:00", "2026-09-20T05:40:00.005-03:30", "2026-09-20T05:40:00"] { + let m = Metadata { capture_time: DateTime::parse_iso(iso), ..Default::default() }; + assert_eq!(read_exif(&write_exif(&m)).capture_time, m.capture_time, "{iso}"); + } + } + + /// A deserialized offset can be any i16, `i16::MIN` included: writing must not overflow. + #[test] + fn extreme_capture_time_offsets_do_not_panic() { + for o in [i16::MIN, i16::MAX, -1081, 1081] { + let mut dt = DateTime::parse_iso("2026-09-20T05:40:00").unwrap(); + dt.offset_minutes = Some(o); + let m = Metadata { capture_time: Some(dt), ..Default::default() }; + assert_eq!(read_exif(&write_exif(&m)).capture_time.map(|d| d.offset_minutes), Some(None), "{o}"); + } + } + + #[test] + fn exif_round_trips() { + let m = Metadata { + make: Some("Synthetic".into()), + model: Some("X2".into()), + copyright: Some("© 2026 A. Person".into()), + artist: Some("A. Person".into()), + capture_time: DateTime::parse_iso("2026-09-20T05:40:00"), + exposure_time: Some(1.0 / 250.0), + f_number: Some(2.8), + iso: Some(400), + focal_length: Some(35.0), + lens_model: Some("35mm F2".into()), + gps: Some(Gps { latitude: 43.7904, longitude: -110.6818, altitude: Some(2000.0) }), + ..Default::default() + }; + let back = read_exif(&write_exif(&m)); + assert_eq!(back.make, m.make); + assert_eq!(back.model, m.model); + assert_eq!(back.copyright, m.copyright); + assert_eq!(back.artist, m.artist); + assert_eq!(back.capture_time.map(|d| d.to_exif()), m.capture_time.map(|d| d.to_exif())); + assert_eq!(back.iso, Some(400)); + assert!((back.f_number.unwrap() - 2.8).abs() < 1e-6); + assert!((back.exposure_time.unwrap() - 0.004).abs() < 1e-9); + assert_eq!(back.lens_model, m.lens_model); + let g = back.gps.unwrap(); + assert!((g.latitude - 43.7904).abs() < 1e-5 && (g.longitude + 110.6818).abs() < 1e-5, "{g:?}"); + } + + #[test] + fn iso_above_65534_round_trips() { + for iso in [65_534, 65_535, 102_400, 409_600] { + let back = read_exif(&write_exif(&Metadata { iso: Some(iso), ..Default::default() })); + assert_eq!(back.iso, Some(iso)); + } + } + + #[test] + fn fast_shutter_speeds_round_trip_exactly() { + for den in [8000.0, 12_800.0, 16_000.0, 32_000.0] { + let back = read_exif(&write_exif(&Metadata { exposure_time: Some(1.0 / den), ..Default::default() })); + let got = back.exposure_time.unwrap(); + assert!((1.0 / got - den).abs() < 0.5, "1/{den} s came back as 1/{}", 1.0 / got); + } + } +} diff --git a/crates/metadata/src/focus.rs b/crates/metadata/src/focus.rs new file mode 100644 index 000000000..b956e667e --- /dev/null +++ b/crates/metadata/src/focus.rs @@ -0,0 +1,284 @@ +//! Where the camera focused: the autofocus point a camera records in its maker notes, as a box on the +//! upright photo ([`FocusPoint`]). Sony only for now. +//! +//! **Sony** (the MakerNote IFD: header-less in ARWs, after a `SONY DSC` header in JPEGs). Tag identities and value names come from the public ExifTool +//! tag-name documentation (Sony table); every layout below was confirmed on real ILCE-1 (α1) and +//! ILCE-7SM3 (A7S III) ARWs, compared with ExifTool's output as a black box. No third-party code was read. +//! +//! | Tag | Layout (confirmed) | Use | +//! |---|---|---| +//! | `0x2027` FocusLocation | 4 × int16u: the width and height of the frame it is measured in, then the focus centre x, y in pixels of that frame (the sensor's landscape frame: a portrait shot still says 8640 × 5760) | the point | +//! | `0x2037` FocusFrameSize | 6 bytes: int16u width, int16u height of the frame the camera showed, then 2 bytes that are zero when there is no frame (manual focus) | the box size | +//! | `0x201b` FocusMode | int8u: 0 Manual, 2 AF-S, 3 AF-C, 4 AF-A, 6 DMF, 7 AF-D | manual = no point | +//! | `0x201c` AFAreaModeSetting | int8u: 0 Wide, 1 Center, 3 Flexible Spot, 11 Zone, 12 Expanded Flexible Spot, 13 Custom AF Area | shown | +//! | `0x2021` AFTracking | int8u: 0 Off, 1 Face tracking, 2 Lock On AF | shown | +//! | `0x9402` (enciphered block) byte 0x16, 0x17 | FocusMode (low 7 bits) and the AF area mode actually used: 0 Multi, 1 Center, 2 Spot, 3 Flexible Spot, 11 Zone, 12 Expanded Flexible Spot, 13 Custom AF Area, 14 Tracking, 15 Face Tracking, 20 Animal Eye Tracking, 21 Human Eye Tracking, 255 Manual | shown; read only when its focus mode agrees with `0x201b` | +//! +//! The `0x94xx` blocks are enciphered byte by byte (the documentation calls it "a simple substitution +//! cipher"). On the sample files each plain byte `p < 249` is stored as `p³ mod 249` and bytes 249–255 +//! as themselves (e.g. a stored `0x20` reads as AFAreaMode 20, stored `0x77` as `0x83`, FocusMode AF-C +//! with its top bit set); the map is a bijection on 0..249 (cubing is invertible mod 3 and mod 83), so +//! it is undone by a table of its inverse. +//! +//! **No real point** (nothing is stored, so nothing is drawn and Z zooms as before): +//! - a body the layout hasn't been confirmed on ([`SONY_MODELS`]), a missing or malformed tag, a zero +//! frame size, or a centre outside its frame; +//! - manual focus (`0x201b` = 0, or the block's area mode says Manual): the camera still records where +//! the AF frame was parked, not where anything was in focus (on the samples these are off-centre); +//! - the exact frame centre while the area used was Multi (Wide with nothing tracked): on the samples +//! the wide area reports the centre when it picked no particular spot. A centred Spot, Zone or +//! Flexible Spot frame is a real choice and is kept. + +pub use crate::focus_read::{HEADER_PREFIX, HeaderRead, MAX_HEADER_BYTES, read_header}; +use crate::{Orientation, Rect}; +use craft_tiff::{ByteOrder, Ifd, Tiff, makernote, tags as t}; +use serde::{Deserialize, Serialize}; + +/// Where the camera focused, on the upright photo. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct FocusPoint { + /// The focus frame, normalized to the photo's full, EXIF-oriented frame (y down, 0..1 on both + /// axes, clipped to it) — the frame face regions use, so it follows Rotate, flips and crops. + pub rect: Rect, + /// Focus mode as the camera names it (`AF-C`, `AF-S`, `DMF`…). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub mode: Option, + /// The AF area chosen on the camera (`Wide`, `Zone`, `Flexible Spot`…). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub area: Option, + /// What the camera followed, when it says (`Animal Eye Tracking`, `Face tracking`, `Lock On AF`…). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tracking: Option, +} + +impl FocusPoint { + /// The box's centre (normalized, upright). + pub fn center(&self) -> (f64, f64) { + ((self.rect.x0 + self.rect.x1) / 2.0, (self.rect.y0 + self.rect.y1) / 2.0) + } + /// The words that describe it, in order (mode, area, tracking), for "AF-C · Wide · Animal Eye Tracking". + pub fn words(&self) -> Vec<&str> { + [&self.mode, &self.area, &self.tracking].into_iter().filter_map(|s| s.as_deref()).filter(|s| !s.is_empty()).collect() + } + /// The box clipped to the frame and checked: `None` when it is not a usable box (non-finite, + /// empty, or entirely outside the photo). + pub fn sanitized(mut self) -> Option { + let r = self.rect; + if ![r.x0, r.y0, r.x1, r.y1].iter().all(|v| v.is_finite()) { + return None; + } + let (x0, x1) = (r.x0.min(r.x1).clamp(0.0, 1.0), r.x0.max(r.x1).clamp(0.0, 1.0)); + let (y0, y1) = (r.y0.min(r.y1).clamp(0.0, 1.0), r.y0.max(r.y1).clamp(0.0, 1.0)); + if x1 - x0 <= 0.0 || y1 - y0 <= 0.0 { + return None; + } + self.rect = Rect { x0, y0, x1, y1 }; + Some(self) + } +} + +/// Sony bodies whose focus tags have been checked against real files. Another body may store the same +/// tags differently (or not mean the same by them): it shows no focus point until its files are checked. +pub const SONY_MODELS: &[&str] = &["ILCE-1", "ILCE-7SM3"]; + +/// Whether a camera as the catalog names it (`"SONY ILCE-1"`: the Exif Make, a space, the Model) is +/// one whose files can carry a focus point, so a photo from any other camera is never opened to look. +pub fn camera_supported(camera: &str) -> bool { + let camera = camera.trim(); + camera.get(..4).is_some_and(|m| m.eq_ignore_ascii_case("SONY")) && camera.rsplit(' ').next().is_some_and(|model| SONY_MODELS.contains(&model)) +} + +/// Names what [`read`] decodes (its bodies), so a record of "looked, found nothing" made by this +/// decoder is not trusted by one that knows more bodies. +pub fn decoder_tag() -> String { + format!("sony-v1:{}", SONY_MODELS.join(",")) +} + +const FOCUS_MODE: u16 = 0x201b; +const AF_AREA_SETTING: u16 = 0x201c; +const AF_TRACKING: u16 = 0x2021; +const FOCUS_LOCATION: u16 = 0x2027; +const FOCUS_FRAME_SIZE: u16 = 0x2037; +const TAG_9402: u16 = 0x9402; + +/// Side of the box drawn when the camera gave a point but no frame size, as a fraction of the frame's +/// short side (about the size of the α1's smallest frames). +const DEFAULT_SIDE: f64 = 0.03; + +/// The focus point a file's maker note records, or `None` (see the module notes for when). +/// `data` is the TIFF stream the maker note's offsets refer to (a whole raw, or an Exif block). +pub fn read(data: &[u8], tiff: &Tiff, make: Option<&str>, model: Option<&str>, orientation: Option) -> Option { + let make = make?; + if !make.to_ascii_uppercase().starts_with("SONY") || !SONY_MODELS.contains(&model?.trim()) { + return None; + } + let e = tiff.exif()?.get(t::MAKER_NOTE)?; + let mn = makernote::parse_makernote(data, e.offset, e.count() as u64, tiff.order, make)?; + // ARWs from these bodies carry a header-less IFD; their JPEGs the `SONY DSC` header + if !matches!(mn.kind, makernote::MakerNoteKind::Sony | makernote::MakerNoteKind::Plain) { + return None; + } + sony(&mn.ifd, mn.order, orientation.unwrap_or_default()) +} + +fn byte(ifd: &Ifd, tag: u16) -> Option { + ifd.value(tag)?.get_u64(0).and_then(|v| u8::try_from(v).ok()) +} + +/// Decode the Sony tags of a parsed maker-note IFD. +pub(crate) fn sony(ifd: &Ifd, order: ByteOrder, orientation: Orientation) -> Option { + let loc = ifd.value(FOCUS_LOCATION)?; + let [w, h, x, y] = [0, 1, 2, 3].map(|i| loc.get_u64(i)); + let (w, h, x, y) = (w? as f64, h? as f64, x? as f64, y? as f64); + if w < 1.0 || h < 1.0 || x > w || y > h { + return None; + } + let mode = byte(ifd, FOCUS_MODE); + if mode == Some(0) { + return None; + } + // the enciphered block: its focus mode must agree with the plain one, else its layout is not the one we know + let block = ifd.bytes(TAG_9402).and_then(|b| { + let (m, area) = (decipher(*b.get(0x16)?) & 0x7f, decipher(*b.get(0x17)?)); + (Some(m) == mode).then_some(area) + }); + if block == Some(255) { + return None; + } + let setting = byte(ifd, AF_AREA_SETTING); + let tracking = byte(ifd, AF_TRACKING); + let wide_untracked = match block { + Some(a) => a == 0, + None => setting == Some(0) && tracking.is_none_or(|t| t == 0), + }; + if wide_untracked && x * 2.0 == w && y * 2.0 == h { + return None; + } + // frame size: two int16u in the note's byte order, then a non-zero marker when there is a frame + let size = ifd.bytes(FOCUS_FRAME_SIZE).and_then(|b| { + let (fw, fh) = (order.read_u16(b, 0)? as f64, order.read_u16(b, 2)? as f64); + let shown = b.get(4..6).is_none_or(|m| m.iter().any(|&v| v != 0)); + (shown && fw >= 1.0 && fh >= 1.0).then_some((fw, fh)) + }); + let (fw, fh) = size.unwrap_or_else(|| { + let side = w.min(h) * DEFAULT_SIDE; + (side, side) + }); + let sensor = Rect { x0: (x - fw / 2.0) / w, y0: (y - fh / 2.0) / h, x1: (x + fw / 2.0) / w, y1: (y + fh / 2.0) / h }; + let name = |v: Option, f: fn(u8) -> Option<&'static str>| v.and_then(f).map(str::to_string); + let area = name(setting, area_setting_name); + let used = name(block, area_used_name); + // what it followed: the area mode used when it says more than the setting (Animal Eye Tracking on + // Wide), else the tracking setting when it is on + let tracking = used.filter(|u| Some(u) != area.as_ref() && u != "Multi").or_else(|| name(tracking, tracking_name).filter(|t| t != "Off")); + FocusPoint { rect: orientation.map_norm_rect(sensor), mode: name(mode, focus_mode_name), area, tracking }.sanitized() +} + +/// A minimal little-endian TIFF laid out as an α1 ARW's metadata is (IFD0 with Make `SONY` and Model `ILCE-1` → +/// Exif IFD → a header-less maker note with the focus tags: AF-C or the given FocusMode byte, Wide, Face tracking, +/// Animal Eye Tracking at (3226, 1980) of 8640 × 5760), followed by `samples` bytes standing in for sensor data. For +/// tests of the readers and the backfill in other crates; not a real raw (it has no image). +#[doc(hidden)] +pub fn synthetic_arw(mode: u8, samples: usize) -> Vec { + fn entry(f: &mut Vec, tag: u16, typ: u16, count: u32, value: [u8; 4]) { + f.extend_from_slice(&tag.to_le_bytes()); + f.extend_from_slice(&typ.to_le_bytes()); + f.extend_from_slice(&count.to_le_bytes()); + f.extend_from_slice(&value); + } + let encipher = |p: u8| if p >= 249 { p } else { (u32::from(p).pow(3) % 249) as u8 }; + let mut f = b"II*\0".to_vec(); + f.extend_from_slice(&8u32.to_le_bytes()); + // IFD0 at 8: Make (at 62), Model (at 68), Orientation, the Exif IFD (at 76) + f.extend_from_slice(&4u16.to_le_bytes()); + entry(&mut f, t::MAKE, 2, 5, 62u32.to_le_bytes()); + entry(&mut f, t::MODEL, 2, 7, 68u32.to_le_bytes()); + entry(&mut f, t::ORIENTATION, 3, 1, [1, 0, 0, 0]); + entry(&mut f, t::EXIF_IFD, 4, 1, 76u32.to_le_bytes()); + f.extend_from_slice(&[0; 4]); + f.extend_from_slice(b"SONY\0\0ILCE-1\0\0"); + // the Exif IFD: the maker note at 94, 156 bytes + f.extend_from_slice(&1u16.to_le_bytes()); + entry(&mut f, t::MAKER_NOTE, 7, 156, 94u32.to_le_bytes()); + f.extend_from_slice(&[0; 4]); + // the maker note's IFD, its values from 172 + f.extend_from_slice(&6u16.to_le_bytes()); + entry(&mut f, FOCUS_MODE, 1, 1, [mode, 0, 0, 0]); + entry(&mut f, AF_AREA_SETTING, 1, 1, [0, 0, 0, 0]); + entry(&mut f, AF_TRACKING, 1, 1, [1, 0, 0, 0]); + entry(&mut f, FOCUS_LOCATION, 3, 4, 172u32.to_le_bytes()); + entry(&mut f, FOCUS_FRAME_SIZE, 7, 6, 180u32.to_le_bytes()); + entry(&mut f, TAG_9402, 7, 64, 186u32.to_le_bytes()); + f.extend_from_slice(&[0; 4]); + for v in [8640u16, 5760, 3226, 1980] { + f.extend_from_slice(&v.to_le_bytes()); + } + f.extend_from_slice(&[0xaf, 0, 0xb4, 0, 1, 1]); + let mut block = [0u8; 64]; + block[0x16] = encipher(0x80 | mode); + block[0x17] = encipher(20); + f.extend_from_slice(&block); + f.extend((0..samples).map(|i| (i % 251) as u8)); + f +} + +/// Undo the `0x94xx` byte substitution: the stored byte is `p³ mod 249` for `p < 249`. +fn decipher(b: u8) -> u8 { + if b >= 249 { + return b; + } + (0..249u32).find(|p| (p * p * p) % 249 == u32::from(b)).and_then(|p| u8::try_from(p).ok()).unwrap_or(b) +} + +fn focus_mode_name(v: u8) -> Option<&'static str> { + Some(match v { + 0 => "Manual", + 2 => "AF-S", + 3 => "AF-C", + 4 => "AF-A", + 6 => "DMF", + 7 => "AF-D", + _ => return None, + }) +} + +fn area_setting_name(v: u8) -> Option<&'static str> { + Some(match v { + 0 => "Wide", + 1 | 9 => "Center", + 3 | 4 => "Flexible Spot", + 11 => "Zone", + 12 => "Expanded Flexible Spot", + 13 => "Custom AF Area", + _ => return None, + }) +} + +fn tracking_name(v: u8) -> Option<&'static str> { + Some(match v { + 0 => "Off", + 1 => "Face tracking", + 2 => "Lock On AF", + _ => return None, + }) +} + +fn area_used_name(v: u8) -> Option<&'static str> { + Some(match v { + 0 => "Multi", + 1 => "Center", + 2 => "Spot", + 3 => "Flexible Spot", + 11 => "Zone", + 12 => "Expanded Flexible Spot", + 13 => "Custom AF Area", + 14 => "Tracking", + 15 => "Face Tracking", + 20 => "Animal Eye Tracking", + 21 => "Human Eye Tracking", + _ => return None, + }) +} + +#[cfg(test)] +mod tests; diff --git a/crates/metadata/src/focus/tests.rs b/crates/metadata/src/focus/tests.rs new file mode 100644 index 000000000..148f5ed99 --- /dev/null +++ b/crates/metadata/src/focus/tests.rs @@ -0,0 +1,449 @@ +use super::*; +use craft_tiff::{Entry, IfdBuilder, TiffWriter, Value}; + +fn encipher(p: u8) -> u8 { + if p >= 249 { p } else { ((u32::from(p).pow(3)) % 249) as u8 } +} + +fn entry(tag: u16, value: Value) -> Entry { + Entry { tag, value, offset: 0 } +} + +/// The α1 sample `_DSC6954.ARW`: AF-C, Wide, Face tracking, Animal Eye Tracking, 8640×5760 at (3226, 1980), 175×180. +fn alpha1() -> Ifd { + let mut block = vec![0u8; 0x40]; + block[0x16] = encipher(0x83); + block[0x17] = encipher(20); + Ifd { + entries: vec![ + entry(FOCUS_MODE, Value::Byte(vec![3])), + entry(AF_AREA_SETTING, Value::Byte(vec![0])), + entry(AF_TRACKING, Value::Byte(vec![1])), + entry(FOCUS_LOCATION, Value::Short(vec![8640, 5760, 3226, 1980])), + entry(FOCUS_FRAME_SIZE, Value::Undefined(vec![0xaf, 0, 0xb4, 0, 1, 1])), + entry(TAG_9402, Value::Undefined(block)), + ], + ..Default::default() + } +} + +fn set(ifd: &mut Ifd, tag: u16, value: Value) { + ifd.entries.retain(|e| e.tag != tag); + ifd.entries.push(entry(tag, value)); + // IFD entries are kept sorted by tag (lookups rely on it) + ifd.entries.sort_by_key(|e| e.tag); +} + +#[test] +fn the_cipher_matches_the_bytes_seen_in_files() { + // stored → plain, as read from an α1 file's 0x9402 block next to ExifTool's decoded values + assert_eq!(decipher(0x20), 20); + assert_eq!(decipher(0x77), 0x83); + assert_eq!(decipher(0xff), 0xff); + for p in 0..=255u8 { + assert_eq!(decipher(encipher(p)), p); + } +} + +#[test] +fn reads_the_alpha1_sample() { + let f = sony(&alpha1(), ByteOrder::Little, Orientation::Normal).unwrap(); + assert!((f.rect.x0 - (3226.0 - 87.5) / 8640.0).abs() < 1e-12); + assert!((f.rect.y1 - (1980.0 + 90.0) / 5760.0).abs() < 1e-12); + assert_eq!(f.words(), vec!["AF-C", "Wide", "Animal Eye Tracking"]); + let (cx, cy) = f.center(); + assert!((cx - 3226.0 / 8640.0).abs() < 1e-12 && (cy - 1980.0 / 5760.0).abs() < 1e-12); +} + +#[test] +fn big_endian_frame_size() { + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_FRAME_SIZE, Value::Undefined(vec![0, 0xaf, 0, 0xb4, 1, 1])); + let f = sony(&ifd, ByteOrder::Big, Orientation::Normal).unwrap(); + assert!(((f.rect.x1 - f.rect.x0) * 8640.0 - 175.0).abs() < 1e-9); +} + +#[test] +fn follows_the_exif_orientation() { + // a portrait shot (Rotate 270 CW, EXIF 8) still records landscape sensor coordinates + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_LOCATION, Value::Short(vec![8640, 5760, 864, 576])); + let f = sony(&ifd, ByteOrder::Little, Orientation::from_exif(8)).unwrap(); + let (cx, cy) = f.center(); + // left of the sensor (x 0.1) is the bottom of the upright photo; its top (y 0.1) the left + assert!((cx - 0.1).abs() < 1e-9 && (cy - 0.9).abs() < 1e-9, "{cx} {cy}"); + let f = sony(&ifd, ByteOrder::Little, Orientation::from_exif(6)).unwrap(); + let (cx, cy) = f.center(); + assert!((cx - 0.9).abs() < 1e-9 && (cy - 0.1).abs() < 1e-9, "{cx} {cy}"); +} + +#[test] +fn no_real_point() { + // manual focus + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_MODE, Value::Byte(vec![0])); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none()); + // the block says Manual (its focus mode agreeing) + let mut ifd = alpha1(); + let mut block = vec![0u8; 0x18]; + block[0x16] = encipher(3); + block[0x17] = encipher(255); + set(&mut ifd, TAG_9402, Value::Undefined(block)); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none()); + // Multi at the exact centre + let mut ifd = alpha1(); + let mut block = vec![0u8; 0x18]; + block[0x16] = encipher(3); + block[0x17] = encipher(0); + set(&mut ifd, TAG_9402, Value::Undefined(block)); + set(&mut ifd, FOCUS_LOCATION, Value::Short(vec![8640, 5760, 4320, 2880])); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none()); + // …and without the block: Wide, tracking off + ifd.entries.retain(|e| e.tag != TAG_9402); + set(&mut ifd, AF_TRACKING, Value::Byte(vec![0])); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none()); + // missing location, zero frame, centre outside it + let mut ifd = alpha1(); + ifd.entries.retain(|e| e.tag != FOCUS_LOCATION); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none()); + for loc in [vec![0, 5760, 0, 0], vec![8640, 0, 1, 0], vec![8640, 5760, 9000, 10], vec![8640, 5760, 10, 6000], vec![8640, 5760, 10]] { + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_LOCATION, Value::Short(loc.clone())); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_none(), "{loc:?}"); + } +} + +#[test] +fn a_centred_spot_is_a_real_point() { + let mut ifd = alpha1(); + set(&mut ifd, AF_AREA_SETTING, Value::Byte(vec![3])); + set(&mut ifd, AF_TRACKING, Value::Byte(vec![0])); + let mut block = vec![0u8; 0x18]; + block[0x16] = encipher(2); + block[0x17] = encipher(3); + set(&mut ifd, TAG_9402, Value::Undefined(block)); + set(&mut ifd, FOCUS_MODE, Value::Byte(vec![2])); + set(&mut ifd, FOCUS_LOCATION, Value::Short(vec![8640, 5760, 4320, 2880])); + let f = sony(&ifd, ByteOrder::Little, Orientation::Normal).unwrap(); + assert_eq!(f.words(), vec!["AF-S", "Flexible Spot"]); + let (cx, cy) = f.center(); + assert!((cx - 0.5).abs() < 1e-12 && (cy - 0.5).abs() < 1e-12); +} + +#[test] +fn a_block_that_disagrees_is_ignored() { + // another body's block layout: its "focus mode" is not the plain tag's, so its area isn't trusted + let mut ifd = alpha1(); + let mut block = vec![0u8; 0x18]; + block[0x16] = encipher(7); + block[0x17] = encipher(255); + set(&mut ifd, TAG_9402, Value::Undefined(block)); + let f = sony(&ifd, ByteOrder::Little, Orientation::Normal).unwrap(); + assert_eq!(f.words(), vec!["AF-C", "Wide", "Face tracking"]); + // a block too short to hold the bytes is ignored the same way + set(&mut ifd, TAG_9402, Value::Undefined(vec![1, 2, 3])); + assert!(sony(&ifd, ByteOrder::Little, Orientation::Normal).is_some()); +} + +#[test] +fn missing_or_garbage_frame_size_gets_a_small_box_and_edges_are_clipped() { + for fs in + [Value::Undefined(vec![0, 0, 0, 0, 0, 0]), Value::Undefined(vec![1]), Value::Short(vec![5]), Value::Undefined(vec![0xaf, 0, 0xb4, 0, 0, 0])] + { + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_FRAME_SIZE, fs); + let f = sony(&ifd, ByteOrder::Little, Orientation::Normal).unwrap(); + assert!(((f.rect.y1 - f.rect.y0) - DEFAULT_SIDE).abs() < 1e-9); + } + // a frame reaching past the corner is clipped to the photo + let mut ifd = alpha1(); + set(&mut ifd, FOCUS_LOCATION, Value::Short(vec![8640, 5760, 0, 5760])); + set(&mut ifd, FOCUS_FRAME_SIZE, Value::Undefined(vec![0xff, 0xff, 0xff, 0xff, 1, 1])); + let f = sony(&ifd, ByteOrder::Little, Orientation::Normal).unwrap(); + assert_eq!(f.rect, Rect { x0: 0.0, y0: 0.0, x1: 1.0, y1: 1.0 }); + let f = sony(&ifd, ByteOrder::Little, Orientation::Normal).unwrap(); + assert!(f.rect.x0 >= 0.0 && f.rect.y1 <= 1.0); +} + +#[test] +fn wrong_types_and_extreme_values_never_panic() { + let values = [ + Value::Ascii("x".into()), + Value::Byte(vec![]), + Value::SShort(vec![-1, -1, -1, -1]), + Value::Long(vec![u32::MAX; 4]), + Value::Double(vec![f64::NAN; 4]), + Value::Undefined(vec![0xff; 300]), + ]; + for v in &values { + for tag in [FOCUS_MODE, AF_AREA_SETTING, AF_TRACKING, FOCUS_LOCATION, FOCUS_FRAME_SIZE, TAG_9402] { + let mut ifd = alpha1(); + set(&mut ifd, tag, v.clone()); + if let Some(f) = sony(&ifd, ByteOrder::Little, Orientation::Normal) { + assert!(f.rect.x0 >= 0.0 && f.rect.x1 <= 1.0 && f.rect.x0 < f.rect.x1); + } + } + } + assert!(FocusPoint { rect: Rect { x0: f64::NAN, y0: 0.0, x1: 1.0, y1: 1.0 }, mode: None, area: None, tracking: None }.sanitized().is_none()); + assert!(FocusPoint { rect: Rect { x0: 2.0, y0: 2.0, x1: 3.0, y1: 3.0 }, mode: None, area: None, tracking: None }.sanitized().is_none()); +} + +/// A whole TIFF with a `SONY DSC` maker note holding `ifd`'s entries, written for real. +fn sony_file(make: &str, model: &str, orientation: u16, note_entries: &[(u16, Value)]) -> Vec { + sony_file_with(make, model, orientation, note_entries, None) +} + +/// [`sony_file`], with the `0x9402` block's value placed `far` bytes past the end of the TIFF +/// (the gap filled with "sample data") when `far` is given. +fn sony_file_with(make: &str, model: &str, orientation: u16, note_entries: &[(u16, Value)], far: Option) -> Vec { + let order = ByteOrder::Little; + // the note: 12-byte header, then an IFD whose out-of-line values are offset from the TIFF header, + // which we only know after writing; write once to learn where the note lands, then rebuild + let build = |note_at: u64, end: u64| -> (Vec, Vec) { + let mut note = b"SONY DSC \0\0\0".to_vec(); + let ifd_len = 2 + note_entries.len() * 12 + 4; + let mut tail: Vec = Vec::new(); + let mut outside: Vec = Vec::new(); + order.put_u16(&mut note, note_entries.len() as u16); + for (tag, v) in note_entries { + let raw = craft_tiff::writer::encode(order, v); + order.put_u16(&mut note, *tag); + order.put_u16(&mut note, v.field_type() as u16); + order.put_u32(&mut note, v.count() as u32); + if raw.len() <= 4 { + let mut r = raw.clone(); + r.resize(4, 0); + note.extend(r); + } else if let (Some(gap), TAG_9402) = (far, *tag) { + order.put_u32(&mut note, (end + gap as u64) as u32); + outside = raw; + } else { + order.put_u32(&mut note, (note_at + 12 + ifd_len as u64 + tail.len() as u64) as u32); + tail.extend(raw); + if tail.len() % 2 == 1 { + tail.push(0); + } + } + } + order.put_u32(&mut note, 0); + note.extend(tail); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note)); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii(make.into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); + ifd0.set(t::ORIENTATION, Value::Short(vec![orientation])); + ifd0.set_child(t::EXIF_IFD, exif); + (TiffWriter { order, bigtiff: false }.write(&[ifd0]).unwrap(), outside) + }; + let (first, _) = build(0, 0); + let at = Tiff::parse(&first).unwrap().exif().unwrap().get(t::MAKER_NOTE).unwrap().offset; + let (mut file, outside) = build(at, first.len() as u64); + assert_eq!(file.len(), first.len()); + assert_eq!(Tiff::parse(&file).unwrap().exif().unwrap().get(t::MAKER_NOTE).unwrap().offset, at); + if let Some(gap) = far { + file.extend((0..gap).map(|i| (i * 13 % 241) as u8)); + file.extend(outside); + } + file +} + +fn alpha1_entries() -> Vec<(u16, Value)> { + alpha1().entries.into_iter().map(|e| (e.tag, e.value)).collect() +} + +#[test] +fn read_exif_finds_it_through_the_maker_note_and_orientation() { + let file = sony_file("SONY", "ILCE-1", 8, &alpha1_entries()); + let m = crate::read_exif(&file); + let f = m.focus.expect("focus point"); + assert_eq!(f.words(), vec!["AF-C", "Wide", "Animal Eye Tracking"]); + // EXIF 8: sensor x becomes 1 - upright y + let (cx, cy) = f.center(); + assert!((cy - (1.0 - 3226.0 / 8640.0)).abs() < 1e-9 && (cx - 1980.0 / 5760.0).abs() < 1e-9, "{cx} {cy}"); + // an unconfirmed body or another maker: nothing + assert!(crate::read_exif(&sony_file("SONY", "ILCE-7M4", 1, &alpha1_entries())).focus.is_none()); + assert!(crate::read_exif(&sony_file("Canon", "ILCE-1", 1, &alpha1_entries())).focus.is_none()); +} + +#[test] +fn truncated_and_garbage_maker_notes_never_panic() { + let file = sony_file("SONY", "ILCE-1", 1, &alpha1_entries()); + let note_at = Tiff::parse(&file).unwrap().exif().unwrap().get(t::MAKER_NOTE).unwrap().offset as usize; + for cut in (0..file.len()).step_by(7) { + let _ = crate::read_exif(&file[..cut]); + } + let mut seed = 0x2545_f491_u32; + for _ in 0..300 { + let mut f = file.clone(); + for _ in 0..8 { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let i = note_at + (seed as usize % (file.len() - note_at)); + f[i] = (seed >> 8) as u8; + } + if let Some(fp) = crate::read_exif(&f).focus { + assert!(fp.rect.x0 >= 0.0 && fp.rect.x1 <= 1.0 && fp.rect.y0 >= 0.0 && fp.rect.y1 <= 1.0); + } + } +} + +/// A whole-file read and a header-only read of the same file agree, and the header read reads a +/// small part of it (the sample data after the metadata is never touched). +#[test] +fn header_read_matches_the_whole_file_and_reads_little() { + let read = |file: &[u8]| { + let mut asked = 0u64; + let r = read_header(file.len() as u64, &mut |off, buf: &mut [u8]| { + asked += buf.len() as u64; + let s = off as usize; + buf.copy_from_slice(&file[s..s + buf.len()]); + Ok(()) + }) + .unwrap(); + assert_eq!(r.bytes_read, asked); + r + }; + // a raw: the metadata, then 8 MB of "samples" + let mut file = sony_file("SONY", "ILCE-1", 8, &alpha1_entries()); + let meta_len = file.len(); + file.extend((0..8_000_000u32).map(|i| (i * 31 % 251) as u8)); + let whole = crate::extract(&file).focus; + assert!(whole.is_some()); + let r = read(&file); + assert_eq!(r.focus, whole); + assert!(r.bytes_read <= HEADER_PREFIX && r.bytes_read < file.len() as u64 / 100, "{} of {}", r.bytes_read, file.len()); + assert!(meta_len < HEADER_PREFIX as usize); + // the area-mode block placed 3 MB further on: found in a further round, still a small part + let mut far = sony_file_with("SONY", "ILCE-1", 1, &alpha1_entries(), Some(3_000_000)); + far.extend((0..4_000_000u32).map(|i| (i * 7 % 253) as u8)); + let whole = crate::extract(&far).focus; + assert!(whole.is_some()); + let r = read(&far); + assert_eq!(r.focus, whole); + assert!(r.bytes_read < 64 * 1024, "{}", r.bytes_read); + // a JPEG: the Exif segment after another segment + let exif = sony_file("SONY", "ILCE-1", 6, &alpha1_entries()); + let mut jpeg = vec![0xff, 0xd8, 0xff, 0xe0, 0, 16]; + jpeg.extend_from_slice(b"JFIF\0\x01\x01\0\0\x01\0\x01\0\0"); + jpeg.extend_from_slice(&[0xff, 0xe1]); + jpeg.extend_from_slice(&((exif.len() + 8) as u16).to_be_bytes()); + jpeg.extend_from_slice(b"Exif\0\0"); + jpeg.extend_from_slice(&exif); + jpeg.extend_from_slice(&[0xff, 0xda, 0, 2]); + jpeg.extend(std::iter::repeat_n(0x55u8, 3_000_000)); + let whole = crate::extract(&jpeg).focus; + assert!(whole.is_some()); + let r = read(&jpeg); + assert_eq!(r.focus, whole); + assert!(r.bytes_read <= HEADER_PREFIX); + // unsupported bodies agree too (no point either way) + let other = sony_file("SONY", "ILCE-7M4", 1, &alpha1_entries()); + assert_eq!(read(&other).focus, None); +} + +/// Every truncation and corruption of a file gives the header read the whole-file read's answer +/// (or an error when the file is shorter than it said), never a panic. +#[test] +fn header_read_of_damaged_files_agrees_or_fails_cleanly() { + let file = sony_file("SONY", "ILCE-1", 1, &alpha1_entries()); + for cut in (0..file.len()).step_by(5) { + let part = &file[..cut]; + let r = read_header(part.len() as u64, &mut |off, buf: &mut [u8]| { + let s = off as usize; + buf.copy_from_slice(part.get(s..s + buf.len()).ok_or_else(|| std::io::Error::from(std::io::ErrorKind::UnexpectedEof))?); + Ok(()) + }) + .unwrap(); + assert_eq!(r.focus, crate::extract(part).focus, "cut at {cut}"); + } + let mut seed = 0x9e37_79b9_u32; + for _ in 0..400 { + let mut f = file.clone(); + for _ in 0..6 { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + let i = seed as usize % f.len(); + f[i] = (seed >> 8) as u8; + } + let r = read_header(f.len() as u64, &mut |off, buf: &mut [u8]| { + let s = off as usize; + buf.copy_from_slice(&f[s..s + buf.len()]); + Ok(()) + }) + .unwrap(); + assert_eq!(r.focus, crate::extract(&f).focus); + } + // a reader told the file is longer than it is (replaced since it was measured): an error + let r = read_header(file.len() as u64 + 100_000, &mut |off, buf: &mut [u8]| { + let s = off as usize; + buf.copy_from_slice(file.get(s..s + buf.len()).ok_or_else(|| std::io::Error::from(std::io::ErrorKind::UnexpectedEof))?); + Ok(()) + }); + assert!(r.is_err()); +} + +#[test] +fn the_synthetic_arw_is_laid_out_as_described() { + let f = synthetic_arw(3, 10); + assert_eq!(f.len(), 260); + let p = crate::extract(&f).focus.expect("a point"); + assert_eq!(p.words(), vec!["AF-C", "Wide", "Animal Eye Tracking"]); + assert!(crate::extract(&synthetic_arw(0, 0)).focus.is_none(), "manual focus"); +} + +#[test] +fn cameras_that_can_have_a_point() { + assert!(camera_supported("SONY ILCE-1") && camera_supported("Sony ILCE-7SM3 ")); + for c in ["SONY ILCE-7M4", "Canon EOS R5", "", "SONY", "ILCE-1", "SON"] { + assert!(!camera_supported(c), "{c}"); + } +} + +/// Opt-in: decode real α1 / A7S III files and compare with the values ExifTool (run as a black box) +/// reports for them. `LIGHTCRAFT_FOCUS_SAMPLES=` is the photo folder the paths below are relative +/// to; skipped without it. No user file is part of the repository. +#[test] +fn real_sony_files_match_exiftool() { + let Ok(dir) = std::env::var("LIGHTCRAFT_FOCUS_SAMPLES") else { return }; + let dir = std::path::Path::new(&dir); + type Want = Option<([f64; 4], [f64; 2], &'static [&'static str])>; + // (file, EXIF orientation, ExifTool's FocusLocation and FocusFrameSize, the words we show; None = no point) + let cases: [(&str, u16, Want); 8] = [ + // α1, AF-C, Wide, Face tracking, AFAreaMode Animal Eye Tracking + ("2023/11-november/17th/_DSC6954.ARW", 1, Some(([8640.0, 5760.0, 3226.0, 1980.0], [175.0, 180.0], &["AF-C", "Wide", "Animal Eye Tracking"]))), + ("2023/11-november/17th/_DSC6894.ARW", 1, Some(([8640.0, 5760.0, 4725.0, 1860.0], [135.0, 132.0], &["AF-C", "Wide", "Animal Eye Tracking"]))), + // α1, manual focus (the location 4441 3060 is where the frame was parked) + ("2023/11-november/15th/_DSC6613.ARW", 1, None), + // α1 portrait, AF-S, Wide, Multi at the exact centre 4320 2880 + ("2022/01-january/9th/_DSC6079.ARW", 8, None), + // α1 APS-C crop mode portrait, AF-C, Wide, Lock On AF, AFAreaMode Tracking + ("2022/12-december/16th/_DSC1575.ARW", 8, Some(([5616.0, 3744.0, 4071.0, 1848.0], [280.0, 280.0], &["AF-C", "Wide", "Tracking"]))), + // A7S III portrait, AF-A, Wide, Face tracking, Animal Eye Tracking + ("2020/10-october/31st/DSC00065.ARW", 8, Some(([4240.0, 2832.0, 1192.0, 1675.0], [278.0, 277.0], &["AF-A", "Wide", "Animal Eye Tracking"]))), + // A7S III upside down, AF-C, a centred Flexible Spot: a real point + ("2021/11-november/7th/DSC05975.ARW", 3, Some(([4240.0, 2832.0, 2120.0, 1416.0], [251.0, 224.0], &["AF-C", "Flexible Spot"]))), + // A7S III, manual focus + ("2021/06-june/12th/DSC06806.ARW", 1, None), + ]; + for (name, exif_orientation, want) in cases { + let bytes = std::fs::read(dir.join(name)).unwrap_or_else(|e| panic!("{name}: {e}")); + let got = crate::extract(&bytes).focus; + match (want, got) { + (None, None) => {} + (Some((loc, size, words)), Some(f)) => { + let [w, h, x, y] = loc; + let sensor = + Rect { x0: (x - size[0] / 2.0) / w, y0: (y - size[1] / 2.0) / h, x1: (x + size[0] / 2.0) / w, y1: (y + size[1] / 2.0) / h }; + let r = Orientation::from_exif(exif_orientation).map_norm_rect(sensor); + for (a, b) in [(f.rect.x0, r.x0), (f.rect.y0, r.y0), (f.rect.x1, r.x1), (f.rect.y1, r.y1)] { + assert!((a - b).abs() < 1e-9, "{name}: {:?} vs {r:?}", f.rect); + } + assert_eq!(f.words(), words, "{name}"); + } + (w, g) => panic!("{name}: want {w:?}, got {g:?}"), + } + } +} diff --git a/crates/metadata/src/focus_read.rs b/crates/metadata/src/focus_read.rs new file mode 100644 index 000000000..14547b051 --- /dev/null +++ b/crates/metadata/src/focus_read.rs @@ -0,0 +1,472 @@ +//! Reading a file's focus point without reading the file: [`read_header`] asks for the few byte +//! ranges [`crate::focus::read`] looks at (a TIFF raw's IFD0, its Exif IFD, the maker-note IFD and +//! the values of the focus tags; a JPEG's Exif segment) and decodes them exactly as a whole-file +//! read would. On the α1 and the A7S III all of it sits in the first 16 KB of a 50–120 MB ARW, so a +//! library of them can be backfilled from a slow disk by reading one small block per photo. +//! +//! The ranges are found by walking the structure as the readers do (the IFD tables and the value +//! offsets they hold), round by round: each round asks only for what the bytes loaded so far point +//! at. A TIFF is decoded from a buffer of the file's length in which only the loaded ranges hold the +//! file's bytes (the rest stays zero and its pages are never touched), so every offset and bound the +//! reader checks is the one it checks on the whole file. Rounds, bytes and the buffer are bounded: +//! a file that wants more is reported as having no focus point. + +use std::io; + +use crate::focus::FocusPoint; +use craft_tiff::{ByteOrder, FieldType, Tiff, tags as t}; + +/// Bytes read first: the α1's and A7S III's focus data ends before 16 KB, so one read covers it. +pub const HEADER_PREFIX: u64 = 32 * 1024; +/// Later reads are widened to this granularity (fewer, larger reads on a spinning disk). +const ALIGN: u64 = 4 * 1024; +/// The most rounds of reads one file gets (a real file needs one; the walk takes at most nine). +const MAX_ROUNDS: usize = 12; +/// The most bytes read from one file: past this the layout is not one the decoder knows. +pub const MAX_HEADER_BYTES: u64 = 2 << 20; +/// The largest TIFF given a buffer of its length (its untouched pages are never written, but the +/// allocation must succeed: the engine's planned preview reads use the same bound). Raws are far below it. +const MAX_SPARSE_FILE: u64 = 1 << 30; +/// JPEG segments walked before giving up. +const MAX_SEGMENTS: usize = 1024; +/// Entries read from one IFD (the TIFF reader's limit). +const MAX_ENTRIES: u64 = 8192; + +/// The focus tags whose values [`crate::focus`] reads from a Sony maker note. +const FOCUS_TAGS: [u16; 6] = [0x201b, 0x201c, 0x2021, 0x2027, 0x2037, 0x9402]; + +/// What [`read_header`] found, and what it cost. +#[derive(Clone, Debug, PartialEq)] +pub struct HeaderRead { + /// The focus point a whole-file read gives ([`crate::extract`]`(file).focus`). + pub focus: Option, + /// Bytes read from the file. + pub bytes_read: u64, +} + +/// The focus point of a file of `len` bytes, reading only what it needs through `read_at(offset, +/// buf)`, which fills `buf` with the file's bytes at `offset` (an error — a file that shrank or +/// went away — is returned). A malformed or unknown file is `Ok` with no point. +pub fn read_header(len: u64, read_at: &mut dyn FnMut(u64, &mut [u8]) -> io::Result<()>) -> io::Result { + let mut reader = Reader { len, read_at, bytes: 0 }; + let head = reader.fetch(0, HEADER_PREFIX.min(len))?; + let focus = if head.starts_with(&[0xff, 0xd8]) { + jpeg(&mut reader, &head)? + } else if matches!(Tiff::sniff(&head), Some((_, false))) { + tiff(&mut reader, head)? + } else { + None + }; + Ok(HeaderRead { focus, bytes_read: reader.bytes }) +} + +struct Reader<'a> { + len: u64, + read_at: &'a mut dyn FnMut(u64, &mut [u8]) -> io::Result<()>, + bytes: u64, +} + +impl Reader<'_> { + /// `[start, end)` of the file (clipped to it), counted against [`MAX_HEADER_BYTES`]. + fn fetch(&mut self, start: u64, end: u64) -> io::Result> { + let end = end.min(self.len); + let n = end.saturating_sub(start); + if self.bytes.saturating_add(n) > MAX_HEADER_BYTES { + return Err(io::Error::other(OverBudget)); + } + let mut buf = vec![0u8; usize::try_from(n).map_err(|_| io::Error::other("range too large"))?]; + if n > 0 { + (self.read_at)(start, &mut buf)?; + } + self.bytes = self.bytes.saturating_add(n); + Ok(buf) + } +} + +/// A file whose focus data would take more than [`MAX_HEADER_BYTES`] to read. +#[derive(Debug)] +struct OverBudget; + +impl std::fmt::Display for OverBudget { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("focus data larger than a header") + } +} + +impl std::error::Error for OverBudget {} + +fn over_budget(e: &io::Error) -> bool { + e.get_ref().is_some_and(|i| i.is::()) +} + +/// An over-budget read is "no focus point here", not a failure to report. +fn budget_is_none(r: io::Result>) -> io::Result> { + match r { + Err(e) if over_budget(&e) => Ok(None), + r => r, + } +} + +/// A JPEG's first `Exif` APP1 segment, found as [`crate::jpeg_segments`] finds it. +fn jpeg(r: &mut Reader, head: &[u8]) -> io::Result> { + budget_is_none(jpeg_inner(r, head)) +} + +fn jpeg_inner(r: &mut Reader, head: &[u8]) -> io::Result> { + let len = r.len; + let at = |r: &mut Reader, start: u64, n: u64| -> io::Result> { + let local = usize::try_from(start).ok().zip(usize::try_from(start.saturating_add(n)).ok()).and_then(|(a, b)| head.get(a..b)); + match local { + Some(s) => Ok(s.to_vec()), + None => r.fetch(start, start.saturating_add(n)), + } + }; + let mut i = 2u64; + for _ in 0..MAX_SEGMENTS { + if i.saturating_add(4) > len { + break; + } + let h = at(r, i, 4)?; + let (Some(&ff), Some(&marker), Some(&a), Some(&b)) = (h.first(), h.get(1), h.get(2), h.get(3)) else { break }; + if ff != 0xff { + break; + } + if marker == 0xff { + i += 1; + continue; + } + if marker == 0xd8 || (0xd0..=0xd7).contains(&marker) || marker == 0x01 { + i += 2; + continue; + } + if marker == 0xd9 || marker == 0xda { + break; + } + let seg = u64::from(u16::from_be_bytes([a, b])); + if seg < 2 { + break; + } + let end = i.saturating_add(2).saturating_add(seg).min(len); + if marker == 0xe1 { + let payload = at(r, i + 4, end.saturating_sub(i + 4))?; + if payload.starts_with(b"Exif\0") { + return Ok(crate::read_exif(payload.get(6..).unwrap_or(&[])).focus); + } + } + i = end; + } + Ok(None) +} + +/// A TIFF raw, decoded from a buffer of its length holding only the ranges [`plan`] asked for. +fn tiff(r: &mut Reader, head: Vec) -> io::Result> { + if r.len > MAX_SPARSE_FILE { + return Ok(None); + } + let Ok(len) = usize::try_from(r.len) else { return Ok(None) }; + let mut buf = vec![0u8; len]; + let Some(dst) = buf.get_mut(..head.len()) else { return Ok(None) }; + dst.copy_from_slice(&head); + let mut loaded = Loaded::default(); + loaded.insert(0, head.len() as u64); + for _ in 0..MAX_ROUNDS { + match plan(&buf, &loaded) { + Step::Ready => { + // The decoder's allocation budget follows the bytes actually fetched, not the logical + // file length: only the loaded ranges are data, and an unrelated entry that claims a + // huge value must be cut by what leaving it cost, not by what the buffer claims to hold. + // The same 8 × / + 1 MiB shape the general TIFF reader uses, over `reader.bytes` instead. + let opts = craft_tiff::ParseOptions { value_budget: Some(r.bytes.saturating_mul(8).saturating_add(1 << 20)), ..Default::default() }; + return Ok(crate::try_read_exif_with(&buf, &opts).ok().and_then(|m| m.focus)); + } + Step::Nothing => return Ok(None), + Step::Need(ranges) => { + for (a, b) in ranges { + // widen to the read granularity, but never re-read what is loaded + let (wa, wb) = ((a / ALIGN) * ALIGN, b.div_ceil(ALIGN).saturating_mul(ALIGN).min(r.len)); + for (s, e) in loaded.gaps(wa, wb) { + let bytes = match r.fetch(s, e) { + Err(e) if over_budget(&e) => return Ok(None), + other => other?, + }; + let (Ok(su), Ok(eu)) = (usize::try_from(s), usize::try_from(e)) else { return Ok(None) }; + let Some(dst) = buf.get_mut(su..eu) else { return Ok(None) }; + dst.copy_from_slice(&bytes); + loaded.insert(s, e); + } + } + } + } + } + Ok(None) +} + +/// Sorted, disjoint, non-touching byte ranges. +#[derive(Default)] +struct Loaded(Vec<(u64, u64)>); + +impl Loaded { + fn insert(&mut self, start: u64, end: u64) { + if start >= end { + return; + } + let (mut s, mut e) = (start, end); + let mut out = Vec::with_capacity(self.0.len() + 1); + for &(a, b) in &self.0 { + if b < s || a > e { + out.push((a, b)); + } else { + s = s.min(a); + e = e.max(b); + } + } + out.push((s, e)); + out.sort_unstable(); + self.0 = out; + } + + fn contains(&self, start: u64, end: u64) -> bool { + start >= end || self.0.iter().any(|&(a, b)| a <= start && end <= b) + } + + /// The parts of `[start, end)` not loaded. + fn gaps(&self, start: u64, end: u64) -> Vec<(u64, u64)> { + let mut out = Vec::new(); + let mut at = start; + for &(a, b) in &self.0 { + if b <= at || a >= end { + continue; + } + if a > at { + out.push((at, a.min(end))); + } + at = at.max(b); + if at >= end { + break; + } + } + if at < end { + out.push((at, end)); + } + out + } +} + +enum Step { + /// Load these and ask again. + Need(Vec<(u64, u64)>), + /// Everything the decoder reads is loaded. + Ready, + /// The file has nothing the decoder could read (no Exif, no maker note, a structure it rejects). + Nothing, +} + +/// One IFD entry as the TIFF reader sees it: where its value is (absolute), if it fits the file. +struct RawEntry { + tag: u16, + typ: FieldType, + start: u64, + end: u64, +} + +struct Want<'a> { + buf: &'a [u8], + order: ByteOrder, + loaded: &'a Loaded, + need: Vec<(u64, u64)>, +} + +impl Want<'_> { + fn len(&self) -> u64 { + self.buf.len() as u64 + } + + /// Whether `[start, end)` (clipped to the file) is loaded; asks for it if not. + fn want(&mut self, start: u64, end: u64) -> bool { + let end = end.min(self.len()); + if self.loaded.contains(start, end) { + return true; + } + self.need.push((start, end)); + false + } + + /// The entries of the IFD at `off` (values relative to 0), once its table is loaded; `Some(empty)` + /// for one the reader would reject. + fn table(&mut self, off: u64) -> Option> { + let len = self.len(); + if off == 0 || off >= len { + return Some(Vec::new()); + } + if !self.want(off, off.saturating_add(2)) { + return None; + } + let pos = usize::try_from(off).ok()?; + let n = u64::from(self.order.read_u16(self.buf, pos)?); + let table = off + 2; + let n_read = n.min(len.saturating_sub(table) / 12).min(MAX_ENTRIES); + let end = table + n_read * 12 + if n_read == n { 4 } else { 0 }; + if !self.want(off, end) { + return None; + } + let mut out = Vec::new(); + for i in 0..n_read { + let Ok(p) = usize::try_from(table + i * 12) else { continue }; + let (Some(tag), Some(typ), Some(count)) = ( + self.order.read_u16(self.buf, p), + self.order.read_u16(self.buf, p + 2).and_then(FieldType::from_u16), + self.order.read_u32(self.buf, p + 4), + ) else { + continue; + }; + let Some(total) = u64::from(count).checked_mul(typ.size() as u64) else { continue }; + let field = p as u64 + 8; + let start = if total <= 4 { + field + } else { + match self.order.read_u32(self.buf, p + 8) { + Some(v) => u64::from(v), + None => continue, + } + }; + let Some(end) = start.checked_add(total).filter(|&e| e <= len) else { continue }; + out.push(RawEntry { tag, typ, start, end }); + } + Some(out) + } + + /// The first unsigned value of an entry (loaded). + fn first_value(&self, e: &RawEntry) -> Option { + let pos = usize::try_from(e.start).ok()?; + match e.typ.size() { + 1 => self.buf.get(pos).map(|&b| u64::from(b)), + 2 => self.order.read_u16(self.buf, pos).map(u64::from), + 4 => self.order.read_u32(self.buf, pos).map(u64::from), + _ => self.order.read_u64(self.buf, pos), + } + } + + fn step(self) -> Step { + if self.need.is_empty() { Step::Ready } else { Step::Need(self.need) } + } +} + +/// What of a classic TIFF the focus decoder still needs, given what is `loaded`: IFD0 (Make, Model, +/// the Exif pointer), the Exif IFD (the maker-note entry), the maker note's header and IFD as +/// `craft_tiff::makernote` locates it, and the focus tags' values. +fn plan(buf: &[u8], loaded: &Loaded) -> Step { + let Some((order, false)) = Tiff::sniff(buf) else { return Step::Nothing }; + let mut w = Want { buf, order, loaded, need: Vec::new() }; + let Some(ifd0) = order.read_u32(buf, 4).map(u64::from) else { return Step::Nothing }; + let Some(e0) = w.table(ifd0) else { return w.step() }; + let mut exif = None; + for e in &e0 { + if matches!(e.tag, t::MAKE | t::MODEL | t::UNIQUE_CAMERA_MODEL | t::ORIENTATION) { + w.want(e.start, e.end); + } + if e.tag == t::EXIF_IFD && exif.is_none() && w.want(e.start, e.end) { + exif = w.first_value(e); + } + } + if !w.need.is_empty() { + return w.step(); + } + let Some(exif) = exif.filter(|&o| o != 0) else { return Step::Nothing }; + let Some(ee) = w.table(exif) else { return w.step() }; + let Some(note) = ee.iter().find(|e| e.tag == t::MAKER_NOTE) else { return Step::Nothing }; + let head_end = note.start.saturating_add(12).min(note.end); + if !w.want(note.start, head_end) { + return w.step(); + } + let head = usize::try_from(note.start).ok().zip(usize::try_from(head_end).ok()).and_then(|(a, b)| buf.get(a..b)).unwrap_or(&[]); + let sony = head.starts_with(b"SONY") || head.starts_with(b"VHAB \0"); + // another maker's header: `focus::read` only decodes Sony and header-less notes + let foreign: [&[u8]; 13] = [ + b"Nikon\0", + b"FUJIFILM", + b"GENERALE", + b"OLYMPUS\0", + b"OM SYSTEM\0", + b"OLYMP\0", + b"EPSON\0", + b"MINOL\0", + b"CAMER\0", + b"SANYO\0", + b"Panasonic\0", + b"AOC\0", + b"PENTAX \0", + ]; + if !sony && foreign.iter().any(|p| head.starts_with(p)) { + return Step::Nothing; + } + let mut entries = None; + if sony { + match w.table(note.start.saturating_add(12)) { + None => return w.step(), + Some(e) if !e.is_empty() => entries = Some(e), + Some(_) => {} + } + } + let entries = match entries { + Some(e) => e, + None => match w.table(note.start) { + Some(e) => e, + None => return w.step(), + }, + }; + for e in entries.iter().filter(|e| FOCUS_TAGS.contains(&e.tag)) { + w.want(e.start, e.end); + } + w.step() +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A reader over `file` that counts what it is asked for. + fn read(file: &[u8]) -> (io::Result, u64) { + let mut asked = 0u64; + let r = read_header(file.len() as u64, &mut |off, buf: &mut [u8]| { + asked += buf.len() as u64; + let start = usize::try_from(off).map_err(io::Error::other)?; + let src = file.get(start..start + buf.len()).ok_or_else(|| io::Error::from(io::ErrorKind::UnexpectedEof))?; + buf.copy_from_slice(src); + Ok(()) + }); + (r, asked) + } + + #[test] + fn loaded_ranges() { + let mut l = Loaded::default(); + l.insert(10, 20); + l.insert(30, 40); + assert_eq!(l.gaps(0, 50), vec![(0, 10), (20, 30), (40, 50)]); + l.insert(20, 30); + assert_eq!(l.0, vec![(10, 40)]); + assert!(l.contains(12, 38) && !l.contains(5, 12)); + assert!(l.gaps(15, 35).is_empty()); + } + + #[test] + fn garbage_and_empty_files_are_no_point() { + for file in [vec![], vec![0xff, 0xd8], b"II*\0\x08\0\0\0".to_vec(), b"MM\0*\xff\xff\xff\xff".to_vec(), vec![0x42; 100_000]] { + let (r, _) = read(&file); + assert_eq!(r.unwrap().focus, None); + } + // a JPEG of nothing but fill bytes and tiny segments stops at its bounds + let mut j = vec![0xff, 0xd8]; + for _ in 0..5000 { + j.extend_from_slice(&[0xff, 0xe2, 0, 2]); + } + assert_eq!(read(&j).0.unwrap().focus, None); + } + + #[test] + fn a_file_that_shrinks_mid_read_is_an_error() { + // the reader is told the file is longer than it is (replaced between the stat and the read) + let r = read_header(1 << 20, &mut |_, _| Err(io::Error::from(io::ErrorKind::UnexpectedEof))); + assert!(r.is_err()); + } +} diff --git a/crates/metadata/src/geom.rs b/crates/metadata/src/geom.rs new file mode 100644 index 000000000..804b7d643 --- /dev/null +++ b/crates/metadata/src/geom.rs @@ -0,0 +1,180 @@ +//! The geometry metadata is stated in: a point, an axis-aligned rectangle, and the eight Exif orientations. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +impl Point { + pub const fn new(x: f64, y: f64) -> Self { + Self { x, y } + } +} + +/// Axis-aligned rectangle `[x0, x1) × [y0, y1)`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Rect { + pub x0: f64, + pub y0: f64, + pub x1: f64, + pub y1: f64, +} + +impl Rect { + pub const UNIT: Rect = Rect { x0: 0.0, y0: 0.0, x1: 1.0, y1: 1.0 }; + pub const fn new(x0: f64, y0: f64, x1: f64, y1: f64) -> Self { + Self { x0, y0, x1, y1 } + } + pub fn from_xywh(x: f64, y: f64, w: f64, h: f64) -> Self { + Self::new(x, y, x + w, y + h) + } + pub fn from_center(c: Point, w: f64, h: f64) -> Self { + Self::new(c.x - w / 2.0, c.y - h / 2.0, c.x + w / 2.0, c.y + h / 2.0) + } + pub fn from_points(a: Point, b: Point) -> Self { + Self::new(a.x.min(b.x), a.y.min(b.y), a.x.max(b.x), a.y.max(b.y)) + } + pub fn width(&self) -> f64 { + self.x1 - self.x0 + } + pub fn height(&self) -> f64 { + self.y1 - self.y0 + } + pub fn area(&self) -> f64 { + self.width().max(0.0) * self.height().max(0.0) + } + pub fn center(&self) -> Point { + Point::new((self.x0 + self.x1) / 2.0, (self.y0 + self.y1) / 2.0) + } + pub fn aspect(&self) -> f64 { + if self.height() != 0.0 { self.width() / self.height() } else { 0.0 } + } + pub fn contains(&self, p: Point) -> bool { + p.x >= self.x0 && p.x < self.x1 && p.y >= self.y0 && p.y < self.y1 + } + pub fn is_empty(&self) -> bool { + self.x1 <= self.x0 || self.y1 <= self.y0 + } + pub fn intersect(&self, o: &Rect) -> Rect { + Rect::new(self.x0.max(o.x0), self.y0.max(o.y0), self.x1.min(o.x1), self.y1.min(o.y1)) + } +} + +/// EXIF orientation (1..=8) as a transform of the stored pixel grid. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum Orientation { + #[default] + Normal, + FlipH, + Rotate180, + FlipV, + Transpose, + Rotate90, + Transverse, + Rotate270, +} + +impl Orientation { + pub fn from_exif(v: u16) -> Orientation { + match v { + 2 => Orientation::FlipH, + 3 => Orientation::Rotate180, + 4 => Orientation::FlipV, + 5 => Orientation::Transpose, + 6 => Orientation::Rotate90, + 7 => Orientation::Transverse, + 8 => Orientation::Rotate270, + _ => Orientation::Normal, + } + } + pub fn to_exif(self) -> u16 { + match self { + Orientation::Normal => 1, + Orientation::FlipH => 2, + Orientation::Rotate180 => 3, + Orientation::FlipV => 4, + Orientation::Transpose => 5, + Orientation::Rotate90 => 6, + Orientation::Transverse => 7, + Orientation::Rotate270 => 8, + } + } + /// Whether width and height swap. + pub fn swaps_axes(self) -> bool { + matches!(self, Orientation::Transpose | Orientation::Rotate90 | Orientation::Transverse | Orientation::Rotate270) + } + /// Decompose into (clockwise quarter turns applied after an optional horizontal flip). + pub fn to_parts(self) -> (bool, u8) { + match self { + Orientation::Normal => (false, 0), + Orientation::Rotate90 => (false, 1), + Orientation::Rotate180 => (false, 2), + Orientation::Rotate270 => (false, 3), + Orientation::FlipH => (true, 0), + Orientation::Transverse => (true, 1), + Orientation::FlipV => (true, 2), + Orientation::Transpose => (true, 3), + } + } + pub fn from_parts(flip: bool, quarter_turns_cw: u8) -> Orientation { + match (flip, quarter_turns_cw % 4) { + (false, 0) => Orientation::Normal, + (false, 1) => Orientation::Rotate90, + (false, 2) => Orientation::Rotate180, + (false, _) => Orientation::Rotate270, + (true, 0) => Orientation::FlipH, + (true, 1) => Orientation::Transverse, + (true, 2) => Orientation::FlipV, + (true, _) => Orientation::Transpose, + } + } + /// Rotate the (already oriented) image a further quarter turn clockwise (`cw`) or counter-clockwise. + pub fn rotated(self, cw: bool) -> Orientation { + let (f, t) = self.to_parts(); + Orientation::from_parts(f, if cw { t + 1 } else { t + 3 }) + } + /// Flip the displayed image horizontally. + pub fn flipped_h(self) -> Orientation { + let (f, t) = self.to_parts(); + // flipH ∘ rot(t) = rot(-t) ∘ flipH + Orientation::from_parts(!f, (4 - t) % 4) + } + /// The orientation that undoes this one (`o.inverse().map(o.map(p)) == p`). Flips and half turns + /// are their own inverses; quarter turns swap. + pub fn inverse(self) -> Orientation { + match self { + Orientation::Rotate90 => Orientation::Rotate270, + Orientation::Rotate270 => Orientation::Rotate90, + o => o, + } + } + /// Map a rectangle in normalized (0..1) coordinates of the stored image to normalized coordinates + /// of the oriented one (e.g. a face region given on the upright photo → the frame after the user's + /// Rotate Left/Right). + pub fn map_norm_rect(self, r: Rect) -> Rect { + let (ax, ay) = self.map(r.x0, r.y0, 1.0, 1.0); + let (bx, by) = self.map(r.x1, r.y1, 1.0, 1.0); + Rect::from_points(Point::new(ax, ay), Point::new(bx, by)) + } + /// Map a point in stored pixel space (w×h stored dims) to oriented space. + pub fn map(self, x: f64, y: f64, w: f64, h: f64) -> (f64, f64) { + let (flip, t) = self.to_parts(); + let (mut x, y2, mut w2, mut h2) = (x, y, w, h); + if flip { + x = w2 - x; + } + let mut y = y2; + for _ in 0..t { + // rotate 90° cw: (x, y) -> (h - y, x) + let nx = h2 - y; + let ny = x; + x = nx; + y = ny; + std::mem::swap(&mut w2, &mut h2); + } + (x, y) + } +} diff --git a/crates/metadata/src/gpx.rs b/crates/metadata/src/gpx.rs new file mode 100644 index 000000000..691a2b7e1 --- /dev/null +++ b/crates/metadata/src/gpx.rs @@ -0,0 +1,491 @@ +//! GPS track logs in GPX 1.0 / 1.1 (the open GPS Exchange Format, ): +//! the timed track points of every ``, and the position at any moment for geotagging photos. +//! +//! What we use from the format: `` → `` → `` with optional `` +//! (metres) and `"#; + assert!(matches!(parse_gpx(wrong_close), Err(GpxError::Xml(_)))); + assert!( + parse_gpx("") + .unwrap() + .points + .is_empty() + ); + let deep = "".to_string() + &"".repeat(100); + assert_eq!(parse_gpx(&deep), Err(GpxError::TooComplex)); + } +} diff --git a/crates/metadata/src/iptc.rs b/crates/metadata/src/iptc.rs new file mode 100644 index 000000000..a398c6fc4 --- /dev/null +++ b/crates/metadata/src/iptc.rs @@ -0,0 +1,151 @@ +//! IPTC-IIM (Information Interchange Model 4.2) — the record-2 datasets people still carry in JPEGs. +//! +//! Stream of `0x1C record dataset size(u16 BE, or extended) data` tags. Text is UTF-8 when the record-1 +//! `CodedCharacterSet` (1:90) is `ESC % G`, otherwise decoded as UTF-8 when valid and Latin-1 otherwise. + +use crate::{DateTime, Metadata}; + +fn decode(b: &[u8], utf8: bool) -> String { + match std::str::from_utf8(b) { + Ok(s) => s.trim_end_matches('\0').trim().to_string(), + Err(_) if utf8 => String::from_utf8_lossy(b).trim_end_matches('\0').trim().to_string(), + Err(_) => b.iter().map(|&c| c as char).collect::().trim_end_matches('\0').trim().to_string(), + } +} + +/// Parse an IIM stream into the fields [`Metadata`] carries (title, keywords, artist, copyright, caption, +/// creation date/time). +pub fn parse_iptc(b: &[u8]) -> Metadata { + let mut m = Metadata::default(); + let mut utf8 = false; + let mut date: Option = None; + let mut time: Option = None; + let mut seen_keywords = std::collections::HashSet::new(); + let mut i = 0usize; + while i + 5 <= b.len() { + if b[i] != 0x1c { + break; + } + let (rec, ds) = (b[i + 1], b[i + 2]); + let mut size = u16::from_be_bytes([b[i + 3], b[i + 4]]) as usize; + let mut at = i + 5; + if size & 0x8000 != 0 { + // extended dataset: next (size & 0x7fff) bytes hold the length + let n = size & 0x7fff; + if n == 0 || n > 4 || at + n > b.len() { + break; + } + size = b[at..at + n].iter().fold(0usize, |a, &c| (a << 8) | c as usize); + at += n; + } + let Some(data) = b.get(at..at.saturating_add(size)) else { break }; + match (rec, ds) { + (1, 90) => utf8 = data == b"\x1b%G", + (2, 5) => m.title = Some(decode(data, utf8)).filter(|s| !s.is_empty()), + (2, 25) => { + let k = decode(data, utf8); + if !k.is_empty() && seen_keywords.insert(k.clone()) { + m.keywords.push(k); + } + } + (2, 80) => m.artist = Some(decode(data, utf8)).filter(|s| !s.is_empty()), + (2, 116) => m.copyright = Some(decode(data, utf8)).filter(|s| !s.is_empty()), + (2, 120) => m.caption = Some(decode(data, utf8)).filter(|s| !s.is_empty()), + (2, 55) => date = Some(decode(data, utf8)), + (2, 60) => time = Some(decode(data, utf8)), + _ => {} + } + i = at + size; + } + if let Some(d) = date.filter(|d| d.len() == 8 && d.bytes().all(|c| c.is_ascii_digit())) { + let mut s = format!("{}-{}-{}", &d[..4], &d[4..6], &d[6..8]); + if let Some(t) = time.filter(|t| t.is_ascii() && t.len() >= 6 && t[..6].bytes().all(|c| c.is_ascii_digit())) { + s += &format!("T{}:{}:{}", &t[..2], &t[2..4], &t[4..6]); + if t.len() >= 11 { + s += &format!("{}:{}", &t[6..9], &t[9..11]); + } + } + m.capture_time = DateTime::parse_iso(&s); + } + m +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + fn ds(out: &mut Vec, rec: u8, ds: u8, data: &[u8]) { + out.extend_from_slice(&[0x1c, rec, ds]); + out.extend_from_slice(&(data.len() as u16).to_be_bytes()); + out.extend_from_slice(data); + } + + #[test] + fn distinct_keywords_are_kept_once_each_without_rescanning() { + let mut b = Vec::new(); + for i in 0..50_000u32 { + ds(&mut b, 2, 25, format!("k{i}").as_bytes()); + ds(&mut b, 2, 25, b"same"); + } + let m = parse_iptc(&b); + assert_eq!(m.keywords.len(), 50_001); + assert_eq!(m.keywords.iter().filter(|k| *k == "same").count(), 1); + } + + pub(crate) fn sample_iim() -> Vec { + let mut v = Vec::new(); + ds(&mut v, 1, 90, b"\x1b%G"); + ds(&mut v, 2, 0, &[0, 4]); + ds(&mut v, 2, 5, "Tïtle".as_bytes()); + ds(&mut v, 2, 25, b"alpha"); + ds(&mut v, 2, 25, b"beta"); + ds(&mut v, 2, 25, b"alpha"); + ds(&mut v, 2, 80, b"Photographer"); + ds(&mut v, 2, 116, b"CC0"); + ds(&mut v, 2, 120, b"A caption"); + ds(&mut v, 2, 55, b"20210304"); + ds(&mut v, 2, 60, b"101112+0100"); + v + } + + #[test] + fn parses_record2() { + let m = parse_iptc(&sample_iim()); + assert_eq!(m.title.as_deref(), Some("Tïtle")); + assert_eq!(m.keywords, vec!["alpha".to_string(), "beta".to_string()]); + assert_eq!(m.artist.as_deref(), Some("Photographer")); + assert_eq!(m.copyright.as_deref(), Some("CC0")); + assert_eq!(m.caption.as_deref(), Some("A caption")); + assert_eq!(m.capture_time.unwrap().to_iso(), "2021-03-04T10:11:12+01:00"); + } + + #[test] + fn non_utf8_text_drops_trailing_nul() { + let mut v = Vec::new(); + ds(&mut v, 2, 5, &[0x43, 0xe9, 0x00]); + ds(&mut v, 2, 25, &[0x6b, 0xe9, 0x00]); + let m = parse_iptc(&v); + assert_eq!(m.title.as_deref(), Some("Cé")); + assert_eq!(m.keywords, vec!["ké".to_string()]); + let mut u = Vec::new(); + ds(&mut u, 1, 90, b"\x1b%G"); + ds(&mut u, 2, 5, &[0x43, 0xff, 0x00]); + assert_eq!(parse_iptc(&u).title.as_deref(), Some("C\u{fffd}")); + } + + #[test] + fn latin1_and_extended_and_garbage() { + let mut v = Vec::new(); + ds(&mut v, 2, 5, &[0x43, 0xe9]); // "Cé" in Latin-1 + v.extend_from_slice(&[0x1c, 2, 120, 0x80, 0x02, 0x00, 0x03]); + v.extend_from_slice(b"abc"); + let m = parse_iptc(&v); + assert_eq!(m.title.as_deref(), Some("Cé")); + assert_eq!(m.caption.as_deref(), Some("abc")); + let s = sample_iim(); + for n in 0..s.len() { + let _ = parse_iptc(&s[..n]); + } + assert_eq!(parse_iptc(&[0x1c, 2, 5, 0xff, 0xff]), Metadata::default()); + } +} diff --git a/crates/metadata/src/lib.rs b/crates/metadata/src/lib.rs new file mode 100644 index 000000000..e485e4a85 --- /dev/null +++ b/crates/metadata/src/lib.rs @@ -0,0 +1,606 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Photo metadata: EXIF, IPTC-IIM, XMP packets and GPX track logs, read into one [`Metadata`] model. +//! +//! One module per format: +//! - EXIF: [`read_exif`] reads a TIFF-structured Exif block, [`from_tiff`] an already parsed stream, +//! [`write_exif`] writes one. Maker notes supply the lens and, in [`focus`], where the camera focused. +//! [`tags`] lists every tag by name. +//! - IPTC-IIM: [`parse_iptc`] reads record 2 datasets. +//! - XMP: [`parse_xmp`] and [`write_xmp`] read and write packets; [`merge_xmp`] updates a packet +//! another program wrote without losing what it does not understand. +//! - GPX: [`parse_gpx`] reads track logs for geotagging by capture time. +//! - Containers: [`embedded`], [`jpeg_segments`], [`png_chunks`] and [`webp_chunks`] locate the Exif, +//! XMP, ICC and IPTC blocks of JPEG, PNG, WebP, TIFF-based, Fujifilm RAF and Canon CR3 ([`cr3`]) files. +//! [`extract`] does all of the above for a whole file. +//! +//! HEIF, PSD and MRW files are read by their own crates. Put the blocks they return into an +//! [`Embedded`] and call [`from_embedded`]. + +mod container; +pub mod cr3; +mod datetime; +mod exif; +pub mod focus; +mod focus_read; +mod geom; +mod gpx; +mod iptc; +pub mod tags; +mod xmp; +mod xmp_merge; + +pub use container::{Embedded, embedded, jpeg_segments, png_chunks, webp_chunks}; +pub use datetime::DateTime; +pub use exif::{from_tiff, read_exif, strip_exif_header, try_read_exif, try_read_exif_with, write_exif}; +pub use focus::FocusPoint; +pub use geom::{Orientation, Point, Rect}; +pub use gpx::{GpxError, Match, TrackPoint, Tracklog, parse_gpx}; +pub use iptc::parse_iptc; +pub use tags::{TagRow, file_tag_rows, tag_rows}; +pub use xmp::{CRS_NS, XmpData, XmpError, XmpNamespace, XmpValue, XmpWrite, parse_xmp, parse_xmp_with, write_xmp, write_xmp_with}; +pub use xmp_merge::{MergeRules, merge_xmp}; + +use serde::{Deserialize, Serialize}; + +/// GPS position (WGS 84). Latitude/longitude in signed decimal degrees (north / east positive). +#[derive(Clone, Copy, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Gps { + pub latitude: f64, + pub longitude: f64, + /// Metres above sea level (negative below). + pub altitude: Option, +} + +/// Exif `Flash` value decoded. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Flash { + pub fired: bool, + /// The raw Exif `Flash` bit field. + pub raw: u16, +} + +/// A named region of interest on a photo (MWG Region Guidelines, `mwg-rs:Regions`): most often a face, +/// drawn by Lightroom, digiKam, Picasa or similar tools. Read only: [`write_xmp`] does not write regions. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Region { + /// Normalized to the image's full, oriented frame (`Metadata::width`/`height`), y-down, 0..1 on + /// both axes — MWG's own convention, and what `AppliedToDimensions` areas are converted to. + pub rect: Rect, + pub kind: RegionKind, + /// `mwg-rs:Name` (e.g. the person's name). + pub name: Option, + pub description: Option, +} + +/// `mwg-rs:Type`. Unrecognised values are kept verbatim rather than dropped. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum RegionKind { + Face, + Pet, + Focus, + BarCode, + /// `mwg-rs:Type` was missing, empty, or a value we don't recognise. + Other(String), +} + +/// What a photo catalogue shows or searches about a photo. All fields are optional; unknown = `None`/empty. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Metadata { + // camera + pub make: Option, + pub model: Option, + pub serial_number: Option, + pub software: Option, + // lens + pub lens_make: Option, + pub lens_model: Option, + pub lens_serial_number: Option, + /// Min/max focal length (mm) and min/max f-number at those, as in Exif `LensSpecification`. + pub lens_spec: Option<[f64; 4]>, + // capture + pub capture_time: Option, + /// Seconds. + pub exposure_time: Option, + pub f_number: Option, + pub iso: Option, + /// Millimetres. + pub focal_length: Option, + pub focal_length_35mm: Option, + pub flash: Option, + /// EV. + pub exposure_bias: Option, + /// Exif `ExposureProgram` code. + pub exposure_program: Option, + /// Exif `MeteringMode` code. + pub metering_mode: Option, + /// Exif `WhiteBalance` (0 auto, 1 manual). + pub white_balance: Option, + pub orientation: Option, + pub gps: Option, + /// Pixel dimensions as recorded (not oriented). + pub width: Option, + pub height: Option, + // descriptive / user + pub artist: Option, + pub copyright: Option, + /// Copyright status (`xmpRights:Marked`): `true` = copyrighted, `false` = public domain, `None` = unknown. + pub copyright_marked: Option, + /// Rights usage terms (`xmpRights:UsageTerms`). + pub usage_terms: Option, + /// Copyright info URL (`xmpRights:WebStatement`). + pub copyright_url: Option, + pub title: Option, + pub caption: Option, + /// Accessibility text (`Iptc4xmpCore:AltTextAccessibility`). + pub alt_text: Option, + /// Long accessibility description (`Iptc4xmpCore:ExtDescrAccessibility`). + pub extended_description: Option, + /// Place within the city (`Iptc4xmpCore:Location`). + pub sublocation: Option, + /// `photoshop:City`, `photoshop:State`, `photoshop:Country`. + pub city: Option, + pub state: Option, + pub country: Option, + pub keywords: Vec, + /// Hierarchical keywords, `|`-separated paths (`lr:hierarchicalSubject` convention). + pub hierarchical_keywords: Vec, + /// -1 = rejected, 0 = unrated, 1..=5 stars. + pub rating: Option, + /// Colour label name (e.g. "Red"). + pub label: Option, + /// Face/pet/focus/barcode regions (`mwg-rs:Regions`), read from XMP only. + pub regions: Vec, + /// Where the camera focused, from its maker notes ([`focus`]; Sony only for now), on the upright photo. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub focus: Option, +} + +macro_rules! fill { + ($self:ident, $o:ident, $($f:ident),*) => { $( if $self.$f.is_none() { $self.$f = $o.$f.clone(); } )* }; +} +macro_rules! overlay { + ($self:ident, $o:ident, $($f:ident),*) => { $( if $o.$f.is_some() { $self.$f = $o.$f.clone(); } )* }; +} + +impl Metadata { + /// Fill every empty field of `self` from `other`. + pub fn fill_missing(&mut self, other: &Metadata) { + fill!( + self, + other, + make, + model, + serial_number, + software, + lens_make, + lens_model, + lens_serial_number, + lens_spec, + capture_time, + exposure_time, + f_number, + iso, + focal_length, + focal_length_35mm, + flash, + exposure_bias, + exposure_program, + metering_mode, + white_balance, + orientation, + gps, + width, + height + ); + self.fill_missing_descriptive(other); + } + + /// Fill descriptive metadata separately from the camera and image properties. + fn fill_missing_descriptive(&mut self, other: &Metadata) { + fill!( + self, + other, + artist, + copyright, + copyright_marked, + usage_terms, + copyright_url, + title, + caption, + alt_text, + extended_description, + sublocation, + city, + state, + country, + rating, + label + ); + if self.keywords.is_empty() { + self.keywords = other.keywords.clone(); + } + if self.hierarchical_keywords.is_empty() { + self.hierarchical_keywords = other.hierarchical_keywords.clone(); + } + if self.regions.is_empty() { + self.regions = other.regions.clone(); + } + if self.focus.is_none() { + self.focus = other.focus.clone(); + } + } + + /// Replace the user-editable fields (rating, label, title, caption, artist, copyright, keywords, GPS) with + /// `other`'s where `other` has them — the XMP-over-EXIF precedence rule. + pub fn overlay_user_fields(&mut self, other: &Metadata) { + overlay!(self, other, rating, label, title, caption, artist, copyright, copyright_marked, usage_terms, copyright_url, gps); + if !other.keywords.is_empty() { + self.keywords = other.keywords.clone(); + } + if !other.hierarchical_keywords.is_empty() { + self.hierarchical_keywords = other.hierarchical_keywords.clone(); + } + if !other.regions.is_empty() { + self.regions = other.regions.clone(); + } + } + + /// Exposure time formatted like cameras show it: `1/250`, `0.5`, `2″`. + pub fn exposure_display(&self) -> Option { + let t = self.exposure_time?; + if t.is_nan() || t <= 0.0 { + return None; + } + Some(if t < 0.3 { + format!("1/{}", (1.0 / t).round()) + } else if t < 1.0 { + format!("{}", (t * 10.0).round() / 10.0) + } else { + format!("{}″", (t * 10.0).round() / 10.0) + }) + } + + /// Human name of the metering mode. + pub fn metering_name(&self) -> Option<&'static str> { + Some(match self.metering_mode? { + 1 => "Average", + 2 => "Center-weighted average", + 3 => "Spot", + 4 => "Multi-spot", + 5 => "Pattern", + 6 => "Partial", + 255 => "Other", + _ => "Unknown", + }) + } + + /// Human name of the exposure program. + pub fn exposure_program_name(&self) -> Option<&'static str> { + Some(match self.exposure_program? { + 1 => "Manual", + 2 => "Program AE", + 3 => "Aperture-priority AE", + 4 => "Shutter speed priority AE", + 5 => "Creative (slow speed)", + 6 => "Action (high speed)", + 7 => "Portrait", + 8 => "Landscape", + 9 => "Bulb", + _ => "Not defined", + }) + } +} + +/// Read everything understood from a whole file (JPEG, PNG, WebP, TIFF-based incl. DNG and most raws, RAF, CR3). +/// Precedence: EXIF, then IPTC fills gaps, then XMP overrides user fields and fills the rest. +pub fn extract(bytes: &[u8]) -> Metadata { + from_embedded(&embedded(bytes)) +} + +/// [`extract`] for blocks that are already located, for example by a HEIF or PSD reader. Best effort: +/// a block that cannot be read counts as absent. A caller that must tell "no location" from +/// "unreadable" uses [`try_read_exif`] and [`parse_xmp`] on the blocks first. +pub fn from_embedded(e: &Embedded) -> Metadata { + let mut m = e.exif.as_deref().map(read_exif).unwrap_or_default(); + let xmp = e.xmp.as_deref().and_then(|xmp| parse_xmp(xmp).ok()); + // the size the Exif states: a size only the XMP states may be the upright one, so it can't tell how the photo is stored + let stored = m.width.zip(m.height); + // and the orientation the Exif states: the photo is displayed by that alone, so an orientation only the XMP + // states ([`Metadata::fill_missing`] reports it below) can't tell how the photo is stored either + let exif_orientation = e.exif.as_deref().and_then(exif::ifd0_orientation); + if let Some(i) = e.iptc.as_deref() { + m.fill_missing(&parse_iptc(i)); + } + if let Some(xmp) = xmp { + m.overlay_user_fields(&xmp.metadata); + m.fill_missing(&xmp.metadata); + // boxes another app drew on the photo as stored go where the upright photo has them + if let (Some(drawn_on), Some(orientation), Some(stored)) = (xmp.regions_drawn_on(), exif_orientation, stored) { + upright_regions(&mut m.regions, drawn_on, orientation, stored); + } + } + m +} + +/// Turn regions drawn on a photo as it is stored onto the upright photo. MWG gives each box as a fraction of +/// `AppliedToDimensions` (`drawn_on`); for a photo whose Exif `orientation` turns it a quarter turn, some apps give the +/// stored, sideways size there and boxes on the stored pixels to match (Lightroom says so per box with +/// `mwg-rs:Rotation` instead, which [`parse_xmp`] reads). When `drawn_on` has the shape of the photo as stored +/// (`stored`, before `orientation`) rather than the upright one, every box is turned with `orientation`. Returns +/// whether they were. A square photo or `drawn_on` looks the same both ways, and a half turn or a mirror doesn't +/// change the shape: their boxes are left as they are. +pub fn upright_regions(regions: &mut [Region], drawn_on: (f64, f64), orientation: Orientation, stored: (u32, u32)) -> bool { + let ((dw, dh), (sw, sh)) = (drawn_on, stored); + if regions.is_empty() || !orientation.swaps_axes() || sw == sh || dw == dh || (dw > dh) != (sw > sh) { + return false; + } + for r in regions.iter_mut() { + r.rect = orientation.map_norm_rect(r.rect); + } + true +} + +/// A lens name from its focal and aperture range (Exif LensSpecification, or a Nikon maker note's Lens), written the +/// way Lightroom shows a lens it has no name for: `50.0 mm f/1.8`, `18.0-135.0 mm f/3.5-5.6`. +pub fn lens_spec_name(spec: [f64; 4]) -> Option { + let [short, long, wide, narrow] = spec; + if ![short, long, wide, narrow].iter().all(|v| v.is_finite() && *v > 0.0) || long < short { + return None; + } + let focal = if (long - short).abs() < 0.05 { format!("{short:.1}") } else { format!("{short:.1}-{long:.1}") }; + let aperture = if (narrow - wide).abs() < 0.05 { format!("{wide:.1}") } else { format!("{wide:.1}-{narrow:.1}") }; + Some(format!("{focal} mm f/{aperture}")) +} + +/// Lenient number parsing used by XMP/IPTC: `"28/10"`, `"2.8"`, `"+0.7"`. +pub(crate) fn parse_number(s: &str) -> Option { + let s = s.trim(); + if let Some((n, d)) = s.split_once('/') { + let (n, d) = (n.trim().parse::().ok()?, d.trim().parse::().ok()?); + return (d != 0.0).then_some(n / d).filter(|v| v.is_finite()); + } + s.parse::().ok().filter(|v| v.is_finite()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn lens_names_from_a_spec_read_like_lightroom() { + assert_eq!(lens_spec_name([50.0, 50.0, 1.8, 1.8]).as_deref(), Some("50.0 mm f/1.8")); + assert_eq!(lens_spec_name([18.0, 135.0, 3.5, 5.6]).as_deref(), Some("18.0-135.0 mm f/3.5-5.6")); + assert_eq!(lens_spec_name([24.0, 70.0, 2.8, 2.8]).as_deref(), Some("24.0-70.0 mm f/2.8")); + assert_eq!(lens_spec_name([70.0, 210.0, 4.0, 5.6]).as_deref(), Some("70.0-210.0 mm f/4.0-5.6")); + assert_eq!(lens_spec_name([0.0, 0.0, 0.0, 0.0]), None); + assert_eq!(lens_spec_name([200.0, 80.0, 2.8, 2.8]), None); + assert_eq!(lens_spec_name([f64::NAN, 50.0, 1.8, 1.8]), None); + } + + #[test] + fn numbers() { + assert_eq!(parse_number("28/10"), Some(2.8)); + assert_eq!(parse_number(" +0.7 "), Some(0.7)); + assert_eq!(parse_number("1/0"), None); + assert_eq!(parse_number("abc"), None); + assert_eq!(parse_number("inf"), None); + } + + #[test] + fn exposure_display() { + let mut m = Metadata { exposure_time: Some(1.0 / 250.0), ..Default::default() }; + assert_eq!(m.exposure_display().as_deref(), Some("1/250")); + m.exposure_time = Some(0.5); + assert_eq!(m.exposure_display().as_deref(), Some("0.5")); + m.exposure_time = Some(2.0); + assert_eq!(m.exposure_display().as_deref(), Some("2″")); + m.exposure_time = Some(0.0); + assert_eq!(m.exposure_display(), None); + } + + #[test] + fn merge_rules() { + let mut a = Metadata { make: Some("A".into()), title: Some("Original".into()), rating: Some(1), ..Default::default() }; + let b = Metadata { + make: Some("B".into()), + model: Some("M".into()), + title: Some("Replacement".into()), + caption: Some("Caption".into()), + rating: Some(4), + keywords: vec!["k".into()], + ..Default::default() + }; + a.fill_missing(&b); + assert_eq!(a.make.as_deref(), Some("A")); + assert_eq!(a.model.as_deref(), Some("M")); + assert_eq!(a.title.as_deref(), Some("Original")); + assert_eq!(a.caption.as_deref(), Some("Caption")); + assert_eq!(a.keywords, vec!["k".to_string()]); + assert_eq!(a.rating, Some(1)); + a.overlay_user_fields(&b); + assert_eq!(a.rating, Some(4)); + assert_eq!(a.make.as_deref(), Some("A")); + assert_eq!(a.keywords, vec!["k".to_string()]); + } + + #[test] + fn names() { + let m = Metadata { metering_mode: Some(5), exposure_program: Some(3), ..Default::default() }; + assert_eq!(m.metering_name(), Some("Pattern")); + assert_eq!(m.exposure_program_name(), Some("Aperture-priority AE")); + } + + /// An MWG regions packet with one face, its box centred at (0.25, 0.4) and 0.1 × 0.15 of `AppliedToDimensions` + /// `w` × `h`; `extra` goes on the region (e.g. Lightroom's `mwg-rs:Rotation`). + fn regions_packet(w: u32, h: u32, extra: &str) -> String { + format!( + r#" + + + + + + + + + + "# + ) + } + + fn near(r: Rect, b: [f64; 4]) -> bool { + [r.x0 - b[0], r.y0 - b[1], r.x1 - b[2], r.y1 - b[3]].iter().all(|d| d.abs() < 1e-9) + } + + /// The face's box as drawn: on the photo as stored, or on the upright one by apps that draw there. + const DRAWN: [f64; 4] = [0.2, 0.325, 0.3, 0.475]; + /// The same box on the photo turned upright by Exif orientation 6 (a quarter turn clockwise). + const TURNED_6: [f64; 4] = [0.525, 0.2, 0.675, 0.3]; + + #[test] + fn boxes_drawn_on_the_stored_photo_turn_with_its_orientation() { + let region = || Region { + rect: Rect { x0: DRAWN[0], y0: DRAWN[1], x1: DRAWN[2], y1: DRAWN[3] }, + kind: RegionKind::Face, + name: None, + description: None, + }; + let turned = |o: Orientation, drawn_on: (f64, f64), stored: (u32, u32)| { + let mut regions = [region()]; + let did = upright_regions(&mut regions, drawn_on, o, stored); + (did, regions[0].rect) + }; + // drawn on the 6000 × 4000 photo as stored: every quarter turn and its mirror image land elsewhere + for (o, want) in [ + (Orientation::Rotate90, TURNED_6), + (Orientation::Rotate270, [0.325, 0.7, 0.475, 0.8]), + (Orientation::Transpose, [0.325, 0.2, 0.475, 0.3]), + (Orientation::Transverse, [0.525, 0.7, 0.675, 0.8]), + ] { + let (did, r) = turned(o, (6000.0, 4000.0), (6000, 4000)); + assert!(did && near(r, want), "{o:?}: {r:?}"); + // a smaller copy of the photo as stored is the same frame + assert!(near(turned(o, (1500.0, 1000.0), (6000, 4000)).1, want), "{o:?} on a smaller size"); + } + // drawn on the upright photo (MWG's own frame), or where the shape can't tell the two apart: as they are + for (o, drawn_on, stored) in [ + (Orientation::Rotate90, (4000.0, 6000.0), (6000, 4000)), + (Orientation::Rotate90, (4000.0, 4000.0), (6000, 4000)), + (Orientation::Rotate90, (4000.0, 4000.0), (4000, 4000)), + (Orientation::Rotate180, (6000.0, 4000.0), (6000, 4000)), + (Orientation::FlipH, (6000.0, 4000.0), (6000, 4000)), + (Orientation::Normal, (6000.0, 4000.0), (6000, 4000)), + ] { + let (did, r) = turned(o, drawn_on, stored); + assert!(!did && near(r, DRAWN), "{o:?} drawn on {drawn_on:?} of {stored:?}: {r:?}"); + } + } + + /// An app that doesn't say how the photo was turned (no `mwg-rs:Rotation`) drew the faces of a photo + /// with Exif orientation 6 on its stored pixels, with the stored 6000 × 4000 as `AppliedToDimensions`. The boxes + /// land on the faces of the upright photo, not where the stored photo had them. + #[test] + fn regions_drawn_on_the_stored_photo_are_read_onto_the_upright_one() { + let jpeg = |packet: &str| { + let mut j = vec![0xff, 0xd8]; + let mut seg = |payload: Vec| { + j.extend([0xff, 0xe1]); + j.extend(u16::try_from(payload.len() + 2).unwrap().to_be_bytes()); + j.extend(payload); + }; + // orientation 6, 6000 × 4000 as stored + seg([b"Exif\0\0".as_slice(), &crate::exif::tests::sample_exif(craft_tiff::ByteOrder::Little)].concat()); + seg([b"http://ns.adobe.com/xap/1.0/\0".as_slice(), packet.as_bytes()].concat()); + j.extend([0xff, 0xd9]); + j + }; + let read = |packet: &str| { + let m = extract(&jpeg(packet)); + assert_eq!((m.orientation, m.width, m.height), (Some(Orientation::Rotate90), Some(6000), Some(4000))); + assert_eq!(from_embedded(&embedded(&jpeg(packet))).regions, m.regions); + assert_eq!(m.regions.len(), 1); + m.regions[0].rect + }; + let stored = read(®ions_packet(6000, 4000, "")); + assert!(near(stored, TURNED_6), "drawn on the photo as stored: {stored:?}"); + let upright = read(®ions_packet(4000, 6000, "")); + assert!(near(upright, DRAWN), "drawn on the upright photo, MWG's way: {upright:?}"); + // Lightroom's boxes say how the photo was turned: turned once, by that + let lightroom = read(®ions_packet(6000, 4000, r#" mwg-rs:Rotation="-1.57080""#)); + assert!(near(lightroom, TURNED_6), "Lightroom's −π/2: {lightroom:?}"); + } + + /// A size only the XMP states can't tell how the photo is stored: `tiff:ImageWidth` and `tiff:ImageLength` of the + /// upright photo have the shape of boxes drawn on it upright, and those must not be turned. + #[test] + fn a_size_only_the_xmp_states_turns_no_regions() { + // orientation 6, no pixel size + let exif = write_exif(&Metadata { orientation: Some(Orientation::Rotate90), ..Default::default() }); + let exif = strip_exif_header(&exif); + let packet = regions_packet(4000, 6000, "") + .replace(r#"rdf:about="""#, r#"rdf:about="" xmlns:tiff="http://ns.adobe.com/tiff/1.0/" tiff:ImageWidth="4000" tiff:ImageLength="6000""#); + let mut j = vec![0xff, 0xd8]; + let mut seg = |payload: Vec| { + j.extend([0xff, 0xe1]); + j.extend(u16::try_from(payload.len() + 2).unwrap().to_be_bytes()); + j.extend(payload); + }; + seg([b"Exif\0\0".as_slice(), exif].concat()); + seg([b"http://ns.adobe.com/xap/1.0/\0".as_slice(), packet.as_bytes()].concat()); + j.extend([0xff, 0xd9]); + let m = extract(&j); + assert_eq!((m.orientation, m.width, m.height), (Some(Orientation::Rotate90), Some(4000), Some(6000))); + assert_eq!(m.regions.len(), 1); + assert!(near(m.regions[0].rect, DRAWN), "{:?}", m.regions[0].rect); + assert_eq!(from_embedded(&embedded(&j)).regions, m.regions); + // a TIFF-based file: the whole file is the Exif block, with the XMP in its first IFD + let mut ifd0 = craft_tiff::IfdBuilder::new(); + ifd0.set(craft_tiff::tags::ORIENTATION, craft_tiff::Value::Short(vec![6])); + ifd0.set(craft_tiff::tags::XMP, craft_tiff::Value::Byte(packet.into_bytes())); + let m = extract(&craft_tiff::TiffWriter::default().write(&[ifd0]).unwrap()); + assert_eq!((m.orientation, m.width, m.height), (Some(Orientation::Rotate90), Some(4000), Some(6000))); + assert!(near(m.regions[0].rect, DRAWN), "TIFF: {:?}", m.regions[0].rect); + } + + /// A photo whose Exif has a size but no orientation, and whose XMP says `tiff:Orientation=6`: a viewer + /// displays it by the Exif alone, so it stays as stored, and boxes drawn on that stored frame are right as + /// written. Turning them by an orientation only the XMP states put them where the photo isn't. + #[test] + fn an_orientation_only_the_xmp_states_turns_no_regions() { + // 6000 × 4000 in the Exif, and no orientation + let size_ifd = || { + let mut sub = craft_tiff::IfdBuilder::new(); + sub.set(craft_tiff::tags::PIXEL_X_DIMENSION, craft_tiff::Value::Long(vec![6000])); + sub.set(craft_tiff::tags::PIXEL_Y_DIMENSION, craft_tiff::Value::Long(vec![4000])); + sub + }; + let mut exif_ifd0 = craft_tiff::IfdBuilder::new(); + exif_ifd0.set_child(craft_tiff::tags::EXIF_IFD, size_ifd()); + let exif = craft_tiff::TiffWriter::new(craft_tiff::ByteOrder::Little, false).write(&[exif_ifd0]).unwrap(); + let packet = regions_packet(6000, 4000, "") + .replace(r#"rdf:about="""#, r#"rdf:about="" xmlns:tiff="http://ns.adobe.com/tiff/1.0/" tiff:Orientation="6""#); + let mut j = vec![0xff, 0xd8]; + let mut seg = |payload: Vec| { + j.extend([0xff, 0xe1]); + j.extend(u16::try_from(payload.len() + 2).unwrap().to_be_bytes()); + j.extend(payload); + }; + seg([b"Exif\0\0".as_slice(), &exif].concat()); + seg([b"http://ns.adobe.com/xap/1.0/\0".as_slice(), packet.as_bytes()].concat()); + j.extend([0xff, 0xd9]); + let m = extract(&j); + assert_eq!((m.width, m.height), (Some(6000), Some(4000))); + assert_eq!(m.orientation, Some(Orientation::Rotate90), "the XMP's orientation is still reported"); + assert_eq!(m.regions.len(), 1); + assert!(near(m.regions[0].rect, DRAWN), "turned by an orientation the Exif doesn't state: {:?}", m.regions[0].rect); + assert_eq!(from_embedded(&embedded(&j)).regions, m.regions); + // a TIFF-based file: the first IFD has no orientation, its XMP (tag 700) has + let mut ifd0 = craft_tiff::IfdBuilder::new(); + ifd0.set(craft_tiff::tags::XMP, craft_tiff::Value::Byte(packet.into_bytes())); + ifd0.set_child(craft_tiff::tags::EXIF_IFD, size_ifd()); + let m = extract(&craft_tiff::TiffWriter::default().write(&[ifd0]).unwrap()); + assert_eq!(m.orientation, Some(Orientation::Rotate90)); + assert!(near(m.regions[0].rect, DRAWN), "TIFF: {:?}", m.regions[0].rect); + } +} diff --git a/crates/metadata/src/tags.rs b/crates/metadata/src/tags.rs new file mode 100644 index 000000000..6e5930922 --- /dev/null +++ b/crates/metadata/src/tags.rs @@ -0,0 +1,529 @@ +//! Every EXIF / TIFF / GPS tag of a file as readable `(group, name, value)` rows (the Info +//! panel's "All Metadata"). Names follow the public TIFF 6.0 / EXIF 2.3 / DNG specifications; +//! common enumerations are spelled out, everything else is shown as stored. Maker notes and +//! binary blobs are summarised by size; of a maker note only the lens name is listed (the +//! Panasonic one, as Panasonic writes no Exif LensModel). + +use craft_tiff::{Ifd, Tiff, Value}; + +/// One metadata row. +#[derive(Clone, Debug, PartialEq)] +pub struct TagRow { + /// `TIFF`, `EXIF`, `GPS`, `Interop`, or a maker note's (`Panasonic`). + pub group: &'static str, + pub tag: u16, + pub name: String, + pub value: String, +} + +const NAMES: &[(u16, &str)] = &[ + // TIFF / IFD0 + (0x00fe, "Subfile Type"), + (0x0100, "Image Width"), + (0x0101, "Image Height"), + (0x0102, "Bits Per Sample"), + (0x0103, "Compression"), + (0x0106, "Photometric Interpretation"), + (0x010e, "Image Description"), + (0x010f, "Make"), + (0x0110, "Model"), + (0x0112, "Orientation"), + (0x0115, "Samples Per Pixel"), + (0x0116, "Rows Per Strip"), + (0x0214, "Reference Black/White"), + (0x9216, "TIFF/EP Standard ID"), + (0x828d, "CFA Repeat Pattern Dim"), + (0x828e, "CFA Pattern"), + (0xa302, "CFA Pattern"), + (0x9217, "Sensing Method"), + (0xc4a5, "Print Image Matching"), + (0xa500, "Gamma"), + (0x011a, "X Resolution"), + (0x011b, "Y Resolution"), + (0x011c, "Planar Configuration"), + (0x0128, "Resolution Unit"), + (0x0131, "Software"), + (0x0132, "Date/Time Modified"), + (0x013b, "Artist"), + (0x013e, "White Point"), + (0x013f, "Primary Chromaticities"), + (0x0213, "YCbCr Positioning"), + (0x8298, "Copyright"), + (0xc612, "DNG Version"), + (0xc614, "Unique Camera Model"), + (0xc62f, "Camera Serial Number"), + // EXIF + (0x829a, "Exposure Time"), + (0x829d, "F-Number"), + (0x8822, "Exposure Program"), + (0x8824, "Spectral Sensitivity"), + (0x8827, "ISO Speed"), + (0x8830, "Sensitivity Type"), + (0x8832, "Recommended Exposure Index"), + (0x9000, "Exif Version"), + (0x9003, "Date/Time Original"), + (0x9004, "Date/Time Digitized"), + (0x9010, "Offset Time"), + (0x9011, "Offset Time Original"), + (0x9012, "Offset Time Digitized"), + (0x9101, "Components Configuration"), + (0x9102, "Compressed Bits Per Pixel"), + (0x9201, "Shutter Speed Value"), + (0x9202, "Aperture Value"), + (0x9203, "Brightness Value"), + (0x9204, "Exposure Compensation"), + (0x9205, "Max Aperture Value"), + (0x9206, "Subject Distance"), + (0x9207, "Metering Mode"), + (0x9208, "Light Source"), + (0x9209, "Flash"), + (0x920a, "Focal Length"), + (0x9214, "Subject Area"), + (0x927c, "Maker Note"), + (0x9286, "User Comment"), + (0x9290, "Sub-second Time"), + (0x9291, "Sub-second Time Original"), + (0x9292, "Sub-second Time Digitized"), + (0xa000, "FlashPix Version"), + (0xa001, "Color Space"), + (0xa002, "Pixel X Dimension"), + (0xa003, "Pixel Y Dimension"), + (0xa20e, "Focal Plane X Resolution"), + (0xa20f, "Focal Plane Y Resolution"), + (0xa210, "Focal Plane Resolution Unit"), + (0xa215, "Exposure Index"), + (0xa217, "Sensing Method"), + (0xa300, "File Source"), + (0xa301, "Scene Type"), + (0xa401, "Custom Rendered"), + (0xa402, "Exposure Mode"), + (0xa403, "White Balance"), + (0xa404, "Digital Zoom Ratio"), + (0xa405, "Focal Length in 35mm Film"), + (0xa406, "Scene Capture Type"), + (0xa407, "Gain Control"), + (0xa408, "Contrast"), + (0xa409, "Saturation"), + (0xa40a, "Sharpness"), + (0xa40c, "Subject Distance Range"), + (0xa420, "Image Unique ID"), + (0xa430, "Camera Owner Name"), + (0xa431, "Body Serial Number"), + (0xa432, "Lens Specification"), + (0xa433, "Lens Make"), + (0xa434, "Lens Model"), + (0xa435, "Lens Serial Number"), + // GPS + (0x0000, "GPS Version"), + (0x0001, "GPS Latitude Ref"), + (0x0002, "GPS Latitude"), + (0x0003, "GPS Longitude Ref"), + (0x0004, "GPS Longitude"), + (0x0005, "GPS Altitude Ref"), + (0x0006, "GPS Altitude"), + (0x0007, "GPS Time Stamp"), + (0x000c, "GPS Speed Ref"), + (0x000d, "GPS Speed"), + (0x0010, "GPS Img Direction Ref"), + (0x0011, "GPS Img Direction"), + (0x0012, "GPS Map Datum"), + (0x001d, "GPS Date Stamp"), +]; + +/// GPS tags reuse low numbers, so they are looked up separately. +fn name(group: &str, tag: u16) -> String { + let gps = group == "GPS"; + NAMES + .iter() + .find(|(t, n)| *t == tag && (n.starts_with("GPS") == gps || (gps && tag > 0x1f))) + .map(|(_, n)| n.to_string()) + .unwrap_or_else(|| format!("Tag 0x{tag:04X}")) +} + +fn rational(n: f64, d: f64) -> String { + if d == 0.0 { + return "—".into(); + } + let v = n / d; + if (v - v.round()).abs() < 1e-9 { + let r = v.round(); + if r == 0.0 { "0".to_string() } else { format!("{}", r) } + } else { + let s = format!("{:.4}", v); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" { "0".to_string() } else { s.to_string() } + } +} + +fn list(v: &[T]) -> String { + list_with(v, T::to_string) +} + +/// The first 16 values of `v` written by `f`, and how many there are when that's not all of them. +fn list_with(v: &[T], f: impl Fn(&T) -> String) -> String { + let shown: Vec = v.iter().take(16).map(f).collect(); + if v.len() > 16 { format!("{} … ({} values)", shown.join(", "), v.len()) } else { shown.join(", ") } +} + +/// Spelled-out enumerations for the commonest tags. +fn enumerated(group: &str, tag: u16, v: i64) -> Option<&'static str> { + if group == "GPS" { + return None; + } + Some(match (tag, v) { + (0x0112, 1) => "Horizontal (normal)", + (0x0112, 3) => "Rotated 180°", + (0x0112, 6) => "Rotated 90° CW", + (0x0112, 8) => "Rotated 90° CCW", + (0x0112, 2) => "Mirrored horizontally", + (0x0112, 4) => "Mirrored vertically", + (0x0103, 1) => "Uncompressed", + (0x0103, 6) => "JPEG (old-style)", + (0x0103, 7) => "JPEG", + (0x0103, 8) => "Deflate", + (0x0103, 32773) => "PackBits", + (0x0103, 34892) => "Lossy JPEG", + (0x0103, 34713) => "Nikon NEF Compressed", + (0x0103, 32767) => "Sony ARW Compressed", + (0x0103, 52546) => "JPEG XL", + (0x0106, 0) => "WhiteIsZero", + (0x0106, 1) => "BlackIsZero", + (0x0106, 2) => "RGB", + (0x0106, 6) => "YCbCr", + (0x0106, 32803) => "Color Filter Array", + (0x0106, 34892) => "Linear Raw", + (0x011c, 1) => "Chunky", + (0x011c, 2) => "Planar", + (0x0128, 2) => "inches", + (0x0128, 3) => "centimetres", + (0x8822, 1) => "Manual", + (0x8822, 2) => "Program", + (0x8822, 3) => "Aperture priority", + (0x8822, 4) => "Shutter priority", + (0x9207, 2) => "Center-weighted average", + (0x9207, 3) => "Spot", + (0x9207, 5) => "Pattern", + (0xa001, 1) => "sRGB", + (0xa001, 0xffff) => "Uncalibrated", + (0xa402, 0) => "Auto", + (0xa402, 1) => "Manual", + (0xa402, 2) => "Auto bracket", + (0xa403, 0) => "Auto", + (0xa403, 1) => "Manual", + (0xa406, 0) => "Standard", + (0xa406, 1) => "Landscape", + (0xa406, 2) => "Portrait", + (0xa406, 3) => "Night", + (0x9209, 0) => "No flash", + (0x9209, 1) => "Fired", + (0x9209, 5) => "Fired, return not detected", + (0x9209, 7) => "Fired, return detected", + (0x9209, 9) => "On, fired", + (0x9209, 16) => "Off, did not fire", + (0x9209, 24) => "Auto, did not fire", + (0x9209, 25) => "Auto, fired", + (0x9209, 32) => "No flash function", + (0x9208, 0) => "Unknown", + (0x9208, 1) => "Daylight", + (0x9208, 2) => "Fluorescent", + (0x9208, 3) => "Tungsten", + (0x9208, 4) => "Flash", + _ => return None, + }) +} + +fn value_text(group: &str, tag: u16, v: &Value) -> String { + let single = |x: i64| enumerated(group, tag, x).map(str::to_string).unwrap_or_else(|| x.to_string()); + match v { + Value::Ascii(s) => s.trim().to_string(), + Value::Short(a) if a.len() == 1 => single(a[0] as i64), + Value::Long(a) if a.len() == 1 => single(a[0] as i64), + Value::Short(a) => list(a), + Value::Long(a) => list(a), + Value::SShort(a) => list(a), + Value::SLong(a) => list(a), + Value::Long8(a) => list(a), + Value::SLong8(a) => list(a), + Value::Float(a) => list(a), + Value::Double(a) => list(a), + Value::Rational(a) if tag == 0x829a && group != "GPS" && a.len() == 1 && a[0].1 != 0 => { + // exposure time as a fraction of a second + let (n, d) = a[0]; + let s = n as f64 / d as f64; + if s >= 1.0 || n == 0 { format!("{} s", rational(n as f64, d as f64)) } else { format!("1/{} s", (1.0 / s).round()) } + } + Value::Rational(a) if tag == 0x829d && group != "GPS" && a.len() == 1 => format!("f/{}", rational(a[0].0 as f64, a[0].1 as f64)), + Value::Rational(a) if tag == 0x920a && group != "GPS" && a.len() == 1 => format!("{} mm", rational(a[0].0 as f64, a[0].1 as f64)), + Value::Rational(a) => list_with(a, |(n, d)| rational(*n as f64, *d as f64)), + Value::SRational(a) => list_with(a, |(n, d)| rational(*n as f64, *d as f64)), + Value::Byte(b) | Value::Undefined(b) => { + // version tags and short printable values read as text + let printable = b.len() <= 64 && b.iter().all(|c| c.is_ascii_graphic() || *c == b' ' || *c == 0); + if printable && !b.is_empty() { + String::from_utf8_lossy(b).trim_end_matches('\0').trim().to_string() + } else if b.len() <= 8 { + list(b) + } else { + format!("({} bytes)", b.len()) + } + } + Value::SByte(a) => list(a), + Value::Ifd(a) => format!("(IFD at {})", list(a)), + Value::Ifd8(a) => format!("(IFD at {})", list(a)), + } +} + +fn rows_of(group: &'static str, ifd: &Ifd, out: &mut Vec) { + for e in &ifd.entries { + // pointers to other IFDs and image data are structure, not metadata + if matches!(e.tag, 0x8769 | 0x8825 | 0xa005 | 0x014a | 0x0111 | 0x0117 | 0x0144 | 0x0145 | 0x0201 | 0x0202 | 0x02bc | 0x83bb | 0x8773) + && group != "GPS" + { + continue; + } + out.push(TagRow { group, tag: e.tag, name: name(group, e.tag), value: value_text(group, e.tag, &e.value) }); + } +} + +/// The IFD holding a raw file's sensor data when it is not IFD0 (a colour-filter-array or linear-raw +/// image in a sub-IFD). IFD0 of such files describes a thumbnail, so its Compression is not the raw's. +fn raw_image_ifd(ifd0: &Ifd) -> Option<&Ifd> { + ifd0.sub_ifds.iter().find(|s| matches!(s.u32(0x0106), Some(32803 | 34892))) +} + +/// Nikon maker note `NEFCompression` (0x0093) spelled out, per the public Nikon tag documentation. +fn nef_compression(v: u32) -> Option<&'static str> { + Some(match v { + 1 => "Lossy (type 1)", + 2 => "Uncompressed", + 3 => "Lossless", + 4 => "Lossy (type 2)", + 5 => "Striped packed 12 bits", + 6 => "Uncompressed (reduced to 12 bit)", + 7 => "Unpacked 12 bits", + 8 => "Small", + 9 => "Packed 12 bits", + 10 => "Packed 14 bits", + 13 => "High Efficiency", + 14 => "High Efficiency*", + _ => return None, + }) +} + +/// Nikon maker note NEFCompression of a raw file, spelled out; `None` for other files or values. +fn nikon_nef_compression(tiff: &Tiff, bytes: &[u8]) -> Option<&'static str> { + let ifd0 = tiff.ifds.first()?; + let make = ifd0.string(0x010f)?; + if !make.to_ascii_uppercase().starts_with("NIKON") { + return None; + } + let e = ifd0.exif.as_ref()?.get(0x927c)?; + let mn = craft_tiff::makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make)?; + nef_compression(mn.ifd.u32(0x0093)?) +} + +/// The Panasonic maker note's lens name (LensType, `0x0051`) as a row, the name `photo.inspect` shows as the lens. +fn panasonic_lens_row(data: &[u8], tiff: &Tiff, out: &mut Vec) { + if let Some(lens) = crate::exif::panasonic_maker_note_lens(data, tiff) { + out.push(TagRow { group: "Panasonic", tag: 0x0051, name: "Lens Type".into(), value: lens.trim().to_string() }); + } +} + +/// Every tag of a parsed TIFF structure (a TIFF-based raw, a TIFF, or an EXIF block): IFD0, +/// then the EXIF, GPS and interoperability IFDs. A raw file's Compression row is the sensor +/// image's, not IFD0's thumbnail's. +pub fn tag_rows(tiff: &Tiff) -> Vec { + let mut out = Vec::new(); + if let Some(ifd0) = tiff.ifds.first() { + rows_of("TIFF", ifd0, &mut out); + if let Some(c) = raw_image_ifd(ifd0).and_then(|r| r.value(0x0103)) + && let Some(row) = out.iter_mut().find(|r| r.tag == 0x0103) + { + // lossless JPEG (DNG, CR2) on the sensor image; plain "JPEG" would read as lossy + row.value = if c.get_u64(0) == Some(7) { "Lossless JPEG".to_string() } else { value_text("TIFF", 0x0103, c) }; + } + if let Some(e) = &ifd0.exif { + rows_of("EXIF", e, &mut out); + if let Some(i) = &e.interop { + rows_of("Interop", i, &mut out); + } + } + if let Some(g) = &ifd0.gps { + rows_of("GPS", g, &mut out); + } + } + out +} + +/// [`tag_rows`] of a file's bytes: a TIFF-based file directly, else its embedded EXIF block +/// (JPEG, PNG, WebP, HEIF, PSD/PSB…). Empty when there is none. +pub fn file_tag_rows(bytes: &[u8]) -> Vec { + if Tiff::sniff(bytes).is_some() + && let Ok(t) = Tiff::parse(bytes) + { + let mut rows = tag_rows(&t); + if let Some(words) = nikon_nef_compression(&t, bytes) + && let Some(row) = rows.iter_mut().find(|r| r.group == "TIFF" && r.tag == 0x0103) + { + row.value = words.to_string(); + } + panasonic_lens_row(bytes, &t, &mut rows); + return rows; + } + let Some(exif) = crate::embedded(bytes).exif else { return Vec::new() }; + let data = crate::strip_exif_header(&exif); + let Ok(t) = Tiff::parse(data) else { return Vec::new() }; + let mut rows = tag_rows(&t); + panasonic_lens_row(data, &t, &mut rows); + rows +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn rational_has_no_trailing_dot() { + assert_eq!(rational(1.0, 100000.0), "0"); + assert_eq!(rational(-1.0, 100000.0), "0"); + assert_eq!(rational(-1.0, 1e10), "0"); + assert_eq!(rational(1000001.0, 100000.0), "10"); + assert_eq!(rational(1.0, 3.0), "0.3333"); + assert_eq!(rational(5.0, 2.0), "2.5"); + } + + /// Rational lists were joined whole, unlike every other list: a tag of millions of + /// fractions became a string as long in the tag panel. + #[test] + fn rational_lists_show_sixteen_values_like_other_lists() { + let first16 = (1..=16).map(|i| i.to_string()).collect::>().join(", "); + let many: Vec<(u32, u32)> = (1..=1000).map(|i| (i, 1)).collect(); + assert_eq!(value_text("EXIF", 0x9999, &Value::Rational(many)), format!("{first16} … (1000 values)")); + let signed: Vec<(i32, i32)> = (1..=1000).map(|i| (-i, 1)).collect(); + assert!(value_text("EXIF", 0x9999, &Value::SRational(signed)).ends_with("-16 … (1000 values)")); + assert_eq!(value_text("EXIF", 0x9999, &Value::Rational(vec![(1, 2), (3, 4)])), "0.5, 0.75"); + } + + #[test] + fn rows_from_written_exif() { + let m = crate::Metadata { + make: Some("Maker".into()), + model: Some("Model X".into()), + exposure_time: Some(1.0 / 250.0), + f_number: Some(2.8), + iso: Some(400), + focal_length: Some(35.0), + gps: Some(crate::Gps { latitude: 51.5, longitude: -0.12, altitude: Some(20.0) }), + ..Default::default() + }; + let exif = crate::write_exif(&m); + let t = Tiff::parse(crate::strip_exif_header(&exif)).unwrap(); + let rows = tag_rows(&t); + let get = |n: &str| rows.iter().find(|r| r.name == n).map(|r| r.value.clone()); + assert_eq!(get("Make").as_deref(), Some("Maker")); + assert_eq!(get("Exposure Time").as_deref(), Some("1/250 s")); + assert_eq!(get("F-Number").as_deref(), Some("f/2.8")); + assert_eq!(get("Focal Length").as_deref(), Some("35 mm")); + assert_eq!(get("ISO Speed").as_deref(), Some("400")); + assert!(rows.iter().any(|r| r.group == "GPS" && r.name == "GPS Latitude"), "{rows:?}"); + assert!(!rows.iter().any(|r| r.tag == 0x8769), "IFD pointers are left out"); + } + + fn entry(tag: u16, ty: u16, count: u32, val: u32) -> Vec { + let mut e = Vec::new(); + e.extend(tag.to_le_bytes()); + e.extend(ty.to_le_bytes()); + e.extend(count.to_le_bytes()); + e.extend(val.to_le_bytes()); + e + } + + fn ifd(entries: &[Vec]) -> Vec { + let mut b = (entries.len() as u16).to_le_bytes().to_vec(); + entries.iter().for_each(|e| b.extend(e)); + b.extend(0u32.to_le_bytes()); + b + } + + /// A little-endian raw with a thumbnail IFD0 (`Compression` 1), a CFA sub-IFD (`Compression` + /// 34713) and, when `nef` is set, a Nikon maker note whose NEFCompression is that value. + fn synthetic_raw(nef: Option) -> Vec { + let make = b"NIKON CORPORATION"; + let mut note = b"NikonII*".to_vec(); + note.extend(ifd(&[entry(0x0093, 3, 1, nef.unwrap_or(0) as u32)])); + let ifd0_len = 2 + 4 * 12 + 4; + let sub_at = 8 + ifd0_len; + let sub = ifd(&[entry(0x0103, 3, 1, 34713), entry(0x0106, 3, 1, 32803)]); + let exif_at = sub_at + sub.len(); + let exif_len = 2 + 12 + 4; + let make_at = exif_at + exif_len; + let note_at = make_at + make.len(); + let mut b = b"II*".to_vec(); + b.extend(ifd(&[ + entry(0x0103, 3, 1, 1), + entry(0x010f, 2, make.len() as u32, make_at as u32), + entry(0x014a, 4, 1, sub_at as u32), + entry(0x8769, 4, 1, exif_at as u32), + ])); + b.extend(sub); + b.extend(ifd(&[entry(0x927c, 7, note.len() as u32, if nef.is_some() { note_at as u32 } else { 0 })])); + b.extend(make); + b.extend(note); + b + } + + fn compression(rows: &[TagRow]) -> Vec<&str> { + rows.iter().filter(|r| r.group == "TIFF" && r.tag == 0x0103).map(|r| r.value.as_str()).collect() + } + + #[test] + fn raw_compression_is_the_sensor_images_not_the_thumbnails() { + let b = synthetic_raw(None); + assert_eq!(compression(&file_tag_rows(&b)), ["Nikon NEF Compressed"]); + } + + #[test] + fn nef_compression_is_spelled_out_from_the_maker_note() { + for (v, words) in [(3, "Lossless"), (10, "Packed 14 bits"), (2, "Uncompressed"), (14, "High Efficiency*")] { + let b = synthetic_raw(Some(v)); + assert_eq!(compression(&file_tag_rows(&b)), [words]); + } + } + + #[test] + fn dng_raw_sub_ifd_jpeg_is_lossless_but_a_plain_jpeg_tiff_is_not() { + let b = synthetic_raw(None); + // switch the sensor sub-IFD's Compression from 34713 to 7 + let pos = b.windows(12).position(|w| w == entry(0x0103, 3, 1, 34713).as_slice()).unwrap(); + let mut dng = b.clone(); + dng[pos + 8] = 7; + dng[pos + 9] = 0; + assert_eq!(compression(&file_tag_rows(&dng)), ["Lossless JPEG"]); + let plain = [b"II*".to_vec(), ifd(&[entry(0x0103, 3, 1, 7)])].concat(); + assert_eq!(compression(&file_tag_rows(&plain)), ["JPEG"]); + } + + #[test] + fn plain_tiff_keeps_its_own_compression() { + let b = [b"II*".to_vec(), ifd(&[entry(0x0103, 3, 1, 8)])].concat(); + assert_eq!(compression(&file_tag_rows(&b)), ["Deflate"]); + } + + #[test] + fn panasonic_lens_is_listed_from_the_maker_note() { + let lens = |rows: &[TagRow]| rows.iter().filter(|r| r.group == "Panasonic" && r.tag == 0x0051).map(|r| r.value.clone()).collect::>(); + // a RW2: the note is only in the JpgFromRaw's Exif + let rw2 = crate::exif::tests::panasonic_rw2("LUMIX S 85/F1.8"); + assert_eq!(lens(&file_tag_rows(&rw2)), ["LUMIX S 85/F1.8"]); + // a JPEG: the note is in its own Exif + let exif = crate::exif::tests::panasonic_exif(craft_tiff::ByteOrder::Big, Some("LUMIX G 25/F1.7"), None); + let mut jpg = vec![0xff, 0xd8, 0xff, 0xe1]; + jpg.extend_from_slice(&((exif.len() + 8) as u16).to_be_bytes()); + jpg.extend(b"Exif\0\0"); + jpg.extend(exif); + jpg.extend_from_slice(&[0xff, 0xda, 0, 2, 0xff, 0xd9]); + assert_eq!(lens(&file_tag_rows(&jpg)), ["LUMIX G 25/F1.7"]); + // no Panasonic note, no row + assert!(lens(&file_tag_rows(&synthetic_raw(Some(3)))).is_empty()); + } +} diff --git a/crates/metadata/src/xmp.rs b/crates/metadata/src/xmp.rs new file mode 100644 index 000000000..e2af65fb5 --- /dev/null +++ b/crates/metadata/src/xmp.rs @@ -0,0 +1,813 @@ +//! XMP packets (ISO 16684-1 / Adobe XMP Specification Part 1–2): read and write. +//! +//! Reading builds a small namespace-resolved tree with `quick-xml`, then collects every property of every +//! `rdf:Description` (attribute form, element form, `rdf:Seq`/`rdf:Bag`/`rdf:Alt` arrays, `rdf:parseType="Resource"` +//! and nested-description structs, flattened as `parent/field`). Prefixes are canonicalised by namespace URI, so +//! a packet that binds `http://purl.org/dc/elements/1.1/` to `foo:` still yields `dc:title`. +//! +//! Writing emits the interchange subset in standard namespaces (dc, xmp, photoshop, exif, exifEX, tiff, lr) and +//! any simple properties of one further namespace the caller names ([`XmpWrite`]). + +use crate::geom::{Point, Rect}; +use crate::{DateTime, Flash, Gps, Metadata, Orientation, Region, RegionKind, parse_number}; +use quick_xml::escape::{escape, partial_escape}; +use quick_xml::events::Event; +use std::collections::BTreeMap; + +/// A namespace beyond the standard ones, such as a program's own: properties in it are reported +/// under `prefix`, whatever prefix the packet binds the URI to. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct XmpNamespace<'a> { + pub prefix: &'a str, + pub uri: &'a str, +} + +/// What [`write_xmp_with`] writes beside the metadata. +#[derive(Clone, Copy, Debug)] +pub struct XmpWrite<'a> { + /// The `x:xmptk` attribute: the name of the writing program. + pub toolkit: &'a str, + /// The namespace of `properties`. Without one, `properties` are not written. + pub namespace: Option>, + /// Simple properties in `namespace`, as `(local name, text)`. A name that is not ASCII letters, + /// digits and `_` is left out. + pub properties: &'a [(&'a str, &'a str)], +} + +impl Default for XmpWrite<'_> { + fn default() -> Self { + Self { toolkit: "craft-metadata", namespace: None, properties: &[] } + } +} + +const NAMESPACES: &[(&str, &str)] = &[ + ("x", "adobe:ns:meta/"), + ("rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"), + ("dc", "http://purl.org/dc/elements/1.1/"), + ("xmp", "http://ns.adobe.com/xap/1.0/"), + ("xmpRights", "http://ns.adobe.com/xap/1.0/rights/"), + ("photoshop", "http://ns.adobe.com/photoshop/1.0/"), + ("exif", "http://ns.adobe.com/exif/1.0/"), + ("exifEX", "http://cipa.jp/exif/1.0/"), + ("tiff", "http://ns.adobe.com/tiff/1.0/"), + ("aux", "http://ns.adobe.com/exif/1.0/aux/"), + ("lr", "http://ns.adobe.com/lightroom/1.0/"), + ("Iptc4xmpCore", "http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/"), + // Read-only: develop settings written by other raw developers. + ("crs", CRS_NS), + // Read-only: face/pet/focus regions written by Lightroom, digiKam, Picasa and others (MWG Region + // Guidelines v2.0). Canonicalising these three lets a packet use any prefix for them. + ("mwg-rs", "http://www.metadataworkinggroup.com/schemas/regions/"), + ("stArea", "http://ns.adobe.com/xmp/sType/Area#"), + ("stDim", "http://ns.adobe.com/xap/1.0/sType/Dimensions#"), +]; + +/// The camera-raw-settings namespace URI (prefix `crs`), read for interchange only. +pub const CRS_NS: &str = "http://ns.adobe.com/camera-raw-settings/1.0/"; + +/// XMP read failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum XmpError { + Xml(String), + /// Nesting / size limits exceeded. + TooComplex, +} + +impl std::fmt::Display for XmpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + XmpError::Xml(e) => write!(f, "XMP is not well-formed XML: {e}"), + XmpError::TooComplex => write!(f, "XMP packet too deeply nested or too large"), + } + } +} + +impl std::error::Error for XmpError {} + +/// Result of [`parse_xmp`]. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct XmpData { + pub metadata: Metadata, + /// Every property found, keyed `prefix:name` (struct fields as `prefix:name/prefix:field`); arrays keep + /// item order, language alternatives put `x-default` first. Foreign namespaces that reuse a recognized + /// prefix use `Q{hex-encoded namespace URI}local` instead, so they cannot impersonate known properties. + pub properties: BTreeMap>, + /// Every top-level property with its full structure (arrays of structs such as local + /// corrections, which [`XmpData::properties`] flattens). + pub values: BTreeMap, +} + +impl XmpData { + /// The pixel size the packet's regions were drawn on (`mwg-rs:AppliedToDimensions`), unless one of them says how + /// the photo was turned (`mwg-rs:Rotation`: Lightroom's boxes, which the reader already turns upright). It tells + /// boxes drawn on the photo as stored from boxes drawn on it upright: see [`crate::upright_regions`]. + pub fn regions_drawn_on(&self) -> Option<(f64, f64)> { + let regions = self.values.get("mwg-rs:Regions")?; + let turned = regions.field("mwg-rs:RegionList").is_some_and(|l| l.items().iter().any(|r| r.field("mwg-rs:Rotation").is_some())); + if turned { None } else { applied_to(regions) } + } +} + +/// A structured XMP value. +#[derive(Clone, Debug, PartialEq)] +pub enum XmpValue { + Text(String), + Array(Vec), + Struct(BTreeMap), +} + +impl XmpValue { + /// A struct field (`prefix:name`). + pub fn field(&self, k: &str) -> Option<&XmpValue> { + match self { + XmpValue::Struct(m) => m.get(k), + _ => None, + } + } + pub fn text(&self) -> Option<&str> { + match self { + XmpValue::Text(s) => Some(s.trim()), + XmpValue::Array(a) => a.first().and_then(XmpValue::text), + XmpValue::Struct(_) => None, + } + } + /// Array items (a lone value is a one-item array). + pub fn items(&self) -> &[XmpValue] { + match self { + XmpValue::Array(a) => a, + other => std::slice::from_ref(other), + } + } +} + +fn node_struct(n: &Node) -> XmpValue { + let mut m = BTreeMap::new(); + for (k, v) in &n.attrs { + if !is_rdf_meta_attr(k) { + m.insert(k.clone(), XmpValue::Text(v.clone())); + } + } + for c in &n.children { + m.insert(c.name.clone(), node_value(c)); + } + XmpValue::Struct(m) +} + +fn node_value(n: &Node) -> XmpValue { + if let Some(arr) = n.children.iter().find(|c| matches!(c.name.as_str(), "rdf:Seq" | "rdf:Bag" | "rdf:Alt")) { + return XmpValue::Array(arr.children.iter().filter(|li| li.name == "rdf:li").map(node_value).collect()); + } + if let Some(d) = n.children.iter().find(|c| c.name == "rdf:Description") { + return node_struct(d); + } + if !n.children.is_empty() || n.attrs.iter().any(|(k, _)| !is_rdf_meta_attr(k)) { + return node_struct(n); + } + XmpValue::Text(n.attr("rdf:resource").map(str::to_string).unwrap_or_else(|| n.text.trim().to_string())) +} + +#[derive(Debug, Default)] +struct Node { + name: String, + attrs: Vec<(String, String)>, + children: Vec, + text: String, +} + +impl Node { + fn attr(&self, n: &str) -> Option<&str> { + self.attrs.iter().find(|(k, _)| k == n).map(|(_, v)| v.as_str()) + } +} + +const MAX_DEPTH: usize = 64; +const MAX_NODES: usize = 200_000; +// Expanding a long namespace URI on many short names must not amplify a small packet without bound. +pub(crate) const MAX_OPAQUE_NAME_BYTES: usize = 8 * 1024 * 1024; + +/// Most text a converter's flattened property keys may add up to. Keys repeat their whole ancestor +/// path (`parent/field`), so a document with a long nested name and a wide child fan-out otherwise +/// retains its prefix once per property: node limits alone cannot bound that duplication. +const MAX_KEY_BYTES: usize = 1 << 20; + +/// Names canonicalised by namespace URI (see [`NAMESPACES`]); conflicting foreign prefixes are opaque. +pub(crate) fn canonical_name( + qname: &str, + scopes: &[Vec<(String, String)>], + extra: &[XmpNamespace], + opaque_budget: &mut usize, +) -> Result { + let known = || NAMESPACES.iter().copied().chain(extra.iter().map(|n| (n.prefix, n.uri))); + let (prefix, local) = qname.split_once(':').unwrap_or(("", qname)); + if prefix == "xml" || prefix == "xmlns" { + return Ok(qname.to_string()); + } + let uri = scopes.iter().rev().flat_map(|s| s.iter().rev()).find(|(p, _)| p == prefix).map(|(_, u)| u.as_str()); + Ok(match uri { + Some(u) => match known().find(|(_, nu)| *nu == u) { + Some((p, _)) => format!("{p}:{local}"), + None if known().any(|(p, _)| p == prefix) => { + // A foreign `app:settings` must never match the caller's `app:*` ownership rule (nor may a + // foreign `xmp:Rating` become a rating). Braces cannot occur in an XML QName; + // hex keeps URI slashes out of the flattened property's `/` path separators. + let size = + u.len().checked_mul(2).and_then(|n| n.checked_add(3)).and_then(|n| n.checked_add(local.len())).ok_or(XmpError::TooComplex)?; + *opaque_budget = opaque_budget.checked_sub(size).ok_or(XmpError::TooComplex)?; + let mut key = String::with_capacity(size); + key.push_str("Q{"); + for byte in u.bytes() { + for nibble in [byte >> 4, byte & 15] { + key.push(char::from(if nibble < 10 { b'0' + nibble } else { b'a' + (nibble - 10) })); + } + } + key.push('}'); + key.push_str(local); + key + } + None if prefix.is_empty() => local.to_string(), + None => qname.to_string(), + }, + None => qname.to_string(), + }) +} + +fn entity(name: &str) -> Option { + Some(match name { + "lt" => '<', + "gt" => '>', + "amp" => '&', + "quot" => '"', + "apos" => '\'', + _ => { + let n = name.strip_prefix('#')?; + let v = match n.strip_prefix(['x', 'X']) { + Some(h) if h.bytes().all(|b| b.is_ascii_hexdigit()) => u32::from_str_radix(h, 16).ok()?, + Some(_) => return None, + None if n.bytes().all(|b| b.is_ascii_digit()) => n.parse().ok()?, + None => return None, + }; + char::from_u32(v).filter(|&c| is_xml_char(c))? + } + }) +} + +/// The XML 1.0 `Char` production (the surrogate range can't occur in a `char`). +fn is_xml_char(c: char) -> bool { + matches!(c, '\t' | '\n' | '\r') || (c >= '\u{20}' && c != '\u{FFFE}' && c != '\u{FFFF}') +} + +/// Drop the characters XML 1.0 forbids, which strict parsers (Adobe XMP Toolkit, expat, libxml2) reject the +/// whole document for. +fn xml_safe(s: &str) -> String { + s.chars().filter(|&c| is_xml_char(c)).collect() +} + +/// Index of the `;` that ends a reference starting at `rest` (its `&`). The scan stops at the first +/// byte that cannot appear in a name or a numeric reference, so zero-padded references such as +/// `a` are found and each run of name bytes is read once, not once per `&`. +fn entity_end(rest: &str) -> Option { + let n = rest.bytes().skip(1).take_while(|b| b.is_ascii_alphanumeric() || *b == b'#').count(); + (rest.as_bytes().get(n + 1) == Some(&b';')).then_some(n + 1) +} + +pub(crate) fn unescape_attr(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(i) = rest.find('&') { + out.push_str(&rest[..i]); + rest = &rest[i..]; + match entity_end(rest).and_then(|j| entity(&rest[1..j]).map(|c| (c, j))) { + Some((c, j)) => { + out.push(c); + rest = &rest[j + 1..]; + } + None => { + out.push('&'); + rest = &rest[1..]; + } + } + } + out.push_str(rest); + out +} + +fn parse_tree(s: &str, extra: &[XmpNamespace]) -> Result { + let mut reader = quick_xml::Reader::from_str(s); + reader.config_mut().check_end_names = false; + let mut stack: Vec = vec![Node { name: "#root".into(), ..Default::default() }]; + let mut scopes: Vec> = vec![vec![("xml".into(), "http://www.w3.org/XML/1998/namespace".into())]]; + let mut count = 0usize; + let mut opaque_budget = MAX_OPAQUE_NAME_BYTES; + loop { + let ev = reader.read_event().map_err(|e| XmpError::Xml(e.to_string()))?; + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + count += 1; + if count > MAX_NODES || stack.len() > MAX_DEPTH { + return Err(XmpError::TooComplex); + } + let mut scope = Vec::new(); + let mut raw_attrs = Vec::new(); + for a in e.attributes().with_checks(false).flatten() { + let k = String::from_utf8_lossy(a.key.as_ref()).into_owned(); + let v = unescape_attr(&String::from_utf8_lossy(&a.value)); + if k == "xmlns" { + scope.push((String::new(), v)); + } else if let Some(p) = k.strip_prefix("xmlns:") { + scope.push((p.to_string(), v)); + } else { + raw_attrs.push((k, v)); + } + } + scopes.push(scope); + let name = canonical_name(&String::from_utf8_lossy(e.name().as_ref()), &scopes, extra, &mut opaque_budget)?; + let attrs = raw_attrs.into_iter().map(|(k, v)| Ok((canonical_name(&k, &scopes, extra, &mut opaque_budget)?, v))).collect::>( + )?; + let node = Node { name, attrs, ..Default::default() }; + if matches!(ev, Event::Empty(_)) { + scopes.pop(); + if let Some(parent) = stack.last_mut() { + parent.children.push(node); + } + } else { + stack.push(node); + } + } + Event::End(_) if stack.len() > 1 => { + let Some(n) = stack.pop() else { return Err(XmpError::Xml("end tag without an open element".into())) }; + scopes.pop(); + if let Some(parent) = stack.last_mut() { + parent.children.push(n); + } + } + Event::Text(t) => { + if let Some(top) = stack.last_mut() { + top.text.push_str(&t.decode().map_err(|e| XmpError::Xml(e.to_string()))?); + } + } + Event::CData(t) => { + if let Some(top) = stack.last_mut() { + top.text.push_str(&String::from_utf8_lossy(&t)); + } + } + Event::GeneralRef(r) => { + let name = String::from_utf8_lossy(&r).into_owned(); + if let Some(top) = stack.last_mut() { + match entity(&name) { + Some(c) => top.text.push(c), + None => { + top.text.push('&'); + top.text.push_str(&name); + top.text.push(';'); + } + } + } + } + Event::Eof => break, + _ => {} + } + } + // close unterminated elements leniently + while stack.len() > 1 { + let Some(n) = stack.pop() else { return Err(XmpError::Xml("unterminated element without a node".into())) }; + let Some(parent) = stack.last_mut() else { return Err(XmpError::Xml("unterminated element without a parent".into())) }; + parent.children.push(n); + } + stack.pop().ok_or_else(|| XmpError::Xml("xmp packet without a root element".into())) +} + +fn is_rdf_meta_attr(k: &str) -> bool { + matches!(k, "rdf:about" | "rdf:ID" | "rdf:nodeID" | "xml:lang" | "rdf:parseType" | "rdf:resource" | "rdf:datatype") || k.starts_with("xmlns") +} + +fn collect_description(d: &Node, prefix: &str, props: &mut BTreeMap>, budget: &mut usize) { + for (k, v) in &d.attrs { + // the budget is checked before the path is built, not after + if is_rdf_meta_attr(k) || *budget < prefix.len().saturating_add(k.len()) { + continue; + } + let key = format!("{prefix}{k}"); + if take_key_bytes(&key, budget) { + props.entry(key).or_default().push(v.clone()); + } + } + for c in &d.children { + collect_property(c, prefix, props, budget); + } +} + +/// Charge one flattened key's length to the key budget; `false` when the budget is spent (the +/// property is then left out, like other elements a limit has cut). +fn take_key_bytes(key: &str, budget: &mut usize) -> bool { + let n = key.len(); + if *budget < n { + return false; + } + *budget -= n; + true +} + +fn collect_property(p: &Node, prefix: &str, props: &mut BTreeMap>, budget: &mut usize) { + // a path that can't fit the budget isn't built (a long ancestor path shared by many children + // would otherwise be copied for each of them after the budget is spent) + if *budget < prefix.len().saturating_add(p.name.len()) { + return; + } + let key = format!("{prefix}{}", p.name); + if let Some(arr) = p.children.iter().find(|c| matches!(c.name.as_str(), "rdf:Seq" | "rdf:Bag" | "rdf:Alt")) { + let mut items: Vec<(bool, String)> = arr + .children + .iter() + .filter(|li| li.name == "rdf:li") + .map(|li| { + (li.attr("xml:lang") == Some("x-default"), li.attr("rdf:resource").map(str::to_string).unwrap_or_else(|| li.text.trim().to_string())) + }) + .collect(); + if arr.name == "rdf:Alt" { + items.sort_by_key(|(default, _)| !*default); + } + if take_key_bytes(&key, budget) { + props.entry(key).or_default().extend(items.into_iter().map(|(_, v)| v)); + } + return; + } + let struct_prefix = format!("{key}/"); + // a struct's path is charged like a key before its fields copy it: that work too is bounded + // by the budget, kept fields or not + let is_struct = p.attr("rdf:parseType") == Some("Resource") || p.children.iter().any(|c| c.name == "rdf:Description"); + if is_struct && !take_key_bytes(&key, budget) { + return; + } + if p.attr("rdf:parseType") == Some("Resource") { + for c in &p.children { + collect_property(c, &struct_prefix, props, budget); + } + return; + } + if let Some(d) = p.children.iter().find(|c| c.name == "rdf:Description") { + collect_description(d, &struct_prefix, props, budget); + return; + } + let field_attrs: Vec<&(String, String)> = p.attrs.iter().filter(|(k, _)| !is_rdf_meta_attr(k)).collect(); + if !field_attrs.is_empty() && p.children.is_empty() && p.text.trim().is_empty() { + for (k, v) in field_attrs { + let full = format!("{struct_prefix}{k}"); + if take_key_bytes(&full, budget) { + props.entry(full).or_default().push(v.clone()); + } + } + return; + } + let v = p.attr("rdf:resource").map(str::to_string).unwrap_or_else(|| p.text.trim().to_string()); + if take_key_bytes(&key, budget) { + props.entry(key).or_default().push(v); + } +} + +fn find_descriptions<'a>(n: &'a Node, out: &mut Vec<&'a Node>) { + if n.name == "rdf:Description" { + out.push(n); + return; + } + for c in &n.children { + find_descriptions(c, out); + } +} + +/// XMP GPS coordinate `DDD,MM,SSk` or `DDD,MM.mmk` (k = N/S/E/W) → signed degrees. +fn parse_gps_coord(s: &str) -> Option { + let s = s.trim(); + let dir = s.chars().last()?; + let body = &s[..s.len() - dir.len_utf8()]; + let parts: Vec = body.split(',').map(|p| p.trim().parse::().ok()).collect::>()?; + let v = match parts.as_slice() { + [d] => *d, + [d, m] => d + m / 60.0, + [d, m, sec] => d + m / 60.0 + sec / 3600.0, + _ => return None, + }; + let v = match dir.to_ascii_uppercase() { + 'N' | 'E' => v, + 'S' | 'W' => -v, + _ => return None, + }; + v.is_finite().then_some(v) +} + +/// `mwg-rs:Regions` → every region in its `RegionList`, converted to the full-image normalized frame. +/// Malformed or out-of-range entries are skipped rather than rejecting the whole list: one bad region +/// (from Lightroom, digiKam, Picasa, or a hand-edited file) shouldn't hide the rest. +fn parse_regions(regions: &XmpValue) -> Vec { + // `AppliedToDimensions` documents the pixel size regions were authored against. Here it's only consulted + // for the rare region whose own Area is in pixel units instead of MWG's normalized default below. + let px_dims = applied_to(regions); + let Some(list) = regions.field("mwg-rs:RegionList") else { return Vec::new() }; + list.items().iter().filter_map(|item| parse_one_region(item, px_dims)).collect() +} + +/// `mwg-rs:AppliedToDimensions` of a `mwg-rs:Regions` struct: (width, height) in pixels, both positive. +fn applied_to(regions: &XmpValue) -> Option<(f64, f64)> { + regions.field("mwg-rs:AppliedToDimensions").and_then(|d| { + let w = d.field("stDim:w").and_then(XmpValue::text).and_then(parse_number)?; + let h = d.field("stDim:h").and_then(XmpValue::text).and_then(parse_number)?; + (w > 0.0 && h > 0.0).then_some((w, h)) + }) +} + +fn parse_one_region(item: &XmpValue, px_dims: Option<(f64, f64)>) -> Option { + let area = item.field("mwg-rs:Area")?; + let num_attr = |k: &str| area.field(k).and_then(XmpValue::text).and_then(parse_number); + // stArea:x/y are the region's CENTER, not its top-left corner (MWG Region Guidelines §Area). + let (cx, cy, w, h) = (num_attr("stArea:x")?, num_attr("stArea:y")?, num_attr("stArea:w")?, num_attr("stArea:h")?); + let finite_and_sane = [cx, cy, w, h].iter().all(|v| v.is_finite() && v.abs() <= 1.0e6); + if !finite_and_sane || w <= 0.0 || h <= 0.0 { + return None; + } + // `stArea:unit` is "normalized" (a fraction of AppliedToDimensions) in every writer we know of, and + // that needs no cross-referencing at all. Only a region that explicitly says "pixel" gets divided + // down, and only if AppliedToDimensions is there to divide by; otherwise it's dropped rather than + // stored as a nonsense fraction. + let is_pixels = area.field("stArea:unit").and_then(XmpValue::text).is_some_and(|u| u.eq_ignore_ascii_case("pixel")); + let (cx, cy, w, h) = match (is_pixels, px_dims) { + (false, _) => (cx, cy, w, h), + (true, Some((dw, dh))) => (cx / dw, cy / dh, w / dw, h / dh), + (true, None) => return None, + }; + // Lightroom Classic stores the box of a photo with Exif orientation 3 / 6 / 8 in the sensor's own + // frame and writes the orientation as the region's `mwg-rs:Rotation` (π for 3, −π/2 for 6, +π/2 for + // 8; 0 for 1) while `AppliedToDimensions` names the upright size. Turn it into the upright frame. + // MWG has no way to say "mirrored", so orientations 2/4/5/7 can't be told apart from their unmirrored + // twins here; boxes on mirrored photos are taken as written . + let rotation = item.field("mwg-rs:Rotation").and_then(XmpValue::text).and_then(parse_number).filter(|r| r.is_finite()); + let (quarter, half) = (std::f64::consts::FRAC_PI_2, std::f64::consts::PI); + let (cx, cy, w, h) = match rotation { + Some(r) if (r + quarter).abs() < 0.05 => (1.0 - cy, cx, h, w), + Some(r) if (r - quarter).abs() < 0.05 => (cy, 1.0 - cx, h, w), + Some(r) if (r.abs() - half).abs() < 0.05 => (1.0 - cx, 1.0 - cy, w, h), + _ => (cx, cy, w, h), + }; + // Keep only the part of the box inside the photo: a region hanging off an edge (common for faces + // cut by the frame) is clipped, and one entirely outside it (hostile or stale) is dropped, so every + // stored rect is within 0..1 and non-empty. + let rect = Rect::from_center(Point::new(cx, cy), w, h).intersect(&Rect::UNIT); + if rect.is_empty() { + return None; + } + let name = item.field("mwg-rs:Name").and_then(XmpValue::text).map(str::to_string).filter(|s| !s.is_empty()); + let description = item.field("mwg-rs:Description").and_then(XmpValue::text).map(str::to_string).filter(|s| !s.is_empty()); + let kind = match item.field("mwg-rs:Type").and_then(XmpValue::text).unwrap_or("") { + t if t.eq_ignore_ascii_case("face") => RegionKind::Face, + t if t.eq_ignore_ascii_case("pet") => RegionKind::Pet, + t if t.eq_ignore_ascii_case("focus") => RegionKind::Focus, + t if t.eq_ignore_ascii_case("barcode") => RegionKind::BarCode, + t => RegionKind::Other(t.to_string()), + }; + Some(Region { rect, kind, name, description }) +} + +fn fmt_gps_coord(v: f64, pos: char, neg: char) -> String { + let a = v.abs(); + let mut d = a.trunc(); + let mut m = ((a - d) * 60.0 * 1e6).round() / 1e6; + if m >= 60.0 { + d += 1.0; + m = 0.0; + } + format!("{},{:.6}{}", d as i64, m, if v < 0.0 { neg } else { pos }) +} + +/// Parse an XMP packet (with or without the `` wrapper / `x:xmpmeta` element). +pub fn parse_xmp(s: &str) -> Result { + parse_xmp_with(s, &[]) +} + +/// [`parse_xmp`] that also knows the namespaces in `extra` by their URI. +pub fn parse_xmp_with(s: &str, extra: &[XmpNamespace]) -> Result { + let s = s.trim_start_matches('\u{feff}'); + let tree = parse_tree(s, extra)?; + let mut descs = Vec::new(); + find_descriptions(&tree, &mut descs); + let mut props = BTreeMap::new(); + let mut values = BTreeMap::new(); + // One budget for the whole document: several descriptions cannot each spend a fresh one. + let mut key_budget = MAX_KEY_BYTES; + for d in descs { + collect_description(d, "", &mut props, &mut key_budget); + if let XmpValue::Struct(m) = node_struct(d) { + values.extend(m); + } + } + let first = |k: &str| props.get(k).and_then(|v| v.first()).map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + let num = |k: &str| first(k).and_then(|s| parse_number(&s)); + let mut m = Metadata { + make: first("tiff:Make"), + model: first("tiff:Model"), + serial_number: first("exifEX:BodySerialNumber").or_else(|| first("aux:SerialNumber")), + software: first("xmp:CreatorTool"), + lens_make: first("exifEX:LensMake"), + lens_model: first("exifEX:LensModel").or_else(|| first("aux:Lens")), + lens_serial_number: first("exifEX:LensSerialNumber").or_else(|| first("aux:LensSerialNumber")), + exposure_time: num("exif:ExposureTime").filter(|v| *v > 0.0), + f_number: num("exif:FNumber").filter(|v| *v > 0.0), + focal_length: num("exif:FocalLength").filter(|v| *v > 0.0), + focal_length_35mm: num("exif:FocalLengthIn35mmFilm").filter(|v| *v > 0.0), + exposure_bias: num("exif:ExposureBiasValue"), + exposure_program: num("exif:ExposureProgram").map(|v| v as u16), + metering_mode: num("exif:MeteringMode").map(|v| v as u16), + white_balance: num("exif:WhiteBalance").map(|v| v as u16), + orientation: num("tiff:Orientation").map(|v| v as u16).filter(|v| (1..=8).contains(v)).map(Orientation::from_exif), + width: num("tiff:ImageWidth").filter(|v| *v >= 0.0).or_else(|| num("exif:PixelXDimension").filter(|v| *v >= 0.0)).map(|v| v as u32), + height: num("tiff:ImageLength").filter(|v| *v >= 0.0).or_else(|| num("exif:PixelYDimension").filter(|v| *v >= 0.0)).map(|v| v as u32), + title: first("dc:title"), + caption: first("dc:description"), + alt_text: first("Iptc4xmpCore:AltTextAccessibility"), + extended_description: first("Iptc4xmpCore:ExtDescrAccessibility"), + sublocation: first("Iptc4xmpCore:Location"), + city: first("photoshop:City"), + state: first("photoshop:State"), + country: first("photoshop:Country"), + copyright: first("dc:rights"), + copyright_marked: first("xmpRights:Marked").and_then(|s| match s.to_ascii_lowercase().as_str() { + "true" => Some(true), + "false" => Some(false), + _ => None, + }), + usage_terms: first("xmpRights:UsageTerms"), + copyright_url: first("xmpRights:WebStatement"), + rating: num("xmp:Rating").map(|v| v.round().clamp(-1.0, 5.0) as i8), + label: first("xmp:Label"), + ..Default::default() + }; + m.iso = [num("exif:ISOSpeedRatings"), num("exifEX:PhotographicSensitivity")].into_iter().flatten().find(|v| *v > 0.0).map(|v| v as u32); + if let Some(spec) = props.get("exifEX:LensSpecification").filter(|v| v.len() == 4) { + let v: Vec = spec.iter().filter_map(|s| parse_number(s)).collect(); + if v.len() == 4 { + m.lens_spec = Some([v[0], v[1], v[2], v[3]]); + } + } + if let Some(creators) = props.get("dc:creator").filter(|c| !c.is_empty()) { + m.artist = Some(creators.join("; ")); + } + m.keywords = props.get("dc:subject").cloned().unwrap_or_default().into_iter().filter(|k| !k.is_empty()).collect(); + m.hierarchical_keywords = props.get("lr:hierarchicalSubject").cloned().unwrap_or_default().into_iter().filter(|k| !k.is_empty()).collect(); + m.capture_time = + ["exif:DateTimeOriginal", "photoshop:DateCreated", "xmp:CreateDate"].iter().find_map(|k| first(k).and_then(|s| DateTime::parse_iso(&s))); + if let Some(fired) = first("exif:Flash/exif:Fired") { + let b = |k: &str| first(k).is_some_and(|s| s.eq_ignore_ascii_case("true")); + let n = |k: &str| num(k).unwrap_or(0.0) as u16; + let fired = fired.eq_ignore_ascii_case("true"); + let raw = fired as u16 + | (n("exif:Flash/exif:Return") & 3) << 1 + | (n("exif:Flash/exif:Mode") & 3) << 3 + | (b("exif:Flash/exif:Function") as u16) << 5 + | (b("exif:Flash/exif:RedEyeMode") as u16) << 6; + m.flash = Some(Flash { fired, raw }); + } + if let (Some(lat), Some(lon)) = + (first("exif:GPSLatitude").and_then(|s| parse_gps_coord(&s)), first("exif:GPSLongitude").and_then(|s| parse_gps_coord(&s))) + && (-90.0..=90.0).contains(&lat) + && (-180.0..=180.0).contains(&lon) + { + let altitude = num("exif:GPSAltitude").map(|a| if first("exif:GPSAltitudeRef").as_deref() == Some("1") { -a } else { a }); + m.gps = Some(Gps { latitude: lat, longitude: lon, altitude }); + } + m.regions = values.get("mwg-rs:Regions").map(parse_regions).unwrap_or_default(); + Ok(XmpData { metadata: m, properties: props, values }) +} + +fn frac(v: f64) -> String { + if v > 0.0 && v < 1.0 { + let inv = 1.0 / v; + if (inv - inv.round()).abs() < 1e-6 { + return format!("1/{}", inv.round() as u64); + } + } + let (n, d) = craft_tiff::writer::rational(v); + format!("{n}/{d}") +} + +fn sfrac(v: f64) -> String { + let (n, d) = craft_tiff::writer::srational(v); + format!("{n}/{d}") +} + +/// Serialise the interchange subset of `meta` as a complete XMP packet. +pub fn write_xmp(meta: &Metadata) -> String { + write_xmp_with(meta, &XmpWrite::default()) +} + +/// Like [`write_xmp`], with the writing program's name and its own simple properties. +pub fn write_xmp_with(meta: &Metadata, with: &XmpWrite) -> String { + let mut simple: Vec<(&str, String)> = Vec::new(); + let mut push = |k: &'static str, v: Option| { + if let Some(v) = v { + simple.push((k, v)); + } + }; + push("tiff:Make", meta.make.clone()); + push("tiff:Model", meta.model.clone()); + push("tiff:Orientation", meta.orientation.map(|o| o.to_exif().to_string())); + push("tiff:ImageWidth", meta.width.map(|v| v.to_string())); + push("tiff:ImageLength", meta.height.map(|v| v.to_string())); + push("xmp:CreatorTool", meta.software.clone()); + push("xmp:Rating", meta.rating.map(|v| v.to_string())); + push("xmp:Label", meta.label.clone()); + push("exif:DateTimeOriginal", meta.capture_time.map(|d| d.to_iso())); + push("photoshop:DateCreated", meta.capture_time.map(|d| d.to_iso())); + push("exif:ExposureTime", meta.exposure_time.map(frac)); + push("exif:FNumber", meta.f_number.map(frac)); + push("exif:FocalLength", meta.focal_length.map(frac)); + push("exif:FocalLengthIn35mmFilm", meta.focal_length_35mm.map(|v| format!("{}", v.round() as i64))); + push("exif:ExposureBiasValue", meta.exposure_bias.map(sfrac)); + push("exif:ExposureProgram", meta.exposure_program.map(|v| v.to_string())); + push("exif:MeteringMode", meta.metering_mode.map(|v| v.to_string())); + push("exif:WhiteBalance", meta.white_balance.map(|v| v.to_string())); + push("exifEX:BodySerialNumber", meta.serial_number.clone()); + push("exifEX:LensMake", meta.lens_make.clone()); + push("exifEX:LensModel", meta.lens_model.clone()); + push("exifEX:LensSerialNumber", meta.lens_serial_number.clone()); + push("Iptc4xmpCore:Location", meta.sublocation.clone()); + push("photoshop:City", meta.city.clone()); + push("photoshop:State", meta.state.clone()); + push("photoshop:Country", meta.country.clone()); + push("xmpRights:Marked", meta.copyright_marked.map(|m| if m { "True" } else { "False" }.to_string())); + push("xmpRights:WebStatement", meta.copyright_url.clone()); + if let Some(g) = meta.gps { + push("exif:GPSLatitude", Some(fmt_gps_coord(g.latitude, 'N', 'S'))); + push("exif:GPSLongitude", Some(fmt_gps_coord(g.longitude, 'E', 'W'))); + if let Some(a) = g.altitude { + push("exif:GPSAltitude", Some(frac(a.abs()))); + push("exif:GPSAltitudeRef", Some(if a < 0.0 { "1" } else { "0" }.into())); + } + } + + let mut x = String::new(); + x.push_str("\n"); + x.push_str(&format!("\n", escape(xml_safe(with.toolkit)))); + x.push_str(" \n"); + x.push_str(" \n"); + for (k, v) in &simple { + x.push_str(&format!(" <{k}>{}\n", escape(xml_safe(v)))); + } + for (k, v) in with.properties.iter().filter(|_| own.is_some()) { + let p = own.map_or("", |n| n.0); + if !k.is_empty() && k.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { + // element content: quotes needn't be escaped, which keeps JSON readable + x.push_str(&format!(" <{p}:{k}>{}\n", partial_escape(xml_safe(v)))); + } + } + let array = |x: &mut String, k: &str, kind: &str, items: &[String], lang: bool| { + if items.is_empty() { + return; + } + x.push_str(&format!(" <{k}>\n \n")); + for it in items { + if lang { + x.push_str(&format!(" {}\n", escape(xml_safe(it)))); + } else { + x.push_str(&format!(" {}\n", escape(xml_safe(it)))); + } + } + x.push_str(&format!(" \n \n")); + }; + let one = |v: &Option| v.iter().cloned().collect::>(); + array(&mut x, "dc:title", "Alt", &one(&meta.title), true); + array(&mut x, "dc:description", "Alt", &one(&meta.caption), true); + array(&mut x, "Iptc4xmpCore:AltTextAccessibility", "Alt", &one(&meta.alt_text), true); + array(&mut x, "Iptc4xmpCore:ExtDescrAccessibility", "Alt", &one(&meta.extended_description), true); + array(&mut x, "dc:rights", "Alt", &one(&meta.copyright), true); + array(&mut x, "xmpRights:UsageTerms", "Alt", &one(&meta.usage_terms), true); + array(&mut x, "dc:creator", "Seq", &one(&meta.artist), false); + array(&mut x, "dc:subject", "Bag", &meta.keywords, false); + array(&mut x, "lr:hierarchicalSubject", "Bag", &meta.hierarchical_keywords, false); + array(&mut x, "exif:ISOSpeedRatings", "Seq", &meta.iso.map(|v| vec![v.to_string()]).unwrap_or_default(), false); + if let Some(s) = meta.lens_spec { + array(&mut x, "exifEX:LensSpecification", "Seq", &s.iter().map(|&v| frac(v)).collect::>(), false); + } + if let Some(f) = meta.flash { + let r = f.raw; + let tf = |b: bool| if b { "True" } else { "False" }; + x.push_str(&format!( + " \n {}\n {}\n {}\n {}\n {}\n \n", + tf(f.fired), + (r >> 1) & 3, + (r >> 3) & 3, + tf(r & 0x20 != 0), + tf(r & 0x40 != 0) + )); + } + x.push_str(" \n \n\n"); + x +} + +#[cfg(test)] +mod tests; diff --git a/crates/metadata/src/xmp/tests.rs b/crates/metadata/src/xmp/tests.rs new file mode 100644 index 000000000..7d7656fba --- /dev/null +++ b/crates/metadata/src/xmp/tests.rs @@ -0,0 +1,631 @@ +use super::*; + +/// A program's own namespace, as an application would pass it. +const LC: XmpNamespace = XmpNamespace { prefix: "lc", uri: "http://ns.example.com/lc/1.0/" }; + +fn write_lc(meta: &Metadata, properties: &[(&str, &str)]) -> String { + write_xmp_with(meta, &XmpWrite { toolkit: "Editor", namespace: Some(LC), properties }) +} + +fn parse_xmp(s: &str) -> Result { + parse_xmp_with(s, &[LC]) +} + +fn settings(d: &XmpData) -> Option<&str> { + d.properties.get("lc:settings").and_then(|v| v.first()).map(String::as_str) +} + +fn full() -> Metadata { + Metadata { + // write_xmp doesn't emit regions yet, so the + // roundtrip fixture must leave this empty or `roundtrip_all_fields` can't round-trip it. + regions: Vec::new(), + focus: None, + make: Some("Maker & Sons".into()), + model: Some("X <1>".into()), + serial_number: Some("SN1".into()), + software: Some("Editor".into()), + lens_make: Some("L".into()), + lens_model: Some("24-70mm".into()), + lens_serial_number: Some("LS".into()), + lens_spec: Some([24.0, 70.0, 2.8, 2.8]), + capture_time: DateTime::parse_iso("2022-05-06T07:08:09.123-04:00"), + exposure_time: Some(1.0 / 125.0), + f_number: Some(5.6), + iso: Some(800), + focal_length: Some(35.0), + focal_length_35mm: Some(52.0), + flash: Some(Flash { fired: true, raw: 0x59 }), + exposure_bias: Some(-1.0 / 3.0), + exposure_program: Some(2), + metering_mode: Some(3), + white_balance: Some(1), + orientation: Some(Orientation::Rotate270), + gps: Some(Gps { latitude: 48.858222, longitude: -2.2945, altitude: Some(-3.5) }), + width: Some(8000), + height: Some(6000), + artist: Some("Ann \"Photo\" Lee".into()), + copyright: Some("© 2022".into()), + copyright_marked: Some(true), + usage_terms: Some("Editorial use only; no ".into()), + copyright_url: Some("https://example.com/licence?a=1&b=2".into()), + title: Some("Tïtle".into()), + caption: Some("Line one\nline two".into()), + alt_text: Some("A tree by the sea".into()), + extended_description: Some("Long ".into()), + sublocation: Some("Champ de Mars".into()), + city: Some("Paris".into()), + state: Some("Île-de-France".into()), + country: Some("France".into()), + keywords: vec!["tree".into(), "sky & sea".into()], + hierarchical_keywords: vec!["Places|France|Paris".into()], + rating: Some(-1), + label: Some("Green".into()), + } +} + +#[test] +fn roundtrip_all_fields() { + let m = full(); + let json = r#"{"version":1,"exposure":0.5,"note":"<&>"}"#; + let x = write_lc(&m, &[("settings", json)]); + let d = parse_xmp(&x).unwrap(); + assert_eq!(settings(&d), Some(json)); + let mut got = d.metadata.clone(); + // floats go through rationals + assert!((got.exposure_bias.unwrap() + 1.0 / 3.0).abs() < 1e-5); + got.exposure_bias = m.exposure_bias; + let g = got.gps.unwrap(); + assert!((g.latitude - 48.858222).abs() < 1e-6 && (g.longitude + 2.2945).abs() < 1e-6, "{g:?}"); + got.gps = m.gps; + assert_eq!(got, m); +} + +#[test] +fn empty_metadata_roundtrip() { + let x = write_xmp(&Metadata::default()); + let d = parse_xmp(&x).unwrap(); + assert_eq!(d.metadata, Metadata::default()); + assert_eq!(settings(&d), None); +} + +#[test] +fn foreign_namespace_cannot_impersonate_known_metadata() { + for body in [ + r#"xmp:Rating="2" lc:settings="foreign settings"/>"#, + r#">2foreign settings"#, + ] { + let packet = format!( + r#""# + ); + let d = parse_xmp(&packet).unwrap(); + assert_eq!(d.metadata.rating, None, "foreign rating: {packet}"); + assert_eq!(settings(&d), None, "foreign settings: {packet}"); + assert!(!d.properties.contains_key("xmp:Rating")); + assert!(!d.values.contains_key("lc:settings")); + assert!(d.properties.values().any(|v| v == &["foreign settings"])); + assert!(d.properties.keys().all(|key| !key.contains('/')), "opaque names must not create struct paths"); + } +} + +#[test] +fn foreign_namespace_names_are_distinct_and_scoped() { + let packet = r#" + + 1 + 2 + 4 + "#; + let d = parse_xmp(packet).unwrap(); + assert_eq!(d.metadata.rating, Some(4), "a local rebinding must not escape its scope"); + assert_eq!(d.properties.len(), 3, "different foreign URIs must not collide"); + assert_eq!(d.values.len(), 3); + assert_eq!(d.properties["xmp:Rating"], vec!["4"]); + assert!(d.properties.values().any(|v| v == &["1"])); + assert!(d.properties.values().any(|v| v == &["2"])); +} + +#[test] +fn attribute_form_foreign_prefixes_and_structs() { + let x = r#" + + TitelTitle & more é + + k1k2 + "#; + let d = parse_xmp(x).unwrap(); + let m = d.metadata; + assert_eq!(m.rating, Some(4)); + assert_eq!(m.label.as_deref(), Some("Blue")); + assert_eq!(m.f_number, Some(4.0)); + assert_eq!(m.title.as_deref(), Some("Title & more é")); + assert_eq!(m.keywords, vec!["k1".to_string(), "k2".to_string()]); + assert_eq!(m.flash, Some(Flash { fired: false, raw: 0x10 })); + let g = m.gps.unwrap(); + assert!((g.latitude - 10.5).abs() < 1e-9); + assert!((g.longitude + 20.26).abs() < 1e-9); + assert_eq!(d.properties["dc:title"], vec!["Title & more é".to_string(), "Titel".to_string()]); +} + +/// Copyright status, usage terms and info URL (XMP Rights Management schema): written as +/// `xmpRights:Marked` (Boolean), `xmpRights:UsageTerms` (Lang Alt) and `xmpRights:WebStatement` (URI). +#[test] +fn rights_management_fields() { + let m = Metadata { copyright_marked: Some(false), usage_terms: Some("CC0".into()), ..Default::default() }; + let x = write_xmp(&m); + assert!(x.contains("False"), "{x}"); + assert!(x.contains("\n \n CC0"), "{x}"); + assert!(!x.contains("WebStatement"), "unset fields are left out"); + assert_eq!(parse_xmp(&x).unwrap().metadata, m); + // attribute form, other prefixes; anything but True / False is "unknown" + let read = |marked: &str| { + let x = format!( + r#" + + "# + ); + parse_xmp(&x).unwrap().metadata + }; + assert_eq!(read("True").copyright_marked, Some(true)); + assert_eq!(read("false").copyright_marked, Some(false)); + assert_eq!(read("").copyright_marked, None); + assert_eq!(read("maybe").copyright_marked, None); + assert_eq!(read("True").copyright_url.as_deref(), Some("https://example.org/c")); +} + +/// A zero or negative `exif:ISOSpeedRatings` is "no ISO" (as in the EXIF reader), not `Some(0)`, and +/// must not hide a valid `exifEX:PhotographicSensitivity`. +#[test] +fn non_positive_iso_is_ignored() { + let read = |attrs: &str| { + let x = format!( + r#" + + "# + ); + parse_xmp(&x).unwrap().metadata.iso + }; + assert_eq!(read(r#"exif:ISOSpeedRatings="0" exifEX:PhotographicSensitivity="400""#), Some(400)); + assert_eq!(read(r#"exif:ISOSpeedRatings="-100""#), None); + assert_eq!(read(r#"exif:ISOSpeedRatings="0""#), None); + assert_eq!(read(r#"exif:ISOSpeedRatings="200""#), Some(200)); +} + +#[test] +fn negative_xmp_pixel_width_is_ignored() { + let read = |attrs: &str| { + let x = format!( + r#" + + "# + ); + parse_xmp(&x).unwrap().metadata.width + }; + + assert_eq!(read(r#"tiff:ImageWidth="-1""#), None); + assert_eq!(read(r#"tiff:ImageWidth="-1" exif:PixelXDimension="640""#), Some(640)); + assert_eq!(read(r#"exif:PixelXDimension="-1""#), None); +} + +#[test] +fn malformed_inputs() { + assert!(parse_xmp("").is_ok() || parse_xmp("").is_err()); + assert_eq!(parse_xmp("").unwrap().metadata, Metadata::default()); + assert_eq!(parse_xmp("not xml at all").unwrap().metadata, Metadata::default()); + let deep = "".repeat(1000); + assert_eq!(parse_xmp(&deep), Err(XmpError::TooComplex)); + let x = write_lc(&full(), &[("settings", "{}")]); + for n in (0..x.len()).step_by(7) { + if x.is_char_boundary(n) { + let _ = parse_xmp(&x[..n]); + } + } +} + +#[test] +fn lc_extras_and_crs_prefix() { + let x = write_lc(&Metadata::default(), &[("settings", "{\"a\":1}"), ("flag", "pick"), ("bad name", "x")]); + assert!(!x.contains("xmlns:crs"), "we never write crs:"); + let d = parse_xmp(&x).unwrap(); + assert_eq!(settings(&d), Some("{\"a\":1}")); + assert_eq!(d.properties["lc:flag"], vec!["pick".to_string()]); + assert!(!x.contains("bad name")); + let foreign = r#""#; + assert_eq!(parse_xmp(foreign).unwrap().properties["crs:Exposure2012"], vec!["+0.5".to_string()]); +} + +#[test] +fn gps_coords() { + assert_eq!(parse_gps_coord("45,30N"), Some(45.5)); + assert_eq!(parse_gps_coord("45,30,36S"), Some(-45.51)); + assert_eq!(parse_gps_coord("45X"), None); + assert_eq!(parse_gps_coord(""), None); + assert_eq!(parse_gps_coord("é"), None); + assert_eq!(fmt_gps_coord(-45.51, 'N', 'S'), "45,30.600000S"); + assert_eq!(fmt_gps_coord(10.9999999999, 'N', 'S'), "11,0.000000N"); + assert_eq!(fmt_gps_coord(-0.9999999999, 'E', 'W'), "1,0.000000W"); +} + +/// Like the EXIF and GPX readers, XMP drops a position outside latitude ±90° / longitude ±180°. +#[test] +fn xmp_gps_out_of_range_rejected() { + let gps = |lat: &str, lon: &str| { + let x = format!( + r#""# + ); + parse_xmp(&x).unwrap().metadata.gps + }; + assert_eq!(gps("95,0N", "200,30E"), None); + assert_eq!(gps("95,0N", "10,0E"), None); + assert_eq!(gps("10,0S", "200,30W"), None); + assert_eq!(gps("90,0,1N", "0,0E"), None); + assert_eq!(gps("90,0N", "180,0E"), Some(Gps { latitude: 90.0, longitude: 180.0, altitude: None })); + assert_eq!(gps("90,0S", "180,0W"), Some(Gps { latitude: -90.0, longitude: -180.0, altitude: None })); +} + +/// MWG Region Guidelines v2.0, element form (`rdf:parseType="Resource"`, self-closing `Area`) — how +/// Lightroom typically writes it. One named face, one untyped/unnamed region (still usable), one +/// region with a description. `AppliedToDimensions` is present but irrelevant: normalized areas +/// (the default, and the only unit real files use) don't need it. +#[test] +fn mwg_regions_element_form() { + let x = r#" + + + + + + + + Jane Doe + Face + + + + + + + Pet + Rex + Good boy + + + + + "#; + let regions = parse_xmp(x).unwrap().metadata.regions; + assert_eq!(regions.len(), 3, "{regions:?}"); + + assert_eq!(regions[0].kind, RegionKind::Face); + assert_eq!(regions[0].name.as_deref(), Some("Jane Doe")); + assert_eq!(regions[0].description, None); + let r = regions[0].rect; + assert!((r.x0 - 0.4).abs() < 1e-9 && (r.x1 - 0.6).abs() < 1e-9, "{r:?}"); + assert!((r.y0 - 0.25).abs() < 1e-9 && (r.y1 - 0.55).abs() < 1e-9, "{r:?}"); + + // No mwg-rs:Type or mwg-rs:Name at all: still a usable region, kind falls back to `Other("")`. + assert_eq!(regions[1].kind, RegionKind::Other(String::new())); + assert_eq!(regions[1].name, None); + + assert_eq!(regions[2].kind, RegionKind::Pet); + assert_eq!(regions[2].name.as_deref(), Some("Rex")); + assert_eq!(regions[2].description.as_deref(), Some("Good boy")); +} + +/// Lightroom Classic writes a rotated photo's (Exif orientation 3 / 6 / 8) box in the sensor's frame +/// and records the orientation as `mwg-rs:Rotation` (±π / −π/2 / +π/2); the reader returns the +/// upright frame. +#[test] +fn mwg_regions_quarter_turn_rotation_is_mapped_to_the_upright_frame() { + let x = r#" + + + + + + + + + + + + + + + + + + + + + + + "#; + let regions = parse_xmp(x).unwrap().metadata.regions; + assert_eq!(regions.len(), 5, "{regions:?}"); + let near = |r: Rect, b: [f64; 4]| [r.x0 - b[0], r.y0 - b[1], r.x1 - b[2], r.y1 - b[3]].iter().all(|d| d.abs() < 1e-9); + assert!(near(regions[0].rect, [0.45, 0.2, 0.75, 0.4]), "−π/2 (orientation 6): {:?}", regions[0].rect); + assert!(near(regions[1].rect, [0.25, 0.6, 0.55, 0.8]), "+π/2 (orientation 8): {:?}", regions[1].rect); + assert!(near(regions[2].rect, [0.2, 0.25, 0.4, 0.55]), "0 is untouched: {:?}", regions[2].rect); + assert!(near(regions[3].rect, [0.6, 0.45, 0.8, 0.75]), "π (orientation 3): {:?}", regions[3].rect); + assert!(near(regions[4].rect, [0.6, 0.45, 0.8, 0.75]), "−π (orientation 3): {:?}", regions[4].rect); +} + +/// The size a packet's boxes were drawn on is its `AppliedToDimensions`, unless a box says how the photo was turned +/// (Lightroom's `mwg-rs:Rotation`: those are upright already) or there is no size to go by. +#[test] +fn regions_drawn_on_is_the_applied_to_size_unless_a_box_says_how_it_was_turned() { + let packet = |dims: &str, rotation: &str| { + format!( + r#" + + {dims} + + + + + + + + + + "# + ) + }; + let dims = r#""#; + let drawn_on = |x: &str| parse_xmp(x).unwrap().regions_drawn_on(); + assert_eq!(drawn_on(&packet(dims, "")), Some((6000.0, 4000.0))); + assert_eq!(drawn_on(&packet(dims, r#" mwg-rs:Rotation="-1.57080""#)), None, "one box turned by Lightroom"); + assert_eq!(drawn_on(&packet("", "")), None, "no AppliedToDimensions"); + assert_eq!(drawn_on(r#""#), None); +} + +/// A second, differently-shaped encoding: each region is its own `rdf:Description` (rather than an +/// `rdf:li rdf:parseType="Resource"`), `mwg-rs:Type` as an attribute instead of an element, and an +/// unrecognised `mwg-rs:Type` value kept verbatim. +#[test] +fn mwg_regions_attribute_form_and_unknown_type() { + let x = r#" + + + + + + + + + + + + + + + + + + "#; + let regions = parse_xmp(x).unwrap().metadata.regions; + assert_eq!(regions.len(), 2, "{regions:?}"); + assert_eq!(regions[0].kind, RegionKind::BarCode); + assert_eq!(regions[1].kind, RegionKind::Other("Sticker".to_string())); +} + +/// Hostile/malformed regions are dropped individually, never panic, and never poison the rest of +/// the list: missing Area, non-finite or absurd numbers, zero/negative size, and a pixel-unit area +/// with no `AppliedToDimensions` to convert it (so it can't be stored as a nonsense fraction). +#[test] +fn mwg_regions_hostile_inputs_are_dropped_not_panicking() { + let x = r#" + + + + + No Area + + + + + + Face + + + + "#; + let regions = parse_xmp(x).unwrap().metadata.regions; + assert_eq!(regions.len(), 1, "only the last, well-formed region should survive: {regions:?}"); + assert_eq!(regions[0].kind, RegionKind::Face); +} + +/// Regions are clipped to the photo frame, and ones that miss the photo entirely are dropped: a +/// hostile sidecar can't park a box at x=1e5 (or a 1e5-wide one) that the loupe or the People view +/// would then draw or crop far outside the image. +#[test] +fn mwg_regions_are_clipped_to_the_frame_and_off_photo_ones_dropped() { + let x = r#" + + + + + Far right + Far left + Touching edge + Huge + Corner + + + + "#; + let regions = parse_xmp(x).unwrap().metadata.regions; + let names: Vec<_> = regions.iter().map(|r| r.name.as_deref().unwrap_or("")).collect(); + assert_eq!(names, ["Huge", "Corner"], "{regions:?}"); + for r in ®ions { + let q = r.rect; + assert!(q.x0 >= 0.0 && q.y0 >= 0.0 && q.x1 <= 1.0 && q.y1 <= 1.0 && !q.is_empty(), "{q:?}"); + } + assert_eq!(regions[0].rect, Rect::UNIT); + let c = regions[1].rect; + assert!((c.x0 - 0.85).abs() < 1e-9 && c.x1 == 1.0 && c.y0 == 0.0 && (c.y1 - 0.15).abs() < 1e-9, "{c:?}"); +} + +/// `extract()`'s precedence (EXIF/IPTC never carry regions; XMP is read and filled in like keywords). +#[test] +fn regions_merge_like_keywords() { + let with_region = Region { rect: Rect::from_center(Point::new(0.5, 0.5), 0.2, 0.2), kind: RegionKind::Face, name: None, description: None }; + let mut a = Metadata::default(); + let b = Metadata { regions: vec![with_region.clone()], ..Default::default() }; + a.fill_missing(&b); + assert_eq!(a.regions, vec![with_region.clone()]); + + let mut a = Metadata::default(); + a.overlay_user_fields(&b); + assert_eq!(a.regions, vec![with_region]); +} + +#[test] +fn writer_drops_characters_xml_1_0_forbids() { + let bad = "a\u{1}b\u{0}c\u{FFFE}d\u{FFFF}e\u{1F}\tf\ng\rh"; + let m = Metadata { make: Some(bad.into()), caption: Some(bad.into()), keywords: vec![bad.into()], ..Default::default() }; + let x = write_lc(&m, &[("settings", bad)]); + let illegal: Vec = + x.chars().filter(|&c| (c < '\u{20}' && !matches!(c, '\t' | '\n' | '\r')) || c == '\u{FFFE}' || c == '\u{FFFF}').collect(); + assert!(illegal.is_empty(), "illegal XML chars in output: {illegal:?}"); + assert!(x.contains(">abcde\tf\ng\rh<"), "allowed characters must survive: {x}"); +} + +#[test] +fn numeric_references_to_forbidden_characters_are_not_decoded() { + assert_eq!(entity("#1"), None); + assert_eq!(entity("#0"), None); + assert_eq!(entity("#xFFFE"), None); + assert_eq!(entity("#x41"), Some('A')); + assert_eq!(entity("#9"), Some('\t')); +} + +/// Flattened keys repeat their whole ancestor path, so a long nested name with a wide child +/// fan-out would otherwise retain its prefix once per leaf: the cumulative key budget stops it +/// (a 65 KB document with 3000 leaves must not retain ~2.4 MB of key text). +#[test] +fn flattened_keys_stay_within_the_cumulative_budget() { + let level = "n".repeat(200); + let mut doc = String::from( + "", + ); + for _ in 0..4 { + doc.push_str(&format!("")); + } + // each leaf has a name of its own, so each one adds a key + for i in 0..3000 { + doc.push_str(&format!("")); + } + doc.push_str(""); + for _ in 0..4 { + doc.push_str(&format!("")); + } + doc.push_str(""); + let d = parse_xmp(&doc).unwrap(); + let total: usize = d.properties.keys().map(String::len).sum(); + assert!(total <= MAX_KEY_BYTES, "flattened-key text {total} is over the {MAX_KEY_BYTES} budget"); + assert!(d.properties.len() < 3000, "keys stop being added once the budget is spent"); + assert!(!d.properties.is_empty(), "properties parsed before the budget are kept"); +} + +#[test] +fn zero_padded_numeric_references_decode() { + assert_eq!(unescape_attr("Canon A A"), "Canon A A"); + assert_eq!(unescape_attr("&abc&#xyz;&;"), "&abc&#xyz;&;"); +} + +#[test] +fn entity_rejects_signed_numeric_references() { + assert_eq!(entity("#+65"), None); + assert_eq!(entity("#x+41"), None); + assert_eq!(entity("#65"), Some('A')); + assert_eq!(entity("#x41"), Some('A')); +} + +#[test] +fn malformed_capture_time_falls_back_and_keeps_raw_property() { + let packet = r#""#; + let parsed = parse_xmp(packet).unwrap(); + assert_eq!( + parsed.metadata.capture_time.map(|d| (d.year, d.month, d.day, d.hour, d.minute, d.second, d.offset_minutes)), + Some((2026, 1, 2, 10, 0, 0, Some(0))) + ); + assert_eq!(parsed.properties["exif:DateTimeOriginal"], vec!["2026-01-01T".to_string()]); +} + +#[test] +fn malformed_capture_time_without_fallback_is_not_projected() { + let packet = r#""#; + let parsed = parse_xmp(packet).unwrap(); + assert_eq!(parsed.metadata.capture_time, None); + assert_eq!(parsed.properties["exif:DateTimeOriginal"], vec!["2026-01-01T".to_string()]); +} + +#[test] +fn many_ampersands_before_one_semicolon_are_read_once() { + // 200 000 `&` then a `;`: each `&` once searched the whole rest of the attribute for the `;` + let hostile = format!("{};&", "&".repeat(200_000)); + assert_eq!(unescape_attr(&hostile), format!("{};&", "&".repeat(200_000))); + assert_eq!(unescape_attr("<a> 􏿿 &unknownreference;"), " \u{10FFFF} &unknownreference;"); +} + +/// Children that add no key (empty structs) still had their whole ancestor path copied, each, +/// after the budget was spent: a 250 KB path under 100 000 such children was 25 GB of copying. +#[test] +fn paths_are_not_built_past_the_key_budget() { + let level = "n".repeat(60_000); + let mut doc = String::from( + "", + ); + for _ in 0..4 { + doc.push_str(&format!("")); + } + for _ in 0..100_000 { + doc.push_str(""); + } + for _ in 0..4 { + doc.push_str(&format!("")); + } + doc.push_str(""); + let t = std::time::Instant::now(); + let _ = parse_xmp(&doc); + assert!(t.elapsed().as_secs() < 10, "took {:?}", t.elapsed()); +} + +/// The caller's namespace is declared where the read-only `crs` sits in the table, is known to the +/// reader only when passed, and is canonicalised by its URI whatever prefix a packet binds it to. +#[test] +fn the_callers_namespace_is_written_and_read_only_when_passed() { + let x = write_lc(&Metadata::default(), &[("flag", "pick")]); + assert!(x.contains("x:xmptk=\"Editor\"")); + assert!(x.contains( + "xmlns:Iptc4xmpCore=\"http://iptc.org/std/Iptc4xmpCore/1.0/xmlns/\"\n xmlns:lc=\"http://ns.example.com/lc/1.0/\"\n xmlns:mwg-rs=" + )); + assert!(!x.contains("xmlns:crs")); + let renamed = x.replace("xmlns:lc=", "xmlns:zz=").replace("lc:flag", "zz:flag"); + assert_eq!(parse_xmp_with(&renamed, &[LC]).unwrap().properties["lc:flag"], vec!["pick".to_string()]); + assert!(!crate::parse_xmp(&renamed).unwrap().properties.contains_key("lc:flag")); + + let plain = write_xmp(&Metadata::default()); + assert!(plain.contains("x:xmptk=\"craft-metadata\"") && !plain.contains("xmlns:lc") && !plain.contains("xmlns:crs")); + // properties without a namespace to put them in are not written + let orphan = write_xmp_with(&Metadata::default(), &XmpWrite { properties: &[("flag", "pick")], ..Default::default() }); + assert_eq!(orphan, plain); +} diff --git a/crates/metadata/src/xmp_merge.rs b/crates/metadata/src/xmp_merge.rs new file mode 100644 index 000000000..b52e81c26 --- /dev/null +++ b/crates/metadata/src/xmp_merge.rs @@ -0,0 +1,505 @@ +//! Merging a freshly written XMP packet into an existing one (a sidecar another application +//! wrote, e.g. with its develop settings and edit history), keeping everything the writer does not +//! own byte for byte. +//! +//! The existing packet is scanned once (namespace-resolved, with byte positions). Only top-level +//! properties of the `rdf:Description` elements directly under `rdf:RDF` are considered, in +//! attribute form (`xmp:Rating="3"`) and element form (`…`): +//! - properties the writer owns ([`MergeRules::owned`]) are removed, and written from the fresh +//! packet when it has them (so a cleared field is cleared in the file too); +//! - properties owned only when stated ([`MergeRules::owned_if_present`]) are replaced when the +//! fresh packet has them and kept otherwise; +//! - any other property of the fresh packet (e.g. `xmp:CreatorTool`) is not written: the file +//! keeps describing itself as its creator left it; +//! - everything else — other namespaces (e.g. `crs:`), `xmpMM:History`, unknown properties, +//! comments, the packet wrapper and padding — is copied unchanged. +//! +//! The written properties go into one new `rdf:Description` (declaring its own namespaces, so +//! the existing prefix bindings never matter) just before ``. A description left empty +//! by the removals is dropped, so repeated saves don't accumulate. The result is checked by +//! parsing it again: every written property must read back as in the fresh packet. + +use std::collections::{BTreeSet, HashSet}; + +use quick_xml::events::Event; + +use crate::xmp::{MAX_OPAQUE_NAME_BYTES, XmpError, XmpNamespace, canonical_name, parse_xmp_with, unescape_attr}; + +/// Which properties [`merge_xmp`] takes from the fresh packet. Names are canonical +/// `prefix:name` (prefixes as [`crate::parse_xmp`] reports them); `prefix:*` is a whole namespace. +#[derive(Clone, Copy, Debug, Default)] +pub struct MergeRules<'a> { + /// Always replaced: removed from the existing packet, written from the fresh one if stated. + pub owned: &'a [&'a str], + /// Replaced only when the fresh packet states them; otherwise the existing value stays. + pub owned_if_present: &'a [&'a str], + /// Namespaces beyond the standard ones that the names above use, as for [`crate::parse_xmp_with`]. + pub namespaces: &'a [XmpNamespace<'a>], +} + +fn listed(list: &[&str], name: &str) -> bool { + list.iter().any(|p| match p.strip_suffix('*') { + Some(ns) => name.starts_with(ns), + None => *p == name, + }) +} + +#[derive(Debug)] +struct Attr { + raw: String, + name: String, + /// The value as written (still escaped). + value: String, +} + +#[derive(Debug)] +struct Prop { + name: String, + start: usize, + end: usize, +} + +#[derive(Debug, Default)] +struct Desc { + tag_start: usize, + tag_end: usize, + /// The element name as written (`rdf:Description`, or another prefix for the RDF namespace). + raw_name: String, + empty: bool, + attrs: Vec, + props: Vec, + /// The end of `` (`tag_end` when self-closing). + end: usize, +} + +#[derive(Debug, Default)] +struct Scan { + descs: Vec, + /// Where the first `` starts. + rdf_close: Option, +} + +enum Kind { + Other, + Rdf, + Desc(usize), + Prop { desc: usize, start: usize, name: String }, +} + +fn is_meta_attr(name: &str) -> bool { + name.starts_with("xmlns") + || matches!(name, "rdf:about" | "rdf:ID" | "rdf:nodeID" | "xml:lang" | "rdf:parseType" | "rdf:resource" | "rdf:datatype") +} + +fn pos(r: &quick_xml::Reader<&[u8]>) -> usize { + usize::try_from(r.buffer_position()).unwrap_or(usize::MAX) +} + +/// Locate the descriptions, their top-level properties and `` in a packet. +fn scan(s: &str, extra: &[XmpNamespace]) -> Result { + let mut reader = quick_xml::Reader::from_reader(s.as_bytes()); + reader.config_mut().check_end_names = true; + let mut out = Scan::default(); + let mut stack: Vec = Vec::new(); + let mut scopes: Vec> = vec![vec![("xml".into(), "http://www.w3.org/XML/1998/namespace".into())]]; + let mut count = 0usize; + let mut opaque_budget = MAX_OPAQUE_NAME_BYTES; + loop { + let before = pos(&reader); + let ev = reader.read_event().map_err(|e| XmpError::Xml(e.to_string()))?; + let after = pos(&reader); + match ev { + Event::Start(ref e) | Event::Empty(ref e) => { + count += 1; + if count > 200_000 || stack.len() > 64 { + return Err(XmpError::TooComplex); + } + let empty = matches!(ev, Event::Empty(_)); + let mut scope = Vec::new(); + let mut raw_attrs = Vec::new(); + // repeated names are refused with a set, in linear time (quick-xml's own check is quadratic) + let mut seen = HashSet::new(); + for a in e.attributes().with_checks(false) { + let a = a.map_err(|e| XmpError::Xml(e.to_string()))?; + let k = String::from_utf8_lossy(a.key.as_ref()).into_owned(); + if !seen.insert(k.clone()) { + return Err(XmpError::Xml(format!("repeated attribute {k}"))); + } + let v = String::from_utf8_lossy(&a.value).into_owned(); + if k == "xmlns" { + scope.push((String::new(), unescape_attr(&v))); + } else if let Some(p) = k.strip_prefix("xmlns:") { + scope.push((p.to_string(), unescape_attr(&v))); + } + raw_attrs.push((k, v)); + } + scopes.push(scope); + let raw_name = String::from_utf8_lossy(e.name().as_ref()).into_owned(); + let name = canonical_name(&raw_name, &scopes, extra, &mut opaque_budget)?; + let kind = match stack.last() { + Some(Kind::Rdf) if name == "rdf:Description" => { + let attrs = raw_attrs + .into_iter() + .map(|(raw, value)| Ok(Attr { name: canonical_name(&raw, &scopes, extra, &mut opaque_budget)?, raw, value })) + .collect::>()?; + out.descs.push(Desc { tag_start: before, tag_end: after, raw_name, empty, attrs, props: Vec::new(), end: after }); + Kind::Desc(out.descs.len() - 1) + } + Some(Kind::Desc(i)) => Kind::Prop { desc: *i, start: before, name }, + _ if name == "rdf:RDF" && out.rdf_close.is_none() => Kind::Rdf, + _ => Kind::Other, + }; + if empty { + scopes.pop(); + if let Kind::Prop { desc, start, name } = kind + && let Some(d) = out.descs.get_mut(desc) + { + d.props.push(Prop { name, start, end: after }); + } + } else { + stack.push(kind); + } + } + Event::End(_) => { + scopes.pop(); + match stack.pop() { + Some(Kind::Prop { desc, start, name }) => { + if let Some(d) = out.descs.get_mut(desc) { + d.props.push(Prop { name, start, end: after }); + } + } + Some(Kind::Desc(i)) => { + if let Some(d) = out.descs.get_mut(i) { + d.end = after; + } + } + Some(Kind::Rdf) => out.rdf_close = out.rdf_close.or(Some(before)), + Some(Kind::Other) => {} + None => return Err(XmpError::Xml("unbalanced end tag".into())), + } + } + Event::Eof => break, + _ => {} + } + } + if !stack.is_empty() { + return Err(XmpError::Xml("unclosed element".into())); + } + Ok(out) +} + +/// `start` moved back over the indentation and line break before it (so a removed property +/// takes its line with it). +fn line_start(s: &str, start: usize) -> usize { + let b = s.as_bytes(); + let mut i = start; + while i > 0 && matches!(b.get(i - 1), Some(b' ' | b'\t')) { + i -= 1; + } + if i > 0 && b.get(i - 1) == Some(&b'\n') { + i -= 1; + if i > 0 && b.get(i - 1) == Some(&b'\r') { + i -= 1; + } + } + i +} + +/// Merge `fresh` (a packet as [`crate::write_xmp_with`] writes it) into `existing` following +/// `rules` (see the module docs). Fails — and the caller must not overwrite `existing` blindly — +/// when `existing` isn't a well-formed XMP packet. +pub fn merge_xmp(existing: &str, fresh: &str, rules: MergeRules) -> Result { + let (bom, old) = match existing.strip_prefix('\u{feff}') { + Some(rest) => ("\u{feff}", rest), + None => ("", existing), + }; + parse_xmp_with(old, rules.namespaces)?; + let ex = scan(old, rules.namespaces)?; + let rdf_close = ex.rdf_close.ok_or_else(|| XmpError::Xml("no rdf:RDF element".into()))?; + let new = scan(fresh, rules.namespaces)?; + let fresh_desc = new.descs.first().ok_or_else(|| XmpError::Xml("the new packet has no rdf:Description".into()))?; + let fresh_names: BTreeSet<&str> = fresh_desc.props.iter().map(|p| p.name.as_str()).collect(); + let remove = |name: &str| listed(rules.owned, name) || (listed(rules.owned_if_present, name) && fresh_names.contains(name)); + let write = |name: &str| listed(rules.owned, name) || listed(rules.owned_if_present, name); + + // edits on `old`: (start, end, replacement), applied back to front + let mut edits: Vec<(usize, usize, String)> = Vec::new(); + for d in &ex.descs { + let gone_attrs: Vec<&Attr> = d.attrs.iter().filter(|a| !is_meta_attr(&a.name) && remove(&a.name)).collect(); + let gone_props: Vec<&Prop> = d.props.iter().filter(|p| remove(&p.name)).collect(); + if gone_attrs.is_empty() && gone_props.is_empty() { + continue; + } + let attrs_left = d.attrs.iter().any(|a| !is_meta_attr(&a.name) && !remove(&a.name)); + if !attrs_left && gone_props.len() == d.props.len() { + // nothing of its own left: drop the whole description + edits.push((line_start(old, d.tag_start), d.end, String::new())); + continue; + } + if !gone_attrs.is_empty() { + let mut tag = format!("<{}", d.raw_name); + for a in d.attrs.iter().filter(|a| is_meta_attr(&a.name) || !remove(&a.name)) { + tag.push_str(&format!("\n {}=\"{}\"", a.raw, a.value.replace('"', """))); + } + tag.push_str(if d.empty { "/>" } else { ">" }); + edits.push((d.tag_start, d.tag_end, tag)); + } + for p in gone_props { + edits.push((line_start(old, p.start), p.end, String::new())); + } + } + // the new description: its own namespace declarations, the properties to write + let mut block = String::from("\n '); + let mut written = 0; + for p in fresh_desc.props.iter().filter(|p| write(&p.name)) { + let text = fresh.get(p.start..p.end).ok_or_else(|| XmpError::Xml("bad property span".into()))?; + block.push_str("\n "); + block.push_str(text); + written += 1; + } + block.push_str("\n "); + if written > 0 { + let at = old.get(..rdf_close).map_or(rdf_close, |h| h.trim_end().len()); + edits.push((at, at, block)); + } + edits.sort_by_key(|e| (e.0, e.1)); + // overlapping edits can't happen (properties sit inside their description, which is either + // dropped whole or edited piecewise), but never splice blindly + let mut out = String::with_capacity(old.len() + 4096); + out.push_str(bom); + let mut at = 0; + for (s, e, r) in &edits { + if *s < at { + return Err(XmpError::Xml("overlapping edits".into())); + } + out.push_str(old.get(at..*s).ok_or_else(|| XmpError::Xml("bad span".into()))?); + out.push_str(r); + at = *e; + } + out.push_str(old.get(at..).ok_or_else(|| XmpError::Xml("bad span".into()))?); + + // read back: everything written must come out as written + let merged = parse_xmp_with(&out, rules.namespaces)?; + let want = parse_xmp_with(fresh, rules.namespaces)?; + for (k, v) in &want.properties { + let top = k.split('/').next().unwrap_or(k); + if write(top) && merged.properties.get(k) != Some(v) { + return Err(XmpError::Xml(format!("{k} did not survive the merge"))); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::xmp::parse_xmp_with; + use crate::{Metadata, XmpData, XmpWrite, write_xmp_with}; + + /// A program's own namespace, as an application would pass it. + const LC: XmpNamespace = XmpNamespace { prefix: "lc", uri: "http://ns.example.com/lc/1.0/" }; + + fn write_lc(meta: &Metadata, properties: &[(&str, &str)]) -> String { + write_xmp_with(meta, &XmpWrite { toolkit: "Editor", namespace: Some(LC), properties }) + } + + fn parse_xmp(s: &str) -> Result { + parse_xmp_with(s, &[LC]) + } + + const RULES: MergeRules = MergeRules { + owned: &["xmp:Rating", "xmp:Label", "dc:subject", "dc:title", "lc:*"], + owned_if_present: &["exif:DateTimeOriginal"], + namespaces: &[LC], + }; + + /// A sidecar as another raw developer writes it: attribute-form simple properties, develop + /// settings, a structured history, an unknown namespace. + const FOREIGN: &str = " + + + + + + + + + + + 0, 0 + 255, 255 + + + + + old + + + + + +"; + + /// With the packet's whitespace padding and trailer. + fn foreign() -> String { + format!("{FOREIGN}{}\n", " ".repeat(100)) + } + + fn fresh(rating: i8, keywords: &[&str]) -> String { + let m = crate::Metadata { + software: Some("Editor".into()), + rating: Some(rating), + keywords: keywords.iter().map(|k| k.to_string()).collect(), + ..Default::default() + }; + write_lc(&m, &[("settings", "{\"a\":1}"), ("flag", "pick")]) + } + + #[test] + fn keeps_foreign_properties_and_replaces_owned_ones() { + let out = merge_xmp(&foreign(), &fresh(4, &["new", "two"]), RULES).unwrap(); + let d = parse_xmp(&out).unwrap(); + let p = |k: &str| d.properties.get(k).cloned().unwrap_or_default(); + assert_eq!(p("xmp:Rating"), vec!["4"]); + assert_eq!(p("dc:subject"), vec!["new", "two"]); + assert_eq!(p("lc:settings"), vec!["{\"a\":1}"]); + assert_eq!(p("lc:flag"), vec!["pick"]); + assert_eq!(p("xmp:CreatorTool"), vec!["Other App"], "not ours: kept"); + assert_eq!(p("crs:Exposure2012"), vec!["+0.50"]); + assert_eq!(p("foo:Thing").len(), 1); + // untouched parts are byte for byte the same + for part in [ + " \n \n \n \n ", + " \n \n 0, 0", + "crs:Contrast2012=\"-12\"", + "foo:Thing=\"a & b\"", + "\n", + &format!("\n{}\n", " ".repeat(100)), + ] { + assert!(out.contains(part), "lost {part:?} in\n{out}"); + } + assert!(!out.contains("xmp:Rating=\"2\"")); + assert!(!out.contains("old")); + // saving again changes only our part, and doesn't pile up descriptions + let again = merge_xmp(&out, &fresh(1, &[]), RULES).unwrap(); + let d = parse_xmp(&again).unwrap(); + assert_eq!(d.properties.get("xmp:Rating"), Some(&vec!["1".to_string()])); + assert!(!d.properties.contains_key("dc:subject"), "a cleared field is cleared"); + assert_eq!(again.matches("\ + "; + let out = merge_xmp(ex, &fresh(3, &[]), RULES).unwrap(); + let d = parse_xmp(&out).unwrap(); + assert_eq!(d.properties.get("xmp:Rating"), Some(&vec!["3".to_string()])); + assert_eq!(d.properties.get("exif:DateTimeOriginal"), Some(&vec!["2020-01-01T00:00:00".to_string()]), "fresh has none: kept"); + let mut m = crate::Metadata { rating: Some(3), ..Default::default() }; + m.capture_time = crate::DateTime::parse_iso("2024-05-06T07:08:09"); + let out = merge_xmp(ex, &write_lc(&m, &[]), RULES).unwrap(); + let d = parse_xmp(&out).unwrap(); + assert_eq!(d.properties.get("exif:DateTimeOriginal"), Some(&vec!["2024-05-06T07:08:09".to_string()])); + } + + #[test] + fn preserves_foreign_properties_using_known_prefixes() { + for property in [r#"lc:Secret="keep-me"/>"#, r#">keep-me"#] { + let desc = format!(r#"{desc}"#); + let out = merge_xmp(&old, &fresh(4, &[]), RULES).unwrap(); + assert!(out.contains(&desc), "lost foreign description: {out}"); + assert_eq!(parse_xmp(&out).unwrap().metadata.rating, Some(4)); + assert_eq!(merge_xmp(&out, &fresh(4, &[]), RULES).unwrap(), out); + } + } + + #[test] + fn namespace_aliases_remain_distinct_during_merge() { + // The alias URI includes a character reference: scanning and parsing must resolve it identically. + let foreign = r#""#; + let old = format!( + r#"{foreign} + + "# + ); + let out = merge_xmp(&old, &fresh(5, &[]), RULES).unwrap(); + assert!(out.contains(foreign), "lost foreign rating: {out}"); + assert!(!out.contains("a:Rating="), "genuine alias must be replaced: {out}"); + assert_eq!(parse_xmp(&out).unwrap().metadata.rating, Some(5)); + assert_eq!(merge_xmp(&out, &fresh(5, &[]), RULES).unwrap(), out); + } + + #[test] + fn opaque_namespace_expansion_is_bounded() { + // A roughly 40 KiB packet must not expand the same long URI into every property name indefinitely. + let packet = format!( + r#"{}"#, + "a".repeat(1024), + "".repeat(5000) + ); + assert_eq!(parse_xmp(&packet).unwrap_err(), XmpError::TooComplex); + assert!(matches!(scan(&packet, &[LC]), Err(XmpError::TooComplex))); + } + + #[test] + fn many_attributes_scan_in_linear_time() { + // quick-xml's duplicate-name check compares each attribute with all earlier ones: 200 000 attributes on + // one description took over 20 s to scan, about 30 ms with the set + let n = 200_000; + let mut s = String::from( + "\ + "); + let t = std::time::Instant::now(); + let found = scan(&s, &[LC]).unwrap(); + let took = t.elapsed(); + assert_eq!(found.descs[0].attrs.len(), n + 2, "every attribute is read"); + assert!(took < std::time::Duration::from_secs(5), "{n} attributes took {took:?}"); + } + + #[test] + fn refuses_what_it_cannot_read() { + for bad in ["", "not xml at all", "", ""] { + assert!(merge_xmp(bad, &fresh(1, &[]), RULES).is_err(), "{bad:?}"); + } + } + + #[test] + fn refuses_a_repeated_attribute() { + // not well-formed XML (strict readers such as expat refuse it): no merge, so the caller keeps a backup. + // An owned name, a foreign one and one on a nested element (a merge would keep both copies of the last two). + for (once, twice) in [ + ("xmp:Rating=\"2\"", "xmp:Rating=\"2\" xmp:Rating=\"3\""), + ("foo:Thing=\"a & b\"", "foo:Thing=\"a & b\" foo:Thing=\"c\""), + ("stEvt:action=\"saved\"", "stEvt:action=\"saved\" stEvt:action=\"saved\""), + ] { + let ex = foreign().replace(once, twice); + assert!(merge_xmp(&ex, &fresh(1, &[]), RULES).is_err(), "{twice}"); + } + } +} diff --git a/crates/metadata/tests/it/main.rs b/crates/metadata/tests/it/main.rs new file mode 100644 index 000000000..f56fd47f0 --- /dev/null +++ b/crates/metadata/tests/it/main.rs @@ -0,0 +1,3 @@ +//! The integration tests, one module per file, linked into one test binary. + +mod robust; diff --git a/crates/metadata/tests/it/robust.rs b/crates/metadata/tests/it/robust.rs new file mode 100644 index 000000000..ce2b8f40e --- /dev/null +++ b/crates/metadata/tests/it/robust.rs @@ -0,0 +1,70 @@ +use craft_metadata::{MergeRules, Metadata, XmpNamespace, XmpWrite, extract, merge_xmp, parse_iptc, parse_xmp_with, read_exif, write_xmp_with}; + +const LC: XmpNamespace = XmpNamespace { prefix: "lc", uri: "http://ns.example.com/lc/1.0/" }; + +fn write_xmp(meta: &Metadata, settings: Option<&str>) -> String { + let properties: Vec<(&str, &str)> = settings.map(|s| ("settings", s)).into_iter().collect(); + write_xmp_with(meta, &XmpWrite { toolkit: "Editor", namespace: Some(LC), properties: &properties }) +} + +fn parse_xmp(s: &str) -> Result { + parse_xmp_with(s, &[LC]) +} + +fn settings(d: &craft_metadata::XmpData) -> Option { + d.properties.get("lc:settings").and_then(|v| v.first()).cloned() +} +use proptest::prelude::*; + +proptest! { + #![proptest_config(ProptestConfig { cases: 1500, .. ProptestConfig::default() })] + + /// Random bytes behind each container's magic: reading them yields whatever can be read and + /// never panics. + #[test] + fn random_containers_never_panic(mut data in proptest::collection::vec(any::(), 0..600), kind in 0usize..7) { + let heads: [&[u8]; 7] = [b"\xff\xd8\xff\xe1", b"\x89PNG\r\n\x1a\n", b"RIFF\0\0\0\0WEBP", b"II*\0", b"MM\0*", b"\0\0\0\x18ftypcrx ", b"\0\0\0\x18ftypheic"]; + let h = heads[kind]; + if data.len() >= h.len() { + data[..h.len()].copy_from_slice(h); + } + let _ = extract(&data); + let _ = read_exif(&data); + let _ = parse_iptc(&data); + } + + #[test] + fn random_text_xmp_never_panics(s in ".{0,400}") { + let _ = parse_xmp(&s); + } + + /// Merging into arbitrary (also damaged) packets never panics, and a merge that succeeds + /// reads back with the written values. + #[test] + fn merge_never_panics(cut in 0usize..900, junk in ".{0,20}", rating in 0i8..=5) { + let base = write_xmp(&Metadata { title: Some("t".into()), rating: Some(1), ..Default::default() }, Some("{}")); + let mut s: String = base.chars().take(cut).collect(); + s.push_str(&junk); + s.extend(base.chars().skip(cut)); + let fresh = write_xmp(&Metadata { rating: Some(rating), ..Default::default() }, Some("{\"x\":1}")); + let rules = MergeRules { owned: &["xmp:Rating", "dc:title", "lc:*"], owned_if_present: &[], namespaces: &[LC] }; + if let Ok(out) = merge_xmp(&s, &fresh, rules) { + let d = parse_xmp(&out).unwrap(); + prop_assert_eq!(d.metadata.rating, Some(rating)); + prop_assert_eq!(settings(&d), Some("{\"x\":1}".to_string())); + } + } + + #[test] + fn user_text_roundtrips(title in "[a-zA-Z0-9 &<>\"'éü€;|/\\\\]{1,40}", kw in proptest::collection::vec("[a-zA-Z0-9 &<>\"'|]{1,12}", 0..5), rating in -1i8..=5) { + let title = title.trim().to_string(); + prop_assume!(!title.is_empty()); + let kw: Vec = kw.into_iter().map(|k| k.trim().to_string()).filter(|k| !k.is_empty()).collect(); + let m = Metadata { title: Some(title.clone()), keywords: kw.clone(), rating: Some(rating), ..Default::default() }; + let d = parse_xmp(&write_xmp(&m, Some(&title))).unwrap(); + prop_assert_eq!(settings(&d), Some(title.clone())); + prop_assert_eq!(d.metadata.title, Some(title)); + prop_assert_eq!(d.metadata.keywords, kw); + prop_assert_eq!(d.metadata.rating, Some(rating)); + } +} From 58f6c4b9e8a543130f55f7ea948cd55b28945b09 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:23:15 -0500 Subject: [PATCH 11/31] craft-idml, craft-epub: read and write through craft-zip (#950) * craft-zip: ZIP preflight, bounded entry read and Budget (from craft-archive, #129) The ZIP half of the craft-archive proposal (#126, #129): an allocation-free check of the end records and central directory that runs before a ZIP reader indexes the archive, a bounded entry read that distrusts the declared size, and the named Budget both charge. The XML half of #129 is not carried: craft-xml bounds XML now. * craft-zip: Archive (entry-count guard, duplicate names, budgeted reads) and Writer check_entry_count is craft-opc's guard: the declared entry count, ZIP64 included, checked on the raw bytes before zip parses the central directory. Archive reads entries within a per-entry limit and a total Budget through read_entry. Duplicate names are found by counting the central directory headers against the entries zip kept. Writer is craft-opc's ZipOut: stored or deflated entries, ZIP64 for large ones, one fixed timestamp. * craft-idml, craft-epub: read and write through craft-zip Both crates drop their own bounded ZIP reads and zip writers. They gain the entry-count check before the archive is indexed (ZIP64 included) and archives whose bytes depend only on their content (one fixed timestamp). --------- Co-authored-by: skylar taylor-barrick --- Cargo.lock | 4 +- crates/epub/Cargo.toml | 9 +++-- crates/epub/README.md | 3 +- crates/epub/src/read.rs | 46 ++++++++++------------ crates/epub/src/write.rs | 14 ++----- crates/epub/tests/read.rs | 8 ++++ crates/idml/Cargo.toml | 9 +++-- crates/idml/README.md | 3 +- crates/idml/src/package.rs | 76 +++++++++++++++--------------------- crates/idml/tests/package.rs | 13 ++++++ 10 files changed, 92 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b531fce60..c7fe9c680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1025,7 +1025,7 @@ name = "craft-epub" version = "0.1.0" dependencies = [ "craft-xml", - "flate2", + "craft-zip", "thiserror 2.0.21", "zip", ] @@ -1185,7 +1185,7 @@ name = "craft-idml" version = "0.1.0" dependencies = [ "craft-xml", - "flate2", + "craft-zip", "thiserror 2.0.21", "zip", ] diff --git a/crates/epub/Cargo.toml b/crates/epub/Cargo.toml index 690a89778..67875fe89 100644 --- a/crates/epub/Cargo.toml +++ b/crates/epub/Cargo.toml @@ -13,8 +13,9 @@ workspace = true [dependencies] craft-xml = { workspace = true } thiserror = { workspace = true } -# `deflate-flate2` with flate2's default miniz_oxide backend, as craft-update pins it (zip's own -# `deflate` feature would switch a whole app's flate2 to zlib-rs). +craft-zip = { workspace = true } + +[dev-dependencies] +# The tests build hostile archives by hand. zip = { version = "9", default-features = false, features = ["deflate-flate2"] } -# The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +craft-zip = { workspace = true, features = ["test-support"] } diff --git a/crates/epub/README.md b/crates/epub/README.md index c15c13876..c9ba7d134 100644 --- a/crates/epub/README.md +++ b/crates/epub/README.md @@ -94,7 +94,8 @@ for path in &spine { | `max_total_bytes` (all entries read from one package) | 256 MiB | `Error::TooLarge` | | `max_manifest`, `max_spine` | 10,000 each | further items are ignored | -Sizes are checked against what an entry really expands to, not what the archive declares. The two +The ZIP layer is craft-zip: the declared entry count is checked before the archive is indexed, and +sizes are checked against what an entry really expands to, not what the archive declares. The two XML documents are read with craft-xml's limits (depth 256, 2,000,000 nodes) and `max_entry_bytes`. ## Where it came from diff --git a/crates/epub/src/read.rs b/crates/epub/src/read.rs index ac93c5b0e..0540e70f8 100644 --- a/crates/epub/src/read.rs +++ b/crates/epub/src/read.rs @@ -1,7 +1,6 @@ //! An EPUB 2 or 3 package, read: container, package document, and bounded entry reads. use std::collections::HashMap; -use std::io::{Cursor, Read}; use craft_xml::{El, Local, Options}; @@ -224,10 +223,8 @@ pub fn parse_package(xml: &[u8], opf_dir: &str, limits: &Limits) -> Result { - zip: zip::ZipArchive>, + zip: craft_zip::Archive<'a>, limits: Limits, - /// Bytes the entries still to be read may expand to. - left: u64, /// The archive path of the package document. pub opf_path: String, pub package: Package, @@ -242,11 +239,16 @@ impl<'a> Epub<'a> { /// Reads the container and the package document. The `mimetype` entry is not required: /// packages without one, or with it compressed, are common. pub fn open_with(bytes: &'a [u8], limits: Limits) -> Result { - let zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| Error::NotEpub(e.to_string()))?; - if zip.len() > limits.max_entries { - return Err(Error::TooManyEntries { found: zip.len(), limit: limits.max_entries }); - } - let mut epub = Epub { zip, limits, left: limits.max_total_bytes, opf_path: String::new(), package: Package::default() }; + let zip = + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: limits.max_total_bytes }) + .map_err(|e| match e { + craft_zip::Error::TooManyEntries { declared, limit } => Error::TooManyEntries { + found: usize::try_from(declared).unwrap_or(usize::MAX), + limit: usize::try_from(limit).unwrap_or(usize::MAX), + }, + e => Error::NotEpub(e.to_string()), + })?; + let mut epub = Epub { zip, limits, opf_path: String::new(), package: Package::default() }; let container = epub.read("META-INF/container.xml")?.ok_or_else(|| Error::NotEpub("META-INF/container.xml is missing".into()))?; let opf_path = parse_container(&container)?.ok_or_else(|| Error::NotEpub("container.xml names no package document".into()))?; // The container gives a path from the archive's root; resolving it also drops a leading `/`. @@ -262,28 +264,20 @@ impl<'a> Epub<'a> { /// Does the archive have an entry `path`? pub fn contains(&self, path: &str) -> bool { - self.zip.index_for_name(path).is_some() + self.zip.contains(path) } /// The bytes of archive entry `path`, or `None` when the archive has no such entry. An entry /// past the limits is an error, and so is one that expands to more than it declared. pub fn read(&mut self, path: &str) -> Result>, Error> { - let Some(index) = self.zip.index_for_name(path).filter(|_| !path.ends_with('/')) else { return Ok(None) }; - let part = |msg: String| Error::Part { part: path.to_string(), msg }; - let f = self.zip.by_index(index).map_err(|e| part(e.to_string()))?; - let limit = self.limits.max_entry_bytes.min(self.left); - let too_large = || Error::TooLarge { part: path.to_string(), limit }; - if f.size() > limit { - return Err(too_large()); + if path.ends_with('/') { + return Ok(None); } - let mut buf = Vec::new(); - f.take(limit.saturating_add(1)).read_to_end(&mut buf).map_err(|e| part(e.to_string()))?; - let n = buf.len() as u64; - if n > limit { - return Err(too_large()); - } - self.left = self.left.saturating_sub(n); - Ok(Some(buf)) + let limit = self.limits.max_entry_bytes.min(self.zip.budget().remaining()); + self.zip.read(path, self.limits.max_entry_bytes).map_err(|e| match e { + craft_zip::Error::EntryTooLarge { .. } | craft_zip::Error::BudgetSpent { .. } => Error::TooLarge { part: path.to_string(), limit }, + e => Error::Part { part: path.to_string(), msg: e.to_string() }, + }) } /// [`Epub::read`] for what a file of the package refers to: `href` as written in the file at @@ -295,6 +289,6 @@ impl<'a> Epub<'a> { /// Bytes the entries still to be read may expand to. pub fn budget_left(&self) -> u64 { - self.left + self.zip.budget().remaining() } } diff --git a/crates/epub/src/write.rs b/crates/epub/src/write.rs index 0c77951a8..fbc741b55 100644 --- a/crates/epub/src/write.rs +++ b/crates/epub/src/write.rs @@ -3,7 +3,6 @@ use std::borrow::Cow; use std::collections::HashSet; use std::fmt::Write as _; -use std::io::Write as _; use craft_xml::{esc, esc_attr, is_name}; @@ -316,10 +315,8 @@ pub fn write(book: &Book) -> Result, Error> { w3cdtf(&book.metadata.modified).ok_or_else(|| Error::Invalid(format!("`{}` is not a modification date", book.metadata.modified)))?; let opf = package_document(book, &modified); let nav = nav_document(book); - let fail = |e: &dyn std::fmt::Display| Error::Invalid(format!("the archive could not be written: {e}")); - let mut z = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); - let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - let deflated = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let fail = |e: craft_zip::Error| Error::Invalid(e.to_string()); + let mut z = craft_zip::Writer::new(); let own = [ ("mimetype".to_string(), MIMETYPE.as_bytes(), true), ("META-INF/container.xml".to_string(), CONTAINER.as_bytes(), false), @@ -328,10 +325,7 @@ pub fn write(book: &Book) -> Result, Error> { ]; let resources = book.resources.iter().map(|r| (format!("{ROOT}/{}", r.path), r.data.as_ref(), r.stored)); for (name, data, store) in own.into_iter().chain(resources) { - // More than 4 GiB needs the ZIP64 form. - let options = if store { stored } else { deflated }.large_file(data.len() >= u32::MAX as usize); - z.start_file(name, options).map_err(|e| fail(&e))?; - z.write_all(data).map_err(|e| fail(&e))?; + z.add(&name, data, if store { craft_zip::Compression::Stored } else { craft_zip::Compression::Deflated }).map_err(fail)?; } - z.finish().map(std::io::Cursor::into_inner).map_err(|e| fail(&e)) + z.finish().map_err(fail) } diff --git a/crates/epub/tests/read.rs b/crates/epub/tests/read.rs index 88fb27df6..dd1999ce4 100644 --- a/crates/epub/tests/read.rs +++ b/crates/epub/tests/read.rs @@ -252,3 +252,11 @@ fn a_package_document_past_the_entry_limit_is_refused() { let limits = Limits { max_entry_bytes: (xml.len() - 1) as u64, ..Limits::DEFAULT }; assert!(matches!(parse_package(&xml, "", &limits), Err(Error::Part { .. }))); } + +/// A ZIP64 end record can declare millions of entries in a small file; the count is refused on +/// the raw bytes, before the archive is indexed. +#[test] +fn an_archive_declaring_too_many_entries_is_refused_before_it_is_indexed() { + let bomb = craft_zip::zip64_entry_count_sample(100_001); + assert_eq!(Epub::open(&bomb).err(), Some(Error::TooManyEntries { found: 100_001, limit: 100_000 })); +} diff --git a/crates/idml/Cargo.toml b/crates/idml/Cargo.toml index 660456229..41df76d7e 100644 --- a/crates/idml/Cargo.toml +++ b/crates/idml/Cargo.toml @@ -13,8 +13,9 @@ workspace = true [dependencies] craft-xml = { workspace = true } thiserror = { workspace = true } -# `deflate-flate2` with flate2's default miniz_oxide backend, as craft-update pins it (zip's own -# `deflate` feature would switch a whole app's flate2 to zlib-rs). +craft-zip = { workspace = true } + +[dev-dependencies] +# The tests build hostile archives by hand. zip = { version = "9", default-features = false, features = ["deflate-flate2"] } -# The backend of zip's deflate: `deflate-flate2` leaves the choice to the build. -flate2 = { version = "1", default-features = false, features = ["rust_backend"] } +craft-zip = { workspace = true, features = ["test-support"] } diff --git a/crates/idml/README.md b/crates/idml/README.md index 0ef741bf2..16435a361 100644 --- a/crates/idml/README.md +++ b/crates/idml/README.md @@ -53,7 +53,8 @@ for line in &doc.warnings { /* a part designmap.xml lists and the package lacks | `xml.max_depth` | 256 | `Error::Part` | | `xml.max_nodes`, `xml.max_attrs_total` (one part) | 8,000,000 and 32,000,000 | `Error::Part` | -Sizes are checked against what a part really expands to, not what the archive declares. +The ZIP layer is craft-zip: the declared entry count is checked before the archive is indexed, and +sizes are checked against what a part really expands to, not what the archive declares. ## Writing diff --git a/crates/idml/src/package.rs b/crates/idml/src/package.rs index c8b6ebec2..dab9d3626 100644 --- a/crates/idml/src/package.rs +++ b/crates/idml/src/package.rs @@ -1,7 +1,6 @@ //! The ZIP container: bounded part reads, lenient part names, designmap includes, and the writer. use std::collections::HashMap; -use std::io::{Cursor, Read, Write}; use craft_xml::El; @@ -34,12 +33,10 @@ impl Default for Limits { /// An IDML package whose parts are decompressed only when asked for, within byte budgets that /// don't trust the sizes the archive claims. pub struct Package<'a> { - zip: zip::ZipArchive>, + zip: craft_zip::Archive<'a>, /// Normalised part name → entry name, first entry wins (zip order, so the choice is stable). lenient: HashMap, limits: Limits, - /// Bytes the parts still to be read may expand to. - left: u64, } /// `designmap.xml` with the parts it includes. @@ -63,46 +60,40 @@ impl<'a> Package<'a> { Self::open_with(bytes, Limits::DEFAULT) } + /// The declared entry count is checked against [`Limits::max_entries`] before the archive + /// is indexed. pub fn open_with(bytes: &'a [u8], limits: Limits) -> Result { - let zip = zip::ZipArchive::new(Cursor::new(bytes)).map_err(|e| Error::NotIdml(e.to_string()))?; - if zip.len() > limits.max_entries { - return Err(Error::TooManyEntries { found: zip.len(), limit: limits.max_entries }); - } + let zip = + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: limits.max_total_bytes }) + .map_err(|e| match e { + craft_zip::Error::TooManyEntries { declared, limit } => Error::TooManyEntries { + found: usize::try_from(declared).unwrap_or(usize::MAX), + limit: usize::try_from(limit).unwrap_or(usize::MAX), + }, + e => Error::NotIdml(e.to_string()), + })?; let mut lenient: HashMap = HashMap::new(); - for i in 0..zip.len() { - let Some(Ok(name)) = zip.name_for_index(i) else { continue }; - if name.ends_with('/') { - continue; - } + for name in zip.names().filter(|n| !n.ends_with('/')) { lenient.entry(normalize_part_name(&name)).or_insert_with(|| name.into_owned()); } - Ok(Package { zip, lenient, limits, left: limits.max_total_bytes }) + Ok(Package { zip, lenient, limits }) } /// The bytes of part `name` (matched exactly, then leniently: writers differ in /// percent-encoding, case, path separators and a leading `./` or `/`), or `None` if the /// package has no such part. pub fn part(&mut self, name: &str) -> Result>> { - let exact = self.zip.index_for_name(name).filter(|_| !name.ends_with('/')); - let Some(i) = exact.or_else(|| self.lenient.get(&normalize_part_name(name)).and_then(|n| self.zip.index_for_name(n))) else { + let exact = (self.zip.contains(name) && !name.ends_with('/')).then_some(name); + let Some(stored) = exact.or_else(|| self.lenient.get(&normalize_part_name(name)).map(String::as_str)) else { return Ok(None); }; - let part = self.zip.name_for_index(i).and_then(|n| n.ok()).map_or_else(|| name.to_string(), |n| n.into_owned()); - let f = self.zip.by_index(i).map_err(|e| Error::NotIdml(e.to_string()))?; - let limit = self.limits.max_part_bytes.min(self.left); - if f.size() > limit { - return Err(Error::TooLarge { part, limit }); - } - let mut buf = Vec::new(); - if let Err(e) = f.take(limit.saturating_add(1)).read_to_end(&mut buf) { - return Err(Error::Part { part, msg: e.to_string() }); - } - let n = buf.len() as u64; - if n > limit { - return Err(Error::TooLarge { part, limit }); - } - self.left = self.left.saturating_sub(n); - Ok(Some(buf)) + let limit = self.limits.max_part_bytes.min(self.zip.budget().remaining()); + let stored = stored.to_string(); + self.zip.read(&stored, self.limits.max_part_bytes).map_err(|e| match e { + craft_zip::Error::EntryTooLarge { .. } | craft_zip::Error::BudgetSpent { .. } => Error::TooLarge { part: stored.clone(), limit }, + craft_zip::Error::Entry { msg, .. } => Error::Part { part: stored.clone(), msg }, + e => Error::NotIdml(e.to_string()), + }) } /// Part `name` as an element tree ([`crate::parse_part`]). @@ -114,7 +105,7 @@ impl<'a> Package<'a> { /// Bytes the parts still to be read may expand to. pub fn budget_left(&self) -> u64 { - self.left + self.zip.budget().remaining() } /// Reads `designmap.xml` and every part it includes. @@ -173,19 +164,14 @@ fn normalize_part_name(name: &str) -> String { } /// Zips an IDML package: the stored `mimetype` first, then `parts` in order as (name, bytes). +/// Equal parts give equal bytes. pub fn write_package, D: AsRef<[u8]>>(parts: &[(N, D)]) -> Result> { - use zip::write::SimpleFileOptions; - let fail = |e: &dyn std::fmt::Display| Error::Write(e.to_string()); - let mut w = zip::ZipWriter::new(Cursor::new(Vec::new())); - let stored = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - let deflated = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); - w.start_file("mimetype", stored).map_err(|e| fail(&e))?; - w.write_all(MIMETYPE.as_bytes()).map_err(|e| fail(&e))?; + use craft_zip::Compression; + let fail = |e: craft_zip::Error| Error::Write(e.to_string()); + let mut w = craft_zip::Writer::new(); + w.add("mimetype", MIMETYPE.as_bytes(), Compression::Stored).map_err(fail)?; for (name, data) in parts { - let data = data.as_ref(); - // More than 4 GiB needs the ZIP64 form. - w.start_file(name.as_ref(), deflated.large_file(data.len() >= u32::MAX as usize)).map_err(|e| fail(&e))?; - w.write_all(data).map_err(|e| fail(&e))?; + w.add(name.as_ref(), data.as_ref(), Compression::Deflated).map_err(fail)?; } - w.finish().map(Cursor::into_inner).map_err(|e| fail(&e)) + w.finish().map_err(fail) } diff --git a/crates/idml/tests/package.rs b/crates/idml/tests/package.rs index 1c7ffd353..03a42d088 100644 --- a/crates/idml/tests/package.rs +++ b/crates/idml/tests/package.rs @@ -244,3 +244,16 @@ fn parts_are_written_one_element_per_line_without_touching_text() { ); assert!(with_props(El::new("A"), Vec::new()).children.is_empty()); } + +/// A ZIP64 end record can declare millions of entries in a small file; the count is refused on +/// the raw bytes, before the archive is indexed. +#[test] +fn a_package_declaring_too_many_entries_is_refused_before_it_is_indexed() { + let bomb = craft_zip::zip64_entry_count_sample(100_001); + assert_eq!(Package::open(&bomb).err(), Some(Error::TooManyEntries { found: 100_001, limit: 100_000 })); +} + +#[test] +fn equal_parts_give_equal_bytes() { + assert_eq!(sample(), sample()); +} From 16936716aa960dda9a7f1532bb04a53caa3a5c75 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:24:06 -0500 Subject: [PATCH 12/31] Add craft-stream-codecs: the codec modules of PdfCraft's stream filters (#953) LZW, ASCII85, ASCIIHex, RunLength, the TIFF and PNG predictors, bounded CCITT fax decoding (through hayro-ccitt) with Group 4 encoding (through fax), and the lossless JBIG2 encoder with its MQ coder, from pdfcraft-filters at PdfCraft revision 80e26e674d1aa33078e83c8139875047f3289eb6. The crate sits below craft-pdf and craft-tiff: it knows no container. The PDF filter chain, Flate and the bounded JBIG2 decoding entry are not part of it. --- Cargo.lock | 22 +- Cargo.toml | 1 + README.md | 5 + crates/stream-codecs/Cargo.toml | 24 + crates/stream-codecs/README.md | 78 ++ crates/stream-codecs/src/ascii85.rs | 156 +++ crates/stream-codecs/src/asciihex.rs | 106 ++ crates/stream-codecs/src/ccitt.rs | 546 +++++++++ crates/stream-codecs/src/jbig2.rs | 468 ++++++++ crates/stream-codecs/src/jbig2/symbols.rs | 1329 +++++++++++++++++++++ crates/stream-codecs/src/lib.rs | 117 ++ crates/stream-codecs/src/lzw.rs | 314 +++++ crates/stream-codecs/src/mq.rs | 209 ++++ crates/stream-codecs/src/predictor.rs | 356 ++++++ crates/stream-codecs/src/runlength.rs | 117 ++ 15 files changed, 3846 insertions(+), 2 deletions(-) create mode 100644 crates/stream-codecs/Cargo.toml create mode 100644 crates/stream-codecs/README.md create mode 100644 crates/stream-codecs/src/ascii85.rs create mode 100644 crates/stream-codecs/src/asciihex.rs create mode 100644 crates/stream-codecs/src/ccitt.rs create mode 100644 crates/stream-codecs/src/jbig2.rs create mode 100644 crates/stream-codecs/src/jbig2/symbols.rs create mode 100644 crates/stream-codecs/src/lib.rs create mode 100644 crates/stream-codecs/src/lzw.rs create mode 100644 crates/stream-codecs/src/mq.rs create mode 100644 crates/stream-codecs/src/predictor.rs create mode 100644 crates/stream-codecs/src/runlength.rs diff --git a/Cargo.lock b/Cargo.lock index c7fe9c680..2fc2f8292 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1588,6 +1588,18 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-stream-codecs" +version = "0.1.0" +dependencies = [ + "fax", + "hayro-ccitt", + "hayro-jbig2", + "proptest", + "thiserror 2.0.21", + "weezl 0.2.1", +] + [[package]] name = "craft-svg" version = "0.1.0" @@ -2777,7 +2789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" dependencies = [ "color_quant", - "weezl", + "weezl 0.1.12", ] [[package]] @@ -6731,7 +6743,7 @@ dependencies = [ "flate2", "half", "quick-error 2.0.1", - "weezl", + "weezl 0.1.12", "zune-jpeg", ] @@ -7590,6 +7602,12 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "weezl" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4ca08e5ef825b65b056d9efbd95c8750683f0a6d0466d02e96dc2e4e360f3d2" + [[package]] name = "wgpu" version = "30.0.1" diff --git a/Cargo.toml b/Cargo.toml index 7059505ca..3634d3b8a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,6 +91,7 @@ craft-settings = { path = "crates/settings" } craft-silhouette-studio = { path = "crates/silhouette-studio" } craft-svg = { path = "crates/svg" } craft-svg-paths = { path = "crates/svg-paths" } +craft-stream-codecs = { path = "crates/stream-codecs" } craft-text = { path = "crates/text" } craft-tga = { path = "crates/tga" } craft-timeline = { path = "crates/timeline" } diff --git a/README.md b/README.md index 3776d0b56..b48364a81 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,11 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. +- [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that + PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, + CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding. Every + decoder is bounded and returns its partial output on damage. From PdfCraft; a PDF filter chain + is built on it in `craft-pdf`. - [`craft-timeline`](crates/timeline/README.md): editorial timeline interchange: exact media time and SMPTE timecode, import/export reports, media path and `file://` URL handling, Avid Log Exchange (ALE) and CMX 3600 EDL documents, format detection. From FilmCraft. - [`craft-metadata`](crates/metadata/README.md): photo metadata, from LightCraft: EXIF (read and write, maker-note lens and focus point, every tag by name), IPTC-IIM, XMP packets (read, write, and a merge diff --git a/crates/stream-codecs/Cargo.toml b/crates/stream-codecs/Cargo.toml new file mode 100644 index 000000000..7624050e2 --- /dev/null +++ b/crates/stream-codecs/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "craft-stream-codecs" +description = "Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +# CCITT Group 4 encoding (ITU-T T.6). MIT. +fax = "0.2.7" +# CCITT fax decoding, the decoder the vendored PDF reader uses. MIT OR Apache-2.0. +hayro-ccitt = { version = "0.3", default-features = false } +thiserror = { workspace = true } + +[dev-dependencies] +# The JBIG2 decoder the vendored PDF reader uses: reads back what the encoder writes. +hayro-jbig2 = { version = "=0.3.0", default-features = false, features = ["std"] } +proptest = { workspace = true } +weezl = "0.2" diff --git a/crates/stream-codecs/README.md b/crates/stream-codecs/README.md new file mode 100644 index 000000000..3a6c73208 --- /dev/null +++ b/crates/stream-codecs/README.md @@ -0,0 +1,78 @@ +# craft-stream-codecs + +Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share. The crate knows no +container: a caller hands it bytes and parameters. A PDF `/Filter` chain (stage order, limits +across stages, Flate, accounting) belongs to `craft-pdf`, which builds on this crate; a TIFF +reader can use the same LZW, predictor, CCITT and JBIG2 code. + +- **Licence:** MIT OR Apache-2.0. Written from ISO 32000-2 §7.4, TIFF 6.0 §14 (predictor 2) and + RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT). CCITT + decoding (T.4 and T.6) is `hayro-ccitt`'s (MIT OR Apache-2.0), bounded here: this crate has + no CCITT decoder of its own, and no JBIG2 decoder (that is `hayro-jbig2`, which the tests + read the encoder's output back with). JBIG2 encoding + is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 + checks the arithmetic coder byte for byte. +- No `unsafe`. Never panics on input. +- From PdfCraft's `pdfcraft-filters` crate (revision `80e26e674d1aa33078e83c8139875047f3289eb6`): + its codec modules, moved whole. Its filter chain and Flate stay with the PDF code. + +## API + +```rust +pub struct Params { predictor, colors, bits_per_component, columns, early_change } // i64s; Default = 1, 1, 8, 1, 1 +pub enum CodecError { Corrupt { filter, detail }, LimitExceeded(usize), Unsupported(String), Spent } +pub struct Failure { error, partial, discarded } // what a decoder decoded before it failed +pub type Step = Result, Failure>; +pay(work: &mut u64, steps) -> Result<(), CodecError> // an allowance shared across a run of images + +lzw::decode(data, early_change, max) -> Step; lzw::encode(data, early_change) -> Vec +ascii85::decode(data, max) -> Step; ascii85::encode(data) -> Vec +asciihex::decode(data, max) -> Step; asciihex::encode(data) -> Vec +runlength::decode(data, max) -> Step; runlength::encode(data) -> Vec +predictor::decode(¶ms, data, name, rows) -> Step; predictor::encode(¶ms, data) -> Result, CodecError> + +pub struct CcittParams { k, columns, rows, end_of_line, encoded_byte_align, end_of_block, black_is_1 } +decode_ccitt_within(data, ¶ms, max_rows, work: &mut u64) -> Result, CodecError> +encode_g4(data, columns, rows) -> Result, CodecError> + +encode_jbig2_generic(data, columns, rows) -> Result, CodecError> +encode_jbig2(data, columns, rows) -> Result, CodecError> +encode_jbig2_shared(&[(data, columns, rows)]) -> Result, CodecError> +``` + +`max` bounds a decoder's output and gives `LimitExceeded`. `Corrupt` names the codec in `filter` +by its PDF filter name. Bi-level images are rows of 1-bit samples, most significant bit first, +each row on a new byte. + +## Behaviour + +| Codec | Decode | Encode | +|---|---|---| +| LZW | 9–12-bit MSB-first codes, clear (256), end (257), early change on or off; a missing end code or initial clear is tolerated; a full table stops growing | greedy, leading clear, clear when the table fills, end code | +| Predictors | 2 = TIFF (1/2/4/8/16 bits per component), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad parameters or tags are `Corrupt`; with `rows`, data that is exactly that many rows without tag bytes is taken as the samples and reported as damage | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum | +| ASCIIHex | white space ignored, odd digit padded with 0, `>` ends (optional), other characters are `Corrupt` | upper case, newline every 64 digits, `>` | +| ASCII85 | `<~` prefix, `z`, white space, `~>` (or bare `~`, or none), final partial group; `z` inside a group, a 1-character final group, overflow or bad characters are `Corrupt` | `z` for zero groups, newline every 75 characters, `~>` | +| RunLength | 0–127 literal, 129–255 repeat, 128 ends (optional); a truncated run is `Corrupt` | runs of 2 or more repeat, literals up to 128, 128 | +| CCITT fax | Group 3 (1-D and 2-D) and Group 4 through `hayro-ccitt`, as rows of 1-bit samples, black 0 unless `black_is_1`, at most `max_rows` rows (fewer with `rows`), the rows before any damage, up to `MAX_CCITT_COLUMNS` (1,048,576) samples a row; the rows are paid for from `work` before they are decoded | `encode_g4`: Group 4 with an end-of-block mark, 0 bits coded black; up to `MAX_G4_COLUMNS` (32,767) samples a row | +| JBIG2 | not here: `hayro-jbig2` | `encode_jbig2_generic`: one immediate lossless generic region (template 0, typical prediction, MQ coder). `encode_jbig2`: the smaller of that and symbol coding (exact copies only, so the stream decodes to the same bits). `encode_jbig2_shared`: a group of images sharing one symbol dictionary (the PDF `/JBIG2Globals` data) | + +The limits of the JBIG2 encoder (symbols per plan, instances per text region, marks, runs) are +those `hayro-jbig2` reads back; they are documented on the constants in `src/jbig2/symbols.rs`. + +## Tests + +- Unit tests per codec with hand-computed vectors (the ISO 32000-2 LZW example, PNG and TIFF rows). +- LZW is cross-checked against `weezl` (dev-dependency) in both directions for both early-change settings. +- `ccitt.rs`: decoding gives back the rows `encode_g4` was given; Group 3 layouts over seeded + pictures; Group 4 output decoded by `hayro-ccitt` matches the input bits, with property tests + over bitmaps of noise and of runs; any input returns rather than panics. +- `mq.rs`: the arithmetic coder reproduces Annex H.2's 30 bytes from its 256 decisions. +- `jbig2.rs`, `jbig2/symbols.rs`: output decoded by `hayro-jbig2` matches the input bits for + every coding, plan and strip size, with property tests; size and limit refusals. + +## Not here + +A bounded JBIG2 decoding entry (`decode_jbig2_within` in PdfCraft): it needs the caller-owned +work allowance PdfCraft added to its `hayro-jbig2` (`Image::decode_within`, `charge_reading`), +which the copy vendored here does not have yet. Flate (a container's choice of zlib settings and tolerances), DCT and JPEG 2000, CCITT Group 3 +encoding. diff --git a/crates/stream-codecs/src/ascii85.rs b/crates/stream-codecs/src/ascii85.rs new file mode 100644 index 000000000..beb0098fd --- /dev/null +++ b/crates/stream-codecs/src/ascii85.rs @@ -0,0 +1,156 @@ +//! ASCII85Decode (ISO 32000-2 §7.4.3): groups of five characters `!`..`u` encode four +//! bytes base-85, `z` stands for four zero bytes, `~>` is EOD, white space is ignored and +//! a final partial group of n characters (2–5) yields n − 1 bytes. A leading `<~` and a +//! missing EOD are tolerated. + +use crate::{Failure, Step, is_pdf_whitespace}; + +const NAME: &str = "ASCII85Decode"; + +fn push(out: &mut Vec, bytes: &[u8], max: usize) -> Result<(), Failure> { + if out.len() + bytes.len() > max { + return Err(Failure::limit(max)); + } + out.extend_from_slice(bytes); + Ok(()) +} + +pub fn decode(data: &[u8], max: usize) -> Step { + let start = data.iter().position(|&b| !is_pdf_whitespace(b)).unwrap_or(data.len()); + let body = data[start..].strip_prefix(b"<~").unwrap_or(&data[start..]); + let mut out = Vec::with_capacity((body.len() / 5 * 4 + 4).min(max)); + let mut acc: u64 = 0; + let mut n = 0usize; + for (i, &b) in body.iter().enumerate() { + match b { + b'!'..=b'u' => { + acc = acc * 85 + u64::from(b - b'!'); + n += 1; + if n == 5 { + let Ok(v) = u32::try_from(acc) else { + return Err(Failure::corrupt(NAME, format!("group ending at {i} exceeds 2^32"), out)); + }; + push(&mut out, &v.to_be_bytes(), max)?; + acc = 0; + n = 0; + } + } + b'z' if n == 0 => push(&mut out, &[0; 4], max)?, + b'z' => return Err(Failure::corrupt(NAME, format!("'z' inside a group at {i}"), out)), + // `~` starts the EOD marker; tolerate it without the following `>`. + b'~' => break, + b if is_pdf_whitespace(b) => {} + b => return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at {i}"), out)), + } + } + match n { + 0 => {} + 1 => return Err(Failure::corrupt(NAME, "final group has a single character", out)), + _ => { + // Pad with `u` (84) to a full group and keep n − 1 bytes. + for _ in n..5 { + acc = acc * 85 + 84; + } + let Ok(v) = u32::try_from(acc) else { + return Err(Failure::corrupt(NAME, "final group exceeds 2^32", out)); + }; + push(&mut out, &v.to_be_bytes()[..n - 1], max)?; + } + } + Ok(out) +} + +/// Encodes with `z` for zero groups, a newline every 75 characters, and a `~>` EOD. +pub fn encode(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() * 5 / 4 + data.len() / 60 + 8); + let mut col = 0; + let mut put = |out: &mut Vec, s: &[u8]| { + for &c in s { + if col == 75 { + out.push(b'\n'); + col = 0; + } + out.push(c); + col += 1; + } + }; + for chunk in data.chunks(4) { + let mut word = [0u8; 4]; + word[..chunk.len()].copy_from_slice(chunk); + let mut v = u32::from_be_bytes(word); + if v == 0 && chunk.len() == 4 { + put(&mut out, b"z"); + continue; + } + let mut digits = [0u8; 5]; + for d in digits.iter_mut().rev() { + *d = (v % 85) as u8 + b'!'; + v /= 85; + } + put(&mut out, &digits[..chunk.len() + 1]); + } + out.extend_from_slice(b"~>"); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 1000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + // "Man " = 0x4D616E20 = 24·85⁴ + 73·85³ + 80·85² + 78·85 + 61 -> "9jqo^". + assert_eq!(dec(b"9jqo^~>").unwrap(), b"Man "); + assert_eq!(encode(b"Man "), b"9jqo^~>"); + // "M" padded to 0x4D000000 -> "9`" (24, 63); decoding pads with 'u'. + assert_eq!(encode(b"M"), b"9`~>"); + assert_eq!(dec(b"9`~>").unwrap(), b"M"); + assert_eq!(dec(b"z~>").unwrap(), [0; 4]); + assert_eq!(encode(&[0; 4]), b"z~>"); + // A partial zero group is not `z`. + assert_eq!(encode(&[0; 3]), b"!!!!~>"); + assert_eq!(dec(b"!!!!~>").unwrap(), [0; 3]); + assert_eq!(dec(b"s8W-!~>").unwrap(), [0xff; 4]); + } + + #[test] + fn syntax_tolerance() { + assert_eq!(dec(b"<~9jqo^~>").unwrap(), b"Man "); + assert_eq!(dec(b" \n<~9j\r\nq o\t^z~>").unwrap(), b"Man \0\0\0\0"); + assert_eq!(dec(b"9jqo^").unwrap(), b"Man "); // missing EOD + assert_eq!(dec(b"9jqo^~").unwrap(), b"Man "); // bare ~ + assert_eq!(dec(b"9jqo^~>garbage{").unwrap(), b"Man "); + assert_eq!(dec(b"~>").unwrap(), b""); + assert_eq!(dec(b"").unwrap(), b""); + } + + #[test] + fn corrupt() { + assert!(matches!(dec(b"9jz~>"), Err(CodecError::Corrupt { .. }))); + assert!(matches!(dec(b"9jqo^9~>"), Err(CodecError::Corrupt { .. }))); + assert!(matches!(dec(b"uuuuu~>"), Err(CodecError::Corrupt { .. }))); // > 2^32 - 1 + assert!(matches!(dec(b"9jqo^v~>"), Err(CodecError::Corrupt { .. }))); + let f = decode(b"9jqo^{", 100).err().unwrap(); + assert_eq!(f.partial, b"Man "); + } + + #[test] + fn limit() { + assert!(matches!(decode(b"zz", 7).map_err(|f| f.error), Err(CodecError::LimitExceeded(7)))); + assert!(matches!(decode(b"z9`", 4).map_err(|f| f.error), Err(CodecError::LimitExceeded(4)))); + assert_eq!(decode(b"zz", 8).ok().unwrap().len(), 8); + } + + #[test] + fn line_wrapping() { + let data: Vec = (0..=255u8).collect(); + let e = encode(&data); + assert!(e.split(|&b| b == b'\n').all(|l| l.len() <= 77)); + assert_eq!(dec(&e).unwrap_or_default().len(), 256); + } +} diff --git a/crates/stream-codecs/src/asciihex.rs b/crates/stream-codecs/src/asciihex.rs new file mode 100644 index 000000000..60e962216 --- /dev/null +++ b/crates/stream-codecs/src/asciihex.rs @@ -0,0 +1,106 @@ +//! ASCIIHexDecode (ISO 32000-2 §7.4.2): pairs of hex digits, white space ignored, +//! `>` is EOD, an odd final digit is padded with 0. A missing EOD is tolerated. + +use crate::{Failure, Step, is_pdf_whitespace}; + +const NAME: &str = "ASCIIHexDecode"; + +fn nibble(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} + +pub fn decode(data: &[u8], max: usize) -> Step { + let mut out = Vec::with_capacity((data.len() / 2).min(max)); + let mut hi: Option = None; + for (i, &b) in data.iter().enumerate() { + if is_pdf_whitespace(b) { + continue; + } + if b == b'>' { + break; + } + let Some(n) = nibble(b) else { + return Err(Failure::corrupt(NAME, format!("invalid character 0x{b:02x} at offset {i}"), out)); + }; + match hi.take() { + None => hi = Some(n), + Some(h) => { + if out.len() == max { + return Err(Failure::limit(max)); + } + out.push(h << 4 | n); + } + } + } + if let Some(h) = hi { + if out.len() == max { + return Err(Failure::limit(max)); + } + out.push(h << 4); + } + Ok(out) +} + +/// Upper-case hex, a newline every 64 digits, terminated by `>`. +pub fn encode(data: &[u8]) -> Vec { + const HEX: &[u8; 16] = b"0123456789ABCDEF"; + let mut out = Vec::with_capacity(data.len() * 2 + data.len() / 32 + 1); + for (i, &b) in data.iter().enumerate() { + if i > 0 && i % 32 == 0 { + out.push(b'\n'); + } + out.push(HEX[usize::from(b >> 4)]); + out.push(HEX[usize::from(b & 15)]); + } + out.push(b'>'); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 1000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + assert_eq!(dec(b"48656c6C6F>").unwrap(), b"Hello"); + assert_eq!(dec(b" 4 8\n6\r5\t6\x0c c\x006c 6f >").unwrap(), b"Hello"); + assert_eq!(dec(b"901FA>").unwrap(), [0x90, 0x1f, 0xa0]); + assert_eq!(dec(b"901FA").unwrap(), [0x90, 0x1f, 0xa0]); // missing EOD + assert_eq!(dec(b"41>zz garbage after EOD").unwrap(), b"A"); + assert_eq!(dec(b">").unwrap(), b""); + assert_eq!(dec(b"").unwrap(), b""); + } + + #[test] + fn invalid_char() { + let f = decode(b"4142G43>", 1000).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"AB"); + } + + #[test] + fn limit() { + assert!(matches!(decode(b"414243", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2)))); + assert!(matches!(decode(b"41424", 2).map_err(|f| f.error), Err(CodecError::LimitExceeded(2)))); + assert_eq!(decode(b"4142", 2).ok().unwrap(), b"AB"); + } + + #[test] + fn encoding() { + assert_eq!(encode(b"Hello"), b"48656C6C6F>"); + assert_eq!(encode(b""), b">"); + let long = encode(&[0xab; 40]); + assert_eq!(long[64], b'\n'); + assert_eq!(dec(&long).unwrap(), [0xab; 40]); + } +} diff --git a/crates/stream-codecs/src/ccitt.rs b/crates/stream-codecs/src/ccitt.rs new file mode 100644 index 000000000..2733cf98b --- /dev/null +++ b/crates/stream-codecs/src/ccitt.rs @@ -0,0 +1,546 @@ +//! CCITT fax for `CCITTFaxDecode` (ISO 32000-2 §7.4.6): Group 4 (ITU-T T.6) encoding, with `/K -1`, +//! and decoding of Group 3 (T.4) and Group 4. The coding is the `fax` crate's and the decoding the +//! `hayro-ccitt` crate's; this module checks what they are given and bounds what they give. + +use fax::encoder::Encoder; +use fax::{Color, VecWriter}; + +use crate::CodecError; + +/// The widest row [`encode_g4`] encodes. The encoder counts the positions in a row in 16 bits +/// and adds two of them, so a row stays within half that range. +pub const MAX_G4_COLUMNS: u32 = 32_767; + +/// `rows` rows of `columns` 1-bit samples, most significant bit first, each row starting on a new +/// byte (as a PDF image's samples are packed), encoded as CCITT Group 4 with an end-of-block mark. +/// +/// A 0 bit is coded black and a 1 bit white, so the stream decodes to the same bits with +/// `/BlackIs1 false`, the default: an image keeps its colour space and decode array. Bytes after +/// the last row are ignored, as are the bits that pad a row to a whole byte. +/// +/// Errors: no rows or columns, or rows wider than [`MAX_G4_COLUMNS`] ([`CodecError::Unsupported`]); +/// fewer bytes than the rows need ([`CodecError::Corrupt`]). +pub fn encode_g4(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let width = match u16::try_from(columns) { + Ok(w) if w > 0 && columns <= MAX_G4_COLUMNS => w, + _ => return Err(CodecError::Unsupported(format!("CCITT Group 4 encoding of rows {columns} samples wide"))), + }; + if rows == 0 { + return Err(CodecError::Unsupported("CCITT Group 4 encoding of an image without rows".into())); + } + let stride = usize::from(width).div_ceil(8); + let samples = usize::try_from(rows).ok().and_then(|r| r.checked_mul(stride)).and_then(|n| data.get(..n)).ok_or_else(|| CodecError::Corrupt { + filter: "CCITTFaxDecode", + detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()), + })?; + let mut encoder = Encoder::new(VecWriter::new()); + for row in samples.chunks_exact(stride) { + let pels = (0..usize::from(width)).map(|x| { + let white = row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + if white { Color::White } else { Color::Black } + }); + // `VecWriter` writes to memory: its error type has no values. + if let Err(never) = encoder.encode_line(pels, width) { + match never {} + } + } + match encoder.finish() { + Ok(writer) => Ok(writer.finish()), + Err(never) => match never {}, + } +} + +/// The widest row [`decode_ccitt_within`] decodes: its row buffer stays within 128 KiB. +pub const MAX_CCITT_COLUMNS: u32 = 1 << 20; + +/// A `CCITTFaxDecode` stream's `/DecodeParms` (ISO 32000-2 Table 11). [`Default`] gives the +/// table's defaults. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CcittParams { + /// `/K`: negative for Group 4, 0 for Group 3 one-dimensional, positive for Group 3 mixed. + pub k: i64, + pub columns: i64, + /// `/Rows`: 0 when not given. + pub rows: i64, + pub end_of_line: bool, + pub encoded_byte_align: bool, + pub end_of_block: bool, + pub black_is_1: bool, +} + +impl Default for CcittParams { + fn default() -> Self { + CcittParams { k: 0, columns: 1728, rows: 0, end_of_line: false, encoded_byte_align: false, end_of_block: true, black_is_1: false } + } +} + +/// What [`decode_ccitt_within`] decodes once it is paid for. +fn decode_ccitt(data: &[u8], p: &CcittParams, max_rows: u32) -> Result, CodecError> { + let columns = u32::try_from(p.columns) + .ok() + .filter(|c| (1..=MAX_CCITT_COLUMNS).contains(c)) + .ok_or_else(|| CodecError::Unsupported(format!("CCITT fax rows {} samples wide", p.columns)))?; + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + if rows == 0 { + return Ok(Vec::new()); + } + let settings = hayro_ccitt::DecodeSettings { + columns, + rows, + end_of_block: p.end_of_block, + end_of_line: p.end_of_line, + // Group 3 rows on bytes may have end-of-line codes whatever `end_of_line` says (the filter + // always accepts them), with the fill starting each code on a byte or ending it on one (T.4's + // fill, as TIFF writes it). Read unaligned, the codes mark the rows either way; without + // codes, the rows are read on bytes. A reading that comes up short isn't tried the other + // way: data cut short would then read as other rows. + rows_are_byte_aligned: p.encoded_byte_align && (p.k < 0 || !starts_with_end_of_line(data)), + encoding: match p.k { + ..0 => hayro_ccitt::EncodingMode::Group4, + 0 => hayro_ccitt::EncodingMode::Group3_1D, + k => hayro_ccitt::EncodingMode::Group3_2D { k: u32::try_from(k).unwrap_or(u32::MAX) }, + }, + invert_black: false, + }; + // u32 to usize: wasm32 and up have at least 32-bit usize. + let mut out = Rows::new(columns as usize, rows as usize, p.black_is_1); + // The rows decoded before damaged data are kept. + let _ = hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)); + Ok(out.out) +} + +/// The steps of reading a byte of fax code ([`decode_ccitt_within`]): noise takes 114 ns a byte of +/// rows and about two bytes of code to a byte of rows, so that and a step a byte of rows is about +/// what a step of JBIG2 decoding takes. +const CODE_WORK: u64 = 4; + +/// CCITT fax `data` decoded as `CCITTFaxDecode` gives it: rows of `columns` 1-bit samples, most +/// significant bit first, each starting on a new byte, with black as 0 (as 1 with `black_is_1`). +/// +/// At most `max_rows` rows are decoded (fewer when `rows` says so), so the output is bounded by +/// the caller. Damaged data gives the rows decoded before the damage, as viewers show them: a +/// caller that needs a number of rows checks it got them. +/// +/// Group 3 rows on bytes (`k` from 0, `encoded_byte_align`) are read by their end-of-line codes, +/// whether the fill starts each code on a byte or ends it on one, when `data` starts with one (T.4 +/// sends one before the first row when it sends them at all), and on bytes otherwise, whatever +/// `end_of_line` says. Group 4 is read as its parameters say. Data with codes but none before the +/// first row is read on bytes, as hayro's renderer draws it; with `k` 0 and the fill ending each +/// code on a byte, that may be other rows than were coded. +/// +/// Errors: `columns` that aren't from 1 to [`MAX_CCITT_COLUMNS`] ([`CodecError::Unsupported`]). +/// +/// Decoding spends from `work`, what decoding may still cost across a run of images, and there is +/// no way in without it: a +/// byte of rows a step, whether or not the data holds them, and [`CODE_WORK`] a byte of `data`. +/// A blank row takes a bit of code, so the rows are what a few bytes can ask for. It is all paid +/// before anything is decoded, and what `work` can't pay for spends all of it. +pub fn decode_ccitt_within(data: &[u8], p: &CcittParams, max_rows: u32, work: &mut u64) -> Result, CodecError> { + let rows = u32::try_from(p.rows).ok().filter(|r| *r > 0).map_or(max_rows, |r| r.min(max_rows)); + let stride = u64::try_from(p.columns).unwrap_or(0).div_ceil(8); + crate::pay(work, stride.saturating_mul(u64::from(rows)).saturating_add((data.len() as u64).saturating_mul(CODE_WORK)))?; + decode_ccitt(data, p, max_rows) +} + +/// Whether CCITT fax `data` starts with an end-of-line code: a 1 after 11 zeros or more, fill +/// included. T.4 sends one before the first row when it sends them at all, and no other code starts +/// with more than eight zeros. +fn starts_with_end_of_line(data: &[u8]) -> bool { + let Some((i, b)) = data.iter().enumerate().find(|(_, b)| **b != 0) else { return false }; + i.saturating_mul(8).saturating_add(b.leading_zeros() as usize) >= 11 +} + +/// Rows of 1-bit samples filled from a decoder's pixels: `columns` a row, each row on a new byte, at +/// most `rows` rows. Pixels past a row's end and rows past the last are dropped. +pub(crate) struct Rows { + pub(crate) out: Vec, + row: Vec, + x: usize, + columns: usize, + rows: usize, + done: usize, + black_is_1: bool, +} + +impl Rows { + pub(crate) fn new(columns: usize, rows: usize, black_is_1: bool) -> Self { + Rows { out: Vec::new(), row: vec![0; columns.div_ceil(8)], x: 0, columns, rows, done: 0, black_is_1 } + } + + /// `count` pixels, white or black. + pub(crate) fn push(&mut self, white: bool, count: usize) { + let end = self.x.saturating_add(count); + if white != self.black_is_1 { + for x in self.x..end.min(self.columns) { + if let Some(b) = self.row.get_mut(x / 8) { + *b |= 0x80 >> (x % 8); + } + } + } + self.x = end; + } + + /// The row filled so far kept, if fewer than `rows` are, and the next one begun. + pub(crate) fn end_row(&mut self) { + if self.done < self.rows { + self.out.extend_from_slice(&self.row); + self.done += 1; + } + self.row.fill(0); + self.x = 0; + } +} + +impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.push(white, 1); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.push(white, (chunks as usize).saturating_mul(8)); + } + fn next_line(&mut self) { + self.end_row(); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A run pays for the rows an image asks for and the code it reads, before any of it is + /// decoded: rows the data doesn't hold are paid for too, as a few bytes can ask for any number. + #[test] + fn a_run_of_images_pays_for_the_rows_it_asks_for() { + let (w, h) = (4096u32, 64u32); + let stride = (w as usize).div_ceil(8); + let data = encode_g4(&vec![0xFF; stride * h as usize], w, h).unwrap(); + let p = CcittParams { k: -1, columns: i64::from(w), ..CcittParams::default() }; + let whole = decode_ccitt(&data, &p, h).unwrap(); + let cost = (whole.len() + 4 * data.len()) as u64; + let mut work = cost; + assert_eq!(decode_ccitt_within(&data, &p, h, &mut work).unwrap(), whole, "exactly enough"); + assert_eq!(work, 0); + let mut work = cost + 7; + decode_ccitt_within(&data, &p, h, &mut work).unwrap(); + assert_eq!(work, 7, "only what it took"); + let mut work = cost - 1; + let refused = decode_ccitt_within(&data, &p, h, &mut work).unwrap_err(); + assert!(matches!(refused, CodecError::Spent), "a step short: {refused}"); + assert_eq!(work, 0, "spends all that was left"); + // Nothing left: not read at all, so bad columns aren't reported as bad columns. + let bad = CcittParams { columns: 0, ..p }; + assert!(decode_ccitt_within(&data, &bad, h, &mut 0).is_err_and(|e| matches!(e, CodecError::Spent))); + // No code at all, for the same rows. + let mut work = u64::MAX; + decode_ccitt_within(&[], &p, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, whole.len() as u64); + // `/Rows` below what is asked for, and no rows. + let mut work = u64::MAX; + decode_ccitt_within(&data, &CcittParams { rows: 3, ..p }, h, &mut work).unwrap(); + assert_eq!(u64::MAX - work, (3 * stride + 4 * data.len()) as u64); + let mut work = u64::MAX; + decode_ccitt_within(&data, &p, 0, &mut work).unwrap(); + assert_eq!(u64::MAX - work, 4 * data.len() as u64); + } + + /// The rows decoded by an independent decoder (`hayro-ccitt`), as 1 for white and 0 for black. + pub(crate) fn decode(data: &[u8], columns: u32, rows: u32) -> Vec> { + struct Rows(Vec>, Vec, usize); + impl hayro_ccitt::Decoder for Rows { + fn push_pixel(&mut self, white: bool) { + self.1.push(white); + } + fn push_pixel_chunk(&mut self, white: bool, chunks: u32) { + self.1.extend(std::iter::repeat_n(white, chunks as usize * 8)); + } + fn next_line(&mut self) { + let mut row = std::mem::take(&mut self.1); + row.truncate(self.2); + self.0.push(row); + } + } + let settings = hayro_ccitt::DecodeSettings { + columns, + rows, + end_of_block: true, + end_of_line: false, + rows_are_byte_aligned: false, + encoding: hayro_ccitt::EncodingMode::Group4, + invert_black: false, + }; + let mut out = Rows(Vec::new(), Vec::new(), columns as usize); + hayro_ccitt::decode(data, &mut out, &mut hayro_ccitt::DecoderContext::new(settings)).unwrap(); + out.0 + } + + /// The rows of packed samples `data`, as 1 for a 1 bit. + pub(crate) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { + let stride = (columns as usize).div_ceil(8); + (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() + } + + #[test] + fn a_page_of_text_like_marks_round_trips_and_shrinks() { + let (w, h) = (1700u32, 220u32); + let stride = (w as usize).div_ceil(8); + let mut data = vec![0xFF; stride * h as usize]; + // Short black strokes in lines, like glyphs on a white page. + for y in (20..h as usize).step_by(40) { + for dy in 0..12 { + for x in (100..1600).step_by(9) { + if (x / 9 + dy) % 5 != 0 { + data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + } + let g4 = encode_g4(&data, w, h).unwrap(); + assert_eq!(decode(&g4, w, h), unpack(&data, w, h)); + assert!(g4.len() * 4 < data.len(), "{} bytes from {}", g4.len(), data.len()); + } + + #[test] + fn padding_bits_and_trailing_bytes_are_ignored() { + // 3 columns: the last 5 bits of each byte pad the row; set them to anything. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; + let g4 = encode_g4(&data, 3, 3).unwrap(); + assert_eq!(decode(&g4, 3, 3), [[true, false, true], [false, true, false], [true, true, true]]); + } + + #[test] + fn what_it_cannot_encode_is_an_error() { + assert!(matches!(encode_g4(&[0; 8], 0, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&[0; 8], 8, 0), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&vec![0; 4097], MAX_G4_COLUMNS + 1, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_g4(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes"); + assert!(matches!(encode_g4(&[], 1, u32::MAX), Err(CodecError::Corrupt { .. }))); + let widest = vec![0x55; (MAX_G4_COLUMNS as usize).div_ceil(8) * 2]; + let g4 = encode_g4(&widest, MAX_G4_COLUMNS, 2).unwrap(); + assert_eq!(decode(&g4, MAX_G4_COLUMNS, 2), unpack(&widest, MAX_G4_COLUMNS, 2)); + } + + /// Rows of `columns` samples built from `runs`: alternating white and black runs one longer + /// than those values (so never empty, even when shrinking makes them all 0), repeated until the + /// rows are full; or, with `noise`, the bytes of `runs` as they are. + fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut data = vec![0u8; stride * rows as usize]; + if runs.is_empty() { + return data; + } + if noise { + data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); + return data; + } + let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); + for y in 0..rows as usize { + for x in 0..columns as usize { + if left == 0 { + white = !white; + left = 1 + usize::from(runs[next % runs.len()]); + next += 1; + } + if white { + data[y * stride + x / 8] |= 0x80 >> (x % 8); + } + left -= 1; + } + } + data + } + + fn g4(columns: u32) -> CcittParams { + CcittParams { k: -1, columns: i64::from(columns), ..CcittParams::default() } + } + + #[test] + fn decoding_gives_the_rows_back() { + // 3 columns, padded with anything: the decoded padding is 0. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010]; + let coded = encode_g4(&data, 3, 3).unwrap(); + assert_eq!(decode_ccitt(&coded, &g4(3), 3).unwrap(), [0b1010_0000, 0b0100_0000, 0b1110_0000]); + // /BlackIs1 gives black as 1. + let p = CcittParams { black_is_1: true, ..g4(3) }; + assert_eq!(decode_ccitt(&coded, &p, 3).unwrap(), [0b0100_0000, 0b1010_0000, 0b0000_0000]); + // At most /Rows rows, and at most max_rows. + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 2, ..g4(3) }, 3).unwrap().len(), 2); + assert_eq!(decode_ccitt(&coded, &g4(3), 1).unwrap().len(), 1); + assert_eq!(decode_ccitt(&coded, &CcittParams { rows: 3, ..g4(3) }, 1).unwrap().len(), 1, "max_rows bounds /Rows"); + assert_eq!(decode_ccitt(&coded, &g4(3), 0).unwrap(), [0u8; 0]); + // Damaged data gives the rows before the damage; none here. + assert!(decode_ccitt(&coded[..1], &g4(3), 3).unwrap().len() < 3); + } + + #[test] + fn what_it_cannot_decode_is_an_error() { + for columns in [0, -1, i64::from(MAX_CCITT_COLUMNS) + 1, i64::MAX] { + let p = CcittParams { columns, ..CcittParams::default() }; + assert!(matches!(decode_ccitt(&[0; 8], &p, 4), Err(CodecError::Unsupported(_))), "{columns}"); + } + assert_eq!(CcittParams::default().columns, 1728, "Table 11"); + } + + /// Group 3 data: rows of run lengths below 64, white first, coded one-dimensionally with T.4's + /// terminating codes (ITU-T T.4 Tables 2 and 3). Each row starts with an end-of-line code with + /// `eol`, then with `tag` a 1 bit (the next row one-dimensional, as `k` above 0 has it). Fill + /// bits (0s) go where `fill` says. + fn group_3(rows: &[Vec], eol: bool, tag: bool, fill: Fill) -> Vec { + // The codes for runs 0 to 63, in order. + const WHITE: &str = concat!( + "00110101 000111 0111 1000 1011 1100 1110 1111 10011 10100 00111 01000 001000 000011 110100 110101 101010 101011 ", + "0100111 0001100 0001000 0010111 0000011 0000100 0101000 0101011 0010011 0100100 0011000 00000010 00000011 00011010 ", + "00011011 00010010 00010011 00010100 00010101 00010110 00010111 00101000 00101001 00101010 00101011 00101100 00101101 ", + "00000100 00000101 00001010 00001011 01010010 01010011 01010100 01010101 00100100 00100101 01011000 01011001 01011010 ", + "01011011 01001010 01001011 00110010 00110011 00110100", + ); + const BLACK: &str = concat!( + "0000110111 010 11 10 011 0011 0010 00011 000101 000100 0000100 0000101 0000111 00000100 00000111 000011000 0000010111 ", + "0000011000 0000001000 00001100111 00001101000 00001101100 00000110111 00000101000 00000010111 00000011000 ", + "000011001010 000011001011 000011001100 000011001101 000001101000 000001101001 000001101010 000001101011 000011010010 ", + "000011010011 000011010100 000011010101 000011010110 000011010111 000001101100 000001101101 000011011010 000011011011 ", + "000001010100 000001010101 000001010110 000001010111 000001100100 000001100101 000001010010 000001010011 000000100100 ", + "000000110111 000000111000 000000100111 000000101000 000001011000 000001011001 000000101011 000000101100 000001011010 ", + "000001100110 000001100111", + ); + let code = |white: bool, run: u32| (if white { WHITE } else { BLACK }).split(' ').nth(run as usize).expect("a run below 64"); + let mut bits = String::new(); + for row in rows { + // The end-of-line code is 12 bits. + let ends_code = if fill == Fill::CodeEnd && eol { 12 } else { 0 }; + while fill != Fill::None && !(bits.len() + ends_code).is_multiple_of(8) { + bits.push('0'); + } + if eol { + bits.push_str("000000000001"); + } + if tag { + bits.push('1'); + } + row.iter().enumerate().for_each(|(i, run)| bits.push_str(code(i % 2 == 0, *run))); + } + bits.as_bytes().chunks(8).map(|c| c.iter().enumerate().fold(0u8, |b, (i, d)| b | (u8::from(*d == b'1') << (7 - i)))).collect() + } + + /// Where [`group_3`] puts fill bits: none; so each row, its end-of-line code first, starts on a + /// byte; or so each end-of-line code ends on one (T.4's fill, as TIFF writes it). + #[derive(Clone, Copy, Debug, PartialEq)] + enum Fill { + None, + RowStart, + CodeEnd, + } + + /// `rows` text-like rows of `columns` samples as runs, white first, from `seed`: long white runs + /// and short black ones; a row may start black. + fn text_like(seed: u64, columns: u32, rows: u32) -> Vec> { + let mut seed = seed.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + let mut random = |n: u64| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + (seed % n) as u32 + }; + (0..rows) + .map(|_| { + let (mut runs, mut left) = (Vec::new(), columns); + while left > 0 { + let run = match runs.len() % 2 { + 0 if runs.is_empty() => random(31), + 0 => 1 + random(30), + _ => 1 + random(5), + }; + runs.push(run.min(left)); + left -= run.min(left); + } + runs + }) + .collect() + } + + /// Rows of runs, white first, packed as `decode_ccitt` gives them (white 1). + fn pack(runs: &[Vec], columns: u32) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut out = vec![0u8; stride * runs.len()]; + for (y, row) in runs.iter().enumerate() { + let mut x = 0; + for (i, run) in row.iter().enumerate() { + if i % 2 == 0 { + (x..x + *run as usize).for_each(|x| out[y * stride + x / 8] |= 0x80 >> (x % 8)); + } + x += *run as usize; + } + } + out + } + + /// Group 3 rows on bytes in each layout, over many pictures, as how a misreading comes out + /// depends on the picture: no codes (as TIFF's Modified Huffman has it), and `end_of_line` + /// without them; codes, each starting on a byte or ending on one, `end_of_line` or not. Each + /// decodes to its picture, and a byte short to fewer rows, not to other rows read another way. + /// Group 4 rows on bytes keep their aligned read, `end_of_line` or not. + #[test] + fn group_3_rows_on_bytes_read_by_the_codes_in_the_data() { + for seed in 0..32u64 { + let (columns, rows) = (8 + (seed as u32 * 37) % 200, 2 + seed as u32 % 20); + let runs = text_like(seed, columns, rows); + let picture = pack(&runs, columns); + for k in [0, 1] { + for (codes, fill, end_of_line) in [ + (false, Fill::RowStart, false), + (false, Fill::RowStart, true), + (true, Fill::RowStart, false), + (true, Fill::RowStart, true), + (true, Fill::CodeEnd, false), + (true, Fill::CodeEnd, true), + ] { + let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align: true, ..CcittParams::default() }; + let mut data = group_3(&runs, codes, k > 0, fill); + let case = format!("seed {seed}, {columns} × {rows}, k {k}, codes {codes}, fill {fill:?}, end_of_line {end_of_line}"); + assert_eq!(decode_ccitt(&data, &p, rows).unwrap(), picture, "{case}"); + data.pop(); + assert!(decode_ccitt(&data, &p, rows).unwrap().len() < picture.len(), "{case}, a byte short"); + } + } + } + // White rows, each the one code 1 (vertical, no change) and its fill. + for end_of_line in [false, true] { + let p = CcittParams { k: -1, columns: 64, end_of_line, encoded_byte_align: true, ..CcittParams::default() }; + assert_eq!(decode_ccitt(&[0x80; 16], &p, 16).unwrap(), [0xFF; 8 * 16], "Group 4, end_of_line {end_of_line}"); + } + } + + proptest::proptest! { + /// Any bitmap, of noise or of runs, decodes to its own bits, through an independent decoder + /// and through `decode_ccitt`. + #[test] + fn bitmaps_round_trip( + columns in 1u32..400, + rows in 1u32..24, + runs in proptest::collection::vec(0u8..=255, 1..48), + noise in proptest::bool::ANY, + ) { + let data = bitmap(columns, rows, &runs, noise); + let coded = encode_g4(&data, columns, rows).unwrap(); + proptest::prop_assert_eq!(decode(&coded, columns, rows), unpack(&data, columns, rows)); + let decoded = decode_ccitt(&coded, &g4(columns), rows).unwrap(); + proptest::prop_assert_eq!(unpack(&decoded, columns, rows), unpack(&data, columns, rows)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns( + data in proptest::collection::vec(0u8..=255, 0..600), + columns in 0u32..40_000, + rows in 0u32..80, + k in -2i64..3, + end_of_line in proptest::bool::ANY, + encoded_byte_align in proptest::bool::ANY, + ) { + let _ = encode_g4(&data, columns, rows); + let p = CcittParams { k, columns: i64::from(columns), end_of_line, encoded_byte_align, ..CcittParams::default() }; + if let Ok(out) = decode_ccitt(&data, &p, rows) { + proptest::prop_assert!(out.len() <= (columns as usize).div_ceil(8) * rows as usize); + } + } + } +} diff --git a/crates/stream-codecs/src/jbig2.rs b/crates/stream-codecs/src/jbig2.rs new file mode 100644 index 000000000..0b8b76720 --- /dev/null +++ b/crates/stream-codecs/src/jbig2.rs @@ -0,0 +1,468 @@ +//! JBIG2 encoding (ITU-T T.88 | ISO/IEC 14492), for `JBIG2Decode` (ISO 32000-2 §7.4.7), lossless: +//! a generic region, template 0 with its nominal adaptive pixels, typical prediction on, MQ +//! arithmetic coding ([`crate::mq`]); or the image's marks as symbols ([`symbols`]), which a group +//! of images may share through `/JBIG2Globals`. Written from the standard's 6.2.5 (coding a generic +//! region), 7.2 (segment headers), 7.4.1, 7.4.6 and 7.4.8 (the segments) and D.3 (the embedded +//! organisation PDF uses). Decoding is in [`decode`]. + +mod symbols; + +/// The largest image the encoders take, in pixels: the largest JBIG2 page `hayro-jbig2` decodes +/// (it decodes a page whole, and takes 20,000 × 10,000), so what they store reads back. +pub const MAX_JBIG2_PIXELS: u64 = 1 << 28; + +use crate::CodecError; +use crate::mq::{Context, MqEncoder}; + +/// The widest row, and the most rows, [`encode_jbig2_generic`] encodes: what `hayro-jbig2`, the +/// decoder behind `JBIG2Decode` here, takes on a side. JBIG2 allows 32-bit sizes, but past this +/// nothing here could read the stream, and the optimizer, which doesn't read back what it stores, +/// replaced such an image with one that draws nothing. +pub const MAX_JBIG2_COLUMNS: u32 = 65_535; + +/// Template 0's pixels (Figure 3), as (x, y) offsets from the pixel being coded, each a bit of the +/// context in this order: the four on the current row, the five fixed ones on the row above, the +/// three on the row above that, then the adaptive pixels A1–A4 at their nominal places (Table 5). +const TEMPLATE: [(i32, i32); 16] = [ + (-1, 0), + (-2, 0), + (-3, 0), + (-4, 0), + (-2, -1), + (-1, -1), + (0, -1), + (1, -1), + (2, -1), + (-1, -2), + (0, -2), + (1, -2), + (3, -1), + (-3, -1), + (2, -2), + (-2, -2), +]; + +/// The context that codes SLTP when template 0 is used (Figure 8): those pixel values, in +/// [`TEMPLATE`]'s order. Figure 8 reads, by row from x = -4: `0 1 0 1` on the current row, +/// `0 1 1 0 0 1 0` (x = -3 to 3) above it, `1 0 0 1 1` (x = -2 to 2) above that. +const SLTP: [u8; 16] = [1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1]; + +/// Columns of padding either side of a row, so every template pixel of a row's pixels is in it. +const PAD: usize = 4; + +/// `rows` rows of `columns` 1-bit PDF samples as lossless JBIG2, the smaller of two codings: +/// [`encode_jbig2_generic`]'s, or one that stores each distinct mark of the image once, as a +/// symbol, and places its copies (see [`symbols`]). Either way the stream is in the embedded +/// organisation and decodes to the same bits without `/JBIG2Globals`. +/// +/// Errors: as [`encode_jbig2_generic`]. +pub fn encode_jbig2(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let generic = encode_jbig2_generic(data, columns, rows)?; + let samples = rows_to_encode(data, columns, rows).ok(); + Ok(match samples.and_then(|s| symbols::encode(s, columns, rows)) { + Some(coded) if coded.len() < generic.len() => coded, + _ => generic, + }) +} + +/// `rows` rows of `columns` 1-bit PDF samples (most significant bit first, each row starting on +/// a new byte), as a JBIG2 stream in the embedded organisation: a page information segment and an +/// immediate lossless generic region segment, which `JBIG2Decode` decodes to the same bits +/// without `/JBIG2Globals`. +/// +/// A 0 sample is coded as a JBIG2 1 (black), since `JBIG2Decode` gives black as 0, as a grey image's +/// samples have it. Bytes after the last row are ignored, as are the bits that pad a row to a whole +/// byte. +/// +/// Errors: no rows or columns, more of either than [`MAX_JBIG2_COLUMNS`], or more samples than +/// [`MAX_JBIG2_PIXELS`], what the decoder takes ([`CodecError::Unsupported`]); fewer bytes than the +/// rows need ([`CodecError::Corrupt`]). +pub fn encode_jbig2_generic(data: &[u8], columns: u32, rows: u32) -> Result, CodecError> { + let samples = rows_to_encode(data, columns, rows)?; + let width = columns as usize; + let stride = width.div_ceil(8); + let mut contexts: Vec = vec![0; 1 << 16]; + let mut mq = MqEncoder::new(); + let height = samples.len() / stride; + code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { + let packed = samples.get(y * stride..).unwrap_or_default(); + for (x, pixel) in row.iter_mut().enumerate() { + let sample = packed.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0); + *pixel = u8::from(!sample); + } + }); + let region = mq.finish(); + let mut out = Vec::with_capacity(region.len() + 64); + segment(&mut out, 0, 48, &[], false, &page_information(columns, rows)); + segment(&mut out, 1, 39, &[], false, &generic_region(columns, rows, 0, 0, ®ion)); + Ok(out) +} + +/// The samples of `rows` rows of `columns` 1-bit samples in `data`, when the encoders here take +/// that size; the errors are [`encode_jbig2_generic`]'s. +fn rows_to_encode(data: &[u8], columns: u32, rows: u32) -> Result<&[u8], CodecError> { + let too_large = columns > MAX_JBIG2_COLUMNS || rows > MAX_JBIG2_COLUMNS || u64::from(columns) * u64::from(rows) > MAX_JBIG2_PIXELS; + if columns == 0 || rows == 0 || too_large { + return Err(CodecError::Unsupported(format!("JBIG2 encoding of {columns} × {rows} samples"))); + } + let stride = (columns as usize).div_ceil(8); + usize::try_from(rows) + .ok() + .and_then(|r| r.checked_mul(stride)) + .and_then(|n| data.get(..n)) + .ok_or_else(|| CodecError::Corrupt { filter: "JBIG2Decode", detail: format!("{} bytes for {rows} rows of {columns} samples", data.len()) }) +} + +/// A group of images coded by [`encode_jbig2_shared`]: the data of the `/JBIG2Globals` stream +/// they share, and each image's own stream, in the order given, or `None` for one that shares +/// nothing with the others. +#[derive(Debug)] +pub struct SharedJbig2 { + pub globals: Vec, + pub images: Vec>>, +} + +/// Images, each `rows` rows of `columns` 1-bit PDF samples, as lossless JBIG2 sharing one symbol +/// dictionary: the marks found in more than one of them are stored once, in a symbol dictionary +/// segment not associated with a page, which is the `/JBIG2Globals` stream's data, and each image's +/// stream, in the embedded organisation, places them from there, with its other marks coded as +/// [`encode_jbig2`]'s symbol coding codes them (see [`symbols`]). With the globals, each decodes to +/// its own bits. An image that holds none of those marks, or that symbol coding doesn't take, has +/// no stream here; it is stored alone. `None` when the images share fewer than two marks. +/// +/// What each image's stream saves is left for the caller to weigh against what the globals take, +/// which the images that use them share. +/// +/// Errors: as [`encode_jbig2_generic`], for the first image it would refuse. +pub fn encode_jbig2_shared(images: &[(&[u8], u32, u32)]) -> Result, CodecError> { + let images = + images.iter().map(|(data, columns, rows)| Ok((rows_to_encode(data, *columns, *rows)?, *columns, *rows))).collect::, _>>()?; + Ok(symbols::encode_shared(&images).map(|(globals, images)| SharedJbig2 { globals, images })) +} + +/// A page information segment's data (7.4.8): the page's size, resolution unknown, eventually +/// lossless, default pixel 0 (white), OR combination, not striped. +fn page_information(columns: u32, rows: u32) -> Vec { + let mut page = Vec::with_capacity(19); + page.extend_from_slice(&columns.to_be_bytes()); + page.extend_from_slice(&rows.to_be_bytes()); + page.extend_from_slice(&[0; 8]); + page.push(0x01); + page.extend_from_slice(&[0, 0]); + page +} + +/// A region segment information field (7.4.1): a `width` × `height` region at (`x`, `y`) on the +/// page, combined with it by OR. +fn region_information(out: &mut Vec, width: u32, height: u32, x: u32, y: u32) { + for v in [width, height, x, y] { + out.extend_from_slice(&v.to_be_bytes()); + } + out.push(0); +} + +/// An immediate lossless generic region segment's data (7.4.6): its region information, flags +/// (MMR 0, template 0, TPGDON 1), the AT pixels at their nominal places, then `coded`. +fn generic_region(width: u32, height: u32, x: u32, y: u32, coded: &[u8]) -> Vec { + let mut data = Vec::with_capacity(26 + coded.len()); + region_information(&mut data, width, height, x, y); + data.push(0x08); + at_pixels(&mut data); + data.extend_from_slice(coded); + data +} + +/// Template 0's four AT pixels at their nominal places, as AT flags (7.4.6.3, 7.4.2.1.2). +fn at_pixels(out: &mut Vec) { + for (x, y) in &TEMPLATE[12..] { + out.push(*x as i8 as u8); + out.push(*y as i8 as u8); + } +} + +/// A segment (7.2) of page 1: its header (number, type, the segments it refers to, the page, data +/// length), then its data. `retained`: a later segment refers to this one. `refers_to` holds each +/// referred-to segment's number and whether a segment after this one refers to it too (its +/// retain bit, 7.2.4). Segment numbers here stay below 256, so each referred-to one takes a byte +/// (7.2.5); a segment refers to at most four, so their count and retain bits take one byte too. +fn segment(out: &mut Vec, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { + segment_of(out, 1, number, kind, refers_to, retained, data); +} + +/// [`segment`], of page `page`: 0 for a segment associated with no page (7.2.6), such as the +/// global ones `/JBIG2Globals` holds, which a page's segments may refer to. +fn segment_of(out: &mut Vec, page: u8, number: u32, kind: u8, refers_to: &[(u8, bool)], retained: bool, data: &[u8]) { + out.extend_from_slice(&number.to_be_bytes()); + out.push(kind); + // At most four, so the count fits its three bits. + let refers_to = refers_to.get(..refers_to.len().min(4)).unwrap_or_default(); + let kept = refers_to.iter().enumerate().fold(0u8, |bits, (k, (_, kept))| bits | u8::from(*kept) << (k + 1)); + out.push((refers_to.len() as u8) << 5 | kept | u8::from(retained)); + out.extend(refers_to.iter().map(|(n, _)| *n)); + out.push(page); + // A segment's data never comes near 4 GiB here: an image is at most 2^28 samples. + out.extend_from_slice(&u32::try_from(data.len()).unwrap_or(u32::MAX).to_be_bytes()); + out.extend_from_slice(data); +} + +/// Codes a `width` × `height` bitmap as a generic region (6.2.5.7) with template 0 and the nominal +/// AT pixels, its decisions in `contexts` (2^16 of them), with typical prediction when `typical` +/// (TPGDON). `fill(y, row)` sets row `y`'s `width` pixels, 1 for black. Pixels outside the bitmap +/// count as 0 (white), as the decoder takes them. +fn code_generic(mq: &mut MqEncoder, contexts: &mut [Context], width: usize, height: usize, typical: bool, mut fill: impl FnMut(usize, &mut [u8])) { + let sltp = SLTP.iter().enumerate().fold(0usize, |cx, (k, v)| cx | usize::from(*v) << k); + // The two rows above the current one and the current one, as JBIG2 pixels (1 black), padded. + let blank = vec![0u8; width + 2 * PAD]; + let (mut above2, mut above1) = (blank.clone(), blank.clone()); + let mut row = blank; + let mut ltp = false; + for y in 0..height { + if let Some(pixels) = row.get_mut(PAD..PAD + width) { + fill(y, pixels); + } + if typical { + // Typical prediction (6.2.5.5): a row the same as the one above is coded as one decision. + let same = row == above1; + if let Some(cx) = contexts.get_mut(sltp) { + mq.encode(cx, same != ltp); + } + ltp = same; + } + if !ltp { + for x in 0..width { + let at = |dx: i32, dy: i32| { + let line = match dy { + 0 => &row, + -1 => &above1, + _ => &above2, + }; + let i = (PAD + x).checked_add_signed(dx as isize); + i.and_then(|i| line.get(i)).copied().unwrap_or(0) + }; + let cx = TEMPLATE.iter().enumerate().fold(0usize, |cx, (k, (dx, dy))| cx | usize::from(at(*dx, *dy)) << k); + if let Some(state) = contexts.get_mut(cx) { + mq.encode(state, row.get(PAD + x) == Some(&1)); + } + } + } + std::mem::swap(&mut above2, &mut above1); + std::mem::swap(&mut above1, &mut row); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The rows an independent decoder (`hayro-jbig2`) reads from `data`, as PDF samples: true + /// for white (a JBIG2 0). + pub(super) fn decode(data: &[u8]) -> Vec> { + decode_with(data, None) + } + + /// [`decode`], with `/JBIG2Globals`. + pub(super) fn decode_with(data: &[u8], globals: Option<&[u8]>) -> Vec> { + struct Rows(Vec>, Vec, usize); + impl hayro_jbig2::Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1.push(!black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.1.extend(std::iter::repeat_n(!black, chunks as usize * 8)); + } + fn next_line(&mut self) { + let mut row = std::mem::take(&mut self.1); + row.truncate(self.2); + self.0.push(row); + } + } + let image = hayro_jbig2::Image::new_embedded(data, globals).unwrap(); + let mut rows = Rows(Vec::new(), Vec::new(), image.width() as usize); + image.decode(&mut rows).unwrap(); + rows.0 + } + + /// The rows of packed samples `data`, true for a 1 bit. + pub(super) fn unpack(data: &[u8], columns: u32, rows: u32) -> Vec> { + let stride = (columns as usize).div_ceil(8); + (0..rows as usize).map(|y| (0..columns as usize).map(|x| data[y * stride + x / 8] & (0x80 >> (x % 8)) != 0).collect()).collect() + } + + /// Rows of `columns` samples from `runs`: alternating white and black runs one longer than + /// those values (never empty, even when shrinking makes them all 0), until the rows are full; + /// or, with `noise`, the bytes of `runs` as they are. + fn bitmap(columns: u32, rows: u32, runs: &[u8], noise: bool) -> Vec { + let stride = (columns as usize).div_ceil(8); + let mut data = vec![0u8; stride * rows as usize]; + if runs.is_empty() { + return data; + } + if noise { + data.iter_mut().zip(runs.iter().cycle()).for_each(|(d, r)| *d = *r); + return data; + } + let (mut white, mut left, mut next) = (true, 1 + usize::from(runs[0]), 1); + for y in 0..rows as usize { + for x in 0..columns as usize { + if left == 0 { + white = !white; + left = 1 + usize::from(runs[next % runs.len()]); + next += 1; + } + if white { + data[y * stride + x / 8] |= 0x80 >> (x % 8); + } + left -= 1; + } + } + data + } + + #[test] + fn a_page_of_text_like_marks_round_trips_and_shrinks() { + let (w, h) = (1700u32, 220u32); + let stride = (w as usize).div_ceil(8); + let mut data = vec![0xFF; stride * h as usize]; + for y in (20..h as usize).step_by(40) { + for dy in 0..12 { + for x in (100..1600).step_by(9) { + if (x / 9 + dy) % 5 != 0 { + data[(y + dy) * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + } + let jbig2 = encode_jbig2_generic(&data, w, h).unwrap(); + assert_eq!(decode(&jbig2), unpack(&data, w, h)); + let g4 = crate::encode_g4(&data, w, h).unwrap(); + assert!(jbig2.len() < g4.len(), "JBIG2 {} bytes, Group 4 {}", jbig2.len(), g4.len()); + } + + #[test] + fn edges_padding_and_typical_rows_round_trip() { + // 3 columns: the last 5 bits of each byte pad the row; set them to anything. + let data = [0b1010_1111, 0b0101_0000, 0b1110_1010, 0xAB, 0xCD]; + assert_eq!(decode(&encode_jbig2_generic(&data, 3, 3).unwrap()), [[true, false, true], [false, true, false], [true, true, true]]); + for (w, h, fill) in [(1, 1, 0x00), (1, 1, 0xFF), (13, 9, 0x00), (13, 9, 0xFF), (64, 40, 0x5A)] { + let data = vec![fill; (w as usize).div_ceil(8) * h as usize]; + assert_eq!(decode(&encode_jbig2_generic(&data, w, h).unwrap()), unpack(&data, w, h), "{w} × {h} of {fill:#04x}"); + } + } + + /// Packed samples from JBIG2 pixels (1 black): a 0 sample for each 1. + pub(super) fn samples(pixels: &[Vec]) -> Vec { + let width = pixels.first().map_or(0, Vec::len); + let stride = width.div_ceil(8); + let mut data = vec![0xFF; stride * pixels.len()]; + for (y, row) in pixels.iter().enumerate() { + for (x, p) in row.iter().enumerate() { + if *p == 1 { + data[y * stride + x / 8] &= !(0x80 >> (x % 8)); + } + } + } + data + } + + /// A row the same as the one above is typical (6.2.5.5): one decision, not a row of pixels. + #[test] + fn a_repeated_row_is_one_decision() { + let row: Vec = (0..800u32).map(|x| u8::from((x * 7919 + x / 3) % 5 < 2)).collect(); + let pixels = vec![row; 1000]; + let data = samples(&pixels); + crate::mq::DECISIONS.with(|n| n.set(0)); + let jbig2 = encode_jbig2_generic(&data, 800, 1000).unwrap(); + // The first row's 800 pixels, and one typical-row decision for each of the 1000 rows. + assert_eq!(crate::mq::DECISIONS.with(|n| n.get()), 800 + 1000); + assert_eq!(decode(&jbig2), unpack(&data, 800, 1000)); + } + + /// SLTP is coded in the context Figure 8 shows, which ordinary pixels can share: an image + /// tiled with that very neighbourhood decodes only if both coders use the same context. + #[test] + fn the_typical_row_flag_shares_figure_8s_context() { + // Figure 8, around a pixel at x = 4 of an 8-column tile: two rows up x = 2..6, one row up + // x = 1..7, the pixel's own row x = 0..3. + let tile = [[0, 0, 1, 0, 0, 1, 1, 0], [0, 0, 1, 1, 0, 0, 1, 0], [0, 1, 0, 1, 1, 0, 0, 0]]; + let pixels: Vec> = (0..90).map(|y| (0..400).map(|x| tile[y % 3][x % 8]).collect()).collect(); + let data = samples(&pixels); + assert_eq!(decode(&encode_jbig2_generic(&data, 400, 90).unwrap()), unpack(&data, 400, 90)); + } + + #[test] + fn what_it_cannot_encode_is_an_error() { + assert!(matches!(encode_jbig2_generic(&[0; 8], 0, 1), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_jbig2_generic(&[0; 8], 8, 0), Err(CodecError::Unsupported(_)))); + assert!(matches!(encode_jbig2_generic(&[0; 3], 16, 2), Err(CodecError::Corrupt { .. })), "two rows of 16 need 4 bytes"); + assert!(matches!(encode_jbig2_generic(&[], 1, 2), Err(CodecError::Corrupt { .. }))); + } + + /// The encoder stops where the decoder does, on each side and in total: past that it would + /// write a stream nothing here can read. 16,384 squared is exactly the bound on samples. + /// `encode_jbig2_shared` refuses what `encode_jbig2_generic` refuses: an image taller than + /// the decoder takes is `Unsupported`, one at the bound isn't refused. (Its one tall mark is + /// too large to be a symbol, so nothing is shared: `Ok(None)`.) + #[test] + fn shared_coding_takes_only_what_the_decoder_takes() { + let past = vec![0x7F; MAX_JBIG2_COLUMNS as usize + 1]; + let tall = (&past[..], 1, MAX_JBIG2_COLUMNS + 1); + assert!(matches!(encode_jbig2_shared(&[tall, tall]), Err(CodecError::Unsupported(_))), "too tall"); + let at = (&past[..MAX_JBIG2_COLUMNS as usize], 1, MAX_JBIG2_COLUMNS); + assert!(matches!(encode_jbig2_shared(&[at, at]), Ok(None)), "at the bound"); + } + + #[test] + fn it_encodes_only_what_the_decoder_takes() { + let unsupported = |columns, rows| matches!(encode_jbig2_generic(&[], columns, rows), Err(CodecError::Unsupported(_))); + assert!(unsupported(MAX_JBIG2_COLUMNS + 1, 1), "too wide"); + assert!(unsupported(1, MAX_JBIG2_COLUMNS + 1), "too tall"); + assert!(unsupported(16_385, 16_384), "too many samples, each side within the bound"); + assert!(unsupported(u32::MAX, u32::MAX), "the product doesn't overflow"); + // At the bounds what it writes reads back, through the decoder that has to read it. + let wide = vec![0x0F; (MAX_JBIG2_COLUMNS as usize).div_ceil(8) * 4]; + assert_eq!(decode(&encode_jbig2_generic(&wide, MAX_JBIG2_COLUMNS, 4).unwrap()), unpack(&wide, MAX_JBIG2_COLUMNS, 4)); + let tall = vec![0x7F; MAX_JBIG2_COLUMNS as usize]; + assert_eq!(decode(&encode_jbig2_generic(&tall, 1, MAX_JBIG2_COLUMNS).unwrap()), unpack(&tall, 1, MAX_JBIG2_COLUMNS)); + struct Rows(u32, u64); + impl hayro_jbig2::Decoder for Rows { + fn push_pixel(&mut self, black: bool) { + self.1 += u64::from(black); + } + fn push_pixel_chunk(&mut self, black: bool, chunks: u32) { + self.1 += u64::from(black) * u64::from(chunks) * 8; + } + fn next_line(&mut self) { + self.0 += 1; + } + } + // One black pixel, so a page that came back blank wouldn't pass. + let mut largest = vec![0xFF; 2_048 * 16_384]; + largest[0] = 0x7F; + let largest = encode_jbig2_generic(&largest, 16_384, 16_384).unwrap(); + let mut rows = Rows(0, 0); + hayro_jbig2::Image::new_embedded(&largest, None).unwrap().decode(&mut rows).unwrap(); + assert_eq!(rows.0, 16_384); + assert_eq!(rows.1, 1); + } + + proptest::proptest! { + /// Any bitmap, of noise or of runs, decodes to its own bits through an independent decoder. + #[test] + fn bitmaps_round_trip( + columns in 1u32..300, + rows in 1u32..24, + runs in proptest::collection::vec(0u8..=255, 1..48), + noise in proptest::bool::ANY, + ) { + let data = bitmap(columns, rows, &runs, noise); + let jbig2 = encode_jbig2_generic(&data, columns, rows).unwrap(); + proptest::prop_assert_eq!(decode(&jbig2), unpack(&data, columns, rows)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { + let _ = encode_jbig2_generic(&data, columns, rows); + } + } +} diff --git a/crates/stream-codecs/src/jbig2/symbols.rs b/crates/stream-codecs/src/jbig2/symbols.rs new file mode 100644 index 000000000..f56503a84 --- /dev/null +++ b/crates/stream-codecs/src/jbig2/symbols.rs @@ -0,0 +1,1329 @@ +//! JBIG2 symbol coding, lossless: each distinct mark of an image (a group of black pixels that +//! touch, side by side or at a corner) is stored once, as a symbol in a symbol dictionary, and +//! text regions place a copy of it wherever it occurs. Marks too large to be symbols (rules, +//! frames, pictures) are left to a generic region beside them, and so, in [`encode`]'s second +//! plan, are marks that occur only once. Only exact copies share a symbol, so the image decodes to the +//! same bits. Written from the committee draft's 6.4 (the text region decoding procedure), 6.5 +//! (the symbol dictionary decoding procedure), 7.4.2 and 7.4.3 (their segments) and Annex A (the +//! arithmetic integer coders). + +use std::collections::HashMap; +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::ops::Range; + +use super::{at_pixels, code_generic, generic_region, page_information, region_information, segment, segment_of}; +use crate::mq::{Context, MqEncoder}; + +/// The longest side of a mark stored as a symbol. A larger one is left to the generic region. +const MAX_SYMBOL_SIDE: u32 = 256; + +/// The most black runs looked at. Past it (halftones, noise) the image is left to generic coding, +/// which suits it better anyway; this bounds the memory the marks take. +const MAX_RUNS: usize = 1 << 20; + +/// The most distinct marks sorted into symbols, and the most pixels their bitmaps may hold +/// together: past either the image is left to generic coding. These bound the memory they take. +const MAX_SYMBOLS: usize = 1 << 16; +const MAX_SYMBOL_PIXELS: usize = 1 << 24; + +/// The most symbols one plan codes. `hayro-jbig2`, the decoder the Craft apps render JBIG2 with, +/// refuses a dictionary whose new and exported symbols pass 65,535 together, and a plan exports +/// every symbol it codes, so it refuses 32,768 or more; others may too, so a plan with more isn't +/// made. +const MAX_CODED_SYMBOLS: usize = (1 << 15) - 1; + +/// The most symbol instances one text region places. `hayro-jbig2`, the decoder the Craft apps render JBIG2 with, +/// refuses a text region of more than 10,000 (a page it then draws blank), so a page with more is +/// placed by several regions, each over a band of rows, all referring to the one dictionary. +const MAX_REGION_INSTANCES: usize = 10_000; + +/// The most symbols a group's shared dictionary holds ([`encode_shared`]): half of what a page may +/// place, so each page keeps room for its own. +const MAX_SHARED_SYMBOLS: usize = MAX_CODED_SYMBOLS / 2; + +// Segment numbers stay at most 256, so a referred-to one takes a byte (7.2.5): the shared +// dictionary, the page, its own dictionary, a text region for each MAX_REGION_INSTANCES of at most +// MAX_RUNS marks, and the generic region. +const _: () = assert!(MAX_RUNS.div_ceil(MAX_REGION_INSTANCES) + 4 <= 256); + +/// The integer coders' ranges (Table A.1): how many 1 bits pick the range, how many bits give the +/// offset into it, and its first value. The last range has no 0 after its prefix. +const RANGES: [(u32, u32, u32); 6] = [(0, 2, 0), (1, 4, 4), (2, 6, 20), (3, 8, 84), (4, 12, 340), (5, 32, 4436)]; + +/// A run of black pixels on row `y`, from `x0` up to but not including `x1`. +#[derive(Clone, Copy, Debug)] +struct Run { + y: u32, + x0: u32, + x1: u32, +} + +/// A mark: its box (`x1` and `y1` just past it) and its runs, as a range of the run order +/// [`marks`] gives. +#[derive(Debug)] +struct Mark { + x0: u32, + x1: u32, + y0: u32, + y1: u32, + runs: Range, +} + +/// A distinct mark's bitmap, a byte a pixel (1 black), and how many marks are exact copies of it. +#[derive(Debug)] +struct Symbol { + width: u32, + height: u32, + pixels: Vec, + uses: u32, +} + +/// An image's marks, sorted out: its black runs, its marks with the run order they index, the +/// distinct marks of at most [`MAX_SYMBOL_SIDE`] a side as symbols, each such mark as (its +/// symbol, the mark), and the marks too large to be symbols. +struct Page { + runs: Vec, + marks: Vec, + order: Vec, + symbols: Vec, + copies: Vec<(u32, u32)>, + large: Vec, +} + +/// A symbol instance (6.4.5): the symbol, and where its bottom-left pixel goes on the page, as S +/// (the column) and T (the row). `width` and `height` are the symbol's. +#[derive(Clone, Copy, Debug)] +struct Instance { + id: u32, + s: u32, + t: u32, + width: u32, + height: u32, +} + +/// A group of images coded by [`encode_shared`]: the globals' data, and each image's stream. +type SharedCoding = (Vec, Vec>>); + +/// The symbols of a group's shared dictionary, as one image's [`plan`] places them: the ID each of +/// the image's symbols has there, if it is there, and how many symbols that dictionary exports. +struct Shared { + ids: Vec>, + count: u32, +} + +/// `rows` rows of `columns` 1-bit PDF samples (black 0), symbol coded: the smaller of two plans, +/// every mark of at most [`MAX_SYMBOL_SIDE`] a side as a symbol, or only those with a copy. +/// `None` when neither plan has from two to [`MAX_CODED_SYMBOLS`] symbols, or the image is too +/// large or too busy to look at. +pub(super) fn encode(samples: &[u8], columns: u32, rows: u32) -> Option> { + let page = page(samples, columns)?; + [1, 2].into_iter().filter_map(|copies| plan(&page, columns, rows, copies, None, None)).min_by_key(Vec::len) +} + +/// Images, each as [`encode`] takes one, symbol coded sharing one dictionary: the globals' data, a +/// symbol dictionary segment associated with no page (number 0) holding the distinct marks found +/// in more than one image, and each image's stream placing them from there, the smaller of +/// [`encode`]'s two plans for its other marks. The group's marks are held within the limits one +/// image's are ([`MAX_SYMBOLS`], [`MAX_SYMBOL_PIXELS`]): past them, later marks aren't shared. Past +/// [`MAX_SHARED_SYMBOLS`] shared marks, those in the most images are kept. +/// +/// An image too large or too busy to look at, or that places none of the shared marks, gets `None`; +/// `None` overall when fewer than two marks are shared. +pub(super) fn encode_shared(images: &[(&[u8], u32, u32)]) -> Option { + // The group's distinct marks, each with how many images hold it (as its uses), looked at an + // image at a time. + let mut table: Vec = Vec::new(); + let mut by_hash: HashMap> = HashMap::new(); + let mut held = 0usize; + for (samples, columns, _) in images { + let Some(page) = page(samples, *columns) else { continue }; + // An image's symbols are distinct, so each counts an image once. + for symbol in page.symbols { + match find(&table, &by_hash, &symbol) { + Some(t) => { + if let Some(s) = table.get_mut(t as usize) { + s.uses = s.uses.saturating_add(1); + } + } + None => { + let Some(more) = held.checked_add(symbol.pixels.len()).filter(|h| *h <= MAX_SYMBOL_PIXELS && table.len() < MAX_SYMBOLS) else { + continue; + }; + held = more; + let t = u32::try_from(table.len()).ok()?; + by_hash.entry(fingerprint(&symbol)).or_default().push(t); + table.push(Symbol { uses: 1, ..symbol }); + } + } + } + } + let uses = |t: &u32| table.get(*t as usize).map_or(0, |s| s.uses); + let mut shared: Vec = (0..u32::try_from(table.len()).ok()?).filter(|t| uses(t) >= 2).collect(); + if shared.len() < 2 { + return None; + } + shared.sort_by_key(|t| std::cmp::Reverse(uses(t))); + shared.truncate(MAX_SHARED_SYMBOLS); + // The dictionary's order, as `plan` keeps it. + shared.sort_by_key(|t| table.get(*t as usize).map(|s| (s.height, s.width))); + let mut global = vec![None; table.len()]; + for (k, t) in shared.iter().enumerate() { + if let Some(slot) = global.get_mut(*t as usize) { + *slot = u32::try_from(k).ok(); + } + } + let symbols: Vec<&Symbol> = shared.iter().filter_map(|t| table.get(*t as usize)).collect(); + let mut globals = Vec::new(); + segment_of(&mut globals, 0, 0, 0, &[], true, &symbol_dictionary(&symbols)?); + let count = u32::try_from(symbols.len()).ok()?; + let streams = images + .iter() + .map(|(samples, columns, rows)| { + let page = page(samples, *columns)?; + let ids = page.symbols.iter().map(|s| find(&table, &by_hash, s).and_then(|t| global.get(t as usize).copied().flatten())).collect(); + let shared = Shared { ids, count }; + [1, 2].into_iter().filter_map(|copies| plan(&page, *columns, *rows, copies, None, Some(&shared))).min_by_key(Vec::len) + }) + .collect(); + Some((globals, streams)) +} + +/// What sorts a symbol's bitmap ([`symbols`] and [`encode_shared`] look exact copies up by it). +fn fingerprint(s: &Symbol) -> u64 { + let mut hasher = DefaultHasher::new(); + (s.width, s.height, &s.pixels).hash(&mut hasher); + hasher.finish() +} + +/// The index in `table` of the symbol with `s`'s bitmap, looked up by its [`fingerprint`]. +fn find(table: &[Symbol], by_hash: &HashMap>, s: &Symbol) -> Option { + by_hash + .get(&fingerprint(s))? + .iter() + .copied() + .find(|t| table.get(*t as usize).is_some_and(|t| (t.width, t.height) == (s.width, s.height) && t.pixels == s.pixels)) +} + +/// The image's marks and symbols; `None` past the limits above. +fn page(samples: &[u8], columns: u32) -> Option { + let width = usize::try_from(columns).ok()?; + let stride = width.div_ceil(8); + if stride == 0 { + return None; + } + let runs = black_runs(samples, width, stride)?; + let (marks, order) = marks(&runs); + let mut page = Page { runs, marks, order, symbols: Vec::new(), copies: Vec::new(), large: Vec::new() }; + symbols(&mut page)?; + Some(page) +} + +/// The black runs of each row (0 samples), row by row and left to right; `None` past [`MAX_RUNS`]. +fn black_runs(samples: &[u8], width: usize, stride: usize) -> Option> { + let black = |row: &[u8], x: usize| row.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) == 0); + let mut runs = Vec::new(); + for (y, row) in samples.chunks_exact(stride).enumerate() { + let y = u32::try_from(y).ok()?; + let mut x = 0; + while x < width { + // Whole white bytes at once. + if x % 8 == 0 && row.get(x / 8) == Some(&0xFF) { + x += 8; + continue; + } + if !black(row, x) { + x += 1; + continue; + } + let start = x; + while x < width && black(row, x) { + x += 1; + } + if runs.len() == MAX_RUNS { + return None; + } + runs.push(Run { y, x0: u32::try_from(start).ok()?, x1: u32::try_from(x).ok()? }); + } + } + Some(runs) +} + +/// The marks the runs make, in the order their first runs come (so from the top), and the run +/// order: the runs' indexes, mark by mark, each mark's in row order. +fn marks(runs: &[Run]) -> (Vec, Vec) { + // Union–find over the runs; a set's root is its first run, as the larger root always joins + // the smaller. Runs on adjacent rows join when they overlap, counting a corner. + let mut parent: Vec = (0..runs.len()).map(|i| i as u32).collect(); + let (mut above, mut above_at, mut above_y): (&[Run], usize, Option) = (&[], 0, None); + let mut at = 0; + for row in runs.chunk_by(|a, b| a.y == b.y) { + let y = row.first().map(|r| r.y); + if above_y.and_then(|a| a.checked_add(1)) == y { + let mut i = 0; + for (j, b) in row.iter().enumerate() { + while above.get(i).is_some_and(|a| a.x1 < b.x0) { + i += 1; + } + let mut k = i; + while above.get(k).is_some_and(|a| a.x0 <= b.x1) { + union(&mut parent, above_at + k, at + j); + k += 1; + } + } + } + (above, above_at, above_y) = (row, at, y); + at += row.len(); + } + let mut mark_of = vec![0u32; runs.len()]; + let mut marks: Vec = Vec::new(); + let mut counts: Vec = Vec::new(); + for (i, r) in runs.iter().enumerate() { + let root = find_root(&mut parent, i); + let m = if root == i { + marks.push(Mark { x0: r.x0, x1: r.x1, y0: r.y, y1: r.y.saturating_add(1), runs: 0..0 }); + counts.push(0); + marks.len() - 1 + } else { + mark_of.get(root).map_or(0, |m| *m as usize) + }; + if let Some(slot) = mark_of.get_mut(i) { + *slot = m as u32; + } + if let (Some(mark), Some(count)) = (marks.get_mut(m), counts.get_mut(m)) { + mark.x0 = mark.x0.min(r.x0); + mark.x1 = mark.x1.max(r.x1); + mark.y1 = mark.y1.max(r.y.saturating_add(1)); + *count += 1; + } + } + let mut next = 0; + for (mark, count) in marks.iter_mut().zip(&counts) { + mark.runs = next..next + count; + next += count; + } + let mut order = vec![0u32; runs.len()]; + let mut fill: Vec = marks.iter().map(|m| m.runs.start).collect(); + for (i, m) in mark_of.iter().enumerate() { + if let Some(n) = fill.get_mut(*m as usize) { + if let Some(slot) = order.get_mut(*n) { + *slot = i as u32; + } + *n += 1; + } + } + (marks, order) +} + +/// The root of `i`'s set, halving the path to it as it goes. Parents are never above their +/// children, so the walk ends. +fn find_root(parent: &mut [u32], mut i: usize) -> usize { + loop { + let Some(&p) = parent.get(i) else { return i }; + let p = p as usize; + if p == i { + return i; + } + let grandparent = parent.get(p).copied().unwrap_or(p as u32); + if let Some(slot) = parent.get_mut(i) { + *slot = grandparent; + } + i = grandparent as usize; + } +} + +fn union(parent: &mut [u32], a: usize, b: usize) { + let (a, b) = (find_root(parent, a), find_root(parent, b)); + if let Some(slot) = parent.get_mut(a.max(b)) { + *slot = a.min(b) as u32; + } +} + +/// Each mark's own pixels in its box: those of its runs, not of other marks the box overlaps. +fn pixels(mark: &Mark, runs: &[Run], order: &[u32]) -> Vec { + // A mark's box holds its runs, so these don't go below 0; a symbol is at most 256 a side. + let width = mark.x1.saturating_sub(mark.x0) as usize; + let height = mark.y1.saturating_sub(mark.y0) as usize; + let mut pixels = vec![0u8; width * height]; + for r in order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| runs.get(*i as usize)) { + let row = r.y.saturating_sub(mark.y0) as usize * width; + let from = row + r.x0.saturating_sub(mark.x0) as usize; + let to = row + r.x1.saturating_sub(mark.x0) as usize; + pixels.get_mut(from..to).unwrap_or_default().fill(1); + } + pixels +} + +/// Sorts the marks into symbols: exact copies share one. `None` past [`MAX_SYMBOLS`] or +/// [`MAX_SYMBOL_PIXELS`]. +fn symbols(page: &mut Page) -> Option<()> { + let mut by_hash: HashMap> = HashMap::new(); + let mut kept = 0usize; + for (m, mark) in page.marks.iter().enumerate() { + let m = u32::try_from(m).ok()?; + let (width, height) = (mark.x1.saturating_sub(mark.x0), mark.y1.saturating_sub(mark.y0)); + if width > MAX_SYMBOL_SIDE || height > MAX_SYMBOL_SIDE { + page.large.push(m); + continue; + } + let symbol = Symbol { width, height, pixels: pixels(mark, &page.runs, &page.order), uses: 0 }; + let symbols = &mut page.symbols; + let id = match find(symbols, &by_hash, &symbol) { + Some(id) => id, + None => { + kept = kept.checked_add(symbol.pixels.len())?; + if kept > MAX_SYMBOL_PIXELS || symbols.len() == MAX_SYMBOLS { + return None; + } + let id = u32::try_from(symbols.len()).ok()?; + by_hash.entry(fingerprint(&symbol)).or_default().push(id); + symbols.push(symbol); + id + } + }; + if let Some(s) = symbols.get_mut(id as usize) { + s.uses = s.uses.saturating_add(1); + } + page.copies.push((id, m)); + } + Some(()) +} + +/// One plan's stream: the symbols with at least `copies` uses in a symbol dictionary, text +/// regions placing them (in strips of 2^`log_strips` rows, or whichever size codes smallest), and +/// the other marks in a generic region. `None` with fewer than two such symbols (their IDs would +/// take no bits at all, SBSYMCODELEN 0, a corner of the standard left alone here) or more than +/// [`MAX_CODED_SYMBOLS`]. +/// +/// With `shared`, the symbols a group's dictionary holds are placed from there, whatever their uses +/// here, and the page's own dictionary holds only the others (none, when it has none), as +/// [`encode_shared`] writes them; `None` too for a page that places none of the shared ones. +fn plan(page: &Page, columns: u32, rows: u32, copies: u32, log_strips: Option, shared: Option<&Shared>) -> Option> { + let symbol = |s: u32| page.symbols.get(s as usize); + let global = |s: u32| shared.and_then(|g| g.ids.get(s as usize).copied().flatten()); + let globals = shared.map_or(0, |g| g.count); + // The dictionary's order (6.5.5): height classes from the shortest, each narrowest first. + let mut kept: Vec = + (0..page.symbols.len() as u32).filter(|s| global(*s).is_none() && symbol(*s).is_some_and(|s| s.uses >= copies)).collect(); + let count = usize::try_from(globals).ok()?.checked_add(kept.len())?; + if !(2..=MAX_CODED_SYMBOLS).contains(&count) { + return None; + } + kept.sort_by_key(|s| symbol(*s).map(|s| (s.height, s.width))); + // The shared symbols' IDs come first, as their dictionary is referred to first (Table 31's + // SBSYMS). + let mut id: Vec> = (0..page.symbols.len() as u32).map(global).collect(); + for (k, s) in kept.iter().enumerate() { + if let Some(slot) = id.get_mut(*s as usize) { + *slot = u32::try_from(k).ok().and_then(|k| k.checked_add(globals)); + } + } + let mut instances = Vec::new(); + let mut left = page.large.clone(); + for (s, m) in &page.copies { + match (id.get(*s as usize).copied().flatten(), page.marks.get(*m as usize), symbol(*s)) { + (Some(id), Some(mark), Some(symbol)) => { + instances.push(Instance { id, s: mark.x0, t: mark.y1.saturating_sub(1), width: symbol.width, height: symbol.height }) + } + _ => left.push(*m), + } + } + if shared.is_some() && !instances.iter().any(|i| i.id < globals) { + return None; + } + let kept: Vec<&Symbol> = kept.iter().filter_map(|s| symbol(*s)).collect(); + let dictionary = if kept.is_empty() { None } else { Some(symbol_dictionary(&kept)?) }; + let count = u32::try_from(count).ok()?; + // Bands of at most MAX_REGION_INSTANCES, from the top. + instances.sort_by_key(|i| (i.t, i.s)); + let mut regions = Vec::new(); + for band in instances.chunks_mut(MAX_REGION_INSTANCES) { + let top = band.iter().map(|i| i.t.saturating_add(1).saturating_sub(i.height)).min()?; + let height = band.iter().map(|i| i.t).max()?.checked_sub(top)?.checked_add(1)?; + regions.push(match log_strips { + Some(l) => text_region(band, count, columns, top, height, l)?, + None => (0..4).filter_map(|l| text_region(band, count, columns, top, height, l)).min_by_key(Vec::len)?, + }); + } + let mut out = Vec::new(); + // The shared dictionary is segment 0, in the globals, and a segment refers only to lower + // numbers (7.2.5): the page's own follow it. + let mut number = u32::from(shared.is_some()); + segment(&mut out, number, 48, &[], false, &page_information(columns, rows)); + number += 1; + let mut dictionaries: Vec = shared.map(|_| 0).into_iter().collect(); + if let Some(dictionary) = &dictionary { + segment(&mut out, number, 0, &[], true, dictionary); + dictionaries.push(u8::try_from(number).ok()?); + number += 1; + } + for (k, text) in regions.iter().enumerate() { + // The dictionaries stay retained for the next region, until the last (7.2.4). + let refers_to: Vec<(u8, bool)> = dictionaries.iter().map(|n| (*n, k + 1 < regions.len())).collect(); + segment(&mut out, number, 7, &refers_to, false, text); + number += 1; + } + if !left.is_empty() { + segment(&mut out, number, 39, &[], false, &leftover_region(&left, page)?); + } + Some(out) +} + +/// A symbol dictionary segment's data (7.4.2) defining `symbols` (in dictionary order) and +/// exporting them all: arithmetic coded, no refinement or aggregation, each bitmap a generic +/// region with template 0, its nominal AT pixels and no typical prediction (Table 16), the +/// bitmaps sharing one set of contexts. +fn symbol_dictionary(symbols: &[&Symbol]) -> Option> { + let count = u32::try_from(symbols.len()).ok()?; + let mut data = vec![0, 0]; + at_pixels(&mut data); + data.extend_from_slice(&count.to_be_bytes()); + data.extend_from_slice(&count.to_be_bytes()); + let mut mq = MqEncoder::new(); + let mut contexts: Vec = vec![0; 1 << 16]; + let (mut iadh, mut iadw, mut iaex) = (IntCoder::new(), IntCoder::new(), IntCoder::new()); + let mut height = 0; + for class in symbols.chunk_by(|a, b| a.height == b.height) { + let h = class.first()?.height; + iadh.encode(&mut mq, Some(int(i64::from(h) - i64::from(height))?)); + height = h; + let mut width = 0; + for s in class { + iadw.encode(&mut mq, Some(int(i64::from(s.width) - i64::from(width))?)); + width = s.width; + let w = s.width as usize; + code_generic(&mut mq, &mut contexts, w, s.height as usize, false, |y, row| { + let line = s.pixels.get(y * w..).unwrap_or_default(); + row.iter_mut().zip(line).for_each(|(p, v)| *p = *v); + }); + } + iadw.encode(&mut mq, None); + } + // The export flags (6.5.10): a run of no symbols not exported, then a run of all of them. + iaex.encode(&mut mq, Some(0)); + iaex.encode(&mut mq, Some(int(i64::from(count))?)); + data.extend_from_slice(&mq.finish()); + Some(data) +} + +/// An immediate lossless text region segment's data (7.4.3) placing `instances` in the band of +/// `height` rows from row `top` (which holds them): arithmetic coded, no refinement, strips of +/// 2^`log_strips` rows, each symbol's bottom-left pixel at (S, T) (REFCORNER BOTTOMLEFT, so a +/// line's letters share their T), not transposed, OR, default pixel 0, SBDSOFFSET 0. Sorts +/// `instances` into strips, each left to right. +fn text_region(instances: &mut [Instance], symbols: u32, columns: u32, top: u32, height: u32, log_strips: u32) -> Option> { + // LOGSBSTRIPS has two bits: strips of 1, 2, 4 or 8 rows. + if log_strips > 3 { + return None; + } + let strips = 1i64 << log_strips; + // T counts from the band's top row. + let t = |i: &Instance| i.t.saturating_sub(top); + instances.sort_by_key(|i| (t(i) >> log_strips, i.s)); + let mut data = Vec::new(); + region_information(&mut data, columns, height, 0, top); + data.extend_from_slice(&((log_strips as u16) << 2).to_be_bytes()); + data.extend_from_slice(&u32::try_from(instances.len()).ok()?.to_be_bytes()); + // SBSYMCODELEN (Table 31): ⌈log2 SBNUMSYMS⌉. + let length = symbols.checked_next_power_of_two()?.trailing_zeros(); + let mut ids: Vec = vec![0; 1 << length]; + let (mut iadt, mut iafs, mut iads, mut iait) = (IntCoder::new(), IntCoder::new(), IntCoder::new(), IntCoder::new()); + let mut mq = MqEncoder::new(); + // The initial STRIPT (6.4.5 step 2) is 0, so each strip's delta T is its first row over the + // strip size, from the strip before. + iadt.encode(&mut mq, Some(0)); + let (mut strip_t, mut first_s) = (0i64, 0i64); + for strip in instances.chunk_by(|a, b| t(a) >> log_strips == t(b) >> log_strips) { + let first = i64::from(t(strip.first()?) >> log_strips << log_strips); + iadt.encode(&mut mq, Some(int((first - strip_t) / strips)?)); + strip_t = first; + let mut cur_s = 0; + for (k, instance) in strip.iter().enumerate() { + let s = i64::from(instance.s); + if k == 0 { + iafs.encode(&mut mq, Some(int(s - first_s)?)); + first_s = s; + } else { + iads.encode(&mut mq, Some(int(s - cur_s)?)); + } + if strips > 1 { + iait.encode(&mut mq, Some(int(i64::from(t(instance)) - strip_t)?)); + } + encode_id(&mut mq, &mut ids, length, instance.id); + // The next S counts from this symbol's last column (6.4.5 step 3c x). + cur_s = s + i64::from(instance.width) - 1; + } + iads.encode(&mut mq, None); + } + data.extend_from_slice(&mq.finish()); + Some(data) +} + +/// A generic region segment's data holding the marks `left`, over their box. +fn leftover_region(left: &[u32], page: &Page) -> Option> { + let left: Vec<&Mark> = left.iter().filter_map(|m| page.marks.get(*m as usize)).collect(); + let x0 = left.iter().map(|m| m.x0).min()?; + let y0 = left.iter().map(|m| m.y0).min()?; + let x1 = left.iter().map(|m| m.x1).max()?; + let y1 = left.iter().map(|m| m.y1).max()?; + let (width, height) = (x1.saturating_sub(x0) as usize, y1.saturating_sub(y0) as usize); + let stride = width.div_ceil(8); + // Packed, 1 for black: no larger than the image's own samples. + let mut packed = vec![0u8; stride.checked_mul(height)?]; + for mark in &left { + for r in page.order.get(mark.runs.clone()).unwrap_or_default().iter().filter_map(|i| page.runs.get(*i as usize)) { + let row = r.y.saturating_sub(y0) as usize * stride; + for x in r.x0.saturating_sub(x0) as usize..r.x1.saturating_sub(x0) as usize { + if let Some(b) = packed.get_mut(row + x / 8) { + *b |= 0x80 >> (x % 8); + } + } + } + } + let mut mq = MqEncoder::new(); + let mut contexts: Vec = vec![0; 1 << 16]; + code_generic(&mut mq, &mut contexts, width, height, true, |y, row| { + let line = packed.get(y * stride..).unwrap_or_default(); + for (x, p) in row.iter_mut().enumerate() { + *p = u8::from(line.get(x / 8).is_some_and(|b| b & (0x80 >> (x % 8)) != 0)); + } + }); + Some(generic_region(u32::try_from(width).ok()?, u32::try_from(height).ok()?, x0, y0, &mq.finish())) +} + +fn int(v: i64) -> Option { + i32::try_from(v).ok() +} + +/// An arithmetic integer coder other than IAID (A.2): its 512 contexts, indexed by PREV. +struct IntCoder([Context; 512]); + +impl IntCoder { + fn new() -> Self { + IntCoder([0; 512]) + } + + /// Codes `value`, or OOB for `None`. + fn encode(&mut self, mq: &mut MqEncoder, value: Option) { + for (cx, d) in decisions(value) { + if let Some(state) = self.0.get_mut(cx) { + mq.encode(state, d); + } + } + } +} + +/// The decisions that code `value`, or OOB for `None`, with their contexts (A.2, Table A.1): the +/// sign, the prefix that picks the range, then the offset into it, most significant bit first; +/// OOB is a negative 0. Each decision's context is PREV: a leading 1 and the decisions before it, +/// at most the last eight once there are more. +fn decisions(value: Option) -> Vec<(usize, bool)> { + let (negative, magnitude) = value.map_or((true, 0), |v| (v < 0, v.unsigned_abs())); + let (ones, bits, first) = + RANGES.into_iter().find(|(_, bits, first)| *bits == 32 || u64::from(magnitude) < u64::from(*first) + (1 << bits)).unwrap_or(RANGES[5]); + let offset = magnitude - first; + let mut d = vec![negative]; + d.extend(std::iter::repeat_n(true, ones as usize)); + if ones < 5 { + d.push(false); + } + d.extend((0..bits).rev().map(|i| offset >> i & 1 == 1)); + let mut prev = 1usize; + d.into_iter() + .map(|bit| { + let cx = prev; + prev = if prev < 256 { prev << 1 | usize::from(bit) } else { ((prev << 1 | usize::from(bit)) & 511) | 256 }; + (cx, bit) + }) + .collect() +} + +/// Codes symbol ID `id` in `length` bits, most significant first, through IAID (A.3): each +/// decision's context is a leading 1 and all the bits before it. +fn encode_id(mq: &mut MqEncoder, contexts: &mut [Context], length: u32, id: u32) { + let mut prev = 1usize; + for i in (0..length).rev() { + let d = id >> i & 1 == 1; + if let Some(state) = contexts.get_mut(prev) { + mq.encode(state, d); + } + prev = prev << 1 | usize::from(d); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + use crate::jbig2::tests::{decode, decode_with, samples, unpack}; + use crate::jbig2::{encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; + + type Bitmap = Vec>; + + /// A small linear congruential generator, so the pictures are the same every run. + struct Lcg(u64); + + impl Lcg { + fn below(&mut self, n: usize) -> usize { + self.0 = self.0.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + (self.0 >> 33) as usize % n.max(1) + } + } + + /// `count` different letter-like shapes: three strokes two pixels thick in a box of 6 to 18 + /// pixels a side, each after the first starting on one before it, so a shape is one mark; + /// trimmed to its pixels. + fn glyphs(count: usize, seed: u64) -> Vec { + let mut r = Lcg(seed); + let mut out: Vec = Vec::new(); + while out.len() < count { + let (w, h) = (6 + r.below(13), 6 + r.below(13)); + let mut g = vec![vec![0u8; w]; h]; + let (mut x, mut y) = (r.below(w), r.below(h)); + for _ in 0..3 { + let (dx, dy) = [(1, 0), (0, 1), (1, 1)][r.below(3)]; + let mut stroke = Vec::new(); + while x < w && y < h { + g[y][x] = 1; + g[y][(x + 1).min(w - 1)] = 1; + stroke.push((x, y)); + x += dx; + y += dy; + } + (x, y) = stroke[r.below(stroke.len())]; + } + let rows: Vec = (0..h).filter(|y| g[*y].contains(&1)).collect(); + let columns: Vec = (0..w).filter(|x| g.iter().any(|row| row[*x] == 1)).collect(); + let g: Bitmap = g[rows[0]..=rows[rows.len() - 1]].iter().map(|row| row[columns[0]..=columns[columns.len() - 1]].to_vec()).collect(); + if !out.contains(&g) { + out.push(g); + } + } + out + } + + /// Draws `glyph` with its top-left pixel at (`x`, `y`), clipped to the page, by OR. + fn draw(page: &mut Bitmap, glyph: &Bitmap, x: i64, y: i64) { + for (gy, row) in glyph.iter().enumerate() { + for (gx, p) in row.iter().enumerate() { + let (px, py) = (x + gx as i64, y + gy as i64); + if *p == 1 + && px >= 0 + && py >= 0 + && let Some(cell) = page.get_mut(py as usize).and_then(|r| r.get_mut(px as usize)) + { + *cell = 1; + } + } + } + } + + /// Lines of text: glyphs side by side, one to four pixels apart, each line's glyphs on its + /// baseline, the lines 30 rows apart; the first glyphs are the most common, as letters are. + fn text(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64) -> Bitmap { + let mut page = vec![vec![0u8; columns]; rows]; + let mut r = Lcg(seed); + for baseline in (24..rows).step_by(30) { + let mut x = 4 + r.below(8); + loop { + let common = r.below(glyphs.len()) + 1; + let g = &glyphs[r.below(common)]; + if x + g[0].len() > columns { + break; + } + draw(&mut page, g, x as i64, (baseline + 1 - g.len()) as i64); + x += g[0].len() + 1 + r.below(4); + } + } + page + } + + /// `n` glyphs anywhere, some over the edges, some over each other. + fn scatter(columns: usize, rows: usize, glyphs: &[Bitmap], seed: u64, n: usize) -> Bitmap { + let mut page = vec![vec![0u8; columns]; rows]; + let mut r = Lcg(seed); + for _ in 0..n { + let g = &glyphs[r.below(glyphs.len())]; + draw(&mut page, g, r.below(columns + 12) as i64 - 6, r.below(rows + 12) as i64 - 6); + } + page + } + + /// A segment's number and type, whether it is retained, and the segments it refers to, each + /// with its retain bit. + type Segment = (u32, u8, bool, Vec<(u8, bool)>); + + /// The segments of a stream in the embedded organisation, from their headers (7.2). + fn segments(data: &[u8]) -> Vec { + let (mut segments, mut at) = (Vec::new(), 0); + while at < data.len() { + let flags = data[at + 5]; + let refers = usize::from(flags >> 5); + let length_at = at + 6 + refers + 1; + let length = u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; + let referred = data[at + 6..at + 6 + refers].iter().enumerate().map(|(k, n)| (*n, flags >> (k + 1) & 1 == 1)).collect(); + let number = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()); + segments.push((number, data[at + 4] & 0x3F, flags & 1 == 1, referred)); + at = length_at + 4 + length; + } + segments + } + + fn segment_types(data: &[u8]) -> Vec { + segments(data).into_iter().map(|(_, kind, ..)| kind).collect() + } + + /// Every plan, at every strip size, decodes to the picture; returns how many plans there were. + fn every_plan_round_trips(picture: &Bitmap) -> usize { + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let data = samples(picture); + let page = page(&data, columns).unwrap(); + let mut plans = 0; + for copies in [1, 2] { + for log_strips in 0..4 { + if let Some(coded) = plan(&page, columns, rows, copies, Some(log_strips), None) { + assert_eq!(decode(&coded), unpack(&data, columns, rows), "{copies} copies, strips of {}", 1 << log_strips); + plans += 1; + } + } + } + plans + } + + /// Annex A.2's example (12 through IADW: the decisions 0101000, in contexts 1, 10, 101, …) + /// and Table A.1's codes. + #[test] + fn integers_are_coded_as_annex_a_says() { + assert_eq!(decisions(Some(12)), [(1, false), (2, true), (5, false), (10, true), (21, false), (42, false), (84, false)]); + let bits = |v| decisions(v).iter().map(|(_, d)| u8::from(*d)).collect::>(); + assert_eq!(bits(None), [1, 0, 0, 0], "OOB"); + assert_eq!(bits(Some(0)), [0, 0, 0, 0]); + assert_eq!(bits(Some(-1)), [1, 0, 0, 1]); + assert_eq!(bits(Some(-3)), [1, 0, 1, 1]); + assert_eq!(bits(Some(4)), [0, 1, 0, 0, 0, 0, 0]); + assert_eq!(bits(Some(-19)), [1, 1, 0, 1, 1, 1, 1]); + assert_eq!(bits(Some(83)), [0, 1, 1, 0, 1, 1, 1, 1, 1, 1]); + assert_eq!(bits(Some(4435))[..6], [0, 1, 1, 1, 1, 0]); + let big = bits(Some(4436)); + assert_eq!((big.len(), &big[..6]), (6 + 32, &[0, 1, 1, 1, 1, 1][..])); + assert!(big[6..].iter().all(|b| *b == 0)); + // PREV keeps a leading 1 and the last eight decisions, so it stays below 512, also when + // those eight are all 0 (PREV exactly 256), as in 4436's offset of 0. + for value in [Some(i32::MIN), Some(4436), Some(-4436), Some(4436 + 0x100), Some(i32::MAX), None] { + let contexts: Vec = decisions(value).iter().map(|(cx, _)| *cx).collect(); + assert!(contexts.iter().take(8).all(|cx| *cx < 256) && contexts.iter().skip(8).all(|cx| (256..512).contains(cx)), "{value:?}"); + } + assert!(decisions(Some(4436)).iter().any(|(cx, _)| *cx == 256)); + } + + /// Strips and first columns thousands of pixels apart need Annex A's 32-bit range: a mark at + /// the far right of a 12,000-pixel page, then one at the left below it (S back by 11,000), and + /// lines 5,000 rows apart. Each shape occurs twice, so both plans are made. + #[test] + fn coordinates_past_4435_round_trip() { + let shapes = glyphs(3, 4); + let mut picture = vec![vec![0u8; 12_000]; 40]; + for (k, x) in [11_000, 0, 5_000, 11_980].into_iter().enumerate() { + draw(&mut picture, &shapes[k % 2], x, 2 + 9 * k as i64); + } + assert_eq!(every_plan_round_trips(&picture), 8); + let mut tall = vec![vec![0u8; 30]; 10_050]; + for (k, y) in [0, 5_000, 10_030].into_iter().enumerate() { + draw(&mut tall, &shapes[k % 3], 3, y); + draw(&mut tall, &shapes[(k + 1) % 3], 15, y); + } + assert_eq!(every_plan_round_trips(&tall), 8); + } + + /// The padding bits of a row whose width isn't a multiple of 8 are ignored whatever they are: + /// black (0) padding, as many scans have, neither joins the marks nor stops symbol coding. + #[test] + fn black_padding_bits_are_ignored() { + for columns in [13, 241, 901] { + let picture = text(columns, 120, &glyphs(12, 6), 8); + let mut data = samples(&picture); + let stride = columns.div_ceil(8); + for row in data.chunks_mut(stride) { + row[stride - 1] &= 0xFF << (8 * stride - columns); + } + let (c, r) = (columns as u32, 120); + let page = page(&data, c).unwrap(); + assert!(page.marks.iter().all(|m| m.x1 as usize <= columns), "{columns} columns"); + for copies in [1, 2] { + if let Some(coded) = plan(&page, c, r, copies, None, None) { + assert_eq!(decode(&coded), unpack(&data, c, r), "{columns} columns"); + } + } + if columns > 13 { + assert_eq!(segment_types(&encode_jbig2(&data, c, r).unwrap())[1], 0, "{columns} columns: symbol coded"); + } + } + } + + /// Pixels touching only at a corner, down to the right or down to the left, are one mark + /// (8-connected); side by side with a white column between, two. A mark of 256 pixels a side + /// is a symbol; one 257 wide, or 257 tall, is left to the generic region. The diagonal and the + /// dot occur twice, so both plans are made. + #[test] + fn marks_join_at_corners_and_large_ones_are_left_out() { + let mut picture = vec![vec![0u8; 300]; 300]; + for i in 0..6 { + picture[10 + i][10 + i] = 1; + picture[10 + i][100 + i] = 1; + picture[10 + i][60 - i] = 1; + } + for row in &mut picture[0..257] { + row[280] = 1; + } + picture[30][10] = 1; + picture[30][12] = 1; + picture[40][0..256].fill(1); + picture[60][0..257].fill(1); + for row in &mut picture[44..300] { + row[290] = 1; + } + let data = samples(&picture); + let page = page(&data, 300).unwrap(); + let boxes: Vec<(u32, u32, u32, u32)> = page.marks.iter().map(|m| (m.x0, m.y0, m.x1 - m.x0, m.y1 - m.y0)).collect(); + assert_eq!( + boxes, + [ + (280, 0, 1, 257), + (10, 10, 6, 6), + (55, 10, 6, 6), + (100, 10, 6, 6), + (10, 30, 1, 1), + (12, 30, 1, 1), + (0, 40, 256, 1), + (290, 44, 1, 256), + (0, 60, 257, 1) + ] + ); + let large: Vec<(u32, u32)> = page.large.iter().map(|m| (page.marks[*m as usize].x0, page.marks[*m as usize].y0)).collect(); + assert_eq!(large, [(280, 0), (0, 60)]); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Past 2^20 black runs the image is left to generic coding; at the cap it is still looked at. + #[test] + fn busy_images_are_left_to_generic_coding() { + // 0x55: four black pixels a byte, each its own run, under the same ones on the next row. + let columns = 1 << 20; + let row = vec![0x55u8; columns / 8]; + assert!(page(&row.repeat(2), columns as u32).is_some(), "2^20 runs"); + assert!(page(&row.repeat(3), columns as u32).is_none(), "1.5 × 2^20 runs"); + } + + /// A plan codes at most 32,767 symbols, which an independent decoder reads back; with more, + /// the second plan can still code the repeated ones. Past 65,536 distinct marks, or 2^24 + /// pixels of them, the image is left to generic coding; at either cap it is still looked at. + #[test] + fn too_many_symbols_are_left_to_generic_coding() { + // `n` distinct marks of 17 × 3 (full top and bottom rows joined by a code of 1 to n, in + // binary, on the row between), then `again` more with the codes 1 and 2 in turn. + let codes = |n: usize, again: usize| { + let (columns, rows) = (256 * 18, (n + again).div_ceil(256) * 4); + let mut picture = vec![vec![0u8; columns]; rows]; + for k in 0..n + again { + let (x, y) = (k % 256 * 18, k / 256 * 4); + let code = if k < n { k + 1 } else { (k - n) % 2 + 1 }; + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + for b in (0..17).filter(|b| code >> b & 1 == 1) { + picture[y + 1][x + b] = 1; + } + } + (samples(&picture), columns as u32, rows as u32) + }; + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS, 0); + let at_cap = page(&data, columns).unwrap(); + assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); + // One more distinct mark, each once: neither plan is made. + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 0); + let past = page(&data, columns).unwrap(); + assert!(plan(&past, columns, rows, 1, None, None).is_none() && plan(&past, columns, rows, 2, None, None).is_none()); + assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); + // With two of them repeated, the second plan codes those two. + let (data, columns, rows) = codes(MAX_CODED_SYMBOLS + 1, 200); + let past = page(&data, columns).unwrap(); + assert!(plan(&past, columns, rows, 1, None, None).is_none()); + assert_eq!(decode(&plan(&past, columns, rows, 2, None, None).unwrap()), unpack(&data, columns, rows)); + // Above a page of text whose letters repeat, that second plan is the one `encode_jbig2` + // keeps: the first, of every mark, would code too many symbols. + let mut picture: Bitmap = vec![vec![0u8; 256 * 18]; 512]; + for k in 0..=MAX_CODED_SYMBOLS { + let (x, y) = (k % 256 * 18, k / 256 * 4); + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + for b in (0..17).filter(|b| (k + 1) >> b & 1 == 1) { + picture[y + 1][x + b] = 1; + } + } + picture.extend(text(256 * 18, 600, &glyphs(40, 7), 11)); + let data = samples(&picture); + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let coded = encode_jbig2(&data, columns, rows).unwrap(); + assert_eq!(segment_types(&coded)[1], 0, "symbol coded"); + assert_eq!(decode(&coded), unpack(&data, columns, rows)); + let (data, columns, _) = codes(MAX_SYMBOLS, 0); + assert_eq!(page(&data, columns).map(|p| p.symbols.len()), Some(MAX_SYMBOLS)); + let (data, columns, _) = codes(MAX_SYMBOLS + 1, 0); + assert!(page(&data, columns).is_none()); + // `n` black squares of 256 × 256, each with a white pixel in a different place: 2^16 + // pixels a symbol, so 256 of them are exactly the cap. + let squares = |n: usize| { + let (columns, rows) = (16 * 258, n.div_ceil(16) * 258); + let mut picture = vec![vec![0u8; columns]; rows]; + for k in 0..n { + let (x, y) = (k % 16 * 258, k / 16 * 258); + for row in &mut picture[y..y + 256] { + row[x..x + 256].fill(1); + } + picture[y + 1 + k / 254][x + 1 + k % 254] = 0; + } + (samples(&picture), columns as u32, rows as u32) + }; + let (data, columns, rows) = squares(256); + let at_cap = page(&data, columns).unwrap(); + assert_eq!(at_cap.symbols.len(), 256); + assert_eq!(decode(&plan(&at_cap, columns, rows, 1, None, None).unwrap()), unpack(&data, columns, rows)); + let (data, columns, _) = squares(257); + assert!(page(&data, columns).is_none()); + } + + /// More than 10,000 marks are placed by several text regions, each of at most 10,000 (more + /// than `hayro-jbig2` accepts in one), over bands of rows, all referring to + /// the dictionary, which each region but the last keeps retained; the generic region comes + /// after them, every segment numbered one more than the one before. + #[test] + fn more_than_10000_marks_are_placed_by_several_regions() { + // 25,000 copies of two 17 × 3 shapes, 256 a line, the lines 5 rows apart, and a bar too + // tall to be a symbol beside them. + let mut picture = vec![vec![0u8; 256 * 18 + 100]; 25_000usize.div_ceil(256) * 5]; + for k in 0..25_000 { + let (x, y) = (k % 256 * 18, k / 256 * 5); + picture[y][x..x + 17].fill(1); + picture[y + 2][x..x + 17].fill(1); + picture[y + 1][x + k % 2] = 1; + } + for row in &mut picture[0..300] { + row[256 * 18 + 50] = 1; + } + let data = samples(&picture); + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let coded = encode_jbig2(&data, columns, rows).unwrap(); + let region = |number, kept| (number, 7, false, vec![(1, kept)]); + assert_eq!( + segments(&coded), + [(0, 48, false, vec![]), (1, 0, true, vec![]), region(2, true), region(3, true), region(4, false), (5, 39, false, vec![])] + ); + assert_eq!(decode(&coded), unpack(&data, columns, rows)); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// A page of text whose letters repeat exactly, as rendered text does, keeps each letter once: + /// a symbol dictionary and a text region, much smaller than the generic coding. + #[test] + fn repeated_marks_are_stored_once() { + let picture = text(1700, 400, &glyphs(40, 7), 11); + let data = samples(&picture); + let coded = encode_jbig2(&data, 1700, 400).unwrap(); + let generic = encode_jbig2_generic(&data, 1700, 400).unwrap(); + // The dictionary is retained for the text region, which refers to it (7.2.4, 7.2.5). + assert_eq!(segments(&coded), [(0, 48, false, vec![]), (1, 0, true, vec![]), (2, 7, false, vec![(1, false)])]); + assert_eq!(decode(&coded), unpack(&data, 1700, 400)); + assert!(coded.len() * 2 < generic.len(), "symbols {} bytes, generic {}", coded.len(), generic.len()); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Marks too large to be symbols (a frame, a rule) go to a generic region over their box, and + /// with the second plan so do marks that occur once; the page still decodes to its bits. + #[test] + fn large_and_lone_marks_are_left_to_a_generic_region() { + let mut picture = text(900, 300, &glyphs(25, 3), 5); + picture[10][100..800].fill(1); + picture[290][100..800].fill(1); + for row in &mut picture[10..=290] { + row[100] = 1; + row[799] = 1; + } + // A lone blob, unlike any letter. + for (y, row) in picture[150..170].iter_mut().enumerate() { + row[850 + y % 7] = 1; + } + let data = samples(&picture); + let coded = encode_jbig2(&data, 900, 300).unwrap(); + assert_eq!(segment_types(&coded), [48, 0, 7, 39]); + assert_eq!(decode(&coded), unpack(&data, 900, 300)); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// Marks on every edge, marks touching only at a corner (one mark), a mark inside another's + /// box, and marks over each other's boxes. + #[test] + fn marks_at_the_edges_and_corners_round_trip() { + let dot = vec![vec![1u8, 1], vec![1, 1]]; + let ell = vec![vec![1u8, 0, 0], vec![1, 0, 0], vec![1, 1, 1]]; + let mut picture = vec![vec![0u8; 41]; 23]; + for (x, y) in [(0, 0), (39, 0), (0, 21), (39, 21), (20, 0), (0, 10), (39, 10), (20, 21)] { + draw(&mut picture, &dot, x, y); + } + for (x, y) in [(10, 5), (28, 14), (12, 14)] { + draw(&mut picture, &ell, x, y); + } + // Corners touching: one mark (`marks_join_at_corners_and_large_ones_are_left_out` checks it). + picture[3][6] = 1; + picture[4][7] = 1; + picture[5][8] = 1; + // A ring with a dot inside, and an ell over the ring's box. + for i in 0..7 { + picture[8][20 + i] = 1; + picture[14][20 + i] = 1; + picture[8 + i][20] = 1; + picture[8 + i][26] = 1; + } + draw(&mut picture, &dot, 22, 10); + draw(&mut picture, &ell, 25, 15); + assert_eq!(every_plan_round_trips(&picture), 8); + } + + /// One shape, however often, would need symbol IDs of no bits: that image is coded as a generic + /// region, as are a blank page and pure noise. + #[test] + fn one_shape_blank_pages_and_noise_are_coded_generically() { + let letter = &glyphs(1, 9)[..]; + let mut r = Lcg(1); + let noise: Bitmap = (0..60).map(|_| (0..200).map(|_| u8::from(r.below(2) == 1)).collect()).collect(); + for picture in [text(600, 200, letter, 2), vec![vec![0u8; 300]; 40], noise] { + let (columns, rows) = (picture[0].len() as u32, picture.len() as u32); + let data = samples(&picture); + assert_eq!(encode_jbig2(&data, columns, rows).unwrap(), encode_jbig2_generic(&data, columns, rows).unwrap()); + } + } + + /// Each segment's page association (7.2.6): the byte after the numbers it refers to. + fn associations(data: &[u8]) -> Vec { + let (mut pages, mut at) = (Vec::new(), 0); + while at < data.len() { + let refers = usize::from(data[at + 5] >> 5); + pages.push(data[at + 6 + refers]); + let length_at = at + 6 + refers + 1; + at = length_at + 4 + u32::from_be_bytes(data[length_at..length_at + 4].try_into().unwrap()) as usize; + } + pages + } + + /// `pictures`, `columns` × `rows` each, coded sharing a dictionary; each image's stream, which + /// has to be there, decodes with the globals to its picture. + fn shared_round_trip(pictures: &[Bitmap]) -> (Vec, Vec>) { + let (columns, rows) = (pictures[0][0].len() as u32, pictures[0].len() as u32); + let data: Vec> = pictures.iter().map(|p| samples(p)).collect(); + let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], columns, rows)).collect(); + let shared = encode_jbig2_shared(&images).unwrap().unwrap(); + let (globals, coded) = (shared.globals, shared.images); + let coded: Vec> = coded.into_iter().map(Option::unwrap).collect(); + for (d, c) in data.iter().zip(&coded) { + assert_eq!(decode_with(c, Some(&globals)), unpack(d, columns, rows)); + } + (globals, coded) + } + + /// `font` with a stem 20 pixels tall down each glyph's left side: a font sharing no glyph with + /// any [`glyphs`] gives, whose glyphs are at most 18 pixels tall. + fn stemmed(font: &[Bitmap]) -> Vec { + font.iter() + .map(|g| { + (0..g.len().max(20)).map(|y| std::iter::once(1).chain(g.get(y).cloned().unwrap_or_else(|| vec![0; g[0].len()])).collect()).collect() + }) + .collect() + } + + /// Mark `k` of a set of distinct ones: a stroke 33 pixels long with a pixel above every other + /// column for each 1 bit of `k`, below 2^17. + fn comb(k: usize) -> Bitmap { + let mut g = vec![vec![0u8; 33]; 2]; + g[1].fill(1); + for bit in 0..17 { + g[0][2 * bit] = u8::from(k >> bit & 1 == 1); + } + g + } + + /// The combs `ks`, 120 a row, the rows four pixels apart. + fn combs(ks: impl ExactSizeIterator) -> Bitmap { + let mut page = vec![vec![0u8; 120 * 36]; ks.len().div_ceil(120) * 4]; + for (i, k) in ks.enumerate() { + draw(&mut page, &comb(k), (i % 120 * 36) as i64, (i / 120 * 4) as i64); + } + page + } + + /// The symbols a symbol dictionary segment exports (SDNUMEXSYMS, 7.4.2.1.4), in a stream + /// that is that segment alone: after its header, flags and adaptive pixels. + fn exported(globals: &[u8]) -> usize { + u32::from_be_bytes(globals[21..25].try_into().unwrap()) as usize + } + + /// Pages in one font share its letters: one symbol dictionary, segment 0, associated with no + /// page and retained (7.2.4, 7.2.6), which each page's text region refers to first, its own + /// segments numbered after it (7.2.5). Together they take less than they take alone. + #[test] + fn pages_in_one_font_share_one_dictionary() { + let font = glyphs(40, 7); + let pictures: Vec = (0..4).map(|seed| text(1200, 300, &font, 20 + seed)).collect(); + let (globals, coded) = shared_round_trip(&pictures); + assert_eq!(segments(&globals), [(0, 0, true, vec![])]); + assert_eq!(associations(&globals), [0]); + let (mut shared, mut alone) = (globals.len(), 0); + for (picture, c) in pictures.iter().zip(&coded) { + let segments = segments(c); + assert_eq!(segments.first().map(|s| s.0), Some(1), "the page's own segments follow"); + assert!(segments.iter().filter(|s| s.1 == 7).all(|s| s.3.first() == Some(&(0, false))), "{segments:?}"); + assert!(associations(c).iter().all(|page| *page == 1)); + shared += c.len(); + alone += encode_jbig2(&samples(picture), 1200, 300).unwrap().len(); + } + assert!(shared < alone, "shared {shared} bytes, alone {alone}"); + } + + /// Two pages alike place every mark from the shared dictionary and have none of their own; a + /// page with letters of its own as well keeps them in its own dictionary, referred to after the + /// shared one (its symbols' IDs follow), and the page with nothing shared has no stream. + #[test] + fn what_a_page_has_alone_stays_its_own() { + let (font, other) = (glyphs(30, 7), stemmed(&glyphs(20, 99))); + let same = text(900, 240, &font, 3); + let mut more = text(900, 240, &font, 4); + more.extend(text(900, 120, &other, 5)); + let (globals, coded) = shared_round_trip(&[same.clone(), same.clone()]); + assert!(!globals.is_empty()); + assert!(coded.iter().all(|c| segment_types(c) == [48, 7]), "{:?}", coded.iter().map(|c| segment_types(c)).collect::>()); + let with_more = |p: &Bitmap| p.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 900], 120)).collect::(); + let (_, coded) = shared_round_trip(&[with_more(&same), more]); + assert_eq!(segment_types(&coded[1])[..3], [48, 0, 7]); + assert_eq!(segments(&coded[1])[2].3, [(0, false), (2, false)]); + let alone = text(900, 360, &other, 6); + let data: Vec> = [&same, &same, &alone].iter().map(|p| samples(p)).collect(); + let images: Vec<(&[u8], u32, u32)> = vec![(&data[0], 900, 240), (&data[1], 900, 240), (&data[2], 900, 360)]; + let (_, coded) = encode_shared(&images).unwrap(); + assert!(coded[0].is_some() && coded[1].is_some() && coded[2].is_none()); + } + + /// Images that share fewer than two marks share nothing; two are enough. + #[test] + fn images_with_nothing_in_common_share_nothing() { + let font = glyphs(30, 7); + let (a, b) = (samples(&text(600, 200, &font, 1)), samples(&text(600, 200, &stemmed(&font), 1))); + assert!(encode_shared(&[(&a, 600, 200), (&b, 600, 200)]).is_none()); + assert!(encode_shared(&[(&a, 600, 200)]).is_none(), "one image"); + assert!(encode_shared(&[]).is_none()); + // The two commonest letters of the first page, below the last line of the second. + let mut two = text(600, 200, &stemmed(&font), 1); + draw(&mut two, &font[0], 10, 180); + draw(&mut two, &font[1], 100, 180); + let two = samples(&two); + let (globals, coded) = encode_shared(&[(&a, 600, 200), (&two, 600, 200)]).unwrap(); + assert_eq!(exported(&globals), 2); + assert_eq!(decode_with(coded[1].as_ref().unwrap(), Some(&globals)), unpack(&two, 600, 200)); + } + + /// The images of a group are checked as `encode_jbig2_generic` checks one, and an image it + /// would refuse refuses the group. + #[test] + fn a_group_with_an_image_the_encoder_refuses_is_refused() { + let page = samples(&text(600, 200, &glyphs(30, 7), 1)); + assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page[..10], 600, 200)]), Err(CodecError::Corrupt { .. }))); + assert!(matches!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 0, 200)]), Err(CodecError::Unsupported(_)))); + assert!(encode_jbig2_shared(&[(&page, 600, 200), (&page, 600, 200)]).unwrap().is_some()); + } + + /// A group holds its distinct marks within what one image's are held to: past 65,536 a mark + /// first seen later isn't held, so isn't shared, while those held count every image they're + /// in. Ten marks of the first image are in two images after it, with 464 first seen there. + #[test] + fn past_the_marks_a_group_holds_later_ones_arent_shared() { + let (a, b) = (samples(&combs(0..40_000)), samples(&combs(40_000..65_536))); + let late = combs((0..10).chain(65_536..66_000).collect::>().into_iter()); + let (rows, late) = (late.len() as u32, samples(&late)); + let (columns, a_rows, b_rows) = (120 * 36, 40_000u32.div_ceil(120) * 4, 25_536u32.div_ceil(120) * 4); + let images = [(&a[..], columns, a_rows), (&b[..], columns, b_rows), (&late[..], columns, rows), (&late[..], columns, rows)]; + let (globals, coded) = encode_shared(&images).unwrap(); + assert_eq!(exported(&globals), 10); + for c in &coded[2..] { + assert_eq!(decode_with(c.as_ref().unwrap(), Some(&globals)), unpack(&late, columns, rows)); + } + } + + /// Past what one dictionary shares, it keeps the marks the most images hold: 20,000 shared, + /// the first 10,000 in all three images and the rest in two. The third image then places all + /// its marks from it, and the other two code the rest themselves (in a dictionary of their own + /// or a generic region, whichever is smaller). + #[test] + fn past_the_shared_limit_the_most_widely_held_marks_are_shared() { + let (both, first) = (combs(0..20_000), combs(0..10_000)); + let rows = both.len(); + let first: Bitmap = first.iter().cloned().chain(std::iter::repeat_n(vec![0u8; 120 * 36], rows - first.len())).collect(); + let (globals, coded) = shared_round_trip(&[both.clone(), both, first]); + assert_eq!(exported(&globals), MAX_SHARED_SYMBOLS); + assert_eq!(segment_types(&coded[2]), [48, 7]); + assert!(segment_types(&coded[0]).iter().any(|t| *t == 0 || *t == 39), "{:?}", segment_types(&coded[0])); + } + + proptest::proptest! { + /// Glyphs scattered over pages of one font: whatever is shared, every page with a stream + /// decodes with the globals to its picture. + #[test] + fn shared_groups_round_trip( + columns in 1usize..200, + rows in 1usize..100, + shapes in 2usize..8, + n in 0usize..60, + pages in 2usize..5, + seed in proptest::num::u64::ANY, + ) { + let font = glyphs(shapes, seed); + let data: Vec> = (0..pages).map(|p| samples(&scatter(columns, rows, &font, seed ^ p as u64, n))).collect(); + let (c, r) = (columns as u32, rows as u32); + let images: Vec<(&[u8], u32, u32)> = data.iter().map(|d| (&d[..], c, r)).collect(); + if let Some((globals, coded)) = encode_shared(&images) { + for (d, coded) in data.iter().zip(coded) { + if let Some(coded) = coded { + proptest::prop_assert_eq!(decode_with(&coded, Some(&globals)), unpack(d, c, r)); + } + } + } + } + + /// Glyphs anywhere, over the edges and each other, with stray pixels: every plan decodes to + /// the picture, and `encode_jbig2` is never larger than the generic coding. + #[test] + fn scattered_marks_round_trip( + columns in 1usize..240, + rows in 1usize..120, + shapes in 2usize..10, + n in 0usize..80, + seed in proptest::num::u64::ANY, + stray in proptest::collection::vec((0usize..240, 0usize..120), 0..20), + ) { + let mut picture = scatter(columns, rows, &glyphs(shapes, seed), seed, n); + for (x, y) in stray { + if let Some(cell) = picture.get_mut(y).and_then(|r| r.get_mut(x)) { + *cell ^= 1; + } + } + every_plan_round_trips(&picture); + let data = samples(&picture); + let (c, r) = (columns as u32, rows as u32); + let coded = encode_jbig2(&data, c, r).unwrap(); + proptest::prop_assert!(coded.len() <= encode_jbig2_generic(&data, c, r).unwrap().len()); + proptest::prop_assert_eq!(decode(&coded), unpack(&data, c, r)); + } + + /// Whatever it is given, it returns. + #[test] + fn any_input_returns(data in proptest::collection::vec(0u8..=255, 0..600), columns in 0u32..5000, rows in 0u32..80) { + let _ = encode_jbig2(&data, columns, rows); + let _ = encode(&data, columns, rows); + let _ = encode_shared(&[(&data, columns, rows), (&data, columns, rows.saturating_sub(1))]); + } + } +} diff --git a/crates/stream-codecs/src/lib.rs b/crates/stream-codecs/src/lib.rs new file mode 100644 index 000000000..04a935333 --- /dev/null +++ b/crates/stream-codecs/src/lib.rs @@ -0,0 +1,117 @@ +//! Byte-stream and bi-level image codecs that PDF, PostScript and TIFF share. +//! +//! - [`lzw`]: variable-width LZW, 9 to 12 bits, most significant bit first (PDF `LZWDecode`, +//! TIFF compression 5), with or without the early code-width change. +//! - [`ascii85`], [`asciihex`], [`runlength`]: the PostScript and PDF text and run-length +//! encodings (`ASCII85Decode`, `ASCIIHexDecode`, `RunLengthDecode`; PackBits). +//! - [`predictor`]: TIFF horizontal differencing and the PNG row filters. +//! - [`ccitt`]: CCITT Group 3 and Group 4 fax decoding, bounded, and Group 4 encoding. +//! - [`jbig2`]: lossless JBIG2 encoding (generic regions, symbol dictionaries, a dictionary +//! shared by a group of images). Decoding is `hayro-jbig2`'s. +//! +//! The crate knows no container: a caller hands it bytes and parameters. A PDF `/Filter` chain +//! (order, limits across stages, Flate) is `craft_pdf::filters`, built on this crate. +//! +//! Every decoder is bounded by a caller-supplied output limit and never panics on malformed +//! input. A decoder that fails returns what it decoded before the damage ([`Failure`]), so a +//! caller can choose to keep it. CCITT decoding spends from an allowance the caller shares +//! across a run of images ([`pay`]); it does not decode without one: +//! +//! ```compile_fail +//! let _ = craft_stream_codecs::decode_ccitt(&[], &craft_stream_codecs::CcittParams::default(), 1); +//! ``` +//! +//! ``` +//! let mut work = 1 << 20; +//! let p = craft_stream_codecs::CcittParams { columns: 8, ..Default::default() }; +//! assert!(craft_stream_codecs::decode_ccitt_within(&[], &p, 1, &mut work).is_ok()); +//! ``` +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +pub mod ascii85; +pub mod asciihex; +pub mod ccitt; +pub mod jbig2; +pub mod lzw; +mod mq; +pub mod predictor; +pub mod runlength; + +pub use ccitt::{CcittParams, MAX_CCITT_COLUMNS, MAX_G4_COLUMNS, decode_ccitt_within, encode_g4}; +pub use jbig2::{MAX_JBIG2_COLUMNS, MAX_JBIG2_PIXELS, SharedJbig2, encode_jbig2, encode_jbig2_generic, encode_jbig2_shared}; + +/// Parameters of the predictors and of LZW, as a PDF `/DecodeParms` dictionary gives them +/// (ISO 32000-2 Tables 8–9); TIFF's `Predictor`, `SamplesPerPixel`, `BitsPerSample` and +/// `ImageWidth` map onto the same four. +#[derive(Clone, Debug, PartialEq)] +pub struct Params { + /// 1 = none, 2 = TIFF predictor 2, 10–15 = PNG predictors. + pub predictor: i64, + pub colors: i64, + pub bits_per_component: i64, + pub columns: i64, + /// LZW only: 1 (default) switches code width one code early, 0 does not. + pub early_change: i64, +} + +impl Default for Params { + fn default() -> Self { + Params { predictor: 1, colors: 1, bits_per_component: 8, columns: 1, early_change: 1 } + } +} + +/// Errors from decoding or encoding. +#[derive(Debug, thiserror::Error)] +pub enum CodecError { + /// `filter` names the codec, as PDF names its filter (`LZWDecode`, `CCITTFaxDecode`, …). + #[error("{filter}: corrupt data: {detail}")] + Corrupt { filter: &'static str, detail: String }, + #[error("decoded output exceeds the limit of {0} bytes")] + LimitExceeded(usize), + #[error("unsupported filter: {0}")] + Unsupported(String), + /// A run of images has nothing left to spend ([`pay`]). + #[error("more image decoding than one operation may do")] + Spent, +} + +/// A decoder failure carrying whatever was decoded before the error. +pub struct Failure { + pub error: CodecError, + pub partial: Vec, + /// Bytes decoded on the way that `partial` doesn't hold: they took as long. + pub discarded: usize, +} + +impl Failure { + pub fn corrupt(filter: &'static str, detail: impl Into, partial: Vec) -> Self { + Failure { error: CodecError::Corrupt { filter, detail: detail.into() }, partial, discarded: 0 } + } + + pub fn limit(max: usize) -> Self { + Failure { error: CodecError::LimitExceeded(max), partial: Vec::new(), discarded: 0 } + } +} + +/// What a byte-stream decoder returns: the decoded bytes, or the failure with its partial output. +pub type Step = Result, Failure>; + +/// Takes `steps` from `work`, what a run of images may still cost. What it can't pay for spends +/// all of it. +pub fn pay(work: &mut u64, steps: u64) -> Result<(), CodecError> { + match work.checked_sub(steps) { + Some(left) => { + *work = left; + Ok(()) + } + None => { + *work = 0; + Err(CodecError::Spent) + } + } +} + +/// PDF and PostScript white-space characters (ISO 32000-2 Table 1). +pub fn is_pdf_whitespace(b: u8) -> bool { + matches!(b, 0 | 9 | 10 | 12 | 13 | 32) +} diff --git a/crates/stream-codecs/src/lzw.rs b/crates/stream-codecs/src/lzw.rs new file mode 100644 index 000000000..e16d78288 --- /dev/null +++ b/crates/stream-codecs/src/lzw.rs @@ -0,0 +1,314 @@ +//! LZWDecode (ISO 32000-2 §7.4.4.2): variable-width codes of 9–12 bits, MSB first, +//! 256 = clear-table, 257 = EOD, first free code 258. With `EarlyChange` 1 (the +//! default) the code width grows one code early. +//! +//! Tolerated: data ending without EOD, a missing initial clear-table code, a table that +//! fills up without a clear-table code (entries are then no longer added). + +use std::collections::HashMap; + +use crate::{Failure, Step}; + +const NAME: &str = "LZWDecode"; +const CLEAR: u16 = 256; +const EOD: u16 = 257; +const FIRST: usize = 258; +const TABLE: usize = 4096; +/// The encoder emits clear-table when its next free code reaches this value. +const ENCODER_RESET_AT: usize = TABLE; + +/// The code width a decoder uses when its next free code is `next`. +fn width(next: usize, early: bool) -> u32 { + match next + usize::from(early) { + 0..512 => 9, + 512..1024 => 10, + 1024..2048 => 11, + _ => 12, + } +} + +struct BitReader<'a> { + data: &'a [u8], + pos: usize, + acc: u32, + bits: u32, +} + +impl BitReader<'_> { + fn read(&mut self, n: u32) -> Option { + while self.bits < n { + let &b = self.data.get(self.pos)?; + self.pos += 1; + self.acc = (self.acc << 8) | u32::from(b); + self.bits += 8; + } + self.bits -= n; + let v = (self.acc >> self.bits) & ((1 << n) - 1); + self.acc &= (1 << self.bits) - 1; + Some(v as u16) + } +} + +pub fn decode(data: &[u8], early: bool, max: usize) -> Step { + // Table entry = (prefix code, last byte, first byte, length). + let mut prefix = vec![0u16; TABLE]; + let mut last = vec![0u8; TABLE]; + let mut first = vec![0u8; TABLE]; + let mut len = vec![0u16; TABLE]; + for i in 0..256 { + last[i] = i as u8; + first[i] = i as u8; + len[i] = 1; + } + let mut next = FIRST; + let mut w = 9; + let mut prev: Option = None; + let mut r = BitReader { data, pos: 0, acc: 0, bits: 0 }; + let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); + + while let Some(code) = r.read(w) { + match code { + CLEAR => { + next = FIRST; + w = 9; + prev = None; + continue; + } + EOD => break, + _ => {} + } + let code = usize::from(code); + if let Some(p) = prev { + let head = if code < next { + first[code] + } else if code == next && next < TABLE { + first[p] + } else { + return Err(Failure::corrupt(NAME, format!("code {code} not in table (next {next})"), out)); + }; + if next < TABLE { + prefix[next] = p as u16; + last[next] = head; + first[next] = first[p]; + len[next] = len[p] + 1; + next += 1; + } + } else if code >= 256 { + return Err(Failure::corrupt(NAME, format!("code {code} before any table entry"), out)); + } + // Emit the string for `code`, walking the prefix chain backwards. + let n = usize::from(len[code]); + if out.len() + n > max { + return Err(Failure::limit(max)); + } + let start = out.len(); + out.resize(start + n, 0); + let mut c = code; + for slot in out[start..].iter_mut().rev() { + *slot = last[c]; + c = usize::from(prefix[c]); + } + prev = Some(code); + w = width(next, early); + } + Ok(out) +} + +struct BitWriter { + out: Vec, + acc: u32, + bits: u32, +} + +impl BitWriter { + fn write(&mut self, code: u16, n: u32) { + self.acc = (self.acc << n) | u32::from(code); + self.bits += n; + while self.bits >= 8 { + self.bits -= 8; + self.out.push((self.acc >> self.bits) as u8); + } + self.acc &= (1 << self.bits) - 1; + } + + fn finish(mut self) -> Vec { + if self.bits > 0 { + self.out.push((self.acc << (8 - self.bits)) as u8); + } + self.out + } +} + +/// Greedy LZW with a leading clear-table code and a trailing EOD. +pub fn encode(data: &[u8], early: bool) -> Vec { + let mut wr = BitWriter { out: Vec::with_capacity(data.len() / 2 + 8), acc: 0, bits: 0 }; + let mut dict: HashMap<(u16, u8), u16> = HashMap::new(); + // `next` is the encoder's next free code; `dnext` mirrors the decoder's, which lags + // one entry behind (the decoder adds an entry only once it sees the following code). + let mut next = FIRST; + let mut dnext = FIRST; + let mut codes_since_clear = 0usize; + wr.write(CLEAR, 9); + let emit = |wr: &mut BitWriter, code: u16, dnext: &mut usize, since: &mut usize| { + wr.write(code, width(*dnext, early)); + if *since > 0 && *dnext < TABLE { + *dnext += 1; + } + *since += 1; + }; + let Some((&b0, rest)) = data.split_first() else { + wr.write(EOD, 9); + return wr.finish(); + }; + let mut cur = u16::from(b0); + for &b in rest { + if let Some(&c) = dict.get(&(cur, b)) { + cur = c; + continue; + } + emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); + dict.insert((cur, b), next as u16); + next += 1; + if next == ENCODER_RESET_AT { + wr.write(CLEAR, width(dnext, early)); + dict.clear(); + next = FIRST; + dnext = FIRST; + codes_since_clear = 0; + } + cur = u16::from(b); + } + emit(&mut wr, cur, &mut dnext, &mut codes_since_clear); + wr.write(EOD, width(dnext, early)); + wr.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8], early: bool) -> Result, CodecError> { + decode(d, early, 1 << 26).map_err(|f| f.error) + } + + /// ISO 32000-2 §7.4.4.2 example: codes 256 45 258 258 65 259 66 257. + const SPEC_IN: [u8; 10] = [45, 45, 45, 45, 45, 65, 45, 45, 45, 66]; + const SPEC_OUT: [u8; 9] = [0x80, 0x0B, 0x60, 0x50, 0x22, 0x0C, 0x0C, 0x85, 0x01]; + + #[test] + fn spec_example() { + assert_eq!(dec(&SPEC_OUT, true).unwrap(), SPEC_IN); + assert_eq!(encode(&SPEC_IN, true), SPEC_OUT); + // Below 511 codes the early-change setting makes no difference. + assert_eq!(dec(&SPEC_OUT, false).unwrap(), SPEC_IN); + } + + #[test] + fn missing_eod_and_clear() { + // Drop the EOD (last 9 bits span the final 2 bytes; keep bits for 66 only). + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + for c in [45u16, 258, 258, 65, 259, 66] { + w.write(c, 9); + } + assert_eq!(dec(&w.finish(), true).unwrap(), SPEC_IN); + } + + #[test] + fn invalid_codes() { + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + for c in [256u16, 65, 300] { + w.write(c, 9); + } + let f = decode(&w.finish(), true, 1000).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"A"); + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + w.write(258, 9); + assert!(matches!(dec(&w.finish(), true), Err(CodecError::Corrupt { .. }))); + } + + #[test] + fn widths() { + assert_eq!((width(510, true), width(511, true), width(511, false), width(512, false)), (9, 10, 9, 10)); + assert_eq!((width(1022, true), width(1023, true), width(2047, true), width(4095, false)), (10, 11, 12, 12)); + } + + fn noisy(n: usize, seed: u32) -> Vec { + let mut s = seed; + (0..n) + .map(|_| { + s = s.wrapping_mul(1_103_515_245).wrapping_add(12_345); + ((s >> 16) % 7) as u8 * 31 + }) + .collect() + } + + #[test] + fn all_widths_and_clears_round_trip() { + for early in [false, true] { + for data in [noisy(200_000, 1), (0..=255u8).cycle().take(70_000).collect(), vec![9u8; 100_000]] { + let e = encode(&data, early); + assert_eq!(dec(&e, early).unwrap(), data); + } + } + } + + #[test] + fn cross_check_with_weezl() { + use weezl::BitOrder::Msb; + for data in [noisy(150_000, 7), SPEC_IN.to_vec(), vec![]] { + // early change 1 is the TIFF size switch; 0 is the plain variant. + let tiff = weezl::encode::Encoder::with_tiff_size_switch(Msb, 8).encode(&data).unwrap(); + assert_eq!(dec(&tiff, true).unwrap(), data); + let plain = weezl::encode::Encoder::new(Msb, 8).encode(&data).unwrap(); + assert_eq!(dec(&plain, false).unwrap(), data); + let ours = encode(&data, true); + assert_eq!(weezl::decode::Decoder::with_tiff_size_switch(Msb, 8).decode(&ours).unwrap(), data); + let ours = encode(&data, false); + assert_eq!(weezl::decode::Decoder::new(Msb, 8).decode(&ours).unwrap(), data); + } + } + + #[test] + fn full_table_without_clear() { + // Hand-roll a stream that fills the table and keeps going at 12 bits. + let data = noisy(40_000, 3); + let mut w = BitWriter { out: vec![], acc: 0, bits: 0 }; + let mut dict: HashMap, u16> = (0..=255u8).map(|b| (vec![b], u16::from(b))).collect(); + let (mut next, mut dnext, mut since) = (FIRST, FIRST, 0usize); + let mut cur: Vec = vec![]; + let mut put = |w: &mut BitWriter, code: u16, dnext: &mut usize| { + w.write(code, width(*dnext, true)); + if since > 0 && *dnext < TABLE { + *dnext += 1; + } + since += 1; + }; + w.write(CLEAR, 9); + for &b in &data { + let mut ext = cur.clone(); + ext.push(b); + if dict.contains_key(&ext) { + cur = ext; + continue; + } + put(&mut w, dict[&cur], &mut dnext); + if next < TABLE { + dict.insert(ext, next as u16); + next += 1; + } + cur = vec![b]; + } + put(&mut w, dict[&cur], &mut dnext); + w.write(EOD, width(dnext, true)); + assert_eq!(dec(&w.finish(), true).unwrap(), data); + } + + #[test] + fn limit() { + let e = encode(&vec![0u8; 100_000], true); + assert!(matches!(decode(&e, true, 50_000).map_err(|f| f.error), Err(CodecError::LimitExceeded(50_000)))); + assert_eq!(decode(&e, true, 100_000).ok().unwrap().len(), 100_000); + } +} diff --git a/crates/stream-codecs/src/mq.rs b/crates/stream-codecs/src/mq.rs new file mode 100644 index 000000000..04dda2bd9 --- /dev/null +++ b/crates/stream-codecs/src/mq.rs @@ -0,0 +1,209 @@ +//! The MQ arithmetic encoder of JBIG2 (ITU-T T.88 | ISO/IEC 14492, Annex E), written from the +//! procedures in its Figures E.2–E.12 and the probability estimates of Table E.1. + +/// Table E.1: for each index, the LPS probability estimate Qe, the next index after an MPS and +/// after an LPS renormalisation, and whether an LPS switches the MPS sense. +const STATES: [(u32, u8, u8, bool); 47] = [ + (0x5601, 1, 1, true), + (0x3401, 2, 6, false), + (0x1801, 3, 9, false), + (0x0AC1, 4, 12, false), + (0x0521, 5, 29, false), + (0x0221, 38, 33, false), + (0x5601, 7, 6, true), + (0x5401, 8, 14, false), + (0x4801, 9, 14, false), + (0x3801, 10, 14, false), + (0x3001, 11, 17, false), + (0x2401, 12, 18, false), + (0x1C01, 13, 20, false), + (0x1601, 29, 21, false), + (0x5601, 15, 14, true), + (0x5401, 16, 14, false), + (0x5101, 17, 15, false), + (0x4801, 18, 16, false), + (0x3801, 19, 17, false), + (0x3401, 20, 18, false), + (0x3001, 21, 19, false), + (0x2801, 22, 19, false), + (0x2401, 23, 20, false), + (0x2201, 24, 21, false), + (0x1C01, 25, 22, false), + (0x1801, 26, 23, false), + (0x1601, 27, 24, false), + (0x1401, 28, 25, false), + (0x1201, 29, 26, false), + (0x1101, 30, 27, false), + (0x0AC1, 31, 28, false), + (0x09C1, 32, 29, false), + (0x08A1, 33, 30, false), + (0x0521, 34, 31, false), + (0x0441, 35, 32, false), + (0x02A1, 36, 33, false), + (0x0221, 37, 34, false), + (0x0141, 38, 35, false), + (0x0111, 39, 36, false), + (0x0085, 40, 37, false), + (0x0049, 41, 38, false), + (0x0025, 42, 39, false), + (0x0015, 43, 40, false), + (0x0009, 44, 41, false), + (0x0005, 45, 42, false), + (0x0001, 45, 43, false), + (0x5601, 46, 46, false), +]; + +#[cfg(test)] +thread_local! { + /// Decisions coded on this thread. + pub(crate) static DECISIONS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// A context's adaptive state: the index into [`STATES`] (low 7 bits) and the MPS sense (bit 7). +/// 0, the initial value, is index 0 with MPS 0 (E.2.8). +pub(crate) type Context = u8; + +/// The encoder's registers (E.2.1) and the bytes written so far. +pub(crate) struct MqEncoder { + a: u32, + c: u32, + ct: u32, + /// The output, after the byte before the first one (B at INITENC), which is 0x00 and is + /// dropped by [`MqEncoder::finish`]: a carry can't reach it before the first byte is out. + out: Vec, +} + +impl MqEncoder { + /// INITENC (Figure E.10). The byte before the first is 0x00, not 0xFF, so CT starts at 12. + pub(crate) fn new() -> Self { + MqEncoder { a: 0x8000, c: 0, ct: 12, out: vec![0] } + } + + /// The byte B, the last one written (or the one before the first). + fn b(&self) -> u8 { + self.out.last().copied().unwrap_or(0) + } + + /// ENCODE (Figures E.3–E.7): code decision `d` in context `cx`, updating its state. + pub(crate) fn encode(&mut self, cx: &mut Context, d: bool) { + #[cfg(test)] + DECISIONS.with(|n| n.set(n.get() + 1)); + let (index, mps) = (usize::from(*cx & 0x7F), *cx >> 7 == 1); + let Some(&(qe, nmps, nlps, switch)) = STATES.get(index) else { return }; + self.a -= qe; + if d == mps { + // CODEMPS. + if self.a & 0x8000 != 0 { + self.c += qe; + return; + } + if self.a < qe { + self.a = qe; + } else { + self.c += qe; + } + *cx = nmps | (*cx & 0x80); + } else { + // CODELPS. + if self.a < qe { + self.c += qe; + } else { + self.a = qe; + } + let mps = mps != switch; + *cx = nlps | (u8::from(mps) << 7); + } + self.renorm(); + } + + /// RENORME (Figure E.8). + fn renorm(&mut self) { + loop { + self.a <<= 1; + self.c <<= 1; + self.ct -= 1; + if self.ct == 0 { + self.byte_out(); + } + if self.a & 0x8000 != 0 { + break; + } + } + } + + /// BYTEOUT (Figure E.9), with bit stuffing after 0xFF and carry into the byte before. + fn byte_out(&mut self) { + if self.b() == 0xFF { + self.stuffed(); + } else if self.c < 0x800_0000 { + self.plain(); + } else { + let b = self.b().wrapping_add(1); + if let Some(last) = self.out.last_mut() { + *last = b; + } + if b == 0xFF { + self.c &= 0x7FF_FFFF; + self.stuffed(); + } else { + self.plain(); + } + } + } + + fn plain(&mut self) { + self.out.push(((self.c >> 19) & 0xFF) as u8); + self.c &= 0x7_FFFF; + self.ct = 8; + } + + fn stuffed(&mut self) { + self.out.push(((self.c >> 20) & 0xFF) as u8); + self.c &= 0xF_FFFF; + self.ct = 7; + } + + /// FLUSH (Figures E.11 and E.12): the final bytes, then the 0xFF 0xAC marker. + pub(crate) fn finish(mut self) -> Vec { + let top = self.c + self.a; + self.c |= 0xFFFF; + if self.c >= top { + self.c -= 0x8000; + } + self.c <<= self.ct; + self.byte_out(); + self.c <<= self.ct; + self.byte_out(); + if self.b() != 0xFF { + self.out.push(0xFF); + } + self.out.push(0xAC); + self.out.split_off(1) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Annex H.2's test sequence: 256 decisions in one context give these 30 bytes. + #[test] + fn the_annex_h_test_sequence_encodes_to_its_bytes() { + let decisions = [ + 0x00, 0x02, 0x00, 0x51, 0x00, 0x00, 0x00, 0xC0, 0x03, 0x52, 0x87, 0x2A, 0xAA, 0xAA, 0xAA, 0xAA, 0x82, 0xC0, 0x20, 0x00, 0xFC, 0xD7, 0x9E, + 0xF6, 0xBF, 0x7F, 0xED, 0x90, 0x4F, 0x46, 0xA3, 0xBF, + ]; + let mut mq = MqEncoder::new(); + let mut cx = 0; + for byte in decisions { + for bit in (0..8).rev() { + mq.encode(&mut cx, byte >> bit & 1 == 1); + } + } + let expected = [ + 0x84, 0xC7, 0x3B, 0xFC, 0xE1, 0xA1, 0x43, 0x04, 0x02, 0x20, 0x00, 0x00, 0x41, 0x0D, 0xBB, 0x86, 0xF4, 0x31, 0x7F, 0xFF, 0x88, 0xFF, 0x37, + 0x47, 0x1A, 0xDB, 0x6A, 0xDF, 0xFF, 0xAC, + ]; + assert_eq!(mq.finish(), expected); + } +} diff --git a/crates/stream-codecs/src/predictor.rs b/crates/stream-codecs/src/predictor.rs new file mode 100644 index 000000000..8e76e3fc2 --- /dev/null +++ b/crates/stream-codecs/src/predictor.rs @@ -0,0 +1,356 @@ +//! Predictor functions for FlateDecode and LZWDecode (ISO 32000-2 §7.4.4.4). +//! +//! - Predictor 2: TIFF 6.0 horizontal differencing (Section 14), per component, for +//! 1, 2, 4, 8 and 16 bits per component. +//! - Predictors 10–15: PNG filters (RFC 2083 §6): each row carries its own filter-type +//! byte (0 None, 1 Sub, 2 Up, 3 Average, 4 Paeth), whatever the predictor value is. +//! +//! A final partial row is decoded as far as its data goes. +//! +//! The samples of an image whose rows carry no filter-type byte, although a PNG predictor is +//! declared, are taken as they are and reported as damage ([`decode`]). + +use crate::{CodecError, Failure, Params, Step}; + +struct Layout { + bpc: usize, + colors: usize, + /// Bytes per complete pixel, at least 1 (the PNG `bpp`). + bpp: usize, + /// Bytes per row (without the PNG tag byte). + row: usize, + /// Samples per row (colors × columns). + samples: usize, +} + +fn layout(p: &Params) -> Result { + let bpc = match p.bits_per_component { + b @ (1 | 2 | 4 | 8 | 16) => b as usize, + b => return Err(format!("invalid BitsPerComponent {b}")), + }; + if !(1..=255).contains(&p.colors) { + return Err(format!("invalid Colors {}", p.colors)); + } + if p.columns < 1 { + return Err(format!("invalid Columns {}", p.columns)); + } + let colors = p.colors as usize; + let samples = usize::try_from(p.columns).ok().and_then(|c| c.checked_mul(colors)).ok_or_else(|| format!("Columns {} too large", p.columns))?; + let bits = samples.checked_mul(bpc).ok_or_else(|| format!("Columns {} too large", p.columns))?; + Ok(Layout { bpc, colors, bpp: (colors * bpc).div_ceil(8), row: bits.div_ceil(8), samples }) +} + +/// Undoes the predictor named by `p.predictor` (1 = identity). `rows` is the height of the image +/// `data` is the samples of, if it is: with a PNG predictor, data of exactly `rows` rows without +/// their filter-type bytes ([`untagged`]) is given back as it is, as a [`Failure`] whose partial +/// output is all of it, so that tolerant decoding takes the samples and reports the repair, and +/// strict decoding refuses them. +pub fn decode(p: &Params, data: Vec, name: &'static str, rows: Option) -> Step { + match p.predictor { + 1 => Ok(data), + 2 | 10..=15 => { + let l = layout(p).map_err(|e| Failure::corrupt(name, e, Vec::new()))?; + if p.predictor == 2 { + return Ok(tiff_decode(&l, data)); + } + if let Some(rows) = rows.filter(|&rows| untagged(&l, data.len(), rows)) { + let detail = + format!("PNG predictor {} declared, but its {rows} rows carry no filter-type bytes: read as unpredicted samples", p.predictor); + return Err(Failure::corrupt(name, detail, data)); + } + png_decode(&l, &data, name) + } + other => Err(Failure::corrupt(name, format!("invalid Predictor {other}"), Vec::new())), + } +} + +/// Whether `len` bytes are exactly `rows` rows of `l` without the filter-type byte that starts each +/// PNG-predicted row (ISO 32000-2 §7.4.4.4), where predicted data would be `rows` × (row + 1): a +/// writer that declared the predictor and didn't apply it. Read with the predictor, every row's +/// first sample was taken for a filter type and the image came out a row short and shifted. +/// PDFium, MuPDF and Poppler read such data as the samples it is. +fn untagged(l: &Layout, len: usize, rows: usize) -> bool { + rows > 0 && l.row > 0 && rows.checked_mul(l.row) == Some(len) +} + +/// Applies the predictor named by `p.predictor` (1 = identity; 15 = per-row optimum). +pub fn encode(p: &Params, data: &[u8]) -> Result, CodecError> { + match p.predictor { + 1 => Ok(data.to_vec()), + 2 | 10..=15 => { + let l = layout(p).map_err(CodecError::Unsupported)?; + Ok(if p.predictor == 2 { tiff_encode(&l, data) } else { png_encode(&l, data, p.predictor) }) + } + other => Err(CodecError::Unsupported(format!("Predictor {other}"))), + } +} + +// ---- TIFF predictor 2 ---- + +fn get(row: &[u8], i: usize, bpc: usize) -> u16 { + match bpc { + 16 => u16::from_be_bytes([row[2 * i], row[2 * i + 1]]), + 8 => u16::from(row[i]), + _ => { + let bit = i * bpc; + let shift = 8 - bpc - bit % 8; + u16::from(row[bit / 8] >> shift) & ((1 << bpc) - 1) + } + } +} + +fn set(row: &mut [u8], i: usize, bpc: usize, v: u16) { + match bpc { + 16 => row[2 * i..2 * i + 2].copy_from_slice(&v.to_be_bytes()), + 8 => row[i] = v as u8, + _ => { + let bit = i * bpc; + let shift = 8 - bpc - bit % 8; + let mask = (((1u16 << bpc) - 1) << shift) as u8; + row[bit / 8] = (row[bit / 8] & !mask) | (((v << shift) as u8) & mask); + } + } +} + +/// Complete samples available in a (possibly partial) row. +fn samples_in(l: &Layout, row_len: usize) -> usize { + (row_len * 8 / l.bpc).min(l.samples) +} + +fn tiff_decode(l: &Layout, mut data: Vec) -> Vec { + let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; + for row in data.chunks_mut(l.row) { + for i in l.colors..samples_in(l, row.len()) { + let v = get(row, i, l.bpc).wrapping_add(get(row, i - l.colors, l.bpc)) & mask; + set(row, i, l.bpc, v); + } + } + data +} + +fn tiff_encode(l: &Layout, data: &[u8]) -> Vec { + let mask = if l.bpc == 16 { u16::MAX } else { (1u16 << l.bpc) - 1 }; + let mut out = data.to_vec(); + for row in out.chunks_mut(l.row) { + // Descending, so the left neighbour still holds its original value. + for i in (l.colors..samples_in(l, row.len())).rev() { + let v = get(row, i, l.bpc).wrapping_sub(get(row, i - l.colors, l.bpc)) & mask; + set(row, i, l.bpc, v); + } + } + out +} + +// ---- PNG predictors ---- + +fn paeth(a: u8, b: u8, c: u8) -> u8 { + let p = i16::from(a) + i16::from(b) - i16::from(c); + let (pa, pb, pc) = ((p - i16::from(a)).abs(), (p - i16::from(b)).abs(), (p - i16::from(c)).abs()); + if pa <= pb && pa <= pc { + a + } else if pb <= pc { + b + } else { + c + } +} + +/// The PNG predictor value for byte `i` given the reconstructed current row `cur` +/// (bytes before `i`) and the prior row `up`. +fn predict(tag: u8, cur: &[u8], up: &[u8], i: usize, bpp: usize) -> u8 { + let a = if i >= bpp { cur[i - bpp] } else { 0 }; + let b = up[i]; + let c = if i >= bpp { up[i - bpp] } else { 0 }; + match tag { + 1 => a, + 2 => b, + 3 => ((u16::from(a) + u16::from(b)) / 2) as u8, + 4 => paeth(a, b, c), + _ => 0, + } +} + +fn png_decode(l: &Layout, data: &[u8], name: &'static str) -> Step { + let mut out = Vec::with_capacity(data.len()); + // The prior row: never longer than the input, even for absurd Columns values. + let mut up = vec![0u8; l.row.min(data.len())]; + for chunk in data.chunks(l.row.saturating_add(1)) { + let (tag, src) = (chunk[0], &chunk[1..]); + if tag > 4 { + return Err(Failure::corrupt(name, format!("invalid PNG filter type {tag}"), out)); + } + let start = out.len(); + for (i, &x) in src.iter().enumerate() { + let pred = predict(tag, &out[start..], &up, i, l.bpp); + out.push(x.wrapping_add(pred)); + } + up[..src.len()].copy_from_slice(&out[start..]); + } + Ok(out) +} + +fn png_filter(tag: u8, cur: &[u8], up: &[u8], bpp: usize, dst: &mut Vec) { + dst.clear(); + dst.push(tag); + dst.extend(cur.iter().enumerate().map(|(i, &x)| x.wrapping_sub(predict(tag, cur, up, i, bpp)))); +} + +/// Sum of residuals as signed bytes: the usual heuristic for choosing a PNG filter. +fn cost(filtered: &[u8]) -> u64 { + filtered[1..].iter().map(|&b| u64::from((b as i8).unsigned_abs())).sum() +} + +fn png_encode(l: &Layout, data: &[u8], predictor: i64) -> Vec { + let rows = data.len().div_ceil(l.row.max(1)); + let mut out = Vec::with_capacity(data.len() + rows); + let zeros = vec![0u8; l.row.min(data.len())]; + let (mut best, mut trial) = (Vec::new(), Vec::new()); + let mut up: &[u8] = &zeros; + for cur in data.chunks(l.row) { + let up_row = &up[..cur.len()]; + if predictor == 15 { + png_filter(0, cur, up_row, l.bpp, &mut best); + let mut best_cost = cost(&best); + for tag in 1..=4 { + png_filter(tag, cur, up_row, l.bpp, &mut trial); + let c = cost(&trial); + if c < best_cost { + best_cost = c; + std::mem::swap(&mut best, &mut trial); + } + } + } else { + png_filter((predictor - 10) as u8, cur, up_row, l.bpp, &mut best); + } + out.extend_from_slice(&best); + up = cur; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn p(predictor: i64, colors: i64, bpc: i64, columns: i64) -> Params { + Params { predictor, colors, bits_per_component: bpc, columns, early_change: 1 } + } + + fn dec(params: &Params, data: &[u8]) -> Result, CodecError> { + decode(params, data.to_vec(), "FlateDecode", None).map_err(|f| f.error) + } + + #[test] + fn png_hand_vectors() { + // 3 columns, 1 color, 8 bpc. Rows: None, Sub, Up, Average, Paeth. + let enc = [ + 0, 10, 20, 30, // None -> 10 20 30 + 1, 5, 1, 1, // Sub -> 5 6 7 + 2, 1, 1, 1, // Up -> 6 7 8 + 3, 10, 10, 10, // Avg -> 10+3=13, 10+(13+7)/2=20, 10+(20+8)/2=24 + 4, 1, 1, + 1, // Paeth: a=0,b=13,c=0 -> p=13 -> b: 14; a=14,b=20,c=13: p=21 -> pa=7,pb=1,pc=8 -> b: 21; + // a=21,b=24,c=20: p=25 -> pa=4,pb=1,pc=5 -> b: 25 + ]; + let out = dec(&p(15, 1, 8, 3), &enc).unwrap(); + assert_eq!(out, [10, 20, 30, 5, 6, 7, 6, 7, 8, 13, 20, 24, 14, 21, 25]); + } + + #[test] + fn paeth_tie_breaks() { + // RFC 2083 order: a, then b, then c. + assert_eq!(paeth(1, 1, 1), 1); + assert_eq!(paeth(10, 20, 15), 15); // p=15: pa=5 pb=5 pc=0 -> c + assert_eq!(paeth(20, 10, 10), 20); // p=20: pa=0 -> a + assert_eq!(paeth(5, 9, 7), 7); // p=7: pa=2 pb=2 pc=0 -> c + } + + #[test] + fn png_multibyte_pixels() { + // colors=3 bpc=16 -> bpp=6; 1 column; Sub on the first pixel uses a=0 throughout. + let enc = [1, 1, 2, 3, 4, 5, 6, 2, 1, 1, 1, 1, 1, 1]; + let out = dec(&p(11, 3, 16, 1), &enc).unwrap(); + assert_eq!(out, [1, 2, 3, 4, 5, 6, 2, 3, 4, 5, 6, 7]); + // colors=3 bpc=8, 2 columns: Sub looks 3 bytes back. + let enc = [1, 1, 2, 3, 1, 1, 1]; + assert_eq!(dec(&p(11, 3, 8, 2), &enc).unwrap(), [1, 2, 3, 2, 3, 4]); + } + + #[test] + fn png_partial_final_row() { + let enc = [0, 1, 2, 3, 4, 2, 1, 1]; + assert_eq!(dec(&p(12, 1, 8, 4), &enc).unwrap(), [1, 2, 3, 4, 2, 3]); + // A lone tag byte at the end yields nothing. + assert_eq!(dec(&p(12, 1, 8, 4), &[0, 1, 2, 3, 4, 2]).unwrap(), [1, 2, 3, 4]); + } + + #[test] + fn png_invalid_tag() { + let r = decode(&p(10, 1, 8, 2), vec![0, 1, 2, 5, 1, 1], "FlateDecode", None); + let f = r.err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, [1, 2]); + } + + #[test] + fn tiff_8bit() { + // 2 colors, 3 columns: differences per component. + let enc = [10, 20, 1, 2, 3, 4, /* row 2 */ 5, 5, 250, 10, 1, 1]; + assert_eq!(dec(&p(2, 2, 8, 3), &enc).unwrap(), [10, 20, 11, 22, 14, 26, 5, 5, 255, 15, 0, 16]); + } + + #[test] + fn tiff_16bit() { + let enc = [0x01, 0x00, 0xff, 0xff, 0x00, 0x02]; + assert_eq!(dec(&p(2, 1, 16, 3), &enc).unwrap(), [0x01, 0x00, 0x00, 0xff, 0x01, 0x01]); + } + + #[test] + fn tiff_sub_byte() { + // 1 bpc, 1 color, 10 columns: bits 1 0 0 1 1 0 0 0 | 1 1 (pad 000000) + // Cumulative XOR: 1 1 1 0 1 1 1 1 | 0 1 + let enc = [0b1001_1000, 0b1100_0000]; + assert_eq!(dec(&p(2, 1, 1, 10), &enc).unwrap(), [0b1110_1111, 0b0100_0000]); + // 4 bpc, 1 color, 3 columns: samples 5, 15, 3 -> 5, 4, 7 (mod 16). + assert_eq!(dec(&p(2, 1, 4, 3), &[0x5f, 0x30]).unwrap(), [0x54, 0x70]); + // 2 bpc, 2 colors, 2 columns: samples 1 2 3 3 -> 1 2 0 1. + assert_eq!(dec(&p(2, 2, 2, 2), &[0b0110_1111]).unwrap(), [0b0110_0001]); + // Padding bits are left alone. + assert_eq!(dec(&p(2, 1, 4, 1), &[0x5f]).unwrap(), [0x5f]); + } + + #[test] + fn tiff_partial_rows() { + // 16-bit with an odd trailing byte: the lone byte is left as is. + assert_eq!(dec(&p(2, 1, 16, 4), &[0, 1, 0, 1, 9]).unwrap(), [0, 1, 0, 2, 9]); + assert_eq!(dec(&p(2, 1, 8, 4), &[1, 1, 1, 1, 2, 2]).unwrap(), [1, 2, 3, 4, 2, 4]); + } + + #[test] + fn invalid_params() { + for bad in [p(2, 1, 3, 1), p(12, 0, 8, 1), p(12, 1, 8, 0), p(3, 1, 8, 1), p(12, 1, 8, i64::MAX), p(2, 300, 8, 1)] { + assert!(matches!(dec(&bad, &[0, 1, 2]), Err(CodecError::Corrupt { .. })), "{bad:?}"); + assert!(encode(&bad, &[1, 2]).is_err()); + } + // Huge but representable Columns does not allocate a huge row. + assert_eq!(dec(&p(12, 1, 8, 1 << 40), &[2, 7, 8]).unwrap(), [7, 8]); + } + + #[test] + fn encode_choices() { + let data = [1u8, 2, 3, 4, 5, 6]; + let up = encode(&p(12, 1, 8, 3), &data).unwrap(); + assert_eq!(up, [2, 1, 2, 3, 2, 3, 3, 3]); + let paeth = encode(&p(14, 1, 8, 3), &data).unwrap(); + assert_eq!(paeth[0], 4); + assert_eq!(paeth[4], 4); + // Per-row optimum picks Sub for a ramp. + let ramp: Vec = (0..64).collect(); + let opt = encode(&p(15, 1, 8, 64), &ramp).unwrap(); + assert_eq!(opt[0], 1); + for pred in [10, 11, 12, 13, 14, 15] { + let e = encode(&p(pred, 1, 8, 3), &data).unwrap(); + assert_eq!(dec(&p(pred, 1, 8, 3), &e).unwrap(), data); + } + } +} diff --git a/crates/stream-codecs/src/runlength.rs b/crates/stream-codecs/src/runlength.rs new file mode 100644 index 000000000..a6df9e19d --- /dev/null +++ b/crates/stream-codecs/src/runlength.rs @@ -0,0 +1,117 @@ +//! RunLengthDecode (ISO 32000-2 §7.4.5): a length byte L of 0–127 copies the next +//! L + 1 bytes, 129–255 repeats the next byte 257 − L times, 128 is EOD. +//! A missing EOD is tolerated; a truncated run is corrupt. + +use crate::{Failure, Step}; + +const NAME: &str = "RunLengthDecode"; + +pub fn decode(data: &[u8], max: usize) -> Step { + let mut out = Vec::with_capacity(data.len().saturating_mul(2).min(max)); + let mut i = 0; + while let Some(&l) = data.get(i) { + i += 1; + match l { + 128 => break, + 0..=127 => { + let n = usize::from(l) + 1; + let avail = n.min(data.len() - i); + if out.len() + avail > max { + return Err(Failure::limit(max)); + } + out.extend_from_slice(&data[i..i + avail]); + i += avail; + if avail < n { + return Err(Failure::corrupt(NAME, "truncated literal run", out)); + } + } + _ => { + let Some(&b) = data.get(i) else { + return Err(Failure::corrupt(NAME, "truncated repeat run", out)); + }; + i += 1; + let n = 257 - usize::from(l); + if out.len() + n > max { + return Err(Failure::limit(max)); + } + out.resize(out.len() + n, b); + } + } + } + Ok(out) +} + +pub fn encode(data: &[u8]) -> Vec { + let mut out = Vec::with_capacity(data.len() + data.len() / 128 + 2); + let n = data.len(); + let mut i = 0; + while i < n { + let mut run = 1; + while i + run < n && run < 128 && data[i + run] == data[i] { + run += 1; + } + if run >= 2 { + out.push((257 - run) as u8); + out.push(data[i]); + i += run; + continue; + } + // Literal: extend until a repeat begins or 128 bytes. + let start = i; + let mut j = i + 1; + while j < n && j - start < 128 && !(j + 1 < n && data[j] == data[j + 1]) { + j += 1; + } + out.push((j - start - 1) as u8); + out.extend_from_slice(&data[start..j]); + i = j; + } + out.push(128); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::CodecError; + + fn dec(d: &[u8]) -> Result, CodecError> { + decode(d, 10_000).map_err(|f| f.error) + } + + #[test] + fn vectors() { + assert_eq!(dec(&[2, b'a', b'b', b'c', 254, b'x', 128]).unwrap(), b"abcxxx"); + assert_eq!(dec(&[129, 7, 128]).unwrap(), [7; 128]); + assert!(dec(&[127]).is_err()); + assert_eq!(dec(&[0, b'q']).unwrap(), b"q"); // missing EOD + assert_eq!(dec(&[128, 0, b'q']).unwrap(), b""); // data after EOD ignored + assert_eq!(dec(&[]).unwrap(), b""); + } + + #[test] + fn truncated() { + let f = decode(&[4, b'a', b'b'], 100).err().unwrap(); + assert!(matches!(f.error, CodecError::Corrupt { .. })); + assert_eq!(f.partial, b"ab"); + assert!(matches!(dec(&[0, b'a', 200]), Err(CodecError::Corrupt { .. }))); + } + + #[test] + fn limit() { + assert!(matches!(decode(&[129, 0, 129, 0], 200).map_err(|f| f.error), Err(CodecError::LimitExceeded(200)))); + assert!(matches!(decode(&[3, 1, 2, 3, 4], 3).map_err(|f| f.error), Err(CodecError::LimitExceeded(3)))); + assert_eq!(decode(&[129, 0, 129, 0], 256).ok().unwrap().len(), 256); + } + + #[test] + fn encoding() { + assert_eq!(encode(b"abcxxx"), [2, b'a', b'b', b'c', 254, b'x', 128]); + assert_eq!(encode(b""), [128]); + assert_eq!(encode(&[5; 300]), [129, 5, 129, 5, 213, 5, 128]); + let lit: Vec = (0..200u8).collect(); + let e = encode(&lit); + assert_eq!((e[0], e[129]), (127, 71)); + assert_eq!(dec(&e).unwrap(), lit); + } +} From eb640fd9928b17db5a9b796ca7f16280c60d5b1d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:24:39 -0500 Subject: [PATCH 13/31] Add craft-sheet: the cell positions, values and tables the spreadsheet formats share (#956) craft-xls carried its own CellRef, Range, Value, CellError, Visibility, sheet-name quoting, built-in number format table and indexed palette. craft-xlsx and craft-ods need the same ones, so they move to a crate with no dependencies and craft-xls re-exports them: an application gets one Value and one CellRef whichever file it read. Added on the way: A1 parsing (CellRef::parse, Range::parse), Range::count and a1, CellError::parse and the newer error values SpreadsheetML stores as text, builtin_format_id for writers. --- Cargo.lock | 5 + Cargo.toml | 1 + README.md | 1 + crates/sheet/Cargo.toml | 13 ++ crates/sheet/README.md | 18 +++ crates/sheet/src/a1.rs | 252 +++++++++++++++++++++++++++++++++++++ crates/sheet/src/lib.rs | 18 +++ crates/sheet/src/style.rs | 87 +++++++++++++ crates/sheet/src/tables.rs | 94 ++++++++++++++ crates/sheet/src/value.rs | 133 ++++++++++++++++++++ crates/xls/Cargo.toml | 1 + crates/xls/README.md | 3 +- crates/xls/src/expr.rs | 94 +------------- crates/xls/src/lib.rs | 3 +- crates/xls/src/model.rs | 186 +-------------------------- crates/xls/src/styles.rs | 73 +---------- 16 files changed, 630 insertions(+), 352 deletions(-) create mode 100644 crates/sheet/Cargo.toml create mode 100644 crates/sheet/README.md create mode 100644 crates/sheet/src/a1.rs create mode 100644 crates/sheet/src/lib.rs create mode 100644 crates/sheet/src/style.rs create mode 100644 crates/sheet/src/tables.rs create mode 100644 crates/sheet/src/value.rs diff --git a/Cargo.lock b/Cargo.lock index 2fc2f8292..7f405cb29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1570,6 +1570,10 @@ dependencies = [ "toml_edit", ] +[[package]] +name = "craft-sheet" +version = "0.1.0" + [[package]] name = "craft-silhouette-studio" version = "0.1.0" @@ -1794,6 +1798,7 @@ name = "craft-xls" version = "0.1.0" dependencies = [ "craft-cfb", + "craft-sheet", "thiserror 2.0.21", ] diff --git a/Cargo.toml b/Cargo.toml index 3634d3b8a..a611a202a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ craft-pst = { path = "crates/pst" } craft-quit = { path = "crates/quit" } craft-rtf = { path = "crates/rtf" } craft-raw = { path = "crates/raw" } +craft-sheet = { path = "crates/sheet" } craft-sketch = { path = "crates/sketch" } craft-settings = { path = "crates/settings" } craft-silhouette-studio = { path = "crates/silhouette-studio" } diff --git a/README.md b/README.md index b48364a81..4e8f4e78d 100644 --- a/README.md +++ b/README.md @@ -250,6 +250,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Self { + CellRef { row, col } + } + + /// The A1 text (`B3`). + pub fn a1(self) -> String { + let mut s = String::new(); + self.push_a1(&mut s); + s + } + + /// Appends the A1 text. + pub fn push_a1(self, s: &mut String) { + push_col(s, self.col); + let _ = write!(s, "{}", u64::from(self.row) + 1); + } + + /// Reads `B3`, `$B$3`, `b3`: a cell inside the grid, with nothing after it. + pub fn parse(s: &str) -> Option { + match CellRef::parse_prefix(s)? { + (cell, "") => Some(cell), + _ => None, + } + } + + /// Reads an A1 reference at the start of `s` and returns it with the rest of `s`. + pub fn parse_prefix(s: &str) -> Option<(CellRef, &str)> { + let s = s.strip_prefix('$').unwrap_or(s); + let letters = s.bytes().take_while(u8::is_ascii_alphabetic).count(); + let (cols, rest) = s.split_at_checked(letters)?; + let col = col_from_letters(cols)?; + let rest = rest.strip_prefix('$').unwrap_or(rest); + let digits = rest.bytes().take_while(u8::is_ascii_digit).count(); + if digits == 0 || digits > 7 { + return None; + } + let (rows, rest) = rest.split_at_checked(digits)?; + let row: u32 = rows.parse().ok()?; + if row == 0 || row > MAX_ROWS { + return None; + } + Some((CellRef::new(row - 1, col), rest)) + } +} + +impl std::fmt::Display for CellRef { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.a1()) + } +} + +/// A rectangle of cells, both corners included; `first` is the top left one. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Range { + pub first: CellRef, + pub last: CellRef, +} + +impl Range { + /// The rectangle spanned by two corners, in any order. + pub fn new(a: CellRef, b: CellRef) -> Self { + Range { first: CellRef::new(a.row.min(b.row), a.col.min(b.col)), last: CellRef::new(a.row.max(b.row), a.col.max(b.col)) } + } + + pub fn contains(&self, c: CellRef) -> bool { + (self.first.row..=self.last.row).contains(&c.row) && (self.first.col..=self.last.col).contains(&c.col) + } + + pub fn is_single(&self) -> bool { + self.first == self.last + } + + /// How many cells the rectangle holds. + pub fn count(&self) -> u64 { + (u64::from(self.last.row - self.first.row) + 1) * (u64::from(self.last.col - self.first.col) + 1) + } + + /// The A1 text: `B3` for one cell, `A1:C2` otherwise. + pub fn a1(&self) -> String { + let mut s = self.first.a1(); + if !self.is_single() { + s.push(':'); + self.last.push_a1(&mut s); + } + s + } + + /// Reads `A1:C2`, `$A$1:$C$2`, one cell (`B3`), whole columns (`A:C`) and whole rows + /// (`1:3`), with surrounding space ignored. Whole columns and rows span the grid. + pub fn parse(s: &str) -> Option { + let s = s.trim(); + let Some((a, b)) = s.split_once(':') else { + return CellRef::parse(s).map(|c| Range { first: c, last: c }); + }; + if let (Some(x), Some(y)) = (CellRef::parse(a), CellRef::parse(b)) { + return Some(Range::new(x, y)); + } + let (a, b) = (a.strip_prefix('$').unwrap_or(a), b.strip_prefix('$').unwrap_or(b)); + if let (Some(x), Some(y)) = (col_from_letters(a), col_from_letters(b)) { + return Some(Range::new(CellRef::new(0, x), CellRef::new(MAX_ROWS - 1, y))); + } + let row = |t: &str| -> Option { + if t.is_empty() || t.len() > 7 || !t.bytes().all(|c| c.is_ascii_digit()) { + return None; + } + t.parse::().ok().filter(|r| (1..=MAX_ROWS).contains(r)).map(|r| r - 1) + }; + Some(Range::new(CellRef::new(row(a)?, 0), CellRef::new(row(b)?, MAX_COLS - 1))) + } +} + +impl std::fmt::Display for Range { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.a1()) + } +} + +/// Appends a column's letters: 0 is `A`, 25 is `Z`, 26 is `AA`. +pub fn push_col(s: &mut String, col: u32) { + let mut letters = [0u8; 8]; + let mut i = letters.len(); + let mut n = u64::from(col) + 1; + while n > 0 && i > 0 { + i -= 1; + if let Some(l) = letters.get_mut(i) { + *l = b'A' + ((n - 1) % 26) as u8; + } + n = (n - 1) / 26; + } + for &l in letters.get(i..).unwrap_or_default() { + s.push(char::from(l)); + } +} + +/// A column's letters: 0 is `A`, 16383 is `XFD`. +pub fn col_letters(col: u32) -> String { + let mut s = String::new(); + push_col(&mut s, col); + s +} + +/// `A` is 0, `xfd` is 16383. `None` for empty text, anything but one to three letters, or a +/// column beyond `XFD`. +pub fn col_from_letters(s: &str) -> Option { + if s.is_empty() || s.len() > 3 || !s.bytes().all(|b| b.is_ascii_alphabetic()) { + return None; + } + let n = s.bytes().fold(0u32, |n, b| n * 26 + u32::from(b.to_ascii_uppercase() - b'A') + 1); + (n <= MAX_COLS).then(|| n - 1) +} + +/// Whether `name` reads as an R1C1 reference: `R`, `C`, `RC`, `R1`, `C1`, `R1C`, `RC1`, `R1C1` +/// in any case. +fn is_r1c1_token(name: &str) -> bool { + let mut chars = name.chars().peekable(); + let mut letters = 0; + for letter in ['r', 'c'] { + if chars.peek().is_some_and(|c| c.to_ascii_lowercase() == letter) { + chars.next(); + letters += 1; + while chars.peek().is_some_and(char::is_ascii_digit) { + chars.next(); + } + } + } + letters > 0 && chars.next().is_none() +} + +/// Whether Excel quotes a sheet name in a reference (`'My Sheet'!A1`): it starts with a digit or +/// `.`, holds anything but letters (any script), digits, `_` and `.`, reads as an A1 cell or as +/// an R1C1 reference, or is a boolean. +pub fn sheet_needs_quotes(name: &str) -> bool { + let Some(first) = name.chars().next() else { + return true; + }; + if first.is_ascii_digit() || first == '.' || name.chars().any(|c| !(c.is_alphanumeric() || c == '_' || c == '.')) { + return true; + } + // No `$` can be here (it is not alphanumeric), so this is exactly "an A1 cell in the grid". + CellRef::parse(name).is_some() || is_r1c1_token(name) || name.eq_ignore_ascii_case("TRUE") || name.eq_ignore_ascii_case("FALSE") +} + +/// A sheet name as it is written in a reference: quoted, with `'` doubled, when Excel quotes it. +pub fn quote_sheet(name: &str) -> String { + if sheet_needs_quotes(name) { format!("'{}'", name.replace('\'', "''")) } else { name.to_string() } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a1_text_both_ways() { + assert_eq!(CellRef::new(2, 27).a1(), "AB3"); + assert_eq!(CellRef::parse("AB3"), Some(CellRef::new(2, 27))); + assert_eq!(CellRef::parse("$ab$3"), Some(CellRef::new(2, 27))); + assert_eq!(CellRef::parse("XFD1048576"), Some(CellRef::new(MAX_ROWS - 1, MAX_COLS - 1))); + assert_eq!(CellRef::new(MAX_ROWS - 1, MAX_COLS - 1).a1(), "XFD1048576"); + for bad in ["", "A", "1", "A0", "XFE1", "A1048577", "A1x", "A+1", "A-1", "AAAA1", "A12345678", "É1"] { + assert_eq!(CellRef::parse(bad), None, "{bad}"); + } + assert_eq!(CellRef::parse_prefix("B2:C3"), Some((CellRef::new(1, 1), ":C3"))); + assert_eq!(col_letters(16383), "XFD"); + assert_eq!((col_letters(26), col_letters(u32::MAX).len()), ("AA".to_string(), 7)); + assert_eq!(col_from_letters("xfd"), Some(16383)); + assert_eq!(col_from_letters("XFE"), None); + } + + #[test] + fn ranges() { + let r = Range::parse(" $C$2:A1 "); + assert_eq!(r, Some(Range { first: CellRef::new(0, 0), last: CellRef::new(1, 2) })); + assert_eq!(r.map(|r| (r.a1(), r.count())), Some(("A1:C2".to_string(), 6))); + assert_eq!(Range::parse("B3").map(|r| (r.is_single(), r.a1())), Some((true, "B3".to_string()))); + assert_eq!(Range::parse("B:$A"), Some(Range::new(CellRef::new(0, 0), CellRef::new(MAX_ROWS - 1, 1)))); + assert_eq!(Range::parse("3:2"), Some(Range::new(CellRef::new(1, 0), CellRef::new(2, MAX_COLS - 1)))); + assert_eq!(Range::parse("1:1048577"), None); + assert_eq!(Range::parse("0:1"), None); + assert_eq!(Range::parse("A1:"), None); + assert_eq!(Range::parse("A:1"), None); + assert!(Range::new(CellRef::new(0, 0), CellRef::new(MAX_ROWS - 1, MAX_COLS - 1)).count() == 17_179_869_184); + } + + #[test] + fn sheet_names_are_quoted_when_excel_quotes_them() { + for plain in ["Sheet1", "Data_2", "Übersicht", "a.b", "XFE1", "A1048577", "RC1x"] { + assert!(!sheet_needs_quotes(plain), "{plain}"); + } + for quoted in ["", "My Sheet", "1st", ".x", "A1", "xfd1048576", "R1C1", "rc", "C", "TRUE", "false", "a-b", "It's"] { + assert!(sheet_needs_quotes(quoted), "{quoted}"); + } + assert_eq!(quote_sheet("It's"), "'It''s'"); + assert_eq!(quote_sheet("Sheet1"), "Sheet1"); + } +} diff --git a/crates/sheet/src/lib.rs b/crates/sheet/src/lib.rs new file mode 100644 index 000000000..337a3daf8 --- /dev/null +++ b/crates/sheet/src/lib.rs @@ -0,0 +1,18 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Spreadsheet vocabulary shared by the spreadsheet format crates (`craft-xls`, `craft-xlsx`, +//! `craft-ods`): cell positions and their A1 text, cell values and error values, sheet-name +//! quoting, the cell formatting enumerations, Excel's built-in number formats and its indexed +//! colour palette. +//! +//! Nothing here is a file format; each format crate re-exports what it uses, so an application +//! gets one `Value` and one `CellRef` whichever file it read. + +mod a1; +mod style; +mod tables; +mod value; + +pub use a1::{CellRef, MAX_COLS, MAX_ROWS, Range, col_from_letters, col_letters, push_col, quote_sheet, sheet_needs_quotes}; +pub use style::{BorderStyle, HAlign, Pattern, Script, Underline, VAlign}; +pub use tables::{DEFAULT_PALETTE, builtin_format, builtin_format_id}; +pub use value::{CellError, Value, Visibility}; diff --git a/crates/sheet/src/style.rs b/crates/sheet/src/style.rs new file mode 100644 index 000000000..d3c0576bc --- /dev/null +++ b/crates/sheet/src/style.rs @@ -0,0 +1,87 @@ +//! The cell formatting choices BIFF8 and SpreadsheetML share. Each format has its own spelling +//! for them (a number in an XF record, a name in `styles.xml`). + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Underline { + #[default] + None, + Single, + Double, + SingleAccounting, + DoubleAccounting, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Script { + #[default] + Baseline, + Superscript, + Subscript, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum HAlign { + #[default] + General, + Left, + Center, + Right, + Fill, + Justify, + CenterAcross, + Distributed, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum VAlign { + Top, + Center, + #[default] + Bottom, + Justify, + Distributed, +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum BorderStyle { + #[default] + None, + Thin, + Medium, + Dashed, + Dotted, + Thick, + Double, + Hair, + MediumDashed, + DashDot, + MediumDashDot, + DashDotDot, + MediumDashDotDot, + SlantDashDot, +} + +/// Fill patterns, named as SpreadsheetML names them. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Pattern { + #[default] + None, + Solid, + Gray50, + Gray75, + Gray25, + DarkHorizontal, + DarkVertical, + DarkDown, + DarkUp, + DarkGrid, + DarkTrellis, + LightHorizontal, + LightVertical, + LightDown, + LightUp, + LightGrid, + LightTrellis, + Gray125, + Gray0625, +} diff --git a/crates/sheet/src/tables.rs b/crates/sheet/src/tables.rs new file mode 100644 index 000000000..b30935d54 --- /dev/null +++ b/crates/sheet/src/tables.rs @@ -0,0 +1,94 @@ +//! Tables Excel files refer to by number. + +/// Built-in number format codes (en-US) by id. Ids not listed (23–36 are East Asian +/// locale formats) read as General. +const BUILTIN_FORMATS: [(u32, &str); 36] = [ + (0, "General"), + (1, "0"), + (2, "0.00"), + (3, "#,##0"), + (4, "#,##0.00"), + (5, "\"$\"#,##0_);\\(\"$\"#,##0\\)"), + (6, "\"$\"#,##0_);[Red]\\(\"$\"#,##0\\)"), + (7, "\"$\"#,##0.00_);\\(\"$\"#,##0.00\\)"), + (8, "\"$\"#,##0.00_);[Red]\\(\"$\"#,##0.00\\)"), + (9, "0%"), + (10, "0.00%"), + (11, "0.00E+00"), + (12, "# ?/?"), + (13, "# ??/??"), + (14, "m/d/yyyy"), + (15, "d-mmm-yy"), + (16, "d-mmm"), + (17, "mmm-yy"), + (18, "h:mm AM/PM"), + (19, "h:mm:ss AM/PM"), + (20, "h:mm"), + (21, "h:mm:ss"), + (22, "m/d/yyyy h:mm"), + (37, "#,##0 ;(#,##0)"), + (38, "#,##0 ;[Red](#,##0)"), + (39, "#,##0.00;(#,##0.00)"), + (40, "#,##0.00;[Red](#,##0.00)"), + (41, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)"), + (42, "_(\"$\"* #,##0_);_(\"$\"* \\(#,##0\\);_(\"$\"* \"-\"_);_(@_)"), + (43, "_(* #,##0.00_);_(* \\(#,##0.00\\);_(* \"-\"??_);_(@_)"), + (44, "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"), + (45, "mm:ss"), + (46, "[h]:mm:ss"), + (47, "mmss.0"), + (48, "##0.0E+0"), + (49, "@"), +]; + +pub fn builtin_format(id: u32) -> Option<&'static str> { + BUILTIN_FORMATS.iter().find(|(i, _)| *i == id).map(|(_, c)| *c) +} + +/// Id to write for a format code when it is a locale-independent built-in. The currency formats +/// 5 to 8 and the accounting formats with a currency, 42 and 44, are left out (`None`, to be +/// written as custom formats) because their meaning depends on the reader's locale. +pub fn builtin_format_id(code: &str) -> Option { + BUILTIN_FORMATS.iter().find(|(i, c)| *c == code && !matches!(i, 5..=8 | 42 | 44)).map(|(i, _)| *i) +} + +/// The indexed colour table of a workbook that does not replace it, as `0xRRGGBB`: the eight +/// fixed colours, then the 56 a BIFF8 Palette record ([MS-XLS] 2.4.188) or a SpreadsheetML +/// `indexedColors` list (ECMA-376 Part 1, 18.8.27) replaces. +pub const DEFAULT_PALETTE: [u32; 64] = [ + 0x000000, 0xFFFFFF, 0xFF0000, 0x00FF00, 0x0000FF, 0xFFFF00, 0xFF00FF, 0x00FFFF, 0x000000, 0xFFFFFF, 0xFF0000, 0x00FF00, 0x0000FF, 0xFFFF00, + 0xFF00FF, 0x00FFFF, 0x800000, 0x008000, 0x000080, 0x808000, 0x800080, 0x008080, 0xC0C0C0, 0x808080, 0x9999FF, 0x993366, 0xFFFFCC, 0xCCFFFF, + 0x660066, 0xFF8080, 0x0066CC, 0xCCCCFF, 0x000080, 0xFF00FF, 0xFFFF00, 0x00FFFF, 0x800080, 0x800000, 0x008080, 0x0000FF, 0x00CCFF, 0xCCFFFF, + 0xCCFFCC, 0xFFFF99, 0x99CCFF, 0xFF99CC, 0xCC99FF, 0xFFCC99, 0x3366FF, 0x33CCCC, 0x99CC00, 0xFFCC00, 0xFF9900, 0xFF6600, 0x666699, 0x969696, + 0x003366, 0x339966, 0x003300, 0x333300, 0x993300, 0x993366, 0x333399, 0x333333, +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builtin_format_ids() { + assert_eq!((0..=60).filter(|&id| builtin_format(id).is_some()).count(), 36); + assert_eq!(builtin_format(0), Some("General")); + assert_eq!(builtin_format(14), Some("m/d/yyyy")); + assert_eq!(builtin_format(22), Some("m/d/yyyy h:mm")); + assert_eq!(builtin_format(49), Some("@")); + // 23 to 36 are locale formats that are not defined here. + assert_eq!((23..=36).find_map(builtin_format), None); + assert_eq!(builtin_format(50), None); + for id in 0..=60 { + if let Some(code) = builtin_format(id) { + let back = builtin_format_id(code); + assert_eq!(back, (!matches!(id, 5..=8 | 42 | 44)).then_some(id), "{id}"); + } + } + assert_eq!(builtin_format_id("yyyy"), None); + } + + #[test] + fn the_palette_repeats_its_eight_fixed_colours() { + assert_eq!(DEFAULT_PALETTE.get(..8), DEFAULT_PALETTE.get(8..16)); + assert_eq!(DEFAULT_PALETTE.get(63), Some(&0x333333)); + } +} diff --git a/crates/sheet/src/value.rs b/crates/sheet/src/value.rs new file mode 100644 index 000000000..066dc3e95 --- /dev/null +++ b/crates/sheet/src/value.rs @@ -0,0 +1,133 @@ +//! What a cell stores. + +use std::sync::Arc; + +/// The error values a spreadsheet file stores. The first eight are the ones BIFF8 has +/// ([MS-XLS] 2.5.10, BErr); the rest are newer and appear in SpreadsheetML as text. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum CellError { + Null, + Div0, + Value, + Ref, + Name, + Num, + NA, + GettingData, + Spill, + Calc, + Blocked, + Python, + Busy, + Timeout, +} + +impl CellError { + /// Every error value, in the order above. + pub const ALL: [CellError; 14] = [ + CellError::Null, + CellError::Div0, + CellError::Value, + CellError::Ref, + CellError::Name, + CellError::Num, + CellError::NA, + CellError::GettingData, + CellError::Spill, + CellError::Calc, + CellError::Blocked, + CellError::Python, + CellError::Busy, + CellError::Timeout, + ]; + + pub fn as_str(&self) -> &'static str { + match self { + CellError::Null => "#NULL!", + CellError::Div0 => "#DIV/0!", + CellError::Value => "#VALUE!", + CellError::Ref => "#REF!", + CellError::Name => "#NAME?", + CellError::Num => "#NUM!", + CellError::NA => "#N/A", + CellError::GettingData => "#GETTING_DATA", + CellError::Spill => "#SPILL!", + CellError::Calc => "#CALC!", + CellError::Blocked => "#BLOCKED!", + CellError::Python => "#PYTHON!", + CellError::Busy => "#BUSY!", + CellError::Timeout => "#TIMEOUT!", + } + } + + /// The error with this text, in any case. + pub fn parse(s: &str) -> Option { + CellError::ALL.iter().find(|e| e.as_str().eq_ignore_ascii_case(s)).copied() + } +} + +impl std::fmt::Display for CellError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +/// A cell's stored value (for a formula cell, its saved result). +#[derive(Clone, Debug, Default, PartialEq)] +pub enum Value { + #[default] + Empty, + Number(f64), + Text(Arc), + Bool(bool), + Error(CellError), +} + +impl Value { + /// A number, or `#NUM!` when it is NaN or infinite (Excel never stores those). Negative zero + /// becomes zero. + pub fn number(v: f64) -> Value { + if v.is_finite() { Value::Number(if v == 0.0 { 0.0 } else { v }) } else { Value::Error(CellError::Num) } + } + + pub fn text(s: impl Into>) -> Value { + Value::Text(s.into()) + } + + pub fn is_empty(&self) -> bool { + matches!(self, Value::Empty) + } +} + +/// Whether a sheet shows: `VeryHidden` cannot be unhidden from the user interface. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Visibility { + #[default] + Visible, + Hidden, + VeryHidden, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn errors_parse_back_from_their_text() { + for e in CellError::ALL { + assert_eq!(CellError::parse(e.as_str()), Some(e)); + assert_eq!(CellError::parse(&e.as_str().to_ascii_lowercase()), Some(e)); + } + assert_eq!(CellError::parse("#NOPE!"), None); + assert_eq!(CellError::Div0.to_string(), "#DIV/0!"); + } + + #[test] + fn numbers_are_finite() { + assert_eq!(Value::number(f64::NAN), Value::Error(CellError::Num)); + assert_eq!(Value::number(f64::INFINITY), Value::Error(CellError::Num)); + assert!(matches!(Value::number(-0.0), Value::Number(n) if n.is_sign_positive())); + assert!(Value::Empty.is_empty() && !Value::text("").is_empty()); + } +} diff --git a/crates/xls/Cargo.toml b/crates/xls/Cargo.toml index f742798c5..41d3362fb 100644 --- a/crates/xls/Cargo.toml +++ b/crates/xls/Cargo.toml @@ -12,4 +12,5 @@ workspace = true [dependencies] craft-cfb = { workspace = true } +craft-sheet = { workspace = true } thiserror = { workspace = true } diff --git a/crates/xls/README.md b/crates/xls/README.md index c0b6c7489..7fbb93280 100644 --- a/crates/xls/README.md +++ b/crates/xls/README.md @@ -3,7 +3,8 @@ A clean-room reader for Excel 97-2003 workbooks (`.xls`, BIFF8). It turns the bytes of a file into a `Workbook` that keeps the file's own terms: cells, formulas as text, the format table, column and row records. Mapping that onto an application's document is the caller's work. -No `unsafe`; the only dependencies are `craft-cfb` (the compound file container) and `thiserror`. +No `unsafe`; the only dependencies are `craft-cfb` (the compound file container), `craft-sheet` (cell +positions, values and tables shared with the other spreadsheet formats) and `thiserror`. No third-party implementation was consulted. | Document | Used for | diff --git a/crates/xls/src/expr.rs b/crates/xls/src/expr.rs index 704e2a57c..f4e0b114d 100644 --- a/crates/xls/src/expr.rs +++ b/crates/xls/src/expr.rs @@ -3,12 +3,9 @@ use std::fmt::Write; use std::sync::Arc; -use crate::CellError; +use craft_sheet::{push_col, quote_sheet, sheet_needs_quotes}; -/// Rows and columns of the grid a sheet name is tested against when deciding whether it reads -/// as a cell reference (the SpreadsheetML grid, which is the larger one). -const GRID_ROWS: u32 = 1_048_576; -const GRID_COLS: u32 = 16_384; +use crate::CellError; /// One end of a reference, with `$` anchors. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -116,31 +113,6 @@ pub(crate) enum Expr { Paren(Box), } -/// 0 → `A`, 25 → `Z`, 26 → `AA`. -fn push_col(s: &mut String, col: u32) { - let mut letters = [0u8; 8]; - let mut i = letters.len(); - let mut n = u64::from(col) + 1; - while n > 0 && i > 0 { - i -= 1; - if let Some(l) = letters.get_mut(i) { - *l = b'A' + ((n - 1) % 26) as u8; - } - n = (n - 1) / 26; - } - for &l in letters.get(i..).unwrap_or_default() { - s.push(char::from(l)); - } -} - -/// The A1 text of a cell (`B3`), for messages and tests. -pub(crate) fn a1(row: u32, col: u32) -> String { - let mut s = String::new(); - push_col(&mut s, col); - let _ = write!(s, "{}", u64::from(row) + 1); - s -} - fn push_anchor(s: &mut String, a: &Anchor) { if a.col_abs { s.push('$'); @@ -152,53 +124,6 @@ fn push_anchor(s: &mut String, a: &Anchor) { let _ = write!(s, "{}", u64::from(a.row) + 1); } -/// Whether `name` is an A1 cell reference inside the grid (`A1`, `xfd1048576`). -fn is_a1(name: &str) -> bool { - let letters = name.bytes().take_while(u8::is_ascii_alphabetic).count(); - let Some((cols, digits)) = name.split_at_checked(letters) else { return false }; - if cols.is_empty() || cols.len() > 3 || digits.is_empty() || digits.len() > 7 || !digits.bytes().all(|b| b.is_ascii_digit()) { - return false; - } - let col = cols.bytes().fold(0u32, |n, b| n * 26 + u32::from(b.to_ascii_uppercase() - b'A') + 1); - let row: u32 = digits.parse().unwrap_or(0); - (1..=GRID_COLS).contains(&col) && (1..=GRID_ROWS).contains(&row) -} - -/// Whether `name` reads as an R1C1 reference: `R`, `C`, `RC`, `R1`, `C1`, `R1C`, `RC1`, `R1C1` -/// in any case. -fn is_r1c1_token(name: &str) -> bool { - let mut chars = name.chars().peekable(); - let mut letters = 0; - for letter in ['r', 'c'] { - if chars.peek().is_some_and(|c| c.to_ascii_lowercase() == letter) { - chars.next(); - letters += 1; - while chars.peek().is_some_and(char::is_ascii_digit) { - chars.next(); - } - } - } - letters > 0 && chars.next().is_none() -} - -/// Whether Excel quotes a sheet name in a reference (`'My Sheet'!A1`): it starts with a digit or -/// `.`, holds anything but letters (any script), digits, `_` and `.`, reads as an A1 cell or as -/// an R1C1 reference, or is a boolean. -pub fn sheet_needs_quotes(name: &str) -> bool { - let Some(first) = name.chars().next() else { - return true; - }; - if first.is_ascii_digit() || first == '.' || name.chars().any(|c| !(c.is_alphanumeric() || c == '_' || c == '.')) { - return true; - } - is_a1(name) || is_r1c1_token(name) || name.eq_ignore_ascii_case("TRUE") || name.eq_ignore_ascii_case("FALSE") -} - -/// A sheet name as it is written in a reference: quoted, with `'` doubled, when Excel quotes it. -pub fn quote_sheet(name: &str) -> String { - if sheet_needs_quotes(name) { format!("'{}'", name.replace('\'', "''")) } else { name.to_string() } -} - fn push_reference(s: &mut String, r: &Reference) { match &r.sheet { SheetSel::Current => {} @@ -402,19 +327,4 @@ mod tests { assert_eq!(number_to_text(1e-10), "1E-10"); assert!(number(f64::MAX).parse::().is_ok_and(f64::is_finite)); } - - #[test] - fn sheet_names_are_quoted_when_excel_quotes_them() { - for plain in ["Sheet1", "Data_2", "Übersicht", "a.b"] { - assert!(!sheet_needs_quotes(plain), "{plain}"); - } - for quoted in ["", "My Data", "1st", ".x", "A1", "xfd1048576", "R1C1", "rc", "true", "It's"] { - assert!(sheet_needs_quotes(quoted), "{quoted}"); - } - // Outside the grid it is no cell reference. - assert!(!sheet_needs_quotes("XFE1")); - assert!(!sheet_needs_quotes("A1048577")); - assert_eq!(quote_sheet("It's"), "'It''s'"); - assert_eq!(a1(2, 27), "AB3"); - } } diff --git a/crates/xls/src/lib.rs b/crates/xls/src/lib.rs index 20d73ab4c..0dc33f61b 100644 --- a/crates/xls/src/lib.rs +++ b/crates/xls/src/lib.rs @@ -40,13 +40,12 @@ mod styles; #[doc(hidden)] pub mod testing; -pub use expr::{quote_sheet, sheet_needs_quotes}; +pub use craft_sheet::{DEFAULT_PALETTE, quote_sheet, sheet_needs_quotes}; pub use model::{ Alignment, BorderLine, BorderStyle, Borders, Calc, CalcMode, Cell, CellError, CellFormat, CellRef, Color, Column, DefinedName, Fill, Font, Formula, HAlign, Pattern, Range, Row, Script, Sheet, Underline, VAlign, Value, View, Visibility, Workbook, }; pub use read::{is_xls, read, read_with}; -pub use styles::DEFAULT_PALETTE; /// Why a workbook could not be read. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] diff --git a/crates/xls/src/model.rs b/crates/xls/src/model.rs index 3aedf8697..ce88a043d 100644 --- a/crates/xls/src/model.rs +++ b/crates/xls/src/model.rs @@ -2,98 +2,8 @@ //! the format (widths and heights keep the file's units, formulas are text). use std::collections::BTreeMap; -use std::sync::Arc; -/// A cell position, zero-based. Ordered by row, then column. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct CellRef { - pub row: u32, - pub col: u32, -} - -impl CellRef { - pub const fn new(row: u32, col: u32) -> Self { - CellRef { row, col } - } - - /// The A1 text (`B3`). - pub fn a1(self) -> String { - crate::expr::a1(self.row, self.col) - } -} - -/// A rectangle of cells, both corners included; `first` is the top left one. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct Range { - pub first: CellRef, - pub last: CellRef, -} - -impl Range { - /// The rectangle spanned by two corners, in any order. - pub fn new(a: CellRef, b: CellRef) -> Self { - Range { first: CellRef::new(a.row.min(b.row), a.col.min(b.col)), last: CellRef::new(a.row.max(b.row), a.col.max(b.col)) } - } - - pub fn contains(&self, c: CellRef) -> bool { - (self.first.row..=self.last.row).contains(&c.row) && (self.first.col..=self.last.col).contains(&c.col) - } - - pub fn is_single(&self) -> bool { - self.first == self.last - } -} - -/// The error values a BIFF8 file stores ([MS-XLS] 2.5.10, BErr). -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub enum CellError { - Null, - Div0, - Value, - Ref, - Name, - Num, - NA, - GettingData, -} - -impl CellError { - pub fn as_str(&self) -> &'static str { - match self { - CellError::Null => "#NULL!", - CellError::Div0 => "#DIV/0!", - CellError::Value => "#VALUE!", - CellError::Ref => "#REF!", - CellError::Name => "#NAME?", - CellError::Num => "#NUM!", - CellError::NA => "#N/A", - CellError::GettingData => "#GETTING_DATA", - } - } -} - -/// A cell's stored value (for a formula cell, its saved result). -#[derive(Clone, Debug, Default, PartialEq)] -pub enum Value { - #[default] - Empty, - Number(f64), - Text(Arc), - Bool(bool), - Error(CellError), -} - -impl Value { - /// A number, or `#NUM!` when it is NaN or infinite (Excel never stores those). Negative zero - /// becomes zero. - pub fn number(v: f64) -> Value { - if v.is_finite() { Value::Number(if v == 0.0 { 0.0 } else { v }) } else { Value::Error(CellError::Num) } - } - - pub fn is_empty(&self) -> bool { - matches!(self, Value::Empty) - } -} +pub use craft_sheet::{BorderStyle, CellError, CellRef, HAlign, Pattern, Range, Script, Underline, VAlign, Value, Visibility}; /// A cell's formula. #[derive(Clone, Debug, PartialEq, Eq)] @@ -114,15 +24,6 @@ pub struct Cell { pub formula: Option, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum Visibility { - #[default] - Visible, - Hidden, - /// Hidden, and not offered by Excel's Unhide dialog. - VeryHidden, -} - /// A column's ColInfo record ([MS-XLS] 2.4.53). #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Column { @@ -224,24 +125,6 @@ pub enum Color { Rgb(u32), } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum Underline { - #[default] - None, - Single, - Double, - SingleAccounting, - DoubleAccounting, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum Script { - #[default] - Baseline, - Superscript, - Subscript, -} - /// A Font record ([MS-XLS] 2.4.122). #[derive(Clone, Debug, Default, PartialEq)] pub struct Font { @@ -261,29 +144,6 @@ pub struct Font { pub charset: Option, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum HAlign { - #[default] - General, - Left, - Center, - Right, - Fill, - Justify, - CenterAcross, - Distributed, -} - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum VAlign { - Top, - Center, - #[default] - Bottom, - Justify, - Distributed, -} - #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Alignment { pub h: HAlign, @@ -295,25 +155,6 @@ pub struct Alignment { pub shrink: bool, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum BorderStyle { - #[default] - None, - Thin, - Medium, - Dashed, - Dotted, - Thick, - Double, - Hair, - MediumDashed, - DashDot, - MediumDashDot, - DashDotDot, - MediumDashDotDot, - SlantDashDot, -} - /// One border side. A side without a style has the automatic colour. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct BorderLine { @@ -333,31 +174,6 @@ pub struct Borders { pub diag_up: BorderLine, } -/// Fill patterns, named as SpreadsheetML names them. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] -pub enum Pattern { - #[default] - None, - Solid, - Gray50, - Gray75, - Gray25, - DarkHorizontal, - DarkVertical, - DarkDown, - DarkUp, - DarkGrid, - DarkTrellis, - LightHorizontal, - LightVertical, - LightDown, - LightUp, - LightGrid, - LightTrellis, - Gray125, - Gray0625, -} - /// A cell's fill. Without a pattern both colours are automatic. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] pub struct Fill { diff --git a/crates/xls/src/styles.rs b/crates/xls/src/styles.rs index 59d687848..ea0fcae8e 100644 --- a/crates/xls/src/styles.rs +++ b/crates/xls/src/styles.rs @@ -9,61 +9,7 @@ use crate::model::{ Alignment, BorderLine, BorderStyle, Borders, CellFormat, Color, Fill, Font, HAlign, Pattern, Script, Underline, VAlign, Workbook, }; use crate::records::Cursor; - -/// Built-in number format codes (en-US) by id. Ids not listed (23–36 are East Asian -/// locale formats) read as General. -const BUILTIN_FORMATS: [(u32, &str); 36] = [ - (0, "General"), - (1, "0"), - (2, "0.00"), - (3, "#,##0"), - (4, "#,##0.00"), - (5, "\"$\"#,##0_);\\(\"$\"#,##0\\)"), - (6, "\"$\"#,##0_);[Red]\\(\"$\"#,##0\\)"), - (7, "\"$\"#,##0.00_);\\(\"$\"#,##0.00\\)"), - (8, "\"$\"#,##0.00_);[Red]\\(\"$\"#,##0.00\\)"), - (9, "0%"), - (10, "0.00%"), - (11, "0.00E+00"), - (12, "# ?/?"), - (13, "# ??/??"), - (14, "m/d/yyyy"), - (15, "d-mmm-yy"), - (16, "d-mmm"), - (17, "mmm-yy"), - (18, "h:mm AM/PM"), - (19, "h:mm:ss AM/PM"), - (20, "h:mm"), - (21, "h:mm:ss"), - (22, "m/d/yyyy h:mm"), - (37, "#,##0 ;(#,##0)"), - (38, "#,##0 ;[Red](#,##0)"), - (39, "#,##0.00;(#,##0.00)"), - (40, "#,##0.00;[Red](#,##0.00)"), - (41, "_(* #,##0_);_(* \\(#,##0\\);_(* \"-\"_);_(@_)"), - (42, "_(\"$\"* #,##0_);_(\"$\"* \\(#,##0\\);_(\"$\"* \"-\"_);_(@_)"), - (43, "_(* #,##0.00_);_(* \\(#,##0.00\\);_(* \"-\"??_);_(@_)"), - (44, "_(\"$\"* #,##0.00_);_(\"$\"* \\(#,##0.00\\);_(\"$\"* \"-\"??_);_(@_)"), - (45, "mm:ss"), - (46, "[h]:mm:ss"), - (47, "mmss.0"), - (48, "##0.0E+0"), - (49, "@"), -]; - -pub(crate) fn builtin_format(id: u32) -> Option<&'static str> { - BUILTIN_FORMATS.iter().find(|(i, _)| *i == id).map(|(_, c)| *c) -} - -/// The colour table a workbook without a Palette record uses, as `0xRRGGBB`: the eight fixed -/// colours, then the 56 a Palette record replaces ([MS-XLS] 2.4.188). -pub const DEFAULT_PALETTE: [u32; 64] = [ - 0x000000, 0xFFFFFF, 0xFF0000, 0x00FF00, 0x0000FF, 0xFFFF00, 0xFF00FF, 0x00FFFF, 0x000000, 0xFFFFFF, 0xFF0000, 0x00FF00, 0x0000FF, 0xFFFF00, - 0xFF00FF, 0x00FFFF, 0x800000, 0x008000, 0x000080, 0x808000, 0x800080, 0x008080, 0xC0C0C0, 0x808080, 0x9999FF, 0x993366, 0xFFFFCC, 0xCCFFFF, - 0x660066, 0xFF8080, 0x0066CC, 0xCCCCFF, 0x000080, 0xFF00FF, 0xFFFF00, 0x00FFFF, 0x800080, 0x800000, 0x008080, 0x0000FF, 0x00CCFF, 0xCCFFFF, - 0xCCFFCC, 0xFFFF99, 0x99CCFF, 0xFF99CC, 0xCC99FF, 0xFFCC99, 0x3366FF, 0x33CCCC, 0x99CC00, 0xFFCC00, 0xFF9900, 0xFF6600, 0x666699, 0x969696, - 0x003366, 0x339966, 0x003300, 0x333300, 0x993300, 0x993366, 0x333399, 0x333333, -]; +use craft_sheet::{DEFAULT_PALETTE, builtin_format}; /// Font records kept at most ([MS-XLS] FontIndex allows indexes up to 1022). const MAX_FONTS: usize = 1024; @@ -453,20 +399,3 @@ pub(crate) fn build(r: &StyleRecords, formats: &HashMap, wb: &mut W wb.cell_styles = named; wb.formats.iter().map(|f| *f == plain).collect() } - -#[cfg(test)] -mod tests { - use super::builtin_format; - - #[test] - fn builtin_format_ids() { - assert_eq!((0..=60).filter(|&id| builtin_format(id).is_some()).count(), 36); - assert_eq!(builtin_format(0), Some("General")); - assert_eq!(builtin_format(14), Some("m/d/yyyy")); - assert_eq!(builtin_format(22), Some("m/d/yyyy h:mm")); - assert_eq!(builtin_format(49), Some("@")); - // 23-36 are locale formats, above 49 is custom. - assert_eq!((23..=36).find_map(builtin_format), None); - assert_eq!(builtin_format(50), None); - } -} From 219f63f64cd045d9027c31dd3973922e48af2cc8 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:25:01 -0500 Subject: [PATCH 14/31] Add craft-xcf: GIMP XCF reader from PhotoCraft (#960) * Add craft-xcf: GIMP XCF reader from PhotoCraft The format half of PhotoCraft's XCF import (crates/io/src/xcf.rs at abeb16c96): the header, precisions, property lists with their length repairs, parasites, layer structures and item paths, channels, and the uncompressed, RLE and zlib tiles. The mapping onto a document (blend modes, surfaces, profiles, warnings) stays in the app. * craft-xcf: an RLE run of zero length is an error A long run (op 127 or 128) with a 16-bit length of 0 consumed input without producing output, so a tile of them was work the pixel budget never charged. Ported from storytold/photocraft#11138. * craft-xcf: channel names are charged to the metadata budget Layer names already were. Many channel pointers to one channel with a large name made a copy per read, multiplying a small file into gigabytes of retained names. Ported from storytold/photocraft#11138. --- Cargo.lock | 8 + Cargo.toml | 1 + README.md | 1 + crates/xcf/Cargo.toml | 15 + crates/xcf/README.md | 40 ++ crates/xcf/src/lib.rs | 923 ++++++++++++++++++++++++++++++++ crates/xcf/src/tests.rs | 1120 +++++++++++++++++++++++++++++++++++++++ crates/xcf/src/tiles.rs | 70 +++ 8 files changed, 2178 insertions(+) create mode 100644 crates/xcf/Cargo.toml create mode 100644 crates/xcf/README.md create mode 100644 crates/xcf/src/lib.rs create mode 100644 crates/xcf/src/tests.rs create mode 100644 crates/xcf/src/tiles.rs diff --git a/Cargo.lock b/Cargo.lock index 7f405cb29..1be03246b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,6 +1793,14 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "craft-xcf" +version = "0.1.0" +dependencies = [ + "flate2", + "thiserror 2.0.21", +] + [[package]] name = "craft-xls" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index a611a202a..3f361200d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -104,6 +104,7 @@ craft-vcard = { path = "crates/vcard" } craft-vectorcraft-format = { path = "crates/vectorcraft-format" } craft-vectordrawable = { path = "crates/vectordrawable" } craft-xls = { path = "crates/xls" } +craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } craft-y4m = { path = "crates/y4m" } diff --git a/README.md b/README.md index 4e8f4e78d..5b51e5ebd 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = ", versions 0 to 26) and from files GIMP 3.2 +writes; no GIMP code is used. + +- `is_xcf`, `is_gzipped_xcf`, `gunzip`: recognise a file and inflate a compressed one. +- `Xcf::parse`: the header, the image properties and every layer structure, as an `Image`: + version, canvas, base type, `Precision`, `Compression`, colormap, resolution, guides, the ICC + profile and comment parasites, the path count, and each `Layer` with its properties as GIMP + wrote them (mode, opacity, offsets, locks, colour tag, composite mode and spaces, group flag, + item path, text / vector / link flags, effect count). No pixel is decoded. +- `Xcf::layer_pixels`, `Xcf::layer_mask`, `Xcf::channel`: one drawable's pixels as normalised + `f32` samples (`Pixels`), from uncompressed, RLE or zlib tiles at every precision (8, 16 and + 32-bit integer; 16, 32 and 64-bit float). Indexed pixels are expanded through the colormap. + Each call takes its samples from a byte budget the caller owns, and a `keep_going` callback. +- `Image::tree`: the layers arranged by their item paths, with the ones whose path leads nowhere + or nests too deep reported. + +The reader repairs what GIMP writes wrongly: the length words of `PROP_COLORMAP`, +`PROP_USER_UNIT` and `PROP_LINK_LAYER` are measured, not trusted. + +Layer modes, colour tags and composite settings stay numbers: what they mean to an application +is not decided here. Paths, layer effects and text or vector layer data are not read (their +presence is reported). There is no writer. + +## Limits + +| Limit | Value | +| --- | --- | +| Canvas or drawable side | 300,000 px (`MAX_SIDE`) | +| Canvas area | 1 GiB at 16 bytes per pixel (`MAX_PIXEL_BYTES`) | +| Decoded pixels | the caller's budget, charged 4 bytes a sample before allocation | +| Layers | 8192 (`MAX_LAYERS`); channels 1024 (`MAX_CHANNELS`) | +| Layer effects on one layer | 1024 | +| Properties in one list | 4096; a payload 64 MiB (`MAX_PAYLOAD`) | +| A string | 1 MiB (`MAX_STRING`) | +| Copied layer and channel names and item paths | 32 MiB in all (`MAX_METADATA_BYTES`) | +| Group nesting in `Image::tree` | 64 (`MAX_DEPTH`) | +| A gzipped file, inflated | 1 GiB (`MAX_INFLATED`) | diff --git a/crates/xcf/src/lib.rs b/crates/xcf/src/lib.rs new file mode 100644 index 000000000..fa226d1f1 --- /dev/null +++ b/crates/xcf/src/lib.rs @@ -0,0 +1,923 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! GIMP XCF (`.xcf`, `.xcf.gz`), read. Layout from the public specification +//! (, versions 0 to 26): a header, image +//! properties, then pointers to layer, channel and path structures; each drawable's pixels sit in +//! 64×64 tiles, uncompressed, RLE-coded or zlib-coded. Written from the specification and from +//! files GIMP 3.2 writes; no GIMP code. +//! +//! - [`Xcf::parse`] reads the header, the image properties and every layer's structure into an +//! [`Image`]: no pixel is decoded. +//! - [`Xcf::layer_pixels`], [`Xcf::layer_mask`] and [`Xcf::channel`] decode one drawable each +//! into [`Pixels`] (normalised `f32` samples), charging a byte budget the caller owns. +//! - [`Image::tree`] arranges the layers by their item paths. +//! - [`is_xcf`], [`is_gzipped_xcf`] and [`gunzip`] recognise and inflate files. +//! +//! Layer modes, colour tags and composite settings are kept as the numbers GIMP wrote; what +//! they mean to an application is the application's to decide. + +#[cfg(test)] +mod tests; +mod tiles; + +use std::cell::Cell; +use std::collections::HashMap; +use std::io::Read; + +/// Why an XCF file was not read. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The file is not XCF, is damaged, or exceeds a limit. + #[error("invalid XCF file: {0}")] + Invalid(String), + /// The caller's `keep_going` returned `false`. + #[error("cancelled")] + Cancelled, +} + +pub type Result = std::result::Result; + +fn invalid(message: impl Into) -> Error { + Error::Invalid(message.into()) +} + +const MAGIC: &[u8] = b"gimp xcf "; +/// Canvas and drawable side limit. +pub const MAX_SIDE: u32 = 300_000; +/// The canvas may hold at most this many bytes at 16 bytes per pixel (RGBA `f32`). Also the +/// budget a caller would usually give the decoders. +pub const MAX_PIXEL_BYTES: usize = 1 << 30; +/// Layers and groups in one image. +pub const MAX_LAYERS: usize = 8192; +/// Extra channels in one image. +pub const MAX_CHANNELS: usize = 1024; +/// Group nesting [`Image::tree`] builds. +pub const MAX_DEPTH: usize = 64; +/// Total copied layer and channel metadata (names and item paths). A single reused record with a large +/// name is bounded by [`MAX_STRING`], but repeated layer offsets referencing it must not multiply +/// into unbounded retained metadata. +pub const MAX_METADATA_BYTES: usize = 32 << 20; +/// A string field (layer names, parasite names). +pub const MAX_STRING: usize = 1 << 20; +/// A property payload. +pub const MAX_PAYLOAD: usize = 64 << 20; +/// The largest file a gzipped XCF may inflate to. +pub const MAX_INFLATED: usize = 1 << 30; +/// Highest version this reader was written against; newer files still open (unknown properties +/// are skipped by their length). +pub const KNOWN_VERSION: u32 = 26; +/// The side of a pixel tile. +pub const TILE: u32 = 64; + +/// `true` for an XCF file (by its signature). +pub fn is_xcf(bytes: &[u8]) -> bool { + bytes.starts_with(MAGIC) +} + +/// `true` for a gzip stream named `.xcf.gz` / `.xcfgz` (GIMP writes and reads compressed XCF). +pub fn is_gzipped_xcf(name: &str, bytes: &[u8]) -> bool { + let lower = name.rsplit(['/', '\\']).next().unwrap_or(name).to_ascii_lowercase(); + bytes.starts_with(&[0x1f, 0x8b]) && (lower.ends_with(".xcf.gz") || lower.ends_with(".xcfgz")) +} + +/// Inflates a gzipped XCF, refusing one that inflates past [`MAX_INFLATED`]. +pub fn gunzip(bytes: &[u8]) -> Result> { + let mut out = Vec::new(); + let mut dec = flate2::read::GzDecoder::new(bytes).take(MAX_INFLATED as u64 + 1); + dec.read_to_end(&mut out).map_err(|e| invalid(format!("gzip: {e}")))?; + if out.len() > MAX_INFLATED { + return Err(invalid("the compressed file inflates past the 1 GiB limit")); + } + Ok(out) +} + +/// How samples are stored. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Precision { + /// Bytes per sample: 1, 2, 4 or 8. + pub bytes: usize, + /// Floating point (16, 32 or 64 bit) rather than unsigned integer. + pub float: bool, + /// The values are linear light; otherwise gamma-encoded (the profile's own curve). + pub linear: bool, +} + +impl Precision { + /// 8-bit gamma integer: every file before version 4. + pub const GAMMA8: Precision = Precision { bytes: 1, float: false, linear: false }; + + /// The precision a header's code means in a file of `version` (the codes changed twice). + pub fn from_header(version: u32, code: u32) -> Result { + let p = |bytes, float, linear| Ok(Precision { bytes, float, linear }); + match version { + 0..=3 => Ok(Self::GAMMA8), + 4 => match code { + 0 => p(1, false, false), + 1 => p(2, false, false), + 2 => p(4, false, true), + 3 => p(2, true, true), + 4 => p(4, true, true), + c => Err(invalid(format!("unknown precision {c} in an XCF version 4 file"))), + }, + 5 | 6 => match code { + 100 => p(1, false, true), + 150 => p(1, false, false), + 200 => p(2, false, true), + 250 => p(2, false, false), + 300 => p(4, false, true), + 350 => p(4, false, false), + 400 => p(2, true, true), + 450 => p(2, true, false), + 500 => p(4, true, true), + 550 => p(4, true, false), + c => Err(invalid(format!("unknown precision {c} in an XCF version {version} file"))), + }, + _ => match code { + 100 => p(1, false, true), + 150 => p(1, false, false), + 200 => p(2, false, true), + 250 => p(2, false, false), + 300 => p(4, false, true), + 350 => p(4, false, false), + 500 => p(2, true, true), + 550 => p(2, true, false), + 600 => p(4, true, true), + 650 => p(4, true, false), + 700 => p(8, true, true), + 750 => p(8, true, false), + c => Err(invalid(format!("unknown precision {c}"))), + }, + } + } + + /// One sample, big-endian, as a normalised float (integers map to 0..1). + pub fn sample(self, b: &[u8]) -> f32 { + match (self.float, self.bytes, b) { + (false, 1, [v]) => f32::from(*v) / 255.0, + (false, 2, [a, b]) => f32::from(u16::from_be_bytes([*a, *b])) / 65535.0, + (false, 4, [a, b, c, d]) => (u32::from_be_bytes([*a, *b, *c, *d]) as f64 / f64::from(u32::MAX)) as f32, + (true, 2, [a, b]) => half_to_f32(u16::from_be_bytes([*a, *b])), + (true, 4, [a, b, c, d]) => f32::from_be_bytes([*a, *b, *c, *d]), + (true, 8, [a, b, c, d, e, f, g, h]) => f64::from_be_bytes([*a, *b, *c, *d, *e, *f, *g, *h]) as f32, + _ => 0.0, + } + } +} + +/// An IEEE 754 binary16 value as `f32` (exact: every half is representable). +fn half_to_f32(h: u16) -> f32 { + let sign = u32::from(h >> 15) << 31; + let exp = u32::from(h >> 10) & 0x1f; + let frac = u32::from(h) & 0x3ff; + let bits = match (exp, frac) { + (0, 0) => 0, + (0, f) => { + // Subnormal: shift the leading one up to the implicit position. + let shift = f.leading_zeros() - 21; + ((113 - shift) << 23) | ((f << shift) & 0x3ff) << 13 + } + (31, f) => 0x7f80_0000 | f << 13, + (e, f) => ((e + 112) << 23) | f << 13, + }; + f32::from_bits(sign | bits) +} + +/// How tiles are stored (`PROP_COMPRESSION`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Compression { + None, + Rle, + Zlib, +} + +/// The image's colour model (the header's base type). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BaseType { + Rgb, + Gray, + /// One colormap index per pixel. [`Pixels`] of an indexed image are expanded to RGB. + Indexed, +} + +/// Big-endian reads with bounds checks; every failure is an error, never a panic. +#[derive(Clone, Copy)] +struct Cursor<'a> { + b: &'a [u8], + pos: usize, + /// XCF 11 and later store pointers as 64-bit. + wide: bool, +} + +impl<'a> Cursor<'a> { + fn at(&self, pos: usize) -> Cursor<'a> { + Cursor { b: self.b, pos, wide: self.wide } + } + + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let end = self.pos.checked_add(n).ok_or_else(|| invalid("offset overflow"))?; + let s = self.b.get(self.pos..end).ok_or_else(|| invalid(format!("the file ends early (wanted {n} bytes at {})", self.pos)))?; + self.pos = end; + Ok(s) + } + + fn u32(&mut self) -> Result { + Ok(be_u32(self.take(4)?).unwrap_or(0)) + } + + fn ptr(&mut self) -> Result { + if self.wide { + let s = self.take(8)?; + let v = s.first_chunk::<8>().map(|b| u64::from_be_bytes(*b)).unwrap_or(0); + usize::try_from(v).map_err(|_| invalid("pointer does not fit in memory")) + } else { + self.u32().map(|v| v as usize) + } + } + + /// A STRING: `n+1` then `n` bytes and a NUL (0 for the empty string). + fn string(&mut self) -> Result { + Ok(String::from_utf8_lossy(self.string_bytes()?).into_owned()) + } + + /// The same bounded STRING read without allocating or decoding an unused field. + fn string_bytes(&mut self) -> Result<&'a [u8]> { + let n = self.u32()? as usize; + if n > MAX_STRING { + return Err(invalid(format!("a string of {n} bytes exceeds the limit"))); + } + let s = self.take(n)?; + Ok(s.strip_suffix(&[0]).unwrap_or(s)) + } + + fn skip(&mut self, n: usize) -> Result<()> { + self.take(n).map(|_| ()) + } +} + +/// One property record: its type and payload. +struct Prop<'a> { + kind: u32, + data: &'a [u8], +} + +/// Reads a property list, measuring known payloads whose length words GIMP writes incorrectly. +/// Ends at PROP_END (type 0). +fn props<'a>(c: &mut Cursor<'a>) -> Result>> { + let mut out = Vec::new(); + loop { + let kind = c.u32()?; + let len = c.u32()? as usize; + if kind == 0 { + return Ok(out); + } + // PROP_COLORMAP's length word was wrong in some releases (n+4 for 3n+4 bytes); its real + // size follows from the count. PROP_USER_UNIT's length is wrong in every release. + // PROP_LINK_LAYER's length can be four bytes short; read all its fields. + let len = match kind { + 1 => { + let n = c.at(c.pos).u32()? as usize; + 4usize.checked_add(n.checked_mul(3).ok_or_else(|| invalid("colormap overflow"))?).ok_or_else(|| invalid("colormap overflow"))? + } + 24 => user_unit_len(c)?, + 48 => link_layer_len(c)?, + _ => len, + }; + if len > MAX_PAYLOAD { + return Err(invalid(format!("property {kind} declares {len} bytes"))); + } + let data = c.take(len)?; + out.push(Prop { kind, data }); + if out.len() > 4096 { + return Err(invalid("too many properties")); + } + } +} + +/// A PROP_LINK_LAYER payload: flags, a path STRING, then the source width and height. +/// The dimensions are present for raster links too, where their values are unused. +fn link_layer_len(c: &Cursor<'_>) -> Result { + let mut p = c.at(c.pos); + p.skip(4)?; + p.string_bytes()?; + p.skip(8)?; + Ok(p.pos - c.pos) +} + +/// The real size of a PROP_USER_UNIT payload: a float, a count and three (XCF 21+) or five +/// strings. Measured without consuming. +fn user_unit_len(c: &Cursor<'_>) -> Result { + let mut p = c.at(c.pos); + p.skip(8)?; + for _ in 0..3 { + p.string_bytes()?; + } + // The two name forms of older files, when present: another string pair before the next + // property. Decide by whether a plausible property header follows. + let three = p.pos - c.pos; + if p.at(p.pos).u32().map(|k| k <= 64).unwrap_or(false) { + return Ok(three); + } + p.string_bytes()?; + p.string_bytes()?; + Ok(p.pos - c.pos) +} + +fn be_u32(b: &[u8]) -> Option { + b.first_chunk::<4>().map(|s| u32::from_be_bytes(*s)) +} + +fn be_i32(b: &[u8]) -> Option { + be_u32(b).map(|v| v as i32) +} + +fn be_f32(b: &[u8]) -> Option { + be_u32(b).map(f32::from_bits) +} + +/// The parasites of a PROP_PARASITES payload as (name, data), in first-seen order; a later +/// duplicate's data wins. A record cut short ends the list. +fn parasites(data: &[u8]) -> Vec<(String, &[u8])> { + let mut c = Cursor { b: data, pos: 0, wide: false }; + let mut out: Vec<(String, &[u8])> = Vec::new(); + // Name to output slot, so a payload of many distinct names is not rescanned per record. + let mut index: HashMap = HashMap::new(); + while c.pos < data.len() { + let Ok(name) = c.string() else { break }; + let (Ok(_flags), Ok(len)) = (c.u32(), c.u32()) else { break }; + let Ok(payload) = c.take(len as usize) else { break }; + match index.get(&name).and_then(|&i| out.get_mut(i)) { + Some(slot) => slot.1 = payload, + None => { + index.insert(name.clone(), out.len()); + out.push((name, payload)); + } + } + } + out +} + +/// A guide's direction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Orientation { + /// A horizontal line: `position` is a y coordinate. + Horizontal, + /// A vertical line: `position` is an x coordinate. + Vertical, +} + +/// A guide (`PROP_GUIDES`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Guide { + pub position: i32, + pub orientation: Orientation, +} + +/// A layer structure, with its properties as GIMP wrote them. +#[derive(Debug, Clone, PartialEq)] +pub struct Layer { + /// The size its pixels are declared to have. + pub width: u32, + pub height: u32, + /// The layer type: 0 RGB, 1 RGBA, 2 gray, 3 gray + alpha, 4 indexed, 5 indexed + alpha. + pub layer_type: u32, + pub name: String, + /// `PROP_FLOATING_SELECTION`: the layer is a floating selection, not part of the stack. + pub floating_selection: bool, + /// 0 to 1: `PROP_FLOAT_OPACITY` when present (it wins whatever the order), else + /// `PROP_OPACITY` / 255. + pub opacity: f32, + /// `PROP_MODE`: GIMP's layer mode number (0 to 22 legacy, 23 and up since 2.10; 61 is + /// pass-through). + pub mode: u32, + pub visible: bool, + /// `PROP_LOCK_ALPHA`. + pub lock_alpha: bool, + /// `PROP_LOCK_CONTENT`. + pub lock_content: bool, + /// `PROP_LOCK_POSITION`. + pub lock_position: bool, + /// `PROP_APPLY_MASK`: the layer mask is in effect. `true` when the property is absent. + pub apply_mask: bool, + /// `PROP_OFFSETS`: where the layer's (and its mask's) top-left pixel sits on the canvas. + pub x: i32, + pub y: i32, + /// `PROP_TEXT_LAYER_FLAGS` or a `gimp-text-layer` parasite: a text layer (its pixels are the + /// rendered text). + pub text: bool, + /// `PROP_VECTOR_LAYER`. + pub vector: bool, + /// `PROP_LINK_LAYER`. + pub link: bool, + /// `PROP_GROUP_ITEM`: a group; its members follow it with longer item paths. + pub group: bool, + /// `PROP_GROUP_ITEM_FLAGS` bit 0. `true` when the property is absent. + pub expanded: bool, + /// `PROP_ITEM_PATH`: the layer's index at each level of the tree; empty at the top level of + /// a file without groups. See [`Image::tree`]. + pub item_path: Vec, + /// `PROP_COLOR_TAG`: 0 none, 1 blue, 2 green, 3 yellow, 4 orange, 5 brown, 6 red, 7 violet, + /// 8 gray. + pub color_tag: u32, + /// `PROP_COMPOSITE_MODE`: 0 or ±1 union, ±2 clip to backdrop, ±3 clip to layer, + /// ±4 intersection (negative: GIMP's automatic choice). + pub composite_mode: i32, + /// `PROP_COMPOSITE_SPACE`: ±1 linear, ±2 perceptual. + pub composite_space: i32, + /// `PROP_BLEND_SPACE`: ±1 linear, ±2 perceptual. + pub blend_space: i32, + /// The layer has a `gimphoto-clipped` parasite (a clipping-mask convention). + pub clipped: bool, + /// Non-destructive layer effects (version 20 and later); they are not read. + pub effects: usize, + hierarchy: usize, + mask: usize, +} + +impl Layer { + /// Does the layer type carry an alpha channel? + pub fn has_alpha(&self) -> bool { + matches!(self.layer_type, 1 | 3 | 5) + } + /// Does the layer point at pixels? Groups usually do not. + pub fn has_pixels(&self) -> bool { + self.hierarchy != 0 + } + /// Does the layer have a mask? + pub fn has_mask(&self) -> bool { + self.mask != 0 + } +} + +/// A channel structure: a layer mask, the selection or an extra channel. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Channel { + pub name: String, + /// `PROP_SELECTION`: this channel is the image's selection. + pub selection: bool, +} + +/// Pixel data of a drawable: `channels` normalised samples per pixel, row-major. A layer has 1 +/// (gray), 2 (gray, alpha), 3 (RGB) or 4 (RGBA) channels; a channel has 1. Indexed pixels are +/// expanded to RGB through the colormap. +#[derive(Debug, Clone, PartialEq)] +pub struct Pixels { + pub width: u32, + pub height: u32, + pub channels: usize, + pub data: Vec, +} + +/// A layer in [`Image::tree`]. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct TreeNode { + /// Index into [`Image::layers`]. + pub layer: usize, + /// The group's members, topmost first. + pub children: Vec, +} + +/// The layers arranged by their item paths. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct Tree { + /// The top level, topmost first. + pub roots: Vec, + /// Layers (by index) nested deeper than [`MAX_DEPTH`], moved to the top level. + pub too_deep: Vec, + /// Layers (by index) whose item path leads to no group, put at the top level. + pub misplaced: Vec, +} + +/// The header and structure of an XCF file. +#[derive(Debug, Clone, PartialEq)] +pub struct Image { + /// The file version: 0 to [`KNOWN_VERSION`], or newer. + pub version: u32, + /// Canvas size, 1 to [`MAX_SIDE`]. + pub width: u32, + pub height: u32, + pub base_type: BaseType, + pub precision: Precision, + pub compression: Compression, + /// `PROP_COLORMAP`: at most 256 RGB entries. + pub colormap: Vec<[u8; 3]>, + /// `PROP_RESOLUTION`: horizontal and vertical pixels per inch, as stored. + pub resolution: Option<(f32, f32)>, + /// `PROP_GUIDES`. Files before version 15 could hold stray negative guides; those are + /// left out, as are guides of an unknown orientation. + pub guides: Vec, + /// The `icc-profile` parasite, when not empty. + pub icc_profile: Option>, + /// The `gimp-comment` parasite. + pub comment: Option, + /// Paths the file holds (they are not read). + pub path_count: usize, + /// Every layer structure in file order, topmost first, floating selections included. + pub layers: Vec, + /// The number of channel structures (the selection and the extras); read each with + /// [`Xcf::channel`]. + pub channel_count: usize, +} + +impl Image { + /// Arranges the layers by their item paths, skipping floating selections. A layer without a + /// path sits at the top level in file order; one whose path leads to no group (at any + /// depth) is put at the top level, so later siblings keep their indices. + pub fn tree(&self) -> Tree { + let mut tree = Tree::default(); + let stack = self.layers.iter().enumerate().filter(|(_, l)| !l.floating_selection); + for (position, (index, layer)) in stack.enumerate() { + // Default path: the item's position at the top level, as a file without groups has it. + let default = [position as u32]; + let mut path: &[u32] = if layer.item_path.is_empty() { &default } else { &layer.item_path }; + if path.len() > MAX_DEPTH { + tree.too_deep.push(index); + path = path.get(..1).unwrap_or(path); + } + let (parents, _) = path.split_at(path.len().saturating_sub(1)); + let mut probe = &tree.roots; + let reachable = parents.iter().all(|&i| match probe.get(i as usize) { + Some(n) if self.layers.get(n.layer).is_some_and(|l| l.group) => { + probe = &n.children; + true + } + _ => false, + }); + let group = if reachable { parents } else { &[] }; + if !reachable { + tree.misplaced.push(index); + } + if let Some(level) = group.iter().try_fold(&mut tree.roots, |level, &i| level.get_mut(i as usize).map(|n| &mut n.children)) { + level.push(TreeNode { layer: index, children: Vec::new() }); + } + } + tree + } +} + +/// A parsed XCF file: its [`Image`] and the pixels behind it. +pub struct Xcf<'a> { + c: Cursor<'a>, + image: Image, + channels: Vec, + /// What is left of [`MAX_METADATA_BYTES`] after the layers; channel names are charged too. + meta_budget: Cell, +} + +impl<'a> Xcf<'a> { + /// Reads the header, the image properties and every layer structure of an XCF file + /// (already inflated when it was gzipped). + pub fn parse(bytes: &'a [u8]) -> Result> { + if !is_xcf(bytes) { + return Err(invalid("not an XCF file")); + } + let mut c = Cursor { b: bytes, pos: MAGIC.len(), wide: false }; + let version = match c.take(5)? { + b"file\0" => 0, + [b'v', a, b, d, 0] if a.is_ascii_digit() && b.is_ascii_digit() && d.is_ascii_digit() => { + u32::from(a - b'0') * 100 + u32::from(b - b'0') * 10 + u32::from(d - b'0') + } + _ => return Err(invalid("unknown XCF version tag")), + }; + c.wide = version >= 11; + let (width, height, base_type) = (c.u32()?, c.u32()?, c.u32()?); + if width == 0 || height == 0 || width > MAX_SIDE || height > MAX_SIDE { + return Err(invalid(format!("canvas {width}x{height} must be 1 to {MAX_SIDE} pixels a side"))); + } + if (width as usize).saturating_mul(height as usize).saturating_mul(16) > MAX_PIXEL_BYTES { + return Err(invalid("the canvas exceeds the 1 GiB pixel limit")); + } + let precision = if version >= 4 { Precision::from_header(version, c.u32()?)? } else { Precision::GAMMA8 }; + let base_type = match base_type { + 0 => BaseType::Rgb, + 1 => BaseType::Gray, + 2 => BaseType::Indexed, + t => return Err(invalid(format!("unknown image type {t}"))), + }; + let mut image = Image { + version, + width, + height, + base_type, + precision, + compression: Compression::None, + colormap: Vec::new(), + resolution: None, + guides: Vec::new(), + icc_profile: None, + comment: None, + path_count: 0, + layers: Vec::new(), + channel_count: 0, + }; + image_props(&mut c, &mut image)?; + + let pointers = |c: &mut Cursor<'_>, max: usize, what: &str| -> Result> { + let mut out = Vec::new(); + loop { + let p = c.ptr()?; + if p == 0 { + return Ok(out); + } + out.push(p); + if out.len() > max { + return Err(invalid(format!("more than {max} {what}"))); + } + } + }; + let layer_ptrs = pointers(&mut c, MAX_LAYERS, "layers")?; + let channels = pointers(&mut c, MAX_CHANNELS, "channels")?; + if version >= 18 { + let mut n = 0; + while c.ptr().ok().is_some_and(|p| p != 0) { + n += 1; + if n > MAX_CHANNELS { + break; + } + } + image.path_count += n; + } + image.channel_count = channels.len(); + let c = c.at(0); + let mut meta_budget = MAX_METADATA_BYTES; + for ptr in layer_ptrs { + let layer = layer(c.at(ptr), &image, &mut meta_budget)?; + image.layers.push(layer); + } + Ok(Xcf { c, image, channels, meta_budget: Cell::new(meta_budget) }) + } + + /// The file's header and structure. + pub fn image(&self) -> &Image { + &self.image + } + + /// Decodes the pixels of layer `index` of [`Image::layers`]; `None` for a layer that has + /// none (a group). The decoded samples (4 bytes each) are taken from `budget` before + /// anything is allocated. `keep_going` is asked every 64 tiles. + pub fn layer_pixels(&self, index: usize, budget: &mut usize, keep_going: &dyn Fn() -> bool) -> Result> { + let layer = self.image.layers.get(index).ok_or_else(|| invalid("no such layer"))?; + if layer.hierarchy == 0 { + return Ok(None); + } + let color = if self.image.base_type == BaseType::Rgb { 3 } else { 1 }; + let source_channels = color + usize::from(layer.has_alpha()); + let indexed = self.image.base_type == BaseType::Indexed; + let px = self.hierarchy(layer.hierarchy, source_channels, indexed, budget, keep_going)?; + if (px.width, px.height) != (layer.width, layer.height) { + let (name, lw, lh) = (&layer.name, layer.width, layer.height); + return Err(invalid(format!("layer '{name}' pixels are {}x{} where {lw}x{lh} were declared", px.width, px.height))); + } + Ok(Some(px)) + } + + /// Decodes the mask of layer `index`: one channel, placed at the layer's offset. `None` + /// for a layer without one. + pub fn layer_mask(&self, index: usize, budget: &mut usize, keep_going: &dyn Fn() -> bool) -> Result> { + let layer = self.image.layers.get(index).ok_or_else(|| invalid("no such layer"))?; + if layer.mask == 0 { + return Ok(None); + } + self.channel_at(layer.mask, budget, keep_going).map(Some) + } + + /// Decodes channel `index` (below [`Image::channel_count`]): the selection or an extra + /// channel, the size of the canvas. + pub fn channel(&self, index: usize, budget: &mut usize, keep_going: &dyn Fn() -> bool) -> Result<(Channel, Pixels)> { + let ptr = *self.channels.get(index).ok_or_else(|| invalid("no such channel"))?; + self.channel_at(ptr, budget, keep_going) + } + + fn channel_at(&self, ptr: usize, budget: &mut usize, keep_going: &dyn Fn() -> bool) -> Result<(Channel, Pixels)> { + let mut c = self.c.at(ptr); + let (_w, _h) = (c.u32()?, c.u32()?); + let name = c.string()?; + // As for layers: repeated channel pointers must not retain unbounded copies of one name. + self.meta_budget.set(self.meta_budget.get().checked_sub(name.len()).ok_or_else(|| invalid("channel metadata exceeds the budget"))?); + let selection = props(&mut c)?.iter().any(|p| p.kind == 4); + let hptr = c.ptr()?; + // Channels are always one plain sample per pixel, even in indexed images. + let px = self.hierarchy(hptr, 1, false, budget, keep_going)?; + Ok((Channel { name, selection }, px)) + } + + /// Reads a hierarchy: the first level's tiles into floats, `source_channels` as stored + /// (indexed pixels expand to RGB here). + fn hierarchy(&self, hptr: usize, source_channels: usize, indexed: bool, budget: &mut usize, keep_going: &dyn Fn() -> bool) -> Result { + let precision = self.image.precision; + let mut c = self.c.at(hptr); + let (w, h, bpp) = (c.u32()?, c.u32()?, c.u32()? as usize); + if w == 0 || h == 0 || w > MAX_SIDE || h > MAX_SIDE { + return Err(invalid(format!("drawable size {w}x{h} is out of range"))); + } + let expected_bpp = source_channels.checked_mul(precision.bytes).ok_or_else(|| invalid("bpp overflow"))?; + if bpp != expected_bpp { + return Err(invalid(format!("{bpp} bytes per pixel where {expected_bpp} were expected"))); + } + let channels = if indexed { source_channels + 2 } else { source_channels }; + let count = (w as usize).checked_mul(h as usize).and_then(|n| n.checked_mul(channels)).ok_or_else(|| invalid("drawable overflow"))?; + let bytes = count.checked_mul(4).ok_or_else(|| invalid("drawable overflow"))?; + *budget = budget.checked_sub(bytes).ok_or_else(|| invalid("the layers exceed the pixel limit"))?; + let lptr = c.ptr()?; + let mut l = self.c.at(lptr); + if (l.u32()?, l.u32()?) != (w, h) { + return Err(invalid("the level does not match its hierarchy")); + } + let across = w.div_ceil(TILE) as usize; + let down = h.div_ceil(TILE) as usize; + let mut tiles = Vec::with_capacity(across * down); + for _ in 0..across * down { + let p = l.ptr()?; + if p == 0 { + return Err(invalid("the tile list ends before every tile")); + } + tiles.push(p); + } + let mut data = vec![0.0f32; count]; + let stride = w as usize * channels; + let side = TILE as usize; + for (i, &at) in tiles.iter().enumerate() { + if i % 64 == 0 && !keep_going() { + return Err(Error::Cancelled); + } + let (tx, ty) = (i % across, i / across); + let (tw, th) = ((w as usize - tx * side).min(side), (h as usize - ty * side).min(side)); + let end = tiles.get(i + 1).copied().filter(|&n| n > at).unwrap_or(self.c.b.len()); + let input = self.c.b.get(at..end.min(self.c.b.len())).ok_or_else(|| invalid("a tile pointer is outside the file"))?; + let raw = tiles::decode(input, self.image.compression, bpp, tw * th)?; + for (py, row) in raw.chunks_exact(tw * bpp).enumerate() { + let start = (ty * side + py) * stride + tx * side * channels; + let Some(out) = data.get_mut(start..start + tw * channels) else { continue }; + for (pixel, dst) in row.chunks_exact(bpp).zip(out.chunks_exact_mut(channels)) { + if indexed { + let idx = pixel.first().copied().unwrap_or(0); + let rgb = self.image.colormap.get(usize::from(idx)).copied().unwrap_or([idx; 3]); + let alpha = pixel.get(1).copied(); + for (d, v) in dst.iter_mut().zip(rgb.into_iter().chain(alpha)) { + *d = f32::from(v) / 255.0; + } + } else { + for (d, s) in dst.iter_mut().zip(pixel.chunks_exact(precision.bytes)) { + *d = precision.sample(s); + } + } + } + } + } + Ok(Pixels { width: w, height: h, channels, data }) + } +} + +fn image_props(c: &mut Cursor<'_>, image: &mut Image) -> Result<()> { + for p in &props(c)? { + match p.kind { + 1 => { + let n = be_u32(p.data).unwrap_or(0) as usize; + image.colormap = p.data.get(4..).unwrap_or(&[]).as_chunks::<3>().0.iter().take(n.min(256)).copied().collect(); + } + 17 => { + image.compression = match p.data.first() { + Some(0) => Compression::None, + Some(1) => Compression::Rle, + Some(2) => Compression::Zlib, + Some(k) => return Err(invalid(format!("unknown compression {k}"))), + None => return Err(invalid("empty compression property")), + } + } + 18 => { + for g in p.data.as_chunks::<5>().0 { + let position = i32::from_be_bytes([g[0], g[1], g[2], g[3]]); + let orientation = match g[4] { + 1 => Orientation::Horizontal, + 2 => Orientation::Vertical, + _ => continue, + }; + if position >= 0 || image.version >= 15 { + image.guides.push(Guide { position, orientation }); + } + } + } + 19 => { + if let (Some(x), Some(y)) = (be_f32(p.data), p.data.get(4..).and_then(be_f32)) { + image.resolution = Some((x, y)); + } + } + 21 => { + for (name, data) in parasites(p.data) { + match name.as_str() { + "icc-profile" if !data.is_empty() => image.icc_profile = Some(data.to_vec()), + "gimp-comment" => image.comment = Some(String::from_utf8_lossy(data).trim_end_matches('\0').to_string()), + _ => {} + } + } + } + 23 => image.path_count += be_u32(p.data.get(4..).unwrap_or(&[])).unwrap_or(0) as usize, + 25 => image.path_count += be_u32(p.data.get(8..).unwrap_or(&[])).unwrap_or(0) as usize, + _ => {} + } + } + Ok(()) +} + +/// Reads a layer structure at `c`. +fn layer(mut c: Cursor<'_>, image: &Image, meta_budget: &mut usize) -> Result { + let (width, height, layer_type) = (c.u32()?, c.u32()?, c.u32()?); + let name = c.string()?; + // Charge the owned name against the image-wide metadata budget: repeated layer offsets + // referencing one large name must not retain unbounded copies. + *meta_budget = meta_budget.checked_sub(name.len()).ok_or_else(|| invalid("layer metadata exceeds the budget"))?; + let lprops = props(&mut c)?; + let hierarchy = c.ptr()?; + let mask = c.ptr()?; + let mut effects = 0usize; + if image.version >= 20 { + while c.ptr()? != 0 { + effects += 1; + if effects > 1024 { + return Err(invalid("too many layer effects")); + } + } + } + let type_rgb = match layer_type { + 0 | 1 => true, + 2..=5 => false, + k => return Err(invalid(format!("unknown layer type {k} on '{name}'"))), + }; + if type_rgb != (image.base_type != BaseType::Gray) && image.base_type != BaseType::Indexed { + return Err(invalid(format!("layer '{name}' does not match the image's colour model"))); + } + let mut l = Layer { + width, + height, + layer_type, + name, + floating_selection: false, + opacity: 1.0, + mode: 0, + visible: true, + lock_alpha: false, + lock_content: false, + lock_position: false, + apply_mask: true, + x: 0, + y: 0, + text: false, + vector: false, + link: false, + group: false, + expanded: true, + item_path: Vec::new(), + color_tag: 0, + composite_mode: 0, + composite_space: 0, + blend_space: 0, + clipped: false, + effects, + hierarchy, + mask, + }; + // PROP_FLOAT_OPACITY wins over PROP_OPACITY whatever their order in the file. + let mut float_opacity = None; + for p in &lprops { + let v = be_u32(p.data).unwrap_or(0); + match p.kind { + 5 => l.floating_selection = true, + 6 => l.opacity = (v.min(255) as f32) / 255.0, + 33 => float_opacity = be_f32(p.data).filter(|v| v.is_finite()).map(|v| v.clamp(0.0, 1.0)).or(float_opacity), + 7 => l.mode = v, + 8 => l.visible = v != 0, + 10 => l.lock_alpha = v != 0, + 11 => l.apply_mask = v != 0, + 15 => { + l.x = be_i32(p.data).unwrap_or(0); + l.y = p.data.get(4..).and_then(be_i32).unwrap_or(0); + } + 26 => l.text = true, + 28 => l.lock_content = v != 0, + 29 => l.group = true, + 30 => { + l.item_path = p.data.as_chunks::<4>().0.iter().map(|s| u32::from_be_bytes(*s)).collect(); + *meta_budget = + meta_budget.checked_sub(l.item_path.len().saturating_mul(4)).ok_or_else(|| invalid("layer metadata exceeds the budget"))?; + } + 31 => l.expanded = v & 1 != 0, + 32 => l.lock_position = v != 0, + 34 => l.color_tag = v, + 35 => l.composite_mode = be_i32(p.data).unwrap_or(0), + 36 => l.composite_space = be_i32(p.data).unwrap_or(0), + 37 => l.blend_space = be_i32(p.data).unwrap_or(0), + 47 => l.vector = true, + 48 => l.link = true, + 21 => { + for (pname, _) in parasites(p.data) { + match pname.as_str() { + "gimphoto-clipped" => l.clipped = true, + "gimp-text-layer" => l.text = true, + _ => {} + } + } + } + _ => {} + } + } + l.opacity = float_opacity.unwrap_or(l.opacity); + Ok(l) +} diff --git a/crates/xcf/src/tests.rs b/crates/xcf/src/tests.rs new file mode 100644 index 000000000..5ec45620c --- /dev/null +++ b/crates/xcf/src/tests.rs @@ -0,0 +1,1120 @@ +//! Synthetic XCF files (written here, from the specification) for the reader: structure, +//! every compression, precisions, groups, masks, channels, limits and malformed input. + +use super::tiles::rle_decode; +use super::*; + +/// A small XCF writer for tests: version 11+ layout (64-bit pointers) by default. +pub(super) struct W { + pub out: Vec, + pub wide: bool, +} + +impl W { + pub fn new(version: u32) -> Self { + let mut w = W { out: Vec::new(), wide: version >= 11 }; + w.out.extend_from_slice(b"gimp xcf "); + if version == 0 { + w.out.extend_from_slice(b"file\0"); + } else { + w.out.extend_from_slice(format!("v{version:03}\0").as_bytes()); + } + w + } + pub fn u32(&mut self, v: u32) { + self.out.extend_from_slice(&v.to_be_bytes()); + } + pub fn ptr(&mut self, v: usize) { + if self.wide { + self.out.extend_from_slice(&(v as u64).to_be_bytes()); + } else { + self.u32(v as u32); + } + } + pub fn string(&mut self, s: &str) { + if s.is_empty() { + self.u32(0); + } else { + self.u32(s.len() as u32 + 1); + self.out.extend_from_slice(s.as_bytes()); + self.out.push(0); + } + } + pub fn prop(&mut self, kind: u32, payload: &[u8]) { + self.u32(kind); + self.u32(payload.len() as u32); + self.out.extend_from_slice(payload); + } + pub fn prop_u32(&mut self, kind: u32, v: u32) { + self.prop(kind, &v.to_be_bytes()); + } + pub fn end(&mut self) { + self.u32(0); + self.u32(0); + } + /// Reserves a pointer slot, returning where to patch it. + pub fn slot(&mut self) -> usize { + let at = self.out.len(); + self.ptr(0); + at + } + pub fn patch(&mut self, slot: usize, target: usize) { + if self.wide { + self.out[slot..slot + 8].copy_from_slice(&(target as u64).to_be_bytes()); + } else { + self.out[slot..slot + 4].copy_from_slice(&(target as u32).to_be_bytes()); + } + } +} + +/// RLE-encodes one tile of `bpp`-byte pixels, plane by plane, with runs and literals. +pub(super) fn rle(pixels: &[u8], bpp: usize) -> Vec { + let n = pixels.len() / bpp; + let mut out = Vec::new(); + for b in 0..bpp { + let plane: Vec = (0..n).map(|i| pixels[i * bpp + b]).collect(); + let mut i = 0; + while i < n { + let v = plane[i]; + let mut run = 1; + while i + run < n && plane[i + run] == v && run < 65535 { + run += 1; + } + if run >= 2 { + if run <= 127 { + out.push((run - 1) as u8); + } else { + out.push(127); + out.push((run / 256) as u8); + out.push((run % 256) as u8); + } + out.push(v); + i += run; + } else { + let start = i; + while i < n && (i + 1 >= n || plane[i + 1] != plane[i]) && i - start < 127 { + i += 1; + } + let len = i - start; + out.push((256 - len) as u8); + out.extend_from_slice(&plane[start..i]); + } + } + } + out +} + +/// Writes a hierarchy for `w`×`h` pixels of `bpp` bytes (`data` row-major), returning its offset. +pub(super) fn hierarchy(w: &mut W, width: u32, height: u32, bpp: usize, data: &[u8], compression: Compression) -> usize { + let at = w.out.len(); + w.u32(width); + w.u32(height); + w.u32(bpp as u32); + let level_slot = w.slot(); + w.ptr(0); + let level = w.out.len(); + w.patch(level_slot, level); + w.u32(width); + w.u32(height); + let across = width.div_ceil(TILE) as usize; + let down = height.div_ceil(TILE) as usize; + let slots: Vec = (0..across * down).map(|_| w.slot()).collect(); + w.ptr(0); + for (i, slot) in slots.into_iter().enumerate() { + let (tx, ty) = ((i % across) as u32, (i / across) as u32); + let (tw, th) = ((width - tx * TILE).min(TILE), (height - ty * TILE).min(TILE)); + let mut tile = Vec::new(); + for py in 0..th { + for px in 0..tw { + let src = ((ty * TILE + py) as usize * width as usize + (tx * TILE + px) as usize) * bpp; + tile.extend_from_slice(&data[src..src + bpp]); + } + } + let encoded = match compression { + Compression::None => tile, + Compression::Rle => rle(&tile, bpp), + Compression::Zlib => { + use std::io::Write; + let mut e = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(&tile).unwrap(); + e.finish().unwrap() + } + }; + let here = w.out.len(); + w.patch(slot, here); + w.out.extend_from_slice(&encoded); + } + at +} + +/// A layer to write. +pub(super) struct TestLayer { + pub name: &'static str, + pub width: u32, + pub height: u32, + pub kind: u32, + pub x: i32, + pub y: i32, + pub opacity: f32, + pub visible: bool, + pub mode: u32, + pub props: Vec<(u32, Vec)>, + pub data: Vec, + pub mask: Option>, + pub apply_mask: bool, +} + +impl TestLayer { + pub fn rgba(name: &'static str, width: u32, height: u32, data: Vec) -> Self { + TestLayer { + name, + width, + height, + kind: 1, + x: 0, + y: 0, + opacity: 1.0, + visible: true, + mode: 28, + props: Vec::new(), + data, + mask: None, + apply_mask: true, + } + } +} + +pub(super) struct TestImage { + pub version: u32, + pub width: u32, + pub height: u32, + pub base_type: u32, + pub precision: u32, + pub compression: Compression, + pub image_props: Vec<(u32, Vec)>, + pub layers: Vec, + pub channels: Vec<(&'static str, Vec, bool)>, +} + +impl TestImage { + pub fn rgb8(width: u32, height: u32) -> Self { + TestImage { + version: 11, + width, + height, + base_type: 0, + precision: 150, + compression: Compression::Rle, + image_props: Vec::new(), + layers: Vec::new(), + channels: Vec::new(), + } + } + + pub fn write(&self) -> Vec { + let mut w = W::new(self.version); + w.u32(self.width); + w.u32(self.height); + w.u32(self.base_type); + if self.version >= 4 { + w.u32(self.precision); + } + let comp = match self.compression { + Compression::None => 0u8, + Compression::Rle => 1, + Compression::Zlib => 2, + }; + w.prop(17, &[comp]); + for (k, v) in &self.image_props { + w.prop(*k, v); + } + w.end(); + let layer_slots: Vec = self.layers.iter().map(|_| w.slot()).collect(); + w.ptr(0); + let channel_slots: Vec = self.channels.iter().map(|_| w.slot()).collect(); + w.ptr(0); + if self.version >= 18 { + w.ptr(0); + } + let bytes = match self.precision { + 100 | 150 => 1, + 200 | 250 | 500 | 550 => 2, + 300 | 350 | 600 | 650 => 4, + _ => 8, + }; + for (layer, slot) in self.layers.iter().zip(layer_slots) { + let at = w.out.len(); + w.patch(slot, at); + w.u32(layer.width); + w.u32(layer.height); + w.u32(layer.kind); + w.string(layer.name); + w.prop_u32(6, (layer.opacity * 255.0).round() as u32); + w.prop(33, &layer.opacity.to_be_bytes()); + w.prop_u32(8, u32::from(layer.visible)); + w.prop_u32(7, layer.mode); + let mut off = Vec::new(); + off.extend_from_slice(&layer.x.to_be_bytes()); + off.extend_from_slice(&layer.y.to_be_bytes()); + w.prop(15, &off); + if layer.mask.is_some() { + w.prop_u32(11, u32::from(layer.apply_mask)); + } + for (k, v) in &layer.props { + w.prop(*k, v); + } + w.end(); + let hslot = w.slot(); + let mslot = w.slot(); + if self.version >= 20 { + w.ptr(0); + } + let channels = match layer.kind { + 0 => 3, + 1 => 4, + 2 | 4 => 1, + _ => 2, + }; + if !layer.data.is_empty() { + let h = hierarchy(&mut w, layer.width, layer.height, channels * bytes, &layer.data, self.compression); + w.patch(hslot, h); + } + if let Some(mask) = &layer.mask { + let at = w.out.len(); + w.patch(mslot, at); + w.u32(layer.width); + w.u32(layer.height); + w.string("mask"); + w.end(); + let hs = w.slot(); + let h = hierarchy(&mut w, layer.width, layer.height, bytes, mask, self.compression); + w.patch(hs, h); + } + } + for ((name, data, selection), slot) in self.channels.iter().zip(channel_slots) { + let at = w.out.len(); + w.patch(slot, at); + w.u32(self.width); + w.u32(self.height); + w.string(name); + if *selection { + w.prop(4, &[]); + } + w.end(); + let hs = w.slot(); + let h = hierarchy(&mut w, self.width, self.height, bytes, data, self.compression); + w.patch(hs, h); + } + w.out + } +} + +const GO: &dyn Fn() -> bool = &|| true; + +fn open(bytes: &[u8]) -> Result> { + Xcf::parse(bytes) +} + +/// Parses and decodes everything, as an application would. +fn read_all(bytes: &[u8]) -> Result { + let xcf = Xcf::parse(bytes)?; + let mut budget = MAX_PIXEL_BYTES; + let mut samples = 0; + for i in 0..xcf.image().layers.len() { + samples += xcf.layer_pixels(i, &mut budget, GO)?.map_or(0, |p| p.data.len()); + samples += xcf.layer_mask(i, &mut budget, GO)?.map_or(0, |(_, p)| p.data.len()); + } + for i in 0..xcf.image().channel_count { + samples += xcf.channel(i, &mut budget, GO)?.1.data.len(); + } + xcf.image().tree(); + Ok(samples) +} + +fn rgba(w: u32, h: u32, f: impl Fn(u32, u32) -> [u8; 4]) -> Vec { + let f = &f; + (0..h).flat_map(|y| (0..w).flat_map(move |x| f(x, y))).collect() +} + +fn pixels(xcf: &Xcf<'_>, layer: usize) -> Pixels { + xcf.layer_pixels(layer, &mut MAX_PIXEL_BYTES.clone(), GO).unwrap().unwrap() +} + +/// The samples of pixel (x, y), in the drawable's own coordinates. +fn at(px: &Pixels, x: u32, y: u32) -> &[f32] { + let i = (y * px.width + x) as usize * px.channels; + &px.data[i..i + px.channels] +} + +fn path(indices: &[u32]) -> (u32, Vec) { + (30, indices.iter().flat_map(|v| v.to_be_bytes()).collect()) +} + +fn names(image: &Image, nodes: &[TreeNode]) -> Vec { + nodes.iter().map(|n| image.layers[n.layer].name.clone()).collect() +} + +#[test] +fn a_layer_decodes_with_every_compression() { + for compression in [Compression::None, Compression::Rle, Compression::Zlib] { + let mut img = TestImage::rgb8(70, 66); + img.compression = compression; + img.layers.push(TestLayer::rgba("A", 70, 66, rgba(70, 66, |x, y| [x as u8 * 3, y as u8 * 2, (x + y) as u8, 255 - x as u8]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let image = xcf.image(); + assert_eq!((image.width, image.height, image.version, image.compression), (70, 66, 11, compression)); + assert_eq!((image.base_type, image.precision), (BaseType::Rgb, Precision::GAMMA8)); + assert_eq!(image.layers.len(), 1, "{compression:?}"); + let px = pixels(&xcf, 0); + assert_eq!((px.width, px.height, px.channels), (70, 66, 4)); + for (x, y) in [(0, 0), (63, 0), (64, 0), (69, 65), (10, 64)] { + let want = [x as f32 * 3.0, y as f32 * 2.0, (x + y) as f32, 255.0 - x as f32].map(|v| v / 255.0); + assert_eq!(at(&px, x, y), want, "{compression:?} at {x},{y}"); + } + } +} + +#[test] +fn every_truncation_errors_never_panics() { + let mut img = TestImage::rgb8(20, 20); + let mut l = TestLayer::rgba("A", 20, 20, rgba(20, 20, |x, y| [x as u8, y as u8, 0, 255])); + l.mask = Some(vec![9; 400]); + img.layers.push(l); + img.channels.push(("Extra", vec![3; 400], false)); + let bytes = img.write(); + for cut in 0..bytes.len() { + assert!(read_all(&bytes[..cut]).is_err(), "cut at {cut}"); + } + assert_eq!(read_all(&bytes), Ok(1600 + 400 + 400)); +} + +#[test] +fn corrupted_files_never_panic() { + let mut img = TestImage::rgb8(70, 20); + let mut group = TestLayer::rgba("Group", 1, 1, Vec::new()); + group.props = vec![(29, vec![])]; + img.layers.push(group); + let mut l = TestLayer::rgba("A", 70, 20, rgba(70, 20, |x, y| [x as u8, y as u8, 0, 255])); + l.props = vec![path(&[0, 0])]; + l.mask = Some(vec![9; 1400]); + img.layers.push(l); + img.channels.push(("Sel", vec![3; 1400], true)); + for compression in [Compression::None, Compression::Rle, Compression::Zlib] { + img.compression = compression; + let bytes = img.write(); + assert!(read_all(&bytes).is_ok()); + let mut state = 0x9e37_79b9_u32; + for _ in 0..2000 { + let mut b = bytes.clone(); + for _ in 0..3 { + state ^= state << 13; + state ^= state >> 17; + state ^= state << 5; + let i = state as usize % b.len(); + b[i] ^= (state >> 24) as u8 | 1; + } + let _ = read_all(&b); + } + } +} + +#[test] +fn properties_offsets_opacity_mode_visibility_locks_and_tags() { + let mut img = TestImage::rgb8(30, 20); + let mut l = TestLayer::rgba("Top", 10, 8, rgba(10, 8, |x, y| [x as u8 * 20, y as u8 * 30, 0, 200])); + l.x = -3; + l.y = 15; + l.opacity = 0.4; + l.visible = false; + l.mode = 30; // Multiply + let one = 1u32.to_be_bytes().to_vec(); + l.props = vec![ + (10, one.clone()), + (28, one.clone()), + (32, one.clone()), + (34, 2u32.to_be_bytes().to_vec()), + (35, (-2i32).to_be_bytes().to_vec()), + (36, 1i32.to_be_bytes().to_vec()), + (37, (-1i32).to_be_bytes().to_vec()), + ]; + img.layers.push(l); + img.layers.push(TestLayer::rgba("Bottom", 30, 20, rgba(30, 20, |_, _| [9, 9, 9, 255]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let layers = &xcf.image().layers; + assert_eq!(layers.iter().map(|l| l.name.as_str()).collect::>(), ["Top", "Bottom"], "file order: topmost first"); + let top = &layers[0]; + assert_eq!((top.mode, top.opacity, top.visible, top.x, top.y), (30, 0.4, false, -3, 15)); + assert!(top.lock_alpha && top.lock_content && top.lock_position); + assert_eq!((top.color_tag, top.composite_mode, top.composite_space, top.blend_space), (2, -2, 1, -1)); + assert!(top.has_alpha() && top.has_pixels() && !top.has_mask() && top.apply_mask && top.expanded); + assert!(!(top.group || top.text || top.vector || top.link || top.clipped || top.floating_selection)); + assert_eq!((top.width, top.height, top.layer_type, top.effects), (10, 8, 1, 0)); + // The layer's own pixel (5, 2); the offset is the caller's to apply. + assert_eq!(at(&pixels(&xcf, 0), 5, 2), [100.0 / 255.0, 60.0 / 255.0, 0.0, 200.0 / 255.0]); + let bottom = &layers[1]; + assert_eq!((bottom.mode, bottom.opacity, bottom.visible, bottom.color_tag), (28, 1.0, true, 0)); + assert!(!bottom.lock_alpha && !bottom.lock_content && !bottom.lock_position); +} + +#[test] +fn groups_follow_item_paths() { + // File order: Top, Group (0), child A [1,0], Nested group [1,1], nested child [1,1,0], Bottom. + let mut img = TestImage::rgb8(16, 16); + let px = |v: u8| rgba(4, 4, move |_, _| [v, v, v, 255]); + img.layers.push(TestLayer::rgba("Top", 4, 4, px(1))); + let mut group = TestLayer::rgba("Group", 1, 1, Vec::new()); + group.props = vec![(29, vec![]), (31, 0u32.to_be_bytes().to_vec())]; + img.layers.push(group); + let mut a = TestLayer::rgba("A", 4, 4, px(2)); + a.props = vec![path(&[1, 0])]; + img.layers.push(a); + let mut nested = TestLayer::rgba("Nested", 1, 1, Vec::new()); + nested.props = vec![(29, vec![]), path(&[1, 1])]; + nested.mode = 61; // Pass through + img.layers.push(nested); + let mut b = TestLayer::rgba("B", 4, 4, px(3)); + b.props = vec![path(&[1, 1, 0])]; + img.layers.push(b); + img.layers.push(TestLayer::rgba("Bottom", 16, 16, rgba(16, 16, |_, _| [0, 0, 0, 255]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let image = xcf.image(); + let tree = image.tree(); + assert!(tree.misplaced.is_empty() && tree.too_deep.is_empty()); + assert_eq!(names(image, &tree.roots), ["Top", "Group", "Bottom"]); + let g = &image.layers[tree.roots[1].layer]; + assert!(g.group && !g.expanded && !g.has_pixels() && g.mode == 28); + assert_eq!(xcf.layer_pixels(tree.roots[1].layer, &mut 0, GO), Ok(None), "a group has no pixels and costs nothing"); + let children = &tree.roots[1].children; + assert_eq!(names(image, children), ["A", "Nested"], "topmost first inside the group too"); + assert_eq!((image.layers[children[1].layer].mode, image.layers[children[1].layer].item_path.as_slice()), (61, &[1, 1][..])); + assert_eq!(names(image, &children[1].children), ["B"]); + assert_eq!(at(&pixels(&xcf, children[0].layer), 0, 0)[0], 2.0 / 255.0); +} + +#[test] +fn a_path_into_nothing_lands_at_the_top_level() { + let mut img = TestImage::rgb8(8, 8); + let mut stray = TestLayer::rgba("Stray", 2, 2, rgba(2, 2, |_, _| [5, 5, 5, 255])); + stray.props = vec![path(&[7, 0])]; + img.layers.push(stray); + img.layers.push(TestLayer::rgba("Bottom", 8, 8, rgba(8, 8, |_, _| [0, 0, 0, 255]))); + let bytes = img.write(); + let tree = open(&bytes).unwrap().image().tree(); + assert_eq!((tree.roots.len(), tree.misplaced.as_slice()), (2, &[0][..])); +} + +/// A path that fails below the top level (here at depth 2) still lands at the top level, so the +/// group it would have been pushed into keeps its indices for the siblings placed after it. +#[test] +fn a_path_failing_below_the_top_level_lands_at_the_top_level() { + let mut img = TestImage::rgb8(8, 8); + let px = |v: u8| rgba(2, 2, move |_, _| [v, v, v, 255]); + let mut group = TestLayer::rgba("Group", 1, 1, Vec::new()); + group.props = vec![(29, vec![]), (31, 0u32.to_be_bytes().to_vec())]; + img.layers.push(group); + let mut stray = TestLayer::rgba("Stray", 2, 2, px(5)); + stray.props = vec![path(&[0, 4, 0])]; + img.layers.push(stray); + let mut a = TestLayer::rgba("A", 2, 2, px(6)); + a.props = vec![path(&[0, 0])]; + img.layers.push(a); + img.layers.push(TestLayer::rgba("Bottom", 8, 8, rgba(8, 8, |_, _| [0, 0, 0, 255]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let (image, tree) = (xcf.image(), xcf.image().tree()); + assert_eq!(names(image, &tree.roots), ["Group", "Stray", "Bottom"]); + assert_eq!(names(image, &tree.roots[0].children), ["A"], "the valid sibling placed after the stray one still lands in the group"); + assert_eq!(tree.misplaced, [1]); +} + +#[test] +fn a_path_deeper_than_the_limit_moves_to_the_top_level() { + let mut img = TestImage::rgb8(8, 8); + let mut deep = TestLayer::rgba("Deep", 1, 1, vec![0; 4]); + deep.props = vec![path(&[0; MAX_DEPTH + 1])]; + let mut edge = TestLayer::rgba("Edge", 1, 1, vec![0; 4]); + edge.props = vec![path(&[0; MAX_DEPTH])]; + img.layers.extend([deep, edge]); + let bytes = img.write(); + let tree = open(&bytes).unwrap().image().tree(); + assert_eq!((tree.roots.len(), tree.too_deep.as_slice(), tree.misplaced.as_slice()), (2, &[0][..], &[1][..])); +} + +#[test] +fn masks_are_read_applied_or_not() { + for apply in [true, false] { + let mut img = TestImage::rgb8(20, 20); + let mut l = TestLayer::rgba("Masked", 6, 6, rgba(6, 6, |_, _| [255, 0, 0, 255])); + l.x = 4; + l.y = 5; + l.mask = Some((0..36).map(|i| (i * 7) as u8).collect()); + l.apply_mask = apply; + img.layers.push(l); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let layer = &xcf.image().layers[0]; + assert!(layer.has_mask()); + assert_eq!((layer.apply_mask, layer.x, layer.y), (apply, 4, 5)); + let (channel, px) = xcf.layer_mask(0, &mut MAX_PIXEL_BYTES.clone(), GO).unwrap().unwrap(); + assert_eq!((channel, px.width, px.height, px.channels), (Channel { name: "mask".into(), selection: false }, 6, 6, 1)); + assert_eq!(at(&px, 3, 2), [(2 * 6 + 3) as f32 * 7.0 / 255.0]); + } +} + +#[test] +fn sixteen_bit_and_float_samples_are_big_endian() { + // 16-bit gamma integer (250): one 2x1 gray+alpha layer. + let mut img = TestImage::rgb8(2, 1); + img.base_type = 1; + img.precision = 250; + let data: Vec = [0x1234u16, 0xffff, 0x8000, 0x0001].into_iter().flat_map(u16::to_be_bytes).collect(); + let mut l = TestLayer::rgba("G", 2, 1, data); + l.kind = 3; // gray with alpha + img.layers.push(l); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + assert_eq!((xcf.image().base_type, xcf.image().precision), (BaseType::Gray, Precision { bytes: 2, float: false, linear: false })); + let px = pixels(&xcf, 0); + assert_eq!(px.data, [0x1234 as f32 / 65535.0, 1.0, 0x8000 as f32 / 65535.0, 1.0 / 65535.0]); + + // 32-bit float linear (600): RGB without alpha, values above 1 kept. + let mut img = TestImage::rgb8(1, 1); + img.precision = 600; + let mut l = TestLayer::rgba("F", 1, 1, [0.25f32, 1.5, -0.5].into_iter().flat_map(f32::to_be_bytes).collect()); + l.kind = 0; + img.layers.push(l); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + assert_eq!(xcf.image().precision, Precision { bytes: 4, float: true, linear: true }); + assert!(!xcf.image().layers[0].has_alpha()); + let px = pixels(&xcf, 0); + assert_eq!((px.channels, px.data.as_slice()), (3, &[0.25, 1.5, -0.5][..])); + + // 16-bit float (500), 64-bit float (700) and 32-bit integer (350) decode too. + for (precision, bytes, want) in + [(500u32, 0x3a00u16.to_be_bytes().to_vec(), 0.75), (700, 0.75f64.to_be_bytes().to_vec(), 0.75), (350, u32::MAX.to_be_bytes().to_vec(), 1.0)] + { + let mut img = TestImage::rgb8(1, 1); + img.base_type = 1; + img.precision = precision; + let mut l = TestLayer::rgba("F", 1, 1, bytes); + l.kind = 2; + img.layers.push(l); + let bytes = img.write(); + assert_eq!(pixels(&open(&bytes).unwrap(), 0).data, [want], "precision {precision}"); + } +} + +#[test] +fn half_floats_convert_exactly() { + for (bits, want) in [ + (0x0000u16, 0.0f32), + (0x3c00, 1.0), + (0xc000, -2.0), + (0x3555, (1024.0 + 341.0) / 4096.0), + (0x7bff, 65504.0), + (0x0400, 2f32.powi(-14)), + (0x0001, 2f32.powi(-24)), + (0x03ff, 1023.0 * 2f32.powi(-24)), + (0x7c00, f32::INFINITY), + (0xfc00, f32::NEG_INFINITY), + ] { + assert_eq!(half_to_f32(bits), want, "{bits:#06x}"); + } + assert!(half_to_f32(0x8000).is_sign_negative() && half_to_f32(0x8000) == 0.0); + assert!(half_to_f32(0x7e00).is_nan()); +} + +#[test] +fn precision_codes_depend_on_the_version() { + let p = |bytes, float, linear| Precision { bytes, float, linear }; + assert_eq!(Precision::from_header(3, 999), Ok(Precision::GAMMA8)); + assert_eq!(Precision::from_header(4, 3), Ok(p(2, true, true))); + assert_eq!(Precision::from_header(5, 500), Ok(p(4, true, true))); + assert_eq!(Precision::from_header(6, 450), Ok(p(2, true, false))); + assert_eq!(Precision::from_header(7, 500), Ok(p(2, true, true))); + assert_eq!(Precision::from_header(12, 750), Ok(p(8, true, false))); + for (version, code) in [(4, 5), (5, 600), (7, 400), (12, 0)] { + assert!(Precision::from_header(version, code).is_err(), "{version} {code}"); + } +} + +#[test] +fn indexed_images_use_the_colormap() { + let mut img = TestImage::rgb8(2, 1); + img.base_type = 2; + let mut cmap = 3u32.to_be_bytes().to_vec(); + cmap.extend_from_slice(&[255, 0, 0, 0, 255, 0, 0, 0, 255]); + img.image_props.push((1, cmap)); + let mut l = TestLayer::rgba("I", 2, 1, vec![2, 255, 1, 128]); + l.kind = 5; // indexed with alpha + img.layers.push(l); + let mut plain = TestLayer::rgba("P", 2, 1, vec![0, 9]); + plain.kind = 4; // indexed, no alpha; index 9 is past the colormap + img.layers.push(plain); + img.channels.push(("C", vec![255, 0], false)); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + assert_eq!((xcf.image().base_type, xcf.image().colormap.len()), (BaseType::Indexed, 3)); + assert_eq!(pixels(&xcf, 0).data, [0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0, 128.0 / 255.0]); + let p = pixels(&xcf, 1); + assert_eq!((p.channels, p.data.as_slice()), (3, &[1.0, 0.0, 0.0, 9.0 / 255.0, 9.0 / 255.0, 9.0 / 255.0][..])); + // Channels are plain samples even in an indexed image. + assert_eq!(xcf.channel(0, &mut MAX_PIXEL_BYTES.clone(), GO).unwrap().1.data, [1.0, 0.0]); +} + +#[test] +fn channels_are_the_extras_and_the_selection() { + let mut img = TestImage::rgb8(4, 4); + img.layers.push(TestLayer::rgba("L", 4, 4, rgba(4, 4, |_, _| [0, 0, 0, 255]))); + img.channels.push(("Extra", (0..16).map(|i| i as u8 * 16).collect(), false)); + img.channels.push(("Selection Mask", vec![255; 16], true)); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + assert_eq!(xcf.image().channel_count, 2); + let mut budget = 200; + let (extra, px) = xcf.channel(0, &mut budget, GO).unwrap(); + assert_eq!((extra, at(&px, 3, 1)), (Channel { name: "Extra".into(), selection: false }, &[7.0 * 16.0 / 255.0][..])); + let (selection, px) = xcf.channel(1, &mut budget, GO).unwrap(); + assert_eq!((selection.selection, px.data[0], budget), (true, 1.0, 200 - 2 * 64)); + assert!(xcf.channel(2, &mut budget, GO).is_err()); + assert!(xcf.layer_pixels(1, &mut budget, GO).is_err() && xcf.layer_mask(1, &mut budget, GO).is_err()); +} + +#[test] +fn image_parasites_resolution_guides_and_comment() { + let mut img = TestImage::rgb8(10, 10); + img.image_props.push((19, [300.0f32, 299.0].into_iter().flat_map(f32::to_be_bytes).collect())); + let mut guides = Vec::new(); + for (position, orientation) in [(4i32, 1u8), (7, 2), (-3, 1), (5, 9)] { + guides.extend_from_slice(&position.to_be_bytes()); + guides.push(orientation); + } + img.image_props.push((18, guides)); + let mut parasites = Vec::new(); + parasite_record(&mut parasites, "gimp-comment", b"hello\0"); + parasite_record(&mut parasites, "icc-profile", &[1, 2, 3]); + img.image_props.push((21, parasites)); + // An unknown property is skipped by its length. + img.image_props.push((99, vec![1, 2, 3, 4, 5])); + img.layers.push(TestLayer::rgba("L", 10, 10, rgba(10, 10, |_, _| [0, 0, 0, 255]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let image = xcf.image(); + assert_eq!(image.resolution, Some((300.0, 299.0))); + let (h, v) = (Orientation::Horizontal, Orientation::Vertical); + assert_eq!( + image.guides, + [Guide { position: 4, orientation: h }, Guide { position: 7, orientation: v }], + "a negative guide before version 15 is stray" + ); + assert_eq!((image.comment.as_deref(), image.icc_profile.as_deref()), (Some("hello"), Some(&[1u8, 2, 3][..]))); + assert_eq!(image.path_count, 0); + img.version = 15; + assert_eq!(open(&img.write()).unwrap().image().guides.len(), 3, "from version 15 a negative guide is off canvas, not stray"); +} + +#[test] +fn clipping_text_vector_effects_and_floating_selections_are_flagged() { + let mut img = TestImage::rgb8(4, 4); + img.version = 20; + let mut clipped = TestLayer::rgba("Clipped", 4, 4, rgba(4, 4, |_, _| [1, 1, 1, 255])); + let mut parasite = Vec::new(); + parasite_record(&mut parasite, "gimphoto-clipped", &[1]); + clipped.props = vec![(21, parasite), (26, 1u32.to_be_bytes().to_vec()), (47, vec![0; 8])]; + img.layers.push(clipped); + let mut floating = TestLayer::rgba("Floating Selection", 2, 2, rgba(2, 2, |_, _| [2, 2, 2, 255])); + floating.props = vec![(5, vec![0; 8])]; + img.layers.push(floating); + let mut text = TestLayer::rgba("Bottom", 4, 4, rgba(4, 4, |_, _| [0, 0, 0, 255])); + let mut parasite = Vec::new(); + parasite_record(&mut parasite, "gimp-text-layer", b"(text \"x\")"); + text.props = vec![(21, parasite)]; + img.layers.push(text); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let image = xcf.image(); + let l = &image.layers; + assert!(l[0].clipped && l[0].text && l[0].vector && !l[0].floating_selection); + assert!(l[1].floating_selection && !l[1].clipped); + assert!(l[2].text && !l[2].vector); + assert_eq!(names(image, &image.tree().roots), ["Clipped", "Bottom"], "the floating selection is not part of the stack"); + // Version 20 layers end with a list of effect pointers. + let effects_end = bytes.windows(8).rposition(|w| w == [0; 8]).unwrap(); + assert!(effects_end > 0); +} + +#[test] +fn layer_effect_pointers_are_counted_and_bounded() { + let layer = |effects: usize| { + let mut w = W::new(20); + for v in [1, 1, 0, 150] { + w.u32(v); + } + w.end(); + let slot = w.slot(); + w.ptr(0); + w.ptr(0); + w.ptr(0); // paths + let at = w.out.len(); + w.patch(slot, at); + for v in [1, 1, 1] { + w.u32(v); + } + w.string("fx"); + w.end(); + w.ptr(0); + w.ptr(0); + for _ in 0..effects { + w.ptr(1); + } + w.ptr(0); + w.out + }; + assert_eq!(open(&layer(1024)).unwrap().image().layers[0].effects, 1024); + assert!(open(&layer(1025)).is_err()); +} + +/// GIMP's link-layer length omits one u32. Trusting it shifts the hierarchy pointer by four +/// bytes, even though the cached pixels and every offset in the file are valid. +#[test] +fn linked_layers_open_cached_pixels_with_short_or_correct_property_lengths() { + for (precision, data) in [ + (150, vec![51, 102, 153, 255]), + (250, [13107u16, 26214, 39321, 65535].into_iter().flat_map(u16::to_be_bytes).collect()), + (600, [0.2f32, 0.4, 0.6, 1.0].into_iter().flat_map(f32::to_be_bytes).collect()), + ] { + for (path, width, height) in [("assets/étiquette.svg", 2000, 1270), ("assets/label.png", 0, 0), ("", 0, 0)] { + let mut payload = W { out: Vec::new(), wide: true }; + payload.u32(1); + payload.string(path); + payload.u32(width); + payload.u32(height); + let payload = payload.out; + let marker = [48u32.to_be_bytes(), (payload.len() as u32).to_be_bytes()].concat(); + let mut img = TestImage::rgb8(1, 1); + img.version = 25; + img.precision = precision; + let mut linked = TestLayer::rgba("Linked", 1, 1, data.clone()); + // Later properties must still be read, including unknown ones skipped by length. + linked.props = vec![(48, payload.clone()), (99, vec![1, 2, 3]), (33, 0.375f32.to_be_bytes().to_vec()), (8, 0u32.to_be_bytes().to_vec())]; + img.layers.push(linked); + img.layers.push(TestLayer::rgba("Bottom", 1, 1, data.clone())); + for short in [false, true] { + let mut bytes = img.write(); + let at = bytes.windows(marker.len()).position(|w| w == marker).unwrap(); + if short { + bytes[at + 4..at + 8].copy_from_slice(&((payload.len() - 4) as u32).to_be_bytes()); + } + let xcf = open(&bytes).unwrap_or_else(|e| panic!("precision {precision}, path {path:?}, short {short}: {e}")); + let layers = &xcf.image().layers; + assert_eq!(layers.len(), 2); + assert_eq!((layers[0].link, layers[0].opacity, layers[0].visible, layers[1].link), (true, 0.375, false, false)); + for layer in 0..2 { + for (got, expected) in pixels(&xcf, layer).data.into_iter().zip([0.2, 0.4, 0.6, 1.0]) { + assert!((got - expected).abs() < 1e-5, "precision {precision}: {got} != {expected}"); + } + } + } + } + } +} + +#[test] +fn truncated_and_oversized_link_layer_fields_are_errors() { + let mut payload = W { out: Vec::new(), wide: true }; + payload.u32(1); + payload.string("assets/label.svg"); + payload.u32(2000); + payload.u32(1270); + let payload = payload.out; + let marker = [48u32.to_be_bytes(), (payload.len() as u32).to_be_bytes()].concat(); + let mut img = TestImage::rgb8(1, 1); + img.version = 25; + let mut linked = TestLayer::rgba("Linked", 1, 1, vec![51, 102, 153, 255]); + linked.props.push((48, payload.clone())); + img.layers.push(linked); + let bytes = img.write(); + let start = bytes.windows(marker.len()).position(|w| w == marker).unwrap() + 8; + // Cut within the flags, string length, path and either dimension. + for end in start..start + payload.len() { + assert!(open(&bytes[..end]).is_err(), "truncated at {end}"); + } + for len in [(MAX_STRING + 1) as u32, u32::MAX] { + let mut oversized = bytes.clone(); + oversized[start + 4..start + 8].copy_from_slice(&len.to_be_bytes()); + assert!(open(&oversized).is_err(), "path length {len}"); + } +} + +#[test] +fn version_3_files_use_32_bit_pointers_and_legacy_modes() { + let mut img = TestImage::rgb8(5, 5); + img.version = 3; + let mut l = TestLayer::rgba("Old", 5, 5, rgba(5, 5, |x, _| [x as u8, 0, 0, 255])); + l.mode = 3; // Multiply (legacy) + img.layers.push(l); + let bytes = img.write(); + assert!(bytes.starts_with(b"gimp xcf v003\0")); + let xcf = open(&bytes).unwrap(); + assert_eq!((xcf.image().version, xcf.image().precision, xcf.image().layers[0].mode), (3, Precision::GAMMA8, 3)); + assert_eq!(at(&pixels(&xcf, 0), 4, 0)[0], 4.0 / 255.0); + // Version 0 ("file") as well. + let mut img0 = TestImage::rgb8(2, 2); + img0.version = 0; + img0.layers.push(TestLayer::rgba("Oldest", 2, 2, rgba(2, 2, |_, _| [7, 7, 7, 255]))); + assert_eq!(open(&img0.write()).unwrap().image().version, 0); + // A version newer than the reader knows still opens. + img0.version = KNOWN_VERSION + 1; + assert_eq!(read_all(&img0.write()), Ok(16)); +} + +#[test] +fn rle_long_runs_and_literals_round_trip() { + // A tile with runs over 127 and long literal stretches, in every byte plane. + let w = 64; + let data = rgba(w, 64, |x, y| if y < 40 { [200, 1, 2, 255] } else { [(x * 7 + y) as u8, (x * 3) as u8, y as u8, 255] }); + assert_eq!(rle_decode(&rle(&data, 4), 4, 64 * 64).unwrap(), data); + // Long literal (op 128) and the error cases. + assert_eq!(rle_decode(&[128, 0, 3, 7, 8, 9], 1, 3).unwrap(), [7, 8, 9]); + assert_eq!(rle_decode(&[127, 0, 4, 9], 1, 4).unwrap(), [9, 9, 9, 9]); + assert_eq!(rle_decode(&[254, 1, 2, 0, 5], 1, 3).unwrap(), [1, 2, 5]); + for bad in [&[][..], &[3], &[127, 0], &[128, 0, 3, 7], &[2, 9], &[253, 1]] { + assert!(rle_decode(bad, 1, 2).is_err(), "{bad:?}"); + } +} + +#[test] +fn limits_and_malformed_structures_are_errors() { + // A canvas past the side limit, and one past the area limit. + let mut big = TestImage::rgb8(1, 1); + big.width = MAX_SIDE + 1; + assert!(open(&big.write()).is_err()); + (big.width, big.height) = (MAX_SIDE, 224); + assert!(open(&big.write()).is_err()); + big.height = 223; + assert!(open(&big.write()).is_ok()); + big.height = 0; + assert!(open(&big.write()).is_err()); + // A layer whose declared size has no pixels behind it (hierarchy pointer to garbage). + let mut img = TestImage::rgb8(4, 4); + img.layers.push(TestLayer::rgba("L", 4, 4, rgba(4, 4, |_, _| [0; 4]))); + let mut bytes = img.write(); + let n = bytes.len(); + bytes[n - 10..].fill(0xff); + assert!(read_all(&bytes).is_err()); + // Wrong signature, unknown version tag, unknown compression, image type and precision. + assert!(matches!(open(b"gimp xcf vXYZ\0"), Err(Error::Invalid(_)))); + assert!(open(b"not an xcf").is_err()); + let good = img.write(); + assert!(read_all(&good).is_ok()); + let patched = |at: usize, v: u8| { + let mut b = good.clone(); + b[at] = v; + open(&b).err() + }; + assert!(patched(14 + 12 + 4 + 8, 3).is_some(), "compression"); + assert!(patched(14 + 11, 3).is_some(), "image type"); + assert!(patched(14 + 15, 1).is_some(), "precision"); + // A layer of another colour model, and an unknown layer type. + for kind in [2, 6] { + img.layers[0].kind = kind; + assert!(open(&img.write()).is_err(), "layer type {kind}"); + } + // Pixels of another size than the layer declares. + img.layers[0].kind = 1; + let mut bytes = img.write(); + let at = bytes.windows(12).rposition(|w| w == [0, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 4]).unwrap(); + bytes[at + 3] = 3; + assert!(open(&bytes).is_ok() && read_all(&bytes).is_err()); +} + +#[test] +fn layer_channel_property_and_metadata_counts_are_bounded() { + let header = |w: &mut W| { + for v in [1, 1, 0, 150] { + w.u32(v); + } + w.end(); + }; + // More layer pointers, then more channel pointers, than the limits. + for (layers, channels, ok) in [(MAX_LAYERS + 1, 0, false), (0, MAX_CHANNELS + 1, false), (0, MAX_CHANNELS, true)] { + let mut w = W::new(11); + header(&mut w); + (0..layers).for_each(|_| w.ptr(1)); + w.ptr(0); + (0..channels).for_each(|_| w.ptr(1)); + w.ptr(0); + assert_eq!(open(&w.out).is_ok(), ok, "{layers} layers, {channels} channels"); + } + // 4097 properties. + let mut w = W::new(11); + for v in [1, 1, 0, 150] { + w.u32(v); + } + (0..4097).for_each(|_| w.prop(99, &[])); + w.end(); + w.ptr(0); + w.ptr(0); + assert_eq!(open(&w.out).err(), Some(Error::Invalid("too many properties".into()))); + // A property declaring more than the payload limit. + let mut w = W::new(11); + for v in [1, 1, 0, 150, 99, MAX_PAYLOAD as u32 + 1] { + w.u32(v); + } + assert!(matches!(open(&w.out), Err(Error::Invalid(m)) if m.contains("declares"))); + // Forty pointers to one layer whose name is a megabyte: its copies pass the metadata budget. + let many = |pointers: usize| { + let mut w = W::new(11); + header(&mut w); + let slots: Vec = (0..pointers).map(|_| w.slot()).collect(); + w.ptr(0); + w.ptr(0); + let at = w.out.len(); + slots.into_iter().for_each(|s| w.patch(s, at)); + for v in [1, 1, 1] { + w.u32(v); + } + w.string(&"n".repeat(MAX_STRING - 1)); + w.end(); + w.ptr(0); + w.ptr(0); + w.out + }; + assert!(open(&many(32)).is_ok()); + assert_eq!(open(&many(33)).err(), Some(Error::Invalid("layer metadata exceeds the budget".into()))); +} + +#[test] +fn the_pixel_budget_is_charged_before_decoding_and_decoding_can_be_cancelled() { + let mut img = TestImage::rgb8(200, 200); + img.layers.push(TestLayer::rgba("L", 200, 200, rgba(200, 200, |_, _| [1, 2, 3, 255]))); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + let cost = 200 * 200 * 4 * 4; + let mut budget = cost - 1; + assert!(matches!(xcf.layer_pixels(0, &mut budget, GO), Err(Error::Invalid(m)) if m.contains("pixel limit"))); + let mut budget = cost + 5; + assert!(xcf.layer_pixels(0, &mut budget, GO).is_ok()); + assert_eq!(budget, 5); + assert_eq!(xcf.layer_pixels(0, &mut MAX_PIXEL_BYTES.clone(), &|| false), Err(Error::Cancelled)); +} + +#[test] +fn gzipped_files_are_recognised_by_name_and_inflated() { + use std::io::Write; + let mut img = TestImage::rgb8(3, 3); + img.layers.push(TestLayer::rgba("L", 3, 3, rgba(3, 3, |_, _| [4, 5, 6, 255]))); + let plain = img.write(); + let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + e.write_all(&plain).unwrap(); + let gz = e.finish().unwrap(); + assert!(is_xcf(&plain) && !is_xcf(&gz)); + assert!(is_gzipped_xcf("a/b/photo.XCF.GZ", &gz) && is_gzipped_xcf("x.xcfgz", &gz) && is_gzipped_xcf("c:\\dir.xcf.gz\\y.xcf.gz", &gz)); + assert!(!is_gzipped_xcf("photo.xcf", &gz) && !is_gzipped_xcf("photo.xcf.gz", &plain) && !is_gzipped_xcf("dir.xcf.gz/photo.gz", &gz)); + assert_eq!(gunzip(&gz).unwrap(), plain); + assert!(gunzip(&gz[..gz.len() / 2]).is_err()); +} + +/// `PROP_FLOAT_OPACITY` wins over `PROP_OPACITY` whichever comes first in the file. +#[test] +fn float_opacity_wins_over_opacity_in_either_order() { + let mut img = TestImage::rgb8(4, 4); + // The writer emits OPACITY (0.4 → 102) then FLOAT_OPACITY 0.4; an OPACITY 255 after them must not win. + let mut byte_last = TestLayer::rgba("Float then byte", 4, 4, rgba(4, 4, |_, _| [1, 2, 3, 255])); + byte_last.opacity = 0.4; + byte_last.props = vec![(6, 255u32.to_be_bytes().to_vec())]; + // A later FLOAT_OPACITY replaces the earlier one; one that is not finite does not. + let mut float_last = TestLayer::rgba("Byte then float", 4, 4, rgba(4, 4, |_, _| [1, 2, 3, 255])); + float_last.opacity = 0.4; + float_last.props = vec![(33, 0.25f32.to_be_bytes().to_vec()), (33, f32::NAN.to_be_bytes().to_vec())]; + img.layers.push(byte_last); + img.layers.push(float_last); + let bytes = img.write(); + let xcf = open(&bytes).unwrap(); + assert_eq!((xcf.image().layers[0].opacity, xcf.image().layers[1].opacity), (0.4, 0.25)); +} + +/// One PROP_PARASITES record: name, u32 flags, u32 len, then the payload. +fn parasite_record(out: &mut Vec, name: &str, payload: &[u8]) { + out.extend_from_slice(&((name.len() as u32) + 1).to_be_bytes()); + out.extend_from_slice(name.as_bytes()); + out.push(0); + out.extend_from_slice(&0u32.to_be_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_be_bytes()); + out.extend_from_slice(payload); +} + +#[test] +fn duplicate_parasites_keep_order_and_last_value() { + let mut data = Vec::new(); + parasite_record(&mut data, "gimp-comment", b"first"); + parasite_record(&mut data, "gimp-image-grid", b"grid"); + parasite_record(&mut data, "gimp-comment", b"second"); + data.extend_from_slice(&[0, 0, 0, 9, b'c', b'u', b't']); + let out = parasites(&data); + assert_eq!(out, [("gimp-comment".to_string(), &b"second"[..]), ("gimp-image-grid".to_string(), &b"grid"[..])]); +} + +#[test] +fn a_user_unit_property_is_measured_not_trusted() { + // PROP_USER_UNIT's length word is wrong in every release: three strings (XCF 21+) or five. + // Which it is, is decided by whether a property header follows the third: a fourth string + // shorter than 64 bytes would look like one, so the five-string case here has a long one. + let long = "s".repeat(70); + for strings in [3, 5] { + let mut img = TestImage::rgb8(2, 2); + let mut unit = W { out: Vec::new(), wide: true }; + unit.out.extend_from_slice(&25.4f32.to_be_bytes()); + unit.u32(2); + (0..strings).for_each(|i| unit.string(["id", "sym", "abbr", long.as_str(), "plural"][i])); + img.image_props.push((24, unit.out)); + img.image_props.push((19, [96.0f32, 96.0].into_iter().flat_map(f32::to_be_bytes).collect())); + img.layers.push(TestLayer::rgba("L", 2, 2, rgba(2, 2, |_, _| [4, 5, 6, 255]))); + let mut bytes = img.write(); + let marker = 24u32.to_be_bytes(); + let at = bytes.windows(4).position(|w| w == marker).unwrap(); + bytes[at + 4..at + 8].copy_from_slice(&3u32.to_be_bytes()); + let xcf = open(&bytes).unwrap_or_else(|e| panic!("{strings} strings: {e}")); + assert_eq!(xcf.image().resolution, Some((96.0, 96.0)), "{strings} strings"); + assert_eq!(read_all(&bytes), Ok(16)); + } +} + +#[test] +fn an_rle_run_of_zero_length_is_an_error() { + // Long runs (op 127 / 128) carry a 16-bit length; zero consumes input without producing output. + assert!(rle_decode(&[127, 0, 0, 5, 3, 0, 9], 1, 4).is_err()); + assert!(rle_decode(&[128, 0, 0, 3, 0, 9], 1, 4).is_err()); + assert_eq!(rle_decode(&[127, 0, 4, 9], 1, 4).unwrap(), [9, 9, 9, 9]); +} + +#[test] +fn channel_names_are_charged_to_the_metadata_budget() { + // Channel pointers that all name one channel with a megabyte name: each read copies the name. + let mut w = W::new(11); + for v in [1, 1, 0, 150] { + w.u32(v); + } + w.end(); + w.ptr(0); + let slots: Vec = (0..40).map(|_| w.slot()).collect(); + w.ptr(0); + let at = w.out.len(); + slots.into_iter().for_each(|s| w.patch(s, at)); + w.u32(1); + w.u32(1); + w.string(&"n".repeat(MAX_STRING - 1)); + w.end(); + let hs = w.slot(); + let h = hierarchy(&mut w, 1, 1, 1, &[7], Compression::None); + w.patch(hs, h); + let xcf = open(&w.out).unwrap(); + let mut budget = MAX_PIXEL_BYTES; + let results: Vec<_> = (0..40).map(|i| xcf.channel(i, &mut budget, GO).map(|(c, _)| c.name.len())).collect(); + assert!(results[..32].iter().all(|r| *r == Ok(MAX_STRING - 1)), "32 copies fit the 32 MiB budget"); + assert!(results[32..].iter().all(|r| *r == Err(Error::Invalid("channel metadata exceeds the budget".into())))); +} diff --git a/crates/xcf/src/tiles.rs b/crates/xcf/src/tiles.rs new file mode 100644 index 000000000..5a567558a --- /dev/null +++ b/crates/xcf/src/tiles.rs @@ -0,0 +1,70 @@ +//! One tile's bytes: uncompressed, zlib-coded or RLE-coded. + +use std::io::Read; + +use crate::{Compression, Result, invalid}; + +/// What one tile's bytes decode to, for a drawable of `bpp` bytes per pixel: `pixels × bpp` +/// interleaved bytes. +pub(crate) fn decode(input: &[u8], compression: Compression, bpp: usize, pixels: usize) -> Result> { + let expected = pixels.checked_mul(bpp).ok_or_else(|| invalid("tile size overflow"))?; + match compression { + Compression::None => input.get(..expected).map(<[u8]>::to_vec).ok_or_else(|| invalid("an uncompressed tile is cut short")), + Compression::Zlib => { + let mut out = Vec::with_capacity(expected); + flate2::read::ZlibDecoder::new(input).take(expected as u64).read_to_end(&mut out).map_err(|e| invalid(format!("zlib tile: {e}")))?; + if out.len() != expected { + return Err(invalid("a zlib tile is cut short")); + } + Ok(out) + } + Compression::Rle => rle_decode(input, bpp, pixels), + } +} + +/// RLE tile data: one run-length stream per byte position of the pixel, each of `pixels` bytes. +pub(crate) fn rle_decode(input: &[u8], bpp: usize, pixels: usize) -> Result> { + let short = || invalid("an RLE tile is cut short"); + let mut out = vec![0u8; pixels * bpp]; + let mut pos = 0usize; + let mut plane = Vec::with_capacity(pixels); + for b in 0..bpp { + plane.clear(); + while plane.len() < pixels { + let op = *input.get(pos).ok_or_else(short)?; + pos += 1; + let mut long = || -> Result { + let (p, q) = (input.get(pos).copied().ok_or_else(short)?, input.get(pos + 1).copied().ok_or_else(short)?); + pos += 2; + Ok(usize::from(p) * 256 + usize::from(q)) + }; + let (run, verbatim) = match op { + 0..=126 => (usize::from(op) + 1, false), + 127 => (long()?, false), + 128 => (long()?, true), + _ => (256 - usize::from(op), true), + }; + if plane.len() + run > pixels { + return Err(invalid("an RLE run crosses the end of its tile")); + } + // A long run of length 0 consumes input without producing output: a tile of them + // (reachable from many tile pointers) is work the pixel budget never charges. + if run == 0 { + return Err(invalid("an RLE run has no length")); + } + if verbatim { + let data = input.get(pos..pos + run).ok_or_else(short)?; + pos += run; + plane.extend_from_slice(data); + } else { + let v = *input.get(pos).ok_or_else(short)?; + pos += 1; + plane.resize(plane.len() + run, v); + } + } + for (slot, v) in out.iter_mut().skip(b).step_by(bpp.max(1)).zip(&plane) { + *slot = *v; + } + } + Ok(out) +} From 53e1717523a8a4dd24d7bc3be93963aa2b009ae3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:25:27 -0500 Subject: [PATCH 15/31] craft-rtf: Outlook compressed RTF and body text, from MailCraft (#961) The outlook feature adds MailCraft's reader of PidTagRtfCompressed ([MS-OXRTFCP]) and of the text and encapsulated HTML ([MS-OXRTFEX]) of the RTF inside, from crates/pst/src/rtf.rs at a44a450. Font charsets use this crate's charset_code_page table; code pages are decoded by craft-mime. Co-authored-by: Martin Ball <98639962+benandez@users.noreply.github.com> --- Cargo.lock | 2 + crates/rtf/Cargo.toml | 10 + crates/rtf/README.md | 11 + crates/rtf/src/lib.rs | 2 + crates/rtf/src/outlook.rs | 711 ++++++++++++++++++++++++++++++++ crates/rtf/src/outlook_tests.rs | 385 +++++++++++++++++ 6 files changed, 1121 insertions(+) create mode 100644 crates/rtf/src/outlook.rs create mode 100644 crates/rtf/src/outlook_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1be03246b..0b52a4503 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1556,7 +1556,9 @@ dependencies = [ name = "craft-rtf" version = "0.1.0" dependencies = [ + "craft-mime", "encoding_rs", + "proptest", ] [[package]] diff --git a/crates/rtf/Cargo.toml b/crates/rtf/Cargo.toml index aec1ca5ce..a6724e251 100644 --- a/crates/rtf/Cargo.toml +++ b/crates/rtf/Cargo.toml @@ -10,6 +10,16 @@ repository.workspace = true [lints] workspace = true +[features] +# `outlook`: compressed RTF ([MS-OXRTFCP]) and the text or encapsulated HTML ([MS-OXRTFEX]) of a +# mail item's body. +outlook = ["dep:craft-mime"] + [dependencies] +# Character sets by Windows code page number, for the `outlook` module. +craft-mime = { workspace = true, optional = true } # The East Asian multibyte code pages (932, 936, 949, 950). encoding_rs = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/rtf/README.md b/crates/rtf/README.md index 9ae51fecc..65c378c17 100644 --- a/crates/rtf/README.md +++ b/crates/rtf/README.md @@ -79,6 +79,17 @@ elsewhere are kept. `\sect` ends a paragraph. The reader does not recurse, and what it returns is never larger than a small multiple of the file: each character of text costs the file at least one byte. +## Outlook bodies (feature `outlook`) + +`outlook::decompress_rtf` undoes the compression of `PidTagRtfCompressed` ([MS-OXRTFCP]: `LZFu` +with its CRC checked, or `MELA` stored as is), never allocating past the caller's cap. +`outlook::rtf_text` gives the text a mail body shows (font charsets and `\ansicpg` decoded through +`craft_mime::charset`, `\uN` with its fallback skipped, destinations dropped), and +`outlook::rtf_html` the HTML of an encapsulated-HTML body ([MS-OXRTFEX], `\fromhtml1`). Groups nest +at most `MAX_RTF_DEPTH` (256) deep. From MailCraft `crates/pst/src/rtf.rs` at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors. + ## Not here yet WordCraft's full reader and writer (styles, lists, fields, bookmarks, pictures, notes, diff --git a/crates/rtf/src/lib.rs b/crates/rtf/src/lib.rs index c6bba82ec..3b22e484b 100644 --- a/crates/rtf/src/lib.rs +++ b/crates/rtf/src/lib.rs @@ -33,6 +33,8 @@ pub mod codepage; mod date; mod lex; mod model; +#[cfg(feature = "outlook")] +pub mod outlook; mod read; mod text; mod write; diff --git a/crates/rtf/src/outlook.rs b/crates/rtf/src/outlook.rs new file mode 100644 index 000000000..7ce886b88 --- /dev/null +++ b/crates/rtf/src/outlook.rs @@ -0,0 +1,711 @@ +//! Rich text as Outlook keeps it in `PidTagRtfCompressed` (0x1009), in .pst and .msg files and +//! TNEF attachments: the compression of +//! [MS-OXRTFCP] ([`decompress_rtf`]) and the text of the RTF that comes out ([`rtf_text`]), +//! written from those two specifications and the public Rich Text Format specification. +//! +//! Both read hostile bytes: output is capped by the caller, a header's size is never trusted +//! for an allocation past that cap, nothing recurses (groups are counted, not nested calls), +//! and every loop ends with its input. + +use std::collections::HashMap; + +use craft_mime::charset::{code_page_label, decode}; + +use crate::codepage; + +/// Why rich text was not read. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Error { + /// The bytes are not what the format says; the text names what was wrong. + Corrupt(&'static str), + /// The compressed bytes do not have the CRC their header gives. + BadChecksum, + /// More would come out than the caller's cap allows. + TooLarge, +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Corrupt(what) => f.write_str(what), + Error::BadChecksum => f.write_str("compressed rich text with a wrong checksum"), + Error::TooLarge => f.write_str("rich text larger than allowed"), + } + } +} + +impl std::error::Error for Error {} + +type Result = std::result::Result; + +/// The CRC of [MS-OXRTFCP] 2.1.3.1 and 3.2: the reflected CRC-32 polynomial, seeded with zero and +/// with no final inversion. +pub(crate) fn crc32(data: &[u8]) -> u32 { + let mut crc = 0u32; + for &b in data { + crc ^= u32::from(b); + for _ in 0..8 { + crc = if crc & 1 != 0 { 0xEDB8_8320 ^ (crc >> 1) } else { crc >> 1 }; + } + } + crc +} + +/// The dictionary a stream starts with ([MS-OXRTFCP] 2.1.2.1), 207 bytes. +const DICT_INIT: &[u8] = b"{\\rtf1\\ansi\\mac\\deff0\\deftab720{\\fonttbl;}{\\f0\\fnil \\froman \\fswiss \\fmodern \\fscript \\fdecor MS Sans SerifSymbolArialTimes New RomanCourier{\\colortbl\\red0\\green0\\blue0\r\n\\par \\pard\\plain\\f0\\fs20\\b\\i\\u\\tab\\tx"; +const _: () = assert!(DICT_INIT.len() == 207); +const DICT_SIZE: usize = 4096; +const HEADER: usize = 16; +const LZFU: u32 = 0x7546_5A4C; +const MELA: u32 = 0x414C_454D; + +fn le32(b: &[u8], at: usize) -> Option { + let s = b.get(at..at.checked_add(4)?)?; + Some(u32::from_le_bytes(<[u8; 4]>::try_from(s).ok()?)) +} + +/// The RTF inside `PidTagRtfCompressed` bytes: the 16-byte header, then either the text as it +/// is (`MELA`) or the dictionary-compressed runs (`LZFu`) whose CRC is checked. At most +/// `max_out` bytes come out; past that is [`Error::TooLarge`], never a bigger allocation. +pub fn decompress_rtf(data: &[u8], max_out: usize) -> Result> { + let (Some(comp_size), Some(raw_size), Some(kind), Some(crc)) = (le32(data, 0), le32(data, 4), le32(data, 8), le32(data, 12)) else { + return Err(Error::Corrupt("rich text shorter than its header")); + }; + let raw_size = usize::try_from(raw_size).unwrap_or(usize::MAX); + let body = data.get(HEADER..).unwrap_or_default(); + match kind { + MELA => { + // The reader may take the sizes the header gives; never more than is there. + let take = raw_size.min(body.len()); + if take > max_out { + return Err(Error::TooLarge); + } + Ok(body.get(..take).unwrap_or_default().to_vec()) + } + LZFU => { + if raw_size > max_out { + return Err(Error::TooLarge); + } + // COMPSIZE counts the 12 bytes of the header after itself. + let content_len = usize::try_from(comp_size) + .unwrap_or(usize::MAX) + .checked_sub(12) + .ok_or(Error::Corrupt("rich text with a size smaller than its header"))?; + let content = body.get(..content_len).ok_or(Error::Corrupt("rich text cut short"))?; + let out = lzfu(content, max_out, raw_size)?; + if crc32(content) != crc { + return Err(Error::BadChecksum); + } + Ok(out) + } + _ => Err(Error::Corrupt("rich text of a kind not known")), + } +} + +fn lzfu(content: &[u8], max_out: usize, raw_size: usize) -> Result> { + let mut dict = vec![0u8; DICT_SIZE]; + if let Some(head) = dict.get_mut(..DICT_INIT.len()) { + head.copy_from_slice(DICT_INIT); + } + let mut write = DICT_INIT.len(); + let mut out: Vec = Vec::with_capacity(raw_size.min(max_out).min(1 << 20)); + let mut at = 0usize; + let cut = Error::Corrupt("compressed rich text ends before its end mark"); + loop { + let control = *content.get(at).ok_or(cut)?; + at = at.saturating_add(1); + for bit in 0..8u32 { + if control >> bit & 1 == 0 { + let b = *content.get(at).ok_or(cut)?; + at = at.saturating_add(1); + if out.len() >= max_out { + return Err(Error::TooLarge); + } + out.push(b); + if let Some(d) = dict.get_mut(write) { + *d = b; + } + write = write.saturating_add(1) % DICT_SIZE; + } else { + let (Some(&hi), Some(&lo)) = (content.get(at), content.get(at.saturating_add(1))) else { return Err(cut) }; + at = at.saturating_add(2); + let word = usize::from(hi) << 8 | usize::from(lo); + let mut read = word >> 4; + if read == write { + return Ok(out); + } + for _ in 0..(word & 0xF).saturating_add(2) { + let b = dict.get(read).copied().unwrap_or(0); + if out.len() >= max_out { + return Err(Error::TooLarge); + } + out.push(b); + read = read.saturating_add(1) % DICT_SIZE; + if let Some(d) = dict.get_mut(write) { + *d = b; + } + write = write.saturating_add(1) % DICT_SIZE; + } + } + } + } +} + +/// Deepest nesting of groups read. +pub const MAX_RTF_DEPTH: usize = 256; + +/// Destinations whose content is not the document's text. +const SKIPPED: &[&str] = &[ + "fonttbl", + "colortbl", + "stylesheet", + "info", + "pict", + "object", + "header", + "headerl", + "headerr", + "headerf", + "footer", + "footerl", + "footerr", + "footerf", + "themedata", + "colorschememapping", + "latentstyles", + "datastore", + "listtable", + "listoverridetable", + "revtbl", + "rsidtbl", + "generator", + "filetbl", + "fldinst", + "xmlnstbl", + "mmathpr", + "pgdsctbl", + "template", + "private", + "bkmkstart", + "bkmkend", + "wgrffmtfilter", + "shpinst", + "nonshppict", + "pnseclvl", + "listtext", + "blipuid", + "falt", + "panose", + "fchars", + "lchars", + "xe", + "tc", + "tcn", + "ud", + "upr", + "title", + "subject", + "author", + "operator", + "company", + "keywords", + "doccomm", + "comment", + "category", + "manager", + "hlinkbase", + "buptim", + "creatim", + "revtim", + "printim", + "txe", + "rxe", + "pnseclvl", + "protusertbl", + "defchp", + "defpap", + "fontemb", + "fontfile", + "background", + "docvar", + "userprops", + "mhtmltag", + "htmltag", + "factoidname", + "oleclsid", + "objdata", +]; + +#[derive(Clone, Copy)] +struct Group { + /// Skip characters after `\u` (`\ucN`). + uc: u32, + skip: bool, + /// In the font table: `\fN` names the font that `\fcharsetN` is for. + fonttbl: bool, + /// The font being defined (in the font table). + defining: i64, + /// The font text is in (`\fN`), if one was chosen. + font: Option, + /// Encapsulated HTML: under `\htmlrtf`, what isn't part of the HTML. + suppressed: bool, + /// Encapsulated HTML: inside an `\htmltag` group. + in_tag: bool, +} + +impl Group { + fn hidden(&self) -> bool { + self.skip || self.suppressed + } +} + +/// Most fonts whose character set is kept. +const MAX_FONTS: usize = 1024; +/// Most bytes of `\'hh` kept waiting for a character boundary. +const PENDING: usize = 4096; + +struct Out { + text: String, + max: usize, + /// Text was dropped for the cap. + cut: bool, + pending: Vec, + /// The code page of `pending`. + pending_cp: u32, + high: Option, + ansicpg: Option, + mac: bool, + deff: Option, + /// `\fcharset` of each font in the font table. + fonts: HashMap, + /// Reading encapsulated HTML rather than the text a document shows. + html: bool, +} + +/// The code page a font character set stands for ([MSFT-RTF] `\fcharsetN`): `None` for the +/// document's own (0 is Windows ANSI, 1252, which is not the document's when it says +/// `\ansicpg` otherwise: [MS-OXRTFEX] 3.1 marks one font `\fcharset0` in a 1251 document to +/// have 0xA8 read as U+00A8). Symbol (2), Johab (130) and the OEM sets (254, 255) are read in the document's. +fn charset_code_page(charset: i64) -> Option { + match charset { + 0 => Some(1252), + 2 | 254 | 255 => None, + other => i32::try_from(other).ok().and_then(codepage::charset_code_page).map(u32::from), + } +} + +/// Whether `b` begins a two-byte character in code page `cp`. +fn is_lead(cp: u32, b: u8) -> bool { + match cp { + 932 => matches!(b, 0x81..=0x9F | 0xE0..=0xFC), + 936 | 949 | 950 => matches!(b, 0x81..=0xFE), + _ => false, + } +} + +impl Out { + fn new(max: usize) -> Out { + Out { + text: String::new(), + max, + cut: false, + pending: Vec::new(), + pending_cp: 1252, + high: None, + ansicpg: None, + mac: false, + deff: None, + fonts: HashMap::new(), + html: false, + } + } + + /// The code page of the document: `\ansicpg`, else the Macintosh's for `\mac`, else 1252. + fn document_cp(&self) -> u32 { + self.ansicpg.unwrap_or(if self.mac { 10000 } else { 1252 }) + } + + /// The code page of bytes in font `font` (else the default font). + fn code_page(&self, font: Option) -> u32 { + font.or(self.deff).and_then(|f| self.fonts.get(&f)).and_then(|cs| charset_code_page(*cs)).unwrap_or_else(|| self.document_cp()) + } + + fn full(&self) -> bool { + self.text.len() >= self.max + } + + fn flush(&mut self) { + let bytes = std::mem::take(&mut self.pending); + self.decode(&bytes); + } + + fn decode(&mut self, bytes: &[u8]) { + if !bytes.is_empty() { + let decoded = decode(bytes, code_page_label(i64::from(self.pending_cp)).unwrap_or("windows-1252")); + self.push_str(&decoded); + } + } + + /// Decodes the whole characters waiting, and keeps a lead byte whose second is not here yet. + fn flush_whole(&mut self) { + let mut cut = 0usize; + let mut i = 0usize; + while i < self.pending.len() { + let step = if self.pending.get(i).is_some_and(|b| is_lead(self.pending_cp, *b)) { 2 } else { 1 }; + if i.saturating_add(step) > self.pending.len() { + break; + } + i = i.saturating_add(step); + cut = i; + } + let rest = self.pending.split_off(cut); + self.flush(); + self.pending = rest; + } + + fn push_str(&mut self, s: &str) { + for c in s.chars() { + // Controls other than the line end and the tab are not text. + if c.is_control() && c != '\n' && c != '\t' { + continue; + } + if self.full() { + self.cut = true; + return; + } + self.text.push(c); + } + } + + fn char(&mut self, c: char) { + self.flush_high(); + self.flush(); + self.push_str(c.encode_utf8(&mut [0; 4])); + } + + fn flush_high(&mut self) { + if self.high.take().is_some() { + self.push_str("\u{FFFD}"); + } + } + + /// A byte of 8-bit text in code page `cp`. + fn byte(&mut self, b: u8, cp: u32) { + self.flush_high(); + if cp != self.pending_cp { + self.flush(); + self.pending_cp = cp; + } + if self.pending.len() >= PENDING { + self.flush_whole(); + } + // (A buffer that is full of one character's bytes is not a text any code page here + // has; it is let grow by one rather than dropped.) + self.pending.push(b); + } + + fn unit(&mut self, u: u16) { + self.flush(); + match (self.high.take(), u) { + (Some(h), 0xDC00..=0xDFFF) => { + let c = 0x10000u32.saturating_add((u32::from(h).saturating_sub(0xD800)) << 10).saturating_add(u32::from(u).saturating_sub(0xDC00)); + self.push_str(&char::from_u32(c).unwrap_or('\u{FFFD}').to_string()); + } + (Some(_), _) => { + self.push_str("\u{FFFD}"); + self.unit(u); + } + (None, 0xD800..=0xDBFF) => self.high = Some(u), + (None, 0xDC00..=0xDFFF) => self.push_str("\u{FFFD}"), + (None, u) => self.push_str(&char::from_u32(u32::from(u)).unwrap_or('\u{FFFD}').to_string()), + } + } +} + +/// The text of an RTF document: paragraphs and lines as newlines, tabs, `\'hh` bytes in the +/// code page of the font they are set in (its `\fcharset`, else the document's `\ansicpg`, +/// Windows-1252 when it names none), `\uN` with its `\ucN` fallback skipped, and none of the +/// font table, colours, styles, pictures, objects, headers or footers, nor any `{\*…}` +/// destination (for encapsulated HTML, [MS-OXRTFEX], that leaves the text the document shows +/// and not the `\htmltag` HTML, and the text as it is for `\fromtext`). Control characters +/// other than the line end and the tab are dropped. At most `max` bytes of text; whether more +/// was cut is the second part of the answer. Groups nest at most [`MAX_RTF_DEPTH`] deep, else +/// an error; unbalanced braces are read as they come. +pub fn rtf_text(rtf: &[u8], max: usize) -> Result<(String, bool)> { + read(rtf, max, false).map(|(text, cut)| (tidy(&text), cut)) +} + +/// The HTML a document keeps when it is encapsulated HTML ([MS-OXRTFEX] 2.2.3): what its +/// `\htmltag` groups hold and the text no `\htmlrtf` suppresses, under the same caps as +/// [`rtf_text`]. `None` for any other document. The HTML is as hostile as the file: whoever +/// shows it reads it as any other message's. +pub fn rtf_html(rtf: &[u8], max: usize) -> Result> { + if !encapsulates_html(rtf) { + return Ok(None); + } + read(rtf, max, true).map(Some) +} + +/// Whether one of the document's first ten tokens says it was HTML (2.2.3.1). +fn encapsulates_html(rtf: &[u8]) -> bool { + let mut i = rtf.iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(rtf.len()); + if rtf.get(i..i.saturating_add(5)) != Some(b"{\\rtf") { + return false; + } + for _ in 0..10 { + match rtf.get(i) { + Some(b'{') => i = i.saturating_add(1), + Some(b'\\') => { + let from = i.saturating_add(1); + i = from; + while rtf.get(i).is_some_and(u8::is_ascii_alphabetic) { + i = i.saturating_add(1); + } + let word = rtf.get(from..i).unwrap_or_default(); + if word.is_empty() { + return false; + } + let digits = i; + while rtf.get(i).is_some_and(u8::is_ascii_digit) { + i = i.saturating_add(1); + } + match word { + b"fromhtml" => return rtf.get(digits..i) != Some(b"0"), + b"fromtext" => return false, + _ => {} + } + if rtf.get(i) == Some(&b' ') { + i = i.saturating_add(1); + } + } + _ => return false, + } + } + false +} + +fn read(rtf: &[u8], max: usize, html: bool) -> Result<(String, bool)> { + let start = rtf.iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(rtf.len()); + if rtf.get(start..start.saturating_add(5)) != Some(b"{\\rtf") { + return Err(Error::Corrupt("not rich text")); + } + let mut out = Out::new(max); + out.html = html; + let mut stack: Vec = Vec::new(); + let mut cur = Group { uc: 1, skip: false, fonttbl: false, defining: 0, font: None, suppressed: false, in_tag: false }; + let mut star = false; + let mut first = false; + let mut skip_chars = 0u32; + let mut i = start; + while let Some(&b) = rtf.get(i) { + i = i.saturating_add(1); + match b { + b'{' => { + if stack.len() >= MAX_RTF_DEPTH { + return Err(Error::Corrupt("rich text nested too deeply")); + } + stack.push(cur); + first = true; + skip_chars = 0; + } + b'}' => { + out.flush(); + out.flush_high(); + if let Some(g) = stack.pop() { + cur = g; + } + first = false; + skip_chars = 0; + } + b'\r' | b'\n' => {} + b'\\' => { + let Some(&n) = rtf.get(i) else { break }; + i = i.saturating_add(1); + match n { + b'\\' | b'{' | b'}' => { + first = false; + if !cur.hidden() && !take(&mut skip_chars) { + out.char(char::from(n)); + } + } + // A backslash before a line end is a paragraph end. + b'\r' | b'\n' => { + first = false; + if !cur.hidden() { + out.char('\n'); + } + } + b'\'' => { + first = false; + let hex = + rtf.get(i..i.saturating_add(2)).and_then(|h| std::str::from_utf8(h).ok()).and_then(|h| u8::from_str_radix(h, 16).ok()); + if let Some(v) = hex { + i = i.saturating_add(2); + if !cur.hidden() && !take(&mut skip_chars) { + let cp = out.code_page(cur.font); + out.byte(v, cp); + } + } + } + b'*' => { + cur.skip = true; + first = false; + star = true; + } + b'~' => symbol(&mut out, &cur, '\u{A0}', &mut first), + b'_' => { + let hyphen = if out.html { '\u{AD}' } else { '\u{2011}' }; + symbol(&mut out, &cur, hyphen, &mut first); + } + b'-' | b'|' | b':' => first = false, + n if n.is_ascii_alphabetic() => { + let from = i.saturating_sub(1); + while rtf.get(i).is_some_and(u8::is_ascii_alphabetic) { + i = i.saturating_add(1); + } + let word = rtf.get(from..i).and_then(|w| std::str::from_utf8(w).ok()).unwrap_or_default(); + let word = if word.len() > 32 { word.get(..32).unwrap_or(word) } else { word }; + // An optional parameter, then at most one space of delimiter. + let mut neg = false; + if rtf.get(i) == Some(&b'-') { + neg = true; + i = i.saturating_add(1); + } + let mut num: Option = None; + while let Some(d) = rtf.get(i).filter(|d| d.is_ascii_digit()) { + num = Some(num.unwrap_or(0).saturating_mul(10).saturating_add(i64::from(d.saturating_sub(b'0'))).min(1 << 40)); + i = i.saturating_add(1); + } + let num = num.map(|v| if neg { v.saturating_neg() } else { v }); + if rtf.get(i) == Some(&b' ') { + i = i.saturating_add(1); + } + let was_first = std::mem::replace(&mut first, false); + // The HTML of a tag is read whatever suppresses what is round it, by the + // document's own code page (2.2.3.2). + if html && std::mem::take(&mut star) && word == "htmltag" && !stack.last().is_some_and(|g| g.skip) { + cur.skip = false; + cur.in_tag = true; + cur.suppressed = false; + cur.font = None; + } + if was_first && SKIPPED.contains(&word.to_ascii_lowercase().as_str()) { + cur.skip = true; + cur.fonttbl = word.eq_ignore_ascii_case("fonttbl"); + } + skip_chars = 0; + control(word, num, &mut cur, &mut out, &mut skip_chars, &mut i, rtf); + } + _ => first = false, + } + } + b'\t' => { + first = false; + if !cur.hidden() && !take(&mut skip_chars) { + out.char('\t'); + } + } + 0x00..=0x1F => {} + _ => { + first = false; + if cur.hidden() || take(&mut skip_chars) { + continue; + } + if b.is_ascii() { + out.char(char::from(b)); + } else { + let cp = out.code_page(cur.font); + out.byte(b, cp); + } + } + } + } + out.flush(); + out.flush_high(); + let cut = out.cut; + Ok((out.text, cut)) +} + +fn take(skip: &mut u32) -> bool { + if *skip > 0 { + *skip = skip.saturating_sub(1); + true + } else { + false + } +} + +fn symbol(out: &mut Out, cur: &Group, c: char, first: &mut bool) { + *first = false; + if !cur.hidden() { + out.char(c); + } +} + +fn control(word: &str, num: Option, cur: &mut Group, out: &mut Out, skip_chars: &mut u32, i: &mut usize, rtf: &[u8]) { + match word { + "ansicpg" => out.ansicpg = num.and_then(|n| u32::try_from(n).ok()), + "mac" => out.mac = true, + // The fonts: which character set each has, and which one the text is in. + "f" if cur.fonttbl => cur.defining = num.unwrap_or(0), + "fcharset" if cur.fonttbl => { + if let Some(cs) = num + && (out.fonts.len() < MAX_FONTS || out.fonts.contains_key(&cur.defining)) + { + out.fonts.insert(cur.defining, cs); + } + } + "f" if !cur.skip => cur.font = num, + "deff" => out.deff = num, + "plain" if !cur.skip => cur.font = None, + "uc" => cur.uc = num.and_then(|n| u32::try_from(n).ok()).unwrap_or(1).min(16), + "htmlrtf" if out.html && !cur.in_tag => cur.suppressed = num != Some(0), + "u" if !cur.hidden() => { + if let Some(n) = num { + // A negative number is the unit minus 65536; the fallback after it is skipped. + let unit = if n < 0 { n.saturating_add(65536) } else { n }; + out.unit(u16::try_from(unit & 0xFFFF).unwrap_or(0xFFFD)); + *skip_chars = cur.uc; + } + } + "bin" => { + let n = num.and_then(|n| usize::try_from(n).ok()).unwrap_or(0); + *i = i.saturating_add(n).min(rtf.len()); + } + "par" | "line" if !cur.hidden() => out.char('\n'), + "sect" | "page" | "row" if !cur.hidden() && !out.html => out.char('\n'), + "tab" if !cur.hidden() => out.char('\t'), + "cell" if !cur.hidden() && !out.html => out.char('\t'), + "emdash" if !cur.hidden() => out.char('\u{2014}'), + "endash" if !cur.hidden() => out.char('\u{2013}'), + "lquote" if !cur.hidden() => out.char('\u{2018}'), + "rquote" if !cur.hidden() => out.char('\u{2019}'), + "ldblquote" if !cur.hidden() => out.char('\u{201C}'), + "rdblquote" if !cur.hidden() => out.char('\u{201D}'), + "bullet" if !cur.hidden() => out.char('\u{2022}'), + "emspace" | "enspace" | "qmspace" if !cur.hidden() => out.char(' '), + _ => {} + } +} + +/// At most one blank line in a row, and no blanks at the ends. +fn tidy(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut newlines = 0u32; + for c in s.chars() { + if c == '\n' { + newlines = newlines.saturating_add(1); + if newlines > 2 { + continue; + } + } else { + newlines = 0; + } + out.push(c); + } + out.trim().to_string() +} + +#[cfg(test)] +#[path = "outlook_tests.rs"] +mod tests; diff --git a/crates/rtf/src/outlook_tests.rs b/crates/rtf/src/outlook_tests.rs new file mode 100644 index 000000000..7eb41dbd2 --- /dev/null +++ b/crates/rtf/src/outlook_tests.rs @@ -0,0 +1,385 @@ +//! Compressed rich text ([MS-OXRTFCP] section 3 samples) and the text of RTF. + +use proptest::prelude::*; + +use super::{Error, MAX_RTF_DEPTH, crc32, decompress_rtf, rtf_html, rtf_text}; + +/// [MS-OXRTFCP] 3.1.1: a simple compressed RTF. +const EXAMPLE_1: [u8; 49] = [ + 0x2d, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x4c, 0x5a, 0x46, 0x75, 0xf1, 0xc5, 0xc7, 0xa7, 0x03, 0x00, 0x0a, 0x00, 0x72, 0x63, 0x70, 0x67, + 0x31, 0x32, 0x35, 0x42, 0x32, 0x0a, 0xf3, 0x20, 0x68, 0x65, 0x6c, 0x09, 0x00, 0x20, 0x62, 0x77, 0x05, 0xb0, 0x6c, 0x64, 0x7d, 0x0a, 0x80, 0x0f, + 0xa0, +]; + +/// [MS-OXRTFCP] 3.1.2: a reference that reads past the write position. +const EXAMPLE_2: [u8; 30] = [ + 0x1a, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x4c, 0x5a, 0x46, 0x75, 0xe2, 0xd4, 0x4b, 0x51, 0x41, 0x00, 0x04, 0x20, 0x57, 0x58, 0x59, 0x5a, + 0x0d, 0x6e, 0x7d, 0x01, 0x0e, 0xb0, +]; + +fn mela(rtf: &[u8]) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&u32::try_from(rtf.len() + 12).unwrap().to_le_bytes()); + v.extend_from_slice(&u32::try_from(rtf.len()).unwrap().to_le_bytes()); + v.extend_from_slice(b"MELA"); + v.extend_from_slice(&[0; 4]); + v.extend_from_slice(rtf); + v +} + +#[test] +fn spec_example_1_decompresses() { + let out = decompress_rtf(&EXAMPLE_1, 1 << 20).unwrap(); + assert_eq!(out, b"{\\rtf1\\ansi\\ansicpg1252\\pard hello world}\r\n"); + assert_eq!(crc32(&EXAMPLE_1[16..]), 0xa7c7_c5f1); + assert_eq!(rtf_text(&out, 1000).unwrap(), ("hello world".to_string(), false)); +} + +#[test] +fn spec_example_2_reads_past_the_write_position() { + let out = decompress_rtf(&EXAMPLE_2, 1 << 20).unwrap(); + assert_eq!(out, b"{\\rtf1 WXYZWXYZWXYZWXYZWXYZ}"); + assert_eq!(out.len(), 0x1c); +} + +#[test] +fn uncompressed_is_taken_as_it_is() { + let rtf = b"{\\rtf1 plain}"; + assert_eq!(decompress_rtf(&mela(rtf), 100).unwrap(), rtf); + // A header that lies about its size is held to what is there. + let mut lying = mela(rtf); + lying.get_mut(4..8).unwrap().copy_from_slice(&u32::MAX.to_le_bytes()); + assert_eq!(decompress_rtf(&lying, 100).unwrap(), rtf); + assert!(matches!(decompress_rtf(&mela(rtf), 3), Err(Error::TooLarge))); +} + +#[test] +fn a_bad_crc_is_an_error() { + let mut bad = EXAMPLE_1; + bad[12] ^= 1; + assert!(matches!(decompress_rtf(&bad, 1 << 20), Err(Error::BadChecksum))); + let mut bad = EXAMPLE_1; + bad[20] ^= 0x40; + assert!(decompress_rtf(&bad, 1 << 20).is_err()); +} + +#[test] +fn truncation_is_an_error_at_every_length() { + for n in 0..EXAMPLE_1.len() { + assert!(decompress_rtf(&EXAMPLE_1[..n], 1 << 20).is_err(), "{n}"); + } + // A header whose size is right but whose runs have no end mark. + let mut v = EXAMPLE_1[..40].to_vec(); + v[..4].copy_from_slice(&28u32.to_le_bytes()); + assert!(decompress_rtf(&v, 1 << 20).is_err()); +} + +#[test] +fn a_header_that_lies_about_the_size_costs_nothing() { + let mut huge = EXAMPLE_1; + huge[4..8].copy_from_slice(&u32::MAX.to_le_bytes()); + assert!(matches!(decompress_rtf(&huge, 1 << 20), Err(Error::TooLarge))); + // Under the cap it is the stream that decides. + let mut small = EXAMPLE_1; + small[4..8].copy_from_slice(&1000u32.to_le_bytes()); + assert_eq!(decompress_rtf(&small, 1 << 20).unwrap().len(), 0x2b); +} + +#[test] +fn output_is_capped_while_it_is_made() { + assert!(matches!(decompress_rtf(&EXAMPLE_1, 10), Err(Error::TooLarge))); + // A long stream (no end mark) is stopped at the cap, not at its end. + let mut body = Vec::new(); + for _ in 0..200_000 { + body.push(0x00); + body.extend_from_slice(b"abcdefgh"); + } + let mut v = Vec::new(); + v.extend_from_slice(&u32::try_from(body.len() + 12).unwrap().to_le_bytes()); + v.extend_from_slice(&0u32.to_le_bytes()); + v.extend_from_slice(b"LZFu"); + v.extend_from_slice(&crc32(&body).to_le_bytes()); + v.extend_from_slice(&body); + assert!(matches!(decompress_rtf(&v, 1 << 16), Err(Error::TooLarge))); +} + +#[test] +fn an_unknown_kind_is_an_error() { + let mut v = EXAMPLE_1; + v[8..12].copy_from_slice(b"XXXX"); + assert!(matches!(decompress_rtf(&v, 100), Err(Error::Corrupt(_)))); +} + +fn text(rtf: &str) -> String { + rtf_text(rtf.as_bytes(), 1 << 20).unwrap().0 +} + +#[test] +fn paragraphs_and_tabs() { + assert_eq!(text("{\\rtf1\\ansi\\deff0 One\\par Two\\line Three\\tab x\\par}"), "One\nTwo\nThree\tx"); + assert_eq!(text("{\\rtf1 a\\par\\par\\par\\par\\par b}"), "a\n\nb"); + assert_eq!(text("{\\rtf1 braces \\{ \\} and \\\\ end}"), "braces { } and \\ end"); + assert_eq!(text("{\\rtf1 line\r\nbreaks in the file are not breaks}"), "linebreaks in the file are not breaks"); +} + +#[test] +fn hex_escapes_use_the_code_page() { + assert_eq!(text("{\\rtf1\\ansi\\ansicpg1252 caf\\'e9 \\'80}"), "caf\u{e9} \u{20ac}"); + assert_eq!(text("{\\rtf1\\ansi\\ansicpg1251 \\'cf\\'f0\\'e8\\'e2\\'e5\\'f2}"), "\u{41f}\u{440}\u{438}\u{432}\u{435}\u{442}"); + // Two bytes of one character (Shift JIS). + assert_eq!(text("{\\rtf1\\ansi\\ansicpg932 \\'82\\'a0}"), "\u{3042}"); + // No code page: Windows-1252. + assert_eq!(text("{\\rtf1 \\'e9}"), "\u{e9}"); +} + +#[test] +fn unicode_escapes_skip_their_fallback() { + assert_eq!(text("{\\rtf1\\uc1\\u8364? x}"), "\u{20ac} x"); + assert_eq!(text("{\\rtf1\\uc2\\u8364\\'80\\'80 x}"), "\u{20ac} x"); + assert_eq!(text("{\\rtf1\\uc0\\u8364 x}"), "\u{20ac}x"); + assert_eq!(text("{\\rtf1\\u-3913 ?}"), "\u{f0b7}"); + assert_eq!(text("{\\rtf1\\u-10179?\\u-8704? x}"), "\u{1f600} x"); + // A lone surrogate is a replacement character, not a panic. + assert_eq!(text("{\\rtf1\\u-10179? x}"), "\u{fffd} x"); + // \ucN belongs to its group. + assert_eq!(text("{\\rtf1{\\uc0}\\u65?B}"), "AB"); +} + +#[test] +fn nested_groups_and_skipped_destinations() { + let rtf = "{\\rtf1\\ansi{\\fonttbl{\\f0 Arial;}}{\\colortbl;\\red0\\green0\\blue0;}{\\stylesheet{\\s1 Heading;}}{\\*\\generator Hidden;}{\\info{\\title T}}\ + {\\pict\\wmetafile8 0102030405}{\\header Head}\\pard{\\b bold {\\i both}} plain{\\*\\htmltag64 }}"; + assert_eq!(text(rtf), "bold both plain"); +} + +#[test] +fn encapsulated_text_and_html_give_what_the_document_shows() { + assert_eq!(text("{\\rtf1\\ansi\\fromtext\\deff0{\\fonttbl{\\f0\\fswiss Arial;}}\\pard Hello\\par}"), "Hello"); + let html = "{\\rtf1\\ansi\\fromhtml1 {\\*\\htmltag19 }\\htmlrtf {\\htmlrtf0 Hello {\\*\\htmltag84 }there\\htmlrtf \\par\\htmlrtf0 }}"; + assert_eq!(text(html), "Hello there"); +} + +#[test] +fn bin_data_is_skipped_by_its_count() { + let mut rtf = b"{\\rtf1 a{\\pict\\bin5 }}{b".to_vec(); + let at = rtf.iter().position(|&b| b == b'5').unwrap() + 2; + rtf.splice(at..at, *b"}{\\par"); + assert_eq!(rtf_text(&rtf, 100).unwrap().0, "ab"); + // A count past the end is the end. + assert_eq!(rtf_text(b"{\\rtf1 a\\bin99999999999 bc", 100).unwrap().0, "a"); +} + +#[test] +fn bad_input_is_an_error_or_what_could_be_read() { + assert!(rtf_text(b"plain text", 100).is_err()); + assert!(rtf_text(b"", 100).is_err()); + assert_eq!(text("{\\rtf1 open {group {never closed"), "open group never closed"); + assert_eq!(text("{\\rtf1 closed}} too} many} text"), "closed too many text"); + assert_eq!(text("{\\rtf1 ends in a slash\\"), "ends in a slash"); + assert_eq!(text("{\\rtf1 \\'zz bad hex}"), "zz bad hex"); +} + +#[test] +fn depth_is_capped() { + let deep = format!("{{\\rtf1 {}x{}", "{".repeat(MAX_RTF_DEPTH + 10), "}".repeat(MAX_RTF_DEPTH + 10)); + assert!(matches!(rtf_text(deep.as_bytes(), 100), Err(Error::Corrupt(_)))); + let ok = format!("{{\\rtf1 {}x{}", "{".repeat(100), "}".repeat(101)); + assert_eq!(text(&ok), "x"); +} + +#[test] +fn text_is_capped() { + let long = format!("{{\\rtf1 {}}}", "word ".repeat(100_000)); + let (t, cut) = rtf_text(long.as_bytes(), 1000).unwrap(); + assert!(t.len() <= 1000 && cut); + assert!(!rtf_text(b"{\\rtf1 short}", 1000).unwrap().1); + // Folded blank lines still say the text was cut. + let folded = format!("{{\\rtf1 {}}}", "x\\par\\par\\par\\par ".repeat(1000)); + let (t, cut) = rtf_text(folded.as_bytes(), 1000).unwrap(); + assert!(cut && t.len() < 1000, "{}", t.len()); + let multibyte = format!("{{\\rtf1 {}}}", "\\u233?".repeat(10_000)); + let t = rtf_text(multibyte.as_bytes(), 101).unwrap().0; + assert!(t.len() <= 102 && t.chars().all(|c| c == '\u{e9}')); +} + +#[test] +fn each_font_has_its_code_page() { + // RichEdit's shape: ANSI document, a font of its own character set. + assert_eq!( + text("{\\rtf1\\ansi\\ansicpg1252{\\fonttbl{\\f0\\fnil\\fcharset128 Yu Gothic;}}\\f0\\'93\\'fa\\'96\\'7b\\'8c\\'ea}"), + "\u{65e5}\u{672c}\u{8a9e}" + ); + assert_eq!(text("{\\rtf1\\ansi\\ansicpg1252{\\fonttbl{\\f0\\fnil\\fcharset204 Calibri;}}\\f0\\'cf\\'f0\\'e8 x}"), "\u{41f}\u{440}\u{438} x"); + // The font goes with its group. + assert_eq!( + text("{\\rtf1\\ansi\\ansicpg1252{\\fonttbl{\\f0\\fcharset204 A;}{\\f1\\fcharset161 B;}}{\\f0 \\'cf}\\'cf{\\f1\\'e1}\\f0\\'cf}"), + "\u{41f}\u{cf}\u{3b1}\u{41f}" + ); + // A change of font in the middle of a run of bytes. + assert_eq!(text("{\\rtf1\\ansi{\\fonttbl{\\f0\\fcharset204 A;}{\\f1\\fcharset0 B;}}\\f0\\'cf\\f1\\'cf\\f0\\'cf}"), "\u{41f}\u{cf}\u{41f}"); + // A flat font table, \deff and no charset at all. + assert_eq!(text("{\\rtf1\\ansi\\deff1{\\fonttbl\\f0\\fcharset0 A;\\f1\\fcharset204 B;}\\'cf}"), "\u{41f}"); + assert_eq!(text("{\\rtf1\\ansi\\ansicpg1251{\\fonttbl{\\f0 A;}}\\f0\\'a8\\f9\\'a8}"), "\u{401}\u{401}"); +} + +#[test] +fn the_sample_of_ms_oxrtfex_reads_its_font_charset() { + // [MS-OXRTFEX] 3.1: a 1251 document, and one font marked \fcharset0 for ¨ (0xA8, U+00A8). + let rtf = "{\\rtf1\\ANSI\\ansicpg1251\\fromhtml1 \\deff0\r\n{\\fonttbl {\\f0\\fmodern Courier New;}{\\f1\\fswiss Arial;}{\\f2\\fswiss\\fcharset0 Arial;}}\ + \\uc1\\pard\\plain\\deftab360 \\f0\\fs24 {\\htmlrtf \\f1 \\htmlrtf0 \\'cf {\\*\\htmltag ¨}\\htmlrtf {\\f2\\'a8}\\htmlrtf0 \\'a8}}"; + assert_eq!(text(rtf), "\u{41f} \u{a8}\u{401}"); +} + +#[test] +fn bytes_past_the_buffer_are_not_dropped() { + let dbcs = format!("{{\\rtf1\\ansi\\ansicpg932 {}}}", "\\'82\\'a0".repeat(3000)); + assert_eq!(text(&dbcs), "\u{3042}".repeat(3000)); + // An odd byte first, so the 4096-byte mark falls inside a pair. + let shifted = format!("{{\\rtf1\\ansi\\ansicpg932 \\'41{}}}", "\\'82\\'a0".repeat(3000)); + assert_eq!(text(&shifted), format!("A{}", "\u{3042}".repeat(3000))); + let single = format!("{{\\rtf1\\ansi\\ansicpg1252 {}}}", "\\'e9".repeat(5000)); + assert_eq!(text(&single), "\u{e9}".repeat(5000)); + let raw: Vec = [b"{\\rtf1\\ansi\\ansicpg1252 ".as_slice(), &[0xE9u8; 5000], b"}"].concat(); + assert_eq!(rtf_text(&raw, 1 << 20).unwrap().0, "\u{e9}".repeat(5000)); +} + +#[test] +fn ansicpg_wins_over_mac_in_either_order() { + assert_eq!(text("{\\rtf1\\ansicpg1251\\mac \\'cf}"), "\u{41f}"); + assert_eq!(text("{\\rtf1\\mac\\ansicpg1251 \\'cf}"), "\u{41f}"); + assert_eq!(text("{\\rtf1\\mac \\'a5}"), "\u{2022}"); +} + +#[test] +fn control_characters_are_not_text() { + assert_eq!(text("{\\rtf1 a\\'00b\\u0? c\\'07\\u1\\u9? d\\u10?e}"), "ab c\t d\ne"); +} + +#[test] +fn a_literal_tab_and_a_backslash_line_end() { + assert_eq!(text("{\\rtf1 a\tb\\\r\nc\\\nd\\\re}"), "a\tb\nc\nd\ne"); +} + +#[test] +fn the_code_pages_of_the_mail_are_the_code_pages_of_the_text() { + // Pages the first version of the text reader lacked: KOI8-R, ISO 8859-2, GB 18030's, EUC-KR's other number. + assert_eq!(text("{\\rtf1\\ansi\\ansicpg20866 \\'d0\\'d2}"), "\u{43f}\u{440}"); + assert_eq!(text("{\\rtf1\\ansi\\ansicpg28592 \\'b1}"), "\u{105}"); + assert_eq!(text("{\\rtf1\\ansi\\ansicpg51949 \\'b0\\'a1}"), "\u{ac00}"); + assert_eq!(craft_mime::charset::code_page_label(20866), Some("koi8-r")); +} + +proptest! { + #![proptest_config(ProptestConfig { cases: 400, ..ProptestConfig::default() })] + + #[test] + fn arbitrary_bytes_never_panic(bytes in proptest::collection::vec(any::(), 0..4096)) { + let _ = decompress_rtf(&bytes, 1 << 16); + let _ = rtf_text(&bytes, 1 << 12); + let mut with_start = b"{\\rtf1".to_vec(); + with_start.extend_from_slice(&bytes); + let _ = rtf_text(&with_start, 1 << 12); + } + + #[test] + fn changed_streams_never_panic(at in 0usize..49, to in any::(), cut in 0usize..49) { + let mut v = EXAMPLE_1.to_vec(); + if let Some(b) = v.get_mut(at) { *b = to; } + v.truncate(cut.max(1)); + if let Ok(rtf) = decompress_rtf(&v, 1 << 12) { let _ = rtf_text(&rtf, 100); } + } + + #[test] + fn rtf_with_valid_start_is_bounded(body in "[\\\\{}a-z0-9 '*-]{0,600}", max in 0usize..300) { + let rtf = format!("{{\\rtf1 {body}"); + if let Ok((t, _)) = rtf_text(rtf.as_bytes(), max) { prop_assert!(t.len() <= max.saturating_add(4)); } + } +} + +/// The document of [MS-OXRTFEX] 2.1.3.1.3's kind: tags in groups, what isn't HTML suppressed. +const ENCAPSULATED: &[u8] = + b"{\\rtf1\\ansi\\ansicpg1252\\fromhtml1 \\deff0{\\fonttbl{\\f0\\fswiss Arial;}{\\f1\\fcharset204 Cyr;}}{\\colortbl;\\red0;}\ +{\\*\\htmltag64

}{\\htmlrtf \\f1 \\htmlrtf0 caf\\'e9 {\\*\\htmltag }{\\htmlrtf \\b \\htmlrtf0 bold{\\*\\htmltag }}\ +\\htmlrtf \\par \\htmlrtf0 {\\*\\htmltag &}\\htmlrtf &\\htmlrtf0 \\u8364?{\\*\\mhtmltag }{\\*\\other no}{\\*\\htmltag

}}}"; + +#[test] +fn encapsulated_html_comes_out_as_it_was() { + let (html, cut) = rtf_html(ENCAPSULATED, 1 << 16).unwrap().unwrap(); + assert!(!cut); + assert_eq!(html, "

caf\u{439} bold&\u{20AC}

"); +} + +#[test] +fn a_tag_is_kept_even_where_the_text_round_it_is_suppressed() { + let rtf = b"{\\rtf1\\fromhtml1 \\htmlrtf {\\*\\htmltag }gone\\htmlrtf0 kept}"; + assert_eq!(rtf_html(rtf, 100).unwrap().unwrap().0, "kept"); +} + +#[test] +fn only_a_document_that_says_so_is_html() { + assert_eq!(rtf_html(b"{\\rtf1\\ansi plain}", 100).unwrap(), None); + assert_eq!(rtf_html(b"{\\rtf1\\fromtext hello}", 100).unwrap(), None); + assert_eq!(rtf_html(b"{\\rtf1 text\\fromhtml1 }", 100).unwrap(), None, "past a token that isn't a control word"); + assert_eq!(rtf_html(b"{\\rtf1\\a\\b\\c\\d\\e\\f\\g\\h\\i\\j\\fromhtml1 }", 100).unwrap(), None, "past the first ten tokens"); + assert_eq!(rtf_html(b"not rich text", 100).unwrap(), None); +} + +#[test] +fn encapsulated_html_is_capped_and_a_deep_nest_refused() { + let mut rtf = b"{\\rtf1\\fromhtml1 ".to_vec(); + rtf.extend(std::iter::repeat_n(b'x', 1000)); + let (html, cut) = rtf_html(&rtf, 10).unwrap().unwrap(); + assert!(cut && html.len() <= 10); + let mut deep = b"{\\rtf1\\fromhtml1 ".to_vec(); + deep.extend(std::iter::repeat_n(b'{', MAX_RTF_DEPTH + 2)); + assert!(rtf_html(&deep, 100).is_err()); +} + +#[test] +fn the_text_of_encapsulated_html_is_still_what_it_shows() { + let rtf = b"{\\rtf1\\fromhtml1 {\\*\\htmltag

}\\htmlrtf x\\htmlrtf0 hello}"; + assert_eq!(rtf_text(rtf, 100).unwrap().0, "xhello"); +} + +proptest! { + #[test] + fn any_bytes_read_as_html_never_panic(data in proptest::collection::vec(any::(), 0..2000)) { + let mut rtf = b"{\\rtf1\\fromhtml1 ".to_vec(); + rtf.extend_from_slice(&data); + let _ = rtf_html(&rtf, 1 << 12); + } +} + +/// The font character sets are the crate's table, but for ANSI (0), which is Windows-1252 +/// whatever the document's code page, and the sets with no text code page here. +#[test] +fn font_character_sets_name_their_code_pages() { + let want = [ + (0, 1252), + (77, 10000), + (128, 932), + (129, 949), + (134, 936), + (136, 950), + (161, 1253), + (162, 1254), + (163, 1258), + (177, 1255), + (178, 1256), + (186, 1257), + (204, 1251), + (222, 874), + (238, 1250), + ]; + for charset in -1..=256 { + let expected = want.iter().find(|(c, _)| *c == charset).map(|(_, cp)| *cp); + assert_eq!(super::charset_code_page(charset), expected, "{charset}"); + } +} + +#[test] +fn the_checksum_is_the_specifications() { + // [MS-OXRTFCP] 3.2.3: the CRC of the compressed bytes of the first example is its header's. + assert_eq!(crc32(&EXAMPLE_1[16..]), u32::from_le_bytes([0xf1, 0xc5, 0xc7, 0xa7])); + assert_eq!(crc32(b""), 0); +} From 2b35d243241bc4c1aaa7a1cc89657c1aad5b3d02 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:25:58 -0500 Subject: [PATCH 16/31] =?UTF-8?q?craft-audio:=20an=20AAC=20program=20confi?= =?UTF-8?q?g=20element=20shaped=20like=203.0=E2=80=935.1=20gets=20those=20?= =?UTF-8?q?speakers=20(#964)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * craft-audio: a program config element in the shape of a channel configuration gets its speakers An AAC stream with channel configuration 0 whose program config element lists a centre, a front pair and then exactly the elements of configuration 3 to 6 decoded as C L R Ls Rs LFE and was reported as an unordered layout, so a caller reading it in WAV order sent the centre to the left speaker. Such a stream now has the configuration's layout and WAV order. Any other element list keeps the decoded order, as before. Ported from storytold/filmcraft#4280 (aac_stream_order in filmcraft-codecs). * craft-audio: the ADTS program config oracle expects 5.1 in WAV order ffmpeg 9 writes its 5.1 program config element centre first, so the stream now has 5.1's speakers: each decoded channel must match ffmpeg's channel of the same index. An element list in another order keeps the old check. --------- Co-authored-by: mpratsaki <110930793+mpratsaki@users.noreply.github.com> --- crates/audio/src/aac/mod.rs | 44 +++++++++++++++++++++ crates/audio/src/track.rs | 5 ++- crates/audio/tests/codecs/routing_oracle.rs | 14 +++++-- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/crates/audio/src/aac/mod.rs b/crates/audio/src/aac/mod.rs index eb1c7d40a..34b081ea6 100644 --- a/crates/audio/src/aac/mod.rs +++ b/crates/audio/src/aac/mod.rs @@ -113,6 +113,26 @@ pub fn wav_order(channel_config: u8) -> Option<&'static [usize]> { }) } +/// The channel configuration whose speakers a stream has: its own, or for configuration 0 the +/// configuration 3–6 that its program config element spells out. A program config element lists +/// front, side, back and LFE elements in that order, so one that opens with a centre and a front +/// pair and goes on with exactly the elements of a configuration has that configuration's speakers +/// (and [`wav_order`]). Any other list has no speaker order that can be told, and gives 0: ffmpeg's +/// 5.1(side), for one, lists the pair first and already decodes in WAV order. +#[cfg(feature = "aac-decode")] +pub fn speaker_config(asc: &AudioSpecificConfig) -> u8 { + if asc.channel_config != 0 { + return asc.channel_config; + } + let centre_then_pair = asc.pce.as_ref().is_some_and(|p| p.front.iter().map(|e| e.0).eq([false, true])); + if !centre_then_pair { + return 0; + } + let elements = asc.layout(); + let same = |config: &u8| config_layout(*config).is_some_and(|l| l.iter().map(|e| e.0).eq(elements.iter().map(|e| e.0))); + (3..=6).find(same).unwrap_or(0) +} + /// The order the [`Encoder`] takes its input in, from WAV order: for each encoder input channel /// (AAC element order, see [`EncoderConfig::channels`]), the WAV-order channel to feed it. The /// inverse of [`wav_order`] for 3 to 7 channels; `None` for mono and stereo (already in order) and @@ -279,6 +299,30 @@ mod tests { mod order_tests { use super::*; + /// A program config element in the shape of a channel configuration has that configuration's + /// speakers, with the centre first. ffmpeg's 5.1(side) lists its pair first and already decodes + /// in WAV order, so it stays as it is. + #[cfg(feature = "aac-decode")] + #[test] + fn a_program_config_element_takes_the_configuration_it_matches() { + let with = |pce| AudioSpecificConfig { pce: Some(pce), ..AudioSpecificConfig::lc(3, 0) }; + let centre_first = ProgramConfig { front: vec![(false, 0), (true, 0)], back: vec![(true, 1)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(centre_first)), 6); + let three = ProgramConfig { front: vec![(false, 0), (true, 0)], ..Default::default() }; + assert_eq!(speaker_config(&with(three)), 3); + let ffmpeg = ProgramConfig { front: vec![(true, 0), (false, 0)], side: vec![(false, 1)], back: vec![(true, 1)], ..Default::default() }; + assert_eq!(speaker_config(&with(ffmpeg)), 0); + // the same element types with no front pair: the side pair is not left and right + let no_front_pair = + ProgramConfig { front: vec![(false, 0)], side: vec![(true, 0)], back: vec![(true, 1)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(no_front_pair)), 0); + // 7.1 in a program config element is not one of the unambiguous configurations + let wide = + ProgramConfig { front: vec![(false, 0), (true, 0)], side: vec![(true, 1)], back: vec![(true, 2)], lfe: vec![0], ..Default::default() }; + assert_eq!(speaker_config(&with(wide)), 0); + assert_eq!(speaker_config(&AudioSpecificConfig::lc(3, 6)), 6); + } + /// `aac_order` undoes `wav_order` for each channel count the encoder maps to a configuration /// (3–6 channels: configurations 3–6; 7: configuration 11). #[test] diff --git a/crates/audio/src/track.rs b/crates/audio/src/track.rs index 3d9ed7dfb..af984d718 100644 --- a/crates/audio/src/track.rs +++ b/crates/audio/src/track.rs @@ -156,7 +156,8 @@ impl TrackDecoder { let codec = if up.is_some() { CodecId::HeAac } else { CodecId::Aac }; // Only twice the core rate is upsampled to; any other signalled rate plays the core. let out = if up.is_some() { out } else { dec.sample_rate() }; - let cfg = dec.config().channel_config; + // a program config element that spells out a configuration has its speakers + let cfg = aac::speaker_config(dec.config()); let layout = aac_layout(cfg, dec.channels()); let order = layout.is_ordered().then(|| aac::wav_order(cfg)).flatten(); (Inner::Aac { dec: Box::new(dec), up, order, adts_format }, codec, out, layout, SampleFormat::Compressed) @@ -1079,6 +1080,8 @@ mod tests { let mut dec = TrackDecoder::new(&aac_params(), &[&frame]).unwrap(); assert_eq!(dec.channels(), 3); assert_eq!(dec.decode(&frame).unwrap().channels.len(), 3); + // its element lists a centre and a front pair, as configuration 3 does: L R C, in WAV order + assert_eq!(dec.layout(), ChannelLayout::SURROUND_3_0); // no program config element where one is needed let bare = aac::AdtsHeader::write(2, 3, 0, 1, 0x7FF).unwrap(); let bare = [&bare[..], &[0xE0]].concat(); diff --git a/crates/audio/tests/codecs/routing_oracle.rs b/crates/audio/tests/codecs/routing_oracle.rs index c35a89002..6472979b3 100644 --- a/crates/audio/tests/codecs/routing_oracle.rs +++ b/crates/audio/tests/codecs/routing_oracle.rs @@ -669,10 +669,14 @@ fn adts_with_a_program_config_element_decodes() { assert_eq!(bytes[3] >> 6 | (bytes[2] & 1) << 2, 0, "the fixture signals its layout in a PCE"); let (info, buf) = decode(&bytes, None).unwrap(); assert_eq!((info.container, info.codec, buf.channels.len()), (c.container, c.codec, c.channels)); - assert_eq!(info.layout, c.layout); + // A program config element that lists a centre, a front pair and then the elements of 5.1 has + // 5.1's speakers, in WAV order like ffmpeg's decode; any other list keeps the decoded order, + // which ffmpeg versions write differently. + let ordered = info.layout.is_ordered(); + assert_eq!(info.layout, if ordered { ChannelLayout::SURROUND_5_1_SIDE } else { c.layout }); assert_eq!(buf.frames(), reference(&ff, &path).len() / c.channels); - // A program config's channels come in its own order, and which order ffmpeg writes differs by - // version: each of ours must match a different one of ffmpeg's. + // Each of our channels must match a different one of ffmpeg's: the same one when the layout is + // ordered. let refr = reference(&ff, &path); let mut matched = Vec::new(); for plane in &buf.channels { @@ -686,7 +690,9 @@ fn adts_with_a_program_config_element_decodes() { assert!(s >= c.min_snr, "channel {} matches ffmpeg's {best} at only {s:.1} dB", matched.len()); matched.push(best); } - matched.sort_unstable(); + if !ordered { + matched.sort_unstable(); + } assert_eq!(matched, (0..c.channels).collect::>()); } From 69d812dd7aeff92e94be339a1b1f3c708735a71c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:29:51 -0500 Subject: [PATCH 17/31] craft-raw: Nikon NEF and NRW (#965) * craft-raw: the file-format layer of camera raw files, with DNG and CRW A new crate for #587 and #778: bytes in, the sensor data out as a neutral RawImage (samples, CFA, levels, active area, crop, orientation, colour tags, opcode lists, profile tables), and back for DNG. One Cargo feature per format: dng (read and write), jxl (JPEG XL DNG), crw (through craft-crw). The base is LightCraft's crates/raw at 5ff567a4: dng, dngwrite, tiffraw, unpack, ljpeg, jxl, semantic, preview, the parsing half of opcodes, profile and gaintable, the model and probe. Demosaicing and everything after it stay with the develop pipeline. Ported from PhotoCraft's and FilmCraft's DNG readers, each with a test: the default crop's per-tag defaults, an unusable crop falling back to the active area, non-finite white levels, the white default of a linearized image, the AsShotWhiteXY range, 8-bit-only lossy JPEG, lossy tiles that must cover their part of the image, and the CinemaDNG frame tags. * craft-raw: Hasselblad 3FR and FFF (feature hasselblad) RawFormat::CfaTiff decodes: the DNG-layout reader plus what these files add. Uncompressed data or Hasselblad's lossless JPEG (selection value 8), RGGB when no CFAPattern is stated, an FFF's colour matrix from the 3FR header it embeds, and the 8-bit RGB preview handed on as a baseline TIFF. Moved from LightCraft's vendor/hasselblad.rs at 5ff567a4 with its tests. Ported from PhotoCraft's reader: four BlackLevel values without a BlackLevelRepeatDim are one per CFA site in a file that is not a DNG. * craft-raw: Minolta MRW, Epson ERF, Mamiya MEF and Kodak DCS520C Four formats only LightCraft decodes, one feature each (mrw, erf, mef, kodak), moved from its vendor modules at 5ff567a4 with their tests; the MRW block container comes from lightcraft-meta. src/shared.rs holds the level helpers the maker decoders share. ERF and MEF state no white balance: the estimate LightCraft made from the preview is develop-side work, so wb_multipliers is None for them here. * craft-raw: shared scan budget for tagged previews, tile-sized lossy JPEG budget, bound on Deflate tile padding Ported from storytold/lightcraft#3695. Candidate preview ranges are deduplicated and charged to one byte budget and one header-walk budget; a lossy JPEG decoded for a tile may hold no more samples than the tile rounded up to whole MCUs (and never more than Limits::max_lossy_tile_samples); a Deflate chunk whose nominal size is mostly padding outside the image is refused before it is inflated. * craft-raw: lossless JPEG sample bound per referenced byte; previews larger than the file are passed over Ported from storytold/lightcraft#3910 (the parts whose code is in craft-raw today). check_image refuses a lossless JPEG image that declares more than 32 samples per referenced byte, counted once per chunk that points at the bytes so identical tiles stored once still decode. The Hasselblad RGB preview search passes over an image whose pixel bytes exceed the file and still takes a smaller complete one. * craft-raw: test for the Deflate padding bound * craft-raw: Pentax PEF and Samsung SRW Moved from LightCraft crates/raw (vendor/pef.rs, vendor/srw.rs) at 5ff567a4. The Huffman difference decoder and bit reader that PEF shares with the Nikon, Panasonic and Sony codings become the huffman module. * craft-raw: Canon CR2, with sRAW and mRAW Moved from LightCraft crates/raw (vendor/cr2.rs, vendor/cr2/sraw.rs) at 5ff567a4. The white-balance presets a CR2 lists come back as data in ColorData::kelvin_presets (the ordering from LightCraft's kelvin_presets.rs; the temperature and tint arithmetic stays with the developer). * craft-raw: an sRAW preview larger than the file is refused before its buffer is reserved Ported from storytold/lightcraft#3910 (cr2::sraw::read_preview). * WIP craft-raw: Nikon NEF/NRW (does not compile yet: limits threading, develop() calls in tests) * craft-raw: Nikon NEF/NRW, with PhotoCraft's maker-note compression routing and 12-bit D4 white level --------- Co-authored-by: radialmonster --- NOTICE | 2 +- README.md | 3 +- crates/raw/Cargo.toml | 6 +- crates/raw/README.md | 10 + crates/raw/docs/nikon-lens-corrections.md | 101 ++ crates/raw/src/lib.rs | 10 + crates/raw/src/nef.rs | 1510 +++++++++++++++++++++ crates/raw/src/nef_lens.rs | 259 ++++ crates/raw/src/nefc.rs | 794 +++++++++++ crates/raw/tests/it/vendor_robust.rs | 2 + 10 files changed, 2692 insertions(+), 5 deletions(-) create mode 100644 crates/raw/docs/nikon-lens-corrections.md create mode 100644 crates/raw/src/nef.rs create mode 100644 crates/raw/src/nef_lens.rs create mode 100644 crates/raw/src/nefc.rs diff --git a/NOTICE b/NOTICE index 3bca3ae0d..eb22134a2 100644 --- a/NOTICE +++ b/NOTICE @@ -93,7 +93,7 @@ crates/raw (craft-raw) is moved from LightCraft, with behaviour ported from two - LightCraft, crates/raw (src/lib.rs, dng.rs, dngwrite.rs, tiffraw.rs, unpack.rs, ljpeg.rs, jxl.rs, opcodes.rs, profile.rs, gaintable.rs, semantic.rs, preview.rs, vendor/crw.rs, vendor/hasselblad.rs, vendor.rs (shared level helpers), vendor/mrw.rs, vendor/erf.rs, vendor/mef.rs, vendor/kodak.rs, vendor/pef.rs, - vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), + vendor/srw.rs, vendor/cr2.rs, vendor/cr2/sraw.rs, vendor/nef.rs, vendor/nefc.rs, vendor/nef_lens.rs, docs/nikon-lens-corrections.md, kelvin_presets.rs (the preset ordering), tests/it/vendor_robust.rs and their tests) and crates/meta/src/mrw.rs (the MRW block container), revision 5ff567a4a2e067bef1b6bce152c11e7f727ccbaf. https://github.com/storytold/lightcraft Copyright (c) 2026 ArtCraft Team and the LightCraft contributors. diff --git a/README.md b/README.md index 5b51e5ebd..951fc1ab9 100644 --- a/README.md +++ b/README.md @@ -134,8 +134,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Result { RawFormat::Mef => mef::decode(bytes, mode, limits), #[cfg(feature = "cr2")] RawFormat::Cr2 => cr2::decode(bytes, mode, limits), + #[cfg(feature = "nef")] + RawFormat::Nef | RawFormat::Nrw => nef::decode(bytes, mode, limits), #[cfg(feature = "pef")] RawFormat::Pef => pef::decode(bytes, mode, limits), #[cfg(feature = "srw")] diff --git a/crates/raw/src/nef.rs b/crates/raw/src/nef.rs new file mode 100644 index 000000000..86ef0e1b2 --- /dev/null +++ b/crates/raw/src/nef.rs @@ -0,0 +1,1510 @@ +//! Nikon NEF / NRW — uncompressed and Huffman-compressed (lossless, lossy) variants. +//! +//! Sources: TIFF 6.0 (the raw image is a standard CFA SubIFD), Laurent Clévy's NEF structure notes (prose: IFD +//! layout, SubIFDs, maker note header) and the ExifTool Nikon tag-name documentation (`0x000c` WB_RBLevels, +//! `0x003d` BlackLevel, `0x0096` NEFLinearizationTable). The Huffman-compressed data (compression 34713) is decoded +//! by [`crate::nefc`], which documents its clean-room sources; "lossy after split" files are decoded when their strip follows the +//! rule documented there; files it can't decode are reported as [`RawError::Unsupported`] and their embedded previews still work. So are High Efficiency NEFs +//! (HE / HE★, Z 8, Z 9, Z 6III, Z f): they keep compression 34713 but carry a wavelet codestream that starts with +//! the JPEG XS markers SOC + CAP (ISO/IEC 21122-1, `FF10 FF50`) and have no `0x0096` table (storytold/lightcraft#193). +//! Strips labelled 34713 without a `0x0096` +//! linearization table are packed uncompressed data in several measured layouts ([`PackedLayout`]) or, when the strip +//! is exactly three bytes per pixel, Nikon small raw ([`small_raw`]); the D1X's genuinely Huffman-coded strip stays +//! refused. + +use crate::nefc; +use crate::shared::white_from_data; +use crate::tiffraw::{Packing, check_image, read_image}; +use crate::{BlackLevel, Camera, Cfa, ColorData, Limits, Mode, OpcodeLists, Orientation, RawData, RawError, RawFormat, RawImage, Rect, Result}; +use craft_tiff::image::ImageInfo; +use craft_tiff::tags::{self as t, photometric}; +use craft_tiff::{ByteOrder, Ifd, Tiff, makernote}; + +const WB_RB_LEVELS: u16 = 0x000c; +const BLACK_LEVEL: u16 = 0x003d; +const LINEARIZATION_TABLE: u16 = 0x0096; +const CROP_AREA: u16 = 0x0045; +/// Maker note NEFCompression (ExifTool's Nikon tag documentation): `0x0093`, or in newer bodies the `u16` at +/// byte 10 of `0x0051`. 6 is packed 12-bit, 10 packed 14-bit, 13 and 14 High Efficiency and High Efficiency★. +const NEF_COMPRESSION: u16 = 0x0093; +const NEF_COMPRESSION_RECORD: u16 = 0x0051; + +fn nef_compression(mn: Option<&makernote::MakerNote>) -> Option { + let m = mn?; + m.ifd.u64(NEF_COMPRESSION).or_else(|| { + let v: [u8; 2] = m.ifd.bytes(NEF_COMPRESSION_RECORD)?.get(10..12)?.try_into().ok()?; + Some(u64::from(m.order.u16(v))) + }) +} +/// JPEG XS start of codestream (SOC, `FF10`) followed by its mandatory capabilities marker (CAP, `FF50`). +const JPEG_XS_START: [u8; 4] = [0xff, 0x10, 0xff, 0x50]; +/// Why a High Efficiency NEF shows its embedded preview: the UI shows the part before " (". +pub(crate) const HIGH_EFFICIENCY: &str = + "Nikon High Efficiency NEF (HE / HE★, a JPEG XS-based codec with no public specification; shoot RAW Lossless compressed to edit the raw data)"; +/// Maker note `0x0014`, the "NRW" data block of Coolpix raws: `"NRW "` + a 4-character version, then fields. In +/// version `0104` the `u32` at byte `0x20` is the black level in sample units (200 in the four 12-bit files here +/// that have 200 as their darkest samples, 0 in the one whose samples reach 0). +const NRW_DATA: u16 = 0x0014; +const NRW_BLACK_AT: usize = 0x20; + +/// The NRW data block, when the maker note has one. +fn nrw_data(mn: Option<&makernote::MakerNote>) -> Option<(&[u8], ByteOrder)> { + let m = mn?; + m.ifd.bytes(NRW_DATA).filter(|b| b.starts_with(b"NRW ")).map(|b| (b, m.order)) +} + +/// Black level from the NRW data block, bounded to a quarter of the sample range so an unknown version's field +/// can't turn into a large offset. +fn nrw_black(mn: Option<&makernote::MakerNote>, bits: u32) -> Option { + let (b, order) = nrw_data(mn)?; + let v = order.u32(b.get(NRW_BLACK_AT..NRW_BLACK_AT + 4)?.try_into().ok()?); + (u64::from(v) < (1u64 << bits.min(16)) / 4).then_some(v as f32) +} + +/// Nikon CropArea is `[left, top, width, height]` in sensor pixels (maker-note tag documentation). +/// Keep the active area/CFA origin unchanged: the default crop is applied after demosaicing. +fn default_crop(mn: Option<&makernote::MakerNote>, w: usize, h: usize) -> Rect { + let fallback = Rect::new(0, 0, w, h); + let Some(values) = mn.and_then(|m| m.ifd.u64s(CROP_AREA)) else { return fallback }; + let [x, y, width, height] = values.as_slice() else { return fallback }; + if *width == 0 || *height == 0 || x.checked_add(*width).is_none_or(|v| v > w as u64) || y.checked_add(*height).is_none_or(|v| v > h as u64) { + return fallback; + } + Rect::new(*x as usize, *y as usize, *width as usize, *height as usize) +} + +/// A body that records no CropArea, measured on its own files: the image area inside the sensor data (where the +/// camera's full-size JPEG sits when the two are aligned) and, when the data's own top would be wrong, the level +/// where its channels clip. +struct MeasuredBody { + model: &'static str, + /// Sensor data size and sample depth of the measured files. + size: (usize, usize), + bits: u32, + /// Image area in sensor pixels: x, y, width, height. + crop: (usize, usize, usize, usize), + white: Option, +} + +/// * **NIKON D4** (148 uncompressed 14-bit NEFs from one body, 2023–2026, ISO 100–10000): the data is 4992 × 3292, +/// with 2 columns of optical black left of the image and 42 right of it, then 8 empty ones; the camera's +/// 4928 × 3280 JPEG sits at (8, 6). Without this area the optical black shows as purple stripes on both sides. +/// Green saturates at 15546–15672 while red and blue run on to 15687–16383, so the data's own top (~16300) leaves +/// clipped highlights magenta; one level of 15520 for every channel turns them white and keeps the white balance. +/// * **NIKON D7100** (storytold/lightcraft#1248; 14-bit NEFs from the public zjqc/photographing-art repository, with their +/// Lightroom Classic exports): the data holds picture, not masked lines, beyond the camera's frame. In full frame +/// it is 6036 × 4020 and the camera's JPEG is 6000 × 4000; in the 1.3x crop mode it is 4812 × 3212 and the JPEG +/// 4800 × 3200. Fitting renders of the full data to Lightroom's exports (feature matches, homography) puts +/// Lightroom's frame, the camera JPEG's size, at (18, 10) and (6, 6): within 0.5 px mean on DSC_1316, DSC_2763 +/// and DSC_1486. Without it, renders are larger than Lightroom's and its crops land up to 6 px off (18 px in +/// full frame). +/// * **NIKON D90** (550 compressed 12-bit NEFs from one body, 10 measured): the data is 4352 × 2868 and its last 42 +/// columns (from 4310) are filler at a flat ~254 whatever the exposure, above the dark image samples (black 0), so +/// the masked-column test keeps them and they develop as a pink band on the right. The camera's 4288 × 2848 JPEG +/// sits at (10, 10). Green saturates at 3840–3851 while red and blue run on to 4095; one level of 3830 keeps +/// clipped highlights white. +/// * **NIKON D7000** and **NIKON D5100** (and the Coolpix A, the same readout): the data is 4992 × 3280, the picture its +/// first 4948 columns, then 36 masked ones at one level (~600, with a black level of 0) and 8 of 0. The masked +/// columns sit above the dark-column trim's threshold, so without this area they show as a coloured bar on the right +/// edge (the top in portrait; storytold/lightcraft#976). The camera's 4928 × 3264 JPEG sits at (8, 8): in the D5100's 3 CC0 +/// samples, and in the D7000's one clean one, with the D5100 and Coolpix A as siblings (see below). +/// * **The other bodies** below keep masked or dark lines inside the whole data (a survey of every Nikon raw on raw.pixls.us): +/// each crop is its camera JPEG's frame as `cargo xtask raw-frame` measures it on raw.pixls.us CC0 samples (the +/// D200, D3200 and Nikon 1 V1 also on other publicly posted sample files), with at least 3 clean +/// files agreeing: of the model (bit depths pooled: both read out the same area), or, with at least +/// one of its own, together with models of the same readout (data and JPEG size) that agree on the same frame +/// (*siblings*). The D4's 12-bit mode reads out the 14-bit area measured above; the Df, the same readout, agrees on +/// (8, 6) in 6 files. Bodies whose crop holds only picture beyond the camera frame have no entry (no change). +const MEASURED_BODIES: &[MeasuredBody] = &[ + MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: Some(15520.0) }, + // the 12-bit samples are the 14-bit ones shifted right by two, and so is the level: 15520 >> 2 + MeasuredBody { model: "NIKON D4", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: Some(3880.0) }, + MeasuredBody { model: "NIKON D7100", size: (6036, 4020), bits: 14, crop: (18, 10, 6000, 4000), white: None }, + MeasuredBody { model: "NIKON D7100", size: (4812, 3212), bits: 14, crop: (6, 6, 4800, 3200), white: None }, + MeasuredBody { model: "NIKON D90", size: (4352, 2868), bits: 12, crop: (10, 10, 4288, 2848), white: Some(3830.0) }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D7000", size: (4992, 3280), bits: 12, crop: (8, 8, 4928, 3264), white: None }, + MeasuredBody { model: "NIKON D5100", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // siblings: D7000 (4), D5100 (3); masked: right 36 + MeasuredBody { model: "COOLPIX A", size: (4992, 3280), bits: 14, crop: (8, 8, 4928, 3264), white: None }, + // 6 files each; masked: left 2, right 2 + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D3S", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 12, crop: (15, 6, 4256, 2832), white: None }, + MeasuredBody { model: "NIKON D700", size: (4288, 2844), bits: 14, crop: (15, 6, 4256, 2832), white: None }, + // 6 and 4 files; masked: right up to 32 + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 12, crop: (15, 10, 4288, 2848), white: None }, + MeasuredBody { model: "NIKON D300S", size: (4352, 2868), bits: 14, crop: (15, 10, 4288, 2848), white: None }, + // D2X (1) and D2Xs (2) as siblings; dark: right 1 + MeasuredBody { model: "NIKON D2Xs", size: (4320, 2868), bits: 12, crop: (16, 10, 4288, 2848), white: None }, + // D3000, D40X, D60, D80: one file each, siblings; masked: right 4, bottom 3 + MeasuredBody { model: "NIKON D3000", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D40X", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D60", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + MeasuredBody { model: "NIKON D80", size: (3904, 2616), bits: 12, crop: (14, 9, 3872, 2592), white: None }, + // 3 files; masked: left 1, right 3 (the D80's data size, framed elsewhere) + MeasuredBody { model: "NIKON D200", size: (3904, 2616), bits: 12, crop: (16, 12, 3872, 2592), white: None }, + // 22 files; masked: right 46 + MeasuredBody { model: "NIKON D3200", size: (6080, 4012), bits: 12, crop: (8, 6, 6016, 4000), white: None }, + // 8 clean of 19, the rest ~0.3 px between pixels on the same frame; dark: bottom 2 + MeasuredBody { model: "NIKON 1 V1", size: (3904, 2606), bits: 12, crop: (12, 6, 3872, 2592), white: None }, + // Best guesses, short of the evidence rule; every frame below keeps the masked lines out by several pixels. + // D3100: 3 of 4 clean files at (24, 6), the 4th at (22, 7); masked: left 6, right 22 + MeasuredBody { model: "NIKON D3100", size: (4672, 3084), bits: 12, crop: (24, 6, 4608, 3072), white: None }, + // D5000: 4 of 5 clean files at (11, 10), the 5th at (11, 12); the D90's sensor, whose own files also align at + // x = 11; masked: right 42 (the same filler as the D90's) + MeasuredBody { model: "NIKON D5000", size: (4352, 2868), bits: 12, crop: (11, 10, 4288, 2848), white: None }, + // D40: 2 of 2 clean files at (16, 9); masked: right 1. The D50 / D70 / D70s share its data size but not its frame + MeasuredBody { model: "NIKON D40", size: (3040, 2014), bits: 12, crop: (16, 9, 3008, 2000), white: None }, + // D50, D70, D70s (one readout): one clean D50 and one clean D70 file at (15, 7); the D70s's 2 files align + // between pixels next to it, at (15.4, 6.9) and (15.6, 6.4); masked: right 1 + MeasuredBody { model: "NIKON D50", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + MeasuredBody { model: "NIKON D70s", size: (3040, 2014), bits: 12, crop: (15, 7, 3008, 2000), white: None }, + // 3 files; dark: bottom 4 + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 12, crop: (4, 4, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON D4S", size: (4936, 3288), bits: 14, crop: (4, 4, 4928, 3280), white: None }, + // 4 files; dark: bottom up to 4 + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 12, crop: (8, 8, 6000, 4000), white: None }, + MeasuredBody { model: "NIKON D5600", size: (6016, 4016), bits: 14, crop: (8, 8, 6000, 4000), white: None }, + // 4 files each; masked: right 8 + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D600", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 12, crop: (8, 6, 6016, 4016), white: None }, + MeasuredBody { model: "NIKON D610", size: (6080, 4028), bits: 14, crop: (8, 6, 6016, 4016), white: None }, + // D800: 6 files; D800E: 2, with the D800 as sibling; masked: right 14 (storytold/lightcraft#914, the top in portrait) + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 12, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D800E", size: (7424, 4924), bits: 14, crop: (8, 6, 7360, 4912), white: None }, + // 3 files; dark: bottom 1 + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 12, crop: (12, 8, 7360, 4912), white: None }, + MeasuredBody { model: "NIKON D810", size: (7380, 4928), bits: 14, crop: (12, 8, 7360, 4912), white: None }, + // 6 files; masked: left 2, right 42 (the D4's readout) + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 12, crop: (8, 6, 4928, 3280), white: None }, + MeasuredBody { model: "NIKON Df", size: (4992, 3292), bits: 14, crop: (8, 6, 4928, 3280), white: None }, +]; + +fn measured_body(model: &str, w: usize, h: usize, bits: u32) -> Option<&'static MeasuredBody> { + MEASURED_BODIES.iter().find(|b| b.model == model.trim() && b.size == (w, h) && b.bits == bits) +} + +/// Pixel height over width from the raw IFD's resolution tags: the D1X and D1H write 600 x 300 per inch for a +/// 4028 x 1324 frame, rows twice as tall as the columns are wide. Only ratios from 1.5 to 4 count (a frame +/// squashed 3:1 or 2:1), the rest is rounding or a stray tag. +pub(crate) fn pixel_aspect_of(raw: &Ifd) -> f32 { + match (raw.f64(t::X_RESOLUTION), raw.f64(t::Y_RESOLUTION)) { + (Some(x), Some(y)) if x > 0.0 && y > 0.0 && (1.5..=4.0).contains(&(x / y)) => (x / y).round() as f32, + _ => 1.0, + } +} + +fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> { + tiff.all_ifds() + .into_iter() + .filter(|i| i.u16(t::PHOTOMETRIC) == Some(photometric::CFA)) + .max_by_key(|i| i.u64(t::IMAGE_WIDTH).unwrap_or(0).saturating_mul(i.u64(t::IMAGE_LENGTH).unwrap_or(0))) +} + +/// Rows the masked-column test reads, from the top of the frame. The header probe ([`Mode::Header`]) decodes +/// only these (a compressed strip is one sequential Huffman stream, so it can stop here; an uncompressed one reads +/// just its first strips), and the full decode judges the active width from the same rows, so +/// [`crate::probe_info`] equals [`RawImage::info`] of the decoded image (storytold/lightcraft#708). On every corpus NEF the +/// answer is the same for 8 to 512 rows as for the whole frame: the masked columns decode to 0 (compressed) or +/// ~19 (uncompressed) while live columns sit at the black level (~150 at 12 bit, ~600 at 14 bit) or above. +pub(crate) const MASK_ROWS: usize = 128; + +/// Width without the optically masked columns some bodies append on the right (the D5100 and D7000 write 8, with +/// no maker-note tag that says so, in compressed and uncompressed files alike): trailing columns (at most 64) whose +/// mean over the first [`MASK_ROWS`] rows of `d` is below 1% of the nominal white level and a tenth of the image +/// interior. Kept even for CFA phase. +pub(crate) fn trailing_masked_columns(d: &[u16], w: usize, bits: u32) -> usize { + let rows = d.len().checked_div(w).unwrap_or(0).min(MASK_ROWS); + if w < 128 || rows == 0 { + return w; + } + let col_mean = |x: usize| (0..rows).map(|y| d.get(y * w + x).copied().unwrap_or(0) as f64).sum::() / rows as f64; + let interior = (w / 4..w * 3 / 4).step_by(w / 64).map(col_mean).sum::() / 32.0; + let white = ((1u32 << bits.clamp(1, 16)) - 1) as f64; + let dark = (white * 0.01).min(interior * 0.1); + let mut aw = w; + while aw > w - 64 && col_mean(aw - 1) < dark { + aw -= 1; + } + aw & !1 +} + +pub(crate) fn decode(bytes: &[u8], mode: Mode, limits: &Limits) -> Result { + let tiff = Tiff::parse(bytes)?; + let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("TIFF without IFDs".into()))?; + let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("NEF without a CFA image IFD".into()))?; + let info = raw.image()?; + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let make = ifd0.string(t::MAKE).unwrap_or_default(); + let mn = + tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make)); + if let Some(strip) = small_raw_strip(&info, bytes.len(), mn.as_ref()) { + return small_raw(bytes, &tiff, &info, strip, mode, limits); + } + // the header probe reads only the rows the masked-column test needs + let rows = match mode { + Mode::Full => h, + Mode::Header => h.min(MASK_ROWS), + }; + let data = if info.compression == t::compression::NIKON { + compressed(bytes, &info, mn.as_ref(), rows, limits)? + } else { + uncompressed(bytes, &info, tiff.order, mn.as_ref(), rows, limits)? + }; + let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float NEF".into())) }; + + // Maker note 0x003d is in 14-bit units whatever the sample depth: 12-bit files from the D750, D780, D850, + // D7500 and Z 50 store 600 / 1008 / 400 / 400 / 1008 while their darkest samples are 150 / ~252 / 99 / 99 / ~250 + // (our measurements on CC0 raw.pixls.us samples; the same bodies' 14-bit files match the tag as is). + let black_scale = if bits == 12 { 0.25 } else { 1.0 }; + let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_LEVEL)).as_deref() { + Some([a, b, c, d]) if [a, b, c, d].iter().all(|v| **v < 16384.0) => BlackLevel { + repeat_rows: 2, + repeat_cols: 2, + values: [a, b, c, d].iter().map(|v| (**v * black_scale) as f32).collect(), + ..Default::default() + }, + _ => BlackLevel::uniform(nrw_black(mn.as_ref(), bits).unwrap_or_else(|| be16_black(&info, bytes.len() as u64, mn.as_ref(), samples))), + }; + let wb = mn + .as_ref() + .and_then(|m| m.ifd.f64s(WB_RB_LEVELS)) + .filter(|v| v.len() >= 2 && v[0] > 0.1 && v[1] > 0.1 && v[0] < 10.0 && v[1] < 10.0) + .map(|v| [v[0] as f32, 1.0, v[1] as f32]); + let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) { + (Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() }, + _ => Cfa::bayer_static("RGGB"), + }; + let model = ifd0.string(t::MODEL).unwrap_or_default(); + let body = measured_body(&model, w, h, bits); + let active_w = trailing_masked_columns(samples, w, bits); + // the white level is measured from the whole image (or known for a measured body); the probe (no samples in its + // result) takes the nominal one + let white = match (body.and_then(|b| b.white), mode) { + (Some(white), _) => white, + (None, Mode::Full) => white_from_data(samples, bits), + (None, Mode::Header) => ((1u32 << bits.clamp(1, 16)) - 1) as f32, + }; + let crop = match body { + Some(&MeasuredBody { crop: (x, y, cw, ch), .. }) if x + cw <= active_w && y + ch <= h => Rect::new(x, y, cw, ch), + _ => default_crop(mn.as_ref(), active_w, h), + }; + let pixel_aspect = pixel_aspect_of(raw); + let img = RawImage { + format: RawFormat::Nef, + width: w, + height: h, + cpp: 1, + data: match mode { + Mode::Full => data, + Mode::Header => RawData::U16(Vec::new()), + }, + cfa: Some(cfa), + bits, + black, + white: vec![white], + active_area: Rect::new(0, 0, active_w, h), + crop, + orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)), + color: ColorData::default(), + wb_multipliers: wb, + linearized: false, + pixel_aspect, + // Nikon's in-file distortion correction, about the optical centre (which follows the sensor shift) + opcodes: OpcodeLists { + list3: crate::nef_lens::distortion(&model, raw, Rect::new(0, 0, active_w, h), crop).into_iter().collect(), + ..Default::default() + }, + camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(), + cinema: Default::default(), + }; + img.validate_for(mode)?; + Ok(img) +} + +/// Uncompressed strips: 16-bit words or 12-bit packed, told apart by the strip size. Packed 12-bit data is an MSB-first +/// byte stream, except in files with an NRW data block (Coolpix), where it is MSB-first inside little-endian 32-bit +/// words (the rows are whole words). Reads the first `rows` rows (the strips that hold them) of the `info.height`. +fn uncompressed( + bytes: &[u8], + info: &ImageInfo, + order: ByteOrder, + mn: Option<&makernote::MakerNote>, + rows: usize, + limits: &Limits, +) -> Result { + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let row_samples = w * info.samples_per_pixel as usize; + let chunks = info.chunks(bytes.len() as u64); + let rows_in_first = chunks.first().map(|c| c.height as usize).unwrap_or(h).max(1); + let bytes_per_row = chunks.first().map(|c| c.len as usize / rows_in_first).unwrap_or(0); + let packing = if bytes_per_row >= row_samples * 2 { + Packing::Word16 + } else if bits == 12 && bytes_per_row * 8 >= row_samples * 12 && bytes_per_row * 8 < row_samples * 13 { + if nrw_data(mn).is_some() && bytes_per_row.is_multiple_of(4) { Packing::Words32Msb } else { Packing::Msb } + } else if info.compression == 1 && bits != 8 && bits != 16 { + return Err(RawError::Unsupported(format!("NEF uncompressed packing ({bytes_per_row} bytes per {row_samples}-sample row)"))); + } else { + Packing::Msb + }; + if rows >= h { + return read_image(bytes, info, order, packing, limits); + } + // the whole image's limits apply either way; then only the strips holding the first `rows` rows are unpacked + check_image(bytes, info, limits)?; + let top = ImageInfo { height: rows as u32, ..info.clone() }; + read_image(bytes, &top, order, packing, limits) +} + +/// Packed layouts of strips that are labelled compression 34713 but have no linearization table (maker note +/// `0x0096`): they hold plain packed samples, not Huffman codes. The layouts were measured from CC0 raw.pixls.us +/// files by a clean-room black-box analysis (strip byte-count identity, same-colour smoothness and correlation with +/// the camera JPEG against at least eight alternative packings). Every layout is a single strip with +/// `stride = StripByteCounts / height`, selected only when `stride * height` equals the strip size exactly. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PackedLayout { + /// LSB-first bit stream of `bits` per sample (12: 3 bytes per 2 samples, 14: 7 bytes per 4), rows byte aligned. + /// 12-bit rows are `3w/2` bytes (older bodies) or padded to 16 bytes (Z 6, D6); 14-bit rows are `7w/4` bytes + /// padded to 16 bytes. + Lsb { stride: usize }, + /// D100: 16-byte chunks of fifteen data bytes holding ten MSB-first 12-bit samples, plus one zero pad byte. The + /// rows carry more samples than the IFD width (3040 against 3034); the extra columns are not image. + Chunked16 { stride: usize }, + /// Coolpix NRW: one big-endian 16-bit word per pixel, rows `2w` bytes with no header or padding; the 12-bit sample + /// is the upper 12 bits of the word (the low nibble is zero, rarely 8). + Be16, +} + +fn packed_layout(w: usize, h: usize, bits: u32, strip: u64, stated: bool) -> Option { + if w == 0 || h == 0 || !matches!(bits, 12 | 14) { + return None; + } + let strip = usize::try_from(strip).ok()?; + if !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let tight = w.checked_mul(bits as usize)?.div_ceil(8); + let tight = if bits == 12 && w.is_multiple_of(2) || bits == 14 && w.is_multiple_of(4) { tight } else { return None }; + let padded = tight.checked_next_multiple_of(16)?; + let lsb_ok = match bits { + 12 => stride == tight || stride == padded, + _ => stride == padded || stated && stride == tight, + }; + if lsb_ok { + return Some(PackedLayout::Lsb { stride }); + } + None +} + +fn d100_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + let strip = usize::try_from(strip).ok()?; + if bits != 12 || h == 0 || !strip.is_multiple_of(h) { + return None; + } + let stride = strip / h; + let per_row = stride / 16 * 10; + (stride.is_multiple_of(16) && per_row >= w && per_row < w.checked_add(10)?).then_some(PackedLayout::Chunked16 { stride }) +} + +/// 12-bit strips of exactly `2 * w * h` bytes: big-endian 16-bit words (Coolpix P330, P7700, P7800 NRW, found by a +/// clean-room black-box analysis of the CC0 files). Tried after the other layouts, whose strides (`1.5w` to +/// `1.5w + 15`, or ten samples per 16 bytes) only reach `2w` for rows narrower than 40 samples, so it can't take a +/// strip they handle. +fn be16_layout(w: usize, h: usize, bits: u32, strip: u64) -> Option { + let want = w.checked_mul(h)?.checked_mul(2)?; + (bits == 12 && w > 0 && h > 0 && usize::try_from(strip).ok()? == want).then_some(PackedLayout::Be16) +} + +/// Black level of the big-endian 16-bit layout, which carries neither `0x003d` nor an NRW block and has no masked +/// pixels (the borders of the frame are image). The darkest samples of the three measured bodies are 214 (P330), +/// 217 (P7700) and 201 (P7800, the only one with a dark enough frame: its per-channel 1st percentile is 209 / 225 / +/// 223), and the other 12-bit Coolpix NRWs state 200 in their NRW block; so 200, never above the darkest sample. +const BE16_BLACK: f32 = 200.0; + +fn be16_black(info: &ImageInfo, file_len: u64, mn: Option<&makernote::MakerNote>, samples: &[u16]) -> f32 { + let has_table = mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some()); + let single = matches!(info.chunks(file_len).as_slice(), [c] if be16_layout(info.width as usize, info.height as usize, info.bits() as u32, c.len).is_some()); + if info.compression != t::compression::NIKON || has_table || info.samples_per_pixel != 1 || !single { + return 0.0; + } + let darkest = samples.iter().copied().min().unwrap_or(0); + BE16_BLACK.min(f32::from(darkest)) +} + +/// Unpack a [`PackedLayout`] strip into `w * h` samples. Short input leaves the missing samples zero. +fn unpack_packed(src: &[u8], w: usize, h: usize, bits: u32, layout: PackedLayout) -> Vec { + use rayon::prelude::*; + let mut out = vec![0u16; w * h]; + let stride = match layout { + PackedLayout::Lsb { stride } | PackedLayout::Chunked16 { stride } => stride, + PackedLayout::Be16 => w.saturating_mul(2), + }; + out.par_chunks_mut(w).enumerate().for_each(|(y, row)| { + let Some(line) = src.get(y * stride..) else { return }; + let line = &line[..stride.min(line.len())]; + match layout { + PackedLayout::Lsb { .. } => crate::unpack::unpack_lsb(line, bits, row), + PackedLayout::Be16 => { + let (words, _) = line.as_chunks::<2>(); + for (dst, word) in row.iter_mut().zip(words) { + *dst = u16::from_be_bytes(*word) >> 4; + } + } + PackedLayout::Chunked16 { .. } => { + let mut samples = vec![0u16; line.len() / 16 * 10]; + for (i, dst) in samples.chunks_mut(10).enumerate() { + if let Some(chunk) = line.get(i * 16..i * 16 + 15) { + crate::unpack::unpack_msb(chunk, 12, dst); + } + } + let n = row.len().min(samples.len()); + row[..n].copy_from_slice(&samples[..n]); + } + } + }); + out +} + +/// Nikon Huffman-compressed strip (see [`crate::nefc`]); the decode table is maker note `0x0096`. Without a table the +/// strip may be packed uncompressed data (see [`PackedLayout`]). +fn compressed(bytes: &[u8], info: &ImageInfo, mn: Option<&makernote::MakerNote>, rows: usize, limits: &Limits) -> Result { + let starts_with = |magic: &[u8]| { + let start = info.chunks(bytes.len() as u64).first().and_then(|c| usize::try_from(c.offset).ok()); + start.and_then(|o| bytes.get(o..o.checked_add(magic.len())?)) == Some(magic) + }; + // the maker note's own compression mode decides first: Z bodies label packed data 34713 and may still carry + // a 0x0096 table + let mode = nef_compression(mn); + if starts_with(&JPEG_XS_START) || matches!(mode, Some(13 | 14)) { + return Err(RawError::Unsupported(HIGH_EFFICIENCY.into())); + } + if let Some(depth) = match mode { + Some(6) => Some(12), + Some(10) => Some(14), + _ => None, + } { + if info.bits() as u32 != depth { + return Err(RawError::Unsupported(format!("Nikon packed {depth}-bit NEF with {}-bit samples", info.bits()))); + } + return packed(bytes, info, rows, true, limits); + } + let Some(table) = mn.and_then(|m| Some((m.ifd.bytes(LINEARIZATION_TABLE)?, m.order))) else { + return packed(bytes, info, rows, false, limits); + }; + if info.samples_per_pixel != 1 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {} samples per pixel", info.samples_per_pixel))); + } + let bits = info.bits() as u32; + if u64::from(info.width).saturating_mul(u64::from(info.height)) > limits.samples() as u64 { + return Err(RawError::Limit("image too large")); + } + let table = nefc::parse_table(table.0, table.1, bits)?; + // one strip in every sample; several would be contiguous parts of the same bit stream + let chunks = info.chunks(bytes.len() as u64); + let (Some(first), Some(last)) = (chunks.first(), chunks.last()) else { return Err(RawError::Corrupt("NEF: no image data".into())) }; + let end = last.offset.saturating_add(last.len).min(bytes.len() as u64); + let src = usize::try_from(first.offset).ok().zip(usize::try_from(end).ok()).and_then(|(a, b)| bytes.get(a..b)); + let src = src.ok_or_else(|| RawError::Corrupt("NEF: image data outside the file".into()))?; + Ok(RawData::U16(nefc::decode_rows(src, info.width as usize, info.height as usize, rows, bits, &table)?)) +} + +/// A "compressed" strip of packed samples: the strip size must match a measured layout exactly. `stated` is set +/// when the maker note names the packed mode, which also allows 14-bit rows without padding. +fn packed(bytes: &[u8], info: &ImageInfo, rows: usize, stated: bool, limits: &Limits) -> Result { + let (w, h) = (info.width as usize, info.height as usize); + let bits = info.bits() as u32; + let chunks = info.chunks(bytes.len() as u64); + let layout = match chunks.as_slice() { + [c] if info.samples_per_pixel == 1 => { + packed_layout(w, h, bits, c.len, stated).or_else(|| d100_layout(w, h, bits, c.len)).or_else(|| be16_layout(w, h, bits, c.len)) + } + _ => None, + }; + let (Some(layout), [c]) = (layout, chunks.as_slice()) else { + return Err(RawError::Unsupported("Nikon compressed NEF without a linearization table (maker note 0x96)".into())); + }; + let src = craft_tiff::image::chunk_bytes(bytes, c) + .filter(|src| u64::try_from(src.len()).ok() == Some(c.len)) + .ok_or_else(|| RawError::Corrupt("NEF: incomplete image strip".into()))?; + if (w as u64).saturating_mul(h as u64) > limits.samples() as u64 { + return Err(RawError::Limit("image too large")); + } + Ok(RawData::U16(unpack_packed(src, w, rows.min(h), bits, layout))) +} + +/// Nikon small raw (sRAW), measured clean-room from the CC0 raw.pixls.us D810 (2036) and D4S (2227) samples. The raw SubIFD is labelled compression 34713, +/// 12 bits, one CFA sample per pixel, but it carries no linearization table (maker note `0x0096`) and its single strip +/// is exactly `3 * width * height` bytes. It holds a YCbCr 4:2:2 image of the IFD size, not a CFA mosaic (the CFA +/// tags are dummies): +/// +/// - the strip is an LSB-first 12-bit stream (`s0 = b0 | (b1 & 0x0f) << 8`, `s1 = b1 >> 4 | b2 << 4`), a row is `3 * width` +/// bytes with no padding, i.e. `2 * width` samples; +/// - the samples come in groups of four, `[Y_left, Y_right, Cb, Cr]`, for the pixel pair `(2k, 2k + 1)`: chroma is +/// halved horizontally and not vertically; +/// - luma is linear and already black-subtracted (the darkest samples are near 0) and never uses the top two bits +/// (maxima 873 and 1004 of 4095), so the white level is 1023; chroma is centred on 2048 (fitted neutral 2045-2049). +/// +/// The chroma is a camera colour-difference scale, not JFIF Cb/Cr. In linear terms the stored values behave as +/// `R - Y = R_GAIN * (Cr - 2048)` and `B - Y = B_GAIN * (Cb - 2048)` on the luma scale: regressing the camera JPEG's +/// linear `(R - Y) / Y` and `(B - Y) / Y` on `(Cr - 2048) / Y` and `(Cb - 2048) / Y` over mid-tone pixels gave slopes +/// 4.92 / 5.09 (D810, correlation 0.94 / 0.88) and 3.52 / 3.73 (D4S, 0.98 / 0.97); the gains below are the rounded +/// means. They are approximate (the two bodies differ by about 30 %), the luma is not tone-mapped, and the data is +/// treated as already white balanced (neutral D4S areas have `Cb`, `Cr` within 2 of 2048 and equal JPEG R, G, B). G +/// follows from the luma equation with the BT.601 weights. The chroma is upsampled horizontally with linear +/// interpolation, centred between the two pixels of a pair. Output is linear RGB scaled by 16 (to a 14-bit range), +/// like Sony's downsized ARWs. +const SMALL_RAW_WHITE: f32 = 1023.0; +const SMALL_RAW_SCALE: f32 = 16.0; +const SMALL_RAW_CHROMA_ZERO: f32 = 2048.0; +const SMALL_RAW_R_GAIN: f32 = 4.2; +const SMALL_RAW_B_GAIN: f32 = 4.4; + +/// The strip of a small raw: compression 34713, 12 bits, no maker note `0x0096`, one strip of exactly +/// `3 * width * height` bytes (even width). +fn small_raw_strip(info: &ImageInfo, file_len: usize, mn: Option<&makernote::MakerNote>) -> Option { + if info.compression != t::compression::NIKON || info.bits() != 12 || info.samples_per_pixel != 1 { + return None; + } + if mn.is_some_and(|m| m.ifd.bytes(LINEARIZATION_TABLE).is_some()) { + return None; + } + let (w, h) = (info.width as u64, info.height as u64); + let chunks = info.chunks(file_len as u64); + match chunks.as_slice() { + [c] if w.is_multiple_of(2) && w > 0 && h > 0 && w.checked_mul(h).and_then(|n| n.checked_mul(3)) == Some(c.len) => Some(*c), + _ => None, + } +} + +/// Decode one row of a small raw (`3 * width` bytes) into `3 * width` linear RGB values on the 14-bit scale. +/// Short input reads as zero bytes. +fn small_raw_row(src: &[u8], w: usize, out: &mut [u16]) { + let Some(n) = w.checked_mul(3).filter(|n| *n <= out.len()) else { return }; + if w == 0 || !w.is_multiple_of(2) { + return; + } + let mut s = vec![0u16; 2 * w]; + crate::unpack::unpack_lsb(&src[..src.len().min(n)], 12, &mut s); + let pairs = w / 2; + let chroma = |k: usize| { + let g = &s[4 * k + 2..4 * k + 4]; + (f32::from(g[0]) - SMALL_RAW_CHROMA_ZERO, f32::from(g[1]) - SMALL_RAW_CHROMA_ZERO) + }; + for k in 0..pairs { + let (cb, cr) = chroma(k); + for side in 0..2 { + let (nb, nr) = if side == 0 { chroma(k.saturating_sub(1)) } else { chroma((k + 1).min(pairs - 1)) }; + let (u, v) = (0.75 * cb + 0.25 * nb, 0.75 * cr + 0.25 * nr); + let y = f32::from(s[4 * k + side]); + let r = y + SMALL_RAW_R_GAIN * v; + let b = y + SMALL_RAW_B_GAIN * u; + let g = (y - 0.299 * r - 0.114 * b) / 0.587; + let dst = (2 * k + side) * 3; + for (i, value) in [r, g, b].into_iter().enumerate() { + out[dst + i] = (value * SMALL_RAW_SCALE).round().clamp(0.0, 65535.0) as u16; + } + } + } +} + +fn small_raw(bytes: &[u8], tiff: &Tiff, info: &ImageInfo, strip: craft_tiff::image::Chunk, mode: Mode, limits: &Limits) -> Result { + use rayon::prelude::*; + let (w, h) = (info.width as usize, info.height as usize); + let stride = w.checked_mul(3).ok_or(RawError::Limit("image too large"))?; + let count = stride.checked_mul(h).filter(|n| *n > 0 && *n <= limits.samples()).ok_or(RawError::Limit("image too large"))?; + let src = craft_tiff::image::chunk_bytes(bytes, &strip) + .filter(|src| u64::try_from(src.len()).ok() == Some(strip.len)) + .ok_or_else(|| RawError::Corrupt("NEF: incomplete image strip".into()))?; + let data = match mode { + Mode::Full => { + let mut data = vec![0u16; count]; + data.par_chunks_mut(stride).enumerate().for_each(|(y, out)| { + small_raw_row(src.get(y * stride..).unwrap_or(&[]), w, out); + }); + data + } + Mode::Header => Vec::new(), + }; + let img = RawImage { + format: RawFormat::Nef, + width: w, + height: h, + cpp: 3, + data: RawData::U16(data), + cfa: None, + bits: 14, + black: BlackLevel::uniform(0.0), + white: vec![SMALL_RAW_WHITE * SMALL_RAW_SCALE], + active_area: Rect::new(0, 0, w, h), + crop: Rect::new(0, 0, w, h), + orientation: Orientation::from_exif(tiff.ifds.first().and_then(|i| i.u16(t::ORIENTATION)).unwrap_or(1)), + color: ColorData::default(), + // the stored colour is already white balanced (see above) + wb_multipliers: Some([1.0; 3]), + linearized: false, + pixel_aspect: 1.0, + opcodes: OpcodeLists::default(), + camera: tiff.ifds.first().map(Camera::from_ifd0).unwrap_or_default(), + cinema: Default::default(), + }; + img.validate_for(mode)?; + Ok(img) +} + +#[cfg(test)] +mod tests { + use super::*; + use craft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter, Value}; + + fn nef(compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32) -> Vec { + nef_with_note(compression, bits, strips, w, h, rps, None) + } + + /// [`nef`] with an optional maker note (`Nikon\0` v2 header + embedded big-endian TIFF holding `note`). + fn nef_with_note(compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32, note: Option) -> Vec { + nef_in(ByteOrder::Big, compression, bits, strips, w, h, rps, note) + } + + /// [`nef_with_note`] in either byte order (file and maker note). + #[allow(clippy::too_many_arguments)] + fn nef_in(order: ByteOrder, compression: u16, bits: u16, strips: Vec>, w: u32, h: u32, rps: u32, note: Option) -> Vec { + nef_model_in(order, "NIKON TEST", compression, bits, strips, w, h, rps, note) + } + + /// [`nef_in`] from camera `model`. + #[allow(clippy::too_many_arguments)] + fn nef_model_in( + order: ByteOrder, + model: &str, + compression: u16, + bits: u16, + strips: Vec>, + w: u32, + h: u32, + rps: u32, + note: Option, + ) -> Vec { + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits])); + raw.set(t::COMPRESSION, Value::Short(vec![compression])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![2, 1, 1, 0])); + raw.set_image(ImageData::Strips { rows_per_strip: rps, strips }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); + if let Some(mn) = note { + let mut bytes = b"Nikon\0\x02\x10\0\0".to_vec(); + bytes.extend(TiffWriter::new(order, false).write(&[mn]).unwrap()); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(bytes)); + ifd0.set_child(t::EXIF_IFD, exif); + } + ifd0.add_sub_ifd(raw); + TiffWriter::new(order, false).write(&[ifd0]).unwrap() + } + + /// A Z 6II NEF carrying NikonNEFInfo gets Nikon's distortion correction as one warp, in the header probe and + /// the full decode alike, and keeps it through DNG export; another body with the same data gets none. + #[test] + fn nikon_in_file_distortion_becomes_a_warp() { + use crate::nef_lens::tests::{distortion_block, nef_info_blob, offset_block}; + let (w, h) = (48u32, 32u32); + let file = |model: &str| { + let words: Vec = (0..w * h).flat_map(|i| (600 + (i % 97) as u16 * 50).to_be_bytes()).collect(); + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![14])); + raw.set(t::COMPRESSION, Value::Short(vec![1])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2])); + let info = nef_info_blob(&[ + (0x0005, distortion_block(3, 21.6, 4, [0.039895, 0.019211, -0.127218], -0.025043)), + (0x0008, offset_block(1.5, -0.75)), + ]); + raw.set(0xc7d5, Value::Undefined(info)); + raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![words] }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii(model.into())); + ifd0.add_sub_ifd(raw); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + }; + let bytes = file("NIKON Z 6_2"); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!(header.info(), full.info()); + assert_eq!(header.opcodes.list3, full.opcodes.list3); + let [crate::Opcode::WarpRectilinear { planes, center }] = full.opcodes.list3.as_slice() else { panic!("{:?}", full.opcodes.list3) }; + assert!((planes[0][0] - 1.0 / (1.0 - 0.025043)).abs() < 1e-5, "{planes:?}"); + assert!((center[0] * 47.0 - (23.5 + 1.5)).abs() < 0.01 && (center[1] * 31.0 - (15.5 - 0.75)).abs() < 0.01, "{center:?}"); + #[cfg(feature = "dng")] + { + let dng = crate::write_dng(&full, &Default::default()).unwrap(); + assert_eq!(crate::decode(&dng).unwrap().opcodes.list3, full.opcodes.list3); + } + assert!(decode(&file("NIKON D850"), Mode::Full, &Limits::default()).unwrap().opcodes.list3.is_empty()); + } + + #[test] + fn preview_color_space_is_read_from_the_nikon_note() { + for (value, expected) in [(1, Some(crate::PreviewColorSpace::Srgb)), (2, Some(crate::PreviewColorSpace::AdobeRgb)), (3, None)] { + let mut mn = IfdBuilder::new(); + mn.set(0x001e, Value::Short(vec![value])); + let bytes = nef_with_note(1, 14, vec![vec![0; 128]], 8, 8, 8, Some(mn)); + assert_eq!(crate::embedded_preview_color_space(&bytes), expected); + } + assert_eq!(crate::embedded_preview_color_space(&nef(1, 14, vec![vec![0; 128]], 8, 8, 8)), None); + assert_eq!(crate::embedded_preview_color_space(b"not a TIFF"), None); + } + + /// storytold/lightcraft#778: any make's raw falls back to its EXIF ColorSpace, with Uncalibrated + InteropIndex "R03" read as + /// Adobe RGB; the Nikon note wins; a DNG is left alone. + #[test] + fn preview_color_space_falls_back_to_the_exif_color_space() { + let file = |make: &str, cs: u16, interop: Option<&str>, note: Option, dng: bool| { + let mut exif = IfdBuilder::new(); + exif.set(0xA001, Value::Short(vec![cs])); + if let Some(i) = interop { + let mut io = IfdBuilder::new(); + io.set(0x0001, Value::Ascii(i.into())); + exif.set_child(t::INTEROP_IFD, io); + } + if let Some(v) = note { + let mut mn = IfdBuilder::new(); + mn.set(0x001e, Value::Short(vec![v])); + let mut bytes = b"Nikon".to_vec(); + bytes.extend(TiffWriter::new(ByteOrder::Little, false).write(&[mn]).unwrap()); + exif.set(t::MAKER_NOTE, Value::Undefined(bytes)); + } + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii(make.into())); + if dng { + ifd0.set(t::DNG_VERSION, Value::Byte(vec![1, 4, 0, 0])); + } + ifd0.set_child(t::EXIF_IFD, exif); + TiffWriter::new(ByteOrder::Little, false).write(&[ifd0]).unwrap() + }; + use crate::PreviewColorSpace::{AdobeRgb, Srgb}; + let f = crate::embedded_preview_color_space; + assert_eq!(f(&file("SONY", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("SONY", 1, Some("R98"), None, false)), Some(Srgb)); + assert_eq!(f(&file("SONY", 0xFFFF, Some("R98"), None, false)), None); + assert_eq!(f(&file("SONY", 0xFFFF, None, None, false)), None); + assert_eq!(f(&file("Canon", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("NIKON CORPORATION", 1, Some("R98"), Some(2), false)), Some(AdobeRgb)); + assert_eq!(f(&file("NIKON CORPORATION", 0xFFFF, Some("R03"), None, false)), Some(AdobeRgb)); + assert_eq!(f(&file("Canon", 0xFFFF, Some("R03"), None, true)), None); + } + + #[test] + fn nikon_crop_area_preserves_sensor_origin_and_rejects_invalid_rectangles() { + let (w, h) = (32, 24); + let words: Vec = (0..w * h).flat_map(|i| (100 + i as u16).to_be_bytes()).collect(); + for (values, expected) in [ + (vec![2, 4, 28, 16], Rect::new(2, 4, 28, 16)), + (vec![31, 4, 28, 16], Rect::new(0, 0, 32, 24)), + (vec![2, 4, 0, 16], Rect::new(0, 0, 32, 24)), + (vec![u32::MAX, 4, 28, 16], Rect::new(0, 0, 32, 24)), + (vec![2, 4, 28], Rect::new(0, 0, 32, 24)), + ] { + let mut mn = IfdBuilder::new(); + mn.set(CROP_AREA, Value::Long(values)); + let bytes = nef_with_note(1, 14, vec![words.clone()], w, h, h, Some(mn)); + let raw = crate::decode(&bytes).unwrap(); + assert_eq!(raw.crop, expected); + assert_eq!(raw.active_area, Rect::new(0, 0, 32, 24)); + assert_eq!((raw.crop.width, raw.crop.height), (expected.width, expected.height)); + assert_eq!(raw.info().developed_size(), (expected.width, expected.height)); + } + } + + /// The D4 records no CropArea: its measured image area and saturation level apply to files of its data size and + /// depth only. + #[test] + fn measured_d4_image_area_and_white_level() { + let (w, h) = (4992usize, 3292usize); + let decode_as = |model: &str, bits: u16| { + let mut nef = nef(1, bits, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32); + let at = nef.windows(10).position(|b| b == b"NIKON TEST").unwrap(); + let name = format!("{model:<10}"); + nef[at..at + 10].copy_from_slice(name.as_bytes()); + crate::decode(&nef).unwrap() + }; + let d4 = decode_as("NIKON D4", 14); + assert_eq!(d4.crop, Rect::new(8, 6, 4928, 3280)); + assert_eq!(d4.white, vec![15520.0]); + assert_eq!((d4.crop.width, d4.crop.height), (4928, 3280)); + assert_eq!(d4.info().developed_size(), (4928, 3280)); + // the 12-bit mode reads out the same area, and its level is the 14-bit one shifted like its samples + let d4_12 = decode_as("NIKON D4", 12); + assert_eq!(d4_12.crop, Rect::new(8, 6, 4928, 3280)); + assert_eq!(d4_12.white, vec![3880.0]); + let other = decode_as("NIKON D5", 14); + assert_eq!(other.crop, Rect::new(0, 0, w, h)); + assert_ne!(other.white, vec![15520.0]); + } + + /// storytold/lightcraft#976: the D7000 and D5100 record no CropArea, and their 36 masked columns at ~600 were developed as a + /// coloured bar. Their measured area applies to the measured models, data sizes and depths only. + #[test] + fn measured_d7000_and_d5100_image_area() { + let (w, h) = (4992usize, 3280usize); + let decode_as = |model: &str, bits: u16| { + let nef = nef_model_in(ByteOrder::Big, model, 1, bits, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, bits) in [("NIKON D7000", 14), ("NIKON D7000", 12), ("NIKON D5100", 14)] { + let raw = decode_as(model, bits); + assert_eq!(raw.crop, Rect::new(8, 8, 4928, 3264), "{model} {bits}-bit"); + assert_eq!((raw.crop.width, raw.crop.height), (4928, 3264)); + assert_eq!(raw.info().developed_size(), (4928, 3264)); + } + // another model, or the D5100's unmeasured 12-bit mode, keeps the whole data + for (model, bits) in [("NIKON D7100", 14), ("NIKON D5100", 12), ("NIKON D7000 ", 16)] { + assert_eq!(decode_as(model, bits).crop, Rect::new(0, 0, w, h), "{model} {bits}-bit"); + } + } + + /// The D200, D3200 and Nikon 1 V1 keep masked or dark lines in their whole data (the D200 shares the D80's data + /// size but frames it elsewhere), measured on public sample files beyond the CC0 ones; the D3100, D5000, D40, D50, + /// D70 and D70s get best-guess frames (the D40 shares the D50 / D70 / D70s data size but not their frame). + #[test] + fn measured_older_dx_bodies_image_area() { + let decode_as = |model: &str, (w, h): (usize, usize)| { + let nef = nef_model_in(ByteOrder::Big, model, 1, 12, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32, None); + crate::decode(&nef).unwrap() + }; + for (model, size, crop) in [ + ("NIKON D200", (3904, 2616), Rect::new(16, 12, 3872, 2592)), + ("NIKON D80", (3904, 2616), Rect::new(14, 9, 3872, 2592)), + ("NIKON D3200", (6080, 4012), Rect::new(8, 6, 6016, 4000)), + ("NIKON 1 V1", (3904, 2606), Rect::new(12, 6, 3872, 2592)), + ("NIKON D3100", (4672, 3084), Rect::new(24, 6, 4608, 3072)), + ("NIKON D5000", (4352, 2868), Rect::new(11, 10, 4288, 2848)), + ("NIKON D40", (3040, 2014), Rect::new(16, 9, 3008, 2000)), + ("NIKON D50", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ("NIKON D70s", (3040, 2014), Rect::new(15, 7, 3008, 2000)), + ] { + assert_eq!(decode_as(model, size).crop, crop, "{model}"); + } + // the V1's area on the V2's data size: no entry, the whole data + assert_eq!(decode_as("NIKON 1 V1", (4620, 3084)).crop, Rect::new(0, 0, 4620, 3084)); + } + + /// Every measured entry is a crop inside its data, once per model, data size and depth, with a white level (if + /// any) inside its bit range. + #[test] + fn measured_bodies_are_consistent() { + let mut seen = std::collections::HashSet::new(); + for b in MEASURED_BODIES { + let (x, y, w, h) = b.crop; + assert!(w > 0 && h > 0 && x + w <= b.size.0 && y + h <= b.size.1, "{}: crop outside its data", b.model); + assert!(seen.insert((b.model, b.size, b.bits)), "{} {:?} {}-bit listed twice", b.model, b.size, b.bits); + assert!(matches!(b.bits, 12 | 14), "{}: {}-bit", b.model, b.bits); + assert!(b.white.is_none_or(|v| v > 0.0 && v < (1u32 << b.bits) as f32), "{}: white level", b.model); + assert_eq!(b.model.trim(), b.model, "{}: model names are matched trimmed", b.model); + } + } + + /// storytold/lightcraft#1248: the D7100 records no CropArea, and its data holds picture beyond Lightroom's frame in both + /// image areas (full frame and 1.3x crop), so renders came out larger and Lightroom crops landed off. + #[test] + fn measured_d7100_image_area() { + let decode_as = |model: &str, (w, h): (usize, usize)| { + let mut nef = nef(1, 14, vec![vec![0x10; w * h * 2]], w as u32, h as u32, h as u32); + // the fixture's model is "NIKON TEST" plus its NUL: 11 bytes to overwrite + let at = nef.windows(11).position(|b| b == b"NIKON TEST\0").unwrap(); + let name = format!("{model:\0<11}"); + nef[at..at + 11].copy_from_slice(name.as_bytes()); + crate::decode(&nef).unwrap() + }; + for (size, crop) in [((6036, 4020), Rect::new(18, 10, 6000, 4000)), ((4812, 3212), Rect::new(6, 6, 4800, 3200))] { + let d7100 = decode_as("NIKON D7100", size); + assert_eq!(d7100.crop, crop); + assert_eq!((d7100.crop.width, d7100.crop.height), (crop.width, crop.height)); + assert_ne!(d7100.white, vec![15520.0]); + assert_eq!(d7100.info().developed_size(), (crop.width, crop.height)); + assert_eq!(decode_as("NIKON D7200", size).crop, Rect::new(0, 0, size.0, size.1)); + } + } + + /// The D90's last 42 columns are filler at a flat ~254, above its dark image samples: without the measured + /// area they stay in the image and develop as a pink band on the right. + #[test] + fn measured_d90_drops_the_filler_columns() { + let (w, h) = (4352usize, 2868usize); + let mut px = vec![40u16; w * h]; + for row in px.chunks_mut(w) { + row[4310..].fill(254); + } + let mut nef = nef(1, 12, vec![px.iter().flat_map(|v| v.to_le_bytes()).collect()], w as u32, h as u32, h as u32); + let at = nef.windows(10).position(|b| b == b"NIKON TEST").unwrap(); + nef[at..at + 10].copy_from_slice(b"NIKON D90 "); + let d90 = crate::decode(&nef).unwrap(); + assert_eq!(d90.crop, Rect::new(10, 10, 4288, 2848)); + assert!(d90.crop.x + d90.crop.width <= 4310, "the filler columns are outside the image"); + assert_eq!(d90.white, vec![3830.0]); + assert_eq!((d90.crop.width, d90.crop.height), (4288, 2848)); + } + + /// storytold/lightcraft#708: the active width is judged from the first [`MASK_ROWS`] rows in both modes, so the header probe + /// (which reads only those rows' strips) and the full decode agree whether the masked columns are there or not. + #[test] + fn masked_columns_are_judged_from_the_top_rows_in_both_modes() { + let (w, h, rps) = (256usize, MASK_ROWS + 64, 8u32); + let file = |masked_rows: std::ops::Range| { + let words: Vec = (0..w * h) + .map(|i| if masked_rows.contains(&(i / w)) && i % w >= w - 8 { 0u16 } else { 1000 + (i % 7) as u16 }) + .flat_map(|v| v.to_be_bytes()) + .collect(); + nef(1, 14, words.chunks(w * 2 * rps as usize).map(|c| c.to_vec()).collect(), w as u32, h as u32, rps) + }; + for (masked_rows, expected_width) in [(0..h, w - 8), (0..MASK_ROWS, w - 8), (MASK_ROWS..h, w), (0..0, w)] { + let bytes = file(masked_rows.clone()); + let full = crate::decode(&bytes).unwrap(); + assert_eq!(full.active_area.width, expected_width, "masked rows {masked_rows:?}"); + assert_eq!(full.crop.width, expected_width); + assert_eq!(crate::probe_info(&bytes).unwrap(), full.info(), "masked rows {masked_rows:?}"); + } + // the shared rule on its own: nominal white, trailing columns at most 64, even width + let d: Vec = (0..w * 4).map(|i| if i % w >= w - 9 { 0 } else { 600 }).collect(); + assert_eq!(trailing_masked_columns(&d, w, 14), w - 10, "nine dark columns trim to an even width"); + assert_eq!(trailing_masked_columns(&d, w, 12), w - 10); + assert_eq!(trailing_masked_columns(&d[..w - 1], w, 14), w, "less than a row of samples leaves the width alone"); + assert_eq!(trailing_masked_columns(&[0; 64], 64, 14), 64, "narrow images are left alone"); + assert_eq!(trailing_masked_columns(&[], 0, 14), 0); + } + + #[test] + fn black_level_tag_is_in_14_bit_units() { + let (w, h) = (16usize, 4usize); + let words: Vec = (0..w * h).flat_map(|i| (100 + i as u16).to_be_bytes()).collect(); + let note = || { + let mut mn = IfdBuilder::new(); + mn.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412])); + mn + }; + let black = |bits| crate::decode(&nef_with_note(1, bits, vec![words.clone()], w as u32, h as u32, h as u32, Some(note()))).unwrap().black; + assert_eq!(black(12).values, vec![100.0, 101.0, 102.0, 103.0]); + assert_eq!(black(14).values, vec![400.0, 404.0, 408.0, 412.0]); + } + + #[test] + fn uncompressed_word16_and_packed12() { + let (w, h) = (16usize, 6usize); + let px: Vec = (0..w * h).map(|i| (i * 131 % 4096) as u16).collect(); + let words: Vec = px.iter().flat_map(|v| v.to_be_bytes()).collect(); + let bytes = nef(1, 12, words.chunks(w * 2 * 3).map(|c| c.to_vec()).collect(), w as u32, h as u32, 3); + assert_eq!(crate::probe(&bytes), Some(RawFormat::Nef)); + let r = crate::decode(&bytes).unwrap(); + assert_eq!(r.data, RawData::U16(px.clone())); + assert_eq!(r.cfa.as_ref().unwrap().name(), "BGGR"); + // 12-bit MSB packed + let mut packed = Vec::new(); + for pair in px.chunks(2) { + let (a, b) = (pair[0] as u32, pair[1] as u32); + let v = (a << 12) | b; + packed.extend_from_slice(&[(v >> 16) as u8, (v >> 8) as u8, v as u8]); + } + let bytes = nef(1, 12, vec![packed], w as u32, h as u32, h as u32); + assert_eq!(crate::decode(&bytes).unwrap().data, RawData::U16(px)); + } + + /// An NRW data block (`"NRW 0104"`, fields from byte 8) with `black` at byte `0x20`, as a maker note. + fn nrw_note(black: u32) -> IfdBuilder { + let mut block = b"NRW 0104".to_vec(); + block.resize(0x20, 0); + block.extend_from_slice(&black.to_le_bytes()); + block.resize(0x40, 0); + let mut mn = IfdBuilder::new(); + mn.set(NRW_DATA, Value::Undefined(block)); + mn + } + + /// 12-bit samples as an MSB-first bit stream inside 32-bit words in `order` (rows are whole words). + fn words32(px: &[u16], w: usize, order: ByteOrder) -> Vec { + let mut out = Vec::new(); + for row in px.chunks(w) { + let stream = row.iter().fold(Vec::new(), |mut bits: Vec, v| { + bits.extend((0..12).rev().map(|b| v >> b & 1 == 1)); + bits + }); + for word in stream.chunks(32) { + let v = word.iter().enumerate().fold(0u32, |a, (i, b)| a | (*b as u32) << (31 - i)); + out.extend_from_slice(&match order { + ByteOrder::Little => v.to_le_bytes(), + ByteOrder::Big => v.to_be_bytes(), + }); + } + } + out + } + + #[test] + fn nrw_block_selects_32_bit_word_packing_and_supplies_the_black_level() { + let (w, h) = (16usize, 6usize); + let px: Vec = (0..w * h).map(|i| (200 + i * 131 % 3800) as u16).collect(); + let packed = words32(&px, w, ByteOrder::Little); + assert_eq!(packed.len(), w * h * 3 / 2); + let decode = |note: Option| { + crate::decode(&nef_in(ByteOrder::Little, 1, 12, vec![packed.clone()], w as u32, h as u32, h as u32, note)).unwrap() + }; + let r = decode(Some(nrw_note(200))); + assert_eq!(r.data, RawData::U16(px.clone())); + assert_eq!(r.black.values, vec![200.0]); + assert_eq!(decode(Some(nrw_note(0))).black.values, vec![0.0]); + // an implausible field is not a black level + assert_eq!(decode(Some(nrw_note(60000))).black.values, vec![0.0]); + // without the block the same bytes are an MSB-first byte stream + let plain = decode(None); + assert_ne!(plain.data, RawData::U16(px)); + assert_eq!(plain.black.values, vec![0.0]); + // the maker-note black level (14-bit units) wins when present + let mut both = nrw_note(200); + both.set(BLACK_LEVEL, Value::Short(vec![400, 404, 408, 412])); + assert_eq!(decode(Some(both)).black.values, vec![100.0, 101.0, 102.0, 103.0]); + } + + #[test] + fn compressed_without_table_is_unsupported() { + let bytes = nef(34713, 14, vec![vec![0; 64]], 8, 8, 8); + let Err(RawError::Unsupported(why)) = crate::decode(&bytes) else { panic!("expected unsupported") }; + assert!(why.contains("linearization table"), "{why}"); + } + + /// storytold/lightcraft#193: a High Efficiency NEF (Z f, Z 8: compression 34713, no `0x0096` table, a JPEG XS codestream) is + /// named as such instead of being blamed on the missing table, and the header probe reports the same reason. + #[test] + fn high_efficiency_nef_is_named() { + let mut strip = vec![0xff, 0x10, 0xff, 0x50, 0x00, 0x22]; + strip.resize(64, 0); + let bytes = nef(34713, 14, vec![strip], 8, 8, 8); + for result in [crate::decode(&bytes).map(|_| ()), crate::probe_info(&bytes).map(|_| ())] { + let Err(RawError::Unsupported(why)) = result else { panic!("expected unsupported") }; + assert_eq!(why, HIGH_EFFICIENCY); + assert!(why.starts_with("Nikon High Efficiency NEF ("), "{why}"); + } + // a strip too short to hold the markers is still the missing-table case, not a panic + let short = nef(34713, 14, vec![vec![0xff, 0x10]], 1, 1, 1); + assert!(matches!(crate::decode(&short), Err(RawError::Unsupported(w)) if w.contains("linearization table"))); + } + + fn samples(n: usize, bits: u32) -> Vec { + (0..n).map(|i| (((i * 2654435761usize) >> 7) as u32 % (1 << bits)) as u16).collect() + } + + fn pack_lsb(row: &[u16], bits: u32) -> Vec { + let (mut out, mut acc, mut n) = (Vec::new(), 0u64, 0u32); + for &v in row { + acc |= (v as u64) << n; + n += bits; + while n >= 8 { + out.push(acc as u8); + acc >>= 8; + n -= 8; + } + } + if n > 0 { + out.push(acc as u8); + } + out + } + + fn pack_msb12(row: &[u16]) -> Vec { + let mut out = Vec::new(); + for pair in row.chunks(2) { + let (a, b) = (pair[0], pair.get(1).copied().unwrap_or(0)); + out.extend([(a >> 4) as u8, ((a & 15) << 4) as u8 | (b >> 8) as u8, b as u8]); + } + out + } + + /// Rows of `samples` packed by `row`, each padded with `pad` bytes of 0xA5 (never decoded). + fn strip(data: &[u16], w: usize, pad: usize, row: impl Fn(&[u16]) -> Vec) -> Vec { + data.chunks(w).flat_map(|r| row(r).into_iter().chain(std::iter::repeat_n(0xA5, pad))).collect() + } + + fn decoded(strip: Vec, bits: u16, w: u32, h: u32) -> Result> { + let bytes = nef(34713, bits, vec![strip], w, h, h); + match crate::decode(&bytes)?.data { + RawData::U16(d) => Ok(d), + _ => Err(RawError::Unsupported("float".into())), + } + } + + #[test] + fn packed_34713_layouts_round_trip() { + let (w, h) = (40usize, 6usize); + let d12 = samples(w * h, 12); + let d14 = samples(w * h, 14); + // group A: 12-bit, 3w/2 bytes per row + assert_eq!(decoded(strip(&d12, w, 0, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group B: 12-bit, rows padded to 16 bytes (60 -> 64) + assert_eq!(decoded(strip(&d12, w, 4, |r| pack_lsb(r, 12)), 12, 40, 6).unwrap(), d12); + // group C: 14-bit, 70 bytes -> 80 + assert_eq!(decoded(strip(&d14, w, 10, |r| pack_lsb(r, 14)), 14, 40, 6).unwrap(), d14); + // group E: 16-byte chunks, 15 data bytes (ten samples) + a zero byte; 16 chunks = 160 samples (at 40 wide the + // 64-byte stride would be group B's) + let (w, h) = (160usize, 3usize); + let d12 = samples(w * h, 12); + let e = strip(&d12, w, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + assert_eq!(e.len(), 256 * h); + assert_eq!(decoded(e, 12, 160, 3).unwrap(), d12); + } + + /// Z 6II "Packed 14 bits" (maker note 0x93 = 10, storytold/lightcraft#769): 6064x4040, 14-bit, one strip of 4040 rows of 10624 + /// bytes (6064 * 14 / 8 = 10612, padded to a multiple of 16). Four rows keep the real stride. + #[test] + fn packed_14_bit_z6ii_geometry_with_row_padding() { + let (w, h) = (6064usize, 4usize); + let d14 = samples(w * h, 14); + let s = strip(&d14, w, 12, |r| pack_lsb(r, 14)); + assert_eq!(s.len(), 10624 * h); + assert_eq!(decoded(s, 14, w as u32, h as u32).unwrap(), d14); + assert_eq!(packed_layout(6064, 4040, 14, 42_920_960, false), Some(PackedLayout::Lsb { stride: 10624 })); + } + + /// The maker note's compression mode routes the strip: 6 and 10 are packed samples even when a `0x0096` + /// table is present (which the Huffman reader would otherwise take), in the old tag and in the `0x0051` + /// record, in either byte order; a depth that contradicts the mode, and High Efficiency (13, 14), are refused. + #[test] + fn maker_note_compression_mode_routes_packed_and_high_efficiency() { + let (w, h) = (40usize, 6usize); + let note = |order: ByteOrder, modern: bool, mode: u16| { + let mut n = IfdBuilder::new(); + n.set(LINEARIZATION_TABLE, Value::Undefined(vec![0x44, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])); + if modern { + let mut record = b"01000400\0\0".to_vec(); + record.extend(if order == ByteOrder::Big { mode.to_be_bytes() } else { mode.to_le_bytes() }); + n.set(NEF_COMPRESSION_RECORD, Value::Undefined(record)); + } else { + n.set(NEF_COMPRESSION, Value::Short(vec![mode])); + } + n + }; + let file = |order, modern, mode, bits: u16, image: Vec| { + nef_in(order, 34713, bits, vec![image], w as u32, h as u32, h as u32, Some(note(order, modern, mode))) + }; + for (bits, mode) in [(12u32, 6u16), (14, 10)] { + let data = samples(w * h, bits); + // rows padded to 16 bytes, and tight rows (14-bit tight rows only when the mode is stated) + for pad in [(16 - (w * bits as usize / 8) % 16) % 16, 0] { + let image = strip(&data, w, pad, |r| pack_lsb(r, bits)); + for order in [ByteOrder::Big, ByteOrder::Little] { + for modern in [false, true] { + let RawData::U16(got) = crate::decode(&file(order, modern, mode, bits as u16, image.clone())).unwrap().data else { + panic!("integer samples") + }; + assert_eq!(got, data, "{bits}-bit, pad {pad}, {order:?}, modern {modern}"); + } + } + } + } + let d14 = samples(w * h, 14); + let image = strip(&d14, w, 10, |r| pack_lsb(r, 14)); + // tight 14-bit rows without a stated mode stay refused + assert!(decoded(strip(&d14, w, 0, |r| pack_lsb(r, 14)), 14, w as u32, h as u32).is_err()); + // the mode says 12-bit packed, the IFD 14 bits + assert!(matches!(crate::decode(&file(ByteOrder::Big, false, 6, 14, image.clone())), Err(RawError::Unsupported(_)))); + for mode in [13, 14] { + for modern in [false, true] { + let err = crate::decode(&file(ByteOrder::Big, modern, mode, 14, image.clone())).unwrap_err(); + assert!(matches!(&err, RawError::Unsupported(m) if m == HIGH_EFFICIENCY), "{err:?}"); + } + } + } + + #[test] + fn packed_layout_refuses_overflowing_dimensions() { + for bits in [12, 14] { + assert!(packed_layout(usize::MAX - 3, 1, bits, u64::MAX, false).is_none()); + } + assert!(d100_layout(usize::MAX, 1, 12, u64::MAX).is_none()); + } + + /// Header probes inspect only the same top rows that the masked-column test uses. Routing + /// still validates the complete strip's layout, so header and full decode keep the same crop. + #[test] + fn packed_header_probe_reads_only_the_mask_rows() { + let (w, h) = (160usize, MASK_ROWS + 7); + let data = samples(w * h, 12); + for image in [strip(&data, w, 0, |r| pack_lsb(r, 12)), be16(&data)] { + let bytes = nef(34713, 12, vec![image], w as u32, h as u32, h as u32); + let tiff = Tiff::parse(&bytes).unwrap(); + let info = raw_ifd(&tiff).unwrap().image().unwrap(); + let RawData::U16(top) = packed(&bytes, &info, MASK_ROWS, false, &Limits::default()).unwrap() else { panic!("integer samples") }; + assert_eq!(top, data[..w * MASK_ROWS]); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!((header.active_area, header.crop), (full.active_area, full.crop)); + assert!(matches!(header.data, RawData::U16(ref samples) if samples.is_empty())); + } + } + + #[test] + fn packed_d100_rows_carry_extra_columns() { + // IFD width 154, rows hold 160 samples (16 chunks); the last 6 are dropped + let (w, wide, h) = (154usize, 160usize, 3usize); + let wide_data = samples(wide * h, 12); + let e = strip(&wide_data, wide, 0, |r| r.chunks(10).flat_map(|c| pack_msb12(c).into_iter().chain([0u8])).collect()); + let got = decoded(e, 12, w as u32, h as u32).unwrap(); + let want: Vec = wide_data.chunks(wide).flat_map(|r| r[..w].iter().copied()).collect(); + assert_eq!(got, want); + } + + #[test] + fn packed_34713_wrong_sizes_stay_refused() { + let d12 = samples(40 * 6, 12); + let good = strip(&d12, 40, 0, |r| pack_lsb(r, 12)); + // one byte more or fewer, a stride that is neither tight nor 16-aligned, a 13-bit file + for len in [good.len() - 1, good.len() + 1, good.len() + 6] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_)))); + // 14-bit rows must be padded to 16 bytes + let d14 = samples(40 * 6, 14); + assert!(matches!(decoded(strip(&d14, 40, 0, |r| pack_lsb(r, 14)), 14, 40, 6), Err(RawError::Unsupported(_)))); + } + + /// A small raw strip of `[Y0 Y1 Cb Cr]` groups, rows of `3 * w` bytes, LSB-first 12-bit. + fn small_strip(groups: &[[u16; 4]], w: usize) -> Vec { + let flat: Vec = groups.iter().flatten().copied().collect(); + assert_eq!(flat.len() % (2 * w), 0); + flat.chunks(2 * w).flat_map(|r| pack_lsb(r, 12)).collect() + } + + fn small(strip: Vec, bits: u16, w: u32, h: u32) -> Result { + crate::decode(&nef(34713, bits, vec![strip], w, h, h)) + } + + #[test] + fn small_raw_422_round_trips() { + let (w, h) = (8usize, 3usize); + // flat chroma per row so the interpolation is exact; distinct luma everywhere + let mut groups = Vec::new(); + for y in 0..h { + for k in 0..w / 2 { + let (cb, cr) = (2048 + 10 * y as u16, 2048 - 5 * y as u16); + groups.push([100 + (y * w + 2 * k) as u16, 100 + (y * w + 2 * k + 1) as u16, cb, cr]); + } + } + let raw = small(small_strip(&groups, w), 12, w as u32, h as u32).unwrap(); + assert_eq!((raw.width, raw.height, raw.cpp, raw.cfa.is_none()), (w, h, 3, true)); + assert_eq!(raw.wb_multipliers, Some([1.0; 3])); + let RawData::U16(d) = &raw.data else { panic!() }; + for y in 0..h { + for x in 0..w { + let luma = (100 + y * w + x) as f32; + let (u, v) = (10.0 * y as f32, -5.0 * y as f32); + let (r, b) = (luma + SMALL_RAW_R_GAIN * v, luma + SMALL_RAW_B_GAIN * u); + let g = (luma - 0.299 * r - 0.114 * b) / 0.587; + for (i, want) in [r, g, b].into_iter().enumerate() { + let got = f32::from(d[(y * w + x) * 3 + i]) / SMALL_RAW_SCALE; + assert!((got - want).abs() < 0.1, "({x},{y}) channel {i}: {got} vs {want}"); + } + } + } + } + + #[test] + fn small_raw_neutral_chroma_is_grey_and_chroma_is_interpolated() { + let w = 8usize; + let groups: Vec<[u16; 4]> = (0..w / 2).map(|k| [200 + 40 * k as u16, 200 + 40 * k as u16, 2048, 2048]).collect(); + let raw = small(small_strip(&groups, w), 12, w as u32, 1).unwrap(); + let RawData::U16(d) = &raw.data else { panic!() }; + for x in 0..w { + let px = &d[x * 3..x * 3 + 3]; + assert!(px.iter().all(|&v| v == px[0]), "{px:?}"); + assert_eq!(f32::from(px[0]), (200 + 40 * (x / 2)) as f32 * SMALL_RAW_SCALE); + } + // a step in Cb between the first two pairs is shared 3:1 by the pixels next to the boundary + let mut groups = groups; + groups[1][2] = 2048 + 100; + let raw = small(small_strip(&groups, w), 12, w as u32, 1).unwrap(); + let RawData::U16(d) = &raw.data else { panic!() }; + let blue_minus_luma = |x: usize, luma: f32| f32::from(d[x * 3 + 2]) / SMALL_RAW_SCALE - luma; + assert!((blue_minus_luma(1, 200.0) - SMALL_RAW_B_GAIN * 25.0).abs() < 0.1); + assert!((blue_minus_luma(2, 240.0) - SMALL_RAW_B_GAIN * 75.0).abs() < 0.1); + } + + #[test] + fn small_raw_wrong_sizes_and_depths_are_refused() { + let (w, h) = (8usize, 3usize); + let good = small_strip(&vec![[100, 100, 2048, 2048]; w / 2 * h], w); + assert!(small(good.clone(), 12, w as u32, h as u32).is_ok()); + for len in [good.len() - 1, good.len() + 1] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(small(s, 12, w as u32, h as u32), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(small(good.clone(), 14, w as u32, h as u32), Err(RawError::Unsupported(_)))); + // odd width: 3 bytes per pixel does not split into pixel pairs + assert!(matches!(small(vec![0; 3 * 7 * 3], 12, 7, 3), Err(RawError::Unsupported(_)))); + } + + #[test] + fn small_raw_short_input_does_not_panic() { + for len in [0, 1, 2, 3, 5, 11, 12, 13] { + let mut out = vec![0u16; 12]; + small_raw_row(&vec![0xFF; len], 4, &mut out); + } + let mut bytes = nef(34713, 12, vec![vec![0x55; 3 * 8 * 3]], 8, 3, 3); + bytes.truncate(bytes.len() - 40); + assert!(crate::decode(&bytes).is_err(), "a truncated small-raw strip is corrupt"); + } + + #[test] + fn small_raw_header_has_full_geometry_without_allocating_pixels() { + let (w, h) = (8usize, 3usize); + let bytes = nef(34713, 12, vec![small_strip(&vec![[100, 110, 2048, 2048]; w / 2 * h], w)], w as u32, h as u32, h as u32); + let header = decode(&bytes, Mode::Header, &Limits::default()).unwrap(); + let full = decode(&bytes, Mode::Full, &Limits::default()).unwrap(); + assert_eq!(header.info(), full.info()); + assert!(matches!(header.data, RawData::U16(ref samples) if samples.is_empty())); + assert!(matches!(full.data, RawData::U16(ref samples) if samples.len() == 3 * w * h)); + } + + #[test] + fn small_raw_rejects_excessive_dimensions_before_allocation() { + let bytes = nef(34713, 12, vec![vec![0; 72]], 8, 3, 3); + let tiff = Tiff::parse(&bytes).unwrap(); + let mut info = raw_ifd(&tiff).unwrap().image().unwrap(); + let strip = info.chunks(bytes.len() as u64)[0]; + info.width = u32::MAX - 1; + info.height = u32::MAX; + assert!(matches!(small_raw(&bytes, &tiff, &info, strip, Mode::Header, &Limits::default()), Err(RawError::Limit(_)))); + let mut out = [0; 12]; + small_raw_row(&[], usize::MAX, &mut out); + small_raw_row(&[], 3, &mut out); + assert_eq!(out, [0; 12]); + } + + /// Big-endian 16-bit words holding the 12-bit samples in their upper bits. + fn be16(data: &[u16]) -> Vec { + data.iter().flat_map(|v| (v << 4).to_be_bytes()).collect() + } + + #[test] + fn coolpix_be16_words_decode_and_wrong_sizes_are_refused() { + let (w, h) = (40usize, 6usize); + let d12 = samples(w * h, 12); + let good = be16(&d12); + assert_eq!(good.len(), 2 * w * h); + assert_eq!(decoded(good.clone(), 12, 40, 6).unwrap(), d12); + // a half-LSB in the low nibble does not change the sample + let mut half = good.clone(); + for b in half.iter_mut().skip(1).step_by(2) { + *b |= 8; + } + assert_eq!(decoded(half, 12, 40, 6).unwrap(), d12); + // any other strip size, or another bit depth, is not this layout + for len in [good.len() - 2, good.len() - 1, good.len() + 1, good.len() + 2, good.len() + 16] { + let mut s = good.clone(); + s.resize(len, 0); + assert!(matches!(decoded(s, 12, 40, 6), Err(RawError::Unsupported(_))), "len {len}"); + } + assert!(matches!(decoded(good.clone(), 13, 40, 6), Err(RawError::Unsupported(_)))); + // the black level is 200, but never above the darkest sample + let black = |data: &[u16]| crate::decode(&nef(34713, 12, vec![be16(data)], 40, 6, 6)).unwrap().black.values[0]; + let mut d = d12.clone(); + for v in &mut d { + *v = (*v).max(300); + } + assert_eq!(black(&d), 200.0); + d[5] = 150; + assert_eq!(black(&d), 150.0); + } + + #[test] + fn coolpix_be16_does_not_take_other_layouts() { + // every width where another group's stride equals 2w must still go to that group (the narrow rows where they + // coincide are below the real-world widths; the other groups are tried first) + for w in (2..=400usize).step_by(2) { + let strip = 2 * w as u64 * 6; + let other = packed_layout(w, 6, 12, strip, false).or_else(|| d100_layout(w, 6, 12, strip)); + if other.is_some() { + assert_eq!( + packed_layout(w, 6, 12, strip, false).or_else(|| d100_layout(w, 6, 12, strip)).or_else(|| be16_layout(w, 6, 12, strip)), + other + ); + } + } + assert_eq!(be16_layout(4032, 3024, 12, 24385536), Some(PackedLayout::Be16)); + assert!(packed_layout(4032, 3024, 12, 24385536, false).is_none() && d100_layout(4032, 3024, 12, 24385536).is_none()); + assert!(be16_layout(4032, 3024, 14, 24385536).is_none()); + assert!(be16_layout(usize::MAX, usize::MAX, 12, u64::MAX).is_none()); + } + + #[test] + fn packed_34713_short_input_does_not_panic() { + let (w, h) = (40usize, 6usize); + for layout in [PackedLayout::Lsb { stride: 64 }, PackedLayout::Lsb { stride: 80 }, PackedLayout::Chunked16 { stride: 64 }, PackedLayout::Be16] + { + for len in [0, 1, 15, 16, 17, 63, 64, 65, 200] { + let bits = if matches!(layout, PackedLayout::Lsb { stride: 80 }) { 14 } else { 12 }; + assert_eq!(unpack_packed(&vec![0xFF; len], w, h, bits, layout).len(), w * h); + } + } + // a strip that claims more bytes than the file has + let mut bytes = nef(34713, 12, vec![vec![0x55; 60 * 6]], 40, 6, 6); + bytes.truncate(bytes.len() - 100); + assert!(crate::decode(&bytes).is_err(), "a truncated packed strip is corrupt"); + } + /// Keep every directory intact before a relocated tail strip, then remove its last rows. + fn truncated_tail_strip(mut bytes: Vec, keep: usize) -> Vec { + let tiff = Tiff::parse(&bytes).unwrap(); + let raw = raw_ifd(&tiff).unwrap(); + let info = raw.image().unwrap(); + let chunk = info.chunks(bytes.len() as u64)[0]; + let data = craft_tiff::image::chunk_bytes(&bytes, &chunk).unwrap().to_vec(); + let offset_entry = raw.get(t::STRIP_OFFSETS).unwrap().offset as usize; + let start = bytes.len() as u32; + bytes[offset_entry..offset_entry + 4].copy_from_slice(&start.to_be_bytes()); + bytes.extend_from_slice(&data[..keep]); + bytes + } + + #[test] + fn packed_rejects_tail_truncation_with_intact_directories() { + let bytes = nef(34713, 12, vec![vec![0x55; 360]], 40, 6, 6); + for keep in [0, 60, 359] { + let short = truncated_tail_strip(bytes.clone(), keep); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&short, mode, &Limits::default()), Err(RawError::Corrupt(_)))); + } + } + } + + #[test] + fn small_raw_rejects_tail_truncation_with_intact_directories() { + let bytes = nef(34713, 12, vec![vec![0x55; 72]], 8, 3, 3); + for keep in [0, 24, 71] { + let short = truncated_tail_strip(bytes.clone(), keep); + for mode in [Mode::Header, Mode::Full] { + assert!(matches!(decode(&short, mode, &Limits::default()), Err(RawError::Corrupt(_)))); + } + } + } +} diff --git a/crates/raw/src/nef_lens.rs b/crates/raw/src/nef_lens.rs new file mode 100644 index 000000000..9f8dafb0e --- /dev/null +++ b/crates/raw/src/nef_lens.rs @@ -0,0 +1,259 @@ +//! Nikon's in-file distortion correction (raw SubIFD tag `0xc7d5`, NikonNEFInfo), expressed as a DNG warp for +//! the shared optics pipeline. +//! +//! Tag identity: the ExifTool Nikon tag-name documentation (NikonNEFInfo `0xc7d5`; its DistortionInfo `0x0005` +//! with DistortionCorrection and RadialDistortionCoefficient1..3). The model below (which stored value multiplies +//! which power, the normalisation, the fill-frame zoom and the optical-centre offset in entry `0x0008`) was inferred +//! independently by matching the raw against the camera's own corrected JPEG; see docs/nikon-lens-corrections.md. +//! No lens database or external decoder code. + +use crate::{Opcode, Rect}; +use craft_tiff::{ByteOrder, Ifd, ParseOptions, parse_ifd_at}; + +const NEF_INFO: u16 = 0xc7d5; +const DISTORTION_INFO: u16 = 0x0005; +const CENTRE_OFFSET: u16 = 0x0008; +/// Bodies whose files were checked against the camera's JPEG (docs/nikon-lens-corrections.md). Matching tags on +/// another body do not establish its pixel pitch or image area; add a body only after checking its files. +const VALIDATED: &[&str] = &["NIKON Z 6_2", "NIKON Z 8", "NIKON Z f"]; +/// The normalisation radius every checked file stores: the FX half-diagonal in millimetres. +const FX_HALF_DIAGONAL_MM: f64 = 21.6; +const SAMPLES: usize = 256; + +/// `r_src = r_out * (1 + k3 rho^2 + k2 rho^4 + k1 rho^6) / (1 + zoom)` around the optical centre, `rho` the radius +/// over the half-diagonal of the image area (`k1..k3` as ExifTool names them, in the order they are stored). +#[derive(Clone, Copy, Debug, PartialEq)] +struct Distortion { + k: [f64; 3], + zoom: f64, +} + +/// A signed rational at `at` (`i32 / i32`), when it is one. +fn rational(b: &[u8], at: usize, order: ByteOrder) -> Option { + let n = order.u32(b.get(at..at + 4)?.try_into().ok()?) as i32; + let d = order.u32(b.get(at + 4..at + 8)?.try_into().ok()?) as i32; + (d > 0).then(|| f64::from(n) / f64::from(d)).filter(|v| v.is_finite()) +} + +/// The NikonNEFInfo entries: the blob is `"Nikon\0"`, a 4-byte version, then a little TIFF stream. +fn nef_info(raw: &Ifd) -> Option<(Ifd, ByteOrder)> { + let tiff = raw.bytes(NEF_INFO)?.strip_prefix(b"Nikon\0")?.get(4..)?; + let (order, false) = craft_tiff::Tiff::sniff(tiff)? else { return None }; + let first = order.u32(tiff.get(4..8)?.try_into().ok()?); + let opts = ParseOptions { follow_children: false, ..Default::default() }; + let (ifd, _) = parse_ifd_at(tiff, u64::from(first), order, 0, false, &opts).ok()?; + Some((ifd, order)) +} + +/// DistortionInfo: `"0100"`, the correction mode (0 none), the normalisation radius (mm) at 8, the term count (4) +/// at 16, the coefficients at 20 / 28 / 36 and the zoom at 64, each a signed rational. +fn distortion_info(b: &[u8], order: ByteOrder) -> Option { + if b.get(0..4)? != b"0100" || *b.get(4)? == 0 { + return None; + } + let norm = rational(b, 8, order)?; + let terms = order.u32(b.get(16..20)?.try_into().ok()?); + if (norm - FX_HALF_DIAGONAL_MM).abs() > 0.05 || terms != 4 { + return None; + } + let k = [rational(b, 20, order)?, rational(b, 28, order)?, rational(b, 36, order)?]; + let zoom = rational(b, 64, order)?; + // real lenses stay far inside these (the checked files: |k| < 0.13, |zoom| < 0.09) + (k.iter().all(|v| v.abs() < 1.0) && zoom.abs() < 0.5).then_some(Distortion { k, zoom }) +} + +/// Entry `0x0008`: `"0100"`, then the optical centre's offset from the image-area centre in pixels (x right, +/// y down), two signed rationals. Zero when the lens stabilises (the sensor stays centred). +fn centre_offset(b: &[u8], order: ByteOrder) -> Option<(f64, f64)> { + (b.get(0..4)? == b"0100").then_some(())?; + Some((rational(b, 4, order)?, rational(b, 12, order)?)) +} + +/// The correction of a `model` NEF whose raw IFD is `raw`, for the image area `crop` inside `active` (both in +/// sensor pixels), as an `OpcodeList3` warp in active-area coordinates. `None` for unvalidated bodies, other +/// image areas, missing or implausible data, and maps that fold. +pub(super) fn distortion(model: &str, raw: &Ifd, active: Rect, crop: Rect) -> Option { + if !VALIDATED.contains(&model.trim()) { + return None; + } + let crop = crop.clipped(active.width, active.height); + if active.width < 2 || active.height < 2 || crop.width < 2 || crop.height < 2 { + return None; + } + // the stored radius is the FX half-diagonal: only the full 3:2 image area (not a DX or other crop of the + // sensor) is normalised by it + if (crop.width as f64 / crop.height as f64 - 1.5).abs() > 0.005 || (crop.width as f64) < active.width as f64 * 0.95 { + return None; + } + let (info, order) = nef_info(raw)?; + let d = distortion_info(info.bytes(DISTORTION_INFO)?, order)?; + let (ox, oy) = info.bytes(CENTRE_OFFSET).and_then(|b| centre_offset(b, order)).unwrap_or((0.0, 0.0)); + let (cw, ch) = (crop.width as f64 - 1.0, crop.height as f64 - 1.0); + if ox.abs() > cw * 0.05 || oy.abs() > ch * 0.05 { + return None; + } + let half_diagonal = cw.hypot(ch) / 2.0; + let poly = |rho: f64| { + let x = rho * rho; + 1.0 + x * (d.k[2] + x * (d.k[1] + x * d.k[0])) + }; + // DNG normalises by the farthest active-area corner from the centre; the image area reaches at most there + let (ax, ay) = (active.width as f64 - 1.0, active.height as f64 - 1.0); + let (cx, cy) = (crop.x as f64 + cw / 2.0 + ox, crop.y as f64 + ch / 2.0 + oy); + let farthest = cx.max(ax - cx).hypot(cy.max(ay - cy)); + let a = farthest / half_diagonal; + // the source radius must grow with the output radius out to the farthest corner, or the map folds + let mut previous = -1.0; + for j in 0..=SAMPLES { + let rho = a * j as f64 / SAMPLES as f64; + let source = rho * poly(rho) / (1.0 + d.zoom); + if !source.is_finite() || source <= previous { + return None; + } + previous = source; + } + let s = 1.0 / (1.0 + d.zoom); + let a2 = a * a; + let planes = vec![[s, d.k[2] * a2 * s, d.k[1] * a2 * a2 * s, d.k[0] * a2 * a2 * a2 * s, 0.0, 0.0]]; + Some(Opcode::WarpRectilinear { planes, center: [cx / ax, cy / ay] }) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use craft_tiff::{Entry, IfdBuilder, TiffWriter, Value}; + + const AREA: Rect = Rect { x: 0, y: 0, width: 6064, height: 4040 }; + const CROP: Rect = Rect { x: 8, y: 8, width: 6048, height: 4024 }; + + fn rat(v: f64, den: i32) -> [u8; 8] { + let mut b = [0u8; 8]; + b[..4].copy_from_slice(&((v * f64::from(den)).round() as i32).to_le_bytes()); + b[4..].copy_from_slice(&den.to_le_bytes()); + b + } + + /// DistortionInfo as the checked files lay it out (84 bytes, little-endian). + pub(crate) fn distortion_block(mode: u8, norm: f64, terms: u32, k: [f64; 3], zoom: f64) -> Vec { + let mut b = vec![0u8; 84]; + b[..4].copy_from_slice(b"0100"); + b[4] = mode; + b[5] = 1; + b[8..16].copy_from_slice(&rat(norm, 4096)); + b[16..20].copy_from_slice(&terms.to_le_bytes()); + for (i, v) in k.iter().enumerate() { + b[20 + 8 * i..28 + 8 * i].copy_from_slice(&rat(*v, 1 << 20)); + } + b[44..52].copy_from_slice(&rat(0.0, 1)); + b[52..60].copy_from_slice(&rat(0.0, 1)); + b[64..72].copy_from_slice(&rat(zoom, 1 << 20)); + b + } + + pub(crate) fn offset_block(x: f64, y: f64) -> Vec { + let mut b = b"0100".to_vec(); + b.extend_from_slice(&rat(x, 256)); + b.extend_from_slice(&rat(y, 256)); + b.extend_from_slice(&[0, 0, 0, 0]); + b + } + + /// A NikonNEFInfo (`0xc7d5`) blob with the given entries. + pub(crate) fn nef_info_blob(entries: &[(u16, Vec)]) -> Vec { + let mut info = IfdBuilder::new(); + for (tag, data) in entries { + info.set(*tag, Value::Undefined(data.clone())); + } + let tiff = TiffWriter::new(ByteOrder::Little, false).write(&[info]).unwrap(); + let mut blob = b"Nikon\0\x01\x03\0\0".to_vec(); + blob.extend_from_slice(&tiff); + blob + } + + /// A raw IFD carrying a NikonNEFInfo blob with the given entries. + pub(super) fn raw_with(entries: &[(u16, Vec)]) -> Ifd { + Ifd { entries: vec![Entry { tag: NEF_INFO, value: Value::Undefined(nef_info_blob(entries)), offset: 0 }], ..Default::default() } + } + + // a Z 24-70 f/4 S at 24 mm and at 70 mm, as stored in two of the checked files + const WIDE: ([f64; 3], f64) = ([0.039895, 0.019211, -0.127218], -0.025043); + const TELE: ([f64; 3], f64) = ([0.01681, -0.010601, 0.056758], 0.088301); + + fn warp(model: &str, raw: &Ifd) -> Option<([f64; 6], [f64; 2])> { + match distortion(model, raw, AREA, CROP)? { + Opcode::WarpRectilinear { planes, center } => Some((planes[0], center)), + _ => None, + } + } + + /// The source radius the warp gives for an output radius `rho` (in image-area half-diagonals) about the centre. + fn source_rho(p: [f64; 6], centre: [f64; 2], rho: f64) -> f64 { + let (ax, ay) = (AREA.width as f64 - 1.0, AREA.height as f64 - 1.0); + let (cx, cy) = (centre[0] * ax, centre[1] * ay); + let m = cx.max(ax - cx).hypot(cy.max(ay - cy)); + let hd = (CROP.width as f64 - 1.0).hypot(CROP.height as f64 - 1.0) / 2.0; + let t2 = (rho * hd / m).powi(2); + rho * (p[0] + t2 * (p[1] + t2 * (p[2] + t2 * p[3]))) + } + + #[test] + fn the_warp_is_the_stored_polynomial_over_the_zoom() { + for (k, zoom) in [WIDE, TELE] { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, k, zoom))]); + let (p, c) = warp("NIKON Z 6_2", &raw).unwrap(); + for rho in [0.0, 0.25, 0.5, 0.75, 1.0] { + let x = rho * rho; + let want = rho * (1.0 + k[2] * x + k[1] * x * x + k[0] * x * x * x) / (1.0 + zoom); + assert!((source_rho(p, c, rho) - want).abs() < 2e-6, "{k:?} at {rho}"); + } + // centred on the image area, which sits 8 px in from the sensor's top-left + assert!((c[0] - (8.0 + 6047.0 / 2.0) / 6063.0).abs() < 1e-12 && (c[1] - (8.0 + 4023.0 / 2.0) / 4039.0).abs() < 1e-12); + } + } + + #[test] + fn the_optical_centre_follows_the_sensor_shift() { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, TELE.0, TELE.1)), (CENTRE_OFFSET, offset_block(-90.1, 18.59))]); + let (_, c) = warp("NIKON Z 6_2", &raw).unwrap(); + assert!((c[0] * 6063.0 - (8.0 + 6047.0 / 2.0 - 90.1)).abs() < 0.01, "{c:?}"); + assert!((c[1] * 4039.0 - (8.0 + 4023.0 / 2.0 + 18.59)).abs() < 0.01, "{c:?}"); + } + + #[test] + fn refuses_unvalidated_bodies_and_unknown_layouts() { + let good = distortion_block(3, 21.6, 4, WIDE.0, WIDE.1); + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + assert!(warp("NIKON Z f", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + assert!(warp("NIKON Z 8", &raw_with(&[(DISTORTION_INFO, good.clone())])).is_some()); + for model in ["", "NIKON Z 6", "NIKON Z 6_3", "NIKON Z 9", "NIKON D850", "NIKON Z 6_2 x"] { + assert!(warp(model, &raw_with(&[(DISTORTION_INFO, good.clone())])).is_none(), "{model}"); + } + // correction off, a DX radius, another term count, a wild coefficient or zoom, no NEFInfo at all + for bad in [ + distortion_block(0, 21.6, 4, WIDE.0, WIDE.1), + distortion_block(3, 14.2, 4, WIDE.0, WIDE.1), + distortion_block(3, 21.6, 3, WIDE.0, WIDE.1), + distortion_block(3, 21.6, 4, [0.0, 0.0, 1.5], 0.0), + distortion_block(3, 21.6, 4, WIDE.0, 0.9), + b"0200".iter().copied().chain([0u8; 80]).collect(), + vec![0u8; 12], + ] { + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, bad)])).is_none()); + } + assert!(warp("NIKON Z 6_2", &Ifd::default()).is_none()); + // a map that folds (the source radius turns back) is refused + assert!(warp("NIKON Z 6_2", &raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, [-0.9, 0.0, 0.0], 0.0))])).is_none()); + } + + #[test] + fn only_the_full_3_2_image_area_is_corrected() { + let raw = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, WIDE.0, WIDE.1))]); + // a DX or 1:1 crop of the sensor would need another normalisation + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(1016, 676, 4032, 2688)).is_none(), "DX"); + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(1016, 8, 4024, 4024)).is_none()); + assert!(distortion("NIKON Z 6_2", &raw, AREA, Rect::new(0, 0, 1, 1)).is_none()); + // a sensor-shift offset beyond 5 % of the frame is not trusted + let far = raw_with(&[(DISTORTION_INFO, distortion_block(3, 21.6, 4, WIDE.0, WIDE.1)), (CENTRE_OFFSET, offset_block(900.0, 0.0))]); + assert!(warp("NIKON Z 6_2", &far).is_none()); + } +} diff --git a/crates/raw/src/nefc.rs b/crates/raw/src/nefc.rs new file mode 100644 index 000000000..9904f7cca --- /dev/null +++ b/crates/raw/src/nefc.rs @@ -0,0 +1,794 @@ +//! Nikon Huffman-compressed NEF image data (TIFF compression 34713): "lossless compressed" and "lossy compressed" +//! (type 1 and type 2) at 12 and 14 bits. +//! +//! **Clean-room.** No raw-decoder source code (dcraw, LibRaw, rawspeed, rawloader/rawler, darktable, RawTherapee, +//! libopenraw, ExifTool's Perl code, …) was read or consulted. What we used: +//! +//! - Prose format descriptions: Laurent Clévy, "Nikon Electronic File (NEF) format" (), +//! the maker-note tag table and the "0x96 (linearization table) tag format" table (version bytes, four `u16` +//! predictor seeds at offset 2, curve size at 10, curve at 12, split value at 562 for lossy type 2, "lossy type +//! 2 reads an incomplete table and interpolation is required"); Bill Claff, "NEF Compression" +//! (): a lossy encoding curve followed by a +//! non-adaptive (fixed-table) Huffman stage, and the number of encoded values per camera (683, 769, 1025, 2753, +//! 3073, 4097 …), which confirmed our curve interpolation; the libopenraw format notes +//! (, prose only); ExifTool's Nikon tag-name documentation +//! (`0x0093` NEFCompression, `0x0096` NEFLinearizationTable); ITU-T T.81 Annex F/H for the "difference category +//! + additional bits" coding and canonical (BITS/HUFFVAL) Huffman codes. +//! - Our own black-box analysis of CC0 samples from raw.pixls.us. Nikon's Huffman tables are not stored in the +//! files and no prose source lists them, so they were recovered from the data: several cameras (D5100, D300, +//! D700) were shot with the same scene in compressed and uncompressed modes; a beam search assigned code words +//! to difference categories so that the decoded compressed image matched the uncompressed one (likelihood of +//! the decoded values given the reference, minus the bits consumed, plus a description-length penalty per new +//! code word). Each table was then verified on ~45 files from ~35 bodies (D40 … D850, Df, Z 6, Z 50, 1 J1): with +//! the right table the whole strip decodes and ends within the last byte of the data, with every value inside +//! the encoded range. The predictor and difference coding are the same as Pentax PEF (see `pef.rs`). +//! +//! Format, as established: +//! - One strip, MSB-first bit stream, no byte stuffing, continuous across rows. Per pixel a canonical Huffman code +//! gives the category `n`, followed by `n` additional bits (T.81: a leading 0 means a negative difference). +//! - Each pixel is predicted from the same-colour pixel two to the left; the first two pixels of a row from the +//! first two pixels of the previous row of the same parity, starting at the seeds of maker note `0x0096`. +//! - Lossless (`0x0096` version `0x46`): the decoded values are the samples. Lossy (version `0x44`): they index a +//! curve. Type 1 (`0x44 0x10`) stores the full curve; type 2 (`0x44 0x20`, `0x44 0x40`) stores 257 points, +//! one every `2^bits / 256` codes (`0x20`) or `2^bits / 1024` codes (`0x40`), linearly interpolated. The span +//! was established from the files: in every `0x20` file the first interval of the 257 points rises by +//! `2^bits / 256` (16 at 12 bit, 64 at 14 bit) and in every `0x40` file by `2^bits / 1024` (4 and 16), i.e. +//! the curve is the identity in the dark part, one code per sample. Read with a point every `2^bits / 256` +//! codes instead, the 12 `0x40` files that are not split decoded to values that never exceeded 255 (12 bit) or +//! 1023 (14 bit) and whose darkest 0.01 % sat at a quarter of the black level of maker note `0x003d` (63 against +//! 252 in 12-bit units); with the first-interval step they sit at that black level. +//! - Values in the maker note are in the maker note's byte order (newer bodies write little-endian notes). +//! +//! "Lossy after split" (non-zero split row at `0x0096` offset 562, type 2 tables only, found in 11 of the CC0 +//! samples): derived by black-box analysis of those files. Rows below the split row use the lossy Huffman path +//! above unchanged. Row `split` starts at the very next bit (no alignment); from there every pixel is a word of +//! `bits - 4` bits (8 at 12 bit, 10 at 14 bit), MSB first, with no row padding. A word is a complete prefix code +//! for a difference category `n` followed by the top `W - prefix length` bits of the T.81 additional bits (the low +//! bits are dropped); the difference is the centre of the magnitude bin that was kept. The predictor is the same +//! as above and the running value is not clamped (only the sample index is, before the curve lookup). The camera +//! ends the strip with 0 to 7 one-bits after the last full row, which the decoder checks. +//! +//! Not supported (returned as [`RawError::Unsupported`], the embedded preview is used instead): +//! - "Lossy after split" files whose fixed-rate section does not end within the last byte of the strip (see +//! below): refused rather than decoded wrongly. +//! - Other `0x0096` versions, bit depths other than 12/14. +//! - Code words that never occurred in any sample (the lossy 12-bit 8-bit code word `11111110`, and code words +//! longer than 9 bits in the lossy tables) are rejected as corrupt; they cannot code a difference within the +//! lossy value range anyway. + +use crate::huffman::{Bits, Huffman, diff}; +use crate::{MAX_SAMPLES, RawError, Result}; +use craft_tiff::ByteOrder; + +/// A table code word whose symbol was never observed (see the module docs). +const UNSEEN: u8 = u8::MAX; + +/// Canonical Huffman table: number of codes of length 1..=16 and the symbols (difference categories) in code order. +struct Table { + counts: &'static [u8], + symbols: &'static [u8], +} + +const LOSSLESS_12: Table = Table { counts: &[0, 1, 4, 2, 3, 1, 2], symbols: &[5, 4, 6, 3, 7, 2, 8, 1, 9, 0, 10, 11, 12] }; +const LOSSLESS_14: Table = Table { counts: &[0, 1, 4, 2, 2, 3, 1, 2], symbols: &[7, 6, 8, 5, 9, 4, 10, 3, 11, 12, 2, 0, 1, 13, 14] }; +const LOSSY_12: Table = Table { counts: &[0, 1, 5, 1, 1, 1, 1, 1, 1], symbols: &[5, 4, 3, 6, 2, 7, 1, 0, 8, 9, UNSEEN, 10] }; +const LOSSY_14: Table = Table { counts: &[0, 1, 4, 3, 1, 1, 1, 1, 1], symbols: &[5, 6, 4, 7, 8, 3, 9, 2, 1, 0, 10, 11, 12] }; + +impl Table { + /// `(left-aligned 12-bit code, length)` per symbol, for [`Huffman::new`]. + fn codes(&self) -> Vec<(u16, u8)> { + let mut out = vec![(0u16, 0u8); 17]; + let mut code = 0u32; + let mut symbols = self.symbols.iter(); + for (i, &n) in self.counts.iter().enumerate() { + let len = i as u32 + 1; + for _ in 0..n { + if let Some(&s) = symbols.next() + && let Some(slot) = out.get_mut(s as usize) + { + *slot = ((code << (12 - len)) as u16, len as u8); + } + code += 1; + } + code <<= 1; + } + out + } +} + +/// How the decoded values map to samples. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum Encoding { + /// The decoded values are the samples (`0..2^bits`). + Lossless, + /// The decoded values index this curve. + Lossy(Vec), +} + +/// Maker note `0x0096` (NEF linearization table), parsed. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct DecodeTable { + pub encoding: Encoding, + /// Predictor seeds `[row parity][column]`. + pub seeds: [[i32; 2]; 2], + /// First row of the fixed-rate section ("lossy after split"), 0 when the whole strip is Huffman coded. + pub split: usize, +} + +/// Parse maker note `0x0096` for a `bits`-bit image. `order` is the maker note's byte order. +pub(crate) fn parse_table(t: &[u8], order: ByteOrder, bits: u32) -> Result { + let u16_at = |o: usize| t.get(o..o + 2).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(|b| order.u16(b)); + let short = || RawError::Corrupt(format!("NEF: linearization table too short ({} bytes)", t.len())); + if bits != 12 && bits != 14 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {bits}-bit samples"))); + } + let (v0, v1) = (t.first().copied().ok_or_else(short)?, t.get(1).copied().ok_or_else(short)?); + let mut seeds = [[0i32; 2]; 2]; + for (i, s) in seeds.iter_mut().flatten().enumerate() { + *s = u16_at(2 + 2 * i).ok_or_else(short)? as i32; + } + let mut split = 0; + let encoding = match (v0, v1) { + (0x46, _) => Encoding::Lossless, + (0x44, 0x10 | 0x20 | 0x40) => { + let n = u16_at(10).ok_or_else(short)? as usize; + let points: Vec = (0..n).map(|i| u16_at(12 + 2 * i)).collect::>().ok_or_else(short)?; + if n < 2 { + return Err(RawError::Corrupt(format!("NEF: linearization curve of {n} points"))); + } + if v1 == 0x10 { + Encoding::Lossy(points) + } else { + split = u16_at(562).unwrap_or(0) as usize; + // type 2: `n` points covering `range` codes, one every `range / (n - 1)`; the curve's first interval + // rises by exactly that step (identity in the dark part) and the table version sets the span: + // `0x20` covers all 2^bits codes, `0x40` the first quarter (see the module docs) + let range = if v1 == 0x40 { 1usize << (bits - 2) } else { 1usize << bits }; + if n - 1 > range || !range.is_multiple_of(n - 1) { + return Err(RawError::Unsupported(format!("NEF: {n}-point lossy curve for {bits}-bit data"))); + } + let step = range / (n - 1); + let mut curve = Vec::with_capacity(range + 1); + for pair in points.windows(2) { + let &[a, b] = pair else { continue }; + let (a, b) = (a as u32, b as u32); + for k in 0..step as u32 { + let s = step as u32; + curve.push(((a * (s - k) + b * k) / s) as u16); + } + } + curve.extend(points.last().copied()); + Encoding::Lossy(curve) + } + } + _ => return Err(RawError::Unsupported(format!("Nikon compressed NEF version {v0:#04x} {v1:#04x}"))), + }; + Ok(DecodeTable { encoding, seeds, split }) +} + +/// Difference-category prefix codes of the fixed-rate section: `(prefix bits, prefix length, category n)`. +/// A word of `W` bits is a prefix followed by `W - length` bits of the T.81 additional bits of category `n`. +/// (The 14-bit `010` = category 12 never occurs in the samples; its category is deduced.) +const FIXED_12: &[(u32, u32, u32)] = &[ + (0b00, 2, 9), + (0b010, 3, 10), + (0b011, 3, 8), + (0b100, 3, 7), + (0b101, 3, 6), + (0b110, 3, 5), + (0b1110, 4, 4), + (0b11110, 5, 3), + (0b111110, 6, 2), + (0b1111110, 7, 1), + (0b11111110, 8, 0), +]; +const FIXED_14: &[(u32, u32, u32)] = &[ + (0b00, 2, 8), + (0b010, 3, 12), + (0b011, 3, 11), + (0b100, 3, 10), + (0b101, 3, 9), + (0b110, 3, 7), + (0b1110, 4, 6), + (0b11110, 5, 5), + (0b111110, 6, 4), + (0b1111110, 7, 3), + (0b11111110, 8, 2), + (0b111111110, 9, 1), + (0b1111111110, 10, 0), +]; + +/// Word (`bits - 4` bits) to dequantised difference for the fixed-rate section; `None` marks the unused all-ones word. +fn fixed_word_table(bits: u32) -> Vec> { + let (w, codes) = if bits == 12 { (8, FIXED_12) } else { (10, FIXED_14) }; + let mut lut = vec![None; 1usize << w]; + for &(prefix, len, n) in codes { + let kept = w - len; + for m in 0..1u32 << kept { + let e = n.saturating_sub(kept); + let ex = (m << e) as i32; + let half = (1i32 << e) >> 1; + let d = if n == 0 { + 0 + } else if ex >= 1 << (n - 1) { + ex + half + } else if e > 0 { + ex + half - (1 << n) + } else { + ex - ((1 << n) - 1) + }; + if let Some(slot) = lut.get_mut(((prefix << kept) | m) as usize) { + *slot = Some(d); + } + } + } + lut +} + +/// Decode a whole `w × h` Nikon Huffman-compressed strip `src` with `bits`-bit samples. +#[cfg(test)] +pub(crate) fn decode(src: &[u8], w: usize, h: usize, bits: u32, table: &DecodeTable) -> Result> { + decode_rows(src, w, h, h, bits, table) +} + +/// The first `rows` rows (`w × rows` samples) of a `w × h` strip. The stream is sequential, so a header-only probe +/// decodes just the rows its masked-column test looks at (storytold/lightcraft#708) instead of the whole image. The size and +/// truncation checks still cover the whole image; the "lossy after split" end-of-strip rule (and corruption +/// further down) can only be seen when the whole strip is decoded. +pub(crate) fn decode_rows(src: &[u8], w: usize, h: usize, rows: usize, bits: u32, table: &DecodeTable) -> Result> { + if bits != 12 && bits != 14 { + return Err(RawError::Unsupported(format!("Nikon compressed NEF with {bits}-bit samples"))); + } + let whole = w.checked_mul(h).filter(|&n| n > 0 && n <= MAX_SAMPLES).ok_or(RawError::Limit("NEF image size"))?; + // every code word is at least 2 bits long: a strip that can't hold the image is truncated (also bounds the + // allocation by the file size) + if whole / 4 > src.len() { + return Err(RawError::Corrupt(format!("NEF: {} bytes of compressed data for {w}x{h} pixels", src.len()))); + } + let rows = rows.min(h); + let n = w.saturating_mul(rows); + let huff = Huffman::new( + &match (&table.encoding, bits) { + (Encoding::Lossless, 12) => LOSSLESS_12, + (Encoding::Lossless, _) => LOSSLESS_14, + (Encoding::Lossy(_), 12) => LOSSY_12, + (Encoding::Lossy(_), _) => LOSSY_14, + } + .codes(), + )?; + let (lut, max): (&[u16], i32) = match &table.encoding { + Encoding::Lossless => (&[], (1i32 << bits) - 1), + Encoding::Lossy(curve) => (curve, curve.len() as i32 - 1), + }; + if table.split != 0 && table.split >= h { + return Err(RawError::Corrupt(format!("NEF: split row {} outside the {h} rows", table.split))); + } + let fixed = if table.split != 0 { fixed_word_table(bits) } else { Vec::new() }; + let word = bits - 4; + let mut out = vec![0u16; n]; + let mut stream = Bits::new(src); + let mut vpred = table.seeds; + let mut clipped = 0usize; + for (y, row) in out.chunks_exact_mut(w).enumerate() { + let seeds = vpred.get_mut(y & 1).ok_or(RawError::Limit("NEF row"))?; + let mut hpred = [0i32; 2]; + for (x, o) in row.iter_mut().enumerate() { + let d = if table.split != 0 && y >= table.split { + let code = stream.get(word) as usize; + fixed.get(code).copied().flatten().ok_or_else(|| RawError::Corrupt(format!("NEF: invalid fixed-rate word at row {y}")))? + } else { + diff(&mut stream, &huff).ok_or_else(|| RawError::Corrupt(format!("NEF: invalid Huffman code at row {y}")))? + }; + // a run of large differences in a corrupt stream can drive a predictor past i32 (storytold/lightcraft#1515) + let overflow = || RawError::Corrupt(format!("NEF: predictor out of range at row {y}")); + let p = if x < 2 { + let s = seeds.get_mut(x).ok_or(RawError::Limit("NEF column"))?; + *s = s.checked_add(d).ok_or_else(overflow)?; + hpred[x & 1] = *s; + *s + } else { + let p = hpred[x & 1].checked_add(d).ok_or_else(overflow)?; + hpred[x & 1] = p; + p + }; + let v = p.clamp(0, max); + clipped += (v != p && (table.split == 0 || y < table.split)) as usize; + *o = if lut.is_empty() { v as u16 } else { lut.get(v as usize).copied().unwrap_or(0) }; + } + // the camera pads the strip; reading more than a few bytes past its end means truncated or corrupt data + if stream.consumed_bits() > src.len() * 8 + 64 { + return Err(RawError::Corrupt(format!("NEF: compressed data ends at row {y} of {h}"))); + } + } + if table.split != 0 && rows == h { + // the fixed-rate section must fill the rest of the strip: whole rows of w words and 0..7 bits of padding. + // A strip may hold a few more rows than the image (one body stores 4022 rows for 4020); anything else means + // the file does not follow the rule, so refuse it instead of decoding it wrongly. + let left = (src.len() * 8).checked_sub(stream.consumed_bits()); + let row_bits = w * word as usize; + if !left.is_some_and(|l| l % row_bits < 8 && l / row_bits <= 4) { + return Err(RawError::Unsupported(format!( + "Nikon \"lossy after split\" NEF whose fixed-rate section (from row {}) does not end within the last byte", + table.split + ))); + } + } + // a wrong table / corrupt stream drifts out of range quickly; real files stay inside (a handful of edge pixels) + if clipped > n / 1000 + 16 { + return Err(RawError::Corrupt(format!("NEF: {clipped} decoded values out of range"))); + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// MSB-first bit writer for the test encoder. + #[derive(Default)] + struct Writer { + out: Vec, + acc: u64, + n: u32, + } + + impl Writer { + fn put(&mut self, v: u32, len: u32) { + for i in (0..len).rev() { + self.acc = (self.acc << 1) | ((v >> i) & 1) as u64; + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + self.acc = 0; + self.n = 0; + } + } + } + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push((self.acc << (8 - self.n)) as u8); + } + self.out.extend_from_slice(&[0; 4]); + self.out + } + } + + fn category(d: i32) -> u32 { + 32 - d.unsigned_abs().leading_zeros() + } + + /// Encode `values` (decoded domain: samples or curve indices) the way the camera does. + fn encode(values: &[i32], w: usize, table: &Table, seeds: [[i32; 2]; 2]) -> Vec { + let codes = table.codes(); + let mut wr = Writer::default(); + let mut vpred = seeds; + for (y, row) in values.chunks(w).enumerate() { + for (x, &v) in row.iter().enumerate() { + let pred = if x < 2 { vpred[y & 1][x] } else { row[x - 2] }; + if x < 2 { + vpred[y & 1][x] = v; + } + let d = v - pred; + let k = category(d); + let (code, len) = codes[k as usize]; + assert!(len > 0, "no code for category {k}"); + wr.put(code as u32 >> (12 - len), len as u32); + let extra = if d < 0 { d + (1 << k) - 1 } else { d }; + wr.put(extra as u32, k); + } + } + wr.finish() + } + + /// Deterministic test image: smooth gradient + texture + hard edges, within `0..=max`. + fn image(w: usize, h: usize, max: i32, seed: u32) -> Vec { + let mut s = seed; + (0..w * h) + .map(|i| { + let (x, y) = ((i % w) as i32, (i / w) as i32); + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let noise = (s >> 24) as i32 % 9 - 4; + let base = (x * 37 + y * 11) % (max / 2) + if (x / 7 + y / 5) % 3 == 0 { max / 3 } else { 0 }; + (base + noise).clamp(0, max) + }) + .collect() + } + + fn lossless(bits: u32) -> DecodeTable { + let s = 1 << (bits - 3); + DecodeTable { encoding: Encoding::Lossless, seeds: [[s, s], [s, s]], split: 0 } + } + + #[test] + fn tables_are_complete_prefix_codes() { + for t in [LOSSLESS_12, LOSSLESS_14, LOSSY_12, LOSSY_14] { + assert_eq!(t.counts.iter().map(|&c| c as usize).sum::(), t.symbols.len()); + let kraft: f64 = t.counts.iter().enumerate().map(|(i, &c)| c as f64 / (1u64 << (i + 1)) as f64).sum(); + assert!(kraft <= 1.0); + assert!(Huffman::new(&t.codes()).is_ok()); + } + assert_eq!(LOSSLESS_14.codes()[7], (0b00 << 10, 2)); + assert_eq!(LOSSLESS_14.codes()[0], (0b111110 << 6, 6)); + assert_eq!(LOSSY_12.codes()[10], (0b111111110 << 3, 9)); + } + + #[test] + fn lossless_roundtrip_12_and_14() { + for (bits, table) in [(12, LOSSLESS_12), (14, LOSSLESS_14)] { + let (w, h) = (64, 9); + let max = (1 << bits) - 1; + let mut img = image(w, h, max, bits); + // extreme jumps exercise the longest categories + img[5] = 0; + img[7] = max; + img[w + 9] = max; + let t = lossless(bits); + let src = encode(&img, w, &table, t.seeds); + let out = decode(&src, w, h, bits, &t).unwrap(); + assert_eq!(out, img.iter().map(|&v| v as u16).collect::>(), "{bits}-bit"); + } + } + + fn lossy_table(bits: u32, order: ByteOrder) -> Vec { + // version 0x44 0x20, seeds, 257 points of a concave curve, padding up to the split value (0) + let mut t = vec![0x44, 0x20]; + let put = |t: &mut Vec, v: u16| t.extend_from_slice(&if order == ByteOrder::Big { v.to_be_bytes() } else { v.to_le_bytes() }); + for _ in 0..4 { + put(&mut t, 300); + } + put(&mut t, 257); + let max = ((1u32 << bits) - 1) as f64; + for i in 0..257u32 { + put(&mut t, (max * (i as f64 / 64.0).min(1.0).sqrt()) as u16); + } + t.resize(624, 0); + t + } + + #[test] + fn lossy_type2_curve_and_roundtrip() { + for (bits, table) in [(12u32, LOSSY_12), (14, LOSSY_14)] { + for order in [ByteOrder::Big, ByteOrder::Little] { + let t = parse_table(&lossy_table(bits, order), order, bits).unwrap(); + assert_eq!(t.seeds, [[300, 300], [300, 300]]); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + let step = (1usize << bits) / 256; + assert_eq!(curve.len(), (1 << bits) + 1); + assert_eq!(curve[64 * step], (1 << bits) - 1); + assert!(curve.windows(2).all(|p| p[0] <= p[1])); + // the encoded range is the curve's rising part + let (w, h) = (48, 7); + let img = image(w, h, 64 * step as i32 - 1, 7); + let src = encode(&img, w, &table, t.seeds); + let out = decode(&src, w, h, bits, &t).unwrap(); + assert_eq!(out, img.iter().map(|&v| curve[v as usize]).collect::>()); + } + } + } + + /// Maker note `0x0096` as bytes, written out by hand: version `0x44 `, four seeds, 257 points (little-endian + /// as newer bodies write it) of a curve whose first 256 points rise by `step` and whose last is the white + /// level, and a zero split value at offset 562. + fn hand_table(v1: u8, seed: u16, step: u16, white: u16) -> Vec { + let mut t = vec![0x44, v1]; + for _ in 0..4 { + t.extend_from_slice(&seed.to_le_bytes()); + } + t.extend_from_slice(&257u16.to_le_bytes()); + for k in 0..256u16 { + t.extend_from_slice(&(k * step).to_le_bytes()); + } + t.extend_from_slice(&white.to_le_bytes()); + t.resize(624, 0); + t + } + + /// Known answers for the `0x44 0x40` curve, from bytes and bit strings written by hand (nothing here uses the + /// decoder's span rule or the test encoder): the first interval of the points rises by `2^bits / 1024`, so + /// the curve is the identity for one code per sample and spans `2^bits / 4` codes. The `0x20` curve of the + /// same sizes keeps its span of all `2^bits` codes. + #[test] + fn lossy_type2_version_0x40_spans_a_quarter_of_the_range() { + // 12 bit: points 0, 4, 8 … 1020, 4095 + let t = parse_table(&hand_table(0x40, 252, 4, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 1025); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[252], curve[255], curve[1020]), (252, 255, 1020)); + assert_eq!((curve[1021], curve[1022], curve[1023], curve[1024]), (1788, 2557, 3326, 4095)); + // the pixels p0 = 252 + 4, p1 = 252, p2 = p0 - 1, p3 = p1, coded as 011 100 | 11110 | 1110 0 | 11110 + // (LOSSY_12: category 3 = 011, 0 = 11110, 1 = 1110; extra bits 100 = +4, 0 = -1) + let out = decode(&[0x73, 0xdc, 0xf0], 4, 1, 12, &t).unwrap(); + assert_eq!(out, [256, 252, 255, 252]); + // 14 bit: points 0, 16, 32 … 4080, 16383 + let t = parse_table(&hand_table(0x40, 1008, 16, 16383), ByteOrder::Little, 14).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!(curve.len(), 4097); + assert_eq!(curve[0..6], [0, 1, 2, 3, 4, 5]); + assert_eq!((curve[1008], curve[4080], curve[4096]), (1008, 4080, 16383)); + // the same pixels coded as 1100 100 | 111110 | 11110 0 | 111110 + // (LOSSY_14: category 3 = 1100, 0 = 111110, 1 = 11110) + let out = decode(&[0xc9, 0xf7, 0x9f, 0x00], 4, 1, 14, &t).unwrap(); + assert_eq!(out, [1012, 1008, 1011, 1008]); + // version 0x20 with the same points is a curve over all 2^bits codes: the points are 16 codes apart + let t = parse_table(&hand_table(0x20, 252, 16, 4095), ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + assert_eq!((curve.len(), curve[16], curve[17], curve[4095]), (4097, 16, 17, 4094)); + // 200 points can't be spaced evenly over the 1024 codes of a 12-bit 0x40 table + let mut odd = hand_table(0x40, 252, 4, 4095); + odd[10..12].copy_from_slice(&200u16.to_le_bytes()); + assert!(parse_table(&odd, ByteOrder::Little, 12).is_err()); + } + + /// Pack a string of `0`/`1` (spaces ignored) MSB first, padding the last byte with one-bits like the camera. + fn pack(bits: &str) -> Vec { + let b: Vec = bits.chars().filter(|c| !c.is_whitespace()).map(|c| c == '1').collect(); + b.chunks(8).map(|c| (0..8).fold(0u8, |a, i| (a << 1) | c.get(i).copied().unwrap_or(true) as u8)).collect() + } + + fn split_table(v1: u8, seed: u16, step: u16, white: u16, split: u16) -> DecodeTable { + let mut t = hand_table(v1, seed, step, white); + t[562..564].copy_from_slice(&split.to_le_bytes()); + parse_table(&t, ByteOrder::Little, if step == 4 { 12 } else { 14 }).unwrap() + } + + /// 12-bit "lossy after split": row 0 Huffman coded, rows 1 and 2 as 8-bit words that follow the last Huffman + /// bit directly. Bit strings written by hand from the rule (prefix, then the kept top bits of the extra bits). + const SPLIT_12: &str = "011 100 11110 1110 0 11110 11111110 11110100 11100111 01110000 00000001 11111110 01111111 11111110"; + + #[test] + fn lossy_after_split_known_answers_12_bit() { + let t = split_table(0x40, 252, 4, 4095, 1); + assert_eq!(t.split, 1); + let out = decode(&pack(SPLIT_12), 4, 3, 12, &t).unwrap(); + // row 1: d = 0 (n0), +4 (n3, exact), -8 (n4, exact), +132 (n8 keeps 5 bits: 16 << 3 plus half a bin, 4); + // row 2: -500 (n9 keeps 6 bits: bin 1 of the negative side, centre -508 + 8), then +252 on the running + // value -244 which is not clamped (8, not 252) + assert_eq!(out, [256, 252, 255, 252, 252, 256, 244, 388, 0, 252, 8, 252]); + } + + #[test] + fn lossy_after_split_known_answers_14_bit() { + let t = split_table(0x40, 1008, 16, 16383, 1); + // row 0 as in the 0x40 test above; row 1 in 10-bit words: 0 (n0), +32 (n6, exact), -254 (n8, exact), + // +510 (n9 keeps 7 bits: 127 << 2 plus 2) + let bits = "1100 100 111110 11110 0 111110 1111111110 1110100000 0000000001 1011111111"; + let out = decode(&pack(bits), 4, 2, 14, &t); + // word 3 is `00` + 8 bits: 00 00000001 -> n8 extra 00000001 = -254 + assert_eq!(out.unwrap(), [1012, 1008, 1011, 1008, 1008, 1040, 754, 1550]); + } + + #[test] + fn lossy_after_split_dequantisation_is_symmetric() { + let lut = fixed_word_table(12); + assert_eq!((lut[0b00_000000], lut[0b00_111111]), (Some(-508), Some(508))); + assert_eq!((lut[0b011_00000], lut[0b011_11111]), (Some(-252), Some(252))); + assert_eq!(lut[0b11111110], Some(0)); + assert_eq!(lut[0xff], None); + let lut = fixed_word_table(14); + assert_eq!((lut[0b00_00000000], lut[0b00_11111111]), (Some(-255), Some(255))); + assert_eq!(lut[0b101_0000000], Some(-510)); + assert_eq!(lut[0b1111111110], Some(0)); + assert_eq!(lut[0x3ff], None); + for bits in [12, 14] { + let lut = fixed_word_table(bits); + assert_eq!(lut.iter().filter(|d| d.is_none()).count(), 1, "{bits}-bit: only the all-ones word is unused"); + } + } + + #[test] + fn lossy_after_split_refuses_strips_that_do_not_follow_the_rule() { + let t = split_table(0x40, 252, 4, 4095, 1); + let mut longer = pack(SPLIT_12); + longer.extend_from_slice(&[0xff; 3]); + assert!(matches!(decode(&longer, 4, 3, 12, &t), Err(RawError::Unsupported(_)))); + let shorter = &pack(SPLIT_12)[..10]; + assert!(decode(shorter, 4, 3, 12, &t).is_err()); + // the unused all-ones word + let bad = pack("011 100 11110 1110 0 11110 11111111 11111110 11111110 11111110"); + assert!(matches!(decode(&bad, 4, 2, 12, &t), Err(RawError::Corrupt(_)))); + // a split row beyond the image + assert!(matches!(decode(&pack(SPLIT_12), 4, 1, 12, &t), Err(RawError::Corrupt(_)))); + // through the container + let mut table = hand_table(0x40, 252, 4, 4095); + table[562..564].copy_from_slice(&1u16.to_le_bytes()); + let ok = crate::decode(&nef_file(pack(SPLIT_12), 4, 3, 12, table.clone(), ByteOrder::Little)).unwrap(); + assert_eq!(ok.data, crate::RawData::U16(vec![256, 252, 255, 252, 252, 256, 244, 388, 0, 252, 8, 252])); + let e = crate::decode(&nef_file(longer, 4, 3, 12, table, ByteOrder::Little)); + assert!(matches!(e, Err(RawError::Unsupported(_))), "{e:?}"); + } + + #[test] + fn lossy_type1_full_curve() { + let mut t = vec![0x44, 0x10, 0, 10, 0, 10, 0, 10, 0, 10, 0, 5]; + for v in [0u16, 100, 400, 900, 1600] { + t.extend_from_slice(&v.to_be_bytes()); + } + let parsed = parse_table(&t, ByteOrder::Big, 12).unwrap(); + assert_eq!(parsed.encoding, Encoding::Lossy(vec![0, 100, 400, 900, 1600])); + let img = vec![0, 4, 1, 3, 2, 2, 4, 0]; + let src = encode(&img, 4, &LOSSY_12, parsed.seeds); + assert_eq!(decode(&src, 4, 2, 12, &parsed).unwrap(), vec![0, 1600, 100, 900, 400, 400, 1600, 0]); + } + + #[test] + fn rejects_unsupported_and_malformed_tables() { + let unsupported = |r: Result| matches!(r, Err(RawError::Unsupported(_))); + let corrupt = |r: Result| matches!(r, Err(RawError::Corrupt(_))); + let mut split = lossy_table(12, ByteOrder::Big); + split[562..564].copy_from_slice(&345u16.to_be_bytes()); + assert_eq!(parse_table(&split, ByteOrder::Big, 12).unwrap().split, 345); + assert!(unsupported(parse_table(&[0x49, 0x30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], ByteOrder::Big, 12))); + assert!(unsupported(parse_table(&lossy_table(12, ByteOrder::Big), ByteOrder::Big, 16))); + assert!(corrupt(parse_table(&[], ByteOrder::Big, 12))); + assert!(corrupt(parse_table(&[0x46, 0x30, 8], ByteOrder::Big, 14))); + assert!(corrupt(parse_table(&lossy_table(12, ByteOrder::Big)[..100], ByteOrder::Big, 12))); + // 300 points don't divide the 12-bit range + let mut odd = lossy_table(12, ByteOrder::Big); + odd[10..12].copy_from_slice(&300u16.to_be_bytes()); + assert!(parse_table(&odd, ByteOrder::Big, 12).is_err()); + assert!(parse_table(&[0x44, 0x20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0], ByteOrder::Big, 12).is_err()); + } + + /// A minimal NEF: CFA SubIFD with one compressed strip, Exif maker note (`Nikon\0` v2 header + embedded TIFF in + /// `mn_order`) holding the linearization table. + fn nef_file(strip: Vec, w: u32, h: u32, bits: u16, table: Vec, mn_order: ByteOrder) -> Vec { + use craft_tiff::tags::{self as t, photometric}; + use craft_tiff::{IfdBuilder, ImageData, TiffWriter, Value}; + let mut mn = IfdBuilder::new(); + mn.set(0x0096, Value::Undefined(table)); + let mut note = b"Nikon\0\x02\x10\0\0".to_vec(); + note.extend(TiffWriter::new(mn_order, false).write(&[mn]).unwrap()); + let mut exif = IfdBuilder::new(); + exif.set(t::MAKER_NOTE, Value::Undefined(note)); + let mut raw = IfdBuilder::new(); + raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0])); + raw.set(t::IMAGE_WIDTH, Value::Long(vec![w])); + raw.set(t::IMAGE_LENGTH, Value::Long(vec![h])); + raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![bits])); + raw.set(t::COMPRESSION, Value::Short(vec![t::compression::NIKON])); + raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA])); + raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2])); + raw.set(t::CFA_PATTERN_EP, Value::Byte(vec![0, 1, 1, 2])); + raw.set_image(ImageData::Strips { rows_per_strip: h, strips: vec![strip] }); + let mut ifd0 = IfdBuilder::new(); + ifd0.set(t::MAKE, Value::Ascii("NIKON CORPORATION".into())); + ifd0.set(t::MODEL, Value::Ascii("NIKON TEST".into())); + ifd0.set_child(t::EXIF_IFD, exif); + ifd0.add_sub_ifd(raw); + TiffWriter::new(ByteOrder::Big, false).write(&[ifd0]).unwrap() + } + + #[test] + fn decodes_through_the_nef_container() { + // lossless 14-bit, big-endian maker note + let (w, h) = (40usize, 6usize); + let img = image(w, h, 16383, 11); + let mut table = vec![0x46, 0x30]; + for _ in 0..4 { + table.extend_from_slice(&2048u16.to_be_bytes()); + } + table.resize(46, 0); + let src = encode(&img, w, &LOSSLESS_14, [[2048; 2]; 2]); + let bytes = nef_file(src, w as u32, h as u32, 14, table, ByteOrder::Big); + let r = crate::decode(&bytes).unwrap(); + assert_eq!(r.data, crate::RawData::U16(img.iter().map(|&v| v as u16).collect())); + assert_eq!(r.cfa.as_ref().unwrap().name(), "RGGB"); + assert_eq!(crate::probe_info(&bytes).unwrap(), r.info()); + // lossy 12-bit type 2, little-endian maker note (newer bodies) + let table = lossy_table(12, ByteOrder::Little); + let t = parse_table(&table, ByteOrder::Little, 12).unwrap(); + let Encoding::Lossy(curve) = &t.encoding else { panic!() }; + let img = image(w, h, 1023, 5); + let src = encode(&img, w, &LOSSY_12, t.seeds); + let r = crate::decode(&nef_file(src.clone(), w as u32, h as u32, 12, table.clone(), ByteOrder::Little)).unwrap(); + assert_eq!(r.data, crate::RawData::U16(img.iter().map(|&v| curve[v as usize]).collect())); + // a split file whose strip does not follow the fixed-rate rule is refused, never decoded wrongly + let mut split = table; + split[562..564].copy_from_slice(&3u16.to_le_bytes()); + let e = crate::decode(&nef_file(src, w as u32, h as u32, 12, split, ByteOrder::Little)); + assert!(e.is_err(), "{e:?}"); + } + + /// storytold/lightcraft#708: the header probe of a compressed NEF decodes only the first [`crate::nef::MASK_ROWS`] + /// rows (the ones the masked-column test reads) instead of the whole Huffman stream, and still finds the masked + /// columns the full decode finds. A strip cut off after those rows proves it: the probe succeeds with the same + /// description while the full decode reports the truncation. + #[test] + fn header_probe_stops_after_the_masked_column_rows() { + use crate::nef::MASK_ROWS; + let (w, h) = (160usize, MASK_ROWS * 3); + let mut img = image(w, h, 16383, 7); + // eight optically masked columns at 0, as the D5100 / D7000 write them + for row in img.chunks_mut(w) { + row[w - 8..].fill(0); + } + let t = lossless(14); + let table = { + let mut v = vec![0x46, 0x30]; + for s in t.seeds.iter().flatten() { + v.extend_from_slice(&(*s as u16).to_be_bytes()); + } + v.resize(46, 0); + v + }; + let src = encode(&img, w, &LOSSLESS_14, t.seeds); + let whole = nef_file(src.clone(), w as u32, h as u32, 14, table.clone(), ByteOrder::Big); + let full = crate::decode(&whole).unwrap(); + assert_eq!(full.active_area, crate::Rect::new(0, 0, w - 8, h), "the masked columns are found in a compressed file"); + assert_eq!(crate::probe_info(&whole).unwrap(), full.info()); + // the same stream cut off after two thirds of the rows (the first rows' codes are a prefix of the whole) + let cut = encode(&img[..w * (h * 2 / 3)], w, &LOSSLESS_14, t.seeds).len(); + assert!(cut > w * h / 4, "enough bytes to pass the size check"); + let truncated = nef_file(src[..cut].to_vec(), w as u32, h as u32, 14, table, ByteOrder::Big); + assert!(matches!(crate::decode(&truncated), Err(RawError::Corrupt(_))), "the full decode sees the truncation"); + assert_eq!(crate::probe_info(&truncated).unwrap(), full.info(), "the header probe never reads that far"); + // decode_rows itself: a prefix of the whole decode, and the whole image's size check still applies + let top = decode_rows(&src, w, h, MASK_ROWS, 14, &t).unwrap(); + assert_eq!(top.len(), w * MASK_ROWS); + assert_eq!(top[..], decode(&src, w, h, 14, &t).unwrap()[..w * MASK_ROWS]); + assert_eq!(decode_rows(&src, w, h, h + 5, 14, &t).unwrap().len(), w * h, "rows are clamped to the height"); + assert!(decode_rows(&src[..w * h / 4 - 1], w, h, 1, 14, &t).is_err(), "a strip too short for the image is corrupt in any mode"); + } + + /// A long run of the largest difference drove a predictor past i32: a panic in builds with overflow checks, + /// wrapped values otherwise (storytold/lightcraft#1515). Both the row and the column predictors now stop with an error. + #[test] + fn predictor_overflow_is_an_error_not_a_panic() { + let codes = LOSSLESS_14.codes(); + let k = (0..codes.len()).rev().find(|&k| codes[k].1 > 0).unwrap(); + let d = (1u32 << k) - 1; + let stream = |n: usize| { + let mut wr = Writer::default(); + let (code, len) = codes[k]; + for _ in 0..n { + wr.put(code as u32 >> (12 - len), len as u32); + wr.put(d, k as u32); + } + wr.finish() + }; + // differences one predictor needs to pass i32::MAX + let steps = (i32::MAX as u32 / d) as usize + 2; + let table = DecodeTable { encoding: Encoding::Lossless, seeds: [[0; 2]; 2], split: 0 }; + // along a row: every other sample adds to the same predictor + let w = 2 * steps + 2; + let err = decode(&stream(w), w, 1, 14, &table).unwrap_err(); + assert!(matches!(err, RawError::Corrupt(ref m) if m.contains("predictor")), "{err}"); + // down two columns: each row adds to the seed of its parity + let h = 2 * steps + 2; + let err = decode(&stream(2 * h), 2, h, 14, &table).unwrap_err(); + assert!(matches!(err, RawError::Corrupt(ref m) if m.contains("predictor")), "{err}"); + } + + #[test] + fn truncated_and_corrupted_streams_error_never_panic() { + let (w, h) = (64, 16); + let t = lossless(14); + let img = image(w, h, 16383, 3); + let src = encode(&img, w, &LOSSLESS_14, t.seeds); + // truncation: too short for the image, or ends early + for cut in [0, 1, 10, src.len() / 2, src.len() - 40] { + assert!(decode(&src[..cut], w, h, 14, &t).is_err(), "cut {cut}"); + } + // garbage and bit flips: an error or (rarely) some image, never a panic + let mut s = 12345u32; + for i in 0..200 { + let mut bad = src.clone(); + for _ in 0..1 + i % 8 { + s = s.wrapping_mul(1_103_515_245).wrapping_add(12345); + let at = (s >> 8) as usize % bad.len(); + bad[at] ^= 1 << (s % 8); + } + let _ = decode(&bad, w, h, 14, &t); + let noise: Vec = (0..src.len()).map(|k| ((k as u32).wrapping_mul(2_654_435_761) >> (i % 24)) as u8).collect(); + let _ = decode(&noise, w, h, 14, &t); + let _ = decode(&noise, w, h, 12, &lossless(12)); + } + // all-ones never forms a valid lossy 12-bit code + assert!(decode(&[0xff; 64], 8, 8, 12, &parse_table(&lossy_table(12, ByteOrder::Big), ByteOrder::Big, 12).unwrap()).is_err()); + // absurd dimensions are limited before allocating + assert!(decode(&src, usize::MAX, 2, 14, &t).is_err()); + assert!(decode(&src, 1 << 20, 1 << 20, 14, &t).is_err()); + assert!(decode(&src, 0, 2, 14, &t).is_err()); + } +} diff --git a/crates/raw/tests/it/vendor_robust.rs b/crates/raw/tests/it/vendor_robust.rs index 68eccf554..65fb139fb 100644 --- a/crates/raw/tests/it/vendor_robust.rs +++ b/crates/raw/tests/it/vendor_robust.rs @@ -61,6 +61,8 @@ fn cr2() -> Vec { fn samples() -> Vec> { vec![ cr2(), + cfa_tiff("NIKON CORPORATION", 1, 12, ByteOrder::Big), + cfa_tiff("NIKON CORPORATION", 34713, 14, ByteOrder::Big), pef(1), pef(65535), cfa_tiff("SAMSUNG", 1, 12, ByteOrder::Little), From ed16993b3b24dff15105c5230be928e4c03bb3d5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:30:23 -0500 Subject: [PATCH 18/31] Add craft-ical: iCalendar lines, values, rules and VTIMEZONE from MailCraft (#966) MailCraft crates/ical at a44a450: content lines, DATE/DATE-TIME values, durations and text, the common subset of recurrence rules, and VTIMEZONE definitions read and written, with their tests. Times are chrono values; the zone database the writer needs is the caller's, through ZoneRules. The event and to-do mapping onto MailCraft's model stays in MailCraft. Co-authored-by: Martin Ball <98639962+benandez@users.noreply.github.com> --- Cargo.lock | 8 + Cargo.toml | 1 + README.md | 4 + crates/ical/Cargo.toml | 17 + crates/ical/README.md | 56 +++ crates/ical/proptest-regressions/lines.txt | 7 + crates/ical/src/lib.rs | 33 ++ crates/ical/src/lines.rs | 321 +++++++++++++ crates/ical/src/rrule.rs | 516 +++++++++++++++++++++ crates/ical/src/values.rs | 419 +++++++++++++++++ crates/ical/src/zones.rs | 498 ++++++++++++++++++++ crates/ical/src/zones_tests.rs | 293 ++++++++++++ 12 files changed, 2173 insertions(+) create mode 100644 crates/ical/Cargo.toml create mode 100644 crates/ical/README.md create mode 100644 crates/ical/proptest-regressions/lines.txt create mode 100644 crates/ical/src/lib.rs create mode 100644 crates/ical/src/lines.rs create mode 100644 crates/ical/src/rrule.rs create mode 100644 crates/ical/src/values.rs create mode 100644 crates/ical/src/zones.rs create mode 100644 crates/ical/src/zones_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 0b52a4503..857ed807c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1180,6 +1180,14 @@ dependencies = [ "writeable", ] +[[package]] +name = "craft-ical" +version = "0.1.0" +dependencies = [ + "chrono", + "proptest", +] + [[package]] name = "craft-idml" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 3f361200d..d7a54cda3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,6 +55,7 @@ craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } craft-idml = { path = "crates/idml" } +craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } craft-kiwi = { path = "crates/kiwi" } craft-keymap = { path = "crates/keymap" } diff --git a/README.md b/README.md index 951fc1ab9..678bf7f1c 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,10 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = "` form on craft-xml elements, value codecs and the package writer; and InDesign Tagged Text, read and written. +- [`craft-ical`](crates/ical/README.md): iCalendar (RFC 5545) building blocks, from MailCraft: content + lines, DATE/DATE-TIME values, durations, TEXT escaping, the common subset of recurrence rules, + and `VTIMEZONE` definitions read and written. No time-zone database; what events and to-dos + mean stays in each app. Bounded and panic-free on hostile input. - [`craft-stream-codecs`](crates/stream-codecs/README.md): byte-stream and bi-level image codecs that PDF, PostScript and TIFF share: LZW, ASCII85, ASCIIHex, RunLength, TIFF and PNG predictors, CCITT Group 3 and 4 decoding and Group 4 encoding, lossless JBIG2 encoding. Every diff --git a/crates/ical/Cargo.toml b/crates/ical/Cargo.toml new file mode 100644 index 000000000..b533c1386 --- /dev/null +++ b/crates/ical/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-ical" +description = "iCalendar (RFC 5545) building blocks: content lines, values, recurrence rules and VTIMEZONE definitions, read and written." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +chrono = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/ical/README.md b/crates/ical/README.md new file mode 100644 index 000000000..1781ffc3b --- /dev/null +++ b/crates/ical/README.md @@ -0,0 +1,56 @@ +# craft-ical + +iCalendar (RFC 5545) building blocks, read and written. Times are `chrono` values; instants are +`i64` seconds since 1970-01-01T00:00:00Z. + +- **`lines`**: `Lines` unfolds a text (CRLF, LF or CR; a byte order mark is skipped); + `ContentLine::parse` splits `NAME;PARAM="quoted",x:value` into an upper-cased name, + parameters (RFC 6868 caret escapes undone) and the value as written; `fold` writes a line in + 75-octet pieces without splitting a character; `param_value` quotes and escapes a parameter. +- **`values`**: `When` (`DATE`, floating or zoned `DATE-TIME`, UTC `DATE-TIME`, with the ISO 8601 + punctuation some writers add), `parse_duration`, `escape_text` / `unescape_text`, `split_list`, + `format_date` / `format_local` / `format_utc`, and the wall-clock helpers they rest on. +- **`rrule`**: `Recurrence`, the subset of RRULE calendars show and edit (daily, weekly, monthly, + yearly; interval; weekdays; the nth or last weekday; count; until), `parse_rrule` and + `format_rrule`, and `format_for` / `format_zoned`, which spell out the start's weekday and + month and give `UNTIL` the start's value type. A rule outside the subset is read as the nearest + it holds. +- **`zones`**: `TimeZones::collect` reads a text's `VTIMEZONE`s (yearly rules by weekday or day + of the month, `UNTIL`, `COUNT`, `RDATE`s); a `VTimezone` answers `offset_at(instant)`, + `offset_for_wall(wall)` and `instant(wall)`, with RFC 5545's reading of skipped and repeated + hours. `write_vtimezone` writes a zone from the caller's `ZoneRules` (offset at an instant, + transitions): one `DAYLIGHT` and one `STANDARD` part with yearly rules where the zone keeps one + rule, else change by change. + +No time-zone database is here: matching a `TZID` to an IANA or Windows zone, and which zone an +event is shown in, are the caller's. So is what events and to-dos mean: an app reads them from +`ContentLine`s with these values and maps them onto its own model. + +## Hostile input + +Nothing panics on any text. + +| Limit | Value | +|---|---| +| unfolded line (`lines::MAX_LINE`) | 1 MiB | +| parameters on a line, values of a parameter | 64 each | +| recurrence rule | 4096 bytes, 64 parts | +| durations | about 10,000 years | +| zones per text (`zones::MAX_ZONES`) | 64 | +| parts per zone, dates per part | 64 each | +| `TZID` kept (`zones::MAX_TZID`) | 128 bytes | +| offsets | at most 18 hours | +| work on zone definitions per text (`zones::WORK_BUDGET`) | 20,000,000 units, spent by the caller through `TimeZones::spend` | + +Every date and time written is in the years 1 to 9999; a value outside them is moved to the +nearest end. + +## Origin + +MailCraft `crates/ical` (`lines.rs`, `values.rs`, `rrule.rs`, `zones.rs`) at +[a44a450](https://github.com/UCS-Care-Group/mailcraft/commit/a44a45013d1c6396229c40ab60172d1882b368d5), +by Martin Ball and the MailCraft contributors (MIT OR Apache-2.0), with their tests. MailCraft's +own date types are replaced by `chrono`, its time-zone database by the `ZoneRules` trait (the +writer's tests use rule zones built in the test), and its recurrence model by `Recurrence`. +MailCraft's event and to-do reader and writer (`read.rs`, `write.rs`), and the matching of a +`TZID` against its zone database, stay in MailCraft as the mapping onto its model. diff --git a/crates/ical/proptest-regressions/lines.txt b/crates/ical/proptest-regressions/lines.txt new file mode 100644 index 000000000..bd55905b3 --- /dev/null +++ b/crates/ical/proptest-regressions/lines.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 960cb45b7c8511f655aea177ca4d647864fc5cf713e5fbdd611ddba6ed077047 # shrinks to line = "\u{feff}\u{feff}" diff --git a/crates/ical/src/lib.rs b/crates/ical/src/lib.rs new file mode 100644 index 000000000..be77be9e7 --- /dev/null +++ b/crates/ical/src/lib.rs @@ -0,0 +1,33 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! iCalendar (RFC 5545): the layers a calendar program reads and writes `.ics` text with. +//! +//! - [`lines`]: content lines. [`Lines`] unfolds a text, [`ContentLine::parse`] splits a line +//! into its name, parameters (RFC 6868 caret escapes undone) and value, [`fold`] and +//! [`param_value`] write them. +//! - [`values`]: DATE and DATE-TIME values ([`When`]), durations, TEXT escaping, lists. +//! - [`rrule`]: the common subset of recurrence rules ([`Recurrence`]), read and written, with the +//! start's weekday and month spelled out where a rule leans on them. +//! - [`zones`]: `VTIMEZONE` definitions a text brings ([`TimeZones`]), which answer the offset at +//! an instant and the instant of a wall-clock time, and [`write_vtimezone`], which writes a +//! definition from the caller's zone rules ([`ZoneRules`]). +//! +//! Times are `chrono` values; instants are seconds since 1970-01-01T00:00:00Z. There is no +//! time-zone database here: what a `TZID` means beyond the text is the caller's. +//! +//! What events and to-dos mean (which properties become which fields, which zone a time is shown +//! in) is the caller's too: it reads them from [`ContentLine`]s with these values. +//! +//! Input is hostile. Nothing here panics on any text: lines, parameters, rules, zones, parts and +//! dates are capped, and work on zone definitions is budgeted ([`TimeZones::spend`]). + +pub mod lines; +pub mod rrule; +pub mod values; +pub mod zones; + +pub use lines::{ContentLine, Lines, Param, fold, param_value}; +pub use rrule::{Freq, Recurrence, format_rrule, parse_rrule}; +pub use values::When; +pub use zones::{TimeZones, Transition, VTimezone, ZoneRules, write_vtimezone}; diff --git a/crates/ical/src/lines.rs b/crates/ical/src/lines.rs new file mode 100644 index 000000000..9c82a7c94 --- /dev/null +++ b/crates/ical/src/lines.rs @@ -0,0 +1,321 @@ +//! Content lines (RFC 5545 §3.1–3.2): unfolding on the way in, folding on the way out, and the +//! `NAME;PARAM=value;PARAM="quoted":value` shape in between. + +/// Longest logical line kept, in bytes; the rest of a longer line is dropped. +pub const MAX_LINE: usize = 1 << 20; +/// Most parameters read from one line, and most values of one parameter. +const MAX_PARAMS: usize = 64; +/// Octets per physical line when writing. +const FOLD_AT: usize = 75; + +/// The longest prefix of `s` of at most `max` bytes that ends on a char boundary. +pub fn prefix(s: &str, max: usize) -> &str { + if s.len() <= max { + return s; + } + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + s.get(..end).unwrap_or("") +} + +/// The logical (unfolded) lines of a text. Lines end at CRLF, LF or CR; a line that starts with a +/// space or a tab continues the one before. +pub struct Lines<'a> { + rest: &'a str, +} + +impl<'a> Lines<'a> { + pub fn new(text: &'a str) -> Lines<'a> { + Lines { rest: text.strip_prefix('\u{feff}').unwrap_or(text) } + } + + /// The next physical line, without its line break. + fn physical(&mut self) -> Option<&'a str> { + if self.rest.is_empty() { + return None; + } + let bytes = self.rest.as_bytes(); + let end = bytes.iter().position(|b| matches!(b, b'\r' | b'\n')).unwrap_or(bytes.len()); + let skip = match (bytes.get(end), bytes.get(end.saturating_add(1))) { + (Some(b'\r'), Some(b'\n')) => 2, + (Some(_), _) => 1, + (None, _) => 0, + }; + let line = self.rest.get(..end).unwrap_or(""); + self.rest = self.rest.get(end.saturating_add(skip)..).unwrap_or(""); + Some(line) + } +} + +impl Iterator for Lines<'_> { + type Item = String; + + fn next(&mut self) -> Option { + let mut line = String::new(); + push_capped(&mut line, self.physical()?); + while self.rest.starts_with([' ', '\t']) { + let Some(more) = self.physical() else { break }; + push_capped(&mut line, more.get(1..).unwrap_or("")); + } + Some(line) + } +} + +fn push_capped(line: &mut String, more: &str) { + line.push_str(prefix(more, MAX_LINE.saturating_sub(line.len()))); +} + +/// One property parameter: `CN="Ada Lovelace"`, `ROLE=OPT-PARTICIPANT`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Param { + /// Upper-cased. + pub name: String, + pub values: Vec, +} + +/// One property: `ATTENDEE;CN=Ada:mailto:ada@example.org`. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ContentLine { + /// Upper-cased. + pub name: String, + pub params: Vec, + /// As written: still escaped. + pub value: String, +} + +impl ContentLine { + /// Parse an unfolded line; `None` for a line without a name or without a `:`. + pub fn parse(line: &str) -> Option { + // Every delimiter is ASCII, so the byte positions found here are char boundaries. + let bytes = line.as_bytes(); + let stop = |from: usize, at: &[u8]| { + bytes.get(from..).and_then(|b| b.iter().position(|c| at.contains(c))).map_or(bytes.len(), |i| from.saturating_add(i)) + }; + let mut pos = stop(0, b";:"); + let name = line.get(..pos)?.trim(); + if name.is_empty() { + return None; + } + let mut params = Vec::new(); + while bytes.get(pos) == Some(&b';') { + let start = pos.saturating_add(1); + pos = stop(start, b"=;:"); + let param = line.get(start..pos)?.trim().to_ascii_uppercase(); + let mut values = Vec::new(); + while matches!(bytes.get(pos), Some(b'=' | b',')) { + let from = pos.saturating_add(1); + let value = if bytes.get(from) == Some(&b'"') { + let open = from.saturating_add(1); + let close = stop(open, b"\""); + // A quote that never closes: the line has no value. + bytes.get(close)?; + pos = stop(close, b",;:"); + line.get(open..close)? + } else { + pos = stop(from, b",;:"); + line.get(from..pos)? + }; + if values.len() < MAX_PARAMS { + values.push(caret_decode(value)); + } + } + if !param.is_empty() && params.len() < MAX_PARAMS { + params.push(Param { name: param, values }); + } + } + if bytes.get(pos) != Some(&b':') { + return None; + } + Some(ContentLine { name: name.to_ascii_uppercase(), params, value: line.get(pos.saturating_add(1)..)?.to_string() }) + } + + /// The first value of the parameter `name` (upper case). + pub fn param(&self, name: &str) -> Option<&str> { + self.params.iter().find(|p| p.name == name).and_then(|p| p.values.first()).map(String::as_str) + } +} + +/// Undo RFC 6868: `^n` is a line break, `^'` a double quote, `^^` a caret. +fn caret_decode(s: &str) -> String { + if !s.contains('^') { + return s.to_string(); + } + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars().peekable(); + while let Some(c) = chars.next() { + let decoded = match (c, chars.peek()) { + ('^', Some('n')) => Some('\n'), + ('^', Some('\'')) => Some('"'), + ('^', Some('^')) => Some('^'), + _ => None, + }; + match decoded { + Some(d) => { + out.push(d); + chars.next(); + } + None => out.push(c), + } + } + out +} + +/// A parameter value ready to write: RFC 6868 escapes for what a parameter can't hold, other +/// control characters dropped, and quotes around it when it needs them. +pub fn param_value(s: &str) -> String { + let mut out = String::with_capacity(s.len().saturating_add(2)); + for c in s.chars() { + match c { + '^' => out.push_str("^^"), + '"' => out.push_str("^'"), + '\n' => out.push_str("^n"), + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + } + if out.contains([';', ':', ',']) || out.starts_with(' ') || out.ends_with(' ') { format!("\"{out}\"") } else { out } +} + +/// Append `line` to `out` with CRLF, folded so no physical line is longer than 75 octets and no +/// character is split. +pub fn fold(out: &mut String, line: &str) { + let mut rest = line; + let mut room = FOLD_AT; + while rest.len() > room { + let head = prefix(rest, room); + if head.is_empty() { + break; + } + out.push_str(head); + out.push_str("\r\n "); + rest = rest.get(head.len()..).unwrap_or(""); + // The leading space counts. + room = FOLD_AT - 1; + } + out.push_str(rest); + out.push_str("\r\n"); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lines(text: &str) -> Vec { + Lines::new(text).collect() + } + + #[test] + fn unfolds_every_kind_of_line_break() { + assert_eq!(lines("A:1\r\nB:2\nC:3\rD:4"), ["A:1", "B:2", "C:3", "D:4"]); + assert_eq!(lines("SUMMARY:Lunch \r\n with Ada\r\n"), ["SUMMARY:Lunch with Ada"]); + assert_eq!(lines("SUMMARY:Lun\n\tch\n twice\n"), ["SUMMARY:Lunch twice"]); + // Only the one space is the fold. + assert_eq!(lines("X:a\r\n b"), ["X:a b"]); + assert_eq!(lines("X:a \r\nY:b"), ["X:a ", "Y:b"]); + assert!(lines("").is_empty()); + } + + #[test] + fn a_byte_order_mark_and_blank_lines_are_harmless() { + assert_eq!(lines("\u{feff}BEGIN:VCALENDAR\r\n\r\nEND:VCALENDAR"), ["BEGIN:VCALENDAR", "", "END:VCALENDAR"]); + assert_eq!(ContentLine::parse(""), None); + } + + #[test] + fn parses_names_parameters_and_values() { + let l = ContentLine::parse("dtstart;tzid=Example/Riverton;Value=DATE-TIME:20261012T090000").unwrap(); + assert_eq!(l.name, "DTSTART"); + assert_eq!(l.param("TZID"), Some("Example/Riverton")); + assert_eq!(l.param("VALUE"), Some("DATE-TIME")); + assert_eq!(l.param("CN"), None); + assert_eq!(l.value, "20261012T090000"); + let plain = ContentLine::parse("SUMMARY:Budget: phase two; hiring").unwrap(); + assert_eq!((plain.name.as_str(), plain.params.len(), plain.value.as_str()), ("SUMMARY", 0, "Budget: phase two; hiring")); + assert_eq!(ContentLine::parse("DESCRIPTION:").unwrap().value, ""); + } + + #[test] + fn quoted_parameters_hide_delimiters() { + let l = ContentLine::parse("ATTENDEE;CN=\"Okafor, Nadia: lead; planning\";ROLE=REQ-PARTICIPANT:mailto:nadia@example.org").unwrap(); + assert_eq!(l.param("CN"), Some("Okafor, Nadia: lead; planning")); + assert_eq!(l.param("ROLE"), Some("REQ-PARTICIPANT")); + assert_eq!(l.value, "mailto:nadia@example.org"); + let several = ContentLine::parse("X-TEST;MEMBER=\"mailto:a@example.org\",\"mailto:b@example.org\",c:v").unwrap(); + assert_eq!(several.params[0].values, ["mailto:a@example.org", "mailto:b@example.org", "c"]); + // A parameter without a value, and an empty one. + let bare = ContentLine::parse("X-TEST;FLAG;EMPTY=:v").unwrap(); + assert_eq!(bare.params.len(), 2); + assert_eq!(bare.param("FLAG"), None); + assert_eq!(bare.param("EMPTY"), Some("")); + } + + #[test] + fn broken_lines_are_refused() { + for bad in ["no colon here", ":value without a name", ";CN=x:y", "ATTENDEE;CN=\"never closed:mailto:a@example.org", "ATTENDEE;CN=x", " "] { + assert_eq!(ContentLine::parse(bad), None, "{bad}"); + } + } + + #[test] + fn parameter_values_escape_and_quote() { + assert_eq!(param_value("Ada Lovelace"), "Ada Lovelace"); + assert_eq!(param_value("Lovelace, Ada"), "\"Lovelace, Ada\""); + assert_eq!(param_value("Ada \"the Countess\" ^1\nLine\r\u{7}"), "Ada ^'the Countess^' ^^1^nLine"); + for name in ["Ada", "Lovelace, Ada", "a\"b^c\nd", "x:y;z", "^n^'^^", "Zoë — 東京"] { + let l = ContentLine::parse(&format!("ORGANIZER;CN={}:mailto:a@example.org", param_value(name))).unwrap(); + assert_eq!(l.param("CN"), Some(name)); + } + } + + #[test] + fn folds_at_75_octets_without_splitting_characters() { + let line = format!("SUMMARY:{}", "é東🎉x".repeat(40)); + let mut out = String::new(); + fold(&mut out, &line); + assert!(out.ends_with("\r\n")); + let physical: Vec<&str> = out.trim_end_matches("\r\n").split("\r\n").collect(); + assert!(physical.len() > 3); + assert!(physical.iter().all(|l| l.len() <= 75), "{physical:?}"); + assert!(physical.iter().skip(1).all(|l| l.starts_with(' '))); + assert_eq!(lines(&out), [line]); + // A short line is left alone, and an exact fit isn't folded. + let mut short = String::new(); + fold(&mut short, &"x".repeat(75)); + assert_eq!(short, format!("{}\r\n", "x".repeat(75))); + let mut one_more = String::new(); + fold(&mut one_more, &"x".repeat(76)); + assert_eq!(one_more, format!("{}\r\n x\r\n", "x".repeat(75))); + } + + #[test] + fn enormous_lines_are_capped() { + let text = format!("SUMMARY:{}\r\n {}\r\nUID:after", "é".repeat(MAX_LINE), "x".repeat(1000)); + let got = lines(&text); + assert_eq!(got.len(), 2); + assert!(got[0].len() <= MAX_LINE); + assert_eq!(got[1], "UID:after"); + assert_eq!(prefix("héllo", 2), "h"); + assert_eq!(prefix("héllo", 0), ""); + assert_eq!(prefix("héllo", 99), "héllo"); + } + + proptest::proptest! { + #[test] + fn folding_then_unfolding_gives_the_line_back(line in "[^\r\n]{0,400}") { + let mut out = String::new(); + fold(&mut out, &line); + proptest::prop_assert!(out.split("\r\n").all(|l| l.len() <= 75)); + // A byte order mark at the very start of a text isn't part of its first line. + proptest::prop_assert_eq!(Lines::new(&out).collect::>(), vec![line.strip_prefix('\u{feff}').unwrap_or(&line).to_string()]); + } + + #[test] + fn any_line_parses_or_is_refused(line in proptest::prelude::any::()) { + let _ = ContentLine::parse(&line); + let _ = Lines::new(&line).count(); + } + } +} diff --git a/crates/ical/src/rrule.rs b/crates/ical/src/rrule.rs new file mode 100644 index 000000000..2833e5a29 --- /dev/null +++ b/crates/ical/src/rrule.rs @@ -0,0 +1,516 @@ +//! Recurrence rules (RFC 5545 §3.3.10): the common subset a calendar keeps. + +use chrono::{Datelike, NaiveDate, NaiveDateTime, Weekday}; + +use crate::lines::prefix; +use crate::values::{When, format_date, valid_time}; + +type Timestamp = i64; + +/// How often a rule repeats. `SECONDLY`, `MINUTELY` and `HOURLY` are not held. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum Freq { + Daily, + #[default] + Weekly, + Monthly, + Yearly, +} + +impl Freq { + /// From `DAILY`, `weekly`…: any letter case. + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_uppercase().as_str() { + "DAILY" => Some(Freq::Daily), + "WEEKLY" => Some(Freq::Weekly), + "MONTHLY" => Some(Freq::Monthly), + "YEARLY" => Some(Freq::Yearly), + _ => None, + } + } + + /// `DAILY`, `WEEKLY`, `MONTHLY`, `YEARLY`. + pub fn as_str(self) -> &'static str { + match self { + Freq::Daily => "DAILY", + Freq::Weekly => "WEEKLY", + Freq::Monthly => "MONTHLY", + Freq::Yearly => "YEARLY", + } + } +} + +/// How an event repeats: the subset of RRULE that calendars show and edit. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Recurrence { + pub freq: Freq, + /// Every `interval` days/weeks/months/years; 0 reads as 1. + pub interval: u32, + /// Weekly: the days of the week (none: the start's own). Daily: only on these days. Monthly and + /// yearly with `nth`: the weekday(s) counted. + pub by_day: Vec, + /// Monthly and yearly: the `nth` `by_day` weekday of the month (1–5, or -1 for the last) + /// instead of the start's day of the month. + pub nth: Option, + /// Ends after this many occurrences. + pub count: Option, + /// Ends on this day (inclusive). + pub until: Option, +} + +impl Recurrence { + /// Every day, week, month or year. + pub fn every(freq: Freq) -> Recurrence { + Recurrence { freq, interval: 1, ..Default::default() } + } +} + +/// The days of the week, Monday first. +pub const WEEKDAYS: [Weekday; 7] = [Weekday::Mon, Weekday::Tue, Weekday::Wed, Weekday::Thu, Weekday::Fri, Weekday::Sat, Weekday::Sun]; + +/// `MO`, `TU`… as iCalendar writes a weekday. +pub fn weekday_code(day: Weekday) -> &'static str { + match day { + Weekday::Mon => "MO", + Weekday::Tue => "TU", + Weekday::Wed => "WE", + Weekday::Thu => "TH", + Weekday::Fri => "FR", + Weekday::Sat => "SA", + Weekday::Sun => "SU", + } +} + +/// The weekday of `MO`, `tu`…; `None` for anything else. +pub fn weekday_of_code(s: &str) -> Option { + WEEKDAYS.into_iter().find(|d| weekday_code(*d).eq_ignore_ascii_case(s.trim())) +} + +/// Longest rule read, in bytes, and most parts (and most `BYDAY` days) of one rule. +const MAX_RULE: usize = 4096; +const MAX_PARTS: usize = 64; + +/// `FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE` as a [`Recurrence`]; `None` for a rule the model can't hold +/// (`SECONDLY`, `MINUTELY`, `HOURLY`, junk). +/// +/// `UNTIL` gives its date, whether it is written as a date or a date-time. `BYDAY=2TU`, `BYDAY=-1FR` +/// and `BYDAY=TU;BYSETPOS=2` give `nth`. Parts that only restate the start (`BYMONTH`, +/// `BYMONTHDAY`, `WKST`) and parts the model has no place for are ignored. +pub fn parse_rrule(rule: &str) -> Option { + parse_with(rule, None) +} + +/// [`parse_rrule`] for a rule read from a file: with `offset_at` (minutes east of UTC at an +/// instant), a UTC `UNTIL` gives its local day. +pub fn parse_with(rule: &str, offset_at: Option<&dyn Fn(Timestamp) -> i32>) -> Option { + let rule = prefix(rule.trim(), MAX_RULE); + let rule = match rule.get(..6) { + Some(name) if name.eq_ignore_ascii_case("RRULE:") => rule.get(6..).unwrap_or(""), + _ => rule, + }; + let mut freq = None; + let mut interval = 1u32; + let (mut count, mut until, mut set_pos) = (None, None, None); + let mut days: Vec<(Option, Weekday)> = Vec::new(); + // Parts that pick days in a way the model can't: with them, a bare BYDAY means something else. + let mut limited = false; + for part in rule.split(';').take(MAX_PARTS) { + let Some((key, value)) = part.split_once('=') else { continue }; + let value = value.trim(); + match key.trim().to_ascii_uppercase().as_str() { + "FREQ" => freq = Some(Freq::parse(value)?), + "INTERVAL" => interval = value.parse().ok().filter(|n| *n > 0).unwrap_or(1), + "COUNT" => count = value.parse::().ok(), + "UNTIL" => { + until = When::parse(value).map(|w| match offset_at { + Some(offset_at) => w.local(offset_at).date(), + None => w.wall().date(), + }) + } + "BYDAY" => days = value.split(',').take(MAX_PARTS).filter_map(by_day).collect(), + "BYSETPOS" => set_pos = value.parse::().ok(), + "BYMONTH" | "BYMONTHDAY" | "BYYEARDAY" | "BYWEEKNO" => limited = true, + _ => {} + } + } + let mut freq = freq?; + let by_month = matches!(freq, Freq::Monthly | Freq::Yearly); + // `1MO,3MO` isn't one position: the first one is kept. + let ordinal = days.iter().find_map(|(ordinal, _)| *ordinal); + if by_month && ordinal.is_some() { + days.retain(|(o, _)| *o == ordinal); + } + let mut by_day: Vec = Vec::new(); + for (_, day) in &days { + if !by_day.contains(day) { + by_day.push(*day); + } + } + let mut nth = None; + if by_month { + nth = match (ordinal, set_pos) { + (Some(ordinal), _) => position(ordinal), + // A position in a set of several weekdays ("the last working day") isn't a weekday's position. + (None, Some(pos)) if by_day.len() == 1 || (by_day.is_empty() && !limited) => position(pos), + _ => None, + }; + if nth.is_none() { + if freq == Freq::Monthly && interval == 1 && !by_day.is_empty() && ordinal.is_none() && set_pos.is_none() && !limited { + // Every Monday of every month is every Monday. + freq = Freq::Weekly; + } else { + by_day.clear(); + } + } + } + Some(Recurrence { freq, interval, by_day, nth, count, until }) +} + +/// `MO`, `2TU`, `-1FR`: the position, when there is one, and the day. +fn by_day(item: &str) -> Option<(Option, Weekday)> { + let item = item.trim(); + let split = item.len().checked_sub(2)?; + let day = weekday_of_code(item.get(split..)?)?; + let ordinal = item.get(..split)?; + Some((if ordinal.is_empty() { None } else { Some(ordinal.parse::().ok()?) }, day)) +} + +/// A position the model holds: the first to the fifth, or the last. +fn position(n: i32) -> Option { + if (1..=5).contains(&n) || n == -1 { i8::try_from(n).ok() } else { None } +} + +/// A [`Recurrence`] as an RRULE value: `FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE`. +/// +/// An `nth` without a `by_day` means "the start's weekday", which a rule alone can't say: it is +/// written as a bare `BYSETPOS`, which [`parse_rrule`] reads back but other programs may not. +/// [`format_for`] knows the start and writes the weekday (and the month of a yearly rule) out. For monthly and yearly rules without an `nth` the model ignores `by_day`, so it isn't +/// written. +pub fn format_rrule(r: &Recurrence) -> String { + format(r, None, None) +} + +/// The rule of an event that starts at `start`: the start's weekday and month are spelled out +/// where the rule leans on them, and `UNTIL` has the start's value type. +pub fn format_for(r: &Recurrence, start: NaiveDateTime, all_day: bool) -> String { + format(r, Some((valid_time(start), all_day)), None) +} + +/// The rule of a timed event whose start is written in a zone: as [`format_for`], with `UNTIL` +/// as the instant `until` in UTC, which is how RFC 5545 has it beside such a start. +pub fn format_zoned(r: &Recurrence, start: NaiveDateTime, until: Option) -> String { + format(r, Some((valid_time(start), false)), until) +} + +fn format(r: &Recurrence, start: Option<(NaiveDateTime, bool)>, until_at: Option) -> String { + let mut out = format!("FREQ={}", r.freq.as_str()); + if r.interval > 1 { + out.push_str(&format!(";INTERVAL={}", r.interval)); + } + let mut days: Vec = Vec::new(); + for day in &r.by_day { + if !days.contains(day) { + days.push(*day); + } + } + let list = |days: &[Weekday], position: &str| days.iter().map(|d| format!("{position}{}", weekday_code(*d))).collect::>().join(","); + match (r.freq, r.nth.filter(|n| *n != 0)) { + (Freq::Daily | Freq::Weekly, _) => { + if !days.is_empty() { + out.push_str(&format!(";BYDAY={}", list(&days, ""))); + } + } + (_, None) => {} + (freq, Some(nth)) => { + // Any negative position is "the last" to the model. + let nth = nth.max(-1); + if let Some((start, _)) = start { + if days.is_empty() { + days.push(start.date().weekday()); + } + if freq == Freq::Yearly { + out.push_str(&format!(";BYMONTH={}", start.date().month())); + } + } + if days.is_empty() { + out.push_str(&format!(";BYSETPOS={nth}")); + } else { + out.push_str(&format!(";BYDAY={}", list(&days, &nth.to_string()))); + } + } + } + if let Some(count) = r.count { + out.push_str(&format!(";COUNT={count}")); + } + if let (Some(_), Some(at)) = (r.until, until_at) { + out.push_str(&format!(";UNTIL={}", crate::values::format_utc(at))); + } else if let Some(until) = r.until { + out.push_str(&format!(";UNTIL={}", format_date(until))); + // The whole last day of a timed series. + if matches!(start, Some((_, false))) { + out.push_str("T235959"); + } + } + out +} + +#[cfg(test)] +mod tests { + use proptest::prelude::*; + + use super::*; + use crate::values::at_midnight; + + fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() + } + + fn rule(freq: Freq) -> Recurrence { + Recurrence::every(freq) + } + + #[test] + fn parses_the_common_rules() { + assert_eq!(parse_rrule("FREQ=DAILY"), Some(rule(Freq::Daily))); + assert_eq!( + parse_rrule("FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE"), + Some(Recurrence { interval: 2, by_day: vec![Weekday::Mon, Weekday::Wed], ..rule(Freq::Weekly) }) + ); + assert_eq!( + parse_rrule("freq=weekly;byday=fr,mo;count=10;wkst=su"), + Some(Recurrence { by_day: vec![Weekday::Fri, Weekday::Mon], count: Some(10), ..rule(Freq::Weekly) }) + ); + assert_eq!( + parse_rrule("FREQ=DAILY;BYDAY=MO,TU,WE,TH,FR"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Tue, Weekday::Wed, Weekday::Thu, Weekday::Fri], ..rule(Freq::Daily) }) + ); + assert_eq!(parse_rrule("FREQ=MONTHLY;INTERVAL=3"), Some(Recurrence { interval: 3, ..rule(Freq::Monthly) })); + assert_eq!(parse_rrule("FREQ=YEARLY"), Some(rule(Freq::Yearly))); + // The property name and stray spaces are let through. + assert_eq!(parse_rrule(" RRULE:FREQ=DAILY; INTERVAL = 4 ;"), Some(Recurrence { interval: 4, ..rule(Freq::Daily) })); + } + + #[test] + fn positions_come_from_ordinals_or_bysetpos() { + let second_tue = Recurrence { by_day: vec![Weekday::Tue], nth: Some(2), ..rule(Freq::Monthly) }; + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=2TU"), Some(second_tue.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=+2TU"), Some(second_tue.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=TU;BYSETPOS=2"), Some(second_tue)); + let last_fri = Recurrence { by_day: vec![Weekday::Fri], nth: Some(-1), ..rule(Freq::Monthly) }; + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=-1FR"), Some(last_fri.clone())); + assert_eq!(parse_rrule("FREQ=MONTHLY;BYSETPOS=-1;BYDAY=FR"), Some(last_fri)); + // The fourth Thursday of November: the month restates the start. + assert_eq!( + parse_rrule("FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"), + Some(Recurrence { by_day: vec![Weekday::Thu], nth: Some(4), ..rule(Freq::Yearly) }) + ); + assert_eq!( + parse_rrule("FREQ=MONTHLY;BYDAY=1MO,1FR"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Fri], nth: Some(1), ..rule(Freq::Monthly) }) + ); + // Positions mean nothing in a weekly rule. + assert_eq!(parse_rrule("FREQ=WEEKLY;BYDAY=2TU"), Some(Recurrence { by_day: vec![Weekday::Tue], ..rule(Freq::Weekly) })); + } + + #[test] + fn rules_the_model_cannot_hold_fall_back() { + // Two positions: the first is kept. + assert_eq!( + parse_rrule("FREQ=MONTHLY;BYDAY=1MO,3MO,1WE"), + Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Wed], nth: Some(1), ..rule(Freq::Monthly) }) + ); + // The last working day, the second-to-last Friday, Friday the 13th and the like: plain monthly. + for r in [ + "FREQ=MONTHLY;BYDAY=MO,TU,WE,TH,FR;BYSETPOS=-1", + "FREQ=MONTHLY;BYDAY=-2FR", + "FREQ=MONTHLY;BYDAY=FR;BYMONTHDAY=13", + "FREQ=MONTHLY;BYMONTHDAY=15", + "FREQ=MONTHLY;BYMONTHDAY=1,2,3;BYSETPOS=1", + ] { + assert_eq!(parse_rrule(r), Some(rule(Freq::Monthly)), "{r}"); + } + // The 20th Monday of the year. + assert_eq!(parse_rrule("FREQ=YEARLY;BYDAY=20MO"), Some(rule(Freq::Yearly))); + // Every Monday and Thursday of every month is weekly; every other month isn't. + assert_eq!(parse_rrule("FREQ=MONTHLY;BYDAY=MO,TH"), Some(Recurrence { by_day: vec![Weekday::Mon, Weekday::Thu], ..rule(Freq::Weekly) })); + assert_eq!(parse_rrule("FREQ=MONTHLY;INTERVAL=2;BYDAY=MO"), Some(Recurrence { interval: 2, ..rule(Freq::Monthly) })); + assert_eq!(parse_rrule("FREQ=YEARLY;BYMONTH=3,6,9"), Some(rule(Freq::Yearly))); + } + + #[test] + fn until_is_a_date_whatever_its_form() { + let until = |r: &str| parse_rrule(r).and_then(|r| r.until); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231T235959"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=20261231T030000Z"), Some(d(2026, 12, 31))); + assert_eq!(until("FREQ=DAILY;UNTIL=someday"), None); + // Read from a file, a UTC time is a local day. + let west = parse_with("FREQ=DAILY;UNTIL=20261231T030000Z", Some(&|_| -300)).unwrap(); + assert_eq!(west.until, Some(d(2026, 12, 30))); + let east = parse_with("FREQ=DAILY;UNTIL=20261230T220000Z", Some(&|_| 600)).unwrap(); + assert_eq!(east.until, Some(d(2026, 12, 31))); + assert_eq!(parse_with("FREQ=DAILY;UNTIL=20261231", Some(&|_| -300)).unwrap().until, Some(d(2026, 12, 31))); + } + + #[test] + fn junk_and_fast_rules_are_none() { + for bad in [ + "", + "FREQ=", + "FREQ=HOURLY", + "FREQ=MINUTELY;INTERVAL=5", + "FREQ=SECONDLY", + "FREQ=SOMETIMES", + "INTERVAL=2;BYDAY=MO", + "weekly", + ";;;===", + "RRULE:", + ] { + assert_eq!(parse_rrule(bad), None, "{bad}"); + } + } + + #[test] + fn junk_parts_are_ignored() { + let got = parse_rrule("FREQ=WEEKLY;INTERVAL=0;COUNT=many;BYDAY=MO,XX,,7,MO,é;BYSETPOS=x;X-THING=1;novalue").unwrap(); + assert_eq!(got, Recurrence { by_day: vec![Weekday::Mon], ..rule(Freq::Weekly) }); + assert_eq!(parse_rrule("FREQ=DAILY;INTERVAL=99999999999;COUNT=99999999999").unwrap(), rule(Freq::Daily)); + assert_eq!(parse_rrule("FREQ=DAILY;COUNT=0").unwrap().count, Some(0)); + let long = format!("FREQ=WEEKLY;BYDAY={}", "MO,".repeat(100_000)); + assert_eq!(parse_rrule(&long).unwrap().by_day, [Weekday::Mon]); + } + + #[test] + fn formats_rules() { + assert_eq!(format_rrule(&rule(Freq::Daily)), "FREQ=DAILY"); + assert_eq!( + format_rrule(&Recurrence { interval: 2, by_day: vec![Weekday::Mon, Weekday::Wed, Weekday::Mon], ..rule(Freq::Weekly) }), + "FREQ=WEEKLY;INTERVAL=2;BYDAY=MO,WE" + ); + assert_eq!(format_rrule(&Recurrence { count: Some(10), ..rule(Freq::Yearly) }), "FREQ=YEARLY;COUNT=10"); + assert_eq!(format_rrule(&Recurrence { until: Some(d(2026, 12, 31)), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;UNTIL=20261231"); + assert_eq!(format_rrule(&Recurrence { by_day: vec![Weekday::Tue], nth: Some(2), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;BYDAY=2TU"); + assert_eq!( + format_rrule(&Recurrence { by_day: vec![Weekday::Mon, Weekday::Fri], nth: Some(-1), ..rule(Freq::Monthly) }), + "FREQ=MONTHLY;BYDAY=-1MO,-1FR" + ); + assert_eq!(format_rrule(&Recurrence { nth: Some(3), ..rule(Freq::Monthly) }), "FREQ=MONTHLY;BYSETPOS=3"); + // Interval 0 reads as 1, and the model ignores the days of a monthly rule without a position. + assert_eq!(format_rrule(&Recurrence { interval: 0, by_day: vec![Weekday::Mon], nth: Some(0), ..rule(Freq::Monthly) }), "FREQ=MONTHLY"); + } + + #[test] + fn an_event_s_rule_spells_out_what_the_start_implies() { + let tuesday = d(2026, 10, 13).and_hms_opt(9, 0, 0).unwrap(); + let r = Recurrence { nth: Some(2), until: Some(d(2027, 6, 30)), ..rule(Freq::Monthly) }; + assert_eq!(format_for(&r, tuesday, false), "FREQ=MONTHLY;BYDAY=2TU;UNTIL=20270630T235959"); + assert_eq!(format_for(&r, tuesday, true), "FREQ=MONTHLY;BYDAY=2TU;UNTIL=20270630"); + let thanksgiving = Recurrence { by_day: vec![Weekday::Thu], nth: Some(4), ..rule(Freq::Yearly) }; + assert_eq!(format_for(&thanksgiving, at_midnight(d(2026, 11, 26)), true), "FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"); + assert_eq!(format_for(&Recurrence { nth: Some(i8::MIN), ..rule(Freq::Yearly) }, tuesday, false), "FREQ=YEARLY;BYMONTH=10;BYDAY=-1TU"); + // What it writes reads back as the same series. + assert_eq!(parse_rrule("FREQ=YEARLY;BYMONTH=11;BYDAY=4TH"), Some(thanksgiving)); + // A start in a year iCalendar can't write is moved into the years it can. + let written = format_for(&Recurrence { nth: Some(1), until: Some(NaiveDate::MAX), ..rule(Freq::Yearly) }, NaiveDateTime::MAX, true); + assert_eq!(written, "FREQ=YEARLY;BYMONTH=12;BYDAY=1FR;UNTIL=99991231"); + } + + /// Every shape the model gives a meaning: weekly and daily rules with any days, monthly and + /// yearly rules with a position (and then any days) or without. + fn shapes() -> Vec { + let day_sets: [&[Weekday]; 5] = [&[], &[Weekday::Tue], &[Weekday::Sun, Weekday::Mon], &[Weekday::Fri, Weekday::Mon, Weekday::Wed], &WEEKDAYS]; + let mut out = Vec::new(); + for freq in [Freq::Daily, Freq::Weekly, Freq::Monthly, Freq::Yearly] { + for interval in [1, 2, 13, u32::MAX] { + for count in [None, Some(0), Some(1), Some(u32::MAX)] { + for until in [None, Some(d(2030, 1, 31)), Some(d(1, 1, 1)), Some(d(9999, 12, 31))] { + for days in day_sets { + let positions: &[Option] = if matches!(freq, Freq::Monthly | Freq::Yearly) { + &[None, Some(1), Some(2), Some(3), Some(4), Some(5), Some(-1)] + } else { + &[None] + }; + for nth in positions { + if nth.is_none() && !days.is_empty() && matches!(freq, Freq::Monthly | Freq::Yearly) { + continue; + } + out.push(Recurrence { freq, interval, by_day: days.to_vec(), nth: *nth, count, until }); + } + } + } + } + } + } + out + } + + #[test] + fn every_shape_the_model_supports_round_trips() { + let shapes = shapes(); + assert!(shapes.len() > 1000); + for r in shapes { + let text = format_rrule(&r); + assert_eq!(parse_rrule(&text), Some(r.clone()), "{text}"); + // And as an event writes it: the start is a Tuesday, so a missing day reads back as Tuesday. + for all_day in [false, true] { + let text = format_for(&r, d(2026, 10, 13).and_hms_opt(9, 30, 0).unwrap(), all_day); + let mut want = r.clone(); + if want.nth.is_some() && want.by_day.is_empty() { + want.by_day = vec![Weekday::Tue]; + } + assert_eq!(parse_rrule(&text), Some(want), "{text}"); + } + } + } + + #[test] + fn shapes_without_a_meaning_are_normalised() { + let normal = |r: &Recurrence| parse_rrule(&format_rrule(r)).unwrap(); + // Interval 0 is 1, duplicate days are one, position 0 is none, any negative position is the last. + assert_eq!(normal(&Recurrence { interval: 0, ..rule(Freq::Weekly) }).interval, 1); + assert_eq!(normal(&Recurrence { by_day: vec![Weekday::Mon; 100], ..rule(Freq::Weekly) }).by_day, [Weekday::Mon]); + assert_eq!(normal(&Recurrence { nth: Some(0), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }), rule(Freq::Monthly)); + assert_eq!(normal(&Recurrence { nth: Some(-7), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }).nth, Some(-1)); + // A position no month has isn't read back. + assert_eq!(normal(&Recurrence { nth: Some(9), by_day: vec![Weekday::Mon], ..rule(Freq::Monthly) }), rule(Freq::Monthly)); + // A position means nothing in a weekly rule. + assert_eq!(normal(&Recurrence { nth: Some(2), ..rule(Freq::Weekly) }), rule(Freq::Weekly)); + } + + fn recurrence() -> impl Strategy { + let freq = prop_oneof![Just(Freq::Daily), Just(Freq::Weekly), Just(Freq::Monthly), Just(Freq::Yearly)]; + let days = proptest::sample::subsequence(WEEKDAYS.to_vec(), 0..=7).prop_shuffle(); + let nth = proptest::option::of(prop_oneof![1i8..=5, Just(-1i8)]); + let until = proptest::option::of((1i32..=9999, 1u32..=12, 1u32..=28).prop_map(|(y, m, day)| d(y, m, day))); + (freq, 1u32.., days, nth, proptest::option::of(any::()), until).prop_map(|(freq, interval, by_day, nth, count, until)| { + let monthly = matches!(freq, Freq::Monthly | Freq::Yearly); + let nth = if monthly { nth } else { None }; + let by_day = if monthly && nth.is_none() { Vec::new() } else { by_day }; + Recurrence { freq, interval, by_day, nth, count, until } + }) + } + + proptest! { + #[test] + fn any_supported_rule_round_trips(r in recurrence()) { + prop_assert_eq!(parse_rrule(&format_rrule(&r)), Some(r)); + } + + #[test] + fn any_rule_at_all_formats_and_reads_back_stably(freq in 0u8..4, interval in any::(), days in proptest::collection::vec(0usize..7, 0..12), nth in proptest::option::of(any::()), count in proptest::option::of(any::())) { + let freq = [Freq::Daily, Freq::Weekly, Freq::Monthly, Freq::Yearly][usize::from(freq)]; + let r = Recurrence { freq, interval, by_day: days.into_iter().filter_map(|i| WEEKDAYS.get(i).copied()).collect(), nth, count, until: None }; + let text = format_rrule(&r); + let back = parse_rrule(&text).unwrap(); + // Reading normalises; a normal rule then round-trips. + prop_assert_eq!(parse_rrule(&format_rrule(&back)), Some(back)); + } + + #[test] + fn rule_text_never_panics(s in any::(), parts in proptest::collection::vec("(FREQ|INTERVAL|COUNT|UNTIL|BYDAY|BYSETPOS|BYMONTH|X)=[-+A-Z0-9,]{0,12}", 0..8)) { + let _ = parse_rrule(&s); + let _ = parse_rrule(&parts.join(";")); + let _ = parse_with(&parts.join(";"), Some(&|_| i32::MIN)); + } + } +} diff --git a/crates/ical/src/values.rs b/crates/ical/src/values.rs new file mode 100644 index 000000000..1f75a804d --- /dev/null +++ b/crates/ical/src/values.rs @@ -0,0 +1,419 @@ +//! Property values: dates and times, durations and escaped text. +//! +//! Times are `chrono` values. An instant is an `i64` of seconds since 1970-01-01T00:00:00Z, and an +//! offset of a zone is minutes east of UTC where this crate takes one from a caller and seconds +//! where a `VTIMEZONE` gives one. Every date and time this crate makes lies in the years 1 to 9999, +//! the years iCalendar can write. + +use chrono::{DateTime, NaiveDate, NaiveDateTime}; + +/// Seconds in a day. +pub const DAY: i64 = 86_400; + +/// Longest duration read, in seconds (some 10 000 years): sums of it can't overflow. +const MAX_DURATION: i64 = 10_000 * 366 * DAY; + +/// The first and the last second iCalendar can write, as seconds since 1970 on the same clock. +const MIN_CIVIL: i64 = -62_135_596_800; +const MAX_CIVIL: i64 = 253_402_300_799; + +/// Seconds since 1970-01-01T00:00:00 on the same wall clock (no zone applied). +pub fn civil_seconds(t: NaiveDateTime) -> i64 { + t.and_utc().timestamp() +} + +/// The wall-clock time `secs` seconds after 1970-01-01T00:00:00, kept in the years 1 to 9999. +pub fn from_civil_seconds(secs: i64) -> NaiveDateTime { + DateTime::from_timestamp(secs.clamp(MIN_CIVIL, MAX_CIVIL), 0).map(|t| t.naive_utc()).unwrap_or_default() +} + +/// The instant a wall clock shows in a zone `offset_minutes` east of UTC. +pub fn to_timestamp(t: NaiveDateTime, offset_minutes: i32) -> i64 { + civil_seconds(t).saturating_sub(i64::from(offset_minutes).saturating_mul(60)) +} + +/// The wall clock at the instant `ts` in a zone `offset_minutes` east of UTC. +pub fn from_timestamp(ts: i64, offset_minutes: i32) -> NaiveDateTime { + from_civil_seconds(ts.saturating_add(i64::from(offset_minutes).saturating_mul(60))) +} + +/// Midnight at the start of `d`. +pub fn at_midnight(d: NaiveDate) -> NaiveDateTime { + d.and_time(chrono::NaiveTime::MIN) +} + +/// A date and time in the years iCalendar can write: one outside them is moved to the nearest end. +pub(crate) fn valid_time(t: NaiveDateTime) -> NaiveDateTime { + from_civil_seconds(civil_seconds(t)) +} + +/// A date from `YYYY-MM-DD` or `YYYYMMDD`, years 1 to 9999. +fn parse_iso_date(s: &str) -> Option { + let s = s.trim(); + let digits: String = s.chars().filter(char::is_ascii_digit).collect(); + if digits.len() != 8 || s.chars().any(|c| !c.is_ascii_digit() && c != '-') { + return None; + } + let year: i32 = digits.get(0..4)?.parse().ok()?; + if year < 1 { + return None; + } + NaiveDate::from_ymd_opt(year, digits.get(4..6)?.parse().ok()?, digits.get(6..8)?.parse().ok()?) +} + +/// A DATE or DATE-TIME value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum When { + /// `20261012`: a whole day. + Date(NaiveDate), + /// `20261012T090000`: a wall-clock time, floating or in a named zone. + Local(NaiveDateTime), + /// `20261012T090000Z`. + Utc(NaiveDateTime), +} + +impl When { + /// From `YYYYMMDD`, `YYYYMMDDTHHMMSS` or `YYYYMMDDTHHMMSSZ`; the seconds may be missing, and + /// the ISO 8601 punctuation some writers add (`2026-10-12T09:00:00Z`) is let through. + pub fn parse(s: &str) -> Option { + let s = s.trim(); + if s.len() > 32 || !s.is_ascii() { + return None; + } + let (date, time) = match s.find(['T', 't']) { + Some(i) => (s.get(..i)?, Some(s.get(i.saturating_add(1)..)?)), + None => (s, None), + }; + let date = parse_iso_date(date)?; + let Some(time) = time else { return Some(When::Date(date)) }; + let (clock, utc) = match time.strip_suffix(['Z', 'z']) { + Some(clock) => (clock, true), + None => (time, false), + }; + let digits: String = clock.chars().filter(|c| *c != ':').collect(); + if !matches!(digits.len(), 4 | 6) || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let part = |at: usize| digits.get(at..at.saturating_add(2)).and_then(|d| d.parse::().ok()); + // A leap second (60) is read as the last second of the minute. + let second = if digits.len() == 6 { part(4)?.min(59) } else { 0 }; + let t = date.and_hms_opt(part(0)?, part(2)?, second)?; + Some(if utc { When::Utc(t) } else { When::Local(t) }) + } + + /// The date and time as written, whatever the zone. + pub fn wall(self) -> NaiveDateTime { + match self { + When::Date(d) => at_midnight(d), + When::Local(t) | When::Utc(t) => t, + } + } + + /// The local wall-clock time: a UTC time is converted, anything else is taken as written. + /// `offset_at(instant)` is minutes east of UTC. + pub fn local(self, offset_at: &dyn Fn(i64) -> i32) -> NaiveDateTime { + match self { + When::Utc(t) => { + let ts = to_timestamp(t, 0); + from_timestamp(ts, offset_at(ts)) + } + other => other.wall(), + } + } + + /// The instant: anything but a UTC time is read in the local zone. + pub fn timestamp(self, offset_at: &dyn Fn(i64) -> i32) -> i64 { + match self { + When::Utc(t) => to_timestamp(t, 0), + other => { + let t = other.wall(); + // Asked twice, so a time near a daylight-saving change still gets its own offset. + let guess = to_timestamp(t, offset_at(to_timestamp(t, 0))); + to_timestamp(t, offset_at(guess)) + } + } + } +} + +/// A duration in seconds from `PT15M`, `-PT1H30M`, `P1D`, `P1DT12H`, `P2W`; `None` for junk. +pub fn parse_duration(s: &str) -> Option { + let s = s.trim(); + let (negative, body) = match s.strip_prefix('-') { + Some(body) => (true, body), + None => (false, s.strip_prefix('+').unwrap_or(s)), + }; + let body = body.strip_prefix(['P', 'p'])?; + let (mut total, mut number, mut in_time, mut any) = (0i64, None::, false, false); + for c in body.chars() { + match c.to_ascii_uppercase() { + '0'..='9' => { + let digit = i64::from(c.to_digit(10)?); + number = Some(number.unwrap_or(0).checked_mul(10)?.checked_add(digit)?); + } + 'T' if number.is_none() => in_time = true, + unit => { + let seconds = match (unit, in_time) { + ('W', false) => 7 * DAY, + ('D', false) => DAY, + ('H', true) => 3600, + ('M', true) => 60, + ('S', true) => 1, + _ => return None, + }; + total = total.checked_add(number.take()?.checked_mul(seconds)?)?; + any = true; + } + } + } + if number.is_some() || !any || total > MAX_DURATION { + return None; + } + Some(if negative { -total } else { total }) +} + +/// `20261012`. +pub fn format_date(d: NaiveDate) -> String { + valid_time(at_midnight(d)).format("%Y%m%d").to_string() +} + +/// `20261012T090000` (floating). +pub fn format_local(t: NaiveDateTime) -> String { + valid_time(t).format("%Y%m%dT%H%M%S").to_string() +} + +/// `20261012T130000Z`. +pub fn format_utc(ts: i64) -> String { + format!("{}Z", format_local(from_timestamp(ts, 0))) +} + +/// A TEXT value ready to write: `\`, `;`, `,` and line breaks escaped (`\r\n` and `\r` count as +/// one line break), other control characters except tab dropped. +pub fn escape_text(s: &str) -> String { + let mut out = String::with_capacity(s.len().saturating_add(8)); + let mut after_cr = false; + for c in s.chars() { + let cr = c == '\r'; + match c { + '\n' if after_cr => {} + '\r' | '\n' => out.push_str("\\n"), + '\\' => out.push_str("\\\\"), + ';' => out.push_str("\\;"), + ',' => out.push_str("\\,"), + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + after_cr = cr; + } + out +} + +/// Undo [`escape_text`]: `\n` and `\N` are line breaks, `\\`, `\;`, `\,` (and the `\:` and `\"` +/// some writers add) their second character; any other backslash is kept. Control characters +/// except tab are dropped. +pub fn unescape_text(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + match c { + '\\' => match chars.next() { + Some('n' | 'N') => out.push('\n'), + Some(x @ ('\\' | ';' | ',' | ':' | '"')) => out.push(x), + Some(x) => { + out.push('\\'); + if !x.is_control() || x == '\t' { + out.push(x); + } + } + None => out.push('\\'), + }, + '\t' => out.push(c), + c if c.is_control() => {} + c => out.push(c), + } + } + out +} + +/// The first `max` items of a list value, split on its unescaped commas; the items are still +/// escaped. +pub fn split_list(s: &str, max: usize) -> Vec<&str> { + let mut out = Vec::new(); + let (mut start, mut escaped) = (0usize, false); + for (i, c) in s.char_indices() { + if out.len() >= max { + return out; + } + match c { + _ if escaped => escaped = false, + '\\' => escaped = true, + ',' => { + out.push(s.get(start..i).unwrap_or("")); + start = i.saturating_add(1); + } + _ => {} + } + } + if out.len() < max { + out.push(s.get(start..).unwrap_or("")); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() + } + + fn at(y: i32, m: u32, day: u32, h: u32, min: u32, s: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, s).unwrap() + } + + #[test] + fn dates_and_times() { + assert_eq!(When::parse("20261012"), Some(When::Date(d(2026, 10, 12)))); + assert_eq!(When::parse("20261012T090000"), Some(When::Local(at(2026, 10, 12, 9, 0, 0)))); + assert_eq!(When::parse(" 20261012T133015Z "), Some(When::Utc(at(2026, 10, 12, 13, 30, 15)))); + assert_eq!(When::parse("20261012t0900z"), Some(When::Utc(at(2026, 10, 12, 9, 0, 0)))); + assert_eq!(When::parse("2026-10-12T09:00:00Z"), Some(When::Utc(at(2026, 10, 12, 9, 0, 0)))); + // A leap second. + assert_eq!(When::parse("20161231T235960Z"), Some(When::Utc(at(2016, 12, 31, 23, 59, 59)))); + } + + #[test] + fn junk_dates_are_none() { + for bad in [ + "", + "tomorrow", + "2026101", + "202610123", + "20261312", + "20260230", + "00000101", + "20261012T", + "20261012T25", + "20261012T240000", + "20261012T0960", + "20261012T090000+0200", + "20261012", + "20261012T09000000000000000000000000000000000000", + ] { + assert_eq!(When::parse(bad), None, "{bad}"); + } + } + + #[test] + fn utc_times_become_local_and_local_times_become_instants() { + let utc = When::parse("20261012T130000Z").unwrap(); + assert_eq!(utc.local(&|_| -240), at(2026, 10, 12, 9, 0, 0)); + assert_eq!(utc.local(&|_| 330), at(2026, 10, 12, 18, 30, 0)); + assert_eq!(utc.wall(), at(2026, 10, 12, 13, 0, 0)); + // Across midnight. + assert_eq!(When::parse("20261012T020000Z").unwrap().local(&|_| -300).date(), d(2026, 10, 11)); + let local = When::parse("20261012T090000").unwrap(); + assert_eq!(local.local(&|_| -240), at(2026, 10, 12, 9, 0, 0)); + assert_eq!(local.timestamp(&|_| -240), utc.timestamp(&|_| 600)); + assert_eq!(When::Date(d(1970, 1, 2)).timestamp(&|_| 60), DAY - 3600); + // A zone that changes its offset at an instant: each side gets its own. + let change = to_timestamp(at(2026, 11, 1, 6, 0, 0), 0); + let zone = |ts: i64| if ts < change { -240 } else { -300 }; + assert_eq!(When::parse("20261101T000000").unwrap().timestamp(&zone), to_timestamp(at(2026, 11, 1, 4, 0, 0), 0)); + assert_eq!(When::parse("20261101T120000").unwrap().timestamp(&zone), to_timestamp(at(2026, 11, 1, 17, 0, 0), 0)); + // A hostile offset is survived. + let _ = utc.local(&|_| i32::MIN); + let _ = local.timestamp(&|_| i32::MAX); + } + + #[test] + fn durations() { + assert_eq!(parse_duration("PT15M"), Some(900)); + assert_eq!(parse_duration("-PT15M"), Some(-900)); + assert_eq!(parse_duration("+PT1H30M"), Some(5400)); + assert_eq!(parse_duration("P1D"), Some(DAY)); + assert_eq!(parse_duration("P1DT12H"), Some(DAY + 12 * 3600)); + assert_eq!(parse_duration("P2W"), Some(14 * DAY)); + assert_eq!(parse_duration("pt0s"), Some(0)); + assert_eq!(parse_duration("-PT0M"), Some(0)); + assert_eq!(parse_duration("PT1H2M3S"), Some(3723)); + for bad in ["", "P", "PT", "15M", "PT15", "P15M", "PT1D", "PTM", "P1X", "P-1D", "P99999999999999999999D", "P9999999999W", "PT1H T"] { + assert_eq!(parse_duration(bad), None, "{bad}"); + } + } + + #[test] + fn formats_and_keeps_dates_in_writable_years() { + assert_eq!(format_date(d(2026, 1, 2)), "20260102"); + assert_eq!(format_date(d(987, 6, 5)), "09870605"); + assert_eq!(format_local(at(2026, 10, 12, 9, 5, 0)), "20261012T090500"); + assert_eq!(format_utc(0), "19700101T000000Z"); + assert_eq!(format_utc(1_760_082_785), "20251010T075305Z"); + assert_eq!(format_utc(i64::MAX), "99991231T235959Z"); + assert_eq!(format_utc(i64::MIN), "00010101T000000Z"); + // Years iCalendar can't write are moved to the nearest it can. + for hostile in [NaiveDate::MIN, NaiveDate::MAX, d(0, 1, 1), d(-4, 2, 29), d(10_000, 2, 29)] { + assert!(When::parse(&format_date(hostile)).is_some(), "{hostile:?}"); + assert!(When::parse(&format_local(at_midnight(hostile))).is_some(), "{hostile:?}"); + } + assert_eq!(format_date(NaiveDate::MAX), "99991231"); + assert_eq!(format_local(NaiveDateTime::MIN), "00010101T000000"); + // Civil seconds are the same clock read as UTC, and are kept in the years 1 to 9999. + assert_eq!(civil_seconds(at(1970, 1, 2, 0, 0, 1)), DAY + 1); + assert_eq!(from_civil_seconds(DAY + 1), at(1970, 1, 2, 0, 0, 1)); + assert_eq!(from_civil_seconds(i64::MAX), at(9999, 12, 31, 23, 59, 59)); + assert_eq!(from_civil_seconds(i64::MIN), at(1, 1, 1, 0, 0, 0)); + assert_eq!(from_timestamp(0, -300), at(1969, 12, 31, 19, 0, 0)); + assert_eq!(to_timestamp(at(1969, 12, 31, 19, 0, 0), -300), 0); + } + + #[test] + fn text_escapes() { + assert_eq!(escape_text("Budget, phase two; \\ done\nNext"), "Budget\\, phase two\\; \\\\ done\\nNext"); + assert_eq!(escape_text("a\r\nb\rc\n\nd\u{0}\u{7f}\te"), "a\\nb\\nc\\n\\nd\te"); + assert_eq!(unescape_text("Budget\\, phase two\\; \\\\ done\\nNext\\NLast"), "Budget, phase two; \\ done\nNext\nLast"); + // What other writers escape without being asked to, and what isn't an escape at all. + assert_eq!(unescape_text("10\\:30 \\\"quoted\\\" C:\\Users\\x tail\\"), "10:30 \"quoted\" C:\\Users\\x tail\\"); + assert_eq!(unescape_text("a\u{0}b\u{1b}c\td"), "abc\td"); + } + + #[test] + fn lists_split_on_unescaped_commas() { + assert_eq!(split_list("Work,Travel\\, abroad,,Home\\\\,x", 10), ["Work", "Travel\\, abroad", "", "Home\\\\", "x"]); + assert_eq!(split_list("", 10), [""]); + assert_eq!(split_list("a,b,c,d", 2), ["a", "b"]); + assert_eq!(split_list("a,b", 2), ["a", "b"]); + assert!(split_list("a,b", 0).is_empty()); + } + + proptest::proptest! { + #[test] + fn clean_text_survives_escaping(s in "(\\PC|[\n\t;,\\\\:\"]){0,80}") { + proptest::prop_assert_eq!(unescape_text(&escape_text(&s)), s); + } + + #[test] + fn any_text_escapes_to_one_clean_line(s in proptest::prelude::any::()) { + let escaped = escape_text(&s); + proptest::prop_assert!(!escaped.contains(['\r', '\n'])); + // Escaping what was read gives the same line again. + proptest::prop_assert_eq!(escape_text(&unescape_text(&escaped)), escaped); + } + + #[test] + fn values_never_panic(s in proptest::prelude::any::(), max in 0usize..5) { + let _ = When::parse(&s); + let _ = parse_duration(&s); + let _ = unescape_text(&s); + let _ = split_list(&s, max); + } + + #[test] + fn any_civil_second_reads_back_in_writable_years(secs in proptest::prelude::any::()) { + let t = from_civil_seconds(secs); + proptest::prop_assert_eq!(civil_seconds(t), secs.clamp(MIN_CIVIL, MAX_CIVIL)); + proptest::prop_assert!(When::parse(&format_local(t)).is_some()); + } + } +} diff --git a/crates/ical/src/zones.rs b/crates/ical/src/zones.rs new file mode 100644 index 000000000..9d01d93b8 --- /dev/null +++ b/crates/ical/src/zones.rs @@ -0,0 +1,498 @@ +//! Time zones in iCalendar text (RFC 5545 §3.6.5): the `VTIMEZONE` definitions a text brings with +//! it, read ([`TimeZones`], [`VTimezone`]) and written ([`write_vtimezone`]). +//! +//! This crate has no time-zone database. A definition read from a text answers for itself (the +//! offset at an instant, the instant of a wall-clock time); what a `TZID` means beyond the text +//! (an IANA name, a Windows name, which database zone keeps the same clock) is the caller's. To +//! write a definition, the caller hands over its zone's rules through [`ZoneRules`]. +//! +//! Definitions are hostile text: offsets of more than eighteen hours are not offsets, and the +//! zones, their parts and their dates are all capped. [`TimeZones::spend`] keeps a budget of the +//! work a caller does with them. + +use std::cell::Cell; + +use chrono::{Datelike, NaiveDate, NaiveDateTime, Weekday}; + +use crate::lines::{ContentLine, Lines, fold, prefix}; +use crate::rrule::{weekday_code, weekday_of_code}; +use crate::values::{When, at_midnight, civil_seconds, format_local, from_civil_seconds, from_timestamp, to_timestamp}; + +/// Most zones one text defines, most parts (`STANDARD`, `DAYLIGHT`) of one zone, and most dates +/// (`RDATE`) of one part. +pub const MAX_ZONES: usize = 64; +pub const MAX_PARTS: usize = 64; +pub const MAX_DATES: usize = 64; +/// The work one text's times may cost, in units of one look at one part or one date of a +/// definition ([`VTimezone::cost`]). Past it, [`TimeZones::spend`] says no. +pub const WORK_BUDGET: u64 = 20_000_000; +/// The furthest from UTC an offset may be, in seconds. +const MAX_OFFSET: i32 = 18 * 3600; +/// Longest `TZID` (and `X-LIC-LOCATION`) kept, in bytes. +pub const MAX_TZID: usize = 128; + +/// On which day of each year a part of a zone begins. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Yearly { + /// The `nth` (1–5, or negative from the end) `weekday` of `month`. + Nth { month: u32, nth: i8, weekday: Weekday }, + /// Day `day` of `month`. + Day { month: u32, day: u32 }, +} + +/// The `nth` (1–5, or negative from the end) `weekday` of a month, when the month has one. +fn nth_weekday(year: i32, month: u32, weekday: Weekday, nth: i8) -> Option { + let first = NaiveDate::from_ymd_opt(year, month, 1)?; + if nth > 0 { + let offset = (i64::from(weekday.num_days_from_monday()) - i64::from(first.weekday().num_days_from_monday())).rem_euclid(7); + NaiveDate::from_ymd_opt(year, month, u32::try_from(1 + offset + 7 * (i64::from(nth) - 1)).ok()?) + } else { + let last = first.checked_add_months(chrono::Months::new(1))?.pred_opt()?; + let back = (i64::from(last.weekday().num_days_from_monday()) - i64::from(weekday.num_days_from_monday())).rem_euclid(7); + let day = last.checked_sub_days(chrono::Days::new(u64::try_from(back + 7 * (i64::from(nth.saturating_neg()) - 1)).ok()?))?; + (day.month() == month).then_some(day) + } +} + +impl Yearly { + fn date(self, year: i32) -> Option { + match self { + Yearly::Day { month, day } => NaiveDate::from_ymd_opt(year, month, day), + Yearly::Nth { month, nth, weekday } => nth_weekday(year, month, weekday, nth), + } + } +} + +/// One `STANDARD` or `DAYLIGHT` part of a definition: from `start` on (a wall-clock time by the +/// offset `from`), and then again every year or on the listed dates, the offset is `to`. +#[derive(Clone, Debug, PartialEq, Eq)] +struct Part { + start: NaiveDateTime, + /// Seconds east of UTC before and after. + from: i32, + to: i32, + yearly: Option, + /// The rule holds up to this instant, and for this many beginnings. + until: Option, + count: Option, + dates: Vec, +} + +impl Part { + /// The wall-clock second from which `to` holds for a beginning written as `start`: a time + /// the clocks skip is still by the offset before, and so is the first of an hour that comes + /// twice (RFC 5545 §3.3.5). + fn wall_from(&self, start: NaiveDateTime) -> i64 { + civil_seconds(start).saturating_add(i64::from(self.to.saturating_sub(self.from).max(0))) + } + + fn instant_of(&self, start: NaiveDateTime) -> i64 { + civil_seconds(start).saturating_sub(i64::from(self.from)) + } + + /// The beginning of the rule's year `year`, when the rule has one then. + fn in_year(&self, year: i32) -> Option { + let rule = self.yearly?; + let first = self.start.year(); + if year < first || self.count.is_some_and(|count| i64::from(year) - i64::from(first) >= i64::from(count)) { + return None; + } + let begins = rule.date(year)?.and_time(self.start.time()); + (begins >= self.start && self.until.is_none_or(|until| self.instant_of(begins) <= until)).then_some(begins) + } + + /// The last beginning at or before `limit`, as `key` measures beginnings. + fn latest(&self, limit: i64, year: i32, key: &dyn Fn(&Part, NaiveDateTime) -> i64) -> Option { + let mut best: Option = None; + let mut take = |begins: NaiveDateTime| { + let at = key(self, begins); + if at <= limit && best.is_none_or(|have| at > have) { + best = Some(at); + } + }; + take(self.start); + self.dates.iter().copied().for_each(&mut take); + // The year of the time asked about and the years beside it; and where the rule ends. + let last = self.until.map(|until| from_timestamp(until, 0).year()); + let counted = self.count.map(|count| self.start.year().saturating_add(i32::try_from(count).unwrap_or(i32::MAX)).saturating_sub(1)); + for year in [Some(year.saturating_add(1)), Some(year), Some(year.saturating_sub(1)), last, last.map(|y| y.saturating_sub(1)), counted] + .into_iter() + .flatten() + { + if let Some(begins) = self.in_year(year) { + take(begins); + } + } + best + } +} + +/// A zone as a text defines it (`VTIMEZONE`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct VTimezone { + tzid: String, + /// `X-LIC-LOCATION`: the place the writer says the zone is (an IANA name, as a rule). + place: String, + parts: Vec, +} + +impl VTimezone { + /// The `TZID`, at most [`MAX_TZID`] bytes. + pub fn tzid(&self) -> &str { + &self.tzid + } + + /// `X-LIC-LOCATION`, empty when the text gives none. + pub fn place(&self) -> &str { + &self.place + } + + /// What one look at the definition costs, in units of [`WORK_BUDGET`]. + pub fn cost(&self) -> u64 { + self.parts.iter().fold(0u64, |sum, part| sum.saturating_add(8).saturating_add(part.dates.len() as u64)) + } + + fn offset_by(&self, limit: i64, year: i32, key: &dyn Fn(&Part, NaiveDateTime) -> i64) -> i32 { + let mut best: Option<(i64, i32)> = None; + for part in &self.parts { + if let Some(at) = part.latest(limit, year, key) + && best.is_none_or(|(have, _)| at >= have) + { + best = Some((at, part.to)); + } + } + match best { + Some((_, to)) => to, + // Before the first beginning: the offset the first one changes from. + None => self.parts.iter().min_by_key(|part| part.start).map_or(0, |part| part.from), + } + } + + /// Seconds east of UTC for a wall-clock time written in the zone. A time the clocks skip is + /// read by the offset before the change; a time that comes twice is the first of the two. + pub fn offset_for_wall(&self, wall: NaiveDateTime) -> i32 { + self.offset_by(civil_seconds(wall), wall.year(), &|part, begins| part.wall_from(begins)) + } + + /// Seconds east of UTC at an instant (seconds since 1970-01-01T00:00:00Z). + pub fn offset_at(&self, instant: i64) -> i32 { + self.offset_by(instant, from_timestamp(instant, 0).year(), &|part, begins| part.instant_of(begins)) + } + + /// The instant of a wall-clock time written in the zone. + pub fn instant(&self, wall: NaiveDateTime) -> i64 { + civil_seconds(wall).saturating_sub(i64::from(self.offset_for_wall(wall))) + } +} + +/// `+0100`, `-0500`, `+053000`: seconds east of UTC; `None` for anything else and for more than +/// an offset can be. +pub fn parse_utc_offset(text: &str) -> Option { + let text = text.trim(); + let (sign, digits) = match (text.strip_prefix('+'), text.strip_prefix('-')) { + (Some(rest), _) => (1, rest), + (None, Some(rest)) => (-1, rest), + (None, None) => return None, + }; + let digits: String = digits.chars().filter(|c| *c != ':').collect(); + if !matches!(digits.len(), 4 | 6) || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + let part = |at: usize| digits.get(at..at.saturating_add(2)).and_then(|d| d.parse::().ok()); + let (hours, minutes, seconds) = (part(0)?, part(2)?, if digits.len() == 6 { part(4)? } else { 0 }); + let total = hours * 3600 + minutes * 60 + seconds; + (minutes < 60 && seconds < 60 && total <= MAX_OFFSET).then_some(sign * total) +} + +/// An offset in seconds as `+0100`, `-0930`, `+045602`. +pub fn format_utc_offset(seconds: i32) -> String { + let sign = if seconds < 0 { '-' } else { '+' }; + let a = seconds.unsigned_abs(); + if a.is_multiple_of(60) { + format!("{sign}{:02}{:02}", a / 3600, a / 60 % 60) + } else { + format!("{sign}{:02}{:02}{:02}", a / 3600, a / 60 % 60, a % 60) + } +} + +/// The rule of a part: `FREQ=YEARLY;BYMONTH=3;BYDAY=2SU`, with the day of the month or the +/// part's own day when no weekday is named. `None` for a rule that isn't yearly. +fn yearly(rule: &str, start: NaiveDateTime) -> Option<(Yearly, Option, Option)> { + let (mut yearly, mut month, mut by_day, mut month_day) = (false, None, None, None); + let (mut until, mut count) = (None, None); + for part in prefix(rule, 512).split(';').take(16) { + let Some((key, value)) = part.split_once('=') else { continue }; + let value = value.trim(); + match key.trim().to_ascii_uppercase().as_str() { + "FREQ" => yearly = value.eq_ignore_ascii_case("YEARLY"), + "INTERVAL" if value != "1" => return None, + "BYMONTH" => month = value.parse::().ok().filter(|m| (1..=12).contains(m)), + "BYMONTHDAY" => month_day = value.parse::().ok().filter(|d| (1..=31).contains(d)), + "BYDAY" => { + let split = value.len().checked_sub(2)?; + let weekday = weekday_of_code(value.get(split..)?)?; + let nth = match value.get(..split)? { + "" => return None, + nth => nth.parse::().ok().filter(|n| (1..=5).contains(n) || (-5..=-1).contains(n))?, + }; + by_day = Some((nth, weekday)); + } + "UNTIL" => until = When::parse(value).map(|at| to_timestamp(at.wall(), 0)), + "COUNT" => count = value.parse::().ok(), + _ => {} + } + } + if !yearly { + return None; + } + let month = month.unwrap_or(start.month()); + let rule = match (by_day, month_day) { + (Some((nth, weekday)), _) => Yearly::Nth { month, nth, weekday }, + (None, day) => Yearly::Day { month, day: day.unwrap_or(start.day()) }, + }; + Some((rule, until, count)) +} + +/// A `STANDARD` or `DAYLIGHT` part being read. +#[derive(Default)] +struct PartBuilder { + start: Option, + from: Option, + to: Option, + rule: Option, + dates: Vec, +} + +impl PartBuilder { + fn finish(self) -> Option { + let (start, from, to) = (self.start?, self.from?, self.to?); + let (yearly, until, count) = match self.rule.as_deref().and_then(|rule| yearly(rule, start)) { + Some((rule, until, count)) => (Some(rule), until, count), + None => (None, None, None), + }; + Some(Part { start, from, to, yearly, until, count, dates: self.dates }) + } +} + +/// The zones a text defines, and the budget of work spent on them. +#[derive(Debug, Default)] +pub struct TimeZones { + zones: Vec, + /// Work spent so far (see [`WORK_BUDGET`]), spends granted so far, and whether the budget ran out. + spent: Cell, + granted: Cell, + stopped: Cell, +} + +impl TimeZones { + /// Read the `VTIMEZONE`s of a text: at most [`MAX_ZONES`], the first of each `TZID`. A zone + /// without a `TZID` or without a part that has a start and both offsets is not one. + pub fn collect(text: &str) -> TimeZones { + let mut out = TimeZones::default(); + // Nothing to look for in most texts. + let named = text.as_bytes().windows(9).any(|bytes| bytes.eq_ignore_ascii_case(b"VTIMEZONE")); + if !named { + return out; + } + let mut zone: Option = None; + let mut part: Option = None; + for raw in Lines::new(text) { + let Some(line) = ContentLine::parse(&raw) else { continue }; + let value = prefix(line.value.trim(), MAX_TZID); + match (line.name.as_str(), value.to_ascii_uppercase().as_str()) { + ("BEGIN", "VTIMEZONE") => { + if out.zones.len() >= MAX_ZONES { + break; + } + zone = Some(VTimezone::default()); + part = None; + } + ("END", "VTIMEZONE") => { + part = None; + if let Some(definition) = zone.take() + && !definition.tzid.is_empty() + && !definition.parts.is_empty() + && !out.zones.iter().any(|have| have.tzid == definition.tzid) + { + out.zones.push(definition); + } + } + ("BEGIN", "STANDARD" | "DAYLIGHT") if zone.is_some() => part = Some(PartBuilder::default()), + ("END", "STANDARD" | "DAYLIGHT") => { + if let (Some(definition), Some(built)) = (&mut zone, part.take().and_then(PartBuilder::finish)) + && definition.parts.len() < MAX_PARTS + { + definition.parts.push(built); + } + } + // Anything else that begins inside a zone is not a part of it. + ("BEGIN", _) => part = None, + (name, _) => match (&mut zone, &mut part) { + (Some(_), Some(part)) => match name { + "DTSTART" => part.start = part.start.or_else(|| When::parse(&line.value).map(When::wall)), + "TZOFFSETFROM" => part.from = part.from.or_else(|| parse_utc_offset(&line.value)), + "TZOFFSETTO" => part.to = part.to.or_else(|| parse_utc_offset(&line.value)), + "RRULE" if part.rule.is_none() => part.rule = Some(prefix(&line.value, 512).to_string()), + "RDATE" => { + let room = MAX_DATES.saturating_sub(part.dates.len()); + part.dates.extend(line.value.split(',').take(room).filter_map(When::parse).map(When::wall)); + } + _ => {} + }, + (Some(definition), None) => match name { + "TZID" if definition.tzid.is_empty() => definition.tzid = value.to_string(), + "X-LIC-LOCATION" if definition.place.is_empty() => definition.place = value.to_string(), + _ => {} + }, + _ => {} + }, + } + } + out + } + + /// The definition of `tzid`: the one of that name, else one whose name differs only in letter + /// case. The name is looked at to [`MAX_TZID`] bytes. + pub fn get(&self, tzid: &str) -> Option<&VTimezone> { + let tzid = prefix(tzid.trim(), MAX_TZID); + self.zones.iter().find(|zone| zone.tzid == tzid).or_else(|| self.zones.iter().find(|zone| zone.tzid.eq_ignore_ascii_case(tzid))) + } + + /// Every definition, in the order of the text. + pub fn iter(&self) -> impl Iterator { + self.zones.iter() + } + + pub fn is_empty(&self) -> bool { + self.zones.is_empty() + } + + /// Spend `units` of the text's work budget; `false` (and the budget is out for good) when + /// there is not enough left. + pub fn spend(&self, units: u64) -> bool { + let total = self.spent.get().saturating_add(units); + if self.stopped.get() || total > WORK_BUDGET { + self.stopped.set(true); + return false; + } + self.spent.set(total); + self.granted.set(self.granted.get().saturating_add(1)); + true + } + + /// When the budget ran out: how many spends were granted before it did. + pub fn stopped_after(&self) -> Option { + self.stopped.get().then(|| self.granted.get()) + } +} + +// ---- writing ---- + +/// A change of a zone's offset: at the instant `at` (seconds since 1970-01-01T00:00:00Z), from +/// `before` to `after` seconds east of UTC. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Transition { + pub at: i64, + pub before: i32, + pub after: i32, +} + +/// What [`write_vtimezone`] needs to know of a zone: the caller's time-zone database. +pub trait ZoneRules { + /// Seconds east of UTC at an instant. + fn offset_at(&self, instant: i64) -> i32; + /// The changes of offset from `from` up to `to` (instants), in order. + fn transitions(&self, from: i64, to: i64) -> Vec; +} + +/// The rule a change of clocks follows when it falls on the same weekday of the same month each +/// year at the same time of day: the `nth` weekday (the last, when it is in the month's last week +/// in each of the years). +fn yearly_rule(changes: &[(NaiveDateTime, i32, i32)]) -> Option { + let (first, first_from, first_to) = changes.first().copied()?; + let alike = changes.iter().all(|(at, from, to)| { + at.month() == first.month() && at.weekday() == first.weekday() && at.time() == first.time() && (*from, *to) == (first_from, first_to) + }); + if !alike { + return None; + } + let nth = |at: &NaiveDateTime| (at.day() - 1) / 7 + 1; + let last = |at: &NaiveDateTime| at.date().checked_add_days(chrono::Days::new(7)).is_none_or(|week_on| week_on.month() != at.month()); + let (month, weekday) = (first.month(), first.weekday()); + if changes.iter().all(|(at, _, _)| nth(at) == nth(&first)) && nth(&first) < 5 && !changes.iter().all(|(at, _, _)| last(at)) { + Some(Yearly::Nth { month, nth: i8::try_from(nth(&first)).unwrap_or(1), weekday }) + } else if changes.iter().all(|(at, _, _)| last(at)) { + Some(Yearly::Nth { month, nth: -1, weekday }) + } else { + None + } +} + +fn write_part(out: &mut String, kind: &str, start: NaiveDateTime, from: i32, to: i32, rule: Option) { + fold(out, &format!("BEGIN:{kind}")); + fold(out, &format!("DTSTART:{}", format_local(start))); + fold(out, &format!("TZOFFSETFROM:{}", format_utc_offset(from))); + fold(out, &format!("TZOFFSETTO:{}", format_utc_offset(to))); + if let Some(Yearly::Nth { month, nth, weekday }) = rule { + fold(out, &format!("RRULE:FREQ=YEARLY;BYMONTH={month};BYDAY={nth}{}", weekday_code(weekday))); + } + fold(out, &format!("END:{kind}")); +} + +/// Write the definition of a zone named `tzid` (control characters dropped, at most [`MAX_TZID`] +/// bytes) for times in the years `from`–`to` (at most 120 +/// years are written; the years are kept in 2 to 9990). +/// +/// A zone that changes its clocks twice a year by one rule is written as most programs write it, +/// and all of them read it: one `STANDARD` and one `DAYLIGHT` part, each with its yearly rule. A +/// zone that doesn't change is one `STANDARD` part. Any other (a year in which the rule changed, +/// a zone that follows the moon) is written out change by change for those years. +pub fn write_vtimezone(out: &mut String, tzid: &str, zone: &dyn ZoneRules, from: i32, to: i32) { + let (from, to) = (from.clamp(2, 9_990), to.clamp(2, 9_990).max(from.clamp(2, 9_990))); + // A few hundred changes at most are looked at. + let to = to.min(from.saturating_add(120)); + let instant = |year: i32| NaiveDate::from_ymd_opt(year, 1, 1).map_or(0, |day| to_timestamp(at_midnight(day), 0)); + let before = zone.offset_at(instant(from - 1)); + // Each change as it is written: the wall-clock time by the offset before it. + let changes: Vec<(NaiveDateTime, i32, i32)> = zone + .transitions(instant(from - 1), instant(to + 1)) + .into_iter() + .map(|change| (from_civil_seconds(change.at.saturating_add(i64::from(change.before))), change.before, change.after)) + .collect(); + let midnight = |year: i32| NaiveDate::from_ymd_opt(year, 1, 1).map(at_midnight).unwrap_or_default(); + fold(out, "BEGIN:VTIMEZONE"); + // A name can't start another line. + let tzid: String = prefix(tzid, MAX_TZID).chars().filter(|c| !c.is_control()).collect(); + fold(out, &format!("TZID:{tzid}")); + if changes.is_empty() { + write_part(out, "STANDARD", midnight(1601), before, before, None); + fold(out, "END:VTIMEZONE"); + return; + } + // Clocks forward is the beginning of daylight time, clocks back the end of it. + let (forward, back): (Vec<_>, Vec<_>) = changes.iter().copied().partition(|(_, from, to)| to > from); + let years = usize::try_from(i64::from(to) - i64::from(from) + 2).unwrap_or(0); + let regular = forward.len() == years && back.len() == years; + match (yearly_rule(&forward).filter(|_| regular), yearly_rule(&back).filter(|_| regular)) { + (Some(spring), Some(autumn)) => { + for (kind, list, rule) in [("DAYLIGHT", &forward, spring), ("STANDARD", &back, autumn)] { + if let Some((start, from, to)) = list.first().copied() { + write_part(out, kind, start, from, to, Some(rule)); + } + } + } + _ => { + // What holds before the first change, then each change. + let first = changes.first().map_or(NaiveDateTime::default(), |(at, _, _)| *at); + write_part(out, "STANDARD", midnight(first.year().saturating_sub(1).max(1)), before, before, None); + for (start, from, to) in changes { + write_part(out, if to > from { "DAYLIGHT" } else { "STANDARD" }, start, from, to, None); + } + } + } + fold(out, "END:VTIMEZONE"); +} + +#[cfg(test)] +#[path = "zones_tests.rs"] +mod tests; diff --git a/crates/ical/src/zones_tests.rs b/crates/ical/src/zones_tests.rs new file mode 100644 index 000000000..5c1ec9eba --- /dev/null +++ b/crates/ical/src/zones_tests.rs @@ -0,0 +1,293 @@ +//! `VTIMEZONE` reading and writing. The zones written are rule zones built here (the United States' +//! rules, the European Union's, Sydney's, a zone without changes, the United States' change of +//! rule in 2007), standing in for a caller's time-zone database. + +use proptest::prelude::*; + +use super::*; + +fn d(y: i32, m: u32, day: u32) -> NaiveDate { + NaiveDate::from_ymd_opt(y, m, day).unwrap() +} + +fn at(y: i32, m: u32, day: u32, h: u32, min: u32) -> NaiveDateTime { + d(y, m, day).and_hms_opt(h, min, 0).unwrap() +} + +fn utc(t: NaiveDateTime) -> i64 { + to_timestamp(t, 0) +} + +/// A zone by its list of changes. +struct Rules { + before: i32, + changes: Vec, +} + +impl ZoneRules for Rules { + fn offset_at(&self, instant: i64) -> i32 { + self.changes.iter().rev().find(|c| c.at <= instant).map_or(self.before, |c| c.after) + } + + fn transitions(&self, from: i64, to: i64) -> Vec { + self.changes.iter().copied().filter(|c| c.at >= from && c.at < to).collect() + } +} + +/// The change on the `nth` `weekday` of `month` at `hour` UTC (`day_before`: the evening before). +fn change(year: i32, month: u32, nth: i8, hour: u32, day_before: bool, before: i32, after: i32) -> Transition { + let day = nth_weekday(year, month, Weekday::Sun, nth).unwrap(); + let day = if day_before { day.pred_opt().unwrap() } else { day }; + Transition { at: utc(day.and_hms_opt(hour, 0, 0).unwrap()), before, after } +} + +const H: i32 = 3600; + +/// The United States' rule since 2007 (and, with `history`, the one before it up to 2006). +fn us_eastern(years: std::ops::RangeInclusive, history: bool) -> Rules { + let mut changes = Vec::new(); + for y in years { + if history && y < 2007 { + changes.push(change(y, 4, 1, 7, false, -5 * H, -4 * H)); + changes.push(change(y, 10, -1, 6, false, -4 * H, -5 * H)); + } else { + changes.push(change(y, 3, 2, 7, false, -5 * H, -4 * H)); + changes.push(change(y, 11, 1, 6, false, -4 * H, -5 * H)); + } + } + Rules { before: -5 * H, changes } +} + +fn berlin(years: std::ops::RangeInclusive) -> Rules { + let changes = years.flat_map(|y| [change(y, 3, -1, 1, false, H, 2 * H), change(y, 10, -1, 1, false, 2 * H, H)]).collect(); + Rules { before: H, changes } +} + +fn sydney(years: std::ops::RangeInclusive) -> Rules { + let changes = years.flat_map(|y| [change(y, 4, 1, 16, true, 11 * H, 10 * H), change(y, 10, 1, 16, true, 10 * H, 11 * H)]).collect(); + Rules { before: 11 * H, changes } +} + +fn fixed(seconds: i32) -> Rules { + Rules { before: seconds, changes: Vec::new() } +} + +/// The zone as a big mail program writes it: today's rule, from the year 1601. +const EASTERN: &str = "BEGIN:VTIMEZONE\r\nTZID:Eastern Standard Time\r\nBEGIN:STANDARD\r\nDTSTART:16011104T020000\r\nRRULE:FREQ=YEARLY;BYDAY=1SU;BYMONTH=11\r\n\ +TZOFFSETFROM:-0400\r\nTZOFFSETTO:-0500\r\nEND:STANDARD\r\nBEGIN:DAYLIGHT\r\nDTSTART:16010311T020000\r\nRRULE:FREQ=YEARLY;BYDAY=2SU;BYMONTH=3\r\n\ +TZOFFSETFROM:-0500\r\nTZOFFSETTO:-0400\r\nEND:DAYLIGHT\r\nEND:VTIMEZONE\r\n"; + +fn only(text: &str) -> VTimezone { + let zones = TimeZones::collect(text); + assert_eq!(zones.iter().count(), 1, "{text}"); + zones.iter().next().unwrap().clone() +} + +#[test] +fn a_definition_gives_each_time_its_offset() { + let eastern = only(EASTERN); + assert_eq!(eastern.tzid(), "Eastern Standard Time"); + assert_eq!(eastern.offset_for_wall(at(2026, 1, 15, 9, 0)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 7, 15, 9, 0)), -4 * 3600); + // March 8, 2026: 1:59 is winter time, 3:00 summer time, and 2:30 (which no clock shows) + // is still read by the offset before. + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 1, 59)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 2, 30)), -5 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 3, 8, 3, 0)), -4 * 3600); + // November 1: 1:30 comes twice, and is the first of the two. + assert_eq!(eastern.offset_for_wall(at(2026, 11, 1, 1, 30)), -4 * 3600); + assert_eq!(eastern.offset_for_wall(at(2026, 11, 1, 2, 0)), -5 * 3600); + // By instants, every half hour of two years agrees with the rules. + let rules = us_eastern(2025..=2028, false); + let mut wall = at(2026, 1, 1, 0, 0); + while wall < at(2028, 1, 1, 0, 0) { + let instant = utc(wall); + assert_eq!(eastern.offset_at(instant), rules.offset_at(instant), "{wall}"); + wall += chrono::Duration::minutes(30); + } + assert_eq!(eastern.instant(at(2026, 7, 15, 9, 0)), utc(at(2026, 7, 15, 13, 0))); + // Before there were any rules: what the first change changes from. + assert_eq!(eastern.offset_for_wall(at(1500, 6, 1, 0, 0)), -5 * 3600); +} + +#[test] +fn definitions_of_every_shape_are_read() { + // Listed dates, a rule that ends, a rule by the day of the month, a part with no rule. + let text = "BEGIN:VTIMEZONE\nTZID:Example/Odd\nX-LIC-LOCATION:Europe/Berlin\n\ +BEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +BEGIN:DAYLIGHT\nDTSTART:20200329T020000\nRDATE:20210328T020000,20220327T020000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0200\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20201025T030000\nRRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU;UNTIL=20221030T010000Z\nTZOFFSETFROM:+0200\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +BEGIN:DAYLIGHT\nDTSTART:20240401T000000\nRRULE:FREQ=YEARLY;BYMONTHDAY=1;BYMONTH=4;COUNT=2\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0130\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20241001T000000\nRRULE:FREQ=YEARLY\nTZOFFSETFROM:+0130\nTZOFFSETTO:+0100\nEND:STANDARD\n\ +END:VTIMEZONE\n"; + let odd = only(text); + let offset = |wall: NaiveDateTime| odd.offset_for_wall(wall) / 60; + assert_eq!(offset(at(2019, 7, 1, 12, 0)), 60); + assert_eq!(offset(at(2020, 7, 1, 12, 0)), 120); + assert_eq!(offset(at(2021, 1, 1, 12, 0)), 60); + assert_eq!(offset(at(2022, 7, 1, 12, 0)), 120); + assert_eq!(offset(at(2023, 7, 1, 12, 0)), 60); + // Two Aprils of half an hour, and no third. + assert_eq!(offset(at(2024, 7, 1, 12, 0)), 90); + assert_eq!(offset(at(2025, 7, 1, 12, 0)), 90); + assert_eq!(offset(at(2025, 12, 1, 12, 0)), 60); + assert_eq!(offset(at(2026, 7, 1, 12, 0)), 60); + // It says where it is, and is found by its name in any letter case. + let zones = TimeZones::collect(text); + assert_eq!(zones.get("Example/Odd").map(VTimezone::place), Some("Europe/Berlin")); + assert!(zones.get("example/odd").is_some()); + assert!(zones.get("Example/Even").is_none()); +} + +#[test] +fn nonsense_definitions_are_no_definitions() { + for text in [ + "BEGIN:VTIMEZONE\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:Empty\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + // Offsets of days, and no offsets at all. + "BEGIN:VTIMEZONE\nTZID:Far\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+9900\nTZOFFSETTO:-7200\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:Far\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+1801\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:None\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:soon\nTZOFFSETTO:later\nEND:STANDARD\nEND:VTIMEZONE\n", + "BEGIN:VTIMEZONE\nTZID:NoStart\nBEGIN:STANDARD\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + // A part outside a zone, and a zone that never ends. + "BEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + "BEGIN:VTIMEZONE\nTZID:Open\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + ] { + assert!(TimeZones::collect(text).is_empty(), "{text}"); + } + assert_eq!(parse_utc_offset("+053000"), Some(5 * 3600 + 30 * 60)); + assert_eq!(parse_utc_offset("-00:30"), Some(-30 * 60)); + for bad in ["", "0100", "+1", "+01", "+0160", "+010060", "+2400", "+١٢٠٠", "++0100"] { + assert_eq!(parse_utc_offset(bad), None, "{bad}"); + } + assert_eq!((format_utc_offset(-570 * 60), format_utc_offset(0), format_utc_offset(16_562)), ("-0930".into(), "+0000".into(), "+043602".into())); + // A megabyte of name, hundreds of zones, thousands of parts and dates: all bounded. + let long = format!( + "BEGIN:VTIMEZONE\nTZID:{}\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n", + "Z".repeat(1 << 20) + ); + let zones = TimeZones::collect(&long); + assert!(only(&long).tzid().len() <= MAX_TZID); + assert!(zones.get(&"Z".repeat(1 << 20)).is_some()); + let many: String = (0..500) + .map(|n| format!("BEGIN:VTIMEZONE\nTZID:Zone{n}\nBEGIN:STANDARD\nDTSTART:19700101T000000\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\nEND:VTIMEZONE\n")) + .collect(); + assert_eq!(TimeZones::collect(&many).iter().count(), MAX_ZONES); + let part = format!( + "BEGIN:STANDARD\nDTSTART:19700101T000000\nRDATE:{}\nTZOFFSETFROM:+0100\nTZOFFSETTO:+0100\nEND:STANDARD\n", + "19800101T000000,".repeat(5_000) + ); + let crowded = only(&format!("BEGIN:VTIMEZONE\nTZID:Crowded\n{}END:VTIMEZONE\n", part.repeat(1_000))); + assert_eq!((crowded.parts.len(), crowded.parts[0].dates.len()), (MAX_PARTS, MAX_DATES)); + assert_eq!(crowded.cost(), (MAX_PARTS * (8 + MAX_DATES)) as u64); + let _ = crowded.offset_for_wall(at(2026, 1, 1, 0, 0)); + // Hostile times ask nothing that panics. + let eastern = only(EASTERN); + for wall in [NaiveDateTime::MAX, NaiveDateTime::MIN, NaiveDateTime::default()] { + let _ = (eastern.offset_for_wall(wall), eastern.instant(wall)); + } + let _ = (eastern.offset_at(i64::MIN), eastern.offset_at(i64::MAX)); +} + +#[test] +fn the_budget_runs_out_for_good() { + let zones = TimeZones::collect(EASTERN); + assert_eq!(zones.stopped_after(), None); + assert!(zones.spend(WORK_BUDGET - 10)); + assert!(zones.spend(10)); + assert!(!zones.spend(1)); + // Once out, out: even a spend of nothing is refused. + assert!(!zones.spend(0)); + assert_eq!(zones.stopped_after(), Some(2)); +} + +/// What a written definition says, read back. +fn written(name: &str, zone: &dyn ZoneRules, from: i32, to: i32) -> (String, VTimezone) { + let mut out = String::new(); + write_vtimezone(&mut out, name, zone, from, to); + let read = only(&out); + (out, read) +} + +#[test] +fn written_definitions_are_the_usual_two_rules_where_a_zone_has_them() { + let (text, _) = written("America/New_York", &us_eastern(2025..=2027, false), 2026, 2026); + assert_eq!( + text.replace("\r\n", "\n"), + "BEGIN:VTIMEZONE\nTZID:America/New_York\n\ +BEGIN:DAYLIGHT\nDTSTART:20250309T020000\nTZOFFSETFROM:-0500\nTZOFFSETTO:-0400\nRRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=2SU\nEND:DAYLIGHT\n\ +BEGIN:STANDARD\nDTSTART:20251102T020000\nTZOFFSETFROM:-0400\nTZOFFSETTO:-0500\nRRULE:FREQ=YEARLY;BYMONTH=11;BYDAY=1SU\nEND:STANDARD\n\ +END:VTIMEZONE\n" + ); + let (text, _) = written("Europe/Berlin", &berlin(2025..=2031), 2026, 2030); + assert!(text.contains("RRULE:FREQ=YEARLY;BYMONTH=3;BYDAY=-1SU") && text.contains("RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=-1SU"), "{text}"); + assert!(text.contains("DTSTART:20250330T020000") && text.contains("DTSTART:20251026T030000"), "{text}"); + // The south: summer time begins in October. + let (text, _) = written("Australia/Sydney", &sydney(2025..=2027), 2026, 2026); + assert!(text.contains("RRULE:FREQ=YEARLY;BYMONTH=10;BYDAY=1SU") && text.contains("RRULE:FREQ=YEARLY;BYMONTH=4;BYDAY=1SU"), "{text}"); + // No changes: one part. An offset is a zone too. + let (text, _) = written("Asia/Kolkata", &fixed(5 * H + 1800), 2026, 2026); + assert_eq!( + text.replace("\r\n", "\n"), + "BEGIN:VTIMEZONE\nTZID:Asia/Kolkata\nBEGIN:STANDARD\nDTSTART:16010101T000000\nTZOFFSETFROM:+0530\nTZOFFSETTO:+0530\nEND:STANDARD\nEND:VTIMEZONE\n" + ); + let (text, _) = written("UTC-09:30", &fixed(-570 * 60), 2026, 2026); + assert!(text.contains("TZID:UTC-09:30") && text.contains("TZOFFSETTO:-0930"), "{text}"); + // The year the United States changed its rule: written out change by change. + let history = us_eastern(2004..=2009, true); + let (text, read) = written("America/New_York", &history, 2006, 2007); + assert!(!text.contains("RRULE") && text.contains("DTSTART:20060402T020000") && text.contains("DTSTART:20070311T020000"), "{text}"); + assert_eq!(read.instant(at(2006, 3, 20, 9, 0)), utc(at(2006, 3, 20, 14, 0))); + assert_eq!(read.instant(at(2007, 3, 20, 9, 0)), utc(at(2007, 3, 20, 13, 0))); + // Hostile years are years, and a hostile name can't start a line. + for (from, to) in [(i32::MIN, i32::MAX), (9_999, 1), (0, 0)] { + let _ = written("Europe/Berlin", &berlin(2020..=2030), from, to); + } + let (text, read) = written("Evil\r\nEND:VTIMEZONE\r\nBEGIN:VEVENT", &fixed(0), 2026, 2026); + assert_eq!(read.tzid(), "EvilEND:VTIMEZONEBEGIN:VEVENT"); + assert_eq!(text.matches("VTIMEZONE").count(), 3, "{text}"); +} + +fn any_zone() -> impl Strategy { + (0usize..6, -12 * 60i32..=14 * 60) +} + +fn zone_of(which: usize, minutes: i32) -> Rules { + match which { + 0 => us_eastern(1960..=2100, false), + 1 => us_eastern(1960..=2100, true), + 2 => berlin(1960..=2100), + 3 => sydney(1960..=2100), + 4 => fixed(minutes * 60), + _ => fixed(0), + } +} + +proptest! { + #[test] + fn a_written_definition_reads_as_the_zone_it_was_written_for((which, minutes_east) in any_zone(), year in 1971i32..2090, span in 0i32..4, minutes in 0i64..365 * 24 * 60) { + let zone = zone_of(which, minutes_east); + let (text, read) = written("Example/Zone", &zone, year, year + span); + // Any time of those years is the instant the rules say. + let wall = at(year, 1, 1, 0, 0) + chrono::Duration::minutes(minutes) + chrono::Duration::days(i64::from(span) * 365); + let instant = utc(wall); + prop_assert_eq!(read.offset_at(instant), zone.offset_at(instant), "{} at {}\n{}", which, wall, text); + // And a wall-clock time is the instant it shows, where the clock shows it once. + let wall_instant = read.instant(wall); + if read.offset_at(wall_instant) == zone.offset_at(wall_instant) && zone.offset_at(wall_instant - 2 * i64::from(H)) == zone.offset_at(wall_instant + 2 * i64::from(H)) { + prop_assert_eq!(wall_instant + i64::from(zone.offset_at(wall_instant)), civil_seconds(wall)); + } + } + + #[test] + fn any_text_is_definitions_or_none(text in proptest::prelude::any::(), parts in proptest::collection::vec("(BEGIN|END|DTSTART|TZOFFSETFROM|TZOFFSETTO|RRULE|RDATE|TZID):[-+A-Z0-9=;,]{0,24}", 0..40), wall in proptest::num::i64::ANY) { + let _ = TimeZones::collect(&text); + let zones = TimeZones::collect(&format!("BEGIN:VTIMEZONE\nTZID:X\n{}\nEND:VTIMEZONE\n", parts.join("\n"))); + let wall = from_civil_seconds(wall); + if let Some(definition) = zones.get("X") { + let _ = (definition.offset_for_wall(wall), definition.offset_at(utc(wall)), definition.instant(wall)); + } + let _ = zones.get(&text); + } +} From fa26a2a4c6732ded9f86c411ad7158e159a854fe Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:30:55 -0500 Subject: [PATCH 19/31] Add craft-pdn: Paint.NET PDN3 reader and a data-only MS-NRBF reader, from PhotoCraft (#968) The format half of PhotoCraft's Paint.NET import (crates/io/src/pdn.rs and pdn/nrbf.rs at abeb16c96): the PDN3 header, the .NET binary serialization graph, the layer list and blend modes, and the chunked gzip pixel blocks. Surfaces, blend-mode mapping and warnings stay in the app. --- Cargo.lock | 9 + Cargo.toml | 1 + README.md | 1 + crates/pdn/Cargo.toml | 18 ++ crates/pdn/README.md | 33 +++ crates/pdn/src/lib.rs | 381 ++++++++++++++++++++++++++++++++ crates/pdn/src/nrbf.rs | 473 ++++++++++++++++++++++++++++++++++++++++ crates/pdn/src/tests.rs | 444 +++++++++++++++++++++++++++++++++++++ 8 files changed, 1360 insertions(+) create mode 100644 crates/pdn/Cargo.toml create mode 100644 crates/pdn/README.md create mode 100644 crates/pdn/src/lib.rs create mode 100644 crates/pdn/src/nrbf.rs create mode 100644 crates/pdn/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 857ed807c..7b227c217 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1409,6 +1409,15 @@ dependencies = [ "zeroize", ] +[[package]] +name = "craft-pdn" +version = "0.1.0" +dependencies = [ + "flate2", + "proptest", + "thiserror 2.0.21", +] + [[package]] name = "craft-penpot" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index d7a54cda3..6b5a23328 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,7 @@ craft-aep = { path = "crates/aep" } craft-affinity = { path = "crates/affinity" } craft-ai-core = { path = "crates/ai-core" } craft-pdf = { path = "crates/pdf", version = "=0.1.0" } +craft-pdn = { path = "crates/pdn" } craft-plate-test = { path = "crates/plate-test" } krilla = { version = "=0.8.2", default-features = false, features = ["raster-images"] } hayro = { version = "=0.7.1", default-features = false } diff --git a/README.md b/README.md index 678bf7f1c..33f37171c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " = std::result::Result; + +fn invalid(message: impl Into) -> Error { + Error::Invalid(message.into()) +} + +fn check(keep_going: &dyn Fn() -> bool) -> Result<()> { + if keep_going() { Ok(()) } else { Err(Error::Cancelled) } +} + +/// The canvas, and all pixel blocks together, may hold at most this many bytes. +pub const MAX_PIXEL_BYTES: usize = 1 << 30; +/// Layers in one document. +pub const MAX_LAYERS: usize = 1024; +/// Longest layer name, in bytes. +pub const MAX_NAME_BYTES: usize = 64 * 1024; +/// Most bytes of layer names in one document. The graph holds a shared string once; every layer +/// that refers to it gets its own copy. +pub const MAX_TOTAL_NAME_BYTES: usize = 4 * 1024 * 1024; + +/// `true` for a PDN3 document (by its signature). +pub fn is_pdn(bytes: &[u8]) -> bool { + bytes.starts_with(b"PDN3") +} + +/// A layer's blend mode: Paint.NET's `LayerBlendMode`, in its numeric order. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum BlendMode { + Normal, + Multiply, + Additive, + /// Paint.NET's own formula, not Photoshop's Color Burn. + ColorBurn, + /// Paint.NET's own formula, not Photoshop's Color Dodge. + ColorDodge, + Reflect, + Glow, + Overlay, + Difference, + Negation, + Lighten, + Darken, + Screen, + Xor, +} + +impl BlendMode { + /// Every mode, indexed by its `LayerBlendMode` value. + pub const ALL: [BlendMode; 14] = [ + BlendMode::Normal, + BlendMode::Multiply, + BlendMode::Additive, + BlendMode::ColorBurn, + BlendMode::ColorDodge, + BlendMode::Reflect, + BlendMode::Glow, + BlendMode::Overlay, + BlendMode::Difference, + BlendMode::Negation, + BlendMode::Lighten, + BlendMode::Darken, + BlendMode::Screen, + BlendMode::Xor, + ]; + + /// The name older files use in `PaintDotNet.UserBlendOps+BlendOp`. + pub fn name(self) -> &'static str { + match self { + BlendMode::Normal => "Normal", + BlendMode::Multiply => "Multiply", + BlendMode::Additive => "Additive", + BlendMode::ColorBurn => "ColorBurn", + BlendMode::ColorDodge => "ColorDodge", + BlendMode::Reflect => "Reflect", + BlendMode::Glow => "Glow", + BlendMode::Overlay => "Overlay", + BlendMode::Difference => "Difference", + BlendMode::Negation => "Negation", + BlendMode::Lighten => "Lighten", + BlendMode::Darken => "Darken", + BlendMode::Screen => "Screen", + BlendMode::Xor => "Xor", + } + } +} + +/// A bitmap layer, bottom first in [`Document::layers`]. Every layer is the size of the canvas. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Layer { + pub name: String, + pub visible: bool, + /// 0 to 255. + pub opacity: u8, + pub blend: BlendMode, + /// Which surface [`Document::read_surfaces`] hands out holds this layer's pixels. Layers may + /// share one. + pub surface: usize, +} + +/// One decoded pixel block: `height` rows of `stride` bytes, each starting with `width` pixels +/// of B, G, R, A bytes (straight alpha). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Surface { + pub width: usize, + pub stride: usize, + pub data: Vec, +} + +impl Surface { + /// The rows, top first, each cut to its `width × 4` pixel bytes (B, G, R, A). + pub fn rows(&self) -> impl Iterator { + self.data.chunks_exact(self.stride.max(1)).filter_map(|row| row.get(..self.width * 4)) + } +} + +struct Block { + length: usize, + stride: usize, +} + +/// The structure of a PDN3 document. Parse first, then read the pixels. +pub struct Document<'a> { + /// Canvas size in pixels. + pub width: usize, + pub height: usize, + /// The layers, bottom first. + pub layers: Vec, + /// The document carries metadata items (`userMetadataItems`); they are not read. + pub has_user_metadata: bool, + bytes: &'a [u8], + /// Where the pixel blocks start. + pixels_at: usize, + blocks: Vec, +} + +fn count(graph: &Graph, object: &Value, field: &str) -> Result { + usize::try_from(graph.integer(graph.field(object, field)?)?).map_err(|_| invalid(format!("invalid {field}"))) +} + +fn blend(graph: &Graph, layer: &Value, props: &Value) -> Result { + if let Some(mode) = graph.optional_field(props, "blendMode")? { + let mode = match graph.resolve(mode)? { + Value::Int(n) => *n, + _ => graph.integer(graph.field(mode, "value__")?)?, + }; + let mode = usize::try_from(mode).map_err(|_| invalid("invalid blend mode"))?; + return BlendMode::ALL.get(mode).copied().ok_or_else(|| invalid(format!("unsupported blend mode {mode}"))); + } + let props = graph.field(layer, "properties")?; + let mode = graph.class_name(graph.field(props, "blendOp")?)?; + let mode = mode + .strip_prefix("PaintDotNet.UserBlendOps+") + .and_then(|s| s.strip_suffix("BlendOp")) + .ok_or_else(|| invalid(format!("unsupported blend operation {mode}")))?; + BlendMode::ALL.iter().copied().find(|m| m.name() == mode).ok_or_else(|| invalid(format!("unsupported blend operation {mode}"))) +} + +impl<'a> Document<'a> { + /// Reads the header and the serialized document. `keep_going` is asked throughout; `false` + /// ends the read with [`Error::Cancelled`]. + pub fn parse(bytes: &'a [u8], keep_going: &dyn Fn() -> bool) -> Result> { + let mut reader = Reader { bytes, pos: 0 }; + if reader.take(4)? != b"PDN3" { + return Err(invalid("unsupported document signature (expected PDN3)")); + } + let header_size = match reader.take(3)? { + [a, b, c] => u32::from_le_bytes([*a, *b, *c, 0]) as usize, + _ => return Err(invalid("invalid header")), + }; + reader.take(header_size)?; + if reader.take(2)? != [0, 1] { + return Err(invalid("unsupported PDN3 serialization version")); + } + let graph = nrbf::parse_from(&mut reader, keep_going)?; + let root = &graph.root; + if graph.class_name(root)? != "PaintDotNet.Document" { + return Err(invalid("root object is not a Paint.NET document")); + } + let width = count(&graph, root, "width")?; + let height = count(&graph, root, "height")?; + let row_bytes = width.checked_mul(4).ok_or_else(|| invalid("canvas size overflow"))?; + let canvas_bytes = row_bytes.checked_mul(height).ok_or_else(|| invalid("canvas size overflow"))?; + if width == 0 || height == 0 || width > i32::MAX as usize || height > i32::MAX as usize || canvas_bytes > MAX_PIXEL_BYTES { + return Err(invalid("canvas is empty or exceeds the 1 GiB pixel limit")); + } + let list = graph.field(root, "layers")?; + let items = graph.array(graph.field(list, "ArrayList+_items")?)?; + let len = count(&graph, list, "ArrayList+_size")?; + if len == 0 || len > MAX_LAYERS { + return Err(invalid("document must have 1..1024 bitmap layers")); + } + let items = items.get(..len).ok_or_else(|| invalid("layer list count exceeds its array"))?; + // Memory block id to the stride of the surfaces that use it. + let mut layouts = HashMap::new(); + let mut layers = Vec::with_capacity(len); + let mut name_bytes = 0usize; + for layer in items { + check(keep_going)?; + if graph.class_name(layer)? != "PaintDotNet.BitmapLayer" { + return Err(invalid("only PDN3 bitmap layers are supported")); + } + if count(&graph, layer, "Layer+width")? != width || count(&graph, layer, "Layer+height")? != height { + return Err(invalid("layer dimensions do not match the canvas")); + } + let surface = graph.field(layer, "surface")?; + if graph.class_name(surface)? != "PaintDotNet.Surface" + || count(&graph, surface, "width")? != width + || count(&graph, surface, "height")? != height + { + return Err(invalid("invalid bitmap surface")); + } + let stride = count(&graph, surface, "stride")?; + if stride < row_bytes { + return Err(invalid("surface stride is too short for BGRA32 pixels")); + } + let Value::Ref(id) = graph.field(surface, "scan0")? else { return Err(invalid("missing pixel memory reference")) }; + if let Some(old) = layouts.insert(*id, stride) + && old != stride + { + return Err(invalid("shared pixel memory has inconsistent strides")); + } + let props = graph.field(layer, "Layer+properties")?; + let opacity = u8::try_from(count(&graph, props, "opacity")?).map_err(|_| invalid("opacity exceeds 255"))?; + let name = graph.string(graph.field(props, "name")?)?; + if name.len() > MAX_NAME_BYTES { + return Err(invalid("layer name is too long")); + } + name_bytes = name_bytes.saturating_add(name.len()); + if name_bytes > MAX_TOTAL_NAME_BYTES { + return Err(invalid("layer names exceed the total metadata budget")); + } + let visible = graph.boolean(graph.field(props, "visible")?)?; + layers.push((*id, Layer { name: name.to_string(), visible, opacity, blend: blend(&graph, layer, props)?, surface: 0 })); + } + // Validate every allocation before any block is decoded. + let mut blocks = Vec::new(); + let mut index_of = HashMap::new(); + let mut total = 0usize; + for (id, block) in graph.objects_of_class("PaintDotNet.MemoryBlock") { + let length = count(&graph, block, "length64")?; + let stride = *layouts.get(&id).ok_or_else(|| invalid("unreferenced pixel memory block"))?; + if length != stride.checked_mul(height).ok_or_else(|| invalid("pixel length overflow"))? { + return Err(invalid("pixel memory length does not match the surface")); + } + if graph.boolean(graph.field(block, "hasParent")?)? || !graph.boolean(graph.field(block, "deferred")?)? { + return Err(invalid("unsupported non-deferred or parented pixel memory")); + } + total = total.checked_add(length).ok_or_else(|| invalid("pixel length overflow"))?; + if total > MAX_PIXEL_BYTES { + return Err(invalid("decoded pixels exceed the 1 GiB limit")); + } + index_of.insert(id, blocks.len()); + blocks.push(Block { length, stride }); + } + let layers = layers + .into_iter() + .map(|(id, layer)| Ok(Layer { surface: *index_of.get(&id).ok_or_else(|| invalid("missing pixel memory block"))?, ..layer })) + .collect::>>()?; + let has_user_metadata = match graph.optional_field(root, "userMetadataItems")? { + Some(meta) if !matches!(graph.resolve(meta)?, Value::Null) => !graph.array(meta)?.is_empty(), + _ => false, + }; + Ok(Document { width, height, layers, has_user_metadata, bytes, pixels_at: reader.pos, blocks }) + } + + /// How many surfaces [`Self::read_surfaces`] hands out. + pub fn surface_count(&self) -> usize { + self.blocks.len() + } + + /// Bytes all surfaces take once decoded: at most [`MAX_PIXEL_BYTES`]. + pub fn decoded_bytes(&self) -> usize { + self.blocks.iter().map(|b| b.length).sum() + } + + /// Decodes the pixel blocks in file order and hands each to `each` with its index (what + /// [`Layer::surface`] names). Only one decoded surface is held at a time unless `each` keeps + /// them. Data after the last block is an error. + pub fn read_surfaces(&self, keep_going: &dyn Fn() -> bool, each: &mut dyn FnMut(usize, Surface)) -> Result<()> { + let mut reader = Reader { bytes: self.bytes, pos: self.pixels_at }; + for (i, block) in self.blocks.iter().enumerate() { + let data = pixels(&mut reader, block.length, keep_going)?; + each(i, Surface { width: self.width, stride: block.stride, data }); + } + if reader.pos != self.bytes.len() { + return Err(invalid("unexpected trailing data")); + } + Ok(()) + } +} + +/// Reads a memory block's numbered chunks in their on-disk order; each chunk has its own gzip +/// stream and checksum. Inflates directly into the bounded destination, checking its exact size. +fn pixels(reader: &mut Reader<'_>, length: usize, keep_going: &dyn Fn() -> bool) -> Result> { + let version = reader.byte()?; + if version > 1 { + return Err(invalid(format!("unsupported pixel compression {version}"))); + } + let chunk_size = reader.be_u32()?; + if chunk_size == 0 { + return Err(invalid("zero pixel chunk size")); + } + let chunks = length.div_ceil(chunk_size); + if chunks > 1_000_000 { + return Err(invalid("too many pixel chunks")); + } + // Uncompressed chunks hold every pixel byte: a file shorter than that is truncated, and is + // rejected before the buffer is allocated. + if version == 1 && reader.bytes.len().saturating_sub(reader.pos) < length { + return Err(invalid("uncompressed pixel data is truncated")); + } + let mut seen = vec![false; chunks]; + let mut data = vec![0; length]; + for _ in 0..chunks { + check(keep_going)?; + let index = reader.be_u32()?; + let found = seen.get_mut(index).ok_or_else(|| invalid("pixel chunk index out of bounds"))?; + if *found { + return Err(invalid("duplicate pixel chunk")); + } + *found = true; + let size = reader.be_u32()?; + let raw = reader.take(size)?; + let start = index.checked_mul(chunk_size).ok_or_else(|| invalid("pixel offset overflow"))?; + let end = start.checked_add(chunk_size.min(length.saturating_sub(start))).ok_or_else(|| invalid("pixel offset overflow"))?; + let dst = data.get_mut(start..end).ok_or_else(|| invalid("invalid pixel chunk range"))?; + if version == 0 { + let mut decoder = flate2::bufread::GzDecoder::new(raw); + for band in dst.chunks_mut(1 << 20) { + check(keep_going)?; + decoder.read_exact(band).map_err(|e| invalid(format!("invalid gzip pixel chunk: {e}")))?; + } + let mut extra = [0]; + if decoder.read(&mut extra).map_err(|e| invalid(format!("invalid gzip pixel checksum: {e}")))? != 0 || !decoder.into_inner().is_empty() { + return Err(invalid("pixel chunk has excess data")); + } + } else { + if raw.len() != dst.len() { + return Err(invalid("incorrect uncompressed pixel chunk size")); + } + dst.copy_from_slice(raw); + } + } + Ok(data) +} diff --git a/crates/pdn/src/nrbf.rs b/crates/pdn/src/nrbf.rs new file mode 100644 index 000000000..ff06b7b6b --- /dev/null +++ b/crates/pdn/src/nrbf.rs @@ -0,0 +1,473 @@ +//! The data-only subset of the .NET Remoting Binary Format (MS-NRBF) that PDN3 uses, from +//! Microsoft's public specification +//! (). The stream becomes +//! a [`Graph`] of [`Value`]s; references stay as ids and no .NET type is instantiated or run. +//! +//! Limits: [`MAX_ITEMS`] records and members, [`MAX_STRINGS`] bytes of strings, [`MAX_DEPTH`] +//! levels of nesting and of reference chains. + +use std::collections::HashMap; +use std::sync::Arc; + +use crate::{Error, Result, invalid}; + +/// Most records, array items and class members in one stream. +pub const MAX_ITEMS: usize = 1_000_000; +/// Most bytes of strings in one stream. +pub const MAX_STRINGS: usize = 64 << 20; +/// Deepest nesting of records, and longest chain of references [`Graph::resolve`] follows. +pub const MAX_DEPTH: usize = 64; + +pub(crate) struct Reader<'a> { + pub bytes: &'a [u8], + pub pos: usize, +} + +impl<'a> Reader<'a> { + pub fn take(&mut self, len: usize) -> Result<&'a [u8]> { + let end = self.pos.checked_add(len).ok_or_else(|| invalid("length overflow"))?; + let b = self.bytes.get(self.pos..end).ok_or_else(|| invalid("truncated document"))?; + self.pos = end; + Ok(b) + } + + pub fn byte(&mut self) -> Result { + Ok(self.take(1)?.first().copied().unwrap_or_default()) + } + + pub fn le_i32(&mut self) -> Result { + Ok(i32::from_le_bytes(self.take(4)?.try_into().map_err(|_| invalid("invalid integer"))?)) + } + + pub fn be_u32(&mut self) -> Result { + usize::try_from(u32::from_be_bytes(self.take(4)?.try_into().map_err(|_| invalid("invalid integer"))?)).map_err(|_| invalid("length overflow")) + } + + fn count(&mut self) -> Result { + let n = usize::try_from(self.le_i32()?).map_err(|_| invalid("negative count"))?; + if n > MAX_ITEMS { + return Err(invalid("serialized array or class is too large")); + } + Ok(n) + } +} + +/// A serialized class: its name and its member names, in order. +#[derive(Debug)] +pub struct Class { + name: String, + names: Vec, + primitives: Vec>, +} + +impl Class { + /// The class name as serialized (`PaintDotNet.Document`). + pub fn name(&self) -> &str { + &self.name + } + /// The member names, in the order [`Value::Object`] holds their values. + pub fn member_names(&self) -> &[String] { + &self.names + } +} + +/// A value of the graph. +#[derive(Debug)] +pub enum Value { + /// A null reference, and every primitive this reader skips (doubles, decimals, time spans). + Null, + Bool(bool), + /// Every integer type, widened. An unsigned 64-bit value and a date keep their bit pattern. + Int(i64), + /// A string or a single character. + String(String), + /// A reference to the object with this id; [`Graph::resolve`] follows it. + Ref(i32), + /// An array's items. A byte array's content is not kept: its value stays [`Value::Null`]. + Array(Vec), + Object { + class: Arc, + members: Vec, + }, +} + +/// The objects of a stream, by id. +pub struct Graph { + /// A reference to the root object. + pub root: Value, + objects: HashMap, + order: Vec, +} + +impl Graph { + /// Every object of the class `name` with its id, in stream order. + pub fn objects_of_class<'a>(&'a self, name: &'a str) -> impl Iterator { + self.order + .iter() + .filter_map(|id| self.objects.get(id).map(|v| (*id, v))) + .filter(move |(_, v)| matches!(v, Value::Object { class, .. } if class.name == name)) + } + /// Follows references to the value they name. + pub fn resolve<'a>(&'a self, mut value: &'a Value) -> Result<&'a Value> { + for _ in 0..MAX_DEPTH { + match value { + Value::Ref(id) => value = self.objects.get(id).ok_or_else(|| invalid("missing object reference"))?, + _ => return Ok(value), + } + } + Err(invalid("cyclic or excessively deep object reference")) + } + + /// The class name of an object. + pub fn class_name<'a>(&'a self, value: &'a Value) -> Result<&'a str> { + match self.resolve(value)? { + Value::Object { class, .. } => Ok(&class.name), + _ => Err(invalid("expected a serialized object")), + } + } + + /// A member of an object by name; an error when the class has none of that name. + pub fn field<'a>(&'a self, value: &'a Value, name: &str) -> Result<&'a Value> { + self.optional_field(value, name)?.ok_or_else(|| invalid(format!("missing field {name}"))) + } + + /// A member of an object by name; `None` when the class has none of that name. + pub fn optional_field<'a>(&'a self, value: &'a Value, name: &str) -> Result> { + match self.resolve(value)? { + Value::Object { class, members } => Ok(class.names.iter().position(|n| n == name).and_then(|i| members.get(i))), + _ => Err(invalid("expected a serialized object")), + } + } + + /// An integer value. + pub fn integer(&self, value: &Value) -> Result { + match self.resolve(value)? { + Value::Int(n) => Ok(*n), + _ => Err(invalid("expected an integer")), + } + } + + /// A boolean value. + pub fn boolean(&self, value: &Value) -> Result { + match self.resolve(value)? { + Value::Bool(b) => Ok(*b), + _ => Err(invalid("expected a boolean")), + } + } + + /// A string value. + pub fn string<'a>(&'a self, value: &'a Value) -> Result<&'a str> { + match self.resolve(value)? { + Value::String(s) => Ok(s), + _ => Err(invalid("expected a string")), + } + } + + /// An array's items. + pub fn array<'a>(&'a self, value: &'a Value) -> Result<&'a [Value]> { + match self.resolve(value)? { + Value::Array(a) => Ok(a), + _ => Err(invalid("expected an array")), + } + } +} + +struct Parser<'a, 'b> { + reader: &'b mut Reader<'a>, + keep_going: &'b dyn Fn() -> bool, + graph: Graph, + classes: HashMap>, + items: usize, + strings: usize, +} + +/// Reads a stream from its header record to its end record. Returns the graph and the number of +/// bytes the stream took, so whatever follows it can be read. `keep_going` is asked at every +/// record and array item; `false` ends the read with [`Error::Cancelled`]. +pub fn parse(bytes: &[u8], keep_going: &dyn Fn() -> bool) -> Result<(Graph, usize)> { + let mut reader = Reader { bytes, pos: 0 }; + let graph = parse_from(&mut reader, keep_going)?; + Ok((graph, reader.pos)) +} + +pub(crate) fn parse_from(reader: &mut Reader<'_>, keep_going: &dyn Fn() -> bool) -> Result { + if reader.byte()? != 0 { + return Err(invalid("missing NRBF stream header")); + } + let root = reader.le_i32()?; + let _header = reader.le_i32()?; + if root == 0 || reader.le_i32()? != 1 || reader.le_i32()? != 0 { + return Err(invalid("unsupported NRBF stream version")); + } + let mut parser = Parser { + reader, + keep_going, + graph: Graph { root: Value::Ref(root), objects: HashMap::new(), order: Vec::new() }, + classes: HashMap::new(), + items: 0, + strings: 0, + }; + loop { + let tag = parser.reader.byte()?; + if tag == 11 { + break; + } + parser.record(tag, 0)?; + } + parser.graph.resolve(&parser.graph.root)?; + Ok(parser.graph) +} + +impl Parser<'_, '_> { + fn check(&self) -> Result<()> { + if (self.keep_going)() { Ok(()) } else { Err(Error::Cancelled) } + } + + fn budget(&mut self, n: usize) -> Result<()> { + self.items = self.items.checked_add(n).ok_or_else(|| invalid("object count overflow"))?; + if self.items > MAX_ITEMS { + return Err(invalid("too many serialized objects or members")); + } + Ok(()) + } + + fn string(&mut self) -> Result { + let mut len = 0usize; + for shift in (0..35).step_by(7) { + let b = self.reader.byte()?; + if shift == 28 && b > 7 { + return Err(invalid("invalid string length")); + } + len |= usize::from(b & 127) << shift; + if b & 128 == 0 { + self.strings = self.strings.checked_add(len).ok_or_else(|| invalid("string length overflow"))?; + if self.strings > MAX_STRINGS { + return Err(invalid("serialized strings are too large")); + } + return String::from_utf8(self.reader.take(len)?.to_vec()).map_err(|_| invalid("invalid UTF-8 string")); + } + } + Err(invalid("invalid string length")) + } + + fn primitive(&mut self, tag: u8) -> Result { + macro_rules! number { + ($ty:ty, $variant:ident) => { + Value::$variant( + <$ty>::from_le_bytes(self.reader.take(size_of::<$ty>())?.try_into().map_err(|_| invalid("invalid primitive"))?).into(), + ) + }; + } + Ok(match tag { + 1 => match self.reader.byte()? { + 0 => Value::Bool(false), + 1 => Value::Bool(true), + _ => return Err(invalid("invalid boolean")), + }, + 2 => Value::Int(i64::from(self.reader.byte()?)), + 3 => { + let first = self.reader.byte()?; + let len = match first { + 0..=0x7f => 1, + 0xc2..=0xdf => 2, + 0xe0..=0xef => 3, + _ => return Err(invalid("invalid character")), + }; + let mut b = vec![first]; + b.extend_from_slice(self.reader.take(len - 1)?); + Value::String(String::from_utf8(b).map_err(|_| invalid("invalid character"))?) + } + 5 | 18 => Value::String(self.string()?), + 6 => { + self.reader.take(8)?; + Value::Null + } + 7 => number!(i16, Int), + 8 => number!(i32, Int), + 9 | 12 => number!(i64, Int), + 10 => Value::Int(i64::from(self.reader.byte()? as i8)), + 11 => { + self.reader.take(4)?; + Value::Null + } + 13 | 16 => { + let n = u64::from_le_bytes(self.reader.take(8)?.try_into().map_err(|_| invalid("invalid integer"))?); + // Metadata dates may use the high bits; their bit pattern stays intact. + Value::Int(n as i64) + } + 14 => number!(u16, Int), + 15 => number!(u32, Int), + 17 => Value::Null, + _ => return Err(invalid(format!("unsupported primitive type {tag}"))), + }) + } + + fn type_info(&mut self, tag: u8) -> Result> { + match tag { + 0 => Ok(Some(self.reader.byte()?)), + 1 | 2 | 5 | 6 => Ok(None), + 3 => { + self.string()?; + Ok(None) + } + 4 => { + self.string()?; + self.reader.le_i32()?; + Ok(None) + } + 7 => { + self.reader.byte()?; + Ok(None) + } + _ => Err(invalid(format!("unsupported binary type {tag}"))), + } + } + + fn reserve(&mut self, id: i32) -> Result<()> { + if id == 0 || self.graph.objects.insert(id, Value::Null).is_some() { + return Err(invalid("duplicate or zero object id")); + } + self.graph.order.push(id); + Ok(()) + } + + fn value(&mut self, depth: usize) -> Result { + loop { + let tag = self.reader.byte()?; + if tag == 12 { + self.record(tag, depth)?; + } else { + return self.record(tag, depth); + } + } + } + + fn object(&mut self, id: i32, class: Arc, depth: usize) -> Result { + self.reserve(id)?; + self.budget(class.names.len())?; + let mut members = Vec::with_capacity(class.names.len()); + for primitive in &class.primitives { + members.push(match primitive { + Some(p) => self.primitive(*p)?, + None => self.value(depth + 1)?, + }); + } + self.graph.objects.insert(id, Value::Object { class, members }); + Ok(Value::Ref(id)) + } + + fn array(&mut self, id: i32, len: usize, primitive: Option, depth: usize) -> Result { + self.reserve(id)?; + self.budget(len)?; + if primitive == Some(2) { + self.reader.take(len)?; + return Ok(Value::Ref(id)); + } + let mut values = Vec::with_capacity(len); + while values.len() < len { + self.check()?; + if let Some(p) = primitive { + values.push(self.primitive(p)?); + continue; + } + let tag = self.reader.byte()?; + if matches!(tag, 13 | 14) { + let count = if tag == 13 { usize::from(self.reader.byte()?) } else { self.reader.count()? }; + if count == 0 || count > len - values.len() { + return Err(invalid("invalid array null run")); + } + values.resize_with(values.len() + count, || Value::Null); + } else if tag == 12 { + self.record(tag, depth + 1)?; + } else { + values.push(self.record(tag, depth + 1)?); + } + } + self.graph.objects.insert(id, Value::Array(values)); + Ok(Value::Ref(id)) + } + + fn record(&mut self, tag: u8, depth: usize) -> Result { + self.check()?; + self.budget(1)?; + if depth > MAX_DEPTH { + return Err(invalid("serialized objects nested too deeply")); + } + match tag { + 1 => { + let id = self.reader.le_i32()?; + let metadata = self.reader.le_i32()?; + let class = self.classes.get(&metadata).cloned().ok_or_else(|| invalid("missing class metadata"))?; + self.object(id, class, depth) + } + 2..=5 => { + let id = self.reader.le_i32()?; + let name = self.string()?; + let count = self.reader.count()?; + self.budget(count)?; + let names = (0..count).map(|_| self.string()).collect::>>()?; + let primitives = if tag >= 4 { + let types = self.reader.take(count)?.to_vec(); + types.into_iter().map(|t| self.type_info(t)).collect::>>()? + } else { + vec![None; count] + }; + if matches!(tag, 3 | 5) { + self.reader.le_i32()?; + } + let class = Arc::new(Class { name, names, primitives }); + if self.classes.insert(id, class.clone()).is_some() { + return Err(invalid("duplicate class metadata")); + } + self.object(id, class, depth) + } + 6 => { + let id = self.reader.le_i32()?; + self.reserve(id)?; + let value = Value::String(self.string()?); + self.graph.objects.insert(id, value); + Ok(Value::Ref(id)) + } + 7 => { + let id = self.reader.le_i32()?; + let kind = self.reader.byte()?; + let rank = self.reader.count()?; + if kind > 5 || rank == 0 || rank > 32 { + return Err(invalid("invalid array dimensions")); + } + let mut len = 1usize; + for _ in 0..rank { + len = len.checked_mul(self.reader.count()?).ok_or_else(|| invalid("array size overflow"))?; + } + if kind >= 3 { + for _ in 0..rank { + if self.reader.le_i32()? != 0 { + return Err(invalid("nonzero array lower bounds are unsupported")); + } + } + } + let tag = self.reader.byte()?; + let primitive = self.type_info(tag)?; + self.array(id, len, primitive, depth) + } + 8 => { + let primitive = self.reader.byte()?; + self.primitive(primitive) + } + 9 => Ok(Value::Ref(self.reader.le_i32()?)), + 10 => Ok(Value::Null), + 12 => { + self.reader.le_i32()?; + self.string()?; + Ok(Value::Null) + } + 15..=17 => { + let id = self.reader.le_i32()?; + let len = self.reader.count()?; + let primitive = if tag == 15 { Some(self.reader.byte()?) } else { None }; + self.array(id, len, primitive, depth) + } + _ => Err(invalid(format!("unsupported NRBF record {tag}"))), + } + } +} diff --git a/crates/pdn/src/tests.rs b/crates/pdn/src/tests.rs new file mode 100644 index 000000000..bf6c8586b --- /dev/null +++ b/crates/pdn/src/tests.rs @@ -0,0 +1,444 @@ +use super::*; +use std::io::Write; + +enum Field { + Int(i32), + Long(i64), + Byte(u8), + Bool(bool), + Ref(i32), +} + +struct Writer(Vec); + +impl Writer { + fn int(&mut self, n: i32) { + self.0.extend(n.to_le_bytes()); + } + fn string(&mut self, s: &str) { + let mut len = s.len(); + while len >= 128 { + self.0.push((len as u8 & 127) | 128); + len >>= 7; + } + self.0.push(len as u8); + self.0.extend(s.as_bytes()); + } + fn object(&mut self, id: i32, class: &str, fields: &[(&str, Field)]) { + self.0.push(5); + self.int(id); + self.string(class); + self.int(fields.len() as i32); + for (name, _) in fields { + self.string(name); + } + for (_, f) in fields { + self.0.push(if matches!(f, Field::Ref(_)) { 2 } else { 0 }); + } + for (_, f) in fields { + match f { + Field::Int(_) => self.0.push(8), + Field::Long(_) => self.0.push(9), + Field::Byte(_) => self.0.push(2), + Field::Bool(_) => self.0.push(1), + Field::Ref(_) => {} + } + } + self.int(1); + for (_, f) in fields { + match f { + Field::Int(n) => self.int(*n), + Field::Long(n) => self.0.extend(n.to_le_bytes()), + Field::Byte(n) => self.0.push(*n), + Field::Bool(b) => self.0.push(u8::from(*b)), + Field::Ref(id) => { + self.0.push(9); + self.int(*id); + } + } + } + } +} + +const GO: &dyn Fn() -> bool = &|| true; + +/// Parses and decodes everything, as an application would. +fn read_all(bytes: &[u8]) -> Result<(Vec, Vec)> { + let doc = Document::parse(bytes, GO)?; + let mut surfaces = Vec::new(); + doc.read_surfaces(GO, &mut |i, s| { + assert_eq!(i, surfaces.len()); + surfaces.push(s); + })?; + assert_eq!(surfaces.iter().map(|s| s.data.len()).sum::(), doc.decoded_bytes()); + Ok((doc.layers, surfaces)) +} + +/// Independent synthetic PDN writer: padded rows, forward references, a root cycle, +/// spare ArrayList capacity, and reversed gzip/raw chunks. There is no production PDN writer. +fn fixture(modes: &[i32], legacy: bool, compressed: bool) -> Vec { + use Field::*; + let mut w = Writer(b"PDN3\0\0\0\0\x01".to_vec()); + w.0.push(0); + for n in [1, -1, 1, 0] { + w.int(n); + } + w.0.push(12); + w.int(1); + w.string("PaintDotNet.Data"); + w.object(1, "PaintDotNet.Document", &[("width", Int(2)), ("height", Int(2)), ("layers", Ref(2))]); + w.object(2, "PaintDotNet.LayerList", &[("parent", Ref(1)), ("ArrayList+_size", Int(modes.len() as i32)), ("ArrayList+_items", Ref(3))]); + w.0.push(16); + w.int(3); + w.int(modes.len() as i32 + 2); + for i in 0..modes.len() { + w.0.push(9); + w.int(16 + i as i32 * 10); + } + w.0.extend([13, 2]); + for (i, mode) in modes.iter().enumerate() { + let id = 16 + i as i32 * 10; + w.object( + id, + "PaintDotNet.BitmapLayer", + &[ + ("Layer+width", Int(2)), + ("Layer+height", Int(2)), + ("surface", Ref(id + 1)), + ("Layer+properties", Ref(id + 2)), + ("properties", Ref(id + 3)), + ], + ); + w.object(id + 1, "PaintDotNet.Surface", &[("width", Int(2)), ("height", Int(2)), ("stride", Int(12)), ("scan0", Ref(id + 7))]); + let mut fields = + vec![("name", Ref(id + 4)), ("opacity", Byte(if i == 0 { 255 } else { 128 })), ("visible", Bool(i != 1)), ("isBackground", Bool(i == 0))]; + if !legacy { + fields.push(("blendMode", Ref(id + 5))); + } + w.object(id + 2, "PaintDotNet.Layer+LayerProperties", &fields); + w.object(id + 3, "PaintDotNet.BitmapLayer+BitmapLayerProperties", &[("blendOp", Ref(id + 6))]); + w.0.push(6); + w.int(id + 4); + w.string(&format!("Läyer {i}")); + w.object(id + 5, "PaintDotNet.LayerBlendMode", &[("value__", Int(*mode))]); + w.object(id + 6, &format!("PaintDotNet.UserBlendOps+{}BlendOp", BlendMode::ALL.get(*mode as usize).map_or("Unknown", |m| m.name())), &[]); + w.object(id + 7, "PaintDotNet.MemoryBlock", &[("length64", Long(24)), ("hasParent", Bool(false)), ("deferred", Bool(true))]); + } + w.0.push(11); + for _ in modes { + w.0.push(if compressed { 0 } else { 1 }); + w.0.extend(12u32.to_be_bytes()); + for index in [1u32, 0] { + let raw = if index == 0 { + vec![30, 20, 10, 255, 60, 50, 40, 128, 99, 99, 99, 99] + } else { + vec![90, 80, 70, 64, 120, 110, 100, 0, 99, 99, 99, 99] + }; + let data = if compressed { + let mut enc = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + enc.write_all(&raw).unwrap(); + enc.finish().unwrap() + } else { + raw + }; + w.0.extend(index.to_be_bytes()); + w.0.extend((data.len() as u32).to_be_bytes()); + w.0.extend(data); + } + } + w.0 +} + +#[test] +fn all_blend_modes_and_layer_properties_are_read() { + let modes: Vec<_> = (0..14).collect(); + for legacy in [false, true] { + for compressed in [false, true] { + let bytes = fixture(&modes, legacy, compressed); + assert!(is_pdn(&bytes)); + let doc = Document::parse(&bytes, GO).unwrap(); + assert_eq!((doc.width, doc.height, doc.layers.len(), doc.surface_count(), doc.has_user_metadata), (2, 2, 14, 14, false)); + assert_eq!(doc.decoded_bytes(), 14 * 24); + let (layers, surfaces) = read_all(&bytes).unwrap(); + for (i, layer) in layers.iter().enumerate() { + assert_eq!( + layer, + &Layer { + name: format!("Läyer {i}"), + visible: i != 1, + opacity: if i == 0 { 255 } else { 128 }, + blend: BlendMode::ALL[i], + surface: i + } + ); + // Rows are padded to the stride (12 bytes for 2 pixels); the padding is cut off. + let surface = &surfaces[layer.surface]; + assert_eq!((surface.width, surface.stride, surface.data.len()), (2, 12, 24)); + let rows: Vec<&[u8]> = surface.rows().collect(); + assert_eq!(rows, [&[30, 20, 10, 255, 60, 50, 40, 128][..], &[90, 80, 70, 64, 120, 110, 100, 0]], "B, G, R, A"); + } + } + } +} + +#[test] +fn truncation_unknown_modes_and_corrupt_chunks_fail_without_panicking() { + let raw = fixture(&[0], false, false); + for len in 0..raw.len() { + assert!(read_all(&raw[..len]).is_err(), "length {len}"); + } + assert!(read_all(&raw).is_ok()); + let mut trailing = raw.clone(); + trailing.push(0); + assert_eq!(read_all(&trailing).err(), Some(Error::Invalid("unexpected trailing data".into()))); + for legacy in [false, true] { + assert!(Document::parse(&fixture(&[999], legacy, false), GO).is_err()); + assert!(Document::parse(&fixture(&[14], legacy, false), GO).is_err()); + } + let mut corrupt = raw.clone(); + let block = corrupt.len() - 45; + // Zero chunk size. + corrupt[block + 1..block + 5].fill(0); + assert!(read_all(&corrupt).is_err()); + let mut duplicate = raw.clone(); + // Both row chunks claim index 1. + duplicate[block + 25..block + 29].copy_from_slice(&1u32.to_be_bytes()); + assert!(read_all(&duplicate).is_err()); + let mut version = raw.clone(); + version[block] = 2; + assert!(read_all(&version).is_err()); + let mut zipped = fixture(&[0], false, true); + let last = zipped.len() - 1; + zipped[last] ^= 1; + assert!(Document::parse(&zipped, GO).is_ok() && read_all(&zipped).is_err(), "gzip checksum"); +} + +#[test] +fn reading_can_be_cancelled_during_parsing_and_pixel_decode() { + use std::cell::Cell; + let bytes = fixture(&[0, 1], false, true); + for limit in [0, 10] { + let checks = Cell::new(0); + let keep_going = || { + checks.set(checks.get() + 1); + checks.get() <= limit + }; + assert_eq!(Document::parse(&bytes, &keep_going).err(), Some(Error::Cancelled), "{limit}"); + } + let doc = Document::parse(&bytes, GO).unwrap(); + let seen = Cell::new(0); + let result = doc.read_surfaces(&|| seen.get() == 0, &mut |_, _| seen.set(seen.get() + 1)); + assert_eq!((result, seen.get()), (Err(Error::Cancelled), 1), "cancelled after the first surface"); +} + +proptest::proptest! { + #[test] + fn arbitrary_serialized_data_returns_an_error_without_panicking(data in proptest::collection::vec(proptest::prelude::any::(), 0..4096)) { + let mut bytes = b"PDN3\0\0\0\0\x01".to_vec(); + bytes.extend(data); + let _ = read_all(&bytes); + } + + #[test] + fn mutated_layered_documents_never_panic(edits in proptest::collection::vec((proptest::prelude::any::(), proptest::prelude::any::()), 1..8)) { + let mut bytes = fixture(&[0, 5, 13], false, true); + for (offset, value) in edits { + let index = offset % bytes.len(); + bytes[index] = value; + } + let _ = read_all(&bytes); + } +} + +#[test] +fn malformed_headers_are_errors() { + for bytes in [b"".as_slice(), b"PDN3", b"PDN3\xff\xff\xff", b"PDN4\0\0\0", b"PDN3\0\0\0\0\x02"] { + assert!(matches!(Document::parse(bytes, GO), Err(Error::Invalid(_))), "{bytes:?}"); + } + assert!(!is_pdn(b"PDN4\0\0\0") && !is_pdn(b"PD")); + // The XML header between the signature and the stream is skipped by its 24-bit length. + let plain = fixture(&[0], false, true); + let mut with_header = b"PDN3\x05\0\0".to_vec(); + with_header.extend_from_slice(&plain[7..]); + assert_eq!(read_all(&with_header), read_all(&plain)); +} + +/// A stream of `records` after the header record. +fn stream(records: impl FnOnce(&mut Writer)) -> Vec { + let mut w = Writer(vec![0]); + for n in [1, -1, 1, 0] { + w.int(n); + } + records(&mut w); + w.0 +} + +#[test] +fn serialized_counts_depth_and_references_are_bounded() { + let array = |w: &mut Writer, id: i32, len: i32| { + w.0.push(16); + w.int(id); + w.int(len); + }; + let nested = |depth: i32| { + stream(|w| { + (1..=depth).for_each(|id| array(w, id, 1)); + w.0.extend([10, 11]); + }) + }; + let strings = |total: usize| { + stream(|w| { + for id in 1..=(total >> 20) as i32 { + w.0.push(6); + w.int(id); + w.string(&"s".repeat(1 << 20)); + } + w.0.push(11); + }) + }; + let cases: Vec<(&str, Vec)> = vec![ + ("an array over the item limit", stream(|w| array(w, 1, nrbf::MAX_ITEMS as i32 + 1))), + ("nesting past the depth limit", nested(nrbf::MAX_DEPTH as i32 + 2)), + ("strings past the limit", strings(nrbf::MAX_STRINGS + (1 << 20))), + ( + "a duplicate object id", + stream(|w| { + for _ in 0..2 { + w.0.push(6); + w.int(1); + w.string("duplicate"); + } + w.0.push(11); + }), + ), + ( + "a null run longer than its array", + stream(|w| { + array(w, 1, 1); + w.0.extend([13, 2, 11]); + }), + ), + ( + "an object of an undeclared class", + stream(|w| { + w.0.push(1); + w.int(1); + w.int(999); + }), + ), + ("no root object", stream(|w| w.0.push(11))), + ("an unknown record", stream(|w| w.0.push(99))), + ("a negative count", stream(|w| array(w, 1, -1))), + ]; + for (what, bytes) in cases { + let result = nrbf::parse(&bytes, GO).map(|(g, _)| g.array(&g.root).map(|a| a.len())); + assert!(matches!(result, Err(Error::Invalid(_)) | Ok(Err(Error::Invalid(_)))), "{what}"); + } + // An array holding a reference to itself is a legal graph: resolving stops at the array. + let own = stream(|w| { + array(w, 1, 1); + w.0.push(9); + w.int(1); + w.0.push(11); + }); + let (graph, _) = nrbf::parse(&own, GO).unwrap(); + let items = graph.array(&graph.root).unwrap(); + assert_eq!(graph.array(&items[0]).map(<[Value]>::len), Ok(1)); + assert!(nrbf::parse(&nested(nrbf::MAX_DEPTH as i32), GO).is_ok()); + assert!(nrbf::parse(&strings(nrbf::MAX_STRINGS), GO).is_ok()); +} + +#[test] +fn a_graph_is_read_by_class_field_and_reference() { + use Field::*; + let mut bytes = stream(|w| { + w.object( + 1, + "Root", + &[("count", Int(-7)), ("big", Long(1 << 40)), ("flag", Bool(true)), ("byte", Byte(200)), ("child", Ref(2)), ("text", Ref(3))], + ); + w.object(2, "Child", &[("parent", Ref(1))]); + w.0.push(6); + w.int(3); + w.string("héllo"); + w.0.push(11); + }); + let len = bytes.len(); + bytes.extend_from_slice(b"pixels follow"); + let (graph, used) = nrbf::parse(&bytes, GO).unwrap(); + assert_eq!(used, len, "what follows the stream is left for the caller"); + let root = &graph.root; + assert_eq!(graph.class_name(root).unwrap(), "Root"); + assert_eq!(graph.integer(graph.field(root, "count").unwrap()), Ok(-7)); + assert_eq!(graph.integer(graph.field(root, "big").unwrap()), Ok(1 << 40)); + assert_eq!(graph.integer(graph.field(root, "byte").unwrap()), Ok(200)); + assert_eq!(graph.boolean(graph.field(root, "flag").unwrap()), Ok(true)); + assert_eq!(graph.string(graph.field(root, "text").unwrap()), Ok("héllo")); + let child = graph.field(root, "child").unwrap(); + assert_eq!(graph.class_name(graph.field(child, "parent").unwrap()).unwrap(), "Root", "a cycle through objects is fine"); + assert!(graph.optional_field(root, "missing").unwrap().is_none() && graph.field(root, "missing").is_err()); + assert!(graph.integer(graph.field(root, "flag").unwrap()).is_err() && graph.string(root).is_err() && graph.array(root).is_err()); + assert_eq!(graph.objects_of_class("Child").map(|(id, _)| id).collect::>(), [2]); + match graph.resolve(root).unwrap() { + Value::Object { class, members } => assert_eq!((class.name(), class.member_names().len(), members.len()), ("Root", 6, 6)), + other => panic!("{other:?}"), + } + assert_eq!(nrbf::parse(&bytes, &|| false).err(), Some(Error::Cancelled)); +} + +#[test] +fn truncated_uncompressed_pixels_are_rejected_before_allocating() { + // A raw (version 1) block of 256 MiB chunks that declares 1 GiB of pixels but holds a few bytes. + let mut reader = Reader { bytes: &[1, 0x10, 0, 0, 0, 0, 0, 0, 0], pos: 0 }; + assert_eq!(pixels(&mut reader, 1 << 30, GO).err(), Some(Error::Invalid("uncompressed pixel data is truncated".into()))); +} + +/// The fixture with `edit` applied to the serialized text: same-length replacements only. +fn edited(from: &[u8], to: &[u8]) -> Vec { + assert_eq!(from.len(), to.len()); + let mut bytes = fixture(&[0, 1], false, false); + let at = bytes.windows(from.len()).position(|w| w == from).unwrap_or_else(|| panic!("{from:?} not in the fixture")); + bytes[at..at + from.len()].copy_from_slice(to); + bytes +} + +#[test] +fn documents_this_reader_does_not_cover_are_refused() { + let cases: [(&str, &[u8], &[u8]); 6] = [ + ("root object is not a Paint.NET document", b"PaintDotNet.Document", b"PaintDotNet.Documenx"), + ("only PDN3 bitmap layers are supported", b"PaintDotNet.BitmapLayer\x05", b"PaintDotNet.VectorLayer\x05"), + ("invalid bitmap surface", b"PaintDotNet.Surface", b"PaintDotNet.Surfacf"), + ("unsupported non-deferred or parented pixel memory", b"\x18\0\0\0\0\0\0\0\0\x01", b"\x18\0\0\0\0\0\0\0\x01\x01"), + ("pixel memory length does not match the surface", b"\x18\0\0\0\0\0\0\0\0\x01", b"\x19\0\0\0\0\0\0\0\0\x01"), + ("surface stride is too short for BGRA32 pixels", b"\x02\0\0\0\x02\0\0\0\x0c\0\0\0", b"\x02\0\0\0\x02\0\0\0\x07\0\0\0"), + ]; + for (why, from, to) in cases { + assert_eq!(Document::parse(&edited(from, to), GO).err(), Some(Error::Invalid(why.into()))); + } + // The canvas: 2 × 2 in the fixture's Document object, right after its class header. + let canvas = |w: i32, h: i32| { + let mut bytes = fixture(&[0], false, false); + let at = bytes.windows(8).position(|w| w == [2, 0, 0, 0, 2, 0, 0, 0]).unwrap(); + bytes[at..at + 4].copy_from_slice(&w.to_le_bytes()); + bytes[at + 4..at + 8].copy_from_slice(&h.to_le_bytes()); + Document::parse(&bytes, GO).err() + }; + let limit = Some(Error::Invalid("canvas is empty or exceeds the 1 GiB pixel limit".into())); + assert_eq!(canvas(0, 2), limit); + assert_eq!(canvas(16385, 16384), limit); + assert_eq!(canvas(-1, 2), Some(Error::Invalid("invalid width".into()))); + assert_eq!( + canvas(16384, 16384), + Some(Error::Invalid("layer dimensions do not match the canvas".into())), + "1 GiB exactly passes the canvas check" + ); +} + +#[test] +fn layer_counts_and_names_are_bounded() { + assert!(Document::parse(&fixture(&[], false, false), GO).is_err(), "no layers"); + // 1025 layers: over the limit. The fixture's ids step by ten, so this stays far below i32. + let modes = vec![0; MAX_LAYERS + 1]; + assert_eq!(Document::parse(&fixture(&modes, false, false), GO).err(), Some(Error::Invalid("document must have 1..1024 bitmap layers".into()))); + assert_eq!(Document::parse(&fixture(&modes[..MAX_LAYERS], false, false), GO).map(|d| d.layers.len()), Ok(MAX_LAYERS)); +} From 712e6fa5e39e49448e5b705eefadd663beff8fb5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:31:29 -0500 Subject: [PATCH 20/31] craft-odf: Manifest::parse_strict and Package::manifest_strict (#972) A strict read of the manifest for a reader that refuses a damaged package instead of opening what it can, as GridCraft's ODS reader does today (crates/xlsx/src/ods.rs check_manifest): malformed or unbalanced XML, any document type declaration, undefined entities, content outside the root, a root that is not manifest:manifest by namespace, a file entry without a full-path, or more entries than the reader keeps are errors (Error::Manifest). --- crates/odf/README.md | 2 +- crates/odf/src/lib.rs | 4 ++++ crates/odf/src/manifest.rs | 27 +++++++++++++++++++++++- crates/odf/src/package.rs | 7 +++++++ crates/odf/src/tests.rs | 42 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 80 insertions(+), 2 deletions(-) diff --git a/crates/odf/README.md b/crates/odf/README.md index e91d0d94d..7e82a42ba 100644 --- a/crates/odf/README.md +++ b/crates/odf/README.md @@ -7,7 +7,7 @@ this crate reads and writes that container and nothing inside it. | Module | What it does | | --- | --- | | `Package`, `PackageWriter` | The ZIP container: the `mimetype` file, the manifest, the files, within `Limits`. | -| `Manifest` | `META-INF/manifest.xml`: each file's media type, and how an encrypted file is encrypted. Read by namespace, whatever prefix the file uses. | +| `Manifest` | `META-INF/manifest.xml`: each file's media type, and how an encrypted file is encrypted. Read by namespace, whatever prefix the file uses; `parse_strict` / `Package::manifest_strict` refuse a damaged one instead of reading what they can. | | `crypt` | Password-protected packages: AES-256-CBC, SHA-256 start key, PBKDF2 (HMAC-SHA-1), SHA-256 or SHA-1 checksums. A package in, a package out. | | `flat` | Flat single-file documents (`.fodt`, `.fods`, `.fodp`, `.fodg`): one `office:document` taken apart into a package's files, and a package put together into one. | | `zip_guard` | Refuses a ZIP whose end record declares too many entries, before `zip` reserves room for them. | diff --git a/crates/odf/src/lib.rs b/crates/odf/src/lib.rs index 174eb50cb..ff41b3518 100644 --- a/crates/odf/src/lib.rs +++ b/crates/odf/src/lib.rs @@ -76,6 +76,10 @@ pub enum Error { /// A flat document that is no `office:document`, or not UTF-8. #[error("not a flat OpenDocument file ({0})")] Flat(&'static str), + /// A manifest that [`Manifest::parse_strict`] refuses: malformed XML, a document type + /// declaration, a root that is no `manifest:manifest`, or more entries than it reads. + #[error("the package's manifest is not valid ({0})")] + Manifest(String), } impl Error { diff --git a/crates/odf/src/manifest.rs b/crates/odf/src/manifest.rs index d355c6047..89d4f0631 100644 --- a/crates/odf/src/manifest.rs +++ b/crates/odf/src/manifest.rs @@ -5,7 +5,9 @@ use std::fmt::Write as _; use base64::Engine as _; use base64::engine::general_purpose::STANDARD; -use craft_xml::{Canonical, El, Options, esc_attr}; +use craft_xml::{Canonical, Doctype, El, Options, esc_attr}; + +use crate::Error; /// The manifest's path in a package. pub const MANIFEST_PATH: &str = "META-INF/manifest.xml"; @@ -76,6 +78,29 @@ impl Manifest { Manifest { version: attr(&root, "version").map(str::to_string), entries } } + /// Reads a manifest, refusing one a lenient read would repair: malformed or unbalanced + /// XML, a document type declaration (of any kind), an undefined entity, text or a second + /// element outside the root, a root that is not `manifest` in the manifest namespace, a + /// `file-entry` without a `full-path`, or more than [`MAX_ENTRIES`] entries. For a reader + /// that refuses a damaged package rather than open what it can. + pub fn parse_strict(xml: &[u8]) -> Result { + let bad = |why: String| Error::Manifest(why); + let canonical = Canonical::new(&[("manifest", MANIFEST_NS)]); + let opts = Options { doctype: Doctype::Reject, ..Options::canonical(&canonical).strict() }; + let root = opts.parse(xml).map_err(|e| bad(e.to_string()))?; + if root.name != "manifest:manifest" { + return Err(bad(format!("the root element is {}, not manifest:manifest", root.name))); + } + let mut entries = Vec::new(); + for e in root.elements().filter(|e| e.name == "manifest:file-entry") { + if entries.len() == MAX_ENTRIES { + return Err(bad(format!("more than {MAX_ENTRIES} file entries"))); + } + entries.push(file_entry(e).ok_or_else(|| bad("a file entry has no full-path".into()))?); + } + Ok(Manifest { version: attr(&root, "version").map(str::to_string), entries }) + } + /// The entry for `path`. pub fn entry(&self, path: &str) -> Option<&FileEntry> { self.entries.iter().find(|e| e.path == path) diff --git a/crates/odf/src/package.rs b/crates/odf/src/package.rs index 89977223b..48272e85a 100644 --- a/crates/odf/src/package.rs +++ b/crates/odf/src/package.rs @@ -135,6 +135,13 @@ impl<'a> Package<'a> { self.read(MANIFEST_PATH).ok().flatten().map(|xml| Manifest::parse(&xml)).unwrap_or_default() } + /// The package's manifest read with [`Manifest::parse_strict`]: an error when the package + /// has none, or one a strict read refuses, or it is over a limit. + pub fn manifest_strict(&mut self) -> Result { + let xml = self.read(MANIFEST_PATH)?.ok_or_else(|| Error::Manifest("the package has no manifest".into()))?; + Manifest::parse_strict(&xml) + } + /// Does the manifest say any file is encrypted ([`crate::crypt`])? pub fn is_encrypted(&mut self) -> bool { self.manifest().is_encrypted() diff --git a/crates/odf/src/tests.rs b/crates/odf/src/tests.rs index 0894271ae..7a7d2eaca 100644 --- a/crates/odf/src/tests.rs +++ b/crates/odf/src/tests.rs @@ -68,6 +68,48 @@ fn a_manifest_is_read_by_namespace_and_survives_damage() { assert_eq!(Manifest::parse(many.as_bytes()).entries.len(), crate::manifest::MAX_ENTRIES); } +#[test] +fn a_strict_manifest_refuses_what_a_lenient_one_repairs() { + const NS: &str = r#"xmlns:manifest="urn:oasis:names:tc:opendocument:xmlns:manifest:1.0""#; + let good = format!( + r#""# + ); + let m = Manifest::parse_strict(good.as_bytes()).unwrap(); + assert_eq!((m.version.as_deref(), m.media_type("/")), (Some("1.3"), Some("a/b"))); + // Another prefix for the same namespace is the same manifest. + let other = r#""#; + assert_eq!(Manifest::parse_strict(other.as_bytes()), Ok(m.clone())); + for bad in [ + format!(r#""#), + format!(r#""#), + format!(r#"]>"#), + format!(r#""#), + format!(r#"trailing"#), + format!(r#""#), + format!(r#""#), + format!(r#""#), + r#""#.to_string(), + r#""#.to_string(), + String::new(), + ] { + assert!(matches!(Manifest::parse_strict(bad.as_bytes()), Err(Error::Manifest(_))), "{bad}"); + let _ = Manifest::parse(bad.as_bytes()); + } + // The lenient read takes what it can of the first. + assert_eq!(Manifest::parse(format!(r#""#).as_bytes()).entries.len(), 1); + // Entries: up to the cap, not past it. + let many = |n: usize| format!(r#"{}"#, r#""#.repeat(n)); + assert_eq!(Manifest::parse_strict(many(crate::manifest::MAX_ENTRIES).as_bytes()).map(|m| m.entries.len()), Ok(crate::manifest::MAX_ENTRIES)); + assert!(Manifest::parse_strict(many(crate::manifest::MAX_ENTRIES + 1).as_bytes()).is_err()); + // Through a package: missing and present. + let mut p_bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes())]); + assert!(matches!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().manifest_strict(), Err(Error::Manifest(_)))); + p_bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes()), ("META-INF/manifest.xml", good.as_bytes())]); + assert_eq!(Package::open(&p_bytes, &Limits::DEFAULT).unwrap().manifest_strict(), Ok(m)); + // A written package passes. + assert!(Package::open(&secret(), &Limits::DEFAULT).unwrap().manifest_strict().is_ok_and(|m| m.entries.len() == 4)); +} + #[test] fn a_package_is_held_to_its_limits() { let bytes = zip_of(&[("mimetype", mime::SPREADSHEET.as_bytes()), ("a.xml", &[b'a'; 100]), ("b.xml", &[b'b'; 100])]); From 119415f46f601e20b9bc6b1b4d6979ceda86d88e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 19:32:04 -0500 Subject: [PATCH 21/31] Add craft-xlsx: XLSX and XLSB part layer (workbook, strings, styles, streamed rows) (#976) * Add craft-sheet: the cell positions, values and tables the spreadsheet formats share craft-xls carried its own CellRef, Range, Value, CellError, Visibility, sheet-name quoting, built-in number format table and indexed palette. craft-xlsx and craft-ods need the same ones, so they move to a crate with no dependencies and craft-xls re-exports them: an application gets one Value and one CellRef whichever file it read. Added on the way: A1 parsing (CellRef::parse, Range::parse), Range::count and a1, CellError::parse and the newer error values SpreadsheetML stores as text, builtin_format_id for writers. * Add craft-xlsx: the part layer of XLSX and XLSB workbooks Package::open finds the workbook part through craft-opc and reads the workbook part, shared strings (runs and phonetic readings), the styles tables as indexed tables, and a worksheet's rows one at a time with the rest of the part as a tree. Binary workbooks give the same workbook, strings, rows and merged cells from their records. Every other part stays reachable through Package::opc. From GridCraft crates/xlsx (7e0d96d7): read.rs, sheet_read.rs, styles.rs, xml.rs (string items, xstring escapes), xlsb.rs and its tests, split from GridCraft's document model. * craft-xlsx: XLSB opens up to 4 million cells, records counted once over the workbook Binary workbooks over 1 million cells were refused, a quarter of what .xls allows, and an allowance of 5 million records per part refused ordinary files: Excel writes four records per non-empty row besides the cells, and one BrtCellBlank for every formatted empty cell. Cells are now 4 million over the workbook (also the most shared strings), text 256 MiB, and records 40 million for every part together, as in craft-xls. Ported from storytold/gridcraft#3807. --------- Co-authored-by: kiwi4boy --- Cargo.lock | 10 + Cargo.toml | 1 + README.md | 1 + crates/xlsx/Cargo.toml | 17 + crates/xlsx/README.md | 41 +++ crates/xlsx/src/lib.rs | 148 +++++++++ crates/xlsx/src/package.rs | 275 ++++++++++++++++ crates/xlsx/src/sheet.rs | 387 ++++++++++++++++++++++ crates/xlsx/src/strings.rs | 196 +++++++++++ crates/xlsx/src/styles.rs | 555 +++++++++++++++++++++++++++++++ crates/xlsx/src/testing.rs | 50 +++ crates/xlsx/src/workbook.rs | 117 +++++++ crates/xlsx/src/xlsb.rs | 563 ++++++++++++++++++++++++++++++++ crates/xlsx/tests/common/mod.rs | 57 ++++ crates/xlsx/tests/xlsb.rs | 444 +++++++++++++++++++++++++ crates/xlsx/tests/xlsx.rs | 177 ++++++++++ 16 files changed, 3039 insertions(+) create mode 100644 crates/xlsx/Cargo.toml create mode 100644 crates/xlsx/README.md create mode 100644 crates/xlsx/src/lib.rs create mode 100644 crates/xlsx/src/package.rs create mode 100644 crates/xlsx/src/sheet.rs create mode 100644 crates/xlsx/src/strings.rs create mode 100644 crates/xlsx/src/styles.rs create mode 100644 crates/xlsx/src/testing.rs create mode 100644 crates/xlsx/src/workbook.rs create mode 100644 crates/xlsx/src/xlsb.rs create mode 100644 crates/xlsx/tests/common/mod.rs create mode 100644 crates/xlsx/tests/xlsb.rs create mode 100644 crates/xlsx/tests/xlsx.rs diff --git a/Cargo.lock b/Cargo.lock index 7b227c217..cb35096ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1829,6 +1829,16 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-xlsx" +version = "0.1.0" +dependencies = [ + "craft-opc", + "craft-sheet", + "craft-xml", + "thiserror 2.0.21", +] + [[package]] name = "craft-xml" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 6b5a23328..9f46a4741 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ craft-vcard = { path = "crates/vcard" } craft-vectorcraft-format = { path = "crates/vectorcraft-format" } craft-vectordrawable = { path = "crates/vectordrawable" } craft-xls = { path = "crates/xls" } +craft-xlsx = { path = "crates/xlsx" } craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } diff --git a/README.md b/README.md index 33f37171c..1784d1e47 100644 --- a/README.md +++ b/README.md @@ -290,6 +290,7 @@ craft- = { git = "https://github.com/storytold/craft-libs", rev = " Result<(), craft_xlsx::Error> { +//! # let bytes = craft_xlsx::testing::workbook(&[("Prices", r#"02.5"#)], &["Tea"]); +//! let mut package = craft_xlsx::Package::open(&bytes, &craft_xlsx::Limits::default())?; +//! let workbook = package.workbook()?; +//! let strings = package.shared_strings()?; +//! let mut values = Vec::new(); +//! package.sheet(&workbook.sheets[0], |row| { +//! values.extend(row.cells.iter().map(|c| c.value.resolve(&strings, workbook.date_1904))); +//! Ok(()) +//! })?; +//! assert_eq!(values, [craft_xlsx::Value::text("Tea"), craft_xlsx::Value::Number(2.5)]); +//! # Ok(()) +//! # } +//! ``` + +mod package; +mod sheet; +mod strings; +mod styles; +#[doc(hidden)] +pub mod testing; +mod workbook; +mod xlsb; + +pub use craft_sheet::{ + BorderStyle, CellError, CellRef, DEFAULT_PALETTE, HAlign, MAX_COLS, MAX_ROWS, Pattern, Range, Script, Underline, VAlign, Value, Visibility, + builtin_format, builtin_format_id, quote_sheet, sheet_needs_quotes, +}; +pub use package::{Package, is_xlsb}; +pub use sheet::{Cell, CellFormula, CellValue, Column, FormulaKind, Row, SheetPart, iso_date_serial}; +pub use strings::{PhoneticProps, PhoneticRun, StringItem, TextRun, decode_xstring, encode_xstring, rich_text}; +pub use styles::{ + Alignment, Border, BorderEdge, CellStyle, Color, ColorSource, Dxf, Fill, Font, FontScheme, GradientFill, PatternFill, Protection, Styles, Xf, +}; +pub use workbook::{DefinedName, SheetEntry, SheetKind, Workbook}; + +/// Why a workbook could not be read. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bytes are not a ZIP package, or an entry is damaged. + #[error("the file is not a valid zip package: {0}")] + Zip(String), + /// A part that must be read holds XML that cannot be. + #[error("the file contains invalid XML: {0}")] + Xml(String), + /// The package is not a workbook, or a binary part is damaged. + #[error("the file is not a valid spreadsheet: {0}")] + Invalid(String), + /// More than [`Limits`] or a fixed cap allows. + #[error("the file is too large: {0}")] + TooLarge(String), +} + +impl From for Error { + fn from(e: craft_opc::Error) -> Self { + match e { + e if e.is_limit() => Error::TooLarge(e.to_string()), + craft_opc::Error::Zip(m) => Error::Zip(m), + craft_opc::Error::Xml { .. } => Error::Xml(e.to_string()), + e => Error::Invalid(e.to_string()), + } + } +} + +impl From for Error { + fn from(e: craft_xml::Error) -> Self { + if e.is_limit() { Error::TooLarge(e.to_string()) } else { Error::Xml(e.to_string()) } + } +} + +/// What a read refuses to exceed. The defaults suit a spreadsheet application; a caller that +/// only wants a table of values can lower them. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// The ZIP package: entries, bytes per part and bytes in all. + pub package: craft_opc::Limits, + /// One XML part. Rows and shared strings are streamed, so `max_nodes` bounds one row or one + /// string and what is left of a part, not a sheet's length. + pub xml: craft_xml::Limits, + /// Records read from a binary workbook, every part together. Records are skipped or decoded + /// without being kept, so this bounds parse time, not memory. Excel writes four records per + /// row besides the cells, and one per formatted empty cell, so a sheet needs many more + /// records than cells. + pub binary_records: usize, + /// Cells with a value read from a binary workbook, over all its sheets; also the most + /// shared-string items. + pub binary_cells: usize, + /// Bytes of decoded text (shared and inline strings) in a binary workbook, over all its parts. + pub binary_text: usize, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + package: craft_opc::Limits::new(200_000, 512 << 20, 1 << 30), + xml: craft_xml::Limits { max_input: 512 << 20, ..craft_xml::Limits::DEFAULT }, + // The limits of `craft-xls`. + binary_records: 40_000_000, + binary_cells: 4_000_000, + binary_text: 256 << 20, + } + } +} + +/// What a read skipped or repaired, each distinct message once and at most 1000 of them. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(crate) struct Warnings(pub(crate) Vec); + +impl Warnings { + pub(crate) fn warn(&mut self, m: impl Into) { + let m = m.into(); + if self.0.len() < 1000 && !self.0.contains(&m) { + self.0.push(m); + } + } +} diff --git a/crates/xlsx/src/package.rs b/crates/xlsx/src/package.rs new file mode 100644 index 000000000..dc772e46e --- /dev/null +++ b/crates/xlsx/src/package.rs @@ -0,0 +1,275 @@ +//! The OPC package of a workbook and the parts this crate reads from it. + +use craft_opc::{ContentTypes, LazyPackage, Rels}; +use craft_xml::{El, Options, UnknownEntity}; + +use crate::sheet::{Row, RowReader, SheetPart}; +use crate::strings::StringItem; +use crate::styles::Styles; +use crate::workbook::{SheetEntry, Workbook}; +use crate::{Error, Limits, Warnings, xlsb}; + +/// Most bytes of `[Content_Types].xml` read from a binary workbook. +const MAX_CONTENT_TYPES: u64 = 1 << 20; + +/// An open workbook package. Parts are inflated when they are asked for, against the +/// package's [`Limits`]. +pub struct Package<'a> { + opc: LazyPackage<'a>, + limits: Limits, + warnings: Warnings, + workbook_part: String, + /// Content types, for a binary workbook (which checks every part it reads against them). + binary: Option, + workbook_rels: Option, + budget: xlsb::Budget, + /// Shared strings in a binary workbook, once counted: a cell's index is checked against it. + binary_strings: Option, +} + +/// Is this a binary workbook (`.xlsb`): a package whose one workbook relationship points at a +/// part of the XLSB workbook content type? +pub fn is_xlsb(bytes: &[u8]) -> bool { + Package::open(bytes, &Limits::default()).is_ok_and(|p| p.is_binary()) +} + +impl<'a> Package<'a> { + /// Opens the package and finds the workbook part. Nothing else is read yet. + pub fn open(bytes: &'a [u8], limits: &Limits) -> Result, Error> { + let mut opc = LazyPackage::open(bytes, &limits.package)?; + let root = opc.rels("")?; + let documents: Vec<_> = root.all("officeDocument").collect(); + let mut binary = None; + let workbook_part = match documents.first() { + Some(rel) if !rel.external && rel.target.to_ascii_lowercase().ends_with(".bin") => { + let types = opc.read_limited("[Content_Types].xml", MAX_CONTENT_TYPES)?.map(|b| ContentTypes::parse(&b)).unwrap_or_default(); + if documents.len() != 1 { + return Err(xlsb::invalid("expected one workbook relationship")); + } + xlsb::check_type(&types, &rel.target, xlsb::MAIN)?; + binary = Some(types); + rel.target.clone() + } + Some(rel) if !rel.external => rel.target.clone(), + _ => "xl/workbook.xml".to_string(), + }; + Ok(Package { + opc, + limits: *limits, + warnings: Warnings::default(), + workbook_part, + binary, + workbook_rels: None, + budget: xlsb::Budget::default(), + binary_strings: None, + }) + } + + /// A binary workbook (`.xlsb`). + pub fn is_binary(&self) -> bool { + self.binary.is_some() + } + + /// The workbook part's name. + pub fn workbook_part(&self) -> &str { + &self.workbook_part + } + + /// What reading skipped or repaired so far, each message once. + pub fn warnings(&self) -> &[String] { + &self.warnings.0 + } + + /// Adds a warning (a caller reading further parts keeps one list). + pub fn warn(&mut self, m: impl Into) { + self.warnings.warn(m); + } + + /// The package underneath, for the parts this crate does not read. + pub fn opc(&mut self) -> &mut LazyPackage<'a> { + &mut self.opc + } + + /// The XML options parts are read with: local names, an undefined entity kept as written, + /// [`Limits::xml`]. + pub fn xml_options(&self) -> Options<'static> { + Options { unknown_entity: UnknownEntity::Literal, ..Options::local() }.with_limits(self.limits.xml) + } + + /// Part `part` as XML; `Ok(None)` when there is no such part. + pub fn read_xml(&mut self, part: &str) -> Result, Error> { + let options = self.xml_options(); + Ok(self.opc.read_xml(part, &options)?) + } + + /// A part the workbook can do without: missing or unreadable is a warning and `None`; + /// only a limit is an error. + pub fn optional_xml(&mut self, part: &str) -> Result, Error> { + match self.read_xml(part) { + Ok(Some(e)) => Ok(Some(e)), + Ok(None) => { + self.warn(format!("missing part {part}")); + Ok(None) + } + Err(Error::TooLarge(m)) => Err(Error::TooLarge(m)), + Err(e) => { + self.warn(format!("skipped {part}: {e}")); + Ok(None) + } + } + } + + /// The relationships of `part`. + pub fn rels(&mut self, part: &str) -> Result { + Ok(self.opc.rels(part)?) + } + + /// The workbook part's relationships. + pub fn workbook_rels(&mut self) -> Result { + if let Some(r) = &self.workbook_rels { + return Ok(r.clone()); + } + let rels = self.opc.rels(&self.workbook_part)?; + self.workbook_rels = Some(rels.clone()); + Ok(rels) + } + + /// The workbook part. + pub fn workbook(&mut self) -> Result { + if self.binary.is_some() { + let bytes = self.required(&self.workbook_part.clone())?; + let rels = self.workbook_rels()?; + return xlsb::workbook(&bytes, &rels, &self.limits, &self.budget, &mut self.warnings); + } + let options = self.xml_options(); + let root = match self.opc.read_xml(&self.workbook_part, &options)? { + Some(root) => root, + None if self.workbook_part != "xl/workbook.xml" && self.opc.has("xl/workbook.xml") => { + self.workbook_part = "xl/workbook.xml".into(); + self.workbook_rels = None; + self.opc.read_xml("xl/workbook.xml", &options)?.ok_or_else(|| Error::Invalid("no workbook part".into()))? + } + None if self.opc.has("content.xml") => return Err(Error::Invalid("this is an OpenDocument file, not XLSX".into())), + None => return Err(Error::Invalid("no workbook part".into())), + }; + if root.name != "workbook" { + return Err(Error::Invalid(format!("unexpected root element <{}> in workbook part", root.name))); + } + let rels = self.workbook_rels()?; + Ok(Workbook::from_xml(root, &rels)) + } + + /// The shared strings, empty when the workbook has none. + pub fn shared_strings(&mut self) -> Result, Error> { + let rels = self.workbook_rels()?; + if self.binary.is_some() { + let found: Vec<_> = rels.all("sharedStrings").collect(); + let Some(rel) = found.first() else { + self.binary_strings = Some(0); + return Ok(Vec::new()); + }; + if found.len() > 1 { + return Err(xlsb::invalid("multiple shared-string relationships")); + } + if rel.external { + return Err(xlsb::invalid("external shared-string relationship")); + } + if let Some(types) = &self.binary { + xlsb::check_type(types, &rel.target, xlsb::STRINGS)?; + } + let bytes = self.required(&rel.target)?; + let strings = xlsb::shared_strings(&bytes, &self.limits, &self.budget, &mut self.warnings)?; + self.binary_strings = Some(strings.len()); + return Ok(strings); + } + let Some(rel) = rels.first("sharedStrings").filter(|r| !r.external) else { return Ok(Vec::new()) }; + let Some(bytes) = self.opc.read(&rel.target)? else { + self.warn(format!("missing part {}", rel.target)); + return Ok(Vec::new()); + }; + let mut items = Vec::new(); + let (_, err) = self.xml_options().parse_streaming_partial(&bytes, &["si"], |si| { + items.push(StringItem::read(&si)); + Ok::<(), craft_xml::Error>(()) + }); + if let Some(e) = err { + self.warn(format!("shared strings are damaged ({e}); kept {} strings", items.len())); + } + let dropped: usize = items.iter().map(|s| s.dropped_phonetic_runs).sum(); + if dropped > 0 { + self.warn(format!("shared strings: {dropped} phonetic runs were dropped (bad sb/eb or no reading)")); + } + Ok(items) + } + + /// The styles part, `None` when the workbook has none (or is binary: its styles are not read). + pub fn styles(&mut self) -> Result, Error> { + if self.binary.is_some() { + return Ok(None); + } + let rels = self.workbook_rels()?; + let Some(rel) = rels.first("styles").filter(|r| !r.external) else { + self.warn("no styles part"); + return Ok(None); + }; + Ok(self.optional_xml(&rel.target.clone())?.map(Styles::read)) + } + + /// Reads a worksheet, handing each row to `each` in file order; an error from `each` stops the + /// read and is returned. A damaged worksheet gives the rows before the damage and says so + /// in [`SheetPart::damaged`] and in the warnings. + pub fn sheet(&mut self, entry: &SheetEntry, mut each: impl FnMut(Row) -> Result<(), Error>) -> Result { + let part = entry.part.clone().ok_or_else(|| Error::Invalid(format!("sheet \"{}\" has no part", entry.name)))?; + if self.binary.is_some() { + if let Some(types) = &self.binary { + xlsb::check_type(types, &part, xlsb::WORKSHEET)?; + } + let strings = match self.binary_strings { + Some(n) => n, + None => self.shared_strings()?.len(), + }; + let bytes = self.required(&part)?; + let mut sheet = xlsb::worksheet(&bytes, strings, &self.limits, &self.budget, &mut self.warnings, &mut each)?; + sheet.part = part; + return Ok(sheet); + } + let bytes = self.required(&part)?; + let rels = self.opc.rels(&part)?; + let mut rows = RowReader::default(); + let (root, err) = self.xml_options().parse_streaming_partial(&bytes, &["row"], |e| match rows.row(&e) { + Some(row) => each(row).map_err(Stop::Caller), + None => Ok(()), + }); + let damaged = match err { + Some(Stop::Caller(e)) => return Err(e), + Some(Stop::Xml(e)) => { + self.warn(format!("sheet \"{}\" is damaged ({e}); kept {} rows", entry.name, rows.rows)); + Some(e.to_string()) + } + None => None, + }; + if rows.dropped > 0 { + self.warn(format!("sheet \"{}\": {} cells or rows outside the 1,048,576 × 16,384 grid were dropped", entry.name, rows.dropped)); + } + let mut sheet = + SheetPart { part, root: root.unwrap_or_default(), rels, rows: rows.rows, dropped: rows.dropped, damaged, ..SheetPart::default() }; + sheet.lift(); + Ok(sheet) + } + + fn required(&mut self, part: &str) -> Result, Error> { + self.opc.read(part)?.ok_or_else(|| Error::Invalid(format!("missing part {part}"))) + } +} + +/// Why a streamed worksheet stopped: the XML, or the caller. +enum Stop { + Xml(craft_xml::Error), + Caller(Error), +} + +impl From for Stop { + fn from(e: craft_xml::Error) -> Self { + Stop::Xml(e) + } +} diff --git a/crates/xlsx/src/sheet.rs b/crates/xlsx/src/sheet.rs new file mode 100644 index 000000000..5375436e8 --- /dev/null +++ b/crates/xlsx/src/sheet.rs @@ -0,0 +1,387 @@ +//! A worksheet part (`CT_Worksheet`, ECMA-376 Part 1, 18.3): rows and cells as a stream, the +//! rest of the part as a tree. + +use std::sync::Arc; + +use craft_opc::Rels; +use craft_sheet::{CellError, CellRef, MAX_COLS, MAX_ROWS, Range, Value}; +use craft_xml::El; + +use crate::strings::{StringItem, decode_xstring}; + +/// What a cell's `` holds, by its type (`t`, 18.18.11). +#[derive(Clone, Debug, Default, PartialEq)] +pub enum CellValue { + #[default] + Empty, + Number(f64), + /// `t="s"`: an index into the shared strings. + Shared(u32), + /// `t="str"` (a formula's text result), `_xHHHH_` escapes decoded. + Text(Arc), + /// `t="inlineStr"`: the `` item, or the `` text when there is no ``. + Inline(StringItem), + Bool(bool), + /// `t="e"`. An error text no [`CellError`] has reads as `#VALUE!`. + Error(CellError), + /// `t="d"`: an ISO 8601 date, time or both, as written. See [`iso_date_serial`]. + Date(String), + /// A `` that does not read as its type says (a number that is not one, a shared-string + /// index that is not a number). Kept for the caller to report. + Invalid(String), +} + +impl CellValue { + /// The value a cell shows: shared strings looked up, dates as serial numbers in the + /// workbook's date system. A missing shared string, an unreadable date or an invalid value + /// is empty. + pub fn resolve(&self, strings: &[StringItem], date_1904: bool) -> Value { + match self { + CellValue::Empty | CellValue::Invalid(_) => Value::Empty, + CellValue::Number(n) => Value::number(*n), + CellValue::Shared(i) => strings.get(*i as usize).map_or(Value::Empty, |s| Value::Text(s.text.clone())), + CellValue::Text(t) => Value::Text(t.clone()), + CellValue::Inline(s) => Value::Text(s.text.clone()), + CellValue::Bool(b) => Value::Bool(*b), + CellValue::Error(e) => Value::Error(*e), + CellValue::Date(d) => iso_date_serial(d, date_1904).map_or(Value::Empty, Value::number), + } + } +} + +/// ``'s `t` (18.18.6). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum FormulaKind { + #[default] + Normal, + Shared, + Array, + /// A what-if data table: `r1`, `r2` and its flags are in [`CellFormula::attrs`]. + DataTable, +} + +/// A cell's ``. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct CellFormula { + pub kind: FormulaKind, + /// The formula as the file spells it (`_xlfn.` prefixes and all), trimmed. Empty for a + /// cell that shares the formula of its group's first cell. + pub text: String, + /// `si`: the shared formula group. + pub shared_index: Option, + /// `ref`: the range of an array formula or of a shared formula group. + pub range: Option, + /// Every attribute as written. + pub attrs: Vec<(String, String)>, +} + +/// One ``. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Cell { + pub at: CellRef, + /// `s`: an index into [`crate::Styles::cell_xfs`], 0 when absent. + pub style: u32, + pub value: CellValue, + pub formula: Option, + /// `vm`: value metadata (a rich value such as a picture in the cell), 1-based. + pub value_metadata: Option, + /// `cm`: cell metadata (a dynamic array formula), 1-based. + pub cell_metadata: Option, + /// `ph`: show the phonetic reading. + pub phonetic: bool, +} + +/// One `` and its cells. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Row { + /// Zero-based. + pub index: u32, + /// `ht`, points. + pub height: Option, + pub custom_height: bool, + pub hidden: bool, + pub outline_level: u8, + pub collapsed: bool, + /// `s` when `customFormat` is set: the format of the row's empty cells. + pub style: Option, + pub cells: Vec, +} + +/// One ``: columns `first..=last` (zero-based). +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Column { + pub first: u32, + pub last: u32, + /// Characters of the default font's maximum digit width, padding included. + pub width: Option, + pub custom_width: bool, + pub best_fit: bool, + pub hidden: bool, + pub outline_level: u8, + pub collapsed: bool, + pub style: Option, +} + +/// A worksheet after its rows were handed out: the rest of the part and what was lifted out of it. +#[derive(Clone, Debug, Default)] +pub struct SheetPart { + pub part: String, + /// The worksheet element without its rows: properties, views, columns, merged cells, + /// conditional formats, validation, hyperlinks, page setup, extensions. Empty for a binary + /// worksheet. + pub root: El, + pub rels: Rels, + pub columns: Vec, + pub merges: Vec, + /// Rows read. + pub rows: u64, + /// Rows and cells dropped because their position was outside the grid or unreadable. + pub dropped: u64, + /// Why reading stopped early, when the part is damaged; the rows before the damage were + /// handed out. + pub damaged: Option, +} + +impl SheetPart { + pub(crate) fn lift(&mut self) { + if let Some(cols) = self.root.child("cols") { + for c in cols.children("col") { + let (Some(min), Some(max)) = (c.attr_u32("min"), c.attr_u32("max")) else { continue }; + if min == 0 || min > MAX_COLS || max < min { + continue; + } + self.columns.push(Column { + first: min - 1, + last: max.min(MAX_COLS) - 1, + width: c.attr_f64("width"), + custom_width: c.flag("customWidth", false), + best_fit: c.flag("bestFit", false), + hidden: c.flag("hidden", false), + outline_level: c.attr_u32("outlineLevel").unwrap_or(0).min(7) as u8, + collapsed: c.flag("collapsed", false), + style: c.attr_u32("style"), + }); + } + } + if let Some(m) = self.root.child("mergeCells") { + self.merges.extend(m.children("mergeCell").filter_map(|m| m.attr("ref").and_then(Range::parse)).filter(|r| !r.is_single())); + } + } +} + +/// Reads rows one at a time, remembering where the last one ended. +#[derive(Default)] +pub(crate) struct RowReader { + next_row: u32, + pub(crate) rows: u64, + pub(crate) dropped: u64, +} + +impl RowReader { + /// `None` for a row outside the grid (counted in `dropped`). + pub(crate) fn row(&mut self, e: &El) -> Option { + self.rows += 1; + let index = match e.attr_u32("r") { + Some(r) if (1..=MAX_ROWS).contains(&r) => r - 1, + Some(_) => { + self.dropped += 1; + return None; + } + None if self.next_row < MAX_ROWS => self.next_row, + None => { + self.dropped += 1; + return None; + } + }; + self.next_row = index + 1; + let mut row = Row { + index, + height: e.attr_f64("ht"), + custom_height: e.flag("customHeight", false), + hidden: e.flag("hidden", false), + outline_level: e.attr_u32("outlineLevel").unwrap_or(0).min(7) as u8, + collapsed: e.flag("collapsed", false), + style: if e.flag("customFormat", false) { e.attr_u32("s") } else { None }, + cells: Vec::new(), + }; + let mut next_col = 0u32; + for c in e.children("c") { + let at = match c.attr("r") { + Some(a) => match CellRef::parse(a.trim()) { + Some(p) => p, + None => { + self.dropped += 1; + continue; + } + }, + None if next_col < MAX_COLS => CellRef::new(index, next_col), + None => { + self.dropped += 1; + continue; + } + }; + next_col = at.col + 1; + row.cells.push(cell(c, at)); + } + Some(row) + } +} + +fn cell(c: &El, at: CellRef) -> Cell { + let v = c.child("v").map(|v| v.text()); + let v = v.as_deref(); + let value = match c.attr("t").unwrap_or("n") { + "s" => match v.map(str::trim) { + Some(t) => t.parse().map_or_else(|_| CellValue::Invalid(t.to_string()), CellValue::Shared), + None => CellValue::Empty, + }, + "str" => v.map_or(CellValue::Empty, |v| CellValue::Text(decode_xstring(v).into())), + "inlineStr" => match c.child("is") { + Some(is) => CellValue::Inline(StringItem::read(is)), + None => v.map_or(CellValue::Empty, |v| CellValue::Text(decode_xstring(v).into())), + }, + "b" => v.map_or(CellValue::Empty, |v| CellValue::Bool(matches!(v.trim(), "1" | "true" | "TRUE"))), + "e" => v.map_or(CellValue::Empty, |v| CellValue::Error(CellError::parse(v.trim()).unwrap_or(CellError::Value))), + "d" => match v.map(str::trim).filter(|v| !v.is_empty()) { + Some(d) => CellValue::Date(d.to_string()), + None => CellValue::Empty, + }, + _ => match v.map(str::trim).filter(|v| !v.is_empty()) { + Some(t) => match t.parse::() { + Ok(n) if n.is_finite() => CellValue::Number(if n == 0.0 { 0.0 } else { n }), + _ => CellValue::Invalid(t.to_string()), + }, + None => CellValue::Empty, + }, + }; + let formula = c.child("f").map(|f| CellFormula { + kind: match f.attr("t") { + Some("shared") => FormulaKind::Shared, + Some("array") => FormulaKind::Array, + Some("dataTable") => FormulaKind::DataTable, + _ => FormulaKind::Normal, + }, + text: f.text().trim().to_string(), + shared_index: f.attr_u32("si"), + range: f.attr("ref").and_then(Range::parse), + attrs: f.attrs.clone(), + }); + Cell { + at, + style: c.attr_u32("s").unwrap_or(0), + value, + formula, + value_metadata: c.attr_u32("vm"), + cell_metadata: c.attr_u32("cm"), + phonetic: c.flag("ph", false), + } +} + +/// Days from 1970-01-01 to a proleptic Gregorian date (the civil-from-days inverse). +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let mp = (m + 9) % 12; + let doy = (153 * mp + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146_097 + doe - 719_468 +} + +/// The serial number of a `t="d"` value: `2024-02-29`, `2024-02-29T13:45:30`, `13:45` (a time +/// alone), a trailing `Z` or `+hh:mm` ignored. Years 1900 (1904 in the 1904 system) to 9999; +/// a day past the month's end counts on into the next month, as Excel's `DATE` does. In the +/// 1900 system, dates before 1 March 1900 are one lower than the day count, for the 29 February +/// 1900 that Excel counts. +pub fn iso_date_serial(s: &str, date_1904: bool) -> Option { + let s = s.trim().trim_end_matches('Z'); + let (date, time) = match s.split_once('T') { + Some((d, t)) => (d, Some(t)), + None if s.contains(':') => ("", Some(s)), + None => (s, None), + }; + let mut serial = 0.0; + if !date.is_empty() { + let mut it = date.split('-'); + let y: i64 = it.next()?.parse().ok()?; + let m: i64 = it.next()?.parse().ok()?; + let d: i64 = it.next()?.parse().ok()?; + if it.next().is_some() || !(1..=12).contains(&m) || !(1..=31).contains(&d) || y > 9999 || y < if date_1904 { 1904 } else { 1900 } { + return None; + } + let first = days_from_civil(y, m, 1); + let first = + if date_1904 { first - days_from_civil(1904, 1, 1) } else { first - days_from_civil(1899, 12, 30) - i64::from(y == 1900 && m <= 2) }; + let n = first + d - 1; + let max = days_from_civil(9999, 12, 31) - if date_1904 { days_from_civil(1904, 1, 1) } else { days_from_civil(1899, 12, 30) }; + if !(0..=max).contains(&n) { + return None; + } + serial = n as f64; + } + if let Some(t) = time { + let t = t.split('+').next().unwrap_or(t); + let mut it = t.split(':'); + let h: f64 = it.next()?.parse().ok()?; + let mi: f64 = it.next().unwrap_or("0").parse().ok()?; + let se: f64 = it.next().unwrap_or("0").parse().ok()?; + if !(0.0..24.0).contains(&h) || !(0.0..60.0).contains(&mi) || !(0.0..61.0).contains(&se) { + return None; + } + serial += (h * 3600.0 + mi * 60.0 + se) / 86400.0; + } + Some(serial) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn iso_dates() { + assert_eq!(iso_date_serial("1900-01-01", false), Some(1.0)); + assert_eq!(iso_date_serial("1900-02-28", false), Some(59.0)); + assert_eq!(iso_date_serial("1900-03-01", false), Some(61.0)); + assert_eq!(iso_date_serial("2024-02-29T12:00:00Z", false), Some(45351.5)); + assert_eq!(iso_date_serial("1904-01-02", true), Some(1.0)); + assert_eq!(iso_date_serial("06:00", false), Some(0.25)); + assert_eq!(iso_date_serial("2023-02-31", false), iso_date_serial("2023-03-03", false)); + assert_eq!(iso_date_serial("9999-12-31", false), Some(2_958_465.0)); + for bad in ["1899-12-31", "10000-01-01", "2024-13-01", "2024-01-00", "x", "2024-01-01T25:00", "2024-01-01-01"] { + assert_eq!(iso_date_serial(bad, false), None, "{bad}"); + } + assert_eq!(iso_date_serial("1903-12-31", true), None); + } + + #[test] + fn rows_and_cells() { + let xml = r#"7A1*24 + 1#SPILL!#WHAT1in_x000A_lineabca_x0009_2024-01-01"#; + let mut r = RowReader::default(); + let row = r.row(&craft_xml::Options::local().parse(xml.as_bytes()).unwrap_or_default()).unwrap_or_default(); + assert_eq!((row.index, row.height, row.custom_height, row.style, row.outline_level), (2, Some(20.0), true, Some(4), 7)); + assert_eq!(r.dropped, 1); + let at: Vec = row.cells.iter().map(|c| c.at.a1()).collect(); + assert_eq!(at, ["B3", "C3", "E3", "F3", "G3", "H3", "I3", "J3", "K3", "L3"]); + let values: Vec<&CellValue> = row.cells.iter().map(|c| &c.value).collect(); + assert_eq!(values[0], &CellValue::Shared(7)); + assert_eq!(values[2], &CellValue::Error(CellError::Spill)); + assert_eq!(values[3], &CellValue::Error(CellError::Value)); + assert_eq!(values[4], &CellValue::Bool(true)); + assert!(matches!(values[5], CellValue::Inline(s) if &*s.text == "in\nline")); + assert_eq!(values[6], &CellValue::Invalid("abc".into())); + assert_eq!(values[7], &CellValue::Text("a\t".into())); + assert_eq!(values[8].resolve(&[], false), Value::Number(45292.0)); + let f = row.cells[1].formula.clone().unwrap_or_default(); + assert_eq!((f.kind, f.text.as_str(), f.shared_index, f.range.map(|r| r.a1())), (FormulaKind::Shared, "A1*2", Some(0), Some("C3:C9".into()))); + let last = &row.cells[9]; + assert_eq!((last.value_metadata, last.cell_metadata, last.phonetic, last.style), (Some(1), Some(2), true, 0)); + assert_eq!(row.cells[0].value.resolve(&[], false), Value::Empty); + // Without `r`, the next row follows the last one. + let next = r.row(&craft_xml::El::new("row")).map(|r| r.index); + assert_eq!(next, Some(3)); + assert!(r.row(&craft_xml::El::new("row").with_attr("r", "1048577")).is_none()); + assert!(r.row(&craft_xml::El::new("row").with_attr("r", "0")).is_none()); + assert_eq!((r.rows, r.dropped), (4, 3)); + } +} diff --git a/crates/xlsx/src/strings.rs b/crates/xlsx/src/strings.rs new file mode 100644 index 000000000..641975032 --- /dev/null +++ b/crates/xlsx/src/strings.rs @@ -0,0 +1,196 @@ +//! String items (`CT_Rst`, ECMA-376 Part 1, 18.4): shared strings, inline strings and comment +//! text, with the `_xHHHH_` escapes of `ST_Xstring` (22.9.2.19). + +use std::sync::Arc; + +use craft_xml::El; + +use crate::styles::Font; + +/// A formatted run of a string item (``). +#[derive(Clone, Debug, PartialEq)] +pub struct TextRun { + pub text: String, + /// ``, `None` when the run has none. + pub font: Option, +} + +/// A phonetic reading over part of a string's text (``, 18.4.6). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PhoneticRun { + /// Offsets into the base text in UTF-16 code units, `start < end`, both within the text. + pub start: u32, + pub end: u32, + pub text: String, +} + +/// `` (18.4.7). `kind` and `alignment` are kept only when they are values the +/// schema defines (`ST_PhoneticType`, `ST_PhoneticAlignment`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PhoneticProps { + pub font_id: u32, + pub kind: Option, + pub alignment: Option, +} + +/// One string item: a shared string (``), an inline string (``) or comment text. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct StringItem { + /// The whole text, runs joined, phonetic readings left out. + pub text: Arc, + /// The formatted runs, empty for a plain string. + pub runs: Vec, + pub phonetic_runs: Vec, + pub phonetic: Option, + /// `` elements that were dropped: no offsets, `sb >= eb`, past the end of the text, or + /// no reading. + pub dropped_phonetic_runs: usize, +} + +impl StringItem { + pub(crate) fn read(e: &El) -> StringItem { + let text = rich_text(e); + let runs = e + .children("r") + .map(|r| TextRun { text: decode_xstring(&r.children("t").map(|t| t.text()).collect::()), font: r.child("rPr").map(Font::read) }) + .collect(); + // The length in UTF-16 code units: the widest reading of "the total length of the base + // text", so a run is only dropped when it cannot refer to the text at all. + let len = u32::try_from(text.encode_utf16().count()).unwrap_or(u32::MAX); + let mut phonetic_runs = Vec::new(); + let mut seen = 0usize; + for r in e.children("rPh") { + seen += 1; + let (Some(sb), Some(eb)) = (r.attr_u32("sb"), r.attr_u32("eb")) else { continue }; + let (start, end) = (sb.min(len), eb.min(len)); + let reading = r.child("t").map(|t| decode_xstring(&t.text())).unwrap_or_default(); + if start < end && !reading.is_empty() { + phonetic_runs.push(PhoneticRun { start, end, text: reading }); + } + } + const KINDS: &[&str] = &["halfwidthKatakana", "fullwidthKatakana", "Hiragana", "noConversion"]; + const ALIGNMENTS: &[&str] = &["noControl", "left", "center", "distributed"]; + let phonetic = e.child("phoneticPr").map(|p| PhoneticProps { + font_id: p.attr_u32("fontId").unwrap_or(0), + kind: p.attr("type").map(str::trim).filter(|v| KINDS.contains(v)).map(str::to_string), + alignment: p.attr("alignment").map(str::trim).filter(|v| ALIGNMENTS.contains(v)).map(str::to_string), + }); + StringItem { dropped_phonetic_runs: seen - phonetic_runs.len(), text: Arc::from(text), runs, phonetic_runs, phonetic } + } +} + +/// The text of a string item: `` and the `` of each ``, in order, with `_xHHHH_` +/// escapes decoded. Phonetic runs are not part of the text. +pub fn rich_text(e: &El) -> String { + let mut s = String::new(); + for c in e.elements() { + match c.name.as_str() { + "t" => s.push_str(&c.text()), + "r" => c.children("t").for_each(|t| s.push_str(&t.text())), + _ => {} + } + } + decode_xstring(&s) +} + +fn is_escape(chars: &[char], i: usize) -> bool { + chars.get(i) == Some(&'_') + && chars.get(i + 1) == Some(&'x') + && chars.get(i + 6) == Some(&'_') + && (i + 2..i + 6).all(|k| chars.get(k).is_some_and(char::is_ascii_hexdigit)) +} + +fn escape_unit(chars: &[char], i: usize) -> Option { + let hex: String = chars.get(i + 2..i + 6)?.iter().collect(); + u32::from_str_radix(&hex, 16).ok() +} + +/// Text for a `` or ``: characters XML cannot carry and control characters other than tab +/// and line feed become `_xHHHH_`, and a literal `_xHHHH_` is protected with `_x005F_`. The +/// result still needs XML escaping. +pub fn encode_xstring(s: &str) -> String { + let chars: Vec = s.chars().collect(); + let mut o = String::with_capacity(s.len()); + for (i, &c) in chars.iter().enumerate() { + if is_escape(&chars, i) { + o.push_str("_x005F_"); + } else if !craft_xml::is_xml_char(c) || (c.is_control() && !matches!(c, '\t' | '\n')) { + o.push_str(&format!("_x{:04X}_", c as u32 & 0xFFFF)); + } else { + o.push(c); + } + } + o +} + +/// The inverse of [`encode_xstring`]. Two escapes that form a UTF-16 surrogate pair are one +/// character; an escape that is no character on its own stays as written. +pub fn decode_xstring(s: &str) -> String { + if !s.contains("_x") { + return s.to_string(); + } + let chars: Vec = s.chars().collect(); + let mut o = String::with_capacity(s.len()); + let mut i = 0; + while let Some(&c) = chars.get(i) { + if is_escape(&chars, i) { + let unit = escape_unit(&chars, i); + if let Some(ch) = unit.and_then(char::from_u32) { + o.push(ch); + i += 7; + continue; + } + let pair = unit.filter(|h| (0xD800..=0xDBFF).contains(h) && is_escape(&chars, i + 7)).and_then(|high| { + let low = escape_unit(&chars, i + 7).filter(|l| (0xDC00..=0xDFFF).contains(l))?; + char::from_u32(0x10000 + ((high - 0xD800) << 10) + (low - 0xDC00)) + }); + if let Some(ch) = pair { + o.push(ch); + i += 14; + continue; + } + } + o.push(c); + i += 1; + } + o +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn xstring() { + assert_eq!(decode_xstring("a_x000D_b"), "a\rb"); + assert_eq!(encode_xstring("a\rb"), "a_x000D_b"); + assert_eq!(encode_xstring("_x000D_"), "_x005F_x000D_"); + assert_eq!(decode_xstring("_x005F_x000D_"), "_x000D_"); + assert_eq!(decode_xstring("_x12"), "_x12"); + assert_eq!(decode_xstring("_xD83E__xDD80_"), "\u{1F980}"); + // A lone surrogate is no character: kept as written. + assert_eq!(decode_xstring("_xD83E_!"), "_xD83E_!"); + for s in ["plain", "tab\tline\n", "\u{1}\u{1F}", "_x0041_ _xZZZZ_ é"] { + assert_eq!(decode_xstring(&encode_xstring(s)), s); + } + } + + fn item(xml: &str) -> StringItem { + StringItem::read(&craft_xml::Options::local().parse(xml.as_bytes()).unwrap_or_default()) + } + + #[test] + fn runs_and_phonetics() { + let si = item( + r#"東京 トウキョウxx"#, + ); + assert_eq!(&*si.text, "東京 "); + assert_eq!(si.runs.len(), 2); + assert_eq!(si.runs[1].font.as_ref().map(|f| (f.bold, f.name.as_deref(), f.size)), Some((Some(true), Some("Arial"), Some(9.0)))); + assert_eq!(si.phonetic_runs, [PhoneticRun { start: 0, end: 2, text: "トウキョウ".into() }]); + assert_eq!(si.dropped_phonetic_runs, 3); + assert_eq!(si.phonetic, Some(PhoneticProps { font_id: 3, kind: Some("Hiragana".into()), alignment: None })); + let plain = item("a_x000A_b"); + assert_eq!((&*plain.text, plain.runs.len()), ("a\nb", 0)); + } +} diff --git a/crates/xlsx/src/styles.rs b/crates/xlsx/src/styles.rs new file mode 100644 index 000000000..ceb9547f0 --- /dev/null +++ b/crates/xlsx/src/styles.rs @@ -0,0 +1,555 @@ +//! The styles part (`xl/styles.xml`, ECMA-376 Part 1, 18.8) as the file has it: tables that +//! cells refer to by index. Attributes a file may leave out are `Option`s, because a +//! differential format (``) states only what it changes. + +use craft_sheet::{BorderStyle, DEFAULT_PALETTE, HAlign, Pattern, Script, Underline, VAlign, builtin_format}; +use craft_xml::El; + +/// Characters of a font name kept (Excel allows 31; damaged files have far more). +const MAX_FONT_NAME_CHARS: usize = 255; + +/// Where a colour comes from (`CT_Color`, 18.8.9). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ColorSource { + /// `auto="1"`, or no attribute that names a colour. + #[default] + Auto, + /// `rgb`: `0xAARRGGBB`. Six hex digits read as opaque. + Rgb(u32), + /// `theme`: an index into the theme's colour scheme. + Theme(u32), + /// `indexed`: an index into [`Styles::palette`]. + Indexed(u32), +} + +/// A colour with its tint (`-1.0` darkest to `1.0` lightest, 0 for none). +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct Color { + pub source: ColorSource, + pub tint: f64, +} + +impl Color { + pub fn read(e: &El) -> Color { + let source = if e.flag("auto", false) { + ColorSource::Auto + } else if let Some(rgb) = e.attr("rgb") { + parse_argb(rgb).map_or(ColorSource::Auto, ColorSource::Rgb) + } else if let Some(t) = e.attr_u32("theme") { + ColorSource::Theme(t) + } else if let Some(i) = e.attr_u32("indexed") { + ColorSource::Indexed(i) + } else { + ColorSource::Auto + }; + Color { source, tint: e.attr_f64("tint").map_or(0.0, |t| t.clamp(-1.0, 1.0)) } + } + + /// `0xRRGGBB` for an RGB colour, or an indexed one in `palette`; `None` for automatic and + /// theme colours. The tint is not applied. + pub fn rgb(&self, palette: &[u32]) -> Option { + match self.source { + ColorSource::Rgb(v) => Some(v & 0xFF_FFFF), + ColorSource::Indexed(i) => palette.get(i as usize).copied(), + ColorSource::Auto | ColorSource::Theme(_) => None, + } + } +} + +fn parse_argb(s: &str) -> Option { + let h = s.trim().trim_start_matches('#'); + let v = u32::from_str_radix(h, 16).ok().filter(|_| h.bytes().all(|b| b.is_ascii_hexdigit()))?; + match h.len() { + 8 => Some(v), + 6 => Some(0xFF00_0000 | v), + _ => None, + } +} + +/// `scheme` of a font: which theme font it follows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FontScheme { + None, + Major, + Minor, +} + +/// A font (``, or a run's ``, 18.8.22 and 18.4.7). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Font { + pub name: Option, + /// Points. + pub size: Option, + pub bold: Option, + pub italic: Option, + pub strike: Option, + pub outline: Option, + pub shadow: Option, + pub condense: Option, + pub extend: Option, + pub underline: Option, + pub vert_align: Option